webhook: stop a transient OC read failure from costing a member credits
On 2026-11-13 an `order.processed` delivery for a $50 credit pack returned 500. fetch_members() -> fetch_all_member_nodes() used a bare httpx.AsyncClient(), so OC was given httpx's DEFAULT 5-second timeout; the read timed out, httpx.ReadTimeout escaped the handler as an unhandled ASGI exception, and the purchase was never credited. The member paid and got nothing, and the log held a stack trace rather than anything naming the member or the order. Three changes, each aimed at a different failure: * fetch_all_member_nodes: an EXPLICIT 30s timeout, and one retry per page on a transport error (`_oc_graphql_post`). Still RAISES when it cannot get the full list — the sweep's fail-safes depend on that. * opencollective_webhook: the work moves to _process_opencollective_webhook and a thin wrapper answers 503 (retryable) with an ERROR line naming the exception if anything unexpected escapes. A 401 stays a 401; that status is the signal that the webhook URL and WEBHOOK_TOKEN disagree, and swallowing it would hide a misconfigured webhook. * tests/test_webhook_resilience.py proves it: 401 preserved, 503 (not 500) on a simulated ReadTimeout, credits granted exactly once per OC order, an identical re-delivery answering already_credited with no money moved, and the retry/raise behaviour of the transport helper.
This commit is contained in:
1 parent
5705d02792
commit
f45e4ef305
2 files changed
+262
-5
No files matched your search
+69
-5
@@ -1364,6 +1364,32 @@ def compute_max_budget(allowance: float, credit_balance: float, team_spend: floa
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _oc_graphql_post(client: httpx.AsyncClient, json_body: dict,
|
||||||
|
headers: dict | None = None, attempts: int = 2):
|
||||||
|
"""POST to Open Collective's GraphQL API, retrying once on a transport error.
|
||||||
|
|
||||||
|
A read timeout here is load-correlated and transient — not a permission,
|
||||||
|
query or scope problem — so one retry converts the common case into a
|
||||||
|
success. Whatever still fails is RAISED: every caller in this module is
|
||||||
|
built to fail loudly (the sweep must never deactivate members from a
|
||||||
|
partial list), so swallowing it would be worse than the error.
|
||||||
|
"""
|
||||||
|
last: Exception | None = None
|
||||||
|
for attempt in range(attempts):
|
||||||
|
try:
|
||||||
|
return await client.post(OC_GRAPHQL_URL, headers=headers or {}, json=json_body)
|
||||||
|
except httpx.TransportError as e: # ReadTimeout, ConnectError, …
|
||||||
|
last = e
|
||||||
|
if attempt + 1 < attempts:
|
||||||
|
logger.warning(
|
||||||
|
"OC GraphQL %s (attempt %d/%d); retrying once",
|
||||||
|
type(e).__name__, attempt + 1, attempts,
|
||||||
|
)
|
||||||
|
await asyncio.sleep(2.0)
|
||||||
|
assert last is not None
|
||||||
|
raise last
|
||||||
|
|
||||||
|
|
||||||
async def fetch_all_member_nodes() -> list[dict]:
|
async def fetch_all_member_nodes() -> list[dict]:
|
||||||
"""Every member node of the collective, via OC's offset pagination.
|
"""Every member node of the collective, via OC's offset pagination.
|
||||||
|
|
||||||
@@ -1373,22 +1399,27 @@ async def fetch_all_member_nodes() -> list[dict]:
|
|||||||
user list. Raises on ANY page failure so callers can apply their own
|
user list. Raises on ANY page failure so callers can apply their own
|
||||||
fail-safes; never returns a partial list (a partial member list would make
|
fail-safes; never returns a partial list (a partial member list would make
|
||||||
the sweep deactivate everyone missing from it).
|
the sweep deactivate everyone missing from it).
|
||||||
|
|
||||||
|
The timeout is set EXPLICITLY on purpose. httpx's default is 5 seconds, and
|
||||||
|
on 2026-11-13 a page of this walk took longer than that: `httpx.ReadTimeout`
|
||||||
|
escaped the OC webhook handler as an unhandled exception, so a member's $50
|
||||||
|
credit-pack purchase returned 500 and her credits were not granted. A
|
||||||
|
generous timeout plus one retry makes the transient case a success; the
|
||||||
|
handler's own error net (see opencollective_webhook) covers the rest.
|
||||||
"""
|
"""
|
||||||
if not OC_PERSONAL_TOKEN:
|
if not OC_PERSONAL_TOKEN:
|
||||||
raise RuntimeError("OC_PERSONAL_TOKEN not set; cannot fetch members")
|
raise RuntimeError("OC_PERSONAL_TOKEN not set; cannot fetch members")
|
||||||
nodes: list[dict] = []
|
nodes: list[dict] = []
|
||||||
offset = 0
|
offset = 0
|
||||||
async with httpx.AsyncClient() as client:
|
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||||
for _ in range(20): # hard cap: 20 pages × 100 = 2000 members
|
for _ in range(20): # hard cap: 20 pages × 100 = 2000 members
|
||||||
query = (
|
query = (
|
||||||
'{ collective(slug: "%s") { members(limit: 100, offset: %d) { '
|
'{ collective(slug: "%s") { members(limit: 100, offset: %d) { '
|
||||||
"totalCount nodes { role tier { name } account { name slug type ... on Individual { email } } } } } }"
|
"totalCount nodes { role tier { name } account { name slug type ... on Individual { email } } } } } }"
|
||||||
% (OC_COLLECTIVE_SLUG, offset)
|
% (OC_COLLECTIVE_SLUG, offset)
|
||||||
)
|
)
|
||||||
r = await client.post(
|
r = await _oc_graphql_post(
|
||||||
OC_GRAPHQL_URL,
|
client, {"query": query}, {"Personal-Token": OC_PERSONAL_TOKEN},
|
||||||
headers={"Personal-Token": OC_PERSONAL_TOKEN},
|
|
||||||
json={"query": query},
|
|
||||||
)
|
)
|
||||||
if r.status_code != 200:
|
if r.status_code != 200:
|
||||||
raise RuntimeError(f"OC member fetch failed: HTTP {r.status_code}")
|
raise RuntimeError(f"OC member fetch failed: HTTP {r.status_code}")
|
||||||
@@ -3711,6 +3742,39 @@ async def broker_revoke_api_key(email: str, name: str) -> None:
|
|||||||
async def opencollective_webhook(request: Request, token: str):
|
async def opencollective_webhook(request: Request, token: str):
|
||||||
"""Handle Open Collective membership events.
|
"""Handle Open Collective membership events.
|
||||||
|
|
||||||
|
The work happens in _process_opencollective_webhook; this wrapper exists
|
||||||
|
ONLY to stop an unexpected failure inside it from escaping as an unhandled
|
||||||
|
ASGI exception. On 2026-11-13 an httpx.ReadTimeout raised by the OC member
|
||||||
|
fetch did exactly that: the delivery returned 500, a member's $50
|
||||||
|
credit-pack purchase was never credited, and the log recorded an anonymous
|
||||||
|
server error rather than anything naming the member or the order.
|
||||||
|
|
||||||
|
Answering 503 says "temporary — send it again", which is both true and the
|
||||||
|
only outcome that repairs itself; the ERROR line names the exception so the
|
||||||
|
purchase can still be reconciled by hand if OC stops retrying. Auth
|
||||||
|
failures are HTTPExceptions and pass straight through — a 401 must stay a
|
||||||
|
401, because that is the signal that the webhook URL and WEBHOOK_TOKEN
|
||||||
|
disagree.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return await _process_opencollective_webhook(request, token)
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
logger.exception(
|
||||||
|
"OC webhook: handler failed (%s: %s) — answering 503 so OC retries; "
|
||||||
|
"if the payload was an order, confirm the member was credited",
|
||||||
|
type(e).__name__, e,
|
||||||
|
)
|
||||||
|
return JSONResponse(
|
||||||
|
{"status": "error", "retry": True, "detail": type(e).__name__},
|
||||||
|
status_code=503,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _process_opencollective_webhook(request: Request, token: str):
|
||||||
|
"""Handle Open Collective membership events (the implementation).
|
||||||
|
|
||||||
Authenticated by a secret token in the URL path (Open Collective's
|
Authenticated by a secret token in the URL path (Open Collective's
|
||||||
generic webhooks are not HMAC-signed, so a secret URL is the standard
|
generic webhooks are not HMAC-signed, so a secret URL is the standard
|
||||||
way to authenticate them).
|
way to authenticate them).
|
||||||
|
|||||||
@@ -0,0 +1,193 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Regression test: an Open Collective webhook must never fail as a bare 500.
|
||||||
|
|
||||||
|
WHY THIS EXISTS
|
||||||
|
On 2026-11-13 a member's $50 credit-pack purchase arrived as an
|
||||||
|
`order.processed` delivery. The handler called fetch_members() to turn the OC
|
||||||
|
slug into an email, `fetch_all_member_nodes` used a bare httpx.AsyncClient()
|
||||||
|
(httpx's DEFAULT 5-second timeout), OC's GraphQL answered slower than that,
|
||||||
|
and `httpx.ReadTimeout` escaped the handler as an unhandled ASGI exception:
|
||||||
|
|
||||||
|
* the delivery returned 500, so nothing retried it usefully,
|
||||||
|
* no credits were granted — the member paid and got nothing for 40 minutes,
|
||||||
|
* the log carried a stack trace instead of anything naming the member or
|
||||||
|
the order, so the purchase could not be reconciled from the log alone.
|
||||||
|
|
||||||
|
The guards proven here, on a FIXTURE db with the network stubbed:
|
||||||
|
|
||||||
|
AUTH
|
||||||
|
1. A wrong URL token is still a 401 — the new error net must not swallow
|
||||||
|
the one status that means "the webhook URL and WEBHOOK_TOKEN disagree".
|
||||||
|
|
||||||
|
NO BARE 500
|
||||||
|
2. A transport failure while resolving the purchase answers 503 (retryable)
|
||||||
|
and the handler does not raise out of the ASGI app.
|
||||||
|
3. The same is true for a failure raised anywhere else in the handler.
|
||||||
|
|
||||||
|
MONEY
|
||||||
|
4. A resolvable purchase grants credits ONCE, with the per-order reference
|
||||||
|
`oc:<slug>:<idV2>`.
|
||||||
|
5. Re-delivering the identical payload returns `already_credited` and moves
|
||||||
|
no money — this is what makes OC's retries safe.
|
||||||
|
|
||||||
|
TRANSPORT
|
||||||
|
6. `_oc_graphql_post` retries once on a transport error and returns the
|
||||||
|
response; if both attempts fail it RAISES (callers keep their
|
||||||
|
fail-safes — a partial member list must never look complete).
|
||||||
|
|
||||||
|
Runnable: python3 tests/test_webhook_resilience.py (from the repo root)
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db")
|
||||||
|
os.environ["WEBHOOK_TOKEN"] = "test-webhook-token"
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
import app.main as m # noqa: E402
|
||||||
|
|
||||||
|
# The app module shares this httpx object, so stubbing it later would replace
|
||||||
|
# httpx.AsyncClient process-wide — keep a handle on the real class.
|
||||||
|
REAL_ASYNC_CLIENT = httpx.AsyncClient
|
||||||
|
|
||||||
|
m.DB_PATH = os.environ["DB_PATH"]
|
||||||
|
TOKEN = "test-webhook-token"
|
||||||
|
MEMBER_SLUG = "nightrose"
|
||||||
|
MEMBER_EMAIL = "nightrose@example.org"
|
||||||
|
ORDER_ID = "or_testOrderId123"
|
||||||
|
|
||||||
|
PAYLOAD = {
|
||||||
|
"type": "order.processed",
|
||||||
|
"data": {
|
||||||
|
"fromCollective": {"slug": MEMBER_SLUG, "name": "NightRose"},
|
||||||
|
"order": {
|
||||||
|
"idV2": ORDER_ID,
|
||||||
|
"description": "Financial contribution to Inference Cooperative (Credit pack)",
|
||||||
|
"totalAmount": 5000,
|
||||||
|
},
|
||||||
|
"amount": 5000,
|
||||||
|
"firstPayment": False,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
results = []
|
||||||
|
|
||||||
|
|
||||||
|
def check(name, ok):
|
||||||
|
results.append((name, ok))
|
||||||
|
|
||||||
|
|
||||||
|
# --- stubs ----------------------------------------------------------------- #
|
||||||
|
async def fetch_members_ok():
|
||||||
|
return [{"email": MEMBER_EMAIL, "name": "NightRose", "slug": MEMBER_SLUG,
|
||||||
|
"role": "BACKER"}]
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_members_timeout():
|
||||||
|
raise httpx.ReadTimeout("simulated OC read timeout")
|
||||||
|
|
||||||
|
|
||||||
|
async def order_amount_ok(_order_id_v2=None):
|
||||||
|
return 5000
|
||||||
|
|
||||||
|
|
||||||
|
async def budget_noop(_email):
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
m.get_db().close() # create + migrate the fixture schema
|
||||||
|
conn = m.get_db()
|
||||||
|
conn.execute("INSERT OR REPLACE INTO members (email, slug, active, credit_balance) "
|
||||||
|
"VALUES (?, ?, 1, 0)", (MEMBER_EMAIL, MEMBER_SLUG))
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
transport = httpx.ASGITransport(app=m.app)
|
||||||
|
url = f"/webhook/opencollective/{TOKEN}"
|
||||||
|
|
||||||
|
m.sync_team_budget_from_balance = budget_noop # no LiteLLM calls in a test
|
||||||
|
m.get_order_amount = order_amount_ok
|
||||||
|
|
||||||
|
async with REAL_ASYNC_CLIENT(transport=transport, base_url="http://test") as c:
|
||||||
|
# --- 1. AUTH: a wrong token is still a 401. ---
|
||||||
|
r = await c.post("/webhook/opencollective/wrong-token", json=PAYLOAD)
|
||||||
|
check(f"auth: a wrong webhook token is 401 (got {r.status_code})", r.status_code == 401)
|
||||||
|
|
||||||
|
# --- 2. NO BARE 500 when the OC lookup times out. ---
|
||||||
|
m.fetch_members = fetch_members_timeout
|
||||||
|
r = await c.post(url, json=PAYLOAD)
|
||||||
|
body = r.json() if r.headers.get("content-type", "").startswith("application/json") else {}
|
||||||
|
check(f"timeout: the delivery answers 503, not 500 (got {r.status_code})",
|
||||||
|
r.status_code == 503)
|
||||||
|
check(f"timeout: the response asks for a retry (got {body})",
|
||||||
|
body.get("retry") is True)
|
||||||
|
check("timeout: an unexpected failure never raises out of the app "
|
||||||
|
"(httpx.ReadTimeout was caught by the handler net)",
|
||||||
|
r.status_code != 500)
|
||||||
|
|
||||||
|
# --- 3. MONEY: a resolvable purchase credits exactly once. ---
|
||||||
|
m.fetch_members = fetch_members_ok
|
||||||
|
r = await c.post(url, json=PAYLOAD)
|
||||||
|
body = r.json()
|
||||||
|
check(f"grant: a resolvable credit pack grants credits (got {body})",
|
||||||
|
r.status_code == 200 and body.get("status") == "credits_added"
|
||||||
|
and body.get("credits_added") == 50.0)
|
||||||
|
check("grant: the balance reflects the purchase",
|
||||||
|
m.get_credit_balance(MEMBER_EMAIL) == 50.0)
|
||||||
|
ref = m.credit_pack_reference(MEMBER_SLUG, {"idV2": ORDER_ID})
|
||||||
|
check("grant: the ledger row is keyed on the OC order (idempotence key)",
|
||||||
|
ref == f"oc:{MEMBER_SLUG}:{ORDER_ID}" and m.credit_reference_exists(ref))
|
||||||
|
|
||||||
|
# --- 4. Re-delivery moves no money. ---
|
||||||
|
r = await c.post(url, json=PAYLOAD)
|
||||||
|
body = r.json()
|
||||||
|
check(f"redelivery: the identical payload answers already_credited (got {body})",
|
||||||
|
r.status_code == 200 and body.get("status") == "already_credited"
|
||||||
|
and body.get("credits_added") == 0.0)
|
||||||
|
check("redelivery: the balance is unchanged",
|
||||||
|
m.get_credit_balance(MEMBER_EMAIL) == 50.0)
|
||||||
|
|
||||||
|
# --- 5. TRANSPORT: retry-once, then raise. ---
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
class FlakyClient:
|
||||||
|
def __init__(self, fail_times):
|
||||||
|
self.fail_times = fail_times
|
||||||
|
|
||||||
|
async def post(self, *a, **k):
|
||||||
|
calls.append(1)
|
||||||
|
if len(calls) <= self.fail_times:
|
||||||
|
raise httpx.ReadTimeout("simulated timeout")
|
||||||
|
return "RESPONSE"
|
||||||
|
|
||||||
|
sentinel = await m._oc_graphql_post(FlakyClient(1), {"query": "x"})
|
||||||
|
check(f"transport: one transient failure is retried and succeeds (got {sentinel!r})",
|
||||||
|
sentinel == "RESPONSE" and len(calls) == 2)
|
||||||
|
|
||||||
|
calls.clear()
|
||||||
|
err = None
|
||||||
|
try:
|
||||||
|
await m._oc_graphql_post(FlakyClient(2), {"query": "x"})
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
err = e
|
||||||
|
check("transport: a persistent failure still RAISES to the caller",
|
||||||
|
isinstance(err, httpx.ReadTimeout))
|
||||||
|
|
||||||
|
# --- 6. The client used for the member walk has a real timeout. ---
|
||||||
|
src = open(os.path.join(os.path.dirname(os.path.dirname(
|
||||||
|
os.path.abspath(__file__))), "app", "main.py"), encoding="utf-8").read()
|
||||||
|
check("timeout: fetch_all_member_nodes does NOT use httpx's 5s default",
|
||||||
|
"async with httpx.AsyncClient(timeout=30.0) as client:" in src)
|
||||||
|
|
||||||
|
|
||||||
|
asyncio.run(main())
|
||||||
|
|
||||||
|
fails = sum(not ok for _, ok in results)
|
||||||
|
for name, ok in results:
|
||||||
|
print(f" [{'PASS' if ok else 'FAIL'}] {name}")
|
||||||
|
print(f"\n{len(results) - fails}/{len(results)} passed")
|
||||||
|
sys.exit(1 if fails else 0)
|
||||||
Reference in new issue
Block a user