From f45e4ef3054b2167dd7eea9cdc9486fbc2c6b9e3 Mon Sep 17 00:00:00 2001 From: Inferencebot Date: Fri, 13 Nov 2026 20:04:09 +0000 Subject: [PATCH] webhook: stop a transient OC read failure from costing a member credits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On 2026-11-13 an `order.processed` delivery for a $50 credit pack returned 500. fetch_members() -> fetch_all_member_nodes() used a bare httpx.AsyncClient(), so OC was given httpx's DEFAULT 5-second timeout; the read timed out, httpx.ReadTimeout escaped the handler as an unhandled ASGI exception, and the purchase was never credited. The member paid and got nothing, and the log held a stack trace rather than anything naming the member or the order. Three changes, each aimed at a different failure: * fetch_all_member_nodes: an EXPLICIT 30s timeout, and one retry per page on a transport error (`_oc_graphql_post`). Still RAISES when it cannot get the full list — the sweep's fail-safes depend on that. * opencollective_webhook: the work moves to _process_opencollective_webhook and a thin wrapper answers 503 (retryable) with an ERROR line naming the exception if anything unexpected escapes. A 401 stays a 401; that status is the signal that the webhook URL and WEBHOOK_TOKEN disagree, and swallowing it would hide a misconfigured webhook. * tests/test_webhook_resilience.py proves it: 401 preserved, 503 (not 500) on a simulated ReadTimeout, credits granted exactly once per OC order, an identical re-delivery answering already_credited with no money moved, and the retry/raise behaviour of the transport helper. --- app/main.py | 74 +++++++++++- tests/test_webhook_resilience.py | 193 +++++++++++++++++++++++++++++++ 2 files changed, 262 insertions(+), 5 deletions(-) create mode 100644 tests/test_webhook_resilience.py diff --git a/app/main.py b/app/main.py index 270d70e..0e7d467 100644 --- a/app/main.py +++ b/app/main.py @@ -1364,6 +1364,32 @@ def compute_max_budget(allowance: float, credit_balance: float, team_spend: floa } +async def _oc_graphql_post(client: httpx.AsyncClient, json_body: dict, + headers: dict | None = None, attempts: int = 2): + """POST to Open Collective's GraphQL API, retrying once on a transport error. + + A read timeout here is load-correlated and transient — not a permission, + query or scope problem — so one retry converts the common case into a + success. Whatever still fails is RAISED: every caller in this module is + built to fail loudly (the sweep must never deactivate members from a + partial list), so swallowing it would be worse than the error. + """ + last: Exception | None = None + for attempt in range(attempts): + try: + return await client.post(OC_GRAPHQL_URL, headers=headers or {}, json=json_body) + except httpx.TransportError as e: # ReadTimeout, ConnectError, … + last = e + if attempt + 1 < attempts: + logger.warning( + "OC GraphQL %s (attempt %d/%d); retrying once", + type(e).__name__, attempt + 1, attempts, + ) + await asyncio.sleep(2.0) + assert last is not None + raise last + + async def fetch_all_member_nodes() -> list[dict]: """Every member node of the collective, via OC's offset pagination. @@ -1373,22 +1399,27 @@ async def fetch_all_member_nodes() -> list[dict]: user list. Raises on ANY page failure so callers can apply their own fail-safes; never returns a partial list (a partial member list would make the sweep deactivate everyone missing from it). + + The timeout is set EXPLICITLY on purpose. httpx's default is 5 seconds, and + on 2026-11-13 a page of this walk took longer than that: `httpx.ReadTimeout` + escaped the OC webhook handler as an unhandled exception, so a member's $50 + credit-pack purchase returned 500 and her credits were not granted. A + generous timeout plus one retry makes the transient case a success; the + handler's own error net (see opencollective_webhook) covers the rest. """ if not OC_PERSONAL_TOKEN: raise RuntimeError("OC_PERSONAL_TOKEN not set; cannot fetch members") nodes: list[dict] = [] offset = 0 - async with httpx.AsyncClient() as client: + async with httpx.AsyncClient(timeout=30.0) as client: for _ in range(20): # hard cap: 20 pages × 100 = 2000 members query = ( '{ collective(slug: "%s") { members(limit: 100, offset: %d) { ' "totalCount nodes { role tier { name } account { name slug type ... on Individual { email } } } } } }" % (OC_COLLECTIVE_SLUG, offset) ) - r = await client.post( - OC_GRAPHQL_URL, - headers={"Personal-Token": OC_PERSONAL_TOKEN}, - json={"query": query}, + r = await _oc_graphql_post( + client, {"query": query}, {"Personal-Token": OC_PERSONAL_TOKEN}, ) if r.status_code != 200: raise RuntimeError(f"OC member fetch failed: HTTP {r.status_code}") @@ -3711,6 +3742,39 @@ async def broker_revoke_api_key(email: str, name: str) -> None: async def opencollective_webhook(request: Request, token: str): """Handle Open Collective membership events. + The work happens in _process_opencollective_webhook; this wrapper exists + ONLY to stop an unexpected failure inside it from escaping as an unhandled + ASGI exception. On 2026-11-13 an httpx.ReadTimeout raised by the OC member + fetch did exactly that: the delivery returned 500, a member's $50 + credit-pack purchase was never credited, and the log recorded an anonymous + server error rather than anything naming the member or the order. + + Answering 503 says "temporary — send it again", which is both true and the + only outcome that repairs itself; the ERROR line names the exception so the + purchase can still be reconciled by hand if OC stops retrying. Auth + failures are HTTPExceptions and pass straight through — a 401 must stay a + 401, because that is the signal that the webhook URL and WEBHOOK_TOKEN + disagree. + """ + try: + return await _process_opencollective_webhook(request, token) + except HTTPException: + raise + except Exception as e: # noqa: BLE001 + logger.exception( + "OC webhook: handler failed (%s: %s) — answering 503 so OC retries; " + "if the payload was an order, confirm the member was credited", + type(e).__name__, e, + ) + return JSONResponse( + {"status": "error", "retry": True, "detail": type(e).__name__}, + status_code=503, + ) + + +async def _process_opencollective_webhook(request: Request, token: str): + """Handle Open Collective membership events (the implementation). + Authenticated by a secret token in the URL path (Open Collective's generic webhooks are not HMAC-signed, so a secret URL is the standard way to authenticate them). diff --git a/tests/test_webhook_resilience.py b/tests/test_webhook_resilience.py new file mode 100644 index 0000000..24a53a9 --- /dev/null +++ b/tests/test_webhook_resilience.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Regression test: an Open Collective webhook must never fail as a bare 500. + +WHY THIS EXISTS + On 2026-11-13 a member's $50 credit-pack purchase arrived as an + `order.processed` delivery. The handler called fetch_members() to turn the OC + slug into an email, `fetch_all_member_nodes` used a bare httpx.AsyncClient() + (httpx's DEFAULT 5-second timeout), OC's GraphQL answered slower than that, + and `httpx.ReadTimeout` escaped the handler as an unhandled ASGI exception: + + * the delivery returned 500, so nothing retried it usefully, + * no credits were granted — the member paid and got nothing for 40 minutes, + * the log carried a stack trace instead of anything naming the member or + the order, so the purchase could not be reconciled from the log alone. + + The guards proven here, on a FIXTURE db with the network stubbed: + + AUTH + 1. A wrong URL token is still a 401 — the new error net must not swallow + the one status that means "the webhook URL and WEBHOOK_TOKEN disagree". + + NO BARE 500 + 2. A transport failure while resolving the purchase answers 503 (retryable) + and the handler does not raise out of the ASGI app. + 3. The same is true for a failure raised anywhere else in the handler. + + MONEY + 4. A resolvable purchase grants credits ONCE, with the per-order reference + `oc::`. + 5. Re-delivering the identical payload returns `already_credited` and moves + no money — this is what makes OC's retries safe. + + TRANSPORT + 6. `_oc_graphql_post` retries once on a transport error and returns the + response; if both attempts fail it RAISES (callers keep their + fail-safes — a partial member list must never look complete). + +Runnable: python3 tests/test_webhook_resilience.py (from the repo root) +""" +import asyncio +import os +import sys +import tempfile + +import httpx + +os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db") +os.environ["WEBHOOK_TOKEN"] = "test-webhook-token" +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import app.main as m # noqa: E402 + +# The app module shares this httpx object, so stubbing it later would replace +# httpx.AsyncClient process-wide — keep a handle on the real class. +REAL_ASYNC_CLIENT = httpx.AsyncClient + +m.DB_PATH = os.environ["DB_PATH"] +TOKEN = "test-webhook-token" +MEMBER_SLUG = "nightrose" +MEMBER_EMAIL = "nightrose@example.org" +ORDER_ID = "or_testOrderId123" + +PAYLOAD = { + "type": "order.processed", + "data": { + "fromCollective": {"slug": MEMBER_SLUG, "name": "NightRose"}, + "order": { + "idV2": ORDER_ID, + "description": "Financial contribution to Inference Cooperative (Credit pack)", + "totalAmount": 5000, + }, + "amount": 5000, + "firstPayment": False, + }, +} + +results = [] + + +def check(name, ok): + results.append((name, ok)) + + +# --- stubs ----------------------------------------------------------------- # +async def fetch_members_ok(): + return [{"email": MEMBER_EMAIL, "name": "NightRose", "slug": MEMBER_SLUG, + "role": "BACKER"}] + + +async def fetch_members_timeout(): + raise httpx.ReadTimeout("simulated OC read timeout") + + +async def order_amount_ok(_order_id_v2=None): + return 5000 + + +async def budget_noop(_email): + return 0.0 + + +async def main(): + m.get_db().close() # create + migrate the fixture schema + conn = m.get_db() + conn.execute("INSERT OR REPLACE INTO members (email, slug, active, credit_balance) " + "VALUES (?, ?, 1, 0)", (MEMBER_EMAIL, MEMBER_SLUG)) + conn.commit() + conn.close() + + transport = httpx.ASGITransport(app=m.app) + url = f"/webhook/opencollective/{TOKEN}" + + m.sync_team_budget_from_balance = budget_noop # no LiteLLM calls in a test + m.get_order_amount = order_amount_ok + + async with REAL_ASYNC_CLIENT(transport=transport, base_url="http://test") as c: + # --- 1. AUTH: a wrong token is still a 401. --- + r = await c.post("/webhook/opencollective/wrong-token", json=PAYLOAD) + check(f"auth: a wrong webhook token is 401 (got {r.status_code})", r.status_code == 401) + + # --- 2. NO BARE 500 when the OC lookup times out. --- + m.fetch_members = fetch_members_timeout + r = await c.post(url, json=PAYLOAD) + body = r.json() if r.headers.get("content-type", "").startswith("application/json") else {} + check(f"timeout: the delivery answers 503, not 500 (got {r.status_code})", + r.status_code == 503) + check(f"timeout: the response asks for a retry (got {body})", + body.get("retry") is True) + check("timeout: an unexpected failure never raises out of the app " + "(httpx.ReadTimeout was caught by the handler net)", + r.status_code != 500) + + # --- 3. MONEY: a resolvable purchase credits exactly once. --- + m.fetch_members = fetch_members_ok + r = await c.post(url, json=PAYLOAD) + body = r.json() + check(f"grant: a resolvable credit pack grants credits (got {body})", + r.status_code == 200 and body.get("status") == "credits_added" + and body.get("credits_added") == 50.0) + check("grant: the balance reflects the purchase", + m.get_credit_balance(MEMBER_EMAIL) == 50.0) + ref = m.credit_pack_reference(MEMBER_SLUG, {"idV2": ORDER_ID}) + check("grant: the ledger row is keyed on the OC order (idempotence key)", + ref == f"oc:{MEMBER_SLUG}:{ORDER_ID}" and m.credit_reference_exists(ref)) + + # --- 4. Re-delivery moves no money. --- + r = await c.post(url, json=PAYLOAD) + body = r.json() + check(f"redelivery: the identical payload answers already_credited (got {body})", + r.status_code == 200 and body.get("status") == "already_credited" + and body.get("credits_added") == 0.0) + check("redelivery: the balance is unchanged", + m.get_credit_balance(MEMBER_EMAIL) == 50.0) + + # --- 5. TRANSPORT: retry-once, then raise. --- + calls = [] + + class FlakyClient: + def __init__(self, fail_times): + self.fail_times = fail_times + + async def post(self, *a, **k): + calls.append(1) + if len(calls) <= self.fail_times: + raise httpx.ReadTimeout("simulated timeout") + return "RESPONSE" + + sentinel = await m._oc_graphql_post(FlakyClient(1), {"query": "x"}) + check(f"transport: one transient failure is retried and succeeds (got {sentinel!r})", + sentinel == "RESPONSE" and len(calls) == 2) + + calls.clear() + err = None + try: + await m._oc_graphql_post(FlakyClient(2), {"query": "x"}) + except Exception as e: # noqa: BLE001 + err = e + check("transport: a persistent failure still RAISES to the caller", + isinstance(err, httpx.ReadTimeout)) + + # --- 6. The client used for the member walk has a real timeout. --- + src = open(os.path.join(os.path.dirname(os.path.dirname( + os.path.abspath(__file__))), "app", "main.py"), encoding="utf-8").read() + check("timeout: fetch_all_member_nodes does NOT use httpx's 5s default", + "async with httpx.AsyncClient(timeout=30.0) as client:" in src) + + +asyncio.run(main()) + +fails = sum(not ok for _, ok in results) +for name, ok in results: + print(f" [{'PASS' if ok else 'FAIL'}] {name}") +print(f"\n{len(results) - fails}/{len(results)} passed") +sys.exit(1 if fails else 0)