#!/usr/bin/env python3 """Regression test: an Open Collective webhook must never fail as a bare 500. WHY THIS EXISTS On 2026-11-13 a member's $50 credit-pack purchase arrived as an `order.processed` delivery. The handler called fetch_members() to turn the OC slug into an email, `fetch_all_member_nodes` used a bare httpx.AsyncClient() (httpx's DEFAULT 5-second timeout), OC's GraphQL answered slower than that, and `httpx.ReadTimeout` escaped the handler as an unhandled ASGI exception: * the delivery returned 500, so nothing retried it usefully, * no credits were granted — the member paid and got nothing for 40 minutes, * the log carried a stack trace instead of anything naming the member or the order, so the purchase could not be reconciled from the log alone. The guards proven here, on a FIXTURE db with the network stubbed: AUTH 1. A wrong URL token is still a 401 — the new error net must not swallow the one status that means "the webhook URL and WEBHOOK_TOKEN disagree". NO BARE 500 2. A transport failure while resolving the purchase answers 503 (retryable) and the handler does not raise out of the ASGI app. 3. The same is true for a failure raised anywhere else in the handler. MONEY 4. A resolvable purchase grants credits ONCE, with the per-order reference `oc::`. 5. Re-delivering the identical payload returns `already_credited` and moves no money — this is what makes OC's retries safe. TRANSPORT 6. `_oc_graphql_post` retries once on a transport error and returns the response; if both attempts fail it RAISES (callers keep their fail-safes — a partial member list must never look complete). Runnable: python3 tests/test_webhook_resilience.py (from the repo root) """ import asyncio import os import sys import tempfile import httpx os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db") os.environ["WEBHOOK_TOKEN"] = "test-webhook-token" sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import app.main as m # noqa: E402 # The app module shares this httpx object, so stubbing it later would replace # httpx.AsyncClient process-wide — keep a handle on the real class. REAL_ASYNC_CLIENT = httpx.AsyncClient m.DB_PATH = os.environ["DB_PATH"] TOKEN = "test-webhook-token" MEMBER_SLUG = "nightrose" MEMBER_EMAIL = "nightrose@example.org" ORDER_ID = "or_testOrderId123" PAYLOAD = { "type": "order.processed", "data": { "fromCollective": {"slug": MEMBER_SLUG, "name": "NightRose"}, "order": { "idV2": ORDER_ID, "description": "Financial contribution to Inference Cooperative (Credit pack)", "totalAmount": 5000, }, "amount": 5000, "firstPayment": False, }, } results = [] def check(name, ok): results.append((name, ok)) SENT = [] def fake_send(to, subject, body, *a, **k): SENT.append({"to": to, "subject": subject, "body": body}) # --- stubs ----------------------------------------------------------------- # async def fetch_members_ok(): return [{"email": MEMBER_EMAIL, "name": "NightRose", "slug": MEMBER_SLUG, "role": "BACKER"}] async def fetch_members_timeout(): raise httpx.ReadTimeout("simulated OC read timeout") async def order_amount_ok(_order_id_v2=None): return 5000 async def budget_noop(_email): return 0.0 async def main(): m.get_db().close() # create + migrate the fixture schema conn = m.get_db() conn.execute("INSERT OR REPLACE INTO members (email, slug, active, credit_balance) " "VALUES (?, ?, 1, 0)", (MEMBER_EMAIL, MEMBER_SLUG)) conn.commit() conn.close() transport = httpx.ASGITransport(app=m.app) url = f"/webhook/opencollective/{TOKEN}" m.sync_team_budget_from_balance = budget_noop # no LiteLLM calls in a test m.get_order_amount = order_amount_ok async with REAL_ASYNC_CLIENT(transport=transport, base_url="http://test") as c: # --- 1. AUTH: a wrong token is still a 401. --- r = await c.post("/webhook/opencollective/wrong-token", json=PAYLOAD) check(f"auth: a wrong webhook token is 401 (got {r.status_code})", r.status_code == 401) # --- 2. NO BARE 500 when the OC lookup times out. --- m.fetch_members = fetch_members_timeout r = await c.post(url, json=PAYLOAD) body = r.json() if r.headers.get("content-type", "").startswith("application/json") else {} check(f"timeout: the delivery answers 503, not 500 (got {r.status_code})", r.status_code == 503) check(f"timeout: the response asks for a retry (got {body})", body.get("retry") is True) check("timeout: an unexpected failure never raises out of the app " "(httpx.ReadTimeout was caught by the handler net)", r.status_code != 500) # --- 2b. The SYSTEM records the failures it refuses. --- # The money-critical alert used to be a log scraper on the operator's own # machine, reading a buffer with hours of memory — whole hours were missing # from it during the 2026-11-13 investigation. The portal handles every # delivery, so the portal is what must notice and say. conn = m.get_db() rows = [dict(zip(("id", "at", "status", "event_type", "detail"), r)) for r in conn.execute("SELECT id, at, status, event_type, detail FROM webhook_failures " "ORDER BY id")] conn.close() check(f"record: the refused 401 delivery is recorded by the portal (got {rows})", any(r["status"] == 401 for r in rows)) check(f"record: the 503 is recorded with its event type (got {rows})", any(r["status"] == 503 and r["event_type"] == "order.processed" for r in rows)) check("record: the 503 row names the exception that caused it", any(r["status"] == 503 and "ReadTimeout" in (r["detail"] or "") for r in rows)) SENT.clear() m.send_email = fake_send rep = m.webhook_failure_report() check(f"report: new failures are summarised (got {rep.get('new_failures')} rows)", rep.get("status") == "reported" and rep.get("new_failures") == len(rows)) check(f"report: info@ is told, naming both kinds of failure (got {SENT})", len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop" and "401" in SENT[0]["body"] and "503" in SENT[0]["body"] and "WEBHOOK_TOKEN" in SENT[0]["body"]) rep2 = m.webhook_failure_report() check(f"report: the SAME batch is not re-reported (got {rep2})", rep2.get("new_failures") == 0 and len(SENT) == 1) await c.post("/webhook/opencollective/wrong-token", json=PAYLOAD) rep3 = m.webhook_failure_report() check(f"report: a LATER failure is reported again (got {rep3.get('new_failures')})", rep3.get("new_failures") == 1 and len(SENT) == 2) # --- 3. MONEY: a resolvable purchase credits exactly once. --- m.fetch_members = fetch_members_ok r = await c.post(url, json=PAYLOAD) body = r.json() check(f"grant: a resolvable credit pack grants credits (got {body})", r.status_code == 200 and body.get("status") == "credits_added" and body.get("credits_added") == 50.0) check("grant: the balance reflects the purchase", m.get_credit_balance(MEMBER_EMAIL) == 50.0) ref = m.credit_pack_reference(MEMBER_SLUG, {"idV2": ORDER_ID}) check("grant: the ledger row is keyed on the OC order (idempotence key)", ref == f"oc:{MEMBER_SLUG}:{ORDER_ID}" and m.credit_reference_exists(ref)) # --- 4. Re-delivery moves no money. --- r = await c.post(url, json=PAYLOAD) body = r.json() check(f"redelivery: the identical payload answers already_credited (got {body})", r.status_code == 200 and body.get("status") == "already_credited" and body.get("credits_added") == 0.0) check("redelivery: the balance is unchanged", m.get_credit_balance(MEMBER_EMAIL) == 50.0) # --- 5. TRANSPORT: retry-once, then raise. --- calls = [] class FlakyClient: def __init__(self, fail_times): self.fail_times = fail_times async def post(self, *a, **k): calls.append(1) if len(calls) <= self.fail_times: raise httpx.ReadTimeout("simulated timeout") return "RESPONSE" sentinel = await m._oc_graphql_post(FlakyClient(1), {"query": "x"}) check(f"transport: one transient failure is retried and succeeds (got {sentinel!r})", sentinel == "RESPONSE" and len(calls) == 2) calls.clear() err = None try: await m._oc_graphql_post(FlakyClient(2), {"query": "x"}) except Exception as e: # noqa: BLE001 err = e check("transport: a persistent failure still RAISES to the caller", isinstance(err, httpx.ReadTimeout)) # --- 6. The client used for the member walk has a real timeout. --- src = open(os.path.join(os.path.dirname(os.path.dirname( os.path.abspath(__file__))), "app", "main.py"), encoding="utf-8").read() check("timeout: fetch_all_member_nodes does NOT use httpx's 5s default", "async with httpx.AsyncClient(timeout=30.0) as client:" in src) asyncio.run(main()) fails = sum(not ok for _, ok in results) for name, ok in results: print(f" [{'PASS' if ok else 'FAIL'}] {name}") print(f"\n{len(results) - fails}/{len(results)} passed") sys.exit(1 if fails else 0)