webhook: stop a transient OC read failure from costing a member credits

On 2026-11-13 an `order.processed` delivery for a $50 credit pack returned 500.
fetch_members() -> fetch_all_member_nodes() used a bare httpx.AsyncClient(), so OC
was given httpx's DEFAULT 5-second timeout; the read timed out, httpx.ReadTimeout
escaped the handler as an unhandled ASGI exception, and the purchase was never
credited. The member paid and got nothing, and the log held a stack trace rather
than anything naming the member or the order.

Three changes, each aimed at a different failure:

* fetch_all_member_nodes: an EXPLICIT 30s timeout, and one retry per page on a
  transport error (`_oc_graphql_post`). Still RAISES when it cannot get the full
  list — the sweep's fail-safes depend on that.
* opencollective_webhook: the work moves to _process_opencollective_webhook and a
  thin wrapper answers 503 (retryable) with an ERROR line naming the exception if
  anything unexpected escapes. A 401 stays a 401; that status is the signal that
  the webhook URL and WEBHOOK_TOKEN disagree, and swallowing it would hide a
  misconfigured webhook.
* tests/test_webhook_resilience.py proves it: 401 preserved, 503 (not 500) on a
  simulated ReadTimeout, credits granted exactly once per OC order, an identical
  re-delivery answering already_credited with no money moved, and the retry/raise
  behaviour of the transport helper.
This commit is contained in:
inference-admin committed 2026-11-13 20:04:09 +00:00
1 parent 5705d02792
commit f45e4ef305
2 files changed
+262 -5

No files matched your search

+69 -5
View File
@@ -1364,6 +1364,32 @@ def compute_max_budget(allowance: float, credit_balance: float, team_spend: floa
} }
async def _oc_graphql_post(client: httpx.AsyncClient, json_body: dict,
headers: dict | None = None, attempts: int = 2):
"""POST to Open Collective's GraphQL API, retrying once on a transport error.
A read timeout here is load-correlated and transient — not a permission,
query or scope problem — so one retry converts the common case into a
success. Whatever still fails is RAISED: every caller in this module is
built to fail loudly (the sweep must never deactivate members from a
partial list), so swallowing it would be worse than the error.
"""
last: Exception | None = None
for attempt in range(attempts):
try:
return await client.post(OC_GRAPHQL_URL, headers=headers or {}, json=json_body)
except httpx.TransportError as e: # ReadTimeout, ConnectError, …
last = e
if attempt + 1 < attempts:
logger.warning(
"OC GraphQL %s (attempt %d/%d); retrying once",
type(e).__name__, attempt + 1, attempts,
)
await asyncio.sleep(2.0)
assert last is not None
raise last
async def fetch_all_member_nodes() -> list[dict]: async def fetch_all_member_nodes() -> list[dict]:
"""Every member node of the collective, via OC's offset pagination. """Every member node of the collective, via OC's offset pagination.
@@ -1373,22 +1399,27 @@ async def fetch_all_member_nodes() -> list[dict]:
user list. Raises on ANY page failure so callers can apply their own user list. Raises on ANY page failure so callers can apply their own
fail-safes; never returns a partial list (a partial member list would make fail-safes; never returns a partial list (a partial member list would make
the sweep deactivate everyone missing from it). the sweep deactivate everyone missing from it).
The timeout is set EXPLICITLY on purpose. httpx's default is 5 seconds, and
on 2026-11-13 a page of this walk took longer than that: `httpx.ReadTimeout`
escaped the OC webhook handler as an unhandled exception, so a member's $50
credit-pack purchase returned 500 and her credits were not granted. A
generous timeout plus one retry makes the transient case a success; the
handler's own error net (see opencollective_webhook) covers the rest.
""" """
if not OC_PERSONAL_TOKEN: if not OC_PERSONAL_TOKEN:
raise RuntimeError("OC_PERSONAL_TOKEN not set; cannot fetch members") raise RuntimeError("OC_PERSONAL_TOKEN not set; cannot fetch members")
nodes: list[dict] = [] nodes: list[dict] = []
offset = 0 offset = 0
async with httpx.AsyncClient() as client: async with httpx.AsyncClient(timeout=30.0) as client:
for _ in range(20): # hard cap: 20 pages × 100 = 2000 members for _ in range(20): # hard cap: 20 pages × 100 = 2000 members
query = ( query = (
'{ collective(slug: "%s") { members(limit: 100, offset: %d) { ' '{ collective(slug: "%s") { members(limit: 100, offset: %d) { '
"totalCount nodes { role tier { name } account { name slug type ... on Individual { email } } } } } }" "totalCount nodes { role tier { name } account { name slug type ... on Individual { email } } } } } }"
% (OC_COLLECTIVE_SLUG, offset) % (OC_COLLECTIVE_SLUG, offset)
) )
r = await client.post( r = await _oc_graphql_post(
OC_GRAPHQL_URL, client, {"query": query}, {"Personal-Token": OC_PERSONAL_TOKEN},
headers={"Personal-Token": OC_PERSONAL_TOKEN},
json={"query": query},
) )
if r.status_code != 200: if r.status_code != 200:
raise RuntimeError(f"OC member fetch failed: HTTP {r.status_code}") raise RuntimeError(f"OC member fetch failed: HTTP {r.status_code}")
@@ -3711,6 +3742,39 @@ async def broker_revoke_api_key(email: str, name: str) -> None:
async def opencollective_webhook(request: Request, token: str): async def opencollective_webhook(request: Request, token: str):
"""Handle Open Collective membership events. """Handle Open Collective membership events.
The work happens in _process_opencollective_webhook; this wrapper exists
ONLY to stop an unexpected failure inside it from escaping as an unhandled
ASGI exception. On 2026-11-13 an httpx.ReadTimeout raised by the OC member
fetch did exactly that: the delivery returned 500, a member's $50
credit-pack purchase was never credited, and the log recorded an anonymous
server error rather than anything naming the member or the order.
Answering 503 says "temporary — send it again", which is both true and the
only outcome that repairs itself; the ERROR line names the exception so the
purchase can still be reconciled by hand if OC stops retrying. Auth
failures are HTTPExceptions and pass straight through — a 401 must stay a
401, because that is the signal that the webhook URL and WEBHOOK_TOKEN
disagree.
"""
try:
return await _process_opencollective_webhook(request, token)
except HTTPException:
raise
except Exception as e: # noqa: BLE001
logger.exception(
"OC webhook: handler failed (%s: %s) — answering 503 so OC retries; "
"if the payload was an order, confirm the member was credited",
type(e).__name__, e,
)
return JSONResponse(
{"status": "error", "retry": True, "detail": type(e).__name__},
status_code=503,
)
async def _process_opencollective_webhook(request: Request, token: str):
"""Handle Open Collective membership events (the implementation).
Authenticated by a secret token in the URL path (Open Collective's Authenticated by a secret token in the URL path (Open Collective's
generic webhooks are not HMAC-signed, so a secret URL is the standard generic webhooks are not HMAC-signed, so a secret URL is the standard
way to authenticate them). way to authenticate them).
+193
View File
@@ -0,0 +1,193 @@
#!/usr/bin/env python3
"""Regression test: an Open Collective webhook must never fail as a bare 500.
WHY THIS EXISTS
On 2026-11-13 a member's $50 credit-pack purchase arrived as an
`order.processed` delivery. The handler called fetch_members() to turn the OC
slug into an email, `fetch_all_member_nodes` used a bare httpx.AsyncClient()
(httpx's DEFAULT 5-second timeout), OC's GraphQL answered slower than that,
and `httpx.ReadTimeout` escaped the handler as an unhandled ASGI exception:
* the delivery returned 500, so nothing retried it usefully,
* no credits were granted — the member paid and got nothing for 40 minutes,
* the log carried a stack trace instead of anything naming the member or
the order, so the purchase could not be reconciled from the log alone.
The guards proven here, on a FIXTURE db with the network stubbed:
AUTH
1. A wrong URL token is still a 401 — the new error net must not swallow
the one status that means "the webhook URL and WEBHOOK_TOKEN disagree".
NO BARE 500
2. A transport failure while resolving the purchase answers 503 (retryable)
and the handler does not raise out of the ASGI app.
3. The same is true for a failure raised anywhere else in the handler.
MONEY
4. A resolvable purchase grants credits ONCE, with the per-order reference
`oc:<slug>:<idV2>`.
5. Re-delivering the identical payload returns `already_credited` and moves
no money — this is what makes OC's retries safe.
TRANSPORT
6. `_oc_graphql_post` retries once on a transport error and returns the
response; if both attempts fail it RAISES (callers keep their
fail-safes — a partial member list must never look complete).
Runnable: python3 tests/test_webhook_resilience.py (from the repo root)
"""
import asyncio
import os
import sys
import tempfile
import httpx
os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db")
os.environ["WEBHOOK_TOKEN"] = "test-webhook-token"
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import app.main as m # noqa: E402
# The app module shares this httpx object, so stubbing it later would replace
# httpx.AsyncClient process-wide — keep a handle on the real class.
REAL_ASYNC_CLIENT = httpx.AsyncClient
m.DB_PATH = os.environ["DB_PATH"]
TOKEN = "test-webhook-token"
MEMBER_SLUG = "nightrose"
MEMBER_EMAIL = "nightrose@example.org"
ORDER_ID = "or_testOrderId123"
PAYLOAD = {
"type": "order.processed",
"data": {
"fromCollective": {"slug": MEMBER_SLUG, "name": "NightRose"},
"order": {
"idV2": ORDER_ID,
"description": "Financial contribution to Inference Cooperative (Credit pack)",
"totalAmount": 5000,
},
"amount": 5000,
"firstPayment": False,
},
}
results = []
def check(name, ok):
results.append((name, ok))
# --- stubs ----------------------------------------------------------------- #
async def fetch_members_ok():
return [{"email": MEMBER_EMAIL, "name": "NightRose", "slug": MEMBER_SLUG,
"role": "BACKER"}]
async def fetch_members_timeout():
raise httpx.ReadTimeout("simulated OC read timeout")
async def order_amount_ok(_order_id_v2=None):
return 5000
async def budget_noop(_email):
return 0.0
async def main():
m.get_db().close() # create + migrate the fixture schema
conn = m.get_db()
conn.execute("INSERT OR REPLACE INTO members (email, slug, active, credit_balance) "
"VALUES (?, ?, 1, 0)", (MEMBER_EMAIL, MEMBER_SLUG))
conn.commit()
conn.close()
transport = httpx.ASGITransport(app=m.app)
url = f"/webhook/opencollective/{TOKEN}"
m.sync_team_budget_from_balance = budget_noop # no LiteLLM calls in a test
m.get_order_amount = order_amount_ok
async with REAL_ASYNC_CLIENT(transport=transport, base_url="http://test") as c:
# --- 1. AUTH: a wrong token is still a 401. ---
r = await c.post("/webhook/opencollective/wrong-token", json=PAYLOAD)
check(f"auth: a wrong webhook token is 401 (got {r.status_code})", r.status_code == 401)
# --- 2. NO BARE 500 when the OC lookup times out. ---
m.fetch_members = fetch_members_timeout
r = await c.post(url, json=PAYLOAD)
body = r.json() if r.headers.get("content-type", "").startswith("application/json") else {}
check(f"timeout: the delivery answers 503, not 500 (got {r.status_code})",
r.status_code == 503)
check(f"timeout: the response asks for a retry (got {body})",
body.get("retry") is True)
check("timeout: an unexpected failure never raises out of the app "
"(httpx.ReadTimeout was caught by the handler net)",
r.status_code != 500)
# --- 3. MONEY: a resolvable purchase credits exactly once. ---
m.fetch_members = fetch_members_ok
r = await c.post(url, json=PAYLOAD)
body = r.json()
check(f"grant: a resolvable credit pack grants credits (got {body})",
r.status_code == 200 and body.get("status") == "credits_added"
and body.get("credits_added") == 50.0)
check("grant: the balance reflects the purchase",
m.get_credit_balance(MEMBER_EMAIL) == 50.0)
ref = m.credit_pack_reference(MEMBER_SLUG, {"idV2": ORDER_ID})
check("grant: the ledger row is keyed on the OC order (idempotence key)",
ref == f"oc:{MEMBER_SLUG}:{ORDER_ID}" and m.credit_reference_exists(ref))
# --- 4. Re-delivery moves no money. ---
r = await c.post(url, json=PAYLOAD)
body = r.json()
check(f"redelivery: the identical payload answers already_credited (got {body})",
r.status_code == 200 and body.get("status") == "already_credited"
and body.get("credits_added") == 0.0)
check("redelivery: the balance is unchanged",
m.get_credit_balance(MEMBER_EMAIL) == 50.0)
# --- 5. TRANSPORT: retry-once, then raise. ---
calls = []
class FlakyClient:
def __init__(self, fail_times):
self.fail_times = fail_times
async def post(self, *a, **k):
calls.append(1)
if len(calls) <= self.fail_times:
raise httpx.ReadTimeout("simulated timeout")
return "RESPONSE"
sentinel = await m._oc_graphql_post(FlakyClient(1), {"query": "x"})
check(f"transport: one transient failure is retried and succeeds (got {sentinel!r})",
sentinel == "RESPONSE" and len(calls) == 2)
calls.clear()
err = None
try:
await m._oc_graphql_post(FlakyClient(2), {"query": "x"})
except Exception as e: # noqa: BLE001
err = e
check("transport: a persistent failure still RAISES to the caller",
isinstance(err, httpx.ReadTimeout))
# --- 6. The client used for the member walk has a real timeout. ---
src = open(os.path.join(os.path.dirname(os.path.dirname(
os.path.abspath(__file__))), "app", "main.py"), encoding="utf-8").read()
check("timeout: fetch_all_member_nodes does NOT use httpx's 5s default",
"async with httpx.AsyncClient(timeout=30.0) as client:" in src)
asyncio.run(main())
fails = sum(not ok for _, ok in results)
for name, ok in results:
print(f" [{'PASS' if ok else 'FAIL'}] {name}")
print(f"\n{len(results) - fails}/{len(results)} passed")
sys.exit(1 if fails else 0)