Files
member-portal/tests/test_webhook_resilience.py
T
inference-admin 2e40387245
ci/woodpecker/push/woodpecker Pipeline was successful
webhook failures: the portal records and reports its own, not a scraper on my host
The alert for a failed Open Collective delivery lived in portal_webhook_watch.py on
the operator's machine, polling this app's log through the Cloudron API. That put a
money-critical signal outside the system, and on a log buffer with only hours of
memory — during the 2026-11-13 investigation whole hours (06:00-13:00, 15:00) were
simply absent, so 'no failed delivery in the log' could not be distinguished from
'no delivery at all'.

The portal handles every delivery, so the portal now knows: a new webhook_failures
table records every non-2xx (including the 401 that means the URL and WEBHOOK_TOKEN
disagree, whose event type is read from a body cached before the token check), and
webhook_failure_report() in the sweep emails info@ ONCE per new batch via a marker
in the new portal_state table, so a wrong token does not mail every sweep while the
rows stay queryable. The delivery path is unchanged: bookkeeping failures are
logged, never raised, and the response stays 401/503.

19 checks in test_webhook_resilience.py; all 18 suites green.
2026-11-13 20:23:33 +00:00

235 lines
9.7 KiB
Python

#!/usr/bin/env python3
"""Regression test: an Open Collective webhook must never fail as a bare 500.
WHY THIS EXISTS
On 2026-11-13 a member's $50 credit-pack purchase arrived as an
`order.processed` delivery. The handler called fetch_members() to turn the OC
slug into an email, `fetch_all_member_nodes` used a bare httpx.AsyncClient()
(httpx's DEFAULT 5-second timeout), OC's GraphQL answered slower than that,
and `httpx.ReadTimeout` escaped the handler as an unhandled ASGI exception:
* the delivery returned 500, so nothing retried it usefully,
* no credits were granted — the member paid and got nothing for 40 minutes,
* the log carried a stack trace instead of anything naming the member or
the order, so the purchase could not be reconciled from the log alone.
The guards proven here, on a FIXTURE db with the network stubbed:
AUTH
1. A wrong URL token is still a 401 — the new error net must not swallow
the one status that means "the webhook URL and WEBHOOK_TOKEN disagree".
NO BARE 500
2. A transport failure while resolving the purchase answers 503 (retryable)
and the handler does not raise out of the ASGI app.
3. The same is true for a failure raised anywhere else in the handler.
MONEY
4. A resolvable purchase grants credits ONCE, with the per-order reference
`oc:<slug>:<idV2>`.
5. Re-delivering the identical payload returns `already_credited` and moves
no money — this is what makes OC's retries safe.
TRANSPORT
6. `_oc_graphql_post` retries once on a transport error and returns the
response; if both attempts fail it RAISES (callers keep their
fail-safes — a partial member list must never look complete).
Runnable: python3 tests/test_webhook_resilience.py (from the repo root)
"""
import asyncio
import os
import sys
import tempfile
import httpx
os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db")
os.environ["WEBHOOK_TOKEN"] = "test-webhook-token"
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import app.main as m # noqa: E402
# The app module shares this httpx object, so stubbing it later would replace
# httpx.AsyncClient process-wide — keep a handle on the real class.
REAL_ASYNC_CLIENT = httpx.AsyncClient
m.DB_PATH = os.environ["DB_PATH"]
TOKEN = "test-webhook-token"
MEMBER_SLUG = "nightrose"
MEMBER_EMAIL = "nightrose@example.org"
ORDER_ID = "or_testOrderId123"
PAYLOAD = {
"type": "order.processed",
"data": {
"fromCollective": {"slug": MEMBER_SLUG, "name": "NightRose"},
"order": {
"idV2": ORDER_ID,
"description": "Financial contribution to Inference Cooperative (Credit pack)",
"totalAmount": 5000,
},
"amount": 5000,
"firstPayment": False,
},
}
results = []
def check(name, ok):
results.append((name, ok))
SENT = []
def fake_send(to, subject, body, *a, **k):
SENT.append({"to": to, "subject": subject, "body": body})
# --- stubs ----------------------------------------------------------------- #
async def fetch_members_ok():
return [{"email": MEMBER_EMAIL, "name": "NightRose", "slug": MEMBER_SLUG,
"role": "BACKER"}]
async def fetch_members_timeout():
raise httpx.ReadTimeout("simulated OC read timeout")
async def order_amount_ok(_order_id_v2=None):
return 5000
async def budget_noop(_email):
return 0.0
async def main():
m.get_db().close() # create + migrate the fixture schema
conn = m.get_db()
conn.execute("INSERT OR REPLACE INTO members (email, slug, active, credit_balance) "
"VALUES (?, ?, 1, 0)", (MEMBER_EMAIL, MEMBER_SLUG))
conn.commit()
conn.close()
transport = httpx.ASGITransport(app=m.app)
url = f"/webhook/opencollective/{TOKEN}"
m.sync_team_budget_from_balance = budget_noop # no LiteLLM calls in a test
m.get_order_amount = order_amount_ok
async with REAL_ASYNC_CLIENT(transport=transport, base_url="http://test") as c:
# --- 1. AUTH: a wrong token is still a 401. ---
r = await c.post("/webhook/opencollective/wrong-token", json=PAYLOAD)
check(f"auth: a wrong webhook token is 401 (got {r.status_code})", r.status_code == 401)
# --- 2. NO BARE 500 when the OC lookup times out. ---
m.fetch_members = fetch_members_timeout
r = await c.post(url, json=PAYLOAD)
body = r.json() if r.headers.get("content-type", "").startswith("application/json") else {}
check(f"timeout: the delivery answers 503, not 500 (got {r.status_code})",
r.status_code == 503)
check(f"timeout: the response asks for a retry (got {body})",
body.get("retry") is True)
check("timeout: an unexpected failure never raises out of the app "
"(httpx.ReadTimeout was caught by the handler net)",
r.status_code != 500)
# --- 2b. The SYSTEM records the failures it refuses. ---
# The money-critical alert used to be a log scraper on the operator's own
# machine, reading a buffer with hours of memory — whole hours were missing
# from it during the 2026-11-13 investigation. The portal handles every
# delivery, so the portal is what must notice and say.
conn = m.get_db()
rows = [dict(zip(("id", "at", "status", "event_type", "detail"), r)) for r in
conn.execute("SELECT id, at, status, event_type, detail FROM webhook_failures "
"ORDER BY id")]
conn.close()
check(f"record: the refused 401 delivery is recorded by the portal (got {rows})",
any(r["status"] == 401 for r in rows))
check(f"record: the 503 is recorded with its event type (got {rows})",
any(r["status"] == 503 and r["event_type"] == "order.processed" for r in rows))
check("record: the 503 row names the exception that caused it",
any(r["status"] == 503 and "ReadTimeout" in (r["detail"] or "") for r in rows))
SENT.clear()
m.send_email = fake_send
rep = m.webhook_failure_report()
check(f"report: new failures are summarised (got {rep.get('new_failures')} rows)",
rep.get("status") == "reported" and rep.get("new_failures") == len(rows))
check(f"report: info@ is told, naming both kinds of failure (got {SENT})",
len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop"
and "401" in SENT[0]["body"] and "503" in SENT[0]["body"]
and "WEBHOOK_TOKEN" in SENT[0]["body"])
rep2 = m.webhook_failure_report()
check(f"report: the SAME batch is not re-reported (got {rep2})",
rep2.get("new_failures") == 0 and len(SENT) == 1)
await c.post("/webhook/opencollective/wrong-token", json=PAYLOAD)
rep3 = m.webhook_failure_report()
check(f"report: a LATER failure is reported again (got {rep3.get('new_failures')})",
rep3.get("new_failures") == 1 and len(SENT) == 2)
# --- 3. MONEY: a resolvable purchase credits exactly once. ---
m.fetch_members = fetch_members_ok
r = await c.post(url, json=PAYLOAD)
body = r.json()
check(f"grant: a resolvable credit pack grants credits (got {body})",
r.status_code == 200 and body.get("status") == "credits_added"
and body.get("credits_added") == 50.0)
check("grant: the balance reflects the purchase",
m.get_credit_balance(MEMBER_EMAIL) == 50.0)
ref = m.credit_pack_reference(MEMBER_SLUG, {"idV2": ORDER_ID})
check("grant: the ledger row is keyed on the OC order (idempotence key)",
ref == f"oc:{MEMBER_SLUG}:{ORDER_ID}" and m.credit_reference_exists(ref))
# --- 4. Re-delivery moves no money. ---
r = await c.post(url, json=PAYLOAD)
body = r.json()
check(f"redelivery: the identical payload answers already_credited (got {body})",
r.status_code == 200 and body.get("status") == "already_credited"
and body.get("credits_added") == 0.0)
check("redelivery: the balance is unchanged",
m.get_credit_balance(MEMBER_EMAIL) == 50.0)
# --- 5. TRANSPORT: retry-once, then raise. ---
calls = []
class FlakyClient:
def __init__(self, fail_times):
self.fail_times = fail_times
async def post(self, *a, **k):
calls.append(1)
if len(calls) <= self.fail_times:
raise httpx.ReadTimeout("simulated timeout")
return "RESPONSE"
sentinel = await m._oc_graphql_post(FlakyClient(1), {"query": "x"})
check(f"transport: one transient failure is retried and succeeds (got {sentinel!r})",
sentinel == "RESPONSE" and len(calls) == 2)
calls.clear()
err = None
try:
await m._oc_graphql_post(FlakyClient(2), {"query": "x"})
except Exception as e: # noqa: BLE001
err = e
check("transport: a persistent failure still RAISES to the caller",
isinstance(err, httpx.ReadTimeout))
# --- 6. The client used for the member walk has a real timeout. ---
src = open(os.path.join(os.path.dirname(os.path.dirname(
os.path.abspath(__file__))), "app", "main.py"), encoding="utf-8").read()
check("timeout: fetch_all_member_nodes does NOT use httpx's 5s default",
"async with httpx.AsyncClient(timeout=30.0) as client:" in src)
asyncio.run(main())
fails = sum(not ok for _, ok in results)
for name, ok in results:
print(f" [{'PASS' if ok else 'FAIL'}] {name}")
print(f"\n{len(results) - fails}/{len(results)} passed")
sys.exit(1 if fails else 0)