webhook failures: the portal records and reports its own, not a scraper on my host
ci/woodpecker/push/woodpecker Pipeline was successful

The alert for a failed Open Collective delivery lived in portal_webhook_watch.py on
the operator's machine, polling this app's log through the Cloudron API. That put a
money-critical signal outside the system, and on a log buffer with only hours of
memory — during the 2026-11-13 investigation whole hours (06:00-13:00, 15:00) were
simply absent, so 'no failed delivery in the log' could not be distinguished from
'no delivery at all'.

The portal handles every delivery, so the portal now knows: a new webhook_failures
table records every non-2xx (including the 401 that means the URL and WEBHOOK_TOKEN
disagree, whose event type is read from a body cached before the token check), and
webhook_failure_report() in the sweep emails info@ ONCE per new batch via a marker
in the new portal_state table, so a wrong token does not mail every sweep while the
rows stay queryable. The delivery path is unchanged: bookkeeping failures are
logged, never raised, and the response stays 401/503.

19 checks in test_webhook_resilience.py; all 18 suites green.
This commit is contained in:
inference-admin committed 2026-11-13 20:23:33 +00:00
1 parent 6c82d7d689
commit 2e40387245
2 files changed
+215 -1

No files matched your search

+174 -1
View File
@@ -1123,6 +1123,31 @@ def get_db() -> sqlite3.Connection:
"resolved_at TEXT"
")"
)
# Deliveries the portal did not accept. Recorded BY THE PORTAL because it
# handles every delivery, so noticing a failed one is the system's job and
# not a scraper's: the alert used to live on the operator's own machine,
# reading this app's log through the API — a money-critical signal depending
# on an outside host, and on a buffer with hours of memory (whole hours were
# missing from it when a $10 purchase was investigated on 2026-11-13). The
# sweep summarises new rows here and emails info@ when there are any.
conn.execute(
"CREATE TABLE IF NOT EXISTS webhook_failures ("
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
"at TEXT NOT NULL, "
"status INTEGER NOT NULL, "
"event_type TEXT, "
"detail TEXT"
")"
)
# Small key/value store for the portal's own bookkeeping (e.g. "the last
# webhook failure id we have already reported"). Deliberately not a settings
# table: nothing here is configuration.
conn.execute(
"CREATE TABLE IF NOT EXISTS portal_state ("
"key TEXT PRIMARY KEY, "
"value TEXT"
")"
)
# Unique oc_reference (audit H1, 2026-11-05): an OC re-delivery must never
# create a second hold (or, via the webhook grant path's exists-check, a
# second credit) for the same purchase. Holds recorded before the fix used
@@ -3737,6 +3762,131 @@ async def broker_revoke_api_key(email: str, name: str) -> None:
# --- A. Open Collective webhook ---
def _state_get(conn, key: str, default: str = "") -> str:
try:
row = conn.execute("SELECT value FROM portal_state WHERE key = ?", (key,)).fetchone()
return (row[0] if row and row[0] is not None else default)
except sqlite3.Error:
return default
def _state_set(conn, key: str, value: str) -> None:
conn.execute(
"INSERT INTO portal_state (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value", (key, str(value)))
def record_webhook_failure(status: int, event_type: str = "", detail: str = "") -> None:
"""Record a delivery the portal did not accept — the SYSTEM owns noticing these.
WHY THIS EXISTS: the alert for a failed Open Collective delivery used to live
in a script on the operator's own machine, polling this app's log through the
Cloudron API. That made a money-critical signal depend on a host outside the
system, and on a log buffer with only hours of memory — when the 2026-11-13
purchases were investigated the buffer was missing whole hours (06:00–13:00
and 15:00), so "no failed delivery in the log" could not be told apart from
"no delivery at all".
The portal handles every delivery; it should be the thing that knows, records
and reports a failure. Rows are summarised by `webhook_failure_report()` in the
sweep, which emails info@ once per new batch. Capped at 200 rows — this is a
recent-history record, not an audit log (the credit ledger and the hold table
are the audit trails).
"""
try:
conn = get_db()
conn.execute(
"INSERT INTO webhook_failures (at, status, event_type, detail) VALUES (?,?,?,?)",
(datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"), int(status),
(event_type or "")[:64], (detail or "")[:400]))
conn.execute("DELETE FROM webhook_failures WHERE id NOT IN "
"(SELECT id FROM webhook_failures ORDER BY id DESC LIMIT 200)")
conn.commit()
conn.close()
except Exception as e: # noqa: BLE001
# Never let bookkeeping break the delivery path.
logger.warning("webhook failure record failed: %s", e)
def _peek_event_type(request) -> str:
"""Best-effort event type from an already-read body, for the failure record only."""
try:
raw = getattr(request, "_body", None) or b""
if not raw:
return ""
data = json.loads(raw.decode("utf-8", "replace"))
return str(data.get("type") or "")[:64]
except Exception: # noqa: BLE001
return ""
WEBHOOK_FAILURE_MARKER = "webhook_failures_reported_id"
def webhook_failure_report(notify: bool = True) -> dict:
"""Summarise delivery failures since the last report; email info@ once per batch.
Runs in the sweep, so the notice is the system's, not a scraper's: the same
process that refused a delivery is the one that says so. A delivery that never
arrives at all is a different failure and is covered where it matters — the
credit-pack reconciliation reads OC's own orders, and the membership sweep
reconciles the roster.
Reported ONCE per batch: the marker advances to the newest row id, so a
persistent misconfiguration (a wrong WEBHOOK_TOKEN) does not mail info@ every
sweep, while the rows stay queryable.
"""
rows: list[dict] = []
try:
conn = get_db()
last_id = int(_state_get(conn, WEBHOOK_FAILURE_MARKER, "0") or "0")
rows = [
{"id": r[0], "at": r[1], "status": r[2], "event_type": r[3], "detail": r[4]}
for r in conn.execute(
"SELECT id, at, status, event_type, detail FROM webhook_failures "
"WHERE id > ? ORDER BY id", (last_id,))]
if rows:
_state_set(conn, WEBHOOK_FAILURE_MARKER, rows[-1]["id"])
conn.commit()
conn.close()
except Exception as e: # noqa: BLE001
logger.warning("webhook failure report failed: %s", e)
return {"status": "error", "detail": str(e)[:200]}
if not rows:
return {"status": "ok", "new_failures": 0}
lines = [
f" {r['at']} HTTP {r['status']} "
f"{r['event_type'] or '(event type unknown)'}"
+ (f" — {r['detail']}" if r["detail"] else "")
for r in rows
]
logger.error("OC webhook: %d delivery(ies) were not accepted since the last report:\n%s",
len(rows), "\n".join(lines))
if notify:
body = (
"The portal did not accept these Open Collective deliveries:\n\n"
+ "\n".join(lines)
+ "\n\nA 401/403/404 means the webhook URL and WEBHOOK_TOKEN disagree — the "
"delivery never reached the handler. Any other status means we took it and "
"then errored; if the payload was an order, confirm the member was credited. "
"The sweep's credit-pack reconciliation reads Open Collective's own orders "
"and grants anything missing, so a failed purchase repairs itself; this notice "
"is for the failures it cannot see.\n"
)
try:
send_email("info@inference.coop",
f"Inference Cooperative — {len(rows)} Open Collective "
f"delivery(ies) failed",
body)
except Exception as e: # noqa: BLE001
logger.warning("webhook failure notice email failed: %s", e)
return {"status": "reported_no_email", "new_failures": len(rows),
"failures": rows, "email_error": str(e)[:120]}
return {"status": "reported", "new_failures": len(rows), "failures": rows}
@app.post("/webhook/opencollective/{token}")
@limiter.limit("30/minute")
async def opencollective_webhook(request: Request, token: str):
@@ -3756,11 +3906,25 @@ async def opencollective_webhook(request: Request, token: str):
401, because that is the signal that the webhook URL and WEBHOOK_TOKEN
disagree.
"""
try:
# Cache the body BEFORE delegating: Starlette buffers it on first read, so
# this costs nothing, and it means the failure record can name the event
# type even on the 401 path, where the token check rejects before the
# handler ever looks at the payload.
await request.body()
except Exception: # noqa: BLE001
pass
try:
return await _process_opencollective_webhook(request, token)
except HTTPException:
except HTTPException as e:
# A 401/403/404 means the delivery never reached the handler: the webhook
# URL and WEBHOOK_TOKEN disagree. Still a 401 to OC, but now the SYSTEM
# records it, so a misconfigured webhook stops being invisible.
if e.status_code in (401, 403, 404):
record_webhook_failure(e.status_code, _peek_event_type(request), str(e.detail))
raise
except Exception as e: # noqa: BLE001
record_webhook_failure(503, _peek_event_type(request), f"{type(e).__name__}: {e}")
logger.exception(
"OC webhook: handler failed (%s: %s) — answering 503 so OC retries; "
"if the payload was an order, confirm the member was credited",
@@ -8187,6 +8351,15 @@ async def admin_sweep(request: Request):
except Exception as exc: # noqa: BLE001
result["credit_reconcile"] = {"error": str(exc)[:200]}
# Deliveries the portal refused. The portal records these itself (see
# record_webhook_failure) and reports them here ONCE per batch, so the alert
# for a failed delivery belongs to the system rather than to a log scraper on
# somebody's laptop. Wrapped so it can never break reconciliation.
try:
result["webhook_failures"] = webhook_failure_report()
except Exception as exc: # noqa: BLE001
result["webhook_failures"] = {"error": str(exc)[:200]}
return result
+41
View File
@@ -80,6 +80,13 @@ def check(name, ok):
results.append((name, ok))
SENT = []
def fake_send(to, subject, body, *a, **k):
SENT.append({"to": to, "subject": subject, "body": body})
# --- stubs ----------------------------------------------------------------- #
async def fetch_members_ok():
return [{"email": MEMBER_EMAIL, "name": "NightRose", "slug": MEMBER_SLUG,
@@ -129,6 +136,40 @@ async def main():
"(httpx.ReadTimeout was caught by the handler net)",
r.status_code != 500)
# --- 2b. The SYSTEM records the failures it refuses. ---
# The money-critical alert used to be a log scraper on the operator's own
# machine, reading a buffer with hours of memory — whole hours were missing
# from it during the 2026-11-13 investigation. The portal handles every
# delivery, so the portal is what must notice and say.
conn = m.get_db()
rows = [dict(zip(("id", "at", "status", "event_type", "detail"), r)) for r in
conn.execute("SELECT id, at, status, event_type, detail FROM webhook_failures "
"ORDER BY id")]
conn.close()
check(f"record: the refused 401 delivery is recorded by the portal (got {rows})",
any(r["status"] == 401 for r in rows))
check(f"record: the 503 is recorded with its event type (got {rows})",
any(r["status"] == 503 and r["event_type"] == "order.processed" for r in rows))
check("record: the 503 row names the exception that caused it",
any(r["status"] == 503 and "ReadTimeout" in (r["detail"] or "") for r in rows))
SENT.clear()
m.send_email = fake_send
rep = m.webhook_failure_report()
check(f"report: new failures are summarised (got {rep.get('new_failures')} rows)",
rep.get("status") == "reported" and rep.get("new_failures") == len(rows))
check(f"report: info@ is told, naming both kinds of failure (got {SENT})",
len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop"
and "401" in SENT[0]["body"] and "503" in SENT[0]["body"]
and "WEBHOOK_TOKEN" in SENT[0]["body"])
rep2 = m.webhook_failure_report()
check(f"report: the SAME batch is not re-reported (got {rep2})",
rep2.get("new_failures") == 0 and len(SENT) == 1)
await c.post("/webhook/opencollective/wrong-token", json=PAYLOAD)
rep3 = m.webhook_failure_report()
check(f"report: a LATER failure is reported again (got {rep3.get('new_failures')})",
rep3.get("new_failures") == 1 and len(SENT) == 2)
# --- 3. MONEY: a resolvable purchase credits exactly once. ---
m.fetch_members = fetch_members_ok
r = await c.post(url, json=PAYLOAD)