webhook failures: the portal records and reports its own, not a scraper on my host
ci/woodpecker/push/woodpecker Pipeline was successful
ci/woodpecker/push/woodpecker Pipeline was successful
The alert for a failed Open Collective delivery lived in portal_webhook_watch.py on the operator's machine, polling this app's log through the Cloudron API. That put a money-critical signal outside the system, and on a log buffer with only hours of memory — during the 2026-11-13 investigation whole hours (06:00-13:00, 15:00) were simply absent, so 'no failed delivery in the log' could not be distinguished from 'no delivery at all'. The portal handles every delivery, so the portal now knows: a new webhook_failures table records every non-2xx (including the 401 that means the URL and WEBHOOK_TOKEN disagree, whose event type is read from a body cached before the token check), and webhook_failure_report() in the sweep emails info@ ONCE per new batch via a marker in the new portal_state table, so a wrong token does not mail every sweep while the rows stay queryable. The delivery path is unchanged: bookkeeping failures are logged, never raised, and the response stays 401/503. 19 checks in test_webhook_resilience.py; all 18 suites green.
This commit is contained in:
1 parent
6c82d7d689
commit
2e40387245
2 files changed
+215
-1
No files matched your search
+174
-1
@@ -1123,6 +1123,31 @@ def get_db() -> sqlite3.Connection:
|
||||
"resolved_at TEXT"
|
||||
")"
|
||||
)
|
||||
# Deliveries the portal did not accept. Recorded BY THE PORTAL because it
|
||||
# handles every delivery, so noticing a failed one is the system's job and
|
||||
# not a scraper's: the alert used to live on the operator's own machine,
|
||||
# reading this app's log through the API — a money-critical signal depending
|
||||
# on an outside host, and on a buffer with hours of memory (whole hours were
|
||||
# missing from it when a $10 purchase was investigated on 2026-11-13). The
|
||||
# sweep summarises new rows here and emails info@ when there are any.
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS webhook_failures ("
|
||||
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
||||
"at TEXT NOT NULL, "
|
||||
"status INTEGER NOT NULL, "
|
||||
"event_type TEXT, "
|
||||
"detail TEXT"
|
||||
")"
|
||||
)
|
||||
# Small key/value store for the portal's own bookkeeping (e.g. "the last
|
||||
# webhook failure id we have already reported"). Deliberately not a settings
|
||||
# table: nothing here is configuration.
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS portal_state ("
|
||||
"key TEXT PRIMARY KEY, "
|
||||
"value TEXT"
|
||||
")"
|
||||
)
|
||||
# Unique oc_reference (audit H1, 2026-11-05): an OC re-delivery must never
|
||||
# create a second hold (or, via the webhook grant path's exists-check, a
|
||||
# second credit) for the same purchase. Holds recorded before the fix used
|
||||
@@ -3737,6 +3762,131 @@ async def broker_revoke_api_key(email: str, name: str) -> None:
|
||||
|
||||
# --- A. Open Collective webhook ---
|
||||
|
||||
def _state_get(conn, key: str, default: str = "") -> str:
|
||||
try:
|
||||
row = conn.execute("SELECT value FROM portal_state WHERE key = ?", (key,)).fetchone()
|
||||
return (row[0] if row and row[0] is not None else default)
|
||||
except sqlite3.Error:
|
||||
return default
|
||||
|
||||
|
||||
def _state_set(conn, key: str, value: str) -> None:
|
||||
conn.execute(
|
||||
"INSERT INTO portal_state (key, value) VALUES (?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value", (key, str(value)))
|
||||
|
||||
|
||||
def record_webhook_failure(status: int, event_type: str = "", detail: str = "") -> None:
|
||||
"""Record a delivery the portal did not accept — the SYSTEM owns noticing these.
|
||||
|
||||
WHY THIS EXISTS: the alert for a failed Open Collective delivery used to live
|
||||
in a script on the operator's own machine, polling this app's log through the
|
||||
Cloudron API. That made a money-critical signal depend on a host outside the
|
||||
system, and on a log buffer with only hours of memory — when the 2026-11-13
|
||||
purchases were investigated the buffer was missing whole hours (06:00–13:00
|
||||
and 15:00), so "no failed delivery in the log" could not be told apart from
|
||||
"no delivery at all".
|
||||
|
||||
The portal handles every delivery; it should be the thing that knows, records
|
||||
and reports a failure. Rows are summarised by `webhook_failure_report()` in the
|
||||
sweep, which emails info@ once per new batch. Capped at 200 rows — this is a
|
||||
recent-history record, not an audit log (the credit ledger and the hold table
|
||||
are the audit trails).
|
||||
"""
|
||||
try:
|
||||
conn = get_db()
|
||||
conn.execute(
|
||||
"INSERT INTO webhook_failures (at, status, event_type, detail) VALUES (?,?,?,?)",
|
||||
(datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"), int(status),
|
||||
(event_type or "")[:64], (detail or "")[:400]))
|
||||
conn.execute("DELETE FROM webhook_failures WHERE id NOT IN "
|
||||
"(SELECT id FROM webhook_failures ORDER BY id DESC LIMIT 200)")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e: # noqa: BLE001
|
||||
# Never let bookkeeping break the delivery path.
|
||||
logger.warning("webhook failure record failed: %s", e)
|
||||
|
||||
|
||||
def _peek_event_type(request) -> str:
|
||||
"""Best-effort event type from an already-read body, for the failure record only."""
|
||||
try:
|
||||
raw = getattr(request, "_body", None) or b""
|
||||
if not raw:
|
||||
return ""
|
||||
data = json.loads(raw.decode("utf-8", "replace"))
|
||||
return str(data.get("type") or "")[:64]
|
||||
except Exception: # noqa: BLE001
|
||||
return ""
|
||||
|
||||
|
||||
WEBHOOK_FAILURE_MARKER = "webhook_failures_reported_id"
|
||||
|
||||
|
||||
def webhook_failure_report(notify: bool = True) -> dict:
|
||||
"""Summarise delivery failures since the last report; email info@ once per batch.
|
||||
|
||||
Runs in the sweep, so the notice is the system's, not a scraper's: the same
|
||||
process that refused a delivery is the one that says so. A delivery that never
|
||||
arrives at all is a different failure and is covered where it matters — the
|
||||
credit-pack reconciliation reads OC's own orders, and the membership sweep
|
||||
reconciles the roster.
|
||||
|
||||
Reported ONCE per batch: the marker advances to the newest row id, so a
|
||||
persistent misconfiguration (a wrong WEBHOOK_TOKEN) does not mail info@ every
|
||||
sweep, while the rows stay queryable.
|
||||
"""
|
||||
rows: list[dict] = []
|
||||
try:
|
||||
conn = get_db()
|
||||
last_id = int(_state_get(conn, WEBHOOK_FAILURE_MARKER, "0") or "0")
|
||||
rows = [
|
||||
{"id": r[0], "at": r[1], "status": r[2], "event_type": r[3], "detail": r[4]}
|
||||
for r in conn.execute(
|
||||
"SELECT id, at, status, event_type, detail FROM webhook_failures "
|
||||
"WHERE id > ? ORDER BY id", (last_id,))]
|
||||
if rows:
|
||||
_state_set(conn, WEBHOOK_FAILURE_MARKER, rows[-1]["id"])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("webhook failure report failed: %s", e)
|
||||
return {"status": "error", "detail": str(e)[:200]}
|
||||
|
||||
if not rows:
|
||||
return {"status": "ok", "new_failures": 0}
|
||||
|
||||
lines = [
|
||||
f" {r['at']} HTTP {r['status']} "
|
||||
f"{r['event_type'] or '(event type unknown)'}"
|
||||
+ (f" — {r['detail']}" if r["detail"] else "")
|
||||
for r in rows
|
||||
]
|
||||
logger.error("OC webhook: %d delivery(ies) were not accepted since the last report:\n%s",
|
||||
len(rows), "\n".join(lines))
|
||||
if notify:
|
||||
body = (
|
||||
"The portal did not accept these Open Collective deliveries:\n\n"
|
||||
+ "\n".join(lines)
|
||||
+ "\n\nA 401/403/404 means the webhook URL and WEBHOOK_TOKEN disagree — the "
|
||||
"delivery never reached the handler. Any other status means we took it and "
|
||||
"then errored; if the payload was an order, confirm the member was credited. "
|
||||
"The sweep's credit-pack reconciliation reads Open Collective's own orders "
|
||||
"and grants anything missing, so a failed purchase repairs itself; this notice "
|
||||
"is for the failures it cannot see.\n"
|
||||
)
|
||||
try:
|
||||
send_email("info@inference.coop",
|
||||
f"Inference Cooperative — {len(rows)} Open Collective "
|
||||
f"delivery(ies) failed",
|
||||
body)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("webhook failure notice email failed: %s", e)
|
||||
return {"status": "reported_no_email", "new_failures": len(rows),
|
||||
"failures": rows, "email_error": str(e)[:120]}
|
||||
return {"status": "reported", "new_failures": len(rows), "failures": rows}
|
||||
|
||||
|
||||
@app.post("/webhook/opencollective/{token}")
|
||||
@limiter.limit("30/minute")
|
||||
async def opencollective_webhook(request: Request, token: str):
|
||||
@@ -3756,11 +3906,25 @@ async def opencollective_webhook(request: Request, token: str):
|
||||
401, because that is the signal that the webhook URL and WEBHOOK_TOKEN
|
||||
disagree.
|
||||
"""
|
||||
try:
|
||||
# Cache the body BEFORE delegating: Starlette buffers it on first read, so
|
||||
# this costs nothing, and it means the failure record can name the event
|
||||
# type even on the 401 path, where the token check rejects before the
|
||||
# handler ever looks at the payload.
|
||||
await request.body()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
try:
|
||||
return await _process_opencollective_webhook(request, token)
|
||||
except HTTPException:
|
||||
except HTTPException as e:
|
||||
# A 401/403/404 means the delivery never reached the handler: the webhook
|
||||
# URL and WEBHOOK_TOKEN disagree. Still a 401 to OC, but now the SYSTEM
|
||||
# records it, so a misconfigured webhook stops being invisible.
|
||||
if e.status_code in (401, 403, 404):
|
||||
record_webhook_failure(e.status_code, _peek_event_type(request), str(e.detail))
|
||||
raise
|
||||
except Exception as e: # noqa: BLE001
|
||||
record_webhook_failure(503, _peek_event_type(request), f"{type(e).__name__}: {e}")
|
||||
logger.exception(
|
||||
"OC webhook: handler failed (%s: %s) — answering 503 so OC retries; "
|
||||
"if the payload was an order, confirm the member was credited",
|
||||
@@ -8187,6 +8351,15 @@ async def admin_sweep(request: Request):
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result["credit_reconcile"] = {"error": str(exc)[:200]}
|
||||
|
||||
# Deliveries the portal refused. The portal records these itself (see
|
||||
# record_webhook_failure) and reports them here ONCE per batch, so the alert
|
||||
# for a failed delivery belongs to the system rather than to a log scraper on
|
||||
# somebody's laptop. Wrapped so it can never break reconciliation.
|
||||
try:
|
||||
result["webhook_failures"] = webhook_failure_report()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result["webhook_failures"] = {"error": str(exc)[:200]}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
|
||||
@@ -80,6 +80,13 @@ def check(name, ok):
|
||||
results.append((name, ok))
|
||||
|
||||
|
||||
SENT = []
|
||||
|
||||
|
||||
def fake_send(to, subject, body, *a, **k):
|
||||
SENT.append({"to": to, "subject": subject, "body": body})
|
||||
|
||||
|
||||
# --- stubs ----------------------------------------------------------------- #
|
||||
async def fetch_members_ok():
|
||||
return [{"email": MEMBER_EMAIL, "name": "NightRose", "slug": MEMBER_SLUG,
|
||||
@@ -129,6 +136,40 @@ async def main():
|
||||
"(httpx.ReadTimeout was caught by the handler net)",
|
||||
r.status_code != 500)
|
||||
|
||||
# --- 2b. The SYSTEM records the failures it refuses. ---
|
||||
# The money-critical alert used to be a log scraper on the operator's own
|
||||
# machine, reading a buffer with hours of memory — whole hours were missing
|
||||
# from it during the 2026-11-13 investigation. The portal handles every
|
||||
# delivery, so the portal is what must notice and say.
|
||||
conn = m.get_db()
|
||||
rows = [dict(zip(("id", "at", "status", "event_type", "detail"), r)) for r in
|
||||
conn.execute("SELECT id, at, status, event_type, detail FROM webhook_failures "
|
||||
"ORDER BY id")]
|
||||
conn.close()
|
||||
check(f"record: the refused 401 delivery is recorded by the portal (got {rows})",
|
||||
any(r["status"] == 401 for r in rows))
|
||||
check(f"record: the 503 is recorded with its event type (got {rows})",
|
||||
any(r["status"] == 503 and r["event_type"] == "order.processed" for r in rows))
|
||||
check("record: the 503 row names the exception that caused it",
|
||||
any(r["status"] == 503 and "ReadTimeout" in (r["detail"] or "") for r in rows))
|
||||
|
||||
SENT.clear()
|
||||
m.send_email = fake_send
|
||||
rep = m.webhook_failure_report()
|
||||
check(f"report: new failures are summarised (got {rep.get('new_failures')} rows)",
|
||||
rep.get("status") == "reported" and rep.get("new_failures") == len(rows))
|
||||
check(f"report: info@ is told, naming both kinds of failure (got {SENT})",
|
||||
len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop"
|
||||
and "401" in SENT[0]["body"] and "503" in SENT[0]["body"]
|
||||
and "WEBHOOK_TOKEN" in SENT[0]["body"])
|
||||
rep2 = m.webhook_failure_report()
|
||||
check(f"report: the SAME batch is not re-reported (got {rep2})",
|
||||
rep2.get("new_failures") == 0 and len(SENT) == 1)
|
||||
await c.post("/webhook/opencollective/wrong-token", json=PAYLOAD)
|
||||
rep3 = m.webhook_failure_report()
|
||||
check(f"report: a LATER failure is reported again (got {rep3.get('new_failures')})",
|
||||
rep3.get("new_failures") == 1 and len(SENT) == 2)
|
||||
|
||||
# --- 3. MONEY: a resolvable purchase credits exactly once. ---
|
||||
m.fetch_members = fetch_members_ok
|
||||
r = await c.post(url, json=PAYLOAD)
|
||||
|
||||
Reference in new issue
Block a user