From 2e4038724588650ac81e566247c53651a9b5632a Mon Sep 17 00:00:00 2001 From: Inferencebot Date: Fri, 13 Nov 2026 20:23:33 +0000 Subject: [PATCH] webhook failures: the portal records and reports its own, not a scraper on my host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The alert for a failed Open Collective delivery lived in portal_webhook_watch.py on the operator's machine, polling this app's log through the Cloudron API. That put a money-critical signal outside the system, and on a log buffer with only hours of memory — during the 2026-11-13 investigation whole hours (06:00-13:00, 15:00) were simply absent, so 'no failed delivery in the log' could not be distinguished from 'no delivery at all'. The portal handles every delivery, so the portal now knows: a new webhook_failures table records every non-2xx (including the 401 that means the URL and WEBHOOK_TOKEN disagree, whose event type is read from a body cached before the token check), and webhook_failure_report() in the sweep emails info@ ONCE per new batch via a marker in the new portal_state table, so a wrong token does not mail every sweep while the rows stay queryable. The delivery path is unchanged: bookkeeping failures are logged, never raised, and the response stays 401/503. 19 checks in test_webhook_resilience.py; all 18 suites green. --- app/main.py | 175 ++++++++++++++++++++++++++++++- tests/test_webhook_resilience.py | 41 ++++++++ 2 files changed, 215 insertions(+), 1 deletion(-) diff --git a/app/main.py b/app/main.py index d605a8d..0417cb9 100644 --- a/app/main.py +++ b/app/main.py @@ -1123,6 +1123,31 @@ def get_db() -> sqlite3.Connection: "resolved_at TEXT" ")" ) + # Deliveries the portal did not accept. Recorded BY THE PORTAL because it + # handles every delivery, so noticing a failed one is the system's job and + # not a scraper's: the alert used to live on the operator's own machine, + # reading this app's log through the API — a money-critical signal depending + # on an outside host, and on a buffer with hours of memory (whole hours were + # missing from it when a $10 purchase was investigated on 2026-11-13). The + # sweep summarises new rows here and emails info@ when there are any. + conn.execute( + "CREATE TABLE IF NOT EXISTS webhook_failures (" + "id INTEGER PRIMARY KEY AUTOINCREMENT, " + "at TEXT NOT NULL, " + "status INTEGER NOT NULL, " + "event_type TEXT, " + "detail TEXT" + ")" + ) + # Small key/value store for the portal's own bookkeeping (e.g. "the last + # webhook failure id we have already reported"). Deliberately not a settings + # table: nothing here is configuration. + conn.execute( + "CREATE TABLE IF NOT EXISTS portal_state (" + "key TEXT PRIMARY KEY, " + "value TEXT" + ")" + ) # Unique oc_reference (audit H1, 2026-11-05): an OC re-delivery must never # create a second hold (or, via the webhook grant path's exists-check, a # second credit) for the same purchase. Holds recorded before the fix used @@ -3737,6 +3762,131 @@ async def broker_revoke_api_key(email: str, name: str) -> None: # --- A. Open Collective webhook --- +def _state_get(conn, key: str, default: str = "") -> str: + try: + row = conn.execute("SELECT value FROM portal_state WHERE key = ?", (key,)).fetchone() + return (row[0] if row and row[0] is not None else default) + except sqlite3.Error: + return default + + +def _state_set(conn, key: str, value: str) -> None: + conn.execute( + "INSERT INTO portal_state (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", (key, str(value))) + + +def record_webhook_failure(status: int, event_type: str = "", detail: str = "") -> None: + """Record a delivery the portal did not accept — the SYSTEM owns noticing these. + + WHY THIS EXISTS: the alert for a failed Open Collective delivery used to live + in a script on the operator's own machine, polling this app's log through the + Cloudron API. That made a money-critical signal depend on a host outside the + system, and on a log buffer with only hours of memory — when the 2026-11-13 + purchases were investigated the buffer was missing whole hours (06:00–13:00 + and 15:00), so "no failed delivery in the log" could not be told apart from + "no delivery at all". + + The portal handles every delivery; it should be the thing that knows, records + and reports a failure. Rows are summarised by `webhook_failure_report()` in the + sweep, which emails info@ once per new batch. Capped at 200 rows — this is a + recent-history record, not an audit log (the credit ledger and the hold table + are the audit trails). + """ + try: + conn = get_db() + conn.execute( + "INSERT INTO webhook_failures (at, status, event_type, detail) VALUES (?,?,?,?)", + (datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"), int(status), + (event_type or "")[:64], (detail or "")[:400])) + conn.execute("DELETE FROM webhook_failures WHERE id NOT IN " + "(SELECT id FROM webhook_failures ORDER BY id DESC LIMIT 200)") + conn.commit() + conn.close() + except Exception as e: # noqa: BLE001 + # Never let bookkeeping break the delivery path. + logger.warning("webhook failure record failed: %s", e) + + +def _peek_event_type(request) -> str: + """Best-effort event type from an already-read body, for the failure record only.""" + try: + raw = getattr(request, "_body", None) or b"" + if not raw: + return "" + data = json.loads(raw.decode("utf-8", "replace")) + return str(data.get("type") or "")[:64] + except Exception: # noqa: BLE001 + return "" + + +WEBHOOK_FAILURE_MARKER = "webhook_failures_reported_id" + + +def webhook_failure_report(notify: bool = True) -> dict: + """Summarise delivery failures since the last report; email info@ once per batch. + + Runs in the sweep, so the notice is the system's, not a scraper's: the same + process that refused a delivery is the one that says so. A delivery that never + arrives at all is a different failure and is covered where it matters — the + credit-pack reconciliation reads OC's own orders, and the membership sweep + reconciles the roster. + + Reported ONCE per batch: the marker advances to the newest row id, so a + persistent misconfiguration (a wrong WEBHOOK_TOKEN) does not mail info@ every + sweep, while the rows stay queryable. + """ + rows: list[dict] = [] + try: + conn = get_db() + last_id = int(_state_get(conn, WEBHOOK_FAILURE_MARKER, "0") or "0") + rows = [ + {"id": r[0], "at": r[1], "status": r[2], "event_type": r[3], "detail": r[4]} + for r in conn.execute( + "SELECT id, at, status, event_type, detail FROM webhook_failures " + "WHERE id > ? ORDER BY id", (last_id,))] + if rows: + _state_set(conn, WEBHOOK_FAILURE_MARKER, rows[-1]["id"]) + conn.commit() + conn.close() + except Exception as e: # noqa: BLE001 + logger.warning("webhook failure report failed: %s", e) + return {"status": "error", "detail": str(e)[:200]} + + if not rows: + return {"status": "ok", "new_failures": 0} + + lines = [ + f" {r['at']} HTTP {r['status']} " + f"{r['event_type'] or '(event type unknown)'}" + + (f" — {r['detail']}" if r["detail"] else "") + for r in rows + ] + logger.error("OC webhook: %d delivery(ies) were not accepted since the last report:\n%s", + len(rows), "\n".join(lines)) + if notify: + body = ( + "The portal did not accept these Open Collective deliveries:\n\n" + + "\n".join(lines) + + "\n\nA 401/403/404 means the webhook URL and WEBHOOK_TOKEN disagree — the " + "delivery never reached the handler. Any other status means we took it and " + "then errored; if the payload was an order, confirm the member was credited. " + "The sweep's credit-pack reconciliation reads Open Collective's own orders " + "and grants anything missing, so a failed purchase repairs itself; this notice " + "is for the failures it cannot see.\n" + ) + try: + send_email("info@inference.coop", + f"Inference Cooperative — {len(rows)} Open Collective " + f"delivery(ies) failed", + body) + except Exception as e: # noqa: BLE001 + logger.warning("webhook failure notice email failed: %s", e) + return {"status": "reported_no_email", "new_failures": len(rows), + "failures": rows, "email_error": str(e)[:120]} + return {"status": "reported", "new_failures": len(rows), "failures": rows} + + @app.post("/webhook/opencollective/{token}") @limiter.limit("30/minute") async def opencollective_webhook(request: Request, token: str): @@ -3756,11 +3906,25 @@ async def opencollective_webhook(request: Request, token: str): 401, because that is the signal that the webhook URL and WEBHOOK_TOKEN disagree. """ + try: + # Cache the body BEFORE delegating: Starlette buffers it on first read, so + # this costs nothing, and it means the failure record can name the event + # type even on the 401 path, where the token check rejects before the + # handler ever looks at the payload. + await request.body() + except Exception: # noqa: BLE001 + pass try: return await _process_opencollective_webhook(request, token) - except HTTPException: + except HTTPException as e: + # A 401/403/404 means the delivery never reached the handler: the webhook + # URL and WEBHOOK_TOKEN disagree. Still a 401 to OC, but now the SYSTEM + # records it, so a misconfigured webhook stops being invisible. + if e.status_code in (401, 403, 404): + record_webhook_failure(e.status_code, _peek_event_type(request), str(e.detail)) raise except Exception as e: # noqa: BLE001 + record_webhook_failure(503, _peek_event_type(request), f"{type(e).__name__}: {e}") logger.exception( "OC webhook: handler failed (%s: %s) — answering 503 so OC retries; " "if the payload was an order, confirm the member was credited", @@ -8187,6 +8351,15 @@ async def admin_sweep(request: Request): except Exception as exc: # noqa: BLE001 result["credit_reconcile"] = {"error": str(exc)[:200]} + # Deliveries the portal refused. The portal records these itself (see + # record_webhook_failure) and reports them here ONCE per batch, so the alert + # for a failed delivery belongs to the system rather than to a log scraper on + # somebody's laptop. Wrapped so it can never break reconciliation. + try: + result["webhook_failures"] = webhook_failure_report() + except Exception as exc: # noqa: BLE001 + result["webhook_failures"] = {"error": str(exc)[:200]} + return result diff --git a/tests/test_webhook_resilience.py b/tests/test_webhook_resilience.py index 24a53a9..80f27c7 100644 --- a/tests/test_webhook_resilience.py +++ b/tests/test_webhook_resilience.py @@ -80,6 +80,13 @@ def check(name, ok): results.append((name, ok)) +SENT = [] + + +def fake_send(to, subject, body, *a, **k): + SENT.append({"to": to, "subject": subject, "body": body}) + + # --- stubs ----------------------------------------------------------------- # async def fetch_members_ok(): return [{"email": MEMBER_EMAIL, "name": "NightRose", "slug": MEMBER_SLUG, @@ -129,6 +136,40 @@ async def main(): "(httpx.ReadTimeout was caught by the handler net)", r.status_code != 500) + # --- 2b. The SYSTEM records the failures it refuses. --- + # The money-critical alert used to be a log scraper on the operator's own + # machine, reading a buffer with hours of memory — whole hours were missing + # from it during the 2026-11-13 investigation. The portal handles every + # delivery, so the portal is what must notice and say. + conn = m.get_db() + rows = [dict(zip(("id", "at", "status", "event_type", "detail"), r)) for r in + conn.execute("SELECT id, at, status, event_type, detail FROM webhook_failures " + "ORDER BY id")] + conn.close() + check(f"record: the refused 401 delivery is recorded by the portal (got {rows})", + any(r["status"] == 401 for r in rows)) + check(f"record: the 503 is recorded with its event type (got {rows})", + any(r["status"] == 503 and r["event_type"] == "order.processed" for r in rows)) + check("record: the 503 row names the exception that caused it", + any(r["status"] == 503 and "ReadTimeout" in (r["detail"] or "") for r in rows)) + + SENT.clear() + m.send_email = fake_send + rep = m.webhook_failure_report() + check(f"report: new failures are summarised (got {rep.get('new_failures')} rows)", + rep.get("status") == "reported" and rep.get("new_failures") == len(rows)) + check(f"report: info@ is told, naming both kinds of failure (got {SENT})", + len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop" + and "401" in SENT[0]["body"] and "503" in SENT[0]["body"] + and "WEBHOOK_TOKEN" in SENT[0]["body"]) + rep2 = m.webhook_failure_report() + check(f"report: the SAME batch is not re-reported (got {rep2})", + rep2.get("new_failures") == 0 and len(SENT) == 1) + await c.post("/webhook/opencollective/wrong-token", json=PAYLOAD) + rep3 = m.webhook_failure_report() + check(f"report: a LATER failure is reported again (got {rep3.get('new_failures')})", + rep3.get("new_failures") == 1 and len(SENT) == 2) + # --- 3. MONEY: a resolvable purchase credits exactly once. --- m.fetch_members = fetch_members_ok r = await c.post(url, json=PAYLOAD)