diff --git a/app/main.py b/app/main.py index d19c6e1..33c6bb3 100644 --- a/app/main.py +++ b/app/main.py @@ -8239,6 +8239,149 @@ async def reconcile_unmapped_backers(notify: bool = True) -> dict: "detail": findings} +# --- Spend-consistency check (was a script on the bot host) ----------------- # +# The failure it exists for (2026-11-05): the portal aggregates all-time spend +# from LiteLLM's Postgres using a COPY of the gateway's DB credential. When the +# gateway rotated that credential the copy went stale, the aggregation failed +# SOFT (empty -> $0), and the panel and dashboard showed $0 while every surface +# looked clean. Nobody would have noticed from inside the portal, because the +# portal WAS the thing that was wrong. +# +# The old check lived on the bot's host and read LiteLLM's own SpendLogs through +# the gateway's credential. This port keeps the two independent reads — one over +# the portal's DB copy, one over LiteLLM's HTTP API with the master key (a +# different transport AND a different credential) — and adds the guard that +# catches the silent-clean signature directly: a $0 (or empty) aggregate while +# LiteLLM's own API reports real spend. +# +# It does NOT compare against the figure /admin/overview computes inline: that +# aggregation is embedded in the handler, and the DB aggregate it is built from is +# what this checks. Extracting it into a shared helper is the next step if the two +# ever need to be compared directly. +SPEND_CHECK_TOLERANCE = float(os.environ.get("SPEND_CHECK_TOLERANCE", "5.00") or "5.00") +SPEND_CHECK_MARKER = "spend_check_reported" + + +async def litellm_api_total_spend() -> tuple[float, int]: + """Total spend from LiteLLM's OWN HTTP API (master key). Raises on failure. + + `/global/spend/keys` returns one entry per API key with its spend; summing them + is the closest HTTP equivalent of COUNT/SUM over the SpendLogs table. A different + credential path from the portal's DB copy — which is the whole point, since the + DB copy is the thing that goes stale. + """ + if not LITELLM_MASTER_KEY: + raise RuntimeError("LITELLM_MASTER_KEY not set; cannot verify against LiteLLM") + async with httpx.AsyncClient(timeout=30.0) as client: + r = await client.get(f"{LITELLM_BASE}/global/spend/keys", + headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"}) + if r.status_code != 200: + raise RuntimeError(f"LiteLLM spend API: HTTP {r.status_code}") + rows = r.json() + if not isinstance(rows, list): + raise RuntimeError(f"LiteLLM spend API returned {type(rows).__name__}, not a list") + total = 0.0 + for row in rows: + try: + total += float((row or {}).get("spend") or 0) + except (TypeError, ValueError): + continue + return round(total, 6), len(rows) + + +async def check_spend_consistency(notify: bool = True) -> dict: + """Compare the portal's SpendLogs aggregate with LiteLLM's own API. Emails only on FAIL. + + Returns status="ok" with a verdict, or "failed" with the reason — and a read that + cannot be made is `skipped`, never a clean bill: "could not look" is an anomaly. + """ + try: + _by_day, portal_total, portal_rows = litellm_spend_by_day() + except Exception as e: # noqa: BLE001 + logger.warning("spend check: portal SpendLogs read failed: %s", e) + return {"status": "skipped", "reason": "portal_read_failed", "detail": str(e)[:200]} + try: + api_total, api_keys = await litellm_api_total_spend() + except Exception as e: # noqa: BLE001 + logger.warning("spend check: LiteLLM API read failed: %s", e) + return {"status": "skipped", "reason": "litellm_api_failed", "detail": str(e)[:200]} + + delta = round(api_total - portal_total, 2) + verdict = { + "portal_total": round(portal_total, 2), + "portal_rows": portal_rows, + "litellm_api_total": round(api_total, 2), + "litellm_api_keys": api_keys, + "delta": delta, + "tolerance": SPEND_CHECK_TOLERANCE, + } + + reason = "" + if portal_rows == 0 and api_total > 1: + # The 2026-11-05 signature, verbatim: our copy of the credential went stale + # and the aggregation collapsed to $0 while LiteLLM's own totals are real. + reason = (f"the portal's SpendLogs aggregate is EMPTY/zero while LiteLLM's own API " + f"reports ${api_total:.2f} — the silent-clean signature of a stale LiteLLM " + f"DB credential (the panel/dashboard $0 is the same failure)") + elif api_keys == 0 and portal_rows > 0: + reason = (f"LiteLLM's spend API returned no keys while the portal read {portal_rows} " + f"SpendLogs rows — the comparison has no independent side") + elif abs(delta) > SPEND_CHECK_TOLERANCE: + reason = (f"portal SpendLogs total ${portal_total:.2f} ({portal_rows} rows) vs LiteLLM " + f"API ${api_total:.2f} over {api_keys} keys: delta ${delta:+.2f} exceeds the " + f"${SPEND_CHECK_TOLERANCE:.2f} tolerance — the two are not reading the same data") + + if not reason: + verdict["status"] = "ok" + logger.info("spend check PASS: portal $%.2f (%d rows) vs LiteLLM API $%.2f (%d keys), " + "delta $%+.2f within $%.2f", portal_total, portal_rows, api_total, + api_keys, delta, SPEND_CHECK_TOLERANCE) + # A clean result clears the marker, so a recurrence is reported again. + try: + conn = get_db() + _state_set(conn, SPEND_CHECK_MARKER, "") + conn.commit() + conn.close() + except Exception as e: # noqa: BLE001 + logger.warning("spend check: could not clear the report marker: %s", e) + return verdict + + verdict["status"] = "failed" + verdict["reason"] = reason + logger.error("spend check FAILED: %s", reason) + + # Reported once per failing episode: the marker holds what a human has been told. + conn = get_db() + reported = (_state_get(conn, SPEND_CHECK_MARKER, "") or "") == reason + conn.close() + verdict["reported"] = False + if notify and not reported: + try: + send_email( + "info@inference.coop", + "Inference Cooperative — spend figures disagree (portal vs LiteLLM)", + "The portal's own spend aggregation and LiteLLM's API do not agree. This " + "check runs on every sweep and is SILENT when they match.\n\n" + f" portal : ${verdict['portal_total']:.2f} over {verdict['portal_rows']} rows\n" + f" LiteLLM : ${verdict['litellm_api_total']:.2f} over " + f"{verdict['litellm_api_keys']} keys\n" + f" delta : ${verdict['delta']:+.2f} (tolerance ${SPEND_CHECK_TOLERANCE:.2f})\n\n" + f"Reason: {reason}\n\n" + "The likely cause is the portal's copy of the LiteLLM DB credential going " + "stale after a rotation: the aggregation then fails soft and every surface " + "shows $0 while looking clean. Check the credential in the portal's env " + "against the gateway app's own.\n", + ) + verdict["reported"] = True + conn = get_db() + _state_set(conn, SPEND_CHECK_MARKER, reason[:400]) + conn.commit() + conn.close() + except Exception as e: # noqa: BLE001 + logger.warning("spend check: notice email failed: %s", e) + return verdict + + async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> dict: """Credit PAID credit packs that never reached our books (issue #7). @@ -8647,6 +8790,15 @@ async def admin_sweep(request: Request): except Exception as exc: # noqa: BLE001 result["unmapped_backers"] = {"error": str(exc)[:200]} + # Spend consistency: the portal's SpendLogs aggregate vs LiteLLM's own API. + # Silent when they agree; the reason it exists is that the portal's copy of the + # LiteLLM DB credential once went stale and every surface showed $0 while looking + # clean. Was a daily script on the bot's host. + try: + result["spend_check"] = await check_spend_consistency() + except Exception as exc: # noqa: BLE001 + result["spend_check"] = {"error": str(exc)[:200]} + # Deliveries the portal refused. The portal records these itself (see # record_webhook_failure) and reports them here ONCE per batch, so the alert # for a failed delivery belongs to the system rather than to a log scraper on @@ -8689,6 +8841,17 @@ async def admin_unmapped_check(request: Request, notify: int = 1): return await reconcile_unmapped_backers(notify=bool(notify)) +@app.post("/admin/spend-check") +@limiter.limit("20/minute") +async def admin_spend_check(request: Request, notify: int = 1): + """Compare the portal's spend aggregate with LiteLLM's own API (X-Admin-Token). + + Silent when they agree; `?notify=0` runs it without any chance of a notice. + """ + _verify_admin_token(request) + return await check_spend_consistency(notify=bool(notify)) + + def _service_sync_pending() -> list[str]: """Active members still waiting on their Loomio invitation. diff --git a/tests/test_spend_check.py b/tests/test_spend_check.py new file mode 100644 index 0000000..90a8e73 --- /dev/null +++ b/tests/test_spend_check.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""Regression test: the spend-consistency check (portal aggregate vs LiteLLM's API). + +WHY THIS EXISTS + On 2026-11-05 the portal's aggregation of all-time spend failed SOFT: its copy of + the gateway's LiteLLM DB credential had gone stale, so the query returned nothing, + every surface showed $0, and nothing looked wrong — the portal WAS the thing that + was wrong. The guard ran as a daily script on the bot's host. It now runs in the + portal's sweep. + +WHAT IS PINNED DOWN HERE + * two independent reads are actually two: the portal's SpendLogs aggregate over + its own DB credential, and LiteLLM's HTTP API over the master key, + * agreement is silent (no email), disagreement and the $0-while-real-spend + signature email info@ exactly once per failing episode, + * a read that cannot be made is `skipped` — never a clean bill of health, + * a clean result clears the marker so a recurrence is reported again. + +Runnable: python3 tests/test_spend_check.py (from the repo root) +""" +import os +import sys +import asyncio +import tempfile + +import httpx + +os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db") +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import app.main as m # noqa: E402 + +m.DB_PATH = os.environ["DB_PATH"] +m.ADMIN_TOKEN = "test-admin-token" +m.ADMIN_TOKEN_LEGACY = "" + +SENT = [] + + +def fake_send(to, subject, body, *a, **k): + SENT.append({"to": to, "subject": subject, "body": body}) + + +# --- stubs ----------------------------------------------------------------- # +PORTAL = {"rows": 100, "total": 250.0} +API = {"total": 250.0, "keys": 5} +FAIL_PORTAL = False +FAIL_API = False + + +def fake_portal_spend(): + if FAIL_PORTAL: + raise RuntimeError("psycopg2: connection refused") + return {"2026-11-01": PORTAL["total"]}, PORTAL["total"], PORTAL["rows"] + + +async def fake_api_spend(): + if FAIL_API: + raise RuntimeError("LiteLLM spend API: HTTP 401") + return API["total"], API["keys"] + + +results = [] + + +def check(name, ok): + results.append((name, bool(ok))) + + +async def main(): + global FAIL_PORTAL, FAIL_API + m.litellm_spend_by_day = fake_portal_spend + m.litellm_api_total_spend = fake_api_spend + m.send_email = fake_send + m.get_db().close() # create + migrate the fixture schema + + def marker(): + conn = m.get_db() + row = conn.execute("SELECT value FROM portal_state WHERE key = ?", + (m.SPEND_CHECK_MARKER,)).fetchone() + conn.close() + return row[0] if row else None + + # --- agreement: silent --- + SENT.clear() + r = await m.check_spend_consistency() + check(f"agreement is ok (got {r.get('status')})", r.get("status") == "ok") + check(f"agreement reports both sides (got {r})", + r.get("portal_total") == 250.0 and r.get("litellm_api_total") == 250.0 + and r.get("portal_rows") == 100 and r.get("litellm_api_keys") == 5) + check(f"agreement sends NO email (got {len(SENT)})", len(SENT) == 0) + check(f"agreement clears the marker (got {marker()!r})", not marker()) + + # --- the 2026-11-05 signature: portal $0 / empty while LiteLLM is real --- + PORTAL["rows"], PORTAL["total"] = 0, 0.0 + SENT.clear() + r = await m.check_spend_consistency() + check(f"a $0 portal aggregate against real spend FAILS (got {r.get('status')})", + r.get("status") == "failed") + check("the reason names the stale-credential signature", + "silent-clean signature" in (r.get("reason") or "")) + check(f"info@ is told once (got {len(SENT)})", + len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop" + and "credential" in SENT[0]["body"]) + r2 = await m.check_spend_consistency() + check(f"the SAME failure is not re-mailed (got {r2.get('reported')}, {len(SENT)} emails)", + r2.get("reported") is False and len(SENT) == 1) + check(f"the marker holds what was reported (got {str(marker())[:50]})", bool(marker())) + + # --- a fixed failure: clean again, marker cleared, recurrence speaks --- + PORTAL["rows"], PORTAL["total"] = 100, 250.0 + r = await m.check_spend_consistency() + check(f"recovery is clean (got {r.get('status')})", r.get("status") == "ok") + check(f"recovery clears the marker (got {marker()!r})", not marker()) + PORTAL["rows"], PORTAL["total"] = 0, 0.0 + SENT.clear() + r = await m.check_spend_consistency() + check(f"a recurrence after a clean run mails again (got {len(SENT)})", + r.get("status") == "failed" and len(SENT) == 1) + PORTAL["rows"], PORTAL["total"] = 100, 250.0 + + # --- drift beyond tolerance --- + API["total"] = 400.0 + SENT.clear() + r = await m.check_spend_consistency() + check(f"a delta beyond tolerance FAILS (got {r.get('status')}, delta {r.get('delta')})", + r.get("status") == "failed" and abs(r.get("delta") - 150.0) < 0.01) + check("the reason names the tolerance", "tolerance" in (r.get("reason") or "")) + API["total"] = 250.0 + r = await m.check_spend_consistency() + check(f"and clearing it is clean again (got {r.get('status')})", r.get("status") == "ok") + + # --- the API side returning nothing is not agreement --- + API["keys"], API["total"] = 0, 0.0 + SENT.clear() + r = await m.check_spend_consistency() + check(f"an empty API side FAILS rather than looking agreed (got {r.get('status')})", + r.get("status") == "failed" and "no independent side" in (r.get("reason") or "")) + API["keys"], API["total"] = 5, 250.0 + + # --- a read that cannot be made is UNKNOWN, not clean --- + FAIL_PORTAL = True + SENT.clear() + r = await m.check_spend_consistency() + check(f"an unreadable portal side is skipped (got {r})", + r.get("status") == "skipped" and r.get("reason") == "portal_read_failed" + and len(SENT) == 0) + FAIL_PORTAL = False + FAIL_API = True + r = await m.check_spend_consistency() + check(f"an unreadable LiteLLM side is skipped (got {r})", + r.get("status") == "skipped" and r.get("reason") == "litellm_api_failed") + FAIL_API = False + + # --- notify=False never mails and never consumes the report --- + PORTAL["rows"], PORTAL["total"] = 0, 0.0 + m.get_db() + SENT.clear() + conn = m.get_db() + conn.execute("DELETE FROM portal_state") + conn.commit() + conn.close() + r = await m.check_spend_consistency(notify=False) + check(f"notify=False reports nothing (got {r.get('reported')}, {len(SENT)} emails)", + r.get("status") == "failed" and r.get("reported") is False and len(SENT) == 0) + r = await m.check_spend_consistency() + check(f"...and a later notifying run still speaks (got {len(SENT)} emails)", + r.get("reported") is True and len(SENT) == 1) + PORTAL["rows"], PORTAL["total"] = 100, 250.0 + + # --- the endpoint --- + SENT.clear() + transport = httpx.ASGITransport(app=m.app) + async with httpx.AsyncClient(transport=transport, base_url="http://t") as c: + r = await c.post("/admin/spend-check") + check(f"endpoint needs the admin token (got {r.status_code})", + r.status_code in (401, 403)) + r = await c.post("/admin/spend-check", headers={"X-Admin-Token": "test-admin-token"}) + body = r.json() + check(f"endpoint reports the comparison (got {body.get('status')})", + r.status_code == 200 and body.get("status") == "ok" + and body.get("litellm_api_total") == 250.0) + check(f"endpoint: a clean result mails nothing (got {len(SENT)})", len(SENT) == 0) + + +asyncio.run(main()) + +fails = sum(not ok for _, ok in results) +for name, ok in results: + print(f" [{'PASS' if ok else 'FAIL'}] {name}") +print(f"\n{len(results) - fails}/{len(results)} passed") +sys.exit(1 if fails else 0)