#!/usr/bin/env python3 """Regression test: the spend-consistency check (portal aggregate vs LiteLLM's API). WHY THIS EXISTS On 2026-11-05 the portal's aggregation of all-time spend failed SOFT: its copy of the gateway's LiteLLM DB credential had gone stale, so the query returned nothing, every surface showed $0, and nothing looked wrong — the portal WAS the thing that was wrong. The guard ran as a daily script on the bot's host. It now runs in the portal's sweep. WHAT IS PINNED DOWN HERE * two independent reads are actually two: the portal's SpendLogs aggregate over its own DB credential, and LiteLLM's HTTP API over the master key, * agreement is silent (no email), disagreement and the $0-while-real-spend signature email info@ exactly once per failing episode, * a read that cannot be made is `skipped` — never a clean bill of health, * a clean result clears the marker so a recurrence is reported again. Runnable: python3 tests/test_spend_check.py (from the repo root) """ import os import sys import asyncio import tempfile import httpx os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db") sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import app.main as m # noqa: E402 m.DB_PATH = os.environ["DB_PATH"] m.ADMIN_TOKEN = "test-admin-token" m.ADMIN_TOKEN_LEGACY = "" SENT = [] def fake_send(to, subject, body, *a, **k): SENT.append({"to": to, "subject": subject, "body": body}) # --- stubs ----------------------------------------------------------------- # PORTAL = {"rows": 100, "total": 250.0} API = {"total": 250.0, "keys": 5} FAIL_PORTAL = False FAIL_API = False def fake_portal_spend(): if FAIL_PORTAL: raise RuntimeError("psycopg2: connection refused") return {"2026-11-01": PORTAL["total"]}, PORTAL["total"], PORTAL["rows"] async def fake_api_spend(): if FAIL_API: raise RuntimeError("LiteLLM spend API: HTTP 401") return API["total"], API["keys"] results = [] def check(name, ok): results.append((name, bool(ok))) async def main(): global FAIL_PORTAL, FAIL_API m.litellm_spend_by_day = fake_portal_spend m.litellm_api_total_spend = fake_api_spend m.send_email = fake_send m.get_db().close() # create + migrate the fixture schema def marker(): conn = m.get_db() row = conn.execute("SELECT value FROM portal_state WHERE key = ?", (m.SPEND_CHECK_MARKER,)).fetchone() conn.close() return row[0] if row else None # --- agreement: silent --- SENT.clear() r = await m.check_spend_consistency() check(f"agreement is ok (got {r.get('status')})", r.get("status") == "ok") check(f"agreement reports both sides (got {r})", r.get("portal_total") == 250.0 and r.get("litellm_api_total") == 250.0 and r.get("portal_rows") == 100 and r.get("litellm_api_keys") == 5) check(f"agreement sends NO email (got {len(SENT)})", len(SENT) == 0) check(f"agreement clears the marker (got {marker()!r})", not marker()) # --- the 2026-11-05 signature: portal $0 / empty while LiteLLM is real --- PORTAL["rows"], PORTAL["total"] = 0, 0.0 SENT.clear() r = await m.check_spend_consistency() check(f"a $0 portal aggregate against real spend FAILS (got {r.get('status')})", r.get("status") == "failed") check("the reason names the stale-credential signature", "silent-clean signature" in (r.get("reason") or "")) check(f"info@ is told once (got {len(SENT)})", len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop" and "credential" in SENT[0]["body"]) r2 = await m.check_spend_consistency() check(f"the SAME failure is not re-mailed (got {r2.get('reported')}, {len(SENT)} emails)", r2.get("reported") is False and len(SENT) == 1) check(f"the marker holds what was reported (got {str(marker())[:50]})", bool(marker())) # --- a fixed failure: clean again, marker cleared, recurrence speaks --- PORTAL["rows"], PORTAL["total"] = 100, 250.0 r = await m.check_spend_consistency() check(f"recovery is clean (got {r.get('status')})", r.get("status") == "ok") check(f"recovery clears the marker (got {marker()!r})", not marker()) PORTAL["rows"], PORTAL["total"] = 0, 0.0 SENT.clear() r = await m.check_spend_consistency() check(f"a recurrence after a clean run mails again (got {len(SENT)})", r.get("status") == "failed" and len(SENT) == 1) PORTAL["rows"], PORTAL["total"] = 100, 250.0 # --- drift beyond tolerance --- API["total"] = 400.0 SENT.clear() r = await m.check_spend_consistency() check(f"a delta beyond tolerance FAILS (got {r.get('status')}, delta {r.get('delta')})", r.get("status") == "failed" and abs(r.get("delta") - 150.0) < 0.01) check("the reason names the tolerance", "tolerance" in (r.get("reason") or "")) API["total"] = 250.0 r = await m.check_spend_consistency() check(f"and clearing it is clean again (got {r.get('status')})", r.get("status") == "ok") # --- the API side returning nothing is not agreement --- API["keys"], API["total"] = 0, 0.0 SENT.clear() r = await m.check_spend_consistency() check(f"an empty API side FAILS rather than looking agreed (got {r.get('status')})", r.get("status") == "failed" and "no independent side" in (r.get("reason") or "")) API["keys"], API["total"] = 5, 250.0 # --- a read that cannot be made is UNKNOWN, not clean --- FAIL_PORTAL = True SENT.clear() r = await m.check_spend_consistency() check(f"an unreadable portal side is skipped (got {r})", r.get("status") == "skipped" and r.get("reason") == "portal_read_failed" and len(SENT) == 0) FAIL_PORTAL = False FAIL_API = True r = await m.check_spend_consistency() check(f"an unreadable LiteLLM side is skipped (got {r})", r.get("status") == "skipped" and r.get("reason") == "litellm_api_failed") FAIL_API = False # --- notify=False never mails and never consumes the report --- PORTAL["rows"], PORTAL["total"] = 0, 0.0 m.get_db() SENT.clear() conn = m.get_db() conn.execute("DELETE FROM portal_state") conn.commit() conn.close() r = await m.check_spend_consistency(notify=False) check(f"notify=False reports nothing (got {r.get('reported')}, {len(SENT)} emails)", r.get("status") == "failed" and r.get("reported") is False and len(SENT) == 0) r = await m.check_spend_consistency() check(f"...and a later notifying run still speaks (got {len(SENT)} emails)", r.get("reported") is True and len(SENT) == 1) PORTAL["rows"], PORTAL["total"] = 100, 250.0 # --- the endpoint --- SENT.clear() transport = httpx.ASGITransport(app=m.app) async with httpx.AsyncClient(transport=transport, base_url="http://t") as c: r = await c.post("/admin/spend-check") check(f"endpoint needs the admin token (got {r.status_code})", r.status_code in (401, 403)) r = await c.post("/admin/spend-check", headers={"X-Admin-Token": "test-admin-token"}) body = r.json() check(f"endpoint reports the comparison (got {body.get('status')})", r.status_code == 200 and body.get("status") == "ok" and body.get("litellm_api_total") == 250.0) check(f"endpoint: a clean result mails nothing (got {len(SENT)})", len(SENT) == 0) asyncio.run(main()) fails = sum(not ok for _, ok in results) for name, ok in results: print(f" [{'PASS' if ok else 'FAIL'}] {name}") print(f"\n{len(results) - fails}/{len(results)} passed") sys.exit(1 if fails else 0)