spend consistency: the check moves into the portal's sweep
ci/woodpecker/push/woodpecker Pipeline was canceled
ci/woodpecker/push/woodpecker Pipeline was canceled
The failure this exists for (2026-11-05): the portal's own all-time-spend aggregate failed SOFT when its copy of the gateway's LiteLLM DB credential went stale — the query returned nothing, every surface showed $0, and nothing looked wrong, because the portal WAS the thing that was wrong. The guard ran as a daily script on the bot's host. It now runs in the portal, and keeps two genuinely independent reads: the portal's SpendLogs aggregate over its own DB credential, and LiteLLM's own HTTP API (/global/spend/keys) over the master key — a different transport AND a different credential, which is the point, since the DB copy is what goes stale. It adds the signature guard the host script could only infer from outside: an EMPTY//usr/bin/bash portal aggregate while LiteLLM reports real spend is a FAILURE, named as the stale-credential case; an API side that returns no keys is a failure too, not agreement. A read that cannot be made is skipped — never a clean bill. Silent when they agree; info@ once per failing episode, marker cleared on recovery. POST /admin/spend-check runs it on demand. 23 checks; all 20 suites green.
This commit is contained in:
1 parent
bb0a02dcd9
commit
e2eaa7ae40
2 files changed
+354
No files matched your search
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regression test: the spend-consistency check (portal aggregate vs LiteLLM's API).
|
||||
|
||||
WHY THIS EXISTS
|
||||
On 2026-11-05 the portal's aggregation of all-time spend failed SOFT: its copy of
|
||||
the gateway's LiteLLM DB credential had gone stale, so the query returned nothing,
|
||||
every surface showed $0, and nothing looked wrong — the portal WAS the thing that
|
||||
was wrong. The guard ran as a daily script on the bot's host. It now runs in the
|
||||
portal's sweep.
|
||||
|
||||
WHAT IS PINNED DOWN HERE
|
||||
* two independent reads are actually two: the portal's SpendLogs aggregate over
|
||||
its own DB credential, and LiteLLM's HTTP API over the master key,
|
||||
* agreement is silent (no email), disagreement and the $0-while-real-spend
|
||||
signature email info@ exactly once per failing episode,
|
||||
* a read that cannot be made is `skipped` — never a clean bill of health,
|
||||
* a clean result clears the marker so a recurrence is reported again.
|
||||
|
||||
Runnable: python3 tests/test_spend_check.py (from the repo root)
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import asyncio
|
||||
import tempfile
|
||||
|
||||
import httpx
|
||||
|
||||
os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db")
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
import app.main as m # noqa: E402
|
||||
|
||||
m.DB_PATH = os.environ["DB_PATH"]
|
||||
m.ADMIN_TOKEN = "test-admin-token"
|
||||
m.ADMIN_TOKEN_LEGACY = ""
|
||||
|
||||
SENT = []
|
||||
|
||||
|
||||
def fake_send(to, subject, body, *a, **k):
|
||||
SENT.append({"to": to, "subject": subject, "body": body})
|
||||
|
||||
|
||||
# --- stubs ----------------------------------------------------------------- #
|
||||
PORTAL = {"rows": 100, "total": 250.0}
|
||||
API = {"total": 250.0, "keys": 5}
|
||||
FAIL_PORTAL = False
|
||||
FAIL_API = False
|
||||
|
||||
|
||||
def fake_portal_spend():
|
||||
if FAIL_PORTAL:
|
||||
raise RuntimeError("psycopg2: connection refused")
|
||||
return {"2026-11-01": PORTAL["total"]}, PORTAL["total"], PORTAL["rows"]
|
||||
|
||||
|
||||
async def fake_api_spend():
|
||||
if FAIL_API:
|
||||
raise RuntimeError("LiteLLM spend API: HTTP 401")
|
||||
return API["total"], API["keys"]
|
||||
|
||||
|
||||
results = []
|
||||
|
||||
|
||||
def check(name, ok):
|
||||
results.append((name, bool(ok)))
|
||||
|
||||
|
||||
async def main():
|
||||
global FAIL_PORTAL, FAIL_API
|
||||
m.litellm_spend_by_day = fake_portal_spend
|
||||
m.litellm_api_total_spend = fake_api_spend
|
||||
m.send_email = fake_send
|
||||
m.get_db().close() # create + migrate the fixture schema
|
||||
|
||||
def marker():
|
||||
conn = m.get_db()
|
||||
row = conn.execute("SELECT value FROM portal_state WHERE key = ?",
|
||||
(m.SPEND_CHECK_MARKER,)).fetchone()
|
||||
conn.close()
|
||||
return row[0] if row else None
|
||||
|
||||
# --- agreement: silent ---
|
||||
SENT.clear()
|
||||
r = await m.check_spend_consistency()
|
||||
check(f"agreement is ok (got {r.get('status')})", r.get("status") == "ok")
|
||||
check(f"agreement reports both sides (got {r})",
|
||||
r.get("portal_total") == 250.0 and r.get("litellm_api_total") == 250.0
|
||||
and r.get("portal_rows") == 100 and r.get("litellm_api_keys") == 5)
|
||||
check(f"agreement sends NO email (got {len(SENT)})", len(SENT) == 0)
|
||||
check(f"agreement clears the marker (got {marker()!r})", not marker())
|
||||
|
||||
# --- the 2026-11-05 signature: portal $0 / empty while LiteLLM is real ---
|
||||
PORTAL["rows"], PORTAL["total"] = 0, 0.0
|
||||
SENT.clear()
|
||||
r = await m.check_spend_consistency()
|
||||
check(f"a $0 portal aggregate against real spend FAILS (got {r.get('status')})",
|
||||
r.get("status") == "failed")
|
||||
check("the reason names the stale-credential signature",
|
||||
"silent-clean signature" in (r.get("reason") or ""))
|
||||
check(f"info@ is told once (got {len(SENT)})",
|
||||
len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop"
|
||||
and "credential" in SENT[0]["body"])
|
||||
r2 = await m.check_spend_consistency()
|
||||
check(f"the SAME failure is not re-mailed (got {r2.get('reported')}, {len(SENT)} emails)",
|
||||
r2.get("reported") is False and len(SENT) == 1)
|
||||
check(f"the marker holds what was reported (got {str(marker())[:50]})", bool(marker()))
|
||||
|
||||
# --- a fixed failure: clean again, marker cleared, recurrence speaks ---
|
||||
PORTAL["rows"], PORTAL["total"] = 100, 250.0
|
||||
r = await m.check_spend_consistency()
|
||||
check(f"recovery is clean (got {r.get('status')})", r.get("status") == "ok")
|
||||
check(f"recovery clears the marker (got {marker()!r})", not marker())
|
||||
PORTAL["rows"], PORTAL["total"] = 0, 0.0
|
||||
SENT.clear()
|
||||
r = await m.check_spend_consistency()
|
||||
check(f"a recurrence after a clean run mails again (got {len(SENT)})",
|
||||
r.get("status") == "failed" and len(SENT) == 1)
|
||||
PORTAL["rows"], PORTAL["total"] = 100, 250.0
|
||||
|
||||
# --- drift beyond tolerance ---
|
||||
API["total"] = 400.0
|
||||
SENT.clear()
|
||||
r = await m.check_spend_consistency()
|
||||
check(f"a delta beyond tolerance FAILS (got {r.get('status')}, delta {r.get('delta')})",
|
||||
r.get("status") == "failed" and abs(r.get("delta") - 150.0) < 0.01)
|
||||
check("the reason names the tolerance", "tolerance" in (r.get("reason") or ""))
|
||||
API["total"] = 250.0
|
||||
r = await m.check_spend_consistency()
|
||||
check(f"and clearing it is clean again (got {r.get('status')})", r.get("status") == "ok")
|
||||
|
||||
# --- the API side returning nothing is not agreement ---
|
||||
API["keys"], API["total"] = 0, 0.0
|
||||
SENT.clear()
|
||||
r = await m.check_spend_consistency()
|
||||
check(f"an empty API side FAILS rather than looking agreed (got {r.get('status')})",
|
||||
r.get("status") == "failed" and "no independent side" in (r.get("reason") or ""))
|
||||
API["keys"], API["total"] = 5, 250.0
|
||||
|
||||
# --- a read that cannot be made is UNKNOWN, not clean ---
|
||||
FAIL_PORTAL = True
|
||||
SENT.clear()
|
||||
r = await m.check_spend_consistency()
|
||||
check(f"an unreadable portal side is skipped (got {r})",
|
||||
r.get("status") == "skipped" and r.get("reason") == "portal_read_failed"
|
||||
and len(SENT) == 0)
|
||||
FAIL_PORTAL = False
|
||||
FAIL_API = True
|
||||
r = await m.check_spend_consistency()
|
||||
check(f"an unreadable LiteLLM side is skipped (got {r})",
|
||||
r.get("status") == "skipped" and r.get("reason") == "litellm_api_failed")
|
||||
FAIL_API = False
|
||||
|
||||
# --- notify=False never mails and never consumes the report ---
|
||||
PORTAL["rows"], PORTAL["total"] = 0, 0.0
|
||||
m.get_db()
|
||||
SENT.clear()
|
||||
conn = m.get_db()
|
||||
conn.execute("DELETE FROM portal_state")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
r = await m.check_spend_consistency(notify=False)
|
||||
check(f"notify=False reports nothing (got {r.get('reported')}, {len(SENT)} emails)",
|
||||
r.get("status") == "failed" and r.get("reported") is False and len(SENT) == 0)
|
||||
r = await m.check_spend_consistency()
|
||||
check(f"...and a later notifying run still speaks (got {len(SENT)} emails)",
|
||||
r.get("reported") is True and len(SENT) == 1)
|
||||
PORTAL["rows"], PORTAL["total"] = 100, 250.0
|
||||
|
||||
# --- the endpoint ---
|
||||
SENT.clear()
|
||||
transport = httpx.ASGITransport(app=m.app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://t") as c:
|
||||
r = await c.post("/admin/spend-check")
|
||||
check(f"endpoint needs the admin token (got {r.status_code})",
|
||||
r.status_code in (401, 403))
|
||||
r = await c.post("/admin/spend-check", headers={"X-Admin-Token": "test-admin-token"})
|
||||
body = r.json()
|
||||
check(f"endpoint reports the comparison (got {body.get('status')})",
|
||||
r.status_code == 200 and body.get("status") == "ok"
|
||||
and body.get("litellm_api_total") == 250.0)
|
||||
check(f"endpoint: a clean result mails nothing (got {len(SENT)})", len(SENT) == 0)
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
|
||||
fails = sum(not ok for _, ok in results)
|
||||
for name, ok in results:
|
||||
print(f" [{'PASS' if ok else 'FAIL'}] {name}")
|
||||
print(f"\n{len(results) - fails}/{len(results)} passed")
|
||||
sys.exit(1 if fails else 0)
|
||||
Reference in new issue
Block a user