ci/woodpecker/push/woodpecker Pipeline was canceled
The first live run FAILED with api $0.00 over 306 keys against the portal's $935.35. The disagreement was real, the verdict was right, and the cause was mine: LiteLLM's /global/spend/keys returns each key's all-time spend as `total_spend`, not `spend`. A checker that trusts its own parse then reports "the portal is wrong" — the exact mirror of the failure this check exists to catch, which is why the guard now names both possibilities when the independent side reads zero against real portal spend. Parser accepts either field name; a regression test exercises the real function against a stubbed httpx (the module attribute is stubbed for the other checks, so the real one is captured at import). 24 checks; all 20 suites green.
232 lines
8.9 KiB
Python
232 lines
8.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Regression test: the spend-consistency check (portal aggregate vs LiteLLM's API).
|
|
|
|
WHY THIS EXISTS
|
|
On 2026-11-05 the portal's aggregation of all-time spend failed SOFT: its copy of
|
|
the gateway's LiteLLM DB credential had gone stale, so the query returned nothing,
|
|
every surface showed $0, and nothing looked wrong — the portal WAS the thing that
|
|
was wrong. The guard ran as a daily script on the bot's host. It now runs in the
|
|
portal's sweep.
|
|
|
|
WHAT IS PINNED DOWN HERE
|
|
* two independent reads are actually two: the portal's SpendLogs aggregate over
|
|
its own DB credential, and LiteLLM's HTTP API over the master key,
|
|
* agreement is silent (no email), disagreement and the $0-while-real-spend
|
|
signature email info@ exactly once per failing episode,
|
|
* a read that cannot be made is `skipped` — never a clean bill of health,
|
|
* a clean result clears the marker so a recurrence is reported again.
|
|
|
|
Runnable: python3 tests/test_spend_check.py (from the repo root)
|
|
"""
|
|
import os
|
|
import sys
|
|
import asyncio
|
|
import tempfile
|
|
|
|
import types
|
|
|
|
import httpx
|
|
|
|
os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db")
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
import app.main as m # noqa: E402
|
|
|
|
m.DB_PATH = os.environ["DB_PATH"]
|
|
m.ADMIN_TOKEN = "test-admin-token"
|
|
m.ADMIN_TOKEN_LEGACY = ""
|
|
m.LITELLM_MASTER_KEY = "test-master-key" # the parser test calls the real function
|
|
|
|
SENT = []
|
|
|
|
# main() replaces the module attribute with a stub; keep the REAL implementation so
|
|
# the parser itself can be exercised.
|
|
REAL_API_TOTAL_SPEND = m.litellm_api_total_spend
|
|
|
|
|
|
def fake_send(to, subject, body, *a, **k):
|
|
SENT.append({"to": to, "subject": subject, "body": body})
|
|
|
|
|
|
# --- stubs ----------------------------------------------------------------- #
|
|
PORTAL = {"rows": 100, "total": 250.0}
|
|
API = {"total": 250.0, "keys": 5}
|
|
FAIL_PORTAL = False
|
|
FAIL_API = False
|
|
|
|
|
|
def fake_portal_spend():
|
|
if FAIL_PORTAL:
|
|
raise RuntimeError("psycopg2: connection refused")
|
|
return {"2026-11-01": PORTAL["total"]}, PORTAL["total"], PORTAL["rows"]
|
|
|
|
|
|
async def fake_api_spend():
|
|
if FAIL_API:
|
|
raise RuntimeError("LiteLLM spend API: HTTP 401")
|
|
return API["total"], API["keys"]
|
|
|
|
|
|
results = []
|
|
|
|
|
|
def check(name, ok):
|
|
results.append((name, bool(ok)))
|
|
|
|
|
|
async def main():
|
|
global FAIL_PORTAL, FAIL_API
|
|
m.litellm_spend_by_day = fake_portal_spend
|
|
m.litellm_api_total_spend = fake_api_spend
|
|
m.send_email = fake_send
|
|
m.get_db().close() # create + migrate the fixture schema
|
|
|
|
def marker():
|
|
conn = m.get_db()
|
|
row = conn.execute("SELECT value FROM portal_state WHERE key = ?",
|
|
(m.SPEND_CHECK_MARKER,)).fetchone()
|
|
conn.close()
|
|
return row[0] if row else None
|
|
|
|
# --- agreement: silent ---
|
|
SENT.clear()
|
|
r = await m.check_spend_consistency()
|
|
check(f"agreement is ok (got {r.get('status')})", r.get("status") == "ok")
|
|
check(f"agreement reports both sides (got {r})",
|
|
r.get("portal_total") == 250.0 and r.get("litellm_api_total") == 250.0
|
|
and r.get("portal_rows") == 100 and r.get("litellm_api_keys") == 5)
|
|
check(f"agreement sends NO email (got {len(SENT)})", len(SENT) == 0)
|
|
check(f"agreement clears the marker (got {marker()!r})", not marker())
|
|
|
|
# --- the 2026-11-05 signature: portal $0 / empty while LiteLLM is real ---
|
|
PORTAL["rows"], PORTAL["total"] = 0, 0.0
|
|
SENT.clear()
|
|
r = await m.check_spend_consistency()
|
|
check(f"a $0 portal aggregate against real spend FAILS (got {r.get('status')})",
|
|
r.get("status") == "failed")
|
|
check("the reason names the stale-credential signature",
|
|
"silent-clean signature" in (r.get("reason") or ""))
|
|
check(f"info@ is told once (got {len(SENT)})",
|
|
len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop"
|
|
and "credential" in SENT[0]["body"])
|
|
r2 = await m.check_spend_consistency()
|
|
check(f"the SAME failure is not re-mailed (got {r2.get('reported')}, {len(SENT)} emails)",
|
|
r2.get("reported") is False and len(SENT) == 1)
|
|
check(f"the marker holds what was reported (got {str(marker())[:50]})", bool(marker()))
|
|
|
|
# --- a fixed failure: clean again, marker cleared, recurrence speaks ---
|
|
PORTAL["rows"], PORTAL["total"] = 100, 250.0
|
|
r = await m.check_spend_consistency()
|
|
check(f"recovery is clean (got {r.get('status')})", r.get("status") == "ok")
|
|
check(f"recovery clears the marker (got {marker()!r})", not marker())
|
|
PORTAL["rows"], PORTAL["total"] = 0, 0.0
|
|
SENT.clear()
|
|
r = await m.check_spend_consistency()
|
|
check(f"a recurrence after a clean run mails again (got {len(SENT)})",
|
|
r.get("status") == "failed" and len(SENT) == 1)
|
|
PORTAL["rows"], PORTAL["total"] = 100, 250.0
|
|
|
|
# --- drift beyond tolerance ---
|
|
API["total"] = 400.0
|
|
SENT.clear()
|
|
r = await m.check_spend_consistency()
|
|
check(f"a delta beyond tolerance FAILS (got {r.get('status')}, delta {r.get('delta')})",
|
|
r.get("status") == "failed" and abs(r.get("delta") - 150.0) < 0.01)
|
|
check("the reason names the tolerance", "tolerance" in (r.get("reason") or ""))
|
|
API["total"] = 250.0
|
|
r = await m.check_spend_consistency()
|
|
check(f"and clearing it is clean again (got {r.get('status')})", r.get("status") == "ok")
|
|
|
|
# --- the API side returning nothing is not agreement ---
|
|
API["keys"], API["total"] = 0, 0.0
|
|
SENT.clear()
|
|
r = await m.check_spend_consistency()
|
|
check(f"an empty API side FAILS rather than looking agreed (got {r.get('status')})",
|
|
r.get("status") == "failed" and "no independent side" in (r.get("reason") or ""))
|
|
API["keys"], API["total"] = 5, 250.0
|
|
|
|
# --- a read that cannot be made is UNKNOWN, not clean ---
|
|
FAIL_PORTAL = True
|
|
SENT.clear()
|
|
r = await m.check_spend_consistency()
|
|
check(f"an unreadable portal side is skipped (got {r})",
|
|
r.get("status") == "skipped" and r.get("reason") == "portal_read_failed"
|
|
and len(SENT) == 0)
|
|
FAIL_PORTAL = False
|
|
FAIL_API = True
|
|
r = await m.check_spend_consistency()
|
|
check(f"an unreadable LiteLLM side is skipped (got {r})",
|
|
r.get("status") == "skipped" and r.get("reason") == "litellm_api_failed")
|
|
FAIL_API = False
|
|
|
|
# --- notify=False never mails and never consumes the report ---
|
|
PORTAL["rows"], PORTAL["total"] = 0, 0.0
|
|
m.get_db()
|
|
SENT.clear()
|
|
conn = m.get_db()
|
|
conn.execute("DELETE FROM portal_state")
|
|
conn.commit()
|
|
conn.close()
|
|
r = await m.check_spend_consistency(notify=False)
|
|
check(f"notify=False reports nothing (got {r.get('reported')}, {len(SENT)} emails)",
|
|
r.get("status") == "failed" and r.get("reported") is False and len(SENT) == 0)
|
|
r = await m.check_spend_consistency()
|
|
check(f"...and a later notifying run still speaks (got {len(SENT)} emails)",
|
|
r.get("reported") is True and len(SENT) == 1)
|
|
PORTAL["rows"], PORTAL["total"] = 100, 250.0
|
|
|
|
# --- the parser: the API calls the field `total_spend`, not `spend` ---
|
|
# Reading only `spend` returned $0 for all 306 live keys on 2026-11-13, and a
|
|
# checker that trusts its own parse then reports "the portal is wrong" — the
|
|
# exact mirror of the failure this check exists to catch.
|
|
class _Resp:
|
|
status_code = 200
|
|
|
|
def json(self):
|
|
return [{"api_key": "a", "total_spend": 10.5}, {"api_key": "b", "spend": 2.5},
|
|
{"api_key": "c"}, {"api_key": "d", "total_spend": None}]
|
|
|
|
class _FakeClient:
|
|
def __init__(self, *a, **k):
|
|
pass
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, *a):
|
|
return False
|
|
|
|
async def get(self, *a, **k):
|
|
return _Resp()
|
|
|
|
real_httpx = m.httpx
|
|
m.httpx = types.SimpleNamespace(AsyncClient=_FakeClient)
|
|
try:
|
|
total, n = await REAL_API_TOTAL_SPEND()
|
|
finally:
|
|
m.httpx = real_httpx
|
|
check(f"the API total sums `total_spend` (got {total} over {n} keys)",
|
|
abs(total - 13.0) < 1e-6 and n == 4)
|
|
|
|
# --- the endpoint ---
|
|
SENT.clear()
|
|
transport = httpx.ASGITransport(app=m.app)
|
|
async with httpx.AsyncClient(transport=transport, base_url="http://t") as c:
|
|
r = await c.post("/admin/spend-check")
|
|
check(f"endpoint needs the admin token (got {r.status_code})",
|
|
r.status_code in (401, 403))
|
|
r = await c.post("/admin/spend-check", headers={"X-Admin-Token": "test-admin-token"})
|
|
body = r.json()
|
|
check(f"endpoint reports the comparison (got {body.get('status')})",
|
|
r.status_code == 200 and body.get("status") == "ok"
|
|
and body.get("litellm_api_total") == 250.0)
|
|
check(f"endpoint: a clean result mails nothing (got {len(SENT)})", len(SENT) == 0)
|
|
|
|
|
|
asyncio.run(main())
|
|
|
|
fails = sum(not ok for _, ok in results)
|
|
for name, ok in results:
|
|
print(f" [{'PASS' if ok else 'FAIL'}] {name}")
|
|
print(f"\n{len(results) - fails}/{len(results)} passed")
|
|
sys.exit(1 if fails else 0)
|