Files
member-portal/tests/test_spend_check.py
T
inference-admin c0ef046b0d
ci/woodpecker/push/woodpecker Pipeline was canceled
spend check: read the field LiteLLM actually sends (total_spend), and fail loudly either way
The first live run FAILED with api $0.00 over 306 keys against the portal's $935.35.
The disagreement was real, the verdict was right, and the cause was mine: LiteLLM's
/global/spend/keys returns each key's all-time spend as `total_spend`, not `spend`.
A checker that trusts its own parse then reports "the portal is wrong" — the exact
mirror of the failure this check exists to catch, which is why the guard now names
both possibilities when the independent side reads zero against real portal spend.

Parser accepts either field name; a regression test exercises the real function
against a stubbed httpx (the module attribute is stubbed for the other checks, so
the real one is captured at import). 24 checks; all 20 suites green.
2026-11-13 20:43:41 +00:00

232 lines
8.9 KiB
Python

#!/usr/bin/env python3
"""Regression test: the spend-consistency check (portal aggregate vs LiteLLM's API).
WHY THIS EXISTS
On 2026-11-05 the portal's aggregation of all-time spend failed SOFT: its copy of
the gateway's LiteLLM DB credential had gone stale, so the query returned nothing,
every surface showed $0, and nothing looked wrong — the portal WAS the thing that
was wrong. The guard ran as a daily script on the bot's host. It now runs in the
portal's sweep.
WHAT IS PINNED DOWN HERE
* two independent reads are actually two: the portal's SpendLogs aggregate over
its own DB credential, and LiteLLM's HTTP API over the master key,
* agreement is silent (no email), disagreement and the $0-while-real-spend
signature email info@ exactly once per failing episode,
* a read that cannot be made is `skipped` — never a clean bill of health,
* a clean result clears the marker so a recurrence is reported again.
Runnable: python3 tests/test_spend_check.py (from the repo root)
"""
import os
import sys
import asyncio
import tempfile
import types
import httpx
os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db")
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import app.main as m # noqa: E402
m.DB_PATH = os.environ["DB_PATH"]
m.ADMIN_TOKEN = "test-admin-token"
m.ADMIN_TOKEN_LEGACY = ""
m.LITELLM_MASTER_KEY = "test-master-key" # the parser test calls the real function
SENT = []
# main() replaces the module attribute with a stub; keep the REAL implementation so
# the parser itself can be exercised.
REAL_API_TOTAL_SPEND = m.litellm_api_total_spend
def fake_send(to, subject, body, *a, **k):
SENT.append({"to": to, "subject": subject, "body": body})
# --- stubs ----------------------------------------------------------------- #
PORTAL = {"rows": 100, "total": 250.0}
API = {"total": 250.0, "keys": 5}
FAIL_PORTAL = False
FAIL_API = False
def fake_portal_spend():
if FAIL_PORTAL:
raise RuntimeError("psycopg2: connection refused")
return {"2026-11-01": PORTAL["total"]}, PORTAL["total"], PORTAL["rows"]
async def fake_api_spend():
if FAIL_API:
raise RuntimeError("LiteLLM spend API: HTTP 401")
return API["total"], API["keys"]
results = []
def check(name, ok):
results.append((name, bool(ok)))
async def main():
global FAIL_PORTAL, FAIL_API
m.litellm_spend_by_day = fake_portal_spend
m.litellm_api_total_spend = fake_api_spend
m.send_email = fake_send
m.get_db().close() # create + migrate the fixture schema
def marker():
conn = m.get_db()
row = conn.execute("SELECT value FROM portal_state WHERE key = ?",
(m.SPEND_CHECK_MARKER,)).fetchone()
conn.close()
return row[0] if row else None
# --- agreement: silent ---
SENT.clear()
r = await m.check_spend_consistency()
check(f"agreement is ok (got {r.get('status')})", r.get("status") == "ok")
check(f"agreement reports both sides (got {r})",
r.get("portal_total") == 250.0 and r.get("litellm_api_total") == 250.0
and r.get("portal_rows") == 100 and r.get("litellm_api_keys") == 5)
check(f"agreement sends NO email (got {len(SENT)})", len(SENT) == 0)
check(f"agreement clears the marker (got {marker()!r})", not marker())
# --- the 2026-11-05 signature: portal $0 / empty while LiteLLM is real ---
PORTAL["rows"], PORTAL["total"] = 0, 0.0
SENT.clear()
r = await m.check_spend_consistency()
check(f"a $0 portal aggregate against real spend FAILS (got {r.get('status')})",
r.get("status") == "failed")
check("the reason names the stale-credential signature",
"silent-clean signature" in (r.get("reason") or ""))
check(f"info@ is told once (got {len(SENT)})",
len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop"
and "credential" in SENT[0]["body"])
r2 = await m.check_spend_consistency()
check(f"the SAME failure is not re-mailed (got {r2.get('reported')}, {len(SENT)} emails)",
r2.get("reported") is False and len(SENT) == 1)
check(f"the marker holds what was reported (got {str(marker())[:50]})", bool(marker()))
# --- a fixed failure: clean again, marker cleared, recurrence speaks ---
PORTAL["rows"], PORTAL["total"] = 100, 250.0
r = await m.check_spend_consistency()
check(f"recovery is clean (got {r.get('status')})", r.get("status") == "ok")
check(f"recovery clears the marker (got {marker()!r})", not marker())
PORTAL["rows"], PORTAL["total"] = 0, 0.0
SENT.clear()
r = await m.check_spend_consistency()
check(f"a recurrence after a clean run mails again (got {len(SENT)})",
r.get("status") == "failed" and len(SENT) == 1)
PORTAL["rows"], PORTAL["total"] = 100, 250.0
# --- drift beyond tolerance ---
API["total"] = 400.0
SENT.clear()
r = await m.check_spend_consistency()
check(f"a delta beyond tolerance FAILS (got {r.get('status')}, delta {r.get('delta')})",
r.get("status") == "failed" and abs(r.get("delta") - 150.0) < 0.01)
check("the reason names the tolerance", "tolerance" in (r.get("reason") or ""))
API["total"] = 250.0
r = await m.check_spend_consistency()
check(f"and clearing it is clean again (got {r.get('status')})", r.get("status") == "ok")
# --- the API side returning nothing is not agreement ---
API["keys"], API["total"] = 0, 0.0
SENT.clear()
r = await m.check_spend_consistency()
check(f"an empty API side FAILS rather than looking agreed (got {r.get('status')})",
r.get("status") == "failed" and "no independent side" in (r.get("reason") or ""))
API["keys"], API["total"] = 5, 250.0
# --- a read that cannot be made is UNKNOWN, not clean ---
FAIL_PORTAL = True
SENT.clear()
r = await m.check_spend_consistency()
check(f"an unreadable portal side is skipped (got {r})",
r.get("status") == "skipped" and r.get("reason") == "portal_read_failed"
and len(SENT) == 0)
FAIL_PORTAL = False
FAIL_API = True
r = await m.check_spend_consistency()
check(f"an unreadable LiteLLM side is skipped (got {r})",
r.get("status") == "skipped" and r.get("reason") == "litellm_api_failed")
FAIL_API = False
# --- notify=False never mails and never consumes the report ---
PORTAL["rows"], PORTAL["total"] = 0, 0.0
m.get_db()
SENT.clear()
conn = m.get_db()
conn.execute("DELETE FROM portal_state")
conn.commit()
conn.close()
r = await m.check_spend_consistency(notify=False)
check(f"notify=False reports nothing (got {r.get('reported')}, {len(SENT)} emails)",
r.get("status") == "failed" and r.get("reported") is False and len(SENT) == 0)
r = await m.check_spend_consistency()
check(f"...and a later notifying run still speaks (got {len(SENT)} emails)",
r.get("reported") is True and len(SENT) == 1)
PORTAL["rows"], PORTAL["total"] = 100, 250.0
# --- the parser: the API calls the field `total_spend`, not `spend` ---
# Reading only `spend` returned $0 for all 306 live keys on 2026-11-13, and a
# checker that trusts its own parse then reports "the portal is wrong" — the
# exact mirror of the failure this check exists to catch.
class _Resp:
status_code = 200
def json(self):
return [{"api_key": "a", "total_spend": 10.5}, {"api_key": "b", "spend": 2.5},
{"api_key": "c"}, {"api_key": "d", "total_spend": None}]
class _FakeClient:
def __init__(self, *a, **k):
pass
async def __aenter__(self):
return self
async def __aexit__(self, *a):
return False
async def get(self, *a, **k):
return _Resp()
real_httpx = m.httpx
m.httpx = types.SimpleNamespace(AsyncClient=_FakeClient)
try:
total, n = await REAL_API_TOTAL_SPEND()
finally:
m.httpx = real_httpx
check(f"the API total sums `total_spend` (got {total} over {n} keys)",
abs(total - 13.0) < 1e-6 and n == 4)
# --- the endpoint ---
SENT.clear()
transport = httpx.ASGITransport(app=m.app)
async with httpx.AsyncClient(transport=transport, base_url="http://t") as c:
r = await c.post("/admin/spend-check")
check(f"endpoint needs the admin token (got {r.status_code})",
r.status_code in (401, 403))
r = await c.post("/admin/spend-check", headers={"X-Admin-Token": "test-admin-token"})
body = r.json()
check(f"endpoint reports the comparison (got {body.get('status')})",
r.status_code == 200 and body.get("status") == "ok"
and body.get("litellm_api_total") == 250.0)
check(f"endpoint: a clean result mails nothing (got {len(SENT)})", len(SENT) == 0)
asyncio.run(main())
fails = sum(not ok for _, ok in results)
for name, ok in results:
print(f" [{'PASS' if ok else 'FAIL'}] {name}")
print(f"\n{len(results) - fails}/{len(results)} passed")
sys.exit(1 if fails else 0)