Files
member-portal/tests/test_spend_check.py
T
inference-admin c92d8979eb
ci/woodpecker/push/woodpecker Pipeline was canceled
spend check: compare like for like (same window, same kind of number)
The second live run FAILED again, correctly and informatively: portal $935.52
all-time vs LiteLLM $827.89 — and the $107.63 'drift' was the WINDOW. LiteLLM's
per-day view reaches back a fixed number of days (31 on 2026-11-13); our table
holds 69. Over the days both sides cover the two agree to $0.07, with no day
differing by more than $0.022 (the API rounds to cents).

So the check now compares DAY BY DAY over the shared window, which localises drift
instead of hiding it in a sum, with a per-day tolerance ($0.10) plus a window-total
tolerance ($5). It reads /global/spend/teams rather than /global/spend/keys: the
per-key figure follows each key's own budget cycle, so summing it is not an all-time
number at all (and /global/spend/report, the all-time one, is Enterprise-gated).

Degenerate reads are judged BEFORE the window overlap, because in the failure this
check exists for the portal's aggregate comes back EMPTY — there would be no shared
days to compare, and the overlap logic would report 'cannot compare' instead of the
stale-credential signature. All failure reporting goes through one helper that mails
info@ once per episode and advances the marker only after a notice is sent.

23 checks; all 20 suites green.
2026-11-13 20:46:50 +00:00

259 lines
10 KiB
Python

#!/usr/bin/env python3
"""Regression test: the spend-consistency check (portal aggregate vs LiteLLM's API).
WHY THIS EXISTS
On 2026-11-05 the portal's aggregation of all-time spend failed SOFT: its copy of
the gateway's LiteLLM DB credential had gone stale, so the query returned nothing,
every surface showed $0, and nothing looked wrong — the portal WAS the thing that
was wrong. The guard ran as a daily script on the bot's host. It now runs in the
portal's sweep.
The first TWO live runs also taught the shape of the comparison, and both lessons
are pinned here:
* window. LiteLLM's per-day view reaches back a fixed number of days (31 on
2026-11-13) while our table holds everything, so comparing our ALL-TIME figure
against it reported $107.63 of "drift" that was only the missing window. Compare
the days BOTH sides cover.
* like-for-like source. `/global/spend/keys` reports each key's spend over the
key's own budget cycle (`total_spend` — and reading `spend` returned $0 for all
306 keys), so it is NOT an all-time figure. The per-day teams view is comparable,
and comparing day by day localises drift instead of hiding it in a sum.
Runnable: python3 tests/test_spend_check.py (from the repo root)
"""
import os
import sys
import types
import asyncio
import tempfile
import httpx
os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db")
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import app.main as m # noqa: E402
m.DB_PATH = os.environ["DB_PATH"]
m.ADMIN_TOKEN = "test-admin-token"
m.ADMIN_TOKEN_LEGACY = ""
m.LITELLM_MASTER_KEY = "test-master-key" # the parser test calls the real function
SENT = []
# main() replaces the module attribute with a stub; keep the REAL implementation so
# the parser itself can be exercised.
REAL_API_DAILY_SPEND = m.litellm_api_daily_spend
def fake_send(to, subject, body, *a, **k):
SENT.append({"to": to, "subject": subject, "body": body})
# --- stubs ----------------------------------------------------------------- #
DAYS = {"2026-11-10": 10.0, "2026-11-11": 20.0, "2026-11-12": 30.0}
API_DAYS = dict(DAYS)
PORTAL_DAYS = dict(DAYS)
PORTAL_ROWS = 1000
FAIL_PORTAL = False
FAIL_API = False
def fake_portal_days():
if FAIL_PORTAL:
raise RuntimeError("psycopg2: connection refused")
return dict(PORTAL_DAYS), round(sum(PORTAL_DAYS.values()), 6), PORTAL_ROWS
async def fake_api_days():
if FAIL_API:
raise RuntimeError("LiteLLM spend API: HTTP 401")
return dict(API_DAYS)
results = []
def check(name, ok):
results.append((name, bool(ok)))
async def main():
global FAIL_PORTAL, FAIL_API, PORTAL_DAYS, PORTAL_ROWS, API_DAYS
m.litellm_spend_by_day = fake_portal_days
m.litellm_api_daily_spend = fake_api_days
m.send_email = fake_send
m.get_db().close() # create + migrate the fixture schema
def marker():
conn = m.get_db()
row = conn.execute("SELECT value FROM portal_state WHERE key = ?",
(m.SPEND_CHECK_MARKER,)).fetchone()
conn.close()
return row[0] if row else None
def reset():
global PORTAL_DAYS, PORTAL_ROWS, API_DAYS
PORTAL_DAYS, PORTAL_ROWS, API_DAYS = dict(DAYS), 1000, dict(DAYS)
# --- agreement over the shared window: silent ---
reset()
SENT.clear()
r = await m.check_spend_consistency()
check(f"agreement is ok (got {r.get('status')})", r.get("status") == "ok")
check(f"agreement reports the shared window (got {r})",
r.get("window") == "2026-11-10..2026-11-12" and r.get("days_compared") == 3
and r.get("portal_window") == 60.0 and r.get("api_window") == 60.0)
check(f"agreement sends NO email (got {len(SENT)})", len(SENT) == 0)
check(f"agreement clears the marker (got {marker()!r})", not marker())
# --- days only ONE side has are NOT drift ---
PORTAL_DAYS = {"2026-09-04": 99.0, **DAYS} # 69-day history vs a 31-day view
SENT.clear()
r = await m.check_spend_consistency()
check(f"a longer portal history is not reported as drift (got {r.get('status')}, "
f"all-time {r.get('portal_all_time')}, delta {r.get('delta')})",
r.get("status") == "ok" and r.get("portal_all_time") == 159.0
and r.get("delta") == 0.0 and len(SENT) == 0)
reset()
# --- the 2026-11-05 signature: portal empty while LiteLLM is real ---
PORTAL_DAYS, PORTAL_ROWS = {}, 0
SENT.clear()
r = await m.check_spend_consistency()
check(f"an EMPTY portal aggregate against real spend FAILS (got {r.get('status')})",
r.get("status") == "failed" and "silent-clean signature" in (r.get("reason") or ""))
check(f"info@ is told once (got {len(SENT)})",
len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop"
and "credential" in SENT[0]["body"])
r2 = await m.check_spend_consistency()
check(f"the SAME failure is not re-mailed (got {len(SENT)} emails)",
r2.get("reported") is False and len(SENT) == 1)
# --- recovery is clean again, and a recurrence speaks ---
PORTAL_DAYS, PORTAL_ROWS = dict(DAYS), 1000
r = await m.check_spend_consistency()
check(f"recovery is clean (got {r.get('status')})", r.get("status") == "ok")
check(f"recovery clears the marker (got {marker()!r})", not marker())
PORTAL_DAYS, PORTAL_ROWS = {}, 0
SENT.clear()
r = await m.check_spend_consistency()
check(f"a recurrence mails again (got {r.get('status')}, {len(SENT)} emails)",
r.get("status") == "failed" and len(SENT) == 1)
reset()
await m.check_spend_consistency() # back to clean
# --- a DAY that drifts beyond the per-day tolerance is named ---
API_DAYS = {"2026-11-10": 10.0, "2026-11-11": 20.0, "2026-11-12": 44.0}
SENT.clear()
r = await m.check_spend_consistency()
check(f"a drifting day FAILS and is named (got {r.get('status')}, {r.get('drifting_days')})",
r.get("status") == "failed" and bool(r.get("drifting_days"))
and r["drifting_days"][0]["date"] == "2026-11-12"
and "2026-11-12" in (r.get("reason") or ""))
check(f"the notice lists the worst day (got {len(SENT)})",
len(SENT) == 1 and "2026-11-12" in SENT[0]["body"])
reset()
await m.check_spend_consistency()
# --- cents-level rounding is NOT drift ---
API_DAYS = {"2026-11-10": 10.02, "2026-11-11": 19.98, "2026-11-12": 30.01}
SENT.clear()
r = await m.check_spend_consistency()
check(f"cent-level rounding passes (got {r.get('status')}, delta {r.get('delta')})",
r.get("status") == "ok" and len(SENT) == 0)
reset()
# --- no shared window: cannot compare, must NOT read as clean ---
API_DAYS = {"2026-10-01": 5.0}
r = await m.check_spend_consistency()
check(f"no shared window is skipped, not clean (got {r})",
r.get("status") == "skipped" and r.get("reason") == "no_overlapping_window")
reset()
# --- a read that cannot be made is UNKNOWN ---
FAIL_PORTAL = True
SENT.clear()
r = await m.check_spend_consistency()
check(f"an unreadable portal side is skipped (got {r})",
r.get("status") == "skipped" and r.get("reason") == "portal_read_failed"
and len(SENT) == 0)
FAIL_PORTAL = False
FAIL_API = True
r = await m.check_spend_consistency()
check(f"an unreadable LiteLLM side is skipped (got {r})",
r.get("status") == "skipped" and r.get("reason") == "litellm_api_failed")
FAIL_API = False
# --- notify=False never mails and never consumes ---
conn = m.get_db()
conn.execute("DELETE FROM portal_state")
conn.commit()
conn.close()
PORTAL_DAYS, PORTAL_ROWS = {}, 0
SENT.clear()
r = await m.check_spend_consistency(notify=False)
check(f"notify=False reports nothing (got {r.get('reported')}, {len(SENT)} emails)",
r.get("status") == "failed" and r.get("reported") is False and len(SENT) == 0)
r = await m.check_spend_consistency()
check(f"...and a later notifying run still speaks (got {len(SENT)} emails)",
r.get("reported") is True and len(SENT) == 1)
reset()
# --- the parser: the API's shape is per-day, per-team ---
class _Resp:
status_code = 200
def json(self):
return {"daily_spend": [
{"date": "2026-11-12", "member:a@x.org": 1.5, "member:b@x.org": 2.25},
{"date": "2026-11-13", "member:a@x.org": 0.5},
{"date": "", "member:a@x.org": 9.0}, # no date -> ignored
{"member:a@x.org": 7.0}, # no date -> ignored
]}
class _FakeClient:
def __init__(self, *a, **k):
pass
async def __aenter__(self):
return self
async def __aexit__(self, *a):
return False
async def get(self, *a, **k):
return _Resp()
real_httpx = m.httpx
m.httpx = types.SimpleNamespace(AsyncClient=_FakeClient)
try:
days = await REAL_API_DAILY_SPEND()
finally:
m.httpx = real_httpx
check(f"the parser sums each day's teams (got {days})",
days == {"2026-11-12": 3.75, "2026-11-13": 0.5})
# --- the endpoint ---
SENT.clear()
transport = httpx.ASGITransport(app=m.app)
async with httpx.AsyncClient(transport=transport, base_url="http://t") as c:
r = await c.post("/admin/spend-check")
check(f"endpoint needs the admin token (got {r.status_code})",
r.status_code in (401, 403))
r = await c.post("/admin/spend-check", headers={"X-Admin-Token": "test-admin-token"})
body = r.json()
check(f"endpoint reports the comparison (got {body.get('status')})",
r.status_code == 200 and body.get("status") == "ok"
and body.get("api_window") == 60.0)
check(f"endpoint: a clean result mails nothing (got {len(SENT)})", len(SENT) == 0)
asyncio.run(main())
fails = sum(not ok for _, ok in results)
for name, ok in results:
print(f" [{'PASS' if ok else 'FAIL'}] {name}")
print(f"\n{len(results) - fails}/{len(results)} passed")
sys.exit(1 if fails else 0)