spend check: compare like for like (same window, same kind of number)
ci/woodpecker/push/woodpecker Pipeline was canceled

The second live run FAILED again, correctly and informatively: portal $935.52
all-time vs LiteLLM $827.89 — and the $107.63 'drift' was the WINDOW. LiteLLM's
per-day view reaches back a fixed number of days (31 on 2026-11-13); our table
holds 69. Over the days both sides cover the two agree to $0.07, with no day
differing by more than $0.022 (the API rounds to cents).

So the check now compares DAY BY DAY over the shared window, which localises drift
instead of hiding it in a sum, with a per-day tolerance ($0.10) plus a window-total
tolerance ($5). It reads /global/spend/teams rather than /global/spend/keys: the
per-key figure follows each key's own budget cycle, so summing it is not an all-time
number at all (and /global/spend/report, the all-time one, is Enterprise-gated).

Degenerate reads are judged BEFORE the window overlap, because in the failure this
check exists for the portal's aggregate comes back EMPTY — there would be no shared
days to compare, and the overlap logic would report 'cannot compare' instead of the
stale-credential signature. All failure reporting goes through one helper that mails
info@ once per episode and advances the marker only after a notice is sent.

23 checks; all 20 suites green.
This commit is contained in:
inference-admin committed 2026-11-13 20:46:50 +00:00
1 parent c13880cfab
commit c92d8979eb
2 files changed
+242 -153

No files matched your search

+93 -66
View File
@@ -8,23 +8,26 @@ WHY THIS EXISTS
was wrong. The guard ran as a daily script on the bot's host. It now runs in the
portal's sweep.
WHAT IS PINNED DOWN HERE
* two independent reads are actually two: the portal's SpendLogs aggregate over
its own DB credential, and LiteLLM's HTTP API over the master key,
* agreement is silent (no email), disagreement and the $0-while-real-spend
signature email info@ exactly once per failing episode,
* a read that cannot be made is `skipped` — never a clean bill of health,
* a clean result clears the marker so a recurrence is reported again.
The first TWO live runs also taught the shape of the comparison, and both lessons
are pinned here:
* window. LiteLLM's per-day view reaches back a fixed number of days (31 on
2026-11-13) while our table holds everything, so comparing our ALL-TIME figure
against it reported $107.63 of "drift" that was only the missing window. Compare
the days BOTH sides cover.
* like-for-like source. `/global/spend/keys` reports each key's spend over the
key's own budget cycle (`total_spend` — and reading `spend` returned $0 for all
306 keys), so it is NOT an all-time figure. The per-day teams view is comparable,
and comparing day by day localises drift instead of hiding it in a sum.
Runnable: python3 tests/test_spend_check.py (from the repo root)
"""
import os
import sys
import types
import asyncio
import tempfile
import types
import httpx
os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db")
@@ -34,13 +37,13 @@ import app.main as m # noqa: E402
m.DB_PATH = os.environ["DB_PATH"]
m.ADMIN_TOKEN = "test-admin-token"
m.ADMIN_TOKEN_LEGACY = ""
m.LITELLM_MASTER_KEY = "test-master-key" # the parser test calls the real function
m.LITELLM_MASTER_KEY = "test-master-key" # the parser test calls the real function
SENT = []
# main() replaces the module attribute with a stub; keep the REAL implementation so
# the parser itself can be exercised.
REAL_API_TOTAL_SPEND = m.litellm_api_total_spend
REAL_API_DAILY_SPEND = m.litellm_api_daily_spend
def fake_send(to, subject, body, *a, **k):
@@ -48,22 +51,24 @@ def fake_send(to, subject, body, *a, **k):
# --- stubs ----------------------------------------------------------------- #
PORTAL = {"rows": 100, "total": 250.0}
API = {"total": 250.0, "keys": 5}
DAYS = {"2026-11-10": 10.0, "2026-11-11": 20.0, "2026-11-12": 30.0}
API_DAYS = dict(DAYS)
PORTAL_DAYS = dict(DAYS)
PORTAL_ROWS = 1000
FAIL_PORTAL = False
FAIL_API = False
def fake_portal_spend():
def fake_portal_days():
if FAIL_PORTAL:
raise RuntimeError("psycopg2: connection refused")
return {"2026-11-01": PORTAL["total"]}, PORTAL["total"], PORTAL["rows"]
return dict(PORTAL_DAYS), round(sum(PORTAL_DAYS.values()), 6), PORTAL_ROWS
async def fake_api_spend():
async def fake_api_days():
if FAIL_API:
raise RuntimeError("LiteLLM spend API: HTTP 401")
return API["total"], API["keys"]
return dict(API_DAYS)
results = []
@@ -74,9 +79,9 @@ def check(name, ok):
async def main():
global FAIL_PORTAL, FAIL_API
m.litellm_spend_by_day = fake_portal_spend
m.litellm_api_total_spend = fake_api_spend
global FAIL_PORTAL, FAIL_API, PORTAL_DAYS, PORTAL_ROWS, API_DAYS
m.litellm_spend_by_day = fake_portal_days
m.litellm_api_daily_spend = fake_api_days
m.send_email = fake_send
m.get_db().close() # create + migrate the fixture schema
@@ -87,64 +92,86 @@ async def main():
conn.close()
return row[0] if row else None
# --- agreement: silent ---
def reset():
global PORTAL_DAYS, PORTAL_ROWS, API_DAYS
PORTAL_DAYS, PORTAL_ROWS, API_DAYS = dict(DAYS), 1000, dict(DAYS)
# --- agreement over the shared window: silent ---
reset()
SENT.clear()
r = await m.check_spend_consistency()
check(f"agreement is ok (got {r.get('status')})", r.get("status") == "ok")
check(f"agreement reports both sides (got {r})",
r.get("portal_total") == 250.0 and r.get("litellm_api_total") == 250.0
and r.get("portal_rows") == 100 and r.get("litellm_api_keys") == 5)
check(f"agreement reports the shared window (got {r})",
r.get("window") == "2026-11-10..2026-11-12" and r.get("days_compared") == 3
and r.get("portal_window") == 60.0 and r.get("api_window") == 60.0)
check(f"agreement sends NO email (got {len(SENT)})", len(SENT) == 0)
check(f"agreement clears the marker (got {marker()!r})", not marker())
# --- the 2026-11-05 signature: portal $0 / empty while LiteLLM is real ---
PORTAL["rows"], PORTAL["total"] = 0, 0.0
# --- days only ONE side has are NOT drift ---
PORTAL_DAYS = {"2026-09-04": 99.0, **DAYS} # 69-day history vs a 31-day view
SENT.clear()
r = await m.check_spend_consistency()
check(f"a $0 portal aggregate against real spend FAILS (got {r.get('status')})",
r.get("status") == "failed")
check("the reason names the stale-credential signature",
"silent-clean signature" in (r.get("reason") or ""))
check(f"a longer portal history is not reported as drift (got {r.get('status')}, "
f"all-time {r.get('portal_all_time')}, delta {r.get('delta')})",
r.get("status") == "ok" and r.get("portal_all_time") == 159.0
and r.get("delta") == 0.0 and len(SENT) == 0)
reset()
# --- the 2026-11-05 signature: portal empty while LiteLLM is real ---
PORTAL_DAYS, PORTAL_ROWS = {}, 0
SENT.clear()
r = await m.check_spend_consistency()
check(f"an EMPTY portal aggregate against real spend FAILS (got {r.get('status')})",
r.get("status") == "failed" and "silent-clean signature" in (r.get("reason") or ""))
check(f"info@ is told once (got {len(SENT)})",
len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop"
and "credential" in SENT[0]["body"])
r2 = await m.check_spend_consistency()
check(f"the SAME failure is not re-mailed (got {r2.get('reported')}, {len(SENT)} emails)",
check(f"the SAME failure is not re-mailed (got {len(SENT)} emails)",
r2.get("reported") is False and len(SENT) == 1)
check(f"the marker holds what was reported (got {str(marker())[:50]})", bool(marker()))
# --- a fixed failure: clean again, marker cleared, recurrence speaks ---
PORTAL["rows"], PORTAL["total"] = 100, 250.0
# --- recovery is clean again, and a recurrence speaks ---
PORTAL_DAYS, PORTAL_ROWS = dict(DAYS), 1000
r = await m.check_spend_consistency()
check(f"recovery is clean (got {r.get('status')})", r.get("status") == "ok")
check(f"recovery clears the marker (got {marker()!r})", not marker())
PORTAL["rows"], PORTAL["total"] = 0, 0.0
PORTAL_DAYS, PORTAL_ROWS = {}, 0
SENT.clear()
r = await m.check_spend_consistency()
check(f"a recurrence after a clean run mails again (got {len(SENT)})",
check(f"a recurrence mails again (got {r.get('status')}, {len(SENT)} emails)",
r.get("status") == "failed" and len(SENT) == 1)
PORTAL["rows"], PORTAL["total"] = 100, 250.0
reset()
await m.check_spend_consistency() # back to clean
# --- drift beyond tolerance ---
API["total"] = 400.0
# --- a DAY that drifts beyond the per-day tolerance is named ---
API_DAYS = {"2026-11-10": 10.0, "2026-11-11": 20.0, "2026-11-12": 44.0}
SENT.clear()
r = await m.check_spend_consistency()
check(f"a delta beyond tolerance FAILS (got {r.get('status')}, delta {r.get('delta')})",
r.get("status") == "failed" and abs(r.get("delta") - 150.0) < 0.01)
check("the reason names the tolerance", "tolerance" in (r.get("reason") or ""))
API["total"] = 250.0
r = await m.check_spend_consistency()
check(f"and clearing it is clean again (got {r.get('status')})", r.get("status") == "ok")
check(f"a drifting day FAILS and is named (got {r.get('status')}, {r.get('drifting_days')})",
r.get("status") == "failed" and bool(r.get("drifting_days"))
and r["drifting_days"][0]["date"] == "2026-11-12"
and "2026-11-12" in (r.get("reason") or ""))
check(f"the notice lists the worst day (got {len(SENT)})",
len(SENT) == 1 and "2026-11-12" in SENT[0]["body"])
reset()
await m.check_spend_consistency()
# --- the API side returning nothing is not agreement ---
API["keys"], API["total"] = 0, 0.0
# --- cents-level rounding is NOT drift ---
API_DAYS = {"2026-11-10": 10.02, "2026-11-11": 19.98, "2026-11-12": 30.01}
SENT.clear()
r = await m.check_spend_consistency()
check(f"an empty API side FAILS rather than looking agreed (got {r.get('status')})",
r.get("status") == "failed" and "no independent side" in (r.get("reason") or ""))
API["keys"], API["total"] = 5, 250.0
check(f"cent-level rounding passes (got {r.get('status')}, delta {r.get('delta')})",
r.get("status") == "ok" and len(SENT) == 0)
reset()
# --- a read that cannot be made is UNKNOWN, not clean ---
# --- no shared window: cannot compare, must NOT read as clean ---
API_DAYS = {"2026-10-01": 5.0}
r = await m.check_spend_consistency()
check(f"no shared window is skipped, not clean (got {r})",
r.get("status") == "skipped" and r.get("reason") == "no_overlapping_window")
reset()
# --- a read that cannot be made is UNKNOWN ---
FAIL_PORTAL = True
SENT.clear()
r = await m.check_spend_consistency()
@@ -158,32 +185,32 @@ async def main():
r.get("status") == "skipped" and r.get("reason") == "litellm_api_failed")
FAIL_API = False
# --- notify=False never mails and never consumes the report ---
PORTAL["rows"], PORTAL["total"] = 0, 0.0
m.get_db()
SENT.clear()
# --- notify=False never mails and never consumes ---
conn = m.get_db()
conn.execute("DELETE FROM portal_state")
conn.commit()
conn.close()
PORTAL_DAYS, PORTAL_ROWS = {}, 0
SENT.clear()
r = await m.check_spend_consistency(notify=False)
check(f"notify=False reports nothing (got {r.get('reported')}, {len(SENT)} emails)",
r.get("status") == "failed" and r.get("reported") is False and len(SENT) == 0)
r = await m.check_spend_consistency()
check(f"...and a later notifying run still speaks (got {len(SENT)} emails)",
r.get("reported") is True and len(SENT) == 1)
PORTAL["rows"], PORTAL["total"] = 100, 250.0
reset()
# --- the parser: the API calls the field `total_spend`, not `spend` ---
# Reading only `spend` returned $0 for all 306 live keys on 2026-11-13, and a
# checker that trusts its own parse then reports "the portal is wrong" — the
# exact mirror of the failure this check exists to catch.
# --- the parser: the API's shape is per-day, per-team ---
class _Resp:
status_code = 200
def json(self):
return [{"api_key": "a", "total_spend": 10.5}, {"api_key": "b", "spend": 2.5},
{"api_key": "c"}, {"api_key": "d", "total_spend": None}]
return {"daily_spend": [
{"date": "2026-11-12", "member:a@x.org": 1.5, "member:b@x.org": 2.25},
{"date": "2026-11-13", "member:a@x.org": 0.5},
{"date": "", "member:a@x.org": 9.0}, # no date -> ignored
{"member:a@x.org": 7.0}, # no date -> ignored
]}
class _FakeClient:
def __init__(self, *a, **k):
@@ -201,11 +228,11 @@ async def main():
real_httpx = m.httpx
m.httpx = types.SimpleNamespace(AsyncClient=_FakeClient)
try:
total, n = await REAL_API_TOTAL_SPEND()
days = await REAL_API_DAILY_SPEND()
finally:
m.httpx = real_httpx
check(f"the API total sums `total_spend` (got {total} over {n} keys)",
abs(total - 13.0) < 1e-6 and n == 4)
check(f"the parser sums each day's teams (got {days})",
days == {"2026-11-12": 3.75, "2026-11-13": 0.5})
# --- the endpoint ---
SENT.clear()
@@ -218,7 +245,7 @@ async def main():
body = r.json()
check(f"endpoint reports the comparison (got {body.get('status')})",
r.status_code == 200 and body.get("status") == "ok"
and body.get("litellm_api_total") == 250.0)
and body.get("api_window") == 60.0)
check(f"endpoint: a clean result mails nothing (got {len(SENT)})", len(SENT) == 0)