ci/woodpecker/push/woodpecker Pipeline was canceled
The second live run FAILED again, correctly and informatively: portal $935.52 all-time vs LiteLLM $827.89 — and the $107.63 'drift' was the WINDOW. LiteLLM's per-day view reaches back a fixed number of days (31 on 2026-11-13); our table holds 69. Over the days both sides cover the two agree to $0.07, with no day differing by more than $0.022 (the API rounds to cents). So the check now compares DAY BY DAY over the shared window, which localises drift instead of hiding it in a sum, with a per-day tolerance ($0.10) plus a window-total tolerance ($5). It reads /global/spend/teams rather than /global/spend/keys: the per-key figure follows each key's own budget cycle, so summing it is not an all-time number at all (and /global/spend/report, the all-time one, is Enterprise-gated). Degenerate reads are judged BEFORE the window overlap, because in the failure this check exists for the portal's aggregate comes back EMPTY — there would be no shared days to compare, and the overlap logic would report 'cannot compare' instead of the stale-credential signature. All failure reporting goes through one helper that mails info@ once per episode and advances the marker only after a notice is sent. 23 checks; all 20 suites green.
259 lines
10 KiB
Python
259 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Regression test: the spend-consistency check (portal aggregate vs LiteLLM's API).
|
|
|
|
WHY THIS EXISTS
|
|
On 2026-11-05 the portal's aggregation of all-time spend failed SOFT: its copy of
|
|
the gateway's LiteLLM DB credential had gone stale, so the query returned nothing,
|
|
every surface showed $0, and nothing looked wrong — the portal WAS the thing that
|
|
was wrong. The guard ran as a daily script on the bot's host. It now runs in the
|
|
portal's sweep.
|
|
|
|
The first TWO live runs also taught the shape of the comparison, and both lessons
|
|
are pinned here:
|
|
|
|
* window. LiteLLM's per-day view reaches back a fixed number of days (31 on
|
|
2026-11-13) while our table holds everything, so comparing our ALL-TIME figure
|
|
against it reported $107.63 of "drift" that was only the missing window. Compare
|
|
the days BOTH sides cover.
|
|
* like-for-like source. `/global/spend/keys` reports each key's spend over the
|
|
key's own budget cycle (`total_spend` — and reading `spend` returned $0 for all
|
|
306 keys), so it is NOT an all-time figure. The per-day teams view is comparable,
|
|
and comparing day by day localises drift instead of hiding it in a sum.
|
|
|
|
Runnable: python3 tests/test_spend_check.py (from the repo root)
|
|
"""
|
|
import os
|
|
import sys
|
|
import types
|
|
import asyncio
|
|
import tempfile
|
|
|
|
import httpx
|
|
|
|
os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db")
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
import app.main as m # noqa: E402
|
|
|
|
m.DB_PATH = os.environ["DB_PATH"]
|
|
m.ADMIN_TOKEN = "test-admin-token"
|
|
m.ADMIN_TOKEN_LEGACY = ""
|
|
m.LITELLM_MASTER_KEY = "test-master-key" # the parser test calls the real function
|
|
|
|
SENT = []
|
|
|
|
# main() replaces the module attribute with a stub; keep the REAL implementation so
|
|
# the parser itself can be exercised.
|
|
REAL_API_DAILY_SPEND = m.litellm_api_daily_spend
|
|
|
|
|
|
def fake_send(to, subject, body, *a, **k):
|
|
SENT.append({"to": to, "subject": subject, "body": body})
|
|
|
|
|
|
# --- stubs ----------------------------------------------------------------- #
|
|
DAYS = {"2026-11-10": 10.0, "2026-11-11": 20.0, "2026-11-12": 30.0}
|
|
API_DAYS = dict(DAYS)
|
|
PORTAL_DAYS = dict(DAYS)
|
|
PORTAL_ROWS = 1000
|
|
FAIL_PORTAL = False
|
|
FAIL_API = False
|
|
|
|
|
|
def fake_portal_days():
|
|
if FAIL_PORTAL:
|
|
raise RuntimeError("psycopg2: connection refused")
|
|
return dict(PORTAL_DAYS), round(sum(PORTAL_DAYS.values()), 6), PORTAL_ROWS
|
|
|
|
|
|
async def fake_api_days():
|
|
if FAIL_API:
|
|
raise RuntimeError("LiteLLM spend API: HTTP 401")
|
|
return dict(API_DAYS)
|
|
|
|
|
|
results = []
|
|
|
|
|
|
def check(name, ok):
|
|
results.append((name, bool(ok)))
|
|
|
|
|
|
async def main():
|
|
global FAIL_PORTAL, FAIL_API, PORTAL_DAYS, PORTAL_ROWS, API_DAYS
|
|
m.litellm_spend_by_day = fake_portal_days
|
|
m.litellm_api_daily_spend = fake_api_days
|
|
m.send_email = fake_send
|
|
m.get_db().close() # create + migrate the fixture schema
|
|
|
|
def marker():
|
|
conn = m.get_db()
|
|
row = conn.execute("SELECT value FROM portal_state WHERE key = ?",
|
|
(m.SPEND_CHECK_MARKER,)).fetchone()
|
|
conn.close()
|
|
return row[0] if row else None
|
|
|
|
def reset():
|
|
global PORTAL_DAYS, PORTAL_ROWS, API_DAYS
|
|
PORTAL_DAYS, PORTAL_ROWS, API_DAYS = dict(DAYS), 1000, dict(DAYS)
|
|
|
|
# --- agreement over the shared window: silent ---
|
|
reset()
|
|
SENT.clear()
|
|
r = await m.check_spend_consistency()
|
|
check(f"agreement is ok (got {r.get('status')})", r.get("status") == "ok")
|
|
check(f"agreement reports the shared window (got {r})",
|
|
r.get("window") == "2026-11-10..2026-11-12" and r.get("days_compared") == 3
|
|
and r.get("portal_window") == 60.0 and r.get("api_window") == 60.0)
|
|
check(f"agreement sends NO email (got {len(SENT)})", len(SENT) == 0)
|
|
check(f"agreement clears the marker (got {marker()!r})", not marker())
|
|
|
|
# --- days only ONE side has are NOT drift ---
|
|
PORTAL_DAYS = {"2026-09-04": 99.0, **DAYS} # 69-day history vs a 31-day view
|
|
SENT.clear()
|
|
r = await m.check_spend_consistency()
|
|
check(f"a longer portal history is not reported as drift (got {r.get('status')}, "
|
|
f"all-time {r.get('portal_all_time')}, delta {r.get('delta')})",
|
|
r.get("status") == "ok" and r.get("portal_all_time") == 159.0
|
|
and r.get("delta") == 0.0 and len(SENT) == 0)
|
|
reset()
|
|
|
|
# --- the 2026-11-05 signature: portal empty while LiteLLM is real ---
|
|
PORTAL_DAYS, PORTAL_ROWS = {}, 0
|
|
SENT.clear()
|
|
r = await m.check_spend_consistency()
|
|
check(f"an EMPTY portal aggregate against real spend FAILS (got {r.get('status')})",
|
|
r.get("status") == "failed" and "silent-clean signature" in (r.get("reason") or ""))
|
|
check(f"info@ is told once (got {len(SENT)})",
|
|
len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop"
|
|
and "credential" in SENT[0]["body"])
|
|
r2 = await m.check_spend_consistency()
|
|
check(f"the SAME failure is not re-mailed (got {len(SENT)} emails)",
|
|
r2.get("reported") is False and len(SENT) == 1)
|
|
|
|
# --- recovery is clean again, and a recurrence speaks ---
|
|
PORTAL_DAYS, PORTAL_ROWS = dict(DAYS), 1000
|
|
r = await m.check_spend_consistency()
|
|
check(f"recovery is clean (got {r.get('status')})", r.get("status") == "ok")
|
|
check(f"recovery clears the marker (got {marker()!r})", not marker())
|
|
PORTAL_DAYS, PORTAL_ROWS = {}, 0
|
|
SENT.clear()
|
|
r = await m.check_spend_consistency()
|
|
check(f"a recurrence mails again (got {r.get('status')}, {len(SENT)} emails)",
|
|
r.get("status") == "failed" and len(SENT) == 1)
|
|
reset()
|
|
await m.check_spend_consistency() # back to clean
|
|
|
|
# --- a DAY that drifts beyond the per-day tolerance is named ---
|
|
API_DAYS = {"2026-11-10": 10.0, "2026-11-11": 20.0, "2026-11-12": 44.0}
|
|
SENT.clear()
|
|
r = await m.check_spend_consistency()
|
|
check(f"a drifting day FAILS and is named (got {r.get('status')}, {r.get('drifting_days')})",
|
|
r.get("status") == "failed" and bool(r.get("drifting_days"))
|
|
and r["drifting_days"][0]["date"] == "2026-11-12"
|
|
and "2026-11-12" in (r.get("reason") or ""))
|
|
check(f"the notice lists the worst day (got {len(SENT)})",
|
|
len(SENT) == 1 and "2026-11-12" in SENT[0]["body"])
|
|
reset()
|
|
await m.check_spend_consistency()
|
|
|
|
# --- cents-level rounding is NOT drift ---
|
|
API_DAYS = {"2026-11-10": 10.02, "2026-11-11": 19.98, "2026-11-12": 30.01}
|
|
SENT.clear()
|
|
r = await m.check_spend_consistency()
|
|
check(f"cent-level rounding passes (got {r.get('status')}, delta {r.get('delta')})",
|
|
r.get("status") == "ok" and len(SENT) == 0)
|
|
reset()
|
|
|
|
# --- no shared window: cannot compare, must NOT read as clean ---
|
|
API_DAYS = {"2026-10-01": 5.0}
|
|
r = await m.check_spend_consistency()
|
|
check(f"no shared window is skipped, not clean (got {r})",
|
|
r.get("status") == "skipped" and r.get("reason") == "no_overlapping_window")
|
|
reset()
|
|
|
|
# --- a read that cannot be made is UNKNOWN ---
|
|
FAIL_PORTAL = True
|
|
SENT.clear()
|
|
r = await m.check_spend_consistency()
|
|
check(f"an unreadable portal side is skipped (got {r})",
|
|
r.get("status") == "skipped" and r.get("reason") == "portal_read_failed"
|
|
and len(SENT) == 0)
|
|
FAIL_PORTAL = False
|
|
FAIL_API = True
|
|
r = await m.check_spend_consistency()
|
|
check(f"an unreadable LiteLLM side is skipped (got {r})",
|
|
r.get("status") == "skipped" and r.get("reason") == "litellm_api_failed")
|
|
FAIL_API = False
|
|
|
|
# --- notify=False never mails and never consumes ---
|
|
conn = m.get_db()
|
|
conn.execute("DELETE FROM portal_state")
|
|
conn.commit()
|
|
conn.close()
|
|
PORTAL_DAYS, PORTAL_ROWS = {}, 0
|
|
SENT.clear()
|
|
r = await m.check_spend_consistency(notify=False)
|
|
check(f"notify=False reports nothing (got {r.get('reported')}, {len(SENT)} emails)",
|
|
r.get("status") == "failed" and r.get("reported") is False and len(SENT) == 0)
|
|
r = await m.check_spend_consistency()
|
|
check(f"...and a later notifying run still speaks (got {len(SENT)} emails)",
|
|
r.get("reported") is True and len(SENT) == 1)
|
|
reset()
|
|
|
|
# --- the parser: the API's shape is per-day, per-team ---
|
|
class _Resp:
|
|
status_code = 200
|
|
|
|
def json(self):
|
|
return {"daily_spend": [
|
|
{"date": "2026-11-12", "member:a@x.org": 1.5, "member:b@x.org": 2.25},
|
|
{"date": "2026-11-13", "member:a@x.org": 0.5},
|
|
{"date": "", "member:a@x.org": 9.0}, # no date -> ignored
|
|
{"member:a@x.org": 7.0}, # no date -> ignored
|
|
]}
|
|
|
|
class _FakeClient:
|
|
def __init__(self, *a, **k):
|
|
pass
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, *a):
|
|
return False
|
|
|
|
async def get(self, *a, **k):
|
|
return _Resp()
|
|
|
|
real_httpx = m.httpx
|
|
m.httpx = types.SimpleNamespace(AsyncClient=_FakeClient)
|
|
try:
|
|
days = await REAL_API_DAILY_SPEND()
|
|
finally:
|
|
m.httpx = real_httpx
|
|
check(f"the parser sums each day's teams (got {days})",
|
|
days == {"2026-11-12": 3.75, "2026-11-13": 0.5})
|
|
|
|
# --- the endpoint ---
|
|
SENT.clear()
|
|
transport = httpx.ASGITransport(app=m.app)
|
|
async with httpx.AsyncClient(transport=transport, base_url="http://t") as c:
|
|
r = await c.post("/admin/spend-check")
|
|
check(f"endpoint needs the admin token (got {r.status_code})",
|
|
r.status_code in (401, 403))
|
|
r = await c.post("/admin/spend-check", headers={"X-Admin-Token": "test-admin-token"})
|
|
body = r.json()
|
|
check(f"endpoint reports the comparison (got {body.get('status')})",
|
|
r.status_code == 200 and body.get("status") == "ok"
|
|
and body.get("api_window") == 60.0)
|
|
check(f"endpoint: a clean result mails nothing (got {len(SENT)})", len(SENT) == 0)
|
|
|
|
|
|
asyncio.run(main())
|
|
|
|
fails = sum(not ok for _, ok in results)
|
|
for name, ok in results:
|
|
print(f" [{'PASS' if ok else 'FAIL'}] {name}")
|
|
print(f"\n{len(results) - fails}/{len(results)} passed")
|
|
sys.exit(1 if fails else 0)
|