spend check: compare like for like (same window, same kind of number)
ci/woodpecker/push/woodpecker Pipeline was canceled

The second live run FAILED again, correctly and informatively: portal $935.52
all-time vs LiteLLM $827.89 — and the $107.63 'drift' was the WINDOW. LiteLLM's
per-day view reaches back a fixed number of days (31 on 2026-11-13); our table
holds 69. Over the days both sides cover the two agree to $0.07, with no day
differing by more than $0.022 (the API rounds to cents).

So the check now compares DAY BY DAY over the shared window, which localises drift
instead of hiding it in a sum, with a per-day tolerance ($0.10) plus a window-total
tolerance ($5). It reads /global/spend/teams rather than /global/spend/keys: the
per-key figure follows each key's own budget cycle, so summing it is not an all-time
number at all (and /global/spend/report, the all-time one, is Enterprise-gated).

Degenerate reads are judged BEFORE the window overlap, because in the failure this
check exists for the portal's aggregate comes back EMPTY — there would be no shared
days to compare, and the overlap logic would report 'cannot compare' instead of the
stale-credential signature. All failure reporting goes through one helper that mails
info@ once per episode and advances the marker only after a notice is sent.

23 checks; all 20 suites green.
This commit is contained in:
inference-admin committed 2026-11-13 20:46:50 +00:00
1 parent c13880cfab
commit c92d8979eb
2 files changed
+252 -163

No files matched your search

+160 -98
View File
@@ -8259,132 +8259,91 @@ async def reconcile_unmapped_backers(notify: bool = True) -> dict:
# what this checks. Extracting it into a shared helper is the next step if the two # what this checks. Extracting it into a shared helper is the next step if the two
# ever need to be compared directly. # ever need to be compared directly.
SPEND_CHECK_TOLERANCE = float(os.environ.get("SPEND_CHECK_TOLERANCE", "5.00") or "5.00") SPEND_CHECK_TOLERANCE = float(os.environ.get("SPEND_CHECK_TOLERANCE", "5.00") or "5.00")
# LiteLLM's API reports spend per day per team rounded to cents, so a day can differ
# by a cent or two legitimately (the largest seen on live data was $0.022).
SPEND_CHECK_DAILY_TOLERANCE = float(os.environ.get("SPEND_CHECK_DAILY_TOLERANCE", "0.10") or "0.10")
SPEND_CHECK_MARKER = "spend_check_reported" SPEND_CHECK_MARKER = "spend_check_reported"
async def litellm_api_total_spend() -> tuple[float, int]: async def litellm_api_daily_spend() -> dict[str, float]:
"""Total spend from LiteLLM's OWN HTTP API (master key). Raises on failure. """{UTC date: $} from LiteLLM's OWN API (`/global/spend/teams`). Raises on failure.
`/global/spend/keys` returns one entry per API key with its spend; summing them WHY NOT `/global/spend/keys`: its per-key `total_spend` follows the key's own
is the closest HTTP equivalent of COUNT/SUM over the SpendLogs table. A different budget cycle, so summing it compares the last cycle against our all-time figure
credential path from the portal's DB copy — which is the whole point, since the ($827.89 vs $935.52 on 2026-11-13 — a $107 window mismatch that looks exactly
DB copy is the thing that goes stale. like drift). `/global/spend/report` would be all-time but is gated behind a
LiteLLM Enterprise licence (HTTP 400). The per-day teams view is the comparable
one, and comparing DAY BY DAY is sharper than comparing sums: it localises drift
instead of hiding it in an aggregate.
""" """
if not LITELLM_MASTER_KEY: if not LITELLM_MASTER_KEY:
raise RuntimeError("LITELLM_MASTER_KEY not set; cannot verify against LiteLLM") raise RuntimeError("LITELLM_MASTER_KEY not set; cannot verify against LiteLLM")
async with httpx.AsyncClient(timeout=30.0) as client: async with httpx.AsyncClient(timeout=30.0) as client:
r = await client.get(f"{LITELLM_BASE}/global/spend/keys", r = await client.get(f"{LITELLM_BASE}/global/spend/teams",
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"}) headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"})
if r.status_code != 200: if r.status_code != 200:
raise RuntimeError(f"LiteLLM spend API: HTTP {r.status_code}") raise RuntimeError(f"LiteLLM spend API: HTTP {r.status_code}")
rows = r.json() body = r.json()
if not isinstance(rows, list): if not isinstance(body, dict):
raise RuntimeError(f"LiteLLM spend API returned {type(rows).__name__}, not a list") raise RuntimeError(f"LiteLLM spend API returned {type(body).__name__}, not an object")
out: dict[str, float] = {}
for day in body.get("daily_spend") or []:
day = day or {}
d = str(day.get("date") or "")[:10]
if not d:
continue
total = 0.0 total = 0.0
for row in rows: for k, v in day.items():
row = row or {} if k == "date":
# LiteLLM reports each key's all-time spend as `total_spend`; older builds continue
# called it `spend`. Reading only `spend` returned $0 for all 306 keys on
# 2026-11-13 — a failure that a naive checker reports as "the portal is wrong".
for field in ("total_spend", "spend"):
try: try:
val = float(row.get(field) or 0) total += float(v or 0)
except (TypeError, ValueError): except (TypeError, ValueError):
val = 0.0 continue
if val: out[d] = round(total, 6)
total += val return out
break
return round(total, 6), len(rows)
async def check_spend_consistency(notify: bool = True) -> dict: async def _spend_failure(notify: bool, reason: str, verdict: dict) -> dict:
"""Compare the portal's SpendLogs aggregate with LiteLLM's own API. Emails only on FAIL. """Report a spend-check failure to info@ ONCE per failing episode.
Returns status="ok" with a verdict, or "failed" with the reason — and a read that The marker holds the reason a human was told, so a persistent failure does not
cannot be made is `skipped`, never a clean bill: "could not look" is an anomaly. mail every sweep while a changed one always does. It is written only after a
notice actually went out: a silent (`notify=False`) run or a failed send must not
swallow the alert a failure gets.
""" """
try: verdict = dict(verdict)
_by_day, portal_total, portal_rows = litellm_spend_by_day()
except Exception as e: # noqa: BLE001
logger.warning("spend check: portal SpendLogs read failed: %s", e)
return {"status": "skipped", "reason": "portal_read_failed", "detail": str(e)[:200]}
try:
api_total, api_keys = await litellm_api_total_spend()
except Exception as e: # noqa: BLE001
logger.warning("spend check: LiteLLM API read failed: %s", e)
return {"status": "skipped", "reason": "litellm_api_failed", "detail": str(e)[:200]}
delta = round(api_total - portal_total, 2)
verdict = {
"portal_total": round(portal_total, 2),
"portal_rows": portal_rows,
"litellm_api_total": round(api_total, 2),
"litellm_api_keys": api_keys,
"delta": delta,
"tolerance": SPEND_CHECK_TOLERANCE,
}
reason = ""
if portal_rows == 0 and api_total > 1:
# The 2026-11-05 signature, verbatim: our copy of the credential went stale
# and the aggregation collapsed to $0 while LiteLLM's own totals are real.
reason = (f"the portal's SpendLogs aggregate is EMPTY/zero while LiteLLM's own API "
f"reports ${api_total:.2f} — the silent-clean signature of a stale LiteLLM "
f"DB credential (the panel/dashboard $0 is the same failure)")
elif api_keys == 0 and portal_rows > 0:
reason = (f"LiteLLM's spend API returned no keys while the portal read {portal_rows} "
f"SpendLogs rows — the comparison has no independent side")
elif api_total == 0 and portal_total > 0:
# Both directions matter: an independent side that reads zero while the portal
# reads real spend is a disagreement, and on 2026-11-13 it was OUR parse (the
# API calls the field `total_spend`, not `spend`). Say both possibilities.
reason = (f"LiteLLM's spend API reports ${api_total:.2f} across {api_keys} keys while "
f"the portal reads ${portal_total:.2f} over {portal_rows} rows — either "
f"LiteLLM's own accounting is broken or this check is reading the wrong "
f"field (it must accept `total_spend`); do not believe either side until "
f"that is settled")
elif abs(delta) > SPEND_CHECK_TOLERANCE:
reason = (f"portal SpendLogs total ${portal_total:.2f} ({portal_rows} rows) vs LiteLLM "
f"API ${api_total:.2f} over {api_keys} keys: delta ${delta:+.2f} exceeds the "
f"${SPEND_CHECK_TOLERANCE:.2f} tolerance — the two are not reading the same data")
if not reason:
verdict["status"] = "ok"
logger.info("spend check PASS: portal $%.2f (%d rows) vs LiteLLM API $%.2f (%d keys), "
"delta $%+.2f within $%.2f", portal_total, portal_rows, api_total,
api_keys, delta, SPEND_CHECK_TOLERANCE)
# A clean result clears the marker, so a recurrence is reported again.
try:
conn = get_db()
_state_set(conn, SPEND_CHECK_MARKER, "")
conn.commit()
conn.close()
except Exception as e: # noqa: BLE001
logger.warning("spend check: could not clear the report marker: %s", e)
return verdict
verdict["status"] = "failed" verdict["status"] = "failed"
verdict["reason"] = reason verdict["reason"] = reason
logger.error("spend check FAILED: %s", reason) verdict["tolerance"] = SPEND_CHECK_TOLERANCE
verdict["daily_tolerance"] = SPEND_CHECK_DAILY_TOLERANCE
# Reported once per failing episode: the marker holds what a human has been told.
conn = get_db()
reported = (_state_get(conn, SPEND_CHECK_MARKER, "") or "") == reason
conn.close()
verdict["reported"] = False verdict["reported"] = False
if notify and not reported: conn = get_db()
already = (_state_get(conn, SPEND_CHECK_MARKER, "") or "") == reason
conn.close()
if not notify or already:
return verdict
try: try:
send_email( send_email(
"info@inference.coop", "info@inference.coop",
"Inference Cooperative — spend figures disagree (portal vs LiteLLM)", "Inference Cooperative — spend figures disagree (portal vs LiteLLM)",
"The portal's own spend aggregation and LiteLLM's API do not agree. This " "The portal's own spend aggregation and LiteLLM's API do not agree. This "
"check runs on every sweep and is SILENT when they match.\n\n" "check runs on every sweep and is SILENT when they match.\n\n"
f" portal : ${verdict['portal_total']:.2f} over {verdict['portal_rows']} rows\n" f" window : {verdict.get('window')} ({verdict.get('days_compared')} day(s))\n"
f" LiteLLM : ${verdict['litellm_api_total']:.2f} over " f" portal : ${float(verdict.get('portal_window') or 0):.2f}\n"
f"{verdict['litellm_api_keys']} keys\n" f" LiteLLM : ${float(verdict.get('api_window') or 0):.2f}\n"
f" delta : ${verdict['delta']:+.2f} (tolerance ${SPEND_CHECK_TOLERANCE:.2f})\n\n" f" delta : ${float(verdict.get('delta') or 0):+.2f} "
f"(tolerance ${SPEND_CHECK_TOLERANCE:.2f}, per-day "
f"${SPEND_CHECK_DAILY_TOLERANCE:.2f})\n"
f" all-time portal: ${float(verdict.get('portal_all_time') or 0):.2f} over "
f"{verdict.get('portal_rows')} rows\n\n"
f"Reason: {reason}\n\n" f"Reason: {reason}\n\n"
"The likely cause is the portal's copy of the LiteLLM DB credential going " "Worst days, if any:\n"
+ ("\n".join(f" {d['date']}: portal ${d['portal']:.2f} vs LiteLLM "
f"${d['api']:.2f} (${d['delta']:+.2f})"
for d in (verdict.get("drifting_days") or []))
or " (none beyond the per-day tolerance)")
+ "\n\nThe likely cause is the portal's copy of the LiteLLM DB credential going "
"stale after a rotation: the aggregation then fails soft and every surface " "stale after a rotation: the aggregation then fails soft and every surface "
"shows $0 while looking clean. Check the credential in the portal's env " "shows $0 while looking clean. Check the credential in the portal's env "
"against the gateway app's own.\n", "against the gateway app's own.\n",
@@ -8399,6 +8358,109 @@ async def check_spend_consistency(notify: bool = True) -> dict:
return verdict return verdict
async def check_spend_consistency(notify: bool = True) -> dict:
"""Compare the portal's SpendLogs aggregate with LiteLLM's own API. Emails only on FAIL.
Returns status="ok" with a verdict, or "failed" with the reason — and a read that
cannot be made is `skipped`, never a clean bill: "could not look" is an anomaly.
"""
try:
portal_by_day, portal_all_time, portal_rows = litellm_spend_by_day()
except Exception as e: # noqa: BLE001
logger.warning("spend check: portal SpendLogs read failed: %s", e)
return {"status": "skipped", "reason": "portal_read_failed", "detail": str(e)[:200]}
try:
api_by_day = await litellm_api_daily_spend()
except Exception as e: # noqa: BLE001
logger.warning("spend check: LiteLLM API read failed: %s", e)
return {"status": "skipped", "reason": "litellm_api_failed", "detail": str(e)[:200]}
# Degenerate reads come FIRST: in the failure this check exists for, the portal's
# aggregate comes back EMPTY, so there are no shared days to compare and the
# overlap logic would call it "cannot compare" instead of naming the real problem.
api_all_days_total = round(sum(api_by_day.values()), 2)
if portal_rows == 0 and api_all_days_total > 1:
reason = (f"the portal's SpendLogs aggregate is EMPTY ($0 over 0 rows) while LiteLLM's "
f"own API reports ${api_all_days_total:.2f} over {len(api_by_day)} day(s) — "
f"the silent-clean signature of a stale LiteLLM DB credential (the "
f"panel/dashboard $0 is the same failure)")
logger.error("spend check FAILED: %s", reason)
return await _spend_failure(
notify, reason,
{"portal_rows": 0, "portal_window": 0.0, "api_window": api_all_days_total,
"delta": -api_all_days_total, "window": "none (the portal read returned nothing)",
"days_compared": 0, "portal_all_time": 0.0, "drifting_days": []})
if not api_by_day and portal_rows > 0:
reason = ("LiteLLM's spend API returned no days while the portal read "
f"{portal_rows} SpendLogs rows — the comparison has no independent side")
logger.error("spend check FAILED: %s", reason)
return await _spend_failure(
notify, reason,
{"portal_rows": portal_rows, "portal_window": 0.0, "api_window": 0.0,
"delta": 0.0, "window": "none", "days_compared": 0,
"portal_all_time": portal_all_time, "drifting_days": []})
# Compare only the days BOTH sides cover. LiteLLM's per-day view reaches back a
# fixed number of days (31 on 2026-11-13) while our table holds everything, so a
# sum-vs-all-time comparison reports the missing window as drift — $107.63 of it.
overlap = sorted(set(api_by_day) & set(portal_by_day))
if not overlap:
return {"status": "skipped", "reason": "no_overlapping_window",
"detail": f"LiteLLM reports {len(api_by_day)} day(s), the portal "
f"{len(portal_by_day)}; they share none"}
per_day = [(d, portal_by_day[d], api_by_day[d],
round(api_by_day[d] - portal_by_day[d], 4)) for d in overlap]
portal_window = round(sum(p for _d, p, _a, _x in per_day), 2)
api_window = round(sum(a for _d, _p, a, _x in per_day), 2)
delta = round(api_window - portal_window, 2)
bad_days = sorted((x for x in per_day if abs(x[3]) > SPEND_CHECK_DAILY_TOLERANCE),
key=lambda x: -abs(x[3]))
verdict = {
"portal_window": portal_window,
"api_window": api_window,
"delta": delta,
"window": f"{overlap[0]}..{overlap[-1]}",
"days_compared": len(overlap),
"portal_all_time": round(portal_all_time, 2),
"portal_rows": portal_rows,
"drifting_days": [{"date": d, "portal": p, "api": a, "delta": x}
for d, p, a, x in bad_days[:5]],
"tolerance": SPEND_CHECK_TOLERANCE,
"daily_tolerance": SPEND_CHECK_DAILY_TOLERANCE,
}
reason = ""
if bad_days:
d, p, a, x = bad_days[0]
reason = (f"{len(bad_days)} day(s) differ by more than ${SPEND_CHECK_DAILY_TOLERANCE:.2f} "
f"across {verdict['window']} — worst {d}: portal ${p:.2f} vs LiteLLM "
f"${a:.2f} (${x:+.2f}); the two are not reading the same data")
elif abs(delta) > SPEND_CHECK_TOLERANCE:
reason = (f"window totals differ by ${delta:+.2f} (portal ${portal_window:.2f} vs "
f"LiteLLM ${api_window:.2f} over {len(overlap)} day(s)) — beyond the "
f"${SPEND_CHECK_TOLERANCE:.2f} tolerance")
if not reason:
verdict["status"] = "ok"
logger.info("spend check PASS: portal $%.2f vs LiteLLM $%.2f over %s (%d days), "
"delta $%+.2f within $%.2f; all-time portal $%.2f over %d rows",
portal_window, api_window, verdict["window"], len(overlap), delta,
SPEND_CHECK_TOLERANCE, portal_all_time, portal_rows)
# A clean result clears the marker, so a recurrence is reported again.
try:
conn = get_db()
_state_set(conn, SPEND_CHECK_MARKER, "")
conn.commit()
conn.close()
except Exception as e: # noqa: BLE001
logger.warning("spend check: could not clear the report marker: %s", e)
return verdict
logger.error("spend check FAILED: %s", reason)
return await _spend_failure(notify, reason, verdict)
async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> dict: async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> dict:
"""Credit PAID credit packs that never reached our books (issue #7). """Credit PAID credit packs that never reached our books (issue #7).
+92 -65
View File
@@ -8,23 +8,26 @@ WHY THIS EXISTS
was wrong. The guard ran as a daily script on the bot's host. It now runs in the was wrong. The guard ran as a daily script on the bot's host. It now runs in the
portal's sweep. portal's sweep.
WHAT IS PINNED DOWN HERE The first TWO live runs also taught the shape of the comparison, and both lessons
* two independent reads are actually two: the portal's SpendLogs aggregate over are pinned here:
its own DB credential, and LiteLLM's HTTP API over the master key,
* agreement is silent (no email), disagreement and the $0-while-real-spend * window. LiteLLM's per-day view reaches back a fixed number of days (31 on
signature email info@ exactly once per failing episode, 2026-11-13) while our table holds everything, so comparing our ALL-TIME figure
* a read that cannot be made is `skipped` — never a clean bill of health, against it reported $107.63 of "drift" that was only the missing window. Compare
* a clean result clears the marker so a recurrence is reported again. the days BOTH sides cover.
* like-for-like source. `/global/spend/keys` reports each key's spend over the
key's own budget cycle (`total_spend` — and reading `spend` returned $0 for all
306 keys), so it is NOT an all-time figure. The per-day teams view is comparable,
and comparing day by day localises drift instead of hiding it in a sum.
Runnable: python3 tests/test_spend_check.py (from the repo root) Runnable: python3 tests/test_spend_check.py (from the repo root)
""" """
import os import os
import sys import sys
import types
import asyncio import asyncio
import tempfile import tempfile
import types
import httpx import httpx
os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db") os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db")
@@ -40,7 +43,7 @@ SENT = []
# main() replaces the module attribute with a stub; keep the REAL implementation so # main() replaces the module attribute with a stub; keep the REAL implementation so
# the parser itself can be exercised. # the parser itself can be exercised.
REAL_API_TOTAL_SPEND = m.litellm_api_total_spend REAL_API_DAILY_SPEND = m.litellm_api_daily_spend
def fake_send(to, subject, body, *a, **k): def fake_send(to, subject, body, *a, **k):
@@ -48,22 +51,24 @@ def fake_send(to, subject, body, *a, **k):
# --- stubs ----------------------------------------------------------------- # # --- stubs ----------------------------------------------------------------- #
PORTAL = {"rows": 100, "total": 250.0} DAYS = {"2026-11-10": 10.0, "2026-11-11": 20.0, "2026-11-12": 30.0}
API = {"total": 250.0, "keys": 5} API_DAYS = dict(DAYS)
PORTAL_DAYS = dict(DAYS)
PORTAL_ROWS = 1000
FAIL_PORTAL = False FAIL_PORTAL = False
FAIL_API = False FAIL_API = False
def fake_portal_spend(): def fake_portal_days():
if FAIL_PORTAL: if FAIL_PORTAL:
raise RuntimeError("psycopg2: connection refused") raise RuntimeError("psycopg2: connection refused")
return {"2026-11-01": PORTAL["total"]}, PORTAL["total"], PORTAL["rows"] return dict(PORTAL_DAYS), round(sum(PORTAL_DAYS.values()), 6), PORTAL_ROWS
async def fake_api_spend(): async def fake_api_days():
if FAIL_API: if FAIL_API:
raise RuntimeError("LiteLLM spend API: HTTP 401") raise RuntimeError("LiteLLM spend API: HTTP 401")
return API["total"], API["keys"] return dict(API_DAYS)
results = [] results = []
@@ -74,9 +79,9 @@ def check(name, ok):
async def main(): async def main():
global FAIL_PORTAL, FAIL_API global FAIL_PORTAL, FAIL_API, PORTAL_DAYS, PORTAL_ROWS, API_DAYS
m.litellm_spend_by_day = fake_portal_spend m.litellm_spend_by_day = fake_portal_days
m.litellm_api_total_spend = fake_api_spend m.litellm_api_daily_spend = fake_api_days
m.send_email = fake_send m.send_email = fake_send
m.get_db().close() # create + migrate the fixture schema m.get_db().close() # create + migrate the fixture schema
@@ -87,64 +92,86 @@ async def main():
conn.close() conn.close()
return row[0] if row else None return row[0] if row else None
# --- agreement: silent --- def reset():
global PORTAL_DAYS, PORTAL_ROWS, API_DAYS
PORTAL_DAYS, PORTAL_ROWS, API_DAYS = dict(DAYS), 1000, dict(DAYS)
# --- agreement over the shared window: silent ---
reset()
SENT.clear() SENT.clear()
r = await m.check_spend_consistency() r = await m.check_spend_consistency()
check(f"agreement is ok (got {r.get('status')})", r.get("status") == "ok") check(f"agreement is ok (got {r.get('status')})", r.get("status") == "ok")
check(f"agreement reports both sides (got {r})", check(f"agreement reports the shared window (got {r})",
r.get("portal_total") == 250.0 and r.get("litellm_api_total") == 250.0 r.get("window") == "2026-11-10..2026-11-12" and r.get("days_compared") == 3
and r.get("portal_rows") == 100 and r.get("litellm_api_keys") == 5) and r.get("portal_window") == 60.0 and r.get("api_window") == 60.0)
check(f"agreement sends NO email (got {len(SENT)})", len(SENT) == 0) check(f"agreement sends NO email (got {len(SENT)})", len(SENT) == 0)
check(f"agreement clears the marker (got {marker()!r})", not marker()) check(f"agreement clears the marker (got {marker()!r})", not marker())
# --- the 2026-11-05 signature: portal $0 / empty while LiteLLM is real --- # --- days only ONE side has are NOT drift ---
PORTAL["rows"], PORTAL["total"] = 0, 0.0 PORTAL_DAYS = {"2026-09-04": 99.0, **DAYS} # 69-day history vs a 31-day view
SENT.clear() SENT.clear()
r = await m.check_spend_consistency() r = await m.check_spend_consistency()
check(f"a $0 portal aggregate against real spend FAILS (got {r.get('status')})", check(f"a longer portal history is not reported as drift (got {r.get('status')}, "
r.get("status") == "failed") f"all-time {r.get('portal_all_time')}, delta {r.get('delta')})",
check("the reason names the stale-credential signature", r.get("status") == "ok" and r.get("portal_all_time") == 159.0
"silent-clean signature" in (r.get("reason") or "")) and r.get("delta") == 0.0 and len(SENT) == 0)
reset()
# --- the 2026-11-05 signature: portal empty while LiteLLM is real ---
PORTAL_DAYS, PORTAL_ROWS = {}, 0
SENT.clear()
r = await m.check_spend_consistency()
check(f"an EMPTY portal aggregate against real spend FAILS (got {r.get('status')})",
r.get("status") == "failed" and "silent-clean signature" in (r.get("reason") or ""))
check(f"info@ is told once (got {len(SENT)})", check(f"info@ is told once (got {len(SENT)})",
len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop" len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop"
and "credential" in SENT[0]["body"]) and "credential" in SENT[0]["body"])
r2 = await m.check_spend_consistency() r2 = await m.check_spend_consistency()
check(f"the SAME failure is not re-mailed (got {r2.get('reported')}, {len(SENT)} emails)", check(f"the SAME failure is not re-mailed (got {len(SENT)} emails)",
r2.get("reported") is False and len(SENT) == 1) r2.get("reported") is False and len(SENT) == 1)
check(f"the marker holds what was reported (got {str(marker())[:50]})", bool(marker()))
# --- a fixed failure: clean again, marker cleared, recurrence speaks --- # --- recovery is clean again, and a recurrence speaks ---
PORTAL["rows"], PORTAL["total"] = 100, 250.0 PORTAL_DAYS, PORTAL_ROWS = dict(DAYS), 1000
r = await m.check_spend_consistency() r = await m.check_spend_consistency()
check(f"recovery is clean (got {r.get('status')})", r.get("status") == "ok") check(f"recovery is clean (got {r.get('status')})", r.get("status") == "ok")
check(f"recovery clears the marker (got {marker()!r})", not marker()) check(f"recovery clears the marker (got {marker()!r})", not marker())
PORTAL["rows"], PORTAL["total"] = 0, 0.0 PORTAL_DAYS, PORTAL_ROWS = {}, 0
SENT.clear() SENT.clear()
r = await m.check_spend_consistency() r = await m.check_spend_consistency()
check(f"a recurrence after a clean run mails again (got {len(SENT)})", check(f"a recurrence mails again (got {r.get('status')}, {len(SENT)} emails)",
r.get("status") == "failed" and len(SENT) == 1) r.get("status") == "failed" and len(SENT) == 1)
PORTAL["rows"], PORTAL["total"] = 100, 250.0 reset()
await m.check_spend_consistency() # back to clean
# --- drift beyond tolerance --- # --- a DAY that drifts beyond the per-day tolerance is named ---
API["total"] = 400.0 API_DAYS = {"2026-11-10": 10.0, "2026-11-11": 20.0, "2026-11-12": 44.0}
SENT.clear() SENT.clear()
r = await m.check_spend_consistency() r = await m.check_spend_consistency()
check(f"a delta beyond tolerance FAILS (got {r.get('status')}, delta {r.get('delta')})", check(f"a drifting day FAILS and is named (got {r.get('status')}, {r.get('drifting_days')})",
r.get("status") == "failed" and abs(r.get("delta") - 150.0) < 0.01) r.get("status") == "failed" and bool(r.get("drifting_days"))
check("the reason names the tolerance", "tolerance" in (r.get("reason") or "")) and r["drifting_days"][0]["date"] == "2026-11-12"
API["total"] = 250.0 and "2026-11-12" in (r.get("reason") or ""))
r = await m.check_spend_consistency() check(f"the notice lists the worst day (got {len(SENT)})",
check(f"and clearing it is clean again (got {r.get('status')})", r.get("status") == "ok") len(SENT) == 1 and "2026-11-12" in SENT[0]["body"])
reset()
await m.check_spend_consistency()
# --- the API side returning nothing is not agreement --- # --- cents-level rounding is NOT drift ---
API["keys"], API["total"] = 0, 0.0 API_DAYS = {"2026-11-10": 10.02, "2026-11-11": 19.98, "2026-11-12": 30.01}
SENT.clear() SENT.clear()
r = await m.check_spend_consistency() r = await m.check_spend_consistency()
check(f"an empty API side FAILS rather than looking agreed (got {r.get('status')})", check(f"cent-level rounding passes (got {r.get('status')}, delta {r.get('delta')})",
r.get("status") == "failed" and "no independent side" in (r.get("reason") or "")) r.get("status") == "ok" and len(SENT) == 0)
API["keys"], API["total"] = 5, 250.0 reset()
# --- a read that cannot be made is UNKNOWN, not clean --- # --- no shared window: cannot compare, must NOT read as clean ---
API_DAYS = {"2026-10-01": 5.0}
r = await m.check_spend_consistency()
check(f"no shared window is skipped, not clean (got {r})",
r.get("status") == "skipped" and r.get("reason") == "no_overlapping_window")
reset()
# --- a read that cannot be made is UNKNOWN ---
FAIL_PORTAL = True FAIL_PORTAL = True
SENT.clear() SENT.clear()
r = await m.check_spend_consistency() r = await m.check_spend_consistency()
@@ -158,32 +185,32 @@ async def main():
r.get("status") == "skipped" and r.get("reason") == "litellm_api_failed") r.get("status") == "skipped" and r.get("reason") == "litellm_api_failed")
FAIL_API = False FAIL_API = False
# --- notify=False never mails and never consumes the report --- # --- notify=False never mails and never consumes ---
PORTAL["rows"], PORTAL["total"] = 0, 0.0
m.get_db()
SENT.clear()
conn = m.get_db() conn = m.get_db()
conn.execute("DELETE FROM portal_state") conn.execute("DELETE FROM portal_state")
conn.commit() conn.commit()
conn.close() conn.close()
PORTAL_DAYS, PORTAL_ROWS = {}, 0
SENT.clear()
r = await m.check_spend_consistency(notify=False) r = await m.check_spend_consistency(notify=False)
check(f"notify=False reports nothing (got {r.get('reported')}, {len(SENT)} emails)", check(f"notify=False reports nothing (got {r.get('reported')}, {len(SENT)} emails)",
r.get("status") == "failed" and r.get("reported") is False and len(SENT) == 0) r.get("status") == "failed" and r.get("reported") is False and len(SENT) == 0)
r = await m.check_spend_consistency() r = await m.check_spend_consistency()
check(f"...and a later notifying run still speaks (got {len(SENT)} emails)", check(f"...and a later notifying run still speaks (got {len(SENT)} emails)",
r.get("reported") is True and len(SENT) == 1) r.get("reported") is True and len(SENT) == 1)
PORTAL["rows"], PORTAL["total"] = 100, 250.0 reset()
# --- the parser: the API calls the field `total_spend`, not `spend` --- # --- the parser: the API's shape is per-day, per-team ---
# Reading only `spend` returned $0 for all 306 live keys on 2026-11-13, and a
# checker that trusts its own parse then reports "the portal is wrong" — the
# exact mirror of the failure this check exists to catch.
class _Resp: class _Resp:
status_code = 200 status_code = 200
def json(self): def json(self):
return [{"api_key": "a", "total_spend": 10.5}, {"api_key": "b", "spend": 2.5}, return {"daily_spend": [
{"api_key": "c"}, {"api_key": "d", "total_spend": None}] {"date": "2026-11-12", "member:a@x.org": 1.5, "member:b@x.org": 2.25},
{"date": "2026-11-13", "member:a@x.org": 0.5},
{"date": "", "member:a@x.org": 9.0}, # no date -> ignored
{"member:a@x.org": 7.0}, # no date -> ignored
]}
class _FakeClient: class _FakeClient:
def __init__(self, *a, **k): def __init__(self, *a, **k):
@@ -201,11 +228,11 @@ async def main():
real_httpx = m.httpx real_httpx = m.httpx
m.httpx = types.SimpleNamespace(AsyncClient=_FakeClient) m.httpx = types.SimpleNamespace(AsyncClient=_FakeClient)
try: try:
total, n = await REAL_API_TOTAL_SPEND() days = await REAL_API_DAILY_SPEND()
finally: finally:
m.httpx = real_httpx m.httpx = real_httpx
check(f"the API total sums `total_spend` (got {total} over {n} keys)", check(f"the parser sums each day's teams (got {days})",
abs(total - 13.0) < 1e-6 and n == 4) days == {"2026-11-12": 3.75, "2026-11-13": 0.5})
# --- the endpoint --- # --- the endpoint ---
SENT.clear() SENT.clear()
@@ -218,7 +245,7 @@ async def main():
body = r.json() body = r.json()
check(f"endpoint reports the comparison (got {body.get('status')})", check(f"endpoint reports the comparison (got {body.get('status')})",
r.status_code == 200 and body.get("status") == "ok" r.status_code == 200 and body.get("status") == "ok"
and body.get("litellm_api_total") == 250.0) and body.get("api_window") == 60.0)
check(f"endpoint: a clean result mails nothing (got {len(SENT)})", len(SENT) == 0) check(f"endpoint: a clean result mails nothing (got {len(SENT)})", len(SENT) == 0)