diff --git a/app/main.py b/app/main.py index 33c6bb3..729fad8 100644 --- a/app/main.py +++ b/app/main.py @@ -8282,10 +8282,18 @@ async def litellm_api_total_spend() -> tuple[float, int]: raise RuntimeError(f"LiteLLM spend API returned {type(rows).__name__}, not a list") total = 0.0 for row in rows: - try: - total += float((row or {}).get("spend") or 0) - except (TypeError, ValueError): - continue + row = row or {} + # LiteLLM reports each key's all-time spend as `total_spend`; older builds + # called it `spend`. Reading only `spend` returned $0 for all 306 keys on + # 2026-11-13 — a failure that a naive checker reports as "the portal is wrong". + for field in ("total_spend", "spend"): + try: + val = float(row.get(field) or 0) + except (TypeError, ValueError): + val = 0.0 + if val: + total += val + break return round(total, 6), len(rows) @@ -8326,6 +8334,15 @@ async def check_spend_consistency(notify: bool = True) -> dict: elif api_keys == 0 and portal_rows > 0: reason = (f"LiteLLM's spend API returned no keys while the portal read {portal_rows} " f"SpendLogs rows — the comparison has no independent side") + elif api_total == 0 and portal_total > 0: + # Both directions matter: an independent side that reads zero while the portal + # reads real spend is a disagreement, and on 2026-11-13 it was OUR parse (the + # API calls the field `total_spend`, not `spend`). Say both possibilities. + reason = (f"LiteLLM's spend API reports ${api_total:.2f} across {api_keys} keys while " + f"the portal reads ${portal_total:.2f} over {portal_rows} rows — either " + f"LiteLLM's own accounting is broken or this check is reading the wrong " + f"field (it must accept `total_spend`); do not believe either side until " + f"that is settled") elif abs(delta) > SPEND_CHECK_TOLERANCE: reason = (f"portal SpendLogs total ${portal_total:.2f} ({portal_rows} rows) vs LiteLLM " f"API ${api_total:.2f} over {api_keys} keys: delta ${delta:+.2f} exceeds the " diff --git a/tests/test_spend_check.py b/tests/test_spend_check.py index 90a8e73..62bd7be 100644 --- a/tests/test_spend_check.py +++ b/tests/test_spend_check.py @@ -23,6 +23,8 @@ import sys import asyncio import tempfile +import types + import httpx os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db") @@ -32,9 +34,14 @@ import app.main as m # noqa: E402 m.DB_PATH = os.environ["DB_PATH"] m.ADMIN_TOKEN = "test-admin-token" m.ADMIN_TOKEN_LEGACY = "" +m.LITELLM_MASTER_KEY = "test-master-key" # the parser test calls the real function SENT = [] +# main() replaces the module attribute with a stub; keep the REAL implementation so +# the parser itself can be exercised. +REAL_API_TOTAL_SPEND = m.litellm_api_total_spend + def fake_send(to, subject, body, *a, **k): SENT.append({"to": to, "subject": subject, "body": body}) @@ -167,6 +174,39 @@ async def main(): r.get("reported") is True and len(SENT) == 1) PORTAL["rows"], PORTAL["total"] = 100, 250.0 + # --- the parser: the API calls the field `total_spend`, not `spend` --- + # Reading only `spend` returned $0 for all 306 live keys on 2026-11-13, and a + # checker that trusts its own parse then reports "the portal is wrong" — the + # exact mirror of the failure this check exists to catch. + class _Resp: + status_code = 200 + + def json(self): + return [{"api_key": "a", "total_spend": 10.5}, {"api_key": "b", "spend": 2.5}, + {"api_key": "c"}, {"api_key": "d", "total_spend": None}] + + class _FakeClient: + def __init__(self, *a, **k): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + async def get(self, *a, **k): + return _Resp() + + real_httpx = m.httpx + m.httpx = types.SimpleNamespace(AsyncClient=_FakeClient) + try: + total, n = await REAL_API_TOTAL_SPEND() + finally: + m.httpx = real_httpx + check(f"the API total sums `total_spend` (got {total} over {n} keys)", + abs(total - 13.0) < 1e-6 and n == 4) + # --- the endpoint --- SENT.clear() transport = httpx.ASGITransport(app=m.app)