spend check: read the field LiteLLM actually sends (total_spend), and fail loudly either way
ci/woodpecker/push/woodpecker Pipeline was canceled
ci/woodpecker/push/woodpecker Pipeline was canceled
The first live run FAILED with api $0.00 over 306 keys against the portal's $935.35. The disagreement was real, the verdict was right, and the cause was mine: LiteLLM's /global/spend/keys returns each key's all-time spend as `total_spend`, not `spend`. A checker that trusts its own parse then reports "the portal is wrong" — the exact mirror of the failure this check exists to catch, which is why the guard now names both possibilities when the independent side reads zero against real portal spend. Parser accepts either field name; a regression test exercises the real function against a stubbed httpx (the module attribute is stubbed for the other checks, so the real one is captured at import). 24 checks; all 20 suites green.
This commit is contained in:
1 parent
1c1fced5ba
commit
c0ef046b0d
2 files changed
+59
-2
No files matched your search
+19
-2
@@ -8282,10 +8282,18 @@ async def litellm_api_total_spend() -> tuple[float, int]:
|
||||
raise RuntimeError(f"LiteLLM spend API returned {type(rows).__name__}, not a list")
|
||||
total = 0.0
|
||||
for row in rows:
|
||||
row = row or {}
|
||||
# LiteLLM reports each key's all-time spend as `total_spend`; older builds
|
||||
# called it `spend`. Reading only `spend` returned $0 for all 306 keys on
|
||||
# 2026-11-13 — a failure that a naive checker reports as "the portal is wrong".
|
||||
for field in ("total_spend", "spend"):
|
||||
try:
|
||||
total += float((row or {}).get("spend") or 0)
|
||||
val = float(row.get(field) or 0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
val = 0.0
|
||||
if val:
|
||||
total += val
|
||||
break
|
||||
return round(total, 6), len(rows)
|
||||
|
||||
|
||||
@@ -8326,6 +8334,15 @@ async def check_spend_consistency(notify: bool = True) -> dict:
|
||||
elif api_keys == 0 and portal_rows > 0:
|
||||
reason = (f"LiteLLM's spend API returned no keys while the portal read {portal_rows} "
|
||||
f"SpendLogs rows — the comparison has no independent side")
|
||||
elif api_total == 0 and portal_total > 0:
|
||||
# Both directions matter: an independent side that reads zero while the portal
|
||||
# reads real spend is a disagreement, and on 2026-11-13 it was OUR parse (the
|
||||
# API calls the field `total_spend`, not `spend`). Say both possibilities.
|
||||
reason = (f"LiteLLM's spend API reports ${api_total:.2f} across {api_keys} keys while "
|
||||
f"the portal reads ${portal_total:.2f} over {portal_rows} rows — either "
|
||||
f"LiteLLM's own accounting is broken or this check is reading the wrong "
|
||||
f"field (it must accept `total_spend`); do not believe either side until "
|
||||
f"that is settled")
|
||||
elif abs(delta) > SPEND_CHECK_TOLERANCE:
|
||||
reason = (f"portal SpendLogs total ${portal_total:.2f} ({portal_rows} rows) vs LiteLLM "
|
||||
f"API ${api_total:.2f} over {api_keys} keys: delta ${delta:+.2f} exceeds the "
|
||||
|
||||
@@ -23,6 +23,8 @@ import sys
|
||||
import asyncio
|
||||
import tempfile
|
||||
|
||||
import types
|
||||
|
||||
import httpx
|
||||
|
||||
os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db")
|
||||
@@ -32,9 +34,14 @@ import app.main as m # noqa: E402
|
||||
m.DB_PATH = os.environ["DB_PATH"]
|
||||
m.ADMIN_TOKEN = "test-admin-token"
|
||||
m.ADMIN_TOKEN_LEGACY = ""
|
||||
m.LITELLM_MASTER_KEY = "test-master-key" # the parser test calls the real function
|
||||
|
||||
SENT = []
|
||||
|
||||
# main() replaces the module attribute with a stub; keep the REAL implementation so
|
||||
# the parser itself can be exercised.
|
||||
REAL_API_TOTAL_SPEND = m.litellm_api_total_spend
|
||||
|
||||
|
||||
def fake_send(to, subject, body, *a, **k):
|
||||
SENT.append({"to": to, "subject": subject, "body": body})
|
||||
@@ -167,6 +174,39 @@ async def main():
|
||||
r.get("reported") is True and len(SENT) == 1)
|
||||
PORTAL["rows"], PORTAL["total"] = 100, 250.0
|
||||
|
||||
# --- the parser: the API calls the field `total_spend`, not `spend` ---
|
||||
# Reading only `spend` returned $0 for all 306 live keys on 2026-11-13, and a
|
||||
# checker that trusts its own parse then reports "the portal is wrong" — the
|
||||
# exact mirror of the failure this check exists to catch.
|
||||
class _Resp:
|
||||
status_code = 200
|
||||
|
||||
def json(self):
|
||||
return [{"api_key": "a", "total_spend": 10.5}, {"api_key": "b", "spend": 2.5},
|
||||
{"api_key": "c"}, {"api_key": "d", "total_spend": None}]
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, *a, **k):
|
||||
pass
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *a):
|
||||
return False
|
||||
|
||||
async def get(self, *a, **k):
|
||||
return _Resp()
|
||||
|
||||
real_httpx = m.httpx
|
||||
m.httpx = types.SimpleNamespace(AsyncClient=_FakeClient)
|
||||
try:
|
||||
total, n = await REAL_API_TOTAL_SPEND()
|
||||
finally:
|
||||
m.httpx = real_httpx
|
||||
check(f"the API total sums `total_spend` (got {total} over {n} keys)",
|
||||
abs(total - 13.0) < 1e-6 and n == 4)
|
||||
|
||||
# --- the endpoint ---
|
||||
SENT.clear()
|
||||
transport = httpx.ASGITransport(app=m.app)
|
||||
|
||||
Reference in new issue
Block a user