spend check: read the field LiteLLM actually sends (total_spend), and fail loudly either way
ci/woodpecker/push/woodpecker Pipeline was canceled
ci/woodpecker/push/woodpecker Pipeline was canceled
The first live run FAILED with api $0.00 over 306 keys against the portal's $935.35. The disagreement was real, the verdict was right, and the cause was mine: LiteLLM's /global/spend/keys returns each key's all-time spend as `total_spend`, not `spend`. A checker that trusts its own parse then reports "the portal is wrong" — the exact mirror of the failure this check exists to catch, which is why the guard now names both possibilities when the independent side reads zero against real portal spend. Parser accepts either field name; a regression test exercises the real function against a stubbed httpx (the module attribute is stubbed for the other checks, so the real one is captured at import). 24 checks; all 20 suites green.
This commit is contained in:
1 parent
1c1fced5ba
commit
c0ef046b0d
2 files changed
+61
-4
No files matched your search
@@ -23,6 +23,8 @@ import sys
|
||||
import asyncio
|
||||
import tempfile
|
||||
|
||||
import types
|
||||
|
||||
import httpx
|
||||
|
||||
os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db")
|
||||
@@ -32,9 +34,14 @@ import app.main as m # noqa: E402
|
||||
m.DB_PATH = os.environ["DB_PATH"]
|
||||
m.ADMIN_TOKEN = "test-admin-token"
|
||||
m.ADMIN_TOKEN_LEGACY = ""
|
||||
m.LITELLM_MASTER_KEY = "test-master-key" # the parser test calls the real function
|
||||
|
||||
SENT = []
|
||||
|
||||
# main() replaces the module attribute with a stub; keep the REAL implementation so
|
||||
# the parser itself can be exercised.
|
||||
REAL_API_TOTAL_SPEND = m.litellm_api_total_spend
|
||||
|
||||
|
||||
def fake_send(to, subject, body, *a, **k):
|
||||
SENT.append({"to": to, "subject": subject, "body": body})
|
||||
@@ -167,6 +174,39 @@ async def main():
|
||||
r.get("reported") is True and len(SENT) == 1)
|
||||
PORTAL["rows"], PORTAL["total"] = 100, 250.0
|
||||
|
||||
# --- the parser: the API calls the field `total_spend`, not `spend` ---
|
||||
# Reading only `spend` returned $0 for all 306 live keys on 2026-11-13, and a
|
||||
# checker that trusts its own parse then reports "the portal is wrong" — the
|
||||
# exact mirror of the failure this check exists to catch.
|
||||
class _Resp:
|
||||
status_code = 200
|
||||
|
||||
def json(self):
|
||||
return [{"api_key": "a", "total_spend": 10.5}, {"api_key": "b", "spend": 2.5},
|
||||
{"api_key": "c"}, {"api_key": "d", "total_spend": None}]
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, *a, **k):
|
||||
pass
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *a):
|
||||
return False
|
||||
|
||||
async def get(self, *a, **k):
|
||||
return _Resp()
|
||||
|
||||
real_httpx = m.httpx
|
||||
m.httpx = types.SimpleNamespace(AsyncClient=_FakeClient)
|
||||
try:
|
||||
total, n = await REAL_API_TOTAL_SPEND()
|
||||
finally:
|
||||
m.httpx = real_httpx
|
||||
check(f"the API total sums `total_spend` (got {total} over {n} keys)",
|
||||
abs(total - 13.0) < 1e-6 and n == 4)
|
||||
|
||||
# --- the endpoint ---
|
||||
SENT.clear()
|
||||
transport = httpx.ASGITransport(app=m.app)
|
||||
|
||||
Reference in new issue
Block a user