From c92d8979ebca0292528dac1e11b759b4c90223be Mon Sep 17 00:00:00 2001 From: Inferencebot Date: Fri, 13 Nov 2026 20:46:50 +0000 Subject: [PATCH] spend check: compare like for like (same window, same kind of number) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The second live run FAILED again, correctly and informatively: portal $935.52 all-time vs LiteLLM $827.89 — and the $107.63 'drift' was the WINDOW. LiteLLM's per-day view reaches back a fixed number of days (31 on 2026-11-13); our table holds 69. Over the days both sides cover the two agree to $0.07, with no day differing by more than $0.022 (the API rounds to cents). So the check now compares DAY BY DAY over the shared window, which localises drift instead of hiding it in a sum, with a per-day tolerance ($0.10) plus a window-total tolerance ($5). It reads /global/spend/teams rather than /global/spend/keys: the per-key figure follows each key's own budget cycle, so summing it is not an all-time number at all (and /global/spend/report, the all-time one, is Enterprise-gated). Degenerate reads are judged BEFORE the window overlap, because in the failure this check exists for the portal's aggregate comes back EMPTY — there would be no shared days to compare, and the overlap logic would report 'cannot compare' instead of the stale-credential signature. All failure reporting goes through one helper that mails info@ once per episode and advances the marker only after a notice is sent. 23 checks; all 20 suites green. --- app/main.py | 236 ++++++++++++++++++++++++-------------- tests/test_spend_check.py | 159 ++++++++++++++----------- 2 files changed, 242 insertions(+), 153 deletions(-) diff --git a/app/main.py b/app/main.py index 729fad8..cc59041 100644 --- a/app/main.py +++ b/app/main.py @@ -8259,42 +8259,103 @@ async def reconcile_unmapped_backers(notify: bool = True) -> dict: # what this checks. Extracting it into a shared helper is the next step if the two # ever need to be compared directly. SPEND_CHECK_TOLERANCE = float(os.environ.get("SPEND_CHECK_TOLERANCE", "5.00") or "5.00") +# LiteLLM's API reports spend per day per team rounded to cents, so a day can differ +# by a cent or two legitimately (the largest seen on live data was $0.022). +SPEND_CHECK_DAILY_TOLERANCE = float(os.environ.get("SPEND_CHECK_DAILY_TOLERANCE", "0.10") or "0.10") SPEND_CHECK_MARKER = "spend_check_reported" -async def litellm_api_total_spend() -> tuple[float, int]: - """Total spend from LiteLLM's OWN HTTP API (master key). Raises on failure. +async def litellm_api_daily_spend() -> dict[str, float]: + """{UTC date: $} from LiteLLM's OWN API (`/global/spend/teams`). Raises on failure. - `/global/spend/keys` returns one entry per API key with its spend; summing them - is the closest HTTP equivalent of COUNT/SUM over the SpendLogs table. A different - credential path from the portal's DB copy — which is the whole point, since the - DB copy is the thing that goes stale. + WHY NOT `/global/spend/keys`: its per-key `total_spend` follows the key's own + budget cycle, so summing it compares the last cycle against our all-time figure + ($827.89 vs $935.52 on 2026-11-13 — a $107 window mismatch that looks exactly + like drift). `/global/spend/report` would be all-time but is gated behind a + LiteLLM Enterprise licence (HTTP 400). The per-day teams view is the comparable + one, and comparing DAY BY DAY is sharper than comparing sums: it localises drift + instead of hiding it in an aggregate. """ if not LITELLM_MASTER_KEY: raise RuntimeError("LITELLM_MASTER_KEY not set; cannot verify against LiteLLM") async with httpx.AsyncClient(timeout=30.0) as client: - r = await client.get(f"{LITELLM_BASE}/global/spend/keys", + r = await client.get(f"{LITELLM_BASE}/global/spend/teams", headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"}) if r.status_code != 200: raise RuntimeError(f"LiteLLM spend API: HTTP {r.status_code}") - rows = r.json() - if not isinstance(rows, list): - raise RuntimeError(f"LiteLLM spend API returned {type(rows).__name__}, not a list") - total = 0.0 - for row in rows: - row = row or {} - # LiteLLM reports each key's all-time spend as `total_spend`; older builds - # called it `spend`. Reading only `spend` returned $0 for all 306 keys on - # 2026-11-13 — a failure that a naive checker reports as "the portal is wrong". - for field in ("total_spend", "spend"): + body = r.json() + if not isinstance(body, dict): + raise RuntimeError(f"LiteLLM spend API returned {type(body).__name__}, not an object") + out: dict[str, float] = {} + for day in body.get("daily_spend") or []: + day = day or {} + d = str(day.get("date") or "")[:10] + if not d: + continue + total = 0.0 + for k, v in day.items(): + if k == "date": + continue try: - val = float(row.get(field) or 0) + total += float(v or 0) except (TypeError, ValueError): - val = 0.0 - if val: - total += val - break - return round(total, 6), len(rows) + continue + out[d] = round(total, 6) + return out + + +async def _spend_failure(notify: bool, reason: str, verdict: dict) -> dict: + """Report a spend-check failure to info@ ONCE per failing episode. + + The marker holds the reason a human was told, so a persistent failure does not + mail every sweep while a changed one always does. It is written only after a + notice actually went out: a silent (`notify=False`) run or a failed send must not + swallow the alert a failure gets. + """ + verdict = dict(verdict) + verdict["status"] = "failed" + verdict["reason"] = reason + verdict["tolerance"] = SPEND_CHECK_TOLERANCE + verdict["daily_tolerance"] = SPEND_CHECK_DAILY_TOLERANCE + verdict["reported"] = False + conn = get_db() + already = (_state_get(conn, SPEND_CHECK_MARKER, "") or "") == reason + conn.close() + if not notify or already: + return verdict + try: + send_email( + "info@inference.coop", + "Inference Cooperative — spend figures disagree (portal vs LiteLLM)", + "The portal's own spend aggregation and LiteLLM's API do not agree. This " + "check runs on every sweep and is SILENT when they match.\n\n" + f" window : {verdict.get('window')} ({verdict.get('days_compared')} day(s))\n" + f" portal : ${float(verdict.get('portal_window') or 0):.2f}\n" + f" LiteLLM : ${float(verdict.get('api_window') or 0):.2f}\n" + f" delta : ${float(verdict.get('delta') or 0):+.2f} " + f"(tolerance ${SPEND_CHECK_TOLERANCE:.2f}, per-day " + f"${SPEND_CHECK_DAILY_TOLERANCE:.2f})\n" + f" all-time portal: ${float(verdict.get('portal_all_time') or 0):.2f} over " + f"{verdict.get('portal_rows')} rows\n\n" + f"Reason: {reason}\n\n" + "Worst days, if any:\n" + + ("\n".join(f" {d['date']}: portal ${d['portal']:.2f} vs LiteLLM " + f"${d['api']:.2f} (${d['delta']:+.2f})" + for d in (verdict.get("drifting_days") or [])) + or " (none beyond the per-day tolerance)") + + "\n\nThe likely cause is the portal's copy of the LiteLLM DB credential going " + "stale after a rotation: the aggregation then fails soft and every surface " + "shows $0 while looking clean. Check the credential in the portal's env " + "against the gateway app's own.\n", + ) + verdict["reported"] = True + conn = get_db() + _state_set(conn, SPEND_CHECK_MARKER, reason[:400]) + conn.commit() + conn.close() + except Exception as e: # noqa: BLE001 + logger.warning("spend check: notice email failed: %s", e) + return verdict async def check_spend_consistency(notify: bool = True) -> dict: @@ -8304,55 +8365,88 @@ async def check_spend_consistency(notify: bool = True) -> dict: cannot be made is `skipped`, never a clean bill: "could not look" is an anomaly. """ try: - _by_day, portal_total, portal_rows = litellm_spend_by_day() + portal_by_day, portal_all_time, portal_rows = litellm_spend_by_day() except Exception as e: # noqa: BLE001 logger.warning("spend check: portal SpendLogs read failed: %s", e) return {"status": "skipped", "reason": "portal_read_failed", "detail": str(e)[:200]} try: - api_total, api_keys = await litellm_api_total_spend() + api_by_day = await litellm_api_daily_spend() except Exception as e: # noqa: BLE001 logger.warning("spend check: LiteLLM API read failed: %s", e) return {"status": "skipped", "reason": "litellm_api_failed", "detail": str(e)[:200]} - delta = round(api_total - portal_total, 2) + # Degenerate reads come FIRST: in the failure this check exists for, the portal's + # aggregate comes back EMPTY, so there are no shared days to compare and the + # overlap logic would call it "cannot compare" instead of naming the real problem. + api_all_days_total = round(sum(api_by_day.values()), 2) + if portal_rows == 0 and api_all_days_total > 1: + reason = (f"the portal's SpendLogs aggregate is EMPTY ($0 over 0 rows) while LiteLLM's " + f"own API reports ${api_all_days_total:.2f} over {len(api_by_day)} day(s) — " + f"the silent-clean signature of a stale LiteLLM DB credential (the " + f"panel/dashboard $0 is the same failure)") + logger.error("spend check FAILED: %s", reason) + return await _spend_failure( + notify, reason, + {"portal_rows": 0, "portal_window": 0.0, "api_window": api_all_days_total, + "delta": -api_all_days_total, "window": "none (the portal read returned nothing)", + "days_compared": 0, "portal_all_time": 0.0, "drifting_days": []}) + if not api_by_day and portal_rows > 0: + reason = ("LiteLLM's spend API returned no days while the portal read " + f"{portal_rows} SpendLogs rows — the comparison has no independent side") + logger.error("spend check FAILED: %s", reason) + return await _spend_failure( + notify, reason, + {"portal_rows": portal_rows, "portal_window": 0.0, "api_window": 0.0, + "delta": 0.0, "window": "none", "days_compared": 0, + "portal_all_time": portal_all_time, "drifting_days": []}) + + # Compare only the days BOTH sides cover. LiteLLM's per-day view reaches back a + # fixed number of days (31 on 2026-11-13) while our table holds everything, so a + # sum-vs-all-time comparison reports the missing window as drift — $107.63 of it. + overlap = sorted(set(api_by_day) & set(portal_by_day)) + if not overlap: + return {"status": "skipped", "reason": "no_overlapping_window", + "detail": f"LiteLLM reports {len(api_by_day)} day(s), the portal " + f"{len(portal_by_day)}; they share none"} + + per_day = [(d, portal_by_day[d], api_by_day[d], + round(api_by_day[d] - portal_by_day[d], 4)) for d in overlap] + portal_window = round(sum(p for _d, p, _a, _x in per_day), 2) + api_window = round(sum(a for _d, _p, a, _x in per_day), 2) + delta = round(api_window - portal_window, 2) + bad_days = sorted((x for x in per_day if abs(x[3]) > SPEND_CHECK_DAILY_TOLERANCE), + key=lambda x: -abs(x[3])) verdict = { - "portal_total": round(portal_total, 2), - "portal_rows": portal_rows, - "litellm_api_total": round(api_total, 2), - "litellm_api_keys": api_keys, + "portal_window": portal_window, + "api_window": api_window, "delta": delta, + "window": f"{overlap[0]}..{overlap[-1]}", + "days_compared": len(overlap), + "portal_all_time": round(portal_all_time, 2), + "portal_rows": portal_rows, + "drifting_days": [{"date": d, "portal": p, "api": a, "delta": x} + for d, p, a, x in bad_days[:5]], "tolerance": SPEND_CHECK_TOLERANCE, + "daily_tolerance": SPEND_CHECK_DAILY_TOLERANCE, } reason = "" - if portal_rows == 0 and api_total > 1: - # The 2026-11-05 signature, verbatim: our copy of the credential went stale - # and the aggregation collapsed to $0 while LiteLLM's own totals are real. - reason = (f"the portal's SpendLogs aggregate is EMPTY/zero while LiteLLM's own API " - f"reports ${api_total:.2f} — the silent-clean signature of a stale LiteLLM " - f"DB credential (the panel/dashboard $0 is the same failure)") - elif api_keys == 0 and portal_rows > 0: - reason = (f"LiteLLM's spend API returned no keys while the portal read {portal_rows} " - f"SpendLogs rows — the comparison has no independent side") - elif api_total == 0 and portal_total > 0: - # Both directions matter: an independent side that reads zero while the portal - # reads real spend is a disagreement, and on 2026-11-13 it was OUR parse (the - # API calls the field `total_spend`, not `spend`). Say both possibilities. - reason = (f"LiteLLM's spend API reports ${api_total:.2f} across {api_keys} keys while " - f"the portal reads ${portal_total:.2f} over {portal_rows} rows — either " - f"LiteLLM's own accounting is broken or this check is reading the wrong " - f"field (it must accept `total_spend`); do not believe either side until " - f"that is settled") + if bad_days: + d, p, a, x = bad_days[0] + reason = (f"{len(bad_days)} day(s) differ by more than ${SPEND_CHECK_DAILY_TOLERANCE:.2f} " + f"across {verdict['window']} — worst {d}: portal ${p:.2f} vs LiteLLM " + f"${a:.2f} (${x:+.2f}); the two are not reading the same data") elif abs(delta) > SPEND_CHECK_TOLERANCE: - reason = (f"portal SpendLogs total ${portal_total:.2f} ({portal_rows} rows) vs LiteLLM " - f"API ${api_total:.2f} over {api_keys} keys: delta ${delta:+.2f} exceeds the " - f"${SPEND_CHECK_TOLERANCE:.2f} tolerance — the two are not reading the same data") + reason = (f"window totals differ by ${delta:+.2f} (portal ${portal_window:.2f} vs " + f"LiteLLM ${api_window:.2f} over {len(overlap)} day(s)) — beyond the " + f"${SPEND_CHECK_TOLERANCE:.2f} tolerance") if not reason: verdict["status"] = "ok" - logger.info("spend check PASS: portal $%.2f (%d rows) vs LiteLLM API $%.2f (%d keys), " - "delta $%+.2f within $%.2f", portal_total, portal_rows, api_total, - api_keys, delta, SPEND_CHECK_TOLERANCE) + logger.info("spend check PASS: portal $%.2f vs LiteLLM $%.2f over %s (%d days), " + "delta $%+.2f within $%.2f; all-time portal $%.2f over %d rows", + portal_window, api_window, verdict["window"], len(overlap), delta, + SPEND_CHECK_TOLERANCE, portal_all_time, portal_rows) # A clean result clears the marker, so a recurrence is reported again. try: conn = get_db() @@ -8363,40 +8457,8 @@ async def check_spend_consistency(notify: bool = True) -> dict: logger.warning("spend check: could not clear the report marker: %s", e) return verdict - verdict["status"] = "failed" - verdict["reason"] = reason logger.error("spend check FAILED: %s", reason) - - # Reported once per failing episode: the marker holds what a human has been told. - conn = get_db() - reported = (_state_get(conn, SPEND_CHECK_MARKER, "") or "") == reason - conn.close() - verdict["reported"] = False - if notify and not reported: - try: - send_email( - "info@inference.coop", - "Inference Cooperative — spend figures disagree (portal vs LiteLLM)", - "The portal's own spend aggregation and LiteLLM's API do not agree. This " - "check runs on every sweep and is SILENT when they match.\n\n" - f" portal : ${verdict['portal_total']:.2f} over {verdict['portal_rows']} rows\n" - f" LiteLLM : ${verdict['litellm_api_total']:.2f} over " - f"{verdict['litellm_api_keys']} keys\n" - f" delta : ${verdict['delta']:+.2f} (tolerance ${SPEND_CHECK_TOLERANCE:.2f})\n\n" - f"Reason: {reason}\n\n" - "The likely cause is the portal's copy of the LiteLLM DB credential going " - "stale after a rotation: the aggregation then fails soft and every surface " - "shows $0 while looking clean. Check the credential in the portal's env " - "against the gateway app's own.\n", - ) - verdict["reported"] = True - conn = get_db() - _state_set(conn, SPEND_CHECK_MARKER, reason[:400]) - conn.commit() - conn.close() - except Exception as e: # noqa: BLE001 - logger.warning("spend check: notice email failed: %s", e) - return verdict + return await _spend_failure(notify, reason, verdict) async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> dict: diff --git a/tests/test_spend_check.py b/tests/test_spend_check.py index 62bd7be..34f22b5 100644 --- a/tests/test_spend_check.py +++ b/tests/test_spend_check.py @@ -8,23 +8,26 @@ WHY THIS EXISTS was wrong. The guard ran as a daily script on the bot's host. It now runs in the portal's sweep. -WHAT IS PINNED DOWN HERE - * two independent reads are actually two: the portal's SpendLogs aggregate over - its own DB credential, and LiteLLM's HTTP API over the master key, - * agreement is silent (no email), disagreement and the $0-while-real-spend - signature email info@ exactly once per failing episode, - * a read that cannot be made is `skipped` — never a clean bill of health, - * a clean result clears the marker so a recurrence is reported again. + The first TWO live runs also taught the shape of the comparison, and both lessons + are pinned here: + + * window. LiteLLM's per-day view reaches back a fixed number of days (31 on + 2026-11-13) while our table holds everything, so comparing our ALL-TIME figure + against it reported $107.63 of "drift" that was only the missing window. Compare + the days BOTH sides cover. + * like-for-like source. `/global/spend/keys` reports each key's spend over the + key's own budget cycle (`total_spend` — and reading `spend` returned $0 for all + 306 keys), so it is NOT an all-time figure. The per-day teams view is comparable, + and comparing day by day localises drift instead of hiding it in a sum. Runnable: python3 tests/test_spend_check.py (from the repo root) """ import os import sys +import types import asyncio import tempfile -import types - import httpx os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db") @@ -34,13 +37,13 @@ import app.main as m # noqa: E402 m.DB_PATH = os.environ["DB_PATH"] m.ADMIN_TOKEN = "test-admin-token" m.ADMIN_TOKEN_LEGACY = "" -m.LITELLM_MASTER_KEY = "test-master-key" # the parser test calls the real function +m.LITELLM_MASTER_KEY = "test-master-key" # the parser test calls the real function SENT = [] # main() replaces the module attribute with a stub; keep the REAL implementation so # the parser itself can be exercised. -REAL_API_TOTAL_SPEND = m.litellm_api_total_spend +REAL_API_DAILY_SPEND = m.litellm_api_daily_spend def fake_send(to, subject, body, *a, **k): @@ -48,22 +51,24 @@ def fake_send(to, subject, body, *a, **k): # --- stubs ----------------------------------------------------------------- # -PORTAL = {"rows": 100, "total": 250.0} -API = {"total": 250.0, "keys": 5} +DAYS = {"2026-11-10": 10.0, "2026-11-11": 20.0, "2026-11-12": 30.0} +API_DAYS = dict(DAYS) +PORTAL_DAYS = dict(DAYS) +PORTAL_ROWS = 1000 FAIL_PORTAL = False FAIL_API = False -def fake_portal_spend(): +def fake_portal_days(): if FAIL_PORTAL: raise RuntimeError("psycopg2: connection refused") - return {"2026-11-01": PORTAL["total"]}, PORTAL["total"], PORTAL["rows"] + return dict(PORTAL_DAYS), round(sum(PORTAL_DAYS.values()), 6), PORTAL_ROWS -async def fake_api_spend(): +async def fake_api_days(): if FAIL_API: raise RuntimeError("LiteLLM spend API: HTTP 401") - return API["total"], API["keys"] + return dict(API_DAYS) results = [] @@ -74,9 +79,9 @@ def check(name, ok): async def main(): - global FAIL_PORTAL, FAIL_API - m.litellm_spend_by_day = fake_portal_spend - m.litellm_api_total_spend = fake_api_spend + global FAIL_PORTAL, FAIL_API, PORTAL_DAYS, PORTAL_ROWS, API_DAYS + m.litellm_spend_by_day = fake_portal_days + m.litellm_api_daily_spend = fake_api_days m.send_email = fake_send m.get_db().close() # create + migrate the fixture schema @@ -87,64 +92,86 @@ async def main(): conn.close() return row[0] if row else None - # --- agreement: silent --- + def reset(): + global PORTAL_DAYS, PORTAL_ROWS, API_DAYS + PORTAL_DAYS, PORTAL_ROWS, API_DAYS = dict(DAYS), 1000, dict(DAYS) + + # --- agreement over the shared window: silent --- + reset() SENT.clear() r = await m.check_spend_consistency() check(f"agreement is ok (got {r.get('status')})", r.get("status") == "ok") - check(f"agreement reports both sides (got {r})", - r.get("portal_total") == 250.0 and r.get("litellm_api_total") == 250.0 - and r.get("portal_rows") == 100 and r.get("litellm_api_keys") == 5) + check(f"agreement reports the shared window (got {r})", + r.get("window") == "2026-11-10..2026-11-12" and r.get("days_compared") == 3 + and r.get("portal_window") == 60.0 and r.get("api_window") == 60.0) check(f"agreement sends NO email (got {len(SENT)})", len(SENT) == 0) check(f"agreement clears the marker (got {marker()!r})", not marker()) - # --- the 2026-11-05 signature: portal $0 / empty while LiteLLM is real --- - PORTAL["rows"], PORTAL["total"] = 0, 0.0 + # --- days only ONE side has are NOT drift --- + PORTAL_DAYS = {"2026-09-04": 99.0, **DAYS} # 69-day history vs a 31-day view SENT.clear() r = await m.check_spend_consistency() - check(f"a $0 portal aggregate against real spend FAILS (got {r.get('status')})", - r.get("status") == "failed") - check("the reason names the stale-credential signature", - "silent-clean signature" in (r.get("reason") or "")) + check(f"a longer portal history is not reported as drift (got {r.get('status')}, " + f"all-time {r.get('portal_all_time')}, delta {r.get('delta')})", + r.get("status") == "ok" and r.get("portal_all_time") == 159.0 + and r.get("delta") == 0.0 and len(SENT) == 0) + reset() + + # --- the 2026-11-05 signature: portal empty while LiteLLM is real --- + PORTAL_DAYS, PORTAL_ROWS = {}, 0 + SENT.clear() + r = await m.check_spend_consistency() + check(f"an EMPTY portal aggregate against real spend FAILS (got {r.get('status')})", + r.get("status") == "failed" and "silent-clean signature" in (r.get("reason") or "")) check(f"info@ is told once (got {len(SENT)})", len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop" and "credential" in SENT[0]["body"]) r2 = await m.check_spend_consistency() - check(f"the SAME failure is not re-mailed (got {r2.get('reported')}, {len(SENT)} emails)", + check(f"the SAME failure is not re-mailed (got {len(SENT)} emails)", r2.get("reported") is False and len(SENT) == 1) - check(f"the marker holds what was reported (got {str(marker())[:50]})", bool(marker())) - # --- a fixed failure: clean again, marker cleared, recurrence speaks --- - PORTAL["rows"], PORTAL["total"] = 100, 250.0 + # --- recovery is clean again, and a recurrence speaks --- + PORTAL_DAYS, PORTAL_ROWS = dict(DAYS), 1000 r = await m.check_spend_consistency() check(f"recovery is clean (got {r.get('status')})", r.get("status") == "ok") check(f"recovery clears the marker (got {marker()!r})", not marker()) - PORTAL["rows"], PORTAL["total"] = 0, 0.0 + PORTAL_DAYS, PORTAL_ROWS = {}, 0 SENT.clear() r = await m.check_spend_consistency() - check(f"a recurrence after a clean run mails again (got {len(SENT)})", + check(f"a recurrence mails again (got {r.get('status')}, {len(SENT)} emails)", r.get("status") == "failed" and len(SENT) == 1) - PORTAL["rows"], PORTAL["total"] = 100, 250.0 + reset() + await m.check_spend_consistency() # back to clean - # --- drift beyond tolerance --- - API["total"] = 400.0 + # --- a DAY that drifts beyond the per-day tolerance is named --- + API_DAYS = {"2026-11-10": 10.0, "2026-11-11": 20.0, "2026-11-12": 44.0} SENT.clear() r = await m.check_spend_consistency() - check(f"a delta beyond tolerance FAILS (got {r.get('status')}, delta {r.get('delta')})", - r.get("status") == "failed" and abs(r.get("delta") - 150.0) < 0.01) - check("the reason names the tolerance", "tolerance" in (r.get("reason") or "")) - API["total"] = 250.0 - r = await m.check_spend_consistency() - check(f"and clearing it is clean again (got {r.get('status')})", r.get("status") == "ok") + check(f"a drifting day FAILS and is named (got {r.get('status')}, {r.get('drifting_days')})", + r.get("status") == "failed" and bool(r.get("drifting_days")) + and r["drifting_days"][0]["date"] == "2026-11-12" + and "2026-11-12" in (r.get("reason") or "")) + check(f"the notice lists the worst day (got {len(SENT)})", + len(SENT) == 1 and "2026-11-12" in SENT[0]["body"]) + reset() + await m.check_spend_consistency() - # --- the API side returning nothing is not agreement --- - API["keys"], API["total"] = 0, 0.0 + # --- cents-level rounding is NOT drift --- + API_DAYS = {"2026-11-10": 10.02, "2026-11-11": 19.98, "2026-11-12": 30.01} SENT.clear() r = await m.check_spend_consistency() - check(f"an empty API side FAILS rather than looking agreed (got {r.get('status')})", - r.get("status") == "failed" and "no independent side" in (r.get("reason") or "")) - API["keys"], API["total"] = 5, 250.0 + check(f"cent-level rounding passes (got {r.get('status')}, delta {r.get('delta')})", + r.get("status") == "ok" and len(SENT) == 0) + reset() - # --- a read that cannot be made is UNKNOWN, not clean --- + # --- no shared window: cannot compare, must NOT read as clean --- + API_DAYS = {"2026-10-01": 5.0} + r = await m.check_spend_consistency() + check(f"no shared window is skipped, not clean (got {r})", + r.get("status") == "skipped" and r.get("reason") == "no_overlapping_window") + reset() + + # --- a read that cannot be made is UNKNOWN --- FAIL_PORTAL = True SENT.clear() r = await m.check_spend_consistency() @@ -158,32 +185,32 @@ async def main(): r.get("status") == "skipped" and r.get("reason") == "litellm_api_failed") FAIL_API = False - # --- notify=False never mails and never consumes the report --- - PORTAL["rows"], PORTAL["total"] = 0, 0.0 - m.get_db() - SENT.clear() + # --- notify=False never mails and never consumes --- conn = m.get_db() conn.execute("DELETE FROM portal_state") conn.commit() conn.close() + PORTAL_DAYS, PORTAL_ROWS = {}, 0 + SENT.clear() r = await m.check_spend_consistency(notify=False) check(f"notify=False reports nothing (got {r.get('reported')}, {len(SENT)} emails)", r.get("status") == "failed" and r.get("reported") is False and len(SENT) == 0) r = await m.check_spend_consistency() check(f"...and a later notifying run still speaks (got {len(SENT)} emails)", r.get("reported") is True and len(SENT) == 1) - PORTAL["rows"], PORTAL["total"] = 100, 250.0 + reset() - # --- the parser: the API calls the field `total_spend`, not `spend` --- - # Reading only `spend` returned $0 for all 306 live keys on 2026-11-13, and a - # checker that trusts its own parse then reports "the portal is wrong" — the - # exact mirror of the failure this check exists to catch. + # --- the parser: the API's shape is per-day, per-team --- class _Resp: status_code = 200 def json(self): - return [{"api_key": "a", "total_spend": 10.5}, {"api_key": "b", "spend": 2.5}, - {"api_key": "c"}, {"api_key": "d", "total_spend": None}] + return {"daily_spend": [ + {"date": "2026-11-12", "member:a@x.org": 1.5, "member:b@x.org": 2.25}, + {"date": "2026-11-13", "member:a@x.org": 0.5}, + {"date": "", "member:a@x.org": 9.0}, # no date -> ignored + {"member:a@x.org": 7.0}, # no date -> ignored + ]} class _FakeClient: def __init__(self, *a, **k): @@ -201,11 +228,11 @@ async def main(): real_httpx = m.httpx m.httpx = types.SimpleNamespace(AsyncClient=_FakeClient) try: - total, n = await REAL_API_TOTAL_SPEND() + days = await REAL_API_DAILY_SPEND() finally: m.httpx = real_httpx - check(f"the API total sums `total_spend` (got {total} over {n} keys)", - abs(total - 13.0) < 1e-6 and n == 4) + check(f"the parser sums each day's teams (got {days})", + days == {"2026-11-12": 3.75, "2026-11-13": 0.5}) # --- the endpoint --- SENT.clear() @@ -218,7 +245,7 @@ async def main(): body = r.json() check(f"endpoint reports the comparison (got {body.get('status')})", r.status_code == 200 and body.get("status") == "ok" - and body.get("litellm_api_total") == 250.0) + and body.get("api_window") == 60.0) check(f"endpoint: a clean result mails nothing (got {len(SENT)})", len(SENT) == 0)