spend check: compare like for like (same window, same kind of number)
ci/woodpecker/push/woodpecker Pipeline was canceled
ci/woodpecker/push/woodpecker Pipeline was canceled
The second live run FAILED again, correctly and informatively: portal $935.52 all-time vs LiteLLM $827.89 — and the $107.63 'drift' was the WINDOW. LiteLLM's per-day view reaches back a fixed number of days (31 on 2026-11-13); our table holds 69. Over the days both sides cover the two agree to $0.07, with no day differing by more than $0.022 (the API rounds to cents). So the check now compares DAY BY DAY over the shared window, which localises drift instead of hiding it in a sum, with a per-day tolerance ($0.10) plus a window-total tolerance ($5). It reads /global/spend/teams rather than /global/spend/keys: the per-key figure follows each key's own budget cycle, so summing it is not an all-time number at all (and /global/spend/report, the all-time one, is Enterprise-gated). Degenerate reads are judged BEFORE the window overlap, because in the failure this check exists for the portal's aggregate comes back EMPTY — there would be no shared days to compare, and the overlap logic would report 'cannot compare' instead of the stale-credential signature. All failure reporting goes through one helper that mails info@ once per episode and advances the marker only after a notice is sent. 23 checks; all 20 suites green.
This commit is contained in:
1 parent
c13880cfab
commit
c92d8979eb
2 files changed
+242
-153
No files matched your search
+149
-87
@@ -8259,42 +8259,103 @@ async def reconcile_unmapped_backers(notify: bool = True) -> dict:
|
||||
# what this checks. Extracting it into a shared helper is the next step if the two
|
||||
# ever need to be compared directly.
|
||||
SPEND_CHECK_TOLERANCE = float(os.environ.get("SPEND_CHECK_TOLERANCE", "5.00") or "5.00")
|
||||
# LiteLLM's API reports spend per day per team rounded to cents, so a day can differ
|
||||
# by a cent or two legitimately (the largest seen on live data was $0.022).
|
||||
SPEND_CHECK_DAILY_TOLERANCE = float(os.environ.get("SPEND_CHECK_DAILY_TOLERANCE", "0.10") or "0.10")
|
||||
SPEND_CHECK_MARKER = "spend_check_reported"
|
||||
|
||||
|
||||
async def litellm_api_total_spend() -> tuple[float, int]:
|
||||
"""Total spend from LiteLLM's OWN HTTP API (master key). Raises on failure.
|
||||
async def litellm_api_daily_spend() -> dict[str, float]:
|
||||
"""{UTC date: $} from LiteLLM's OWN API (`/global/spend/teams`). Raises on failure.
|
||||
|
||||
`/global/spend/keys` returns one entry per API key with its spend; summing them
|
||||
is the closest HTTP equivalent of COUNT/SUM over the SpendLogs table. A different
|
||||
credential path from the portal's DB copy — which is the whole point, since the
|
||||
DB copy is the thing that goes stale.
|
||||
WHY NOT `/global/spend/keys`: its per-key `total_spend` follows the key's own
|
||||
budget cycle, so summing it compares the last cycle against our all-time figure
|
||||
($827.89 vs $935.52 on 2026-11-13 — a $107 window mismatch that looks exactly
|
||||
like drift). `/global/spend/report` would be all-time but is gated behind a
|
||||
LiteLLM Enterprise licence (HTTP 400). The per-day teams view is the comparable
|
||||
one, and comparing DAY BY DAY is sharper than comparing sums: it localises drift
|
||||
instead of hiding it in an aggregate.
|
||||
"""
|
||||
if not LITELLM_MASTER_KEY:
|
||||
raise RuntimeError("LITELLM_MASTER_KEY not set; cannot verify against LiteLLM")
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
r = await client.get(f"{LITELLM_BASE}/global/spend/keys",
|
||||
r = await client.get(f"{LITELLM_BASE}/global/spend/teams",
|
||||
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"})
|
||||
if r.status_code != 200:
|
||||
raise RuntimeError(f"LiteLLM spend API: HTTP {r.status_code}")
|
||||
rows = r.json()
|
||||
if not isinstance(rows, list):
|
||||
raise RuntimeError(f"LiteLLM spend API returned {type(rows).__name__}, not a list")
|
||||
total = 0.0
|
||||
for row in rows:
|
||||
row = row or {}
|
||||
# LiteLLM reports each key's all-time spend as `total_spend`; older builds
|
||||
# called it `spend`. Reading only `spend` returned $0 for all 306 keys on
|
||||
# 2026-11-13 — a failure that a naive checker reports as "the portal is wrong".
|
||||
for field in ("total_spend", "spend"):
|
||||
body = r.json()
|
||||
if not isinstance(body, dict):
|
||||
raise RuntimeError(f"LiteLLM spend API returned {type(body).__name__}, not an object")
|
||||
out: dict[str, float] = {}
|
||||
for day in body.get("daily_spend") or []:
|
||||
day = day or {}
|
||||
d = str(day.get("date") or "")[:10]
|
||||
if not d:
|
||||
continue
|
||||
total = 0.0
|
||||
for k, v in day.items():
|
||||
if k == "date":
|
||||
continue
|
||||
try:
|
||||
val = float(row.get(field) or 0)
|
||||
total += float(v or 0)
|
||||
except (TypeError, ValueError):
|
||||
val = 0.0
|
||||
if val:
|
||||
total += val
|
||||
break
|
||||
return round(total, 6), len(rows)
|
||||
continue
|
||||
out[d] = round(total, 6)
|
||||
return out
|
||||
|
||||
|
||||
async def _spend_failure(notify: bool, reason: str, verdict: dict) -> dict:
|
||||
"""Report a spend-check failure to info@ ONCE per failing episode.
|
||||
|
||||
The marker holds the reason a human was told, so a persistent failure does not
|
||||
mail every sweep while a changed one always does. It is written only after a
|
||||
notice actually went out: a silent (`notify=False`) run or a failed send must not
|
||||
swallow the alert a failure gets.
|
||||
"""
|
||||
verdict = dict(verdict)
|
||||
verdict["status"] = "failed"
|
||||
verdict["reason"] = reason
|
||||
verdict["tolerance"] = SPEND_CHECK_TOLERANCE
|
||||
verdict["daily_tolerance"] = SPEND_CHECK_DAILY_TOLERANCE
|
||||
verdict["reported"] = False
|
||||
conn = get_db()
|
||||
already = (_state_get(conn, SPEND_CHECK_MARKER, "") or "") == reason
|
||||
conn.close()
|
||||
if not notify or already:
|
||||
return verdict
|
||||
try:
|
||||
send_email(
|
||||
"info@inference.coop",
|
||||
"Inference Cooperative — spend figures disagree (portal vs LiteLLM)",
|
||||
"The portal's own spend aggregation and LiteLLM's API do not agree. This "
|
||||
"check runs on every sweep and is SILENT when they match.\n\n"
|
||||
f" window : {verdict.get('window')} ({verdict.get('days_compared')} day(s))\n"
|
||||
f" portal : ${float(verdict.get('portal_window') or 0):.2f}\n"
|
||||
f" LiteLLM : ${float(verdict.get('api_window') or 0):.2f}\n"
|
||||
f" delta : ${float(verdict.get('delta') or 0):+.2f} "
|
||||
f"(tolerance ${SPEND_CHECK_TOLERANCE:.2f}, per-day "
|
||||
f"${SPEND_CHECK_DAILY_TOLERANCE:.2f})\n"
|
||||
f" all-time portal: ${float(verdict.get('portal_all_time') or 0):.2f} over "
|
||||
f"{verdict.get('portal_rows')} rows\n\n"
|
||||
f"Reason: {reason}\n\n"
|
||||
"Worst days, if any:\n"
|
||||
+ ("\n".join(f" {d['date']}: portal ${d['portal']:.2f} vs LiteLLM "
|
||||
f"${d['api']:.2f} (${d['delta']:+.2f})"
|
||||
for d in (verdict.get("drifting_days") or []))
|
||||
or " (none beyond the per-day tolerance)")
|
||||
+ "\n\nThe likely cause is the portal's copy of the LiteLLM DB credential going "
|
||||
"stale after a rotation: the aggregation then fails soft and every surface "
|
||||
"shows $0 while looking clean. Check the credential in the portal's env "
|
||||
"against the gateway app's own.\n",
|
||||
)
|
||||
verdict["reported"] = True
|
||||
conn = get_db()
|
||||
_state_set(conn, SPEND_CHECK_MARKER, reason[:400])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("spend check: notice email failed: %s", e)
|
||||
return verdict
|
||||
|
||||
|
||||
async def check_spend_consistency(notify: bool = True) -> dict:
|
||||
@@ -8304,55 +8365,88 @@ async def check_spend_consistency(notify: bool = True) -> dict:
|
||||
cannot be made is `skipped`, never a clean bill: "could not look" is an anomaly.
|
||||
"""
|
||||
try:
|
||||
_by_day, portal_total, portal_rows = litellm_spend_by_day()
|
||||
portal_by_day, portal_all_time, portal_rows = litellm_spend_by_day()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("spend check: portal SpendLogs read failed: %s", e)
|
||||
return {"status": "skipped", "reason": "portal_read_failed", "detail": str(e)[:200]}
|
||||
try:
|
||||
api_total, api_keys = await litellm_api_total_spend()
|
||||
api_by_day = await litellm_api_daily_spend()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("spend check: LiteLLM API read failed: %s", e)
|
||||
return {"status": "skipped", "reason": "litellm_api_failed", "detail": str(e)[:200]}
|
||||
|
||||
delta = round(api_total - portal_total, 2)
|
||||
# Degenerate reads come FIRST: in the failure this check exists for, the portal's
|
||||
# aggregate comes back EMPTY, so there are no shared days to compare and the
|
||||
# overlap logic would call it "cannot compare" instead of naming the real problem.
|
||||
api_all_days_total = round(sum(api_by_day.values()), 2)
|
||||
if portal_rows == 0 and api_all_days_total > 1:
|
||||
reason = (f"the portal's SpendLogs aggregate is EMPTY ($0 over 0 rows) while LiteLLM's "
|
||||
f"own API reports ${api_all_days_total:.2f} over {len(api_by_day)} day(s) — "
|
||||
f"the silent-clean signature of a stale LiteLLM DB credential (the "
|
||||
f"panel/dashboard $0 is the same failure)")
|
||||
logger.error("spend check FAILED: %s", reason)
|
||||
return await _spend_failure(
|
||||
notify, reason,
|
||||
{"portal_rows": 0, "portal_window": 0.0, "api_window": api_all_days_total,
|
||||
"delta": -api_all_days_total, "window": "none (the portal read returned nothing)",
|
||||
"days_compared": 0, "portal_all_time": 0.0, "drifting_days": []})
|
||||
if not api_by_day and portal_rows > 0:
|
||||
reason = ("LiteLLM's spend API returned no days while the portal read "
|
||||
f"{portal_rows} SpendLogs rows — the comparison has no independent side")
|
||||
logger.error("spend check FAILED: %s", reason)
|
||||
return await _spend_failure(
|
||||
notify, reason,
|
||||
{"portal_rows": portal_rows, "portal_window": 0.0, "api_window": 0.0,
|
||||
"delta": 0.0, "window": "none", "days_compared": 0,
|
||||
"portal_all_time": portal_all_time, "drifting_days": []})
|
||||
|
||||
# Compare only the days BOTH sides cover. LiteLLM's per-day view reaches back a
|
||||
# fixed number of days (31 on 2026-11-13) while our table holds everything, so a
|
||||
# sum-vs-all-time comparison reports the missing window as drift — $107.63 of it.
|
||||
overlap = sorted(set(api_by_day) & set(portal_by_day))
|
||||
if not overlap:
|
||||
return {"status": "skipped", "reason": "no_overlapping_window",
|
||||
"detail": f"LiteLLM reports {len(api_by_day)} day(s), the portal "
|
||||
f"{len(portal_by_day)}; they share none"}
|
||||
|
||||
per_day = [(d, portal_by_day[d], api_by_day[d],
|
||||
round(api_by_day[d] - portal_by_day[d], 4)) for d in overlap]
|
||||
portal_window = round(sum(p for _d, p, _a, _x in per_day), 2)
|
||||
api_window = round(sum(a for _d, _p, a, _x in per_day), 2)
|
||||
delta = round(api_window - portal_window, 2)
|
||||
bad_days = sorted((x for x in per_day if abs(x[3]) > SPEND_CHECK_DAILY_TOLERANCE),
|
||||
key=lambda x: -abs(x[3]))
|
||||
verdict = {
|
||||
"portal_total": round(portal_total, 2),
|
||||
"portal_rows": portal_rows,
|
||||
"litellm_api_total": round(api_total, 2),
|
||||
"litellm_api_keys": api_keys,
|
||||
"portal_window": portal_window,
|
||||
"api_window": api_window,
|
||||
"delta": delta,
|
||||
"window": f"{overlap[0]}..{overlap[-1]}",
|
||||
"days_compared": len(overlap),
|
||||
"portal_all_time": round(portal_all_time, 2),
|
||||
"portal_rows": portal_rows,
|
||||
"drifting_days": [{"date": d, "portal": p, "api": a, "delta": x}
|
||||
for d, p, a, x in bad_days[:5]],
|
||||
"tolerance": SPEND_CHECK_TOLERANCE,
|
||||
"daily_tolerance": SPEND_CHECK_DAILY_TOLERANCE,
|
||||
}
|
||||
|
||||
reason = ""
|
||||
if portal_rows == 0 and api_total > 1:
|
||||
# The 2026-11-05 signature, verbatim: our copy of the credential went stale
|
||||
# and the aggregation collapsed to $0 while LiteLLM's own totals are real.
|
||||
reason = (f"the portal's SpendLogs aggregate is EMPTY/zero while LiteLLM's own API "
|
||||
f"reports ${api_total:.2f} — the silent-clean signature of a stale LiteLLM "
|
||||
f"DB credential (the panel/dashboard $0 is the same failure)")
|
||||
elif api_keys == 0 and portal_rows > 0:
|
||||
reason = (f"LiteLLM's spend API returned no keys while the portal read {portal_rows} "
|
||||
f"SpendLogs rows — the comparison has no independent side")
|
||||
elif api_total == 0 and portal_total > 0:
|
||||
# Both directions matter: an independent side that reads zero while the portal
|
||||
# reads real spend is a disagreement, and on 2026-11-13 it was OUR parse (the
|
||||
# API calls the field `total_spend`, not `spend`). Say both possibilities.
|
||||
reason = (f"LiteLLM's spend API reports ${api_total:.2f} across {api_keys} keys while "
|
||||
f"the portal reads ${portal_total:.2f} over {portal_rows} rows — either "
|
||||
f"LiteLLM's own accounting is broken or this check is reading the wrong "
|
||||
f"field (it must accept `total_spend`); do not believe either side until "
|
||||
f"that is settled")
|
||||
if bad_days:
|
||||
d, p, a, x = bad_days[0]
|
||||
reason = (f"{len(bad_days)} day(s) differ by more than ${SPEND_CHECK_DAILY_TOLERANCE:.2f} "
|
||||
f"across {verdict['window']} — worst {d}: portal ${p:.2f} vs LiteLLM "
|
||||
f"${a:.2f} (${x:+.2f}); the two are not reading the same data")
|
||||
elif abs(delta) > SPEND_CHECK_TOLERANCE:
|
||||
reason = (f"portal SpendLogs total ${portal_total:.2f} ({portal_rows} rows) vs LiteLLM "
|
||||
f"API ${api_total:.2f} over {api_keys} keys: delta ${delta:+.2f} exceeds the "
|
||||
f"${SPEND_CHECK_TOLERANCE:.2f} tolerance — the two are not reading the same data")
|
||||
reason = (f"window totals differ by ${delta:+.2f} (portal ${portal_window:.2f} vs "
|
||||
f"LiteLLM ${api_window:.2f} over {len(overlap)} day(s)) — beyond the "
|
||||
f"${SPEND_CHECK_TOLERANCE:.2f} tolerance")
|
||||
|
||||
if not reason:
|
||||
verdict["status"] = "ok"
|
||||
logger.info("spend check PASS: portal $%.2f (%d rows) vs LiteLLM API $%.2f (%d keys), "
|
||||
"delta $%+.2f within $%.2f", portal_total, portal_rows, api_total,
|
||||
api_keys, delta, SPEND_CHECK_TOLERANCE)
|
||||
logger.info("spend check PASS: portal $%.2f vs LiteLLM $%.2f over %s (%d days), "
|
||||
"delta $%+.2f within $%.2f; all-time portal $%.2f over %d rows",
|
||||
portal_window, api_window, verdict["window"], len(overlap), delta,
|
||||
SPEND_CHECK_TOLERANCE, portal_all_time, portal_rows)
|
||||
# A clean result clears the marker, so a recurrence is reported again.
|
||||
try:
|
||||
conn = get_db()
|
||||
@@ -8363,40 +8457,8 @@ async def check_spend_consistency(notify: bool = True) -> dict:
|
||||
logger.warning("spend check: could not clear the report marker: %s", e)
|
||||
return verdict
|
||||
|
||||
verdict["status"] = "failed"
|
||||
verdict["reason"] = reason
|
||||
logger.error("spend check FAILED: %s", reason)
|
||||
|
||||
# Reported once per failing episode: the marker holds what a human has been told.
|
||||
conn = get_db()
|
||||
reported = (_state_get(conn, SPEND_CHECK_MARKER, "") or "") == reason
|
||||
conn.close()
|
||||
verdict["reported"] = False
|
||||
if notify and not reported:
|
||||
try:
|
||||
send_email(
|
||||
"info@inference.coop",
|
||||
"Inference Cooperative — spend figures disagree (portal vs LiteLLM)",
|
||||
"The portal's own spend aggregation and LiteLLM's API do not agree. This "
|
||||
"check runs on every sweep and is SILENT when they match.\n\n"
|
||||
f" portal : ${verdict['portal_total']:.2f} over {verdict['portal_rows']} rows\n"
|
||||
f" LiteLLM : ${verdict['litellm_api_total']:.2f} over "
|
||||
f"{verdict['litellm_api_keys']} keys\n"
|
||||
f" delta : ${verdict['delta']:+.2f} (tolerance ${SPEND_CHECK_TOLERANCE:.2f})\n\n"
|
||||
f"Reason: {reason}\n\n"
|
||||
"The likely cause is the portal's copy of the LiteLLM DB credential going "
|
||||
"stale after a rotation: the aggregation then fails soft and every surface "
|
||||
"shows $0 while looking clean. Check the credential in the portal's env "
|
||||
"against the gateway app's own.\n",
|
||||
)
|
||||
verdict["reported"] = True
|
||||
conn = get_db()
|
||||
_state_set(conn, SPEND_CHECK_MARKER, reason[:400])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("spend check: notice email failed: %s", e)
|
||||
return verdict
|
||||
return await _spend_failure(notify, reason, verdict)
|
||||
|
||||
|
||||
async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> dict:
|
||||
|
||||
Reference in new issue
Block a user