spend consistency: the check moves into the portal's sweep
ci/woodpecker/push/woodpecker Pipeline was canceled
ci/woodpecker/push/woodpecker Pipeline was canceled
The failure this exists for (2026-11-05): the portal's own all-time-spend aggregate failed SOFT when its copy of the gateway's LiteLLM DB credential went stale — the query returned nothing, every surface showed $0, and nothing looked wrong, because the portal WAS the thing that was wrong. The guard ran as a daily script on the bot's host. It now runs in the portal, and keeps two genuinely independent reads: the portal's SpendLogs aggregate over its own DB credential, and LiteLLM's own HTTP API (/global/spend/keys) over the master key — a different transport AND a different credential, which is the point, since the DB copy is what goes stale. It adds the signature guard the host script could only infer from outside: an EMPTY//usr/bin/bash portal aggregate while LiteLLM reports real spend is a FAILURE, named as the stale-credential case; an API side that returns no keys is a failure too, not agreement. A read that cannot be made is skipped — never a clean bill. Silent when they agree; info@ once per failing episode, marker cleared on recovery. POST /admin/spend-check runs it on demand. 23 checks; all 20 suites green.
This commit is contained in:
1 parent
bb0a02dcd9
commit
e2eaa7ae40
2 files changed
+354
No files matched your search
+163
@@ -8239,6 +8239,149 @@ async def reconcile_unmapped_backers(notify: bool = True) -> dict:
|
||||
"detail": findings}
|
||||
|
||||
|
||||
# --- Spend-consistency check (was a script on the bot host) ----------------- #
|
||||
# The failure it exists for (2026-11-05): the portal aggregates all-time spend
|
||||
# from LiteLLM's Postgres using a COPY of the gateway's DB credential. When the
|
||||
# gateway rotated that credential the copy went stale, the aggregation failed
|
||||
# SOFT (empty -> $0), and the panel and dashboard showed $0 while every surface
|
||||
# looked clean. Nobody would have noticed from inside the portal, because the
|
||||
# portal WAS the thing that was wrong.
|
||||
#
|
||||
# The old check lived on the bot's host and read LiteLLM's own SpendLogs through
|
||||
# the gateway's credential. This port keeps the two independent reads — one over
|
||||
# the portal's DB copy, one over LiteLLM's HTTP API with the master key (a
|
||||
# different transport AND a different credential) — and adds the guard that
|
||||
# catches the silent-clean signature directly: a $0 (or empty) aggregate while
|
||||
# LiteLLM's own API reports real spend.
|
||||
#
|
||||
# It does NOT compare against the figure /admin/overview computes inline: that
|
||||
# aggregation is embedded in the handler, and the DB aggregate it is built from is
|
||||
# what this checks. Extracting it into a shared helper is the next step if the two
|
||||
# ever need to be compared directly.
|
||||
SPEND_CHECK_TOLERANCE = float(os.environ.get("SPEND_CHECK_TOLERANCE", "5.00") or "5.00")
|
||||
SPEND_CHECK_MARKER = "spend_check_reported"
|
||||
|
||||
|
||||
async def litellm_api_total_spend() -> tuple[float, int]:
|
||||
"""Total spend from LiteLLM's OWN HTTP API (master key). Raises on failure.
|
||||
|
||||
`/global/spend/keys` returns one entry per API key with its spend; summing them
|
||||
is the closest HTTP equivalent of COUNT/SUM over the SpendLogs table. A different
|
||||
credential path from the portal's DB copy — which is the whole point, since the
|
||||
DB copy is the thing that goes stale.
|
||||
"""
|
||||
if not LITELLM_MASTER_KEY:
|
||||
raise RuntimeError("LITELLM_MASTER_KEY not set; cannot verify against LiteLLM")
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
r = await client.get(f"{LITELLM_BASE}/global/spend/keys",
|
||||
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"})
|
||||
if r.status_code != 200:
|
||||
raise RuntimeError(f"LiteLLM spend API: HTTP {r.status_code}")
|
||||
rows = r.json()
|
||||
if not isinstance(rows, list):
|
||||
raise RuntimeError(f"LiteLLM spend API returned {type(rows).__name__}, not a list")
|
||||
total = 0.0
|
||||
for row in rows:
|
||||
try:
|
||||
total += float((row or {}).get("spend") or 0)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return round(total, 6), len(rows)
|
||||
|
||||
|
||||
async def check_spend_consistency(notify: bool = True) -> dict:
|
||||
"""Compare the portal's SpendLogs aggregate with LiteLLM's own API. Emails only on FAIL.
|
||||
|
||||
Returns status="ok" with a verdict, or "failed" with the reason — and a read that
|
||||
cannot be made is `skipped`, never a clean bill: "could not look" is an anomaly.
|
||||
"""
|
||||
try:
|
||||
_by_day, portal_total, portal_rows = litellm_spend_by_day()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("spend check: portal SpendLogs read failed: %s", e)
|
||||
return {"status": "skipped", "reason": "portal_read_failed", "detail": str(e)[:200]}
|
||||
try:
|
||||
api_total, api_keys = await litellm_api_total_spend()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("spend check: LiteLLM API read failed: %s", e)
|
||||
return {"status": "skipped", "reason": "litellm_api_failed", "detail": str(e)[:200]}
|
||||
|
||||
delta = round(api_total - portal_total, 2)
|
||||
verdict = {
|
||||
"portal_total": round(portal_total, 2),
|
||||
"portal_rows": portal_rows,
|
||||
"litellm_api_total": round(api_total, 2),
|
||||
"litellm_api_keys": api_keys,
|
||||
"delta": delta,
|
||||
"tolerance": SPEND_CHECK_TOLERANCE,
|
||||
}
|
||||
|
||||
reason = ""
|
||||
if portal_rows == 0 and api_total > 1:
|
||||
# The 2026-11-05 signature, verbatim: our copy of the credential went stale
|
||||
# and the aggregation collapsed to $0 while LiteLLM's own totals are real.
|
||||
reason = (f"the portal's SpendLogs aggregate is EMPTY/zero while LiteLLM's own API "
|
||||
f"reports ${api_total:.2f} — the silent-clean signature of a stale LiteLLM "
|
||||
f"DB credential (the panel/dashboard $0 is the same failure)")
|
||||
elif api_keys == 0 and portal_rows > 0:
|
||||
reason = (f"LiteLLM's spend API returned no keys while the portal read {portal_rows} "
|
||||
f"SpendLogs rows — the comparison has no independent side")
|
||||
elif abs(delta) > SPEND_CHECK_TOLERANCE:
|
||||
reason = (f"portal SpendLogs total ${portal_total:.2f} ({portal_rows} rows) vs LiteLLM "
|
||||
f"API ${api_total:.2f} over {api_keys} keys: delta ${delta:+.2f} exceeds the "
|
||||
f"${SPEND_CHECK_TOLERANCE:.2f} tolerance — the two are not reading the same data")
|
||||
|
||||
if not reason:
|
||||
verdict["status"] = "ok"
|
||||
logger.info("spend check PASS: portal $%.2f (%d rows) vs LiteLLM API $%.2f (%d keys), "
|
||||
"delta $%+.2f within $%.2f", portal_total, portal_rows, api_total,
|
||||
api_keys, delta, SPEND_CHECK_TOLERANCE)
|
||||
# A clean result clears the marker, so a recurrence is reported again.
|
||||
try:
|
||||
conn = get_db()
|
||||
_state_set(conn, SPEND_CHECK_MARKER, "")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("spend check: could not clear the report marker: %s", e)
|
||||
return verdict
|
||||
|
||||
verdict["status"] = "failed"
|
||||
verdict["reason"] = reason
|
||||
logger.error("spend check FAILED: %s", reason)
|
||||
|
||||
# Reported once per failing episode: the marker holds what a human has been told.
|
||||
conn = get_db()
|
||||
reported = (_state_get(conn, SPEND_CHECK_MARKER, "") or "") == reason
|
||||
conn.close()
|
||||
verdict["reported"] = False
|
||||
if notify and not reported:
|
||||
try:
|
||||
send_email(
|
||||
"info@inference.coop",
|
||||
"Inference Cooperative — spend figures disagree (portal vs LiteLLM)",
|
||||
"The portal's own spend aggregation and LiteLLM's API do not agree. This "
|
||||
"check runs on every sweep and is SILENT when they match.\n\n"
|
||||
f" portal : ${verdict['portal_total']:.2f} over {verdict['portal_rows']} rows\n"
|
||||
f" LiteLLM : ${verdict['litellm_api_total']:.2f} over "
|
||||
f"{verdict['litellm_api_keys']} keys\n"
|
||||
f" delta : ${verdict['delta']:+.2f} (tolerance ${SPEND_CHECK_TOLERANCE:.2f})\n\n"
|
||||
f"Reason: {reason}\n\n"
|
||||
"The likely cause is the portal's copy of the LiteLLM DB credential going "
|
||||
"stale after a rotation: the aggregation then fails soft and every surface "
|
||||
"shows $0 while looking clean. Check the credential in the portal's env "
|
||||
"against the gateway app's own.\n",
|
||||
)
|
||||
verdict["reported"] = True
|
||||
conn = get_db()
|
||||
_state_set(conn, SPEND_CHECK_MARKER, reason[:400])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("spend check: notice email failed: %s", e)
|
||||
return verdict
|
||||
|
||||
|
||||
async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> dict:
|
||||
"""Credit PAID credit packs that never reached our books (issue #7).
|
||||
|
||||
@@ -8647,6 +8790,15 @@ async def admin_sweep(request: Request):
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result["unmapped_backers"] = {"error": str(exc)[:200]}
|
||||
|
||||
# Spend consistency: the portal's SpendLogs aggregate vs LiteLLM's own API.
|
||||
# Silent when they agree; the reason it exists is that the portal's copy of the
|
||||
# LiteLLM DB credential once went stale and every surface showed $0 while looking
|
||||
# clean. Was a daily script on the bot's host.
|
||||
try:
|
||||
result["spend_check"] = await check_spend_consistency()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result["spend_check"] = {"error": str(exc)[:200]}
|
||||
|
||||
# Deliveries the portal refused. The portal records these itself (see
|
||||
# record_webhook_failure) and reports them here ONCE per batch, so the alert
|
||||
# for a failed delivery belongs to the system rather than to a log scraper on
|
||||
@@ -8689,6 +8841,17 @@ async def admin_unmapped_check(request: Request, notify: int = 1):
|
||||
return await reconcile_unmapped_backers(notify=bool(notify))
|
||||
|
||||
|
||||
@app.post("/admin/spend-check")
|
||||
@limiter.limit("20/minute")
|
||||
async def admin_spend_check(request: Request, notify: int = 1):
|
||||
"""Compare the portal's spend aggregate with LiteLLM's own API (X-Admin-Token).
|
||||
|
||||
Silent when they agree; `?notify=0` runs it without any chance of a notice.
|
||||
"""
|
||||
_verify_admin_token(request)
|
||||
return await check_spend_consistency(notify=bool(notify))
|
||||
|
||||
|
||||
def _service_sync_pending() -> list[str]:
|
||||
"""Active members still waiting on their Loomio invitation.
|
||||
|
||||
|
||||
Reference in new issue
Block a user