spend consistency: the check moves into the portal's sweep
ci/woodpecker/push/woodpecker Pipeline was canceled

The failure this exists for (2026-11-05): the portal's own all-time-spend aggregate
failed SOFT when its copy of the gateway's LiteLLM DB credential went stale — the
query returned nothing, every surface showed $0, and nothing looked wrong, because
the portal WAS the thing that was wrong. The guard ran as a daily script on the
bot's host.

It now runs in the portal, and keeps two genuinely independent reads: the portal's
SpendLogs aggregate over its own DB credential, and LiteLLM's own HTTP API
(/global/spend/keys) over the master key — a different transport AND a different
credential, which is the point, since the DB copy is what goes stale.

It adds the signature guard the host script could only infer from outside: an
EMPTY//usr/bin/bash portal aggregate while LiteLLM reports real spend is a FAILURE, named as
the stale-credential case; an API side that returns no keys is a failure too, not
agreement. A read that cannot be made is skipped — never a clean bill. Silent when
they agree; info@ once per failing episode, marker cleared on recovery.
POST /admin/spend-check runs it on demand. 23 checks; all 20 suites green.
This commit is contained in:
inference-admin committed 2026-11-13 20:41:27 +00:00
1 parent bb0a02dcd9
commit e2eaa7ae40
2 files changed
+354

No files matched your search

+163
View File
@@ -8239,6 +8239,149 @@ async def reconcile_unmapped_backers(notify: bool = True) -> dict:
"detail": findings}
# --- Spend-consistency check (was a script on the bot host) ----------------- #
# The failure it exists for (2026-11-05): the portal aggregates all-time spend
# from LiteLLM's Postgres using a COPY of the gateway's DB credential. When the
# gateway rotated that credential the copy went stale, the aggregation failed
# SOFT (empty -> $0), and the panel and dashboard showed $0 while every surface
# looked clean. Nobody would have noticed from inside the portal, because the
# portal WAS the thing that was wrong.
#
# The old check lived on the bot's host and read LiteLLM's own SpendLogs through
# the gateway's credential. This port keeps the two independent reads — one over
# the portal's DB copy, one over LiteLLM's HTTP API with the master key (a
# different transport AND a different credential) — and adds the guard that
# catches the silent-clean signature directly: a $0 (or empty) aggregate while
# LiteLLM's own API reports real spend.
#
# It does NOT compare against the figure /admin/overview computes inline: that
# aggregation is embedded in the handler, and the DB aggregate it is built from is
# what this checks. Extracting it into a shared helper is the next step if the two
# ever need to be compared directly.
SPEND_CHECK_TOLERANCE = float(os.environ.get("SPEND_CHECK_TOLERANCE", "5.00") or "5.00")
SPEND_CHECK_MARKER = "spend_check_reported"
async def litellm_api_total_spend() -> tuple[float, int]:
"""Total spend from LiteLLM's OWN HTTP API (master key). Raises on failure.
`/global/spend/keys` returns one entry per API key with its spend; summing them
is the closest HTTP equivalent of COUNT/SUM over the SpendLogs table. A different
credential path from the portal's DB copy — which is the whole point, since the
DB copy is the thing that goes stale.
"""
if not LITELLM_MASTER_KEY:
raise RuntimeError("LITELLM_MASTER_KEY not set; cannot verify against LiteLLM")
async with httpx.AsyncClient(timeout=30.0) as client:
r = await client.get(f"{LITELLM_BASE}/global/spend/keys",
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"})
if r.status_code != 200:
raise RuntimeError(f"LiteLLM spend API: HTTP {r.status_code}")
rows = r.json()
if not isinstance(rows, list):
raise RuntimeError(f"LiteLLM spend API returned {type(rows).__name__}, not a list")
total = 0.0
for row in rows:
try:
total += float((row or {}).get("spend") or 0)
except (TypeError, ValueError):
continue
return round(total, 6), len(rows)
async def check_spend_consistency(notify: bool = True) -> dict:
"""Compare the portal's SpendLogs aggregate with LiteLLM's own API. Emails only on FAIL.
Returns status="ok" with a verdict, or "failed" with the reason — and a read that
cannot be made is `skipped`, never a clean bill: "could not look" is an anomaly.
"""
try:
_by_day, portal_total, portal_rows = litellm_spend_by_day()
except Exception as e: # noqa: BLE001
logger.warning("spend check: portal SpendLogs read failed: %s", e)
return {"status": "skipped", "reason": "portal_read_failed", "detail": str(e)[:200]}
try:
api_total, api_keys = await litellm_api_total_spend()
except Exception as e: # noqa: BLE001
logger.warning("spend check: LiteLLM API read failed: %s", e)
return {"status": "skipped", "reason": "litellm_api_failed", "detail": str(e)[:200]}
delta = round(api_total - portal_total, 2)
verdict = {
"portal_total": round(portal_total, 2),
"portal_rows": portal_rows,
"litellm_api_total": round(api_total, 2),
"litellm_api_keys": api_keys,
"delta": delta,
"tolerance": SPEND_CHECK_TOLERANCE,
}
reason = ""
if portal_rows == 0 and api_total > 1:
# The 2026-11-05 signature, verbatim: our copy of the credential went stale
# and the aggregation collapsed to $0 while LiteLLM's own totals are real.
reason = (f"the portal's SpendLogs aggregate is EMPTY/zero while LiteLLM's own API "
f"reports ${api_total:.2f} — the silent-clean signature of a stale LiteLLM "
f"DB credential (the panel/dashboard $0 is the same failure)")
elif api_keys == 0 and portal_rows > 0:
reason = (f"LiteLLM's spend API returned no keys while the portal read {portal_rows} "
f"SpendLogs rows — the comparison has no independent side")
elif abs(delta) > SPEND_CHECK_TOLERANCE:
reason = (f"portal SpendLogs total ${portal_total:.2f} ({portal_rows} rows) vs LiteLLM "
f"API ${api_total:.2f} over {api_keys} keys: delta ${delta:+.2f} exceeds the "
f"${SPEND_CHECK_TOLERANCE:.2f} tolerance — the two are not reading the same data")
if not reason:
verdict["status"] = "ok"
logger.info("spend check PASS: portal $%.2f (%d rows) vs LiteLLM API $%.2f (%d keys), "
"delta $%+.2f within $%.2f", portal_total, portal_rows, api_total,
api_keys, delta, SPEND_CHECK_TOLERANCE)
# A clean result clears the marker, so a recurrence is reported again.
try:
conn = get_db()
_state_set(conn, SPEND_CHECK_MARKER, "")
conn.commit()
conn.close()
except Exception as e: # noqa: BLE001
logger.warning("spend check: could not clear the report marker: %s", e)
return verdict
verdict["status"] = "failed"
verdict["reason"] = reason
logger.error("spend check FAILED: %s", reason)
# Reported once per failing episode: the marker holds what a human has been told.
conn = get_db()
reported = (_state_get(conn, SPEND_CHECK_MARKER, "") or "") == reason
conn.close()
verdict["reported"] = False
if notify and not reported:
try:
send_email(
"info@inference.coop",
"Inference Cooperative — spend figures disagree (portal vs LiteLLM)",
"The portal's own spend aggregation and LiteLLM's API do not agree. This "
"check runs on every sweep and is SILENT when they match.\n\n"
f" portal : ${verdict['portal_total']:.2f} over {verdict['portal_rows']} rows\n"
f" LiteLLM : ${verdict['litellm_api_total']:.2f} over "
f"{verdict['litellm_api_keys']} keys\n"
f" delta : ${verdict['delta']:+.2f} (tolerance ${SPEND_CHECK_TOLERANCE:.2f})\n\n"
f"Reason: {reason}\n\n"
"The likely cause is the portal's copy of the LiteLLM DB credential going "
"stale after a rotation: the aggregation then fails soft and every surface "
"shows $0 while looking clean. Check the credential in the portal's env "
"against the gateway app's own.\n",
)
verdict["reported"] = True
conn = get_db()
_state_set(conn, SPEND_CHECK_MARKER, reason[:400])
conn.commit()
conn.close()
except Exception as e: # noqa: BLE001
logger.warning("spend check: notice email failed: %s", e)
return verdict
async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> dict:
"""Credit PAID credit packs that never reached our books (issue #7).
@@ -8647,6 +8790,15 @@ async def admin_sweep(request: Request):
except Exception as exc: # noqa: BLE001
result["unmapped_backers"] = {"error": str(exc)[:200]}
# Spend consistency: the portal's SpendLogs aggregate vs LiteLLM's own API.
# Silent when they agree; the reason it exists is that the portal's copy of the
# LiteLLM DB credential once went stale and every surface showed $0 while looking
# clean. Was a daily script on the bot's host.
try:
result["spend_check"] = await check_spend_consistency()
except Exception as exc: # noqa: BLE001
result["spend_check"] = {"error": str(exc)[:200]}
# Deliveries the portal refused. The portal records these itself (see
# record_webhook_failure) and reports them here ONCE per batch, so the alert
# for a failed delivery belongs to the system rather than to a log scraper on
@@ -8689,6 +8841,17 @@ async def admin_unmapped_check(request: Request, notify: int = 1):
return await reconcile_unmapped_backers(notify=bool(notify))
@app.post("/admin/spend-check")
@limiter.limit("20/minute")
async def admin_spend_check(request: Request, notify: int = 1):
"""Compare the portal's spend aggregate with LiteLLM's own API (X-Admin-Token).
Silent when they agree; `?notify=0` runs it without any chance of a notice.
"""
_verify_admin_token(request)
return await check_spend_consistency(notify=bool(notify))
def _service_sync_pending() -> list[str]:
"""Active members still waiting on their Loomio invitation.
+191
View File
@@ -0,0 +1,191 @@
#!/usr/bin/env python3
"""Regression test: the spend-consistency check (portal aggregate vs LiteLLM's API).
WHY THIS EXISTS
On 2026-11-05 the portal's aggregation of all-time spend failed SOFT: its copy of
the gateway's LiteLLM DB credential had gone stale, so the query returned nothing,
every surface showed $0, and nothing looked wrong — the portal WAS the thing that
was wrong. The guard ran as a daily script on the bot's host. It now runs in the
portal's sweep.
WHAT IS PINNED DOWN HERE
* two independent reads are actually two: the portal's SpendLogs aggregate over
its own DB credential, and LiteLLM's HTTP API over the master key,
* agreement is silent (no email), disagreement and the $0-while-real-spend
signature email info@ exactly once per failing episode,
* a read that cannot be made is `skipped` — never a clean bill of health,
* a clean result clears the marker so a recurrence is reported again.
Runnable: python3 tests/test_spend_check.py (from the repo root)
"""
import os
import sys
import asyncio
import tempfile
import httpx
os.environ["DB_PATH"] = os.path.join(tempfile.mkdtemp(), "fixture.db")
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import app.main as m # noqa: E402
m.DB_PATH = os.environ["DB_PATH"]
m.ADMIN_TOKEN = "test-admin-token"
m.ADMIN_TOKEN_LEGACY = ""
SENT = []
def fake_send(to, subject, body, *a, **k):
SENT.append({"to": to, "subject": subject, "body": body})
# --- stubs ----------------------------------------------------------------- #
PORTAL = {"rows": 100, "total": 250.0}
API = {"total": 250.0, "keys": 5}
FAIL_PORTAL = False
FAIL_API = False
def fake_portal_spend():
if FAIL_PORTAL:
raise RuntimeError("psycopg2: connection refused")
return {"2026-11-01": PORTAL["total"]}, PORTAL["total"], PORTAL["rows"]
async def fake_api_spend():
if FAIL_API:
raise RuntimeError("LiteLLM spend API: HTTP 401")
return API["total"], API["keys"]
results = []
def check(name, ok):
results.append((name, bool(ok)))
async def main():
global FAIL_PORTAL, FAIL_API
m.litellm_spend_by_day = fake_portal_spend
m.litellm_api_total_spend = fake_api_spend
m.send_email = fake_send
m.get_db().close() # create + migrate the fixture schema
def marker():
conn = m.get_db()
row = conn.execute("SELECT value FROM portal_state WHERE key = ?",
(m.SPEND_CHECK_MARKER,)).fetchone()
conn.close()
return row[0] if row else None
# --- agreement: silent ---
SENT.clear()
r = await m.check_spend_consistency()
check(f"agreement is ok (got {r.get('status')})", r.get("status") == "ok")
check(f"agreement reports both sides (got {r})",
r.get("portal_total") == 250.0 and r.get("litellm_api_total") == 250.0
and r.get("portal_rows") == 100 and r.get("litellm_api_keys") == 5)
check(f"agreement sends NO email (got {len(SENT)})", len(SENT) == 0)
check(f"agreement clears the marker (got {marker()!r})", not marker())
# --- the 2026-11-05 signature: portal $0 / empty while LiteLLM is real ---
PORTAL["rows"], PORTAL["total"] = 0, 0.0
SENT.clear()
r = await m.check_spend_consistency()
check(f"a $0 portal aggregate against real spend FAILS (got {r.get('status')})",
r.get("status") == "failed")
check("the reason names the stale-credential signature",
"silent-clean signature" in (r.get("reason") or ""))
check(f"info@ is told once (got {len(SENT)})",
len(SENT) == 1 and SENT[0]["to"] == "info@inference.coop"
and "credential" in SENT[0]["body"])
r2 = await m.check_spend_consistency()
check(f"the SAME failure is not re-mailed (got {r2.get('reported')}, {len(SENT)} emails)",
r2.get("reported") is False and len(SENT) == 1)
check(f"the marker holds what was reported (got {str(marker())[:50]})", bool(marker()))
# --- a fixed failure: clean again, marker cleared, recurrence speaks ---
PORTAL["rows"], PORTAL["total"] = 100, 250.0
r = await m.check_spend_consistency()
check(f"recovery is clean (got {r.get('status')})", r.get("status") == "ok")
check(f"recovery clears the marker (got {marker()!r})", not marker())
PORTAL["rows"], PORTAL["total"] = 0, 0.0
SENT.clear()
r = await m.check_spend_consistency()
check(f"a recurrence after a clean run mails again (got {len(SENT)})",
r.get("status") == "failed" and len(SENT) == 1)
PORTAL["rows"], PORTAL["total"] = 100, 250.0
# --- drift beyond tolerance ---
API["total"] = 400.0
SENT.clear()
r = await m.check_spend_consistency()
check(f"a delta beyond tolerance FAILS (got {r.get('status')}, delta {r.get('delta')})",
r.get("status") == "failed" and abs(r.get("delta") - 150.0) < 0.01)
check("the reason names the tolerance", "tolerance" in (r.get("reason") or ""))
API["total"] = 250.0
r = await m.check_spend_consistency()
check(f"and clearing it is clean again (got {r.get('status')})", r.get("status") == "ok")
# --- the API side returning nothing is not agreement ---
API["keys"], API["total"] = 0, 0.0
SENT.clear()
r = await m.check_spend_consistency()
check(f"an empty API side FAILS rather than looking agreed (got {r.get('status')})",
r.get("status") == "failed" and "no independent side" in (r.get("reason") or ""))
API["keys"], API["total"] = 5, 250.0
# --- a read that cannot be made is UNKNOWN, not clean ---
FAIL_PORTAL = True
SENT.clear()
r = await m.check_spend_consistency()
check(f"an unreadable portal side is skipped (got {r})",
r.get("status") == "skipped" and r.get("reason") == "portal_read_failed"
and len(SENT) == 0)
FAIL_PORTAL = False
FAIL_API = True
r = await m.check_spend_consistency()
check(f"an unreadable LiteLLM side is skipped (got {r})",
r.get("status") == "skipped" and r.get("reason") == "litellm_api_failed")
FAIL_API = False
# --- notify=False never mails and never consumes the report ---
PORTAL["rows"], PORTAL["total"] = 0, 0.0
m.get_db()
SENT.clear()
conn = m.get_db()
conn.execute("DELETE FROM portal_state")
conn.commit()
conn.close()
r = await m.check_spend_consistency(notify=False)
check(f"notify=False reports nothing (got {r.get('reported')}, {len(SENT)} emails)",
r.get("status") == "failed" and r.get("reported") is False and len(SENT) == 0)
r = await m.check_spend_consistency()
check(f"...and a later notifying run still speaks (got {len(SENT)} emails)",
r.get("reported") is True and len(SENT) == 1)
PORTAL["rows"], PORTAL["total"] = 100, 250.0
# --- the endpoint ---
SENT.clear()
transport = httpx.ASGITransport(app=m.app)
async with httpx.AsyncClient(transport=transport, base_url="http://t") as c:
r = await c.post("/admin/spend-check")
check(f"endpoint needs the admin token (got {r.status_code})",
r.status_code in (401, 403))
r = await c.post("/admin/spend-check", headers={"X-Admin-Token": "test-admin-token"})
body = r.json()
check(f"endpoint reports the comparison (got {body.get('status')})",
r.status_code == 200 and body.get("status") == "ok"
and body.get("litellm_api_total") == 250.0)
check(f"endpoint: a clean result mails nothing (got {len(SENT)})", len(SENT) == 0)
asyncio.run(main())
fails = sum(not ok for _, ok in results)
for name, ok in results:
print(f" [{'PASS' if ok else 'FAIL'}] {name}")
print(f"\n{len(results) - fails}/{len(results)} passed")
sys.exit(1 if fails else 0)