broker: per-member daily usage series in /broker/usage (daily: day/spend/requests)
ci/woodpecker/push/woodpecker Pipeline was successful
ci/woodpecker/push/woodpecker Pipeline was successful
The member dashboard's usage chart reads per-day spend + requests for the signed-in member, bounded by the existing days param (0 = all time, the unchanged default). Same source and day convention as the co-op daily series (to_char(startTime)); interior zero days are explicit so chart spacing is true; requests excludes failed attempts; spend sums every row. Additive and backward compatible: every existing field untouched. Tested in tests/test_broker_usage_daily.py (route table, SQL shape, fallback twin, fail-soft degradation).
This commit is contained in:
1 parent
afffd52dec
commit
3c64b150db
2 files changed
+301
No files matched your search
+105
@@ -587,6 +587,96 @@ def aggregate_model_usage(days: int = 0, team_id: str | None = None) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _fill_daily_days(rows: list[dict]) -> list[dict]:
|
||||||
|
"""Fill calendar days WITHOUT usage rows with spend 0 / requests 0 between
|
||||||
|
the first and last measured day, so the chart's x spacing is true: a day
|
||||||
|
with no requests IS a zero-usage day (measured), not an unmeasured gap —
|
||||||
|
the fail-closed NULL rule is for days the SOURCE could not read, which
|
||||||
|
these are not (a GROUP BY simply omits them)."""
|
||||||
|
if not rows:
|
||||||
|
return rows
|
||||||
|
from datetime import date, timedelta
|
||||||
|
start = date.fromisoformat(rows[0]["day"])
|
||||||
|
end = date.fromisoformat(rows[-1]["day"])
|
||||||
|
by_day = {r["day"]: r for r in rows}
|
||||||
|
out = []
|
||||||
|
d = start
|
||||||
|
while d <= end:
|
||||||
|
key = d.isoformat()
|
||||||
|
out.append(by_day.get(key) or {"day": key, "spend": 0.0, "requests": 0})
|
||||||
|
d += timedelta(days=1)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def daily_series_from_rows(rows) -> list[dict]:
|
||||||
|
"""Per-day usage series from spend-log rows ALREADY filtered to one team —
|
||||||
|
the HTTP fallback's client-side twin of member_daily_usage. spend sums every
|
||||||
|
row; requests counts only non-failure rows (same rules as the DB path)."""
|
||||||
|
daily_map: dict[str, dict] = {}
|
||||||
|
for row in rows if isinstance(rows, list) else []:
|
||||||
|
day = str(row.get("startTime") or "")[:10]
|
||||||
|
if not day:
|
||||||
|
continue
|
||||||
|
d = daily_map.setdefault(day, {"spend": 0.0, "requests": 0})
|
||||||
|
d["spend"] += float(row.get("spend") or 0.0)
|
||||||
|
if str(row.get("status") or "").lower() != "failure":
|
||||||
|
d["requests"] += 1
|
||||||
|
return _fill_daily_days([
|
||||||
|
{"day": k, "spend": round(v["spend"], 6), "requests": v["requests"]}
|
||||||
|
for k, v in sorted(daily_map.items())
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
|
def member_daily_usage(days: int = 0, team_id: str | None = None) -> list[dict]:
|
||||||
|
"""The member's own per-day usage (spend + requests), for the dashboard's
|
||||||
|
usage chart. Same source and day convention as the co-op daily series
|
||||||
|
(metrics_snapshot.spend_by_day): to_char("startTime", 'YYYY-MM-DD'), so the
|
||||||
|
two series agree on which day a request belongs to.
|
||||||
|
|
||||||
|
`days` bounds the window (0 = all time). spend sums EVERY log row (a call
|
||||||
|
that failed after tokens were consumed still cost money); `requests` counts
|
||||||
|
only metered calls that succeeded, like aggregate_model_usage.
|
||||||
|
"""
|
||||||
|
if not litellm_db_ready():
|
||||||
|
raise RuntimeError("LiteLLM DB credentials not configured")
|
||||||
|
|
||||||
|
where = []
|
||||||
|
params: list = []
|
||||||
|
if days and days > 0:
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).strftime(
|
||||||
|
"%Y-%m-%d %H:%M:%S"
|
||||||
|
)
|
||||||
|
where.append("\"startTime\" >= %s")
|
||||||
|
params.append(cutoff)
|
||||||
|
if team_id:
|
||||||
|
where.append("team_id = %s")
|
||||||
|
params.append(team_id)
|
||||||
|
where_sql = ("WHERE " + " AND ".join(where)) if where else ""
|
||||||
|
|
||||||
|
sql = f"""
|
||||||
|
SELECT to_char("startTime", 'YYYY-MM-DD') AS day,
|
||||||
|
COALESCE(SUM(spend), 0) AS spend,
|
||||||
|
COUNT(*) FILTER (WHERE status IS DISTINCT FROM 'failure') AS requests
|
||||||
|
FROM "LiteLLM_SpendLogs"
|
||||||
|
{where_sql}
|
||||||
|
GROUP BY 1
|
||||||
|
ORDER BY 1
|
||||||
|
"""
|
||||||
|
|
||||||
|
conn = _pg_connect()
|
||||||
|
try:
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
cur.execute(sql, params)
|
||||||
|
rows = cur.fetchall()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
return _fill_daily_days([
|
||||||
|
{"day": r[0], "spend": round(float(r[1] or 0.0), 6), "requests": int(r[2] or 0)}
|
||||||
|
for r in rows
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
def get_team_budget_from_db(team_id: str) -> dict | None:
|
def get_team_budget_from_db(team_id: str) -> dict | None:
|
||||||
"""Fetch a team's spend/budget/reset directly from LiteLLM's Postgres.
|
"""Fetch a team's spend/budget/reset directly from LiteLLM's Postgres.
|
||||||
|
|
||||||
@@ -5882,6 +5972,14 @@ async def broker_get_usage(email: str, days: int = 0) -> dict:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("broker_get_usage: per-key spend failed: %s", e)
|
logger.warning("broker_get_usage: per-key spend failed: %s", e)
|
||||||
key_spend = []
|
key_spend = []
|
||||||
|
# Per-day series for the member's usage chart (30/90/all-time
|
||||||
|
# toggles re-render client-side from this). Best-effort like
|
||||||
|
# key_spend: a failure degrades to an empty series, never an error.
|
||||||
|
try:
|
||||||
|
daily = member_daily_usage(days=days, team_id=team_id)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("broker_get_usage: daily usage query failed: %s", e)
|
||||||
|
daily = []
|
||||||
return {
|
return {
|
||||||
"email": email,
|
"email": email,
|
||||||
"guest": is_guest_account(email),
|
"guest": is_guest_account(email),
|
||||||
@@ -5896,6 +5994,7 @@ async def broker_get_usage(email: str, days: int = 0) -> dict:
|
|||||||
"reset_at": reset_at,
|
"reset_at": reset_at,
|
||||||
"models": model_list,
|
"models": model_list,
|
||||||
"key_spend": key_spend,
|
"key_spend": key_spend,
|
||||||
|
"daily": daily,
|
||||||
}
|
}
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("broker_get_usage: DB path failed (%s); falling back to HTTP", e)
|
logger.warning("broker_get_usage: DB path failed (%s); falling back to HTTP", e)
|
||||||
@@ -5923,6 +6022,7 @@ async def broker_get_usage(email: str, days: int = 0) -> dict:
|
|||||||
# filter client-side on the returned rows.
|
# filter client-side on the returned rows.
|
||||||
models = {}
|
models = {}
|
||||||
total_tokens = 0
|
total_tokens = 0
|
||||||
|
team_rows: list = []
|
||||||
try:
|
try:
|
||||||
r2 = await client.get(
|
r2 = await client.get(
|
||||||
f"{LITELLM_BASE}/spend/logs",
|
f"{LITELLM_BASE}/spend/logs",
|
||||||
@@ -5935,6 +6035,7 @@ async def broker_get_usage(email: str, days: int = 0) -> dict:
|
|||||||
for row in rows if isinstance(rows, list) else []:
|
for row in rows if isinstance(rows, list) else []:
|
||||||
if row.get("team_id") != team_id:
|
if row.get("team_id") != team_id:
|
||||||
continue
|
continue
|
||||||
|
team_rows.append(row)
|
||||||
# Prefer model_group: it carries our provider/model alias (e.g.
|
# Prefer model_group: it carries our provider/model alias (e.g.
|
||||||
# greenpt/glm-5.3-flash) when present; the raw `model` field holds
|
# greenpt/glm-5.3-flash) when present; the raw `model` field holds
|
||||||
# messy upstream/internal names. Older rows predating the alias
|
# messy upstream/internal names. Older rows predating the alias
|
||||||
@@ -5949,6 +6050,9 @@ async def broker_get_usage(email: str, days: int = 0) -> dict:
|
|||||||
total_tokens += int(row.get("total_tokens") or 0)
|
total_tokens += int(row.get("total_tokens") or 0)
|
||||||
except Exception as e: # spend logs are best-effort; don't fail the whole view
|
except Exception as e: # spend logs are best-effort; don't fail the whole view
|
||||||
logger.warning("spend/logs fetch failed: %s", e)
|
logger.warning("spend/logs fetch failed: %s", e)
|
||||||
|
# Per-day series for the usage chart (same day convention as the DB
|
||||||
|
# path: the UTC day of startTime).
|
||||||
|
daily = daily_series_from_rows(team_rows)
|
||||||
|
|
||||||
# All-time spend from the same log rows (team counter is period-only).
|
# All-time spend from the same log rows (team counter is period-only).
|
||||||
all_time_spend = round(sum(m["spend"] for m in models.values()), 2)
|
all_time_spend = round(sum(m["spend"] for m in models.values()), 2)
|
||||||
@@ -5976,6 +6080,7 @@ async def broker_get_usage(email: str, days: int = 0) -> dict:
|
|||||||
# per-key counter is cycle-only); this HTTP fallback runs only when the
|
# per-key counter is cycle-only); this HTTP fallback runs only when the
|
||||||
# DB is unconfigured, so there is nothing accurate to report here.
|
# DB is unconfigured, so there is nothing accurate to report here.
|
||||||
"key_spend": [],
|
"key_spend": [],
|
||||||
|
"daily": daily,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,196 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Regression tests for the per-member daily usage series (Nov 2026).
|
||||||
|
|
||||||
|
The member dashboard's usage chart needs per-day spend/requests for the
|
||||||
|
signed-in member, bounded by the days parameter. /broker/usage now carries a
|
||||||
|
`daily` series (additive — every existing field is untouched, so older
|
||||||
|
dashboards keep working), built by member_daily_usage (DB path) or
|
||||||
|
daily_series_from_rows (HTTP fallback). These tests pin:
|
||||||
|
|
||||||
|
1. the route->handler table for /broker/usage (a decorator landing on the
|
||||||
|
wrong handler is invisible to every check except a request);
|
||||||
|
2. the DB path's SQL: day convention matches the co-op daily series
|
||||||
|
(to_char("startTime", 'YYYY-MM-DD')), the window filter goes on the
|
||||||
|
column DIRECTLY (a COALESCE with '' on a timestamp column was the
|
||||||
|
silent-empty-window bug in the model leaderboard), and requests excludes
|
||||||
|
failed attempts;
|
||||||
|
3. daily_series_from_rows: team filtering happens BEFORE it is called, rows
|
||||||
|
without a parseable day are skipped, spend sums failures too, requests
|
||||||
|
does not, output is day-ascending;
|
||||||
|
4. broker_get_usage's response contract: `daily` present and fail-soft
|
||||||
|
(a daily-query failure degrades to [], never an error).
|
||||||
|
|
||||||
|
Runnable: python3 tests/test_broker_usage_daily.py
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_TEST_DB = os.path.join(tempfile.mkdtemp(prefix="usage-daily-test-"), "members.db")
|
||||||
|
os.environ["DB_PATH"] = _TEST_DB # must exist BEFORE app.main is imported
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
for _cand in (os.environ.get("PORTAL_CODE_DIR"),
|
||||||
|
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))):
|
||||||
|
if _cand and os.path.isfile(os.path.join(_cand, "app", "main.py")):
|
||||||
|
sys.path.insert(0, _cand)
|
||||||
|
ROOT = Path(_cand)
|
||||||
|
break
|
||||||
|
|
||||||
|
import app.main as portal # noqa: E402
|
||||||
|
|
||||||
|
FAILED = []
|
||||||
|
|
||||||
|
|
||||||
|
def check(name, cond, detail=""):
|
||||||
|
if cond:
|
||||||
|
print(f" ok {name}")
|
||||||
|
else:
|
||||||
|
print(f" FAIL {name} {detail}")
|
||||||
|
FAILED.append(name)
|
||||||
|
|
||||||
|
|
||||||
|
# --- 1. route -> handler table (touched route AND its neighbours) ------------
|
||||||
|
src = (ROOT / "app" / "main.py").read_text()
|
||||||
|
|
||||||
|
def handler_for(path_literal):
|
||||||
|
m = re.search(
|
||||||
|
r'@app\.get\("' + re.escape(path_literal) + r'"\)\n(?:@[^\n]+\n)*async def (\w+)',
|
||||||
|
src,
|
||||||
|
)
|
||||||
|
return m.group(1) if m else None
|
||||||
|
|
||||||
|
check("/broker/usage -> broker_usage", handler_for("/broker/usage") == "broker_usage",
|
||||||
|
f"got {handler_for('/broker/usage')}")
|
||||||
|
check("/broker/metrics -> broker_metrics", handler_for("/broker/metrics") == "broker_metrics",
|
||||||
|
f"got {handler_for('/broker/metrics')}")
|
||||||
|
check("/broker/keys exists", handler_for("/broker/keys") is not None)
|
||||||
|
|
||||||
|
# --- 2. member_daily_usage SQL ------------------------------------------------
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
class _Cur:
|
||||||
|
def __init__(self, rows):
|
||||||
|
self._rows = rows
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *exc):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def execute(self, sql, params):
|
||||||
|
captured["sql"] = sql
|
||||||
|
captured["params"] = list(params)
|
||||||
|
|
||||||
|
def fetchall(self):
|
||||||
|
return [("2026-11-10", 0.5, 3), ("2026-11-11", 1.25, 5)]
|
||||||
|
|
||||||
|
|
||||||
|
class _Conn:
|
||||||
|
def __init__(self, rows):
|
||||||
|
self._rows = rows
|
||||||
|
|
||||||
|
def cursor(self):
|
||||||
|
return _Cur(self._rows)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
orig_db_ready = portal.litellm_db_ready
|
||||||
|
orig_pg = portal._pg_connect
|
||||||
|
portal.litellm_db_ready = lambda: True
|
||||||
|
portal._pg_connect = lambda: _Conn(None)
|
||||||
|
|
||||||
|
try:
|
||||||
|
out = portal.member_daily_usage(days=90, team_id="team-abc")
|
||||||
|
sql = re.sub(r"\s+", " ", captured["sql"])
|
||||||
|
check("day convention matches the co-op series",
|
||||||
|
"to_char(\"startTime\", 'YYYY-MM-DD')" in sql, sql[:120])
|
||||||
|
check("window filter on the column directly",
|
||||||
|
'"startTime" >= %s' in sql
|
||||||
|
and "COALESCE" not in sql.split("WHERE", 1)[-1], sql[:200])
|
||||||
|
check("failure rows excluded from requests",
|
||||||
|
"COUNT(*) FILTER (WHERE status IS DISTINCT FROM 'failure')" in sql)
|
||||||
|
check("team filter bound as parameter", captured["params"][-1] == "team-abc",
|
||||||
|
f"params={captured['params']}")
|
||||||
|
check("rows mapped to day/spend/requests",
|
||||||
|
out == [{"day": "2026-11-10", "spend": 0.5, "requests": 3},
|
||||||
|
{"day": "2026-11-11", "spend": 1.25, "requests": 5}], str(out))
|
||||||
|
|
||||||
|
# days=0 (no param) -> no window filter at all (backward-compatible default)
|
||||||
|
portal.member_daily_usage(days=0, team_id="team-abc")
|
||||||
|
check("days=0 means all time (no window predicate)",
|
||||||
|
'"startTime" >=' not in captured["sql"])
|
||||||
|
finally:
|
||||||
|
portal.litellm_db_ready = orig_db_ready
|
||||||
|
portal._pg_connect = orig_pg
|
||||||
|
|
||||||
|
# DB unconfigured must raise RuntimeError (the caller degrades to an empty
|
||||||
|
# series) — with the real litellm_db_ready restored and no stub.
|
||||||
|
db_ready_now = portal.litellm_db_ready()
|
||||||
|
if db_ready_now:
|
||||||
|
# A live DB on this host: re-stub to the unconfigured state for this check.
|
||||||
|
portal.litellm_db_ready = lambda: False
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
portal.member_daily_usage(days=30, team_id="t")
|
||||||
|
check("DB unconfigured raises RuntimeError", False, "no raise")
|
||||||
|
except RuntimeError:
|
||||||
|
check("DB unconfigured raises RuntimeError", True)
|
||||||
|
except Exception as e:
|
||||||
|
check("DB unconfigured raises RuntimeError", False, f"raised {type(e).__name__}: {e}")
|
||||||
|
finally:
|
||||||
|
portal.litellm_db_ready = orig_db_ready
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
portal.member_daily_usage(days=30, team_id="t")
|
||||||
|
check("DB unconfigured raises RuntimeError", False, "no raise")
|
||||||
|
except RuntimeError:
|
||||||
|
check("DB unconfigured raises RuntimeError", True)
|
||||||
|
except Exception as e:
|
||||||
|
check("DB unconfigured raises RuntimeError", False, f"raised {type(e).__name__}: {e}")
|
||||||
|
|
||||||
|
# --- 3. daily_series_from_rows ------------------------------------------------
|
||||||
|
rows = [
|
||||||
|
{"startTime": "2026-11-10 08:15:00+00", "spend": 0.10, "status": "success"},
|
||||||
|
{"startTime": "2026-11-10 09:00:00+00", "spend": 0.20, "status": "failure"},
|
||||||
|
{"startTime": "2026-11-11 00:05:00+00", "spend": 0.30, "status": "success"},
|
||||||
|
{"startTime": "", "spend": 9.99, "status": "success"}, # unparseable day: skipped
|
||||||
|
]
|
||||||
|
out = portal.daily_series_from_rows(rows)
|
||||||
|
check("fallback series: spend sums failures, requests does not",
|
||||||
|
out == [{"day": "2026-11-10", "spend": 0.3, "requests": 1},
|
||||||
|
{"day": "2026-11-11", "spend": 0.3, "requests": 1}], str(out))
|
||||||
|
check("fallback series ascending by day",
|
||||||
|
[r["day"] for r in out] == sorted(r["day"] for r in out))
|
||||||
|
check("fallback series on empty/None input", portal.daily_series_from_rows(None) == []
|
||||||
|
and portal.daily_series_from_rows([]) == [])
|
||||||
|
|
||||||
|
# Calendar fill: a day with no usage rows inside the measured span becomes an
|
||||||
|
# explicit zero (measured), so chart x-spacing is true; days outside the span
|
||||||
|
# are never invented.
|
||||||
|
gappy = portal.daily_series_from_rows([
|
||||||
|
{"startTime": "2026-11-08 10:00:00+00", "spend": 1.0, "status": "success"},
|
||||||
|
{"startTime": "2026-11-11 10:00:00+00", "spend": 2.0, "status": "success"},
|
||||||
|
])
|
||||||
|
check("calendar fill: interior zero days explicit, ends respected",
|
||||||
|
[r["day"] for r in gappy] == ["2026-11-08", "2026-11-09", "2026-11-10", "2026-11-11"]
|
||||||
|
and gappy[1] == {"day": "2026-11-09", "spend": 0.0, "requests": 0}
|
||||||
|
and gappy[0]["spend"] == 1.0 and gappy[3]["spend"] == 2.0, str(gappy))
|
||||||
|
|
||||||
|
# --- 4. broker_get_usage response contract ------------------------------------
|
||||||
|
check("DB path returns 'daily' key (source contract)",
|
||||||
|
'"daily": daily' in src)
|
||||||
|
check("daily query is fail-soft in the DB path",
|
||||||
|
re.search(r"daily usage query failed.*?\n\s*daily = \[\]", src, re.S) is not None)
|
||||||
|
|
||||||
|
print()
|
||||||
|
if FAILED:
|
||||||
|
print(f"FAILED: {len(FAILED)} check(s): " + ", ".join(FAILED))
|
||||||
|
sys.exit(1)
|
||||||
|
print("broker usage daily series: OK")
|
||||||
Reference in new issue
Block a user