From 3c64b150db97b8de5428eda0f78316bd5b28f0b4 Mon Sep 17 00:00:00 2001 From: Inferencebot Date: Fri, 13 Nov 2026 16:11:16 +0000 Subject: [PATCH] broker: per-member daily usage series in /broker/usage (daily: day/spend/requests) The member dashboard's usage chart reads per-day spend + requests for the signed-in member, bounded by the existing days param (0 = all time, the unchanged default). Same source and day convention as the co-op daily series (to_char(startTime)); interior zero days are explicit so chart spacing is true; requests excludes failed attempts; spend sums every row. Additive and backward compatible: every existing field untouched. Tested in tests/test_broker_usage_daily.py (route table, SQL shape, fallback twin, fail-soft degradation). --- app/main.py | 105 +++++++++++++++++ tests/test_broker_usage_daily.py | 196 +++++++++++++++++++++++++++++++ 2 files changed, 301 insertions(+) create mode 100644 tests/test_broker_usage_daily.py diff --git a/app/main.py b/app/main.py index 0e3138e..eb9b390 100644 --- a/app/main.py +++ b/app/main.py @@ -587,6 +587,96 @@ def aggregate_model_usage(days: int = 0, team_id: str | None = None) -> dict: } +def _fill_daily_days(rows: list[dict]) -> list[dict]: + """Fill calendar days WITHOUT usage rows with spend 0 / requests 0 between + the first and last measured day, so the chart's x spacing is true: a day + with no requests IS a zero-usage day (measured), not an unmeasured gap — + the fail-closed NULL rule is for days the SOURCE could not read, which + these are not (a GROUP BY simply omits them).""" + if not rows: + return rows + from datetime import date, timedelta + start = date.fromisoformat(rows[0]["day"]) + end = date.fromisoformat(rows[-1]["day"]) + by_day = {r["day"]: r for r in rows} + out = [] + d = start + while d <= end: + key = d.isoformat() + out.append(by_day.get(key) or {"day": key, "spend": 0.0, "requests": 0}) + d += timedelta(days=1) + return out + + +def daily_series_from_rows(rows) -> list[dict]: + """Per-day usage series from spend-log rows ALREADY filtered to one team — + the HTTP fallback's client-side twin of member_daily_usage. spend sums every + row; requests counts only non-failure rows (same rules as the DB path).""" + daily_map: dict[str, dict] = {} + for row in rows if isinstance(rows, list) else []: + day = str(row.get("startTime") or "")[:10] + if not day: + continue + d = daily_map.setdefault(day, {"spend": 0.0, "requests": 0}) + d["spend"] += float(row.get("spend") or 0.0) + if str(row.get("status") or "").lower() != "failure": + d["requests"] += 1 + return _fill_daily_days([ + {"day": k, "spend": round(v["spend"], 6), "requests": v["requests"]} + for k, v in sorted(daily_map.items()) + ]) + + +def member_daily_usage(days: int = 0, team_id: str | None = None) -> list[dict]: + """The member's own per-day usage (spend + requests), for the dashboard's + usage chart. Same source and day convention as the co-op daily series + (metrics_snapshot.spend_by_day): to_char("startTime", 'YYYY-MM-DD'), so the + two series agree on which day a request belongs to. + + `days` bounds the window (0 = all time). spend sums EVERY log row (a call + that failed after tokens were consumed still cost money); `requests` counts + only metered calls that succeeded, like aggregate_model_usage. + """ + if not litellm_db_ready(): + raise RuntimeError("LiteLLM DB credentials not configured") + + where = [] + params: list = [] + if days and days > 0: + from datetime import datetime, timedelta, timezone + cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).strftime( + "%Y-%m-%d %H:%M:%S" + ) + where.append("\"startTime\" >= %s") + params.append(cutoff) + if team_id: + where.append("team_id = %s") + params.append(team_id) + where_sql = ("WHERE " + " AND ".join(where)) if where else "" + + sql = f""" + SELECT to_char("startTime", 'YYYY-MM-DD') AS day, + COALESCE(SUM(spend), 0) AS spend, + COUNT(*) FILTER (WHERE status IS DISTINCT FROM 'failure') AS requests + FROM "LiteLLM_SpendLogs" + {where_sql} + GROUP BY 1 + ORDER BY 1 + """ + + conn = _pg_connect() + try: + with conn.cursor() as cur: + cur.execute(sql, params) + rows = cur.fetchall() + finally: + conn.close() + return _fill_daily_days([ + {"day": r[0], "spend": round(float(r[1] or 0.0), 6), "requests": int(r[2] or 0)} + for r in rows + ]) + + def get_team_budget_from_db(team_id: str) -> dict | None: """Fetch a team's spend/budget/reset directly from LiteLLM's Postgres. @@ -5882,6 +5972,14 @@ async def broker_get_usage(email: str, days: int = 0) -> dict: except Exception as e: logger.warning("broker_get_usage: per-key spend failed: %s", e) key_spend = [] + # Per-day series for the member's usage chart (30/90/all-time + # toggles re-render client-side from this). Best-effort like + # key_spend: a failure degrades to an empty series, never an error. + try: + daily = member_daily_usage(days=days, team_id=team_id) + except Exception as e: + logger.warning("broker_get_usage: daily usage query failed: %s", e) + daily = [] return { "email": email, "guest": is_guest_account(email), @@ -5896,6 +5994,7 @@ async def broker_get_usage(email: str, days: int = 0) -> dict: "reset_at": reset_at, "models": model_list, "key_spend": key_spend, + "daily": daily, } except Exception as e: logger.warning("broker_get_usage: DB path failed (%s); falling back to HTTP", e) @@ -5923,6 +6022,7 @@ async def broker_get_usage(email: str, days: int = 0) -> dict: # filter client-side on the returned rows. models = {} total_tokens = 0 + team_rows: list = [] try: r2 = await client.get( f"{LITELLM_BASE}/spend/logs", @@ -5935,6 +6035,7 @@ async def broker_get_usage(email: str, days: int = 0) -> dict: for row in rows if isinstance(rows, list) else []: if row.get("team_id") != team_id: continue + team_rows.append(row) # Prefer model_group: it carries our provider/model alias (e.g. # greenpt/glm-5.3-flash) when present; the raw `model` field holds # messy upstream/internal names. Older rows predating the alias @@ -5949,6 +6050,9 @@ async def broker_get_usage(email: str, days: int = 0) -> dict: total_tokens += int(row.get("total_tokens") or 0) except Exception as e: # spend logs are best-effort; don't fail the whole view logger.warning("spend/logs fetch failed: %s", e) + # Per-day series for the usage chart (same day convention as the DB + # path: the UTC day of startTime). + daily = daily_series_from_rows(team_rows) # All-time spend from the same log rows (team counter is period-only). all_time_spend = round(sum(m["spend"] for m in models.values()), 2) @@ -5976,6 +6080,7 @@ async def broker_get_usage(email: str, days: int = 0) -> dict: # per-key counter is cycle-only); this HTTP fallback runs only when the # DB is unconfigured, so there is nothing accurate to report here. "key_spend": [], + "daily": daily, } diff --git a/tests/test_broker_usage_daily.py b/tests/test_broker_usage_daily.py new file mode 100644 index 0000000..3634317 --- /dev/null +++ b/tests/test_broker_usage_daily.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Regression tests for the per-member daily usage series (Nov 2026). + +The member dashboard's usage chart needs per-day spend/requests for the +signed-in member, bounded by the days parameter. /broker/usage now carries a +`daily` series (additive — every existing field is untouched, so older +dashboards keep working), built by member_daily_usage (DB path) or +daily_series_from_rows (HTTP fallback). These tests pin: + + 1. the route->handler table for /broker/usage (a decorator landing on the + wrong handler is invisible to every check except a request); + 2. the DB path's SQL: day convention matches the co-op daily series + (to_char("startTime", 'YYYY-MM-DD')), the window filter goes on the + column DIRECTLY (a COALESCE with '' on a timestamp column was the + silent-empty-window bug in the model leaderboard), and requests excludes + failed attempts; + 3. daily_series_from_rows: team filtering happens BEFORE it is called, rows + without a parseable day are skipped, spend sums failures too, requests + does not, output is day-ascending; + 4. broker_get_usage's response contract: `daily` present and fail-soft + (a daily-query failure degrades to [], never an error). + +Runnable: python3 tests/test_broker_usage_daily.py +""" +import asyncio +import os +import re +import sys +import tempfile +from pathlib import Path + +_TEST_DB = os.path.join(tempfile.mkdtemp(prefix="usage-daily-test-"), "members.db") +os.environ["DB_PATH"] = _TEST_DB # must exist BEFORE app.main is imported + +ROOT = Path(__file__).resolve().parent.parent +for _cand in (os.environ.get("PORTAL_CODE_DIR"), + os.path.dirname(os.path.dirname(os.path.abspath(__file__)))): + if _cand and os.path.isfile(os.path.join(_cand, "app", "main.py")): + sys.path.insert(0, _cand) + ROOT = Path(_cand) + break + +import app.main as portal # noqa: E402 + +FAILED = [] + + +def check(name, cond, detail=""): + if cond: + print(f" ok {name}") + else: + print(f" FAIL {name} {detail}") + FAILED.append(name) + + +# --- 1. route -> handler table (touched route AND its neighbours) ------------ +src = (ROOT / "app" / "main.py").read_text() + +def handler_for(path_literal): + m = re.search( + r'@app\.get\("' + re.escape(path_literal) + r'"\)\n(?:@[^\n]+\n)*async def (\w+)', + src, + ) + return m.group(1) if m else None + +check("/broker/usage -> broker_usage", handler_for("/broker/usage") == "broker_usage", + f"got {handler_for('/broker/usage')}") +check("/broker/metrics -> broker_metrics", handler_for("/broker/metrics") == "broker_metrics", + f"got {handler_for('/broker/metrics')}") +check("/broker/keys exists", handler_for("/broker/keys") is not None) + +# --- 2. member_daily_usage SQL ------------------------------------------------ +captured = {} + +class _Cur: + def __init__(self, rows): + self._rows = rows + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def execute(self, sql, params): + captured["sql"] = sql + captured["params"] = list(params) + + def fetchall(self): + return [("2026-11-10", 0.5, 3), ("2026-11-11", 1.25, 5)] + + +class _Conn: + def __init__(self, rows): + self._rows = rows + + def cursor(self): + return _Cur(self._rows) + + def close(self): + pass + + +orig_db_ready = portal.litellm_db_ready +orig_pg = portal._pg_connect +portal.litellm_db_ready = lambda: True +portal._pg_connect = lambda: _Conn(None) + +try: + out = portal.member_daily_usage(days=90, team_id="team-abc") + sql = re.sub(r"\s+", " ", captured["sql"]) + check("day convention matches the co-op series", + "to_char(\"startTime\", 'YYYY-MM-DD')" in sql, sql[:120]) + check("window filter on the column directly", + '"startTime" >= %s' in sql + and "COALESCE" not in sql.split("WHERE", 1)[-1], sql[:200]) + check("failure rows excluded from requests", + "COUNT(*) FILTER (WHERE status IS DISTINCT FROM 'failure')" in sql) + check("team filter bound as parameter", captured["params"][-1] == "team-abc", + f"params={captured['params']}") + check("rows mapped to day/spend/requests", + out == [{"day": "2026-11-10", "spend": 0.5, "requests": 3}, + {"day": "2026-11-11", "spend": 1.25, "requests": 5}], str(out)) + + # days=0 (no param) -> no window filter at all (backward-compatible default) + portal.member_daily_usage(days=0, team_id="team-abc") + check("days=0 means all time (no window predicate)", + '"startTime" >=' not in captured["sql"]) +finally: + portal.litellm_db_ready = orig_db_ready + portal._pg_connect = orig_pg + +# DB unconfigured must raise RuntimeError (the caller degrades to an empty +# series) — with the real litellm_db_ready restored and no stub. +db_ready_now = portal.litellm_db_ready() +if db_ready_now: + # A live DB on this host: re-stub to the unconfigured state for this check. + portal.litellm_db_ready = lambda: False + try: + try: + portal.member_daily_usage(days=30, team_id="t") + check("DB unconfigured raises RuntimeError", False, "no raise") + except RuntimeError: + check("DB unconfigured raises RuntimeError", True) + except Exception as e: + check("DB unconfigured raises RuntimeError", False, f"raised {type(e).__name__}: {e}") + finally: + portal.litellm_db_ready = orig_db_ready +else: + try: + portal.member_daily_usage(days=30, team_id="t") + check("DB unconfigured raises RuntimeError", False, "no raise") + except RuntimeError: + check("DB unconfigured raises RuntimeError", True) + except Exception as e: + check("DB unconfigured raises RuntimeError", False, f"raised {type(e).__name__}: {e}") + +# --- 3. daily_series_from_rows ------------------------------------------------ +rows = [ + {"startTime": "2026-11-10 08:15:00+00", "spend": 0.10, "status": "success"}, + {"startTime": "2026-11-10 09:00:00+00", "spend": 0.20, "status": "failure"}, + {"startTime": "2026-11-11 00:05:00+00", "spend": 0.30, "status": "success"}, + {"startTime": "", "spend": 9.99, "status": "success"}, # unparseable day: skipped +] +out = portal.daily_series_from_rows(rows) +check("fallback series: spend sums failures, requests does not", + out == [{"day": "2026-11-10", "spend": 0.3, "requests": 1}, + {"day": "2026-11-11", "spend": 0.3, "requests": 1}], str(out)) +check("fallback series ascending by day", + [r["day"] for r in out] == sorted(r["day"] for r in out)) +check("fallback series on empty/None input", portal.daily_series_from_rows(None) == [] + and portal.daily_series_from_rows([]) == []) + +# Calendar fill: a day with no usage rows inside the measured span becomes an +# explicit zero (measured), so chart x-spacing is true; days outside the span +# are never invented. +gappy = portal.daily_series_from_rows([ + {"startTime": "2026-11-08 10:00:00+00", "spend": 1.0, "status": "success"}, + {"startTime": "2026-11-11 10:00:00+00", "spend": 2.0, "status": "success"}, +]) +check("calendar fill: interior zero days explicit, ends respected", + [r["day"] for r in gappy] == ["2026-11-08", "2026-11-09", "2026-11-10", "2026-11-11"] + and gappy[1] == {"day": "2026-11-09", "spend": 0.0, "requests": 0} + and gappy[0]["spend"] == 1.0 and gappy[3]["spend"] == 2.0, str(gappy)) + +# --- 4. broker_get_usage response contract ------------------------------------ +check("DB path returns 'daily' key (source contract)", + '"daily": daily' in src) +check("daily query is fail-soft in the DB path", + re.search(r"daily usage query failed.*?\n\s*daily = \[\]", src, re.S) is not None) + +print() +if FAILED: + print(f"FAILED: {len(FAILED)} check(s): " + ", ".join(FAILED)) + sys.exit(1) +print("broker usage daily series: OK")