#!/usr/bin/env python3 """Regression tests for the per-member daily usage series (Nov 2026). The member dashboard's usage chart needs per-day spend/requests for the signed-in member, bounded by the days parameter. /broker/usage now carries a `daily` series (additive — every existing field is untouched, so older dashboards keep working), built by member_daily_usage (DB path) or daily_series_from_rows (HTTP fallback). These tests pin: 1. the route->handler table for /broker/usage (a decorator landing on the wrong handler is invisible to every check except a request); 2. the DB path's SQL: day convention matches the co-op daily series (to_char("startTime", 'YYYY-MM-DD')), the window filter goes on the column DIRECTLY (a COALESCE with '' on a timestamp column was the silent-empty-window bug in the model leaderboard), and requests excludes failed attempts; 3. daily_series_from_rows: team filtering happens BEFORE it is called, rows without a parseable day are skipped, spend sums failures too, requests does not, output is day-ascending; 4. broker_get_usage's response contract: `daily` present and fail-soft (a daily-query failure degrades to [], never an error). Runnable: python3 tests/test_broker_usage_daily.py """ import asyncio import os import re import sys import tempfile from pathlib import Path _TEST_DB = os.path.join(tempfile.mkdtemp(prefix="usage-daily-test-"), "members.db") os.environ["DB_PATH"] = _TEST_DB # must exist BEFORE app.main is imported ROOT = Path(__file__).resolve().parent.parent for _cand in (os.environ.get("PORTAL_CODE_DIR"), os.path.dirname(os.path.dirname(os.path.abspath(__file__)))): if _cand and os.path.isfile(os.path.join(_cand, "app", "main.py")): sys.path.insert(0, _cand) ROOT = Path(_cand) break import app.main as portal # noqa: E402 FAILED = [] def check(name, cond, detail=""): if cond: print(f" ok {name}") else: print(f" FAIL {name} {detail}") FAILED.append(name) # --- 1. route -> handler table (touched route AND its neighbours) ------------ src = (ROOT / "app" / "main.py").read_text() def handler_for(path_literal): m = re.search( r'@app\.get\("' + re.escape(path_literal) + r'"\)\n(?:@[^\n]+\n)*async def (\w+)', src, ) return m.group(1) if m else None check("/broker/usage -> broker_usage", handler_for("/broker/usage") == "broker_usage", f"got {handler_for('/broker/usage')}") check("/broker/metrics -> broker_metrics", handler_for("/broker/metrics") == "broker_metrics", f"got {handler_for('/broker/metrics')}") check("/broker/keys exists", handler_for("/broker/keys") is not None) # --- 2. member_daily_usage SQL ------------------------------------------------ captured = {} class _Cur: def __init__(self, rows): self._rows = rows def __enter__(self): return self def __exit__(self, *exc): return False def execute(self, sql, params): captured["sql"] = sql captured["params"] = list(params) def fetchall(self): return [("2026-11-10", 0.5, 3), ("2026-11-11", 1.25, 5)] class _Conn: def __init__(self, rows): self._rows = rows def cursor(self): return _Cur(self._rows) def close(self): pass orig_db_ready = portal.litellm_db_ready orig_pg = portal._pg_connect portal.litellm_db_ready = lambda: True portal._pg_connect = lambda: _Conn(None) try: out = portal.member_daily_usage(days=90, team_id="team-abc") sql = re.sub(r"\s+", " ", captured["sql"]) check("day convention matches the co-op series", "to_char(\"startTime\", 'YYYY-MM-DD')" in sql, sql[:120]) check("window filter on the column directly", '"startTime" >= %s' in sql and "COALESCE" not in sql.split("WHERE", 1)[-1], sql[:200]) check("failure rows excluded from requests", "COUNT(*) FILTER (WHERE status IS DISTINCT FROM 'failure')" in sql) check("team filter bound as parameter", captured["params"][-1] == "team-abc", f"params={captured['params']}") check("rows mapped to day/spend/requests", out == [{"day": "2026-11-10", "spend": 0.5, "requests": 3}, {"day": "2026-11-11", "spend": 1.25, "requests": 5}], str(out)) # days=0 (no param) -> no window filter at all (backward-compatible default) portal.member_daily_usage(days=0, team_id="team-abc") check("days=0 means all time (no window predicate)", '"startTime" >=' not in captured["sql"]) finally: portal.litellm_db_ready = orig_db_ready portal._pg_connect = orig_pg # DB unconfigured must raise RuntimeError (the caller degrades to an empty # series) — with the real litellm_db_ready restored and no stub. db_ready_now = portal.litellm_db_ready() if db_ready_now: # A live DB on this host: re-stub to the unconfigured state for this check. portal.litellm_db_ready = lambda: False try: try: portal.member_daily_usage(days=30, team_id="t") check("DB unconfigured raises RuntimeError", False, "no raise") except RuntimeError: check("DB unconfigured raises RuntimeError", True) except Exception as e: check("DB unconfigured raises RuntimeError", False, f"raised {type(e).__name__}: {e}") finally: portal.litellm_db_ready = orig_db_ready else: try: portal.member_daily_usage(days=30, team_id="t") check("DB unconfigured raises RuntimeError", False, "no raise") except RuntimeError: check("DB unconfigured raises RuntimeError", True) except Exception as e: check("DB unconfigured raises RuntimeError", False, f"raised {type(e).__name__}: {e}") # --- 3. daily_series_from_rows ------------------------------------------------ rows = [ {"startTime": "2026-11-10 08:15:00+00", "spend": 0.10, "status": "success"}, {"startTime": "2026-11-10 09:00:00+00", "spend": 0.20, "status": "failure"}, {"startTime": "2026-11-11 00:05:00+00", "spend": 0.30, "status": "success"}, {"startTime": "", "spend": 9.99, "status": "success"}, # unparseable day: skipped ] out = portal.daily_series_from_rows(rows) check("fallback series: spend sums failures, requests does not", out == [{"day": "2026-11-10", "spend": 0.3, "requests": 1}, {"day": "2026-11-11", "spend": 0.3, "requests": 1}], str(out)) check("fallback series ascending by day", [r["day"] for r in out] == sorted(r["day"] for r in out)) check("fallback series on empty/None input", portal.daily_series_from_rows(None) == [] and portal.daily_series_from_rows([]) == []) # Calendar fill: a day with no usage rows inside the measured span becomes an # explicit zero (measured), so chart x-spacing is true; days outside the span # are never invented. gappy = portal.daily_series_from_rows([ {"startTime": "2026-11-08 10:00:00+00", "spend": 1.0, "status": "success"}, {"startTime": "2026-11-11 10:00:00+00", "spend": 2.0, "status": "success"}, ]) check("calendar fill: interior zero days explicit, ends respected", [r["day"] for r in gappy] == ["2026-11-08", "2026-11-09", "2026-11-10", "2026-11-11"] and gappy[1] == {"day": "2026-11-09", "spend": 0.0, "requests": 0} and gappy[0]["spend"] == 1.0 and gappy[3]["spend"] == 2.0, str(gappy)) # --- 4. broker_get_usage response contract ------------------------------------ check("DB path returns 'daily' key (source contract)", '"daily": daily' in src) check("daily query is fail-soft in the DB path", re.search(r"daily usage query failed.*?\n\s*daily = \[\]", src, re.S) is not None) print() if FAILED: print(f"FAILED: {len(FAILED)} check(s): " + ", ".join(FAILED)) sys.exit(1) print("broker usage daily series: OK")