Files
member-portal/tests/test_broker_usage_daily.py
T
inference-bot 3c64b150db
ci/woodpecker/push/woodpecker Pipeline was successful
broker: per-member daily usage series in /broker/usage (daily: day/spend/requests)
The member dashboard's usage chart reads per-day spend + requests for the
signed-in member, bounded by the existing days param (0 = all time, the
unchanged default). Same source and day convention as the co-op daily series
(to_char(startTime)); interior zero days are explicit so chart spacing is
true; requests excludes failed attempts; spend sums every row. Additive and
backward compatible: every existing field untouched. Tested in
tests/test_broker_usage_daily.py (route table, SQL shape, fallback twin,
fail-soft degradation).
2026-11-13 16:11:16 +00:00

197 lines
7.7 KiB
Python

#!/usr/bin/env python3
"""Regression tests for the per-member daily usage series (Nov 2026).
The member dashboard's usage chart needs per-day spend/requests for the
signed-in member, bounded by the days parameter. /broker/usage now carries a
`daily` series (additive — every existing field is untouched, so older
dashboards keep working), built by member_daily_usage (DB path) or
daily_series_from_rows (HTTP fallback). These tests pin:
1. the route->handler table for /broker/usage (a decorator landing on the
wrong handler is invisible to every check except a request);
2. the DB path's SQL: day convention matches the co-op daily series
(to_char("startTime", 'YYYY-MM-DD')), the window filter goes on the
column DIRECTLY (a COALESCE with '' on a timestamp column was the
silent-empty-window bug in the model leaderboard), and requests excludes
failed attempts;
3. daily_series_from_rows: team filtering happens BEFORE it is called, rows
without a parseable day are skipped, spend sums failures too, requests
does not, output is day-ascending;
4. broker_get_usage's response contract: `daily` present and fail-soft
(a daily-query failure degrades to [], never an error).
Runnable: python3 tests/test_broker_usage_daily.py
"""
import asyncio
import os
import re
import sys
import tempfile
from pathlib import Path
_TEST_DB = os.path.join(tempfile.mkdtemp(prefix="usage-daily-test-"), "members.db")
os.environ["DB_PATH"] = _TEST_DB # must exist BEFORE app.main is imported
ROOT = Path(__file__).resolve().parent.parent
for _cand in (os.environ.get("PORTAL_CODE_DIR"),
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))):
if _cand and os.path.isfile(os.path.join(_cand, "app", "main.py")):
sys.path.insert(0, _cand)
ROOT = Path(_cand)
break
import app.main as portal # noqa: E402
FAILED = []
def check(name, cond, detail=""):
if cond:
print(f" ok {name}")
else:
print(f" FAIL {name} {detail}")
FAILED.append(name)
# --- 1. route -> handler table (touched route AND its neighbours) ------------
src = (ROOT / "app" / "main.py").read_text()
def handler_for(path_literal):
m = re.search(
r'@app\.get\("' + re.escape(path_literal) + r'"\)\n(?:@[^\n]+\n)*async def (\w+)',
src,
)
return m.group(1) if m else None
check("/broker/usage -> broker_usage", handler_for("/broker/usage") == "broker_usage",
f"got {handler_for('/broker/usage')}")
check("/broker/metrics -> broker_metrics", handler_for("/broker/metrics") == "broker_metrics",
f"got {handler_for('/broker/metrics')}")
check("/broker/keys exists", handler_for("/broker/keys") is not None)
# --- 2. member_daily_usage SQL ------------------------------------------------
captured = {}
class _Cur:
def __init__(self, rows):
self._rows = rows
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def execute(self, sql, params):
captured["sql"] = sql
captured["params"] = list(params)
def fetchall(self):
return [("2026-11-10", 0.5, 3), ("2026-11-11", 1.25, 5)]
class _Conn:
def __init__(self, rows):
self._rows = rows
def cursor(self):
return _Cur(self._rows)
def close(self):
pass
orig_db_ready = portal.litellm_db_ready
orig_pg = portal._pg_connect
portal.litellm_db_ready = lambda: True
portal._pg_connect = lambda: _Conn(None)
try:
out = portal.member_daily_usage(days=90, team_id="team-abc")
sql = re.sub(r"\s+", " ", captured["sql"])
check("day convention matches the co-op series",
"to_char(\"startTime\", 'YYYY-MM-DD')" in sql, sql[:120])
check("window filter on the column directly",
'"startTime" >= %s' in sql
and "COALESCE" not in sql.split("WHERE", 1)[-1], sql[:200])
check("failure rows excluded from requests",
"COUNT(*) FILTER (WHERE status IS DISTINCT FROM 'failure')" in sql)
check("team filter bound as parameter", captured["params"][-1] == "team-abc",
f"params={captured['params']}")
check("rows mapped to day/spend/requests",
out == [{"day": "2026-11-10", "spend": 0.5, "requests": 3},
{"day": "2026-11-11", "spend": 1.25, "requests": 5}], str(out))
# days=0 (no param) -> no window filter at all (backward-compatible default)
portal.member_daily_usage(days=0, team_id="team-abc")
check("days=0 means all time (no window predicate)",
'"startTime" >=' not in captured["sql"])
finally:
portal.litellm_db_ready = orig_db_ready
portal._pg_connect = orig_pg
# DB unconfigured must raise RuntimeError (the caller degrades to an empty
# series) — with the real litellm_db_ready restored and no stub.
db_ready_now = portal.litellm_db_ready()
if db_ready_now:
# A live DB on this host: re-stub to the unconfigured state for this check.
portal.litellm_db_ready = lambda: False
try:
try:
portal.member_daily_usage(days=30, team_id="t")
check("DB unconfigured raises RuntimeError", False, "no raise")
except RuntimeError:
check("DB unconfigured raises RuntimeError", True)
except Exception as e:
check("DB unconfigured raises RuntimeError", False, f"raised {type(e).__name__}: {e}")
finally:
portal.litellm_db_ready = orig_db_ready
else:
try:
portal.member_daily_usage(days=30, team_id="t")
check("DB unconfigured raises RuntimeError", False, "no raise")
except RuntimeError:
check("DB unconfigured raises RuntimeError", True)
except Exception as e:
check("DB unconfigured raises RuntimeError", False, f"raised {type(e).__name__}: {e}")
# --- 3. daily_series_from_rows ------------------------------------------------
rows = [
{"startTime": "2026-11-10 08:15:00+00", "spend": 0.10, "status": "success"},
{"startTime": "2026-11-10 09:00:00+00", "spend": 0.20, "status": "failure"},
{"startTime": "2026-11-11 00:05:00+00", "spend": 0.30, "status": "success"},
{"startTime": "", "spend": 9.99, "status": "success"}, # unparseable day: skipped
]
out = portal.daily_series_from_rows(rows)
check("fallback series: spend sums failures, requests does not",
out == [{"day": "2026-11-10", "spend": 0.3, "requests": 1},
{"day": "2026-11-11", "spend": 0.3, "requests": 1}], str(out))
check("fallback series ascending by day",
[r["day"] for r in out] == sorted(r["day"] for r in out))
check("fallback series on empty/None input", portal.daily_series_from_rows(None) == []
and portal.daily_series_from_rows([]) == [])
# Calendar fill: a day with no usage rows inside the measured span becomes an
# explicit zero (measured), so chart x-spacing is true; days outside the span
# are never invented.
gappy = portal.daily_series_from_rows([
{"startTime": "2026-11-08 10:00:00+00", "spend": 1.0, "status": "success"},
{"startTime": "2026-11-11 10:00:00+00", "spend": 2.0, "status": "success"},
])
check("calendar fill: interior zero days explicit, ends respected",
[r["day"] for r in gappy] == ["2026-11-08", "2026-11-09", "2026-11-10", "2026-11-11"]
and gappy[1] == {"day": "2026-11-09", "spend": 0.0, "requests": 0}
and gappy[0]["spend"] == 1.0 and gappy[3]["spend"] == 2.0, str(gappy))
# --- 4. broker_get_usage response contract ------------------------------------
check("DB path returns 'daily' key (source contract)",
'"daily": daily' in src)
check("daily query is fail-soft in the DB path",
re.search(r"daily usage query failed.*?\n\s*daily = \[\]", src, re.S) is not None)
print()
if FAILED:
print(f"FAILED: {len(FAILED)} check(s): " + ", ".join(FAILED))
sys.exit(1)
print("broker usage daily series: OK")