#!/usr/bin/env python3 """Regression tests for the metrics day-completion fix (Nov 2026 under-count). Bug: snapshot() wrote only TODAY's row at 03:15 UTC — the first ~3 hours of that day's traffic — and never revisited it, so every metrics_daily row was frozen at its first hours (Nov 6 read $1.20/93 requests vs a true $34.26/4508). Fix: each run completes YESTERDAY's row with its full-day usage/impact aggregates (usage columns ONLY — membership columns are point-in-time measurements that must survive), then records today live as before. backfill() gains --since: a targeted usage-only repair that leaves the published membership curve untouched. Runnable: * from the repo root: python3 tests/test_metrics_completion.py * inside the portal app: PORTAL_CODE_DIR=/app/code python3 tests/test_metrics_completion.py """ import datetime as dt import os import sqlite3 import sys import tempfile _TEST_DB = os.path.join(tempfile.mkdtemp(prefix="metrics-test-"), "members.db") os.environ["DB_PATH"] = _TEST_DB # must exist BEFORE metrics_snapshot is imported for _cand in (os.environ.get("PORTAL_CODE_DIR"), os.path.dirname(os.path.dirname(os.path.abspath(__file__)))): if _cand and os.path.isfile(os.path.join(_cand, "metrics_snapshot.py")): sys.path.insert(0, _cand) break import metrics_snapshot as ms # noqa: E402 TODAY = ms.today() YESTERDAY = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=1)).isoformat() DAY_BEFORE = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=2)).isoformat() FAILED = [] def check(name, cond, detail=""): if cond: print(f" ok {name}") else: print(f" FAIL {name} {detail}") FAILED.append(name) def stub_sources(spend, op_spend, tokens, joins=(), active=40, total=71, subs=71, matrix=65, balance=123.45): """Replace every network source with fixed in-memory data.""" ms.from_sqlite = lambda: {"active": active, "total": total, "joins": list(joins)} ms.from_listmonk = lambda: subs ms.listmonk_joins = lambda: list(joins) ms.from_matrix = lambda: matrix ms.oc_collective_balance = lambda: balance ms.spend_by_day = lambda since=None: dict(spend) ms.operator_spend_by_day = lambda since=None: None if op_spend is None else dict(op_spend) ms.tokens_by_day_model = lambda since=None: {k: dict(v) for k, v in tokens.items()} def read_row(day): conn = sqlite3.connect(_TEST_DB) try: conn.row_factory = sqlite3.Row return conn.execute("SELECT * FROM metrics_daily WHERE day = ?", (day,)).fetchone() finally: conn.close() def seed(day, **cols): """Insert a row with arbitrary columns (the nightly snapshot's shape).""" ms.upsert(day, {"spend_usd": None, "requests": None, "out_tokens": None, "energy_wh": None, "co2e_g": None, "tokens_measured": None, "tokens_estimated": None, "factor_revision": None, **cols}) # ---------------------------------------------------------------------- print("1. snapshot() completes YESTERDAY with full-day usage, records today live") stub_sources( spend={YESTERDAY: (29.0604, 3384), TODAY: (2.19, 691)}, op_spend={YESTERDAY: 1.1613, TODAY: 0.5}, tokens={YESTERDAY: {"greenpt/glm-5.3-flash": 500000}, TODAY: {"greenpt/glm-5.3-flash": 100000}}, joins=["2026-11-01"], ) row = ms.snapshot() y = read_row(YESTERDAY) t = read_row(TODAY) check("yesterday spend_usd = full-day total", y["spend_usd"] == 29.0604, y["spend_usd"]) check("yesterday requests = full-day count", y["requests"] == 3384, y["requests"]) check("yesterday operator_spend carried", y["operator_spend_usd"] == 1.1613, y["operator_spend_usd"]) check("yesterday out_tokens from full day", y["out_tokens"] == 500000, y["out_tokens"]) check("yesterday energy computed from tokens", y["energy_wh"] is not None and y["energy_wh"] > 0) check("yesterday co2e_g computed (>0, unit tripwire band)", y["co2e_g"] is not None and y["co2e_g"] > 0) check("today's row keeps the live partial", t["spend_usd"] == 2.19 and t["requests"] == 691) check("today's row carries live membership", t["members_total"] == 71 and t["members_active"] == 40) check("today's row carries oc_balance", t["oc_balance_usd"] == 123.45) check("yesterday's membership columns are NULL (not measured, never 0)", y["members_active"] is None and y["members_total"] is None and y["newsletter_subs"] is None) print("2. completion overwrites a stale partial WITHOUT touching membership columns") conn = sqlite3.connect(_TEST_DB) conn.execute("DELETE FROM metrics_daily") conn.commit() conn.close() # What the OLD nightly code left behind: partial usage + that morning's counts. seed(YESTERDAY, spend_usd=2.192, requests=691, members_active=39, members_total=70, newsletter_subs=70, matrix_space_members=64) stub_sources( spend={YESTERDAY: (29.0604, 3384), TODAY: (0.5, 100)}, op_spend={YESTERDAY: 1.1613, TODAY: None}, tokens={YESTERDAY: {"greenpt/glm-5.3-flash": 500000}}, ) ms.snapshot() y = read_row(YESTERDAY) check("stale partial spend replaced with full-day value", y["spend_usd"] == 29.0604, y["spend_usd"]) check("stale partial requests replaced", y["requests"] == 3384, y["requests"]) check("members_active preserved from that morning", y["members_active"] == 39, y["members_active"]) check("members_total preserved", y["members_total"] == 70, y["members_total"]) check("newsletter_subs preserved", y["newsletter_subs"] == 70) check("matrix_space_members preserved", y["matrix_space_members"] == 64) print("3. upsert_usage on a missing row INSERTs usage + NULL membership (fail-closed)") conn = sqlite3.connect(_TEST_DB) conn.execute("DELETE FROM metrics_daily") conn.commit() conn.close() ms.upsert_usage(DAY_BEFORE, {"spend_usd": 5.0, "requests": 42, "out_tokens": None, "energy_wh": None, "co2e_g": None, "tokens_measured": None, "tokens_estimated": None, "factor_revision": None, "operator_spend_usd": None}) y = read_row(DAY_BEFORE) check("usage columns written", y["spend_usd"] == 5.0 and y["requests"] == 42) check("membership columns NULL, never zero", y["members_active"] is None and y["members_total"] is None and y["members_new"] is None) print("4. backfill(since=...) repairs usage only — membership curve untouched") conn = sqlite3.connect(_TEST_DB) conn.execute("DELETE FROM metrics_daily") conn.commit() conn.close() # Three days as the nightly snapshots wrote them: membership present, usage partial. for d, sp, rq in ((DAY_BEFORE, 1.0, 50), (YESTERDAY, 2.0, 60)): seed(d, spend_usd=sp, requests=rq, members_active=39, members_total=70, members_new=3, newsletter_subs=70, matrix_space_members=64) seed(TODAY, spend_usd=0.5, requests=10, members_active=40, members_total=71) stub_sources( spend={DAY_BEFORE: (17.8791, 4493), YESTERDAY: (29.0604, 3384), TODAY: (0.5, 10)}, op_spend={DAY_BEFORE: 0.2, YESTERDAY: 1.1613, TODAY: None}, tokens={DAY_BEFORE: {"greenpt/deepseek-v4.1-flash": 200000}, YESTERDAY: {"greenpt/glm-5.3-flash": 500000}}, joins=["2026-11-01"], ) ms.backfill(since=YESTERDAY) y = read_row(YESTERDAY) db = read_row(DAY_BEFORE) t = read_row(TODAY) check("repaired day matches ground truth spend", y["spend_usd"] == 29.0604, y["spend_usd"]) check("repaired day matches ground truth requests", y["requests"] == 3384, y["requests"]) check("repaired day keeps its membership columns", y["members_active"] == 39 and y["members_total"] == 70 and y["members_new"] == 3) check("day BEFORE --since untouched", db["spend_usd"] == 1.0 and db["requests"] == 50) check("today still gets the full live update", t["members_total"] == 71) print("5. full backfill (no --since) rebuilds membership columns as before") conn = sqlite3.connect(_TEST_DB) conn.execute("DELETE FROM metrics_daily") conn.commit() conn.close() ms.backfill() y = read_row(YESTERDAY) check("full backfill writes joins-derived members_total", y["members_total"] == 1, y["members_total"]) # one join on 2026-11-01 check("full backfill still writes full-day usage", y["spend_usd"] == 29.0604) print("6. provenance: a day with tokens but no measured factor reports estimated, not measured") ms.upsert_usage(DAY_BEFORE, {"spend_usd": 1.0, "requests": 5, "out_tokens": 1000, "energy_wh": 0.5, "co2e_g": 0.125, "tokens_measured": 0, "tokens_estimated": 1000, "factor_revision": "x"}) y = read_row(DAY_BEFORE) check("split persisted", y["tokens_measured"] == 0 and y["tokens_estimated"] == 1000) print() if FAILED: print(f"FAILED: {len(FAILED)} check(s): {FAILED}") sys.exit(1) print("ALL CHECKS PASSED")