Files
member-portal/tests/test_metrics_completion.py
inference-bot cf3dbb7489
ci/woodpecker/push/woodpecker Pipeline was successful
metrics: complete YESTERDAY's row at 03:15 — every day was frozen at its first ~3 hours
snapshot() fetched the full spend window (since = yesterday) but kept only
spend.get(TODAY): the day's first hours, written at 03:15 and never revisited.
Every metrics_daily row after the last backfill under-counted its day by the
hours after 03:15 (Nov 7 recorded $2.19/691 requests against a true
$29.06/3384). The completed day's aggregates were computed and thrown away.

Now each run completes the day that just ENDED — full-day spend, requests,
tokens and impact, written to the usage columns only (upsert_usage: membership
columns are point-in-time measurements and are never rewritten) — and then
records today's live partial as before.

backfill() gains --since YYYY-MM-DD: a targeted usage-only repair that cannot
touch the published membership curve.

Regression test: tests/test_metrics_completion.py (day completion, membership
preservation, since-mode repair boundaries, full-backfill parity).
2026-11-12 18:40:31 +00:00

191 lines
8.6 KiB
Python

#!/usr/bin/env python3
"""Regression tests for the metrics day-completion fix (Nov 2026 under-count).
Bug: snapshot() wrote only TODAY's row at 03:15 UTC — the first ~3 hours of
that day's traffic — and never revisited it, so every metrics_daily row was
frozen at its first hours (Nov 6 read $1.20/93 requests vs a true $34.26/4508).
Fix: each run completes YESTERDAY's row with its full-day usage/impact
aggregates (usage columns ONLY — membership columns are point-in-time
measurements that must survive), then records today live as before.
backfill() gains --since: a targeted usage-only repair that leaves the
published membership curve untouched.
Runnable:
* from the repo root: python3 tests/test_metrics_completion.py
* inside the portal app: PORTAL_CODE_DIR=/app/code python3 tests/test_metrics_completion.py
"""
import datetime as dt
import os
import sqlite3
import sys
import tempfile
_TEST_DB = os.path.join(tempfile.mkdtemp(prefix="metrics-test-"), "members.db")
os.environ["DB_PATH"] = _TEST_DB # must exist BEFORE metrics_snapshot is imported
for _cand in (os.environ.get("PORTAL_CODE_DIR"),
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))):
if _cand and os.path.isfile(os.path.join(_cand, "metrics_snapshot.py")):
sys.path.insert(0, _cand)
break
import metrics_snapshot as ms # noqa: E402
TODAY = ms.today()
YESTERDAY = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=1)).isoformat()
DAY_BEFORE = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=2)).isoformat()
FAILED = []
def check(name, cond, detail=""):
if cond:
print(f" ok {name}")
else:
print(f" FAIL {name} {detail}")
FAILED.append(name)
def stub_sources(spend, op_spend, tokens, joins=(), active=40, total=71,
subs=71, matrix=65, balance=123.45):
"""Replace every network source with fixed in-memory data."""
ms.from_sqlite = lambda: {"active": active, "total": total, "joins": list(joins)}
ms.from_listmonk = lambda: subs
ms.listmonk_joins = lambda: list(joins)
ms.from_matrix = lambda: matrix
ms.oc_collective_balance = lambda: balance
ms.spend_by_day = lambda since=None: dict(spend)
ms.operator_spend_by_day = lambda since=None: None if op_spend is None else dict(op_spend)
ms.tokens_by_day_model = lambda since=None: {k: dict(v) for k, v in tokens.items()}
def read_row(day):
conn = sqlite3.connect(_TEST_DB)
try:
conn.row_factory = sqlite3.Row
return conn.execute("SELECT * FROM metrics_daily WHERE day = ?", (day,)).fetchone()
finally:
conn.close()
def seed(day, **cols):
"""Insert a row with arbitrary columns (the nightly snapshot's shape)."""
ms.upsert(day, {"spend_usd": None, "requests": None, "out_tokens": None,
"energy_wh": None, "co2e_g": None, "tokens_measured": None,
"tokens_estimated": None, "factor_revision": None, **cols})
# ----------------------------------------------------------------------
print("1. snapshot() completes YESTERDAY with full-day usage, records today live")
stub_sources(
spend={YESTERDAY: (29.0604, 3384), TODAY: (2.19, 691)},
op_spend={YESTERDAY: 1.1613, TODAY: 0.5},
tokens={YESTERDAY: {"greenpt/glm-5.3-flash": 500000},
TODAY: {"greenpt/glm-5.3-flash": 100000}},
joins=["2026-11-01"],
)
row = ms.snapshot()
y = read_row(YESTERDAY)
t = read_row(TODAY)
check("yesterday spend_usd = full-day total", y["spend_usd"] == 29.0604, y["spend_usd"])
check("yesterday requests = full-day count", y["requests"] == 3384, y["requests"])
check("yesterday operator_spend carried", y["operator_spend_usd"] == 1.1613, y["operator_spend_usd"])
check("yesterday out_tokens from full day", y["out_tokens"] == 500000, y["out_tokens"])
check("yesterday energy computed from tokens", y["energy_wh"] is not None and y["energy_wh"] > 0)
check("yesterday co2e_g computed (>0, unit tripwire band)", y["co2e_g"] is not None and y["co2e_g"] > 0)
check("today's row keeps the live partial", t["spend_usd"] == 2.19 and t["requests"] == 691)
check("today's row carries live membership", t["members_total"] == 71 and t["members_active"] == 40)
check("today's row carries oc_balance", t["oc_balance_usd"] == 123.45)
check("yesterday's membership columns are NULL (not measured, never 0)",
y["members_active"] is None and y["members_total"] is None and y["newsletter_subs"] is None)
print("2. completion overwrites a stale partial WITHOUT touching membership columns")
conn = sqlite3.connect(_TEST_DB)
conn.execute("DELETE FROM metrics_daily")
conn.commit()
conn.close()
# What the OLD nightly code left behind: partial usage + that morning's counts.
seed(YESTERDAY, spend_usd=2.192, requests=691, members_active=39, members_total=70,
newsletter_subs=70, matrix_space_members=64)
stub_sources(
spend={YESTERDAY: (29.0604, 3384), TODAY: (0.5, 100)},
op_spend={YESTERDAY: 1.1613, TODAY: None},
tokens={YESTERDAY: {"greenpt/glm-5.3-flash": 500000}},
)
ms.snapshot()
y = read_row(YESTERDAY)
check("stale partial spend replaced with full-day value", y["spend_usd"] == 29.0604, y["spend_usd"])
check("stale partial requests replaced", y["requests"] == 3384, y["requests"])
check("members_active preserved from that morning", y["members_active"] == 39, y["members_active"])
check("members_total preserved", y["members_total"] == 70, y["members_total"])
check("newsletter_subs preserved", y["newsletter_subs"] == 70)
check("matrix_space_members preserved", y["matrix_space_members"] == 64)
print("3. upsert_usage on a missing row INSERTs usage + NULL membership (fail-closed)")
conn = sqlite3.connect(_TEST_DB)
conn.execute("DELETE FROM metrics_daily")
conn.commit()
conn.close()
ms.upsert_usage(DAY_BEFORE, {"spend_usd": 5.0, "requests": 42, "out_tokens": None,
"energy_wh": None, "co2e_g": None, "tokens_measured": None,
"tokens_estimated": None, "factor_revision": None,
"operator_spend_usd": None})
y = read_row(DAY_BEFORE)
check("usage columns written", y["spend_usd"] == 5.0 and y["requests"] == 42)
check("membership columns NULL, never zero",
y["members_active"] is None and y["members_total"] is None and y["members_new"] is None)
print("4. backfill(since=...) repairs usage only — membership curve untouched")
conn = sqlite3.connect(_TEST_DB)
conn.execute("DELETE FROM metrics_daily")
conn.commit()
conn.close()
# Three days as the nightly snapshots wrote them: membership present, usage partial.
for d, sp, rq in ((DAY_BEFORE, 1.0, 50), (YESTERDAY, 2.0, 60)):
seed(d, spend_usd=sp, requests=rq, members_active=39, members_total=70,
members_new=3, newsletter_subs=70, matrix_space_members=64)
seed(TODAY, spend_usd=0.5, requests=10, members_active=40, members_total=71)
stub_sources(
spend={DAY_BEFORE: (17.8791, 4493), YESTERDAY: (29.0604, 3384), TODAY: (0.5, 10)},
op_spend={DAY_BEFORE: 0.2, YESTERDAY: 1.1613, TODAY: None},
tokens={DAY_BEFORE: {"greenpt/deepseek-v4.1-flash": 200000},
YESTERDAY: {"greenpt/glm-5.3-flash": 500000}},
joins=["2026-11-01"],
)
ms.backfill(since=YESTERDAY)
y = read_row(YESTERDAY)
db = read_row(DAY_BEFORE)
t = read_row(TODAY)
check("repaired day matches ground truth spend", y["spend_usd"] == 29.0604, y["spend_usd"])
check("repaired day matches ground truth requests", y["requests"] == 3384, y["requests"])
check("repaired day keeps its membership columns",
y["members_active"] == 39 and y["members_total"] == 70 and y["members_new"] == 3)
check("day BEFORE --since untouched", db["spend_usd"] == 1.0 and db["requests"] == 50)
check("today still gets the full live update", t["members_total"] == 71)
print("5. full backfill (no --since) rebuilds membership columns as before")
conn = sqlite3.connect(_TEST_DB)
conn.execute("DELETE FROM metrics_daily")
conn.commit()
conn.close()
ms.backfill()
y = read_row(YESTERDAY)
check("full backfill writes joins-derived members_total",
y["members_total"] == 1, y["members_total"]) # one join on 2026-11-01
check("full backfill still writes full-day usage", y["spend_usd"] == 29.0604)
print("6. provenance: a day with tokens but no measured factor reports estimated, not measured")
ms.upsert_usage(DAY_BEFORE, {"spend_usd": 1.0, "requests": 5, "out_tokens": 1000,
"energy_wh": 0.5, "co2e_g": 0.125,
"tokens_measured": 0, "tokens_estimated": 1000,
"factor_revision": "x"})
y = read_row(DAY_BEFORE)
check("split persisted", y["tokens_measured"] == 0 and y["tokens_estimated"] == 1000)
print()
if FAILED:
print(f"FAILED: {len(FAILED)} check(s): {FAILED}")
sys.exit(1)
print("ALL CHECKS PASSED")