ci/woodpecker/push/woodpecker Pipeline was successful
snapshot() fetched the full spend window (since = yesterday) but kept only spend.get(TODAY): the day's first hours, written at 03:15 and never revisited. Every metrics_daily row after the last backfill under-counted its day by the hours after 03:15 (Nov 7 recorded $2.19/691 requests against a true $29.06/3384). The completed day's aggregates were computed and thrown away. Now each run completes the day that just ENDED — full-day spend, requests, tokens and impact, written to the usage columns only (upsert_usage: membership columns are point-in-time measurements and are never rewritten) — and then records today's live partial as before. backfill() gains --since YYYY-MM-DD: a targeted usage-only repair that cannot touch the published membership curve. Regression test: tests/test_metrics_completion.py (day completion, membership preservation, since-mode repair boundaries, full-backfill parity).
191 lines
8.6 KiB
Python
191 lines
8.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Regression tests for the metrics day-completion fix (Nov 2026 under-count).
|
|
|
|
Bug: snapshot() wrote only TODAY's row at 03:15 UTC — the first ~3 hours of
|
|
that day's traffic — and never revisited it, so every metrics_daily row was
|
|
frozen at its first hours (Nov 6 read $1.20/93 requests vs a true $34.26/4508).
|
|
|
|
Fix: each run completes YESTERDAY's row with its full-day usage/impact
|
|
aggregates (usage columns ONLY — membership columns are point-in-time
|
|
measurements that must survive), then records today live as before.
|
|
backfill() gains --since: a targeted usage-only repair that leaves the
|
|
published membership curve untouched.
|
|
|
|
Runnable:
|
|
* from the repo root: python3 tests/test_metrics_completion.py
|
|
* inside the portal app: PORTAL_CODE_DIR=/app/code python3 tests/test_metrics_completion.py
|
|
"""
|
|
import datetime as dt
|
|
import os
|
|
import sqlite3
|
|
import sys
|
|
import tempfile
|
|
|
|
_TEST_DB = os.path.join(tempfile.mkdtemp(prefix="metrics-test-"), "members.db")
|
|
os.environ["DB_PATH"] = _TEST_DB # must exist BEFORE metrics_snapshot is imported
|
|
|
|
for _cand in (os.environ.get("PORTAL_CODE_DIR"),
|
|
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))):
|
|
if _cand and os.path.isfile(os.path.join(_cand, "metrics_snapshot.py")):
|
|
sys.path.insert(0, _cand)
|
|
break
|
|
|
|
import metrics_snapshot as ms # noqa: E402
|
|
|
|
TODAY = ms.today()
|
|
YESTERDAY = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=1)).isoformat()
|
|
DAY_BEFORE = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=2)).isoformat()
|
|
|
|
FAILED = []
|
|
|
|
|
|
def check(name, cond, detail=""):
|
|
if cond:
|
|
print(f" ok {name}")
|
|
else:
|
|
print(f" FAIL {name} {detail}")
|
|
FAILED.append(name)
|
|
|
|
|
|
def stub_sources(spend, op_spend, tokens, joins=(), active=40, total=71,
|
|
subs=71, matrix=65, balance=123.45):
|
|
"""Replace every network source with fixed in-memory data."""
|
|
ms.from_sqlite = lambda: {"active": active, "total": total, "joins": list(joins)}
|
|
ms.from_listmonk = lambda: subs
|
|
ms.listmonk_joins = lambda: list(joins)
|
|
ms.from_matrix = lambda: matrix
|
|
ms.oc_collective_balance = lambda: balance
|
|
ms.spend_by_day = lambda since=None: dict(spend)
|
|
ms.operator_spend_by_day = lambda since=None: None if op_spend is None else dict(op_spend)
|
|
ms.tokens_by_day_model = lambda since=None: {k: dict(v) for k, v in tokens.items()}
|
|
|
|
|
|
def read_row(day):
|
|
conn = sqlite3.connect(_TEST_DB)
|
|
try:
|
|
conn.row_factory = sqlite3.Row
|
|
return conn.execute("SELECT * FROM metrics_daily WHERE day = ?", (day,)).fetchone()
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def seed(day, **cols):
|
|
"""Insert a row with arbitrary columns (the nightly snapshot's shape)."""
|
|
ms.upsert(day, {"spend_usd": None, "requests": None, "out_tokens": None,
|
|
"energy_wh": None, "co2e_g": None, "tokens_measured": None,
|
|
"tokens_estimated": None, "factor_revision": None, **cols})
|
|
|
|
|
|
# ----------------------------------------------------------------------
|
|
print("1. snapshot() completes YESTERDAY with full-day usage, records today live")
|
|
stub_sources(
|
|
spend={YESTERDAY: (29.0604, 3384), TODAY: (2.19, 691)},
|
|
op_spend={YESTERDAY: 1.1613, TODAY: 0.5},
|
|
tokens={YESTERDAY: {"greenpt/glm-5.3-flash": 500000},
|
|
TODAY: {"greenpt/glm-5.3-flash": 100000}},
|
|
joins=["2026-11-01"],
|
|
)
|
|
row = ms.snapshot()
|
|
y = read_row(YESTERDAY)
|
|
t = read_row(TODAY)
|
|
check("yesterday spend_usd = full-day total", y["spend_usd"] == 29.0604, y["spend_usd"])
|
|
check("yesterday requests = full-day count", y["requests"] == 3384, y["requests"])
|
|
check("yesterday operator_spend carried", y["operator_spend_usd"] == 1.1613, y["operator_spend_usd"])
|
|
check("yesterday out_tokens from full day", y["out_tokens"] == 500000, y["out_tokens"])
|
|
check("yesterday energy computed from tokens", y["energy_wh"] is not None and y["energy_wh"] > 0)
|
|
check("yesterday co2e_g computed (>0, unit tripwire band)", y["co2e_g"] is not None and y["co2e_g"] > 0)
|
|
check("today's row keeps the live partial", t["spend_usd"] == 2.19 and t["requests"] == 691)
|
|
check("today's row carries live membership", t["members_total"] == 71 and t["members_active"] == 40)
|
|
check("today's row carries oc_balance", t["oc_balance_usd"] == 123.45)
|
|
check("yesterday's membership columns are NULL (not measured, never 0)",
|
|
y["members_active"] is None and y["members_total"] is None and y["newsletter_subs"] is None)
|
|
|
|
print("2. completion overwrites a stale partial WITHOUT touching membership columns")
|
|
conn = sqlite3.connect(_TEST_DB)
|
|
conn.execute("DELETE FROM metrics_daily")
|
|
conn.commit()
|
|
conn.close()
|
|
# What the OLD nightly code left behind: partial usage + that morning's counts.
|
|
seed(YESTERDAY, spend_usd=2.192, requests=691, members_active=39, members_total=70,
|
|
newsletter_subs=70, matrix_space_members=64)
|
|
stub_sources(
|
|
spend={YESTERDAY: (29.0604, 3384), TODAY: (0.5, 100)},
|
|
op_spend={YESTERDAY: 1.1613, TODAY: None},
|
|
tokens={YESTERDAY: {"greenpt/glm-5.3-flash": 500000}},
|
|
)
|
|
ms.snapshot()
|
|
y = read_row(YESTERDAY)
|
|
check("stale partial spend replaced with full-day value", y["spend_usd"] == 29.0604, y["spend_usd"])
|
|
check("stale partial requests replaced", y["requests"] == 3384, y["requests"])
|
|
check("members_active preserved from that morning", y["members_active"] == 39, y["members_active"])
|
|
check("members_total preserved", y["members_total"] == 70, y["members_total"])
|
|
check("newsletter_subs preserved", y["newsletter_subs"] == 70)
|
|
check("matrix_space_members preserved", y["matrix_space_members"] == 64)
|
|
|
|
print("3. upsert_usage on a missing row INSERTs usage + NULL membership (fail-closed)")
|
|
conn = sqlite3.connect(_TEST_DB)
|
|
conn.execute("DELETE FROM metrics_daily")
|
|
conn.commit()
|
|
conn.close()
|
|
ms.upsert_usage(DAY_BEFORE, {"spend_usd": 5.0, "requests": 42, "out_tokens": None,
|
|
"energy_wh": None, "co2e_g": None, "tokens_measured": None,
|
|
"tokens_estimated": None, "factor_revision": None,
|
|
"operator_spend_usd": None})
|
|
y = read_row(DAY_BEFORE)
|
|
check("usage columns written", y["spend_usd"] == 5.0 and y["requests"] == 42)
|
|
check("membership columns NULL, never zero",
|
|
y["members_active"] is None and y["members_total"] is None and y["members_new"] is None)
|
|
|
|
print("4. backfill(since=...) repairs usage only — membership curve untouched")
|
|
conn = sqlite3.connect(_TEST_DB)
|
|
conn.execute("DELETE FROM metrics_daily")
|
|
conn.commit()
|
|
conn.close()
|
|
# Three days as the nightly snapshots wrote them: membership present, usage partial.
|
|
for d, sp, rq in ((DAY_BEFORE, 1.0, 50), (YESTERDAY, 2.0, 60)):
|
|
seed(d, spend_usd=sp, requests=rq, members_active=39, members_total=70,
|
|
members_new=3, newsletter_subs=70, matrix_space_members=64)
|
|
seed(TODAY, spend_usd=0.5, requests=10, members_active=40, members_total=71)
|
|
stub_sources(
|
|
spend={DAY_BEFORE: (17.8791, 4493), YESTERDAY: (29.0604, 3384), TODAY: (0.5, 10)},
|
|
op_spend={DAY_BEFORE: 0.2, YESTERDAY: 1.1613, TODAY: None},
|
|
tokens={DAY_BEFORE: {"greenpt/deepseek-v4.1-flash": 200000},
|
|
YESTERDAY: {"greenpt/glm-5.3-flash": 500000}},
|
|
joins=["2026-11-01"],
|
|
)
|
|
ms.backfill(since=YESTERDAY)
|
|
y = read_row(YESTERDAY)
|
|
db = read_row(DAY_BEFORE)
|
|
t = read_row(TODAY)
|
|
check("repaired day matches ground truth spend", y["spend_usd"] == 29.0604, y["spend_usd"])
|
|
check("repaired day matches ground truth requests", y["requests"] == 3384, y["requests"])
|
|
check("repaired day keeps its membership columns",
|
|
y["members_active"] == 39 and y["members_total"] == 70 and y["members_new"] == 3)
|
|
check("day BEFORE --since untouched", db["spend_usd"] == 1.0 and db["requests"] == 50)
|
|
check("today still gets the full live update", t["members_total"] == 71)
|
|
|
|
print("5. full backfill (no --since) rebuilds membership columns as before")
|
|
conn = sqlite3.connect(_TEST_DB)
|
|
conn.execute("DELETE FROM metrics_daily")
|
|
conn.commit()
|
|
conn.close()
|
|
ms.backfill()
|
|
y = read_row(YESTERDAY)
|
|
check("full backfill writes joins-derived members_total",
|
|
y["members_total"] == 1, y["members_total"]) # one join on 2026-11-01
|
|
check("full backfill still writes full-day usage", y["spend_usd"] == 29.0604)
|
|
|
|
print("6. provenance: a day with tokens but no measured factor reports estimated, not measured")
|
|
ms.upsert_usage(DAY_BEFORE, {"spend_usd": 1.0, "requests": 5, "out_tokens": 1000,
|
|
"energy_wh": 0.5, "co2e_g": 0.125,
|
|
"tokens_measured": 0, "tokens_estimated": 1000,
|
|
"factor_revision": "x"})
|
|
y = read_row(DAY_BEFORE)
|
|
check("split persisted", y["tokens_measured"] == 0 and y["tokens_estimated"] == 1000)
|
|
|
|
print()
|
|
if FAILED:
|
|
print(f"FAILED: {len(FAILED)} check(s): {FAILED}")
|
|
sys.exit(1)
|
|
print("ALL CHECKS PASSED")
|