#!/usr/bin/env python3 """Regression tests for the OC-balance self-heal (the Nov 2026 chart hole). Incident: the 03:15 snapshot on 2026-11-12 DID record oc_balance_usd=1357.39, but a repair backfill run hours later rewrote the day through backfill()'s full-row upsert — whose row dict carries no balance key — and ON CONFLICT wrote NULL over it. Nothing revisited a day's balance afterwards, so the hole was permanent and the money chart broke mid-line. Fixes under test here: * upsert() never erases a balance it did not measure (COALESCE on conflict); * the nightly snapshot heals any RECENT day with a missing balance from OC's transaction ledger (balance column ONLY — membership/usage never touched); * days before the series' first recorded balance stay NULL by design; * an OC failure is fail-soft: hole stays, next night retries, nothing raises. Runnable: * from the repo root: python3 tests/test_balance_backfill.py * inside the portal app: PORTAL_CODE_DIR=/app/code python3 tests/test_balance_backfill.py """ import datetime as dt import os import sqlite3 import sys import tempfile _TEST_DB = os.path.join(tempfile.mkdtemp(prefix="balance-heal-test-"), "members.db") os.environ["DB_PATH"] = _TEST_DB # must exist BEFORE metrics_snapshot is imported for _cand in (os.environ.get("PORTAL_CODE_DIR"), os.path.dirname(os.path.dirname(os.path.abspath(__file__)))): if _cand and os.path.isfile(os.path.join(_cand, "metrics_snapshot.py")): sys.path.insert(0, _cand) break import metrics_snapshot as ms # noqa: E402 TODAY = ms.today() YESTERDAY = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=1)).isoformat() DAY_MINUS2 = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=2)).isoformat() DAY_MINUS5 = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=5)).isoformat() FAILED = [] def check(name, cond, detail=""): if cond: print(f" ok {name}") else: print(f" FAIL {name} {detail}") FAILED.append(name) def read_row(day): conn = sqlite3.connect(_TEST_DB) try: conn.row_factory = sqlite3.Row return conn.execute("SELECT * FROM metrics_daily WHERE day = ?", (day,)).fetchone() finally: conn.close() def wipe(): ms.connect().close() # ensure the table exists (schema init) conn = sqlite3.connect(_TEST_DB) conn.execute("DELETE FROM metrics_daily") conn.commit() conn.close() def stub_sources(**kw): """Replace every network source with fixed in-memory data.""" ms.from_sqlite = lambda: {"active": kw.get("active", 40), "total": kw.get("total", 71), "joins": list(kw.get("joins", []))} ms.from_listmonk = lambda: kw.get("subs", 71) ms.listmonk_joins = lambda: list(kw.get("joins", [])) ms.from_matrix = lambda: kw.get("matrix", 65) ms.oc_collective_balance = lambda: kw.get("balance", 123.45) ms.spend_by_day = lambda since=None: dict(kw.get("spend", {})) ms.operator_spend_by_day = lambda since=None: None if kw.get("op_spend") is None else dict(kw.get("op_spend") or {}) ms.tokens_by_day_model = lambda since=None: {k: dict(v) for k, v in (kw.get("tokens") or {}).items()} def seed(day, **cols): ms.upsert(day, {"spend_usd": None, "requests": None, "out_tokens": None, "energy_wh": None, "co2e_g": None, "tokens_measured": None, "tokens_estimated": None, "factor_revision": None, **cols}) # ---------------------------------------------------------------------- print("1. REGRESSION: a full-row rewrite without a balance never erases one " "(the 2026-11-12 incident)") wipe() # The day as the 03:15 snapshot wrote it: membership + usage + a live balance. seed(TODAY, spend_usd=3.38, requests=674, members_active=113, members_total=113, members_new=2, newsletter_subs=122, matrix_space_members=36, oc_balance_usd=1357.39) # Full backfill (the shape the 2026-11-12 18:42 repair ran): its today-branch # rewrites the day with a dict that has NO oc_balance_usd key. stub_sources(joins=["2026-11-01"], spend={TODAY: (3.38, 674)}, op_spend={TODAY: 0.9}, tokens={TODAY: {"greenpt/glm-5.3-flash": 379040}}) ms.backfill() t = read_row(TODAY) check("backfill rewrote the day (test is live)", t["members_total"] == 71, t["members_total"]) check("the recorded balance SURVIVED the rewrite", t["oc_balance_usd"] == 1357.39, t["oc_balance_usd"]) print("2. snapshot() with a failed OC read keeps a balance recorded earlier that day") wipe() seed(TODAY, spend_usd=0.5, requests=10, members_active=40, members_total=71, oc_balance_usd=1357.39) stub_sources(spend={YESTERDAY: (5.0, 100), TODAY: (0.7, 20)}, op_spend=None, tokens={}, balance=None) ms.snapshot() t = read_row(TODAY) check("today's row still refreshed (spend updated)", t["spend_usd"] == 0.7, t["spend_usd"]) check("earlier balance kept when the fresh read failed", t["oc_balance_usd"] == 1357.39, t["oc_balance_usd"]) print("3. self-heal reconstructs a missing day from the ledger — balance column ONLY") wipe() seed(DAY_MINUS2, spend_usd=1.0, requests=50, members_active=39, members_total=70, oc_balance_usd=1200.0) # The hole: yesterday's row exists, everything but the balance. seed(YESTERDAY, spend_usd=2.0, requests=60, members_active=39, members_total=70, newsletter_subs=71, matrix_space_members=65) seed(TODAY, spend_usd=0.5, requests=10, members_active=40, members_total=71, oc_balance_usd=1470.44) READ_AT = TODAY + "T03:15:58.123Z" # Ledger after the two cutoffs: yesterday +73.05, day-before +113.05 (all days # cumulative toward the anchor). Signed as OC stores them: CREDIT +, DEBIT -. ledger = [ {"createdAt": DAY_MINUS2 + "T10:00:00.000Z", "type": "CREDIT", "amount": {"valueInCents": 5000}}, {"createdAt": DAY_MINUS2 + "T10:00:00.200Z", "type": "DEBIT", "amount": {"valueInCents": -500}}, {"createdAt": YESTERDAY + "T12:00:00.000Z", "type": "CREDIT", "amount": {"valueInCents": 7805}}, {"createdAt": READ_AT, "type": "DEBIT", "amount": {"valueInCents": -1000}}, # exactly at read_at: in the (C, read_at] window ] ms.oc_fetch_transactions = lambda date_from, date_to: list(ledger) ms.heal_missing_balances(1470.44, READ_AT) y = read_row(YESTERDAY) d2 = read_row(DAY_MINUS2) t = read_row(TODAY) check("hole filled: yesterday = anchor - net(yesterday 03:15 .. read_at)", y["oc_balance_usd"] == 1470.44 - (7805 - 1000) / 100.0, y["oc_balance_usd"]) check("existing days untouched by the heal", d2["oc_balance_usd"] == 1200.0 and t["oc_balance_usd"] == 1470.44) check("membership columns of the healed day intact", y["members_active"] == 39 and y["members_total"] == 70 and y["newsletter_subs"] == 71) check("usage columns of the healed day intact", y["spend_usd"] == 2.0 and y["requests"] == 60) print("4. pre-history stays NULL — a hole BEFORE the first recorded day is not invented") wipe() seed(DAY_MINUS5, spend_usd=1.0, requests=5) # before the series' first balance seed(DAY_MINUS2, spend_usd=1.0, requests=5, oc_balance_usd=1200.0) seed(TODAY, oc_balance_usd=1470.44) ms.oc_fetch_transactions = lambda date_from, date_to: [] ms.heal_missing_balances(1470.44, READ_AT) d5 = read_row(DAY_MINUS5) check("day before the first balance stays NULL (gap by design)", d5["oc_balance_usd"] is None, d5["oc_balance_usd"]) print("5. OC ledger failure is fail-soft: hole stays, nothing raises, no zero written") wipe() seed(DAY_MINUS2, oc_balance_usd=1200.0) seed(YESTERDAY, spend_usd=2.0, members_active=39, oc_balance_usd=None) seed(TODAY, oc_balance_usd=1470.44) def _boom(date_from, date_to): raise RuntimeError("OC unreachable (test)") ms.oc_fetch_transactions = _boom ms.heal_missing_balances(1470.44, READ_AT) # must not raise y = read_row(YESTERDAY) check("hole left NULL for the next night", y["oc_balance_usd"] is None, y["oc_balance_usd"]) check("no zero written on failure", y["oc_balance_usd"] != 0.0) print("6. write_balance on a day with NO row inserts balance + NULL everything else") wipe() ms.write_balance(YESTERDAY, 1357.39) y = read_row(YESTERDAY) check("balance written", y["oc_balance_usd"] == 1357.39) check("membership/usage columns NULL (fail-closed, never zero)", y["members_active"] is None and y["spend_usd"] is None and y["requests"] is None) print("7. anchor=None (OC read failed this run) heals nothing") wipe() seed(DAY_MINUS2, oc_balance_usd=1200.0) seed(YESTERDAY, oc_balance_usd=None) seed(TODAY, oc_balance_usd=1470.44) ms.oc_fetch_transactions = lambda date_from, date_to: list(ledger) ms.heal_missing_balances(None, READ_AT) y = read_row(YESTERDAY) check("no heal without an anchor balance", y["oc_balance_usd"] is None, y["oc_balance_usd"]) print() if FAILED: print(f"FAILED: {len(FAILED)} check(s): {FAILED}") sys.exit(1) print("ALL CHECKS PASSED")