diff --git a/metrics_snapshot.py b/metrics_snapshot.py index 1320534..cf3de7a 100644 --- a/metrics_snapshot.py +++ b/metrics_snapshot.py @@ -11,6 +11,12 @@ dashboard can chart trends from one source of truth. repair recent days' usage only (membership columns are left untouched) +The nightly run also SELF-HEALS the OC balance line: any recent day (within a +week, after the series' first recorded balance) whose `oc_balance_usd` is NULL +is reconstructed from Open Collective's transaction ledger and written into the +balance column only, so one failed night no longer leaves a permanent hole in +the money chart. See heal_missing_balances(). + Rules this file follows deliberately: - **Fail closed per metric.** A source that cannot be read yields NULL for its @@ -94,8 +100,10 @@ def connect() -> sqlite3.Connection: ("factor_revision", "TEXT"), # The collective's Open Collective balance, read once a day. The OC API # exposes only the CURRENT balance — there is no historical series — so - # a real balance history can only accrue from these daily snapshots - # (from 2026-11-08 onward; earlier days stay NULL by design). + # a real balance history accrues from these daily snapshots, and a day + # whose sample failed is RECONSTRUCTED from OC's transaction ledger the + # next night (heal_missing_balances; from 2026-11-08 onward — earlier + # days stay NULL by design, a gap in the chart, never a zero). ("oc_balance_usd", "REAL"), ): try: @@ -583,6 +591,167 @@ def oc_collective_balance() -> float | None: return None +def oc_fetch_transactions(date_from: str, date_to: str) -> list[dict]: + """Every transaction in the collective's OC ledger within [date_from, date_to]. + + The raw building block for the balance self-heal. Raises on failure — the + heal caller prints the hole and lets the next night retry; an empty result + must stay distinguishable from a failed read, so nothing is swallowed here. + OC stores DEBIT amounts NEGATIVE and CREDIT positive; callers must not + re-sign them. Same browser-UA requirement as oc_collective_balance above + (the default Python UA is Cloudflare-blocked, 403/1010). + """ + token = os.environ.get("OC_PERSONAL_TOKEN", "") + slug = os.environ.get("OC_COLLECTIVE_SLUG", "inference-cooperative") + if not token: + raise RuntimeError("OC_PERSONAL_TOKEN not set") + query = ("query TxW($slug: String, $from: DateTime!, $to: DateTime!, " + "$limit: Int!, $offset: Int!) { collective(slug: $slug) { " + "transactions(dateFrom: $from, dateTo: $to, limit: $limit, offset: $offset) " + "{ totalCount nodes { createdAt type amount { valueInCents } } } } }") + nodes: list[dict] = [] + offset = 0 + for _page in range(40): # hard cap, mirroring main.py's money pagination + req = urllib.request.Request( + OC_GRAPHQL_URL, + data=json.dumps({"query": query, "variables": { + "slug": slug, "from": date_from, "to": date_to, + "limit": 500, "offset": offset, + }}).encode("utf-8"), + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {token}", + "Api-Version": "2022-01-01", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64)", + }, + ) + with urllib.request.urlopen(req, timeout=30) as resp: + data = json.loads(resp.read() or b"{}") + if data.get("errors"): + raise RuntimeError(f"OC transactions read failed: {str(data['errors'])[:200]}") + coll = ((data.get("data") or {}).get("collective") or {}).get("transactions") or {} + batch = coll.get("nodes") or [] + nodes.extend(batch) + offset += len(batch) + if not batch or offset >= int(coll.get("totalCount") or 0): + break + return nodes + + +def oc_historical_balances(current_balance: float, read_at: str, days: list[str]) -> dict[str, float]: + """Reconstruct missing days' balances from OC's transaction ledger. + + OC exposes only the CURRENT balance, but the collective's balance is exactly + the signed sum of its transactions (verified to the cent: the full-ledger + sum equals stats.balance), so the balance as of a past cutoff C is: + + current_balance - sum(signed amounts of transactions in (C, read_at]) + + The daily sample semantics are preserved: each day's value is the balance as + of that day's 03:15 UTC snapshot moment, matching what the nightly read + would have recorded. `read_at` is the moment `current_balance` was fetched; + the ledger window is clamped to it (dateTo) so a payment landing between the + balance read and the ledger read cannot distort the reconstruction. Days + with no transactions after their cutoff simply carry the current balance. + """ + flows: dict[str, float] = {d: 0.0 for d in days} # in CENTS + if days: + earliest = min(days) + "T03:15:00.000Z" + nodes = oc_fetch_transactions(earliest, read_at) + for n in nodes: + created = str(n.get("createdAt") or "") + cents = int((n.get("amount") or {}).get("valueInCents") or 0) + signed = cents if (n.get("type") or "").upper() == "CREDIT" else -abs(cents) + for d in days: + if d + "T03:15:00.000Z" < created <= read_at: + flows[d] += signed + return { + # cents -> USD before touching the balance, never after + d: round(current_balance - flows.get(d, 0.0) / 100.0, 2) + for d in days + } + + +def missing_balance_days(today_key: str, window_days: int = 7) -> list[str]: + """Recent days whose balance sample is missing — holes in the line, not + pre-history. A day qualifies when it is AFTER the series' first recorded + balance (days before 2026-11-08 stay NULL by design: the chart shows a gap + there, never a zero) and within the last `window_days` days, which bounds + the nightly repair work and keeps a long outage from triggering a huge + backfill. Only PAST days are returned: today's row is written by this very + run.""" + conn = connect() + try: + first = conn.execute( + "SELECT MIN(day) FROM metrics_daily WHERE oc_balance_usd IS NOT NULL" + ).fetchone()[0] + rows = conn.execute( + "SELECT day FROM metrics_daily WHERE oc_balance_usd IS NULL AND day < ? ORDER BY day", + (today_key,), + ).fetchall() + finally: + conn.close() + if not first: + return [] + today_d = dt.date.fromisoformat(today_key) + return [ + d for (d,) in rows + if d > first and (today_d - dt.date.fromisoformat(d)).days <= window_days + ] + + +def write_balance(day: str, value: float) -> None: + """Write ONLY oc_balance_usd for a day — the balance-column twin of + upsert_usage. A missing row is INSERTed with every other column NULL + (fail-closed: "balance known, membership not measured that day", never a + zero); an existing row keeps all its other columns untouched. This is the + only way a healed day's value reaches the store, so no heal can ever + corrupt the membership or usage series.""" + conn = connect() + try: + conn.execute( + """INSERT INTO metrics_daily (day, oc_balance_usd, recorded_at) + VALUES (?, ?, ?) + ON CONFLICT(day) DO UPDATE SET + oc_balance_usd = excluded.oc_balance_usd, + recorded_at = excluded.recorded_at""", + ( + day, + float(value), + dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"), + ), + ) + conn.commit() + finally: + conn.close() + + +def heal_missing_balances(balance: float | None, read_at: str) -> None: + """Self-heal the money chart: fill RECENT days whose balance sample is gone. + + A missed night (OC unreachable at 03:15, a scheduler run killed mid-deploy, + a repair backfill that predated the no-erase rule below) used to leave a + PERMANENT hole in the balance line, because nothing ever revisited a day's + balance. The next night now reconstructs each missing day from OC's own + transaction ledger (same source the nightly sample reads — no guessing by + arithmetic from a remembered number) and writes it balance-column-only. + Fail-soft end to end: any OC read failure prints and leaves the hole for + the next run; nothing here can raise into the snapshot itself. + """ + if balance is None: + return # OC read failed this run — today's heal would have no anchor + try: + days = missing_balance_days(today()) + if not days: + return + healed = oc_historical_balances(balance, read_at, days) + for d in sorted(healed): # oldest first: a later failure leaves earlier days done + write_balance(d, healed[d]) + print(f"metrics: healed missing balance day(s) -> {json.dumps(healed)}") + except Exception as exc: # noqa: BLE001 - the hole waits for the next night + print(f"metrics: balance heal failed ({exc}) — missing day(s) left for the next run") + + # ---------------------------------------------------------------- writes # The columns of metrics_daily that describe THAT DAY'S TRAFFIC and its impact, @@ -693,7 +862,13 @@ def upsert(day: str, row: dict) -> None: tokens_measured=excluded.tokens_measured, tokens_estimated=excluded.tokens_estimated, factor_revision=excluded.factor_revision, - oc_balance_usd=excluded.oc_balance_usd, + -- A NULL oc_balance_usd means "the OC read failed this run", + -- never "the balance is gone": keep the previously recorded + -- value instead of erasing it. (The Nov 2026 chart hole was + -- exactly this: a backfill's full-row upsert whose dict + -- carried no balance NULLed a sample the 03:15 run had + -- recorded hours earlier.) + oc_balance_usd=COALESCE(excluded.oc_balance_usd, metrics_daily.oc_balance_usd), recorded_at=excluded.recorded_at""", ( day, @@ -735,6 +910,14 @@ def snapshot(day: str | None = None) -> dict: op_spend = operator_spend_by_day(since=since) tokens = tokens_by_day_model(since=since) balance = oc_collective_balance() + # The moment the balance above was read. The self-heal clamps its ledger + # window to this instant, so a payment landing between the balance read and + # the ledger read is excluded from the reconstruction (it is not in the + # anchor balance either). Millisecond precision: OC stamps createdAt with + # millis, and the comparison is lexicographic on matching ISO shapes. + balance_read_at = ( + dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + ) # 1. Complete the day that just ENDED. The window above already contains # yesterday's full aggregates — the old code fetched them and threw them # away, keeping only today's first ~3 hours and never revisiting the @@ -760,6 +943,10 @@ def snapshot(day: str | None = None) -> dict: oc_balance_usd=balance, ) upsert(day, today_row) + # 3. Self-heal: any RECENT day whose balance sample is gone (one failed + # night used to leave a permanent hole in the money chart) is + # reconstructed from OC's own transaction ledger, balance column only. + heal_missing_balances(balance, balance_read_at) print(f"metrics: completed {yesterday} -> {json.dumps(yesterday_usage)}") print(f"metrics: recorded {day} -> {json.dumps(today_row)}") return today_row diff --git a/tests/test_balance_backfill.py b/tests/test_balance_backfill.py new file mode 100644 index 0000000..754da27 --- /dev/null +++ b/tests/test_balance_backfill.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Regression tests for the OC-balance self-heal (the Nov 2026 chart hole). + +Incident: the 03:15 snapshot on 2026-11-12 DID record oc_balance_usd=1357.39, +but a repair backfill run hours later rewrote the day through backfill()'s +full-row upsert — whose row dict carries no balance key — and ON CONFLICT wrote +NULL over it. Nothing revisited a day's balance afterwards, so the hole was +permanent and the money chart broke mid-line. + +Fixes under test here: + * upsert() never erases a balance it did not measure (COALESCE on conflict); + * the nightly snapshot heals any RECENT day with a missing balance from OC's + transaction ledger (balance column ONLY — membership/usage never touched); + * days before the series' first recorded balance stay NULL by design; + * an OC failure is fail-soft: hole stays, next night retries, nothing raises. + +Runnable: + * from the repo root: python3 tests/test_balance_backfill.py + * inside the portal app: PORTAL_CODE_DIR=/app/code python3 tests/test_balance_backfill.py +""" +import datetime as dt +import os +import sqlite3 +import sys +import tempfile + +_TEST_DB = os.path.join(tempfile.mkdtemp(prefix="balance-heal-test-"), "members.db") +os.environ["DB_PATH"] = _TEST_DB # must exist BEFORE metrics_snapshot is imported + +for _cand in (os.environ.get("PORTAL_CODE_DIR"), + os.path.dirname(os.path.dirname(os.path.abspath(__file__)))): + if _cand and os.path.isfile(os.path.join(_cand, "metrics_snapshot.py")): + sys.path.insert(0, _cand) + break + +import metrics_snapshot as ms # noqa: E402 + +TODAY = ms.today() +YESTERDAY = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=1)).isoformat() +DAY_MINUS2 = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=2)).isoformat() +DAY_MINUS5 = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=5)).isoformat() + +FAILED = [] + + +def check(name, cond, detail=""): + if cond: + print(f" ok {name}") + else: + print(f" FAIL {name} {detail}") + FAILED.append(name) + + +def read_row(day): + conn = sqlite3.connect(_TEST_DB) + try: + conn.row_factory = sqlite3.Row + return conn.execute("SELECT * FROM metrics_daily WHERE day = ?", (day,)).fetchone() + finally: + conn.close() + + +def wipe(): + ms.connect().close() # ensure the table exists (schema init) + conn = sqlite3.connect(_TEST_DB) + conn.execute("DELETE FROM metrics_daily") + conn.commit() + conn.close() + + +def stub_sources(**kw): + """Replace every network source with fixed in-memory data.""" + ms.from_sqlite = lambda: {"active": kw.get("active", 40), "total": kw.get("total", 71), + "joins": list(kw.get("joins", []))} + ms.from_listmonk = lambda: kw.get("subs", 71) + ms.listmonk_joins = lambda: list(kw.get("joins", [])) + ms.from_matrix = lambda: kw.get("matrix", 65) + ms.oc_collective_balance = lambda: kw.get("balance", 123.45) + ms.spend_by_day = lambda since=None: dict(kw.get("spend", {})) + ms.operator_spend_by_day = lambda since=None: None if kw.get("op_spend") is None else dict(kw.get("op_spend") or {}) + ms.tokens_by_day_model = lambda since=None: {k: dict(v) for k, v in (kw.get("tokens") or {}).items()} + + +def seed(day, **cols): + ms.upsert(day, {"spend_usd": None, "requests": None, "out_tokens": None, + "energy_wh": None, "co2e_g": None, "tokens_measured": None, + "tokens_estimated": None, "factor_revision": None, **cols}) + + +# ---------------------------------------------------------------------- +print("1. REGRESSION: a full-row rewrite without a balance never erases one " + "(the 2026-11-12 incident)") +wipe() +# The day as the 03:15 snapshot wrote it: membership + usage + a live balance. +seed(TODAY, spend_usd=3.38, requests=674, members_active=113, members_total=113, + members_new=2, newsletter_subs=122, matrix_space_members=36, oc_balance_usd=1357.39) +# Full backfill (the shape the 2026-11-12 18:42 repair ran): its today-branch +# rewrites the day with a dict that has NO oc_balance_usd key. +stub_sources(joins=["2026-11-01"], spend={TODAY: (3.38, 674)}, op_spend={TODAY: 0.9}, + tokens={TODAY: {"greenpt/glm-5.3-flash": 379040}}) +ms.backfill() +t = read_row(TODAY) +check("backfill rewrote the day (test is live)", t["members_total"] == 71, t["members_total"]) +check("the recorded balance SURVIVED the rewrite", t["oc_balance_usd"] == 1357.39, + t["oc_balance_usd"]) + +print("2. snapshot() with a failed OC read keeps a balance recorded earlier that day") +wipe() +seed(TODAY, spend_usd=0.5, requests=10, members_active=40, members_total=71, + oc_balance_usd=1357.39) +stub_sources(spend={YESTERDAY: (5.0, 100), TODAY: (0.7, 20)}, op_spend=None, tokens={}, + balance=None) +ms.snapshot() +t = read_row(TODAY) +check("today's row still refreshed (spend updated)", t["spend_usd"] == 0.7, t["spend_usd"]) +check("earlier balance kept when the fresh read failed", t["oc_balance_usd"] == 1357.39, + t["oc_balance_usd"]) + +print("3. self-heal reconstructs a missing day from the ledger — balance column ONLY") +wipe() +seed(DAY_MINUS2, spend_usd=1.0, requests=50, members_active=39, members_total=70, + oc_balance_usd=1200.0) +# The hole: yesterday's row exists, everything but the balance. +seed(YESTERDAY, spend_usd=2.0, requests=60, members_active=39, members_total=70, + newsletter_subs=71, matrix_space_members=65) +seed(TODAY, spend_usd=0.5, requests=10, members_active=40, members_total=71, + oc_balance_usd=1470.44) +READ_AT = TODAY + "T03:15:58.123Z" +# Ledger after the two cutoffs: yesterday +73.05, day-before +113.05 (all days +# cumulative toward the anchor). Signed as OC stores them: CREDIT +, DEBIT -. +ledger = [ + {"createdAt": DAY_MINUS2 + "T10:00:00.000Z", "type": "CREDIT", "amount": {"valueInCents": 5000}}, + {"createdAt": DAY_MINUS2 + "T10:00:00.200Z", "type": "DEBIT", "amount": {"valueInCents": -500}}, + {"createdAt": YESTERDAY + "T12:00:00.000Z", "type": "CREDIT", "amount": {"valueInCents": 7805}}, + {"createdAt": READ_AT, "type": "DEBIT", "amount": {"valueInCents": -1000}}, # exactly at read_at: in the (C, read_at] window +] +ms.oc_fetch_transactions = lambda date_from, date_to: list(ledger) +ms.heal_missing_balances(1470.44, READ_AT) +y = read_row(YESTERDAY) +d2 = read_row(DAY_MINUS2) +t = read_row(TODAY) +check("hole filled: yesterday = anchor - net(yesterday 03:15 .. read_at)", + y["oc_balance_usd"] == 1470.44 - (7805 - 1000) / 100.0, y["oc_balance_usd"]) +check("existing days untouched by the heal", d2["oc_balance_usd"] == 1200.0 and t["oc_balance_usd"] == 1470.44) +check("membership columns of the healed day intact", + y["members_active"] == 39 and y["members_total"] == 70 and y["newsletter_subs"] == 71) +check("usage columns of the healed day intact", y["spend_usd"] == 2.0 and y["requests"] == 60) + +print("4. pre-history stays NULL — a hole BEFORE the first recorded day is not invented") +wipe() +seed(DAY_MINUS5, spend_usd=1.0, requests=5) # before the series' first balance +seed(DAY_MINUS2, spend_usd=1.0, requests=5, oc_balance_usd=1200.0) +seed(TODAY, oc_balance_usd=1470.44) +ms.oc_fetch_transactions = lambda date_from, date_to: [] +ms.heal_missing_balances(1470.44, READ_AT) +d5 = read_row(DAY_MINUS5) +check("day before the first balance stays NULL (gap by design)", d5["oc_balance_usd"] is None, + d5["oc_balance_usd"]) + +print("5. OC ledger failure is fail-soft: hole stays, nothing raises, no zero written") +wipe() +seed(DAY_MINUS2, oc_balance_usd=1200.0) +seed(YESTERDAY, spend_usd=2.0, members_active=39, oc_balance_usd=None) +seed(TODAY, oc_balance_usd=1470.44) +def _boom(date_from, date_to): + raise RuntimeError("OC unreachable (test)") +ms.oc_fetch_transactions = _boom +ms.heal_missing_balances(1470.44, READ_AT) # must not raise +y = read_row(YESTERDAY) +check("hole left NULL for the next night", y["oc_balance_usd"] is None, y["oc_balance_usd"]) +check("no zero written on failure", y["oc_balance_usd"] != 0.0) + +print("6. write_balance on a day with NO row inserts balance + NULL everything else") +wipe() +ms.write_balance(YESTERDAY, 1357.39) +y = read_row(YESTERDAY) +check("balance written", y["oc_balance_usd"] == 1357.39) +check("membership/usage columns NULL (fail-closed, never zero)", + y["members_active"] is None and y["spend_usd"] is None and y["requests"] is None) + +print("7. anchor=None (OC read failed this run) heals nothing") +wipe() +seed(DAY_MINUS2, oc_balance_usd=1200.0) +seed(YESTERDAY, oc_balance_usd=None) +seed(TODAY, oc_balance_usd=1470.44) +ms.oc_fetch_transactions = lambda date_from, date_to: list(ledger) +ms.heal_missing_balances(None, READ_AT) +y = read_row(YESTERDAY) +check("no heal without an anchor balance", y["oc_balance_usd"] is None, y["oc_balance_usd"]) + +print() +if FAILED: + print(f"FAILED: {len(FAILED)} check(s): {FAILED}") + sys.exit(1) +print("ALL CHECKS PASSED")