metrics: self-heal missing OC balance days from the OC ledger

The 2026-11-12 hole was NOT a missed sample: the 03:15 run recorded
oc_balance_usd=1357.39, and the evening repair backfill's full-row
upsert of the then-current day — whose dict carries no balance key —
nulled it on conflict. Nothing ever revisited a day's balance, so the
hole was permanent.

Two fixes:
- upsert() can no longer erase a balance it did not measure: the
  conflict clause COALESCEs, so a NULL (a failed OC read) keeps the
  previously recorded value.
- the nightly snapshot heals any recent day (within a week, after the
  series' first recorded balance) whose balance is missing: the value is
  reconstructed from Open Collective's transaction ledger
  (balance(cutoff) = current - net flow after the 03:15 sample moment,
  ledger window clamped to the balance read), matching the daily sample
  semantics exactly. It is written balance-column-only, so membership
  and usage series are untouched. Days before 2026-11-08 stay NULL by
  design. Fail-soft: an OC failure prints and retries next night.

The reconstruction was verified to the cent against OC's own ledger:
full-ledger signed sum == stats.balance (1470.44 on 2026-11-13), and
1470.44 - 113.05 (net flow 2026-11-12 03:15 -> 2026-11-13 03:15:58)
== 1357.39, exactly the sample the 03:15 run had recorded.
This commit is contained in:
inference-bot committed 2026-11-13 04:04:43 +00:00
1 parent 8405fd9157
commit cad5897e9a
2 files changed
+385 -3

No files matched your search

+190 -3
View File
@@ -11,6 +11,12 @@ dashboard can chart trends from one source of truth.
repair recent days' usage only (membership
columns are left untouched)
The nightly run also SELF-HEALS the OC balance line: any recent day (within a
week, after the series' first recorded balance) whose `oc_balance_usd` is NULL
is reconstructed from Open Collective's transaction ledger and written into the
balance column only, so one failed night no longer leaves a permanent hole in
the money chart. See heal_missing_balances().
Rules this file follows deliberately:
- **Fail closed per metric.** A source that cannot be read yields NULL for its
@@ -94,8 +100,10 @@ def connect() -> sqlite3.Connection:
("factor_revision", "TEXT"),
# The collective's Open Collective balance, read once a day. The OC API
# exposes only the CURRENT balance — there is no historical series — so
# a real balance history can only accrue from these daily snapshots
# (from 2026-11-08 onward; earlier days stay NULL by design).
# a real balance history accrues from these daily snapshots, and a day
# whose sample failed is RECONSTRUCTED from OC's transaction ledger the
# next night (heal_missing_balances; from 2026-11-08 onward — earlier
# days stay NULL by design, a gap in the chart, never a zero).
("oc_balance_usd", "REAL"),
):
try:
@@ -583,6 +591,167 @@ def oc_collective_balance() -> float | None:
return None
def oc_fetch_transactions(date_from: str, date_to: str) -> list[dict]:
"""Every transaction in the collective's OC ledger within [date_from, date_to].
The raw building block for the balance self-heal. Raises on failure — the
heal caller prints the hole and lets the next night retry; an empty result
must stay distinguishable from a failed read, so nothing is swallowed here.
OC stores DEBIT amounts NEGATIVE and CREDIT positive; callers must not
re-sign them. Same browser-UA requirement as oc_collective_balance above
(the default Python UA is Cloudflare-blocked, 403/1010).
"""
token = os.environ.get("OC_PERSONAL_TOKEN", "")
slug = os.environ.get("OC_COLLECTIVE_SLUG", "inference-cooperative")
if not token:
raise RuntimeError("OC_PERSONAL_TOKEN not set")
query = ("query TxW($slug: String, $from: DateTime!, $to: DateTime!, "
"$limit: Int!, $offset: Int!) { collective(slug: $slug) { "
"transactions(dateFrom: $from, dateTo: $to, limit: $limit, offset: $offset) "
"{ totalCount nodes { createdAt type amount { valueInCents } } } } }")
nodes: list[dict] = []
offset = 0
for _page in range(40): # hard cap, mirroring main.py's money pagination
req = urllib.request.Request(
OC_GRAPHQL_URL,
data=json.dumps({"query": query, "variables": {
"slug": slug, "from": date_from, "to": date_to,
"limit": 500, "offset": offset,
}}).encode("utf-8"),
headers={
"Content-Type": "application/json",
"Authorization": f"Bearer {token}",
"Api-Version": "2022-01-01",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)",
},
)
with urllib.request.urlopen(req, timeout=30) as resp:
data = json.loads(resp.read() or b"{}")
if data.get("errors"):
raise RuntimeError(f"OC transactions read failed: {str(data['errors'])[:200]}")
coll = ((data.get("data") or {}).get("collective") or {}).get("transactions") or {}
batch = coll.get("nodes") or []
nodes.extend(batch)
offset += len(batch)
if not batch or offset >= int(coll.get("totalCount") or 0):
break
return nodes
def oc_historical_balances(current_balance: float, read_at: str, days: list[str]) -> dict[str, float]:
"""Reconstruct missing days' balances from OC's transaction ledger.
OC exposes only the CURRENT balance, but the collective's balance is exactly
the signed sum of its transactions (verified to the cent: the full-ledger
sum equals stats.balance), so the balance as of a past cutoff C is:
current_balance - sum(signed amounts of transactions in (C, read_at])
The daily sample semantics are preserved: each day's value is the balance as
of that day's 03:15 UTC snapshot moment, matching what the nightly read
would have recorded. `read_at` is the moment `current_balance` was fetched;
the ledger window is clamped to it (dateTo) so a payment landing between the
balance read and the ledger read cannot distort the reconstruction. Days
with no transactions after their cutoff simply carry the current balance.
"""
flows: dict[str, float] = {d: 0.0 for d in days} # in CENTS
if days:
earliest = min(days) + "T03:15:00.000Z"
nodes = oc_fetch_transactions(earliest, read_at)
for n in nodes:
created = str(n.get("createdAt") or "")
cents = int((n.get("amount") or {}).get("valueInCents") or 0)
signed = cents if (n.get("type") or "").upper() == "CREDIT" else -abs(cents)
for d in days:
if d + "T03:15:00.000Z" < created <= read_at:
flows[d] += signed
return {
# cents -> USD before touching the balance, never after
d: round(current_balance - flows.get(d, 0.0) / 100.0, 2)
for d in days
}
def missing_balance_days(today_key: str, window_days: int = 7) -> list[str]:
"""Recent days whose balance sample is missing — holes in the line, not
pre-history. A day qualifies when it is AFTER the series' first recorded
balance (days before 2026-11-08 stay NULL by design: the chart shows a gap
there, never a zero) and within the last `window_days` days, which bounds
the nightly repair work and keeps a long outage from triggering a huge
backfill. Only PAST days are returned: today's row is written by this very
run."""
conn = connect()
try:
first = conn.execute(
"SELECT MIN(day) FROM metrics_daily WHERE oc_balance_usd IS NOT NULL"
).fetchone()[0]
rows = conn.execute(
"SELECT day FROM metrics_daily WHERE oc_balance_usd IS NULL AND day < ? ORDER BY day",
(today_key,),
).fetchall()
finally:
conn.close()
if not first:
return []
today_d = dt.date.fromisoformat(today_key)
return [
d for (d,) in rows
if d > first and (today_d - dt.date.fromisoformat(d)).days <= window_days
]
def write_balance(day: str, value: float) -> None:
"""Write ONLY oc_balance_usd for a day — the balance-column twin of
upsert_usage. A missing row is INSERTed with every other column NULL
(fail-closed: "balance known, membership not measured that day", never a
zero); an existing row keeps all its other columns untouched. This is the
only way a healed day's value reaches the store, so no heal can ever
corrupt the membership or usage series."""
conn = connect()
try:
conn.execute(
"""INSERT INTO metrics_daily (day, oc_balance_usd, recorded_at)
VALUES (?, ?, ?)
ON CONFLICT(day) DO UPDATE SET
oc_balance_usd = excluded.oc_balance_usd,
recorded_at = excluded.recorded_at""",
(
day,
float(value),
dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
),
)
conn.commit()
finally:
conn.close()
def heal_missing_balances(balance: float | None, read_at: str) -> None:
"""Self-heal the money chart: fill RECENT days whose balance sample is gone.
A missed night (OC unreachable at 03:15, a scheduler run killed mid-deploy,
a repair backfill that predated the no-erase rule below) used to leave a
PERMANENT hole in the balance line, because nothing ever revisited a day's
balance. The next night now reconstructs each missing day from OC's own
transaction ledger (same source the nightly sample reads — no guessing by
arithmetic from a remembered number) and writes it balance-column-only.
Fail-soft end to end: any OC read failure prints and leaves the hole for
the next run; nothing here can raise into the snapshot itself.
"""
if balance is None:
return # OC read failed this run — today's heal would have no anchor
try:
days = missing_balance_days(today())
if not days:
return
healed = oc_historical_balances(balance, read_at, days)
for d in sorted(healed): # oldest first: a later failure leaves earlier days done
write_balance(d, healed[d])
print(f"metrics: healed missing balance day(s) -> {json.dumps(healed)}")
except Exception as exc: # noqa: BLE001 - the hole waits for the next night
print(f"metrics: balance heal failed ({exc}) — missing day(s) left for the next run")
# ---------------------------------------------------------------- writes
# The columns of metrics_daily that describe THAT DAY'S TRAFFIC and its impact,
@@ -693,7 +862,13 @@ def upsert(day: str, row: dict) -> None:
tokens_measured=excluded.tokens_measured,
tokens_estimated=excluded.tokens_estimated,
factor_revision=excluded.factor_revision,
oc_balance_usd=excluded.oc_balance_usd,
-- A NULL oc_balance_usd means "the OC read failed this run",
-- never "the balance is gone": keep the previously recorded
-- value instead of erasing it. (The Nov 2026 chart hole was
-- exactly this: a backfill's full-row upsert whose dict
-- carried no balance NULLed a sample the 03:15 run had
-- recorded hours earlier.)
oc_balance_usd=COALESCE(excluded.oc_balance_usd, metrics_daily.oc_balance_usd),
recorded_at=excluded.recorded_at""",
(
day,
@@ -735,6 +910,14 @@ def snapshot(day: str | None = None) -> dict:
op_spend = operator_spend_by_day(since=since)
tokens = tokens_by_day_model(since=since)
balance = oc_collective_balance()
# The moment the balance above was read. The self-heal clamps its ledger
# window to this instant, so a payment landing between the balance read and
# the ledger read is excluded from the reconstruction (it is not in the
# anchor balance either). Millisecond precision: OC stamps createdAt with
# millis, and the comparison is lexicographic on matching ISO shapes.
balance_read_at = (
dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
)
# 1. Complete the day that just ENDED. The window above already contains
# yesterday's full aggregates — the old code fetched them and threw them
# away, keeping only today's first ~3 hours and never revisiting the
@@ -760,6 +943,10 @@ def snapshot(day: str | None = None) -> dict:
oc_balance_usd=balance,
)
upsert(day, today_row)
# 3. Self-heal: any RECENT day whose balance sample is gone (one failed
# night used to leave a permanent hole in the money chart) is
# reconstructed from OC's own transaction ledger, balance column only.
heal_missing_balances(balance, balance_read_at)
print(f"metrics: completed {yesterday} -> {json.dumps(yesterday_usage)}")
print(f"metrics: recorded {day} -> {json.dumps(today_row)}")
return today_row
+195
View File
@@ -0,0 +1,195 @@
#!/usr/bin/env python3
"""Regression tests for the OC-balance self-heal (the Nov 2026 chart hole).
Incident: the 03:15 snapshot on 2026-11-12 DID record oc_balance_usd=1357.39,
but a repair backfill run hours later rewrote the day through backfill()'s
full-row upsert — whose row dict carries no balance key — and ON CONFLICT wrote
NULL over it. Nothing revisited a day's balance afterwards, so the hole was
permanent and the money chart broke mid-line.
Fixes under test here:
* upsert() never erases a balance it did not measure (COALESCE on conflict);
* the nightly snapshot heals any RECENT day with a missing balance from OC's
transaction ledger (balance column ONLY — membership/usage never touched);
* days before the series' first recorded balance stay NULL by design;
* an OC failure is fail-soft: hole stays, next night retries, nothing raises.
Runnable:
* from the repo root: python3 tests/test_balance_backfill.py
* inside the portal app: PORTAL_CODE_DIR=/app/code python3 tests/test_balance_backfill.py
"""
import datetime as dt
import os
import sqlite3
import sys
import tempfile
_TEST_DB = os.path.join(tempfile.mkdtemp(prefix="balance-heal-test-"), "members.db")
os.environ["DB_PATH"] = _TEST_DB # must exist BEFORE metrics_snapshot is imported
for _cand in (os.environ.get("PORTAL_CODE_DIR"),
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))):
if _cand and os.path.isfile(os.path.join(_cand, "metrics_snapshot.py")):
sys.path.insert(0, _cand)
break
import metrics_snapshot as ms # noqa: E402
TODAY = ms.today()
YESTERDAY = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=1)).isoformat()
DAY_MINUS2 = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=2)).isoformat()
DAY_MINUS5 = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=5)).isoformat()
FAILED = []
def check(name, cond, detail=""):
if cond:
print(f" ok {name}")
else:
print(f" FAIL {name} {detail}")
FAILED.append(name)
def read_row(day):
conn = sqlite3.connect(_TEST_DB)
try:
conn.row_factory = sqlite3.Row
return conn.execute("SELECT * FROM metrics_daily WHERE day = ?", (day,)).fetchone()
finally:
conn.close()
def wipe():
ms.connect().close() # ensure the table exists (schema init)
conn = sqlite3.connect(_TEST_DB)
conn.execute("DELETE FROM metrics_daily")
conn.commit()
conn.close()
def stub_sources(**kw):
"""Replace every network source with fixed in-memory data."""
ms.from_sqlite = lambda: {"active": kw.get("active", 40), "total": kw.get("total", 71),
"joins": list(kw.get("joins", []))}
ms.from_listmonk = lambda: kw.get("subs", 71)
ms.listmonk_joins = lambda: list(kw.get("joins", []))
ms.from_matrix = lambda: kw.get("matrix", 65)
ms.oc_collective_balance = lambda: kw.get("balance", 123.45)
ms.spend_by_day = lambda since=None: dict(kw.get("spend", {}))
ms.operator_spend_by_day = lambda since=None: None if kw.get("op_spend") is None else dict(kw.get("op_spend") or {})
ms.tokens_by_day_model = lambda since=None: {k: dict(v) for k, v in (kw.get("tokens") or {}).items()}
def seed(day, **cols):
ms.upsert(day, {"spend_usd": None, "requests": None, "out_tokens": None,
"energy_wh": None, "co2e_g": None, "tokens_measured": None,
"tokens_estimated": None, "factor_revision": None, **cols})
# ----------------------------------------------------------------------
print("1. REGRESSION: a full-row rewrite without a balance never erases one "
"(the 2026-11-12 incident)")
wipe()
# The day as the 03:15 snapshot wrote it: membership + usage + a live balance.
seed(TODAY, spend_usd=3.38, requests=674, members_active=113, members_total=113,
members_new=2, newsletter_subs=122, matrix_space_members=36, oc_balance_usd=1357.39)
# Full backfill (the shape the 2026-11-12 18:42 repair ran): its today-branch
# rewrites the day with a dict that has NO oc_balance_usd key.
stub_sources(joins=["2026-11-01"], spend={TODAY: (3.38, 674)}, op_spend={TODAY: 0.9},
tokens={TODAY: {"greenpt/glm-5.3-flash": 379040}})
ms.backfill()
t = read_row(TODAY)
check("backfill rewrote the day (test is live)", t["members_total"] == 71, t["members_total"])
check("the recorded balance SURVIVED the rewrite", t["oc_balance_usd"] == 1357.39,
t["oc_balance_usd"])
print("2. snapshot() with a failed OC read keeps a balance recorded earlier that day")
wipe()
seed(TODAY, spend_usd=0.5, requests=10, members_active=40, members_total=71,
oc_balance_usd=1357.39)
stub_sources(spend={YESTERDAY: (5.0, 100), TODAY: (0.7, 20)}, op_spend=None, tokens={},
balance=None)
ms.snapshot()
t = read_row(TODAY)
check("today's row still refreshed (spend updated)", t["spend_usd"] == 0.7, t["spend_usd"])
check("earlier balance kept when the fresh read failed", t["oc_balance_usd"] == 1357.39,
t["oc_balance_usd"])
print("3. self-heal reconstructs a missing day from the ledger — balance column ONLY")
wipe()
seed(DAY_MINUS2, spend_usd=1.0, requests=50, members_active=39, members_total=70,
oc_balance_usd=1200.0)
# The hole: yesterday's row exists, everything but the balance.
seed(YESTERDAY, spend_usd=2.0, requests=60, members_active=39, members_total=70,
newsletter_subs=71, matrix_space_members=65)
seed(TODAY, spend_usd=0.5, requests=10, members_active=40, members_total=71,
oc_balance_usd=1470.44)
READ_AT = TODAY + "T03:15:58.123Z"
# Ledger after the two cutoffs: yesterday +73.05, day-before +113.05 (all days
# cumulative toward the anchor). Signed as OC stores them: CREDIT +, DEBIT -.
ledger = [
{"createdAt": DAY_MINUS2 + "T10:00:00.000Z", "type": "CREDIT", "amount": {"valueInCents": 5000}},
{"createdAt": DAY_MINUS2 + "T10:00:00.200Z", "type": "DEBIT", "amount": {"valueInCents": -500}},
{"createdAt": YESTERDAY + "T12:00:00.000Z", "type": "CREDIT", "amount": {"valueInCents": 7805}},
{"createdAt": READ_AT, "type": "DEBIT", "amount": {"valueInCents": -1000}}, # exactly at read_at: in the (C, read_at] window
]
ms.oc_fetch_transactions = lambda date_from, date_to: list(ledger)
ms.heal_missing_balances(1470.44, READ_AT)
y = read_row(YESTERDAY)
d2 = read_row(DAY_MINUS2)
t = read_row(TODAY)
check("hole filled: yesterday = anchor - net(yesterday 03:15 .. read_at)",
y["oc_balance_usd"] == 1470.44 - (7805 - 1000) / 100.0, y["oc_balance_usd"])
check("existing days untouched by the heal", d2["oc_balance_usd"] == 1200.0 and t["oc_balance_usd"] == 1470.44)
check("membership columns of the healed day intact",
y["members_active"] == 39 and y["members_total"] == 70 and y["newsletter_subs"] == 71)
check("usage columns of the healed day intact", y["spend_usd"] == 2.0 and y["requests"] == 60)
print("4. pre-history stays NULL — a hole BEFORE the first recorded day is not invented")
wipe()
seed(DAY_MINUS5, spend_usd=1.0, requests=5) # before the series' first balance
seed(DAY_MINUS2, spend_usd=1.0, requests=5, oc_balance_usd=1200.0)
seed(TODAY, oc_balance_usd=1470.44)
ms.oc_fetch_transactions = lambda date_from, date_to: []
ms.heal_missing_balances(1470.44, READ_AT)
d5 = read_row(DAY_MINUS5)
check("day before the first balance stays NULL (gap by design)", d5["oc_balance_usd"] is None,
d5["oc_balance_usd"])
print("5. OC ledger failure is fail-soft: hole stays, nothing raises, no zero written")
wipe()
seed(DAY_MINUS2, oc_balance_usd=1200.0)
seed(YESTERDAY, spend_usd=2.0, members_active=39, oc_balance_usd=None)
seed(TODAY, oc_balance_usd=1470.44)
def _boom(date_from, date_to):
raise RuntimeError("OC unreachable (test)")
ms.oc_fetch_transactions = _boom
ms.heal_missing_balances(1470.44, READ_AT) # must not raise
y = read_row(YESTERDAY)
check("hole left NULL for the next night", y["oc_balance_usd"] is None, y["oc_balance_usd"])
check("no zero written on failure", y["oc_balance_usd"] != 0.0)
print("6. write_balance on a day with NO row inserts balance + NULL everything else")
wipe()
ms.write_balance(YESTERDAY, 1357.39)
y = read_row(YESTERDAY)
check("balance written", y["oc_balance_usd"] == 1357.39)
check("membership/usage columns NULL (fail-closed, never zero)",
y["members_active"] is None and y["spend_usd"] is None and y["requests"] is None)
print("7. anchor=None (OC read failed this run) heals nothing")
wipe()
seed(DAY_MINUS2, oc_balance_usd=1200.0)
seed(YESTERDAY, oc_balance_usd=None)
seed(TODAY, oc_balance_usd=1470.44)
ms.oc_fetch_transactions = lambda date_from, date_to: list(ledger)
ms.heal_missing_balances(None, READ_AT)
y = read_row(YESTERDAY)
check("no heal without an anchor balance", y["oc_balance_usd"] is None, y["oc_balance_usd"])
print()
if FAILED:
print(f"FAILED: {len(FAILED)} check(s): {FAILED}")
sys.exit(1)
print("ALL CHECKS PASSED")