metrics: self-heal missing OC balance days from the OC ledger
The 2026-11-12 hole was NOT a missed sample: the 03:15 run recorded oc_balance_usd=1357.39, and the evening repair backfill's full-row upsert of the then-current day — whose dict carries no balance key — nulled it on conflict. Nothing ever revisited a day's balance, so the hole was permanent. Two fixes: - upsert() can no longer erase a balance it did not measure: the conflict clause COALESCEs, so a NULL (a failed OC read) keeps the previously recorded value. - the nightly snapshot heals any recent day (within a week, after the series' first recorded balance) whose balance is missing: the value is reconstructed from Open Collective's transaction ledger (balance(cutoff) = current - net flow after the 03:15 sample moment, ledger window clamped to the balance read), matching the daily sample semantics exactly. It is written balance-column-only, so membership and usage series are untouched. Days before 2026-11-08 stay NULL by design. Fail-soft: an OC failure prints and retries next night. The reconstruction was verified to the cent against OC's own ledger: full-ledger signed sum == stats.balance (1470.44 on 2026-11-13), and 1470.44 - 113.05 (net flow 2026-11-12 03:15 -> 2026-11-13 03:15:58) == 1357.39, exactly the sample the 03:15 run had recorded.
This commit is contained in:
1 parent
8405fd9157
commit
cad5897e9a
2 files changed
+385
-3
No files matched your search
+190
-3
@@ -11,6 +11,12 @@ dashboard can chart trends from one source of truth.
|
|||||||
repair recent days' usage only (membership
|
repair recent days' usage only (membership
|
||||||
columns are left untouched)
|
columns are left untouched)
|
||||||
|
|
||||||
|
The nightly run also SELF-HEALS the OC balance line: any recent day (within a
|
||||||
|
week, after the series' first recorded balance) whose `oc_balance_usd` is NULL
|
||||||
|
is reconstructed from Open Collective's transaction ledger and written into the
|
||||||
|
balance column only, so one failed night no longer leaves a permanent hole in
|
||||||
|
the money chart. See heal_missing_balances().
|
||||||
|
|
||||||
Rules this file follows deliberately:
|
Rules this file follows deliberately:
|
||||||
|
|
||||||
- **Fail closed per metric.** A source that cannot be read yields NULL for its
|
- **Fail closed per metric.** A source that cannot be read yields NULL for its
|
||||||
@@ -94,8 +100,10 @@ def connect() -> sqlite3.Connection:
|
|||||||
("factor_revision", "TEXT"),
|
("factor_revision", "TEXT"),
|
||||||
# The collective's Open Collective balance, read once a day. The OC API
|
# The collective's Open Collective balance, read once a day. The OC API
|
||||||
# exposes only the CURRENT balance — there is no historical series — so
|
# exposes only the CURRENT balance — there is no historical series — so
|
||||||
# a real balance history can only accrue from these daily snapshots
|
# a real balance history accrues from these daily snapshots, and a day
|
||||||
# (from 2026-11-08 onward; earlier days stay NULL by design).
|
# whose sample failed is RECONSTRUCTED from OC's transaction ledger the
|
||||||
|
# next night (heal_missing_balances; from 2026-11-08 onward — earlier
|
||||||
|
# days stay NULL by design, a gap in the chart, never a zero).
|
||||||
("oc_balance_usd", "REAL"),
|
("oc_balance_usd", "REAL"),
|
||||||
):
|
):
|
||||||
try:
|
try:
|
||||||
@@ -583,6 +591,167 @@ def oc_collective_balance() -> float | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def oc_fetch_transactions(date_from: str, date_to: str) -> list[dict]:
|
||||||
|
"""Every transaction in the collective's OC ledger within [date_from, date_to].
|
||||||
|
|
||||||
|
The raw building block for the balance self-heal. Raises on failure — the
|
||||||
|
heal caller prints the hole and lets the next night retry; an empty result
|
||||||
|
must stay distinguishable from a failed read, so nothing is swallowed here.
|
||||||
|
OC stores DEBIT amounts NEGATIVE and CREDIT positive; callers must not
|
||||||
|
re-sign them. Same browser-UA requirement as oc_collective_balance above
|
||||||
|
(the default Python UA is Cloudflare-blocked, 403/1010).
|
||||||
|
"""
|
||||||
|
token = os.environ.get("OC_PERSONAL_TOKEN", "")
|
||||||
|
slug = os.environ.get("OC_COLLECTIVE_SLUG", "inference-cooperative")
|
||||||
|
if not token:
|
||||||
|
raise RuntimeError("OC_PERSONAL_TOKEN not set")
|
||||||
|
query = ("query TxW($slug: String, $from: DateTime!, $to: DateTime!, "
|
||||||
|
"$limit: Int!, $offset: Int!) { collective(slug: $slug) { "
|
||||||
|
"transactions(dateFrom: $from, dateTo: $to, limit: $limit, offset: $offset) "
|
||||||
|
"{ totalCount nodes { createdAt type amount { valueInCents } } } } }")
|
||||||
|
nodes: list[dict] = []
|
||||||
|
offset = 0
|
||||||
|
for _page in range(40): # hard cap, mirroring main.py's money pagination
|
||||||
|
req = urllib.request.Request(
|
||||||
|
OC_GRAPHQL_URL,
|
||||||
|
data=json.dumps({"query": query, "variables": {
|
||||||
|
"slug": slug, "from": date_from, "to": date_to,
|
||||||
|
"limit": 500, "offset": offset,
|
||||||
|
}}).encode("utf-8"),
|
||||||
|
headers={
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Authorization": f"Bearer {token}",
|
||||||
|
"Api-Version": "2022-01-01",
|
||||||
|
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||||
|
data = json.loads(resp.read() or b"{}")
|
||||||
|
if data.get("errors"):
|
||||||
|
raise RuntimeError(f"OC transactions read failed: {str(data['errors'])[:200]}")
|
||||||
|
coll = ((data.get("data") or {}).get("collective") or {}).get("transactions") or {}
|
||||||
|
batch = coll.get("nodes") or []
|
||||||
|
nodes.extend(batch)
|
||||||
|
offset += len(batch)
|
||||||
|
if not batch or offset >= int(coll.get("totalCount") or 0):
|
||||||
|
break
|
||||||
|
return nodes
|
||||||
|
|
||||||
|
|
||||||
|
def oc_historical_balances(current_balance: float, read_at: str, days: list[str]) -> dict[str, float]:
|
||||||
|
"""Reconstruct missing days' balances from OC's transaction ledger.
|
||||||
|
|
||||||
|
OC exposes only the CURRENT balance, but the collective's balance is exactly
|
||||||
|
the signed sum of its transactions (verified to the cent: the full-ledger
|
||||||
|
sum equals stats.balance), so the balance as of a past cutoff C is:
|
||||||
|
|
||||||
|
current_balance - sum(signed amounts of transactions in (C, read_at])
|
||||||
|
|
||||||
|
The daily sample semantics are preserved: each day's value is the balance as
|
||||||
|
of that day's 03:15 UTC snapshot moment, matching what the nightly read
|
||||||
|
would have recorded. `read_at` is the moment `current_balance` was fetched;
|
||||||
|
the ledger window is clamped to it (dateTo) so a payment landing between the
|
||||||
|
balance read and the ledger read cannot distort the reconstruction. Days
|
||||||
|
with no transactions after their cutoff simply carry the current balance.
|
||||||
|
"""
|
||||||
|
flows: dict[str, float] = {d: 0.0 for d in days} # in CENTS
|
||||||
|
if days:
|
||||||
|
earliest = min(days) + "T03:15:00.000Z"
|
||||||
|
nodes = oc_fetch_transactions(earliest, read_at)
|
||||||
|
for n in nodes:
|
||||||
|
created = str(n.get("createdAt") or "")
|
||||||
|
cents = int((n.get("amount") or {}).get("valueInCents") or 0)
|
||||||
|
signed = cents if (n.get("type") or "").upper() == "CREDIT" else -abs(cents)
|
||||||
|
for d in days:
|
||||||
|
if d + "T03:15:00.000Z" < created <= read_at:
|
||||||
|
flows[d] += signed
|
||||||
|
return {
|
||||||
|
# cents -> USD before touching the balance, never after
|
||||||
|
d: round(current_balance - flows.get(d, 0.0) / 100.0, 2)
|
||||||
|
for d in days
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def missing_balance_days(today_key: str, window_days: int = 7) -> list[str]:
|
||||||
|
"""Recent days whose balance sample is missing — holes in the line, not
|
||||||
|
pre-history. A day qualifies when it is AFTER the series' first recorded
|
||||||
|
balance (days before 2026-11-08 stay NULL by design: the chart shows a gap
|
||||||
|
there, never a zero) and within the last `window_days` days, which bounds
|
||||||
|
the nightly repair work and keeps a long outage from triggering a huge
|
||||||
|
backfill. Only PAST days are returned: today's row is written by this very
|
||||||
|
run."""
|
||||||
|
conn = connect()
|
||||||
|
try:
|
||||||
|
first = conn.execute(
|
||||||
|
"SELECT MIN(day) FROM metrics_daily WHERE oc_balance_usd IS NOT NULL"
|
||||||
|
).fetchone()[0]
|
||||||
|
rows = conn.execute(
|
||||||
|
"SELECT day FROM metrics_daily WHERE oc_balance_usd IS NULL AND day < ? ORDER BY day",
|
||||||
|
(today_key,),
|
||||||
|
).fetchall()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
if not first:
|
||||||
|
return []
|
||||||
|
today_d = dt.date.fromisoformat(today_key)
|
||||||
|
return [
|
||||||
|
d for (d,) in rows
|
||||||
|
if d > first and (today_d - dt.date.fromisoformat(d)).days <= window_days
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def write_balance(day: str, value: float) -> None:
|
||||||
|
"""Write ONLY oc_balance_usd for a day — the balance-column twin of
|
||||||
|
upsert_usage. A missing row is INSERTed with every other column NULL
|
||||||
|
(fail-closed: "balance known, membership not measured that day", never a
|
||||||
|
zero); an existing row keeps all its other columns untouched. This is the
|
||||||
|
only way a healed day's value reaches the store, so no heal can ever
|
||||||
|
corrupt the membership or usage series."""
|
||||||
|
conn = connect()
|
||||||
|
try:
|
||||||
|
conn.execute(
|
||||||
|
"""INSERT INTO metrics_daily (day, oc_balance_usd, recorded_at)
|
||||||
|
VALUES (?, ?, ?)
|
||||||
|
ON CONFLICT(day) DO UPDATE SET
|
||||||
|
oc_balance_usd = excluded.oc_balance_usd,
|
||||||
|
recorded_at = excluded.recorded_at""",
|
||||||
|
(
|
||||||
|
day,
|
||||||
|
float(value),
|
||||||
|
dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def heal_missing_balances(balance: float | None, read_at: str) -> None:
|
||||||
|
"""Self-heal the money chart: fill RECENT days whose balance sample is gone.
|
||||||
|
|
||||||
|
A missed night (OC unreachable at 03:15, a scheduler run killed mid-deploy,
|
||||||
|
a repair backfill that predated the no-erase rule below) used to leave a
|
||||||
|
PERMANENT hole in the balance line, because nothing ever revisited a day's
|
||||||
|
balance. The next night now reconstructs each missing day from OC's own
|
||||||
|
transaction ledger (same source the nightly sample reads — no guessing by
|
||||||
|
arithmetic from a remembered number) and writes it balance-column-only.
|
||||||
|
Fail-soft end to end: any OC read failure prints and leaves the hole for
|
||||||
|
the next run; nothing here can raise into the snapshot itself.
|
||||||
|
"""
|
||||||
|
if balance is None:
|
||||||
|
return # OC read failed this run — today's heal would have no anchor
|
||||||
|
try:
|
||||||
|
days = missing_balance_days(today())
|
||||||
|
if not days:
|
||||||
|
return
|
||||||
|
healed = oc_historical_balances(balance, read_at, days)
|
||||||
|
for d in sorted(healed): # oldest first: a later failure leaves earlier days done
|
||||||
|
write_balance(d, healed[d])
|
||||||
|
print(f"metrics: healed missing balance day(s) -> {json.dumps(healed)}")
|
||||||
|
except Exception as exc: # noqa: BLE001 - the hole waits for the next night
|
||||||
|
print(f"metrics: balance heal failed ({exc}) — missing day(s) left for the next run")
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- writes
|
# ---------------------------------------------------------------- writes
|
||||||
|
|
||||||
# The columns of metrics_daily that describe THAT DAY'S TRAFFIC and its impact,
|
# The columns of metrics_daily that describe THAT DAY'S TRAFFIC and its impact,
|
||||||
@@ -693,7 +862,13 @@ def upsert(day: str, row: dict) -> None:
|
|||||||
tokens_measured=excluded.tokens_measured,
|
tokens_measured=excluded.tokens_measured,
|
||||||
tokens_estimated=excluded.tokens_estimated,
|
tokens_estimated=excluded.tokens_estimated,
|
||||||
factor_revision=excluded.factor_revision,
|
factor_revision=excluded.factor_revision,
|
||||||
oc_balance_usd=excluded.oc_balance_usd,
|
-- A NULL oc_balance_usd means "the OC read failed this run",
|
||||||
|
-- never "the balance is gone": keep the previously recorded
|
||||||
|
-- value instead of erasing it. (The Nov 2026 chart hole was
|
||||||
|
-- exactly this: a backfill's full-row upsert whose dict
|
||||||
|
-- carried no balance NULLed a sample the 03:15 run had
|
||||||
|
-- recorded hours earlier.)
|
||||||
|
oc_balance_usd=COALESCE(excluded.oc_balance_usd, metrics_daily.oc_balance_usd),
|
||||||
recorded_at=excluded.recorded_at""",
|
recorded_at=excluded.recorded_at""",
|
||||||
(
|
(
|
||||||
day,
|
day,
|
||||||
@@ -735,6 +910,14 @@ def snapshot(day: str | None = None) -> dict:
|
|||||||
op_spend = operator_spend_by_day(since=since)
|
op_spend = operator_spend_by_day(since=since)
|
||||||
tokens = tokens_by_day_model(since=since)
|
tokens = tokens_by_day_model(since=since)
|
||||||
balance = oc_collective_balance()
|
balance = oc_collective_balance()
|
||||||
|
# The moment the balance above was read. The self-heal clamps its ledger
|
||||||
|
# window to this instant, so a payment landing between the balance read and
|
||||||
|
# the ledger read is excluded from the reconstruction (it is not in the
|
||||||
|
# anchor balance either). Millisecond precision: OC stamps createdAt with
|
||||||
|
# millis, and the comparison is lexicographic on matching ISO shapes.
|
||||||
|
balance_read_at = (
|
||||||
|
dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
||||||
|
)
|
||||||
# 1. Complete the day that just ENDED. The window above already contains
|
# 1. Complete the day that just ENDED. The window above already contains
|
||||||
# yesterday's full aggregates — the old code fetched them and threw them
|
# yesterday's full aggregates — the old code fetched them and threw them
|
||||||
# away, keeping only today's first ~3 hours and never revisiting the
|
# away, keeping only today's first ~3 hours and never revisiting the
|
||||||
@@ -760,6 +943,10 @@ def snapshot(day: str | None = None) -> dict:
|
|||||||
oc_balance_usd=balance,
|
oc_balance_usd=balance,
|
||||||
)
|
)
|
||||||
upsert(day, today_row)
|
upsert(day, today_row)
|
||||||
|
# 3. Self-heal: any RECENT day whose balance sample is gone (one failed
|
||||||
|
# night used to leave a permanent hole in the money chart) is
|
||||||
|
# reconstructed from OC's own transaction ledger, balance column only.
|
||||||
|
heal_missing_balances(balance, balance_read_at)
|
||||||
print(f"metrics: completed {yesterday} -> {json.dumps(yesterday_usage)}")
|
print(f"metrics: completed {yesterday} -> {json.dumps(yesterday_usage)}")
|
||||||
print(f"metrics: recorded {day} -> {json.dumps(today_row)}")
|
print(f"metrics: recorded {day} -> {json.dumps(today_row)}")
|
||||||
return today_row
|
return today_row
|
||||||
|
|||||||
@@ -0,0 +1,195 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Regression tests for the OC-balance self-heal (the Nov 2026 chart hole).
|
||||||
|
|
||||||
|
Incident: the 03:15 snapshot on 2026-11-12 DID record oc_balance_usd=1357.39,
|
||||||
|
but a repair backfill run hours later rewrote the day through backfill()'s
|
||||||
|
full-row upsert — whose row dict carries no balance key — and ON CONFLICT wrote
|
||||||
|
NULL over it. Nothing revisited a day's balance afterwards, so the hole was
|
||||||
|
permanent and the money chart broke mid-line.
|
||||||
|
|
||||||
|
Fixes under test here:
|
||||||
|
* upsert() never erases a balance it did not measure (COALESCE on conflict);
|
||||||
|
* the nightly snapshot heals any RECENT day with a missing balance from OC's
|
||||||
|
transaction ledger (balance column ONLY — membership/usage never touched);
|
||||||
|
* days before the series' first recorded balance stay NULL by design;
|
||||||
|
* an OC failure is fail-soft: hole stays, next night retries, nothing raises.
|
||||||
|
|
||||||
|
Runnable:
|
||||||
|
* from the repo root: python3 tests/test_balance_backfill.py
|
||||||
|
* inside the portal app: PORTAL_CODE_DIR=/app/code python3 tests/test_balance_backfill.py
|
||||||
|
"""
|
||||||
|
import datetime as dt
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
_TEST_DB = os.path.join(tempfile.mkdtemp(prefix="balance-heal-test-"), "members.db")
|
||||||
|
os.environ["DB_PATH"] = _TEST_DB # must exist BEFORE metrics_snapshot is imported
|
||||||
|
|
||||||
|
for _cand in (os.environ.get("PORTAL_CODE_DIR"),
|
||||||
|
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))):
|
||||||
|
if _cand and os.path.isfile(os.path.join(_cand, "metrics_snapshot.py")):
|
||||||
|
sys.path.insert(0, _cand)
|
||||||
|
break
|
||||||
|
|
||||||
|
import metrics_snapshot as ms # noqa: E402
|
||||||
|
|
||||||
|
TODAY = ms.today()
|
||||||
|
YESTERDAY = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=1)).isoformat()
|
||||||
|
DAY_MINUS2 = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=2)).isoformat()
|
||||||
|
DAY_MINUS5 = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=5)).isoformat()
|
||||||
|
|
||||||
|
FAILED = []
|
||||||
|
|
||||||
|
|
||||||
|
def check(name, cond, detail=""):
|
||||||
|
if cond:
|
||||||
|
print(f" ok {name}")
|
||||||
|
else:
|
||||||
|
print(f" FAIL {name} {detail}")
|
||||||
|
FAILED.append(name)
|
||||||
|
|
||||||
|
|
||||||
|
def read_row(day):
|
||||||
|
conn = sqlite3.connect(_TEST_DB)
|
||||||
|
try:
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
return conn.execute("SELECT * FROM metrics_daily WHERE day = ?", (day,)).fetchone()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def wipe():
|
||||||
|
ms.connect().close() # ensure the table exists (schema init)
|
||||||
|
conn = sqlite3.connect(_TEST_DB)
|
||||||
|
conn.execute("DELETE FROM metrics_daily")
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def stub_sources(**kw):
|
||||||
|
"""Replace every network source with fixed in-memory data."""
|
||||||
|
ms.from_sqlite = lambda: {"active": kw.get("active", 40), "total": kw.get("total", 71),
|
||||||
|
"joins": list(kw.get("joins", []))}
|
||||||
|
ms.from_listmonk = lambda: kw.get("subs", 71)
|
||||||
|
ms.listmonk_joins = lambda: list(kw.get("joins", []))
|
||||||
|
ms.from_matrix = lambda: kw.get("matrix", 65)
|
||||||
|
ms.oc_collective_balance = lambda: kw.get("balance", 123.45)
|
||||||
|
ms.spend_by_day = lambda since=None: dict(kw.get("spend", {}))
|
||||||
|
ms.operator_spend_by_day = lambda since=None: None if kw.get("op_spend") is None else dict(kw.get("op_spend") or {})
|
||||||
|
ms.tokens_by_day_model = lambda since=None: {k: dict(v) for k, v in (kw.get("tokens") or {}).items()}
|
||||||
|
|
||||||
|
|
||||||
|
def seed(day, **cols):
|
||||||
|
ms.upsert(day, {"spend_usd": None, "requests": None, "out_tokens": None,
|
||||||
|
"energy_wh": None, "co2e_g": None, "tokens_measured": None,
|
||||||
|
"tokens_estimated": None, "factor_revision": None, **cols})
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------------
|
||||||
|
print("1. REGRESSION: a full-row rewrite without a balance never erases one "
|
||||||
|
"(the 2026-11-12 incident)")
|
||||||
|
wipe()
|
||||||
|
# The day as the 03:15 snapshot wrote it: membership + usage + a live balance.
|
||||||
|
seed(TODAY, spend_usd=3.38, requests=674, members_active=113, members_total=113,
|
||||||
|
members_new=2, newsletter_subs=122, matrix_space_members=36, oc_balance_usd=1357.39)
|
||||||
|
# Full backfill (the shape the 2026-11-12 18:42 repair ran): its today-branch
|
||||||
|
# rewrites the day with a dict that has NO oc_balance_usd key.
|
||||||
|
stub_sources(joins=["2026-11-01"], spend={TODAY: (3.38, 674)}, op_spend={TODAY: 0.9},
|
||||||
|
tokens={TODAY: {"greenpt/glm-5.3-flash": 379040}})
|
||||||
|
ms.backfill()
|
||||||
|
t = read_row(TODAY)
|
||||||
|
check("backfill rewrote the day (test is live)", t["members_total"] == 71, t["members_total"])
|
||||||
|
check("the recorded balance SURVIVED the rewrite", t["oc_balance_usd"] == 1357.39,
|
||||||
|
t["oc_balance_usd"])
|
||||||
|
|
||||||
|
print("2. snapshot() with a failed OC read keeps a balance recorded earlier that day")
|
||||||
|
wipe()
|
||||||
|
seed(TODAY, spend_usd=0.5, requests=10, members_active=40, members_total=71,
|
||||||
|
oc_balance_usd=1357.39)
|
||||||
|
stub_sources(spend={YESTERDAY: (5.0, 100), TODAY: (0.7, 20)}, op_spend=None, tokens={},
|
||||||
|
balance=None)
|
||||||
|
ms.snapshot()
|
||||||
|
t = read_row(TODAY)
|
||||||
|
check("today's row still refreshed (spend updated)", t["spend_usd"] == 0.7, t["spend_usd"])
|
||||||
|
check("earlier balance kept when the fresh read failed", t["oc_balance_usd"] == 1357.39,
|
||||||
|
t["oc_balance_usd"])
|
||||||
|
|
||||||
|
print("3. self-heal reconstructs a missing day from the ledger — balance column ONLY")
|
||||||
|
wipe()
|
||||||
|
seed(DAY_MINUS2, spend_usd=1.0, requests=50, members_active=39, members_total=70,
|
||||||
|
oc_balance_usd=1200.0)
|
||||||
|
# The hole: yesterday's row exists, everything but the balance.
|
||||||
|
seed(YESTERDAY, spend_usd=2.0, requests=60, members_active=39, members_total=70,
|
||||||
|
newsletter_subs=71, matrix_space_members=65)
|
||||||
|
seed(TODAY, spend_usd=0.5, requests=10, members_active=40, members_total=71,
|
||||||
|
oc_balance_usd=1470.44)
|
||||||
|
READ_AT = TODAY + "T03:15:58.123Z"
|
||||||
|
# Ledger after the two cutoffs: yesterday +73.05, day-before +113.05 (all days
|
||||||
|
# cumulative toward the anchor). Signed as OC stores them: CREDIT +, DEBIT -.
|
||||||
|
ledger = [
|
||||||
|
{"createdAt": DAY_MINUS2 + "T10:00:00.000Z", "type": "CREDIT", "amount": {"valueInCents": 5000}},
|
||||||
|
{"createdAt": DAY_MINUS2 + "T10:00:00.200Z", "type": "DEBIT", "amount": {"valueInCents": -500}},
|
||||||
|
{"createdAt": YESTERDAY + "T12:00:00.000Z", "type": "CREDIT", "amount": {"valueInCents": 7805}},
|
||||||
|
{"createdAt": READ_AT, "type": "DEBIT", "amount": {"valueInCents": -1000}}, # exactly at read_at: in the (C, read_at] window
|
||||||
|
]
|
||||||
|
ms.oc_fetch_transactions = lambda date_from, date_to: list(ledger)
|
||||||
|
ms.heal_missing_balances(1470.44, READ_AT)
|
||||||
|
y = read_row(YESTERDAY)
|
||||||
|
d2 = read_row(DAY_MINUS2)
|
||||||
|
t = read_row(TODAY)
|
||||||
|
check("hole filled: yesterday = anchor - net(yesterday 03:15 .. read_at)",
|
||||||
|
y["oc_balance_usd"] == 1470.44 - (7805 - 1000) / 100.0, y["oc_balance_usd"])
|
||||||
|
check("existing days untouched by the heal", d2["oc_balance_usd"] == 1200.0 and t["oc_balance_usd"] == 1470.44)
|
||||||
|
check("membership columns of the healed day intact",
|
||||||
|
y["members_active"] == 39 and y["members_total"] == 70 and y["newsletter_subs"] == 71)
|
||||||
|
check("usage columns of the healed day intact", y["spend_usd"] == 2.0 and y["requests"] == 60)
|
||||||
|
|
||||||
|
print("4. pre-history stays NULL — a hole BEFORE the first recorded day is not invented")
|
||||||
|
wipe()
|
||||||
|
seed(DAY_MINUS5, spend_usd=1.0, requests=5) # before the series' first balance
|
||||||
|
seed(DAY_MINUS2, spend_usd=1.0, requests=5, oc_balance_usd=1200.0)
|
||||||
|
seed(TODAY, oc_balance_usd=1470.44)
|
||||||
|
ms.oc_fetch_transactions = lambda date_from, date_to: []
|
||||||
|
ms.heal_missing_balances(1470.44, READ_AT)
|
||||||
|
d5 = read_row(DAY_MINUS5)
|
||||||
|
check("day before the first balance stays NULL (gap by design)", d5["oc_balance_usd"] is None,
|
||||||
|
d5["oc_balance_usd"])
|
||||||
|
|
||||||
|
print("5. OC ledger failure is fail-soft: hole stays, nothing raises, no zero written")
|
||||||
|
wipe()
|
||||||
|
seed(DAY_MINUS2, oc_balance_usd=1200.0)
|
||||||
|
seed(YESTERDAY, spend_usd=2.0, members_active=39, oc_balance_usd=None)
|
||||||
|
seed(TODAY, oc_balance_usd=1470.44)
|
||||||
|
def _boom(date_from, date_to):
|
||||||
|
raise RuntimeError("OC unreachable (test)")
|
||||||
|
ms.oc_fetch_transactions = _boom
|
||||||
|
ms.heal_missing_balances(1470.44, READ_AT) # must not raise
|
||||||
|
y = read_row(YESTERDAY)
|
||||||
|
check("hole left NULL for the next night", y["oc_balance_usd"] is None, y["oc_balance_usd"])
|
||||||
|
check("no zero written on failure", y["oc_balance_usd"] != 0.0)
|
||||||
|
|
||||||
|
print("6. write_balance on a day with NO row inserts balance + NULL everything else")
|
||||||
|
wipe()
|
||||||
|
ms.write_balance(YESTERDAY, 1357.39)
|
||||||
|
y = read_row(YESTERDAY)
|
||||||
|
check("balance written", y["oc_balance_usd"] == 1357.39)
|
||||||
|
check("membership/usage columns NULL (fail-closed, never zero)",
|
||||||
|
y["members_active"] is None and y["spend_usd"] is None and y["requests"] is None)
|
||||||
|
|
||||||
|
print("7. anchor=None (OC read failed this run) heals nothing")
|
||||||
|
wipe()
|
||||||
|
seed(DAY_MINUS2, oc_balance_usd=1200.0)
|
||||||
|
seed(YESTERDAY, oc_balance_usd=None)
|
||||||
|
seed(TODAY, oc_balance_usd=1470.44)
|
||||||
|
ms.oc_fetch_transactions = lambda date_from, date_to: list(ledger)
|
||||||
|
ms.heal_missing_balances(None, READ_AT)
|
||||||
|
y = read_row(YESTERDAY)
|
||||||
|
check("no heal without an anchor balance", y["oc_balance_usd"] is None, y["oc_balance_usd"])
|
||||||
|
|
||||||
|
print()
|
||||||
|
if FAILED:
|
||||||
|
print(f"FAILED: {len(FAILED)} check(s): {FAILED}")
|
||||||
|
sys.exit(1)
|
||||||
|
print("ALL CHECKS PASSED")
|
||||||
Reference in new issue
Block a user