From cf3dbb748954d935c912bc85e8b5b89c174620ad Mon Sep 17 00:00:00 2001 From: Inferencebot Date: Thu, 12 Nov 2026 18:40:31 +0000 Subject: [PATCH] =?UTF-8?q?metrics:=20complete=20YESTERDAY's=20row=20at=20?= =?UTF-8?q?03:15=20=E2=80=94=20every=20day=20was=20frozen=20at=20its=20fir?= =?UTF-8?q?st=20~3=20hours?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit snapshot() fetched the full spend window (since = yesterday) but kept only spend.get(TODAY): the day's first hours, written at 03:15 and never revisited. Every metrics_daily row after the last backfill under-counted its day by the hours after 03:15 (Nov 7 recorded $2.19/691 requests against a true $29.06/3384). The completed day's aggregates were computed and thrown away. Now each run completes the day that just ENDED — full-day spend, requests, tokens and impact, written to the usage columns only (upsert_usage: membership columns are point-in-time measurements and are never rewritten) — and then records today's live partial as before. backfill() gains --since YYYY-MM-DD: a targeted usage-only repair that cannot touch the published membership curve. Regression test: tests/test_metrics_completion.py (day completion, membership preservation, since-mode repair boundaries, full-backfill parity). --- README.md | 9 +- docs/energy-metrics.md | 11 +- metrics_snapshot.py | 216 ++++++++++++++++++++++--------- tests/test_metrics_completion.py | 190 +++++++++++++++++++++++++++ 4 files changed, 365 insertions(+), 61 deletions(-) create mode 100644 tests/test_metrics_completion.py diff --git a/README.md b/README.md index a889c92..0a1ddc7 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,10 @@ Three jobs run inside the container via `CloudronManifest.json` → `addons.sche - **metrics_snapshot** (daily 3:15am UTC) — `metrics_snapshot.py` records one row per day into the `metrics_daily` table: members, newsletter subscribers, Matrix membership, spend, request counts, energy/CO₂, and the Open Collective - balance. A source that cannot be read + balance. Each run **completes yesterday's row** with its full-day + spend/requests/tokens/impact aggregates, then records today's row live (its + usage columns carry the hours so far and are completed by the next run's + completion pass). A source that cannot be read yields NULL for that metric rather than a zero. `spend_usd` is the co-op-wide total and deliberately spans every team, the co-op's own operator accounts included; `operator_spend_usd` is the breakout of that portion, served through @@ -209,7 +212,9 @@ Three jobs run inside the container via `CloudronManifest.json` → `addons.sche OC balance (the OC API has no historical series, so balance history accrues from these snapshots only — 2026-11-08 onward). `--backfill` rebuilds history idempotently (balance history cannot be backfilled: past balances were never - recorded). Methodology: [Energy and carbon metrics](docs/energy-metrics.md). + recorded); `--backfill --since YYYY-MM-DD` repairs recent days' usage columns + only, leaving the membership curve as the nightly snapshots measured it. + Methodology: [Energy and carbon metrics](docs/energy-metrics.md). Verify after any manifest change: the deploy log must show `setupAddons: ... scheduler ...` listing all three tasks, and each script must diff --git a/docs/energy-metrics.md b/docs/energy-metrics.md index 75b2b42..1112383 100644 --- a/docs/energy-metrics.md +++ b/docs/energy-metrics.md @@ -155,10 +155,19 @@ than by us. `members_total`, `members_new`, `newsletter_subs`, `matrix_space_members`, `spend_usd`, `requests`, `out_tokens`, `energy_wh`, `co2e_g`, `tokens_measured`, `tokens_estimated`, `factor_revision`, `recorded_at`. +- **Each 03:15 run completes YESTERDAY, then records today live.** The day that + just ended is rewritten with its FULL spend/requests/tokens/impact aggregates + (usage columns only — the membership columns are point-in-time measurements + that are not reconstructible afterwards). Today's row carries the first hours + of live data and is completed by the next run. (Until 2026-11-12 the run + wrote only today's partial and never revisited it, which froze every day at + its first ~3 hours — the November under-count.) - **`metrics_snapshot.py --backfill` rebuilds history** from the sources and is idempotent — it rewrites a day rather than duplicating it. It is the tool to run after changing a factor, and the reason a published chart can move: history is - **rewritten**, not appended to. + **rewritten**, not appended to. `--backfill --since YYYY-MM-DD` limits the + rewrite to days from that date and touches ONLY the usage/impact columns, so + a targeted repair of recent days cannot alter the published membership curve. ## What is measured, and what is not diff --git a/metrics_snapshot.py b/metrics_snapshot.py index 9a77030..1320534 100644 --- a/metrics_snapshot.py +++ b/metrics_snapshot.py @@ -7,6 +7,9 @@ dashboard can chart trends from one source of truth. metrics_snapshot.py record / refresh TODAY's row metrics_snapshot.py --backfill rebuild history (idempotent) + metrics_snapshot.py --backfill --since YYYY-MM-DD + repair recent days' usage only (membership + columns are left untouched) Rules this file follows deliberately: @@ -14,6 +17,11 @@ Rules this file follows deliberately: columns, never a zero. A zero reads as "nobody", which is a lie that would survive into a chart; NULL reads as "not measured". Each failure is printed, and the task's stdout lands in the app log (`cloudron logs --app `). +- **The 03:15 run completes YESTERDAY, then records today live.** Each run + writes the day that just ended with its FULL spend/requests/tokens/impact + aggregates, and today's row with the first hours of live data (completed by + the next run). Writing only today's partial and never revisiting it froze + every day at its first ~3 hours — the November 2026 under-count. - **Cumulative vs daily.** `members_active`, `members_total`, `newsletter_subs` and `matrix_space_members` are counts AS OF that day; `spend_usd` and `requests` are THAT day's traffic. Charts must treat them accordingly. @@ -577,6 +585,88 @@ def oc_collective_balance() -> float | None: # ---------------------------------------------------------------- writes +# The columns of metrics_daily that describe THAT DAY'S TRAFFIC and its impact, +# as opposed to the point-in-time membership counts. Completing a day rewrites +# exactly these and nothing else. +USAGE_COLUMNS = ( + "spend_usd", + "operator_spend_usd", + "requests", + "out_tokens", + "energy_wh", + "co2e_g", + "tokens_measured", + "tokens_estimated", + "factor_revision", +) + + +def _usage_fields( + spend: dict[str, tuple[float, int]], + op_spend: dict[str, float] | None, + tokens: dict[str, dict[str, int]], + key: str, +) -> dict: + """The usage/impact columns for one day, from the spend-log aggregates. + + Fail-closed per metric: a day absent from the window reads as NULL ("not + measured"), never a zero — the same convention snapshot() has always used. + """ + sp = spend.get(key, (None, None)) + if op_spend is None or sp[0] is None: + op = None + else: + op = op_spend.get(key, 0.0) + day_models = tokens.get(key) or {} + tot = sum(day_models.values()) + impact = impact_for(day_models) + return { + "spend_usd": sp[0], + "operator_spend_usd": op, + "requests": sp[1], + "out_tokens": tot or None, + "energy_wh": impact["wh"], + "co2e_g": impact["co2e_g"], + "tokens_measured": impact["tokens_measured"], + "tokens_estimated": impact["tokens_estimated"], + "factor_revision": impact["factor_revision"], + } + + +def upsert_usage(day: str, usage: dict) -> None: + """Write ONLY the usage/impact columns of a metrics_daily row. + + This is how a day is COMPLETED after it has ended: spend_usd, requests, + out_tokens and the impact columns are recomputable from the full spend + logs, but members_active / members_total / newsletter_subs / + matrix_space_members are point-in-time measurements taken THAT day and are + not reconstructible afterwards — so this never overwrites them. A row that + does not exist yet is INSERTed with those columns NULL ("not measured", + per the module's fail-closed rule), never a zero. + """ + cols = [c for c in USAGE_COLUMNS if c in usage] + if not cols: + return + conn = connect() + try: + conn.execute( + f"""INSERT INTO metrics_daily + (day, {', '.join(cols)}, recorded_at) + VALUES ({', '.join('?' * (len(cols) + 2))}) + ON CONFLICT(day) DO UPDATE SET + {', '.join(f'{c} = excluded.{c}' for c in cols)}, + recorded_at = excluded.recorded_at""", + ( + day, + *(usage[c] for c in cols), + dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"), + ), + ) + conn.commit() + finally: + conn.close() + + def upsert(day: str, row: dict) -> None: conn = connect() try: @@ -632,50 +722,50 @@ def upsert(day: str, row: dict) -> None: def snapshot(day: str | None = None) -> dict: day = day or today() + yesterday = (dt.date.fromisoformat(day) - dt.timedelta(days=1)).isoformat() m = from_sqlite() subs = from_listmonk() joins = listmonk_joins() space = from_matrix() + # One window covers BOTH rows this run writes: `yesterday` in full (the day + # just ended — what the charts must report) and `day` partially (the hours + # so far, completed by the next run). since = dt.date.fromisoformat(day) - dt.timedelta(days=1) spend = spend_by_day(since=since) - today_spend = spend.get(day, (None, None)) op_spend = operator_spend_by_day(since=since) + tokens = tokens_by_day_model(since=since) + balance = oc_collective_balance() + # 1. Complete the day that just ENDED. The window above already contains + # yesterday's full aggregates — the old code fetched them and threw them + # away, keeping only today's first ~3 hours and never revisiting the + # row, which froze every day at its first ~3 hours (the Nov 2026 + # under-count). Membership columns are deliberately NOT touched here: + # they were measured yesterday morning, as of that day, and cannot be + # reconstructed now. + yesterday_usage = _usage_fields(spend, op_spend, tokens, yesterday) + upsert_usage(yesterday, yesterday_usage) + # 2. Today's live row — full row, as before. Its usage columns carry the + # hours so far; step 1 of the next run completes them. + today_row = _usage_fields(spend, op_spend, tokens, day) # The co-op's own share of today's spend. If the LiteLLM DB could not be # read, both spend and operator_spend are NULL (not measured). If it was # read and operators simply used nothing, that is a measured 0.0 — but only - # when there WAS spend to attribute that day. - if op_spend is None or today_spend[0] is None: - op_today = None - else: - op_today = op_spend.get(day, 0.0) - day_models = (tokens_by_day_model(since=since).get(day)) or {} - tot = sum(day_models.values()) - out_tokens = tot or None - impact = impact_for(day_models) - balance = oc_collective_balance() - row = { - "members_active": m["active"], - "members_total": m["total"], - "members_new": (sum(1 for j in joins if j == day) if joins is not None else None), - "newsletter_subs": subs, - "matrix_space_members": space, - "spend_usd": today_spend[0], - "operator_spend_usd": op_today, - "requests": today_spend[1], - "out_tokens": out_tokens, - "energy_wh": impact["wh"], - "co2e_g": impact["co2e_g"], - "tokens_measured": impact["tokens_measured"], - "tokens_estimated": impact["tokens_estimated"], - "factor_revision": impact["factor_revision"], - "oc_balance_usd": balance, - } - upsert(day, row) - print(f"metrics: recorded {day} -> {json.dumps(row)}") - return row + # when there WAS spend to attribute that day. (_usage_fields enforces this.) + today_row.update( + members_active=m["active"], + members_total=m["total"], + members_new=(sum(1 for j in joins if j == day) if joins is not None else None), + newsletter_subs=subs, + matrix_space_members=space, + oc_balance_usd=balance, + ) + upsert(day, today_row) + print(f"metrics: completed {yesterday} -> {json.dumps(yesterday_usage)}") + print(f"metrics: recorded {day} -> {json.dumps(today_row)}") + return today_row -def backfill() -> None: +def backfill(since: str | None = None) -> None: """Rebuild history: membership from list-join dates, usage from spend logs. Membership uses Listmonk's per-subscription `subscription_created_at` — the @@ -683,6 +773,12 @@ def backfill() -> None: in it (their departure is likewise absent), so past days are the cumulative joins of today's members: an honest floor for "how many of us were here by that day", not a reconstruction of the roster at the time. + + With `since` (YYYY-MM-DD) the rebuild starts at that day and writes ONLY the + usage/impact columns (upsert_usage): a targeted repair of recent days must + not rewrite the published membership curve, which the nightly snapshots + measured at the time. Without it, the full history is rebuilt exactly as + before, membership included. """ m = from_sqlite() joins = sorted(listmonk_joins() or []) @@ -695,49 +791,43 @@ def backfill() -> None: print("metrics: backfill found no history at all") return - candidates = list(joins) + list(spend) - first = min(candidates) + first = min(list(joins) + list(spend)) + if since: + first = max(first, dt.date.fromisoformat(since).isoformat()) last = today() print( f"metrics: backfilling {first}..{last} " f"({len(joins)} join dates, {len(spend)} spend days, {len(tokens)} token days)" + + (f", usage-only from {since}" if since else "") ) day = dt.date.fromisoformat(first) end = dt.date.fromisoformat(last) written = 0 while day <= end: key = day.isoformat() - # Cumulative members AS OF this day: current members whose list-join date - # is on or before it. Days before anyone joined stay 0. - members_total = sum(1 for j in joins if j <= key) if joins else None - row = { - "members_total": members_total, - "members_new": sum(1 for j in joins if j == key), - "spend_usd": spend.get(key, (None, None))[0], - "operator_spend_usd": ( - None if op_spend is None - else (op_spend.get(key, 0.0) if key in spend else None) - ), - "requests": spend.get(key, (None, None))[1], - } - day_models = tokens.get(key) or {} - if sum(day_models.values()): - row["out_tokens"] = sum(day_models.values()) - impact = impact_for(day_models) - row["energy_wh"] = impact["wh"] - row["co2e_g"] = impact["co2e_g"] - row["tokens_measured"] = impact["tokens_measured"] - row["tokens_estimated"] = impact["tokens_estimated"] - row["factor_revision"] = impact["factor_revision"] + usage = _usage_fields(spend, op_spend, tokens, key) if key == today(): # today's row carries the exact live metrics + row = dict(usage) row.update( members_active=m["active"], members_total=m["total"], + members_new=(sum(1 for j in joins if j == key) if joins is not None else None), newsletter_subs=from_listmonk(), matrix_space_members=from_matrix(), ) - upsert(key, row) + upsert(key, row) + elif since: + # targeted repair: rewrite the usage columns only; the membership + # columns keep what the nightly snapshots measured that day + upsert_usage(key, usage) + else: + row = { + "members_total": sum(1 for j in joins if j <= key) if joins else None, + "members_new": sum(1 for j in joins if j == key), + **usage, + } + upsert(key, row) written += 1 day += dt.timedelta(days=1) print(f"metrics: backfill wrote {written} day(s)") @@ -745,6 +835,16 @@ def backfill() -> None: if __name__ == "__main__": if "--backfill" in sys.argv: - backfill() + _since = None + if "--since" in sys.argv: + _i = sys.argv.index("--since") + if _i + 1 >= len(sys.argv): + sys.exit("metrics: --since requires a date (YYYY-MM-DD)") + _since = sys.argv[_i + 1] + try: + dt.date.fromisoformat(_since) + except ValueError: + sys.exit(f"metrics: --since must be YYYY-MM-DD, got {_since!r}") + backfill(_since) else: snapshot() diff --git a/tests/test_metrics_completion.py b/tests/test_metrics_completion.py new file mode 100644 index 0000000..467ec52 --- /dev/null +++ b/tests/test_metrics_completion.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +"""Regression tests for the metrics day-completion fix (Nov 2026 under-count). + +Bug: snapshot() wrote only TODAY's row at 03:15 UTC — the first ~3 hours of +that day's traffic — and never revisited it, so every metrics_daily row was +frozen at its first hours (Nov 6 read $1.20/93 requests vs a true $34.26/4508). + +Fix: each run completes YESTERDAY's row with its full-day usage/impact +aggregates (usage columns ONLY — membership columns are point-in-time +measurements that must survive), then records today live as before. +backfill() gains --since: a targeted usage-only repair that leaves the +published membership curve untouched. + +Runnable: + * from the repo root: python3 tests/test_metrics_completion.py + * inside the portal app: PORTAL_CODE_DIR=/app/code python3 tests/test_metrics_completion.py +""" +import datetime as dt +import os +import sqlite3 +import sys +import tempfile + +_TEST_DB = os.path.join(tempfile.mkdtemp(prefix="metrics-test-"), "members.db") +os.environ["DB_PATH"] = _TEST_DB # must exist BEFORE metrics_snapshot is imported + +for _cand in (os.environ.get("PORTAL_CODE_DIR"), + os.path.dirname(os.path.dirname(os.path.abspath(__file__)))): + if _cand and os.path.isfile(os.path.join(_cand, "metrics_snapshot.py")): + sys.path.insert(0, _cand) + break + +import metrics_snapshot as ms # noqa: E402 + +TODAY = ms.today() +YESTERDAY = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=1)).isoformat() +DAY_BEFORE = (dt.date.fromisoformat(TODAY) - dt.timedelta(days=2)).isoformat() + +FAILED = [] + + +def check(name, cond, detail=""): + if cond: + print(f" ok {name}") + else: + print(f" FAIL {name} {detail}") + FAILED.append(name) + + +def stub_sources(spend, op_spend, tokens, joins=(), active=40, total=71, + subs=71, matrix=65, balance=123.45): + """Replace every network source with fixed in-memory data.""" + ms.from_sqlite = lambda: {"active": active, "total": total, "joins": list(joins)} + ms.from_listmonk = lambda: subs + ms.listmonk_joins = lambda: list(joins) + ms.from_matrix = lambda: matrix + ms.oc_collective_balance = lambda: balance + ms.spend_by_day = lambda since=None: dict(spend) + ms.operator_spend_by_day = lambda since=None: None if op_spend is None else dict(op_spend) + ms.tokens_by_day_model = lambda since=None: {k: dict(v) for k, v in tokens.items()} + + +def read_row(day): + conn = sqlite3.connect(_TEST_DB) + try: + conn.row_factory = sqlite3.Row + return conn.execute("SELECT * FROM metrics_daily WHERE day = ?", (day,)).fetchone() + finally: + conn.close() + + +def seed(day, **cols): + """Insert a row with arbitrary columns (the nightly snapshot's shape).""" + ms.upsert(day, {"spend_usd": None, "requests": None, "out_tokens": None, + "energy_wh": None, "co2e_g": None, "tokens_measured": None, + "tokens_estimated": None, "factor_revision": None, **cols}) + + +# ---------------------------------------------------------------------- +print("1. snapshot() completes YESTERDAY with full-day usage, records today live") +stub_sources( + spend={YESTERDAY: (29.0604, 3384), TODAY: (2.19, 691)}, + op_spend={YESTERDAY: 1.1613, TODAY: 0.5}, + tokens={YESTERDAY: {"greenpt/glm-5.3-flash": 500000}, + TODAY: {"greenpt/glm-5.3-flash": 100000}}, + joins=["2026-11-01"], +) +row = ms.snapshot() +y = read_row(YESTERDAY) +t = read_row(TODAY) +check("yesterday spend_usd = full-day total", y["spend_usd"] == 29.0604, y["spend_usd"]) +check("yesterday requests = full-day count", y["requests"] == 3384, y["requests"]) +check("yesterday operator_spend carried", y["operator_spend_usd"] == 1.1613, y["operator_spend_usd"]) +check("yesterday out_tokens from full day", y["out_tokens"] == 500000, y["out_tokens"]) +check("yesterday energy computed from tokens", y["energy_wh"] is not None and y["energy_wh"] > 0) +check("yesterday co2e_g computed (>0, unit tripwire band)", y["co2e_g"] is not None and y["co2e_g"] > 0) +check("today's row keeps the live partial", t["spend_usd"] == 2.19 and t["requests"] == 691) +check("today's row carries live membership", t["members_total"] == 71 and t["members_active"] == 40) +check("today's row carries oc_balance", t["oc_balance_usd"] == 123.45) +check("yesterday's membership columns are NULL (not measured, never 0)", + y["members_active"] is None and y["members_total"] is None and y["newsletter_subs"] is None) + +print("2. completion overwrites a stale partial WITHOUT touching membership columns") +conn = sqlite3.connect(_TEST_DB) +conn.execute("DELETE FROM metrics_daily") +conn.commit() +conn.close() +# What the OLD nightly code left behind: partial usage + that morning's counts. +seed(YESTERDAY, spend_usd=2.192, requests=691, members_active=39, members_total=70, + newsletter_subs=70, matrix_space_members=64) +stub_sources( + spend={YESTERDAY: (29.0604, 3384), TODAY: (0.5, 100)}, + op_spend={YESTERDAY: 1.1613, TODAY: None}, + tokens={YESTERDAY: {"greenpt/glm-5.3-flash": 500000}}, +) +ms.snapshot() +y = read_row(YESTERDAY) +check("stale partial spend replaced with full-day value", y["spend_usd"] == 29.0604, y["spend_usd"]) +check("stale partial requests replaced", y["requests"] == 3384, y["requests"]) +check("members_active preserved from that morning", y["members_active"] == 39, y["members_active"]) +check("members_total preserved", y["members_total"] == 70, y["members_total"]) +check("newsletter_subs preserved", y["newsletter_subs"] == 70) +check("matrix_space_members preserved", y["matrix_space_members"] == 64) + +print("3. upsert_usage on a missing row INSERTs usage + NULL membership (fail-closed)") +conn = sqlite3.connect(_TEST_DB) +conn.execute("DELETE FROM metrics_daily") +conn.commit() +conn.close() +ms.upsert_usage(DAY_BEFORE, {"spend_usd": 5.0, "requests": 42, "out_tokens": None, + "energy_wh": None, "co2e_g": None, "tokens_measured": None, + "tokens_estimated": None, "factor_revision": None, + "operator_spend_usd": None}) +y = read_row(DAY_BEFORE) +check("usage columns written", y["spend_usd"] == 5.0 and y["requests"] == 42) +check("membership columns NULL, never zero", + y["members_active"] is None and y["members_total"] is None and y["members_new"] is None) + +print("4. backfill(since=...) repairs usage only — membership curve untouched") +conn = sqlite3.connect(_TEST_DB) +conn.execute("DELETE FROM metrics_daily") +conn.commit() +conn.close() +# Three days as the nightly snapshots wrote them: membership present, usage partial. +for d, sp, rq in ((DAY_BEFORE, 1.0, 50), (YESTERDAY, 2.0, 60)): + seed(d, spend_usd=sp, requests=rq, members_active=39, members_total=70, + members_new=3, newsletter_subs=70, matrix_space_members=64) +seed(TODAY, spend_usd=0.5, requests=10, members_active=40, members_total=71) +stub_sources( + spend={DAY_BEFORE: (17.8791, 4493), YESTERDAY: (29.0604, 3384), TODAY: (0.5, 10)}, + op_spend={DAY_BEFORE: 0.2, YESTERDAY: 1.1613, TODAY: None}, + tokens={DAY_BEFORE: {"greenpt/deepseek-v4.1-flash": 200000}, + YESTERDAY: {"greenpt/glm-5.3-flash": 500000}}, + joins=["2026-11-01"], +) +ms.backfill(since=YESTERDAY) +y = read_row(YESTERDAY) +db = read_row(DAY_BEFORE) +t = read_row(TODAY) +check("repaired day matches ground truth spend", y["spend_usd"] == 29.0604, y["spend_usd"]) +check("repaired day matches ground truth requests", y["requests"] == 3384, y["requests"]) +check("repaired day keeps its membership columns", + y["members_active"] == 39 and y["members_total"] == 70 and y["members_new"] == 3) +check("day BEFORE --since untouched", db["spend_usd"] == 1.0 and db["requests"] == 50) +check("today still gets the full live update", t["members_total"] == 71) + +print("5. full backfill (no --since) rebuilds membership columns as before") +conn = sqlite3.connect(_TEST_DB) +conn.execute("DELETE FROM metrics_daily") +conn.commit() +conn.close() +ms.backfill() +y = read_row(YESTERDAY) +check("full backfill writes joins-derived members_total", + y["members_total"] == 1, y["members_total"]) # one join on 2026-11-01 +check("full backfill still writes full-day usage", y["spend_usd"] == 29.0604) + +print("6. provenance: a day with tokens but no measured factor reports estimated, not measured") +ms.upsert_usage(DAY_BEFORE, {"spend_usd": 1.0, "requests": 5, "out_tokens": 1000, + "energy_wh": 0.5, "co2e_g": 0.125, + "tokens_measured": 0, "tokens_estimated": 1000, + "factor_revision": "x"}) +y = read_row(DAY_BEFORE) +check("split persisted", y["tokens_measured"] == 0 and y["tokens_estimated"] == 1000) + +print() +if FAILED: + print(f"FAILED: {len(FAILED)} check(s): {FAILED}") + sys.exit(1) +print("ALL CHECKS PASSED")