metrics: complete YESTERDAY's row at 03:15 — every day was frozen at its first ~3 hours
ci/woodpecker/push/woodpecker Pipeline was successful

snapshot() fetched the full spend window (since = yesterday) but kept only
spend.get(TODAY): the day's first hours, written at 03:15 and never revisited.
Every metrics_daily row after the last backfill under-counted its day by the
hours after 03:15 (Nov 7 recorded $2.19/691 requests against a true
$29.06/3384). The completed day's aggregates were computed and thrown away.

Now each run completes the day that just ENDED — full-day spend, requests,
tokens and impact, written to the usage columns only (upsert_usage: membership
columns are point-in-time measurements and are never rewritten) — and then
records today's live partial as before.

backfill() gains --since YYYY-MM-DD: a targeted usage-only repair that cannot
touch the published membership curve.

Regression test: tests/test_metrics_completion.py (day completion, membership
preservation, since-mode repair boundaries, full-backfill parity).
This commit is contained in:
inference-bot committed 2026-11-12 18:40:31 +00:00
1 parent 022e193cc2
commit cf3dbb7489
4 files changed
+365 -61

No files matched your search

+158 -58
View File
@@ -7,6 +7,9 @@ dashboard can chart trends from one source of truth.
metrics_snapshot.py record / refresh TODAY's row
metrics_snapshot.py --backfill rebuild history (idempotent)
metrics_snapshot.py --backfill --since YYYY-MM-DD
repair recent days' usage only (membership
columns are left untouched)
Rules this file follows deliberately:
@@ -14,6 +17,11 @@ Rules this file follows deliberately:
columns, never a zero. A zero reads as "nobody", which is a lie that would
survive into a chart; NULL reads as "not measured". Each failure is printed,
and the task's stdout lands in the app log (`cloudron logs --app <id>`).
- **The 03:15 run completes YESTERDAY, then records today live.** Each run
writes the day that just ended with its FULL spend/requests/tokens/impact
aggregates, and today's row with the first hours of live data (completed by
the next run). Writing only today's partial and never revisiting it froze
every day at its first ~3 hours — the November 2026 under-count.
- **Cumulative vs daily.** `members_active`, `members_total`, `newsletter_subs`
and `matrix_space_members` are counts AS OF that day; `spend_usd` and
`requests` are THAT day's traffic. Charts must treat them accordingly.
@@ -577,6 +585,88 @@ def oc_collective_balance() -> float | None:
# ---------------------------------------------------------------- writes
# The columns of metrics_daily that describe THAT DAY'S TRAFFIC and its impact,
# as opposed to the point-in-time membership counts. Completing a day rewrites
# exactly these and nothing else.
USAGE_COLUMNS = (
"spend_usd",
"operator_spend_usd",
"requests",
"out_tokens",
"energy_wh",
"co2e_g",
"tokens_measured",
"tokens_estimated",
"factor_revision",
)
def _usage_fields(
spend: dict[str, tuple[float, int]],
op_spend: dict[str, float] | None,
tokens: dict[str, dict[str, int]],
key: str,
) -> dict:
"""The usage/impact columns for one day, from the spend-log aggregates.
Fail-closed per metric: a day absent from the window reads as NULL ("not
measured"), never a zero — the same convention snapshot() has always used.
"""
sp = spend.get(key, (None, None))
if op_spend is None or sp[0] is None:
op = None
else:
op = op_spend.get(key, 0.0)
day_models = tokens.get(key) or {}
tot = sum(day_models.values())
impact = impact_for(day_models)
return {
"spend_usd": sp[0],
"operator_spend_usd": op,
"requests": sp[1],
"out_tokens": tot or None,
"energy_wh": impact["wh"],
"co2e_g": impact["co2e_g"],
"tokens_measured": impact["tokens_measured"],
"tokens_estimated": impact["tokens_estimated"],
"factor_revision": impact["factor_revision"],
}
def upsert_usage(day: str, usage: dict) -> None:
"""Write ONLY the usage/impact columns of a metrics_daily row.
This is how a day is COMPLETED after it has ended: spend_usd, requests,
out_tokens and the impact columns are recomputable from the full spend
logs, but members_active / members_total / newsletter_subs /
matrix_space_members are point-in-time measurements taken THAT day and are
not reconstructible afterwards — so this never overwrites them. A row that
does not exist yet is INSERTed with those columns NULL ("not measured",
per the module's fail-closed rule), never a zero.
"""
cols = [c for c in USAGE_COLUMNS if c in usage]
if not cols:
return
conn = connect()
try:
conn.execute(
f"""INSERT INTO metrics_daily
(day, {', '.join(cols)}, recorded_at)
VALUES ({', '.join('?' * (len(cols) + 2))})
ON CONFLICT(day) DO UPDATE SET
{', '.join(f'{c} = excluded.{c}' for c in cols)},
recorded_at = excluded.recorded_at""",
(
day,
*(usage[c] for c in cols),
dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
),
)
conn.commit()
finally:
conn.close()
def upsert(day: str, row: dict) -> None:
conn = connect()
try:
@@ -632,50 +722,50 @@ def upsert(day: str, row: dict) -> None:
def snapshot(day: str | None = None) -> dict:
day = day or today()
yesterday = (dt.date.fromisoformat(day) - dt.timedelta(days=1)).isoformat()
m = from_sqlite()
subs = from_listmonk()
joins = listmonk_joins()
space = from_matrix()
# One window covers BOTH rows this run writes: `yesterday` in full (the day
# just ended — what the charts must report) and `day` partially (the hours
# so far, completed by the next run).
since = dt.date.fromisoformat(day) - dt.timedelta(days=1)
spend = spend_by_day(since=since)
today_spend = spend.get(day, (None, None))
op_spend = operator_spend_by_day(since=since)
tokens = tokens_by_day_model(since=since)
balance = oc_collective_balance()
# 1. Complete the day that just ENDED. The window above already contains
# yesterday's full aggregates — the old code fetched them and threw them
# away, keeping only today's first ~3 hours and never revisiting the
# row, which froze every day at its first ~3 hours (the Nov 2026
# under-count). Membership columns are deliberately NOT touched here:
# they were measured yesterday morning, as of that day, and cannot be
# reconstructed now.
yesterday_usage = _usage_fields(spend, op_spend, tokens, yesterday)
upsert_usage(yesterday, yesterday_usage)
# 2. Today's live row — full row, as before. Its usage columns carry the
# hours so far; step 1 of the next run completes them.
today_row = _usage_fields(spend, op_spend, tokens, day)
# The co-op's own share of today's spend. If the LiteLLM DB could not be
# read, both spend and operator_spend are NULL (not measured). If it was
# read and operators simply used nothing, that is a measured 0.0 — but only
# when there WAS spend to attribute that day.
if op_spend is None or today_spend[0] is None:
op_today = None
else:
op_today = op_spend.get(day, 0.0)
day_models = (tokens_by_day_model(since=since).get(day)) or {}
tot = sum(day_models.values())
out_tokens = tot or None
impact = impact_for(day_models)
balance = oc_collective_balance()
row = {
"members_active": m["active"],
"members_total": m["total"],
"members_new": (sum(1 for j in joins if j == day) if joins is not None else None),
"newsletter_subs": subs,
"matrix_space_members": space,
"spend_usd": today_spend[0],
"operator_spend_usd": op_today,
"requests": today_spend[1],
"out_tokens": out_tokens,
"energy_wh": impact["wh"],
"co2e_g": impact["co2e_g"],
"tokens_measured": impact["tokens_measured"],
"tokens_estimated": impact["tokens_estimated"],
"factor_revision": impact["factor_revision"],
"oc_balance_usd": balance,
}
upsert(day, row)
print(f"metrics: recorded {day} -> {json.dumps(row)}")
return row
# when there WAS spend to attribute that day. (_usage_fields enforces this.)
today_row.update(
members_active=m["active"],
members_total=m["total"],
members_new=(sum(1 for j in joins if j == day) if joins is not None else None),
newsletter_subs=subs,
matrix_space_members=space,
oc_balance_usd=balance,
)
upsert(day, today_row)
print(f"metrics: completed {yesterday} -> {json.dumps(yesterday_usage)}")
print(f"metrics: recorded {day} -> {json.dumps(today_row)}")
return today_row
def backfill() -> None:
def backfill(since: str | None = None) -> None:
"""Rebuild history: membership from list-join dates, usage from spend logs.
Membership uses Listmonk's per-subscription `subscription_created_at` — the
@@ -683,6 +773,12 @@ def backfill() -> None:
in it (their departure is likewise absent), so past days are the cumulative
joins of today's members: an honest floor for "how many of us were here by
that day", not a reconstruction of the roster at the time.
With `since` (YYYY-MM-DD) the rebuild starts at that day and writes ONLY the
usage/impact columns (upsert_usage): a targeted repair of recent days must
not rewrite the published membership curve, which the nightly snapshots
measured at the time. Without it, the full history is rebuilt exactly as
before, membership included.
"""
m = from_sqlite()
joins = sorted(listmonk_joins() or [])
@@ -695,49 +791,43 @@ def backfill() -> None:
print("metrics: backfill found no history at all")
return
candidates = list(joins) + list(spend)
first = min(candidates)
first = min(list(joins) + list(spend))
if since:
first = max(first, dt.date.fromisoformat(since).isoformat())
last = today()
print(
f"metrics: backfilling {first}..{last} "
f"({len(joins)} join dates, {len(spend)} spend days, {len(tokens)} token days)"
+ (f", usage-only from {since}" if since else "")
)
day = dt.date.fromisoformat(first)
end = dt.date.fromisoformat(last)
written = 0
while day <= end:
key = day.isoformat()
# Cumulative members AS OF this day: current members whose list-join date
# is on or before it. Days before anyone joined stay 0.
members_total = sum(1 for j in joins if j <= key) if joins else None
row = {
"members_total": members_total,
"members_new": sum(1 for j in joins if j == key),
"spend_usd": spend.get(key, (None, None))[0],
"operator_spend_usd": (
None if op_spend is None
else (op_spend.get(key, 0.0) if key in spend else None)
),
"requests": spend.get(key, (None, None))[1],
}
day_models = tokens.get(key) or {}
if sum(day_models.values()):
row["out_tokens"] = sum(day_models.values())
impact = impact_for(day_models)
row["energy_wh"] = impact["wh"]
row["co2e_g"] = impact["co2e_g"]
row["tokens_measured"] = impact["tokens_measured"]
row["tokens_estimated"] = impact["tokens_estimated"]
row["factor_revision"] = impact["factor_revision"]
usage = _usage_fields(spend, op_spend, tokens, key)
if key == today():
# today's row carries the exact live metrics
row = dict(usage)
row.update(
members_active=m["active"],
members_total=m["total"],
members_new=(sum(1 for j in joins if j == key) if joins is not None else None),
newsletter_subs=from_listmonk(),
matrix_space_members=from_matrix(),
)
upsert(key, row)
upsert(key, row)
elif since:
# targeted repair: rewrite the usage columns only; the membership
# columns keep what the nightly snapshots measured that day
upsert_usage(key, usage)
else:
row = {
"members_total": sum(1 for j in joins if j <= key) if joins else None,
"members_new": sum(1 for j in joins if j == key),
**usage,
}
upsert(key, row)
written += 1
day += dt.timedelta(days=1)
print(f"metrics: backfill wrote {written} day(s)")
@@ -745,6 +835,16 @@ def backfill() -> None:
if __name__ == "__main__":
if "--backfill" in sys.argv:
backfill()
_since = None
if "--since" in sys.argv:
_i = sys.argv.index("--since")
if _i + 1 >= len(sys.argv):
sys.exit("metrics: --since requires a date (YYYY-MM-DD)")
_since = sys.argv[_i + 1]
try:
dt.date.fromisoformat(_since)
except ValueError:
sys.exit(f"metrics: --since must be YYYY-MM-DD, got {_since!r}")
backfill(_since)
else:
snapshot()