metrics: complete YESTERDAY's row at 03:15 — every day was frozen at its first ~3 hours
ci/woodpecker/push/woodpecker Pipeline was successful
ci/woodpecker/push/woodpecker Pipeline was successful
snapshot() fetched the full spend window (since = yesterday) but kept only spend.get(TODAY): the day's first hours, written at 03:15 and never revisited. Every metrics_daily row after the last backfill under-counted its day by the hours after 03:15 (Nov 7 recorded $2.19/691 requests against a true $29.06/3384). The completed day's aggregates were computed and thrown away. Now each run completes the day that just ENDED — full-day spend, requests, tokens and impact, written to the usage columns only (upsert_usage: membership columns are point-in-time measurements and are never rewritten) — and then records today's live partial as before. backfill() gains --since YYYY-MM-DD: a targeted usage-only repair that cannot touch the published membership curve. Regression test: tests/test_metrics_completion.py (day completion, membership preservation, since-mode repair boundaries, full-backfill parity).
This commit is contained in:
1 parent
022e193cc2
commit
cf3dbb7489
4 files changed
+365
-61
No files matched your search
+158
-58
@@ -7,6 +7,9 @@ dashboard can chart trends from one source of truth.
|
||||
|
||||
metrics_snapshot.py record / refresh TODAY's row
|
||||
metrics_snapshot.py --backfill rebuild history (idempotent)
|
||||
metrics_snapshot.py --backfill --since YYYY-MM-DD
|
||||
repair recent days' usage only (membership
|
||||
columns are left untouched)
|
||||
|
||||
Rules this file follows deliberately:
|
||||
|
||||
@@ -14,6 +17,11 @@ Rules this file follows deliberately:
|
||||
columns, never a zero. A zero reads as "nobody", which is a lie that would
|
||||
survive into a chart; NULL reads as "not measured". Each failure is printed,
|
||||
and the task's stdout lands in the app log (`cloudron logs --app <id>`).
|
||||
- **The 03:15 run completes YESTERDAY, then records today live.** Each run
|
||||
writes the day that just ended with its FULL spend/requests/tokens/impact
|
||||
aggregates, and today's row with the first hours of live data (completed by
|
||||
the next run). Writing only today's partial and never revisiting it froze
|
||||
every day at its first ~3 hours — the November 2026 under-count.
|
||||
- **Cumulative vs daily.** `members_active`, `members_total`, `newsletter_subs`
|
||||
and `matrix_space_members` are counts AS OF that day; `spend_usd` and
|
||||
`requests` are THAT day's traffic. Charts must treat them accordingly.
|
||||
@@ -577,6 +585,88 @@ def oc_collective_balance() -> float | None:
|
||||
|
||||
# ---------------------------------------------------------------- writes
|
||||
|
||||
# The columns of metrics_daily that describe THAT DAY'S TRAFFIC and its impact,
|
||||
# as opposed to the point-in-time membership counts. Completing a day rewrites
|
||||
# exactly these and nothing else.
|
||||
USAGE_COLUMNS = (
|
||||
"spend_usd",
|
||||
"operator_spend_usd",
|
||||
"requests",
|
||||
"out_tokens",
|
||||
"energy_wh",
|
||||
"co2e_g",
|
||||
"tokens_measured",
|
||||
"tokens_estimated",
|
||||
"factor_revision",
|
||||
)
|
||||
|
||||
|
||||
def _usage_fields(
|
||||
spend: dict[str, tuple[float, int]],
|
||||
op_spend: dict[str, float] | None,
|
||||
tokens: dict[str, dict[str, int]],
|
||||
key: str,
|
||||
) -> dict:
|
||||
"""The usage/impact columns for one day, from the spend-log aggregates.
|
||||
|
||||
Fail-closed per metric: a day absent from the window reads as NULL ("not
|
||||
measured"), never a zero — the same convention snapshot() has always used.
|
||||
"""
|
||||
sp = spend.get(key, (None, None))
|
||||
if op_spend is None or sp[0] is None:
|
||||
op = None
|
||||
else:
|
||||
op = op_spend.get(key, 0.0)
|
||||
day_models = tokens.get(key) or {}
|
||||
tot = sum(day_models.values())
|
||||
impact = impact_for(day_models)
|
||||
return {
|
||||
"spend_usd": sp[0],
|
||||
"operator_spend_usd": op,
|
||||
"requests": sp[1],
|
||||
"out_tokens": tot or None,
|
||||
"energy_wh": impact["wh"],
|
||||
"co2e_g": impact["co2e_g"],
|
||||
"tokens_measured": impact["tokens_measured"],
|
||||
"tokens_estimated": impact["tokens_estimated"],
|
||||
"factor_revision": impact["factor_revision"],
|
||||
}
|
||||
|
||||
|
||||
def upsert_usage(day: str, usage: dict) -> None:
|
||||
"""Write ONLY the usage/impact columns of a metrics_daily row.
|
||||
|
||||
This is how a day is COMPLETED after it has ended: spend_usd, requests,
|
||||
out_tokens and the impact columns are recomputable from the full spend
|
||||
logs, but members_active / members_total / newsletter_subs /
|
||||
matrix_space_members are point-in-time measurements taken THAT day and are
|
||||
not reconstructible afterwards — so this never overwrites them. A row that
|
||||
does not exist yet is INSERTed with those columns NULL ("not measured",
|
||||
per the module's fail-closed rule), never a zero.
|
||||
"""
|
||||
cols = [c for c in USAGE_COLUMNS if c in usage]
|
||||
if not cols:
|
||||
return
|
||||
conn = connect()
|
||||
try:
|
||||
conn.execute(
|
||||
f"""INSERT INTO metrics_daily
|
||||
(day, {', '.join(cols)}, recorded_at)
|
||||
VALUES ({', '.join('?' * (len(cols) + 2))})
|
||||
ON CONFLICT(day) DO UPDATE SET
|
||||
{', '.join(f'{c} = excluded.{c}' for c in cols)},
|
||||
recorded_at = excluded.recorded_at""",
|
||||
(
|
||||
day,
|
||||
*(usage[c] for c in cols),
|
||||
dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def upsert(day: str, row: dict) -> None:
|
||||
conn = connect()
|
||||
try:
|
||||
@@ -632,50 +722,50 @@ def upsert(day: str, row: dict) -> None:
|
||||
|
||||
def snapshot(day: str | None = None) -> dict:
|
||||
day = day or today()
|
||||
yesterday = (dt.date.fromisoformat(day) - dt.timedelta(days=1)).isoformat()
|
||||
m = from_sqlite()
|
||||
subs = from_listmonk()
|
||||
joins = listmonk_joins()
|
||||
space = from_matrix()
|
||||
# One window covers BOTH rows this run writes: `yesterday` in full (the day
|
||||
# just ended — what the charts must report) and `day` partially (the hours
|
||||
# so far, completed by the next run).
|
||||
since = dt.date.fromisoformat(day) - dt.timedelta(days=1)
|
||||
spend = spend_by_day(since=since)
|
||||
today_spend = spend.get(day, (None, None))
|
||||
op_spend = operator_spend_by_day(since=since)
|
||||
tokens = tokens_by_day_model(since=since)
|
||||
balance = oc_collective_balance()
|
||||
# 1. Complete the day that just ENDED. The window above already contains
|
||||
# yesterday's full aggregates — the old code fetched them and threw them
|
||||
# away, keeping only today's first ~3 hours and never revisiting the
|
||||
# row, which froze every day at its first ~3 hours (the Nov 2026
|
||||
# under-count). Membership columns are deliberately NOT touched here:
|
||||
# they were measured yesterday morning, as of that day, and cannot be
|
||||
# reconstructed now.
|
||||
yesterday_usage = _usage_fields(spend, op_spend, tokens, yesterday)
|
||||
upsert_usage(yesterday, yesterday_usage)
|
||||
# 2. Today's live row — full row, as before. Its usage columns carry the
|
||||
# hours so far; step 1 of the next run completes them.
|
||||
today_row = _usage_fields(spend, op_spend, tokens, day)
|
||||
# The co-op's own share of today's spend. If the LiteLLM DB could not be
|
||||
# read, both spend and operator_spend are NULL (not measured). If it was
|
||||
# read and operators simply used nothing, that is a measured 0.0 — but only
|
||||
# when there WAS spend to attribute that day.
|
||||
if op_spend is None or today_spend[0] is None:
|
||||
op_today = None
|
||||
else:
|
||||
op_today = op_spend.get(day, 0.0)
|
||||
day_models = (tokens_by_day_model(since=since).get(day)) or {}
|
||||
tot = sum(day_models.values())
|
||||
out_tokens = tot or None
|
||||
impact = impact_for(day_models)
|
||||
balance = oc_collective_balance()
|
||||
row = {
|
||||
"members_active": m["active"],
|
||||
"members_total": m["total"],
|
||||
"members_new": (sum(1 for j in joins if j == day) if joins is not None else None),
|
||||
"newsletter_subs": subs,
|
||||
"matrix_space_members": space,
|
||||
"spend_usd": today_spend[0],
|
||||
"operator_spend_usd": op_today,
|
||||
"requests": today_spend[1],
|
||||
"out_tokens": out_tokens,
|
||||
"energy_wh": impact["wh"],
|
||||
"co2e_g": impact["co2e_g"],
|
||||
"tokens_measured": impact["tokens_measured"],
|
||||
"tokens_estimated": impact["tokens_estimated"],
|
||||
"factor_revision": impact["factor_revision"],
|
||||
"oc_balance_usd": balance,
|
||||
}
|
||||
upsert(day, row)
|
||||
print(f"metrics: recorded {day} -> {json.dumps(row)}")
|
||||
return row
|
||||
# when there WAS spend to attribute that day. (_usage_fields enforces this.)
|
||||
today_row.update(
|
||||
members_active=m["active"],
|
||||
members_total=m["total"],
|
||||
members_new=(sum(1 for j in joins if j == day) if joins is not None else None),
|
||||
newsletter_subs=subs,
|
||||
matrix_space_members=space,
|
||||
oc_balance_usd=balance,
|
||||
)
|
||||
upsert(day, today_row)
|
||||
print(f"metrics: completed {yesterday} -> {json.dumps(yesterday_usage)}")
|
||||
print(f"metrics: recorded {day} -> {json.dumps(today_row)}")
|
||||
return today_row
|
||||
|
||||
|
||||
def backfill() -> None:
|
||||
def backfill(since: str | None = None) -> None:
|
||||
"""Rebuild history: membership from list-join dates, usage from spend logs.
|
||||
|
||||
Membership uses Listmonk's per-subscription `subscription_created_at` — the
|
||||
@@ -683,6 +773,12 @@ def backfill() -> None:
|
||||
in it (their departure is likewise absent), so past days are the cumulative
|
||||
joins of today's members: an honest floor for "how many of us were here by
|
||||
that day", not a reconstruction of the roster at the time.
|
||||
|
||||
With `since` (YYYY-MM-DD) the rebuild starts at that day and writes ONLY the
|
||||
usage/impact columns (upsert_usage): a targeted repair of recent days must
|
||||
not rewrite the published membership curve, which the nightly snapshots
|
||||
measured at the time. Without it, the full history is rebuilt exactly as
|
||||
before, membership included.
|
||||
"""
|
||||
m = from_sqlite()
|
||||
joins = sorted(listmonk_joins() or [])
|
||||
@@ -695,49 +791,43 @@ def backfill() -> None:
|
||||
print("metrics: backfill found no history at all")
|
||||
return
|
||||
|
||||
candidates = list(joins) + list(spend)
|
||||
first = min(candidates)
|
||||
first = min(list(joins) + list(spend))
|
||||
if since:
|
||||
first = max(first, dt.date.fromisoformat(since).isoformat())
|
||||
last = today()
|
||||
print(
|
||||
f"metrics: backfilling {first}..{last} "
|
||||
f"({len(joins)} join dates, {len(spend)} spend days, {len(tokens)} token days)"
|
||||
+ (f", usage-only from {since}" if since else "")
|
||||
)
|
||||
day = dt.date.fromisoformat(first)
|
||||
end = dt.date.fromisoformat(last)
|
||||
written = 0
|
||||
while day <= end:
|
||||
key = day.isoformat()
|
||||
# Cumulative members AS OF this day: current members whose list-join date
|
||||
# is on or before it. Days before anyone joined stay 0.
|
||||
members_total = sum(1 for j in joins if j <= key) if joins else None
|
||||
row = {
|
||||
"members_total": members_total,
|
||||
"members_new": sum(1 for j in joins if j == key),
|
||||
"spend_usd": spend.get(key, (None, None))[0],
|
||||
"operator_spend_usd": (
|
||||
None if op_spend is None
|
||||
else (op_spend.get(key, 0.0) if key in spend else None)
|
||||
),
|
||||
"requests": spend.get(key, (None, None))[1],
|
||||
}
|
||||
day_models = tokens.get(key) or {}
|
||||
if sum(day_models.values()):
|
||||
row["out_tokens"] = sum(day_models.values())
|
||||
impact = impact_for(day_models)
|
||||
row["energy_wh"] = impact["wh"]
|
||||
row["co2e_g"] = impact["co2e_g"]
|
||||
row["tokens_measured"] = impact["tokens_measured"]
|
||||
row["tokens_estimated"] = impact["tokens_estimated"]
|
||||
row["factor_revision"] = impact["factor_revision"]
|
||||
usage = _usage_fields(spend, op_spend, tokens, key)
|
||||
if key == today():
|
||||
# today's row carries the exact live metrics
|
||||
row = dict(usage)
|
||||
row.update(
|
||||
members_active=m["active"],
|
||||
members_total=m["total"],
|
||||
members_new=(sum(1 for j in joins if j == key) if joins is not None else None),
|
||||
newsletter_subs=from_listmonk(),
|
||||
matrix_space_members=from_matrix(),
|
||||
)
|
||||
upsert(key, row)
|
||||
upsert(key, row)
|
||||
elif since:
|
||||
# targeted repair: rewrite the usage columns only; the membership
|
||||
# columns keep what the nightly snapshots measured that day
|
||||
upsert_usage(key, usage)
|
||||
else:
|
||||
row = {
|
||||
"members_total": sum(1 for j in joins if j <= key) if joins else None,
|
||||
"members_new": sum(1 for j in joins if j == key),
|
||||
**usage,
|
||||
}
|
||||
upsert(key, row)
|
||||
written += 1
|
||||
day += dt.timedelta(days=1)
|
||||
print(f"metrics: backfill wrote {written} day(s)")
|
||||
@@ -745,6 +835,16 @@ def backfill() -> None:
|
||||
|
||||
if __name__ == "__main__":
|
||||
if "--backfill" in sys.argv:
|
||||
backfill()
|
||||
_since = None
|
||||
if "--since" in sys.argv:
|
||||
_i = sys.argv.index("--since")
|
||||
if _i + 1 >= len(sys.argv):
|
||||
sys.exit("metrics: --since requires a date (YYYY-MM-DD)")
|
||||
_since = sys.argv[_i + 1]
|
||||
try:
|
||||
dt.date.fromisoformat(_since)
|
||||
except ValueError:
|
||||
sys.exit(f"metrics: --since must be YYYY-MM-DD, got {_since!r}")
|
||||
backfill(_since)
|
||||
else:
|
||||
snapshot()
|
||||
Reference in new issue
Block a user