metrics: fix the impact columns — group by model_group, never write a zero
Two bugs from the first deploy, both caught by reading the numbers back: 1. Tokens were summed per raw `model`, which carries the UPSTREAM name (openai/deepseek-v4.1-flash), while the factor table is keyed on our provider aliases (greenpt/deepseek-v4.1-flash). Every model therefore matched nothing. Now grouped on model_group, the same convention aggregate_model_usage uses, with model as the fallback for old rows. 2. Every day was written as energy_wh = 0.0 / co2e_g = 0.0 — a false claim that the co-op used no energy, and precisely the fail-open this module is written to avoid. A day with no measured tokens now yields NULL for all three impact columns, and impact_for() refuses to compute from an empty total. The backfill is idempotent, so re-running it overwrote the zero rows.
This commit is contained in:
1 parent
89593cfa74
commit
8e9906d844
1 file changed
+18
-7
+18
-7
@@ -294,16 +294,21 @@ def tokens_by_day_model(since: dt.date | None = None) -> dict[str, dict[str, int
|
||||
if since:
|
||||
where, params = 'WHERE "startTime" >= %s', [since.isoformat()]
|
||||
with conn.cursor() as cur:
|
||||
# model_group carries our provider/model alias (e.g.
|
||||
# greenpt/deepseek-v4.1-flash) — the name the factor table is keyed
|
||||
# on. Raw `model` holds the upstream name (openai/deepseek-v4.1-flash)
|
||||
# and would match nothing. Same convention as aggregate_model_usage.
|
||||
cur.execute(
|
||||
f"""SELECT to_char("startTime", 'YYYY-MM-DD') AS day, model,
|
||||
f"""SELECT to_char("startTime", 'YYYY-MM-DD') AS day,
|
||||
COALESCE(NULLIF(model_group, ''), NULLIF(model, ''), 'unknown') AS alias,
|
||||
COALESCE(SUM(completion_tokens), 0)
|
||||
FROM "LiteLLM_SpendLogs" {where}
|
||||
GROUP BY 1, 2""",
|
||||
params,
|
||||
)
|
||||
out: dict[str, dict[str, int]] = {}
|
||||
for day, model, toks in cur.fetchall():
|
||||
out.setdefault(day, {})[model or "unknown"] = int(toks)
|
||||
for day, alias, toks in cur.fetchall():
|
||||
out.setdefault(day, {})[alias or "unknown"] = int(toks)
|
||||
finally:
|
||||
conn.close()
|
||||
return out
|
||||
@@ -319,6 +324,12 @@ def impact_for(models: dict[str, int]) -> tuple[float | None, float | None]:
|
||||
"""
|
||||
if not models:
|
||||
return None, None
|
||||
if not sum(t for t in models.values() if t):
|
||||
# No measured tokens for this day: report "not measured" (NULL), never a
|
||||
# zero. A zero reads as "the co-op used no energy" — a lie that would
|
||||
# survive into a chart. The first backfill shipped exactly that, because
|
||||
# it summed the wrong column and every model totalled 0.
|
||||
return None, None
|
||||
wh = 0.0
|
||||
co2 = 0.0
|
||||
for model, toks in models.items():
|
||||
@@ -388,9 +399,9 @@ def snapshot(day: str | None = None) -> dict:
|
||||
spend = spend_by_day(since=since)
|
||||
today_spend = spend.get(day, (None, None))
|
||||
day_models = (tokens_by_day_model(since=since).get(day)) or {}
|
||||
out_tokens, energy_wh, co2e_g = (
|
||||
(sum(day_models.values()), *impact_for(day_models)) if day_models else (None, None, None)
|
||||
)
|
||||
tot = sum(day_models.values())
|
||||
out_tokens = tot or None
|
||||
energy_wh, co2e_g = impact_for(day_models) if tot else (None, None)
|
||||
row = {
|
||||
"members_active": m["active"],
|
||||
"members_total": m["total"],
|
||||
@@ -449,7 +460,7 @@ def backfill() -> None:
|
||||
"requests": spend.get(key, (None, None))[1],
|
||||
}
|
||||
day_models = tokens.get(key) or {}
|
||||
if day_models:
|
||||
if sum(day_models.values()):
|
||||
row["out_tokens"] = sum(day_models.values())
|
||||
row["energy_wh"], row["co2e_g"] = impact_for(day_models)
|
||||
if key == today():
|
||||
|
||||
Reference in new issue
Block a user