diff --git a/app/main.py b/app/main.py index 29544ae..5178916 100644 --- a/app/main.py +++ b/app/main.py @@ -698,12 +698,24 @@ def get_db() -> sqlite3.Connection: "out_tokens INTEGER, " "energy_wh REAL, " "co2e_g REAL, " + "tokens_measured INTEGER, " + "tokens_estimated INTEGER, " + "factor_revision TEXT, " "recorded_at TEXT" ")" ) # CREATE IF NOT EXISTS never adds columns to an existing table, so migrate: - # impact columns arrived after the table did (see co-op/support#26). - for _col, _decl in (("out_tokens", "INTEGER"), ("energy_wh", "REAL"), ("co2e_g", "REAL")): + # impact columns arrived after the table did (see co-op/support#26), and the + # measured-vs-estimated provenance columns after those. All additive: existing + # rows keep NULL, and a NULL split reports as unknown — never as "measured". + for _col, _decl in ( + ("out_tokens", "INTEGER"), + ("energy_wh", "REAL"), + ("co2e_g", "REAL"), + ("tokens_measured", "INTEGER"), + ("tokens_estimated", "INTEGER"), + ("factor_revision", "TEXT"), + ): try: conn.execute(f"ALTER TABLE metrics_daily ADD COLUMN {_col} {_decl}") except sqlite3.OperationalError: @@ -3165,6 +3177,31 @@ async def admin_overview(request: Request): } +def _metrics_provenance(tokens_measured: int | None, tokens_estimated: int | None) -> str: + """Classify a day's measured-vs-estimated split for the API. + + "measured" — every token rested on a directly measured factor. + "estimated" — every token rested on a non-measured factor (borrowed + SAME_FAMILY or the flat proxy). + "mixed" — the day's total is a blend of both. + "none" — no provenance recorded: either the day has no output tokens + or the row predates these columns (both split columns NULL). + Deliberately NOT "measured" — an unknown split must never read + as a confident measurement. + """ + if tokens_measured is None and tokens_estimated is None: + return "none" + m = tokens_measured or 0 + e = tokens_estimated or 0 + if m and e: + return "mixed" + if m: + return "measured" + if e: + return "estimated" + return "none" + + def metrics_series(days: int = 90) -> list[dict]: """Daily metrics rows, oldest first. @@ -3176,13 +3213,21 @@ def metrics_series(days: int = 90) -> list[dict]: energy_wh and co2e_g are computed in metrics_snapshot.py from the day's output tokens against a factor table: measured for GreenPT models (their API returns an `impact` object), estimated for the rest. See co-op/support#26. + + Each row also carries its provenance: `tokens_measured`/`tokens_estimated` + (the output-token split across measured vs non-measured factors), + `factor_revision` (a short stable hash of the factor table that produced the + row), and a derived `provenance` indicator — measured | mixed | estimated | + none. Rows written before these columns exist have NULL splits and report + `provenance: "none"`, never a confident "measured". See docs/energy-metrics.md. """ limit = max(1, min(int(days or 90), 730)) conn = get_db() try: rows = conn.execute( "SELECT day, members_active, members_total, members_new, newsletter_subs, " - "matrix_space_members, spend_usd, requests, out_tokens, energy_wh, co2e_g, recorded_at " + "matrix_space_members, spend_usd, requests, out_tokens, energy_wh, co2e_g, " + "tokens_measured, tokens_estimated, factor_revision, recorded_at " "FROM metrics_daily ORDER BY day DESC LIMIT ?", (limit,), ).fetchall() @@ -3200,9 +3245,19 @@ def metrics_series(days: int = 90) -> list[dict]: "out_tokens", "energy_wh", "co2e_g", + "tokens_measured", + "tokens_estimated", + "factor_revision", "recorded_at", ] - return [dict(zip(cols, row)) for row in reversed(rows)] + series = [] + for row in reversed(rows): + rec = dict(zip(cols, row)) + rec["provenance"] = _metrics_provenance( + rec["tokens_measured"], rec["tokens_estimated"] + ) + series.append(rec) + return series @app.get("/admin/metrics/series") @@ -3219,6 +3274,11 @@ async def admin_metrics_series(request: Request, days: int = 90): counts as of that day; spend_usd/requests are that day's traffic only. Backfilled days carry a LOWER BOUND for members_total (members provisioned without a welcome email are absent from the history). + + Each row carries its energy provenance — `provenance` (measured | mixed | + estimated | none) plus `tokens_measured`, `tokens_estimated` and + `factor_revision`. A row written before those columns existed reports + `provenance: "none"`, not "measured". See docs/energy-metrics.md. """ _verify_admin_token(request) series = metrics_series(days) @@ -3607,6 +3667,12 @@ async def broker_metrics(request: Request, days: int = 90): it is AGGREGATE and non-personal — co-op size, requests served, spend — the same class of social-proof data as /broker/model-leaderboard. No per-member figure passes through here, and members asked to see how the co-op is doing. + + Rows carry the energy provenance the admin series does: `provenance` + (measured | mixed | estimated | none) plus `tokens_measured`, + `tokens_estimated` and `factor_revision` — so a member-facing chart can say + how much of a day's figure rested on a measurement. NULL splits (older rows) + report `provenance: "none"`. See docs/energy-metrics.md. """ _verify_broker_secret(request) series = metrics_series(days) diff --git a/docs/energy-metrics.md b/docs/energy-metrics.md index 8a8cc6b..75b2b42 100644 --- a/docs/energy-metrics.md +++ b/docs/energy-metrics.md @@ -49,9 +49,55 @@ Factors live in `metrics_snapshot.py` (`IMPACT_FACTORS`, `IMPACT_PROXY`). Matchi not disclose impact, **deliberately**, so that a day's total is never quietly missing models. It is not a number we know. -**No tier is exposed anywhere.** The endpoints (`/admin/metrics/series`, -`/broker/metrics`) return bare numbers, so a consumer cannot tell measured from -estimated. Exposing provenance is tracked as a separate issue. +**No tier is exposed on its own**, but every row now carries a *provenance* read +derived from the measured/estimated token split — see "Provenance in the API" +below. A consumer can tell how much of a day's figure rested on a measurement. + +## Provenance in the API + +Every row returned by `/admin/metrics/series` and `/broker/metrics` carries the +measured-vs-estimated split, so a consumer can tell how much of a day's figure +rested on a real measurement. Four fields: + +| field | meaning | +|---|---| +| `tokens_measured` | output tokens that day attributed to a factor whose `source` is **MEASURED** — a direct provider measurement. | +| `tokens_estimated` | the rest: **SAME_FAMILY** (borrowed from a sibling) and the flat **proxy**. A borrowed or proxied factor is an assumption, so it is **not** counted as measured. | +| `factor_revision` | a short, stable hash of the factor table (`IMPACT_FACTORS` + `IMPACT_PROXY`) in effect when the row was computed. | +| `provenance` | the derived read, below. | + +`tokens_measured + tokens_estimated = out_tokens` for any day with tokens. A day +with no output tokens has NULL in both (fail-closed, same rule as energy/carbon). + +`provenance` is one of: + +- **`measured`** — every token rested on a measured factor. +- **`estimated`** — every token rested on a non-measured factor (borrowed or proxy). +- **`mixed`** — both are present; the day's total is a blend. +- **`none`** — no provenance. This is **either** a day with no output tokens **or** + a row written before these columns existed (older rows keep NULL). It is + deliberately **not** `measured`: an unknown split must never read as a confident + measurement. This is the whole point of the indicator — failure reads as unknown. + +`factor_revision` is a deterministic hash, **not a timestamp**: two runs against the +same table produce the same string. It changes if any factor value or source label +changes, or if the matching order changes (order is meaningful — longest prefix +first). It is "what the numbers were computed with", not "what was known then". + +### Forward-only, then backfill + +The columns are added by an **additive** migration — a guarded +`ALTER TABLE metrics_daily ADD COLUMN …`, the same idiom used when the impact +columns first arrived. On an existing database this is safe: it never errors, +existing rows survive untouched, and they simply keep NULL in the new columns. +Nothing is guessed for history on migration; those rows read as `provenance: "none"`. + +To give past rows a real split, run `metrics_snapshot.py --backfill` (or re-run the +affected days). Backfill rewrites each day from the current LiteLLM token split and +the **current** factor table, so it also updates `factor_revision` to today's +revision — a backfilled historical row records the revision used to recompute it, +not the one that was current at the time. Honest for a chart label; not a historical +audit trail. ## Formulas @@ -67,8 +113,8 @@ intended — which is why the tripwire below exists. ## NULL, never zero -`impact_for()` returns `None, None` for a day with no measured tokens, and the row is -written with NULL energy and carbon. It never writes `0.0`: +`impact_for()` returns an all-NULL impact block for a day with no measured tokens, +and the row is written with NULL energy and carbon. It never writes `0.0`: > a zero reads as "the co-op used no energy", which is a claim we cannot support, and > it would survive into a chart as a fact. @@ -107,7 +153,8 @@ than by us. - It writes **one row per day** into the `metrics_daily` table, in the same SQLite database as the members table (`members.db`). Columns: `day`, `members_active`, `members_total`, `members_new`, `newsletter_subs`, `matrix_space_members`, - `spend_usd`, `requests`, `out_tokens`, `energy_wh`, `co2e_g`, `recorded_at`. + `spend_usd`, `requests`, `out_tokens`, `energy_wh`, `co2e_g`, `tokens_measured`, + `tokens_estimated`, `factor_revision`, `recorded_at`. - **`metrics_snapshot.py --backfill` rebuilds history** from the sources and is idempotent — it rewrites a day rather than duplicating it. It is the tool to run after changing a factor, and the reason a published chart can move: history is @@ -122,9 +169,11 @@ Stated plainly, because a mixed total is easy to over-read: - **Estimated:** everything else. Currently that means **all Tinfoil and Regolo traffic**, which is a real share of co-op usage. -So a day's figure is a blend, and **the split is not recorded per day** — you cannot -tell from `metrics_daily` how much of a total rested on the proxy. Persisting that -split is a separate follow-up. +So a day's figure is a blend, and **the split is now recorded per day** — +`tokens_measured` and `tokens_estimated` say how much of the total rested on a +measurement, and the API derives a `provenance` read from them. Before those +columns existed the split was not recorded; such rows carry NULL and read as +`provenance: "none"`. See "Provenance in the API". One consequence worth knowing: the proxy assumes a 250 gCO2e/kWh grid, while Regolo runs on 100% renewable energy, so the proxy **overstates** Regolo's carbon. Per-provider diff --git a/metrics_snapshot.py b/metrics_snapshot.py index c06effb..f5255e7 100644 --- a/metrics_snapshot.py +++ b/metrics_snapshot.py @@ -27,6 +27,7 @@ Rules this file follows deliberately: """ import datetime as dt +import hashlib import json import os import sqlite3 @@ -60,10 +61,23 @@ def connect() -> sqlite3.Connection: out_tokens INTEGER, energy_wh REAL, co2e_g REAL, + tokens_measured INTEGER, + tokens_estimated INTEGER, + factor_revision TEXT, recorded_at TEXT)""" ) # CREATE IF NOT EXISTS never adds columns to an existing table, so migrate. - for col, decl in (("out_tokens", "INTEGER"), ("energy_wh", "REAL"), ("co2e_g", "REAL")): + for col, decl in ( + ("out_tokens", "INTEGER"), + ("energy_wh", "REAL"), + ("co2e_g", "REAL"), + # Measured-vs-estimated provenance (see co-op/support#26 follow-ups). + # Additive: existing rows keep NULL, and NULL reports as "unknown", never + # as a confident "measured". + ("tokens_measured", "INTEGER"), + ("tokens_estimated", "INTEGER"), + ("factor_revision", "TEXT"), + ): try: conn.execute(f"ALTER TABLE metrics_daily ADD COLUMN {col} {decl}") except sqlite3.OperationalError: @@ -105,6 +119,30 @@ IMPACT_FACTORS = { } IMPACT_PROXY = {"wh_per_1k_out": 0.5000, "gco2e_per_kwh": 250.0, "source": ESTIMATED} + +def _factor_revision() -> str: + """Short, stable fingerprint of the factor table in force. + + Serialised WITHOUT sort_keys, preserving both the per-factor key order and the + table's longest-prefix-first ordering — because reordering IMPACT_FACTORS + changes which factor a model matches, so the revision must move with it. A + deterministic hash of values rather than a timestamp: two runs against the + same table produce the same string, and a chart label can trust it. + """ + payload = json.dumps( + { + "factors": [[prefix, IMPACT_FACTORS[prefix]] for prefix in IMPACT_FACTORS], + "proxy": IMPACT_PROXY, + }, + separators=(",", ":"), + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:8] + + +# Revision of the CURRENT factor table. Written onto every row computed from it, so +# a chart can tell which factor set produced a number. See docs/energy-metrics.md. +FACTOR_REVISION = _factor_revision() + # ---------------------------------------------------------------- sources def listmonk_joins() -> list[str] | None: @@ -325,8 +363,38 @@ def tokens_by_day_model(since: dt.date | None = None) -> dict[str, dict[str, int return out -def impact_for(models: dict[str, int]) -> tuple[float | None, float | None]: - """(Wh, gCO2e) for a day's {model: output_tokens}. +def _no_impact() -> dict: + """The impact block for a day with no output tokens: all NULL, fail-closed. + + NULL energy/carbon means "not measured" (see the module docstring). The + provenance split is NULL too, so an unknown split cannot be read as a + confident "0 measured". + """ + return { + "wh": None, + "co2e_g": None, + "tokens_measured": None, + "tokens_estimated": None, + "factor_revision": None, + } + + +def impact_for(models: dict[str, int]) -> dict: + """Impact for a day's {model: output_tokens}, and the provenance split. + + Returns a dict: + wh, co2e_g — totals, unchanged from before (same factors, same + matching, same arithmetic, same rounding). + tokens_measured — output tokens attributed to a factor whose `source` is + MEASURED (a direct provider measurement). + tokens_estimated — the rest: SAME_FAMILY (borrowed from a sibling) and the + flat proxy. A borrowed or proxied factor is an + assumption, so it is NOT counted as measured. + factor_revision — the revision of the factor table used. + + The split is reported, not recomputed: the numeric behaviour is identical to + the previous (Wh, gCO2e)-only version. `tokens_measured + tokens_estimated` + equals the day's output tokens for any day with tokens. Factors live in IMPACT_FACTORS. Unknown models fall back to the documented proxy row rather than being skipped, so a total is never quietly missing a @@ -334,15 +402,17 @@ def impact_for(models: dict[str, int]) -> tuple[float | None, float | None]: docs say so. Energy: Wh per 1,000 output tokens. Carbon: gCO2e per kWh. """ if not models: - return None, None + return _no_impact() if not sum(t for t in models.values() if t): # No measured tokens for this day: report "not measured" (NULL), never a # zero. A zero reads as "the co-op used no energy" — a lie that would # survive into a chart. The first backfill shipped exactly that, because # it summed the wrong column and every model totalled 0. - return None, None + return _no_impact() wh = 0.0 co2 = 0.0 + tokens_measured = 0 + tokens_estimated = 0 for model, toks in models.items(): if not toks: continue @@ -352,6 +422,12 @@ def impact_for(models: dict[str, int]) -> tuple[float | None, float | None]: f = factor break f = f or IMPACT_PROXY + # Provenance is a pure reporting side-channel: only MEASURED is a direct + # measurement; SAME_FAMILY (borrowed) and the proxy are not. + if f.get("source") == MEASURED: + tokens_measured += toks + else: + tokens_estimated += toks wh += toks / 1000.0 * f["wh_per_1k_out"] # Wh -> kWh is /1e3, then x gCO2e/kWh. This divided by 1e6 once, which # understated carbon exactly 1000x: the energy column stayed plausible @@ -366,7 +442,13 @@ def impact_for(models: dict[str, int]) -> tuple[float | None, float | None]: f"metrics: WARNING implied grid intensity {implied:.1f} gCO2e/kWh is outside " f"the plausible band — suspect a unit error before trusting these numbers" ) - return round(wh, 6), round(co2, 6) + return { + "wh": round(wh, 6), + "co2e_g": round(co2, 6), + "tokens_measured": tokens_measured, + "tokens_estimated": tokens_estimated, + "factor_revision": FACTOR_REVISION, + } # ---------------------------------------------------------------- writes @@ -378,8 +460,8 @@ def upsert(day: str, row: dict) -> None: """INSERT INTO metrics_daily (day, members_active, members_total, members_new, newsletter_subs, matrix_space_members, spend_usd, requests, out_tokens, energy_wh, - co2e_g, recorded_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + co2e_g, tokens_measured, tokens_estimated, factor_revision, recorded_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(day) DO UPDATE SET members_active=excluded.members_active, members_total=excluded.members_total, @@ -391,6 +473,9 @@ def upsert(day: str, row: dict) -> None: out_tokens=excluded.out_tokens, energy_wh=excluded.energy_wh, co2e_g=excluded.co2e_g, + tokens_measured=excluded.tokens_measured, + tokens_estimated=excluded.tokens_estimated, + factor_revision=excluded.factor_revision, recorded_at=excluded.recorded_at""", ( day, @@ -404,6 +489,9 @@ def upsert(day: str, row: dict) -> None: row.get("out_tokens"), row.get("energy_wh"), row.get("co2e_g"), + row.get("tokens_measured"), + row.get("tokens_estimated"), + row.get("factor_revision"), dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"), ), ) @@ -424,7 +512,7 @@ def snapshot(day: str | None = None) -> dict: day_models = (tokens_by_day_model(since=since).get(day)) or {} tot = sum(day_models.values()) out_tokens = tot or None - energy_wh, co2e_g = impact_for(day_models) if tot else (None, None) + impact = impact_for(day_models) row = { "members_active": m["active"], "members_total": m["total"], @@ -434,8 +522,11 @@ def snapshot(day: str | None = None) -> dict: "spend_usd": today_spend[0], "requests": today_spend[1], "out_tokens": out_tokens, - "energy_wh": energy_wh, - "co2e_g": co2e_g, + "energy_wh": impact["wh"], + "co2e_g": impact["co2e_g"], + "tokens_measured": impact["tokens_measured"], + "tokens_estimated": impact["tokens_estimated"], + "factor_revision": impact["factor_revision"], } upsert(day, row) print(f"metrics: recorded {day} -> {json.dumps(row)}") @@ -485,7 +576,12 @@ def backfill() -> None: day_models = tokens.get(key) or {} if sum(day_models.values()): row["out_tokens"] = sum(day_models.values()) - row["energy_wh"], row["co2e_g"] = impact_for(day_models) + impact = impact_for(day_models) + row["energy_wh"] = impact["wh"] + row["co2e_g"] = impact["co2e_g"] + row["tokens_measured"] = impact["tokens_measured"] + row["tokens_estimated"] = impact["tokens_estimated"] + row["factor_revision"] = impact["factor_revision"] if key == today(): # today's row carries the exact live metrics row.update(