metrics: expose + persist measured-vs-estimated provenance (#4, #6)

Thread the per-factor source through impact_for() instead of discarding it.
Numeric output (energy_wh, co2e_g) is byte-identical — this is a reporting
change, not a recalculation.

#6 persistence: additive columns tokens_measured, tokens_estimated and
factor_revision (a stable hash of the factor table) on metrics_daily, added via
the existing guarded ALTER TABLE idiom in both app/main.py and
metrics_snapshot.py. Existing rows keep NULL.

#4 API: metrics_series() selects the new columns and derives a per-row
provenance indicator (measured | mixed | estimated | none) from the split; both
/admin/metrics/series and /broker/metrics return it. NULL/unknown splits report
'none', never a confident 'measured'.

docs/energy-metrics.md documents the indicator, the new columns, and the
forward-only-vs-backfill behaviour.

app/main.py

@@ -698,12 +698,24 @@ def get_db() -> sqlite3.Connection:
         "out_tokens INTEGER, "
         "energy_wh REAL, "
         "co2e_g REAL, "
+        "tokens_measured INTEGER, "
+        "tokens_estimated INTEGER, "
+        "factor_revision TEXT, "
         "recorded_at TEXT"
         ")"
     )
     # CREATE IF NOT EXISTS never adds columns to an existing table, so migrate:
-    # impact columns arrived after the table did (see co-op/support#26).
+    # impact columns arrived after the table did (see co-op/support#26), and the
-    for _col, _decl in (("out_tokens", "INTEGER"), ("energy_wh", "REAL"), ("co2e_g", "REAL")):
+    # measured-vs-estimated provenance columns after those. All additive: existing
+    # rows keep NULL, and a NULL split reports as unknown — never as "measured".
+    for _col, _decl in (
+        ("out_tokens", "INTEGER"),
+        ("energy_wh", "REAL"),
+        ("co2e_g", "REAL"),
+        ("tokens_measured", "INTEGER"),
+        ("tokens_estimated", "INTEGER"),
+        ("factor_revision", "TEXT"),
+    ):
         try:
             conn.execute(f"ALTER TABLE metrics_daily ADD COLUMN {_col} {_decl}")
         except sqlite3.OperationalError:
@@ -3165,6 +3177,31 @@ async def admin_overview(request: Request):
     }
 
 
+def _metrics_provenance(tokens_measured: int | None, tokens_estimated: int | None) -> str:
+    """Classify a day's measured-vs-estimated split for the API.
+
+    "measured"  — every token rested on a directly measured factor.
+    "estimated" — every token rested on a non-measured factor (borrowed
+                  SAME_FAMILY or the flat proxy).
+    "mixed"     — the day's total is a blend of both.
+    "none"      — no provenance recorded: either the day has no output tokens
+                  or the row predates these columns (both split columns NULL).
+                  Deliberately NOT "measured" — an unknown split must never read
+                  as a confident measurement.
+    """
+    if tokens_measured is None and tokens_estimated is None:
+        return "none"
+    m = tokens_measured or 0
+    e = tokens_estimated or 0
+    if m and e:
+        return "mixed"
+    if m:
+        return "measured"
+    if e:
+        return "estimated"
+    return "none"
+
+
 def metrics_series(days: int = 90) -> list[dict]:
     """Daily metrics rows, oldest first.
 
@@ -3176,13 +3213,21 @@ def metrics_series(days: int = 90) -> list[dict]:
     energy_wh and co2e_g are computed in metrics_snapshot.py from the day's
     output tokens against a factor table: measured for GreenPT models (their API
     returns an `impact` object), estimated for the rest. See co-op/support#26.
+
+    Each row also carries its provenance: `tokens_measured`/`tokens_estimated`
+    (the output-token split across measured vs non-measured factors),
+    `factor_revision` (a short stable hash of the factor table that produced the
+    row), and a derived `provenance` indicator — measured | mixed | estimated |
+    none. Rows written before these columns exist have NULL splits and report
+    `provenance: "none"`, never a confident "measured". See docs/energy-metrics.md.
     """
     limit = max(1, min(int(days or 90), 730))
     conn = get_db()
     try:
         rows = conn.execute(
             "SELECT day, members_active, members_total, members_new, newsletter_subs, "
-            "matrix_space_members, spend_usd, requests, out_tokens, energy_wh, co2e_g, recorded_at "
+            "matrix_space_members, spend_usd, requests, out_tokens, energy_wh, co2e_g, "
+            "tokens_measured, tokens_estimated, factor_revision, recorded_at "
             "FROM metrics_daily ORDER BY day DESC LIMIT ?",
             (limit,),
         ).fetchall()
@@ -3200,9 +3245,19 @@ def metrics_series(days: int = 90) -> list[dict]:
         "out_tokens",
         "energy_wh",
         "co2e_g",
+        "tokens_measured",
+        "tokens_estimated",
+        "factor_revision",
         "recorded_at",
     ]
-    return [dict(zip(cols, row)) for row in reversed(rows)]
+    series = []
+    for row in reversed(rows):
+        rec = dict(zip(cols, row))
+        rec["provenance"] = _metrics_provenance(
+            rec["tokens_measured"], rec["tokens_estimated"]
+        )
+        series.append(rec)
+    return series
 
 
 @app.get("/admin/metrics/series")
@@ -3219,6 +3274,11 @@ async def admin_metrics_series(request: Request, days: int = 90):
     counts as of that day; spend_usd/requests are that day's traffic only.
     Backfilled days carry a LOWER BOUND for members_total (members provisioned
     without a welcome email are absent from the history).
+
+    Each row carries its energy provenance — `provenance` (measured | mixed |
+    estimated | none) plus `tokens_measured`, `tokens_estimated` and
+    `factor_revision`. A row written before those columns existed reports
+    `provenance: "none"`, not "measured". See docs/energy-metrics.md.
     """
     _verify_admin_token(request)
     series = metrics_series(days)
@@ -3607,6 +3667,12 @@ async def broker_metrics(request: Request, days: int = 90):
     it is AGGREGATE and non-personal — co-op size, requests served, spend — the
     same class of social-proof data as /broker/model-leaderboard. No per-member
     figure passes through here, and members asked to see how the co-op is doing.
+
+    Rows carry the energy provenance the admin series does: `provenance`
+    (measured | mixed | estimated | none) plus `tokens_measured`,
+    `tokens_estimated` and `factor_revision` — so a member-facing chart can say
+    how much of a day's figure rested on a measurement. NULL splits (older rows)
+    report `provenance: "none"`. See docs/energy-metrics.md.
     """
     _verify_broker_secret(request)
     series = metrics_series(days)

docs/energy-metrics.md

@@ -49,9 +49,55 @@ Factors live in `metrics_snapshot.py` (`IMPACT_FACTORS`, `IMPACT_PROXY`). Matchi
   not disclose impact, **deliberately**, so that a day's total is never quietly
   missing models. It is not a number we know.
 
-**No tier is exposed anywhere.** The endpoints (`/admin/metrics/series`,
+**No tier is exposed on its own**, but every row now carries a *provenance* read
-`/broker/metrics`) return bare numbers, so a consumer cannot tell measured from
+derived from the measured/estimated token split — see "Provenance in the API"
-estimated. Exposing provenance is tracked as a separate issue.
+below. A consumer can tell how much of a day's figure rested on a measurement.
+
+## Provenance in the API
+
+Every row returned by `/admin/metrics/series` and `/broker/metrics` carries the
+measured-vs-estimated split, so a consumer can tell how much of a day's figure
+rested on a real measurement. Four fields:
+
+| field | meaning |
+|---|---|
+| `tokens_measured` | output tokens that day attributed to a factor whose `source` is **MEASURED** — a direct provider measurement. |
+| `tokens_estimated` | the rest: **SAME_FAMILY** (borrowed from a sibling) and the flat **proxy**. A borrowed or proxied factor is an assumption, so it is **not** counted as measured. |
+| `factor_revision` | a short, stable hash of the factor table (`IMPACT_FACTORS` + `IMPACT_PROXY`) in effect when the row was computed. |
+| `provenance` | the derived read, below. |
+
+`tokens_measured + tokens_estimated = out_tokens` for any day with tokens. A day
+with no output tokens has NULL in both (fail-closed, same rule as energy/carbon).
+
+`provenance` is one of:
+
+- **`measured`** — every token rested on a measured factor.
+- **`estimated`** — every token rested on a non-measured factor (borrowed or proxy).
+- **`mixed`** — both are present; the day's total is a blend.
+- **`none`** — no provenance. This is **either** a day with no output tokens **or**
+  a row written before these columns existed (older rows keep NULL). It is
+  deliberately **not** `measured`: an unknown split must never read as a confident
+  measurement. This is the whole point of the indicator — failure reads as unknown.
+
+`factor_revision` is a deterministic hash, **not a timestamp**: two runs against the
+same table produce the same string. It changes if any factor value or source label
+changes, or if the matching order changes (order is meaningful — longest prefix
+first). It is "what the numbers were computed with", not "what was known then".
+
+### Forward-only, then backfill
+
+The columns are added by an **additive** migration — a guarded
+`ALTER TABLE metrics_daily ADD COLUMN …`, the same idiom used when the impact
+columns first arrived. On an existing database this is safe: it never errors,
+existing rows survive untouched, and they simply keep NULL in the new columns.
+Nothing is guessed for history on migration; those rows read as `provenance: "none"`.
+
+To give past rows a real split, run `metrics_snapshot.py --backfill` (or re-run the
+affected days). Backfill rewrites each day from the current LiteLLM token split and
+the **current** factor table, so it also updates `factor_revision` to today's
+revision — a backfilled historical row records the revision used to recompute it,
+not the one that was current at the time. Honest for a chart label; not a historical
+audit trail.
 
 ## Formulas
 
@@ -67,8 +113,8 @@ intended — which is why the tripwire below exists.
 
 ## NULL, never zero
 
-`impact_for()` returns `None, None` for a day with no measured tokens, and the row is
+`impact_for()` returns an all-NULL impact block for a day with no measured tokens,
-written with NULL energy and carbon. It never writes `0.0`:
+and the row is written with NULL energy and carbon. It never writes `0.0`:
 
 > a zero reads as "the co-op used no energy", which is a claim we cannot support, and
 > it would survive into a chart as a fact.
@@ -107,7 +153,8 @@ than by us.
 - It writes **one row per day** into the `metrics_daily` table, in the same SQLite
   database as the members table (`members.db`). Columns: `day`, `members_active`,
   `members_total`, `members_new`, `newsletter_subs`, `matrix_space_members`,
-  `spend_usd`, `requests`, `out_tokens`, `energy_wh`, `co2e_g`, `recorded_at`.
+  `spend_usd`, `requests`, `out_tokens`, `energy_wh`, `co2e_g`, `tokens_measured`,
+  `tokens_estimated`, `factor_revision`, `recorded_at`.
 - **`metrics_snapshot.py --backfill` rebuilds history** from the sources and is
   idempotent — it rewrites a day rather than duplicating it. It is the tool to run
   after changing a factor, and the reason a published chart can move: history is
@@ -122,9 +169,11 @@ Stated plainly, because a mixed total is easy to over-read:
 - **Estimated:** everything else. Currently that means **all Tinfoil and Regolo
   traffic**, which is a real share of co-op usage.
 
-So a day's figure is a blend, and **the split is not recorded per day** — you cannot
+So a day's figure is a blend, and **the split is now recorded per day** —
-tell from `metrics_daily` how much of a total rested on the proxy. Persisting that
+`tokens_measured` and `tokens_estimated` say how much of the total rested on a
-split is a separate follow-up.
+measurement, and the API derives a `provenance` read from them. Before those
+columns existed the split was not recorded; such rows carry NULL and read as
+`provenance: "none"`. See "Provenance in the API".
 
 One consequence worth knowing: the proxy assumes a 250 gCO2e/kWh grid, while Regolo
 runs on 100% renewable energy, so the proxy **overstates** Regolo's carbon. Per-provider

metrics_snapshot.py

@@ -27,6 +27,7 @@ Rules this file follows deliberately:
 """
 
 import datetime as dt
+import hashlib
 import json
 import os
 import sqlite3
@@ -60,10 +61,23 @@ def connect() -> sqlite3.Connection:
                out_tokens INTEGER,
                energy_wh REAL,
                co2e_g REAL,
+               tokens_measured INTEGER,
+               tokens_estimated INTEGER,
+               factor_revision TEXT,
                recorded_at TEXT)"""
     )
     # CREATE IF NOT EXISTS never adds columns to an existing table, so migrate.
-    for col, decl in (("out_tokens", "INTEGER"), ("energy_wh", "REAL"), ("co2e_g", "REAL")):
+    for col, decl in (
+        ("out_tokens", "INTEGER"),
+        ("energy_wh", "REAL"),
+        ("co2e_g", "REAL"),
+        # Measured-vs-estimated provenance (see co-op/support#26 follow-ups).
+        # Additive: existing rows keep NULL, and NULL reports as "unknown", never
+        # as a confident "measured".
+        ("tokens_measured", "INTEGER"),
+        ("tokens_estimated", "INTEGER"),
+        ("factor_revision", "TEXT"),
+    ):
         try:
             conn.execute(f"ALTER TABLE metrics_daily ADD COLUMN {col} {decl}")
         except sqlite3.OperationalError:
@@ -105,6 +119,30 @@ IMPACT_FACTORS = {
 }
 IMPACT_PROXY = {"wh_per_1k_out": 0.5000, "gco2e_per_kwh": 250.0, "source": ESTIMATED}
 
+
+def _factor_revision() -> str:
+    """Short, stable fingerprint of the factor table in force.
+
+    Serialised WITHOUT sort_keys, preserving both the per-factor key order and the
+    table's longest-prefix-first ordering — because reordering IMPACT_FACTORS
+    changes which factor a model matches, so the revision must move with it. A
+    deterministic hash of values rather than a timestamp: two runs against the
+    same table produce the same string, and a chart label can trust it.
+    """
+    payload = json.dumps(
+        {
+            "factors": [[prefix, IMPACT_FACTORS[prefix]] for prefix in IMPACT_FACTORS],
+            "proxy": IMPACT_PROXY,
+        },
+        separators=(",", ":"),
+    )
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:8]
+
+
+# Revision of the CURRENT factor table. Written onto every row computed from it, so
+# a chart can tell which factor set produced a number. See docs/energy-metrics.md.
+FACTOR_REVISION = _factor_revision()
+
 # ---------------------------------------------------------------- sources
 
 def listmonk_joins() -> list[str] | None:
@@ -325,8 +363,38 @@ def tokens_by_day_model(since: dt.date | None = None) -> dict[str, dict[str, int
     return out
 
 
-def impact_for(models: dict[str, int]) -> tuple[float | None, float | None]:
+def _no_impact() -> dict:
-    """(Wh, gCO2e) for a day's {model: output_tokens}.
+    """The impact block for a day with no output tokens: all NULL, fail-closed.
+
+    NULL energy/carbon means "not measured" (see the module docstring). The
+    provenance split is NULL too, so an unknown split cannot be read as a
+    confident "0 measured".
+    """
+    return {
+        "wh": None,
+        "co2e_g": None,
+        "tokens_measured": None,
+        "tokens_estimated": None,
+        "factor_revision": None,
+    }
+
+
+def impact_for(models: dict[str, int]) -> dict:
+    """Impact for a day's {model: output_tokens}, and the provenance split.
+
+    Returns a dict:
+      wh, co2e_g        — totals, unchanged from before (same factors, same
+                          matching, same arithmetic, same rounding).
+      tokens_measured   — output tokens attributed to a factor whose `source` is
+                          MEASURED (a direct provider measurement).
+      tokens_estimated  — the rest: SAME_FAMILY (borrowed from a sibling) and the
+                          flat proxy. A borrowed or proxied factor is an
+                          assumption, so it is NOT counted as measured.
+      factor_revision   — the revision of the factor table used.
+
+    The split is reported, not recomputed: the numeric behaviour is identical to
+    the previous (Wh, gCO2e)-only version. `tokens_measured + tokens_estimated`
+    equals the day's output tokens for any day with tokens.
 
     Factors live in IMPACT_FACTORS. Unknown models fall back to the documented
     proxy row rather than being skipped, so a total is never quietly missing a
@@ -334,15 +402,17 @@ def impact_for(models: dict[str, int]) -> tuple[float | None, float | None]:
     docs say so. Energy: Wh per 1,000 output tokens. Carbon: gCO2e per kWh.
     """
     if not models:
-        return None, None
+        return _no_impact()
     if not sum(t for t in models.values() if t):
         # No measured tokens for this day: report "not measured" (NULL), never a
         # zero. A zero reads as "the co-op used no energy" — a lie that would
         # survive into a chart. The first backfill shipped exactly that, because
         # it summed the wrong column and every model totalled 0.
-        return None, None
+        return _no_impact()
     wh = 0.0
     co2 = 0.0
+    tokens_measured = 0
+    tokens_estimated = 0
     for model, toks in models.items():
         if not toks:
             continue
@@ -352,6 +422,12 @@ def impact_for(models: dict[str, int]) -> tuple[float | None, float | None]:
                 f = factor
                 break
         f = f or IMPACT_PROXY
+        # Provenance is a pure reporting side-channel: only MEASURED is a direct
+        # measurement; SAME_FAMILY (borrowed) and the proxy are not.
+        if f.get("source") == MEASURED:
+            tokens_measured += toks
+        else:
+            tokens_estimated += toks
         wh += toks / 1000.0 * f["wh_per_1k_out"]
         # Wh -> kWh is /1e3, then x gCO2e/kWh. This divided by 1e6 once, which
         # understated carbon exactly 1000x: the energy column stayed plausible
@@ -366,7 +442,13 @@ def impact_for(models: dict[str, int]) -> tuple[float | None, float | None]:
                 f"metrics: WARNING implied grid intensity {implied:.1f} gCO2e/kWh is outside "
                 f"the plausible band — suspect a unit error before trusting these numbers"
             )
-    return round(wh, 6), round(co2, 6)
+    return {
+        "wh": round(wh, 6),
+        "co2e_g": round(co2, 6),
+        "tokens_measured": tokens_measured,
+        "tokens_estimated": tokens_estimated,
+        "factor_revision": FACTOR_REVISION,
+    }
 
 
 # ---------------------------------------------------------------- writes
@@ -378,8 +460,8 @@ def upsert(day: str, row: dict) -> None:
             """INSERT INTO metrics_daily
                    (day, members_active, members_total, members_new, newsletter_subs,
                     matrix_space_members, spend_usd, requests, out_tokens, energy_wh,
-                    co2e_g, recorded_at)
+                    co2e_g, tokens_measured, tokens_estimated, factor_revision, recorded_at)
-               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                ON CONFLICT(day) DO UPDATE SET
                    members_active=excluded.members_active,
                    members_total=excluded.members_total,
@@ -391,6 +473,9 @@ def upsert(day: str, row: dict) -> None:
                    out_tokens=excluded.out_tokens,
                    energy_wh=excluded.energy_wh,
                    co2e_g=excluded.co2e_g,
+                   tokens_measured=excluded.tokens_measured,
+                   tokens_estimated=excluded.tokens_estimated,
+                   factor_revision=excluded.factor_revision,
                    recorded_at=excluded.recorded_at""",
             (
                 day,
@@ -404,6 +489,9 @@ def upsert(day: str, row: dict) -> None:
                 row.get("out_tokens"),
                 row.get("energy_wh"),
                 row.get("co2e_g"),
+                row.get("tokens_measured"),
+                row.get("tokens_estimated"),
+                row.get("factor_revision"),
                 dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
             ),
         )
@@ -424,7 +512,7 @@ def snapshot(day: str | None = None) -> dict:
     day_models = (tokens_by_day_model(since=since).get(day)) or {}
     tot = sum(day_models.values())
     out_tokens = tot or None
-    energy_wh, co2e_g = impact_for(day_models) if tot else (None, None)
+    impact = impact_for(day_models)
     row = {
         "members_active": m["active"],
         "members_total": m["total"],
@@ -434,8 +522,11 @@ def snapshot(day: str | None = None) -> dict:
         "spend_usd": today_spend[0],
         "requests": today_spend[1],
         "out_tokens": out_tokens,
-        "energy_wh": energy_wh,
+        "energy_wh": impact["wh"],
-        "co2e_g": co2e_g,
+        "co2e_g": impact["co2e_g"],
+        "tokens_measured": impact["tokens_measured"],
+        "tokens_estimated": impact["tokens_estimated"],
+        "factor_revision": impact["factor_revision"],
     }
     upsert(day, row)
     print(f"metrics: recorded {day} -> {json.dumps(row)}")
@@ -485,7 +576,12 @@ def backfill() -> None:
         day_models = tokens.get(key) or {}
         if sum(day_models.values()):
             row["out_tokens"] = sum(day_models.values())
-            row["energy_wh"], row["co2e_g"] = impact_for(day_models)
+            impact = impact_for(day_models)
+            row["energy_wh"] = impact["wh"]
+            row["co2e_g"] = impact["co2e_g"]
+            row["tokens_measured"] = impact["tokens_measured"]
+            row["tokens_estimated"] = impact["tokens_estimated"]
+            row["factor_revision"] = impact["factor_revision"]
         if key == today():
             # today's row carries the exact live metrics
             row.update(