diff --git a/README.md b/README.md index 07cf28e..686ef4b 100644 --- a/README.md +++ b/README.md @@ -125,13 +125,17 @@ Three jobs run inside the container via `CloudronManifest.json` → `addons.sche State persists in `/app/data/rag_refresh_state.json`. - **metrics_snapshot** (daily 3:15am UTC) — `metrics_snapshot.py` records one row per day into the `metrics_daily` table: members, newsletter subscribers, Matrix - membership, spend, request counts, and energy/CO₂. A source that cannot be read + membership, spend, request counts, energy/CO₂, and the Open Collective + balance. A source that cannot be read yields NULL for that metric rather than a zero. `spend_usd` is the co-op-wide total and deliberately spans every team, the co-op's own operator accounts included; `operator_spend_usd` is the breakout of that portion, served through `/broker/metrics` so the member dashboard can show the co-op's own share - alongside the total. `--backfill` rebuilds history idempotently. Methodology: - [Energy and carbon metrics](docs/energy-metrics.md). + alongside the total. `oc_balance_usd` is a daily snapshot of the collective's + OC balance (the OC API has no historical series, so balance history accrues + from these snapshots only — 2026-11-08 onward). `--backfill` rebuilds history + idempotently (balance history cannot be backfilled: past balances were never + recorded). Methodology: [Energy and carbon metrics](docs/energy-metrics.md). Verify after any manifest change: the deploy log must show `setupAddons: ... scheduler ...` listing all three tasks, and each script must diff --git a/app/main.py b/app/main.py index a60827f..10470e6 100644 --- a/app/main.py +++ b/app/main.py @@ -812,6 +812,10 @@ def get_db() -> sqlite3.Connection: ("tokens_measured", "INTEGER"), ("tokens_estimated", "INTEGER"), ("factor_revision", "TEXT"), + # The collective's OC balance, snapshotted daily (see metrics_snapshot.py + # oc_collective_balance). The OC API exposes only the CURRENT balance, so + # a real history accrues from these snapshots — earlier days stay NULL. + ("oc_balance_usd", "REAL"), ): try: conn.execute(f"ALTER TABLE metrics_daily ADD COLUMN {_col} {_decl}") @@ -3768,6 +3772,350 @@ def metrics_series(days: int = 90) -> list[dict]: return series +# --------------------------------------------------------------------------- +# Money series — the admin panel's Money chart: cumulative Open Collective +# revenue, token spend over time, and the OC balance. +# +# The three series have genuinely different provenance, and the endpoint says +# so rather than papering over it: +# revenue — OC contribution transactions (CREDIT / CONTRIBUTION / not a +# refund), each with a createdAt and an amount, summed +# chronologically. Monthly and one-time contributions both count; +# org-backed contributions count through the org account that +# paid. Exact from each contribution's start date (2026-09-06). +# spend — LiteLLM SpendLogs, summed per UTC day in ONE GROUP BY pass over +# the full history (134k+ rows), with no team filter — members, +# operators and agent teams all count, matching spend_by_day() in +# metrics_snapshot.py. +# balance — the OC API exposes only the collective's CURRENT balance; there +# is NO historical series. The current balance is returned live +# and a real history accrues from the daily metrics_snapshot +# snapshots into metrics_daily.oc_balance_usd. Days before that +# pipeline stay NULL — a gap in the chart, never a zero. +# +# Aggregate totals only: no member emails or names are selected or returned. +# +# Caching: the OC queries are slow (network + pagination), so the payload is +# computed at most once per MONEY_SERIES_TTL seconds (default 1h) and served +# from an in-memory cache between refreshes; a failed refresh serves the +# previous payload marked `stale: true` rather than breaking the chart. The +# daily snapshot owns history accrual, so no schedule entry is needed here. +# --------------------------------------------------------------------------- + +MONEY_SERIES_TTL = float(os.environ.get("MONEY_SERIES_TTL", "3600")) +_MONEY_CACHE: dict = {"data": None, "ts": 0.0} +_money_lock = asyncio.Lock() + +# Hard stop for the transactions pagination: 500 rows/page x 40 pages. The +# collective holds ~160 contribution transactions; the cap only exists so a +# surprising totalCount can never turn into an unbounded loop. +_MONEY_TX_PAGE = 500 +_MONEY_TX_MAX_PAGES = 40 + + +async def _oc_graphql(query: str, variables: dict) -> dict: + """One authenticated OC GraphQL call (browser UA — Cloudflare blocks the + default Python UA with a 403/1010 bot signature). Raises on HTTP or + GraphQL errors so callers can fail per-source.""" + async with httpx.AsyncClient(timeout=30.0) as client: + r = await client.post( + OC_GRAPHQL_URL, + headers={"Content-Type": "application/json", + "Authorization": f"Bearer {OC_PERSONAL_TOKEN}", + "Api-Version": "2022-01-01", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"}, + json={"query": query, "variables": variables}, + ) + if r.status_code != 200: + raise RuntimeError(f"OC GraphQL failed: HTTP {r.status_code}") + body = r.json() + if body.get("errors"): + raise RuntimeError(f"OC GraphQL errors: {str(body['errors'])[:200]}") + return body.get("data") or {} + + +async def oc_collective_balance() -> dict: + """The collective's CURRENT balance (there is no historical endpoint).""" + if not OC_PERSONAL_TOKEN: + raise RuntimeError("OC_PERSONAL_TOKEN not set") + data = await _oc_graphql( + "query Bal($slug: String) { collective(slug: $slug) " + "{ stats { balance { valueInCents currency } } } }", + {"slug": OC_COLLECTIVE_SLUG}, + ) + bal = ( + ((data.get("collective") or {}).get("stats") or {}).get("balance") or {} + ) + cents = bal.get("valueInCents") + if cents is None: + raise RuntimeError("OC balance field missing from response") + return { + "value": round(int(cents) / 100.0, 2), + "currency": bal.get("currency") or "USD", + "as_of": datetime.now(timezone.utc).isoformat(timespec="seconds"), + } + + +async def oc_contribution_by_day() -> tuple[dict[str, float], int, float]: + """{UTC day: contribution $} over every CREDIT CONTRIBUTION transaction. + + Paginated (variables, never interpolation — the same injection rule as + get_order_amount). Returns (by_day, count, total). Payer identities are + deliberately NOT selected: this feed is aggregate-only. + """ + if not OC_PERSONAL_TOKEN: + raise RuntimeError("OC_PERSONAL_TOKEN not set") + by_day: dict[str, float] = {} + count = 0 + total_cents = 0 + offset = 0 + for _page in range(_MONEY_TX_MAX_PAGES): + data = await _oc_graphql( + "query Contribs($slug: String, $limit: Int!, $offset: Int!) " + "{ collective(slug: $slug) { transactions(type: CREDIT, limit: $limit, " + "offset: $offset) { totalCount nodes { createdAt type kind isRefund " + "amount { valueInCents } } } } }", + {"slug": OC_COLLECTIVE_SLUG, "limit": _MONEY_TX_PAGE, "offset": offset}, + ) + coll = ((data.get("collective") or {}).get("transactions") or {}) + total_count = int(coll.get("totalCount") or 0) + nodes = coll.get("nodes") or [] + if not nodes: + break + for t in nodes: + if (t.get("type") or "").upper() != "CREDIT": + continue + if (t.get("kind") or "").upper() != "CONTRIBUTION": + continue + if t.get("isRefund"): + continue + dtv = _parse_oc_dt(t.get("createdAt")) + cents = ((t.get("amount") or {}).get("valueInCents")) or 0 + if dtv is None: + continue + day = dtv.astimezone(timezone.utc).strftime("%Y-%m-%d") + by_day[day] = by_day.get(day, 0.0) + int(cents) / 100.0 + count += 1 + total_cents += int(cents) + offset += len(nodes) + if offset >= total_count or len(nodes) < _MONEY_TX_PAGE: + break + return by_day, count, round(total_cents / 100.0, 2) + + +def litellm_spend_by_day() -> tuple[dict[str, float], float, int]: + """{UTC day: $}, total, and row count from LiteLLM's SpendLogs — one + GROUP BY pass over the whole table. No team filter: the co-op's total + spans members, operators and agent teams (same definition as + spend_by_day() in metrics_snapshot.py, so the two agree by construction).""" + if not litellm_db_ready(): + raise RuntimeError("LiteLLM DB credentials not configured") + conn = _pg_connect() + try: + with conn.cursor() as cur: + cur.execute( + """SELECT to_char("startTime", 'YYYY-MM-DD') AS day, + COALESCE(SUM("spend"), 0) AS spend, COUNT(*) AS n + FROM "LiteLLM_SpendLogs" + GROUP BY 1 ORDER BY 1""" + ) + rows = cur.fetchall() + finally: + conn.close() + by_day = {r[0]: round(float(r[1]), 6) for r in rows} + return by_day, round(sum(by_day.values()), 6), int(sum(int(r[2]) for r in rows)) + + +def oc_balance_history() -> dict[str, float]: + """metrics_daily.oc_balance_usd rows — the daily snapshots. Empty before + the pipeline deployed, which is exactly the honest-history caveat.""" + conn = get_db() + try: + rows = conn.execute( + "SELECT day, oc_balance_usd FROM metrics_daily " + "WHERE oc_balance_usd IS NOT NULL ORDER BY day" + ).fetchall() + finally: + conn.close() + return {r[0]: float(r[1]) for r in rows} + + +def _seed_balance_row(day: str, value: float) -> None: + """Fold today's live OC balance into metrics_daily WITHOUT creating a row + and WITHOUT touching any other column. The daily snapshot owns the rest of + that row (a partial INSERT here would punch NULL gaps into the members + chart for the day). When the snapshot has not written today's row yet + there is nothing to seed and this is a no-op; its own 03:15 read then + writes the balance itself.""" + if value is None: + return + conn = get_db() + try: + conn.execute( + "UPDATE metrics_daily SET oc_balance_usd = ? WHERE day = ?", + (float(value), day), + ) + conn.commit() + finally: + conn.close() + + +def _build_money_series(rev_by_day: dict[str, float] | None, + spend_by_day: dict[str, float] | None, + bal_hist: dict[str, float], + today_key: str) -> list[dict]: + """One row per UTC day across the full span, oldest first. + + revenue_cum / spend_cum are running sums; days before a series' first + entry carry 0 — a genuine zero (the sources were readable and nothing had + arrived yet), unlike oc_balance, which is NULL before the first daily + snapshot and must render as a gap. + """ + day_keys = set(rev_by_day or {}) | set(spend_by_day or {}) | set(bal_hist or {}) + day_keys.add(today_key) + cur = datetime.strptime(min(day_keys), "%Y-%m-%d").date() + end = datetime.strptime(today_key, "%Y-%m-%d").date() + rev_cum = 0.0 + spend_cum = 0.0 + out: list[dict] = [] + while cur <= end: + key = cur.strftime("%Y-%m-%d") + rev_today = (rev_by_day or {}).get(key, 0.0) + spend_today = (spend_by_day or {}).get(key, 0.0) + rev_cum += rev_today + spend_cum += spend_today + out.append({ + "day": key, + "revenue_cum": round(rev_cum, 2) if rev_by_day is not None else None, + "spend_day": round(spend_today, 6) if spend_by_day is not None else None, + "spend_cum": round(spend_cum, 2) if spend_by_day is not None else None, + "oc_balance": bal_hist.get(key), + }) + cur += timedelta(days=1) + return out + + +async def _compute_money_payload() -> dict: + """Fetch every source independently; a failed source yields NULL columns + plus an `errors` entry rather than taking the whole payload down. Only an + all-sources failure raises (the caller then serves stale or 503s).""" + errors: list[str] = [] + today_key = datetime.now(timezone.utc).strftime("%Y-%m-%d") + + rev_by_day = None + contrib_count = None + revenue_total = None + try: + rev_by_day, contrib_count, revenue_total = await oc_contribution_by_day() + except Exception as e: + errors.append(f"revenue: {e}") + + spend_by_day = None + spend_total = None + spend_rows = None + try: + spend_by_day, spend_total, spend_rows = litellm_spend_by_day() + except Exception as e: + errors.append(f"spend: {e}") + + balance_current = None + try: + balance_current = await oc_collective_balance() + except Exception as e: + errors.append(f"balance: {e}") + + if rev_by_day is None and spend_by_day is None and balance_current is None: + raise RuntimeError("all money sources failed: " + "; ".join(errors)) + + bal_hist = oc_balance_history() + # Today's balance is the LIVE value just fetched — fresher than the 03:15 + # snapshot — so it overrides (or extends) the history at today's key, and + # is folded into the stored row when one already exists. + if balance_current is not None: + bal_hist = {**bal_hist, today_key: float(balance_current["value"])} + try: + _seed_balance_row(today_key, balance_current["value"]) + except Exception as e: + errors.append(f"balance seed: {e}") + + series = _build_money_series(rev_by_day, spend_by_day, bal_hist, today_key) + balance_history_start = min(bal_hist) if bal_hist else None + return { + "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "currency": "USD", + "revenue_start": min(rev_by_day) if rev_by_day else None, + "spend_start": min(spend_by_day) if spend_by_day else None, + "balance_history_start": balance_history_start, + "balance_current": balance_current, + "totals": { + "revenue_all_time": revenue_total, + "contributions": contrib_count, + "spend_all_time": spend_total, + "spend_rows": spend_rows, + }, + "sources": { + "revenue": "Open Collective contribution transactions (CREDIT, " + "CONTRIBUTION, non-refund) — monthly and one-time, " + "org-backed included via the org account", + "spend": "LiteLLM SpendLogs daily sums (UTC), all teams — members, " + "operators and agent teams included", + "balance": "Open Collective current balance + daily snapshots in " + "metrics_daily.oc_balance_usd", + }, + "caveat": "Balance history begins " + f"{balance_history_start or 'at deployment'}: the Open " + "Collective API exposes only the current balance, so earlier " + "history does not exist and accrues from daily snapshots.", + "errors": errors, + "series": series, + } + + +async def _get_money_payload() -> dict: + """Hourly-cached access to the money payload; serves the previous payload + (marked stale) when a refresh fails.""" + now = datetime.now(timezone.utc).timestamp() + async with _money_lock: + cached = _MONEY_CACHE.get("data") + if cached and now - _MONEY_CACHE.get("ts", 0.0) < MONEY_SERIES_TTL: + out = dict(cached) + out["cached"] = True + out["cache_ttl_seconds"] = MONEY_SERIES_TTL + return out + try: + data = await _compute_money_payload() + except Exception as e: + if cached: + out = dict(cached) + out["cached"] = True + out["stale"] = True + out["errors"] = list(out.get("errors") or []) + [f"refresh failed, serving cached: {e}"] + return out + raise + _MONEY_CACHE["data"] = data + _MONEY_CACHE["ts"] = now + out = dict(data) + out["cached"] = False + out["cache_ttl_seconds"] = MONEY_SERIES_TTL + return out + + +@app.get("/admin/money-series") +@limiter.limit("30/minute") +async def admin_money_series(request: Request): + """Revenue / token-spend / OC-balance series for the admin panel's Money + chart, hourly-cached (the OC queries are slow; never called per refresh). + + Authenticated by the X-Admin-Token header, like the other /admin/* routes. + The full history is always returned (cumulative series need it); the panel + slices client-side so its period toggle never refetches. Aggregate totals + only — no member emails or names in this payload. See the block comment + above for each series' provenance and honest limits. + """ + _verify_admin_token(request) + return await _get_money_payload() + + @app.get("/admin/metrics/series") @limiter.limit("30/minute") async def admin_metrics_series(request: Request, days: int = 90): diff --git a/metrics_snapshot.py b/metrics_snapshot.py index 3a65498..f0030be 100644 --- a/metrics_snapshot.py +++ b/metrics_snapshot.py @@ -20,7 +20,7 @@ Rules this file follows deliberately: - **Runs in Cloudron's scheduler container**, which gets the platform env (declared app env + addon vars) but NOT variables the app's own start.sh sourced. Everything needed here is a declared app env var: LITELLM_DB_*, - LISTMONK_*, MATRIX_BOT_TOKEN, LISTMONK_LIST_ID. + LISTMONK_*, MATRIX_BOT_TOKEN, LISTMONK_LIST_ID, OC_PERSONAL_TOKEN. - `members_new` uses `welcome_sent_at`, the provisioning timestamp — the closest thing the portal stores to a join date. Manual members have no welcome email, so their joins are absent from the growth curve (noted, not guessed). @@ -84,6 +84,11 @@ def connect() -> sqlite3.Connection: ("tokens_measured", "INTEGER"), ("tokens_estimated", "INTEGER"), ("factor_revision", "TEXT"), + # The collective's Open Collective balance, read once a day. The OC API + # exposes only the CURRENT balance — there is no historical series — so + # a real balance history can only accrue from these daily snapshots + # (from 2026-11-08 onward; earlier days stay NULL by design). + ("oc_balance_usd", "REAL"), ): try: conn.execute(f"ALTER TABLE metrics_daily ADD COLUMN {col} {decl}") @@ -513,6 +518,55 @@ def impact_for(models: dict[str, int]) -> dict: } +# ---------------------------------------------------------------- OC balance + +OC_GRAPHQL_URL = "https://opencollective.com/api/graphql/v2" + + +def oc_collective_balance() -> float | None: + """The collective's CURRENT Open Collective balance, in USD. + + OC exposes only the current balance — there is NO historical series — so a + real balance history can only accrue from these daily snapshots (from + 2026-11-08 onward; earlier days stay NULL, which charts must render as a + gap, never a zero). Fail-soft like every other source here: a failed read + prints and returns None. + """ + token = os.environ.get("OC_PERSONAL_TOKEN", "") + slug = os.environ.get("OC_COLLECTIVE_SLUG", "inference-cooperative") + if not token: + print("metrics: OC_PERSONAL_TOKEN not set — oc_balance_usd left NULL") + return None + query = ("query Bal($slug: String) { collective(slug: $slug) " + "{ stats { balance { valueInCents currency } } } }") + req = urllib.request.Request( + OC_GRAPHQL_URL, + data=json.dumps({"query": query, "variables": {"slug": slug}}).encode("utf-8"), + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {token}", + "Api-Version": "2022-01-01", + # The default Python UA is blocked by Cloudflare (error 1010), which + # reads like a scope failure — see references/opencollective-api.md. + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64)", + }, + ) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + data = json.loads(resp.read() or b"{}") + bal = ( + ((data.get("data") or {}).get("collective") or {}).get("stats") or {} + ).get("balance") or {} + cents = bal.get("valueInCents") + if cents is None: + print("metrics: OC balance field missing from response — oc_balance_usd left NULL") + return None + return round(int(cents) / 100.0, 2) + except Exception as exc: # noqa: BLE001 - report, never guess + print(f"metrics: OC balance read failed ({exc}) — oc_balance_usd left NULL") + return None + + # ---------------------------------------------------------------- writes def upsert(day: str, row: dict) -> None: @@ -523,8 +577,9 @@ def upsert(day: str, row: dict) -> None: (day, members_active, members_total, members_new, newsletter_subs, matrix_space_members, spend_usd, operator_spend_usd, requests, out_tokens, energy_wh, - co2e_g, tokens_measured, tokens_estimated, factor_revision, recorded_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + co2e_g, tokens_measured, tokens_estimated, factor_revision, + oc_balance_usd, recorded_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(day) DO UPDATE SET members_active=excluded.members_active, members_total=excluded.members_total, @@ -540,6 +595,7 @@ def upsert(day: str, row: dict) -> None: tokens_measured=excluded.tokens_measured, tokens_estimated=excluded.tokens_estimated, factor_revision=excluded.factor_revision, + oc_balance_usd=excluded.oc_balance_usd, recorded_at=excluded.recorded_at""", ( day, @@ -557,6 +613,7 @@ def upsert(day: str, row: dict) -> None: row.get("tokens_measured"), row.get("tokens_estimated"), row.get("factor_revision"), + row.get("oc_balance_usd"), dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"), ), ) @@ -587,6 +644,7 @@ def snapshot(day: str | None = None) -> dict: tot = sum(day_models.values()) out_tokens = tot or None impact = impact_for(day_models) + balance = oc_collective_balance() row = { "members_active": m["active"], "members_total": m["total"], @@ -602,6 +660,7 @@ def snapshot(day: str | None = None) -> dict: "tokens_measured": impact["tokens_measured"], "tokens_estimated": impact["tokens_estimated"], "factor_revision": impact["factor_revision"], + "oc_balance_usd": balance, } upsert(day, row) print(f"metrics: recorded {day} -> {json.dumps(row)}")