Files
member-portal/app/main.py
T

8045 lines
363 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
Member Portal — membership middleware for the Inference Cooperative.
Reconciles three systems:
1. Open Collective — who is a paying member (billing)
2. Cloudron — who can log in (identity/SSO)
3. LiteLLM — who can use the models, and how much (inference)
Three responsibilities:
A. Webhook handler — listen for Open Collective membership events
B. Key injector — read the member's email from a header, inject their
LiteLLM key, and forward the request to the gateway
C. Admin endpoints — health, status, manual reconciliation
"""
import os
import json
import logging
import re
import sqlite3
import secrets
import smtplib
from datetime import datetime, timedelta, timezone
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from email.utils import make_msgid
# --- Key encryption at rest (inlined from keycrypto.py; see repo history) ---
# Member chat keys are encrypted with AES-256-GCM using a key-encryption key
# (KEK) from the PORTAL_KEK env var. Plaintext exists only in process memory
# when a key is served. Format: "enc1:" + base64(nonce[12] || ct || tag).
# Values without the prefix are legacy plaintext (pre-migration) and pass
# through on decrypt. Fail-closed: nothing new is stored plaintext when
# PORTAL_KEK is unset, and encrypted values cannot be served without it.
import base64 as _b64
from cryptography.hazmat.primitives.ciphers.aead import AESGCM as _AESGCM
_KEYENC_PREFIX = "enc1:"
_KEYENC_AAD = b"member-chat-key-v1"
def _kek() -> bytes:
raw = os.environ.get("PORTAL_KEK", "")
if not raw:
raise RuntimeError(
"PORTAL_KEK is not set - refusing to encrypt/decrypt member keys"
)
key = _b64.b64decode(raw)
if len(key) != 32:
raise RuntimeError("PORTAL_KEK must decode to exactly 32 bytes (base64)")
return key
def encrypt_key(plain: str | None) -> str | None:
"""Encrypt a member key. Already-encrypted and empty values pass through."""
if not plain or plain.startswith(_KEYENC_PREFIX):
return plain
nonce = os.urandom(12)
ct = _AESGCM(_kek()).encrypt(nonce, plain.encode(), _KEYENC_AAD)
return _KEYENC_PREFIX + _b64.b64encode(nonce + ct).decode()
def decrypt_key(stored: str | None) -> str | None:
"""Decrypt a member key. Legacy plaintext rows pass through unchanged."""
if not stored:
return stored
if not stored.startswith(_KEYENC_PREFIX):
return stored # pre-migration plaintext
blob = _b64.b64decode(stored[len(_KEYENC_PREFIX):])
return _AESGCM(_kek()).decrypt(blob[:12], blob[12:], _KEYENC_AAD).decode()
def migrate_plaintext_keys(conn) -> int:
"""Encrypt plaintext key_token rows. Idempotent; returns rows changed."""
if not os.environ.get("PORTAL_KEK"):
logging.getLogger("uvicorn.error").critical(
"PORTAL_KEK unset - member keys remain PLAINTEXT at rest; "
"set PORTAL_KEK to a base64 32-byte key and restart"
)
return 0
rows = conn.execute(
"SELECT email, key_token FROM members "
"WHERE key_token != '' AND key_token IS NOT NULL "
"AND key_token NOT LIKE 'enc1:%'"
).fetchall()
changed = 0
for email, kt in rows:
enc = encrypt_key(kt)
if enc != kt:
conn.execute(
"UPDATE members SET key_token = ? WHERE email = ?", (enc, email)
)
changed += 1
return changed
# --- end key encryption ---
import httpx
import jwt
import asyncio
from fastapi import FastAPI, Request, Response, HTTPException
from fastapi.responses import JSONResponse, StreamingResponse
from slowapi import Limiter
from slowapi.util import get_remote_address
from slowapi.errors import RateLimitExceeded
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("member-portal")
# SECURITY (ops/security#2): OC's generic webhooks cannot send headers, so the shared
# secret must travel in the URL path — which means uvicorn writes it into the access
# log on every delivery, and the log is read by more things than the secret is. The
# token cannot be kept out of the URL; it can be kept out of the retention. Filters
# sit on the access logger and on the root handler so both uvicorn's line and any
# string the app itself logs get scrubbed.
_WEBHOOK_TOKEN_RE = re.compile(r"(/webhook/opencollective/)[^\s\"'?&]+")
# M1 (audit 2026-11-05): member emails persist in portal logs — send_email
# recipients, listmonk failures, Loomio invite lists, provisioning and credit
# lines (17 sites). Container logs are readable via `cloudron logs`, persist
# on the host and ride the backups, so an email is redacted from every record
# the same way the webhook token is. Logs stay useful: counts, slugs, MXIDs
# (matrix @user:server has no address shape) and amounts all survive; only
# the address is dropped. The daily digest's own sanitisation is unchanged.
_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+")
def _redact_emails(value):
if isinstance(value, str) and "@" in value:
return _EMAIL_RE.sub("<email>", value)
return value
def _redact_log_value(value):
if isinstance(value, str) and "/webhook/opencollective/" in value:
value = _WEBHOOK_TOKEN_RE.sub(r"\1<redacted>", value)
return _redact_emails(value)
def _redact_deep(value):
"""Redact every string in a log argument, preserving the argument's type."""
if isinstance(value, str):
return _redact_log_value(value)
if isinstance(value, tuple):
return tuple(_redact_deep(v) for v in value)
if isinstance(value, list):
return [_redact_deep(v) for v in value]
if isinstance(value, dict):
return {k: _redact_deep(v) for k, v in value.items()}
return value
class _RedactWebhookPath(logging.Filter):
"""Scrub secrets and member emails out of any log record.
Replaces the webhook token in a logged request path (ops/security#2) and
redacts email-shaped strings from messages, %-format args (recursing into
tuple/list/dict args) and any exception text already attached to the
record. A logging filter must never break logging.
"""
def filter(self, record: logging.LogRecord) -> bool:
try:
record.msg = _redact_log_value(record.msg)
if isinstance(record.args, tuple):
record.args = tuple(_redact_deep(a) for a in record.args)
elif isinstance(record.args, dict):
record.args = {k: _redact_deep(v) for k, v in record.args.items()}
if isinstance(record.exc_text, str):
record.exc_text = _redact_log_value(record.exc_text)
except Exception:
pass # a logging filter must never break logging
return True
logging.getLogger("uvicorn.access").addFilter(_RedactWebhookPath())
for _handler in logging.getLogger().handlers:
_handler.addFilter(_RedactWebhookPath())
app = FastAPI(title="Inference Cooperative Member Portal")
# --- Rate limiting (control plane only, not the member inference path) ---
# The data plane (chat completions via /v1/*) is capped by each member's
# LiteLLM budget, NOT by rate — a member streaming a long conversation must
# never be throttled. Rate limiting here covers the control-plane surfaces:
# admin endpoints, the broker, the webhook, and the unauthenticated model list.
limiter = Limiter(key_func=get_remote_address)
app.state.limiter = limiter
@app.exception_handler(RateLimitExceeded)
async def _rate_limit_handler(request: Request, exc: RateLimitExceeded):
return JSONResponse(
status_code=429,
content={"detail": "Too many requests. Slow down."},
)
# --- Configuration (from environment) ---
CLOUDRON_API = os.environ.get("CLOUDRON_API_ORIGIN", "https://my.inference.coop")
CLOUDRON_TOKEN = os.environ.get("CLOUDRON_TOKEN", "")
LITELLM_BASE = os.environ.get("LITELLM_BASE", "https://gateway.inference.coop")
LITELLM_MASTER_KEY = os.environ.get("LITELLM_MASTER_KEY", "")
# Shared secret that the chat frontend uses to authenticate requests to the
# portal. LibreChat sends it as an X-Portal-Secret header; OpenWebUI signs the
# user's email as a JWT (see FORWARD_USER_INFO_HEADER_JWT_SECRET below).
PORTAL_SECRET = os.environ.get("PORTAL_SECRET", "")
# OpenWebUI can sign the forwarded user identity as a JWT using this shared
# secret. When set, the portal verifies the JWT (HS256) to confirm the email
# genuinely came from OpenWebUI (behind Cloudron SSO) rather than a spoofed
# header. This is stronger than the plain X-Portal-Secret header because the
# email itself is tamper-proof.
OWUI_JWT_SECRET = os.environ.get("OWUI_JWT_SECRET", "")
# Open WebUI admin API key — used at provisioning time to add the new member to
# Open WebUI's `members` group, which gates read access to the shared
# "Inference Cooperative Docs" RAG collection. Without this, a new member's
# first chat session has no docs grounding (the nightly refresh would only fix
# it hours later). Machine-to-machine only; never logged.
OWUI_API_KEY = os.environ.get("OWUI_API_KEY", "")
OWUI_BASE = os.environ.get("OWUI_BASE", "http://chat:8080") # internal Cloudron network
OWUI_MEMBERS_GROUP_ID = os.environ.get("OWUI_MEMBERS_GROUP_ID", "eb348e1a-059a-4a8f-8920-0f73942f6272")
# Open WebUI knowledge collection id for the co-op docs RAG. Attached to every
# model in /v1/models so answers are grounded by automatic RAG injection (and
# the model's knowledge tools are scoped to the docs rather than browsing all
# knowledge bases). Env-driven; empty disables the attachment.
DOCS_KNOWLEDGE_ID = os.environ.get("DOCS_KNOWLEDGE_ID", "946e18ef-c1bc-4c57-b23e-7f5fc5b66e51")
# Shared secret that the member dashboard uses to authenticate broker requests.
# The dashboard is Cloudron-SSO-gated and passes the authenticated member's
# email; the portal trusts that email only because it carries this secret. Kept
# separate from PORTAL_SECRET so the two can be rotated independently.
BROKER_SECRET = os.environ.get("BROKER_SECRET", "")
# Secret token required in the Open Collective webhook URL path. Open
# Collective's generic webhooks are not HMAC-signed, so a secret in the URL
# is the standard way to authenticate them.
WEBHOOK_TOKEN = os.environ.get("WEBHOOK_TOKEN", "")
# SECURITY (2026-10-27 member disclosure): the /admin/* token is SEPARATE from
# WEBHOOK_TOKEN. WEBHOOK_TOKEN must sit in the Open Collective webhook URL (OC's
# generic webhooks cannot send headers), so anything that learns that URL — an OC
# admin, OC's logs, this app's own access logs, a Referer — used to gain portal
# admin. ADMIN_TOKEN is only ever sent as a header and never appears in a URL.
# Cutover COMPLETE (2026-10-28): ADMIN_TOKEN_LEGACY=0, so the legacy token no
# longer authenticates here (verified: ADMIN_TOKEN -> 200, WEBHOOK_TOKEN -> 403).
# ADMIN_TOKEN was minted new on 2026-10-27 and WEBHOOK_TOKEN was rotated the same
# day, so NO rotation is outstanding — do not "finish" this by rotating again.
# Callers must read ADMIN_TOKEN and fail loudly; never fall back to WEBHOOK_TOKEN,
# which 403s and would turn a missing variable into a mid-run failure.
ADMIN_TOKEN = os.environ.get("ADMIN_TOKEN", "")
ADMIN_TOKEN_LEGACY = os.environ.get("ADMIN_TOKEN_LEGACY", "1") != "0"
OC_GRAPHQL_URL = "https://opencollective.com/api/graphql/v2"
OC_COLLECTIVE_SLUG = os.environ.get("OC_COLLECTIVE_SLUG", "inference-cooperative")
# Cache of order idV2 -> tip-free contribution amount in cents, resolved via
# OC's PUBLIC GraphQL API (no token needed; our personal token lacks the
# orders scope but unauthenticated reads of public orders work).
_order_amount_cache: dict[str, int] = {}
async def get_order_amount(order_id_v2: str | None) -> int | None:
"""Look up an order's contribution amount (EXCLUDING the platform tip).
The webhook's `totalAmount` includes the buyer's optional OC tip, which
the co-op never receives — credits must be granted on the tip-free
amount. Returns cents, or None if unresolvable.
The order id arrives from the OC webhook (untrusted input), so it is sent
as a GraphQL VARIABLE, never interpolated into the query string (audit L9,
2026-11-05 — an interpolated id was a GraphQL injection into the
unauthenticated OC API; the variables form is parameterized by the server).
"""
if not order_id_v2:
return None
if order_id_v2 in _order_amount_cache:
return _order_amount_cache[order_id_v2]
try:
query = ("query OrderAmount($id: String!) { order(order: {id: $id}) "
"{ amount { valueInCents } platformTipAmount { valueInCents } } }")
async with httpx.AsyncClient(timeout=15.0) as client:
r = await client.post(OC_GRAPHQL_URL,
headers={"Content-Type": "application/json",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"},
json={"query": query, "variables": {"id": order_id_v2}})
o = r.json().get("data", {}).get("order") or {}
cents = (o.get("amount") or {}).get("valueInCents")
if cents is not None:
_order_amount_cache[order_id_v2] = int(cents)
return int(cents)
except Exception as e:
logger.warning("Order amount lookup failed for %s: %s", order_id_v2, e)
return None
def credit_pack_reference(slug: str, order: dict | None) -> str:
"""Stable idempotence key for a credit-pack purchase (audit H1, 2026-11-05).
Keyed on the OC order id from the webhook payload (`order.idV2` — the id
`get_order_amount` resolves, so a VERIFIED purchase always carries one).
The webhook's top-level `data.id` is absent from real deliveries; building
the reference from it produced the legacy `oc:<slug>:` empty-suffix
format, which cannot distinguish two purchases by the same member and so
cannot dedupe a re-delivered event. (The shortfall path keys on the OC
transaction publicId — equally unique; the webhook payload does not carry
it, and per this module's contract the collective order list is the
complete source for that shape, so no extra OC query is added here.)
Legacy ledger rows in the empty-suffix format are neither re-keyed nor
rejected: a new-format reference (non-empty id suffix) can never match
one, so the exists-check never blocks a first real grant, and the
historical rows stay exactly as recorded.
"""
ref_id = str((order or {}).get("idV2") or "").strip()
if not ref_id:
raise ValueError(
"credit pack: webhook payload carries no order idV2; refusing to "
"build an ambiguous credit reference"
)
return f"oc:{slug}:{ref_id}"
# Personal token for the "Inference Co-op Bot" account, which is an admin of
# the collective. Authenticated as an admin, the GraphQL API exposes member
# emails (which are hidden from anonymous access). We use this to match a
# member by email — the stable identifier that works even for guest
# contributors (who have no Open Collective account and thus no usable slug).
OC_PERSONAL_TOKEN = os.environ.get("OC_PERSONAL_TOKEN", "")
# Monthly credit budget (in USD of tokens) for all members.
# Single sliding-scale tier: everyone gets the same $15/month in credits,
# regardless of their $10/15/20 contribution. Governance decision (Loomio).
MEMBER_BUDGET = float(os.environ.get("MEMBER_BUDGET", "15.0"))
# --- Guest accounts (operator decisions, 2026-11-08) -------------------------- #
# A member can give a friend access to the co-op with $2 of credits, no Open
# Collective membership required. The guest is a ONE-TIME non-expiring credit
# balance held in the credits model (credit_balance), hard-capped by the
# LiteLLM team budget. Two lapse triggers, both handled by the sweep:
# * the $2 is spent (the cap enforces the limit; the sweep deactivates the
# row once spend reaches the credit balance), and
# * GUEST_DAYS elapse from provisioning — a time box, even with credits left.
# Deactivation is the same semantics as a member's: guests group → inactive,
# team budget zeroed, row active=0, newsletter unsubscribed. The account is
# KEPT so a returning payer resumes cleanly (and the guest→member conversion
# still fires if their OC membership appears later).
GUEST_BUDGET = float(os.environ.get("GUEST_BUDGET", "2.0"))
GUEST_DAYS = float(os.environ.get("GUEST_DAYS", "60"))
# One guest per inviting member per rolling 30 days (abuse protection).
GUEST_PER_INVITER_DAYS = float(os.environ.get("GUEST_PER_INVITER_DAYS", "30"))
# The ADMIN-PANEL guest-invite path records THIS account as the inviter in the
# guests ledger, and admin invites are NOT bound by the member's
# one-per-30-days cap (admins are not members).
ADMIN_INVITER_EMAIL = os.environ.get("ADMIN_INVITER_EMAIL", "info@inference.coop").strip().lower()
# Trial-period approval gate for MEMBER-initiated guest invitations. Off (the
# default): a member's request waits in PENDING until the operator approves or
# rejects it in the panel. On: a member's request provisions immediately,
# through the same pipeline and guards. Flipping it later is an env change,
# not a code change.
GUEST_AUTO_APPROVE = os.environ.get("GUEST_AUTO_APPROVE", "").strip().lower() in (
"1", "true", "yes", "on")
# The Cloudron "guests" group: guest accounts land here. The chat and dashboard
# apps' access restrictions list BOTH members and guests, so a guest reaches
# exactly those two apps — nothing else is granted by group membership.
GUESTS_GROUP_ID = os.environ.get("GUESTS_GROUP_ID", "")
# Where a SELF-CANCELLED guest's Cloudron user goes: out of the guests group,
# so chat (and every other app gated to members+guests) stops admitting them —
# but INTO a group the dashboard app lists in its access restriction, so the
# ended state ("your access has ended", data kept, Become-a-member) stays
# reachable. Gateway access is revoked separately (team budget zeroed), so the
# dashboard view is read-only by construction. Falls back to the shared
# inactive group when unset — then the dashboard itself is also closed off and
# the ended state is visible only in the cancellation response.
GUESTS_ENDED_GROUP_ID = os.environ.get("GUESTS_ENDED_GROUP_ID", "")
# Where a guest's "Become a member" button and email point (the contribution
# page — what actually starts a membership). Deliberately NOT the portal's
# credit-pack URL: a guest needs a membership, not a credit pack.
BECOME_MEMBER_URL = "https://opencollective.com/inference-cooperative/contribute"
# Days to wait before re-sending the welcome/invite email to a member who was
# provisioned but never completed account setup (inviteAccepted=False). The
# nightly sweep re-invites such members, throttled by this interval so they
# aren't emailed every single night.
REINVITE_AFTER_DAYS = float(os.environ.get("REINVITE_AFTER_DAYS", "3"))
# Models every member key/team may access.
# Deliberately REMOVED: members get access to every model the gateway serves.
# The gateway's `model_list` is the single source of truth for what's available;
# teams and keys carry no per-model restriction (empty = "all models allowed" in
# LiteLLM). Adding a provider is a gateway-config change only — no team/key
# migration. (Kept as a documented empty value so the intent is explicit.)
MEMBER_MODELS: list[str] = []
# The "members" group in Cloudron (group-based access control).
# Members are assigned to this group, which grants access to the chat app.
MEMBERS_GROUP_ID = os.environ.get("MEMBERS_GROUP_ID", "")
# The "inactive" group in Cloudron. Lapsed members are moved here (instead of
# being deleted) so they lose access but can reclaim their data on reactivation.
INACTIVE_GROUP_ID = os.environ.get("INACTIVE_GROUP_ID", "")
# Manually-provisioned members (comma-separated emails) who are NOT in the
# Open Collective member list. Used for the free "founder" tier, which OC
# cannot process as a $0 recurring subscription. These members are exempt from
# the sweep's deactivation (they have no OC membership to lapse).
MANUAL_MEMBERS = {
e.strip().lower()
for e in os.environ.get("MANUAL_MEMBERS", "").split(",")
if e.strip()
}
# Loomio integration (forum.inference.coop). The portal reconciles the Loomio
# group to the current active-member list via the User API (/api/b2).
LOOMIO_BASE = os.environ.get("LOOMIO_BASE", "https://forum.inference.coop")
LOOMIO_API_KEY = os.environ.get("LOOMIO_API_KEY", "")
# Server API key (B3) — used to fix auto-generated Loomio usernames after
# invites. Loomio's b2/memberships API only takes emails; new users get a
# mangled auto-handle (email local-part + random suffix). The B3 users API
# lets us rename them to the member's Cloudron username.
LOOMIO_B3_KEY = os.environ.get("LOOMIO_B3_KEY", "")
# Master switch for the Loomio membership sync. It defaults to ENABLED: the
# co-op wants every new member added to Loomio, so restoring LOOMIO_API_KEY
# and deploying is all that is needed. It is expressed as a KILL SWITCH —
# set LOOMIO_SYNC_ENABLED=0 (or false/off/no) to pause the sync; any other
# value, or leaving it unset, keeps it ON. (Incident 2026-11-04 was fixed the
# right way by making the ADD path idempotent, NOT by switching the feature
# off — see sync_loomio_memberships for the guards.)
LOOMIO_SYNC_ENABLED = os.environ.get("LOOMIO_SYNC_ENABLED", "1").strip().lower() not in ("0", "false", "off", "no")
# Matrix homeserver (self-hosted Synapse at matrix.inference.coop). The bot
# account (admin) invites every active member to the co-op Space on sync.
# Members authenticate via Cloudron SSO; their homeserver MXID is
# @<cloudron-username>:inference.coop. Federated MXIDs (external servers)
# can also be invited directly.
MATRIX_HOMESERVER = os.environ.get("MATRIX_HOMESERVER", "https://matrix.inference.coop")
MATRIX_BOT_TOKEN = os.environ.get("MATRIX_BOT_TOKEN", "")
MATRIX_SPACE_ID = os.environ.get("MATRIX_SPACE_ID", "")
MATRIX_GENERAL_ROOM = os.environ.get("MATRIX_GENERAL_ROOM", "")
# The explicit set of rooms every active member is invited to (comma-separated
# room IDs). This is authoritative configuration and is deliberately NOT derived
# from the Space's children — some child rooms (operator/admin-only) must never
# be pushed at every member. When unset, sync_matrix_space() falls back to the
# co-op Space + General room. Restricted rooms are NOT discoverable over
# federation until a member has joined (the space hierarchy omits them for remote
# servers), so member-facing rooms need this explicit invite pass.
MATRIX_INVITE_ROOMS = [r.strip() for r in os.environ.get("MATRIX_INVITE_ROOMS", "").split(",") if r.strip()]
# Legacy alias kept for compatibility: extra member-facing rooms appended to the
# invite set above (superseded by MATRIX_INVITE_ROOMS).
MATRIX_EXTRA_ROOMS = [r.strip() for r in os.environ.get("MATRIX_EXTRA_ROOMS", "").split(",") if r.strip()]
LOOMIO_GROUP_ID = os.environ.get("LOOMIO_GROUP_ID", "")
# Operator accounts that must always remain in the Loomio group (never removed
# by the remove_absent reconciliation), comma-separated.
LOOMIO_ALWAYS_KEEP = [
e.strip()
for e in os.environ.get("LOOMIO_ALWAYS_KEEP", "info@inference.coop,bot@inference.coop").split(",")
if e.strip()
]
# Persistent store (SQLite) for email → LiteLLM key token mapping.
# Lives in /app/data (Cloudron localstorage addon persists this).
DB_PATH = os.environ.get("DB_PATH", "/app/data/members.db")
# --- Email (Cloudron sendmail addon) ---
# The sendmail addon exports SMTP credentials as CLOUDRON_MAIL_* env vars.
SMTP_SERVER = os.environ.get("CLOUDRON_MAIL_SMTP_SERVER", "")
SMTP_PORT = int(os.environ.get("CLOUDRON_MAIL_SMTP_PORT", "2525"))
SMTP_USERNAME = os.environ.get("CLOUDRON_MAIL_SMTP_USERNAME", "")
SMTP_PASSWORD = os.environ.get("CLOUDRON_MAIL_SMTP_PASSWORD", "")
MAIL_FROM = os.environ.get("CLOUDRON_MAIL_FROM", "info@inference.coop")
MAIL_FROM_NAME = os.environ.get("CLOUDRON_MAIL_FROM_DISPLAY_NAME", "Inference Cooperative")
# Reply-to address so member replies land at the co-op inbox.
MAIL_REPLY_TO = os.environ.get("MAIL_REPLY_TO", "info@inference.coop")
# Public base URL for the portal (used in email links).
PORTAL_BASE = os.environ.get("PORTAL_BASE", "https://portal.inference.coop")
# Listmonk newsletter integration. The portal keeps the "Members" list in sync:
# subscribe on provision, unsubscribe on deactivation. Auth uses listmonk's
# native API-user token scheme (Authorization: token <user>:<token>).
LISTMONK_BASE = os.environ.get("LISTMONK_BASE") or "https://newsletter.inference.coop"
# NOTE: `or` (not default-arg) — an env var set to EMPTY must fall back to the
# default. An empty LISTMONK_API_USER made every sync 403 silently (the token
# alone is not a valid listmonk credential) while listmonk_sync's best-effort
# error handling swallowed it — that is how stale accounts stayed subscribed.
LISTMONK_API_USER = os.environ.get("LISTMONK_API_USER") or "portal-sync"
LISTMONK_API_TOKEN = os.environ.get("LISTMONK_API_TOKEN", "")
LISTMONK_LIST_ID = int(os.environ.get("LISTMONK_LIST_ID", "3")) # "Members" list
# Direct read access to LiteLLM's Postgres (for usage aggregations).
# READ-ONLY purpose: never write to LiteLLM's tables — its HTTP API remains the
# only interface for writes/management. Falls back to the HTTP path when unset.
LITELLM_DB_HOST = os.environ.get("LITELLM_DB_HOST", "")
LITELLM_DB_PORT = int(os.environ.get("LITELLM_DB_PORT", "5432"))
LITELLM_DB_USER = os.environ.get("LITELLM_DB_USER", "")
LITELLM_DB_PASSWORD = os.environ.get("LITELLM_DB_PASSWORD", "")
LITELLM_DB_NAME = os.environ.get("LITELLM_DB_NAME", "")
def litellm_db_ready() -> bool:
"""True when LiteLLM DB read credentials are configured."""
return bool(LITELLM_DB_HOST and LITELLM_DB_USER and LITELLM_DB_NAME)
def _pg_connect():
"""Open a short-lived connection to LiteLLM's Postgres (read-only use)."""
import psycopg2
return psycopg2.connect(
host=LITELLM_DB_HOST, port=LITELLM_DB_PORT, user=LITELLM_DB_USER,
password=LITELLM_DB_PASSWORD, dbname=LITELLM_DB_NAME,
connect_timeout=10, sslmode="prefer",
)
def aggregate_model_usage(days: int = 0, team_id: str | None = None) -> dict:
"""Aggregate LiteLLM spend logs per model, straight from Postgres.
Returns {"models": [...], "totals": {...}} in the same shape the HTTP path
produced, so callers don't change. `days` bounds the window (0 = all time);
`team_id` restricts to one member's team (used by the dashboard path).
Prefers the `model_group` column, which carries our provider/model alias
(e.g. greenpt/glm-5.3-flash); falls back to `model` for pre-alias rows — matching
the logic the HTTP path used, expressed in SQL.
`requests` counts only metered calls that SUCCEEDED. Refused/failed attempts
(rows with status = 'failure', e.g. a 401 on the /v1/systemone decision
route) are excluded from that count and reported separately as `refused`,
so a failure cannot inflate a model's call count. `spend` and `tokens` stay
summed over every row: a call that failed after tokens were consumed still
cost money and still processed those tokens.
"""
if not litellm_db_ready():
raise RuntimeError("LiteLLM DB credentials not configured")
where = []
params: list = []
if days and days > 0:
from datetime import datetime, timedelta, timezone
cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).strftime(
"%Y-%m-%d %H:%M:%S"
)
where.append("\"startTime\" >= %s")
params.append(cutoff)
if team_id:
where.append("team_id = %s")
params.append(team_id)
where_sql = ("WHERE " + " AND ".join(where)) if where else ""
sql = f"""
SELECT
COALESCE(NULLIF(model_group, ''), NULLIF(model, ''), 'unknown') AS model,
COALESCE(SUM(spend), 0) AS spend,
COALESCE(SUM(total_tokens), 0) AS tokens,
COUNT(*) FILTER (WHERE status IS DISTINCT FROM 'failure') AS requests,
COUNT(*) FILTER (WHERE status = 'failure') AS refused
FROM "LiteLLM_SpendLogs"
{where_sql}
GROUP BY 1
ORDER BY requests DESC
"""
conn = _pg_connect()
try:
with conn.cursor() as cur:
cur.execute(sql, params)
rows = cur.fetchall()
finally:
conn.close()
models = [
{"model": r[0], "spend": float(r[1] or 0.0), "tokens": int(r[2] or 0),
"requests": int(r[3] or 0), "refused": int(r[4] or 0)}
for r in rows
]
return {
"models": models,
"totals": {
"total_spend": sum(m["spend"] for m in models),
"total_tokens": sum(m["tokens"] for m in models),
"total_requests": sum(m["requests"] for m in models),
"total_refused": sum(m["refused"] for m in models),
},
}
def get_team_budget_from_db(team_id: str) -> dict | None:
"""Fetch a team's spend/budget/reset directly from LiteLLM's Postgres.
Same data as the /team/list HTTP path (LiteLLM computes `spend`), without
downloading the whole team list. Returns None when the team is unknown.
"""
if not litellm_db_ready():
return None
sql = """
SELECT max_budget, spend, budget_reset_at
FROM "LiteLLM_TeamTable"
WHERE team_id = %s
LIMIT 1
"""
conn = _pg_connect()
try:
with conn.cursor() as cur:
cur.execute(sql, (team_id,))
row = cur.fetchone()
finally:
conn.close()
if not row:
return None
return {
"max_budget": float(row[0]) if row[0] is not None else None,
"spend": float(row[1] or 0.0),
"budget_reset_at": row[2],
}
async def listmonk_sync(email: str, subscribe: bool) -> None:
"""Subscribe or unsubscribe a member to the newsletter list.
Best-effort: never raises, so a newsletter outage can't break provisioning.
"""
if not LISTMONK_API_TOKEN:
logger.warning("Listmonk not configured; skipping sync for %s", email)
return
auth = {"Authorization": f"token {LISTMONK_API_USER}:{LISTMONK_API_TOKEN}"}
try:
async with httpx.AsyncClient(timeout=10.0) as client:
if subscribe:
r = await client.post(
f"{LISTMONK_BASE}/api/subscribers",
headers=auth,
json={
"email": email,
"name": "",
"status": "enabled",
"lists": [LISTMONK_LIST_ID],
"preconfirm_subscriptions": True,
},
)
else:
# Look up the subscriber id by email, then blocklist by id.
# (listmonk's blocklist endpoint takes an id, not an email.)
q = await client.get(
f"{LISTMONK_BASE}/api/subscribers",
headers=auth,
params={"query": f"subscribers.email='{email}'"},
)
if q.status_code != 200:
logger.warning("Listmonk lookup for %s failed: %s", email, q.status_code)
return
results = q.json().get("data", {}).get("results", [])
if not results:
return # not subscribed; nothing to do
sid = results[0]["id"]
r = await client.put(
f"{LISTMONK_BASE}/api/subscribers/{sid}/blocklist",
headers=auth,
)
if r.status_code not in (200, 201):
logger.warning(
"Listmonk sync %s for %s failed: %s",
"subscribe" if subscribe else "unsubscribe",
email,
r.status_code,
)
except Exception as e:
logger.warning("Listmonk sync error for %s: %s", email, e)
async def listmonk_reconcile() -> dict:
"""Reconcile the Members newsletter list against the active membership.
The newsletter list mirrors the portal's ACTIVE members. Subscribers who are
not active members fall into two classes, and only one is acted on:
- emails that exist in the members table but are inactive, and emails that
are not members at all, are taken off the list (the subscriber record
stays as a suppression marker). A DELETE would destroy the subscriber's
consent history, and a stray test account is indistinguishable from a
real person who subscribed outside the portal, so nobody is purged
automatically;
- OPERATOR_EMAILS — infrastructure accounts, which the panel surfaces
under "admins" and excludes from member counts — are left exactly as
they are. They are neither members nor stale. The Loomio
reconciliation applies the same rule through LOOMIO_ALWAYS_KEEP.
Best-effort: never raises, so a newsletter outage can't break the sweep.
"""
if not LISTMONK_API_TOKEN:
logger.warning("Listmonk not configured; skipping newsletter reconcile")
return {"removed": 0, "emails": [], "error": "not configured"}
auth = {"Authorization": f"token {LISTMONK_API_USER}:{LISTMONK_API_TOKEN}"}
conn = get_db()
active = {
row[0].lower()
for row in conn.execute("SELECT email FROM members WHERE active = 1")
}
member_emails = {
row[0].lower() for row in conn.execute("SELECT email FROM members")
}
conn.close()
try:
async with httpx.AsyncClient(timeout=20.0) as client:
# Fetch the full subscriber set for the list (paginated).
subs, page = [], 1
while True:
r = await client.get(
f"{LISTMONK_BASE}/api/subscribers",
headers=auth,
params={"list_id": LISTMONK_LIST_ID, "page": page, "per_page": 100},
)
if r.status_code != 200:
logger.warning(
"Listmonk reconcile: subscriber fetch failed: %s", r.status_code
)
return {"removed": 0, "emails": [], "error": f"fetch {r.status_code}"}
data = r.json().get("data", {})
results = data.get("results", [])
subs += results
total = data.get("total")
# Stop on an empty or short page, or once the reported count is
# reached. The short-page test means a missing `total` cannot
# truncate the reconcile at page 1.
if (
not results
or len(results) < 100
or (total is not None and len(subs) >= total)
):
break
page += 1
removed, emails = 0, []
by_reason = {"inactive member": 0, "not a member": 0}
# Operator accounts are neither members nor stale; never touch their
# subscription (LOOMIO_ALWAYS_KEEP is the same rule for the forum).
operators_kept = sum(
1 for s in subs if (s.get("email") or "").lower() in OPERATOR_EMAILS
)
for s in subs:
email = (s.get("email") or "").lower()
if not email or email in active or email in OPERATOR_EMAILS:
continue
sid = s.get("id")
# Both classes leave the list the same way: unsubscribe, keep the
# record. Nothing is deleted.
reason = "inactive member" if email in member_emails else "not a member"
# listmonk validates the PUT body's email field: omitting it is a
# 400 "Invalid email." and the subscriber stays on the list (which
# is why the inactive-member branch never actually worked).
r = await client.put(
f"{LISTMONK_BASE}/api/subscribers/{sid}",
headers=auth,
json={"email": s.get("email"), "lists": []},
)
if r.status_code in (200, 201):
removed += 1
by_reason[reason] += 1
emails.append(email)
else:
logger.warning(
"Listmonk reconcile: unsubscribe %s failed: %s", email, r.status_code
)
if removed:
# Counts only — subscriber addresses don't belong in the log.
logger.info(
"Listmonk reconcile: removed %d stale subscribers (%s)",
removed,
", ".join(f"{k}: {v}" for k, v in by_reason.items() if v),
)
if operators_kept:
logger.info(
"Listmonk reconcile: left %d operator address(es) on the list",
operators_kept,
)
return {"removed": removed, "emails": emails, "operators_kept": operators_kept}
except Exception as e:
logger.warning("Listmonk reconcile error: %s", e)
return {"removed": 0, "emails": [], "error": str(e)}
def _verify_portal_secret(request: Request) -> None:
"""Reject requests that didn't come through LibreChat (shared secret)."""
if not PORTAL_SECRET:
# If no secret is configured, refuse to inject keys (fail closed).
raise HTTPException(503, "Portal secret not configured")
provided = request.headers.get("x-portal-secret", "")
if not secrets.compare_digest(provided, PORTAL_SECRET):
raise HTTPException(401, "Invalid portal secret")
def _verify_admin_token(request: Request) -> None:
"""Authorize a /admin/* caller (SECURITY 2026-10-27).
Admin auth is a HEADER, never a URL token: the webhook URL token lives in
places we do not control, and it used to unlock all of /admin/*.
"""
provided = request.headers.get("x-admin-token", "")
if ADMIN_TOKEN and secrets.compare_digest(provided, ADMIN_TOKEN):
return
if ADMIN_TOKEN_LEGACY and WEBHOOK_TOKEN and secrets.compare_digest(provided, WEBHOOK_TOKEN):
logger.warning(
"admin auth: caller used the legacy WEBHOOK_TOKEN — move it to"
" ADMIN_TOKEN, then set ADMIN_TOKEN_LEGACY=0"
)
return
raise HTTPException(403, "Forbidden")
def get_db() -> sqlite3.Connection:
conn = sqlite3.connect(DB_PATH)
conn.execute(
"CREATE TABLE IF NOT EXISTS members ("
"email TEXT PRIMARY KEY, "
"key_token TEXT, "
"cloudron_user_id TEXT, "
"slug TEXT, "
"active INTEGER DEFAULT 1"
")"
)
# Daily metrics snapshot, written by /app/code/metrics_snapshot.py from the
# scheduler and read by the admin panel + dashboard. Cumulative columns
# (members_*, newsletter_subs, matrix_space_members) are counts AS OF that
# day; spend_usd/requests are THAT day's traffic. A NULL column means the
# source could not be read that day — charts must show a gap, never a zero.
conn.execute(
"CREATE TABLE IF NOT EXISTS metrics_daily ("
"day TEXT PRIMARY KEY, "
"members_active INTEGER, "
"members_total INTEGER, "
"members_new INTEGER, "
"newsletter_subs INTEGER, "
"matrix_space_members INTEGER, "
"spend_usd REAL, "
"operator_spend_usd REAL, "
"requests INTEGER, "
"out_tokens INTEGER, "
"energy_wh REAL, "
"co2e_g REAL, "
"tokens_measured INTEGER, "
"tokens_estimated INTEGER, "
"factor_revision TEXT, "
"recorded_at TEXT"
")"
)
# CREATE IF NOT EXISTS never adds columns to an existing table, so migrate:
# impact columns arrived after the table did (see co-op/support#26), and the
# measured-vs-estimated provenance columns after those. All additive: existing
# rows keep NULL, and a NULL split reports as unknown — never as "measured".
for _col, _decl in (
# The co-op's OWN accounts' share of spend_usd (OPERATOR_EMAILS). The
# total keeps spanning every team; this is the breakout, so a
# member-facing surface can say how much of it is the co-op's own
# usage. NULL = not measured (never a confident 0).
("operator_spend_usd", "REAL"),
("out_tokens", "INTEGER"),
("energy_wh", "REAL"),
("co2e_g", "REAL"),
("tokens_measured", "INTEGER"),
("tokens_estimated", "INTEGER"),
("factor_revision", "TEXT"),
# The collective's OC balance, snapshotted daily (see metrics_snapshot.py
# oc_collective_balance). The OC API exposes only the CURRENT balance, so
# a real history accrues from these snapshots — earlier days stay NULL.
("oc_balance_usd", "REAL"),
):
try:
conn.execute(f"ALTER TABLE metrics_daily ADD COLUMN {_col} {_decl}")
except sqlite3.OperationalError:
pass # already present
# Member-managed API keys (created via the broker). Each row is one API key
# under the member's team, tracked so we can list/revoke by name without
# exposing the chat key. `sk_token` is the full sk- key (revealed once).
conn.execute(
"CREATE TABLE IF NOT EXISTS member_api_keys ("
"email TEXT NOT NULL, "
"name TEXT NOT NULL, "
"sk_token TEXT NOT NULL, "
"hash TEXT NOT NULL, "
"created_at TEXT DEFAULT (datetime('now')), "
"PRIMARY KEY (email, name)"
")"
)
# Migration: add slug column if the members table predates it.
cols = [r[1] for r in conn.execute("PRAGMA table_info(members)").fetchall()]
if "slug" not in cols:
conn.execute("ALTER TABLE members ADD COLUMN slug TEXT")
# Migration: add balance column (USD) — the member's current credit
# allowance. Defaults to MEMBER_BUDGET; can be topped up by payments.
# This is the flexible counter to the fixed $15/month allowance.
if "balance" not in cols:
conn.execute("ALTER TABLE members ADD COLUMN balance REAL")
# Migration: welcome_sent_at — timestamp of the last welcome/invite email,
# used to throttle re-invites of members who never activated.
if "welcome_sent_at" not in cols:
conn.execute("ALTER TABLE members ADD COLUMN welcome_sent_at TEXT")
# Migration: matrix_mxid — an OPTIONAL extra Matrix account a member wants
# invited to the co-op Space (e.g. their existing federated account on
# another homeserver). Empty = only the local @username:inference.coop
# account is invited. At most one extra MXID per member.
if "matrix_mxid" not in cols:
conn.execute("ALTER TABLE members ADD COLUMN matrix_mxid TEXT")
# Migration: credit_balance — purchased, NON-EXPIRING credits (USD).
# Distinct from `balance` (the monthly allowance, which resets): credits
# carry over indefinitely and are spent only after the allowance is
# exhausted within a cycle.
if "credit_balance" not in cols:
conn.execute("ALTER TABLE members ADD COLUMN credit_balance REAL NOT NULL DEFAULT 0")
# Migration: org_slug — for ORGANIZATION-backed memberships. OC orgs never
# expose an email to our integration, so the org's designated contact is
# linked manually (POST /admin/org-link): org_slug holds the OC org slug
# whose Membership-tier dues keep this account active. The nightly sweep
# keeps the account active while the org is a BACKER on the Membership
# tier and deactivates it when the org's dues lapse.
if "org_slug" not in cols:
conn.execute("ALTER TABLE members ADD COLUMN org_slug TEXT")
# Migration: loomio_invited_at — the first time this member's email was
# handed to Loomio's membership API. Non-NULL means "already invited, never
# invite again". This is the positive exclusion enforced by
# sync_loomio_memberships: an existing member (already in Loomio, or already
# invited) can never be sent a Loomio invite a second time.
if "loomio_invited_at" not in cols:
conn.execute("ALTER TABLE members ADD COLUMN loomio_invited_at TEXT")
# Migration: account_type — 'member' (default) or 'guest'. Guest accounts are
# provisioned by /admin/provision-guest with $2 of non-expiring credit, live
# in the Cloudron "guests" group, and sit outside every membership
# obligation: the sweep never deactivates one for lacking an OC
# contribution, they are invited to no Loomio/Matrix rooms, and they lapse
# only by their own rules (credit exhausted, or the GUEST_DAYS time box).
if "account_type" not in cols:
conn.execute("ALTER TABLE members ADD COLUMN account_type TEXT DEFAULT 'member'")
# Migration: oc_email — the Open Collective email a member's contribution
# arrives under when it differs from their account email (set via
# POST /admin/guest-convert when a invited friend joins under another
# address). The sweep and the webhook treat it as an additional matcher so
# renewals under that address keep the account active.
if "oc_email" not in cols:
conn.execute("ALTER TABLE members ADD COLUMN oc_email TEXT")
# Migration: guest naming (2026-11-08). The ledger table, the members
# expiry column, the lapse-notice table and the account_type VALUES all
# carried 'gift' before the feature was renamed to guest access. Every
# database that predates the rename moves once, here — idempotent, and a
# no-op on any database created after it. Runs BEFORE the
# guest_expires_at ADD COLUMN below so an old database renames its
# populated column instead of gaining an empty twin.
_tables = [r[0] for r in conn.execute(
"SELECT name FROM sqlite_master WHERE type='table'").fetchall()]
if "gifts" in _tables and "guests" not in _tables:
conn.execute("ALTER TABLE gifts RENAME TO guests")
if "gift_lapse_notices" in _tables and "guest_lapse_notices" not in _tables:
conn.execute("ALTER TABLE gift_lapse_notices RENAME TO guest_lapse_notices")
# The ledger's own expiry column moves with its table — the provision
# INSERT writes guest_expires_at, so a legacy table keeping the old
# column name would fail on the first provision after the rename.
_gcols = [r[1] for r in conn.execute("PRAGMA table_info(guests)").fetchall()]
if "gift_expires_at" in _gcols and "guest_expires_at" not in _gcols:
conn.execute("ALTER TABLE guests RENAME COLUMN gift_expires_at TO guest_expires_at")
if "gift_expires_at" in cols and "guest_expires_at" not in cols:
conn.execute("ALTER TABLE members RENAME COLUMN gift_expires_at TO guest_expires_at")
conn.execute("UPDATE members SET account_type = 'guest' WHERE account_type = 'gift'")
# Credit-ledger dedupe references move with the rename (a re-grant that
# misses the old prefix could double-credit). Returns first: a
# 'gift-return:…' value also matches 'gift:%'. Only when the table
# already exists — a first-boot database creates it below, empty.
if "credit_ledger" in _tables:
conn.execute("UPDATE credit_ledger SET reference = 'guest-return:' || substr(reference, 13) "
"WHERE reference LIKE 'gift-return:%'")
conn.execute("UPDATE credit_ledger SET reference = 'guest:' || substr(reference, 6) "
"WHERE reference LIKE 'gift:%'")
# Migration: guest_expires_at — when a guest account's GUEST_DAYS time box
# elapses (set at guest provisioning). The sweep deactivates the account
# after this instant even if credits remain.
# NOTE: this must be an EXCLUSIVE branch against the RENAME above — `cols`
# is a snapshot taken before the rename, so a plain `if not in cols` here
# would add a duplicate column on any database the rename just migrated.
if "gift_expires_at" not in cols and "guest_expires_at" not in cols:
conn.execute("ALTER TABLE members ADD COLUMN guest_expires_at TEXT")
# Guests ledger — the abuse-protection audit trail: WHO invited WHOM, when,
# and what became of the guest. One row per guest ever sent.
conn.execute(
"CREATE TABLE IF NOT EXISTS guests ("
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
"inviter_email TEXT NOT NULL, "
"recipient_email TEXT NOT NULL, "
"oc_reference TEXT, "
"created_at TEXT DEFAULT (datetime('now')), "
"guest_expires_at TEXT, "
"status TEXT DEFAULT 'active')"
)
# Guest REQUESTS — the member self-serve path's trial gate. A member's
# invitation lands here first; the operator approves (which fires the
# full guest pipeline and writes the ledger row above) or rejects it.
# PENDING requests are part of the abuse rules: they block the recipient
# address and count against the inviter's rolling window.
conn.execute(
"CREATE TABLE IF NOT EXISTS guest_requests ("
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
"inviter_email TEXT NOT NULL, "
"recipient_email TEXT NOT NULL, "
"recipient_name TEXT, "
"created_at TEXT DEFAULT (datetime('now')), "
"decided_at TEXT, "
"status TEXT DEFAULT 'pending', "
"note TEXT)"
)
# Credit ledger: audit trail for every credit movement (purchases via the
# OC webhook, admin adjustments, spends drawn against credits). Money needs
# an audit trail.
conn.execute(
"CREATE TABLE IF NOT EXISTS credit_ledger ("
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
"email TEXT NOT NULL, "
"delta REAL NOT NULL, "
"reason TEXT NOT NULL, "
"reference TEXT, "
"created_at TEXT DEFAULT (datetime('now'))"
")"
)
# Held credit packs: purchases by NON-members. Money received but credits
# not granted (a membership is required to use credits). Each hold is
# either refunded manually (OC dashboard, 30-day window) or granted
# automatically when the buyer becomes a member.
conn.execute(
"CREATE TABLE IF NOT EXISTS held_credits ("
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
"slug TEXT NOT NULL, "
"email TEXT, "
"name TEXT, "
"amount REAL NOT NULL, "
"oc_reference TEXT, "
"status TEXT NOT NULL DEFAULT 'held', "
"created_at TEXT DEFAULT (datetime('now')), "
"resolved_at TEXT"
")"
)
# Unique oc_reference (audit H1, 2026-11-05): an OC re-delivery must never
# create a second hold (or, via the webhook grant path's exists-check, a
# second credit) for the same purchase. Holds recorded before the fix used
# the legacy empty-suffix format and are distinct as audited, so the index
# applies cleanly to the live table; multiple NULLs stay legal (SQLite).
# If creation fails there are REAL duplicates — log CRITICAL and keep
# serving unguarded rather than boot-looping the portal.
try:
conn.execute(
"CREATE UNIQUE INDEX IF NOT EXISTS idx_held_credits_oc_reference "
"ON held_credits(oc_reference)"
)
except sqlite3.OperationalError as e:
logger.error("held_credits.oc_reference unique index NOT created: %s", e)
# Member-managed API keys: add a fingerprint (last-4 of the key, shown in
# lists) and purge the stored plaintext. Member API keys are revealed ONCE
# (at creation) and never again — the industry standard. Revocation uses the
# LiteLLM hash, not the plaintext, so the portal has no reason to keep it.
# (The `members.key_token` chat key is a different table and IS retained —
# encrypted at rest (AES-256-GCM), decrypted in memory only when served.)
ak_cols = [r[1] for r in conn.execute("PRAGMA table_info(member_api_keys)").fetchall()]
if "fingerprint" not in ak_cols:
conn.execute("ALTER TABLE member_api_keys ADD COLUMN fingerprint TEXT")
# Backfill fingerprints for any key still holding plaintext (fingerprint =
# last-4 of the key), then purge the plaintext. Idempotent — runs every
# time, affects only rows where plaintext is still present.
conn.execute(
"UPDATE member_api_keys SET fingerprint = substr(sk_token, -4) "
"WHERE (fingerprint IS NULL OR fingerprint = '') AND sk_token != ''"
)
conn.execute("UPDATE member_api_keys SET sk_token = '' WHERE sk_token != ''")
# Encrypt member chat keys at rest (AES-256-GCM, KEK from PORTAL_KEK env).
# Idempotent: only touches rows still holding plaintext. Skipped with a
# CRITICAL log if PORTAL_KEK is unset — see the inline key-encryption module.
migrate_plaintext_keys(conn)
# Commit the DML above — ALTER TABLE auto-commits, but the UPDATEs open an
# implicit transaction that would otherwise roll back when get_db() closes.
conn.commit()
return conn
def get_member_balance(email: str) -> float:
"""Return the member's current balance (USD), defaulting to MEMBER_BUDGET.
The balance is the flexible allowance — it starts at MEMBER_BUDGET but can
be adjusted (topped up by payments) without changing code.
"""
conn = get_db()
row = conn.execute(
"SELECT balance FROM members WHERE email = ?", (email,)
).fetchone()
conn.close()
if row and row[0] is not None:
return float(row[0])
return MEMBER_BUDGET
def set_member_balance(email: str, balance: float) -> None:
"""Set the member's balance (USD). Used by top-up / billing adjustments."""
conn = get_db()
conn.execute(
"UPDATE members SET balance = ? WHERE email = ?", (balance, email)
)
conn.commit()
conn.close()
# --- Credits: purchased, non-expiring usage dollars ---
def get_credit_balance(email: str) -> float:
"""Return the member's non-expiring credit balance (USD, default 0)."""
conn = get_db()
row = conn.execute(
"SELECT credit_balance FROM members WHERE email = ?", (email,)
).fetchone()
conn.close()
if row and row[0] is not None:
return float(row[0])
return 0.0
def get_member_account_type(email: str) -> str:
"""Return the row's account type: 'member' (default) or 'guest'."""
conn = get_db()
row = conn.execute(
"SELECT account_type FROM members WHERE email = ?", (email,)
).fetchone()
conn.close()
return (row[0] or "member") if row else "member"
def is_guest_account(email: str) -> bool:
"""True when the row is a guest account (operator guest access feature)."""
return get_member_account_type(email) == "guest"
def guest_provision_guard(
conn, inviter_email: str, recipient_email: str, admin: bool = False
) -> str | None:
"""Decide whether a guest may be sent (pure — runs on any connection).
Abuse-protection rules (operator decision, 2026-11-08):
* the inviter must be an ACTIVE member row (account_type 'member');
* one guest per inviter per rolling GUEST_PER_INVITER_DAYS (default 30 days);
* one guest per recipient EVER — any prior guests-ledger row for the
recipient refuses, as does an existing portal row under that email
(they already have/had a guest account or are a member).
admin=True is the ADMIN-PANEL path: the inviter is the co-op's admin
account, not a member, so the member-side inviter rules (active member,
one-per-30-days) do NOT apply — admins are not bound by the member's
monthly cap. The recipient-side rules still run on every path: one guest
per recipient ever, and no existing portal row.
Returns None when the guest may proceed, else a human-readable refusal.
"""
inviter = (inviter_email or "").strip().lower()
recipient = (recipient_email or "").strip().lower()
if not inviter or "@" not in inviter:
return "The inviting member's email is required"
if not recipient or "@" not in recipient:
return "A valid recipient email is required"
if inviter == recipient:
return "You cannot guest the account you are guest access from"
if not admin:
row = conn.execute(
"SELECT active, account_type FROM members WHERE email = ?", (inviter,)
).fetchone()
if not row or not row[0] or (row[1] or "member") == "guest":
return f"Inviter {inviter} is not an active member account"
recent = conn.execute(
"SELECT COUNT(*) FROM guests WHERE inviter_email = ? AND "
"created_at >= datetime('now', ?)",
(inviter, f"-{int(GUEST_PER_INVITER_DAYS)} days"),
).fetchone()[0]
if recent:
return (f"{inviter} already sent a guest in the last "
f"{int(GUEST_PER_INVITER_DAYS)} days — one guest per member per month")
ever = conn.execute(
"SELECT COUNT(*) FROM guests WHERE recipient_email = ?", (recipient,)
).fetchone()[0]
if ever:
return f"{recipient} has already received a guest account"
existing = conn.execute(
"SELECT account_type FROM members WHERE email = ?", (recipient,)
).fetchone()
if existing:
acct = existing[0] or "member"
return (f"{recipient} already has a portal row ({acct}) — guest access is for "
"new addresses; existing members and past guest accounts are refused")
return None
def guest_request_guard(conn, inviter_email: str, recipient_email: str) -> str | None:
"""Decide whether a MEMBER may send a guest invitation (pure).
Everything the guest pipeline enforces (guest_provision_guard: active
member inviter, one guest per inviter per rolling window, one guest per
recipient ever, no existing portal row) plus the member-path rules:
* the recipient is not on MANUAL_MEMBERS — manually-provisioned
accounts are co-op insiders, never guest targets;
* the recipient has no PENDING request already (from any inviter);
* the inviter's rolling GUEST_PER_INVITER_DAYS window counts REQUESTS
too — a pending or approved request sits inside the window, so the
one-per-30-days limit cannot be outrun by requesting twice while the
first is still waiting.
A REJECTED request counts as neither: nothing was ever sent, and the
operator's gate still sits in front of every actual send.
Returns None when the request may proceed, else a human-readable refusal.
"""
refusal = guest_provision_guard(conn, inviter_email, recipient_email)
if refusal:
return refusal
inviter = (inviter_email or "").strip().lower()
recipient = (recipient_email or "").strip().lower()
if recipient in MANUAL_MEMBERS:
return ("That address belongs to a manually-provisioned co-op account — "
"guest invitations are for people outside the co-op")
pending = conn.execute(
"SELECT COUNT(*) FROM guest_requests WHERE recipient_email = ? AND status = 'pending'",
(recipient,),
).fetchone()[0]
if pending:
return f"{recipient} already has a guest request waiting for approval"
recent = conn.execute(
"SELECT COUNT(*) FROM guest_requests WHERE inviter_email = ? "
"AND status IN ('pending', 'approved') "
"AND created_at >= datetime('now', ?)",
(inviter, f"-{int(GUEST_PER_INVITER_DAYS)} days"),
).fetchone()[0]
if recent:
return (f"You already sent a guest request in the last "
f"{int(GUEST_PER_INVITER_DAYS)} days — one guest per member per month")
return None
def add_credits(email: str, delta: float, reason: str, reference: str | None = None) -> float:
"""Adjust a member's credit balance and record the movement in the ledger.
Returns the new balance. Negative deltas are allowed (draws against
credits); the balance is clamped at >= 0.
"""
conn = get_db()
conn.execute(
"UPDATE members SET credit_balance = MAX(0, COALESCE(credit_balance, 0) + ?) "
"WHERE email = ?",
(delta, email),
)
conn.execute(
"INSERT INTO credit_ledger (email, delta, reason, reference) VALUES (?, ?, ?, ?)",
(email, delta, reason, reference),
)
row = conn.execute(
"SELECT credit_balance FROM members WHERE email = ?", (email,)
).fetchone()
conn.commit()
conn.close()
return float(row[0]) if row and row[0] is not None else 0.0
def compute_max_budget(allowance: float, credit_balance: float, team_spend: float | None) -> dict:
"""Compute the effective LiteLLM max_budget from the two buckets.
Model: the member has a monthly ALLOWANCE (renews on each membership
payment) plus a credit balance that adds spendable headroom ON TOP of the
allowance. The total spendable cap is simply `allowance + credit_balance`
— credits never shrink the allowance; they extend it.
`credit_balance` is the TOTAL purchased (the ledger's lifetime sum); the
amount still available this cycle is `credit_balance - credits_used`.
`team_spend` is the member's spend so far this cycle (from LiteLLM). It is
used only to split the spend between allowance vs credits for display, NOT
to reduce the cap.
Returns {"max_budget", "allowance_used", "credits_used", "credits_remaining"}
for dashboards.
"""
max_budget = allowance + credit_balance
if team_spend is None:
return {
"max_budget": max_budget,
"allowance_used": 0.0,
"credits_used": 0.0,
"credits_remaining": credit_balance,
}
allowance_used = min(max(team_spend, 0.0), allowance)
credits_used = max(0.0, team_spend - allowance)
return {
"max_budget": max_budget,
"allowance_used": allowance_used,
"credits_used": credits_used,
"credits_remaining": max(0.0, credit_balance - credits_used),
}
async def fetch_all_member_nodes() -> list[dict]:
"""Every member node of the collective, via OC's offset pagination.
The members connection has NO cursor/pageInfo pagination (older-style API)
and caps at 100 per page: an unpaginated query silently hides everyone
past the first page — the same failure class as Cloudron's 25-per-page
user list. Raises on ANY page failure so callers can apply their own
fail-safes; never returns a partial list (a partial member list would make
the sweep deactivate everyone missing from it).
"""
if not OC_PERSONAL_TOKEN:
raise RuntimeError("OC_PERSONAL_TOKEN not set; cannot fetch members")
nodes: list[dict] = []
offset = 0
async with httpx.AsyncClient() as client:
for _ in range(20): # hard cap: 20 pages × 100 = 2000 members
query = (
'{ collective(slug: "%s") { members(limit: 100, offset: %d) { '
"totalCount nodes { role tier { name } account { name slug type ... on Individual { email } } } } } }"
% (OC_COLLECTIVE_SLUG, offset)
)
r = await client.post(
OC_GRAPHQL_URL,
headers={"Personal-Token": OC_PERSONAL_TOKEN},
json={"query": query},
)
if r.status_code != 200:
raise RuntimeError(f"OC member fetch failed: HTTP {r.status_code}")
members = ((r.json().get("data") or {}).get("collective") or {}).get("members") or {}
total = members.get("totalCount") or 0
page = members.get("nodes") or []
nodes.extend(page)
offset += len(page)
if not page or offset >= total:
return nodes
raise RuntimeError("OC member pagination exceeded 20 pages")
_MEMBER_SET_MIN_RATIO = 0.5
def _member_set_verdict(fetched: int, local_active: int) -> bool:
"""Is this member list plausible enough to act on? (member-portal#2)
Pure decision so it can be tested without a database.
Guards the case a raise-only fail-safe cannot see: a fetch that SUCCEEDS while
returning nothing, or far less than usual. A narrowed token scope, an expired
credential with partial access, or a renamed Membership tier on Open Collective
all produce that, and the sweep's response to it is to deactivate the membership
rather than to fail. An empty collective is legitimate only when we have no
active members ourselves.
"""
if local_active <= 0:
return True
# Strictly above: a halving of the membership is implausible enough, and refusing
# costs a loud error (no action taken) where allowing it costs the membership.
return fetched > 0 and fetched > local_active * _MEMBER_SET_MIN_RATIO
def _member_set_guard(fetched: int, label: str) -> None:
"""Raise when the fetched member list is implausible, so callers fail safe."""
try:
conn = get_db()
local_active = conn.execute(
"SELECT COUNT(*) FROM members WHERE active = 1"
).fetchone()[0]
conn.close()
except Exception as e: # fresh install / unreadable DB: nothing to compare against
logger.warning("member-set guard: cannot read the local member count (%s); skipping", e)
return
if _member_set_verdict(fetched, local_active):
logger.info("member-set guard OK (%s): fetched %d, %d active locally",
label, fetched, local_active)
return
raise RuntimeError(
f"implausible member list from Open Collective ({label}): fetched {fetched} while "
f"{local_active} members are active locally. Refusing to reconcile — the sweep "
"fail-safe keeps members active instead. Check the OC token's scopes and expiry, "
"and that the Membership tier still exists (fetch_members filters on its name)."
)
async def fetch_members() -> list[dict]:
"""Return the full active-member list: email, name, slug, role, tier.
Uses the bot's personal token (admin) so emails are exposed. This is the
authoritative source of truth for reconciliation (provisioning missing
members, deactivating lapsed ones).
Membership is tied to the "Membership" TIER — a one-time donation grants
BACKER role on Open Collective but is NOT a membership. So this returns
only users whose tier name contains "membership" (case-insensitive) or
who have no tier (legacy: contributors who predate the tier system).
"""
if not OC_PERSONAL_TOKEN:
logger.warning("OC_PERSONAL_TOKEN not set; cannot fetch members")
return []
# Paginated fetch (raises on failure → caller fail-safes apply: the sweep
# skips deactivation entirely when we can't get the full member list).
nodes = await fetch_all_member_nodes()
result: list[dict] = []
for n in nodes:
role = n.get("role", "")
acct = n.get("account", {}) or {}
email = (acct.get("email") or "").strip().lower()
tier = ((n.get("tier") or {}).get("name") or "").strip()
if not email or role not in ("BACKER", "ADMIN"):
continue
# Admins (the co-op itself) always count. BACKERs must be on the
# Membership tier; no-tier BACKERs are legacy contributors from before
# tiers existed — grandfathered as members.
if role == "ADMIN" or "membership" in tier.lower() or tier == "":
result.append({
"email": email,
"name": acct.get("name") or email.split("@")[0],
"slug": acct.get("slug") or "",
"role": role,
"tier": tier,
})
# Guard before ANY caller acts on this: the sweep deactivates members missing
# from the list, so an implausible list must fail rather than propagate.
_member_set_guard(len(result), "oc members query")
return result
async def fetch_org_backers() -> dict[str, dict]:
"""Organization-backed memberships on the Membership tier.
OC Organizations never expose an email to our integration (the
`... on Individual { email }` fragment doesn't match them, and OC does not
return org emails even to collective admins), so these members are LINKED
to portal accounts via members.org_slug (POST /admin/org-link). Keyed by
org slug: {slug: {name, role, tier}}.
"""
if not OC_PERSONAL_TOKEN:
logger.warning("OC_PERSONAL_TOKEN not set; cannot fetch org members")
raise RuntimeError("OC_PERSONAL_TOKEN not set")
# Paginated fetch; RAISES on failure so the sweep's fail-safe keeps linked
# accounts active rather than deactivating them on a fetch error.
nodes = await fetch_all_member_nodes()
result: dict[str, dict] = {}
for n in nodes:
role = n.get("role", "")
acct = n.get("account", {}) or {}
if (acct.get("type") or "") != "ORGANIZATION":
continue
if role not in ("BACKER", "ADMIN"):
continue
tier = ((n.get("tier") or {}).get("name") or "").strip()
if role != "ADMIN" and "membership" not in tier.lower() and tier != "":
continue
slug = acct.get("slug") or ""
if slug:
result[slug] = {
"name": acct.get("name") or slug,
"role": role,
"tier": tier,
}
return result
def send_email(to: str, subject: str, text_body: str, html_body: str | None = None) -> bool:
"""Send an email via Cloudron's sendmail addon (SMTP).
Returns True on success, False on failure (logged, not raised — email is
best-effort and must never break the provisioning flow).
"""
if not SMTP_SERVER:
logger.warning("SMTP not configured; skipping email to %s", to)
return False
try:
msg = MIMEMultipart("alternative")
msg["From"] = f"{MAIL_FROM_NAME} <{MAIL_FROM}>"
msg["To"] = to
msg["Subject"] = subject
msg["Reply-To"] = MAIL_REPLY_TO
# Explicit Message-ID so a send is traceable in the mail system and can be
# named in a report without reading the mailbox.
msg["Message-ID"] = make_msgid(domain=(MAIL_FROM.split("@")[-1] if "@" in MAIL_FROM else "inference.coop"))
msg.attach(MIMEText(text_body, "plain", "utf-8"))
if html_body:
msg.attach(MIMEText(html_body, "html", "utf-8"))
with smtplib.SMTP(SMTP_SERVER, SMTP_PORT, timeout=15) as s:
if SMTP_USERNAME:
s.login(SMTP_USERNAME, SMTP_PASSWORD)
s.sendmail(MAIL_FROM, [to], msg.as_string())
logger.info("Sent email to %s: %s (message-id %s)", to, subject, msg["Message-ID"])
return True
except Exception as e:
logger.warning("Failed to send email to %s: %s", to, e)
return False
def welcome_email(name: str, setup_url: str) -> tuple[str, str, str]:
"""Build the welcome email (subject, text, html) for a new member.
`setup_url` is the Cloudron account-setup link (from invite_link), which
lets the member create their account directly — no OAuth step needed.
The HTML uses a centered, card-style layout matching the landing page
(cream background, forest-green button, Figtree-like system fonts).
Email HTML is table-based for client compatibility; no external assets.
"""
subject = "Welcome to the Inference Cooperative — finish your setup"
text = (
f"Hi {name},\n\n"
"Thanks for joining the Inference Cooperative!\n\n"
"To finish setting up your account and start using private AI chat, "
"click the link below to create your account:\n\n"
f"{setup_url}\n\n"
"This takes about a minute.\n\n"
"Questions? Reply to this email or write to info@inference.coop.\n\n"
"— The Inference Cooperative\n"
)
html = f"""<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"></head>
<body style="margin:0;padding:0;background:#faf8f5;font-family:-apple-system,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;">
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:#faf8f5;padding:40px 16px;">
<tr>
<td align="center">
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="max-width:520px;">
<tr>
<td align="center" style="padding-bottom:24px;">
<!-- Pine-tree circle logo (simplified) -->
<svg width="56" height="56" viewBox="0 0 130 130" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true">
<circle cx="65" cy="14" r="5" fill="#5b8c5a"/>
<circle cx="58" cy="30" r="6" fill="#7ab87a"/>
<circle cx="74" cy="30" r="4" fill="#6ba86b"/>
<circle cx="51.5" cy="46" r="6.5" fill="#5b8c5a"/>
<circle cx="65" cy="46" r="5" fill="#7ab87a"/>
<circle cx="81" cy="46" r="4" fill="#6ba86b"/>
<circle cx="44" cy="62" r="7" fill="#6ba86b"/>
<circle cx="61" cy="62" r="5.5" fill="#5b8c5a"/>
<circle cx="76" cy="62" r="4.5" fill="#7ab87a"/>
<circle cx="89.5" cy="62" r="3.5" fill="#6ba86b"/>
<circle cx="36.5" cy="78" r="7.5" fill="#5b8c5a"/>
<circle cx="54" cy="78" r="6" fill="#7ab87a"/>
<circle cx="70" cy="78" r="5" fill="#6ba86b"/>
<circle cx="84" cy="78" r="4" fill="#5b8c5a"/>
<circle cx="97.5" cy="78" r="3.5" fill="#7ab87a"/>
<circle cx="44" cy="94" r="6.5" fill="#6ba86b"/>
<circle cx="61" cy="94" r="5.5" fill="#5b8c5a"/>
<circle cx="76" cy="94" r="4.5" fill="#7ab87a"/>
<circle cx="89.5" cy="94" r="3.5" fill="#6ba86b"/>
<circle cx="51.5" cy="110" r="5.5" fill="#5b8c5a"/>
<circle cx="65" cy="110" r="4.5" fill="#7ab87a"/>
<circle cx="81" cy="110" r="3.5" fill="#6ba86b"/>
</svg>
</td>
</tr>
<tr>
<td align="center" style="background:#ffffff;border:1px solid #e8e2d8;border-radius:16px;padding:40px 32px;">
<h1 style="margin:0 0 12px;font-size:24px;font-weight:700;color:#2d3327;">Welcome, {name}</h1>
<p style="margin:0 0 8px;font-size:16px;color:#6b7a62;line-height:1.5;">Thanks for joining the <strong style="color:#2d3327;">Inference Cooperative</strong>.</p>
<p style="margin:0 0 28px;font-size:15px;color:#6b7a62;line-height:1.5;">Finish setting up your account to start using private AI chat.</p>
<a href="{setup_url}" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:10px;font-size:16px;font-weight:600;">Finish setup</a>
<p style="margin:28px 0 0;font-size:13px;color:#94a08c;line-height:1.5;">This takes about a minute.</p>
</td>
</tr>
<tr>
<td align="center" style="padding-top:24px;font-size:13px;color:#94a08c;line-height:1.6;">
Questions? Reply to this email or write to <a href="mailto:info@inference.coop" style="color:#6b7a62;">info@inference.coop</a>.<br>
<span style="color:#2d3327;">The Inference Cooperative</span> &middot; inference.coop<br>
<span style="font-size:12px;">By creating your account, you agree to our <a href="https://git.inference.coop/co-op/docs/src/branch/main/terms-of-service.md" style="color:#6b7a62;">Terms of Service</a> and <a href="https://git.inference.coop/co-op/docs/src/branch/main/privacy-policy.md" style="color:#6b7a62;">Privacy Policy</a>.</span>
</td>
</tr>
</table>
</td>
</tr>
</table>
</body>
</html>"""
return subject, text, html
async def get_active_member_emails() -> list[str]:
"""Return the list of active MEMBER emails from the portal's own database.
The local `members` table is the authoritative source of member emails
(populated during provisioning). Used for the Loomio sync and the Matrix
invite pass. GUEST accounts are excluded: they are not members — they hold
$2 credits, live in the Cloudron guests group, and must never be invited
to Loomio, Matrix or (via these lists) any member service.
"""
conn = get_db()
rows = conn.execute(
"SELECT email, account_type FROM members WHERE active = 1"
).fetchall()
conn.close()
return [r[0] for r in rows if (r[1] or "member") != "guest"]
def get_loomio_invited_at(email: str) -> str | None:
"""Return when `email` was first handed to Loomio, or None if never invited."""
conn = get_db()
row = conn.execute(
"SELECT loomio_invited_at FROM members WHERE email = ?", (email,)
).fetchone()
conn.close()
return row[0] if row else None
def mark_loomio_invited(email: str) -> None:
"""Record that `email` has been sent to Loomio at least once (idempotent).
Never overwrites an existing timestamp — the invariant is "has this email
EVER been handed to Loomio", not "when was it last".
"""
conn = get_db()
conn.execute(
"UPDATE members SET loomio_invited_at = COALESCE(loomio_invited_at, datetime('now')) "
"WHERE email = ?",
(email,),
)
conn.commit()
conn.close()
def get_member_matrix_mxid(email: str) -> str:
"""Return the member's optional extra Matrix MXID, or "" if unset."""
conn = get_db()
row = conn.execute(
"SELECT matrix_mxid FROM members WHERE email = ?", (email,)
).fetchone()
conn.close()
return (row[0] or "").strip() if row else ""
def set_member_matrix_mxid(email: str, mxid: str) -> None:
"""Set (or clear) the member's optional extra Matrix MXID."""
conn = get_db()
conn.execute(
"UPDATE members SET matrix_mxid = ? WHERE email = ?",
(mxid.strip() if mxid else "", email),
)
conn.commit()
conn.close()
def _loomio_safe_username(raw) -> str | None:
"""Sanitize a candidate into a Loomio-legal username.
Loomio accepts lowercase letters, numbers, underscores and INTERNAL hyphens
only, so a dot-containing candidate — an OAuth uid or an email local part
such as "joshua.z.tan" — must be normalized rather than passed through.
Passing it through is what made this fix fail with HTTP 422 on every sweep.
"""
u = re.sub(r"[^a-z0-9_-]", "_", (raw or "").lower()).strip("-_")
return u if len(u) >= 2 else None
def _loomio_is_autogenerated(current: str, email: str) -> bool:
"""Is this handle one Loomio generated for us?
Loomio derives a handle from the email local part with non-alphanumerics
stripped, plus a random suffix ("joshuaztan" + "xjan5rzb8j27" for
joshua.z.tan@...). Anything that is not that shape — "amarino", say — was
chosen by a person, and renaming it would be an unrequested member-visible
change. Only auto-generated handles are in scope for this fix.
"""
local = re.sub(r"[^a-z0-9]", "", (email or "").split("@")[0].lower())
cur = (current or "").lower()
return bool(local) and cur.startswith(local) and len(cur) > len(local)
async def loomio_fix_usernames(client: httpx.AsyncClient) -> dict:
"""Rename Loomio users with auto-generated handles to their Cloudron username.
Loomio's b2/memberships invite creates the account with a mangled handle
(email local-part + random suffix, e.g. danishipleydsdk4ibdihka). After
first SSO login the user carries an oauth identity whose uid IS their
Cloudron username — prefer that; fall back to the email local-part.
Uses the server (b3) API. Returns a {fixed, skipped} summary.
"""
H = {"Authorization": f"Bearer {LOOMIO_B3_KEY}", "Content-Type": "application/json"}
r = await client.get(f"{LOOMIO_BASE}/api/b3/users", headers=H)
r.raise_for_status()
users = r.json().get("users", [])
fixed, skipped = 0, 0
for u in users:
# Only Loomio's own auto-generated handles are in scope: a name a human
# chose stays, even when it differs from the Cloudron username.
if not _loomio_is_autogenerated(u.get("username") or "", u.get("email") or ""):
skipped += 1
continue
target = None
for ident in u.get("identities") or []:
if ident.get("identity_type") == "oauth" and ident.get("uid"):
target = _loomio_safe_username(ident["uid"])
if not target:
# uid unusable even after sanitizing — fall back to the email local part
target = _loomio_safe_username((u.get("email") or "").split("@")[0])
break
if not target:
target = _loomio_safe_username((u.get("email") or "").split("@")[0])
if not target or u.get("username") == target:
skipped += 1
continue
rr = await client.patch(
f"{LOOMIO_BASE}/api/b3/users/{u['id']}",
headers=H, json={"username": target},
)
if rr.status_code == 200:
fixed += 1
else:
# log the Loomio user id, never the member's email
logger.warning("Loomio username fix for user %s failed: %s %s",
u.get("id"), rr.status_code, rr.text[:120])
if fixed:
logger.info("Loomio usernames fixed: %s", fixed)
return {"fixed": fixed, "skipped": skipped}
def member_invite_rooms() -> list[str]:
"""The explicit set of rooms every active member of the co-op is invited to.
MATRIX_INVITE_ROOMS is authoritative when set; otherwise we fall back to the
co-op Space and General room. The legacy MATRIX_EXTRA_ROOMS entries are
appended (deduped) so existing configuration keeps working. This is
deliberately explicit configuration and never "every child of the Space":
operator/admin-only child rooms must not be pushed at every member.
"""
if MATRIX_INVITE_ROOMS:
rooms = list(MATRIX_INVITE_ROOMS)
else:
rooms = [MATRIX_SPACE_ID]
if MATRIX_GENERAL_ROOM:
rooms.append(MATRIX_GENERAL_ROOM)
for r in MATRIX_EXTRA_ROOMS:
if r and r not in rooms:
rooms.append(r)
seen: set[str] = set()
out: list[str] = []
for r in rooms:
if r and r not in seen:
seen.add(r)
out.append(r)
return out
async def sync_matrix_space() -> dict:
"""Invite every active member to the configured co-op Matrix rooms.
The room set comes from member_invite_rooms(): MATRIX_INVITE_ROOMS when set,
else the co-op Space + General room, plus the legacy MATRIX_EXTRA_ROOMS.
Members log in via Cloudron SSO, so their local MXID is
@<username>:inference.coop. We resolve the Cloudron username for each
active member email and invite them if not already in the space.
Best-effort: missing config just skips; failures log and continue.
"""
if not (MATRIX_BOT_TOKEN and MATRIX_SPACE_ID):
logger.info("Matrix space sync skipped (not configured)")
return {"skipped": True}
H = {"Authorization": f"Bearer {MATRIX_BOT_TOKEN}", "Content-Type": "application/json"}
async def invite(mxid: str, room: str) -> bool:
import asyncio as _aio
for attempt in range(3):
r = await client.post(
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{room}/invite",
headers=H, json={"user_id": mxid},
)
if r.status_code == 200:
return True
if r.status_code == 429:
wait = (r.json().get("retry_after_ms", 1000) / 1000) + 0.2
await _aio.sleep(wait)
continue
# 403 with 'already in room' is fine
if "already" in r.text:
return True
logger.warning("Matrix invite %s -> %s failed: %s %s",
mxid, room, r.status_code, r.text[:120])
return False
return False
async with httpx.AsyncClient(timeout=30) as client:
# current members of the space (to avoid duplicate invites)
r = await client.get(
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{MATRIX_SPACE_ID}/joined_members",
headers=H)
joined = set(r.json().get("joined", {}).keys()) if r.status_code == 200 else set()
# also pending invites
r = await client.get(
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{MATRIX_SPACE_ID}/members?membership=invite",
headers=H)
invited = {m["state_key"] for m in r.json().get("chunk", [])} if r.status_code == 200 else set()
# One identity per member: when a member sets a dashboard extra MXID,
# their auto-generated local account (@username:inference.coop) should
# NOT also sit in the Space. Track the desired set, then retire the
# local accounts of members whose dashboard MXID replaced them.
invited_count = 0
async for email, mxid in matrix_member_mxids(client):
if not mxid or mxid in joined or mxid in invited:
continue
for room in member_invite_rooms():
await invite(mxid, room)
invited_count += 1
if invited_count:
logger.info("Matrix space invites sent: %s", invited_count)
# Retire local accounts: for each member with a dashboard-set external
# MXID, kick their @username:inference.coop account if it's currently
# in the Space. Only ever touches accounts derived from that member's
# own Cloudron username — never other members' or federated accounts.
conn = get_db()
rows = conn.execute(
"SELECT email, matrix_mxid FROM members WHERE active = 1 AND matrix_mxid IS NOT NULL AND matrix_mxid != ''"
).fetchall()
conn.close()
users: dict[str, str] = {}
page = 1
async with httpx.AsyncClient(timeout=30) as uc:
while True:
r = await uc.get(
f"{CLOUDRON_API}/api/v1/users",
params={"page": page},
headers=cloudron_headers(),
)
if r.status_code != 200:
break
batch = r.json().get("users", [])
for u in batch:
if u.get("email") and u.get("username"):
users[u["email"].lower()] = u["username"]
if len(batch) < 25:
break
page += 1
retired = 0
for email, ext_mxid in rows:
username = users.get(email.lower())
if not username:
continue
local_mxid = f"@{username}:inference.coop"
if local_mxid == ext_mxid:
continue # dashboard MXID IS the local account — nothing to retire
# Retire BOTH joined and invite-state local accounts: a stale invite
# leaves the local account visible in the user directory and
# accept-able from a lingering client notification, which defeats
# the one-identity rule. Revoke invites via state-event membership
# (kick 403s on invite-state users).
import urllib.parse # local import: only needed on this retire path
# Same room set as the invite half (member_invite_rooms()) so the
# two cannot drift apart again: a retired identity must be removed
# from every room we invite members to, not just Space + General.
for room in member_invite_rooms():
enc_room = urllib.parse.quote(room, safe="")
if local_mxid in joined:
await client.post(
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{enc_room}/kick",
headers=H,
json={"user_id": local_mxid,
"reason": f"This co-op account is superseded by {ext_mxid} (set on your dashboard)"},
)
retired += 1
logger.info("Matrix sync: retired local account %s (member uses %s)", local_mxid, ext_mxid)
elif local_mxid in invited:
await client.put(
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{enc_room}/state/m.room.member/{urllib.parse.quote(local_mxid, safe='')}",
headers=H,
json={"membership": "leave",
"reason": f"This co-op account is superseded by {ext_mxid} (set on your dashboard)"},
)
retired += 1
logger.info("Matrix sync: revoked stale invite for local account %s (member uses %s)", local_mxid, ext_mxid)
if retired:
logger.info("Matrix sync: retired %s local account(s)/invite(s)", retired)
return {"invited": invited_count}
async def matrix_member_mxids(client: httpx.AsyncClient):
"""Yield (email, mxid) for each active member.
Local members log in via Cloudron SSO and Synapse names them by Cloudron
username → @<username>:inference.coop. Emails that don't resolve to a
Cloudron user (e.g. federated/external members) are skipped here but can
be added via the MATRIX_EXTRA_MXIDS env (comma list of full MXIDs).
"""
extra = [m.strip() for m in os.environ.get("MATRIX_EXTRA_MXIDS", "").split(",") if m.strip()]
seen: set[str] = set()
# Cloudron paginates /api/v1/users at 25/page — walk every page.
users: dict[str, str] = {}
page = 1
while True:
r = await client.get(
f"{CLOUDRON_API}/api/v1/users",
params={"page": page},
headers=cloudron_headers(),
)
if r.status_code != 200:
logger.warning("Cloudron users lookup page %s failed: %s", page, r.status_code)
break
batch = r.json().get("users", [])
for u in batch:
if u.get("email") and u.get("username"):
users[u["email"].lower()] = u["username"]
if len(batch) < 25:
break
page += 1
for email in await get_active_member_emails():
# A member's own extra MXID (set on the dashboard) takes precedence —
# that's the account they want in the Space. Otherwise use their local
# @username:inference.coop account.
own = get_member_matrix_mxid(email)
if own:
if own not in seen:
seen.add(own)
yield email, own
continue
username = users.get(email.lower())
if username:
mxid = f"@{username}:inference.coop"
seen.add(mxid)
yield email, mxid
for mxid in extra:
if mxid not in seen:
yield mxid, mxid
async def fetch_loomio_members(client: httpx.AsyncClient) -> dict[str, int]:
"""Return {email_lower: membership_id} for everyone in the Loomio group.
Uses the Server API (b3) so we get membership ids in one pass (the id is
what a targeted removal needs) AND a STABLE read: the b3 index is ordered by
id and paginates cleanly, whereas the User API b2 index has no ORDER BY, so
paging it can repeat or skip rows — a skipped row would make the pre-check
wrongly think an existing member is new. Reads the group's ACTIVE
memberships (pending invitees included, already-removed members excluded).
This is the live-membership half of the positive exclusion, and the ONLY
layer that reflects reality rather than our own records. It must succeed
before any invite or removal (fail-closed in sync_loomio_memberships).
"""
members: dict[str, int] = {}
offset = 0
while True:
r = await client.get(
f"{LOOMIO_BASE}/api/b3/memberships",
headers={"Authorization": f"Bearer {LOOMIO_B3_KEY}"},
params={"group_id": int(LOOMIO_GROUP_ID), "limit": 50, "offset": offset},
)
r.raise_for_status()
page = r.json().get("memberships", [])
for m in page:
email = (m.get("user_email") or "").lower()
if email:
members[email] = m.get("id")
if len(page) < 50:
break
offset += 50
return members
async def remove_loomio_membership(client: httpx.AsyncClient, membership_id: int) -> bool:
"""Revoke ONE Loomio membership by id, via the Server API (b3).
Targeted on purpose: the request names only the membership being removed, so
it carries no address of anyone who is staying and therefore cannot trigger
an invite or a group notification — unlike b2's `remove_absent`, which
requires re-sending the whole keep-list.
Idempotent: Loomio answers 404 for an id that is already gone (or never
existed); the end state we want is "not a member", so 404 counts as success
and a repeated run does not error or churn.
"""
r = await client.delete(
f"{LOOMIO_BASE}/api/b3/memberships/{membership_id}",
headers={"Authorization": f"Bearer {LOOMIO_B3_KEY}"},
)
if r.status_code in (200, 204, 404):
return True
logger.warning(
"Loomio remove of membership %s failed: %s %s",
membership_id, r.status_code, r.text[:200],
)
return False
async def sync_loomio_memberships() -> None:
"""Reconcile the Loomio group with the current ACTIVE-member list.
TWO jobs with DIFFERENT guards — a deactivation must still propagate:
1. ADD (invite) — a member is invited only if they are BOTH not already in
the group (live pre-check) AND never handed to Loomio before
(members.loomio_invited_at). Exactly once, ever. This is the dedupe
guard, and it applies to INVITATIONS ONLY.
2. REMOVE — a member no longer active is revoked so a lapse propagates to
Loomio. The removal path is targeted (b3 DELETE by membership id) and
sends nothing for anyone who is staying, so it cannot re-invite or
notify a present member. Operator accounts (LOOMIO_ALWAYS_KEEP) are
never removed.
Guards (incident 2026-11-04: a live run emailed Loomio invites to the whole
membership):
* The invite payload contains ONLY addresses Loomio has never seen and we
have never handed over — an existing member is skipped SILENTLY, no API
call about them at all.
* FAIL CLOSED — if the live group roster cannot be read, NEITHER phase
runs. We never hand Loomio a list we could not check.
Loomio's b2 invite has no "silent" flag, so the only way to guarantee no
invite is to send an address exactly once, ever — hence the record.
"""
if not LOOMIO_SYNC_ENABLED:
logger.info("Loomio sync disabled (LOOMIO_SYNC_ENABLED=0); skipping")
return
if not LOOMIO_GROUP_ID:
logger.warning("Loomio not configured; skipping membership sync")
return
# The set that decides who STAYS in the group: every active member plus the
# operator allowlist. Computed from the UNFILTERED active list on purpose —
# an active member whose Cloudron account is not yet activated is still a
# member and must NOT be removed from Loomio (only their invite is deferred).
keep_emails = {e.lower() for e in await get_active_member_emails()}
keep_emails |= {e.lower() for e in LOOMIO_ALWAYS_KEEP}
# Invite candidates are additionally filtered to activated accounts so a
# Loomio invite email never precedes the member's own account setup.
invite_emails = await get_active_member_emails()
for keep in LOOMIO_ALWAYS_KEEP:
if keep not in invite_emails:
invite_emails.append(keep)
invite_emails = await filter_loomio_activated(invite_emails)
if not LOOMIO_B3_KEY:
# The live roster (and the removal path) needs the Server API key. Without
# it we cannot verify reality, so we do nothing rather than risk a
# blind invite.
logger.warning("Loomio sync: LOOMIO_B3_KEY unset; cannot read group roster (fail-closed)")
return
async with httpx.AsyncClient() as client:
# FAIL CLOSED: never act on a roster we could not read.
try:
current = await fetch_loomio_members(client)
except Exception as e:
logger.warning(
"Loomio sync: could not read current group members; skipping "
"(fail-closed): %s", e,
)
return
# --- REMOVE: reconcile the group to the current active set. ---
removed = 0
for email, mid in current.items():
if email in keep_emails or mid is None:
continue
if await remove_loomio_membership(client, mid):
removed += 1
logger.info("Loomio sync: removed lapsed membership %s", mid)
if removed:
logger.info("Loomio sync: %d lapsed membership(s) removed", removed)
# --- ADD: only addresses Loomio has never seen AND we never handed over. ---
fresh = [
e for e in invite_emails
if e.lower() not in current and not get_loomio_invited_at(e)
]
if not fresh:
logger.info(
"Loomio sync: nobody new (all already in Loomio or already "
"invited); nothing to invite"
)
elif not LOOMIO_API_KEY:
logger.warning(
"Loomio sync: %d member(s) to invite but LOOMIO_API_KEY unset; "
"skipping invites", len(fresh),
)
else:
r = await client.post(
f"{LOOMIO_BASE}/api/b2/memberships",
headers={"Authorization": f"Bearer {LOOMIO_API_KEY}"},
json={"group_id": int(LOOMIO_GROUP_ID), "emails": fresh},
)
if r.status_code != 200:
logger.warning("Loomio membership sync failed: %s %s", r.status_code, r.text[:200])
else:
added = r.json().get("added_emails", []) or []
# Record EVERY address we sent, so it is never sent again — even
# if Loomio didn't add it, or the membership is later revoked. A
# revoked member is NOT a never-invited member, and must not be
# re-invited.
for e in fresh:
mark_loomio_invited(e)
logger.info("Loomio sync: first-time invites sent (%d): %s", len(added), added)
# Fix auto-generated usernames (email+randomsuffix) → Cloudron username.
# Best-effort: no B3 key or an API failure just logs and moves on.
if LOOMIO_B3_KEY:
try:
await loomio_fix_usernames(client)
except Exception as e:
logger.warning("Loomio username fix pass failed: %s", e)
async def filter_loomio_activated(emails: list[str]) -> list[str]:
"""Return the subset of `emails` whose Cloudron account is set up, plus any
operator accounts (LOOMIO_ALWAYS_KEEP), which must always pass through.
Fetches the Cloudron user list once and builds an inviteAccepted set keyed
by email. Members who are provisioned but not yet activated are dropped so
Loomio doesn't email them an invite before they've set up their account.
"""
activated: set[str] = set()
try:
async with httpx.AsyncClient(timeout=15.0) as client:
users = await cloudron_fetch_all_users(client)
if users is None:
raise RuntimeError("cloudron users fetch failed (page 1)")
for u in users:
if u.get("inviteAccepted") and u.get("email"):
activated.add(u["email"])
except Exception as e:
# If we can't determine activation state, fail open (invite everyone)
# rather than silently dropping members from Loomio.
logger.warning("filter_loomio_activated: cloudron users fetch failed: %s", e)
return emails
keep = set(LOOMIO_ALWAYS_KEEP)
return [e for e in emails if e in keep or e in activated]
def store_member(email: str, key_token: str, cloudron_user_id: str, slug: str = "",
account_type: str | None = None) -> None:
conn = get_db()
# Preserve an existing slug if the caller passed none — re-provisioning an
# already-provisioned member must not wipe their OC slug (the deactivation
# webhook maps slugs back to members).
if not slug:
row = conn.execute("SELECT slug FROM members WHERE email = ?", (email,)).fetchone()
if row and row[0]:
slug = row[0]
# account_type: None (every normal path) preserves whatever the row holds —
# re-provisioning a guest via the member pipeline must not un-guest it. Only
# an explicit value (provision_guest_member passes 'guest') writes the column.
conn.execute(
"INSERT INTO members (email, key_token, cloudron_user_id, slug, active, account_type) "
"VALUES (?, ?, ?, ?, 1, COALESCE(?, 'member')) "
"ON CONFLICT(email) DO UPDATE SET key_token=excluded.key_token, "
"cloudron_user_id=excluded.cloudron_user_id, slug=excluded.slug, active=1, "
"account_type=CASE WHEN excluded.account_type='guest' THEN 'guest' "
"ELSE COALESCE(members.account_type, 'member') END",
(email, encrypt_key(key_token), cloudron_user_id, slug, account_type),
)
conn.commit()
conn.close()
def get_member_key(email: str) -> str | None:
conn = get_db()
row = conn.execute(
"SELECT key_token FROM members WHERE email = ? AND active = 1", (email,)
).fetchone()
conn.close()
return decrypt_key(row[0]) if row else None
def get_stored_key(email: str) -> str | None:
"""Return the stored LiteLLM key token regardless of active status.
Unlike get_member_key (which filters active=1), this returns the key even
for a lapsed member being reactivated, so provisioning can reuse it rather
than mint a duplicate alias (LiteLLM rejects a duplicate `member:<email>`
alias with 400).
"""
conn = get_db()
row = conn.execute(
"SELECT key_token FROM members WHERE email = ?", (email,)
).fetchone()
conn.close()
return (decrypt_key(row[0]) if row and row[0] else None)
def touch_welcome_sent(email: str) -> None:
"""Record that a welcome/invite email was just sent for this member."""
conn = get_db()
conn.execute(
"UPDATE members SET welcome_sent_at = datetime('now') WHERE email = ?",
(email,),
)
conn.commit()
conn.close()
def get_member_by_slug(slug: str) -> dict | None:
"""Look up a member (email, key_token, cloudron_user_id) by their OC slug."""
conn = get_db()
row = conn.execute(
"SELECT email, key_token, cloudron_user_id FROM members WHERE slug = ?", (slug,)
).fetchone()
conn.close()
if not row:
return None
return {"email": row[0], "key_token": decrypt_key(row[1]), "cloudron_user_id": row[2]}
def deactivate_member(email: str) -> str | None:
"""Mark member inactive and return their DECRYPTED key token (for deletion)."""
conn = get_db()
row = conn.execute(
"SELECT key_token FROM members WHERE email = ?", (email,)
).fetchone()
conn.execute("UPDATE members SET active = 0 WHERE email = ?", (email,))
conn.commit()
conn.close()
return decrypt_key(row[0]) if row else None
# --- Helpers ---
def cloudron_headers() -> dict:
return {"Authorization": f"Bearer {CLOUDRON_TOKEN}"}
# M6 (audit 2026-11-05): a single-page `?per_page=100` fetch silently truncates
# once the box passes 100 users (the membership sits at the threshold). Both
# walkers below page through /api/v1/users like the Matrix paths already do,
# capped sanely for Cloudron (50 pages x 100 = 5000 users).
_CLOUDRON_USERS_PAGE_CAP = 50
async def cloudron_fetch_all_users(client: httpx.AsyncClient) -> list[dict] | None:
"""Every Cloudron user, walking /api/v1/users page by page (async callers).
Returns None only when page 1 is unresolvable (caller keeps its existing
failure behavior); a failure on a LATER page keeps what was read, which is
strictly more than the truncated single-page fetch returned before.
"""
users: list[dict] = []
for page in range(1, _CLOUDRON_USERS_PAGE_CAP + 1):
r = await client.get(
f"{CLOUDRON_API}/api/v1/users",
params={"page": page, "per_page": 100},
headers=cloudron_headers(),
)
if r.status_code != 200:
if page == 1:
logger.warning("Cloudron users fetch page 1 failed: %s", r.status_code)
return None
logger.warning("Cloudron users fetch page %s failed: %s", page, r.status_code)
break
batch = r.json().get("users", [])
users.extend(batch)
if len(batch) < 100:
break
else:
logger.warning("Cloudron users fetch hit the %s-page cap; list truncated",
_CLOUDRON_USERS_PAGE_CAP)
return users
def cloudron_fetch_all_users_sync() -> list[dict]:
"""Every Cloudron user for the sync (urllib) call sites, same pagination.
Raises on a page-1 failure (the call sites error the request); later-page
failures keep what was read.
"""
import urllib.request
users: list[dict] = []
for page in range(1, _CLOUDRON_USERS_PAGE_CAP + 1):
req = urllib.request.Request(
f"{CLOUDRON_API}/api/v1/users?page={page}&per_page=100",
headers=cloudron_headers(),
)
try:
with urllib.request.urlopen(req, timeout=15) as r:
batch = json.loads(r.read().decode()).get("users", [])
except Exception:
if page == 1:
raise
logger.warning("Cloudron users fetch page %s failed", page, exc_info=True)
break
users.extend(batch)
if len(batch) < 100:
break
else:
logger.warning("Cloudron users fetch hit the %s-page cap; list truncated",
_CLOUDRON_USERS_PAGE_CAP)
return users
async def cloudron_create_user(email: str, name: str) -> str:
"""Create (or return existing) Cloudron user, assigned to the members group.
Verified user shape (from live API):
{id, username, email, fallbackEmail, displayName, role, active, groupIds}
Roles: "owner", "admin", "user".
Group assignment is a SEPARATE call: PUT /api/v1/users/:userId/groups
with body {"groupIds": [...]}.
"""
async with httpx.AsyncClient() as client:
# Check if user exists (walks every page — a single-page fetch
# truncated past 100 users and let duplicate users get created).
users = await cloudron_fetch_all_users(client)
if users is None:
raise RuntimeError("cloudron users fetch failed; refusing to create a duplicate")
for u in users:
if u.get("email") == email:
# If the existing user has no display name (e.g. created with a
# placeholder), fill it in. Otherwise leave it alone — the user
# may have set their own name during account setup.
if not u.get("displayName"):
await cloudron_update_profile(u["id"], email, name)
return u["id"]
# Create user (role "user" = regular member). Only email is required;
# Cloudron lets the user choose their own username during account setup.
r = await client.post(
f"{CLOUDRON_API}/api/v1/users",
headers=cloudron_headers(),
json={
"email": email,
"displayName": name,
"role": "user",
"active": True,
},
)
r.raise_for_status()
return r.json()["id"]
async def cloudron_set_group(user_id: str, group_id: str | None = None) -> None:
"""Assign a user to a group (defaults to the members group)."""
gid = group_id or MEMBERS_GROUP_ID
if not gid:
logger.warning("No group id; skipping group assignment")
return
async with httpx.AsyncClient() as client:
r = await client.put(
f"{CLOUDRON_API}/api/v1/users/{user_id}/groups",
headers=cloudron_headers(),
json={"groupIds": [gid]},
)
r.raise_for_status()
async def cloudron_set_active(user_id: str, active: bool) -> None:
async with httpx.AsyncClient() as client:
r = await client.put(
f"{CLOUDRON_API}/api/v1/users/{user_id}/active",
headers=cloudron_headers(),
json={"active": active},
)
r.raise_for_status()
async def owui_add_to_members_group(cloudron_user_id: str) -> bool:
"""Add a (new) member to Open WebUI's `members` group by their Cloudron uid.
Open WebUI authenticates via Cloudron SSO and stores users keyed by the SSO
`sub` — which is the Cloudron user id (uid-…). The shared docs-RAG knowledge
collection is granted read to the `members` group, so group membership is
what makes the docs visible. Called at provisioning time so a new member's
FIRST chat session already has the docs (the nightly refresh sync is too
slow for that).
Best-effort: any failure logs a warning and returns False — provisioning
must not fail over this; the nightly refresh self-heals within a day.
"""
if not OWUI_API_KEY:
logger.warning("OWUI_API_KEY not set; skipping Open WebUI group add for %s", cloudron_user_id)
return False
try:
async with httpx.AsyncClient(timeout=15.0) as client:
# find the OWUI user (SSO `sub` = Cloudron uid); if they've never
# logged in, the user row may not exist yet — the nightly sync
# covers it, and group-add by uid also works after first login.
r = await client.get(
f"{OWUI_BASE}/api/v1/users/",
headers={"Authorization": f"Bearer {OWUI_API_KEY}"},
params={"query": cloudron_user_id},
)
if r.status_code != 200:
logger.warning("OWUI user lookup failed: %s %s", r.status_code, r.text[:150])
return False
payload = r.json() or {}
users = payload.get("users", []) if isinstance(payload, dict) else (payload or [])
owui_uid = None
for u in users:
if u.get("id") == cloudron_user_id:
owui_uid = u["id"]
break
if not owui_uid:
# user hasn't logged in yet — they'll be added to the group by
# the nightly sync after first login. Not an error.
logger.info("OWUI: no user row yet for %s (not logged in); nightly sync will add", cloudron_user_id)
return False
r = await client.post(
f"{OWUI_BASE}/api/v1/groups/id/{OWUI_MEMBERS_GROUP_ID}/users/add",
headers={"Authorization": f"Bearer {OWUI_API_KEY}"},
json={"user_ids": [owui_uid]},
)
if r.status_code == 200:
logger.info("OWUI: added %s to members group", cloudron_user_id)
return True
# 400 "already in group" is fine
logger.warning("OWUI group add for %s: %s %s", cloudron_user_id, r.status_code, r.text[:150])
return False
except Exception as e:
logger.warning("OWUI group add failed for %s: %s", cloudron_user_id, e)
return False
async def cloudron_update_profile(user_id: str, email: str, display_name: str, fallback_email: str = "") -> None:
"""Update a user's profile (display name, email) after creation.
Uses POST /api/v1/users/:userId/profile (the Cloudron "Update user" route).
Needed because cloudron_create_user() only sets displayName at creation and
returns early if the user already exists — so corrections require this.
"""
async with httpx.AsyncClient() as client:
r = await client.post(
f"{CLOUDRON_API}/api/v1/users/{user_id}/profile",
headers=cloudron_headers(),
json={"email": email, "displayName": display_name, "fallbackEmail": fallback_email},
)
r.raise_for_status()
async def cloudron_get_invite_link(user_id: str) -> str:
"""Return the account-setup link WITHOUT sending Cloudron's own email.
We use this so we can send our own branded email (with the setup link
embedded) instead of Cloudron's default invite email.
"""
async with httpx.AsyncClient() as client:
r = await client.get(
f"{CLOUDRON_API}/api/v1/users/{user_id}/invite_link",
headers=cloudron_headers(),
)
r.raise_for_status()
return r.json().get("inviteLink", "")
async def cloudron_is_activated(user_id: str) -> bool:
"""Return True if the user has completed account setup (inviteAccepted).
`inviteAccepted` flips true once the member clicks their setup link and
chooses a username/password. Until then they're provisioned but not able to
log in — the state we re-invite for.
"""
async with httpx.AsyncClient() as client:
users = await cloudron_fetch_all_users(client)
if users is None:
raise RuntimeError("cloudron users fetch failed")
for u in users:
if u.get("id") == user_id:
return bool(u.get("inviteAccepted"))
return False
async def provision_member(email: str, name: str, slug: str = "") -> str:
"""Provision a member end-to-end and send our own welcome email.
Creates the Cloudron user (members group), issues a LiteLLM key under the
member's team (budget = the member's current balance, not a hardcoded $15),
stores the mapping, and sends our branded welcome email containing the
Cloudron account-setup link (instead of Cloudron's default invite email).
Idempotent: if the member already has a stored LiteLLM key, reuse it rather
than minting a new one (LiteLLM rejects a duplicate `member:<email>` alias
with 400 — the bug that previously made re-provisioning of existing members
fail). Re-sending this way also re-sends the welcome email, so it doubles
as the manual "re-invite" path.
GUEST accounts: every provisioning entry point (webhook, sweep,
/admin/provision) funnels through here. When a guest's OC identity appears
as a live backer, this call CONVERTS the account instead of provisioning —
the automatic guest→member transition.
"""
if is_guest_account(email):
return await convert_guest_to_member(email, name, slug)
user_id = await cloudron_create_user(email, name)
await cloudron_set_group(user_id)
await cloudron_set_active(user_id, True)
balance = get_member_balance(email)
team_id = await litellm_get_or_create_team(email, balance)
# Pre-provision the member into Open WebUI's docs-RAG access group so their
# FIRST chat session has the co-op docs. Best-effort (most new members
# haven't logged in yet, so the OWUI user row doesn't exist — the nightly
# refresh catches them after first login).
try:
await owui_add_to_members_group(user_id)
except Exception as e:
logger.warning("OWUI members-group add skipped for %s: %s", email, e)
# Reuse an existing key if we already have one (the sk- value is stored in
# our DB at creation time and is the authoritative token for this member's
# chat key). Only mint a fresh key when there is none.
key_token = get_stored_key(email)
if not key_token:
key_token = await litellm_create_key(email, balance, team_id)
store_member(email, key_token, user_id, slug)
# Send our own welcome email with the account-setup link (no OAuth step).
setup_url = await cloudron_get_invite_link(user_id)
subject, text, html = welcome_email(name, setup_url)
send_email(email, subject, text, html)
touch_welcome_sent(email)
# Subscribe to the member newsletter (single opt-in; members consented by joining).
await listmonk_sync(email, subscribe=True)
# Auto-grant any HELD credit packs for this member (they pre-purchased
# credits before joining — apply them now as part of provisioning).
granted = 0.0
try:
conn = get_db()
granted = grant_held_credits(conn, email, slug)
conn.close()
if granted > 0:
await sync_team_budget_from_balance(email)
logger.info("Granted %.2f held credits to new member %s (balance %s)",
granted, email, get_credit_balance(email))
except Exception as e:
logger.warning("Held-credit grant for %s failed: %s", email, e)
await sync_loomio_memberships()
await sync_matrix_space()
logger.info("Provisioned member %s (user_id=%s)", email, user_id)
return user_id
# --- Guest accounts (operator feature, 2026-11-08) --------------------------- #
# A member gives a friend access with $2 of credits: Cloudron user in the
# guests group (chat + dashboard only), a one-time LiteLLM budget with NO
# monthly reset, the portal row marked account_type='guest', the guest welcome
# email, and the newsletter subscription. NO Loomio/Matrix invites. See the
# constants block for the lapse rules (credit exhaustion + GUEST_DAYS time box).
def guest_invitation_email(name: str, setup_url: str, inviter_name: str = "") -> tuple[str, str, str]:
"""Build the guest INVITATION email — the one email a guest sends.
This is the guest-time email AND the guest welcome in one: the recipient is
invited in (the inviter named), told what the guest gives ($2 of credits,
how long access runs), and given the account-setup link. The guest welcome
content lives here rather than in a second email — one coherent invitation,
not two mails at once. Same branded shell as welcome_email(): cream
background, white card, forest-green button, pine mark, no external
assets. No Loomio/Matrix — guests receive no member invitations.
"""
subject = "You've been invited to the Inference Cooperative"
# Neutral greeting: no name is guessed from the email — the guest chooses
# their own display name during Cloudron account setup.
if name:
greet = f"Hi {name},"
h1 = f"You're invited, {name}"
else:
greet = "Hi there,"
h1 = "You're invited"
if inviter_name:
invited_line = f"{inviter_name} has invited you to the Inference Cooperative"
html_invited = f"<strong style=\"color:#2d3327;\">{inviter_name}</strong> has invited you to the <strong style=\"color:#2d3327;\">Inference Cooperative</strong>"
else:
invited_line = "A member has invited you to the Inference Cooperative"
html_invited = "A member has invited you to the <strong style=\"color:#2d3327;\">Inference Cooperative</strong>"
text = (
f"{greet}\n\n"
f"{invited_line}.\n\n"
"It's a member-governed AI cooperative that runs open models.\n\n"
"Your invitation comes with $2 of credits for the co-op's chat and\n"
f"API. Access runs for {int(GUEST_DAYS)} days from today; the credits\n"
"themselves don't expire early.\n\n"
"To accept, create your account — it takes about a minute:\n\n"
f"{setup_url}\n\n"
"If you like it, become a member: pay on a sliding scale, and\n"
"receive a $15/month allowance, plus optional credit packs — plus a\n"
"say in how it's run.\n\n"
f"{BECOME_MEMBER_URL}\n\n"
"You're also on our low-volume newsletter — unsubscribe anytime\n"
"from any issue.\n\n"
"Questions? Just reply to this email — it reaches the co-op directly.\n\n"
"— The Inference Cooperative\n"
)
html = f"""<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"></head>
<body style="margin:0;padding:0;background:#faf8f5;font-family:-apple-system,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;">
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:#faf8f5;padding:40px 16px;">
<tr>
<td align="center">
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="max-width:520px;">
<tr>
<td align="center" style="padding-bottom:24px;">
{PINE_LOGO_SVG}
</td>
</tr>
<tr>
<td align="center" style="background:#ffffff;border:1px solid #e8e2d8;border-radius:16px;padding:40px 32px;">
<h1 style="margin:0 0 12px;font-size:24px;font-weight:700;color:#2d3327;">{h1}</h1>
<p style="margin:0 0 8px;font-size:16px;color:#6b7a62;line-height:1.5;">{html_invited} — a member-governed AI cooperative running open models.</p>
<p style="margin:0 0 24px;font-size:15px;color:#6b7a62;line-height:1.5;">Your invitation comes with <strong style="color:#2d3327;">$2 of credits</strong> for the co-op's chat and API. Access runs for {int(GUEST_DAYS)} days from today; the credits themselves don't expire early.</p>
<a href="{setup_url}" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:10px;font-size:16px;font-weight:600;">Create your account</a>
<p style="margin:14px 0 28px;font-size:13px;color:#94a08c;line-height:1.5;">This takes about a minute.</p>
<p style="margin:0 0 10px;font-size:15px;color:#2d3327;line-height:1.5;"><strong>If you like it, become a member.</strong></p>
<p style="margin:0 0 16px;font-size:14px;color:#6b7a62;line-height:1.5;">Pay on a sliding scale, and receive a $15/month allowance, plus optional credit packs — plus a say in how it's run.</p>
<a href="{BECOME_MEMBER_URL}" style="display:inline-block;background:#eaf3ea;color:#2f5c30;text-decoration:none;padding:12px 30px;border-radius:10px;font-size:15px;font-weight:600;border:1px solid #cfe0cf;">Become a member</a>
<p style="margin:24px 0 0;font-size:13px;color:#94a08c;line-height:1.5;">You're also on our low-volume newsletter — unsubscribe anytime from any issue.</p>
</td>
</tr>
<tr>
<td align="center" style="padding-top:24px;font-size:13px;color:#94a08c;line-height:1.6;">
Questions? Just reply to this email — it reaches the co-op directly, or write to <a href="mailto:info@inference.coop" style="color:#6b7a62;">info@inference.coop</a>.<br>
<span style="color:#2d3327;">The Inference Cooperative</span> &middot; inference.coop<br>
<span style="font-size:12px;">By creating your account, you agree to our <a href="https://git.inference.coop/co-op/docs/src/branch/main/terms-of-service.md" style="color:#6b7a62;">Terms of Service</a> and <a href="https://git.inference.coop/co-op/docs/src/branch/main/privacy-policy.md" style="color:#6b7a62;">Privacy Policy</a>.</span>
</td>
</tr>
</table>
</td>
</tr>
</table>
</body>
</html>"""
return subject, text, html
def guest_member_congrats_email(name: str, setup_url: str | None) -> tuple[str, str, str]:
"""Short congratulations email for a guest account converting to a member.
Their account, login and any unspent guest credit carry over unchanged;
they gain the monthly allowance and the member services (Loomio, Matrix
invites follow from the normal syncs).
"""
subject = "Welcome to the Inference Cooperative — you're a member now!"
setup_line = ""
if setup_url:
setup_line = (f"Your account isn't set up yet — one minute and you're in:\n\n"
f"{setup_url}\n\n")
text = (
f"Hi {name},\n\n"
"Great news: your guest account is now a full membership. Nothing about\n"
"your login, dashboard or API keys changes — and any guest credit you\n"
"hadn't spent yet stays on your account on top of your allowance.\n\n"
"As a member you now get:\n"
" • a $15 monthly allowance (renews on your join day)\n"
" • our co-op forum (Loomio) and chat space (Matrix) — invites on the way\n"
" • a say in how the co-op is run\n\n"
f"{setup_line}"
f"Your dashboard: https://dashboard.inference.coop\n\n"
"Questions? Reply to this email — it reaches the co-op directly.\n\n"
"— The Inference Cooperative\n"
)
html = f"""<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"></head>
<body style="margin:0;padding:0;background:#faf8f5;font-family:-apple-system,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;">
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:#faf8f5;padding:40px 16px;">
<tr><td align="center">
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="max-width:520px;">
<tr><td align="center" style="background:#ffffff;border:1px solid #e8e2d8;border-radius:16px;padding:40px 32px;">
<h1 style="margin:0 0 12px;font-size:24px;font-weight:700;color:#2d3327;">You're a member now, {name}</h1>
<p style="margin:0 0 16px;font-size:16px;color:#6b7a62;line-height:1.5;">Your guest account has become a full membership. Nothing about your login, dashboard or API keys changes — <strong style="color:#2d3327;">any unspent guest credit stays on your account</strong>, on top of your new $15 monthly allowance.</p>
<p style="margin:0 0 24px;font-size:15px;color:#6b7a62;line-height:1.5;">Invites to the co-op forum (Loomio) and chat space (Matrix) are on their way.</p>
{'<a href="' + setup_url + '" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:10px;font-size:16px;font-weight:600;">Finish account setup</a>' if setup_url else '<a href="https://dashboard.inference.coop" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:10px;font-size:16px;font-weight:600;">Open your dashboard</a>'}
<p style="margin:24px 0 0;font-size:13px;color:#94a08c;line-height:1.5;">Questions? Reply to this email — it reaches the co-op directly.</p>
</td></tr>
<tr><td align="center" style="padding-top:24px;font-size:13px;color:#94a08c;line-height:1.6;">
<span style="color:#2d3327;">The Inference Cooperative</span> &middot; inference.coop
</td></tr>
</table>
</td></tr>
</table>
</body>
</html>"""
return subject, text, html
def guest_limits_email(name: str, reason: str) -> tuple[str, str, str]:
"""Build the reaching-limits email for a guest account hitting its limit.
Sent ONCE per guest when the sweep deactivates the guest — either the $2
of credits are spent or the GUEST_DAYS time box has run out. Says plainly:
the guest access has ended, the account and data are kept, and becoming a
member ($15/month via Open Collective) restores access with a full
monthly allowance and the community spaces. Same shell as the other guest
emails; matter-of-fact and welcoming — no guilt, no upsell.
"""
subject = "Your guest access has ended — become a member to continue"
if reason == "depleted":
ended = f"Your ${GUEST_BUDGET:g} of guest credits are spent."
else:
ended = f"The guest's {int(GUEST_DAYS)} days are up."
text = (
f"Hi {name},\n\n"
f"{ended} Your guest access to the Inference Cooperative has ended.\n\n"
"Your account and data are kept. If you'd like to continue, become a\n"
"member: $15/month through Open Collective restores full access, with\n"
"a full monthly allowance and the co-op's community spaces.\n\n"
f"{BECOME_MEMBER_URL}\n\n"
"Questions? Just reply to this email — it reaches the co-op directly.\n\n"
"— The Inference Cooperative\n"
)
html = f"""<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"></head>
<body style="margin:0;padding:0;background:#faf8f5;font-family:-apple-system,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;">
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:#faf8f5;padding:40px 16px;">
<tr><td align="center">
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="max-width:520px;">
<tr><td align="center" style="padding-bottom:24px;">
{PINE_LOGO_SVG}
</td></tr>
<tr><td align="center" style="background:#ffffff;border:1px solid #e8e2d8;border-radius:16px;padding:40px 32px;">
<h1 style="margin:0 0 12px;font-size:24px;font-weight:700;color:#2d3327;">Your guest access has ended</h1>
<p style="margin:0 0 16px;font-size:16px;color:#6b7a62;line-height:1.5;">{ended} Your guest access to the <strong style="color:#2d3327;">Inference Cooperative</strong> has ended.</p>
<p style="margin:0 0 24px;font-size:15px;color:#6b7a62;line-height:1.5;"><strong style="color:#2d3327;">Your account and data are kept.</strong> If you'd like to continue, become a member — $15/month through Open Collective restores full access, with a full monthly allowance and the co-op's community spaces.</p>
<a href="{BECOME_MEMBER_URL}" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:10px;font-size:16px;font-weight:600;">Become a member</a>
<p style="margin:24px 0 0;font-size:13px;color:#94a08c;line-height:1.5;">Questions? Reply to this email — it reaches the co-op directly.</p>
</td></tr>
<tr><td align="center" style="padding-top:24px;font-size:13px;color:#94a08c;line-height:1.6;">
<span style="color:#2d3327;">The Inference Cooperative</span> &middot; inference.coop
</td></tr>
</table>
</td></tr>
</table>
</body>
</html>"""
return subject, text, html
async def provision_guest_member(
email: str,
name: str,
inviter_email: str,
admin: bool = False,
inviter_display: str | None = None,
) -> str:
"""Provision a GUEST account end-to-end (operator guest access feature).
A guest gives a friend $2 of credits with no Open Collective membership:
* Cloudron user in the GUESTS group (chat + dashboard only — the two
apps' access restrictions list both groups);
* a LiteLLM team hard-capped at GUEST_BUDGET with NO monthly reset
(budget_duration None — the cap is the guest's whole life);
* the portal row marked account_type='guest' with the $2 held as
non-expiring credit (the credits model) and guest_expires_at set;
* the guest INVITATION email (inviter named, the $2 credits, the
setup link — the invitation and the guest welcome in one) and the
newsletter subscription;
* a guests-ledger row recording inviter → recipient;
* NO Loomio/Matrix invites, no docs-RAG members group.
Reuses provision_member's internals (user create, team, key, store) rather
than duplicating them. Guards (guest_provision_guard) enforce one guest per
inviter per 30 days and one guest per recipient ever — except on the
admin path (admin=True), where the inviter is the co-op's admin account
and the member's monthly cap does not apply.
inviter_display overrides the invitation email's inviter line (the admin
path passes a co-op label instead of resolving the inviter's member slug).
"""
conn = get_db()
refusal = guest_provision_guard(conn, inviter_email, email, admin=admin)
conn.close()
if refusal:
raise HTTPException(429, refusal)
if not GUESTS_GROUP_ID:
raise HTTPException(503, "GUESTS_GROUP_ID is not configured on the portal app")
user_id = await cloudron_create_user(email, name)
await cloudron_set_group(user_id, GUESTS_GROUP_ID)
await cloudron_set_active(user_id, True)
# One-time $2 team: budget_duration None means LiteLLM never resets it.
team_id = await litellm_get_or_create_team(email, GUEST_BUDGET, budget_duration=None)
key_token = get_stored_key(email)
if not key_token:
key_token = await litellm_create_key(email, GUEST_BUDGET, team_id)
# The row FIRST (the credits model updates a row that must exist), then
# the guest credit — non-expiring, ledger-audited, deduped on the guest
# reference.
store_member(email, key_token, user_id, "", account_type="guest")
guest_ref = f"guest:{email}"
if not credit_reference_exists(guest_ref):
add_credits(email, GUEST_BUDGET,
reason="guest account — $2 credit from a member",
reference=guest_ref)
# Guest rows hold NO allowance (balance 0 — the $2 is all credit) and
# anchor their join day at the guest date (kept if they later convert).
conn = get_db()
conn.execute(
"UPDATE members SET balance = 0.0, "
"guest_expires_at = datetime('now', ?), "
"anchor_day = COALESCE(anchor_day, ?) "
"WHERE email = ?",
(f"+{int(GUEST_DAYS)} days", _clamp_anchor_day(datetime.now(timezone.utc).day), email),
)
conn.commit()
conn.close()
await sync_team_budget_from_balance(email) # cap = allowance(0) + credits(5)
setup_url = await cloudron_get_invite_link(user_id)
# Name the inviter in the invitation: their member slug (the username the
# co-op knows them by — the members table has no display-name column).
# The admin path passes inviter_display instead (a co-op label — the
# invitation comes from the co-op, not from a member).
inviter_name = ""
if inviter_display is not None:
inviter_name = inviter_display
else:
conn = get_db()
row = conn.execute(
"SELECT slug FROM members WHERE email = ?", (inviter_email.strip().lower(),)
).fetchone()
conn.close()
if row and row[0]:
inviter_name = row[0]
# ONE email at guest time: the invitation is also the guest welcome (merged,
# not two mails to the same person at once).
subject, text, html = guest_invitation_email(name, setup_url, inviter_name)
send_email(email, subject, text, html)
touch_welcome_sent(email)
# Guests receive the newsletter (same pattern as members) — and nothing
# else: no Loomio/Matrix sync, no docs-RAG group.
await listmonk_sync(email, subscribe=True)
conn = get_db()
conn.execute(
"INSERT INTO guests (inviter_email, recipient_email, oc_reference, guest_expires_at, status) "
"VALUES (?, ?, NULL, (SELECT guest_expires_at FROM members WHERE email = ?), 'active')",
(inviter_email.strip().lower(), email, email),
)
conn.commit()
conn.close()
logger.info("Provisioned GUEST account %s (user_id=%s, inviter=%s, budget=%.2f, "
"expires per GUEST_DAYS=%d)", email, user_id, inviter_email,
GUEST_BUDGET, GUEST_DAYS)
return user_id
async def convert_guest_to_member(email: str, name: str, slug: str = "") -> str:
"""Convert a GUEST account into a full member (the automatic transition).
Fires when the guest's OC identity appears as a live BACKER/ADMIN with the
SAME email (reconcile_memberships or the webhook) — or by the operator via
POST /admin/guest-convert when the contribution arrived under a different
OC email (that path records members.oc_email so later renewals match).
What converts:
* the Cloudron user moves guests → members (regaining Loomio/Matrix
eligibility through the normal syncs);
* the LiteLLM cap lifts from the $2 guest to the member allowance +
credits — any UNSPENT guest credit stays (it already lives in the
non-expiring credit_balance bucket);
* the portal row becomes account_type='member' with the standard
MEMBER_BUDGET allowance; anchor_day (their join day = the guest day) is
KEPT, and the reset is pinned to it like any member's;
* a short congratulations email goes out, then the normal member
onboarding (newsletter, Loomio + Matrix syncs, docs-RAG group,
held-credit grants).
"""
conn = get_db()
row = conn.execute(
"SELECT cloudron_user_id, anchor_day FROM members WHERE email = ?", (email,)
).fetchone()
conn.close()
if not row:
raise HTTPException(404, f"No portal row for {email}")
user_id = row[0]
# 1. Group move: guests → members (idempotent PUT; default = members group).
await cloudron_set_group(user_id)
# 2. Row: full member with the standard allowance. Unspent guest credits
# simply stay in credit_balance. anchor_day is untouched: their join
# day is the day the guest started.
conn = get_db()
conn.execute(
"UPDATE members SET balance = ?, account_type = 'member', active = 1, "
"slug = CASE WHEN ? != '' THEN ? ELSE slug END, "
"guest_expires_at = NULL WHERE email = ?",
(MEMBER_BUDGET, slug, slug, email),
)
conn.commit()
conn.close()
# 3. Lift the LiteLLM cap to allowance + credits and pin the reset to
# their anchor day, exactly as the webhook does for any member.
await sync_team_budget_from_balance(email)
try:
team_id = await litellm_team_id_for_email(email)
if team_id:
anchor = await anchor_day_for_member(email, slug)
await litellm_pin_reset_date(team_id, next_reset_for_anchor_day(anchor))
except Exception as e:
logger.warning("Guest convert: reset pin failed for %s: %s", email, e)
# 4. Full member onboarding — the pieces the guest path skipped.
try:
await owui_add_to_members_group(user_id)
except Exception as e:
logger.warning("Guest convert: OWUI group add skipped for %s: %s", email, e)
await listmonk_sync(email, subscribe=True)
try:
conn = get_db()
granted = grant_held_credits(conn, email, slug)
conn.close()
if granted > 0:
await sync_team_budget_from_balance(email)
logger.info("Guest convert: granted %.2f held credits to %s", granted, email)
except Exception as e:
logger.warning("Guest convert: held-credit grant for %s failed: %s", email, e)
await sync_loomio_memberships()
await sync_matrix_space()
# 5. Congratulations email (include the setup link only if they never
# activated — a 409 from invite_link means they did).
setup_url = None
try:
if not await cloudron_is_activated(user_id):
setup_url = await cloudron_get_invite_link(user_id)
except Exception:
setup_url = None
subject, text, html = guest_member_congrats_email(name or email.split("@")[0], setup_url)
send_email(email, subject, text, html)
touch_welcome_sent(email)
conn = get_db()
conn.execute("UPDATE guests SET status = 'converted' WHERE recipient_email = ? AND status = 'active'",
(email,))
conn.commit()
conn.close()
logger.info("Converted GUEST account %s to member (slug=%s)", email, slug)
return user_id
def guest_lapse_notice_sent(email: str) -> bool:
"""True when the reaching-limits email has already gone out for this guest.
The once-only gate (same shape as shortfall_notice_sent): one row per
guest, ever. The guests ledger records what became of a guest; this table
records what actually left the building.
"""
conn = get_db()
_ensure_guest_notice_table(conn)
row = conn.execute(
"SELECT 1 FROM guest_lapse_notices WHERE email = ? LIMIT 1", (email,)
).fetchone()
conn.close()
return bool(row)
def mark_guest_lapse_notice_sent(email: str, reason: str) -> None:
conn = get_db()
_ensure_guest_notice_table(conn)
conn.execute(
"INSERT OR REPLACE INTO guest_lapse_notices (email, reason, sent_at)"
" VALUES (?, ?, datetime('now'))",
(email, reason),
)
conn.commit()
conn.close()
def _ensure_guest_notice_table(conn) -> None:
"""Records which guests have been told their guest access ended.
Kept apart from the guests ledger on purpose: the ledger answers "what was
invited and what became of it", this answers "what left the building". A
guest is notified at most once, ever.
"""
conn.execute(
"CREATE TABLE IF NOT EXISTS guest_lapse_notices ("
" email TEXT PRIMARY KEY, reason TEXT NOT NULL,"
" sent_at TEXT NOT NULL DEFAULT (datetime('now')))"
)
conn.commit()
async def send_guest_lapse_notice(email: str, user_id: str | None, reason: str) -> bool:
"""Send the reaching-limits email ONCE per guest.
Checks the sent-state row before sending and marks it only after the send
succeeded — an unmarked guest (send failed, or lapsed before this email
shipped) is retried by the sweep's catch-up pass, never double-sent.
Returns True when a send actually left the building.
"""
if guest_lapse_notice_sent(email):
return False
# Greet by name where we can find one: the Cloudron display name, else the
# portal slug, else "there" (never the email local part).
name = ""
if user_id:
try:
async with httpx.AsyncClient() as client:
users = await cloudron_fetch_all_users(client)
if users:
name = next((u.get("displayName") or "" for u in users
if u.get("id") == user_id), "")
except Exception as e:
logger.warning("Guest lapse notice: name lookup failed for %s: %s", email, e)
if not name:
conn = get_db()
row = conn.execute(
"SELECT slug FROM members WHERE email = ?", (email,)
).fetchone()
conn.close()
if row and row[0]:
name = row[0]
subject, text_body, html_body = guest_limits_email(name or "there", reason)
if send_email(email, subject, text_body, html_body):
mark_guest_lapse_notice_sent(email, reason)
return True
return False
async def reconcile_guest_lifecycles() -> dict:
"""Lapse guest accounts by their own rules: time box or spent credits.
Runs inside the nightly sweep. Two deactivation triggers, checked per
ACTIVE guest row:
* guest_expires_at past (the GUEST_DAYS time box) — even with credits left;
* credits exhausted — the team's spend has reached the credit balance
(LiteLLM's cap already blocks further spend; this retires the row).
Deactivation is the same semantics as a member's: guests group → inactive,
team budget zeroed, row active=0, newsletter unsubscribed. The account is
KEPT so a returning payer resumes cleanly; the guests-ledger status records
which rule ended it ('expired' | 'depleted').
Returns {expired, depleted, lapse_notices} counts; never raises past a
logged warning.
"""
expired = 0
depleted = 0
notified = 0
conn = get_db()
rows = conn.execute(
"SELECT email, cloudron_user_id, credit_balance, balance, guest_expires_at "
"FROM members WHERE active = 1 AND account_type = 'guest'"
).fetchall()
conn.close()
now = datetime.now(timezone.utc)
for email, user_id, credits, balance, expires_at in rows:
try:
allowance = float(balance) if balance is not None else 0.0
credits = float(credits or 0.0)
reason = None
if expires_at:
exp = datetime.fromisoformat(str(expires_at))
if exp.tzinfo is None:
exp = exp.replace(tzinfo=timezone.utc)
if now >= exp:
reason = "expired"
if reason is None:
team_id = await litellm_team_id_for_email(email)
spend = None
if team_id:
team = get_team_budget_from_db(team_id)
if team:
spend = team.get("spend")
if spend is not None:
eff = compute_max_budget(allowance, credits, float(spend))
if eff["credits_remaining"] <= 0.0:
reason = "depleted"
if reason is None:
continue
if INACTIVE_GROUP_ID and user_id:
await cloudron_set_group(user_id, INACTIVE_GROUP_ID)
deactivate_member(email)
await revoke_member_litellm_access(email)
await listmonk_sync(email, subscribe=False)
conn = get_db()
conn.execute(
"UPDATE guests SET status = ? WHERE recipient_email = ? AND status = 'active'",
(reason, email),
)
conn.commit()
conn.close()
if reason == "expired":
expired += 1
else:
depleted += 1
logger.info("Guest lifecycle: %s deactivated (%s)", email, reason)
# Reaching-limits email: once per guest, ever, sent AFTER the
# deactivation committed (never tell someone their access ended
# while it hasn't). A failed send stays unmarked and is retried
# by the catch-up pass below on a later sweep.
try:
if await send_guest_lapse_notice(email, user_id, reason):
notified += 1
except Exception as e:
logger.warning("Guest lapse notice failed for %s: %s", email, e)
except Exception as e:
logger.warning("Guest lifecycle: failed for %s: %s", email, e)
# Catch-up: guests already deactivated (lapsed before this email shipped,
# or whose send failed) who were never told. Converted guests are excluded
# by the WHERE — their account_type is 'member' now, and a member is never
# told their guest ended.
conn = get_db()
pending = conn.execute(
"SELECT email, cloudron_user_id, guest_expires_at FROM members "
"WHERE active = 0 AND account_type = 'guest'"
).fetchall()
conn.close()
for email, user_id, expires_at in pending:
if guest_lapse_notice_sent(email):
continue
try:
conn = get_db()
grow = conn.execute(
"SELECT status FROM guests WHERE recipient_email = ? "
"ORDER BY id DESC LIMIT 1", (email,)
).fetchone()
conn.close()
reason = grow[0] if grow and grow[0] in ("expired", "depleted") else None
if reason is None:
# No ledger verdict: read the time box off the row; a row with
# no expiry was ended by its credits.
try:
exp = datetime.fromisoformat(str(expires_at)) if expires_at else None
if exp and exp.tzinfo is None:
exp = exp.replace(tzinfo=timezone.utc)
reason = "expired" if (exp and datetime.now(timezone.utc) >= exp) else "depleted"
except ValueError:
reason = "depleted"
if await send_guest_lapse_notice(email, user_id, reason):
notified += 1
except Exception as e:
logger.warning("Guest lapse catch-up failed for %s: %s", email, e)
return {"expired": expired, "depleted": depleted, "lapse_notices": notified}
async def cancel_guest_account(email: str) -> dict:
"""End a guest account at the recipient's own request (self-service cancel).
The guest-side counterpart to reconcile_guest_lifecycles: the recipient
cancels from their dashboard instead of waiting out the time box. Same
deactivation semantics as a guest lapse — the account and its data are
KEPT so a re-invite (or the guest → member conversion) resumes cleanly —
plus the one thing a lapse does not do: the unspent credit returns to
the co-op.
* LiteLLM team budget zeroed (every key under it stops working);
* the Cloudron user leaves the guests group (chat access ends) and
moves to GUESTS_ENDED_GROUP_ID, which the dashboard app still admits
so the ended state and the Become-a-member path stay visible;
* newsletter unsubscribed;
* unspent credit returned: a credit_ledger row with a NEGATIVE delta,
its reason naming the cancellation and the original guest reference
(traceable), its reference deduping the return so a repeated cancel
can never double-credit the accounting;
* the guests ledger records the full trail: inviter → recipient →
'cancelled by recipient' (rows are never deleted or rewritten);
* the reaching-limits bookkeeping is pre-marked when that subsystem is
deployed: someone who cancelled themselves must never be told their
credits were spent.
Idempotent: a second call is a no-op that reports the recorded outcome.
Refuses (403) anything that is not a guest account — a paying member can
never end a membership through this path, and there is deliberately no
member analogue: cancelling a membership means ending the Open Collective
contribution.
"""
conn = get_db()
row = conn.execute(
"SELECT active, account_type, cloudron_user_id, balance, credit_balance "
"FROM members WHERE email = ?",
(email,),
).fetchone()
conn.close()
if not row:
raise HTTPException(403, "No active membership")
active, account_type, user_id, balance, credits = row
if (account_type or "member") != "guest":
# The hard member guard: this endpoint only ever ends GUEST access.
# A membership is ended by ending its Open Collective contribution.
raise HTTPException(403, "Only guest accounts can be cancelled here")
guest_status = _guest_ledger_status(email)
if not active:
# Already ended (a previous cancel, or the sweep's lapse). No-op —
# and never a second return of credit.
return {
"status": "already_ended",
"guest_status": guest_status or "inactive",
"credits_returned": _guest_returned_credits(email),
}
# Unspent credit = the credits bucket minus what this cycle's spend
# already drew (a guest row holds NO allowance — balance is 0 — so the
# credits model's remaining figure IS the unspent guest).
allowance = float(balance) if balance is not None else 0.0
credit_total = float(credits or 0.0)
spend = None
team_id = await litellm_team_id_for_email(email)
if team_id:
team = get_team_budget_from_db(team_id)
if team:
spend = team.get("spend")
unspent = round(compute_max_budget(allowance, credit_total, spend)["credits_remaining"], 2)
return_ref = f"guest-return:{email}"
# The ledger is the accounting record: when a return row already exists
# (a cancel that failed part-way through), report what IT recorded and
# never write a second one — dedupe lives on the reference.
credits_returned = _guest_returned_credits(email) if credit_reference_exists(return_ref) else 0.0
if credits_returned == 0.0 and unspent > 0:
add_credits(
email, -unspent,
reason=(f"guest cancelled by recipient — unspent credit returned to the "
f"co-op (original guest: guest:{email})"),
reference=return_ref,
)
credits_returned = unspent
# Deactivation, guest-lapse semantics: row inactive, group out of guests,
# gateway budget zeroed, newsletter off.
deactivate_member(email)
ended_group = GUESTS_ENDED_GROUP_ID or INACTIVE_GROUP_ID
if ended_group and user_id:
await cloudron_set_group(user_id, ended_group)
await revoke_member_litellm_access(email)
await listmonk_sync(email, subscribe=False)
conn = get_db()
conn.execute(
"UPDATE guests SET status = 'cancelled by recipient' "
"WHERE recipient_email = ? AND status = 'active'",
(email,),
)
conn.commit()
conn.close()
# Pre-mark the reaching-limits notice: the sweep's catch-up must not
# email a "your credits were spent" notice to someone who cancelled
# themselves — they were told by doing it.
mark_guest_lapse_notice_sent(email, "cancelled by recipient")
logger.info("Guest cancel: %s cancelled by recipient (returned %.2f)", email, credits_returned)
return {
"status": "cancelled",
"guest_status": "cancelled by recipient",
"credits_returned": credits_returned,
}
def _guest_ledger_status(email: str) -> str | None:
"""The latest guests-ledger status for a recipient, or None."""
conn = get_db()
row = conn.execute(
"SELECT status FROM guests WHERE recipient_email = ? ORDER BY id DESC LIMIT 1",
(email,),
).fetchone()
conn.close()
return row[0] if row else None
def _guest_returned_credits(email: str) -> float:
"""Credit returned to the co-op for this guest (0 when there was none).
Read back from the ledger — the return row IS the accounting record, so
the reported amount always matches what was actually written.
"""
conn = get_db()
rows = conn.execute(
"SELECT delta FROM credit_ledger WHERE reference = ?",
(f"guest-return:{email}",),
).fetchall()
conn.close()
return round(sum(-float(r[0] or 0.0) for r in rows), 2)
async def litellm_get_or_create_team(email: str, budget: float,
budget_duration: str | None = "30d") -> str:
"""Return the team_id for a member's budget team, creating it if needed.
Each member gets a LiteLLM "team" (a budget container) with a $15/30d
budget. The team holds the chat key + any member-managed API keys, so all
spend draws from the one pool.
`budget_duration` is the reset cadence: members renew every 30 days; GUEST
accounts pass None — a one-time budget that NEVER resets, so the $2 cap is
the guest's whole life (spend it and it is gone).
Idempotent: looks up the existing team by alias before creating. Note that
/team/new is NOT idempotent (it creates a new team each call), so we must
check /team/list first.
"""
alias = f"member:{email}"
# /team/list has grown slow as usage accumulates; the httpx default 5s
# read timeout was too short (root cause of dashboard 500s on 2026-09-26).
async with httpx.AsyncClient(timeout=httpx.Timeout(connect=15.0, read=60.0, write=15.0, pool=15.0)) as client:
r = await client.get(
f"{LITELLM_BASE}/team/list",
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
)
r.raise_for_status()
for t in r.json():
if t.get("team_alias") == alias:
return t["team_id"]
payload = {
"team_alias": alias,
"max_budget": budget,
"models": MEMBER_MODELS,
}
if budget_duration:
payload["budget_duration"] = budget_duration
r = await client.post(
f"{LITELLM_BASE}/team/new",
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
json=payload,
)
r.raise_for_status()
return r.json()["team_id"]
async def litellm_update_team_budget(team_id: str, budget: float) -> None:
"""Set a team's max_budget (the enforced spend cap).
This is the primitive behind "payment → add to balance": a top-up adjusts
the team's max_budget, which every key under the team (chat + API) draws
from. Keys inherit the team budget (their own max_budget is ignored when a
team_id is set), so the team is the single source of truth for allowance.
"""
async with httpx.AsyncClient() as client:
r = await client.post(
f"{LITELLM_BASE}/team/update",
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
json={"team_id": team_id, "max_budget": budget},
)
r.raise_for_status()
async def litellm_pin_reset_date(team_id: str, reset_at: datetime) -> None:
"""Pin a team's allowance reset to an exact moment (next OC payment date).
LiteLLM's /team/update silently DROPS budget_reset_at (verified v1.74.0),
so this writes the authoritative DB directly. Budgets reset to zero spend
at the pinned instant; the pinned date is honored across restarts.
"""
def _write() -> None:
conn = _pg_connect()
try:
cur = conn.cursor()
cur.execute(
'UPDATE "LiteLLM_TeamTable" SET budget_reset_at = %s WHERE team_id = %s',
(reset_at, team_id),
)
conn.commit()
finally:
conn.close()
await asyncio.get_event_loop().run_in_executor(None, _write)
logger.info("Pinned budget reset for %s to %s", team_id, reset_at.isoformat())
def next_reset_from_now() -> datetime:
"""The member's next allowance reset: 30 days from now, on the hour."""
return datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0) + timedelta(days=30)
async def sync_team_budget_from_balance(email: str) -> float:
"""Apply the member's effective budget to their LiteLLM team.
Effective budget = monthly allowance + remaining non-expiring credits
(credits drawn only after the allowance is exhausted this cycle). Reads
the member's live spend from LiteLLM to determine how much credit has
already been consumed in the current cycle. Returns the applied budget.
"""
balance = get_member_balance(email)
credits = get_credit_balance(email)
team_id = await litellm_get_or_create_team(email, balance)
# Live spend this cycle (DB path preferred; LiteLLM computes the reset).
team = get_team_budget_from_db(team_id) or {}
team_spend = team.get("spend")
eff = compute_max_budget(balance, credits, team_spend)
await litellm_update_team_budget(team_id, eff["max_budget"])
return eff["max_budget"]
async def litellm_create_key(email: str, budget: float, team_id: str) -> str:
"""Create a fresh LiteLLM virtual key under the member's team.
Returns the ACTUAL sk- key (only revealed at generation time). We always
create a new key rather than reusing — the sk- value is unrecoverable from
a stored hash, so "reuse" would store a useless hash (a bug that previously
broke chat for three members).
NOTE: the key does NOT set its own max_budget — it inherits the team's
budget (LiteLLM ignores a key's max_budget when team_id is set anyway).
The team is the single source of truth for allowance, so a balance change
(top-up) updates the team once and every key follows. The `budget` arg is
retained only for the team-creation fallback in the migration path.
"""
alias = f"member:{email}"
async with httpx.AsyncClient() as client:
r = await client.post(
f"{LITELLM_BASE}/key/generate",
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
json={
"key_alias": alias,
"team_id": team_id,
"models": MEMBER_MODELS,
},
)
r.raise_for_status()
return r.json().get("key", "")
async def litellm_disable_key(key_token: str) -> None:
"""Delete a member's LiteLLM key (on payment lapse).
/key/delete accepts either the sk- key or its SHA256 hash, so this works
for both current keys and legacy hash-keyed members.
"""
if not key_token:
return
async with httpx.AsyncClient() as client:
await client.post(
f"{LITELLM_BASE}/key/delete",
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
json={"keys": [key_token]},
)
async def litellm_team_id_for_email(email: str) -> str | None:
"""The team_id of a member's budget team, or None if they have none."""
async with httpx.AsyncClient(timeout=httpx.Timeout(connect=15.0, read=60.0, write=15.0, pool=15.0)) as client:
r = await client.get(f"{LITELLM_BASE}/team/list",
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"})
r.raise_for_status()
for t in r.json():
if t.get("team_alias") == f"member:{email}":
return t["team_id"]
return None
async def revoke_member_litellm_access(email: str) -> bool:
"""Cut a deactivated member's gateway access (SECURITY 2026-10-27).
Zeroing the team budget blocks every key under it: member-managed keys carry
no max_budget of their own (verified live 2026-10-27), so they inherit the
team's. Deleting the key instead would break reactivation, which deliberately
reuses the stored sk- token — and get_or_create_team does NOT update the
budget of an existing team, so reactivation paths must re-sync explicitly.
Best-effort by design: a failure here must not stop deactivation from
completing, but it must be loud, because the member keeps access until it
succeeds.
"""
try:
team_id = await litellm_team_id_for_email(email)
if not team_id:
logger.info("Revoke: %s has no gateway team; nothing to zero", email)
return True
await litellm_update_team_budget(team_id, 0.0)
logger.info("Revoke: %s gateway budget zeroed (team %s)", email, team_id)
return True
except Exception as e:
logger.error("Revoke: FAILED to zero gateway budget for %s: %s", email, e)
try:
send_email("info@inference.coop",
"Deactivation did not revoke gateway access",
f"{email} was deactivated but their LiteLLM team budget could not be zeroed:\n\n{e}\n\n"
"Their existing API keys may still work. Fix by hand: set the team's max_budget to 0.\n")
except Exception:
pass
return False
# --- Broker: member-managed API keys (scoped to the member's team) ---
def _api_key_alias(email: str, name: str) -> str:
return f"api:{email}:{name}"
async def broker_create_api_key(email: str, name: str) -> dict:
"""Create a member-managed API key under the member's team.
Returns {name, sk_token, created_at}. The key draws from the member's team
budget and is tracked in member_api_keys so it can be listed/revoked by name.
"""
if not name or not re.fullmatch(r"[A-Za-z0-9._-]{1,64}", name):
raise HTTPException(400, "Invalid key name (1-64 chars, alphanumeric/._-)")
balance = get_member_balance(email)
team_id = await litellm_get_or_create_team(email, balance)
alias = _api_key_alias(email, name)
async with httpx.AsyncClient() as client:
# Rotation support: LiteLLM requires unique key aliases, so a re-create
# with an existing (email, name) must retire the old key first. Without
# this, rotating a key from the dashboard 400s ("alias already exists").
conn = get_db()
row = conn.execute(
"SELECT hash FROM member_api_keys WHERE email = ? AND name = ?",
(email, name),
).fetchone()
conn.close()
if row and row[0]:
old_hash = row[0]
try:
del_res = await client.post(
f"{LITELLM_BASE}/key/delete",
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
json={"keys": [old_hash]},
)
if del_res.status_code == 200:
logger.info("Rotated API key %s for %s (old key retired)", name, email)
else:
logger.warning(
"Old key retire failed for %s/%s: %s %s",
email, name, del_res.status_code, del_res.text[:200],
)
except Exception as exc: # noqa: BLE001 - rotation is best-effort
logger.warning("Old key retire errored for %s/%s: %s", email, name, exc)
r = await client.post(
f"{LITELLM_BASE}/key/generate",
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
json={
"key_alias": alias,
"team_id": team_id,
"models": MEMBER_MODELS,
},
)
r.raise_for_status()
data = r.json()
sk_token = data.get("key", "")
h = data.get("token") or data.get("token_id") or ""
# Reveal the key ONCE (in the response). Persist only a fingerprint (last-4)
# and the LiteLLM hash — the plaintext is NOT stored. Revocation uses the
# hash, so the full key never needs to be retained.
fingerprint = sk_token[-4:] if sk_token else ""
conn = get_db()
conn.execute(
"INSERT INTO member_api_keys (email, name, sk_token, hash, fingerprint) VALUES (?, ?, ?, ?, ?) "
"ON CONFLICT(email, name) DO UPDATE SET sk_token='', hash=excluded.hash, fingerprint=excluded.fingerprint, "
"created_at=datetime('now')",
(email, name, "", h, fingerprint),
)
conn.commit()
row = conn.execute(
"SELECT created_at FROM member_api_keys WHERE email = ? AND name = ?",
(email, name),
).fetchone()
conn.close()
return {"name": name, "sk_token": sk_token, "created_at": row[0] if row else None}
async def broker_list_api_keys(email: str) -> list[dict]:
conn = get_db()
rows = conn.execute(
"SELECT name, fingerprint, created_at FROM member_api_keys WHERE email = ? ORDER BY created_at DESC",
(email,),
).fetchall()
conn.close()
return [
{"name": r[0], "fingerprint": r[1] or "", "created_at": r[2]}
for r in rows
]
async def broker_revoke_api_key(email: str, name: str) -> None:
conn = get_db()
row = conn.execute(
"SELECT hash FROM member_api_keys WHERE email = ? AND name = ?", (email, name)
).fetchone()
if not row:
conn.close()
raise HTTPException(404, "Key not found")
conn.execute("DELETE FROM member_api_keys WHERE email = ? AND name = ?", (email, name))
conn.commit()
conn.close()
# Delete the underlying LiteLLM key by its hash.
await litellm_disable_key(row[0])
# --- A. Open Collective webhook ---
@app.post("/webhook/opencollective/{token}")
@limiter.limit("30/minute")
async def opencollective_webhook(request: Request, token: str):
"""Handle Open Collective membership events.
Authenticated by a secret token in the URL path (Open Collective's
generic webhooks are not HMAC-signed, so a secret URL is the standard
way to authenticate them).
"""
if not WEBHOOK_TOKEN or not secrets.compare_digest(token, WEBHOOK_TOKEN):
raise HTTPException(401, "Invalid webhook token")
payload = await request.json()
event_type = payload.get("type", "")
data = payload.get("data", {})
# The webhook does NOT include email (Open Collective strips it for
# privacy). We get name + slug, then look up the email via the admin token.
#
# Slug/name live in different places depending on the event type:
# - order.processed / transaction.created → data.fromCollective.{slug,name}
# - collective.member.created → data.member.memberCollective.{slug,name}
from_collective = data.get("fromCollective", {})
member_collective = (data.get("member", {}) or {}).get("memberCollective", {})
slug = from_collective.get("slug") or member_collective.get("slug", "")
name = from_collective.get("name") or member_collective.get("name", "Member")
logger.info("Open Collective event: %s (name=%s, slug=%s)", event_type, name, slug)
# Open Collective webhook events (verified):
# - "order.processed" → fires on EVERY payment (incl. monthly recurring)
# - "new member" → fires on FIRST contribution only
# - payload has "firstPayment" boolean to distinguish new vs recurring
# - "collective.transaction.created" is DEPRECATED (being removed)
if event_type in ("order.processed", "new.member", "collective.member.created"):
# New member → provision immediately and send our own welcome email.
# The webhook strips email, so look it up by slug via the admin token.
# Only provision on firstPayment (order.processed also fires on monthly
# renewals, which should NOT re-send the welcome email). Default to
# False so a missing field never triggers a spurious re-provision; the
# nightly sweep backstops any missed first payment.
first_payment = data.get("firstPayment", False)
# Membership provision: only on payments to the "Membership" tier. A
# one-time donation (no tier, or any other tier) grants BACKER role on
# OC but is NOT a membership — don't provision it.
#
# Tier detection: the webhook payload carries only `data.order.TierId`
# (numeric) — NO tier name. The order description embeds it
# ("Financial contribution to Inference Cooperative (Credit pack)").
order = data.get("order") or {}
order_desc = order.get("description") or ""
m = re.search(r"\(([^)]+)\)\s*$", order_desc.strip())
tier_name = m.group(1).strip() if m else ""
amount_cents = data.get("amount") or 0
if not amount_cents:
amount_cents = order.get("totalAmount") or 0
# Credit-pack detection: one-time contributions to a "Credit pack"
# tier route to the member's NON-EXPIRING credit balance instead of
# provisioning/allowance. Detection: the OC tier name contains
# "credit" (case-insensitive). Recurring subscriptions never route
# here (a credit pack is a one-time purchase).
if "credit" in tier_name.lower() and slug and event_type == "order.processed":
members = await fetch_members()
email = next((m["email"] for m in members if m["slug"] == slug), None)
# The tier is VARIABLE-amount — no fixed face value. Credits are
# granted on the contribution amount EXCLUDING the OC platform
# tip (totalAmount - tip = amount), which we resolve via OC's
# public order API. Fallback: webhook totalAmount (slightly
# overgrants if a tip was included; logged).
tip_free_cents = await get_order_amount(order.get("idV2"))
if tip_free_cents is None:
# SECURITY (2026-10-27): do NOT fall back to the payload's own
# amount. This route is authenticated only by a URL token, so a
# forged POST naming its own totalAmount used to mint credits for
# any slug. Treat the payload as a hint; require a resolvable OC
# order. A forged id cannot resolve; a transient OC failure holds
# for manual reconciliation instead of granting blind.
claimed = round(float(amount_cents) / 100.0, 2)
logger.warning(
"Credit pack REJECTED: OC could not resolve order %r (slug=%s, payload claimed $%.2f)"
" — no credits granted",
order.get("idV2"), slug, claimed,
)
try:
send_email(
"info@inference.coop",
"Credit pack could not be verified — no credits granted",
f"A credit-pack webhook arrived for {slug} claiming ${claimed:.2f}, but\n"
f"Open Collective's order API could not resolve order id {order.get('idV2')!r}.\n\n"
"No credits were granted: the payload's amount is not trusted. If this was a\n"
"real purchase, confirm the order in Open Collective and grant the credits\n"
"by hand; if it was not, this was an attempt to mint credits.\n",
)
except Exception as e:
logger.warning("Credit pack rejection: operator email failed: %s", e)
return JSONResponse({"status": "unverified_order_held", "slug": slug})
amount = round(tip_free_cents / 100.0, 2)
charged = round(float(amount_cents) / 100.0, 2)
if charged > amount:
logger.info(
"Credit pack: charged $%.2f including tip; granting tip-free $%.2f",
charged, amount,
)
oc_ref = credit_pack_reference(slug, order)
if email:
# Idempotence (audit H1, 2026-11-05): OC re-delivers events and
# this handler is slow (OC member fetch + email + Loomio +
# Matrix inside the request), which invites re-delivery. This
# purchase was already recorded once — return the standing
# balance WITHOUT crediting again. The reference is per-order,
# so this never blocks a new purchase, and legacy empty-suffix
# ledger rows can never match a new-format reference.
if credit_reference_exists(oc_ref):
logger.info(
"Credit pack: %s already in the ledger (OC re-delivery) "
"— not crediting again", oc_ref,
)
return JSONResponse({
"status": "already_credited", "email": email,
"credits_added": 0.0,
"credit_balance": get_credit_balance(email),
"reference": oc_ref,
})
# Member purchase → credits land immediately.
new_balance = add_credits(
email, amount,
reason="credit pack purchase (Open Collective)",
reference=oc_ref,
)
# Push the enlarged budget immediately.
await sync_team_budget_from_balance(email)
logger.info("Credit pack: +%s credits for %s (balance %s)",
amount, email, new_balance)
return JSONResponse({
"status": "credits_added", "email": email,
"credits_added": amount, "credit_balance": new_balance,
})
# Non-member purchase → HOLD. Record it, email the buyer with
# their two options (refund or join-and-apply). Credits are NOT
# granted: a membership is required to use them.
buyer_name = name or slug
buyer_email = ""
# The webhook payload strips emails; look up the payer via the
# admin token (bot account sees contributor emails).
try:
# Paginated member fetch (raises on failure → caught below)
for n in await fetch_all_member_nodes():
if (n.get("account") or {}).get("slug") == slug:
buyer_email = ((n.get("account") or {}).get("email") or "").strip()
buyer_name = (n.get("account") or {}).get("name") or buyer_name
break
except Exception as e:
logger.warning("Credit-pack hold: payer lookup failed for %s: %s", slug, e)
conn = get_db()
# Idempotence (audit H1, 2026-11-05): one hold per purchase. The
# unique index on oc_reference is the hard stop; this pre-check
# returns a calm response on re-delivery instead of a 500 (which
# would make OC re-deliver again).
existing = conn.execute(
"SELECT id FROM held_credits WHERE oc_reference = ?", (oc_ref,)
).fetchone()
if existing:
conn.close()
logger.info(
"Credit pack HOLD: %s already recorded (hold #%s); not duplicating",
oc_ref, existing[0],
)
return JSONResponse({
"status": "hold_already_recorded", "slug": slug,
"hold_id": existing[0],
})
conn.execute(
"INSERT INTO held_credits (slug, email, name, amount, oc_reference) "
"VALUES (?, ?, ?, ?, ?)",
(slug, buyer_email or None, buyer_name, amount, oc_ref),
)
try:
conn.commit()
except sqlite3.IntegrityError:
# Lost a race with a concurrent delivery of the same order;
# the unique index kept us from double-holding.
conn.close()
logger.warning(
"Credit pack HOLD: %s raced an existing hold; not duplicating", oc_ref,
)
return JSONResponse({"status": "hold_already_recorded", "slug": slug})
conn.close()
logger.warning(
"Credit pack HELD from non-member %s (%.2f credits); buyer notified",
slug, amount,
)
if buyer_email:
send_email(
buyer_email,
"Inference Cooperative — about your credit pack",
f"Hi {buyer_name},\n\n"
f"Thank you for your ${amount:.2f} credit pack purchase — we've received it.\n\n"
"One thing to flag: credit packs are for members, and our records\n"
"don't show you as one yet. You have two options:\n\n"
"1. Join the co-op ($10-20/month, sliding scale):\n"
" https://opencollective.com/inference-cooperative/contribute\n"
f" Once you're a member, your ${amount:.2f} in credits will be\n"
" applied to your account automatically.\n\n"
"2. Request a refund:\n"
" Reply to this email or write to info@inference.coop and\n"
" we'll refund you in full.\n\n"
"Sorry for the friction — memberships keep the co-op\n"
"member-governed, which is rather the point of the place.\n\n"
"— The Inference Cooperative\n"
"https://inference.coop",
)
else:
logger.warning("Credit-pack hold: no buyer email found for %s; no notification sent", slug)
return JSONResponse({
"status": "held",
"note": "credit pack purchased without membership; hold recorded and buyer notified",
})
if slug and "membership" in tier_name.lower() and event_type == "order.processed":
# Every Membership-tier payment (first or recurring) re-pins the
# member's allowance reset to payment + 30 days — the reset
# follows the member's actual billing rhythm, not a global
# calendar (all members landing on Oct 1 otherwise).
members = await fetch_members()
member = next((m for m in members if m["slug"] == slug), None)
if member:
# A contribution may arrive under a DIFFERENT email than the
# portal account (a invited friend who joined under another OC
# address — linked via /admin/guest-convert): route the payment
# to the mapped account instead of minting a duplicate row.
oc_email = (member.get("email") or "").strip().lower()
if oc_email:
conn = get_db()
mapped = conn.execute(
"SELECT email FROM members WHERE oc_email = ?", (oc_email,)
).fetchone()
conn.close()
if mapped and mapped[0] != member["email"]:
member = {**member, "email": mapped[0]}
logger.info("Webhook: payment under %s mapped to portal account %s",
oc_email, mapped[0])
if first_payment:
try:
await provision_member(member["email"], member["name"], member["slug"])
# A lapsed member's team budget was zeroed on deactivation and
# get_or_create_team does not restore it — re-apply explicitly.
await sync_team_budget_from_balance(member["email"])
except Exception as e:
logger.warning("Webhook: failed to provision %s: %s", member["email"], e)
try:
# POLICY (2026-11-04): the allowance reset is anchored to the
# member's JOIN day, NOT the payment date. OC aligns charges
# onto a common billing day, so re-anchoring to the charge
# moved a renewal and skipped a cycle. A payment must only
# CONFIRM the cycle, never redefine where it falls.
team_id = await litellm_get_or_create_team(
member["email"], get_member_balance(member["email"]))
anchor = await anchor_day_for_member(member["email"], member["slug"])
await litellm_pin_reset_date(team_id, next_reset_for_anchor_day(anchor))
except Exception as e:
logger.warning("Webhook: failed to pin reset date for %s: %s",
member["email"], e)
elif slug:
# Organization-backed membership: OC orgs never expose an email
# to our integration, so fetch_members can't see them. Linked
# orgs (members.org_slug) get their account re-activated on
# dues; unlinked orgs ping the operator to link them.
conn = get_db()
row = conn.execute(
"SELECT email, org_slug FROM members WHERE org_slug = ?", (slug,)
).fetchone()
conn.close()
if row:
org_email, org_slug_col = row
conn = get_db()
conn.execute("UPDATE members SET active = 1 WHERE email = ?", (org_email,))
conn.commit()
conn.close()
logger.info("Webhook: org dues for %s — linked account %s re-activated",
slug, org_email)
if first_payment:
try:
await provision_member(org_email, name, slug)
await sync_team_budget_from_balance(org_email)
except Exception as e:
logger.warning("Webhook: org re-provision failed for %s: %s",
org_email, e)
else:
logger.warning(
"Webhook: Membership payment from ORGANIZATION %s (%s) — "
"no linked account; add via POST /admin/org-link",
name, slug,
)
send_email(
"info@inference.coop",
"Action needed: organizational membership without a linked account",
f"Open Collective reports a Membership-tier payment from the\n"
f"organization \"{name}\" (slug: {slug}).\n\n"
"Organizations don't expose an email to our integration, so the\n"
"membership needs to be linked to a portal account manually:\n\n"
" POST /admin/org-link\n"
" {\"org_slug\": \"" + slug + "\", \"email\": \"<their preferred email>\"}\n\n"
"Until linked, the organization cannot onboard.\n\n"
"— The portal",
)
elif slug and first_payment and event_type == "order.processed":
# One-time donation (no tier or non-membership tier): NOT a
# membership. Log it so we can see it, but don't provision.
logger.info("One-time donation from slug %s (tier=%r) — not a membership, not provisioning",
slug, tier_name)
return JSONResponse({"status": "provisioned", "name": name, "slug": slug})
if event_type in ("collective.member.deleted", "collective.transaction.deleted"):
# Lapsed member → move to the inactive group (lose access, keep data).
# The webhook gives us the slug; we map it to the member's email/user.
if slug:
member = get_member_by_slug(slug)
if member:
# Move to inactive group (revokes chat access, keeps account + data)
if INACTIVE_GROUP_ID and member.get("cloudron_user_id"):
await cloudron_set_group(member["cloudron_user_id"], INACTIVE_GROUP_ID)
# Mark inactive in our DB (key injector will refuse requests)
deactivate_member(member["email"])
await revoke_member_litellm_access(member["email"])
await listmonk_sync(member["email"], subscribe=False)
await sync_loomio_memberships()
await sync_matrix_space()
logger.info("Deactivated member %s (slug=%s)", member["email"], slug)
return JSONResponse({"status": "deactivated", "email": member["email"]})
return JSONResponse({"status": "deactivated", "note": "no matching member"})
return JSONResponse({"status": "ignored", "type": event_type})
# --- B. Key injector (proxy) ---
# --- Model picker: value-based naming ---
# The chat model picker displays each model's `name` field. We set it to a
# friendly name prefixed with an emoji for the provider's value tier, so the
# flat picker list clusters by what members care about (privacy / green /
# openness). The `id` field is left untouched — it remains the canonical key
# used for actual API calls. Same emoji as the docs (models.md).
# Emoji marks the FEATURE tier, not the provider: 🌱 = green/renewable-powered
# (GreenPT and Regolo both), 🔒 = TEE-architected private (Tinfoil).
MODEL_VALUE_EMOJI = {
"tinfoil": "🔒", # architecturally private (TEE)
"greenpt": "🌱", # 100% renewable energy
"regolo": "🌱", # EU-hosted, zero data retention, 100% renewables
}
MODEL_FRIENDLY_NAMES = {
"greenpt/deepseek-v4.1-flash": "DeepSeek V4.1",
"tinfoil/deepseek-v4-1-flash": "DeepSeek V4.1",
"tinfoil/glm-5.3": "GLM-5.3",
"greenpt/glm-5.3-flash": "GLM-5.3 Flash",
"greenpt/glm-5.3": "GLM-5.3",
"tinfoil/gpt-oss-120b": "GPT-OSS 120B",
"regolo/apertus-70b": "Apertus 70B",
"regolo/qwen3.5-122b": "Qwen 3.5 122B",
}
@app.get("/v1/models")
@limiter.limit("60/minute")
async def list_models(request: Request):
"""List models (same for everyone — no per-user auth needed).
Annotates each model with OpenWebUI metadata so web search is enabled by
default (defaultFeatureIds includes 'web_search'). Without this, OpenWebUI
leaves web search off for these models and members must toggle it manually.
"""
async with httpx.AsyncClient() as client:
r = await client.get(
f"{LITELLM_BASE}/v1/models",
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
)
if r.status_code != 200:
return Response(
content=r.content,
status_code=r.status_code,
headers={"content-type": r.headers.get("content-type", "application/json")},
)
payload = r.json()
# Chat-list filter: exclude audio-only models (STT/TTS) and embedding
# models. They are registered and callable via their own routes
# (/v1/audio/* and /v1/embeddings) for API users, but they are not chat
# models — listing them in Open WebUI would just confuse members. The
# gateway's model_info carries `mode` for these; fall back to a name
# check for safety.
audio_modes = {"audio_transcription", "audio_speech"}
audio_name_hints = ("whisper", "voxtral", "tts")
embedding_name_hints = ("embed",)
# Image-generation models are used through Open WebUI's image feature
# (Admin > Images), not as chat models — listing them in the picker
# would fail on chat completions.
image_modes = {"image_generation"}
def is_audio_model(m) -> bool:
mode = ((m.get("info") or {}).get("meta") or {}).get("mode") or ""
if mode in audio_modes:
return True
mid = (m.get("id") or "").lower()
return any(h in mid for h in audio_name_hints)
def is_embedding_model(m) -> bool:
mode = ((m.get("info") or {}).get("meta") or {}).get("mode") or ""
if mode == "embedding":
return True
mid = (m.get("id") or "").lower()
return any(h in mid for h in embedding_name_hints)
image_name_hints = ("qwen-image",)
def is_image_model(m) -> bool:
mode = ((m.get("info") or {}).get("meta") or {}).get("mode") or ""
if mode in image_modes:
return True
mid = (m.get("id") or "").lower()
return any(h in mid for h in image_name_hints)
payload["data"] = [
m for m in payload.get("data", [])
if not is_audio_model(m) and not is_embedding_model(m) and not is_image_model(m)
]
for model in payload.get("data", []):
# Friendly display name: emoji value-tier + short name. The `id`
# stays canonical (Open WebUI keys lookups and requests on it).
mid = model.get("id", "")
provider = mid.split("/", 1)[0] if "/" in mid else ""
emoji = MODEL_VALUE_EMOJI.get(provider, "")
friendly = MODEL_FRIENDLY_NAMES.get(
mid, mid.split("/", 1)[-1] if "/" in mid else mid
)
model["name"] = f"{emoji} {friendly}".strip()
model.setdefault("info", {}).setdefault("meta", {})
model["info"]["meta"].setdefault("capabilities", {})["web_search"] = True
model["info"]["meta"].setdefault("defaultFeatureIds", [])
if "web_search" not in model["info"]["meta"]["defaultFeatureIds"]:
model["info"]["meta"]["defaultFeatureIds"].append("web_search")
# Attach the co-op docs knowledge collection to every model. This
# makes Open WebUI treat the docs as the model's own knowledge:
# automatic RAG injection grounds answers without tool-calling, and
# the knowledge toolset is scoped to the attached collection instead
# of exposing browse-all-knowledge tools (the "explored
# search_knowledge_files… view_knowledge_file" wandering that new
# members saw from the getting-started prompt). Env-driven so the
# collection id can be rotated without a code change.
if DOCS_KNOWLEDGE_ID:
model["info"]["meta"].setdefault("knowledge", [])
kb_entry = {"id": DOCS_KNOWLEDGE_ID, "type": "collection"}
if kb_entry not in model["info"]["meta"]["knowledge"]:
model["info"]["meta"]["knowledge"].append(kb_entry)
# SILENT AUTO-RAG (legacy function-calling mode), chosen 2026-10-09:
# with native FC the model gets a query_knowledge_files tool and
# calls it on every prompt — including off-topic ones — with
# visible "let me check the docs" narration. Prompt steering
# failed twice (soft + firm). Legacy mode instead auto-injects
# relevant docs chunks into context on every message: silent,
# no tool-calls, no announcements, grounding always on. Cost:
# one embedding + vector search per message (~300ms, paid even
# on off-topic messages). builtinTools.knowledge=false removes
# the knowledge tool + <attached_knowledge> tag so the model
# can't narrate or fumble; the injected sources still ground
# co-op answers.
model.setdefault("params", {})
model["params"]["function_calling"] = "legacy"
model["info"]["meta"]["builtinTools"] = {"knowledge": False}
# Steering: sources arrive as plain context. Tell the model the
# docs may be present but are co-op-only, so it never cites or
# mentions them for off-topic answers.
steering = (
"Some messages include excerpts from the Inference "
"Cooperative's own documentation (membership, models, API, "
"governance), added automatically as background context. "
"Use those excerpts only when they are relevant to the "
"user's actual question. For general questions, ignore them "
"and answer from your own knowledge — do not mention the "
"co-op, its documentation, or any knowledge search unless "
"the user explicitly asks about the co-op."
)
existing = model["params"].get("system")
if existing:
if "Inference Cooperative" not in existing:
model["params"]["system"] = existing + "\n\n" + steering
else:
model["params"]["system"] = steering
return JSONResponse(content=payload)
@app.api_route("/v1/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"])
@limiter.limit("120/minute")
async def inject_key(request: Request, path: str):
"""Read the member's email from a header, inject their key, forward to LiteLLM."""
# Verify the request came through the chat frontend, so a direct caller
# can't spoof the user-email header and use another member's key.
#
# Two supported auth paths:
# 1. OpenWebUI signs the user identity as a JWT (X-OpenWebUI-User-Jwt)
# with a shared secret — the email is tamper-proof, so no separate
# secret header is needed.
# 2. LibreChat sends a plain X-User-Email header plus an X-Portal-Secret
# shared-secret header.
email = ""
jwt_header = request.headers.get("x-openwebui-user-jwt", "")
if jwt_header and OWUI_JWT_SECRET:
try:
claims = jwt.decode(jwt_header, OWUI_JWT_SECRET, algorithms=["HS256"])
email = claims.get("email", "")
except jwt.PyJWTError:
raise HTTPException(401, "Invalid user identity token")
else:
_verify_portal_secret(request)
email = (
request.headers.get("x-user-email", "")
or request.headers.get("x-openwebui-user-email", "")
)
if not email:
raise HTTPException(401, "No member identity (user-email header)")
# Look up the member's key from the persistent store
member_key = get_member_key(email)
if not member_key:
raise HTTPException(403, "No active membership key")
# Forward the request to LiteLLM with the member's key, STREAMING the
# response back token-by-token (or byte-by-byte) rather than buffering it.
# Buffering collapsed the model's stream and handed OpenWebUI the full
# answer in one blob, so members saw nothing until generation finished —
# which made chat feel slow even though time-to-first-byte was fine.
body = await request.body()
headers = dict(request.headers)
headers["authorization"] = f"Bearer {member_key}"
headers.pop("host", None)
headers.pop("content-length", None)
headers.pop("accept-encoding", None) # let httpx handle decompression
timeout = httpx.Timeout(connect=15.0, read=600.0, write=30.0, pool=15.0)
# Send the request upstream, but don't buffer the body — use a stream.
client = httpx.AsyncClient(timeout=timeout)
upstream_req = client.build_request(
method=request.method,
url=f"{LITELLM_BASE}/v1/{path}",
headers=headers,
content=body,
)
upstream = await client.send(upstream_req, stream=True)
async def gen():
try:
async for chunk in upstream.aiter_raw():
yield chunk
finally:
await upstream.aclose()
await client.aclose()
# Pass through the content-type (critical for SSE streaming) and status.
return StreamingResponse(
gen(),
status_code=upstream.status_code,
headers={
"content-type": upstream.headers.get("content-type", "application/json"),
"cache-control": "no-cache",
},
)
# --- D. Admin / health ---
@app.get("/health")
async def health():
return {"status": "ok"}
@app.post("/admin/sync-loomio")
@limiter.limit("20/minute")
async def admin_sync_loomio(request: Request):
"""Manually trigger a Loomio membership sync.
Authenticated by the X-Admin-Token header. Useful for reconciling
pre-existing members (accounts created before the sync existed) or
recovering from a missed webhook. Idempotent — safe to call repeatedly.
"""
_verify_admin_token(request)
await sync_loomio_memberships()
await sync_matrix_space()
return {"status": "synced"}
@app.post("/admin/provision")
@limiter.limit("20/minute")
async def admin_provision(request: Request):
"""Manually provision a member by email (full pipeline).
Authenticated by the X-Admin-Token header. Runs the full provisioning
pipeline: Cloudron user + members group + LiteLLM key + our welcome email +
Loomio sync. Body: {"email": "...", "name": "..."}. Idempotent.
"""
_verify_admin_token(request)
body = await request.json()
email = (body.get("email") or "").strip().lower()
name = body.get("name") or email.split("@")[0]
if not email:
raise HTTPException(400, "Missing email")
user_id = await provision_member(email, name, "")
return {"status": "provisioned", "email": email, "user_id": user_id}
@app.post("/admin/provision-guest")
@limiter.limit("20/minute")
async def admin_provision_guest(request: Request):
"""Provision a GUEST account (the admin panel's invite action).
Same auth as /admin/provision (X-Admin-Token). Body: {"email": "..."} —
ONE field, the standard guest-invite form. No name is collected and none
is derived: the guest chooses their own display name and username during
Cloudron account setup, and the invitation email greets neutrally. The
inviter is always the co-op's ADMIN account
(ADMIN_INVITER_EMAIL, default info@inference.coop) — the panel has no
inviter input, and the ledger records the admin account.
Runs the guest pipeline: Cloudron user in the guests group, a $2 LiteLLM
team with NO monthly reset, the portal row marked as a guest with a
GUEST_DAYS expiry, the guest welcome email, the newsletter subscription,
and a guests-ledger row recording inviter → recipient. Admin invites
provision IMMEDIATELY (no pending state — the operator IS the approver)
and are NOT bound by the member's one-per-30-days cap; the recipient-side
guards still run (one guest per recipient ever, no existing portal row).
No Loomio/Matrix.
"""
_verify_admin_token(request)
body = await request.json()
email = (body.get("email") or "").strip().lower()
# No name is collected and none is derived: the guest chooses their own
# display name and username during Cloudron account setup, and the
# invitation email greets neutrally.
name = (body.get("name") or "").strip()
if not email or "@" not in email:
raise HTTPException(400, "A valid recipient email is required")
# The inviter is the admin account, never a caller-supplied member email:
# this endpoint speaks for the operator, and admins have no monthly cap.
inviter = ADMIN_INVITER_EMAIL
user_id = await provision_guest_member(
email, name, inviter, admin=True, inviter_display="The co-op"
)
return {
"status": "provisioned", "account_type": "guest", "email": email,
"inviter": inviter, "user_id": user_id, "budget": GUEST_BUDGET,
"expires_in_days": int(GUEST_DAYS),
}
@app.post("/admin/guest-convert")
@limiter.limit("20/minute")
async def admin_guest_convert(request: Request):
"""Convert a guest account to a member when OC cannot auto-match.
The automatic conversion matches the guest's OWN email against the live OC
backer list. When the membership was paid under a DIFFERENT Open
Collective email, the operator links it here: {"email": "<guest account
email>", "oc_email": "<the OC email the contribution is under>"}. The OC
member record (name, slug) is resolved from oc_email and stored on the row
(members.oc_email) so the sweep and the webhook keep matching renewals
under that address. Omitting oc_email converts on the account's own email
without an OC lookup (the operator asserts the membership exists; the row
is then matched by its own email as usual).
"""
_verify_admin_token(request)
body = await request.json()
email = (body.get("email") or "").strip().lower()
oc_email = (body.get("oc_email") or "").strip().lower()
if not email or "@" not in email:
raise HTTPException(400, "A valid guest account email is required")
if oc_email and "@" not in oc_email:
raise HTTPException(400, "oc_email must be a valid email")
if not is_guest_account(email):
raise HTTPException(409, f"{email} is not a guest account (already converted or a member row)")
name, slug = "", ""
if oc_email:
rec = next((m for m in await fetch_members()
if (m.get("email") or "").strip().lower() == oc_email), None)
if not rec:
raise HTTPException(404, f"No Open Collective member found under {oc_email}")
name = rec.get("name") or ""
slug = rec.get("slug") or ""
conn = get_db()
conn.execute("UPDATE members SET oc_email = ? WHERE email = ?", (oc_email, email))
conn.commit()
conn.close()
user_id = await convert_guest_to_member(email, name or email.split("@")[0], slug)
return {
"status": "converted", "email": email, "oc_email": oc_email or None,
"user_id": user_id,
"note": ("account keeps its own email and login; group moved guests → members; "
"cap lifted to the member allowance with unspent guest credit carried; "
"anchor day (join day) kept"),
}
# TRANSITION ALIASES (guest rename, 2026-11-08): the deployed dashboard and
# panel builds predate the guest rename and still call the old paths. These
# answer identically (auth runs inside the real handler) until the renamed
# builds are deployed; REMOVE once no caller remains.
@app.post("/admin/provision-gift")
@limiter.limit("20/minute")
async def admin_provision_gift_alias(request: Request):
return await admin_provision_guest(request)
@app.post("/admin/gift-convert")
@limiter.limit("20/minute")
async def admin_gift_convert_alias(request: Request):
return await admin_guest_convert(request)
# ---------------------------------------------------------------------------
# Guest-request approval (the member self-serve trial gate).
# ---------------------------------------------------------------------------
def _request_row(row) -> dict:
return {
"id": row[0], "inviter": row[1], "recipient": row[2],
"name": row[3], "created_at": row[4], "decided_at": row[5],
"status": row[6], "note": row[7],
}
@app.get("/admin/guest-requests")
@limiter.limit("60/minute")
async def admin_guest_requests(request: Request):
"""The member guest-request queue and the guests ledger, for the panel.
Authenticated by the X-Admin-Token header. Returns the PENDING requests
awaiting the operator's decision (oldest first), the recently DECIDED
ones (approved/rejected, newest first), and the guests ledger itself —
one view where pending vs approved (and every later state) is visible.
"""
_verify_admin_token(request)
conn = get_db()
pending = conn.execute(
"SELECT id, inviter_email, recipient_email, recipient_name, created_at, "
"decided_at, status, note FROM guest_requests WHERE status = 'pending' ORDER BY id"
).fetchall()
decided = conn.execute(
"SELECT id, inviter_email, recipient_email, recipient_name, created_at, "
"decided_at, status, note FROM guest_requests WHERE status != 'pending' "
"ORDER BY id DESC LIMIT 100"
).fetchall()
ledger = conn.execute(
"SELECT id, inviter_email, recipient_email, created_at, status "
"FROM guests ORDER BY id DESC LIMIT 100"
).fetchall()
conn.close()
return {
"pending": [_request_row(r) for r in pending],
"decided": [_request_row(r) for r in decided],
"ledger": [
{"id": i, "inviter": inv, "recipient": rec, "created_at": created,
"status": status}
for i, inv, rec, created, status in ledger
],
}
@app.post("/admin/guest-requests/{request_id}/approve")
@limiter.limit("20/minute")
async def admin_guest_request_approve(request_id: int, request: Request):
"""Approve a pending guest request — the trial gate's approval action.
Approval fires the FULL guest pipeline (provision_guest_member: Cloudron
user, $2 credit, GUEST_DAYS box, invitation email naming the inviter,
newsletter subscribe, ledger row) — the same path the operator's
/admin/provision-guest runs, so every guard re-runs. If state moved since
the request (the recipient signed up; another invitation landed), the
refusal is recorded on the request and surfaced — the queue never
silently drops a decision.
"""
_verify_admin_token(request)
conn = get_db()
row = conn.execute(
"SELECT id, inviter_email, recipient_email, recipient_name, status "
"FROM guest_requests WHERE id = ?", (request_id,)
).fetchone()
conn.close()
if not row:
raise HTTPException(404, f"No guest request #{request_id}")
_, inviter, recipient, name, status = row
if status != "pending":
raise HTTPException(409, f"Request #{request_id} is already {status}")
try:
user_id = await provision_guest_member(
recipient, name or "", inviter)
except HTTPException as e:
conn = get_db()
conn.execute(
"UPDATE guest_requests SET status = 'rejected', "
"decided_at = datetime('now'), note = ? WHERE id = ?",
(str(e.detail), request_id),
)
conn.commit()
conn.close()
raise HTTPException(e.status_code, e.detail)
conn = get_db()
conn.execute(
"UPDATE guest_requests SET status = 'approved', "
"decided_at = datetime('now') WHERE id = ?", (request_id,)
)
conn.commit()
conn.close()
logger.info("Guest request #%s approved: %s invited %s (provisioned)",
request_id, inviter, recipient)
return {
"status": "approved", "request_id": request_id, "recipient": recipient,
"inviter": inviter, "user_id": user_id,
"budget": GUEST_BUDGET, "expires_in_days": int(GUEST_DAYS),
}
@app.post("/admin/guest-requests/{request_id}/reject")
@limiter.limit("20/minute")
async def admin_guest_request_reject(request_id: int, request: Request):
"""Reject a pending guest request (the trial gate's rejection action).
Nothing was ever sent to the recipient — no account, no email — so the
rejection tells only the INVITING member, through their own guest trail
(status 'not approved'). An optional note records why.
"""
_verify_admin_token(request)
body = await request.json() if request.headers.get("content-length", "0") not in ("", "0") else {}
note = (body.get("note") or "").strip() or None
conn = get_db()
row = conn.execute(
"SELECT id, status FROM guest_requests WHERE id = ?", (request_id,)
).fetchone()
if not row:
conn.close()
raise HTTPException(404, f"No guest request #{request_id}")
if row[1] != "pending":
conn.close()
raise HTTPException(409, f"Request #{request_id} is already {row[1]}")
conn.execute(
"UPDATE guest_requests SET status = 'rejected', "
"decided_at = datetime('now'), note = ? WHERE id = ?", (note, request_id)
)
conn.commit()
conn.close()
logger.info("Guest request #%s rejected", request_id)
return {"status": "rejected", "request_id": request_id, "note": note}
@app.post("/admin/org-link")
@limiter.limit("20/minute")
async def admin_org_link(request: Request):
"""Link an OC organizational membership to a portal account.
OC Organizations never expose an email to our integration, so org-backed
memberships are linked manually: body {"org_slug": "...", "email": "...",
"name": "..."}. Creates (or reuses) the members row for the email with
org_slug set, and provisions the full pipeline — Cloudron user, LiteLLM
key, and the welcome email containing the Cloudron account-setup link, so
the organization's contact can complete signup. The nightly sweep keeps
the account active while the org holds the Membership tier and deactivates
it when the org's dues lapse (dues monitoring, automatic).
"""
_verify_admin_token(request)
body = await request.json()
org_slug = (body.get("org_slug") or "").strip().lower()
email = (body.get("email") or "").strip().lower()
name = (body.get("name") or org_slug).strip()
if not org_slug or not email or "@" not in email:
raise HTTPException(400, "org_slug and a valid email are required")
try:
org = (await fetch_org_backers()).get(org_slug)
except Exception as e:
# OC unreachable: link without provisioning — the sweep provisions and
# activates once OC is reachable again.
logger.warning("org-link: OC status fetch failed for %s (linking anyway): %s", org_slug, e)
org = None
conn = get_db()
existing = conn.execute("SELECT email FROM members WHERE org_slug = ?", (org_slug,)).fetchone()
# slug = the org slug (this row represents the org's contact, not an OC
# individual — provision_member() overwrites slug/cloudron_user_id/key for
# emails that resolve to a real OC account).
conn.execute(
"INSERT INTO members (email, slug, active, org_slug) VALUES (?, ?, 1, ?) "
"ON CONFLICT(email) DO UPDATE SET org_slug = excluded.org_slug, active = 1",
(email, org_slug, org_slug),
)
conn.commit()
conn.close()
provisioned = False
if org: # only provision while the org's membership is active
try:
await provision_member(email, name, org_slug)
provisioned = True
except Exception as e:
logger.warning("org-link: provision failed for %s: %s", email, e)
return {
"status": "linked",
"org_slug": org_slug,
"email": email,
"org_active": org is not None,
"tier": (org or {}).get("tier"),
"provisioned": provisioned,
"replaced_existing_link": bool(existing),
"note": "the nightly sweep deactivates this account automatically if the org's dues lapse",
}
@app.get("/admin/org-links")
@limiter.limit("30/minute")
async def admin_org_links(request: Request):
"""List organization-linked accounts with their live OC dues status."""
_verify_admin_token(request)
org_backers = await fetch_org_backers()
conn = get_db()
rows = conn.execute(
"SELECT email, org_slug, active FROM members WHERE org_slug IS NOT NULL"
).fetchall()
conn.close()
links = []
for email, org_slug, active in rows:
org = org_backers.get(org_slug)
links.append({
"org_slug": org_slug,
"org_name": (org or {}).get("name") or org_slug,
"email": email,
"account_active": bool(active),
"dues_active": org is not None,
"tier": (org or {}).get("tier"),
})
return {"links": links}
@app.post("/admin/set-balance")
@limiter.limit("20/minute")
async def admin_set_balance(request: Request):
"""Set (or add to) a member's credit balance and sync it to LiteLLM.
The flexible alternative to a fixed $15/month allowance. A payment (from OC
or another system) is recorded by adjusting the balance here, which pushes
the new cap to the member's team (every chat/API key under it follows).
Body: {"email": "...", "balance": 25.0} → set absolute balance
{"email": "...", "add": 10.0} → add to current balance
"""
_verify_admin_token(request)
body = await request.json()
email = (body.get("email") or "").strip().lower()
if not email:
raise HTTPException(400, "Missing email")
current = get_member_balance(email)
if "add" in body:
new_balance = current + float(body["add"])
elif "balance" in body:
new_balance = float(body["balance"])
else:
raise HTTPException(400, "Provide 'balance' or 'add'")
if new_balance < 0:
raise HTTPException(400, "Balance cannot be negative")
set_member_balance(email, new_balance)
applied = await sync_team_budget_from_balance(email)
return {"status": "updated", "email": email, "balance": new_balance, "team_budget": applied}
@app.post("/admin/set-credits")
@limiter.limit("20/minute")
async def admin_set_credits(request: Request):
"""Adjust a member's non-expiring credit balance (with ledger entry).
Body: {"email": "...", "add": 10.0, "reason": "goodwill"} → add/subtract
{"email": "...", "set": 0.0, "reason": "correction"} → set absolute
"""
_verify_admin_token(request)
body = await request.json()
email = (body.get("email") or "").strip().lower()
if not email:
raise HTTPException(400, "Missing email")
reason = (body.get("reason") or "admin adjustment").strip()
if "add" in body:
delta = float(body["add"])
new_balance = add_credits(email, delta, reason=reason)
elif "set" in body:
target = float(body["set"])
current = get_credit_balance(email)
new_balance = add_credits(email, target - current, reason=reason)
else:
raise HTTPException(400, "Provide 'add' or 'set'")
applied = await sync_team_budget_from_balance(email)
return {
"status": "updated", "email": email,
"credit_balance": new_balance, "team_budget": applied,
}
@app.get("/admin/credits/{email}")
@limiter.limit("30/minute")
async def admin_credit_ledger(email: str, request: Request):
"""Return a member's credit balance and full ledger (audit trail)."""
_verify_admin_token(request)
conn = get_db()
rows = conn.execute(
"SELECT delta, reason, reference, created_at FROM credit_ledger "
"WHERE email = ? ORDER BY created_at DESC",
(email,),
).fetchall()
conn.close()
return {
"email": email,
"credit_balance": get_credit_balance(email),
"ledger": [
{"delta": r[0], "reason": r[1], "reference": r[2], "created_at": r[3]}
for r in rows
],
}
@app.get("/admin/held-credits")
@limiter.limit("30/minute")
async def admin_held_credits(request: Request):
"""List held credit packs (non-member purchases awaiting refund or grant)."""
_verify_admin_token(request)
conn = get_db()
rows = conn.execute(
"SELECT id, slug, email, name, amount, oc_reference, status, created_at, resolved_at "
"FROM held_credits ORDER BY created_at DESC"
).fetchall()
conn.close()
return {
"held": [
{"id": r[0], "slug": r[1], "email": r[2], "name": r[3], "amount": r[4],
"reference": r[5], "status": r[6], "created_at": r[7], "resolved_at": r[8]}
for r in rows
]
}
@app.post("/admin/held-credits/{hold_id}/refund")
@limiter.limit("20/minute")
async def admin_held_refund(hold_id: int, request: Request):
"""Mark a held credit pack as refunded (the refund itself is done manually
in the Open Collective dashboard — this records it and updates the hold)."""
_verify_admin_token(request)
conn = get_db()
cur = conn.execute(
"UPDATE held_credits SET status = 'refunded', resolved_at = datetime('now') "
"WHERE id = ? AND status = 'held'",
(hold_id,),
)
conn.commit()
updated = cur.rowcount
conn.close()
if not updated:
raise HTTPException(404, "Held credit not found (or already resolved)")
return {"status": "refunded", "id": hold_id}
@app.post("/admin/held-credits/{hold_id}/grant")
@limiter.limit("20/minute")
async def admin_held_grant(hold_id: int, request: Request):
"""Apply a held credit pack to a specified member account.
Body: {"email": "member@..."} — the member account that should receive the
held credits. This covers purchases made by someone other than the member
(e.g. a credit pack bought on another account's behalf), which the
provisioning-time auto-grant cannot match because it keys on the buyer's
own email/slug.
Atomic and idempotent: the balance update and the ledger insert happen in
one transaction, and the hold is flipped `held` → `granted` in the same
transaction. The ledger reference is `held:<hold_id>` (the same shape the
provisioning-time auto-grant uses), so a re-run cannot double-apply.
"""
_verify_admin_token(request)
body = await request.json()
email = (body.get("email") or "").strip().lower()
if not email:
raise HTTPException(400, "Missing email")
conn = get_db()
row = conn.execute(
"SELECT id, slug, email, name, amount, oc_reference, status "
"FROM held_credits WHERE id = ?",
(hold_id,),
).fetchone()
if not row:
conn.close()
raise HTTPException(404, "Held credit not found")
if row[6] != "held":
conn.close()
raise HTTPException(409, f"Held credit already {row[6]}")
dest = conn.execute(
"SELECT active FROM members WHERE LOWER(email) = ?", (email,)
).fetchone()
if not dest:
conn.close()
raise HTTPException(404, f"No member account for {email}")
if not dest[0]:
conn.close()
raise HTTPException(409, f"Member {email} is not active; refusing")
amount = float(row[4] or 0.0)
reference = f"held:{hold_id}"
if credit_reference_exists(reference):
conn.close()
raise HTTPException(409, f"Credit already recorded for {reference}")
buyer = row[2] or row[1]
buyer_name = row[3] or row[1]
reason = (
f"credit pack (${amount:.2f}) bought by {buyer} ({buyer_name}); "
f"held credit transferred to this member account at the operator's "
f"instruction (hold #{hold_id}, {row[5] or 'no oc ref'})"
)
# One transaction: flip the hold, move the balance, write the audit row.
cur = conn.execute(
"UPDATE held_credits SET status = 'granted', resolved_at = datetime('now') "
"WHERE id = ? AND status = 'held'",
(hold_id,),
)
if cur.rowcount != 1:
conn.close()
raise HTTPException(409, "Held credit already resolved")
conn.execute(
"UPDATE members SET credit_balance = MAX(0, COALESCE(credit_balance, 0) + ?) "
"WHERE LOWER(email) = ?",
(amount, email),
)
conn.execute(
"INSERT INTO credit_ledger (email, delta, reason, reference) VALUES (?, ?, ?, ?)",
(email, amount, reason, reference),
)
new_balance = conn.execute(
"SELECT credit_balance FROM members WHERE LOWER(email) = ?", (email,)
).fetchone()[0]
conn.commit()
conn.close()
applied = await sync_team_budget_from_balance(email)
logger.info(
"Held credit #%s ($%.2f) from %s granted to %s (balance %s)",
hold_id, amount, buyer, email, new_balance,
)
return {
"status": "granted",
"hold_id": hold_id,
"email": email,
"credits_added": amount,
"credit_balance": float(new_balance),
"reference": reference,
"oc_reference": row[5],
"team_budget": applied,
}
# Operator accounts — shown under a separate "admin" status in the panel and
# excluded from the member counts. They're infrastructure accounts (the SSO
# owner and the bot), not dues-paying members.
OPERATOR_EMAILS = {
e.strip().lower()
for e in os.environ.get("OPERATOR_EMAILS", "info@inference.coop,bot@inference.coop").split(",")
if e.strip()
}
@app.get("/admin/overview")
@limiter.limit("30/minute")
async def admin_overview(request: Request):
"""Return a co-op-wide overview for the admin dashboard.
Authenticated by the X-Admin-Token header. Returns every member (active +
inactive) with their balance, spend, remaining, and team reset date, plus
co-op aggregates (total members, total spend). Guest (guest) accounts come
back in their own `guests` list — never inside `members` — so no consumer
can count them as members by accident; their spend is broken out in the
totals rather than folded into the member figures.
"""
_verify_admin_token(request)
conn = get_db()
rows = conn.execute(
"SELECT email, active, balance, cloudron_user_id, org_slug, matrix_mxid, "
"account_type, oc_email, guest_expires_at FROM members ORDER BY email"
).fetchall()
# Guest inviter/expiry from the guests ledger (latest row per recipient) —
# the abuse-protection context the panel shows on the guest pill.
guest_info: dict[str, dict] = {}
try:
for r in conn.execute(
"SELECT recipient_email, inviter_email, guest_expires_at, status "
"FROM guests WHERE id IN (SELECT MAX(id) FROM guests GROUP BY recipient_email)"
).fetchall():
guest_info[r[0]] = {"inviter": r[1], "expires": r[2], "status": r[3]}
except Exception as e:
logger.warning("admin overview: guests ledger read failed: %s", e)
conn.close()
# Fetch Cloudron users once to map user_id → username + inviteAccepted.
# username is set only after the member completes account setup (chooses a
# username + password via the setup link); inviteAccepted tracks that same
# step, so "activated" = inviteAccepted (username present).
usernames = {}
invite_accepted = {}
try:
async with httpx.AsyncClient(timeout=15.0) as client:
users = await cloudron_fetch_all_users(client)
if users is None:
raise RuntimeError("cloudron users fetch failed (page 1)")
for u in users:
uid = u.get("id")
if u.get("username"):
usernames[uid] = u["username"]
invite_accepted[uid] = bool(u.get("inviteAccepted"))
except Exception as e:
logger.warning("admin overview: cloudron users fetch failed: %s", e)
# Fetch team spend/budget from LiteLLM once.
teams = {}
try:
async with httpx.AsyncClient(timeout=15.0) as client:
r = await client.get(
f"{LITELLM_BASE}/team/list",
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
)
r.raise_for_status()
for t in r.json():
teams[t.get("team_alias")] = t
except Exception as e:
logger.warning("admin overview: team/list failed: %s", e)
# All-time spend per team, from SpendLogs (the team `spend` counter is
# cycle-only and resets at each team's budget_reset_at — the panel needs both
# to avoid the "month change wiped the numbers" confusion).
# Covers agent teams as well as members: an agent's cycle counter resets the
# same way, and its all-time spend is the only figure that stays meaningful.
all_time_spend: dict[str, float] = {}
try:
conn = _pg_connect()
try:
with conn.cursor() as cur:
cur.execute(
"""
SELECT t.team_alias, COALESCE(SUM(s.spend), 0)
FROM "LiteLLM_TeamTable" t
LEFT JOIN "LiteLLM_SpendLogs" s ON s."team_id" = t.team_id
WHERE t.team_alias LIKE 'member:%' OR t.team_alias LIKE 'agent:%'
GROUP BY 1
"""
)
all_time_spend = {alias: float(v or 0) for alias, v in cur.fetchall()}
finally:
conn.close()
except Exception as e:
logger.warning("admin overview: all-time spend query failed: %s", e)
members = []
guests = []
admins = []
total_spend = 0.0
total_all_time = 0.0
operator_spend = 0.0
operator_all_time = 0.0
guest_spend = 0.0
guest_all_time = 0.0
for email, active, balance, user_id, org_slug, matrix_mxid, acct_type, oc_email, guest_exp in rows:
team = teams.get(f"member:{email}", {})
spend = float(team.get("spend") or 0.0)
member_all_time = float(all_time_spend.get(f"member:{email}", 0.0))
# Normalize floating-point residue (e.g. -5.55e-17 after a cycle reset)
# so the panel/dashboards never render "-0.00".
if abs(spend) < 1e-9:
spend = 0.0
if abs(member_all_time) < 1e-9:
member_all_time = 0.0
credits = get_credit_balance(email)
eff = compute_max_budget(
float(balance) if balance is not None else MEMBER_BUDGET,
credits,
spend,
)
mstate, mstate_reason = member_membership_state(email, bool(active))
entry = {
"email": email,
"all_time_spend": round(member_all_time, 2),
"username": usernames.get(user_id),
"activated": invite_accepted.get(user_id, False),
"active": bool(active),
"state": mstate,
"state_reason": mstate_reason,
"manual": email.lower() in MANUAL_MEMBERS,
"org": org_slug or None,
"matrix_mxid": matrix_mxid or None,
"balance": float(balance) if balance is not None else MEMBER_BUDGET,
"credit_balance": credits,
"credits_remaining": max(0.0, credits - eff["credits_used"]),
"allowance_used": eff["allowance_used"],
"credits_used": eff["credits_used"],
"spend": spend,
"remaining": max(eff["max_budget"] - spend, 0.0),
"reset_at": team.get("budget_reset_at"),
"cloudron_user_id": user_id,
"account_type": acct_type or "member",
"oc_email": oc_email or None,
"guest_expires_at": guest_exp or None,
}
if (acct_type or "member") == "guest":
gi = guest_info.get(email) or {}
entry["guest_inviter"] = gi.get("inviter")
entry["guest_status"] = gi.get("status")
if email.lower() in OPERATOR_EMAILS:
# Operator accounts: surfaced under a separate "admin" category in
# the panel, excluded from member counts and aggregates (their
# usage is a collective expense, not member activity).
entry["admin"] = True
admins.append(entry)
total_spend += spend
total_all_time += member_all_time
operator_spend += spend
operator_all_time += member_all_time
elif (acct_type or "member") == "guest":
# Guest (guest) accounts: their own list, never members. Excluded
# from the member count AND from the spend totals a reader reads
# as member activity — a guest is not a membership. Their spend is
# broken out separately in the totals so it stays visible (the
# same reasoning as the agents below).
guests.append(entry)
guest_spend += spend
guest_all_time += member_all_time
else:
members.append(entry)
total_spend += spend
total_all_time += member_all_time
# Bot and sandbox teams: real spend, but not members. Surfaced in their own
# category so the co-op's own usage is accountable without inflating member
# counts — the same reasoning as the operator accounts above.
agents = [
{
"alias": alias,
# Display name: the panel's section header already says "Agents", so the
# "agent:" prefix is noise there. `alias` keeps the full string.
"name": alias.split(":", 1)[1] if ":" in alias else alias,
"spend": float(t.get("spend") or 0.0),
# Cycle counter resets at budget_reset_at (the 1st, for the agents), so
# the panel shows all-time for these accounts instead of a number that
# reads as zeroed every month.
"all_time_spend": round(all_time_spend.get(alias, 0.0), 2),
"budget": t.get("max_budget"),
"team_members": len(t.get("members_with_roles") or []),
}
for alias, t in teams.items()
# member:* are members; sandbox:* are throwaway test accounts rather than
# agents of the co-op. Their spend stops being visible anywhere once
# filtered here — accepted knowingly.
if alias and not alias.startswith(("member:", "sandbox:"))
]
agents.sort(key=lambda a: a["spend"], reverse=True)
return {
"members": members,
"guests": guests,
"admins": admins,
"agents": agents,
"totals": {
"members": len(members),
"active": sum(1 for m in members if m["active"]),
# Guests are never members: their own count + spend breakout, so a
# consumer never has to guess whether a figure includes them.
"guests": len(guests),
"guests_active": sum(1 for g in guests if g["active"]),
"guest_spend": round(guest_spend, 2),
"guest_all_time_spend": round(guest_all_time, 2),
# total_spend spans members AND the co-op's own operator accounts.
# operator_spend is the breakout of that portion, so a consumer
# never has to guess whether a figure includes the co-op's own use.
"total_spend": round(total_spend, 2),
"total_all_time_spend": round(total_all_time, 2),
"operator_spend": round(operator_spend, 2),
"operator_all_time_spend": round(operator_all_time, 2),
},
}
def _metrics_provenance(tokens_measured: int | None, tokens_estimated: int | None) -> str:
"""Classify a day's measured-vs-estimated split for the API.
"measured" — every token rested on a directly measured factor.
"estimated" — every token rested on a non-measured factor (borrowed
SAME_FAMILY or the flat proxy).
"mixed" — the day's total is a blend of both.
"none" — no provenance recorded: either the day has no output tokens
or the row predates these columns (both split columns NULL).
Deliberately NOT "measured" — an unknown split must never read
as a confident measurement.
"""
if tokens_measured is None and tokens_estimated is None:
return "none"
m = tokens_measured or 0
e = tokens_estimated or 0
if m and e:
return "mixed"
if m:
return "measured"
if e:
return "estimated"
return "none"
def metrics_series(days: int = 90) -> list[dict]:
"""Daily metrics rows, oldest first.
One query, two callers: the admin panel (via /admin/metrics/series) and the
member dashboard (via /broker/metrics). Columns are cumulative as of that
day except spend_usd/requests/out_tokens/energy_wh/co2e_g, which are that
day only; NULL means the source could not be read that day.
energy_wh and co2e_g are computed in metrics_snapshot.py from the day's
output tokens against a factor table: measured for GreenPT models (their API
returns an `impact` object), estimated for the rest. See co-op/support#26.
Each row also carries its provenance: `tokens_measured`/`tokens_estimated`
(the output-token split across measured vs non-measured factors),
`factor_revision` (a short stable hash of the factor table that produced the
row), and a derived `provenance` indicator — measured | mixed | estimated |
none. Rows written before these columns exist have NULL splits and report
`provenance: "none"`, never a confident "measured". See docs/energy-metrics.md.
"""
limit = max(1, min(int(days or 90), 730))
conn = get_db()
try:
rows = conn.execute(
"SELECT day, members_active, members_total, members_new, newsletter_subs, "
"matrix_space_members, spend_usd, operator_spend_usd, requests, out_tokens, energy_wh, co2e_g, "
"tokens_measured, tokens_estimated, factor_revision, recorded_at "
"FROM metrics_daily ORDER BY day DESC LIMIT ?",
(limit,),
).fetchall()
finally:
conn.close()
cols = [
"day",
"members_active",
"members_total",
"members_new",
"newsletter_subs",
"matrix_space_members",
"spend_usd",
"operator_spend_usd",
"requests",
"out_tokens",
"energy_wh",
"co2e_g",
"tokens_measured",
"tokens_estimated",
"factor_revision",
"recorded_at",
]
series = []
for row in reversed(rows):
rec = dict(zip(cols, row))
rec["provenance"] = _metrics_provenance(
rec["tokens_measured"], rec["tokens_estimated"]
)
series.append(rec)
return series
# ---------------------------------------------------------------------------
# Money series — the admin panel's Money chart: cumulative Open Collective
# revenue, token spend over time, and the OC balance.
#
# The three series have genuinely different provenance, and the endpoint says
# so rather than papering over it:
# revenue — OC contribution transactions (CREDIT / CONTRIBUTION / not a
# refund), each with a createdAt and an amount, summed
# chronologically. Monthly and one-time contributions both count;
# org-backed contributions count through the org account that
# paid. Exact from each contribution's start date (2026-09-06).
# spend — LiteLLM SpendLogs, summed per UTC day in ONE GROUP BY pass over
# the full history (134k+ rows), with no team filter — members,
# operators and agent teams all count, matching spend_by_day() in
# metrics_snapshot.py.
# balance — the OC API exposes only the collective's CURRENT balance; there
# is NO historical series. The current balance is returned live
# and a real history accrues from the daily metrics_snapshot
# snapshots into metrics_daily.oc_balance_usd. Days before that
# pipeline stay NULL — a gap in the chart, never a zero.
#
# Aggregate totals only: no member emails or names are selected or returned.
#
# Caching: the OC queries are slow (network + pagination), so the payload is
# computed at most once per MONEY_SERIES_TTL seconds (default 1h) and served
# from an in-memory cache between refreshes; a failed refresh serves the
# previous payload marked `stale: true` rather than breaking the chart. The
# daily snapshot owns history accrual, so no schedule entry is needed here.
# ---------------------------------------------------------------------------
MONEY_SERIES_TTL = float(os.environ.get("MONEY_SERIES_TTL", "3600"))
_MONEY_CACHE: dict = {"data": None, "ts": 0.0}
_money_lock = asyncio.Lock()
# Hard stop for the transactions pagination: 500 rows/page x 40 pages. The
# collective holds ~160 contribution transactions; the cap only exists so a
# surprising totalCount can never turn into an unbounded loop.
_MONEY_TX_PAGE = 500
_MONEY_TX_MAX_PAGES = 40
async def _oc_graphql(query: str, variables: dict) -> dict:
"""One authenticated OC GraphQL call (browser UA — Cloudflare blocks the
default Python UA with a 403/1010 bot signature). Raises on HTTP or
GraphQL errors so callers can fail per-source."""
async with httpx.AsyncClient(timeout=30.0) as client:
r = await client.post(
OC_GRAPHQL_URL,
headers={"Content-Type": "application/json",
"Authorization": f"Bearer {OC_PERSONAL_TOKEN}",
"Api-Version": "2022-01-01",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"},
json={"query": query, "variables": variables},
)
if r.status_code != 200:
raise RuntimeError(f"OC GraphQL failed: HTTP {r.status_code}")
body = r.json()
if body.get("errors"):
raise RuntimeError(f"OC GraphQL errors: {str(body['errors'])[:200]}")
return body.get("data") or {}
async def oc_collective_balance() -> dict:
"""The collective's CURRENT balance (there is no historical endpoint)."""
if not OC_PERSONAL_TOKEN:
raise RuntimeError("OC_PERSONAL_TOKEN not set")
data = await _oc_graphql(
"query Bal($slug: String) { collective(slug: $slug) "
"{ stats { balance { valueInCents currency } } } }",
{"slug": OC_COLLECTIVE_SLUG},
)
bal = (
((data.get("collective") or {}).get("stats") or {}).get("balance") or {}
)
cents = bal.get("valueInCents")
if cents is None:
raise RuntimeError("OC balance field missing from response")
return {
"value": round(int(cents) / 100.0, 2),
"currency": bal.get("currency") or "USD",
"as_of": datetime.now(timezone.utc).isoformat(timespec="seconds"),
}
async def oc_contribution_by_day() -> tuple[dict[str, float], int, float]:
"""{UTC day: contribution $} over every CREDIT CONTRIBUTION transaction.
Paginated (variables, never interpolation — the same injection rule as
get_order_amount). Returns (by_day, count, total). Payer identities are
deliberately NOT selected: this feed is aggregate-only.
"""
if not OC_PERSONAL_TOKEN:
raise RuntimeError("OC_PERSONAL_TOKEN not set")
by_day: dict[str, float] = {}
count = 0
total_cents = 0
offset = 0
for _page in range(_MONEY_TX_MAX_PAGES):
data = await _oc_graphql(
"query Contribs($slug: String, $limit: Int!, $offset: Int!) "
"{ collective(slug: $slug) { transactions(type: CREDIT, limit: $limit, "
"offset: $offset) { totalCount nodes { createdAt type kind isRefund "
"amount { valueInCents } } } } }",
{"slug": OC_COLLECTIVE_SLUG, "limit": _MONEY_TX_PAGE, "offset": offset},
)
coll = ((data.get("collective") or {}).get("transactions") or {})
total_count = int(coll.get("totalCount") or 0)
nodes = coll.get("nodes") or []
if not nodes:
break
for t in nodes:
if (t.get("type") or "").upper() != "CREDIT":
continue
if (t.get("kind") or "").upper() != "CONTRIBUTION":
continue
if t.get("isRefund"):
continue
dtv = _parse_oc_dt(t.get("createdAt"))
cents = ((t.get("amount") or {}).get("valueInCents")) or 0
if dtv is None:
continue
day = dtv.astimezone(timezone.utc).strftime("%Y-%m-%d")
by_day[day] = by_day.get(day, 0.0) + int(cents) / 100.0
count += 1
total_cents += int(cents)
offset += len(nodes)
if offset >= total_count or len(nodes) < _MONEY_TX_PAGE:
break
return by_day, count, round(total_cents / 100.0, 2)
def litellm_spend_by_day() -> tuple[dict[str, float], float, int]:
"""{UTC day: $}, total, and row count from LiteLLM's SpendLogs — one
GROUP BY pass over the whole table. No team filter: the co-op's total
spans members, operators and agent teams (same definition as
spend_by_day() in metrics_snapshot.py, so the two agree by construction)."""
if not litellm_db_ready():
raise RuntimeError("LiteLLM DB credentials not configured")
conn = _pg_connect()
try:
with conn.cursor() as cur:
cur.execute(
"""SELECT to_char("startTime", 'YYYY-MM-DD') AS day,
COALESCE(SUM("spend"), 0) AS spend, COUNT(*) AS n
FROM "LiteLLM_SpendLogs"
GROUP BY 1 ORDER BY 1"""
)
rows = cur.fetchall()
finally:
conn.close()
by_day = {r[0]: round(float(r[1]), 6) for r in rows}
return by_day, round(sum(by_day.values()), 6), int(sum(int(r[2]) for r in rows))
def oc_balance_history() -> dict[str, float]:
"""metrics_daily.oc_balance_usd rows — the daily snapshots. Empty before
the pipeline deployed (earlier balances were never recorded)."""
conn = get_db()
try:
rows = conn.execute(
"SELECT day, oc_balance_usd FROM metrics_daily "
"WHERE oc_balance_usd IS NOT NULL ORDER BY day"
).fetchall()
finally:
conn.close()
return {r[0]: float(r[1]) for r in rows}
def _seed_balance_row(day: str, value: float) -> None:
"""Fold today's live OC balance into metrics_daily WITHOUT creating a row
and WITHOUT touching any other column. The daily snapshot owns the rest of
that row (a partial INSERT here would punch NULL gaps into the members
chart for the day). When the snapshot has not written today's row yet
there is nothing to seed and this is a no-op; its own 03:15 read then
writes the balance itself."""
if value is None:
return
conn = get_db()
try:
conn.execute(
"UPDATE metrics_daily SET oc_balance_usd = ? WHERE day = ?",
(float(value), day),
)
conn.commit()
finally:
conn.close()
def _build_money_series(rev_by_day: dict[str, float] | None,
spend_by_day: dict[str, float] | None,
bal_hist: dict[str, float],
today_key: str) -> list[dict]:
"""One row per UTC day across the full span, oldest first.
revenue_cum / spend_cum are running sums; days before a series' first
entry carry 0 — a genuine zero (the sources were readable and nothing had
arrived yet), unlike oc_balance, which is NULL before the first daily
snapshot and must render as a gap.
"""
day_keys = set(rev_by_day or {}) | set(spend_by_day or {}) | set(bal_hist or {})
day_keys.add(today_key)
cur = datetime.strptime(min(day_keys), "%Y-%m-%d").date()
end = datetime.strptime(today_key, "%Y-%m-%d").date()
rev_cum = 0.0
spend_cum = 0.0
out: list[dict] = []
while cur <= end:
key = cur.strftime("%Y-%m-%d")
rev_today = (rev_by_day or {}).get(key, 0.0)
spend_today = (spend_by_day or {}).get(key, 0.0)
rev_cum += rev_today
spend_cum += spend_today
out.append({
"day": key,
"revenue_cum": round(rev_cum, 2) if rev_by_day is not None else None,
"spend_day": round(spend_today, 6) if spend_by_day is not None else None,
"spend_cum": round(spend_cum, 2) if spend_by_day is not None else None,
"oc_balance": bal_hist.get(key),
})
cur += timedelta(days=1)
return out
async def _compute_money_payload() -> dict:
"""Fetch every source independently; a failed source yields NULL columns
plus an `errors` entry rather than taking the whole payload down. Only an
all-sources failure raises (the caller then serves stale or 503s)."""
errors: list[str] = []
today_key = datetime.now(timezone.utc).strftime("%Y-%m-%d")
rev_by_day = None
contrib_count = None
revenue_total = None
try:
rev_by_day, contrib_count, revenue_total = await oc_contribution_by_day()
except Exception as e:
errors.append(f"revenue: {e}")
spend_by_day = None
spend_total = None
spend_rows = None
try:
spend_by_day, spend_total, spend_rows = litellm_spend_by_day()
except Exception as e:
errors.append(f"spend: {e}")
balance_current = None
try:
balance_current = await oc_collective_balance()
except Exception as e:
errors.append(f"balance: {e}")
if rev_by_day is None and spend_by_day is None and balance_current is None:
raise RuntimeError("all money sources failed: " + "; ".join(errors))
bal_hist = oc_balance_history()
# Today's balance is the LIVE value just fetched — fresher than the 03:15
# snapshot — so it overrides (or extends) the history at today's key, and
# is folded into the stored row when one already exists.
if balance_current is not None:
bal_hist = {**bal_hist, today_key: float(balance_current["value"])}
try:
_seed_balance_row(today_key, balance_current["value"])
except Exception as e:
errors.append(f"balance seed: {e}")
series = _build_money_series(rev_by_day, spend_by_day, bal_hist, today_key)
balance_history_start = min(bal_hist) if bal_hist else None
return {
"generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"currency": "USD",
"revenue_start": min(rev_by_day) if rev_by_day else None,
"spend_start": min(spend_by_day) if spend_by_day else None,
"balance_history_start": balance_history_start,
"balance_current": balance_current,
"totals": {
"revenue_all_time": revenue_total,
"contributions": contrib_count,
"spend_all_time": spend_total,
"spend_rows": spend_rows,
},
"sources": {
"revenue": "Open Collective contribution transactions (CREDIT, "
"CONTRIBUTION, non-refund) — monthly and one-time, "
"org-backed included via the org account",
"spend": "LiteLLM SpendLogs daily sums (UTC), all teams — members, "
"operators and agent teams included",
"balance": "Open Collective current balance + daily snapshots in "
"metrics_daily.oc_balance_usd",
},
"errors": errors,
"series": series,
}
async def _get_money_payload() -> dict:
"""Hourly-cached access to the money payload; serves the previous payload
(marked stale) when a refresh fails."""
now = datetime.now(timezone.utc).timestamp()
async with _money_lock:
cached = _MONEY_CACHE.get("data")
if cached and now - _MONEY_CACHE.get("ts", 0.0) < MONEY_SERIES_TTL:
out = dict(cached)
out["cached"] = True
out["cache_ttl_seconds"] = MONEY_SERIES_TTL
return out
try:
data = await _compute_money_payload()
except Exception as e:
if cached:
out = dict(cached)
out["cached"] = True
out["stale"] = True
out["errors"] = list(out.get("errors") or []) + [f"refresh failed, serving cached: {e}"]
return out
raise
_MONEY_CACHE["data"] = data
_MONEY_CACHE["ts"] = now
out = dict(data)
out["cached"] = False
out["cache_ttl_seconds"] = MONEY_SERIES_TTL
return out
@app.get("/admin/money-series")
@limiter.limit("30/minute")
async def admin_money_series(request: Request):
"""Revenue / token-spend / OC-balance series for the admin panel's Money
chart, hourly-cached (the OC queries are slow; never called per refresh).
Authenticated by the X-Admin-Token header, like the other /admin/* routes.
The full history is always returned (cumulative series need it); the panel
slices client-side so its period toggle never refetches. Aggregate totals
only — no member emails or names in this payload. See the block comment
above for each series' provenance and honest limits.
"""
_verify_admin_token(request)
return await _get_money_payload()
@app.get("/admin/metrics/series")
@limiter.limit("30/minute")
async def admin_metrics_series(request: Request, days: int = 90):
"""Daily metrics series for charts.
Authenticated by the X-Admin-Token header. Returns oldest-first rows from
`metrics_daily` for the last `days` days, for the admin panel and the member
dashboard to chart from one source of truth. A NULL column means that source
could not be read on that day: render a gap, not a zero.
Cumulative columns (members_*, newsletter_subs, matrix_space_members) are
counts as of that day; spend_usd/requests are that day's traffic only.
Backfilled days carry a LOWER BOUND for members_total (members provisioned
without a welcome email are absent from the history).
Each row carries its energy provenance — `provenance` (measured | mixed |
estimated | none) plus `tokens_measured`, `tokens_estimated` and
`factor_revision`. A row written before those columns existed reports
`provenance: "none"`, not "measured". See docs/energy-metrics.md.
"""
_verify_admin_token(request)
series = metrics_series(days)
return {"series": series, "count": len(series), "days": max(1, min(int(days or 90), 730))}
@app.get("/admin/model-usage")
@limiter.limit("30/minute")
async def admin_model_usage(request: Request, days: int = 0):
"""Co-op-wide per-model usage from LiteLLM spend logs.
Authenticated by the X-Admin-Token header. Aggregates spend-log rows
(across ALL members) by model: total spend, tokens, and the count of
successful requests (refused/failed attempts are excluded and reported
separately as `refused`), sorted by request count descending. Optional
?days=N limits to the last N days
(0 or omitted = all-time). No per-member breakdown — this is co-op-level
so admins can see which models the membership actually uses.
Reads go straight to LiteLLM's Postgres when configured (fast, bounded);
otherwise falls back to the HTTP API path.
"""
_verify_admin_token(request)
# Fast path: direct Postgres aggregation.
if litellm_db_ready():
try:
return aggregate_model_usage(days=days)
except Exception as e:
logger.warning("admin model-usage: DB path failed (%s); falling back to HTTP", e)
# Fallback path: LiteLLM HTTP API.
cutoff = None
if days and days > 0:
from datetime import datetime, timedelta, timezone
cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).strftime(
"%Y-%m-%dT%H:%M:%S"
)
models = {}
total_spend = 0.0
total_tokens = 0
total_requests = 0
total_refused = 0
try:
async with httpx.AsyncClient(timeout=30.0) as client:
r = await client.get(
f"{LITELLM_BASE}/spend/logs",
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
)
r.raise_for_status()
rows = r.json()
if isinstance(rows, dict):
rows = rows.get("data", rows.get("logs", []))
for row in rows if isinstance(rows, list) else []:
# Prefer model_group: it carries our provider/model alias (e.g.
# greenpt/glm-5.3-flash, publicai/apertus-v1.5-8b) when present; the
# raw `model` field holds messy upstream/internal names. Older
# rows predating the alias convention have model_group empty,
# so fall back to `model` for those.
if cutoff:
started = str(row.get("startTime") or "")
if started and started < cutoff:
continue
model = row.get("model_group") or row.get("model") or "unknown"
m = models.setdefault(model, {"spend": 0.0, "tokens": 0, "requests": 0, "refused": 0})
m["spend"] += float(row.get("spend") or 0.0)
m["tokens"] += int(row.get("total_tokens") or 0)
# Refused/failed attempts are not served requests: keep them out
# of the request count (reported separately as `refused`).
if str(row.get("status") or "").lower() == "failure":
m["refused"] += 1
total_refused += 1
else:
m["requests"] += 1
total_requests += 1
total_spend += float(row.get("spend") or 0.0)
total_tokens += int(row.get("total_tokens") or 0)
except Exception as e:
logger.warning("admin model-usage: spend/logs fetch failed: %s", e)
model_list = [
{"model": name, **stats}
for name, stats in sorted(models.items(), key=lambda kv: kv[1]["spend"], reverse=True)
]
return {
"models": model_list,
"totals": {
"total_spend": total_spend,
"total_tokens": total_tokens,
"total_requests": total_requests,
"total_refused": total_refused,
},
}
# --- Broker HTTP endpoints (called by the member dashboard) ---
# The dashboard is Cloudron-SSO-gated; it authenticates the member and passes
# their email with a shared secret. The portal trusts the email only because it
# carries BROKER_SECRET. These endpoints expose ONLY the authenticated member's
# own keys — never another member's, and never the chat key.
# --- First-login service sync (activation hook) ---
# A newly provisioned member is deliberately skipped by the Loomio/Matrix syncs
# until their Cloudron account is activated (so a half-created account is never
# invited), and the 3x/day sweep was the only thing that picked them up AFTER
# they did activate — a new member could wait hours for forum/space access.
# The broker identity path is where the member first presents themselves after
# setting up (the dashboard calls it on first login), so it is the natural
# trigger: when a broker-authenticated, active member resolves AND their Loomio
# invite is still outstanding, confirm their Cloudron account is now activated
# and, only then, fire the same idempotent syncs the sweep runs. The syncs run
# as a background task, so no response ever waits on Loomio or Matrix.
_first_login_sync_inflight = False
def _member_slug(email: str) -> str:
"""Member slug for log lines (slugs survive log redaction; emails don't)."""
try:
conn = get_db()
row = conn.execute(
"SELECT slug FROM members WHERE email = ?", (email,)
).fetchone()
conn.close()
return (row[0] if row and row[0] else "") or "<slug-unset>"
except Exception:
return "<slug-unset>"
async def _member_cloudron_activated(email: str) -> bool:
"""True only when the member's Cloudron account is CONFIRMED activated.
Fail-CLOSED on purpose: if the Cloudron user list cannot be read (or the
member is not on it), report not-activated and let the 3x/day sweep try
later. The hook must never be the path that invites a half-created account,
so unlike the sweep's fail-open roster filter, an unverifiable activation
state means "do not fire".
"""
async with httpx.AsyncClient(timeout=15.0) as client:
users = await cloudron_fetch_all_users(client)
if not users:
return False
for u in users:
if (u.get("email") or "").lower() == email.lower():
return bool(u.get("inviteAccepted"))
return False
async def _first_login_service_sync(email: str) -> None:
"""Fire the Loomio + Matrix Space syncs for a newly-activated member.
Runs as a background task off the broker request (a Loomio outage must
never block a dashboard). The syncs are the SAME idempotent functions the
sweep and provisioning run — guarded by members.loomio_invited_at and the
live-roster pre-check — so a re-fire cannot double-invite. Best-effort:
any failure is logged and swallowed; the sweep remains the backstop.
"""
global _first_login_sync_inflight
slug = _member_slug(email)
try:
if not await _member_cloudron_activated(email):
logger.info(
"first-login sync: %s Cloudron account not yet activated; not "
"firing (the sweep covers it later)", slug,
)
return
# Each service independently best-effort: a Loomio outage must never
# stop the Matrix invite (or vice versa), and neither may raise out of
# this background task.
for label, sync_fn in (
("Loomio", sync_loomio_memberships),
("Matrix", sync_matrix_space),
):
try:
logger.info("first-login sync: firing %s sync for %s", label, slug)
await sync_fn()
except Exception:
logger.warning(
"first-login sync: %s sync for %s failed (non-fatal; the "
"sweep covers later)", label, slug, exc_info=True,
)
logger.info(
"first-login sync: done for %s (loomio_invited_at=%s)",
slug, get_loomio_invited_at(email),
)
except Exception:
logger.warning(
"first-login sync for %s failed (non-fatal; sweep covers later)",
slug, exc_info=True,
)
finally:
_first_login_sync_inflight = False
def maybe_first_login_service_sync(email: str) -> None:
"""Cheap gate on the broker hot path; never raises.
Established members (loomio_invited_at set) cost one indexed SELECT and
return. A never-invited member schedules the sync as a background task.
While one first-login sync is in flight, further gate hits do nothing: the
syncs are full-roster reconciles, so the running pass already covers every
pending member.
"""
global _first_login_sync_inflight
try:
if get_loomio_invited_at(email):
return # established member — zero further work
if is_guest_account(email):
return # guest accounts receive no Loomio/Matrix invites
if _first_login_sync_inflight:
return # a sync is already running; it covers every pending member
loop = asyncio.get_running_loop()
except Exception:
return # the gate must never break the broker response
_first_login_sync_inflight = True
try:
loop.create_task(_first_login_service_sync(email))
except Exception:
_first_login_sync_inflight = False
logger.warning(
"first-login sync could not be scheduled (non-fatal)", exc_info=True,
)
def _verify_broker_secret(request: Request) -> str:
"""Return the authenticated member's email, or 401.
The member dashboard sends X-Broker-Secret (shared secret) + X-Member-Email
(or X-Member-User, a Cloudron username). Both must be present and the secret
must match, or we refuse (fail closed).
Cloudron's proxyAuth injects the USERNAME (e.g. "ntnsndr"), not the email.
So when the dashboard sends a username (no "@"), we resolve it to an email
via the Cloudron user list (we already hold a Cloudron admin token).
"""
if not BROKER_SECRET:
raise HTTPException(503, "Broker secret not configured")
provided = request.headers.get("x-broker-secret", "")
if not secrets.compare_digest(provided, BROKER_SECRET):
raise HTTPException(401, "Invalid broker secret")
identity = (
request.headers.get("x-member-email")
or request.headers.get("x-member-user")
or ""
).strip()
if not identity:
raise HTTPException(401, "Missing member identity")
email = resolve_member_email(identity)
# The member must be active in our DB before they can manage keys.
if not get_member_key(email):
raise HTTPException(403, "No active membership")
# Activation hook: if this active member's service invites are still
# outstanding AND their Cloudron account is now activated, fire the Loomio
# + Matrix syncs (background task; established members cost one SELECT).
# Best-effort by construction — it must never affect this response.
maybe_first_login_service_sync(email)
return email
def resolve_member_email(identity: str) -> str:
"""Resolve a member identity (email, Cloudron username, or uid) to email.
Cloudron's OIDC `sub` claim is the user ID (uid-…), and email-invited users
have no username at all. So accept three identity shapes:
- email ("@") → return as-is
- uid-… → match on the user id
- username → match on the username field (legacy accounts)
"""
identity = identity.strip()
if "@" in identity:
return identity.lower()
# uid / username → email via the full (paginated) user list; one walk
# covers both matchers.
users = cloudron_fetch_all_users_sync()
for u in users:
if u.get("id") == identity:
return (u.get("email") or "").lower()
for u in users:
if u.get("username") == identity:
return (u.get("email") or "").lower()
raise HTTPException(403, f"No Cloudron user for identity {identity!r}")
def _verify_broker_guest_identity(request: Request) -> str:
"""Broker auth that also admits ENDED guest accounts.
Same shared-secret + identity contract as _verify_broker_secret, with one
widening scoped to guest accounts only: a guest is admitted in ANY state,
because the self-service cancel must stay a no-op after it takes effect
and the ended state must render on a later page load. Members (and only
members) still require an active row, so this never widens what a member
identity can reach — the routes on top enforce the guest-only rule again.
"""
if not BROKER_SECRET:
raise HTTPException(503, "Broker secret not configured")
provided = request.headers.get("x-broker-secret", "")
if not secrets.compare_digest(provided, BROKER_SECRET):
raise HTTPException(401, "Invalid broker secret")
identity = (
request.headers.get("x-member-email")
or request.headers.get("x-member-user")
or ""
).strip()
if not identity:
raise HTTPException(401, "Missing member identity")
email = resolve_member_email(identity)
conn = get_db()
row = conn.execute(
"SELECT account_type, active FROM members WHERE email = ?", (email,)
).fetchone()
conn.close()
if not row:
raise HTTPException(403, "No active membership")
account_type, active = (row[0] or "member"), row[1]
if account_type == "guest":
return email # any state — the cancel/guest-status route decides
if not active or not get_member_key(email):
raise HTTPException(403, "No active membership")
maybe_first_login_service_sync(email)
return email
@app.get("/broker/keys")
@limiter.limit("30/minute")
async def broker_list(request: Request):
email = _verify_broker_secret(request)
return {"keys": await broker_list_api_keys(email)}
@app.post("/broker/keys")
@limiter.limit("30/minute")
async def broker_create(request: Request):
email = _verify_broker_secret(request)
body = await request.json()
name = (body.get("name") or "").strip()
key = await broker_create_api_key(email, name)
return {"status": "created", **key}
@app.delete("/broker/keys/{name}")
@limiter.limit("30/minute")
async def broker_revoke(name: str, request: Request):
email = _verify_broker_secret(request)
await broker_revoke_api_key(email, name)
return {"status": "revoked", "name": name}
# Matrix MXID (self-serve federated identity). A member can attach ONE extra
# Matrix account (e.g. their existing @user:matrix.org) to be invited to the
# co-op Space. Empty string clears it. Validation is loose on purpose — Synapse
# will simply reject a malformed MXID on invite, and we log that.
_MXID_RE = re.compile(r"^@[^\s:]+:[^\s]+\.[^\s]+$")
@app.get("/broker/matrix-mxid")
@limiter.limit("30/minute")
async def broker_get_matrix_mxid(request: Request):
email = _verify_broker_secret(request)
if is_guest_account(email):
# Guests get no Matrix: their dashboard hides the section and the
# sync never invites them — the endpoint refuses too, in depth.
raise HTTPException(403, "Matrix chat is available to members")
return {"matrix_mxid": get_member_matrix_mxid(email), "local_mxid": local_mxid_for_email(email)}
def local_mxid_for_email(email: str) -> str:
"""Resolve a member's local Matrix ID (@username:inference.coop), or '' if unknown."""
try:
for u in cloudron_fetch_all_users_sync():
if (u.get("email") or "").lower() == email and u.get("username"):
return f"@{u['username']}:inference.coop"
except Exception as e:
logger.warning("local_mxid_for_email lookup failed: %s", e)
return ""
@app.post("/broker/matrix-mxid")
@limiter.limit("30/minute")
async def broker_set_matrix_mxid(request: Request):
email = _verify_broker_secret(request)
if is_guest_account(email):
# Setting an MXID would put a guest's account in the next Matrix sync
# invite pass — exactly what the feature excludes.
raise HTTPException(403, "Matrix chat is available to members")
body = await request.json()
mxid = (body.get("matrix_mxid") or "").strip()
if mxid and not _MXID_RE.match(mxid):
return JSONResponse(
{"error": "Enter a Matrix ID like @username:server.org"}, status_code=400
)
set_member_matrix_mxid(email, mxid)
# Invite the new account into the Space immediately (best-effort; the daily
# sync will also pick it up). Clear = no invite (just removes the mapping).
invite_sent = True
if mxid:
try:
await sync_matrix_space()
except Exception as e:
logger.warning("matrix-mxid: immediate invite failed: %s", e)
invite_sent = False
# invite_sent=False tells the dashboard the nightly sync will retry; the
# mapping is saved either way, so the member gets a confident confirmation.
return {"status": "saved", "matrix_mxid": mxid, "invite_sent": invite_sent}
def member_per_key_spend(email: str, team_id: str) -> list[dict]:
"""Per-key spend for a member, derived from LiteLLM spend LOGS.
The per-key `spend` counter LiteLLM exposes (`LiteLLM_VerificationToken.spend`,
`/spend/keys`) is CYCLE-ONLY: it resets at the team's `budget_reset_at`, so it
cannot answer "how much has this key spent?". We instead sum `spend` over the
raw rows in `LiteLLM_SpendLogs`, grouped by the key identifier in `api_key`
(the key's LiteLLM hash — the same value we store as `member_api_keys.hash`).
A row whose `api_key` matches one of the member's stored `member_api_keys`
rows is labelled with that key's name. Any row that matches no stored key
belongs to the portal-held CHAT key (alias `member:<email>`, minted at
provisioning, never in member_api_keys) and is grouped under the label
"chat" — so a member sees "chat: $X, <keyname>: $Y".
Returns [{"name", "is_chat", "spend", "tokens", "requests"}], API keys by
spend descending with the chat row last (it is secondary in the keys view).
Empty list when the DB is not configured — never raises for the caller.
"""
if not litellm_db_ready():
return []
conn = get_db()
try:
rows = conn.execute(
"SELECT name, hash FROM member_api_keys WHERE email = ?", (email,)
).fetchall()
finally:
conn.close()
by_hash = {r[1]: r[0] for r in rows if r[1]}
pg = _pg_connect()
try:
with pg.cursor() as cur:
cur.execute(
'SELECT api_key, COALESCE(SUM(spend), 0), '
'COALESCE(SUM(completion_tokens), 0), COUNT(*) '
'FROM "LiteLLM_SpendLogs" WHERE team_id = %s GROUP BY api_key',
(team_id,),
)
raw = cur.fetchall()
finally:
pg.close()
# Aggregate by a stable internal id. Rows matching no stored key are the chat
# key; the sentinel can't collide with a real name (key names are
# [A-Za-z0-9._-], so a member key literally named "chat" stays separate).
agg: dict = {}
for api_key, spend, tokens, reqs in raw:
name = by_hash.get(api_key)
cid = name if name else "__chat__"
a = agg.setdefault(
cid,
{"name": name or "chat", "is_chat": name is None,
"spend": 0.0, "tokens": 0, "requests": 0},
)
a["spend"] += float(spend or 0.0)
a["tokens"] += int(tokens or 0)
a["requests"] += int(reqs or 0)
out = list(agg.values())
# API keys by spend desc, then the chat key last.
out.sort(key=lambda x: (x["is_chat"], -x["spend"]))
for r in out:
r["spend"] = round(r["spend"], 2)
return out
async def broker_get_usage(email: str, days: int = 0) -> dict:
"""Return the member's spend vs. balance, plus per-model and token detail.
This is the read-only primitive the member dashboard uses (no master key on
the client). Reads go straight to LiteLLM's Postgres when configured (fast,
bounded); otherwise falls back to the HTTP API path. `days` bounds the
per-model breakdown window (0 = all time); balance/spend always come from
the team's live totals.
"""
team_id = await litellm_get_or_create_team(email, get_member_balance(email))
allowance = get_member_balance(email)
credits = get_credit_balance(email)
spend = 0.0
reset_at = None
# Fast path: direct Postgres reads.
if litellm_db_ready():
try:
team = get_team_budget_from_db(team_id)
if team:
if team["max_budget"] is not None:
balance = float(team["max_budget"])
spend = float(team["spend"] or 0.0)
reset_at = team["budget_reset_at"]
usage = aggregate_model_usage(days=days, team_id=team_id)
# All-time spend for this member: total of their spend-log rows
# (the team `spend` counter resets with their allowance period).
all_time_spend = 0.0
try:
conn = _pg_connect()
try:
with conn.cursor() as cur:
cur.execute(
'SELECT COALESCE(SUM(spend), 0) FROM "LiteLLM_SpendLogs" WHERE team_id = %s',
(team_id,),
)
all_time_spend = float(cur.fetchone()[0] or 0.0)
finally:
conn.close()
except Exception as e:
logger.warning("broker_get_usage: all-time spend query failed: %s", e)
model_list = [
{"model": m["model"], "spend": m["spend"], "tokens": m["tokens"],
"calls": m["requests"]}
for m in usage["models"]
]
# Per-key spend (chat key shown separately), from the spend-log rows —
# never LiteLLM's cycle-only per-key counter. Best-effort: a failure
# must not blank the whole usage view.
try:
key_spend = member_per_key_spend(email, team_id)
except Exception as e:
logger.warning("broker_get_usage: per-key spend failed: %s", e)
key_spend = []
return {
"email": email,
"guest": is_guest_account(email),
"balance": balance,
"allowance": allowance,
"credits": credits,
"credits_remaining": max(0.0, credits - max(0.0, spend - allowance)),
"spend": spend,
"all_time_spend": round(all_time_spend, 2),
"remaining": max(balance - spend, 0.0),
"total_tokens": usage["totals"]["total_tokens"],
"reset_at": reset_at,
"models": model_list,
"key_spend": key_spend,
}
except Exception as e:
logger.warning("broker_get_usage: DB path failed (%s); falling back to HTTP", e)
# Fallback path: LiteLLM HTTP API.
# /team/list and /spend/logs have grown slow as usage accumulates; the
# httpx default 5s read timeout was too short and made /broker/usage 500.
client_timeout = httpx.Timeout(connect=15.0, read=60.0, write=15.0, pool=15.0)
async with httpx.AsyncClient(timeout=client_timeout) as client:
# 1. Team spend + budget
r = await client.get(
f"{LITELLM_BASE}/team/list",
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
)
r.raise_for_status()
for t in r.json():
if t.get("team_id") == team_id:
balance = float(t.get("max_budget") or balance)
spend = float(t.get("spend") or 0.0)
reset_at = t.get("budget_reset_at")
break
# 2. Per-model token/spend breakdown from spend logs (filtered by team).
# /spend/logs does not reliably filter by team_id server-side, so we
# filter client-side on the returned rows.
models = {}
total_tokens = 0
try:
r2 = await client.get(
f"{LITELLM_BASE}/spend/logs",
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
)
r2.raise_for_status()
rows = r2.json()
if isinstance(rows, dict):
rows = rows.get("data", rows.get("logs", []))
for row in rows if isinstance(rows, list) else []:
if row.get("team_id") != team_id:
continue
# Prefer model_group: it carries our provider/model alias (e.g.
# greenpt/glm-5.3-flash) when present; the raw `model` field holds
# messy upstream/internal names. Older rows predating the alias
# convention have model_group empty, so fall back to `model`.
model = row.get("model_group") or row.get("model") or "unknown"
m = models.setdefault(model, {"spend": 0.0, "tokens": 0, "calls": 0})
m["spend"] += float(row.get("spend") or 0.0)
m["tokens"] += int(row.get("total_tokens") or 0)
# Refused/failed attempts are not served requests.
if str(row.get("status") or "").lower() != "failure":
m["calls"] += 1
total_tokens += int(row.get("total_tokens") or 0)
except Exception as e: # spend logs are best-effort; don't fail the whole view
logger.warning("spend/logs fetch failed: %s", e)
# All-time spend from the same log rows (team counter is period-only).
all_time_spend = round(sum(m["spend"] for m in models.values()), 2)
# Sort models by spend descending (which models are draining the most).
model_list = [
{"model": name, **stats}
for name, stats in sorted(models.items(), key=lambda kv: kv[1]["spend"], reverse=True)
]
return {
"email": email,
"guest": is_guest_account(email),
"balance": balance,
"allowance": allowance,
"credits": credits,
"credits_remaining": max(0.0, credits - max(0.0, spend - allowance)),
"spend": spend,
"all_time_spend": all_time_spend,
"remaining": max(balance - spend, 0.0),
"total_tokens": total_tokens,
"reset_at": reset_at,
"models": model_list,
# Per-key spend needs the Postgres spend-log table (LiteLLM's exposed
# per-key counter is cycle-only); this HTTP fallback runs only when the
# DB is unconfigured, so there is nothing accurate to report here.
"key_spend": [],
}
@app.get("/broker/usage")
@limiter.limit("60/minute")
async def broker_usage(request: Request, days: int = 0):
email = _verify_broker_secret(request)
return await broker_get_usage(email, days=days)
@app.get("/broker/guest-status")
@limiter.limit("30/minute")
async def broker_guest_status(request: Request):
"""What became of the caller's guest account (self-service cancel support).
Served through the broker like every dashboard route. The one broker read
that answers for an ENDED guest too (see _verify_broker_guest_identity): a
cancelled or lapsed guest's dashboard loads this so it can say, plainly,
that access ended — the data is kept — and a membership is still one
contribution away. For an active guest it reports what the cancel button
needs (guest, active, ledger status); for a member it simply says
guest: false.
"""
email = _verify_broker_guest_identity(request)
conn = get_db()
row = conn.execute(
"SELECT active, balance, credit_balance FROM members WHERE email = ?",
(email,),
).fetchone()
conn.close()
if not row:
raise HTTPException(403, "No active membership")
active, balance, credits = row
if not is_guest_account(email):
return {"guest": False, "active": bool(active)}
guest_status = _guest_ledger_status(email) or ("active" if active else "inactive")
if active:
# Same remaining-credit figure the usage view shows.
spend = None
team_id = await litellm_team_id_for_email(email)
if team_id:
team = get_team_budget_from_db(team_id)
if team:
spend = team.get("spend")
eff = compute_max_budget(
float(balance) if balance is not None else 0.0,
float(credits or 0.0),
spend,
)
credits_remaining = round(eff["credits_remaining"], 2)
else:
credits_remaining = 0.0
return {
"guest": True,
"active": bool(active),
"ended": not active,
"guest_status": guest_status,
"cancelled": guest_status == "cancelled by recipient",
"credits_remaining": credits_remaining,
"credits_returned": _guest_returned_credits(email),
}
@app.post("/broker/cancel-guest")
@limiter.limit("10/minute")
async def broker_cancel_guest(request: Request):
"""End the caller's own guest account (self-service cancellation).
Authenticated with the member-dashboard broker identity, guest accounts
only (see cancel_guest_account for the semantics and the idempotence).
A paying member calling this gets a 403, never a cancellation.
"""
email = _verify_broker_guest_identity(request)
return await cancel_guest_account(email)
# TRANSITION ALIASES (guest rename, 2026-11-08): see the note above the admin
# aliases. The deployed dashboard build still calls these two paths.
@app.get("/broker/gift-status")
@limiter.limit("30/minute")
async def broker_gift_status_alias(request: Request):
return await broker_guest_status(request)
@app.post("/broker/cancel-gift")
@limiter.limit("10/minute")
async def broker_cancel_gift_alias(request: Request):
return await broker_cancel_guest(request)
# ---------------------------------------------------------------------------
# Member self-serve guest invitations (trial gate: operator approval).
# ---------------------------------------------------------------------------
@app.get("/broker/guests")
@limiter.limit("30/minute")
async def broker_guest_list(request: Request):
"""The caller's own guest-access view: invitation allowance + their trail.
Served through the member broker identity (a guest identity fails it —
only members invite guests). Returns:
* eligible / next_eligible_at — the rolling one-guest-per-member-per-
GUEST_PER_INVITER_DAYS rule, counting pending and approved REQUESTS
as well as sent guests (the same rule guest_request_guard enforces);
* trail — every invitation this member has made, newest first: pending
and rejected REQUESTS (an approved request IS its ledger row) plus
every row in the guests ledger with what became of it (active,
converted, expired, depleted, cancelled by recipient).
"""
email = _verify_broker_secret(request)
conn = get_db()
requests = conn.execute(
"SELECT id, recipient_email, recipient_name, created_at, decided_at, status, note "
"FROM guest_requests WHERE inviter_email = ? ORDER BY id DESC",
(email,),
).fetchall()
sent = conn.execute(
"SELECT recipient_email, created_at, status "
"FROM guests WHERE inviter_email = ? ORDER BY id DESC",
(email,),
).fetchall()
window = f"-{int(GUEST_PER_INVITER_DAYS)} days"
request_recent = conn.execute(
"SELECT MAX(created_at) FROM guest_requests WHERE inviter_email = ? "
"AND status IN ('pending', 'approved') AND created_at >= datetime('now', ?)",
(email, window),
).fetchone()[0]
ledger_recent = conn.execute(
"SELECT MAX(created_at) FROM guests WHERE inviter_email = ? "
"AND created_at >= datetime('now', ?)",
(email, window),
).fetchone()[0]
conn.close()
last_sent = max((t for t in (request_recent, ledger_recent) if t), default=None)
eligible = last_sent is None
next_eligible_at = None
if not eligible:
try:
anchor = datetime.strptime(str(last_sent)[:19], "%Y-%m-%d %H:%M:%S")
next_eligible_at = (anchor + timedelta(days=int(GUEST_PER_INVITER_DAYS))
).strftime("%Y-%m-%dT%H:%M:%SZ")
except ValueError:
pass
trail = []
for rid, recipient, rname, created, decided, status, note in requests:
if status in ("pending", "rejected"):
trail.append({
"kind": "request", "request_id": rid, "recipient": recipient,
"name": rname, "created_at": created, "decided_at": decided,
"status": status, "note": note,
})
for recipient, created, status in sent:
trail.append({"kind": "guest", "recipient": recipient,
"created_at": created, "status": status})
trail.sort(key=lambda t: t.get("decided_at") or t["created_at"] or "", reverse=True)
return {
"eligible": eligible,
"per_inviter_days": int(GUEST_PER_INVITER_DAYS),
"next_eligible_at": next_eligible_at,
"auto_approve": bool(GUEST_AUTO_APPROVE),
"trail": trail,
}
@app.post("/broker/guests")
@limiter.limit("10/minute")
async def broker_guest_invite(request: Request):
"""Invite a guest, from the member dashboard (self-serve guest access).
Authenticated with the standard member broker identity — a GUEST identity
fails it, so only members reach here. Body: {"email": "<recipient>",
"name": "<optional display name>"}. Every protection runs BEFORE anything
is created (guest_request_guard): active-member inviter, one invitation
per member per rolling GUEST_PER_INVITER_DAYS (pending and approved
requests count), recipient never an existing member, a prior guest, a
pending request, or a MANUAL_MEMBERS address. The guest itself is
provisioned ONLY through provision_guest_member — the same pipeline the
operator's /admin/provision-guest runs (Cloudron user in the guests
group, $2 non-expiring credit, GUEST_DAYS time box, invitation email
naming the inviter, newsletter subscribe, ledger row).
Trial gate: with GUEST_AUTO_APPROVE off (the default) the request waits
in PENDING for the operator's approve/reject in the panel — nothing goes
to the recipient until it is approved, and the member sees 'pending
approval' in their dashboard trail. With it on, provisioning fires
immediately. The member is never emailed about a rejection; their trail
carries the status.
"""
email = _verify_broker_secret(request)
body = await request.json()
recipient = (body.get("email") or "").strip().lower()
name = (body.get("name") or "").strip()
if not recipient or "@" not in recipient:
raise HTTPException(400, "A valid guest email is required")
conn = get_db()
refusal = guest_request_guard(conn, email, recipient)
conn.close()
if refusal:
raise HTTPException(429, refusal)
if not GUEST_AUTO_APPROVE:
conn = get_db()
cur = conn.execute(
"INSERT INTO guest_requests (inviter_email, recipient_email, recipient_name) "
"VALUES (?, ?, ?)",
(email, recipient, name or None),
)
conn.commit()
request_id = cur.lastrowid
conn.close()
logger.info("Guest request #%s: %s invited %s (pending approval)",
request_id, email, recipient)
return {
"status": "pending_approval",
"request_id": request_id,
"recipient": recipient,
"name": name or None,
"note": ("Your guest request is waiting for a co-op approval. "
"Nothing has been sent yet — once it's approved, your "
"friend gets their account and invitation email, and "
"the status shows here."),
}
user_id = await provision_guest_member(recipient, name, email)
return {
"status": "provisioned", "account_type": "guest", "email": recipient,
"inviter": email, "user_id": user_id, "budget": GUEST_BUDGET,
"expires_in_days": int(GUEST_DAYS),
}
@app.get("/broker/metrics")
@limiter.limit("30/minute")
async def broker_metrics(request: Request, days: int = 90):
"""Co-op metrics over time, for the member dashboard.
The same series the admin panel reads. It comes through the broker because
it is AGGREGATE and non-personal — co-op size, requests served, spend — the
same class of social-proof data as /broker/model-leaderboard. No per-member
figure passes through here, and members asked to see how the co-op is doing.
Rows carry the energy provenance the admin series does: `provenance`
(measured | mixed | estimated | none) plus `tokens_measured`,
`tokens_estimated` and `factor_revision` — so a member-facing chart can say
how much of a day's figure rested on a measurement. NULL splits (older rows)
report `provenance: "none"`. See docs/energy-metrics.md.
"""
_verify_broker_secret(request)
series = metrics_series(days)
return {"series": series, "count": len(series), "days": max(1, min(int(days or 90), 730))}
@app.get("/broker/model-leaderboard")
@limiter.limit("30/minute")
async def broker_model_leaderboard(request: Request, days: int = 0):
"""Co-op-wide model popularity for the member dashboard.
Same aggregation as the admin model-usage view (spend/tokens/requests per
model, across all members) but served through the broker so members can
see which models the co-op uses most — social guidance for model choice.
No per-member data; aggregate totals only.
"""
_verify_broker_secret(request)
if litellm_db_ready():
try:
usage = aggregate_model_usage(days=days)
except Exception as e:
logger.warning("broker leaderboard: DB path failed (%s)", e)
usage = None
if usage:
return {
"models": [
{"model": m["model"], "spend": m["spend"],
"tokens": m["tokens"], "requests": m["requests"]}
for m in usage["models"][:10]
]
}
# Fallback: empty (rare; the DB path is the norm)
return {"models": []}
async def reconcile_memberships() -> dict:
"""Reconcile the portal against the live Open Collective member list.
- Provisions members who are BACKER/ADMIN on OC but not yet in our DB
(missed webhook, or contributed before the email step existed).
- Deactivates members who are in our DB but no longer BACKER/ADMIN
(lapsed or cancelled membership).
- Syncs Loomio to the resulting active-member set.
Idempotent and safe to run repeatedly. Returns a summary of actions taken.
"""
# fetch_members() raises when OC is unreachable or the query fails; treat
# that like an empty result so the documented fail-safe below actually
# applies, instead of letting the exception escape and 500 the sweep.
try:
active = await fetch_members()
except Exception as e:
logger.warning(
"Sweep: member fetch failed; skipping reconciliation (fail-safe): %s", e
)
return {
"status": "skipped",
"reason": "member_fetch_failed",
"provisioned": 0,
"deactivated": 0,
"reinvited": 0,
"newsletter_removed": 0,
}
active_emails = {m["email"] for m in active}
active_by_email = {m["email"]: m for m in active}
# FAIL-SAFE: if the OC fetch returned nothing (token expired, OC outage,
# or a query error), we must NOT deactivate everyone — that would be a
# catastrophic false-positive. Skip reconciliation entirely in that case.
if not active:
logger.warning("Sweep: fetch_members returned empty; skipping reconciliation (fail-safe)")
return {"status": "skipped", "reason": "empty_member_list", "provisioned": 0, "deactivated": 0}
# Currently provisioned accounts (from our DB), with their type — the guest
# marker drives both the conversion trigger and the deactivation skip.
conn = get_db()
rows = conn.execute(
"SELECT email, cloudron_user_id, account_type FROM members WHERE active = 1"
).fetchall()
conn.close()
provisioned = {
r[0]: {"user_id": r[1], "account_type": (r[2] or "member")}
for r in rows
}
# Members whose contribution arrives under a DIFFERENT OC email (set by
# POST /admin/guest-convert): their oc_email is an additional presence
# matcher — when it appears in the live OC list the account counts as
# active even though their own email does not.
conn = get_db()
oc_rows = conn.execute(
"SELECT email, oc_email FROM members WHERE oc_email IS NOT NULL AND oc_email != ''"
).fetchall()
conn.close()
for row_email, oc in oc_rows:
if (oc or "").strip().lower() in active_emails:
active_emails.add(row_email)
if row_email not in active_by_email:
active_by_email[row_email] = {
"email": row_email, "name": row_email.split("@")[0],
"slug": "", "role": "BACKER",
}
logger.info("Sweep: %s active via OC email %s", row_email, oc)
provisioned_count = 0
deactivated_count = 0
converted_count = 0
# 0. Organization-backed memberships: dues paid by an OC Organization.
# Linked accounts (members.org_slug) stay active while their org holds a
# BACKER/ADMIN role on the Membership tier; lapsed orgs deactivate like
# individuals. Fail-safe: if the org fetch fails, keep linked accounts
# active (never deactivate on our own fetch failure).
try:
org_backers = await fetch_org_backers()
org_fetch_ok = True
except Exception as e:
logger.warning("Sweep: org fetch failed (keeping linked accounts active): %s", e)
org_backers = {}
org_fetch_ok = False
if org_backers or not org_fetch_ok:
conn = get_db()
org_rows = conn.execute(
"SELECT email, org_slug FROM members WHERE org_slug IS NOT NULL"
).fetchall()
conn.close()
for org_email, org_slug in org_rows:
org = org_backers.get(org_slug)
if org_fetch_ok and not org:
continue # org lapsed → falls through to deactivation below
if org_email not in active_emails:
active.append({
"email": org_email,
"name": (org or {}).get("name") or org_slug,
"slug": org_slug,
"role": (org or {}).get("role", "BACKER"),
"tier": ((org or {}).get("tier", "") or "") + " (org)",
})
active_emails.add(org_email)
logger.info("Sweep: org-linked member %s active via org %s", org_email, org_slug)
# 1. Provision missing members — and CONVERT guest accounts whose owner has
# joined (their email now appears in the live OC backer list). The
# conversion itself lives in provision_member, so the webhook and
# /admin/provision trigger the same transition automatically.
for m in active:
email = m["email"]
entry = provisioned.get(email)
if entry is None:
try:
await provision_member(email, m["name"], m["slug"])
provisioned_count += 1
logger.info("Sweep: provisioned %s", email)
except Exception as e:
logger.warning("Sweep: failed to provision %s: %s", email, e)
elif entry["account_type"] == "guest":
try:
await provision_member(email, m["name"], m["slug"])
converted_count += 1
logger.info("Sweep: converted guest account %s to member", email)
except Exception as e:
logger.warning("Sweep: failed to convert guest %s: %s", email, e)
# 1b. Guest lifecycles: time-box expiry + spent-out guests. A guest is never
# deactivated for LACKING a contribution (next loop excludes them) —
# only by its own rules. Runs before the member pass so its results
# are independent of OC state.
try:
guest_counts = await reconcile_guest_lifecycles()
except Exception as e:
logger.warning("Sweep: guest lifecycle pass failed (non-fatal): %s", e)
guest_counts = {"expired": 0, "depleted": 0}
# 2. Deactivate lapsed members (but never manual/founder members, who have
# no OC membership to lapse, and never GUEST accounts — a guest has no OC
# membership by design; it lapses only via its own rules, handled above).
for email, info in provisioned.items():
if info["account_type"] == "guest":
continue
user_id = info["user_id"]
if email not in active_emails and email not in MANUAL_MEMBERS:
try:
if INACTIVE_GROUP_ID and user_id:
await cloudron_set_group(user_id, INACTIVE_GROUP_ID)
deactivate_member(email)
await revoke_member_litellm_access(email)
await listmonk_sync(email, subscribe=False)
deactivated_count += 1
logger.info("Sweep: deactivated %s", email)
except Exception as e:
logger.warning("Sweep: failed to deactivate %s: %s", email, e)
# 2b. Re-invite never-activated members (provisioned but no account setup).
# A member who was provisioned (e.g. webhook fired while the OC token was
# missing the `email` scope, or the invite email went to spam) may sit
# active-but-unactivated indefinitely. Re-send their welcome email, but
# throttle by REINVITE_AFTER_DAYS so we don't spam them nightly.
reinvited_count = 0
conn = get_db()
reinvite_rows = conn.execute(
"SELECT email, cloudron_user_id, welcome_sent_at, account_type FROM members WHERE active = 1"
).fetchall()
conn.close()
now = datetime.now(timezone.utc)
for email, user_id, sent_at, acct_type in reinvite_rows:
if not user_id:
continue
if (acct_type or "member") == "guest":
# Guest accounts are re-invited by re-running /admin/provision-guest
# (it re-sends the guest email idempotently). The MEMBER pipeline
# here would move the user into the members group.
continue
# Skip if already activated.
try:
if await cloudron_is_activated(user_id):
continue
except Exception as e:
logger.warning("Sweep: activation check failed for %s: %s", email, e)
continue
# Throttle: only re-invite if the last send was > REINVITE_AFTER_DAYS ago.
if sent_at:
try:
last = datetime.fromisoformat(sent_at)
# stored as UTC naive (datetime('now') is UTC); attach tz
if last.tzinfo is None:
last = last.replace(tzinfo=timezone.utc)
if (now - last).total_seconds() < REINVITE_AFTER_DAYS * 86400:
continue
except Exception:
pass
# Re-send the welcome email (reuses the stored key — no duplicate alias).
try:
name = active_by_email.get(email, {}).get("name") or email.split("@")[0]
await provision_member(email, name, "")
reinvited_count += 1
logger.info("Sweep: re-invited %s (never activated)", email)
except Exception as e:
logger.warning("Sweep: failed to re-invite %s: %s", email, e)
# 3. Sync Loomio.
await sync_loomio_memberships()
await sync_matrix_space()
# 4. Reconcile the newsletter list against active membership (removes
# deleted accounts, never-activated test accounts, lapsed members).
newsletter = await listmonk_reconcile()
return {
"status": "reconciled",
"provisioned": provisioned_count,
"deactivated": deactivated_count,
"reinvited": reinvited_count,
"converted": converted_count,
"guests_expired": guest_counts.get("expired", 0),
"guests_depleted": guest_counts.get("depleted", 0),
"guest_lapse_notices": guest_counts.get("lapse_notices", 0),
"newsletter_removed": newsletter.get("removed", 0),
}
# --------------------------------------------------------------------------- #
# Allowance renewal — payment-driven, anchored to the member's JOIN day. #
# --------------------------------------------------------------------------- #
# Policy (operator decision 2026-11-04). Open Collective aligns every member's
# charge onto the collective's common billing day, so anchoring an allowance
# reset to the PAYMENT date lets a charge move the boundary and skip a cycle
# entirely: a member who joins mid-month can be charged on the common billing
# day within weeks, and the old reset rule re-anchored them to that new day
# with no reset in between. So the
# reset is anchored to the day of the month the member JOINED, and renewal is
# decided by asking Open Collective directly whether a recurring contribution is
# live — on our own schedule, not inferred from webhook delivery. A webhook that
# never arrives then costs a delayed renewal, never a lost month.
#
# Decisions (per member, per cycle):
# renewed — a recurring contribution is live; the allowance reset is pinned
# to the member's next join-day.
# reminded — no live recurring contribution; one reminder email is sent.
# deactivated — reminded on an earlier pass and still unpaid after the grace
# window; the member loses group access but keeps their account.
# would_* — report mode (and a send run while ALLOWANCE_DEACTIVATE is off)
# writes the `would_` form, including `would_deactivate` for a
# reminder older than the grace window, so the deactivation the
# policy WOULD apply is visible BEFORE the flag is enabled.
# Every decision is written to allowance_log with the Open Collective evidence
# (payment date, or its absence) so a dispute is answered from a record.
ALLOWANCE_POLICY = (os.environ.get("ALLOWANCE_POLICY", "report") or "report").strip().lower()
# Deactivation is gated SEPARATELY and defaults OFF, so the renewal/reminder path
# can be exercised before anyone loses access (operator directive 2026-11-04).
ALLOWANCE_DEACTIVATE = (os.environ.get("ALLOWANCE_DEACTIVATE", "off") or "off").strip().lower()
# Days to wait after the single reminder before deactivating a still-unpaid member.
PAYMENT_GRACE_DAYS = float(os.environ.get("PAYMENT_GRACE_DAYS", "7"))
def _ensure_allowance_tables(conn) -> None:
conn.execute(
"CREATE TABLE IF NOT EXISTS allowance_log ("
" id INTEGER PRIMARY KEY AUTOINCREMENT,"
" email TEXT NOT NULL, cycle TEXT NOT NULL, decision TEXT NOT NULL,"
" evidence TEXT, anchor_day INTEGER,"
" decided_at TEXT DEFAULT (datetime('now')))"
)
conn.execute(
"CREATE TABLE IF NOT EXISTS payment_reminders ("
" email TEXT NOT NULL, cycle TEXT NOT NULL, sent_at TEXT NOT NULL,"
" PRIMARY KEY (email, cycle))"
)
cols = {r[1] for r in conn.execute("PRAGMA table_info(members)")}
if "anchor_day" not in cols:
conn.execute("ALTER TABLE members ADD COLUMN anchor_day INTEGER")
conn.commit()
def member_membership_state(email: str, active: bool) -> tuple[str, str]:
"""Derived membership state for the admin panel — read-only, no side effects.
Mirrors what the allowance reconciliation already knows, so the panel reflects
the portal rather than inventing policy of its own:
inactive — membership deactivated (members.active = 0).
in_grace — a payment reminder has been sent for this member's
current cycle and no contribution has arrived yet;
access continues for PAYMENT_GRACE_DAYS.
awaiting_contribution — the reconciliation flagged the member unpaid but no
reminder has been sent yet.
active — current / no open reminder.
"""
if not active:
return "inactive", "deactivated"
try:
conn = get_db()
rem = conn.execute(
"SELECT sent_at FROM payment_reminders WHERE email = ? "
"ORDER BY sent_at DESC LIMIT 1",
(email,),
).fetchone()
log = conn.execute(
"SELECT decision FROM allowance_log WHERE email = ? ORDER BY id DESC LIMIT 1",
(email,),
).fetchone()
conn.close()
except Exception: # tables absent / unreadable: fail safe
return "active", ""
if rem:
return "in_grace", (f"awaiting contribution — reminder sent "
f"{str(rem[0])[:10]}, grace {int(PAYMENT_GRACE_DAYS)}d")
if log and log[0] in ("reminded", "would_remind", "remind_failed"):
return "awaiting_contribution", "no current contribution at Open Collective"
return "active", ""
def _parse_oc_dt(value):
"""Parse an Open Collective ISO timestamp ('…Z') into an aware datetime."""
if not value:
return None
try:
dt = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
except Exception:
return None
def _clamp_anchor_day(anchor_day: int) -> int:
"""Clamp the anchor day to 28 so short months never skip a renewal."""
try:
d = int(anchor_day)
except Exception:
return 1
return min(max(d, 1), 28)
def next_reset_for_anchor_day(anchor_day: int, now: datetime | None = None) -> datetime:
"""The member's next allowance reset: the next occurrence of their join-day.
The day-of-month is the member's JOIN day (not their payment date), so an OC
charge landing early cannot move the boundary. Always strictly after `now`.
"""
now = now or datetime.now(timezone.utc)
day = _clamp_anchor_day(anchor_day)
y, m = now.year, now.month
cand = datetime(y, m, day, 0, 0, tzinfo=timezone.utc)
if cand <= now:
y, m = (y + 1, 1) if m == 12 else (y, m + 1)
cand = datetime(y, m, day, 0, 0, tzinfo=timezone.utc)
return cand
def cycle_start_for_anchor_day(anchor_day: int, now: datetime | None = None) -> datetime:
"""The most recent occurrence of the member's join-day at/before `now`."""
now = now or datetime.now(timezone.utc)
day = _clamp_anchor_day(anchor_day)
y, m = now.year, now.month
cand = datetime(y, m, day, 0, 0, tzinfo=timezone.utc)
if cand > now:
y, m = (y - 1, 12) if m == 1 else (y, m - 1)
cand = datetime(y, m, day, 0, 0, tzinfo=timezone.utc)
return cand
async def fetch_member_orders() -> dict[str, list[dict]]:
"""Every OC order for the collective, grouped by the payer's slug.
One query for the whole collective (orders(limit: 200)) rather than a call
per member. Raises on failure so the caller can skip rather than mis-decide.
"""
if not OC_PERSONAL_TOKEN:
raise RuntimeError("OC_PERSONAL_TOKEN not set; cannot fetch orders")
query = (
'{ collective(slug: "%s") { orders(limit: 200) { totalCount nodes { '
"frequency status createdAt lastChargedAt nextChargeDate "
"fromAccount { slug } totalAmount { value currency } } } } }"
% OC_COLLECTIVE_SLUG
)
async with httpx.AsyncClient(timeout=30.0) as client:
r = await client.post(
OC_GRAPHQL_URL,
headers={"Content-Type": "application/json",
"Authorization": f"Bearer {OC_PERSONAL_TOKEN}",
"Api-Version": "2022-01-01",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"},
json={"query": query},
)
if r.status_code != 200:
raise RuntimeError(f"OC order fetch failed: HTTP {r.status_code}")
body = r.json()
if body.get("errors"):
raise RuntimeError(f"OC order fetch errors: {str(body['errors'])[:200]}")
orders = (((body.get("data") or {}).get("collective") or {}).get("orders") or {})
by_slug: dict[str, list[dict]] = {}
for o in orders.get("nodes") or []:
slug = ((o.get("fromAccount") or {}).get("slug") or "").strip()
if slug:
by_slug.setdefault(slug, []).append(o)
return by_slug
def _payment_verdict(orders: list[dict], cycle_start: datetime,
now: datetime | None = None) -> tuple[bool, str]:
"""Did this member's recurring contribution pay for the current cycle?
Frequency-aware. Only a RECURRING contribution (MONTHLY or YEARLY) can cover
an allowance cycle; a ONE-TIME payment is credit, not an allowance period, so
it is never treated as covering a month and never masks an unpaid member.
Returns (paid, evidence).
Rules, in order:
* PROCESSING — a payment is in flight; treated as PAID (a payment mid-flight
is not a failure), so a PROCESSING order never triggers a reminder.
* ACTIVE MONTHLY — a live subscription; treated as paid even if its next
charge is still ahead, so a member whose charge falls due tomorrow does
not read as unpaid on every sweep.
* ACTIVE YEARLY — paid only while its coverage window contains the current
cycle: a year from its last charge, or until its nextChargeDate. A yearly
contributor paid once and is current for twelve cycles; a flat monthly
test would read them as unpaid (and, with deactivation on, deactivate
them) for eleven of those months.
* otherwise — judge on the most recent RECURRING charge: within the cycle
(7-day slack) is paid, before it is the remind path.
DORMANT ON TODAY'S DATA: as of 2026-11-04 the collective has no yearly orders
(104 orders: 88 MONTHLY, 16 ONETIME, 0 YEARLY — read from the collective-level
order list, which is complete for our collective). The yearly branch is
defensive robustness for a tier we may add later; it is not a fix for any
current member. Attribution caution: one member's `account.orders` spans EVERY
collective they back, so an order's OWNING collective (`toAccount`) must be
checked before it is credited to us — a foreign YEARLY order (social.coop) was
once misread as a yearly payer here. `now` is injectable for tests.
"""
now = now or datetime.now(timezone.utc)
def _freq(o: dict) -> str:
return (o.get("frequency") or "").upper()
def _status(o: dict) -> str:
return (o.get("status") or "").upper()
recurring = [o for o in orders if _freq(o) in ("MONTHLY", "YEARLY")]
if not recurring:
return False, "no recurring (monthly) contribution on Open Collective"
live = [o for o in recurring if _status(o) in ("ACTIVE", "PROCESSING")]
# An ACTIVE monthly subscription, or any PROCESSING order, is a live
# contribution that covers the cycle (evidence string kept stable so a
# report is comparable run to run).
live_monthly = [o for o in live if _freq(o) == "MONTHLY" or _status(o) == "PROCESSING"]
if live_monthly:
charges = [c for c in (_parse_oc_dt(o.get("lastChargedAt")) for o in live_monthly) if c]
last = max(charges).date().isoformat() if charges else "unknown"
return True, f"live recurring contribution (last charged {last})"
# An ACTIVE yearly subscription covers the cycle only while its coverage
# window (last charge + 1 year, or up to nextChargeDate) still includes now.
for o in live:
if _freq(o) != "YEARLY":
continue
lc = _parse_oc_dt(o.get("lastChargedAt"))
nc = _parse_oc_dt(o.get("nextChargeDate"))
# Coverage runs to the later of (next charge date) and (last charge + 1yr).
covered_until = nc
if lc:
year_end = lc + timedelta(days=365)
if covered_until is None or year_end > covered_until:
covered_until = year_end
if covered_until and now < covered_until:
return True, (f"live yearly contribution covers this cycle "
f"(last charged {lc.date().isoformat() if lc else 'unknown'}, "
f"covers until {covered_until.date().isoformat()})")
charged = [(_parse_oc_dt(o.get("lastChargedAt")), _status(o)) for o in recurring]
charged = [(c, s) for c, s in charged if c]
if charged:
last_dt, _ = max(charged, key=lambda t: t[0])
if last_dt >= cycle_start - timedelta(days=7):
return True, f"charged {last_dt.date().isoformat()} within the current cycle"
return False, f"last charge {last_dt.date().isoformat()} is before this cycle"
statuses = ",".join(sorted({_status(o) or "?" for o in recurring}))
return False, f"no live recurring contribution (order status: {statuses})"
def payment_reminder_email(name: str, contribute_url: str) -> tuple[str, str, str]:
"""Build the one-per-cycle 'we haven't seen your contribution' reminder.
Same shell as welcome_email()/budget_notice_email(): cream background, white
card, forest-green button, inline pine-tree mark, table-based, no external
assets. Plain and direct — it names the action, not the plumbing.
"""
subject = "Your Inference Cooperative membership needs attention"
text = (
f"Hi {name},\n\n"
"Our records at Open Collective don't show a current monthly contribution "
"for your account, so your monthly allowance won't renew.\n\n"
"To keep your access, restart or update your contribution here:\n\n"
f"{contribute_url}\n\n"
"If your payment has already gone through, ignore this — your allowance "
"will renew on its own. If we don't see a contribution after a short grace "
"period, your account stays in place but loses access until your "
"contribution resumes.\n\n"
"Questions? Reply to this email or write to info@inference.coop.\n\n"
"— The Inference Cooperative\n"
)
html = f"""<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"></head>
<body style="margin:0;padding:0;background:#faf8f5;font-family:-apple-system,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;">
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:#faf8f5;padding:40px 16px;">
<tr>
<td align="center">
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="max-width:520px;">
<tr>
<td align="center" style="padding-bottom:24px;">
{PINE_LOGO_SVG}
</td>
</tr>
<tr>
<td align="center" style="background:#ffffff;border:1px solid #e8e2d8;border-radius:16px;padding:40px 32px;">
<h1 style="margin:0 0 12px;font-size:24px;font-weight:700;color:#2d3327;">Your membership needs attention</h1>
<p style="margin:0 0 8px;font-size:16px;color:#6b7a62;line-height:1.5;">Our records at Open Collective don't show a current monthly contribution for your account, so your monthly allowance won't renew.</p>
<p style="margin:0 0 28px;font-size:15px;color:#6b7a62;line-height:1.5;">To keep your access, restart or update your contribution. If your payment has already gone through, ignore this.</p>
<a href="{contribute_url}" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:999px;font-size:16px;font-weight:600;">Update your contribution</a>
<p style="margin:28px 0 0;font-size:13px;color:#94a08c;line-height:1.5;">If we don't see a contribution after a short grace period, your account stays in place but loses access until your contribution resumes.</p>
</td>
</tr>
<tr>
<td align="center" style="padding-top:24px;font-size:13px;color:#94a08c;line-height:1.6;">
Questions? Reply to this email or write to <a href="mailto:info@inference.coop" style="color:#6b7a62;">info@inference.coop</a>.<br>
<span style="color:#2d3327;">The Inference Cooperative</span> &middot; inference.coop
</td>
</tr>
</table>
</td>
</tr>
</table>
</body>
</html>"""
return subject, text, html
async def fetch_member_index() -> dict[str, dict]:
"""email -> {slug, joined, role, tier} from the live OC member list.
Paginated (the members connection caps at 100/page and hides the rest).
The slug here is the member's CURRENT Open Collective slug — the members
table can hold a stale `guest-…` slug from first provisioning, so orders
must be matched on the slug resolved from the member's EMAIL, never on the
stored column (matching on the stored slug falsely read 18 current payers
as unpaid in the first report run).
"""
if not OC_PERSONAL_TOKEN:
raise RuntimeError("OC_PERSONAL_TOKEN not set; cannot fetch the member index")
out: dict[str, dict] = {}
offset = 0
async with httpx.AsyncClient(timeout=30.0) as client:
for _ in range(20):
query = (
'{ collective(slug: "%s") { members(limit: 100, offset: %d) { totalCount '
"nodes { role createdAt tier { name } account { name slug type ... on Individual { email } } } } } }"
% (OC_COLLECTIVE_SLUG, offset)
)
r = await client.post(
OC_GRAPHQL_URL,
headers={"Content-Type": "application/json",
"Personal-Token": OC_PERSONAL_TOKEN,
"Api-Version": "2022-01-01",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"},
json={"query": query},
)
if r.status_code != 200:
raise RuntimeError(f"OC member index fetch failed: HTTP {r.status_code}")
members = (((r.json().get("data") or {}).get("collective") or {}).get("members") or {})
page = members.get("nodes") or []
for n in page:
a = n.get("account") or {}
e = (a.get("email") or "").strip().lower()
if e:
out[e] = {
"slug": a.get("slug") or "",
"joined": n.get("createdAt") or "",
"role": n.get("role") or "",
"tier": ((n.get("tier") or {}).get("name") or ""),
}
offset += len(page)
if not page or offset >= (members.get("totalCount") or 0):
break
return out
async def anchor_day_for_member(email: str, slug: str = "") -> int:
"""The member's allowance anchor day (the day of month they joined).
Stored on the members row; derived from the OC join date (resolved by email)
on first use and persisted so later decisions are stable. Clamped to 28.
"""
conn = get_db()
row = conn.execute("SELECT anchor_day FROM members WHERE email = ?", (email,)).fetchone()
conn.close()
if row and row[0]:
return _clamp_anchor_day(row[0])
try:
rec = (await fetch_member_index()).get(email.strip().lower())
if rec:
dt = _parse_oc_dt(rec.get("joined"))
if dt:
d = _clamp_anchor_day(dt.day)
conn = get_db()
conn.execute("UPDATE members SET anchor_day = ? WHERE email = ?", (d, email))
conn.commit()
conn.close()
return d
except Exception as e:
logger.warning("anchor_day_for_member: OC join-date lookup failed for %s: %s", email, e)
return _clamp_anchor_day(datetime.now(timezone.utc).day)
def allowance_lapse_decision(reminder_sent_at, now, grace_days, acting):
"""What to do with a member who was reminded and is still unpaid (pure).
No I/O — `now`, the reminder time and the grace length are all arguments, so
the lapse logic can be unit-tested against a SYNTHETIC reminder time without
touching any member's rows (see tests/test_allowance_lapse.py). Returns
`(decision, deadline)`:
* ("in_grace", None) — the grace window has not elapsed yet.
* ("would_deactivate", deadline) — elapsed, but this caller may not act
(a report run, or ALLOWANCE_DEACTIVATE still off). NOTHING changes.
* ("deactivated", deadline) — elapsed AND the caller is authorised.
`acting` must be True only when BOTH gates hold — a verified send run
(ALLOWANCE_POLICY=on) AND ALLOWANCE_DEACTIVATE=on — so enabling the preview
never by itself removes anyone's access. `deadline` is the moment the grace
window elapses (reminder sent + grace_days), carried into the decision log.
"""
if reminder_sent_at is None:
return "in_grace", None
deadline = reminder_sent_at + timedelta(days=grace_days)
if now < deadline:
return "in_grace", None
return ("deactivated" if acting else "would_deactivate"), deadline
async def reconcile_allowances(send: bool = False) -> dict:
"""Decide every active member's allowance renewal from OC payment evidence.
`send=False` is a REPORT: it computes and records every decision (would
renew / would remind / would deactivate) and changes NOTHING — no resets,
no emails, no group changes. `send=True` performs renewals and sends the
single reminder. Deactivation still additionally requires
ALLOWANCE_DEACTIVATE=on.
"""
try:
by_slug = await fetch_member_orders()
except Exception as e:
logger.warning("Allowance reconcile: order fetch failed; skipping (fail-safe): %s", e)
return {"status": "skipped", "reason": "order_fetch_failed", "detail": str(e)[:200]}
try:
index = await fetch_member_index()
except Exception as e:
logger.warning("Allowance reconcile: member index fetch failed; skipping (fail-safe): %s", e)
return {"status": "skipped", "reason": "member_index_failed", "detail": str(e)[:200]}
conn = get_db()
_ensure_allowance_tables(conn)
cols = {r[1] for r in conn.execute("PRAGMA table_info(members)")}
extra = ", welcome_sent_at" if "welcome_sent_at" in cols else ""
rows = conn.execute(
f"SELECT email, slug, anchor_day, org_slug, account_type, oc_email{extra} "
"FROM members WHERE active = 1"
).fetchall()
now = datetime.now(timezone.utc)
contribute = credit_pack_url()
verified = "on" if (send and ALLOWANCE_POLICY == "on") else "report"
_APPLIED = {"renew": "renewed", "remind": "reminded", "deactivate": "deactivated"}
_WOULD = {"renew": "would_renew", "remind": "would_remind", "deactivate": "would_deactivate"}
label = (lambda a: _APPLIED[a]) if verified == "on" else (lambda a: _WOULD[a])
decisions: list[dict] = []
counts = {"renewed": 0, "reminded": 0, "deactivated": 0,
"would_renew": 0, "would_remind": 0, "would_deactivate": 0, "in_grace": 0,
"skipped": 0}
for row in rows:
email = row[0]
slug = row[1] or ""
anchor_day = row[2]
org_slug = row[3]
acct_type = row[4] or "member"
oc_email = (row[5] or "").strip().lower()
welcome_sent = row[6] if len(row) > 6 else None
if acct_type == "guest":
# Guest accounts have no OC membership by design — the $2 credit
# balance and the GUEST_DAYS time box govern them instead. No
# renewal decision, no reminder, ever.
counts["skipped"] += 1
continue
if email.lower() in MANUAL_MEMBERS:
counts["skipped"] += 1
continue
if org_slug:
counts["skipped"] += 1 # dues paid by an OC Organization
continue
rec = index.get(email.strip().lower())
if rec is None and oc_email:
# Contribution under a different OC email (guest-convert fallback).
rec = index.get(oc_email)
if rec and rec.get("role") == "ADMIN":
counts["skipped"] += 1 # the co-op's own account
continue
oc_slug = (rec or {}).get("slug") or ""
if not oc_slug:
decisions.append({"email": email, "slug": slug, "anchor_day": anchor_day,
"cycle": "", "decision": "skipped",
"evidence": "not in the live Open Collective member list — cannot verify payment"})
counts["skipped"] += 1
continue
# Keep the stored slug in step with Open Collective (authoritative).
if send and oc_slug != slug:
conn.execute("UPDATE members SET slug = ? WHERE email = ?", (oc_slug, email))
conn.commit()
if not anchor_day:
dt = _parse_oc_dt(rec.get("joined")) or _parse_oc_dt(welcome_sent) or now
anchor_day = _clamp_anchor_day(dt.day)
if send:
conn.execute("UPDATE members SET anchor_day = ? WHERE email = ?", (anchor_day, email))
conn.commit()
anchor_day = _clamp_anchor_day(anchor_day)
cycle_start = cycle_start_for_anchor_day(anchor_day, now)
cycle_key = cycle_start.date().isoformat()
orders = by_slug.get(oc_slug, [])
paid, evidence = _payment_verdict(orders, cycle_start)
if paid:
next_reset = next_reset_for_anchor_day(anchor_day, now)
evidence = f"{evidence}; reset anchored to join-day {anchor_day} → next reset {next_reset.date().isoformat()}"
decision = label("renew")
counts[decision] += 1
if verified == "on":
try:
team_id = await litellm_team_id_for_email(email)
if team_id:
await litellm_pin_reset_date(team_id, next_reset)
except Exception as e:
logger.warning("Allowance renew failed for %s: %s", email, e)
else:
reminded = conn.execute(
"SELECT sent_at FROM payment_reminders WHERE email = ? AND cycle = ?",
(email, cycle_key),
).fetchone()
if not reminded:
decision = label("remind")
counts[decision] += 1
if verified == "on":
subject, text_body, html_body = payment_reminder_email(oc_slug or slug or "there", contribute)
if send_email(email, subject, text_body, html_body):
conn.execute(
"INSERT OR REPLACE INTO payment_reminders (email, cycle, sent_at)"
" VALUES (?, ?, datetime('now'))",
(email, cycle_key),
)
conn.commit()
else:
decision = "remind_failed"
else:
sent_at = _parse_oc_dt(str(reminded[0]))
# Lapse decision is a pure, testable function: past the grace
# window it reports `would_deactivate` in EVERY mode; the real
# action needs `acting` — a verified send run AND the separate
# ALLOWANCE_DEACTIVATE gate. So a report run (and a send run with
# deactivate still off) PREVIEWS the deactivation and changes
# nobody's access.
acting = verified == "on" and ALLOWANCE_DEACTIVATE == "on"
lapse, deadline = allowance_lapse_decision(
sent_at, now, PAYMENT_GRACE_DAYS, acting)
if lapse == "in_grace":
decision = "in_grace"
counts["in_grace"] += 1
evidence = (f"{evidence}; reminder sent "
f"{str(reminded[0])[:10] if reminded[0] else '?'} "
f"(grace {int(PAYMENT_GRACE_DAYS)}d)")
else:
decision = lapse # would_deactivate | deactivated
counts[decision] += 1
dl = deadline.date().isoformat() if deadline else "?"
evidence = (f"{evidence}; reminder sent {str(reminded[0])[:10]}, "
f"grace {int(PAYMENT_GRACE_DAYS)}d elapsed, "
f"deadline {dl}")
if decision == "deactivated":
urow = conn.execute(
"SELECT cloudron_user_id FROM members WHERE email = ?", (email,)
).fetchone()
if INACTIVE_GROUP_ID and urow and urow[0]:
await cloudron_set_group(urow[0], INACTIVE_GROUP_ID)
deactivate_member(email)
await revoke_member_litellm_access(email)
await listmonk_sync(email, subscribe=False)
# L4 (audit 2026-11-05): one row per (member, cycle, decision) — the
# same decision re-derived on a later run UPDATES that row's evidence
# and timestamp instead of appending a duplicate. The sweep runs 3x/day
# and report-mode runs also write here; without this the table grew
# ~500 rows/day (measured 2026-11-05: 1741 -> 2257). A decision CHANGE
# within a cycle still gets its own row, so the "latest decision per
# member" readers are unaffected and no history is rewritten.
cur = conn.execute(
"UPDATE allowance_log SET evidence = ?, anchor_day = ?, "
"decided_at = datetime('now') "
"WHERE id = (SELECT id FROM allowance_log WHERE email = ? AND cycle = ? "
"AND decision = ? ORDER BY id DESC LIMIT 1)",
(evidence, anchor_day, email, cycle_key, decision),
)
if cur.rowcount == 0:
conn.execute(
"INSERT INTO allowance_log (email, cycle, decision, evidence, anchor_day)"
" VALUES (?, ?, ?, ?, ?)",
(email, cycle_key, decision, evidence, anchor_day),
)
conn.commit()
decisions.append({"email": email, "slug": oc_slug, "anchor_day": anchor_day,
"cycle": cycle_key, "decision": decision, "evidence": evidence})
conn.close()
return {
"mode": ALLOWANCE_POLICY,
"verified": verified,
"deactivation_enabled": ALLOWANCE_DEACTIVATE == "on",
"considered": len(rows),
"counts": counts,
"decisions": decisions,
}
# --------------------------------------------------------------------------- #
# Allowance shortfall — OC's billing-day alignment truncates a paid cycle. #
# --------------------------------------------------------------------------- #
# Open Collective aligns every recurring charge onto the collective's COMMON
# billing day, so a member who joins mid-month is charged again well before a
# full month has elapsed: they pay for a full month and receive less than one.
# Operator decision (2026-11-04): make the member whole by crediting the missing
# ALLOWANCE — the flat MEMBER_BUDGET pro-rated for the days lost — as
# non-expiring credits (the same balance that extends the allowance and is spent
# only after it). It is NOT a share of what they paid, and it is NOT rounded.
#
# Scope: EVERY monthly paid cycle shorter than a full month, for any member and
# any cycle (not only the first — OC can re-align an existing charge date too).
# Derived FROM DATA (each recurring order's real charge history); members are
# never asked to claim it. Idempotent: each credit is keyed to the specific OC
# charge that caused it, so a re-run credits nothing twice. A charge whose date
# cannot be determined is SKIPPED and logged — paying a credit that is not owed
# is worse than paying one late.
#
# Member notices go out automatically (the sweep does both in one pass), but they
# default OFF (ALLOWANCE_SHORTFALL_EMAILS=off) so the wording can be reviewed
# before any member is written to about billing. Setting it to `on` sends the
# staged notice on the next sweep; nothing is re-sent per charge.
FULL_PAID_MONTH_DAYS = 30 # the allowance is a flat monthly amount => "a month" = 30 days
ALLOWANCE_SHORTFALL_EMAILS = (os.environ.get("ALLOWANCE_SHORTFALL_EMAILS", "off") or "off").strip().lower()
def shortfall_reason(prev: str, cur: str) -> str:
"""Human-readable ledger reason — a member's question is answerable from it."""
return f"allowance shortfall, truncated paid cycle {prev}→{cur} (OC billing alignment)"
# Charges already made whole OUTSIDE this mechanism, keyed by the OC charge
# reference alone (the same idempotence key the shortfall path uses). The one
# listed charge received a manual allowance reset on 2026-11-04 for the cycle
# OC truncated; it is excluded so a re-run never credits it again. Later
# charges form new cycles and are credited normally. The member's identity and
# the full timeline behind this entry are recorded in the operations docs and
# the ledger — deliberately NOT in this public repository (audit H2,
# 2026-11-05: the exclusion used to embed a member's email and billing dates).
SHORTFALL_REMEDIATED: set[str] = {
"tx_Kr6n3rW577OSjr2bIt5IC",
}
def _usd(x: float) -> str:
"""Format a credit amount for a member without inventing cents (2 -> $2)."""
return f"${x:g}"
def _ensure_shortfall_tables(conn) -> None:
"""Records which charge-triggered notices have actually been sent.
Kept apart from the credit ledger on purpose: the ledger answers "what did we
credit, and why", this answers "what left the building". A notice is sent at
most once per charge reference.
"""
conn.execute(
"CREATE TABLE IF NOT EXISTS shortfall_notices ("
" reference TEXT PRIMARY KEY, email TEXT NOT NULL, amount REAL NOT NULL,"
" sent_at TEXT NOT NULL DEFAULT (datetime('now')))"
)
conn.commit()
def credit_reference_exists(reference: str) -> bool:
"""Has a credit already been recorded against this OC charge reference?"""
conn = get_db()
row = conn.execute(
"SELECT 1 FROM credit_ledger WHERE reference = ? LIMIT 1", (reference,)
).fetchone()
conn.close()
return bool(row)
def grant_held_credits(conn: sqlite3.Connection, email: str, slug: str = "") -> float:
"""Apply every HELD credit pack matched to this member, atomically (M5).
Per hold, in ONE transaction: flip the hold `held` → `granted`, move the
member's balance, and write the ledger row — mirroring admin_held_grant.
Committing the three together is the point: an exception can no longer
leave hold=granted with the member uncredited. The ledger reference is
`held:<hold_id>` (the same shape admin_held_grant uses), so re-running a
hold that was already applied is a no-op and the resolve-then-credit can
never double-apply. Returns the total amount granted.
"""
email = (email or "").strip().lower()
rows = conn.execute(
"SELECT id, amount FROM held_credits "
"WHERE status = 'held' AND (LOWER(email) = ? OR LOWER(slug) = ?)",
(email.lower(), slug.lower() if slug else ""),
).fetchall()
granted = 0.0
for r in rows:
hold_id, amount = r[0], float(r[1] or 0.0)
reference = f"held:{hold_id}"
if credit_reference_exists(reference):
# Already applied by an earlier run (e.g. an admin grant on this
# very hold): reconcile the status, move no money.
conn.execute(
"UPDATE held_credits SET status = 'granted', resolved_at = "
"COALESCE(resolved_at, datetime('now')) WHERE id = ? AND status = 'held'",
(hold_id,),
)
conn.commit()
continue
conn.execute(
"UPDATE held_credits SET status = 'granted', resolved_at = datetime('now') "
"WHERE id = ? AND status = 'held'",
(hold_id,),
)
conn.execute(
"UPDATE members SET credit_balance = "
"MAX(0, COALESCE(credit_balance, 0) + ?) WHERE LOWER(email) = ?",
(amount, email),
)
conn.execute(
"INSERT INTO credit_ledger (email, delta, reason, reference) "
"VALUES (?, ?, ?, ?)",
(email, amount, "held credit pack(s) granted on membership", reference),
)
conn.commit()
granted += amount
return granted
def shortfall_notice_sent(reference: str) -> bool:
conn = get_db()
row = conn.execute(
"SELECT 1 FROM shortfall_notices WHERE reference = ? LIMIT 1", (reference,)
).fetchone()
conn.close()
return bool(row)
def mark_shortfall_notice_sent(reference: str, email: str, amount: float) -> None:
conn = get_db()
_ensure_shortfall_tables(conn)
conn.execute(
"INSERT OR REPLACE INTO shortfall_notices (reference, email, amount, sent_at)"
" VALUES (?, ?, ?, datetime('now'))",
(reference, email, amount),
)
conn.commit()
conn.close()
async def fetch_oc_monthly_order_charges() -> list[dict]:
"""Every MONTHLY order's real contribution charges — one record per order.
A record is {"slug", "order_id", "charges": [(datetime, publicId), ...]} from
ONE order. Truncation is judged per order: merging the charges of two
DIFFERENT subscriptions a member happens to hold invents a short gap that is
not a truncated cycle (one member started a second monthly order six days
after the first; that is not OC re-aligning a charge). Raises on failure so
the caller skips rather than mis-decides. Charges are the CONTRIBUTION
amounts INTO the collective (a non-refund CREDIT), one per charge.
"""
if not OC_PERSONAL_TOKEN:
raise RuntimeError("OC_PERSONAL_TOKEN not set; cannot fetch order charges")
query = (
'{ collective(slug: "%s") { orders(limit: 200) { totalCount nodes { '
"id frequency fromAccount { slug } "
"transactions { publicId createdAt type kind isRefund } } } } }"
% OC_COLLECTIVE_SLUG
)
async with httpx.AsyncClient(timeout=30.0) as client:
r = await client.post(
OC_GRAPHQL_URL,
headers={"Content-Type": "application/json",
"Authorization": f"Bearer {OC_PERSONAL_TOKEN}",
"Api-Version": "2022-01-01",
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"},
json={"query": query},
)
if r.status_code != 200:
raise RuntimeError(f"OC order-charge fetch failed: HTTP {r.status_code}")
body = r.json()
if body.get("errors"):
raise RuntimeError(f"OC order-charge fetch errors: {str(body['errors'])[:200]}")
nodes = (((body.get("data") or {}).get("collective") or {}).get("orders") or {}).get("nodes") or []
out: list[dict] = []
for o in nodes:
if (o.get("frequency") or "").upper() != "MONTHLY":
continue
slug = ((o.get("fromAccount") or {}).get("slug") or "").strip()
if not slug:
continue
charges = []
for t in o.get("transactions") or []:
if t.get("kind") != "CONTRIBUTION" or t.get("isRefund"):
continue
if (t.get("type") or "").upper() != "CREDIT":
continue
dt = _parse_oc_dt(t.get("createdAt"))
if dt and t.get("publicId"):
charges.append((dt, t["publicId"]))
charges.sort()
if charges:
out.append({"slug": slug, "order_id": o.get("id"), "charges": charges})
return out
def derive_shortfalls(order_records: list[dict], budget: float | None = None,
full_month_days: int = FULL_PAID_MONTH_DAYS) -> list[dict]:
"""Pure: truncated paid cycles from per-order monthly charge ladders.
For each consecutive pair of charges on the SAME order, a gap shorter than a
full month means the member paid for a full month and received fewer days of
allowance. Credit = budget * days_lost / full_month_days (no rounding). One
record per truncated cycle; `reference` is the early charge that truncated it
(globally unique, so it is the idempotence key).
"""
budget = MEMBER_BUDGET if budget is None else budget
out: list[dict] = []
for rec in order_records:
charges = rec.get("charges") or []
for i in range(1, len(charges)):
prev_dt, _ = charges[i - 1]
cur_dt, cur_id = charges[i]
gap = (cur_dt - prev_dt).days
if gap >= full_month_days:
continue
lost = full_month_days - gap
out.append({
"slug": rec.get("slug"), "order_id": rec.get("order_id"),
"prev": prev_dt.date().isoformat(), "cur": cur_dt.date().isoformat(),
"gap_days": gap, "days_lost": lost,
"amount": budget * lost / full_month_days, "reference": cur_id,
})
return out
def shortfall_email(name: str, amount: float, prev: str, cur: str,
gap_days: int, days_lost: int, pack_url: str) -> tuple[str, str, str]:
"""Build the shortfall credit notice (subject, text, html).
Same shell as welcome_email()/budget_notice_email(): cream background, white
card, forest-green button, inline pine-tree mark, table-based, no external
assets. Plain and direct — says what happened, why, the amount, and how to get
more. No filler, no apology theatre.
"""
subject = "A credit for a short allowance cycle"
text = (
f"Hi {name},\n\n"
"Your Open Collective contribution is charged on the collective's billing "
"day, not on the day you joined. That can make a charge arrive less than a "
"month after the one before it, so a month you paid for turns out shorter "
"than a month of allowance.\n\n"
f"That happened on your account: charges on {prev} and {cur} were only "
f"{gap_days} days apart, so you paid for a full month and received "
f"{days_lost} days less than one.\n\n"
f"To make that up, we've added {_usd(amount)} to your account as credits. "
"Credits don't expire, and they're used after your monthly allowance.\n\n"
"If you'd like to add more, you can buy credit packs here:\n\n"
f"{pack_url}\n\n"
"Questions? Reply to this email or write to info@inference.coop.\n\n"
"— The Inference Cooperative\n"
)
html = f"""<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"></head>
<body style="margin:0;padding:0;background:#faf8f5;font-family:-apple-system,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;">
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:#faf8f5;padding:40px 16px;">
<tr>
<td align="center">
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="max-width:520px;">
<tr>
<td align="center" style="padding-bottom:24px;">
{PINE_LOGO_SVG}
</td>
</tr>
<tr>
<td align="center" style="background:#ffffff;border:1px solid #e8e2d8;border-radius:16px;padding:40px 32px;">
<h1 style="margin:0 0 12px;font-size:24px;font-weight:700;color:#2d3327;">A credit for a short allowance cycle</h1>
<p style="margin:0 0 8px;font-size:16px;color:#6b7a62;line-height:1.5;">Your Open Collective contribution is charged on the collective's billing day, not on the day you joined. That can make a charge arrive less than a month after the one before it, so a month you paid for turns out shorter than a month of allowance.</p>
<p style="margin:0 0 20px;font-size:15px;color:#6b7a62;line-height:1.5;">That happened on your account: charges on {prev} and {cur} were only {gap_days} days apart, so you paid for a full month and received {days_lost} days less than one.</p>
<p style="margin:0 0 24px;font-size:15px;color:#2d3327;line-height:1.5;"><strong>We've added {_usd(amount)} to your account as credits.</strong> Credits don't expire, and they're used after your monthly allowance.</p>
<a href="{pack_url}" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:999px;font-size:16px;font-weight:600;">Buy credit packs</a>
<p style="margin:28px 0 0;font-size:13px;color:#94a08c;line-height:1.5;">If you'd like to add more, you can buy credit packs at the button above.</p>
</td>
</tr>
<tr>
<td align="center" style="padding-top:24px;font-size:13px;color:#94a08c;line-height:1.6;">
Questions? Reply to this email or write to <a href="mailto:info@inference.coop" style="color:#6b7a62;">info@inference.coop</a>.<br>
<span style="color:#2d3327;">The Inference Cooperative</span> &middot; inference.coop
</td>
</tr>
</table>
</td>
</tr>
</table>
</body>
</html>"""
return subject, text, html
async def reconcile_allowance_shortfalls(send: bool = False) -> dict:
"""Credit members whose PAID cycle Open Collective's billing-day alignment truncated.
`send=False` (report) computes the set and changes NOTHING. `send=True`
(honoured only when ALLOWANCE_POLICY == "on") applies one credit per truncated
charge — idempotent on the charge reference — and, when
ALLOWANCE_SHORTFALL_EMAILS == "on", sends the member notice (once per charge).
Emails default OFF so the wording is reviewed before any member is written to.
"""
try:
records = await fetch_oc_monthly_order_charges()
except Exception as e:
logger.warning("Allowance shortfall: charge fetch failed; skipping (fail-safe): %s", e)
return {"status": "skipped", "reason": "charge_fetch_failed", "detail": str(e)[:200]}
try:
index = await fetch_member_index()
except Exception as e:
logger.warning("Allowance shortfall: member index fetch failed; skipping (fail-safe): %s", e)
return {"status": "skipped", "reason": "member_index_failed", "detail": str(e)[:200]}
cycles = derive_shortfalls(records)
# slug -> email from the LIVE OC member list; only provisioned, active members
# are credited. Orders are matched by the slug resolved from the member's email
# (the stored column can hold a stale guest-… slug from first provisioning).
slug2email: dict[str, str] = {}
for e, rec in index.items():
s = rec.get("slug") or ""
if s:
slug2email.setdefault(s, e)
conn = get_db()
_ensure_shortfall_tables(conn)
active = {r[0].strip().lower() for r in conn.execute("SELECT email FROM members WHERE active = 1")}
meta = {r[0].strip().lower(): {"slug": r[1], "org_slug": r[2]}
for r in conn.execute("SELECT email, slug, org_slug FROM members")}
conn.close()
acting = bool(send and ALLOWANCE_POLICY == "on")
mail_on = ALLOWANCE_SHORTFALL_EMAILS == "on"
pack_url = credit_pack_url()
credited, staged, skipped = [], [], []
seen: set[str] = set()
for c in cycles:
if c["reference"] in seen: # a charge truncates exactly one cycle
continue
email = slug2email.get(c["slug"])
if not email:
skipped.append({**c, "why": "no Open Collective account email for this charge's slug"})
continue
if email not in active:
skipped.append({**c, "email": email, "why": "not an active portal member"})
continue
m = meta.get(email) or {}
if email in MANUAL_MEMBERS or m.get("org_slug"):
skipped.append({**c, "email": email, "why": "exempt member (manual / organization-backed)"})
continue
rec = index.get(email)
if rec and rec.get("role") == "ADMIN":
skipped.append({**c, "email": email, "why": "operator account"})
continue
if c["reference"] in SHORTFALL_REMEDIATED:
skipped.append({**c, "email": email,
"why": "already remediated by a manual allowance reset (2026-11-04)"})
continue
seen.add(c["reference"])
entry = {**c, "email": email, "name": (m.get("slug") or "there")}
already = credit_reference_exists(c["reference"])
entry["already_credited"] = already
if not already and acting:
add_credits(email, c["amount"],
reason=shortfall_reason(c["prev"], c["cur"]),
reference=c["reference"])
try:
await sync_team_budget_from_balance(email)
except Exception as e: # budget push is best-effort
logger.warning("Shortfall: team budget sync failed for %s: %s", email, e)
entry["credited"] = True
elif not already:
entry["credited"] = False # would-credit (report mode / policy off)
if mail_on and not shortfall_notice_sent(c["reference"]):
subject, text_body, html_body = shortfall_email(
entry["name"], c["amount"], c["prev"], c["cur"], c["gap_days"],
c["days_lost"], pack_url)
if send_email(email, subject, text_body, html_body):
mark_shortfall_notice_sent(c["reference"], email, c["amount"])
entry["notified"] = True
else:
entry["notified"] = False
elif not mail_on:
entry["notice"] = "staged (ALLOWANCE_SHORTFALL_EMAILS is off)"
(credited if entry.get("credited") or entry.get("already_credited") else staged).append(entry)
newly = sum(e["amount"] for e in credited if e.get("credited"))
total = sum(e["amount"] for e in credited)
return {
"status": "ok",
"mode": ALLOWANCE_POLICY,
"acting": acting,
"emails": ALLOWANCE_SHORTFALL_EMAILS,
"full_month_days": FULL_PAID_MONTH_DAYS,
"affected": len(credited),
"newly_credited": sum(1 for e in credited if e.get("credited")),
"already_credited": sum(1 for e in credited if e.get("already_credited")),
"new_credit_total": round(newly, 10),
"affected_credit_total": round(total, 10),
"credits": credited,
"would_credit": staged,
"skipped": skipped,
}
@app.post("/admin/shortfall-credits")
@limiter.limit("10/minute")
async def admin_shortfall_credits(request: Request, mode: str = "report"):
"""Preview (default) or apply allowance-shortfall credits.
`mode=report` computes the affected set and changes nothing. `mode=send`
applies the credits (requires ALLOWANCE_POLICY=on) — idempotent on the OC
charge reference, so a second call credits nothing again. Member notices are
sent only when ALLOWANCE_SHORTFALL_EMAILS=on.
"""
_verify_admin_token(request)
if mode == "send":
if ALLOWANCE_POLICY != "on":
return {"refused": f"ALLOWANCE_POLICY is '{ALLOWANCE_POLICY}', not 'on'"}
return await reconcile_allowance_shortfalls(send=True)
return await reconcile_allowance_shortfalls(send=False)
@app.post("/admin/allowance")
@limiter.limit("10/minute")
async def admin_allowance(request: Request, mode: str = "report", to: str | None = None):
"""Reconcile member allowances against Open Collective payments.
`mode=report` (default) changes nothing. `mode=send` performs renewals and
sends the single reminder, and requires ALLOWANCE_POLICY=on on the app.
Deactivation is separately gated by ALLOWANCE_DEACTIVATE=on.
`mode=test&to=<address>` emails one sample reminder so the copy and the
outbound path can be verified without touching the ledger.
"""
_verify_admin_token(request)
if mode == "test":
dest = (to or "").strip()
if not dest:
return {"refused": "mode=test needs &to=<address>"}
subject, text_body, html_body = payment_reminder_email("there", credit_pack_url())
ok = send_email(dest, subject, text_body, html_body)
return {"mode": "test", "to": dest, "sent": bool(ok), "subject": subject}
if mode == "send":
if ALLOWANCE_POLICY != "on":
return {"refused": f"ALLOWANCE_POLICY is '{ALLOWANCE_POLICY}', not 'on' — "
"nothing sent; set it on the portal app first"}
return await reconcile_allowances(send=True)
return await reconcile_allowances(send=False)
@app.post("/admin/sweep")
@limiter.limit("20/minute")
async def admin_sweep(request: Request):
"""Manually trigger a full membership reconciliation.
Authenticated by the X-Admin-Token header. Idempotent — safe to call
repeatedly. This is also what the Cloudron scheduler invokes on a cron
schedule (see the sweep script).
"""
_verify_admin_token(request)
result = await reconcile_memberships()
# Budget notices ride the same cron the Cloudron scheduler already hits. They are OFF
# unless BUDGET_NOTICES=on, and wrapped so a failure here can never break reconciliation.
try:
result["budget_notices"] = await budget_notice_sweep(send=(budget_notices_enabled() == "on"))
except Exception as exc: # noqa: BLE001
result["budget_notices"] = {"error": str(exc)[:200]}
# Payment-driven allowance renewal (join-day anchored). Rides the same sweep.
# REPORT-ONLY unless ALLOWANCE_POLICY=on; deactivation additionally needs
# ALLOWANCE_DEACTIVATE=on. Wrapped so it can never break reconciliation.
try:
result["allowances"] = await reconcile_allowances(send=(ALLOWANCE_POLICY == "on"))
except Exception as exc: # noqa: BLE001
result["allowances"] = {"error": str(exc)[:200]}
# Allowance-shortfall credits: OC's billing-day alignment truncates some paid
# cycles. Rides the same sweep so a new member is compensated automatically;
# credits are idempotent on the OC charge reference. Wrapped so it can never
# break reconciliation; notices only go out when ALLOWANCE_SHORTFALL_EMAILS=on.
try:
result["shortfall_credits"] = await reconcile_allowance_shortfalls(
send=(ALLOWANCE_POLICY == "on"))
except Exception as exc: # noqa: BLE001
result["shortfall_credits"] = {"error": str(exc)[:200]}
return result
@app.post("/admin/newsletter/reconcile")
@limiter.limit("20/minute")
async def admin_newsletter_reconcile(request: Request):
"""Remove stale subscribers from the Members newsletter list.
The newsletter list must mirror ACTIVE members: deleted accounts,
never-activated test accounts, and lapsed members are removed (see
listmonk_reconcile for the exact rules). Also runs automatically as
step 4 of the nightly membership sweep.
Authenticated by the X-Admin-Token header.
"""
_verify_admin_token(request)
return await listmonk_reconcile()
def _verify_sync_token(request: Request) -> None:
"""Authenticate the LibreChat key-sync feed via X-Sync-Token.
Dedicated machine secret (least privilege): it only ever exposes the
members' own chat keys, never the admin API. The same secret is set on
the LibreChat sandbox app (LIBRECHAT_SYNC_TOKEN in /app/data/env).
"""
expected = os.environ.get("LIBRECHAT_SYNC_TOKEN", "")
if not expected:
raise HTTPException(503, "Sync token not configured")
provided = request.headers.get("x-sync-token", "")
if not secrets.compare_digest(provided, expected):
raise HTTPException(401, "Invalid sync token")
@app.get("/admin/librechat/keys")
@limiter.limit("30/minute")
async def admin_librechat_keys(request: Request):
"""Key feed for the LibreChat sandbox's per-member key auto-load.
Returns every ACTIVE member's portal-provisioned chat key so the sandbox
can seed LibreChat's user-key store and members never see an API-key
prompt (chat bills to the member's own LiteLLM team/budget). Auth is the
dedicated X-Sync-Token header — NOT the admin token.
DELIBERATE DESIGN (audit M2, 2026-11-05): one static token returns every
member's decrypted chat key — a total-member-key blast radius. Kept (not
per-member fetch) because the only consumer is the LibreChat sandbox's
in-app key-sync, and scoping the feed per member would require changing
the sandbox's sync identity model; a wrong change there stops key seeding
for every member at once. Recorded in ops-docs
(maintenance/librechat-key-feed.md). Monitoring: every fetch is logged
(count only, never key material) and the daily digest watches the rate —
the sandbox polls ~every 10 min, so anything far above that cadence (or a
401 burst on this route) is surfaced as an anomaly.
"""
_verify_sync_token(request)
conn = get_db()
rows = conn.execute(
"SELECT email, key_token FROM members WHERE active = 1 AND key_token != ''"
).fetchall()
conn.close()
logger.warning(
"librechat key feed served: %d member key(s) to %s (ua=%s)",
len(rows), request.client.host if request.client else "?",
(request.headers.get("user-agent") or "?")[:60],
)
return {
"members": [
{"email": email, "key": decrypt_key(key_token)}
for email, key_token in rows
]
}
@app.get("/")
async def index():
return {"service": "Inference Cooperative Member Portal", "version": "0.1.0"}
# --------------------------------------------------------------------------- #
# Budget notices — email a member as they approach their monthly limit. #
# --------------------------------------------------------------------------- #
# OFF by default: BUDGET_NOTICES = off | dryrun | on. Nothing is ever emailed
# unless it is "on"; the admin endpoint below can preview at any time, including
# while off. One notice per level per budget cycle, so a member hears from us
# once when they cross 80% and once when they reach the cap — never on every
# sweep. Spend and budget are read from LiteLLM's own Postgres, the same source
# the dashboard and admin panel display, so the numbers cannot drift.
#
# CREDIT_PACK_URL is configuration because the website has no dedicated pack
# link today; the Open Collective contribution page is what actually credits a
# member (the webhook settles contributions into credits).
# Generic on purpose: the balance can be short because the monthly allowance is spent,
# because purchased credit is spent, or a mix — the letter says "balance" so it is true
# either way. The trigger is proportional — 90% of the balance used — so it means the same
# thing to a member on $15 and one on $35: the letter arrives when roughly a tenth is left,
# on whichever sweep follows, rather than at an arbitrary absolute amount.
BUDGET_NOTICE_AT = float(os.environ.get("BUDGET_NOTICE_AT", "0.90"))
PINE_LOGO_SVG = """<svg width="56" height="56" viewBox="0 0 130 130" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true">
<circle cx="65" cy="14" r="5" fill="#5b8c5a"/>
<circle cx="58" cy="30" r="6" fill="#7ab87a"/>
<circle cx="74" cy="30" r="4" fill="#6ba86b"/>
<circle cx="51.5" cy="46" r="6.5" fill="#5b8c5a"/>
<circle cx="65" cy="46" r="5" fill="#7ab87a"/>
<circle cx="81" cy="46" r="4" fill="#6ba86b"/>
<circle cx="44" cy="62" r="7" fill="#6ba86b"/>
<circle cx="61" cy="62" r="5.5" fill="#5b8c5a"/>
<circle cx="76" cy="62" r="4.5" fill="#7ab87a"/>
<circle cx="89.5" cy="62" r="3.5" fill="#6ba86b"/>
<circle cx="36.5" cy="78" r="7.5" fill="#5b8c5a"/>
<circle cx="54" cy="78" r="6" fill="#7ab87a"/>
<circle cx="70" cy="78" r="5" fill="#6ba86b"/>
<circle cx="84" cy="78" r="4" fill="#5b8c5a"/>
<circle cx="97.5" cy="78" r="3.5" fill="#7ab87a"/>
<circle cx="44" cy="94" r="6.5" fill="#6ba86b"/>
<circle cx="61" cy="94" r="5.5" fill="#5b8c5a"/>
<circle cx="76" cy="94" r="4.5" fill="#7ab87a"/>
<circle cx="89.5" cy="94" r="3.5" fill="#6ba86b"/>
<circle cx="51.5" cy="110" r="5.5" fill="#5b8c5a"/>
<circle cx="65" cy="110" r="4.5" fill="#7ab87a"/>
<circle cx="81" cy="110" r="3.5" fill="#6ba86b"/>
</svg>"""
def budget_notices_enabled() -> str:
"""BUDGET_NOTICES: off (default) | dryrun | on."""
return (os.environ.get("BUDGET_NOTICES", "off") or "off").strip().lower()
def credit_pack_url() -> str:
return os.environ.get(
"CREDIT_PACK_URL", "https://opencollective.com/inference-cooperative/contribute"
)
def _ensure_budget_notices_table(conn) -> None:
conn.execute(
"CREATE TABLE IF NOT EXISTS budget_notices ("
" email TEXT NOT NULL, level TEXT NOT NULL, cycle TEXT NOT NULL,"
" sent_at TEXT NOT NULL, detail TEXT,"
" PRIMARY KEY (email, level, cycle))"
)
conn.commit()
def budget_notice_email(
name: str, spend: float, budget: float, remaining: float, pack_url: str
) -> tuple[str, str, str]:
"""Build the low-balance notice (subject, text, html).
Deliberately generic about where the balance came from: a member may be short
because the monthly allowance is spent, because credit they bought is spent, or
a mix of both. The wording says "balance" so it is true either way, and it names
no mechanism a member would have to reconcile against their own dashboard.
Same shell as welcome_email(): cream background, white card, forest-green
button, inline pine-tree mark, table-based layout, no external assets.
"""
subject = "Your Inference Cooperative balance is running low"
heading = "Your balance is running low"
lede = (f"You have ${remaining:.2f} of your ${budget:.2f} balance left. When it "
"runs out, chat and API requests stop until you add more.")
money = f"${remaining:.2f} remaining"
text = (
f"Hi {name},\n\n"
f"{lede}\n\n"
f"{money}\n\n"
"If you'd like to keep going, you can buy credit packs here:\n\n"
f"{pack_url}\n\n"
"This is the only notice of its kind you'll get until your balance goes back "
"above the line and runs low again. Questions? Reply to this email or write to "
"info@inference.coop.\n\n"
"— The Inference Cooperative\n"
)
html = f"""<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"></head>
<body style="margin:0;padding:0;background:#faf8f5;font-family:-apple-system,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;">
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:#faf8f5;padding:40px 16px;">
<tr>
<td align="center">
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="max-width:520px;">
<tr>
<td align="center" style="padding-bottom:24px;">
{PINE_LOGO_SVG}
</td>
</tr>
<tr>
<td align="center" style="background:#ffffff;border:1px solid #e8e2d8;border-radius:16px;padding:40px 32px;">
<h1 style="margin:0 0 12px;font-size:24px;font-weight:700;color:#2d3327;">{heading}</h1>
<p style="margin:0 0 8px;font-size:16px;color:#6b7a62;line-height:1.5;">{lede}</p>
<p style="margin:0 0 24px;font-size:15px;color:#2d3327;line-height:1.5;"><strong>{money}</strong></p>
<a href="{pack_url}" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:999px;font-size:16px;font-weight:600;">Buy credit packs</a>
<p style="margin:28px 0 0;font-size:13px;color:#94a08c;line-height:1.5;">This is the only notice of its kind you'll get until your balance goes back above the line and runs low again.</p>
</td>
</tr>
<tr>
<td align="center" style="padding-top:24px;font-size:13px;color:#94a08c;line-height:1.6;">
Questions? Reply to this email or write to <a href="mailto:info@inference.coop" style="color:#6b7a62;">info@inference.coop</a>.<br>
<span style="color:#2d3327;">The Inference Cooperative</span> &middot; inference.coop
</td>
</tr>
</table>
</td>
</tr>
</table>
</body>
</html>"""
return subject, text, html
async def budget_notice_sweep(send: bool = False) -> dict:
"""Find members whose balance has fallen below the low-water mark, and optionally email.
`send=False` is a preview: it computes exactly who would be written to, with the
numbers, and sends nothing. `send=True` emails and records the send — the caller
is responsible for requiring BUDGET_NOTICES=on first.
"""
mode = budget_notices_enabled()
if send and mode != "on":
return {"refused": f"BUDGET_NOTICES is '{mode}', not 'on' — nothing sent"}
conn = get_db()
_ensure_budget_notices_table(conn)
cols = {r[1] for r in conn.execute("PRAGMA table_info(members)")}
# There is no name column in `members` — `slug` is the username (e.g. ntnsndr). Falling
# back to the email local part produced "Hi n," for the operator, which is how this was
# caught. Prefer the username; say "there" rather than inventing a name.
ident = "slug" if "slug" in cols else None
sel = ("SELECT email" + (f", {ident}" if ident else "") + ", account_type"
" FROM members WHERE active = 1")
rows = conn.execute(sel).fetchall()
pack = credit_pack_url()
preview, sent, skipped = [], 0, 0
for row in rows:
email = row[0]
name = ((row[1] or "").strip() if len(row) > 1 else "") or "there"
acct_type = (row[2] or "member") if len(row) > 2 else "member"
if acct_type == "guest":
# A guest's exhaustion is handled by the guest lifecycle pass (the
# sweep deactivates it); member low-balance mail about buying
# credit packs is not for guests.
skipped += 1
continue
team_id = await litellm_team_id_for_email(email)
if not team_id:
skipped += 1
continue
stats = get_team_budget_from_db(team_id) or {}
budget = stats.get("max_budget")
spend = float(stats.get("spend") or 0.0)
if not budget or budget <= 0:
skipped += 1 # no cap configured: nothing to warn about
continue
remaining = float(budget) - spend
if spend < BUDGET_NOTICE_AT * float(budget):
# Above the line — they added funds, or the allowance reset. Clear the latch so the
# NEXT crossing sends a fresh notice. That is what makes it once each time, and it
# is why this is a latch rather than a key on the budget cycle.
conn.execute("DELETE FROM budget_notices WHERE email = ?", (email,))
conn.commit()
continue
if conn.execute("SELECT 1 FROM budget_notices WHERE email = ?", (email,)).fetchone():
skipped += 1 # already told them for this crossing
continue
subject, text_body, html_body = budget_notice_email(name, spend, budget, remaining, pack)
entry = {
"email": email, "remaining": round(remaining, 2),
"spend": round(spend, 2), "budget": round(budget, 2),
"resets": str(stats.get("budget_reset_at") or stats.get("reset_at") or "")[:19],
}
if send:
ok = send_email(email, subject, text_body, html_body)
entry["sent"] = bool(ok)
if ok:
conn.execute(
"INSERT OR REPLACE INTO budget_notices"
" (email, level, cycle, sent_at, detail) VALUES (?, 'low', ?, datetime('now'), ?)",
(email, entry["resets"] or "crossing", f"remaining={remaining:.4f}"),
)
conn.commit()
sent += 1
else:
entry["subject"] = subject
preview.append(entry)
conn.close()
if send:
return {"mode": mode, "sent": sent, "considered": len(rows),
"skipped": skipped, "notices": preview}
return {"mode": mode, "dry_run": True, "considered": len(rows), "skipped": skipped,
"would_notify": len(preview), "notices": preview}
async def budget_notice_test(dest: str) -> dict:
"""Email one sample low-balance letter to `dest`, using that member's real numbers.
Deliberately does NOT write to budget_notices: a test must not consume the real
once-per-crossing latch, or reviewing the copy would silence the actual notice.
Works while BUDGET_NOTICES is off, so copy can be reviewed before arming anything.
"""
conn = get_db()
row = conn.execute("SELECT slug FROM members WHERE email = ?", (dest,)).fetchone()
name = ((row[0] or "").strip() if row else "") or "there"
conn.close()
budget, spend = 15.0, 13.34 # sample numbers if they aren't a member
if row:
team_id = await litellm_team_id_for_email(dest)
stats = (get_team_budget_from_db(team_id) or {}) if team_id else {}
if stats.get("max_budget"):
budget = float(stats["max_budget"])
spend = float(stats.get("spend") or 0.0)
remaining = budget - spend
subject, text_body, html_body = budget_notice_email(
name, spend, budget, remaining, credit_pack_url()
)
ok = send_email(dest, subject, text_body, html_body)
return {
"mode": "test", "to": dest, "sent": bool(ok), "subject": subject,
"rendered": {"name": name, "spend": round(spend, 2), "budget": round(budget, 2),
"remaining": round(remaining, 2)},
"note": "no ledger row written — the real once-per-crossing latch is untouched",
}
@app.post("/admin/budget-notices")
@limiter.limit("10/minute")
async def admin_budget_notices(request: Request, mode: str = "dryrun", to: str | None = None):
"""Preview (default), send, or email a single sample.
Authenticated by X-Admin-Token. `mode=dryrun` never emails; `mode=send` requires
BUDGET_NOTICES=on and refuses otherwise; `mode=test&to=<address>` emails one sample
using that member's real numbers so the copy can be read in a mail client first.
A test works while the flag is off and leaves no trace in the ledger.
"""
_verify_admin_token(request)
if mode == "test":
dest = (to or "").strip()
if not dest:
return {"refused": "mode=test needs &to=<address>"}
return await budget_notice_test(dest)
if mode == "send":
if budget_notices_enabled() != "on":
return {"refused": f"BUDGET_NOTICES is '{budget_notices_enabled()}', not 'on'"
" — set it on the portal app first"}
return await budget_notice_sweep(send=True)
return await budget_notice_sweep(send=False)