ci/woodpecker/push/woodpecker Pipeline was canceled
New read-only /broker/membership-status, authenticated by the broker secret with a verifier that admits a member row in ANY state (usage and keys still require active). Returns active/state, the latest allowance_log decision with its evidence, the OC minimum contribution (MIN_CONTRIBUTION, copy-only — OC stays the authority), credits, allowance and the contribute URL. MIN_CONTRIBUTION is deliberately NOT wired into deactivation: a number we half-know should not cost anyone access.
9725 lines
447 KiB
Python
9725 lines
447 KiB
Python
"""
|
||
Member Portal — membership middleware for the Inference Cooperative.
|
||
|
||
Reconciles three systems:
|
||
1. Open Collective — who is a paying member (billing)
|
||
2. Cloudron — who can log in (identity/SSO)
|
||
3. LiteLLM — who can use the models, and how much (inference)
|
||
|
||
Three responsibilities:
|
||
A. Webhook handler — listen for Open Collective membership events
|
||
B. Key injector — read the member's email from a header, inject their
|
||
LiteLLM key, and forward the request to the gateway
|
||
C. Admin endpoints — health, status, manual reconciliation
|
||
"""
|
||
|
||
import os
|
||
import json
|
||
import logging
|
||
import re
|
||
import sqlite3
|
||
import secrets
|
||
import smtplib
|
||
from datetime import datetime, timedelta, timezone
|
||
from email.mime.text import MIMEText
|
||
from email.mime.multipart import MIMEMultipart
|
||
from email.utils import make_msgid
|
||
|
||
# --- Key encryption at rest (inlined from keycrypto.py; see repo history) ---
|
||
# Member chat keys are encrypted with AES-256-GCM using a key-encryption key
|
||
# (KEK) from the PORTAL_KEK env var. Plaintext exists only in process memory
|
||
# when a key is served. Format: "enc1:" + base64(nonce[12] || ct || tag).
|
||
# Values without the prefix are legacy plaintext (pre-migration) and pass
|
||
# through on decrypt. Fail-closed: nothing new is stored plaintext when
|
||
# PORTAL_KEK is unset, and encrypted values cannot be served without it.
|
||
import base64 as _b64
|
||
|
||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM as _AESGCM
|
||
|
||
_KEYENC_PREFIX = "enc1:"
|
||
_KEYENC_AAD = b"member-chat-key-v1"
|
||
|
||
|
||
def _kek() -> bytes:
|
||
raw = os.environ.get("PORTAL_KEK", "")
|
||
if not raw:
|
||
raise RuntimeError(
|
||
"PORTAL_KEK is not set - refusing to encrypt/decrypt member keys"
|
||
)
|
||
key = _b64.b64decode(raw)
|
||
if len(key) != 32:
|
||
raise RuntimeError("PORTAL_KEK must decode to exactly 32 bytes (base64)")
|
||
return key
|
||
|
||
|
||
def encrypt_key(plain: str | None) -> str | None:
|
||
"""Encrypt a member key. Already-encrypted and empty values pass through."""
|
||
if not plain or plain.startswith(_KEYENC_PREFIX):
|
||
return plain
|
||
nonce = os.urandom(12)
|
||
ct = _AESGCM(_kek()).encrypt(nonce, plain.encode(), _KEYENC_AAD)
|
||
return _KEYENC_PREFIX + _b64.b64encode(nonce + ct).decode()
|
||
|
||
|
||
def decrypt_key(stored: str | None) -> str | None:
|
||
"""Decrypt a member key. Legacy plaintext rows pass through unchanged."""
|
||
if not stored:
|
||
return stored
|
||
if not stored.startswith(_KEYENC_PREFIX):
|
||
return stored # pre-migration plaintext
|
||
blob = _b64.b64decode(stored[len(_KEYENC_PREFIX):])
|
||
return _AESGCM(_kek()).decrypt(blob[:12], blob[12:], _KEYENC_AAD).decode()
|
||
|
||
|
||
def migrate_plaintext_keys(conn) -> int:
|
||
"""Encrypt plaintext key_token rows. Idempotent; returns rows changed."""
|
||
if not os.environ.get("PORTAL_KEK"):
|
||
logging.getLogger("uvicorn.error").critical(
|
||
"PORTAL_KEK unset - member keys remain PLAINTEXT at rest; "
|
||
"set PORTAL_KEK to a base64 32-byte key and restart"
|
||
)
|
||
return 0
|
||
rows = conn.execute(
|
||
"SELECT email, key_token FROM members "
|
||
"WHERE key_token != '' AND key_token IS NOT NULL "
|
||
"AND key_token NOT LIKE 'enc1:%'"
|
||
).fetchall()
|
||
changed = 0
|
||
for email, kt in rows:
|
||
enc = encrypt_key(kt)
|
||
if enc != kt:
|
||
conn.execute(
|
||
"UPDATE members SET key_token = ? WHERE email = ?", (enc, email)
|
||
)
|
||
changed += 1
|
||
return changed
|
||
# --- end key encryption ---
|
||
|
||
import httpx
|
||
import jwt
|
||
import asyncio
|
||
from fastapi import FastAPI, Request, Response, HTTPException
|
||
from fastapi.responses import JSONResponse, StreamingResponse
|
||
from slowapi import Limiter
|
||
from slowapi.util import get_remote_address
|
||
from slowapi.errors import RateLimitExceeded
|
||
|
||
logging.basicConfig(level=logging.INFO)
|
||
logger = logging.getLogger("member-portal")
|
||
|
||
# SECURITY (ops/security#2): OC's generic webhooks cannot send headers, so the shared
|
||
# secret must travel in the URL path — which means uvicorn writes it into the access
|
||
# log on every delivery, and the log is read by more things than the secret is. The
|
||
# token cannot be kept out of the URL; it can be kept out of the retention. Filters
|
||
# sit on the access logger and on the root handler so both uvicorn's line and any
|
||
# string the app itself logs get scrubbed.
|
||
_WEBHOOK_TOKEN_RE = re.compile(r"(/webhook/opencollective/)[^\s\"'?&]+")
|
||
# M1 (audit 2026-11-05): member emails persist in portal logs — send_email
|
||
# recipients, listmonk failures, Loomio invite lists, provisioning and credit
|
||
# lines (17 sites). Container logs are readable via `cloudron logs`, persist
|
||
# on the host and ride the backups, so an email is redacted from every record
|
||
# the same way the webhook token is. Logs stay useful: counts, slugs, MXIDs
|
||
# (matrix @user:server has no address shape) and amounts all survive; only
|
||
# the address is dropped. The daily digest's own sanitisation is unchanged.
|
||
_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+")
|
||
|
||
|
||
def _redact_emails(value):
|
||
if isinstance(value, str) and "@" in value:
|
||
return _EMAIL_RE.sub("<email>", value)
|
||
return value
|
||
|
||
|
||
def _redact_log_value(value):
|
||
if isinstance(value, str) and "/webhook/opencollective/" in value:
|
||
value = _WEBHOOK_TOKEN_RE.sub(r"\1<redacted>", value)
|
||
return _redact_emails(value)
|
||
|
||
|
||
def _redact_deep(value):
|
||
"""Redact every string in a log argument, preserving the argument's type."""
|
||
if isinstance(value, str):
|
||
return _redact_log_value(value)
|
||
if isinstance(value, tuple):
|
||
return tuple(_redact_deep(v) for v in value)
|
||
if isinstance(value, list):
|
||
return [_redact_deep(v) for v in value]
|
||
if isinstance(value, dict):
|
||
return {k: _redact_deep(v) for k, v in value.items()}
|
||
return value
|
||
|
||
|
||
class _RedactWebhookPath(logging.Filter):
|
||
"""Scrub secrets and member emails out of any log record.
|
||
|
||
Replaces the webhook token in a logged request path (ops/security#2) and
|
||
redacts email-shaped strings from messages, %-format args (recursing into
|
||
tuple/list/dict args) and any exception text already attached to the
|
||
record. A logging filter must never break logging.
|
||
"""
|
||
|
||
def filter(self, record: logging.LogRecord) -> bool:
|
||
try:
|
||
record.msg = _redact_log_value(record.msg)
|
||
if isinstance(record.args, tuple):
|
||
record.args = tuple(_redact_deep(a) for a in record.args)
|
||
elif isinstance(record.args, dict):
|
||
record.args = {k: _redact_deep(v) for k, v in record.args.items()}
|
||
if isinstance(record.exc_text, str):
|
||
record.exc_text = _redact_log_value(record.exc_text)
|
||
except Exception:
|
||
pass # a logging filter must never break logging
|
||
return True
|
||
|
||
|
||
logging.getLogger("uvicorn.access").addFilter(_RedactWebhookPath())
|
||
for _handler in logging.getLogger().handlers:
|
||
_handler.addFilter(_RedactWebhookPath())
|
||
|
||
app = FastAPI(title="Inference Cooperative Member Portal")
|
||
|
||
# --- Rate limiting (control plane only, not the member inference path) ---
|
||
# The data plane (chat completions via /v1/*) is capped by each member's
|
||
# LiteLLM budget, NOT by rate — a member streaming a long conversation must
|
||
# never be throttled. Rate limiting here covers the control-plane surfaces:
|
||
# admin endpoints, the broker, the webhook, and the unauthenticated model list.
|
||
# Every request reaches this app through Cloudron's proxy, so get_remote_address
|
||
# returns the SAME address (172.18.0.1) for the panel, the dashboard, our own
|
||
# scheduler jobs and a member's browser alike. An IP-only key therefore makes each
|
||
# limit a co-op-wide budget rather than a per-caller one: on 2026-11-13 a guest
|
||
# invite was refused with 429 because the bucket had been drained by a minute of
|
||
# unrelated admin traffic (my own verification probes among it), and the operator
|
||
# saw a failure for an action nobody had abused. Keying on the PATH as well gives
|
||
# each route its own bucket, which is what a per-route limit is supposed to mean.
|
||
limiter = Limiter(key_func=lambda request: f"{get_remote_address(request)}:{request.url.path}")
|
||
app.state.limiter = limiter
|
||
|
||
|
||
@app.exception_handler(RateLimitExceeded)
|
||
async def _rate_limit_handler(request: Request, exc: RateLimitExceeded):
|
||
return JSONResponse(
|
||
status_code=429,
|
||
content={"detail": "Too many requests. Slow down."},
|
||
)
|
||
|
||
# --- Configuration (from environment) ---
|
||
CLOUDRON_API = os.environ.get("CLOUDRON_API_ORIGIN", "https://my.inference.coop")
|
||
CLOUDRON_TOKEN = os.environ.get("CLOUDRON_TOKEN", "")
|
||
LITELLM_BASE = os.environ.get("LITELLM_BASE", "https://gateway.inference.coop")
|
||
LITELLM_MASTER_KEY = os.environ.get("LITELLM_MASTER_KEY", "")
|
||
|
||
# Shared secret that the chat frontend uses to authenticate requests to the
|
||
# portal. LibreChat sends it as an X-Portal-Secret header; OpenWebUI signs the
|
||
# user's email as a JWT (see FORWARD_USER_INFO_HEADER_JWT_SECRET below).
|
||
PORTAL_SECRET = os.environ.get("PORTAL_SECRET", "")
|
||
|
||
# OpenWebUI can sign the forwarded user identity as a JWT using this shared
|
||
# secret. When set, the portal verifies the JWT (HS256) to confirm the email
|
||
# genuinely came from OpenWebUI (behind Cloudron SSO) rather than a spoofed
|
||
# header. This is stronger than the plain X-Portal-Secret header because the
|
||
# email itself is tamper-proof.
|
||
|
||
# Open WebUI admin API key — used at provisioning time to add the new member to
|
||
# Open WebUI's `members` group, which gates read access to the shared
|
||
# "Inference Cooperative Docs" RAG collection. Without this, a new member's
|
||
# first chat session has no docs grounding (the nightly refresh would only fix
|
||
# it hours later). Machine-to-machine only; never logged.
|
||
|
||
# Open WebUI knowledge collection id for the co-op docs RAG. Attached to every
|
||
# model in /v1/models so answers are grounded by automatic RAG injection (and
|
||
# the model's knowledge tools are scoped to the docs rather than browsing all
|
||
# knowledge bases). Env-driven; empty disables the attachment.
|
||
DOCS_KNOWLEDGE_ID = os.environ.get("DOCS_KNOWLEDGE_ID", "946e18ef-c1bc-4c57-b23e-7f5fc5b66e51")
|
||
|
||
# Shared secret that the member dashboard uses to authenticate broker requests.
|
||
# The dashboard is Cloudron-SSO-gated and passes the authenticated member's
|
||
# email; the portal trusts that email only because it carries this secret. Kept
|
||
# separate from PORTAL_SECRET so the two can be rotated independently.
|
||
BROKER_SECRET = os.environ.get("BROKER_SECRET", "")
|
||
|
||
# Secret token required in the Open Collective webhook URL path. Open
|
||
# Collective's generic webhooks are not HMAC-signed, so a secret in the URL
|
||
# is the standard way to authenticate them.
|
||
WEBHOOK_TOKEN = os.environ.get("WEBHOOK_TOKEN", "")
|
||
|
||
# SECURITY (2026-10-27 member disclosure): the /admin/* token is SEPARATE from
|
||
# WEBHOOK_TOKEN. WEBHOOK_TOKEN must sit in the Open Collective webhook URL (OC's
|
||
# generic webhooks cannot send headers), so anything that learns that URL — an OC
|
||
# admin, OC's logs, this app's own access logs, a Referer — used to gain portal
|
||
# admin. ADMIN_TOKEN is only ever sent as a header and never appears in a URL.
|
||
# Cutover COMPLETE (2026-10-28): ADMIN_TOKEN_LEGACY=0, so the legacy token no
|
||
# longer authenticates here (verified: ADMIN_TOKEN -> 200, WEBHOOK_TOKEN -> 403).
|
||
# ADMIN_TOKEN was minted new on 2026-10-27 and WEBHOOK_TOKEN was rotated the same
|
||
# day, so NO rotation is outstanding — do not "finish" this by rotating again.
|
||
# Callers must read ADMIN_TOKEN and fail loudly; never fall back to WEBHOOK_TOKEN,
|
||
# which 403s and would turn a missing variable into a mid-run failure.
|
||
ADMIN_TOKEN = os.environ.get("ADMIN_TOKEN", "")
|
||
ADMIN_TOKEN_LEGACY = os.environ.get("ADMIN_TOKEN_LEGACY", "1") != "0"
|
||
|
||
OC_GRAPHQL_URL = "https://opencollective.com/api/graphql/v2"
|
||
OC_COLLECTIVE_SLUG = os.environ.get("OC_COLLECTIVE_SLUG", "inference-cooperative")
|
||
|
||
# Cache of order idV2 -> tip-free contribution amount in cents, resolved via
|
||
# OC's PUBLIC GraphQL API (no token needed; our personal token lacks the
|
||
# orders scope but unauthenticated reads of public orders work).
|
||
_order_amount_cache: dict[str, int] = {}
|
||
|
||
|
||
async def get_order_amount(order_id_v2: str | None) -> int | None:
|
||
"""Look up an order's contribution amount (EXCLUDING the platform tip).
|
||
|
||
The webhook's `totalAmount` includes the buyer's optional OC tip, which
|
||
the co-op never receives — credits must be granted on the tip-free
|
||
amount. Returns cents, or None if unresolvable.
|
||
|
||
The order id arrives from the OC webhook (untrusted input), so it is sent
|
||
as a GraphQL VARIABLE, never interpolated into the query string (audit L9,
|
||
2026-11-05 — an interpolated id was a GraphQL injection into the
|
||
unauthenticated OC API; the variables form is parameterized by the server).
|
||
"""
|
||
if not order_id_v2:
|
||
return None
|
||
if order_id_v2 in _order_amount_cache:
|
||
return _order_amount_cache[order_id_v2]
|
||
try:
|
||
query = ("query OrderAmount($id: String!) { order(order: {id: $id}) "
|
||
"{ amount { valueInCents } platformTipAmount { valueInCents } } }")
|
||
async with httpx.AsyncClient(timeout=15.0) as client:
|
||
r = await client.post(OC_GRAPHQL_URL,
|
||
headers={"Content-Type": "application/json",
|
||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"},
|
||
json={"query": query, "variables": {"id": order_id_v2}})
|
||
o = r.json().get("data", {}).get("order") or {}
|
||
cents = (o.get("amount") or {}).get("valueInCents")
|
||
if cents is not None:
|
||
_order_amount_cache[order_id_v2] = int(cents)
|
||
return int(cents)
|
||
except Exception as e:
|
||
logger.warning("Order amount lookup failed for %s: %s", order_id_v2, e)
|
||
return None
|
||
|
||
|
||
def credit_pack_reference(slug: str, order: dict | None) -> str:
|
||
"""Stable idempotence key for a credit-pack purchase (audit H1, 2026-11-05).
|
||
|
||
Keyed on the OC order id from the webhook payload (`order.idV2` — the id
|
||
`get_order_amount` resolves, so a VERIFIED purchase always carries one).
|
||
The webhook's top-level `data.id` is absent from real deliveries; building
|
||
the reference from it produced the legacy `oc:<slug>:` empty-suffix
|
||
format, which cannot distinguish two purchases by the same member and so
|
||
cannot dedupe a re-delivered event. (The shortfall path keys on the OC
|
||
transaction publicId — equally unique; the webhook payload does not carry
|
||
it, and per this module's contract the collective order list is the
|
||
complete source for that shape, so no extra OC query is added here.)
|
||
|
||
Legacy ledger rows in the empty-suffix format are neither re-keyed nor
|
||
rejected: a new-format reference (non-empty id suffix) can never match
|
||
one, so the exists-check never blocks a first real grant, and the
|
||
historical rows stay exactly as recorded.
|
||
"""
|
||
ref_id = str((order or {}).get("idV2") or "").strip()
|
||
if not ref_id:
|
||
raise ValueError(
|
||
"credit pack: webhook payload carries no order idV2; refusing to "
|
||
"build an ambiguous credit reference"
|
||
)
|
||
return f"oc:{slug}:{ref_id}"
|
||
|
||
|
||
|
||
# Personal token for the "Inference Co-op Bot" account, which is an admin of
|
||
# the collective. Authenticated as an admin, the GraphQL API exposes member
|
||
# emails (which are hidden from anonymous access). We use this to match a
|
||
# member by email — the stable identifier that works even for guest
|
||
# contributors (who have no Open Collective account and thus no usable slug).
|
||
OC_PERSONAL_TOKEN = os.environ.get("OC_PERSONAL_TOKEN", "")
|
||
|
||
# Monthly credit budget (in USD of tokens) for all members.
|
||
# Single sliding-scale tier: everyone gets the same $15/month in credits,
|
||
# regardless of their $10/15/20 contribution. Governance decision (Loomio).
|
||
MEMBER_BUDGET = float(os.environ.get("MEMBER_BUDGET", "15.0"))
|
||
|
||
# Open Collective's flexible-tier floor (observed 2026-11-17). Used for COPY and
|
||
# evidence only — never as a deactivation trigger: OC remains the authority on dues.
|
||
MIN_CONTRIBUTION = float(os.environ.get("MIN_CONTRIBUTION", "5.0") or 5.0)
|
||
|
||
# --- Guest accounts (operator decisions, 2026-11-08) -------------------------- #
|
||
# A member can give a friend access to the co-op with $2 of credits, no Open
|
||
# Collective membership required. The guest is a ONE-TIME non-expiring credit
|
||
# balance held in the credits model (credit_balance), hard-capped by the
|
||
# LiteLLM team budget. Two lapse triggers, both handled by the sweep:
|
||
# * the $2 is spent (the cap enforces the limit; the sweep deactivates the
|
||
# row once spend reaches the credit balance), and
|
||
# * GUEST_DAYS elapse from provisioning — a time box, even with credits left.
|
||
# Deactivation is the same semantics as a member's: guests group → inactive,
|
||
# team budget zeroed, row active=0, newsletter unsubscribed. The account is
|
||
# KEPT so a returning payer resumes cleanly (and the guest→member conversion
|
||
# still fires if their OC membership appears later).
|
||
GUEST_BUDGET = float(os.environ.get("GUEST_BUDGET", "2.0"))
|
||
GUEST_DAYS = float(os.environ.get("GUEST_DAYS", "60"))
|
||
# One guest per inviting member per rolling 30 days (abuse protection).
|
||
GUEST_PER_INVITER_DAYS = float(os.environ.get("GUEST_PER_INVITER_DAYS", "30"))
|
||
# The ADMIN-PANEL guest-invite path records THIS account as the inviter in the
|
||
# guests ledger, and admin invites are NOT bound by the member's
|
||
# one-per-30-days cap (admins are not members).
|
||
ADMIN_INVITER_EMAIL = os.environ.get("ADMIN_INVITER_EMAIL", "info@inference.coop").strip().lower()
|
||
# MEMBER-initiated guest invitations provision IMMEDIATELY (operator decision,
|
||
# 2026-11-08 — the trial-era approval gate, including its GUEST_AUTO_APPROVE
|
||
# env flag, was removed). The guest pipeline's own guards are the protections:
|
||
# active-member inviter, one guest per inviter per rolling 30 days, one guest
|
||
# per recipient ever, no existing portal row, plus the member-path rule that
|
||
# the recipient is never a MANUAL_MEMBERS address.
|
||
|
||
# The Cloudron "guests" group: guest accounts land here. The chat and dashboard
|
||
# apps' access restrictions list BOTH members and guests, so a guest reaches
|
||
# exactly those two apps — nothing else is granted by group membership.
|
||
GUESTS_GROUP_ID = os.environ.get("GUESTS_GROUP_ID", "")
|
||
|
||
# Where a SELF-CANCELLED guest's Cloudron user goes: out of the guests group,
|
||
# so chat (and every other app gated to members+guests) stops admitting them —
|
||
# but INTO a group the dashboard app lists in its access restriction, so the
|
||
# ended state ("your access has ended", data kept, Become-a-member) stays
|
||
# reachable. Gateway access is revoked separately (team budget zeroed), so the
|
||
# dashboard view is read-only by construction. Falls back to the shared
|
||
# inactive group when unset — then the dashboard itself is also closed off and
|
||
# the ended state is visible only in the cancellation response.
|
||
GUESTS_ENDED_GROUP_ID = os.environ.get("GUESTS_ENDED_GROUP_ID", "")
|
||
|
||
# Where a guest's "Become a member" button and email point (the contribution
|
||
# page — what actually starts a membership). Deliberately NOT the portal's
|
||
# credit-pack URL: a guest needs a membership, not a credit pack.
|
||
BECOME_MEMBER_URL = "https://opencollective.com/inference-cooperative/contribute"
|
||
|
||
# Days to wait before re-sending the welcome/invite email to a member who was
|
||
# provisioned but never completed account setup (inviteAccepted=False). The
|
||
# nightly sweep re-invites such members, throttled by this interval so they
|
||
# aren't emailed every single night.
|
||
REINVITE_AFTER_DAYS = float(os.environ.get("REINVITE_AFTER_DAYS", "3"))
|
||
|
||
# Models every member key/team may access.
|
||
# Deliberately REMOVED: members get access to every model the gateway serves.
|
||
# The gateway's `model_list` is the single source of truth for what's available;
|
||
# teams and keys carry no per-model restriction (empty = "all models allowed" in
|
||
# LiteLLM). Adding a provider is a gateway-config change only — no team/key
|
||
# migration. (Kept as a documented empty value so the intent is explicit.)
|
||
MEMBER_MODELS: list[str] = []
|
||
|
||
# The "members" group in Cloudron (group-based access control).
|
||
# Members are assigned to this group, which grants access to the chat app.
|
||
MEMBERS_GROUP_ID = os.environ.get("MEMBERS_GROUP_ID", "")
|
||
|
||
# The "inactive" group in Cloudron. Lapsed members are moved here (instead of
|
||
# being deleted) so they lose access but can reclaim their data on reactivation.
|
||
INACTIVE_GROUP_ID = os.environ.get("INACTIVE_GROUP_ID", "")
|
||
|
||
# Manually-provisioned members (comma-separated emails) who are NOT in the
|
||
# Open Collective member list. Used for the free "founder" tier, which OC
|
||
# cannot process as a $0 recurring subscription. These members are exempt from
|
||
# the sweep's deactivation (they have no OC membership to lapse).
|
||
# How a member's membership is ACCOUNTED FOR. This exists because "manual" was
|
||
# doing two incompatible jobs (2026-11-13):
|
||
# oc — OC-backed, matched by email. The default.
|
||
# oc-incognito — OC-backed, but the OC profile HIDES its email (mmoya@mmoya.org),
|
||
# so email matching can never see them. The contribution is real
|
||
# and the reconciliation already matches orders by the payer's
|
||
# SLUG, which is what the members.slug column holds for these
|
||
# people — so they keep lapse logic, dues tracking and order
|
||
# attribution, and are NOT swept into the manual bucket.
|
||
# manual — hand-provisioned (MANUAL_MEMBERS): no OC contribution is
|
||
# expected, so nothing reconciles and nobody lapses.
|
||
# A row with no explicit source falls back to MANUAL_MEMBERS, so this changed
|
||
# nothing for anyone until a source was actually set.
|
||
MEMBER_SOURCES = ("oc", "oc-incognito", "manual")
|
||
|
||
|
||
def member_source(email: str, conn=None) -> str:
|
||
"""This member's accounting source. Never raises: an unreadable row is 'oc'
|
||
(the default), which is the safe direction — it means we keep reconciling them
|
||
rather than quietly exempting them from it."""
|
||
email = (email or "").strip().lower()
|
||
src = ""
|
||
try:
|
||
own = conn is None
|
||
conn = conn or get_db()
|
||
try:
|
||
cols = {r[1] for r in conn.execute("PRAGMA table_info(members)")}
|
||
if "source" in cols:
|
||
row = conn.execute("SELECT source FROM members WHERE lower(email) = ?",
|
||
(email,)).fetchone()
|
||
src = ((row[0] if row else "") or "").strip().lower()
|
||
finally:
|
||
if own:
|
||
conn.close()
|
||
except Exception as e:
|
||
logger.warning("member_source(%s): %s", email, e)
|
||
if src in MEMBER_SOURCES:
|
||
return src
|
||
return "manual" if email in MANUAL_MEMBERS else "oc"
|
||
|
||
|
||
def is_manual_member(email: str, conn=None) -> bool:
|
||
"""Hand-provisioned: the set MANUAL_MEMBERS names, minus anyone given an
|
||
explicit source of their own (an oc-incognito member is OC-backed even though
|
||
email matching cannot see them)."""
|
||
return member_source(email, conn) == "manual"
|
||
|
||
|
||
def set_member_source(email: str, source: str) -> bool:
|
||
"""Record a member's source. Idempotent; returns True when the row was written."""
|
||
source = (source or "").strip().lower()
|
||
if source not in MEMBER_SOURCES:
|
||
raise ValueError(f"unknown source {source!r}; expected one of {MEMBER_SOURCES}")
|
||
conn = get_db()
|
||
try:
|
||
cols = {r[1] for r in conn.execute("PRAGMA table_info(members)")}
|
||
if "source" not in cols:
|
||
conn.execute("ALTER TABLE members ADD COLUMN source TEXT")
|
||
cur = conn.execute("UPDATE members SET source = ? WHERE lower(email) = ?",
|
||
(source, (email or "").strip().lower()))
|
||
conn.commit()
|
||
return cur.rowcount > 0
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
MANUAL_MEMBERS = {
|
||
e.strip().lower()
|
||
for e in os.environ.get("MANUAL_MEMBERS", "").split(",")
|
||
if e.strip()
|
||
}
|
||
|
||
# Loomio integration (forum.inference.coop). The portal reconciles the Loomio
|
||
# group to the current active-member list via the User API (/api/b2).
|
||
LOOMIO_BASE = os.environ.get("LOOMIO_BASE", "https://forum.inference.coop")
|
||
LOOMIO_API_KEY = os.environ.get("LOOMIO_API_KEY", "")
|
||
# Server API key (B3) — used to fix auto-generated Loomio usernames after
|
||
# invites. Loomio's b2/memberships API only takes emails; new users get a
|
||
# mangled auto-handle (email local-part + random suffix). The B3 users API
|
||
# lets us rename them to the member's Cloudron username.
|
||
LOOMIO_B3_KEY = os.environ.get("LOOMIO_B3_KEY", "")
|
||
# Master switch for the Loomio membership sync. It defaults to ENABLED: the
|
||
# co-op wants every new member added to Loomio, so restoring LOOMIO_API_KEY
|
||
# and deploying is all that is needed. It is expressed as a KILL SWITCH —
|
||
# set LOOMIO_SYNC_ENABLED=0 (or false/off/no) to pause the sync; any other
|
||
# value, or leaving it unset, keeps it ON. (Incident 2026-11-04 was fixed the
|
||
# right way by making the ADD path idempotent, NOT by switching the feature
|
||
# off — see sync_loomio_memberships for the guards.)
|
||
LOOMIO_SYNC_ENABLED = os.environ.get("LOOMIO_SYNC_ENABLED", "1").strip().lower() not in ("0", "false", "off", "no")
|
||
|
||
# Matrix homeserver (self-hosted Synapse at matrix.inference.coop). The bot
|
||
# account (admin) invites every active member to the co-op Space on sync.
|
||
# Members authenticate via Cloudron SSO; their homeserver MXID is
|
||
# @<cloudron-username>:inference.coop. Federated MXIDs (external servers)
|
||
# can also be invited directly.
|
||
MATRIX_HOMESERVER = os.environ.get("MATRIX_HOMESERVER", "https://matrix.inference.coop")
|
||
MATRIX_BOT_TOKEN = os.environ.get("MATRIX_BOT_TOKEN", "")
|
||
MATRIX_SPACE_ID = os.environ.get("MATRIX_SPACE_ID", "")
|
||
MATRIX_GENERAL_ROOM = os.environ.get("MATRIX_GENERAL_ROOM", "")
|
||
# The explicit set of rooms every active member is invited to (comma-separated
|
||
# room IDs). This is authoritative configuration and is deliberately NOT derived
|
||
# from the Space's children — some child rooms (operator/admin-only) must never
|
||
# be pushed at every member. When unset, sync_matrix_space() falls back to the
|
||
# co-op Space + General room. Restricted rooms are NOT discoverable over
|
||
# federation until a member has joined (the space hierarchy omits them for remote
|
||
# servers), so member-facing rooms need this explicit invite pass.
|
||
MATRIX_INVITE_ROOMS = [r.strip() for r in os.environ.get("MATRIX_INVITE_ROOMS", "").split(",") if r.strip()]
|
||
# Legacy alias kept for compatibility: extra member-facing rooms appended to the
|
||
# invite set above (superseded by MATRIX_INVITE_ROOMS).
|
||
MATRIX_EXTRA_ROOMS = [r.strip() for r in os.environ.get("MATRIX_EXTRA_ROOMS", "").split(",") if r.strip()]
|
||
LOOMIO_GROUP_ID = os.environ.get("LOOMIO_GROUP_ID", "")
|
||
# Operator accounts that must always remain in the Loomio group (never removed
|
||
# by the remove_absent reconciliation), comma-separated.
|
||
LOOMIO_ALWAYS_KEEP = [
|
||
e.strip()
|
||
for e in os.environ.get("LOOMIO_ALWAYS_KEEP", "info@inference.coop,bot@inference.coop").split(",")
|
||
if e.strip()
|
||
]
|
||
|
||
# Persistent store (SQLite) for email → LiteLLM key token mapping.
|
||
# Lives in /app/data (Cloudron localstorage addon persists this).
|
||
DB_PATH = os.environ.get("DB_PATH", "/app/data/members.db")
|
||
|
||
# --- Email (Cloudron sendmail addon) ---
|
||
# The sendmail addon exports SMTP credentials as CLOUDRON_MAIL_* env vars.
|
||
SMTP_SERVER = os.environ.get("CLOUDRON_MAIL_SMTP_SERVER", "")
|
||
SMTP_PORT = int(os.environ.get("CLOUDRON_MAIL_SMTP_PORT", "2525"))
|
||
SMTP_USERNAME = os.environ.get("CLOUDRON_MAIL_SMTP_USERNAME", "")
|
||
SMTP_PASSWORD = os.environ.get("CLOUDRON_MAIL_SMTP_PASSWORD", "")
|
||
MAIL_FROM = os.environ.get("CLOUDRON_MAIL_FROM", "info@inference.coop")
|
||
MAIL_FROM_NAME = os.environ.get("CLOUDRON_MAIL_FROM_DISPLAY_NAME", "Inference Cooperative")
|
||
# Reply-to address so member replies land at the co-op inbox.
|
||
MAIL_REPLY_TO = os.environ.get("MAIL_REPLY_TO", "info@inference.coop")
|
||
# Public base URL for the portal (used in email links).
|
||
PORTAL_BASE = os.environ.get("PORTAL_BASE", "https://portal.inference.coop")
|
||
|
||
# Listmonk newsletter integration. The portal keeps the "Members" list in sync:
|
||
# subscribe on provision, unsubscribe on deactivation. Auth uses listmonk's
|
||
# native API-user token scheme (Authorization: token <user>:<token>).
|
||
LISTMONK_BASE = os.environ.get("LISTMONK_BASE") or "https://newsletter.inference.coop"
|
||
# NOTE: `or` (not default-arg) — an env var set to EMPTY must fall back to the
|
||
# default. An empty LISTMONK_API_USER made every sync 403 silently (the token
|
||
# alone is not a valid listmonk credential) while listmonk_sync's best-effort
|
||
# error handling swallowed it — that is how stale accounts stayed subscribed.
|
||
LISTMONK_API_USER = os.environ.get("LISTMONK_API_USER") or "portal-sync"
|
||
LISTMONK_API_TOKEN = os.environ.get("LISTMONK_API_TOKEN", "")
|
||
LISTMONK_LIST_ID = int(os.environ.get("LISTMONK_LIST_ID", "3")) # "Members" list
|
||
|
||
# Direct read access to LiteLLM's Postgres (for usage aggregations).
|
||
# READ-ONLY purpose: never write to LiteLLM's tables — its HTTP API remains the
|
||
# only interface for writes/management. Falls back to the HTTP path when unset.
|
||
LITELLM_DB_HOST = os.environ.get("LITELLM_DB_HOST", "")
|
||
LITELLM_DB_PORT = int(os.environ.get("LITELLM_DB_PORT", "5432"))
|
||
LITELLM_DB_USER = os.environ.get("LITELLM_DB_USER", "")
|
||
LITELLM_DB_PASSWORD = os.environ.get("LITELLM_DB_PASSWORD", "")
|
||
LITELLM_DB_NAME = os.environ.get("LITELLM_DB_NAME", "")
|
||
|
||
|
||
def _coop_spend_totals_safe() -> dict:
|
||
"""coop_spend_totals() that degrades to {} — a spend figure is a display
|
||
detail; a failure to compute it must not blank the panel or the dashboard.
|
||
Callers OMIT the key rather than showing a zero, because a zero reads as
|
||
"the co-op has spent nothing", which is a lie."""
|
||
try:
|
||
return coop_spend_totals()
|
||
except Exception as e:
|
||
logger.warning("coop spend totals unavailable: %s", e)
|
||
return {}
|
||
|
||
|
||
def litellm_db_ready() -> bool:
|
||
"""True when LiteLLM DB read credentials are configured."""
|
||
return bool(LITELLM_DB_HOST and LITELLM_DB_USER and LITELLM_DB_NAME)
|
||
|
||
|
||
def _pg_connect():
|
||
"""Open a short-lived connection to LiteLLM's Postgres (read-only use)."""
|
||
import psycopg2
|
||
return psycopg2.connect(
|
||
host=LITELLM_DB_HOST, port=LITELLM_DB_PORT, user=LITELLM_DB_USER,
|
||
password=LITELLM_DB_PASSWORD, dbname=LITELLM_DB_NAME,
|
||
connect_timeout=10, sslmode="prefer",
|
||
)
|
||
|
||
|
||
def aggregate_model_usage(days: int = 0, team_id: str | None = None) -> dict:
|
||
"""Aggregate LiteLLM spend logs per model, straight from Postgres.
|
||
|
||
Returns {"models": [...], "totals": {...}} in the same shape the HTTP path
|
||
produced, so callers don't change. `days` bounds the window (0 = all time);
|
||
`team_id` restricts to one member's team (used by the dashboard path).
|
||
|
||
Prefers the `model_group` column, which carries our provider/model alias
|
||
(e.g. greenpt/glm-5.3-flash); falls back to `model` for pre-alias rows — matching
|
||
the logic the HTTP path used, expressed in SQL.
|
||
|
||
`requests` counts only metered calls that SUCCEEDED. Refused/failed attempts
|
||
(rows with status = 'failure', e.g. a 401 on the /v1/systemone decision
|
||
route) are excluded from that count and reported separately as `refused`,
|
||
so a failure cannot inflate a model's call count. `spend` and `tokens` stay
|
||
summed over every row: a call that failed after tokens were consumed still
|
||
cost money and still processed those tokens.
|
||
"""
|
||
if not litellm_db_ready():
|
||
raise RuntimeError("LiteLLM DB credentials not configured")
|
||
|
||
where = []
|
||
params: list = []
|
||
if days and days > 0:
|
||
from datetime import datetime, timedelta, timezone
|
||
cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).strftime(
|
||
"%Y-%m-%d %H:%M:%S"
|
||
)
|
||
where.append("\"startTime\" >= %s")
|
||
params.append(cutoff)
|
||
if team_id:
|
||
where.append("team_id = %s")
|
||
params.append(team_id)
|
||
where_sql = ("WHERE " + " AND ".join(where)) if where else ""
|
||
|
||
sql = f"""
|
||
SELECT
|
||
COALESCE(NULLIF(model_group, ''), NULLIF(model, ''), 'unknown') AS model,
|
||
COALESCE(SUM(spend), 0) AS spend,
|
||
COALESCE(SUM(total_tokens), 0) AS tokens,
|
||
COUNT(*) FILTER (WHERE status IS DISTINCT FROM 'failure') AS requests,
|
||
COUNT(*) FILTER (WHERE status = 'failure') AS refused
|
||
FROM "LiteLLM_SpendLogs"
|
||
{where_sql}
|
||
GROUP BY 1
|
||
ORDER BY requests DESC
|
||
"""
|
||
|
||
conn = _pg_connect()
|
||
try:
|
||
with conn.cursor() as cur:
|
||
cur.execute(sql, params)
|
||
rows = cur.fetchall()
|
||
finally:
|
||
conn.close()
|
||
|
||
models = [
|
||
{"model": r[0], "spend": float(r[1] or 0.0), "tokens": int(r[2] or 0),
|
||
"requests": int(r[3] or 0), "refused": int(r[4] or 0)}
|
||
for r in rows
|
||
]
|
||
return {
|
||
"models": models,
|
||
"totals": {
|
||
"total_spend": sum(m["spend"] for m in models),
|
||
"total_tokens": sum(m["tokens"] for m in models),
|
||
"total_requests": sum(m["requests"] for m in models),
|
||
"total_refused": sum(m["refused"] for m in models),
|
||
},
|
||
}
|
||
|
||
|
||
def _fill_daily_days(rows: list[dict]) -> list[dict]:
|
||
"""Fill calendar days WITHOUT usage rows with spend 0 / requests 0 between
|
||
the first and last measured day, so the chart's x spacing is true: a day
|
||
with no requests IS a zero-usage day (measured), not an unmeasured gap —
|
||
the fail-closed NULL rule is for days the SOURCE could not read, which
|
||
these are not (a GROUP BY simply omits them)."""
|
||
if not rows:
|
||
return rows
|
||
from datetime import date, timedelta
|
||
start = date.fromisoformat(rows[0]["day"])
|
||
end = date.fromisoformat(rows[-1]["day"])
|
||
by_day = {r["day"]: r for r in rows}
|
||
out = []
|
||
d = start
|
||
while d <= end:
|
||
key = d.isoformat()
|
||
out.append(by_day.get(key) or {"day": key, "spend": 0.0, "requests": 0})
|
||
d += timedelta(days=1)
|
||
return out
|
||
|
||
|
||
def daily_series_from_rows(rows) -> list[dict]:
|
||
"""Per-day usage series from spend-log rows ALREADY filtered to one team —
|
||
the HTTP fallback's client-side twin of member_daily_usage. spend sums every
|
||
row; requests counts only non-failure rows (same rules as the DB path)."""
|
||
daily_map: dict[str, dict] = {}
|
||
for row in rows if isinstance(rows, list) else []:
|
||
day = str(row.get("startTime") or "")[:10]
|
||
if not day:
|
||
continue
|
||
d = daily_map.setdefault(day, {"spend": 0.0, "requests": 0})
|
||
d["spend"] += float(row.get("spend") or 0.0)
|
||
if str(row.get("status") or "").lower() != "failure":
|
||
d["requests"] += 1
|
||
return _fill_daily_days([
|
||
{"day": k, "spend": round(v["spend"], 6), "requests": v["requests"]}
|
||
for k, v in sorted(daily_map.items())
|
||
])
|
||
|
||
|
||
def member_daily_usage(days: int = 0, team_id: str | None = None) -> list[dict]:
|
||
"""The member's own per-day usage (spend + requests), for the dashboard's
|
||
usage chart. Same source and day convention as the co-op daily series
|
||
(metrics_snapshot.spend_by_day): to_char("startTime", 'YYYY-MM-DD'), so the
|
||
two series agree on which day a request belongs to.
|
||
|
||
`days` bounds the window (0 = all time). spend sums EVERY log row (a call
|
||
that failed after tokens were consumed still cost money); `requests` counts
|
||
only metered calls that succeeded, like aggregate_model_usage.
|
||
"""
|
||
if not litellm_db_ready():
|
||
raise RuntimeError("LiteLLM DB credentials not configured")
|
||
|
||
where = []
|
||
params: list = []
|
||
if days and days > 0:
|
||
from datetime import datetime, timedelta, timezone
|
||
cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).strftime(
|
||
"%Y-%m-%d %H:%M:%S"
|
||
)
|
||
where.append("\"startTime\" >= %s")
|
||
params.append(cutoff)
|
||
if team_id:
|
||
where.append("team_id = %s")
|
||
params.append(team_id)
|
||
where_sql = ("WHERE " + " AND ".join(where)) if where else ""
|
||
|
||
sql = f"""
|
||
SELECT to_char("startTime", 'YYYY-MM-DD') AS day,
|
||
COALESCE(SUM(spend), 0) AS spend,
|
||
COUNT(*) FILTER (WHERE status IS DISTINCT FROM 'failure') AS requests
|
||
FROM "LiteLLM_SpendLogs"
|
||
{where_sql}
|
||
GROUP BY 1
|
||
ORDER BY 1
|
||
"""
|
||
|
||
conn = _pg_connect()
|
||
try:
|
||
with conn.cursor() as cur:
|
||
cur.execute(sql, params)
|
||
rows = cur.fetchall()
|
||
finally:
|
||
conn.close()
|
||
return _fill_daily_days([
|
||
{"day": r[0], "spend": round(float(r[1] or 0.0), 6), "requests": int(r[2] or 0)}
|
||
for r in rows
|
||
])
|
||
|
||
|
||
def get_team_budget_from_db(team_id: str) -> dict | None:
|
||
"""Fetch a team's spend/budget/reset directly from LiteLLM's Postgres.
|
||
|
||
Same data as the /team/list HTTP path (LiteLLM computes `spend`), without
|
||
downloading the whole team list. Returns None when the team is unknown.
|
||
"""
|
||
if not litellm_db_ready():
|
||
return None
|
||
sql = """
|
||
SELECT max_budget, spend, budget_reset_at
|
||
FROM "LiteLLM_TeamTable"
|
||
WHERE team_id = %s
|
||
LIMIT 1
|
||
"""
|
||
conn = _pg_connect()
|
||
try:
|
||
with conn.cursor() as cur:
|
||
cur.execute(sql, (team_id,))
|
||
row = cur.fetchone()
|
||
finally:
|
||
conn.close()
|
||
if not row:
|
||
return None
|
||
return {
|
||
"max_budget": float(row[0]) if row[0] is not None else None,
|
||
"spend": float(row[1] or 0.0),
|
||
"budget_reset_at": row[2],
|
||
}
|
||
|
||
|
||
async def listmonk_sync(email: str, subscribe: bool) -> None:
|
||
"""Subscribe or unsubscribe a member to the newsletter list.
|
||
|
||
Best-effort: never raises, so a newsletter outage can't break provisioning.
|
||
"""
|
||
if not LISTMONK_API_TOKEN:
|
||
logger.warning("Listmonk not configured; skipping sync for %s", email)
|
||
return
|
||
auth = {"Authorization": f"token {LISTMONK_API_USER}:{LISTMONK_API_TOKEN}"}
|
||
try:
|
||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||
if subscribe:
|
||
r = await client.post(
|
||
f"{LISTMONK_BASE}/api/subscribers",
|
||
headers=auth,
|
||
json={
|
||
"email": email,
|
||
"name": "",
|
||
"status": "enabled",
|
||
"lists": [LISTMONK_LIST_ID],
|
||
"preconfirm_subscriptions": True,
|
||
},
|
||
)
|
||
else:
|
||
# Look up the subscriber id by email, then blocklist by id.
|
||
# (listmonk's blocklist endpoint takes an id, not an email.)
|
||
q = await client.get(
|
||
f"{LISTMONK_BASE}/api/subscribers",
|
||
headers=auth,
|
||
params={"query": f"subscribers.email='{email}'"},
|
||
)
|
||
if q.status_code != 200:
|
||
logger.warning("Listmonk lookup for %s failed: %s", email, q.status_code)
|
||
return
|
||
results = q.json().get("data", {}).get("results", [])
|
||
if not results:
|
||
return # not subscribed; nothing to do
|
||
sid = results[0]["id"]
|
||
r = await client.put(
|
||
f"{LISTMONK_BASE}/api/subscribers/{sid}/blocklist",
|
||
headers=auth,
|
||
)
|
||
if r.status_code not in (200, 201):
|
||
logger.warning(
|
||
"Listmonk sync %s for %s failed: %s",
|
||
"subscribe" if subscribe else "unsubscribe",
|
||
email,
|
||
r.status_code,
|
||
)
|
||
except Exception as e:
|
||
logger.warning("Listmonk sync error for %s: %s", email, e)
|
||
|
||
|
||
async def listmonk_reconcile() -> dict:
|
||
"""Reconcile the Members newsletter list against the active membership.
|
||
|
||
The newsletter list mirrors the portal's ACTIVE members. Subscribers who are
|
||
not active members fall into two classes, and only one is acted on:
|
||
- emails that exist in the members table but are inactive, and emails that
|
||
are not members at all, are taken off the list (the subscriber record
|
||
stays as a suppression marker). A DELETE would destroy the subscriber's
|
||
consent history, and a stray test account is indistinguishable from a
|
||
real person who subscribed outside the portal, so nobody is purged
|
||
automatically;
|
||
- OPERATOR_EMAILS — infrastructure accounts, which the panel surfaces
|
||
under "admins" and excludes from member counts — are left exactly as
|
||
they are. They are neither members nor stale. The Loomio
|
||
reconciliation applies the same rule through LOOMIO_ALWAYS_KEEP.
|
||
|
||
Best-effort: never raises, so a newsletter outage can't break the sweep.
|
||
"""
|
||
if not LISTMONK_API_TOKEN:
|
||
logger.warning("Listmonk not configured; skipping newsletter reconcile")
|
||
return {"removed": 0, "emails": [], "error": "not configured"}
|
||
auth = {"Authorization": f"token {LISTMONK_API_USER}:{LISTMONK_API_TOKEN}"}
|
||
conn = get_db()
|
||
active = {
|
||
row[0].lower()
|
||
for row in conn.execute("SELECT email FROM members WHERE active = 1")
|
||
}
|
||
member_emails = {
|
||
row[0].lower() for row in conn.execute("SELECT email FROM members")
|
||
}
|
||
conn.close()
|
||
try:
|
||
async with httpx.AsyncClient(timeout=20.0) as client:
|
||
# Fetch the full subscriber set for the list (paginated).
|
||
subs, page = [], 1
|
||
while True:
|
||
r = await client.get(
|
||
f"{LISTMONK_BASE}/api/subscribers",
|
||
headers=auth,
|
||
params={"list_id": LISTMONK_LIST_ID, "page": page, "per_page": 100},
|
||
)
|
||
if r.status_code != 200:
|
||
logger.warning(
|
||
"Listmonk reconcile: subscriber fetch failed: %s", r.status_code
|
||
)
|
||
return {"removed": 0, "emails": [], "error": f"fetch {r.status_code}"}
|
||
data = r.json().get("data", {})
|
||
results = data.get("results", [])
|
||
subs += results
|
||
total = data.get("total")
|
||
# Stop on an empty or short page, or once the reported count is
|
||
# reached. The short-page test means a missing `total` cannot
|
||
# truncate the reconcile at page 1.
|
||
if (
|
||
not results
|
||
or len(results) < 100
|
||
or (total is not None and len(subs) >= total)
|
||
):
|
||
break
|
||
page += 1
|
||
removed, emails = 0, []
|
||
by_reason = {"inactive member": 0, "not a member": 0}
|
||
# Operator accounts are neither members nor stale; never touch their
|
||
# subscription (LOOMIO_ALWAYS_KEEP is the same rule for the forum).
|
||
operators_kept = sum(
|
||
1 for s in subs if (s.get("email") or "").lower() in OPERATOR_EMAILS
|
||
)
|
||
for s in subs:
|
||
email = (s.get("email") or "").lower()
|
||
if not email or email in active or email in OPERATOR_EMAILS:
|
||
continue
|
||
sid = s.get("id")
|
||
# Both classes leave the list the same way: unsubscribe, keep the
|
||
# record. Nothing is deleted.
|
||
reason = "inactive member" if email in member_emails else "not a member"
|
||
# listmonk validates the PUT body's email field: omitting it is a
|
||
# 400 "Invalid email." and the subscriber stays on the list (which
|
||
# is why the inactive-member branch never actually worked).
|
||
r = await client.put(
|
||
f"{LISTMONK_BASE}/api/subscribers/{sid}",
|
||
headers=auth,
|
||
json={"email": s.get("email"), "lists": []},
|
||
)
|
||
if r.status_code in (200, 201):
|
||
removed += 1
|
||
by_reason[reason] += 1
|
||
emails.append(email)
|
||
else:
|
||
logger.warning(
|
||
"Listmonk reconcile: unsubscribe %s failed: %s", email, r.status_code
|
||
)
|
||
if removed:
|
||
# Counts only — subscriber addresses don't belong in the log.
|
||
logger.info(
|
||
"Listmonk reconcile: removed %d stale subscribers (%s)",
|
||
removed,
|
||
", ".join(f"{k}: {v}" for k, v in by_reason.items() if v),
|
||
)
|
||
if operators_kept:
|
||
logger.info(
|
||
"Listmonk reconcile: left %d operator address(es) on the list",
|
||
operators_kept,
|
||
)
|
||
return {"removed": removed, "emails": emails, "operators_kept": operators_kept}
|
||
except Exception as e:
|
||
logger.warning("Listmonk reconcile error: %s", e)
|
||
return {"removed": 0, "emails": [], "error": str(e)}
|
||
|
||
|
||
def _verify_portal_secret(request: Request) -> None:
|
||
"""Reject requests that didn't come through LibreChat (shared secret)."""
|
||
if not PORTAL_SECRET:
|
||
# If no secret is configured, refuse to inject keys (fail closed).
|
||
raise HTTPException(503, "Portal secret not configured")
|
||
provided = request.headers.get("x-portal-secret", "")
|
||
if not secrets.compare_digest(provided, PORTAL_SECRET):
|
||
raise HTTPException(401, "Invalid portal secret")
|
||
|
||
|
||
def _verify_admin_token(request: Request) -> None:
|
||
"""Authorize a /admin/* caller (SECURITY 2026-10-27).
|
||
|
||
Admin auth is a HEADER, never a URL token: the webhook URL token lives in
|
||
places we do not control, and it used to unlock all of /admin/*.
|
||
"""
|
||
provided = request.headers.get("x-admin-token", "")
|
||
if ADMIN_TOKEN and secrets.compare_digest(provided, ADMIN_TOKEN):
|
||
return
|
||
if ADMIN_TOKEN_LEGACY and WEBHOOK_TOKEN and secrets.compare_digest(provided, WEBHOOK_TOKEN):
|
||
logger.warning(
|
||
"admin auth: caller used the legacy WEBHOOK_TOKEN — move it to"
|
||
" ADMIN_TOKEN, then set ADMIN_TOKEN_LEGACY=0"
|
||
)
|
||
return
|
||
raise HTTPException(403, "Forbidden")
|
||
|
||
|
||
def get_db() -> sqlite3.Connection:
|
||
conn = sqlite3.connect(DB_PATH)
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS members ("
|
||
"email TEXT PRIMARY KEY, "
|
||
"key_token TEXT, "
|
||
"cloudron_user_id TEXT, "
|
||
"slug TEXT, "
|
||
"active INTEGER DEFAULT 1"
|
||
")"
|
||
)
|
||
# Daily metrics snapshot, written by /app/code/metrics_snapshot.py from the
|
||
# scheduler and read by the admin panel + dashboard. Cumulative columns
|
||
# (members_*, newsletter_subs, matrix_space_members) are counts AS OF that
|
||
# day; spend_usd/requests are THAT day's traffic. A NULL column means the
|
||
# source could not be read that day — charts must show a gap, never a zero.
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS metrics_daily ("
|
||
"day TEXT PRIMARY KEY, "
|
||
"members_active INTEGER, "
|
||
"members_total INTEGER, "
|
||
"members_new INTEGER, "
|
||
"newsletter_subs INTEGER, "
|
||
"matrix_space_members INTEGER, "
|
||
"spend_usd REAL, "
|
||
"operator_spend_usd REAL, "
|
||
"requests INTEGER, "
|
||
"out_tokens INTEGER, "
|
||
"energy_wh REAL, "
|
||
"co2e_g REAL, "
|
||
"tokens_measured INTEGER, "
|
||
"tokens_estimated INTEGER, "
|
||
"factor_revision TEXT, "
|
||
"recorded_at TEXT"
|
||
")"
|
||
)
|
||
# CREATE IF NOT EXISTS never adds columns to an existing table, so migrate:
|
||
# impact columns arrived after the table did (see co-op/support#26), and the
|
||
# measured-vs-estimated provenance columns after those. All additive: existing
|
||
# rows keep NULL, and a NULL split reports as unknown — never as "measured".
|
||
for _col, _decl in (
|
||
# The co-op's OWN accounts' share of spend_usd (OPERATOR_EMAILS). The
|
||
# total keeps spanning every team; this is the breakout, so a
|
||
# member-facing surface can say how much of it is the co-op's own
|
||
# usage. NULL = not measured (never a confident 0).
|
||
("operator_spend_usd", "REAL"),
|
||
("out_tokens", "INTEGER"),
|
||
("energy_wh", "REAL"),
|
||
("co2e_g", "REAL"),
|
||
("tokens_measured", "INTEGER"),
|
||
("tokens_estimated", "INTEGER"),
|
||
("factor_revision", "TEXT"),
|
||
# The collective's OC balance, snapshotted daily (see metrics_snapshot.py
|
||
# oc_collective_balance). The OC API exposes only the CURRENT balance, so
|
||
# a real history accrues from these snapshots — earlier days stay NULL.
|
||
("oc_balance_usd", "REAL"),
|
||
):
|
||
try:
|
||
conn.execute(f"ALTER TABLE metrics_daily ADD COLUMN {_col} {_decl}")
|
||
except sqlite3.OperationalError:
|
||
pass # already present
|
||
# Member-managed API keys (created via the broker). Each row is one API key
|
||
# under the member's team, tracked so we can list/revoke by name without
|
||
# exposing the chat key. `sk_token` is the full sk- key (revealed once).
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS member_api_keys ("
|
||
"email TEXT NOT NULL, "
|
||
"name TEXT NOT NULL, "
|
||
"sk_token TEXT NOT NULL, "
|
||
"hash TEXT NOT NULL, "
|
||
"created_at TEXT DEFAULT (datetime('now')), "
|
||
"PRIMARY KEY (email, name)"
|
||
")"
|
||
)
|
||
# Migration: add slug column if the members table predates it.
|
||
cols = [r[1] for r in conn.execute("PRAGMA table_info(members)").fetchall()]
|
||
if "slug" not in cols:
|
||
conn.execute("ALTER TABLE members ADD COLUMN slug TEXT")
|
||
# Migration: add balance column (USD) — the member's current credit
|
||
# allowance. Defaults to MEMBER_BUDGET; can be topped up by payments.
|
||
# This is the flexible counter to the fixed $15/month allowance.
|
||
if "balance" not in cols:
|
||
conn.execute("ALTER TABLE members ADD COLUMN balance REAL")
|
||
# Migration: welcome_sent_at — timestamp of the last welcome/invite email,
|
||
# used to throttle re-invites of members who never activated.
|
||
if "welcome_sent_at" not in cols:
|
||
conn.execute("ALTER TABLE members ADD COLUMN welcome_sent_at TEXT")
|
||
# Migration: matrix_mxid — an OPTIONAL extra Matrix account a member wants
|
||
# invited to the co-op Space (e.g. their existing federated account on
|
||
# another homeserver). Empty = only the local @username:inference.coop
|
||
# account is invited. At most one extra MXID per member.
|
||
if "matrix_mxid" not in cols:
|
||
conn.execute("ALTER TABLE members ADD COLUMN matrix_mxid TEXT")
|
||
# Migration: credit_balance — purchased, NON-EXPIRING credits (USD).
|
||
# Distinct from `balance` (the monthly allowance, which resets): credits
|
||
# carry over indefinitely and are spent only after the allowance is
|
||
# exhausted within a cycle.
|
||
if "credit_balance" not in cols:
|
||
conn.execute("ALTER TABLE members ADD COLUMN credit_balance REAL NOT NULL DEFAULT 0")
|
||
# Migration: org_slug — for ORGANIZATION-backed memberships. OC orgs never
|
||
# expose an email to our integration, so the org's designated contact is
|
||
# linked manually (POST /admin/org-link): org_slug holds the OC org slug
|
||
# whose Membership-tier dues keep this account active. The nightly sweep
|
||
# keeps the account active while the org is a BACKER on the Membership
|
||
# tier and deactivates it when the org's dues lapse.
|
||
if "org_slug" not in cols:
|
||
conn.execute("ALTER TABLE members ADD COLUMN org_slug TEXT")
|
||
# Migration: loomio_invited_at — the first time this member's email was
|
||
# handed to Loomio's membership API. Non-NULL means "already invited, never
|
||
# invite again". This is the positive exclusion enforced by
|
||
# sync_loomio_memberships: an existing member (already in Loomio, or already
|
||
# invited) can never be sent a Loomio invite a second time.
|
||
if "loomio_invited_at" not in cols:
|
||
conn.execute("ALTER TABLE members ADD COLUMN loomio_invited_at TEXT")
|
||
# Migration: account_type — 'member' (default) or 'guest'. Guest accounts are
|
||
# provisioned by /admin/provision-guest with $2 of non-expiring credit, live
|
||
# in the Cloudron "guests" group, and sit outside every membership
|
||
# obligation: the sweep never deactivates one for lacking an OC
|
||
# contribution, they are invited to no Loomio/Matrix rooms, and they lapse
|
||
# only by their own rules (credit exhausted, or the GUEST_DAYS time box).
|
||
if "account_type" not in cols:
|
||
conn.execute("ALTER TABLE members ADD COLUMN account_type TEXT DEFAULT 'member'")
|
||
# Migration: source — how this membership is accounted for (see
|
||
# MEMBER_SOURCES). NULL means "not stated": fall back to MANUAL_MEMBERS.
|
||
if "source" not in cols:
|
||
conn.execute("ALTER TABLE members ADD COLUMN source TEXT")
|
||
# Migration: oc_email — the Open Collective email a member's contribution
|
||
# arrives under when it differs from their account email (set via
|
||
# POST /admin/guest-convert when a invited friend joins under another
|
||
# address). The sweep and the webhook treat it as an additional matcher so
|
||
# renewals under that address keep the account active.
|
||
if "oc_email" not in cols:
|
||
conn.execute("ALTER TABLE members ADD COLUMN oc_email TEXT")
|
||
# Migration: guest naming (2026-11-08). The ledger table, the members
|
||
# expiry column, the lapse-notice table and the account_type VALUES all
|
||
# carried 'gift' before the feature was renamed to guest access. Every
|
||
# database that predates the rename moves once, here — idempotent, and a
|
||
# no-op on any database created after it. Runs BEFORE the
|
||
# guest_expires_at ADD COLUMN below so an old database renames its
|
||
# populated column instead of gaining an empty twin.
|
||
_tables = [r[0] for r in conn.execute(
|
||
"SELECT name FROM sqlite_master WHERE type='table'").fetchall()]
|
||
if "gifts" in _tables and "guests" not in _tables:
|
||
conn.execute("ALTER TABLE gifts RENAME TO guests")
|
||
if "gift_lapse_notices" in _tables and "guest_lapse_notices" not in _tables:
|
||
conn.execute("ALTER TABLE gift_lapse_notices RENAME TO guest_lapse_notices")
|
||
# The ledger's own expiry column moves with its table — the provision
|
||
# INSERT writes guest_expires_at, so a legacy table keeping the old
|
||
# column name would fail on the first provision after the rename.
|
||
_gcols = [r[1] for r in conn.execute("PRAGMA table_info(guests)").fetchall()]
|
||
if "gift_expires_at" in _gcols and "guest_expires_at" not in _gcols:
|
||
conn.execute("ALTER TABLE guests RENAME COLUMN gift_expires_at TO guest_expires_at")
|
||
if "gift_expires_at" in cols and "guest_expires_at" not in cols:
|
||
conn.execute("ALTER TABLE members RENAME COLUMN gift_expires_at TO guest_expires_at")
|
||
conn.execute("UPDATE members SET account_type = 'guest' WHERE account_type = 'gift'")
|
||
# Credit-ledger dedupe references move with the rename (a re-grant that
|
||
# misses the old prefix could double-credit). Returns first: a
|
||
# 'gift-return:…' value also matches 'gift:%'. Only when the table
|
||
# already exists — a first-boot database creates it below, empty.
|
||
if "credit_ledger" in _tables:
|
||
conn.execute("UPDATE credit_ledger SET reference = 'guest-return:' || substr(reference, 13) "
|
||
"WHERE reference LIKE 'gift-return:%'")
|
||
conn.execute("UPDATE credit_ledger SET reference = 'guest:' || substr(reference, 6) "
|
||
"WHERE reference LIKE 'gift:%'")
|
||
# Migration: guest_expires_at — when a guest account's GUEST_DAYS time box
|
||
# elapses (set at guest provisioning). The sweep deactivates the account
|
||
# after this instant even if credits remain.
|
||
# NOTE: this must be an EXCLUSIVE branch against the RENAME above — `cols`
|
||
# is a snapshot taken before the rename, so a plain `if not in cols` here
|
||
# would add a duplicate column on any database the rename just migrated.
|
||
if "gift_expires_at" not in cols and "guest_expires_at" not in cols:
|
||
conn.execute("ALTER TABLE members ADD COLUMN guest_expires_at TEXT")
|
||
# Guests ledger — the abuse-protection audit trail: WHO invited WHOM, when,
|
||
# and what became of the guest. One row per guest ever sent.
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS guests ("
|
||
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
||
"inviter_email TEXT NOT NULL, "
|
||
"recipient_email TEXT NOT NULL, "
|
||
"oc_reference TEXT, "
|
||
"created_at TEXT DEFAULT (datetime('now')), "
|
||
"guest_expires_at TEXT, "
|
||
"status TEXT DEFAULT 'active')"
|
||
)
|
||
# Guest REQUESTS — LEGACY history from the trial-approval period. The
|
||
# approval gate was removed on 2026-11-08: a member's invitation
|
||
# provisions immediately, straight into the guests ledger above. The
|
||
# table is kept so the historical trail (decided requests) still reads;
|
||
# nothing writes to it any more.
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS guest_requests ("
|
||
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
||
"inviter_email TEXT NOT NULL, "
|
||
"recipient_email TEXT NOT NULL, "
|
||
"recipient_name TEXT, "
|
||
"created_at TEXT DEFAULT (datetime('now')), "
|
||
"decided_at TEXT, "
|
||
"status TEXT DEFAULT 'pending', "
|
||
"note TEXT)"
|
||
)
|
||
# Credit ledger: audit trail for every credit movement (purchases via the
|
||
# OC webhook, admin adjustments, spends drawn against credits). Money needs
|
||
# an audit trail.
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS credit_ledger ("
|
||
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
||
"email TEXT NOT NULL, "
|
||
"delta REAL NOT NULL, "
|
||
"reason TEXT NOT NULL, "
|
||
"reference TEXT, "
|
||
"created_at TEXT DEFAULT (datetime('now'))"
|
||
")"
|
||
)
|
||
# Held credit packs: purchases by NON-members. Money received but credits
|
||
# not granted (a membership is required to use credits). Each hold is
|
||
# either refunded manually (OC dashboard, 30-day window) or granted
|
||
# automatically when the buyer becomes a member.
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS held_credits ("
|
||
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
||
"slug TEXT NOT NULL, "
|
||
"email TEXT, "
|
||
"name TEXT, "
|
||
"amount REAL NOT NULL, "
|
||
"oc_reference TEXT, "
|
||
"status TEXT NOT NULL DEFAULT 'held', "
|
||
"created_at TEXT DEFAULT (datetime('now')), "
|
||
"resolved_at TEXT"
|
||
")"
|
||
)
|
||
# Deliveries the portal did not accept. Recorded BY THE PORTAL because it
|
||
# handles every delivery, so noticing a failed one is the system's job and
|
||
# not a scraper's: the alert used to live on the operator's own machine,
|
||
# reading this app's log through the API — a money-critical signal depending
|
||
# on an outside host, and on a buffer with hours of memory (whole hours were
|
||
# missing from it when a $10 purchase was investigated on 2026-11-13). The
|
||
# sweep summarises new rows here and emails info@ when there are any.
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS webhook_failures ("
|
||
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
||
"at TEXT NOT NULL, "
|
||
"status INTEGER NOT NULL, "
|
||
"event_type TEXT, "
|
||
"detail TEXT"
|
||
")"
|
||
)
|
||
# Small key/value store for the portal's own bookkeeping (e.g. "the last
|
||
# webhook failure id we have already reported"). Deliberately not a settings
|
||
# table: nothing here is configuration.
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS portal_state ("
|
||
"key TEXT PRIMARY KEY, "
|
||
"value TEXT"
|
||
")"
|
||
)
|
||
# Unique oc_reference (audit H1, 2026-11-05): an OC re-delivery must never
|
||
# create a second hold (or, via the webhook grant path's exists-check, a
|
||
# second credit) for the same purchase. Holds recorded before the fix used
|
||
# the legacy empty-suffix format and are distinct as audited, so the index
|
||
# applies cleanly to the live table; multiple NULLs stay legal (SQLite).
|
||
# If creation fails there are REAL duplicates — log CRITICAL and keep
|
||
# serving unguarded rather than boot-looping the portal.
|
||
try:
|
||
conn.execute(
|
||
"CREATE UNIQUE INDEX IF NOT EXISTS idx_held_credits_oc_reference "
|
||
"ON held_credits(oc_reference)"
|
||
)
|
||
except sqlite3.OperationalError as e:
|
||
logger.error("held_credits.oc_reference unique index NOT created: %s", e)
|
||
|
||
# Member-managed API keys: add a fingerprint (last-4 of the key, shown in
|
||
# lists) and purge the stored plaintext. Member API keys are revealed ONCE
|
||
# (at creation) and never again — the industry standard. Revocation uses the
|
||
# LiteLLM hash, not the plaintext, so the portal has no reason to keep it.
|
||
# (The `members.key_token` chat key is a different table and IS retained —
|
||
# encrypted at rest (AES-256-GCM), decrypted in memory only when served.)
|
||
ak_cols = [r[1] for r in conn.execute("PRAGMA table_info(member_api_keys)").fetchall()]
|
||
if "fingerprint" not in ak_cols:
|
||
conn.execute("ALTER TABLE member_api_keys ADD COLUMN fingerprint TEXT")
|
||
# Backfill fingerprints for any key still holding plaintext (fingerprint =
|
||
# last-4 of the key), then purge the plaintext. Idempotent — runs every
|
||
# time, affects only rows where plaintext is still present.
|
||
conn.execute(
|
||
"UPDATE member_api_keys SET fingerprint = substr(sk_token, -4) "
|
||
"WHERE (fingerprint IS NULL OR fingerprint = '') AND sk_token != ''"
|
||
)
|
||
conn.execute("UPDATE member_api_keys SET sk_token = '' WHERE sk_token != ''")
|
||
|
||
# Encrypt member chat keys at rest (AES-256-GCM, KEK from PORTAL_KEK env).
|
||
# Idempotent: only touches rows still holding plaintext. Skipped with a
|
||
# CRITICAL log if PORTAL_KEK is unset — see the inline key-encryption module.
|
||
migrate_plaintext_keys(conn)
|
||
|
||
# Commit the DML above — ALTER TABLE auto-commits, but the UPDATEs open an
|
||
# implicit transaction that would otherwise roll back when get_db() closes.
|
||
conn.commit()
|
||
return conn
|
||
|
||
|
||
def get_member_balance(email: str) -> float:
|
||
"""Return the member's current balance (USD), defaulting to MEMBER_BUDGET.
|
||
|
||
The balance is the flexible allowance — it starts at MEMBER_BUDGET but can
|
||
be adjusted (topped up by payments) without changing code.
|
||
"""
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT balance FROM members WHERE email = ?", (email,)
|
||
).fetchone()
|
||
conn.close()
|
||
if row and row[0] is not None:
|
||
return float(row[0])
|
||
return MEMBER_BUDGET
|
||
|
||
|
||
def set_member_balance(email: str, balance: float) -> None:
|
||
"""Set the member's balance (USD). Used by top-up / billing adjustments."""
|
||
conn = get_db()
|
||
conn.execute(
|
||
"UPDATE members SET balance = ? WHERE email = ?", (balance, email)
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
# --- Credits: purchased, non-expiring usage dollars ---
|
||
|
||
def get_credit_balance(email: str) -> float:
|
||
"""Return the member's non-expiring credit balance (USD, default 0)."""
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT credit_balance FROM members WHERE email = ?", (email,)
|
||
).fetchone()
|
||
conn.close()
|
||
if row and row[0] is not None:
|
||
return float(row[0])
|
||
return 0.0
|
||
|
||
|
||
def get_member_account_type(email: str) -> str:
|
||
"""Return the row's account type: 'member' (default) or 'guest'."""
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT account_type FROM members WHERE email = ?", (email,)
|
||
).fetchone()
|
||
conn.close()
|
||
return (row[0] or "member") if row else "member"
|
||
|
||
|
||
def is_guest_account(email: str) -> bool:
|
||
"""True when the row is a guest account (operator guest access feature)."""
|
||
return get_member_account_type(email) == "guest"
|
||
|
||
|
||
def guest_provision_guard(
|
||
conn, inviter_email: str, recipient_email: str, admin: bool = False
|
||
) -> str | None:
|
||
"""Decide whether a guest may be sent (pure — runs on any connection).
|
||
|
||
Abuse-protection rules (operator decision, 2026-11-08):
|
||
* the inviter must be an ACTIVE member row (account_type 'member');
|
||
* one guest per inviter per rolling GUEST_PER_INVITER_DAYS (default 30 days);
|
||
* one guest per recipient EVER — any prior guests-ledger row for the
|
||
recipient refuses, as does an existing portal row under that email
|
||
(they already have/had a guest account or are a member).
|
||
|
||
admin=True is the ADMIN-PANEL path: the inviter is the co-op's admin
|
||
account, not a member, so the member-side inviter rules (active member,
|
||
one-per-30-days) do NOT apply — admins are not bound by the member's
|
||
monthly cap. The recipient-side rules still run on every path: one guest
|
||
per recipient ever, and no existing portal row.
|
||
|
||
Returns None when the guest may proceed, else a human-readable refusal.
|
||
"""
|
||
inviter = (inviter_email or "").strip().lower()
|
||
recipient = (recipient_email or "").strip().lower()
|
||
if not inviter or "@" not in inviter:
|
||
return "The inviting member's email is required"
|
||
if not recipient or "@" not in recipient:
|
||
return "A valid recipient email is required"
|
||
if inviter == recipient:
|
||
return "You cannot guest the account you are guest access from"
|
||
if not admin:
|
||
row = conn.execute(
|
||
"SELECT active, account_type FROM members WHERE email = ?", (inviter,)
|
||
).fetchone()
|
||
if not row or not row[0] or (row[1] or "member") == "guest":
|
||
return f"Inviter {inviter} is not an active member account"
|
||
recent = conn.execute(
|
||
"SELECT COUNT(*) FROM guests WHERE inviter_email = ? AND "
|
||
"created_at >= datetime('now', ?)",
|
||
(inviter, f"-{int(GUEST_PER_INVITER_DAYS)} days"),
|
||
).fetchone()[0]
|
||
if recent:
|
||
return (f"{inviter} already sent a guest in the last "
|
||
f"{int(GUEST_PER_INVITER_DAYS)} days — one guest per member per month")
|
||
ever = conn.execute(
|
||
"SELECT COUNT(*) FROM guests WHERE recipient_email = ?", (recipient,)
|
||
).fetchone()[0]
|
||
if ever:
|
||
return f"{recipient} has already received a guest account"
|
||
existing = conn.execute(
|
||
"SELECT account_type FROM members WHERE email = ?", (recipient,)
|
||
).fetchone()
|
||
if existing:
|
||
acct = existing[0] or "member"
|
||
return (f"{recipient} already has a portal row ({acct}) — guest access is for "
|
||
"new addresses; existing members and past guest accounts are refused")
|
||
return None
|
||
|
||
|
||
def guest_request_guard(conn, inviter_email: str, recipient_email: str) -> str | None:
|
||
"""Decide whether a MEMBER may send a guest invitation (pure).
|
||
|
||
The member-invite guard. Runs everything guest_provision_guard enforces
|
||
(active member inviter, one guest per inviter per rolling window, one
|
||
guest per recipient ever, no existing portal row) plus the member-path
|
||
rule: the recipient is not on MANUAL_MEMBERS — manually-provisioned
|
||
accounts are co-op insiders, never guest targets.
|
||
|
||
(The trial-era pending-request rules are gone with the approval gate:
|
||
invitations provision immediately, and the one-per-30-days window is
|
||
counted from the guests ledger by guest_provision_guard.)
|
||
|
||
Returns None when the invitation may proceed, else a human-readable refusal.
|
||
"""
|
||
refusal = guest_provision_guard(conn, inviter_email, recipient_email)
|
||
if refusal:
|
||
return refusal
|
||
recipient = (recipient_email or "").strip().lower()
|
||
if recipient in MANUAL_MEMBERS:
|
||
return ("That address belongs to a manually-provisioned co-op account — "
|
||
"guest invitations are for people outside the co-op")
|
||
return None
|
||
|
||
|
||
def add_credits(email: str, delta: float, reason: str, reference: str | None = None) -> float:
|
||
"""Adjust a member's credit balance and record the movement in the ledger.
|
||
|
||
Returns the new balance. Negative deltas are allowed (draws against
|
||
credits); the balance is clamped at >= 0.
|
||
"""
|
||
conn = get_db()
|
||
conn.execute(
|
||
"UPDATE members SET credit_balance = MAX(0, COALESCE(credit_balance, 0) + ?) "
|
||
"WHERE email = ?",
|
||
(delta, email),
|
||
)
|
||
conn.execute(
|
||
"INSERT INTO credit_ledger (email, delta, reason, reference) VALUES (?, ?, ?, ?)",
|
||
(email, delta, reason, reference),
|
||
)
|
||
row = conn.execute(
|
||
"SELECT credit_balance FROM members WHERE email = ?", (email,)
|
||
).fetchone()
|
||
conn.commit()
|
||
conn.close()
|
||
return float(row[0]) if row and row[0] is not None else 0.0
|
||
|
||
|
||
def compute_max_budget(allowance: float, credit_balance: float, team_spend: float | None) -> dict:
|
||
"""Compute the effective LiteLLM max_budget from the two buckets.
|
||
|
||
Model: the member has a monthly ALLOWANCE (renews on each membership
|
||
payment) plus a credit balance that adds spendable headroom ON TOP of the
|
||
allowance. The total spendable cap is simply `allowance + credit_balance`
|
||
— credits never shrink the allowance; they extend it.
|
||
|
||
`credit_balance` is the TOTAL purchased (the ledger's lifetime sum); the
|
||
amount still available this cycle is `credit_balance - credits_used`.
|
||
|
||
`team_spend` is the member's spend so far this cycle (from LiteLLM). It is
|
||
used only to split the spend between allowance vs credits for display, NOT
|
||
to reduce the cap.
|
||
|
||
Returns {"max_budget", "allowance_used", "credits_used", "credits_remaining"}
|
||
for dashboards.
|
||
"""
|
||
max_budget = allowance + credit_balance
|
||
if team_spend is None:
|
||
return {
|
||
"max_budget": max_budget,
|
||
"allowance_used": 0.0,
|
||
"credits_used": 0.0,
|
||
"credits_remaining": credit_balance,
|
||
}
|
||
allowance_used = min(max(team_spend, 0.0), allowance)
|
||
credits_used = max(0.0, team_spend - allowance)
|
||
return {
|
||
"max_budget": max_budget,
|
||
"allowance_used": allowance_used,
|
||
"credits_used": credits_used,
|
||
"credits_remaining": max(0.0, credit_balance - credits_used),
|
||
}
|
||
|
||
|
||
async def _oc_graphql_post(client: httpx.AsyncClient, json_body: dict,
|
||
headers: dict | None = None, attempts: int = 2):
|
||
"""POST to Open Collective's GraphQL API, retrying once on a transport error.
|
||
|
||
A read timeout here is load-correlated and transient — not a permission,
|
||
query or scope problem — so one retry converts the common case into a
|
||
success. Whatever still fails is RAISED: every caller in this module is
|
||
built to fail loudly (the sweep must never deactivate members from a
|
||
partial list), so swallowing it would be worse than the error.
|
||
"""
|
||
last: Exception | None = None
|
||
for attempt in range(attempts):
|
||
try:
|
||
return await client.post(OC_GRAPHQL_URL, headers=headers or {}, json=json_body)
|
||
except httpx.TransportError as e: # ReadTimeout, ConnectError, …
|
||
last = e
|
||
if attempt + 1 < attempts:
|
||
logger.warning(
|
||
"OC GraphQL %s (attempt %d/%d); retrying once",
|
||
type(e).__name__, attempt + 1, attempts,
|
||
)
|
||
await asyncio.sleep(2.0)
|
||
assert last is not None
|
||
raise last
|
||
|
||
|
||
async def fetch_all_member_nodes() -> list[dict]:
|
||
"""Every member node of the collective, via OC's offset pagination.
|
||
|
||
The members connection has NO cursor/pageInfo pagination (older-style API)
|
||
and caps at 100 per page: an unpaginated query silently hides everyone
|
||
past the first page — the same failure class as Cloudron's 25-per-page
|
||
user list. Raises on ANY page failure so callers can apply their own
|
||
fail-safes; never returns a partial list (a partial member list would make
|
||
the sweep deactivate everyone missing from it).
|
||
|
||
The timeout is set EXPLICITLY on purpose. httpx's default is 5 seconds, and
|
||
on 2026-11-13 a page of this walk took longer than that: `httpx.ReadTimeout`
|
||
escaped the OC webhook handler as an unhandled exception, so a member's $50
|
||
credit-pack purchase returned 500 and her credits were not granted. A
|
||
generous timeout plus one retry makes the transient case a success; the
|
||
handler's own error net (see opencollective_webhook) covers the rest.
|
||
"""
|
||
if not OC_PERSONAL_TOKEN:
|
||
raise RuntimeError("OC_PERSONAL_TOKEN not set; cannot fetch members")
|
||
nodes: list[dict] = []
|
||
offset = 0
|
||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||
for _ in range(20): # hard cap: 20 pages × 100 = 2000 members
|
||
query = (
|
||
'{ collective(slug: "%s") { members(limit: 100, offset: %d) { '
|
||
"totalCount nodes { id role createdAt tier { name } account { name slug type ... on Individual { email } } } } } }"
|
||
% (OC_COLLECTIVE_SLUG, offset)
|
||
)
|
||
r = await _oc_graphql_post(
|
||
client, {"query": query}, {"Personal-Token": OC_PERSONAL_TOKEN},
|
||
)
|
||
if r.status_code != 200:
|
||
raise RuntimeError(f"OC member fetch failed: HTTP {r.status_code}")
|
||
members = ((r.json().get("data") or {}).get("collective") or {}).get("members") or {}
|
||
total = members.get("totalCount") or 0
|
||
page = members.get("nodes") or []
|
||
nodes.extend(page)
|
||
offset += len(page)
|
||
if not page or offset >= total:
|
||
return nodes
|
||
raise RuntimeError("OC member pagination exceeded 20 pages")
|
||
|
||
|
||
_MEMBER_SET_MIN_RATIO = 0.5
|
||
|
||
|
||
def _member_set_verdict(fetched: int, local_active: int) -> bool:
|
||
"""Is this member list plausible enough to act on? (member-portal#2)
|
||
|
||
Pure decision so it can be tested without a database.
|
||
|
||
Guards the case a raise-only fail-safe cannot see: a fetch that SUCCEEDS while
|
||
returning nothing, or far less than usual. A narrowed token scope, an expired
|
||
credential with partial access, or a renamed Membership tier on Open Collective
|
||
all produce that, and the sweep's response to it is to deactivate the membership
|
||
rather than to fail. An empty collective is legitimate only when we have no
|
||
active members ourselves.
|
||
"""
|
||
if local_active <= 0:
|
||
return True
|
||
# Strictly above: a halving of the membership is implausible enough, and refusing
|
||
# costs a loud error (no action taken) where allowing it costs the membership.
|
||
return fetched > 0 and fetched > local_active * _MEMBER_SET_MIN_RATIO
|
||
|
||
|
||
def _member_set_guard(fetched: int, label: str) -> None:
|
||
"""Raise when the fetched member list is implausible, so callers fail safe."""
|
||
try:
|
||
conn = get_db()
|
||
local_active = conn.execute(
|
||
"SELECT COUNT(*) FROM members WHERE active = 1"
|
||
).fetchone()[0]
|
||
conn.close()
|
||
except Exception as e: # fresh install / unreadable DB: nothing to compare against
|
||
logger.warning("member-set guard: cannot read the local member count (%s); skipping", e)
|
||
return
|
||
if _member_set_verdict(fetched, local_active):
|
||
logger.info("member-set guard OK (%s): fetched %d, %d active locally",
|
||
label, fetched, local_active)
|
||
return
|
||
raise RuntimeError(
|
||
f"implausible member list from Open Collective ({label}): fetched {fetched} while "
|
||
f"{local_active} members are active locally. Refusing to reconcile — the sweep "
|
||
"fail-safe keeps members active instead. Check the OC token's scopes and expiry, "
|
||
"and that the Membership tier still exists (fetch_members filters on its name)."
|
||
)
|
||
|
||
|
||
async def fetch_members() -> list[dict]:
|
||
"""Return the full active-member list: email, name, slug, role, tier.
|
||
|
||
Uses the bot's personal token (admin) so emails are exposed. This is the
|
||
authoritative source of truth for reconciliation (provisioning missing
|
||
members, deactivating lapsed ones).
|
||
|
||
Membership is tied to the "Membership" TIER — a one-time donation grants
|
||
BACKER role on Open Collective but is NOT a membership. So this returns
|
||
only users whose tier name contains "membership" (case-insensitive) or
|
||
who have no tier (legacy: contributors who predate the tier system).
|
||
"""
|
||
if not OC_PERSONAL_TOKEN:
|
||
logger.warning("OC_PERSONAL_TOKEN not set; cannot fetch members")
|
||
return []
|
||
# Paginated fetch (raises on failure → caller fail-safes apply: the sweep
|
||
# skips deactivation entirely when we can't get the full member list).
|
||
nodes = await fetch_all_member_nodes()
|
||
result: list[dict] = []
|
||
for n in nodes:
|
||
role = n.get("role", "")
|
||
acct = n.get("account", {}) or {}
|
||
email = (acct.get("email") or "").strip().lower()
|
||
tier = ((n.get("tier") or {}).get("name") or "").strip()
|
||
if not email or role not in ("BACKER", "ADMIN"):
|
||
continue
|
||
# Admins (the co-op itself) always count. BACKERs must be on the
|
||
# Membership tier; no-tier BACKERs are legacy contributors from before
|
||
# tiers existed — grandfathered as members.
|
||
if role == "ADMIN" or "membership" in tier.lower() or tier == "":
|
||
result.append({
|
||
"email": email,
|
||
"name": acct.get("name") or email.split("@")[0],
|
||
"slug": acct.get("slug") or "",
|
||
"role": role,
|
||
"tier": tier,
|
||
})
|
||
# Guard before ANY caller acts on this: the sweep deactivates members missing
|
||
# from the list, so an implausible list must fail rather than propagate.
|
||
_member_set_guard(len(result), "oc members query")
|
||
return result
|
||
|
||
|
||
async def fetch_org_backers() -> dict[str, dict]:
|
||
"""Organization-backed memberships on the Membership tier.
|
||
|
||
OC Organizations never expose an email to our integration (the
|
||
`... on Individual { email }` fragment doesn't match them, and OC does not
|
||
return org emails even to collective admins), so these members are LINKED
|
||
to portal accounts via members.org_slug (POST /admin/org-link). Keyed by
|
||
org slug: {slug: {name, role, tier}}.
|
||
"""
|
||
if not OC_PERSONAL_TOKEN:
|
||
logger.warning("OC_PERSONAL_TOKEN not set; cannot fetch org members")
|
||
raise RuntimeError("OC_PERSONAL_TOKEN not set")
|
||
# Paginated fetch; RAISES on failure so the sweep's fail-safe keeps linked
|
||
# accounts active rather than deactivating them on a fetch error.
|
||
nodes = await fetch_all_member_nodes()
|
||
result: dict[str, dict] = {}
|
||
for n in nodes:
|
||
role = n.get("role", "")
|
||
acct = n.get("account", {}) or {}
|
||
if (acct.get("type") or "") != "ORGANIZATION":
|
||
continue
|
||
if role not in ("BACKER", "ADMIN"):
|
||
continue
|
||
tier = ((n.get("tier") or {}).get("name") or "").strip()
|
||
if role != "ADMIN" and "membership" not in tier.lower() and tier != "":
|
||
continue
|
||
slug = acct.get("slug") or ""
|
||
if slug:
|
||
result[slug] = {
|
||
"name": acct.get("name") or slug,
|
||
"role": role,
|
||
"tier": tier,
|
||
}
|
||
return result
|
||
|
||
|
||
def send_email(to: str, subject: str, text_body: str, html_body: str | None = None) -> bool:
|
||
"""Send an email via Cloudron's sendmail addon (SMTP).
|
||
|
||
Returns True on success, False on failure (logged, not raised — email is
|
||
best-effort and must never break the provisioning flow).
|
||
"""
|
||
if not SMTP_SERVER:
|
||
logger.warning("SMTP not configured; skipping email to %s", to)
|
||
return False
|
||
try:
|
||
msg = MIMEMultipart("alternative")
|
||
msg["From"] = f"{MAIL_FROM_NAME} <{MAIL_FROM}>"
|
||
msg["To"] = to
|
||
msg["Subject"] = subject
|
||
msg["Reply-To"] = MAIL_REPLY_TO
|
||
# Explicit Message-ID so a send is traceable in the mail system and can be
|
||
# named in a report without reading the mailbox.
|
||
msg["Message-ID"] = make_msgid(domain=(MAIL_FROM.split("@")[-1] if "@" in MAIL_FROM else "inference.coop"))
|
||
msg.attach(MIMEText(text_body, "plain", "utf-8"))
|
||
if html_body:
|
||
msg.attach(MIMEText(html_body, "html", "utf-8"))
|
||
|
||
with smtplib.SMTP(SMTP_SERVER, SMTP_PORT, timeout=15) as s:
|
||
if SMTP_USERNAME:
|
||
s.login(SMTP_USERNAME, SMTP_PASSWORD)
|
||
s.sendmail(MAIL_FROM, [to], msg.as_string())
|
||
logger.info("Sent email to %s: %s (message-id %s)", to, subject, msg["Message-ID"])
|
||
return True
|
||
except Exception as e:
|
||
logger.warning("Failed to send email to %s: %s", to, e)
|
||
return False
|
||
|
||
|
||
def welcome_email(name: str, setup_url: str) -> tuple[str, str, str]:
|
||
"""Build the welcome email (subject, text, html) for a new member.
|
||
|
||
`setup_url` is the Cloudron account-setup link (from invite_link), which
|
||
lets the member create their account directly — no OAuth step needed.
|
||
|
||
The HTML uses a centered, card-style layout matching the landing page
|
||
(cream background, forest-green button, Figtree-like system fonts).
|
||
Email HTML is table-based for client compatibility; no external assets.
|
||
"""
|
||
subject = "Welcome to the Inference Cooperative — finish your setup"
|
||
text = (
|
||
f"Hi {name},\n\n"
|
||
"Thanks for joining the Inference Cooperative!\n\n"
|
||
"To finish setting up your account and start using private AI chat, "
|
||
"click the link below to create your account:\n\n"
|
||
f"{setup_url}\n\n"
|
||
"This takes about a minute.\n\n"
|
||
"Questions? Reply to this email or write to info@inference.coop.\n\n"
|
||
"— The Inference Cooperative\n"
|
||
)
|
||
html = f"""<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"></head>
|
||
<body style="margin:0;padding:0;background:#faf8f5;font-family:-apple-system,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;">
|
||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:#faf8f5;padding:40px 16px;">
|
||
<tr>
|
||
<td align="center">
|
||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="max-width:520px;">
|
||
<tr>
|
||
<td align="center" style="padding-bottom:24px;">
|
||
<!-- Pine-tree circle logo (simplified) -->
|
||
<svg width="56" height="56" viewBox="0 0 130 130" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true">
|
||
<circle cx="65" cy="14" r="5" fill="#5b8c5a"/>
|
||
<circle cx="58" cy="30" r="6" fill="#7ab87a"/>
|
||
<circle cx="74" cy="30" r="4" fill="#6ba86b"/>
|
||
<circle cx="51.5" cy="46" r="6.5" fill="#5b8c5a"/>
|
||
<circle cx="65" cy="46" r="5" fill="#7ab87a"/>
|
||
<circle cx="81" cy="46" r="4" fill="#6ba86b"/>
|
||
<circle cx="44" cy="62" r="7" fill="#6ba86b"/>
|
||
<circle cx="61" cy="62" r="5.5" fill="#5b8c5a"/>
|
||
<circle cx="76" cy="62" r="4.5" fill="#7ab87a"/>
|
||
<circle cx="89.5" cy="62" r="3.5" fill="#6ba86b"/>
|
||
<circle cx="36.5" cy="78" r="7.5" fill="#5b8c5a"/>
|
||
<circle cx="54" cy="78" r="6" fill="#7ab87a"/>
|
||
<circle cx="70" cy="78" r="5" fill="#6ba86b"/>
|
||
<circle cx="84" cy="78" r="4" fill="#5b8c5a"/>
|
||
<circle cx="97.5" cy="78" r="3.5" fill="#7ab87a"/>
|
||
<circle cx="44" cy="94" r="6.5" fill="#6ba86b"/>
|
||
<circle cx="61" cy="94" r="5.5" fill="#5b8c5a"/>
|
||
<circle cx="76" cy="94" r="4.5" fill="#7ab87a"/>
|
||
<circle cx="89.5" cy="94" r="3.5" fill="#6ba86b"/>
|
||
<circle cx="51.5" cy="110" r="5.5" fill="#5b8c5a"/>
|
||
<circle cx="65" cy="110" r="4.5" fill="#7ab87a"/>
|
||
<circle cx="81" cy="110" r="3.5" fill="#6ba86b"/>
|
||
</svg>
|
||
</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="center" style="background:#ffffff;border:1px solid #e8e2d8;border-radius:16px;padding:40px 32px;">
|
||
<h1 style="margin:0 0 12px;font-size:24px;font-weight:700;color:#2d3327;">Welcome, {name}</h1>
|
||
<p style="margin:0 0 8px;font-size:16px;color:#6b7a62;line-height:1.5;">Thanks for joining the <strong style="color:#2d3327;">Inference Cooperative</strong>.</p>
|
||
<p style="margin:0 0 28px;font-size:15px;color:#6b7a62;line-height:1.5;">Finish setting up your account to start using private AI chat.</p>
|
||
<a href="{setup_url}" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:10px;font-size:16px;font-weight:600;">Finish setup</a>
|
||
<p style="margin:28px 0 0;font-size:13px;color:#94a08c;line-height:1.5;">This takes about a minute.</p>
|
||
</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="center" style="padding-top:24px;font-size:13px;color:#94a08c;line-height:1.6;">
|
||
Questions? Reply to this email or write to <a href="mailto:info@inference.coop" style="color:#6b7a62;">info@inference.coop</a>.<br>
|
||
<span style="color:#2d3327;">The Inference Cooperative</span> · inference.coop<br>
|
||
<span style="font-size:12px;">By creating your account, you agree to our <a href="https://git.inference.coop/co-op/docs/src/branch/main/terms-of-service.md" style="color:#6b7a62;">Terms of Service</a> and <a href="https://git.inference.coop/co-op/docs/src/branch/main/privacy-policy.md" style="color:#6b7a62;">Privacy Policy</a>.</span>
|
||
</td>
|
||
</tr>
|
||
</table>
|
||
</td>
|
||
</tr>
|
||
</table>
|
||
</body>
|
||
</html>"""
|
||
return subject, text, html
|
||
|
||
|
||
async def get_active_member_emails() -> list[str]:
|
||
"""Return the list of active MEMBER emails from the portal's own database.
|
||
|
||
The local `members` table is the authoritative source of member emails
|
||
(populated during provisioning). Used for the Loomio sync and the Matrix
|
||
invite pass. GUEST accounts are excluded: they are not members — they hold
|
||
$2 credits, live in the Cloudron guests group, and must never be invited
|
||
to Loomio, Matrix or (via these lists) any member service.
|
||
"""
|
||
conn = get_db()
|
||
rows = conn.execute(
|
||
"SELECT email, account_type FROM members WHERE active = 1"
|
||
).fetchall()
|
||
conn.close()
|
||
return [r[0] for r in rows if (r[1] or "member") != "guest"]
|
||
|
||
|
||
def get_loomio_invited_at(email: str) -> str | None:
|
||
"""Return when `email` was first handed to Loomio, or None if never invited."""
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT loomio_invited_at FROM members WHERE email = ?", (email,)
|
||
).fetchone()
|
||
conn.close()
|
||
return row[0] if row else None
|
||
|
||
|
||
def mark_loomio_invited(email: str) -> None:
|
||
"""Record that `email` has been sent to Loomio at least once (idempotent).
|
||
|
||
Never overwrites an existing timestamp — the invariant is "has this email
|
||
EVER been handed to Loomio", not "when was it last".
|
||
"""
|
||
conn = get_db()
|
||
conn.execute(
|
||
"UPDATE members SET loomio_invited_at = COALESCE(loomio_invited_at, datetime('now')) "
|
||
"WHERE email = ?",
|
||
(email,),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def get_member_matrix_mxid(email: str) -> str:
|
||
"""Return the member's optional extra Matrix MXID, or "" if unset."""
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT matrix_mxid FROM members WHERE email = ?", (email,)
|
||
).fetchone()
|
||
conn.close()
|
||
return (row[0] or "").strip() if row else ""
|
||
|
||
|
||
def set_member_matrix_mxid(email: str, mxid: str) -> None:
|
||
"""Set (or clear) the member's optional extra Matrix MXID."""
|
||
conn = get_db()
|
||
conn.execute(
|
||
"UPDATE members SET matrix_mxid = ? WHERE email = ?",
|
||
(mxid.strip() if mxid else "", email),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def _loomio_safe_username(raw) -> str | None:
|
||
"""Sanitize a candidate into a Loomio-legal username.
|
||
|
||
Loomio accepts lowercase letters, numbers, underscores and INTERNAL hyphens
|
||
only, so a dot-containing candidate — an OAuth uid or an email local part
|
||
such as "joshua.z.tan" — must be normalized rather than passed through.
|
||
Passing it through is what made this fix fail with HTTP 422 on every sweep.
|
||
"""
|
||
u = re.sub(r"[^a-z0-9_-]", "_", (raw or "").lower()).strip("-_")
|
||
return u if len(u) >= 2 else None
|
||
|
||
|
||
def _loomio_is_autogenerated(current: str, email: str) -> bool:
|
||
"""Is this handle one Loomio generated for us?
|
||
|
||
Loomio derives a handle from the email local part with non-alphanumerics
|
||
stripped, plus a random suffix ("joshuaztan" + "xjan5rzb8j27" for
|
||
joshua.z.tan@...). Anything that is not that shape — "amarino", say — was
|
||
chosen by a person, and renaming it would be an unrequested member-visible
|
||
change. Only auto-generated handles are in scope for this fix.
|
||
"""
|
||
local = re.sub(r"[^a-z0-9]", "", (email or "").split("@")[0].lower())
|
||
cur = (current or "").lower()
|
||
return bool(local) and cur.startswith(local) and len(cur) > len(local)
|
||
|
||
|
||
async def loomio_fix_usernames(client: httpx.AsyncClient) -> dict:
|
||
"""Rename Loomio users with auto-generated handles to their Cloudron username.
|
||
|
||
Loomio's b2/memberships invite creates the account with a mangled handle
|
||
(email local-part + random suffix, e.g. danishipleydsdk4ibdihka). After
|
||
first SSO login the user carries an oauth identity whose uid IS their
|
||
Cloudron username — prefer that; fall back to the email local-part.
|
||
Uses the server (b3) API. Returns a {fixed, skipped} summary.
|
||
"""
|
||
H = {"Authorization": f"Bearer {LOOMIO_B3_KEY}", "Content-Type": "application/json"}
|
||
r = await client.get(f"{LOOMIO_BASE}/api/b3/users", headers=H)
|
||
r.raise_for_status()
|
||
users = r.json().get("users", [])
|
||
|
||
fixed, skipped = 0, 0
|
||
for u in users:
|
||
# Only Loomio's own auto-generated handles are in scope: a name a human
|
||
# chose stays, even when it differs from the Cloudron username.
|
||
if not _loomio_is_autogenerated(u.get("username") or "", u.get("email") or ""):
|
||
skipped += 1
|
||
continue
|
||
target = None
|
||
for ident in u.get("identities") or []:
|
||
if ident.get("identity_type") == "oauth" and ident.get("uid"):
|
||
target = _loomio_safe_username(ident["uid"])
|
||
if not target:
|
||
# uid unusable even after sanitizing — fall back to the email local part
|
||
target = _loomio_safe_username((u.get("email") or "").split("@")[0])
|
||
break
|
||
if not target:
|
||
target = _loomio_safe_username((u.get("email") or "").split("@")[0])
|
||
if not target or u.get("username") == target:
|
||
skipped += 1
|
||
continue
|
||
rr = await client.patch(
|
||
f"{LOOMIO_BASE}/api/b3/users/{u['id']}",
|
||
headers=H, json={"username": target},
|
||
)
|
||
if rr.status_code == 200:
|
||
fixed += 1
|
||
else:
|
||
# log the Loomio user id, never the member's email
|
||
logger.warning("Loomio username fix for user %s failed: %s %s",
|
||
u.get("id"), rr.status_code, rr.text[:120])
|
||
if fixed:
|
||
logger.info("Loomio usernames fixed: %s", fixed)
|
||
return {"fixed": fixed, "skipped": skipped}
|
||
|
||
|
||
def member_invite_rooms() -> list[str]:
|
||
"""The explicit set of rooms every active member of the co-op is invited to.
|
||
|
||
MATRIX_INVITE_ROOMS is authoritative when set; otherwise we fall back to the
|
||
co-op Space and General room. The legacy MATRIX_EXTRA_ROOMS entries are
|
||
appended (deduped) so existing configuration keeps working. This is
|
||
deliberately explicit configuration and never "every child of the Space":
|
||
operator/admin-only child rooms must not be pushed at every member.
|
||
"""
|
||
if MATRIX_INVITE_ROOMS:
|
||
rooms = list(MATRIX_INVITE_ROOMS)
|
||
else:
|
||
rooms = [MATRIX_SPACE_ID]
|
||
if MATRIX_GENERAL_ROOM:
|
||
rooms.append(MATRIX_GENERAL_ROOM)
|
||
for r in MATRIX_EXTRA_ROOMS:
|
||
if r and r not in rooms:
|
||
rooms.append(r)
|
||
seen: set[str] = set()
|
||
out: list[str] = []
|
||
for r in rooms:
|
||
if r and r not in seen:
|
||
seen.add(r)
|
||
out.append(r)
|
||
return out
|
||
|
||
|
||
async def sync_matrix_space() -> dict:
|
||
"""Invite every active member to the configured co-op Matrix rooms.
|
||
|
||
The room set comes from member_invite_rooms(): MATRIX_INVITE_ROOMS when set,
|
||
else the co-op Space + General room, plus the legacy MATRIX_EXTRA_ROOMS.
|
||
Members log in via Cloudron SSO, so their local MXID is
|
||
@<username>:inference.coop. We resolve the Cloudron username for each
|
||
active member email and invite them if not already in the space.
|
||
Best-effort: missing config just skips; failures log and continue.
|
||
"""
|
||
if not (MATRIX_BOT_TOKEN and MATRIX_SPACE_ID):
|
||
logger.info("Matrix space sync skipped (not configured)")
|
||
return {"skipped": True}
|
||
H = {"Authorization": f"Bearer {MATRIX_BOT_TOKEN}", "Content-Type": "application/json"}
|
||
|
||
class _RateLimited(Exception):
|
||
"""Synapse asked us to back off (429). Ends the pass, not the sync."""
|
||
|
||
def __init__(self, wait_ms):
|
||
super().__init__(f"retry_after_ms={wait_ms}")
|
||
self.wait_ms = wait_ms
|
||
|
||
async def invite(mxid: str, room: str) -> bool:
|
||
r = await client.post(
|
||
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{room}/invite",
|
||
headers=H, json={"user_id": mxid},
|
||
)
|
||
if r.status_code == 200:
|
||
return True
|
||
if r.status_code == 429:
|
||
# Synapse asks for MINUTES here, so sleeping would blow past the
|
||
# scheduled tick that called us. End the pass instead: the next run
|
||
# starts from the top, every already-sent invite is a cheap no-op,
|
||
# and progress accumulates across runs.
|
||
raise _RateLimited(r.json().get("retry_after_ms", 1000))
|
||
# 403 with 'already in room' is fine
|
||
if "already" in r.text:
|
||
return True
|
||
logger.warning("Matrix invite %s -> %s failed: %s %s",
|
||
mxid, room, r.status_code, r.text[:120])
|
||
return False
|
||
|
||
rooms = member_invite_rooms()
|
||
invited_count = 0
|
||
rate_limited = False
|
||
|
||
async with httpx.AsyncClient(timeout=30) as client:
|
||
# Snapshot EVERY room's membership once, up front: who is joined and who
|
||
# has a pending invite. The gate is PER ROOM on purpose — gating on the
|
||
# Space alone meant an account already invited to the Space was skipped
|
||
# for every other room, so a member whose invite pass was interrupted
|
||
# (or who predates a room added to MATRIX_INVITE_ROOMS later) never got
|
||
# that room and could not see it.
|
||
joined_by_room: dict[str, set[str]] = {}
|
||
invited_by_room: dict[str, set[str]] = {}
|
||
for room in rooms:
|
||
r = await client.get(
|
||
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{room}/joined_members",
|
||
headers=H)
|
||
if r.status_code != 200:
|
||
logger.warning(
|
||
"Matrix sync: cannot read members of %s (%s); leaving that room alone",
|
||
room, r.status_code)
|
||
continue
|
||
r2 = await client.get(
|
||
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{room}/members",
|
||
headers=H, params={"membership": "invite"})
|
||
if r2.status_code != 200:
|
||
logger.warning(
|
||
"Matrix sync: cannot read invites of %s (%s); leaving that room alone",
|
||
room, r2.status_code)
|
||
continue
|
||
joined_by_room[room] = set(r.json().get("joined", {}).keys())
|
||
invited_by_room[room] = {m["state_key"] for m in r2.json().get("chunk", [])}
|
||
# Rooms we could NOT inspect are skipped entirely — never invite into a
|
||
# room whose current membership we could not read.
|
||
readable = [room for room in rooms if room in joined_by_room]
|
||
|
||
# One identity per member: when a member sets a dashboard extra MXID,
|
||
# their auto-generated local account (@username:inference.coop) should
|
||
# NOT also sit in the Space. Track the desired set, then retire the
|
||
# local accounts of members whose dashboard MXID replaced them.
|
||
async for email, mxid in matrix_member_mxids(client):
|
||
if not mxid:
|
||
continue
|
||
missing = [
|
||
room for room in readable
|
||
if mxid not in joined_by_room[room]
|
||
and mxid not in invited_by_room.get(room, set())
|
||
]
|
||
if not missing:
|
||
continue
|
||
try:
|
||
for room in missing:
|
||
await invite(mxid, room)
|
||
except _RateLimited as exc:
|
||
rate_limited = True
|
||
logger.info(
|
||
"Matrix sync: rate limited (retry_after_ms=%s); ending this pass — "
|
||
"the next run resumes where this one stopped", exc.wait_ms)
|
||
break
|
||
invited_count += 1
|
||
if invited_count:
|
||
logger.info("Matrix space invites sent: %s", invited_count)
|
||
|
||
# Retire local accounts: for each member with a dashboard-set external
|
||
# MXID, kick their @username:inference.coop account if it's currently
|
||
# in the Space. Only ever touches accounts derived from that member's
|
||
# own Cloudron username — never other members' or federated accounts.
|
||
conn = get_db()
|
||
rows = conn.execute(
|
||
"SELECT email, matrix_mxid FROM members WHERE active = 1 AND matrix_mxid IS NOT NULL AND matrix_mxid != ''"
|
||
).fetchall()
|
||
conn.close()
|
||
users: dict[str, str] = {}
|
||
page = 1
|
||
async with httpx.AsyncClient(timeout=30) as uc:
|
||
while True:
|
||
r = await uc.get(
|
||
f"{CLOUDRON_API}/api/v1/users",
|
||
params={"page": page},
|
||
headers=cloudron_headers(),
|
||
)
|
||
if r.status_code != 200:
|
||
break
|
||
batch = r.json().get("users", [])
|
||
for u in batch:
|
||
if u.get("email") and u.get("username"):
|
||
users[u["email"].lower()] = u["username"]
|
||
if len(batch) < 25:
|
||
break
|
||
page += 1
|
||
retired = 0
|
||
for email, ext_mxid in rows:
|
||
username = users.get(email.lower())
|
||
if not username:
|
||
continue
|
||
local_mxid = f"@{username}:inference.coop"
|
||
if local_mxid == ext_mxid:
|
||
continue # dashboard MXID IS the local account — nothing to retire
|
||
# Retire BOTH joined and invite-state local accounts: a stale invite
|
||
# leaves the local account visible in the user directory and
|
||
# accept-able from a lingering client notification, which defeats
|
||
# the one-identity rule. Revoke invites via state-event membership
|
||
# (kick 403s on invite-state users).
|
||
import urllib.parse # local import: only needed on this retire path
|
||
# Same room set as the invite half (member_invite_rooms()) so the
|
||
# two cannot drift apart again: a retired identity must be removed
|
||
# from every room we invite members to, not just Space + General.
|
||
for room in readable:
|
||
enc_room = urllib.parse.quote(room, safe="")
|
||
if local_mxid in joined_by_room.get(room, set()):
|
||
await client.post(
|
||
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{enc_room}/kick",
|
||
headers=H,
|
||
json={"user_id": local_mxid,
|
||
"reason": f"This co-op account is superseded by {ext_mxid} (set on your dashboard)"},
|
||
)
|
||
retired += 1
|
||
logger.info("Matrix sync: retired local account %s (member uses %s)", local_mxid, ext_mxid)
|
||
elif local_mxid in invited_by_room.get(room, set()):
|
||
await client.put(
|
||
f"{MATRIX_HOMESERVER}/_matrix/client/v3/rooms/{enc_room}/state/m.room.member/{urllib.parse.quote(local_mxid, safe='')}",
|
||
headers=H,
|
||
json={"membership": "leave",
|
||
"reason": f"This co-op account is superseded by {ext_mxid} (set on your dashboard)"},
|
||
)
|
||
retired += 1
|
||
logger.info("Matrix sync: revoked stale invite for local account %s (member uses %s)", local_mxid, ext_mxid)
|
||
if retired:
|
||
logger.info("Matrix sync: retired %s local account(s)/invite(s)", retired)
|
||
return {"invited": invited_count, "rate_limited": rate_limited}
|
||
|
||
|
||
async def matrix_member_mxids(client: httpx.AsyncClient):
|
||
"""Yield (email, mxid) for each active member.
|
||
|
||
Local members log in via Cloudron SSO and Synapse names them by Cloudron
|
||
username → @<username>:inference.coop. Emails that don't resolve to a
|
||
Cloudron user (e.g. federated/external members) are skipped here but can
|
||
be added via the MATRIX_EXTRA_MXIDS env (comma list of full MXIDs).
|
||
"""
|
||
extra = [m.strip() for m in os.environ.get("MATRIX_EXTRA_MXIDS", "").split(",") if m.strip()]
|
||
seen: set[str] = set()
|
||
# Cloudron paginates /api/v1/users at 25/page — walk every page.
|
||
users: dict[str, str] = {}
|
||
page = 1
|
||
while True:
|
||
r = await client.get(
|
||
f"{CLOUDRON_API}/api/v1/users",
|
||
params={"page": page},
|
||
headers=cloudron_headers(),
|
||
)
|
||
if r.status_code != 200:
|
||
logger.warning("Cloudron users lookup page %s failed: %s", page, r.status_code)
|
||
break
|
||
batch = r.json().get("users", [])
|
||
for u in batch:
|
||
if u.get("email") and u.get("username"):
|
||
users[u["email"].lower()] = u["username"]
|
||
if len(batch) < 25:
|
||
break
|
||
page += 1
|
||
for email in await get_active_member_emails():
|
||
# A member's own extra MXID (set on the dashboard) takes precedence —
|
||
# that's the account they want in the Space. Otherwise use their local
|
||
# @username:inference.coop account.
|
||
own = get_member_matrix_mxid(email)
|
||
if own:
|
||
if own not in seen:
|
||
seen.add(own)
|
||
yield email, own
|
||
continue
|
||
username = users.get(email.lower())
|
||
if username:
|
||
mxid = f"@{username}:inference.coop"
|
||
seen.add(mxid)
|
||
yield email, mxid
|
||
for mxid in extra:
|
||
if mxid not in seen:
|
||
yield mxid, mxid
|
||
|
||
|
||
async def fetch_loomio_members(client: httpx.AsyncClient) -> dict[str, int]:
|
||
"""Return {email_lower: membership_id} for everyone in the Loomio group.
|
||
|
||
Uses the Server API (b3) so we get membership ids in one pass (the id is
|
||
what a targeted removal needs) AND a STABLE read: the b3 index is ordered by
|
||
id and paginates cleanly, whereas the User API b2 index has no ORDER BY, so
|
||
paging it can repeat or skip rows — a skipped row would make the pre-check
|
||
wrongly think an existing member is new. Reads the group's ACTIVE
|
||
memberships (pending invitees included, already-removed members excluded).
|
||
|
||
This is the live-membership half of the positive exclusion, and the ONLY
|
||
layer that reflects reality rather than our own records. It must succeed
|
||
before any invite or removal (fail-closed in sync_loomio_memberships).
|
||
"""
|
||
members: dict[str, int] = {}
|
||
offset = 0
|
||
while True:
|
||
r = await client.get(
|
||
f"{LOOMIO_BASE}/api/b3/memberships",
|
||
headers={"Authorization": f"Bearer {LOOMIO_B3_KEY}"},
|
||
params={"group_id": int(LOOMIO_GROUP_ID), "limit": 50, "offset": offset},
|
||
)
|
||
r.raise_for_status()
|
||
page = r.json().get("memberships", [])
|
||
for m in page:
|
||
email = (m.get("user_email") or "").lower()
|
||
if email:
|
||
members[email] = m.get("id")
|
||
if len(page) < 50:
|
||
break
|
||
offset += 50
|
||
return members
|
||
|
||
|
||
async def remove_loomio_membership(client: httpx.AsyncClient, membership_id: int) -> bool:
|
||
"""Revoke ONE Loomio membership by id, via the Server API (b3).
|
||
|
||
Targeted on purpose: the request names only the membership being removed, so
|
||
it carries no address of anyone who is staying and therefore cannot trigger
|
||
an invite or a group notification — unlike b2's `remove_absent`, which
|
||
requires re-sending the whole keep-list.
|
||
|
||
Idempotent: Loomio answers 404 for an id that is already gone (or never
|
||
existed); the end state we want is "not a member", so 404 counts as success
|
||
and a repeated run does not error or churn.
|
||
"""
|
||
r = await client.delete(
|
||
f"{LOOMIO_BASE}/api/b3/memberships/{membership_id}",
|
||
headers={"Authorization": f"Bearer {LOOMIO_B3_KEY}"},
|
||
)
|
||
if r.status_code in (200, 204, 404):
|
||
return True
|
||
logger.warning(
|
||
"Loomio remove of membership %s failed: %s %s",
|
||
membership_id, r.status_code, r.text[:200],
|
||
)
|
||
return False
|
||
|
||
|
||
async def sync_loomio_memberships() -> None:
|
||
"""Reconcile the Loomio group with the current ACTIVE-member list.
|
||
|
||
TWO jobs with DIFFERENT guards — a deactivation must still propagate:
|
||
|
||
1. ADD (invite) — a member is invited only if they are BOTH not already in
|
||
the group (live pre-check) AND never handed to Loomio before
|
||
(members.loomio_invited_at). Exactly once, ever. This is the dedupe
|
||
guard, and it applies to INVITATIONS ONLY.
|
||
2. REMOVE — a member no longer active is revoked so a lapse propagates to
|
||
Loomio. The removal path is targeted (b3 DELETE by membership id) and
|
||
sends nothing for anyone who is staying, so it cannot re-invite or
|
||
notify a present member. Operator accounts (LOOMIO_ALWAYS_KEEP) are
|
||
never removed.
|
||
|
||
Guards (incident 2026-11-04: a live run emailed Loomio invites to the whole
|
||
membership):
|
||
* The invite payload contains ONLY addresses Loomio has never seen and we
|
||
have never handed over — an existing member is skipped SILENTLY, no API
|
||
call about them at all.
|
||
* FAIL CLOSED — if the live group roster cannot be read, NEITHER phase
|
||
runs. We never hand Loomio a list we could not check.
|
||
|
||
Loomio's b2 invite has no "silent" flag, so the only way to guarantee no
|
||
invite is to send an address exactly once, ever — hence the record.
|
||
"""
|
||
if not LOOMIO_SYNC_ENABLED:
|
||
logger.info("Loomio sync disabled (LOOMIO_SYNC_ENABLED=0); skipping")
|
||
return
|
||
if not LOOMIO_GROUP_ID:
|
||
logger.warning("Loomio not configured; skipping membership sync")
|
||
return
|
||
|
||
# The set that decides who STAYS in the group: every active member plus the
|
||
# operator allowlist. Computed from the UNFILTERED active list on purpose —
|
||
# an active member whose Cloudron account is not yet activated is still a
|
||
# member and must NOT be removed from Loomio (only their invite is deferred).
|
||
keep_emails = {e.lower() for e in await get_active_member_emails()}
|
||
keep_emails |= {e.lower() for e in LOOMIO_ALWAYS_KEEP}
|
||
|
||
# Invite candidates are additionally filtered to activated accounts so a
|
||
# Loomio invite email never precedes the member's own account setup.
|
||
invite_emails = await get_active_member_emails()
|
||
for keep in LOOMIO_ALWAYS_KEEP:
|
||
if keep not in invite_emails:
|
||
invite_emails.append(keep)
|
||
invite_emails = await filter_loomio_activated(invite_emails)
|
||
|
||
if not LOOMIO_B3_KEY:
|
||
# The live roster (and the removal path) needs the Server API key. Without
|
||
# it we cannot verify reality, so we do nothing rather than risk a
|
||
# blind invite.
|
||
logger.warning("Loomio sync: LOOMIO_B3_KEY unset; cannot read group roster (fail-closed)")
|
||
return
|
||
|
||
async with httpx.AsyncClient() as client:
|
||
# FAIL CLOSED: never act on a roster we could not read.
|
||
try:
|
||
current = await fetch_loomio_members(client)
|
||
except Exception as e:
|
||
logger.warning(
|
||
"Loomio sync: could not read current group members; skipping "
|
||
"(fail-closed): %s", e,
|
||
)
|
||
return
|
||
|
||
# --- HEAL: reconcile our own dedupe record with the live roster. ---
|
||
# A member sitting in the group with NO loomio_invited_at stamp (added by
|
||
# an earlier sync generation, or by hand) reads as "never invited" to the
|
||
# guard below. They are not: they are already in. Stamp them from the
|
||
# roster so the record reflects reality — otherwise the guard is only as
|
||
# good as the rows it happens to have.
|
||
healed = 0
|
||
for email in current:
|
||
if not get_loomio_invited_at(email):
|
||
mark_loomio_invited(email)
|
||
healed += 1
|
||
if healed:
|
||
logger.info("Loomio sync: stamped %d already-present member(s) as invited", healed)
|
||
|
||
# --- REMOVE: reconcile the group to the current active set. ---
|
||
removed = 0
|
||
for email, mid in current.items():
|
||
if email in keep_emails or mid is None:
|
||
continue
|
||
if await remove_loomio_membership(client, mid):
|
||
removed += 1
|
||
logger.info("Loomio sync: removed lapsed membership %s", mid)
|
||
if removed:
|
||
logger.info("Loomio sync: %d lapsed membership(s) removed", removed)
|
||
|
||
# --- ADD: only addresses Loomio has never seen AND we never handed over. ---
|
||
fresh = [
|
||
e for e in invite_emails
|
||
if e.lower() not in current and not get_loomio_invited_at(e)
|
||
]
|
||
if not fresh:
|
||
logger.info(
|
||
"Loomio sync: nobody new (all already in Loomio or already "
|
||
"invited); nothing to invite"
|
||
)
|
||
elif not LOOMIO_API_KEY:
|
||
logger.warning(
|
||
"Loomio sync: %d member(s) to invite but LOOMIO_API_KEY unset; "
|
||
"skipping invites", len(fresh),
|
||
)
|
||
else:
|
||
r = await client.post(
|
||
f"{LOOMIO_BASE}/api/b2/memberships",
|
||
headers={"Authorization": f"Bearer {LOOMIO_API_KEY}"},
|
||
json={"group_id": int(LOOMIO_GROUP_ID), "emails": fresh},
|
||
)
|
||
if r.status_code != 200:
|
||
logger.warning("Loomio membership sync failed: %s %s", r.status_code, r.text[:200])
|
||
else:
|
||
added = r.json().get("added_emails", []) or []
|
||
# Record EVERY address we sent, so it is never sent again — even
|
||
# if Loomio didn't add it, or the membership is later revoked. A
|
||
# revoked member is NOT a never-invited member, and must not be
|
||
# re-invited.
|
||
for e in fresh:
|
||
mark_loomio_invited(e)
|
||
logger.info("Loomio sync: first-time invites sent (%d): %s", len(added), added)
|
||
|
||
# Fix auto-generated usernames (email+randomsuffix) → Cloudron username.
|
||
# Best-effort: no B3 key or an API failure just logs and moves on.
|
||
if LOOMIO_B3_KEY:
|
||
try:
|
||
await loomio_fix_usernames(client)
|
||
except Exception as e:
|
||
logger.warning("Loomio username fix pass failed: %s", e)
|
||
|
||
|
||
async def filter_loomio_activated(emails: list[str]) -> list[str]:
|
||
"""Return the subset of `emails` whose Cloudron account is set up, plus any
|
||
operator accounts (LOOMIO_ALWAYS_KEEP), which must always pass through.
|
||
|
||
Fetches the Cloudron user list once and builds an inviteAccepted set keyed
|
||
by email. Members who are provisioned but not yet activated are dropped so
|
||
Loomio doesn't email them an invite before they've set up their account.
|
||
"""
|
||
activated: set[str] = set()
|
||
try:
|
||
async with httpx.AsyncClient(timeout=15.0) as client:
|
||
users = await cloudron_fetch_all_users(client)
|
||
if users is None:
|
||
raise RuntimeError("cloudron users fetch failed (page 1)")
|
||
for u in users:
|
||
if u.get("inviteAccepted") and u.get("email"):
|
||
activated.add(u["email"])
|
||
except Exception as e:
|
||
# If we can't determine activation state, fail open (invite everyone)
|
||
# rather than silently dropping members from Loomio.
|
||
logger.warning("filter_loomio_activated: cloudron users fetch failed: %s", e)
|
||
return emails
|
||
|
||
keep = set(LOOMIO_ALWAYS_KEEP)
|
||
return [e for e in emails if e in keep or e in activated]
|
||
|
||
|
||
def store_member(email: str, key_token: str, cloudron_user_id: str, slug: str = "",
|
||
account_type: str | None = None) -> None:
|
||
conn = get_db()
|
||
# Preserve an existing slug if the caller passed none — re-provisioning an
|
||
# already-provisioned member must not wipe their OC slug (the deactivation
|
||
# webhook maps slugs back to members).
|
||
if not slug:
|
||
row = conn.execute("SELECT slug FROM members WHERE email = ?", (email,)).fetchone()
|
||
if row and row[0]:
|
||
slug = row[0]
|
||
# account_type: None (every normal path) preserves whatever the row holds —
|
||
# re-provisioning a guest via the member pipeline must not un-guest it. Only
|
||
# an explicit value (provision_guest_member passes 'guest') writes the column.
|
||
conn.execute(
|
||
"INSERT INTO members (email, key_token, cloudron_user_id, slug, active, account_type) "
|
||
"VALUES (?, ?, ?, ?, 1, COALESCE(?, 'member')) "
|
||
"ON CONFLICT(email) DO UPDATE SET key_token=excluded.key_token, "
|
||
"cloudron_user_id=excluded.cloudron_user_id, slug=excluded.slug, active=1, "
|
||
"account_type=CASE WHEN excluded.account_type='guest' THEN 'guest' "
|
||
"ELSE COALESCE(members.account_type, 'member') END",
|
||
(email, encrypt_key(key_token), cloudron_user_id, slug, account_type),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def get_member_key(email: str) -> str | None:
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT key_token FROM members WHERE email = ? AND active = 1", (email,)
|
||
).fetchone()
|
||
conn.close()
|
||
return decrypt_key(row[0]) if row else None
|
||
|
||
|
||
def get_stored_key(email: str) -> str | None:
|
||
"""Return the stored LiteLLM key token regardless of active status.
|
||
|
||
Unlike get_member_key (which filters active=1), this returns the key even
|
||
for a lapsed member being reactivated, so provisioning can reuse it rather
|
||
than mint a duplicate alias (LiteLLM rejects a duplicate `member:<email>`
|
||
alias with 400).
|
||
"""
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT key_token FROM members WHERE email = ?", (email,)
|
||
).fetchone()
|
||
conn.close()
|
||
return (decrypt_key(row[0]) if row and row[0] else None)
|
||
|
||
|
||
def touch_welcome_sent(email: str) -> None:
|
||
"""Record that a welcome/invite email was just sent for this member."""
|
||
conn = get_db()
|
||
conn.execute(
|
||
"UPDATE members SET welcome_sent_at = datetime('now') WHERE email = ?",
|
||
(email,),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def get_member_by_slug(slug: str) -> dict | None:
|
||
"""Look up a member (email, key_token, cloudron_user_id) by their OC slug."""
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT email, key_token, cloudron_user_id FROM members WHERE slug = ?", (slug,)
|
||
).fetchone()
|
||
conn.close()
|
||
if not row:
|
||
return None
|
||
return {"email": row[0], "key_token": decrypt_key(row[1]), "cloudron_user_id": row[2]}
|
||
|
||
|
||
def deactivate_member(email: str) -> str | None:
|
||
"""Mark member inactive and return their DECRYPTED key token (for deletion)."""
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT key_token FROM members WHERE email = ?", (email,)
|
||
).fetchone()
|
||
conn.execute("UPDATE members SET active = 0 WHERE email = ?", (email,))
|
||
conn.commit()
|
||
conn.close()
|
||
return decrypt_key(row[0]) if row else None
|
||
|
||
|
||
# --- Helpers ---
|
||
|
||
def cloudron_headers() -> dict:
|
||
return {"Authorization": f"Bearer {CLOUDRON_TOKEN}"}
|
||
|
||
|
||
# M6 (audit 2026-11-05): a single-page `?per_page=100` fetch silently truncates
|
||
# once the box passes 100 users (the membership sits at the threshold). Both
|
||
# walkers below page through /api/v1/users like the Matrix paths already do,
|
||
# capped sanely for Cloudron (50 pages x 100 = 5000 users).
|
||
_CLOUDRON_USERS_PAGE_CAP = 50
|
||
|
||
|
||
async def cloudron_fetch_all_users(client: httpx.AsyncClient) -> list[dict] | None:
|
||
"""Every Cloudron user, walking /api/v1/users page by page (async callers).
|
||
|
||
Returns None only when page 1 is unresolvable (caller keeps its existing
|
||
failure behavior); a failure on a LATER page keeps what was read, which is
|
||
strictly more than the truncated single-page fetch returned before.
|
||
"""
|
||
users: list[dict] = []
|
||
for page in range(1, _CLOUDRON_USERS_PAGE_CAP + 1):
|
||
r = await client.get(
|
||
f"{CLOUDRON_API}/api/v1/users",
|
||
params={"page": page, "per_page": 100},
|
||
headers=cloudron_headers(),
|
||
)
|
||
if r.status_code != 200:
|
||
if page == 1:
|
||
logger.warning("Cloudron users fetch page 1 failed: %s", r.status_code)
|
||
return None
|
||
logger.warning("Cloudron users fetch page %s failed: %s", page, r.status_code)
|
||
break
|
||
batch = r.json().get("users", [])
|
||
users.extend(batch)
|
||
if len(batch) < 100:
|
||
break
|
||
else:
|
||
logger.warning("Cloudron users fetch hit the %s-page cap; list truncated",
|
||
_CLOUDRON_USERS_PAGE_CAP)
|
||
return users
|
||
|
||
|
||
def cloudron_fetch_all_users_sync() -> list[dict]:
|
||
"""Every Cloudron user for the sync (urllib) call sites, same pagination.
|
||
|
||
Raises on a page-1 failure (the call sites error the request); later-page
|
||
failures keep what was read.
|
||
"""
|
||
import urllib.request
|
||
|
||
users: list[dict] = []
|
||
for page in range(1, _CLOUDRON_USERS_PAGE_CAP + 1):
|
||
req = urllib.request.Request(
|
||
f"{CLOUDRON_API}/api/v1/users?page={page}&per_page=100",
|
||
headers=cloudron_headers(),
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=15) as r:
|
||
batch = json.loads(r.read().decode()).get("users", [])
|
||
except Exception:
|
||
if page == 1:
|
||
raise
|
||
logger.warning("Cloudron users fetch page %s failed", page, exc_info=True)
|
||
break
|
||
users.extend(batch)
|
||
if len(batch) < 100:
|
||
break
|
||
else:
|
||
logger.warning("Cloudron users fetch hit the %s-page cap; list truncated",
|
||
_CLOUDRON_USERS_PAGE_CAP)
|
||
return users
|
||
|
||
|
||
async def cloudron_create_user(email: str, name: str) -> str:
|
||
"""Create (or return existing) Cloudron user, assigned to the members group.
|
||
|
||
Verified user shape (from live API):
|
||
{id, username, email, fallbackEmail, displayName, role, active, groupIds}
|
||
Roles: "owner", "admin", "user".
|
||
Group assignment is a SEPARATE call: PUT /api/v1/users/:userId/groups
|
||
with body {"groupIds": [...]}.
|
||
"""
|
||
async with httpx.AsyncClient() as client:
|
||
# Check if user exists (walks every page — a single-page fetch
|
||
# truncated past 100 users and let duplicate users get created).
|
||
users = await cloudron_fetch_all_users(client)
|
||
if users is None:
|
||
raise RuntimeError("cloudron users fetch failed; refusing to create a duplicate")
|
||
for u in users:
|
||
if u.get("email") == email:
|
||
# If the existing user has no display name (e.g. created with a
|
||
# placeholder), fill it in. Otherwise leave it alone — the user
|
||
# may have set their own name during account setup.
|
||
if not u.get("displayName"):
|
||
await cloudron_update_profile(u["id"], email, name)
|
||
return u["id"]
|
||
|
||
# Create user (role "user" = regular member). Only email is required;
|
||
# Cloudron lets the user choose their own username during account setup.
|
||
r = await client.post(
|
||
f"{CLOUDRON_API}/api/v1/users",
|
||
headers=cloudron_headers(),
|
||
json={
|
||
"email": email,
|
||
"displayName": name,
|
||
"role": "user",
|
||
"active": True,
|
||
},
|
||
)
|
||
r.raise_for_status()
|
||
return r.json()["id"]
|
||
|
||
|
||
async def cloudron_set_group(user_id: str, group_id: str | None = None) -> None:
|
||
"""Assign a user to a group (defaults to the members group)."""
|
||
gid = group_id or MEMBERS_GROUP_ID
|
||
if not gid:
|
||
logger.warning("No group id; skipping group assignment")
|
||
return
|
||
async with httpx.AsyncClient() as client:
|
||
r = await client.put(
|
||
f"{CLOUDRON_API}/api/v1/users/{user_id}/groups",
|
||
headers=cloudron_headers(),
|
||
json={"groupIds": [gid]},
|
||
)
|
||
r.raise_for_status()
|
||
|
||
|
||
async def cloudron_set_active(user_id: str, active: bool) -> None:
|
||
async with httpx.AsyncClient() as client:
|
||
r = await client.put(
|
||
f"{CLOUDRON_API}/api/v1/users/{user_id}/active",
|
||
headers=cloudron_headers(),
|
||
json={"active": active},
|
||
)
|
||
r.raise_for_status()
|
||
|
||
|
||
# Open WebUI was RETIRED 2026-11-13: the app is stopped (com.openwebui.cloudronapp,
|
||
# installed|stopped) and its "members group" lived inside Open WebUI's own
|
||
# database, never in Cloudron. A stopped container has no DNS entry, so every
|
||
# call failed with 'Temporary failure in name resolution' and no new member
|
||
# ever reached the group. Nothing here may call out to it.
|
||
|
||
async def cloudron_update_profile(user_id: str, email: str, display_name: str, fallback_email: str = "") -> None:
|
||
"""Update a user's profile (display name, email) after creation.
|
||
|
||
Uses POST /api/v1/users/:userId/profile (the Cloudron "Update user" route).
|
||
Needed because cloudron_create_user() only sets displayName at creation and
|
||
returns early if the user already exists — so corrections require this.
|
||
"""
|
||
async with httpx.AsyncClient() as client:
|
||
r = await client.post(
|
||
f"{CLOUDRON_API}/api/v1/users/{user_id}/profile",
|
||
headers=cloudron_headers(),
|
||
json={"email": email, "displayName": display_name, "fallbackEmail": fallback_email},
|
||
)
|
||
r.raise_for_status()
|
||
|
||
|
||
async def cloudron_get_invite_link(user_id: str) -> str:
|
||
"""Return the account-setup link WITHOUT sending Cloudron's own email.
|
||
|
||
We use this so we can send our own branded email (with the setup link
|
||
embedded) instead of Cloudron's default invite email.
|
||
"""
|
||
async with httpx.AsyncClient() as client:
|
||
r = await client.get(
|
||
f"{CLOUDRON_API}/api/v1/users/{user_id}/invite_link",
|
||
headers=cloudron_headers(),
|
||
)
|
||
r.raise_for_status()
|
||
return r.json().get("inviteLink", "")
|
||
|
||
|
||
async def cloudron_is_activated(user_id: str) -> bool:
|
||
"""Return True if the user has completed account setup (inviteAccepted).
|
||
|
||
`inviteAccepted` flips true once the member clicks their setup link and
|
||
chooses a username/password. Until then they're provisioned but not able to
|
||
log in — the state we re-invite for.
|
||
"""
|
||
async with httpx.AsyncClient() as client:
|
||
users = await cloudron_fetch_all_users(client)
|
||
if users is None:
|
||
raise RuntimeError("cloudron users fetch failed")
|
||
for u in users:
|
||
if u.get("id") == user_id:
|
||
return bool(u.get("inviteAccepted"))
|
||
return False
|
||
|
||
|
||
async def provision_member(email: str, name: str, slug: str = "") -> str:
|
||
"""Provision a member end-to-end and send our own welcome email.
|
||
|
||
Creates the Cloudron user (members group), issues a LiteLLM key under the
|
||
member's team (budget = the member's current balance, not a hardcoded $15),
|
||
stores the mapping, and sends our branded welcome email containing the
|
||
Cloudron account-setup link (instead of Cloudron's default invite email).
|
||
|
||
Idempotent: if the member already has a stored LiteLLM key, reuse it rather
|
||
than minting a new one (LiteLLM rejects a duplicate `member:<email>` alias
|
||
with 400 — the bug that previously made re-provisioning of existing members
|
||
fail). Re-sending this way also re-sends the welcome email, so it doubles
|
||
as the manual "re-invite" path.
|
||
|
||
GUEST accounts: every provisioning entry point (webhook, sweep,
|
||
/admin/provision) funnels through here. When a guest's OC identity appears
|
||
as a live backer, this call CONVERTS the account instead of provisioning —
|
||
the automatic guest→member transition.
|
||
"""
|
||
if is_guest_account(email):
|
||
return await convert_guest_to_member(email, name, slug)
|
||
user_id = await cloudron_create_user(email, name)
|
||
await cloudron_set_group(user_id)
|
||
await cloudron_set_active(user_id, True)
|
||
balance = get_member_balance(email)
|
||
team_id = await litellm_get_or_create_team(email, balance)
|
||
|
||
# Reuse an existing key if we already have one (the sk- value is stored in
|
||
# our DB at creation time and is the authoritative token for this member's
|
||
# chat key). Only mint a fresh key when there is none.
|
||
key_token = get_stored_key(email)
|
||
if not key_token:
|
||
key_token = await litellm_create_key(email, balance, team_id)
|
||
store_member(email, key_token, user_id, slug)
|
||
|
||
# Send our own welcome email with the account-setup link (no OAuth step).
|
||
setup_url = await cloudron_get_invite_link(user_id)
|
||
subject, text, html = welcome_email(name, setup_url)
|
||
send_email(email, subject, text, html)
|
||
touch_welcome_sent(email)
|
||
|
||
# Subscribe to the member newsletter (single opt-in; members consented by joining).
|
||
await listmonk_sync(email, subscribe=True)
|
||
|
||
# Auto-grant any HELD credit packs for this member (they pre-purchased
|
||
# credits before joining — apply them now as part of provisioning).
|
||
granted = 0.0
|
||
try:
|
||
conn = get_db()
|
||
granted = grant_held_credits(conn, email, slug)
|
||
conn.close()
|
||
if granted > 0:
|
||
await sync_team_budget_from_balance(email)
|
||
logger.info("Granted %.2f held credits to new member %s (balance %s)",
|
||
granted, email, get_credit_balance(email))
|
||
except Exception as e:
|
||
logger.warning("Held-credit grant for %s failed: %s", email, e)
|
||
|
||
await sync_loomio_memberships()
|
||
await sync_matrix_space()
|
||
logger.info("Provisioned member %s (user_id=%s)", email, user_id)
|
||
return user_id
|
||
|
||
|
||
# --- Guest accounts (operator feature, 2026-11-08) --------------------------- #
|
||
# A member gives a friend access with $2 of credits: Cloudron user in the
|
||
# guests group (chat + dashboard only), a one-time LiteLLM budget with NO
|
||
# monthly reset, the portal row marked account_type='guest', the guest welcome
|
||
# email, and the newsletter subscription. NO Loomio/Matrix invites. See the
|
||
# constants block for the lapse rules (credit exhaustion + GUEST_DAYS time box).
|
||
|
||
def guest_invitation_email(name: str, setup_url: str, inviter_name: str = "") -> tuple[str, str, str]:
|
||
"""Build the guest INVITATION email — the one email a guest sends.
|
||
|
||
This is the guest-time email AND the guest welcome in one: the recipient is
|
||
invited in (the inviter named), told what the guest gives ($2 of credits,
|
||
how long access runs), and given the account-setup link. The guest welcome
|
||
content lives here rather than in a second email — one coherent invitation,
|
||
not two mails at once. Same branded shell as welcome_email(): cream
|
||
background, white card, forest-green button, pine mark, no external
|
||
assets. No Loomio/Matrix — guests receive no member invitations.
|
||
"""
|
||
subject = "You've been invited to the Inference Cooperative"
|
||
# Neutral greeting: no name is guessed from the email — the guest chooses
|
||
# their own display name during Cloudron account setup.
|
||
if name:
|
||
greet = f"Hi {name},"
|
||
h1 = f"You're invited, {name}"
|
||
else:
|
||
greet = "Hi there,"
|
||
h1 = "You're invited"
|
||
if inviter_name:
|
||
invited_line = f"{inviter_name} has invited you to the Inference Cooperative"
|
||
html_invited = f"<strong style=\"color:#2d3327;\">{inviter_name}</strong> has invited you to the <strong style=\"color:#2d3327;\">Inference Cooperative</strong>"
|
||
else:
|
||
invited_line = "A member has invited you to the Inference Cooperative"
|
||
html_invited = "A member has invited you to the <strong style=\"color:#2d3327;\">Inference Cooperative</strong>"
|
||
text = (
|
||
f"{greet}\n\n"
|
||
f"{invited_line}.\n\n"
|
||
"It's a member-governed AI cooperative that runs open models.\n\n"
|
||
"Your invitation comes with $2 of credits for the co-op's chat and\n"
|
||
f"API. Access runs for {int(GUEST_DAYS)} days from today; the credits\n"
|
||
"themselves don't expire early.\n\n"
|
||
"To accept, create your account — it takes about a minute:\n\n"
|
||
f"{setup_url}\n\n"
|
||
"If you like it, become a member: pay on a sliding scale, and\n"
|
||
"receive a $15/month allowance, plus optional credit packs — plus a\n"
|
||
"say in how it's run.\n\n"
|
||
f"{BECOME_MEMBER_URL}\n\n"
|
||
"You're also on our low-volume newsletter — unsubscribe anytime\n"
|
||
"from any issue.\n\n"
|
||
"Questions? Just reply to this email — it reaches the co-op directly.\n\n"
|
||
"— The Inference Cooperative\n"
|
||
)
|
||
html = f"""<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"></head>
|
||
<body style="margin:0;padding:0;background:#faf8f5;font-family:-apple-system,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;">
|
||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:#faf8f5;padding:40px 16px;">
|
||
<tr>
|
||
<td align="center">
|
||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="max-width:520px;">
|
||
<tr>
|
||
<td align="center" style="padding-bottom:24px;">
|
||
{PINE_LOGO_SVG}
|
||
</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="center" style="background:#ffffff;border:1px solid #e8e2d8;border-radius:16px;padding:40px 32px;">
|
||
<h1 style="margin:0 0 12px;font-size:24px;font-weight:700;color:#2d3327;">{h1}</h1>
|
||
<p style="margin:0 0 8px;font-size:16px;color:#6b7a62;line-height:1.5;">{html_invited} — a member-governed AI cooperative running open models.</p>
|
||
<p style="margin:0 0 24px;font-size:15px;color:#6b7a62;line-height:1.5;">Your invitation comes with <strong style="color:#2d3327;">$2 of credits</strong> for the co-op's chat and API. Access runs for {int(GUEST_DAYS)} days from today; the credits themselves don't expire early.</p>
|
||
<a href="{setup_url}" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:10px;font-size:16px;font-weight:600;">Create your account</a>
|
||
<p style="margin:14px 0 28px;font-size:13px;color:#94a08c;line-height:1.5;">This takes about a minute.</p>
|
||
<p style="margin:0 0 10px;font-size:15px;color:#2d3327;line-height:1.5;"><strong>If you like it, become a member.</strong></p>
|
||
<p style="margin:0 0 16px;font-size:14px;color:#6b7a62;line-height:1.5;">Pay on a sliding scale, and receive a $15/month allowance, plus optional credit packs — plus a say in how it's run.</p>
|
||
<a href="{BECOME_MEMBER_URL}" style="display:inline-block;background:#eaf3ea;color:#2f5c30;text-decoration:none;padding:12px 30px;border-radius:10px;font-size:15px;font-weight:600;border:1px solid #cfe0cf;">Become a member</a>
|
||
<p style="margin:24px 0 0;font-size:13px;color:#94a08c;line-height:1.5;">You're also on our low-volume newsletter — unsubscribe anytime from any issue.</p>
|
||
</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="center" style="padding-top:24px;font-size:13px;color:#94a08c;line-height:1.6;">
|
||
Questions? Just reply to this email — it reaches the co-op directly, or write to <a href="mailto:info@inference.coop" style="color:#6b7a62;">info@inference.coop</a>.<br>
|
||
<span style="color:#2d3327;">The Inference Cooperative</span> · inference.coop<br>
|
||
<span style="font-size:12px;">By creating your account, you agree to our <a href="https://git.inference.coop/co-op/docs/src/branch/main/terms-of-service.md" style="color:#6b7a62;">Terms of Service</a> and <a href="https://git.inference.coop/co-op/docs/src/branch/main/privacy-policy.md" style="color:#6b7a62;">Privacy Policy</a>.</span>
|
||
</td>
|
||
</tr>
|
||
</table>
|
||
</td>
|
||
</tr>
|
||
</table>
|
||
</body>
|
||
</html>"""
|
||
return subject, text, html
|
||
|
||
|
||
def guest_member_congrats_email(name: str, setup_url: str | None) -> tuple[str, str, str]:
|
||
"""Short congratulations email for a guest account converting to a member.
|
||
|
||
Their account, login and any unspent guest credit carry over unchanged;
|
||
they gain the monthly allowance and the member services (Loomio, Matrix
|
||
invites follow from the normal syncs).
|
||
"""
|
||
subject = "Welcome to the Inference Cooperative — you're a member now!"
|
||
setup_line = ""
|
||
if setup_url:
|
||
setup_line = (f"Your account isn't set up yet — one minute and you're in:\n\n"
|
||
f"{setup_url}\n\n")
|
||
text = (
|
||
f"Hi {name},\n\n"
|
||
"Great news: your guest account is now a full membership. Nothing about\n"
|
||
"your login, dashboard or API keys changes — and any guest credit you\n"
|
||
"hadn't spent yet stays on your account on top of your allowance.\n\n"
|
||
"As a member you now get:\n"
|
||
" • a $15 monthly allowance (renews on your join day)\n"
|
||
" • our co-op forum (Loomio) and chat space (Matrix) — invites on the way\n"
|
||
" • a say in how the co-op is run\n\n"
|
||
f"{setup_line}"
|
||
f"Your dashboard: https://dashboard.inference.coop\n\n"
|
||
"Questions? Reply to this email — it reaches the co-op directly.\n\n"
|
||
"— The Inference Cooperative\n"
|
||
)
|
||
html = f"""<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"></head>
|
||
<body style="margin:0;padding:0;background:#faf8f5;font-family:-apple-system,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;">
|
||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:#faf8f5;padding:40px 16px;">
|
||
<tr><td align="center">
|
||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="max-width:520px;">
|
||
<tr><td align="center" style="background:#ffffff;border:1px solid #e8e2d8;border-radius:16px;padding:40px 32px;">
|
||
<h1 style="margin:0 0 12px;font-size:24px;font-weight:700;color:#2d3327;">You're a member now, {name}</h1>
|
||
<p style="margin:0 0 16px;font-size:16px;color:#6b7a62;line-height:1.5;">Your guest account has become a full membership. Nothing about your login, dashboard or API keys changes — <strong style="color:#2d3327;">any unspent guest credit stays on your account</strong>, on top of your new $15 monthly allowance.</p>
|
||
<p style="margin:0 0 24px;font-size:15px;color:#6b7a62;line-height:1.5;">Invites to the co-op forum (Loomio) and chat space (Matrix) are on their way.</p>
|
||
{'<a href="' + setup_url + '" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:10px;font-size:16px;font-weight:600;">Finish account setup</a>' if setup_url else '<a href="https://dashboard.inference.coop" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:10px;font-size:16px;font-weight:600;">Open your dashboard</a>'}
|
||
<p style="margin:24px 0 0;font-size:13px;color:#94a08c;line-height:1.5;">Questions? Reply to this email — it reaches the co-op directly.</p>
|
||
</td></tr>
|
||
<tr><td align="center" style="padding-top:24px;font-size:13px;color:#94a08c;line-height:1.6;">
|
||
<span style="color:#2d3327;">The Inference Cooperative</span> · inference.coop
|
||
</td></tr>
|
||
</table>
|
||
</td></tr>
|
||
</table>
|
||
</body>
|
||
</html>"""
|
||
return subject, text, html
|
||
|
||
|
||
def guest_limits_email(name: str, reason: str) -> tuple[str, str, str]:
|
||
"""Build the reaching-limits email for a guest account hitting its limit.
|
||
|
||
Sent ONCE per guest when the sweep deactivates the guest — either the $2
|
||
of credits are spent or the GUEST_DAYS time box has run out. Says plainly:
|
||
the guest access has ended, the account and data are kept, and becoming a
|
||
member ($15/month via Open Collective) restores access with a full
|
||
monthly allowance and the community spaces. Same shell as the other guest
|
||
emails; matter-of-fact and welcoming — no guilt, no upsell.
|
||
"""
|
||
subject = "Your guest access has ended — become a member to continue"
|
||
if reason == "depleted":
|
||
ended = f"Your ${GUEST_BUDGET:g} of guest credits are spent."
|
||
else:
|
||
ended = f"The guest's {int(GUEST_DAYS)} days are up."
|
||
text = (
|
||
f"Hi {name},\n\n"
|
||
f"{ended} Your guest access to the Inference Cooperative has ended.\n\n"
|
||
"Your account and data are kept. If you'd like to continue, become a\n"
|
||
"member: $15/month through Open Collective restores full access, with\n"
|
||
"a full monthly allowance and the co-op's community spaces.\n\n"
|
||
f"{BECOME_MEMBER_URL}\n\n"
|
||
"Questions? Just reply to this email — it reaches the co-op directly.\n\n"
|
||
"— The Inference Cooperative\n"
|
||
)
|
||
html = f"""<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"></head>
|
||
<body style="margin:0;padding:0;background:#faf8f5;font-family:-apple-system,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;">
|
||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:#faf8f5;padding:40px 16px;">
|
||
<tr><td align="center">
|
||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="max-width:520px;">
|
||
<tr><td align="center" style="padding-bottom:24px;">
|
||
{PINE_LOGO_SVG}
|
||
</td></tr>
|
||
<tr><td align="center" style="background:#ffffff;border:1px solid #e8e2d8;border-radius:16px;padding:40px 32px;">
|
||
<h1 style="margin:0 0 12px;font-size:24px;font-weight:700;color:#2d3327;">Your guest access has ended</h1>
|
||
<p style="margin:0 0 16px;font-size:16px;color:#6b7a62;line-height:1.5;">{ended} Your guest access to the <strong style="color:#2d3327;">Inference Cooperative</strong> has ended.</p>
|
||
<p style="margin:0 0 24px;font-size:15px;color:#6b7a62;line-height:1.5;"><strong style="color:#2d3327;">Your account and data are kept.</strong> If you'd like to continue, become a member — $15/month through Open Collective restores full access, with a full monthly allowance and the co-op's community spaces.</p>
|
||
<a href="{BECOME_MEMBER_URL}" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:10px;font-size:16px;font-weight:600;">Become a member</a>
|
||
<p style="margin:24px 0 0;font-size:13px;color:#94a08c;line-height:1.5;">Questions? Reply to this email — it reaches the co-op directly.</p>
|
||
</td></tr>
|
||
<tr><td align="center" style="padding-top:24px;font-size:13px;color:#94a08c;line-height:1.6;">
|
||
<span style="color:#2d3327;">The Inference Cooperative</span> · inference.coop
|
||
</td></tr>
|
||
</table>
|
||
</td></tr>
|
||
</table>
|
||
</body>
|
||
</html>"""
|
||
return subject, text, html
|
||
|
||
|
||
async def provision_guest_member(
|
||
email: str,
|
||
name: str,
|
||
inviter_email: str,
|
||
admin: bool = False,
|
||
inviter_display: str | None = None,
|
||
) -> str:
|
||
"""Provision a GUEST account end-to-end (operator guest access feature).
|
||
|
||
A guest gives a friend $2 of credits with no Open Collective membership:
|
||
* Cloudron user in the GUESTS group (chat + dashboard only — the two
|
||
apps' access restrictions list both groups);
|
||
* a LiteLLM team hard-capped at GUEST_BUDGET with NO monthly reset
|
||
(budget_duration None — the cap is the guest's whole life);
|
||
* the portal row marked account_type='guest' with the $2 held as
|
||
non-expiring credit (the credits model) and guest_expires_at set;
|
||
* the guest INVITATION email (inviter named, the $2 credits, the
|
||
setup link — the invitation and the guest welcome in one) and the
|
||
newsletter subscription;
|
||
* a guests-ledger row recording inviter → recipient;
|
||
* NO Loomio/Matrix invites, no docs-RAG members group.
|
||
|
||
Reuses provision_member's internals (user create, team, key, store) rather
|
||
than duplicating them. Guards (guest_provision_guard) enforce one guest per
|
||
inviter per 30 days and one guest per recipient ever — except on the
|
||
admin path (admin=True), where the inviter is the co-op's admin account
|
||
and the member's monthly cap does not apply.
|
||
|
||
inviter_display overrides the invitation email's inviter line (the admin
|
||
path passes a co-op label instead of resolving the inviter's member slug).
|
||
"""
|
||
conn = get_db()
|
||
refusal = guest_provision_guard(conn, inviter_email, email, admin=admin)
|
||
conn.close()
|
||
if refusal:
|
||
raise HTTPException(429, refusal)
|
||
if not GUESTS_GROUP_ID:
|
||
raise HTTPException(503, "GUESTS_GROUP_ID is not configured on the portal app")
|
||
|
||
user_id = await cloudron_create_user(email, name)
|
||
await cloudron_set_group(user_id, GUESTS_GROUP_ID)
|
||
await cloudron_set_active(user_id, True)
|
||
|
||
# One-time $2 team: budget_duration None means LiteLLM never resets it.
|
||
team_id = await litellm_get_or_create_team(email, GUEST_BUDGET, budget_duration=None)
|
||
key_token = get_stored_key(email)
|
||
if not key_token:
|
||
key_token = await litellm_create_key(email, GUEST_BUDGET, team_id)
|
||
|
||
# The row FIRST (the credits model updates a row that must exist), then
|
||
# the guest credit — non-expiring, ledger-audited, deduped on the guest
|
||
# reference.
|
||
store_member(email, key_token, user_id, "", account_type="guest")
|
||
guest_ref = f"guest:{email}"
|
||
if not credit_reference_exists(guest_ref):
|
||
add_credits(email, GUEST_BUDGET,
|
||
reason="guest account — $2 credit from a member",
|
||
reference=guest_ref)
|
||
# Guest rows hold NO allowance (balance 0 — the $2 is all credit) and
|
||
# anchor their join day at the guest date (kept if they later convert).
|
||
conn = get_db()
|
||
conn.execute(
|
||
"UPDATE members SET balance = 0.0, "
|
||
"guest_expires_at = datetime('now', ?), "
|
||
"anchor_day = COALESCE(anchor_day, ?) "
|
||
"WHERE email = ?",
|
||
(f"+{int(GUEST_DAYS)} days", _clamp_anchor_day(datetime.now(timezone.utc).day), email),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
await sync_team_budget_from_balance(email) # cap = allowance(0) + credits(5)
|
||
|
||
setup_url = await cloudron_get_invite_link(user_id)
|
||
# Name the inviter in the invitation: their member slug (the username the
|
||
# co-op knows them by — the members table has no display-name column).
|
||
# The admin path passes inviter_display instead (a co-op label — the
|
||
# invitation comes from the co-op, not from a member).
|
||
inviter_name = ""
|
||
if inviter_display is not None:
|
||
inviter_name = inviter_display
|
||
else:
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT slug FROM members WHERE email = ?", (inviter_email.strip().lower(),)
|
||
).fetchone()
|
||
conn.close()
|
||
if row and row[0]:
|
||
inviter_name = row[0]
|
||
# ONE email at guest time: the invitation is also the guest welcome (merged,
|
||
# not two mails to the same person at once).
|
||
subject, text, html = guest_invitation_email(name, setup_url, inviter_name)
|
||
send_email(email, subject, text, html)
|
||
touch_welcome_sent(email)
|
||
|
||
# Guests receive the newsletter (same pattern as members) — and nothing
|
||
# else: no Loomio/Matrix sync, no docs-RAG group.
|
||
await listmonk_sync(email, subscribe=True)
|
||
|
||
conn = get_db()
|
||
conn.execute(
|
||
"INSERT INTO guests (inviter_email, recipient_email, oc_reference, guest_expires_at, status) "
|
||
"VALUES (?, ?, NULL, (SELECT guest_expires_at FROM members WHERE email = ?), 'active')",
|
||
(inviter_email.strip().lower(), email, email),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
logger.info("Provisioned GUEST account %s (user_id=%s, inviter=%s, budget=%.2f, "
|
||
"expires per GUEST_DAYS=%d)", email, user_id, inviter_email,
|
||
GUEST_BUDGET, GUEST_DAYS)
|
||
return user_id
|
||
|
||
|
||
async def convert_guest_to_member(email: str, name: str, slug: str = "") -> str:
|
||
"""Convert a GUEST account into a full member (the automatic transition).
|
||
|
||
Fires when the guest's OC identity appears as a live BACKER/ADMIN with the
|
||
SAME email (reconcile_memberships or the webhook) — or by the operator via
|
||
POST /admin/guest-convert when the contribution arrived under a different
|
||
OC email (that path records members.oc_email so later renewals match).
|
||
|
||
What converts:
|
||
* the Cloudron user moves guests → members (regaining Loomio/Matrix
|
||
eligibility through the normal syncs);
|
||
* the LiteLLM cap lifts from the $2 guest to the member allowance +
|
||
credits — any UNSPENT guest credit stays (it already lives in the
|
||
non-expiring credit_balance bucket);
|
||
* the portal row becomes account_type='member' with the standard
|
||
MEMBER_BUDGET allowance; anchor_day (their join day = the guest day) is
|
||
KEPT, and the reset is pinned to it like any member's;
|
||
* a short congratulations email goes out, then the normal member
|
||
onboarding (newsletter, Loomio + Matrix syncs, docs-RAG group,
|
||
held-credit grants).
|
||
"""
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT cloudron_user_id, anchor_day FROM members WHERE email = ?", (email,)
|
||
).fetchone()
|
||
conn.close()
|
||
if not row:
|
||
raise HTTPException(404, f"No portal row for {email}")
|
||
user_id = row[0]
|
||
|
||
# 1. Group move: guests → members (idempotent PUT; default = members group).
|
||
await cloudron_set_group(user_id)
|
||
|
||
# 2. Row: full member with the standard allowance. Unspent guest credits
|
||
# simply stay in credit_balance. anchor_day is untouched: their join
|
||
# day is the day the guest started.
|
||
conn = get_db()
|
||
conn.execute(
|
||
"UPDATE members SET balance = ?, account_type = 'member', active = 1, "
|
||
"slug = CASE WHEN ? != '' THEN ? ELSE slug END, "
|
||
"guest_expires_at = NULL WHERE email = ?",
|
||
(MEMBER_BUDGET, slug, slug, email),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
# 3. Lift the LiteLLM cap to allowance + credits and pin the reset to
|
||
# their anchor day, exactly as the webhook does for any member.
|
||
await sync_team_budget_from_balance(email)
|
||
try:
|
||
team_id = await litellm_team_id_for_email(email)
|
||
if team_id:
|
||
anchor = await anchor_day_for_member(email, slug)
|
||
await litellm_pin_reset_date(team_id, next_reset_for_anchor_day(anchor))
|
||
except Exception as e:
|
||
logger.warning("Guest convert: reset pin failed for %s: %s", email, e)
|
||
|
||
# 4. Full member onboarding — the pieces the guest path skipped.
|
||
await listmonk_sync(email, subscribe=True)
|
||
try:
|
||
conn = get_db()
|
||
granted = grant_held_credits(conn, email, slug)
|
||
conn.close()
|
||
if granted > 0:
|
||
await sync_team_budget_from_balance(email)
|
||
logger.info("Guest convert: granted %.2f held credits to %s", granted, email)
|
||
except Exception as e:
|
||
logger.warning("Guest convert: held-credit grant for %s failed: %s", email, e)
|
||
await sync_loomio_memberships()
|
||
await sync_matrix_space()
|
||
|
||
# 5. Congratulations email (include the setup link only if they never
|
||
# activated — a 409 from invite_link means they did).
|
||
setup_url = None
|
||
try:
|
||
if not await cloudron_is_activated(user_id):
|
||
setup_url = await cloudron_get_invite_link(user_id)
|
||
except Exception:
|
||
setup_url = None
|
||
subject, text, html = guest_member_congrats_email(name or email.split("@")[0], setup_url)
|
||
send_email(email, subject, text, html)
|
||
touch_welcome_sent(email)
|
||
|
||
conn = get_db()
|
||
conn.execute("UPDATE guests SET status = 'converted' WHERE recipient_email = ? AND status = 'active'",
|
||
(email,))
|
||
conn.commit()
|
||
conn.close()
|
||
logger.info("Converted GUEST account %s to member (slug=%s)", email, slug)
|
||
return user_id
|
||
|
||
|
||
def guest_lapse_notice_sent(email: str) -> bool:
|
||
"""True when the reaching-limits email has already gone out for this guest.
|
||
|
||
The once-only gate (same shape as shortfall_notice_sent): one row per
|
||
guest, ever. The guests ledger records what became of a guest; this table
|
||
records what actually left the building.
|
||
"""
|
||
conn = get_db()
|
||
_ensure_guest_notice_table(conn)
|
||
row = conn.execute(
|
||
"SELECT 1 FROM guest_lapse_notices WHERE email = ? LIMIT 1", (email,)
|
||
).fetchone()
|
||
conn.close()
|
||
return bool(row)
|
||
|
||
|
||
def mark_guest_lapse_notice_sent(email: str, reason: str) -> None:
|
||
conn = get_db()
|
||
_ensure_guest_notice_table(conn)
|
||
conn.execute(
|
||
"INSERT OR REPLACE INTO guest_lapse_notices (email, reason, sent_at)"
|
||
" VALUES (?, ?, datetime('now'))",
|
||
(email, reason),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def _ensure_guest_notice_table(conn) -> None:
|
||
"""Records which guests have been told their guest access ended.
|
||
|
||
Kept apart from the guests ledger on purpose: the ledger answers "what was
|
||
invited and what became of it", this answers "what left the building". A
|
||
guest is notified at most once, ever.
|
||
"""
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS guest_lapse_notices ("
|
||
" email TEXT PRIMARY KEY, reason TEXT NOT NULL,"
|
||
" sent_at TEXT NOT NULL DEFAULT (datetime('now')))"
|
||
)
|
||
conn.commit()
|
||
|
||
|
||
async def send_guest_lapse_notice(email: str, user_id: str | None, reason: str) -> bool:
|
||
"""Send the reaching-limits email ONCE per guest.
|
||
|
||
Checks the sent-state row before sending and marks it only after the send
|
||
succeeded — an unmarked guest (send failed, or lapsed before this email
|
||
shipped) is retried by the sweep's catch-up pass, never double-sent.
|
||
Returns True when a send actually left the building.
|
||
"""
|
||
if guest_lapse_notice_sent(email):
|
||
return False
|
||
# Greet by name where we can find one: the Cloudron display name, else the
|
||
# portal slug, else "there" (never the email local part).
|
||
name = ""
|
||
if user_id:
|
||
try:
|
||
async with httpx.AsyncClient() as client:
|
||
users = await cloudron_fetch_all_users(client)
|
||
if users:
|
||
name = next((u.get("displayName") or "" for u in users
|
||
if u.get("id") == user_id), "")
|
||
except Exception as e:
|
||
logger.warning("Guest lapse notice: name lookup failed for %s: %s", email, e)
|
||
if not name:
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT slug FROM members WHERE email = ?", (email,)
|
||
).fetchone()
|
||
conn.close()
|
||
if row and row[0]:
|
||
name = row[0]
|
||
subject, text_body, html_body = guest_limits_email(name or "there", reason)
|
||
if send_email(email, subject, text_body, html_body):
|
||
mark_guest_lapse_notice_sent(email, reason)
|
||
return True
|
||
return False
|
||
|
||
|
||
async def reconcile_guest_lifecycles() -> dict:
|
||
"""Lapse guest accounts by their own rules: time box or spent credits.
|
||
|
||
Runs inside the nightly sweep. Two deactivation triggers, checked per
|
||
ACTIVE guest row:
|
||
* guest_expires_at past (the GUEST_DAYS time box) — even with credits left;
|
||
* credits exhausted — the team's spend has reached the credit balance
|
||
(LiteLLM's cap already blocks further spend; this retires the row).
|
||
Deactivation is the same semantics as a member's: guests group → inactive,
|
||
team budget zeroed, row active=0, newsletter unsubscribed. The account is
|
||
KEPT so a returning payer resumes cleanly; the guests-ledger status records
|
||
which rule ended it ('expired' | 'depleted').
|
||
|
||
Returns {expired, depleted, lapse_notices} counts; never raises past a
|
||
logged warning.
|
||
"""
|
||
expired = 0
|
||
depleted = 0
|
||
notified = 0
|
||
conn = get_db()
|
||
rows = conn.execute(
|
||
"SELECT email, cloudron_user_id, credit_balance, balance, guest_expires_at "
|
||
"FROM members WHERE active = 1 AND account_type = 'guest'"
|
||
).fetchall()
|
||
conn.close()
|
||
now = datetime.now(timezone.utc)
|
||
for email, user_id, credits, balance, expires_at in rows:
|
||
try:
|
||
allowance = float(balance) if balance is not None else 0.0
|
||
credits = float(credits or 0.0)
|
||
reason = None
|
||
if expires_at:
|
||
exp = datetime.fromisoformat(str(expires_at))
|
||
if exp.tzinfo is None:
|
||
exp = exp.replace(tzinfo=timezone.utc)
|
||
if now >= exp:
|
||
reason = "expired"
|
||
if reason is None:
|
||
team_id = await litellm_team_id_for_email(email)
|
||
spend = None
|
||
if team_id:
|
||
team = get_team_budget_from_db(team_id)
|
||
if team:
|
||
spend = team.get("spend")
|
||
if spend is not None:
|
||
eff = compute_max_budget(allowance, credits, float(spend))
|
||
if eff["credits_remaining"] <= 0.0:
|
||
reason = "depleted"
|
||
if reason is None:
|
||
continue
|
||
if INACTIVE_GROUP_ID and user_id:
|
||
await cloudron_set_group(user_id, INACTIVE_GROUP_ID)
|
||
deactivate_member(email)
|
||
await revoke_member_litellm_access(email)
|
||
await listmonk_sync(email, subscribe=False)
|
||
conn = get_db()
|
||
conn.execute(
|
||
"UPDATE guests SET status = ? WHERE recipient_email = ? AND status = 'active'",
|
||
(reason, email),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
if reason == "expired":
|
||
expired += 1
|
||
else:
|
||
depleted += 1
|
||
logger.info("Guest lifecycle: %s deactivated (%s)", email, reason)
|
||
# Reaching-limits email: once per guest, ever, sent AFTER the
|
||
# deactivation committed (never tell someone their access ended
|
||
# while it hasn't). A failed send stays unmarked and is retried
|
||
# by the catch-up pass below on a later sweep.
|
||
try:
|
||
if await send_guest_lapse_notice(email, user_id, reason):
|
||
notified += 1
|
||
except Exception as e:
|
||
logger.warning("Guest lapse notice failed for %s: %s", email, e)
|
||
except Exception as e:
|
||
logger.warning("Guest lifecycle: failed for %s: %s", email, e)
|
||
# Catch-up: guests already deactivated (lapsed before this email shipped,
|
||
# or whose send failed) who were never told. Converted guests are excluded
|
||
# by the WHERE — their account_type is 'member' now, and a member is never
|
||
# told their guest ended.
|
||
conn = get_db()
|
||
pending = conn.execute(
|
||
"SELECT email, cloudron_user_id, guest_expires_at FROM members "
|
||
"WHERE active = 0 AND account_type = 'guest'"
|
||
).fetchall()
|
||
conn.close()
|
||
for email, user_id, expires_at in pending:
|
||
if guest_lapse_notice_sent(email):
|
||
continue
|
||
try:
|
||
conn = get_db()
|
||
grow = conn.execute(
|
||
"SELECT status FROM guests WHERE recipient_email = ? "
|
||
"ORDER BY id DESC LIMIT 1", (email,)
|
||
).fetchone()
|
||
conn.close()
|
||
reason = grow[0] if grow and grow[0] in ("expired", "depleted") else None
|
||
if reason is None:
|
||
# No ledger verdict: read the time box off the row; a row with
|
||
# no expiry was ended by its credits.
|
||
try:
|
||
exp = datetime.fromisoformat(str(expires_at)) if expires_at else None
|
||
if exp and exp.tzinfo is None:
|
||
exp = exp.replace(tzinfo=timezone.utc)
|
||
reason = "expired" if (exp and datetime.now(timezone.utc) >= exp) else "depleted"
|
||
except ValueError:
|
||
reason = "depleted"
|
||
if await send_guest_lapse_notice(email, user_id, reason):
|
||
notified += 1
|
||
except Exception as e:
|
||
logger.warning("Guest lapse catch-up failed for %s: %s", email, e)
|
||
return {"expired": expired, "depleted": depleted, "lapse_notices": notified}
|
||
|
||
|
||
async def cancel_guest_account(email: str) -> dict:
|
||
"""End a guest account at the recipient's own request (self-service cancel).
|
||
|
||
The guest-side counterpart to reconcile_guest_lifecycles: the recipient
|
||
cancels from their dashboard instead of waiting out the time box. Same
|
||
deactivation semantics as a guest lapse — the account and its data are
|
||
KEPT so a re-invite (or the guest → member conversion) resumes cleanly —
|
||
plus the one thing a lapse does not do: the unspent credit returns to
|
||
the co-op.
|
||
|
||
* LiteLLM team budget zeroed (every key under it stops working);
|
||
* the Cloudron user leaves the guests group (chat access ends) and
|
||
moves to GUESTS_ENDED_GROUP_ID, which the dashboard app still admits
|
||
so the ended state and the Become-a-member path stay visible;
|
||
* newsletter unsubscribed;
|
||
* unspent credit returned: a credit_ledger row with a NEGATIVE delta,
|
||
its reason naming the cancellation and the original guest reference
|
||
(traceable), its reference deduping the return so a repeated cancel
|
||
can never double-credit the accounting;
|
||
* the guests ledger records the full trail: inviter → recipient →
|
||
'cancelled by recipient' (rows are never deleted or rewritten);
|
||
* the reaching-limits bookkeeping is pre-marked when that subsystem is
|
||
deployed: someone who cancelled themselves must never be told their
|
||
credits were spent.
|
||
|
||
Idempotent: a second call is a no-op that reports the recorded outcome.
|
||
Refuses (403) anything that is not a guest account — a paying member can
|
||
never end a membership through this path, and there is deliberately no
|
||
member analogue: cancelling a membership means ending the Open Collective
|
||
contribution.
|
||
"""
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT active, account_type, cloudron_user_id, balance, credit_balance "
|
||
"FROM members WHERE email = ?",
|
||
(email,),
|
||
).fetchone()
|
||
conn.close()
|
||
if not row:
|
||
raise HTTPException(403, "No active membership")
|
||
active, account_type, user_id, balance, credits = row
|
||
if (account_type or "member") != "guest":
|
||
# The hard member guard: this endpoint only ever ends GUEST access.
|
||
# A membership is ended by ending its Open Collective contribution.
|
||
raise HTTPException(403, "Only guest accounts can be cancelled here")
|
||
|
||
guest_status = _guest_ledger_status(email)
|
||
if not active:
|
||
# Already ended (a previous cancel, or the sweep's lapse). No-op —
|
||
# and never a second return of credit.
|
||
return {
|
||
"status": "already_ended",
|
||
"guest_status": guest_status or "inactive",
|
||
"credits_returned": _guest_returned_credits(email),
|
||
}
|
||
|
||
# Unspent credit = the credits bucket minus what this cycle's spend
|
||
# already drew (a guest row holds NO allowance — balance is 0 — so the
|
||
# credits model's remaining figure IS the unspent guest).
|
||
allowance = float(balance) if balance is not None else 0.0
|
||
credit_total = float(credits or 0.0)
|
||
spend = None
|
||
team_id = await litellm_team_id_for_email(email)
|
||
if team_id:
|
||
team = get_team_budget_from_db(team_id)
|
||
if team:
|
||
spend = team.get("spend")
|
||
unspent = round(compute_max_budget(allowance, credit_total, spend)["credits_remaining"], 2)
|
||
|
||
return_ref = f"guest-return:{email}"
|
||
# The ledger is the accounting record: when a return row already exists
|
||
# (a cancel that failed part-way through), report what IT recorded and
|
||
# never write a second one — dedupe lives on the reference.
|
||
credits_returned = _guest_returned_credits(email) if credit_reference_exists(return_ref) else 0.0
|
||
if credits_returned == 0.0 and unspent > 0:
|
||
add_credits(
|
||
email, -unspent,
|
||
reason=(f"guest cancelled by recipient — unspent credit returned to the "
|
||
f"co-op (original guest: guest:{email})"),
|
||
reference=return_ref,
|
||
)
|
||
credits_returned = unspent
|
||
|
||
# Deactivation, guest-lapse semantics: row inactive, group out of guests,
|
||
# gateway budget zeroed, newsletter off.
|
||
deactivate_member(email)
|
||
ended_group = GUESTS_ENDED_GROUP_ID or INACTIVE_GROUP_ID
|
||
if ended_group and user_id:
|
||
await cloudron_set_group(user_id, ended_group)
|
||
await revoke_member_litellm_access(email)
|
||
await listmonk_sync(email, subscribe=False)
|
||
|
||
conn = get_db()
|
||
conn.execute(
|
||
"UPDATE guests SET status = 'cancelled by recipient' "
|
||
"WHERE recipient_email = ? AND status = 'active'",
|
||
(email,),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
# Pre-mark the reaching-limits notice: the sweep's catch-up must not
|
||
# email a "your credits were spent" notice to someone who cancelled
|
||
# themselves — they were told by doing it.
|
||
mark_guest_lapse_notice_sent(email, "cancelled by recipient")
|
||
|
||
logger.info("Guest cancel: %s cancelled by recipient (returned %.2f)", email, credits_returned)
|
||
return {
|
||
"status": "cancelled",
|
||
"guest_status": "cancelled by recipient",
|
||
"credits_returned": credits_returned,
|
||
}
|
||
|
||
|
||
def _guest_ledger_status(email: str) -> str | None:
|
||
"""The latest guests-ledger status for a recipient, or None."""
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT status FROM guests WHERE recipient_email = ? ORDER BY id DESC LIMIT 1",
|
||
(email,),
|
||
).fetchone()
|
||
conn.close()
|
||
return row[0] if row else None
|
||
|
||
|
||
def _guest_returned_credits(email: str) -> float:
|
||
"""Credit returned to the co-op for this guest (0 when there was none).
|
||
|
||
Read back from the ledger — the return row IS the accounting record, so
|
||
the reported amount always matches what was actually written.
|
||
"""
|
||
conn = get_db()
|
||
rows = conn.execute(
|
||
"SELECT delta FROM credit_ledger WHERE reference = ?",
|
||
(f"guest-return:{email}",),
|
||
).fetchall()
|
||
conn.close()
|
||
return round(sum(-float(r[0] or 0.0) for r in rows), 2)
|
||
|
||
|
||
async def litellm_get_or_create_team(email: str, budget: float,
|
||
budget_duration: str | None = "30d") -> str:
|
||
"""Return the team_id for a member's budget team, creating it if needed.
|
||
|
||
Each member gets a LiteLLM "team" (a budget container) with a $15/30d
|
||
budget. The team holds the chat key + any member-managed API keys, so all
|
||
spend draws from the one pool.
|
||
|
||
`budget_duration` is the reset cadence: members renew every 30 days; GUEST
|
||
accounts pass None — a one-time budget that NEVER resets, so the $2 cap is
|
||
the guest's whole life (spend it and it is gone).
|
||
|
||
Idempotent: looks up the existing team by alias before creating. Note that
|
||
/team/new is NOT idempotent (it creates a new team each call), so we must
|
||
check /team/list first.
|
||
"""
|
||
alias = f"member:{email}"
|
||
# /team/list has grown slow as usage accumulates; the httpx default 5s
|
||
# read timeout was too short (root cause of dashboard 500s on 2026-09-26).
|
||
async with httpx.AsyncClient(timeout=httpx.Timeout(connect=15.0, read=60.0, write=15.0, pool=15.0)) as client:
|
||
r = await client.get(
|
||
f"{LITELLM_BASE}/team/list",
|
||
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
|
||
)
|
||
r.raise_for_status()
|
||
for t in r.json():
|
||
if t.get("team_alias") == alias:
|
||
return t["team_id"]
|
||
|
||
payload = {
|
||
"team_alias": alias,
|
||
"max_budget": budget,
|
||
"models": MEMBER_MODELS,
|
||
}
|
||
if budget_duration:
|
||
payload["budget_duration"] = budget_duration
|
||
r = await client.post(
|
||
f"{LITELLM_BASE}/team/new",
|
||
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
|
||
json=payload,
|
||
)
|
||
r.raise_for_status()
|
||
return r.json()["team_id"]
|
||
|
||
|
||
async def litellm_update_team_budget(team_id: str, budget: float) -> None:
|
||
"""Set a team's max_budget (the enforced spend cap).
|
||
|
||
This is the primitive behind "payment → add to balance": a top-up adjusts
|
||
the team's max_budget, which every key under the team (chat + API) draws
|
||
from. Keys inherit the team budget (their own max_budget is ignored when a
|
||
team_id is set), so the team is the single source of truth for allowance.
|
||
"""
|
||
async with httpx.AsyncClient() as client:
|
||
r = await client.post(
|
||
f"{LITELLM_BASE}/team/update",
|
||
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
|
||
json={"team_id": team_id, "max_budget": budget},
|
||
)
|
||
r.raise_for_status()
|
||
|
||
|
||
async def litellm_pin_reset_date(team_id: str, reset_at: datetime) -> None:
|
||
"""Pin a team's allowance reset to an exact moment (next OC payment date).
|
||
|
||
LiteLLM's /team/update silently DROPS budget_reset_at (verified v1.74.0),
|
||
so this writes the authoritative DB directly. Budgets reset to zero spend
|
||
at the pinned instant; the pinned date is honored across restarts.
|
||
"""
|
||
def _write() -> None:
|
||
conn = _pg_connect()
|
||
try:
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
'UPDATE "LiteLLM_TeamTable" SET budget_reset_at = %s WHERE team_id = %s',
|
||
(reset_at, team_id),
|
||
)
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
await asyncio.get_event_loop().run_in_executor(None, _write)
|
||
logger.info("Pinned budget reset for %s to %s", team_id, reset_at.isoformat())
|
||
|
||
|
||
def next_reset_from_now() -> datetime:
|
||
"""The member's next allowance reset: 30 days from now, on the hour."""
|
||
return datetime.now(timezone.utc).replace(minute=0, second=0, microsecond=0) + timedelta(days=30)
|
||
|
||
|
||
async def sync_team_budget_from_balance(email: str) -> float:
|
||
"""Apply the member's effective budget to their LiteLLM team.
|
||
|
||
Effective budget = monthly allowance + remaining non-expiring credits
|
||
(credits drawn only after the allowance is exhausted this cycle). Reads
|
||
the member's live spend from LiteLLM to determine how much credit has
|
||
already been consumed in the current cycle. Returns the applied budget.
|
||
"""
|
||
balance = get_member_balance(email)
|
||
credits = get_credit_balance(email)
|
||
team_id = await litellm_get_or_create_team(email, balance)
|
||
|
||
# Live spend this cycle (DB path preferred; LiteLLM computes the reset).
|
||
team = get_team_budget_from_db(team_id) or {}
|
||
team_spend = team.get("spend")
|
||
eff = compute_max_budget(balance, credits, team_spend)
|
||
await litellm_update_team_budget(team_id, eff["max_budget"])
|
||
return eff["max_budget"]
|
||
|
||
|
||
async def litellm_create_key(email: str, budget: float, team_id: str) -> str:
|
||
"""Create a fresh LiteLLM virtual key under the member's team.
|
||
|
||
Returns the ACTUAL sk- key (only revealed at generation time). We always
|
||
create a new key rather than reusing — the sk- value is unrecoverable from
|
||
a stored hash, so "reuse" would store a useless hash (a bug that previously
|
||
broke chat for three members).
|
||
|
||
NOTE: the key does NOT set its own max_budget — it inherits the team's
|
||
budget (LiteLLM ignores a key's max_budget when team_id is set anyway).
|
||
The team is the single source of truth for allowance, so a balance change
|
||
(top-up) updates the team once and every key follows. The `budget` arg is
|
||
retained only for the team-creation fallback in the migration path.
|
||
"""
|
||
alias = f"member:{email}"
|
||
async with httpx.AsyncClient() as client:
|
||
r = await client.post(
|
||
f"{LITELLM_BASE}/key/generate",
|
||
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
|
||
json={
|
||
"key_alias": alias,
|
||
"team_id": team_id,
|
||
"models": MEMBER_MODELS,
|
||
},
|
||
)
|
||
r.raise_for_status()
|
||
return r.json().get("key", "")
|
||
|
||
|
||
async def litellm_disable_key(key_token: str) -> None:
|
||
"""Delete a member's LiteLLM key (on payment lapse).
|
||
|
||
/key/delete accepts either the sk- key or its SHA256 hash, so this works
|
||
for both current keys and legacy hash-keyed members.
|
||
"""
|
||
if not key_token:
|
||
return
|
||
async with httpx.AsyncClient() as client:
|
||
await client.post(
|
||
f"{LITELLM_BASE}/key/delete",
|
||
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
|
||
json={"keys": [key_token]},
|
||
)
|
||
|
||
|
||
async def litellm_team_id_for_email(email: str) -> str | None:
|
||
"""The team_id of a member's budget team, or None if they have none."""
|
||
async with httpx.AsyncClient(timeout=httpx.Timeout(connect=15.0, read=60.0, write=15.0, pool=15.0)) as client:
|
||
r = await client.get(f"{LITELLM_BASE}/team/list",
|
||
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"})
|
||
r.raise_for_status()
|
||
for t in r.json():
|
||
if t.get("team_alias") == f"member:{email}":
|
||
return t["team_id"]
|
||
return None
|
||
|
||
|
||
async def revoke_member_litellm_access(email: str) -> bool:
|
||
"""Cut a deactivated member's gateway access (SECURITY 2026-10-27).
|
||
|
||
Zeroing the team budget blocks every key under it: member-managed keys carry
|
||
no max_budget of their own (verified live 2026-10-27), so they inherit the
|
||
team's. Deleting the key instead would break reactivation, which deliberately
|
||
reuses the stored sk- token — and get_or_create_team does NOT update the
|
||
budget of an existing team, so reactivation paths must re-sync explicitly.
|
||
|
||
Best-effort by design: a failure here must not stop deactivation from
|
||
completing, but it must be loud, because the member keeps access until it
|
||
succeeds.
|
||
"""
|
||
try:
|
||
team_id = await litellm_team_id_for_email(email)
|
||
if not team_id:
|
||
logger.info("Revoke: %s has no gateway team; nothing to zero", email)
|
||
return True
|
||
await litellm_update_team_budget(team_id, 0.0)
|
||
logger.info("Revoke: %s gateway budget zeroed (team %s)", email, team_id)
|
||
return True
|
||
except Exception as e:
|
||
logger.error("Revoke: FAILED to zero gateway budget for %s: %s", email, e)
|
||
try:
|
||
send_email("info@inference.coop",
|
||
"Deactivation did not revoke gateway access",
|
||
f"{email} was deactivated but their LiteLLM team budget could not be zeroed:\n\n{e}\n\n"
|
||
"Their existing API keys may still work. Fix by hand: set the team's max_budget to 0.\n")
|
||
except Exception:
|
||
pass
|
||
return False
|
||
|
||
|
||
# --- Broker: member-managed API keys (scoped to the member's team) ---
|
||
|
||
def _api_key_alias(email: str, name: str) -> str:
|
||
return f"api:{email}:{name}"
|
||
|
||
|
||
async def broker_create_api_key(email: str, name: str) -> dict:
|
||
"""Create a member-managed API key under the member's team.
|
||
|
||
Returns {name, sk_token, created_at}. The key draws from the member's team
|
||
budget and is tracked in member_api_keys so it can be listed/revoked by name.
|
||
"""
|
||
if not name or not re.fullmatch(r"[A-Za-z0-9._-]{1,64}", name):
|
||
raise HTTPException(400, "Invalid key name (1-64 chars, alphanumeric/._-)")
|
||
|
||
balance = get_member_balance(email)
|
||
team_id = await litellm_get_or_create_team(email, balance)
|
||
alias = _api_key_alias(email, name)
|
||
async with httpx.AsyncClient() as client:
|
||
# Rotation support: LiteLLM requires unique key aliases, so a re-create
|
||
# with an existing (email, name) must retire the old key first. Without
|
||
# this, rotating a key from the dashboard 400s ("alias already exists").
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT hash FROM member_api_keys WHERE email = ? AND name = ?",
|
||
(email, name),
|
||
).fetchone()
|
||
conn.close()
|
||
if row and row[0]:
|
||
old_hash = row[0]
|
||
try:
|
||
del_res = await client.post(
|
||
f"{LITELLM_BASE}/key/delete",
|
||
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
|
||
json={"keys": [old_hash]},
|
||
)
|
||
if del_res.status_code == 200:
|
||
logger.info("Rotated API key %s for %s (old key retired)", name, email)
|
||
else:
|
||
logger.warning(
|
||
"Old key retire failed for %s/%s: %s %s",
|
||
email, name, del_res.status_code, del_res.text[:200],
|
||
)
|
||
except Exception as exc: # noqa: BLE001 - rotation is best-effort
|
||
logger.warning("Old key retire errored for %s/%s: %s", email, name, exc)
|
||
r = await client.post(
|
||
f"{LITELLM_BASE}/key/generate",
|
||
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
|
||
json={
|
||
"key_alias": alias,
|
||
"team_id": team_id,
|
||
"models": MEMBER_MODELS,
|
||
},
|
||
)
|
||
r.raise_for_status()
|
||
data = r.json()
|
||
sk_token = data.get("key", "")
|
||
h = data.get("token") or data.get("token_id") or ""
|
||
|
||
# Reveal the key ONCE (in the response). Persist only a fingerprint (last-4)
|
||
# and the LiteLLM hash — the plaintext is NOT stored. Revocation uses the
|
||
# hash, so the full key never needs to be retained.
|
||
fingerprint = sk_token[-4:] if sk_token else ""
|
||
|
||
conn = get_db()
|
||
conn.execute(
|
||
"INSERT INTO member_api_keys (email, name, sk_token, hash, fingerprint) VALUES (?, ?, ?, ?, ?) "
|
||
"ON CONFLICT(email, name) DO UPDATE SET sk_token='', hash=excluded.hash, fingerprint=excluded.fingerprint, "
|
||
"created_at=datetime('now')",
|
||
(email, name, "", h, fingerprint),
|
||
)
|
||
conn.commit()
|
||
row = conn.execute(
|
||
"SELECT created_at FROM member_api_keys WHERE email = ? AND name = ?",
|
||
(email, name),
|
||
).fetchone()
|
||
conn.close()
|
||
return {"name": name, "sk_token": sk_token, "created_at": row[0] if row else None}
|
||
|
||
|
||
async def broker_list_api_keys(email: str) -> list[dict]:
|
||
conn = get_db()
|
||
rows = conn.execute(
|
||
"SELECT name, fingerprint, created_at FROM member_api_keys WHERE email = ? ORDER BY created_at DESC",
|
||
(email,),
|
||
).fetchall()
|
||
conn.close()
|
||
return [
|
||
{"name": r[0], "fingerprint": r[1] or "", "created_at": r[2]}
|
||
for r in rows
|
||
]
|
||
|
||
|
||
async def broker_revoke_api_key(email: str, name: str) -> None:
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT hash FROM member_api_keys WHERE email = ? AND name = ?", (email, name)
|
||
).fetchone()
|
||
if not row:
|
||
conn.close()
|
||
raise HTTPException(404, "Key not found")
|
||
conn.execute("DELETE FROM member_api_keys WHERE email = ? AND name = ?", (email, name))
|
||
conn.commit()
|
||
conn.close()
|
||
# Delete the underlying LiteLLM key by its hash.
|
||
await litellm_disable_key(row[0])
|
||
|
||
|
||
# --- A. Open Collective webhook ---
|
||
|
||
def _state_get(conn, key: str, default: str = "") -> str:
|
||
try:
|
||
row = conn.execute("SELECT value FROM portal_state WHERE key = ?", (key,)).fetchone()
|
||
return (row[0] if row and row[0] is not None else default)
|
||
except sqlite3.Error:
|
||
return default
|
||
|
||
|
||
def _state_set(conn, key: str, value: str) -> None:
|
||
conn.execute(
|
||
"INSERT INTO portal_state (key, value) VALUES (?, ?) "
|
||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value", (key, str(value)))
|
||
|
||
|
||
def record_webhook_failure(status: int, event_type: str = "", detail: str = "") -> None:
|
||
"""Record a delivery the portal did not accept — the SYSTEM owns noticing these.
|
||
|
||
WHY THIS EXISTS: the alert for a failed Open Collective delivery used to live
|
||
in a script on the operator's own machine, polling this app's log through the
|
||
Cloudron API. That made a money-critical signal depend on a host outside the
|
||
system, and on a log buffer with only hours of memory — when the 2026-11-13
|
||
purchases were investigated the buffer was missing whole hours (06:00–13:00
|
||
and 15:00), so "no failed delivery in the log" could not be told apart from
|
||
"no delivery at all".
|
||
|
||
The portal handles every delivery; it should be the thing that knows, records
|
||
and reports a failure. Rows are summarised by `webhook_failure_report()` in the
|
||
sweep, which emails info@ once per new batch. Capped at 200 rows — this is a
|
||
recent-history record, not an audit log (the credit ledger and the hold table
|
||
are the audit trails).
|
||
"""
|
||
try:
|
||
conn = get_db()
|
||
conn.execute(
|
||
"INSERT INTO webhook_failures (at, status, event_type, detail) VALUES (?,?,?,?)",
|
||
(datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S"), int(status),
|
||
(event_type or "")[:64], (detail or "")[:400]))
|
||
conn.execute("DELETE FROM webhook_failures WHERE id NOT IN "
|
||
"(SELECT id FROM webhook_failures ORDER BY id DESC LIMIT 200)")
|
||
conn.commit()
|
||
conn.close()
|
||
except Exception as e: # noqa: BLE001
|
||
# Never let bookkeeping break the delivery path.
|
||
logger.warning("webhook failure record failed: %s", e)
|
||
|
||
|
||
def _peek_event_type(request) -> str:
|
||
"""Best-effort event type from an already-read body, for the failure record only."""
|
||
try:
|
||
raw = getattr(request, "_body", None) or b""
|
||
if not raw:
|
||
return ""
|
||
data = json.loads(raw.decode("utf-8", "replace"))
|
||
return str(data.get("type") or "")[:64]
|
||
except Exception: # noqa: BLE001
|
||
return ""
|
||
|
||
|
||
WEBHOOK_FAILURE_MARKER = "webhook_failures_reported_id"
|
||
|
||
|
||
def webhook_failure_report(notify: bool = True) -> dict:
|
||
"""Summarise delivery failures since the last report; email info@ once per batch.
|
||
|
||
Runs in the sweep, so the notice is the system's, not a scraper's: the same
|
||
process that refused a delivery is the one that says so. A delivery that never
|
||
arrives at all is a different failure and is covered where it matters — the
|
||
credit-pack reconciliation reads OC's own orders, and the membership sweep
|
||
reconciles the roster.
|
||
|
||
Reported ONCE per batch: the marker advances to the newest row id, so a
|
||
persistent misconfiguration (a wrong WEBHOOK_TOKEN) does not mail info@ every
|
||
sweep, while the rows stay queryable.
|
||
"""
|
||
rows: list[dict] = []
|
||
try:
|
||
conn = get_db()
|
||
last_id = int(_state_get(conn, WEBHOOK_FAILURE_MARKER, "0") or "0")
|
||
rows = [
|
||
{"id": r[0], "at": r[1], "status": r[2], "event_type": r[3], "detail": r[4]}
|
||
for r in conn.execute(
|
||
"SELECT id, at, status, event_type, detail FROM webhook_failures "
|
||
"WHERE id > ? ORDER BY id", (last_id,))]
|
||
conn.close()
|
||
except Exception as e: # noqa: BLE001
|
||
logger.warning("webhook failure report failed: %s", e)
|
||
return {"status": "error", "detail": str(e)[:200]}
|
||
|
||
if not rows:
|
||
return {"status": "ok", "new_failures": 0}
|
||
|
||
lines = [
|
||
f" {r['at']} HTTP {r['status']} "
|
||
f"{r['event_type'] or '(event type unknown)'}"
|
||
+ (f" — {r['detail']}" if r["detail"] else "")
|
||
for r in rows
|
||
]
|
||
logger.error("OC webhook: %d delivery(ies) were not accepted since the last report:\n%s",
|
||
len(rows), "\n".join(lines))
|
||
if not notify:
|
||
# A look, not a notice: logged, told to nobody, and the marker is NOT
|
||
# advanced — a silent run must never swallow the alert a failure gets.
|
||
return {"status": "observed", "new_failures": len(rows), "failures": rows}
|
||
|
||
body = (
|
||
"The portal did not accept these Open Collective deliveries:\n\n"
|
||
+ "\n".join(lines)
|
||
+ "\n\nA 401/403/404 means the webhook URL and WEBHOOK_TOKEN disagree — the "
|
||
"delivery never reached the handler. Any other status means we took it and "
|
||
"then errored; if the payload was an order, confirm the member was credited. "
|
||
"The sweep's credit-pack reconciliation reads Open Collective's own orders "
|
||
"and grants anything missing, so a failed purchase repairs itself; this notice "
|
||
"is for the failures it cannot see.\n"
|
||
)
|
||
try:
|
||
send_email("info@inference.coop",
|
||
f"Inference Cooperative — {len(rows)} Open Collective "
|
||
f"delivery(ies) failed",
|
||
body)
|
||
except Exception as e: # noqa: BLE001
|
||
logger.warning("webhook failure notice email failed: %s", e)
|
||
return {"status": "reported_no_email", "new_failures": len(rows),
|
||
"failures": rows, "email_error": str(e)[:120]}
|
||
|
||
# The marker records what someone was TOLD — advanced only after a notice is
|
||
# actually sent, so a failed send retries on the next sweep instead of the
|
||
# failure going quiet forever.
|
||
try:
|
||
conn = get_db()
|
||
_state_set(conn, WEBHOOK_FAILURE_MARKER, rows[-1]["id"])
|
||
conn.commit()
|
||
conn.close()
|
||
except Exception as e: # noqa: BLE001
|
||
logger.warning("webhook failure marker write failed: %s", e)
|
||
return {"status": "reported", "new_failures": len(rows), "failures": rows}
|
||
|
||
|
||
@app.post("/webhook/opencollective/{token}")
|
||
@limiter.limit("30/minute")
|
||
async def opencollective_webhook(request: Request, token: str):
|
||
"""Handle Open Collective membership events.
|
||
|
||
The work happens in _process_opencollective_webhook; this wrapper exists
|
||
ONLY to stop an unexpected failure inside it from escaping as an unhandled
|
||
ASGI exception. On 2026-11-13 an httpx.ReadTimeout raised by the OC member
|
||
fetch did exactly that: the delivery returned 500, a member's $50
|
||
credit-pack purchase was never credited, and the log recorded an anonymous
|
||
server error rather than anything naming the member or the order.
|
||
|
||
Answering 503 says "temporary — send it again", which is both true and the
|
||
only outcome that repairs itself; the ERROR line names the exception so the
|
||
purchase can still be reconciled by hand if OC stops retrying. Auth
|
||
failures are HTTPExceptions and pass straight through — a 401 must stay a
|
||
401, because that is the signal that the webhook URL and WEBHOOK_TOKEN
|
||
disagree.
|
||
"""
|
||
try:
|
||
# Cache the body BEFORE delegating: Starlette buffers it on first read, so
|
||
# this costs nothing, and it means the failure record can name the event
|
||
# type even on the 401 path, where the token check rejects before the
|
||
# handler ever looks at the payload.
|
||
await request.body()
|
||
except Exception: # noqa: BLE001
|
||
pass
|
||
try:
|
||
return await _process_opencollective_webhook(request, token)
|
||
except HTTPException as e:
|
||
# A 401/403/404 means the delivery never reached the handler: the webhook
|
||
# URL and WEBHOOK_TOKEN disagree. Still a 401 to OC, but now the SYSTEM
|
||
# records it, so a misconfigured webhook stops being invisible.
|
||
if e.status_code in (401, 403, 404):
|
||
record_webhook_failure(e.status_code, _peek_event_type(request), str(e.detail))
|
||
raise
|
||
except Exception as e: # noqa: BLE001
|
||
record_webhook_failure(503, _peek_event_type(request), f"{type(e).__name__}: {e}")
|
||
logger.exception(
|
||
"OC webhook: handler failed (%s: %s) — answering 503 so OC retries; "
|
||
"if the payload was an order, confirm the member was credited",
|
||
type(e).__name__, e,
|
||
)
|
||
return JSONResponse(
|
||
{"status": "error", "retry": True, "detail": type(e).__name__},
|
||
status_code=503,
|
||
)
|
||
|
||
|
||
async def _process_opencollective_webhook(request: Request, token: str):
|
||
"""Handle Open Collective membership events (the implementation).
|
||
|
||
Authenticated by a secret token in the URL path (Open Collective's
|
||
generic webhooks are not HMAC-signed, so a secret URL is the standard
|
||
way to authenticate them).
|
||
"""
|
||
if not WEBHOOK_TOKEN or not secrets.compare_digest(token, WEBHOOK_TOKEN):
|
||
raise HTTPException(401, "Invalid webhook token")
|
||
|
||
payload = await request.json()
|
||
|
||
event_type = payload.get("type", "")
|
||
data = payload.get("data", {})
|
||
|
||
# The webhook does NOT include email (Open Collective strips it for
|
||
# privacy). We get name + slug, then look up the email via the admin token.
|
||
#
|
||
# Slug/name live in different places depending on the event type:
|
||
# - order.processed / transaction.created → data.fromCollective.{slug,name}
|
||
# - collective.member.created → data.member.memberCollective.{slug,name}
|
||
from_collective = data.get("fromCollective", {})
|
||
member_collective = (data.get("member", {}) or {}).get("memberCollective", {})
|
||
slug = from_collective.get("slug") or member_collective.get("slug", "")
|
||
name = from_collective.get("name") or member_collective.get("name", "Member")
|
||
|
||
logger.info("Open Collective event: %s (name=%s, slug=%s)", event_type, name, slug)
|
||
|
||
# Open Collective webhook events (verified):
|
||
# - "order.processed" → fires on EVERY payment (incl. monthly recurring)
|
||
# - "new member" → fires on FIRST contribution only
|
||
# - payload has "firstPayment" boolean to distinguish new vs recurring
|
||
# - "collective.transaction.created" is DEPRECATED (being removed)
|
||
if event_type in ("order.processed", "new.member", "collective.member.created"):
|
||
# New member → provision immediately and send our own welcome email.
|
||
# The webhook strips email, so look it up by slug via the admin token.
|
||
# Only provision on firstPayment (order.processed also fires on monthly
|
||
# renewals, which should NOT re-send the welcome email). Default to
|
||
# False so a missing field never triggers a spurious re-provision; the
|
||
# nightly sweep backstops any missed first payment.
|
||
first_payment = data.get("firstPayment", False)
|
||
|
||
# Membership provision: only on payments to the "Membership" tier. A
|
||
# one-time donation (no tier, or any other tier) grants BACKER role on
|
||
# OC but is NOT a membership — don't provision it.
|
||
#
|
||
# Tier detection: the webhook payload carries only `data.order.TierId`
|
||
# (numeric) — NO tier name. The order description embeds it
|
||
# ("Financial contribution to Inference Cooperative (Credit pack)").
|
||
order = data.get("order") or {}
|
||
order_desc = order.get("description") or ""
|
||
m = re.search(r"\(([^)]+)\)\s*$", order_desc.strip())
|
||
tier_name = m.group(1).strip() if m else ""
|
||
amount_cents = data.get("amount") or 0
|
||
if not amount_cents:
|
||
amount_cents = order.get("totalAmount") or 0
|
||
|
||
# Credit-pack detection: one-time contributions to a "Credit pack"
|
||
# tier route to the member's NON-EXPIRING credit balance instead of
|
||
# provisioning/allowance. Detection: the OC tier name contains
|
||
# "credit" (case-insensitive). Recurring subscriptions never route
|
||
# here (a credit pack is a one-time purchase).
|
||
if "credit" in tier_name.lower() and slug and event_type == "order.processed":
|
||
members = await fetch_members()
|
||
email = next((m["email"] for m in members if m["slug"] == slug), None)
|
||
# The tier is VARIABLE-amount — no fixed face value. Credits are
|
||
# granted on the contribution amount EXCLUDING the OC platform
|
||
# tip (totalAmount - tip = amount), which we resolve via OC's
|
||
# public order API. Fallback: webhook totalAmount (slightly
|
||
# overgrants if a tip was included; logged).
|
||
tip_free_cents = await get_order_amount(order.get("idV2"))
|
||
if tip_free_cents is None:
|
||
# SECURITY (2026-10-27): do NOT fall back to the payload's own
|
||
# amount. This route is authenticated only by a URL token, so a
|
||
# forged POST naming its own totalAmount used to mint credits for
|
||
# any slug. Treat the payload as a hint; require a resolvable OC
|
||
# order. A forged id cannot resolve; a transient OC failure holds
|
||
# for manual reconciliation instead of granting blind.
|
||
claimed = round(float(amount_cents) / 100.0, 2)
|
||
logger.warning(
|
||
"Credit pack REJECTED: OC could not resolve order %r (slug=%s, payload claimed $%.2f)"
|
||
" — no credits granted",
|
||
order.get("idV2"), slug, claimed,
|
||
)
|
||
try:
|
||
send_email(
|
||
"info@inference.coop",
|
||
"Credit pack could not be verified — no credits granted",
|
||
f"A credit-pack webhook arrived for {slug} claiming ${claimed:.2f}, but\n"
|
||
f"Open Collective's order API could not resolve order id {order.get('idV2')!r}.\n\n"
|
||
"No credits were granted: the payload's amount is not trusted. If this was a\n"
|
||
"real purchase, confirm the order in Open Collective and grant the credits\n"
|
||
"by hand; if it was not, this was an attempt to mint credits.\n",
|
||
)
|
||
except Exception as e:
|
||
logger.warning("Credit pack rejection: operator email failed: %s", e)
|
||
return JSONResponse({"status": "unverified_order_held", "slug": slug})
|
||
amount = round(tip_free_cents / 100.0, 2)
|
||
charged = round(float(amount_cents) / 100.0, 2)
|
||
if charged > amount:
|
||
logger.info(
|
||
"Credit pack: charged $%.2f including tip; granting tip-free $%.2f",
|
||
charged, amount,
|
||
)
|
||
oc_ref = credit_pack_reference(slug, order)
|
||
|
||
if email:
|
||
# Idempotence (audit H1, 2026-11-05): OC re-delivers events and
|
||
# this handler is slow (OC member fetch + email + Loomio +
|
||
# Matrix inside the request), which invites re-delivery. This
|
||
# purchase was already recorded once — return the standing
|
||
# balance WITHOUT crediting again. The reference is per-order,
|
||
# so this never blocks a new purchase, and legacy empty-suffix
|
||
# ledger rows can never match a new-format reference.
|
||
if credit_reference_exists(oc_ref):
|
||
logger.info(
|
||
"Credit pack: %s already in the ledger (OC re-delivery) "
|
||
"— not crediting again", oc_ref,
|
||
)
|
||
return JSONResponse({
|
||
"status": "already_credited", "email": email,
|
||
"credits_added": 0.0,
|
||
"credit_balance": get_credit_balance(email),
|
||
"reference": oc_ref,
|
||
})
|
||
# Member purchase → credits land immediately.
|
||
new_balance = add_credits(
|
||
email, amount,
|
||
reason="credit pack purchase (Open Collective)",
|
||
reference=oc_ref,
|
||
)
|
||
# Push the enlarged budget immediately.
|
||
await sync_team_budget_from_balance(email)
|
||
logger.info("Credit pack: +%s credits for %s (balance %s)",
|
||
amount, email, new_balance)
|
||
return JSONResponse({
|
||
"status": "credits_added", "email": email,
|
||
"credits_added": amount, "credit_balance": new_balance,
|
||
})
|
||
|
||
# Non-member purchase → HOLD. Record it, email the buyer with
|
||
# their two options (refund or join-and-apply). Credits are NOT
|
||
# granted: a membership is required to use them.
|
||
buyer_name = name or slug
|
||
buyer_email = ""
|
||
# The webhook payload strips emails; look up the payer via the
|
||
# admin token (bot account sees contributor emails).
|
||
try:
|
||
# Paginated member fetch (raises on failure → caught below)
|
||
for n in await fetch_all_member_nodes():
|
||
if (n.get("account") or {}).get("slug") == slug:
|
||
buyer_email = ((n.get("account") or {}).get("email") or "").strip()
|
||
buyer_name = (n.get("account") or {}).get("name") or buyer_name
|
||
break
|
||
except Exception as e:
|
||
logger.warning("Credit-pack hold: payer lookup failed for %s: %s", slug, e)
|
||
|
||
conn = get_db()
|
||
# Idempotence (audit H1, 2026-11-05): one hold per purchase. The
|
||
# unique index on oc_reference is the hard stop; this pre-check
|
||
# returns a calm response on re-delivery instead of a 500 (which
|
||
# would make OC re-deliver again).
|
||
existing = conn.execute(
|
||
"SELECT id FROM held_credits WHERE oc_reference = ?", (oc_ref,)
|
||
).fetchone()
|
||
if existing:
|
||
conn.close()
|
||
logger.info(
|
||
"Credit pack HOLD: %s already recorded (hold #%s); not duplicating",
|
||
oc_ref, existing[0],
|
||
)
|
||
return JSONResponse({
|
||
"status": "hold_already_recorded", "slug": slug,
|
||
"hold_id": existing[0],
|
||
})
|
||
conn.execute(
|
||
"INSERT INTO held_credits (slug, email, name, amount, oc_reference) "
|
||
"VALUES (?, ?, ?, ?, ?)",
|
||
(slug, buyer_email or None, buyer_name, amount, oc_ref),
|
||
)
|
||
try:
|
||
conn.commit()
|
||
except sqlite3.IntegrityError:
|
||
# Lost a race with a concurrent delivery of the same order;
|
||
# the unique index kept us from double-holding.
|
||
conn.close()
|
||
logger.warning(
|
||
"Credit pack HOLD: %s raced an existing hold; not duplicating", oc_ref,
|
||
)
|
||
return JSONResponse({"status": "hold_already_recorded", "slug": slug})
|
||
conn.close()
|
||
|
||
logger.warning(
|
||
"Credit pack HELD from non-member %s (%.2f credits); buyer notified",
|
||
slug, amount,
|
||
)
|
||
if buyer_email:
|
||
send_email(
|
||
buyer_email,
|
||
"Inference Cooperative — about your credit pack",
|
||
f"Hi {buyer_name},\n\n"
|
||
f"Thank you for your ${amount:.2f} credit pack purchase — we've received it.\n\n"
|
||
"One thing to flag: credit packs are for members, and our records\n"
|
||
"don't show you as one yet. You have two options:\n\n"
|
||
"1. Join the co-op ($10-20/month, sliding scale):\n"
|
||
" https://opencollective.com/inference-cooperative/contribute\n"
|
||
f" Once you're a member, your ${amount:.2f} in credits will be\n"
|
||
" applied to your account automatically.\n\n"
|
||
"2. Request a refund:\n"
|
||
" Reply to this email or write to info@inference.coop and\n"
|
||
" we'll refund you in full.\n\n"
|
||
"Sorry for the friction — memberships keep the co-op\n"
|
||
"member-governed, which is rather the point of the place.\n\n"
|
||
"— The Inference Cooperative\n"
|
||
"https://inference.coop",
|
||
)
|
||
else:
|
||
logger.warning("Credit-pack hold: no buyer email found for %s; no notification sent", slug)
|
||
|
||
return JSONResponse({
|
||
"status": "held",
|
||
"note": "credit pack purchased without membership; hold recorded and buyer notified",
|
||
})
|
||
|
||
if slug and "membership" in tier_name.lower() and event_type == "order.processed":
|
||
# Every Membership-tier payment (first or recurring) re-pins the
|
||
# member's allowance reset to payment + 30 days — the reset
|
||
# follows the member's actual billing rhythm, not a global
|
||
# calendar (all members landing on Oct 1 otherwise).
|
||
members = await fetch_members()
|
||
member = next((m for m in members if m["slug"] == slug), None)
|
||
if not member and slug:
|
||
# A FIRST-TIME contributor's OC profile is usually not in the
|
||
# collective's members list yet when their first order lands (OC
|
||
# reports the payer as guest-…/incognito-…). The slug lookup then
|
||
# finds nothing, and this branch used to fall straight through —
|
||
# SILENTLY. The webhook answered 200, nobody was provisioned,
|
||
# nothing was credited and nothing was logged. On 2026-11-13 that
|
||
# cost conor.canaday@gmail.com his account and his welcome email
|
||
# on a live $5/month membership; only the sweep's
|
||
# unmapped-backers check noticed, hours later.
|
||
payer = await fetch_account_by_slug(slug)
|
||
email = (payer or {}).get("email")
|
||
if email:
|
||
member = {"email": email,
|
||
"name": (payer or {}).get("name") or email.split("@")[0],
|
||
"slug": slug}
|
||
logger.warning(
|
||
"Webhook: membership order from %s is not in the OC members "
|
||
"list yet; resolved it to %s by account email", slug, email)
|
||
else:
|
||
logger.error(
|
||
"Webhook: membership order from %s could not be resolved to an "
|
||
"email (incognito profile); NOBODY was provisioned or credited — "
|
||
"the sweep's unmapped-backers check must report this", slug)
|
||
if member:
|
||
# A contribution may arrive under a DIFFERENT email than the
|
||
# portal account (a invited friend who joined under another OC
|
||
# address — linked via /admin/guest-convert): route the payment
|
||
# to the mapped account instead of minting a duplicate row.
|
||
oc_email = (member.get("email") or "").strip().lower()
|
||
if oc_email:
|
||
conn = get_db()
|
||
mapped = conn.execute(
|
||
"SELECT email FROM members WHERE oc_email = ?", (oc_email,)
|
||
).fetchone()
|
||
conn.close()
|
||
if mapped and mapped[0] != member["email"]:
|
||
member = {**member, "email": mapped[0]}
|
||
logger.info("Webhook: payment under %s mapped to portal account %s",
|
||
oc_email, mapped[0])
|
||
if first_payment:
|
||
try:
|
||
await provision_member(member["email"], member["name"], member["slug"])
|
||
# A lapsed member's team budget was zeroed on deactivation and
|
||
# get_or_create_team does not restore it — re-apply explicitly.
|
||
await sync_team_budget_from_balance(member["email"])
|
||
except Exception as e:
|
||
logger.warning("Webhook: failed to provision %s: %s", member["email"], e)
|
||
try:
|
||
# POLICY (2026-11-04): the allowance reset is anchored to the
|
||
# member's JOIN day, NOT the payment date. OC aligns charges
|
||
# onto a common billing day, so re-anchoring to the charge
|
||
# moved a renewal and skipped a cycle. A payment must only
|
||
# CONFIRM the cycle, never redefine where it falls.
|
||
team_id = await litellm_get_or_create_team(
|
||
member["email"], get_member_balance(member["email"]))
|
||
anchor = await anchor_day_for_member(member["email"], member["slug"])
|
||
await litellm_pin_reset_date(team_id, next_reset_for_anchor_day(anchor))
|
||
except Exception as e:
|
||
logger.warning("Webhook: failed to pin reset date for %s: %s",
|
||
member["email"], e)
|
||
elif slug:
|
||
# Organization-backed membership: OC orgs never expose an email
|
||
# to our integration, so fetch_members can't see them. Linked
|
||
# orgs (members.org_slug) get their account re-activated on
|
||
# dues; unlinked orgs ping the operator to link them.
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT email, org_slug FROM members WHERE org_slug = ?", (slug,)
|
||
).fetchone()
|
||
conn.close()
|
||
if row:
|
||
org_email, org_slug_col = row
|
||
conn = get_db()
|
||
conn.execute("UPDATE members SET active = 1 WHERE email = ?", (org_email,))
|
||
conn.commit()
|
||
conn.close()
|
||
logger.info("Webhook: org dues for %s — linked account %s re-activated",
|
||
slug, org_email)
|
||
if first_payment:
|
||
try:
|
||
await provision_member(org_email, name, slug)
|
||
await sync_team_budget_from_balance(org_email)
|
||
except Exception as e:
|
||
logger.warning("Webhook: org re-provision failed for %s: %s",
|
||
org_email, e)
|
||
else:
|
||
logger.warning(
|
||
"Webhook: Membership payment from ORGANIZATION %s (%s) — "
|
||
"no linked account; add via POST /admin/org-link",
|
||
name, slug,
|
||
)
|
||
send_email(
|
||
"info@inference.coop",
|
||
"Action needed: organizational membership without a linked account",
|
||
f"Open Collective reports a Membership-tier payment from the\n"
|
||
f"organization \"{name}\" (slug: {slug}).\n\n"
|
||
"Organizations don't expose an email to our integration, so the\n"
|
||
"membership needs to be linked to a portal account manually:\n\n"
|
||
" POST /admin/org-link\n"
|
||
" {\"org_slug\": \"" + slug + "\", \"email\": \"<their preferred email>\"}\n\n"
|
||
"Until linked, the organization cannot onboard.\n\n"
|
||
"— The portal",
|
||
)
|
||
elif slug and first_payment and event_type == "order.processed":
|
||
# One-time donation (no tier or non-membership tier): NOT a
|
||
# membership. Log it so we can see it, but don't provision.
|
||
logger.info("One-time donation from slug %s (tier=%r) — not a membership, not provisioning",
|
||
slug, tier_name)
|
||
return JSONResponse({"status": "provisioned", "name": name, "slug": slug})
|
||
|
||
if event_type in ("collective.member.deleted", "collective.transaction.deleted"):
|
||
# Lapsed member → move to the inactive group (lose access, keep data).
|
||
# The webhook gives us the slug; we map it to the member's email/user.
|
||
if slug:
|
||
member = get_member_by_slug(slug)
|
||
if member:
|
||
# Move to inactive group (revokes chat access, keeps account + data)
|
||
if INACTIVE_GROUP_ID and member.get("cloudron_user_id"):
|
||
await cloudron_set_group(member["cloudron_user_id"], INACTIVE_GROUP_ID)
|
||
# Mark inactive in our DB (key injector will refuse requests)
|
||
deactivate_member(member["email"])
|
||
await revoke_member_litellm_access(member["email"])
|
||
await listmonk_sync(member["email"], subscribe=False)
|
||
await sync_loomio_memberships()
|
||
await sync_matrix_space()
|
||
logger.info("Deactivated member %s (slug=%s)", member["email"], slug)
|
||
return JSONResponse({"status": "deactivated", "email": member["email"]})
|
||
return JSONResponse({"status": "deactivated", "note": "no matching member"})
|
||
|
||
return JSONResponse({"status": "ignored", "type": event_type})
|
||
|
||
|
||
# --- B. Key injector (proxy) ---
|
||
|
||
# --- Model picker: value-based naming ---
|
||
# The chat model picker displays each model's `name` field. We set it to a
|
||
# friendly name prefixed with an emoji for the provider's value tier, so the
|
||
# flat picker list clusters by what members care about (privacy / green /
|
||
# openness). The `id` field is left untouched — it remains the canonical key
|
||
# used for actual API calls. Same emoji as the docs (models.md).
|
||
# Emoji marks the FEATURE tier, not the provider: 🌱 = green/renewable-powered
|
||
# (GreenPT and Regolo both), 🔒 = TEE-architected private (Tinfoil).
|
||
MODEL_VALUE_EMOJI = {
|
||
"tinfoil": "🔒", # architecturally private (TEE)
|
||
"greenpt": "🌱", # 100% renewable energy
|
||
"regolo": "🌱", # EU-hosted, zero data retention, 100% renewables
|
||
}
|
||
MODEL_FRIENDLY_NAMES = {
|
||
"greenpt/deepseek-v4.1-flash": "DeepSeek V4.1",
|
||
"tinfoil/deepseek-v4-1-flash": "DeepSeek V4.1",
|
||
"tinfoil/glm-5.3": "GLM-5.3",
|
||
"greenpt/glm-5.3-flash": "GLM-5.3 Flash",
|
||
"greenpt/glm-5.3": "GLM-5.3",
|
||
"tinfoil/gpt-oss-120b": "GPT-OSS 120B",
|
||
"regolo/apertus-70b": "Apertus 70B",
|
||
"regolo/qwen3.5-122b": "Qwen 3.5 122B",
|
||
}
|
||
|
||
|
||
@app.get("/v1/models")
|
||
@limiter.limit("60/minute")
|
||
async def list_models(request: Request):
|
||
"""List models (same for everyone — no per-user auth needed).
|
||
|
||
Annotates each model with OpenWebUI metadata so web search is enabled by
|
||
default (defaultFeatureIds includes 'web_search'). Without this, OpenWebUI
|
||
leaves web search off for these models and members must toggle it manually.
|
||
"""
|
||
async with httpx.AsyncClient() as client:
|
||
r = await client.get(
|
||
f"{LITELLM_BASE}/v1/models",
|
||
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
|
||
)
|
||
if r.status_code != 200:
|
||
return Response(
|
||
content=r.content,
|
||
status_code=r.status_code,
|
||
headers={"content-type": r.headers.get("content-type", "application/json")},
|
||
)
|
||
|
||
payload = r.json()
|
||
|
||
# Chat-list filter: exclude audio-only models (STT/TTS) and embedding
|
||
# models. They are registered and callable via their own routes
|
||
# (/v1/audio/* and /v1/embeddings) for API users, but they are not chat
|
||
# models — listing them in Open WebUI would just confuse members. The
|
||
# gateway's model_info carries `mode` for these; fall back to a name
|
||
# check for safety.
|
||
audio_modes = {"audio_transcription", "audio_speech"}
|
||
audio_name_hints = ("whisper", "voxtral", "tts")
|
||
embedding_name_hints = ("embed",)
|
||
# Image-generation models are used through Open WebUI's image feature
|
||
# (Admin > Images), not as chat models — listing them in the picker
|
||
# would fail on chat completions.
|
||
image_modes = {"image_generation"}
|
||
|
||
def is_audio_model(m) -> bool:
|
||
mode = ((m.get("info") or {}).get("meta") or {}).get("mode") or ""
|
||
if mode in audio_modes:
|
||
return True
|
||
mid = (m.get("id") or "").lower()
|
||
return any(h in mid for h in audio_name_hints)
|
||
|
||
def is_embedding_model(m) -> bool:
|
||
mode = ((m.get("info") or {}).get("meta") or {}).get("mode") or ""
|
||
if mode == "embedding":
|
||
return True
|
||
mid = (m.get("id") or "").lower()
|
||
return any(h in mid for h in embedding_name_hints)
|
||
|
||
image_name_hints = ("qwen-image",)
|
||
|
||
def is_image_model(m) -> bool:
|
||
mode = ((m.get("info") or {}).get("meta") or {}).get("mode") or ""
|
||
if mode in image_modes:
|
||
return True
|
||
mid = (m.get("id") or "").lower()
|
||
return any(h in mid for h in image_name_hints)
|
||
|
||
payload["data"] = [
|
||
m for m in payload.get("data", [])
|
||
if not is_audio_model(m) and not is_embedding_model(m) and not is_image_model(m)
|
||
]
|
||
|
||
for model in payload.get("data", []):
|
||
# Friendly display name: emoji value-tier + short name. The `id`
|
||
# stays canonical (Open WebUI keys lookups and requests on it).
|
||
mid = model.get("id", "")
|
||
provider = mid.split("/", 1)[0] if "/" in mid else ""
|
||
emoji = MODEL_VALUE_EMOJI.get(provider, "")
|
||
friendly = MODEL_FRIENDLY_NAMES.get(
|
||
mid, mid.split("/", 1)[-1] if "/" in mid else mid
|
||
)
|
||
model["name"] = f"{emoji} {friendly}".strip()
|
||
|
||
model.setdefault("info", {}).setdefault("meta", {})
|
||
model["info"]["meta"].setdefault("capabilities", {})["web_search"] = True
|
||
model["info"]["meta"].setdefault("defaultFeatureIds", [])
|
||
if "web_search" not in model["info"]["meta"]["defaultFeatureIds"]:
|
||
model["info"]["meta"]["defaultFeatureIds"].append("web_search")
|
||
|
||
# Attach the co-op docs knowledge collection to every model. This
|
||
# makes Open WebUI treat the docs as the model's own knowledge:
|
||
# automatic RAG injection grounds answers without tool-calling, and
|
||
# the knowledge toolset is scoped to the attached collection instead
|
||
# of exposing browse-all-knowledge tools (the "explored
|
||
# search_knowledge_files… view_knowledge_file" wandering that new
|
||
# members saw from the getting-started prompt). Env-driven so the
|
||
# collection id can be rotated without a code change.
|
||
if DOCS_KNOWLEDGE_ID:
|
||
model["info"]["meta"].setdefault("knowledge", [])
|
||
kb_entry = {"id": DOCS_KNOWLEDGE_ID, "type": "collection"}
|
||
if kb_entry not in model["info"]["meta"]["knowledge"]:
|
||
model["info"]["meta"]["knowledge"].append(kb_entry)
|
||
|
||
# SILENT AUTO-RAG (legacy function-calling mode), chosen 2026-10-09:
|
||
# with native FC the model gets a query_knowledge_files tool and
|
||
# calls it on every prompt — including off-topic ones — with
|
||
# visible "let me check the docs" narration. Prompt steering
|
||
# failed twice (soft + firm). Legacy mode instead auto-injects
|
||
# relevant docs chunks into context on every message: silent,
|
||
# no tool-calls, no announcements, grounding always on. Cost:
|
||
# one embedding + vector search per message (~300ms, paid even
|
||
# on off-topic messages). builtinTools.knowledge=false removes
|
||
# the knowledge tool + <attached_knowledge> tag so the model
|
||
# can't narrate or fumble; the injected sources still ground
|
||
# co-op answers.
|
||
model.setdefault("params", {})
|
||
model["params"]["function_calling"] = "legacy"
|
||
model["info"]["meta"]["builtinTools"] = {"knowledge": False}
|
||
|
||
# Steering: sources arrive as plain context. Tell the model the
|
||
# docs may be present but are co-op-only, so it never cites or
|
||
# mentions them for off-topic answers.
|
||
steering = (
|
||
"Some messages include excerpts from the Inference "
|
||
"Cooperative's own documentation (membership, models, API, "
|
||
"governance), added automatically as background context. "
|
||
"Use those excerpts only when they are relevant to the "
|
||
"user's actual question. For general questions, ignore them "
|
||
"and answer from your own knowledge — do not mention the "
|
||
"co-op, its documentation, or any knowledge search unless "
|
||
"the user explicitly asks about the co-op."
|
||
)
|
||
existing = model["params"].get("system")
|
||
if existing:
|
||
if "Inference Cooperative" not in existing:
|
||
model["params"]["system"] = existing + "\n\n" + steering
|
||
else:
|
||
model["params"]["system"] = steering
|
||
|
||
return JSONResponse(content=payload)
|
||
|
||
|
||
@app.api_route("/v1/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH"])
|
||
@limiter.limit("120/minute")
|
||
async def inject_key(request: Request, path: str):
|
||
"""Read the member's email from a header, inject their key, forward to LiteLLM."""
|
||
# Verify the request came through the chat frontend, so a direct caller
|
||
# can't spoof the user-email header and use another member's key.
|
||
#
|
||
# ONE auth path: LibreChat sends a plain X-User-Email header plus an
|
||
# X-Portal-Secret shared-secret header. OpenWebUI's signed-JWT path was
|
||
# retired with the app (2026-11-13) — honouring a shared-secret-signed token
|
||
# for an integration nobody runs is auth surface with no user.
|
||
_verify_portal_secret(request)
|
||
email = (
|
||
request.headers.get("x-user-email", "")
|
||
or request.headers.get("x-openwebui-user-email", "")
|
||
)
|
||
|
||
if not email:
|
||
raise HTTPException(401, "No member identity (user-email header)")
|
||
|
||
# Look up the member's key from the persistent store
|
||
member_key = get_member_key(email)
|
||
|
||
if not member_key:
|
||
raise HTTPException(403, "No active membership key")
|
||
|
||
# Forward the request to LiteLLM with the member's key, STREAMING the
|
||
# response back token-by-token (or byte-by-byte) rather than buffering it.
|
||
# Buffering collapsed the model's stream and handed OpenWebUI the full
|
||
# answer in one blob, so members saw nothing until generation finished —
|
||
# which made chat feel slow even though time-to-first-byte was fine.
|
||
body = await request.body()
|
||
headers = dict(request.headers)
|
||
headers["authorization"] = f"Bearer {member_key}"
|
||
headers.pop("host", None)
|
||
headers.pop("content-length", None)
|
||
headers.pop("accept-encoding", None) # let httpx handle decompression
|
||
|
||
timeout = httpx.Timeout(connect=15.0, read=600.0, write=30.0, pool=15.0)
|
||
|
||
# Send the request upstream, but don't buffer the body — use a stream.
|
||
client = httpx.AsyncClient(timeout=timeout)
|
||
upstream_req = client.build_request(
|
||
method=request.method,
|
||
url=f"{LITELLM_BASE}/v1/{path}",
|
||
headers=headers,
|
||
content=body,
|
||
)
|
||
upstream = await client.send(upstream_req, stream=True)
|
||
|
||
async def gen():
|
||
try:
|
||
async for chunk in upstream.aiter_raw():
|
||
yield chunk
|
||
finally:
|
||
await upstream.aclose()
|
||
await client.aclose()
|
||
|
||
# Pass through the content-type (critical for SSE streaming) and status.
|
||
return StreamingResponse(
|
||
gen(),
|
||
status_code=upstream.status_code,
|
||
headers={
|
||
"content-type": upstream.headers.get("content-type", "application/json"),
|
||
"cache-control": "no-cache",
|
||
},
|
||
)
|
||
|
||
|
||
# --- D. Admin / health ---
|
||
|
||
@app.get("/health")
|
||
async def health():
|
||
return {"status": "ok"}
|
||
|
||
|
||
@app.post("/admin/sync-loomio")
|
||
@limiter.limit("20/minute")
|
||
async def admin_sync_loomio(request: Request):
|
||
"""Manually trigger a Loomio membership sync.
|
||
|
||
Authenticated by the X-Admin-Token header. Useful for reconciling
|
||
pre-existing members (accounts created before the sync existed) or
|
||
recovering from a missed webhook. Idempotent — safe to call repeatedly.
|
||
"""
|
||
_verify_admin_token(request)
|
||
|
||
await sync_loomio_memberships()
|
||
await sync_matrix_space()
|
||
return {"status": "synced"}
|
||
|
||
|
||
@app.post("/admin/provision")
|
||
@limiter.limit("20/minute")
|
||
async def admin_provision(request: Request):
|
||
"""Manually provision a member by email (full pipeline).
|
||
|
||
Authenticated by the X-Admin-Token header. Runs the full provisioning
|
||
pipeline: Cloudron user + members group + LiteLLM key + our welcome email +
|
||
Loomio sync. Body: {"email": "...", "name": "..."}. Idempotent.
|
||
"""
|
||
_verify_admin_token(request)
|
||
|
||
body = await request.json()
|
||
email = (body.get("email") or "").strip().lower()
|
||
name = body.get("name") or email.split("@")[0]
|
||
if not email:
|
||
raise HTTPException(400, "Missing email")
|
||
|
||
user_id = await provision_member(email, name, "")
|
||
return {"status": "provisioned", "email": email, "user_id": user_id}
|
||
|
||
|
||
@app.post("/admin/provision-guest")
|
||
@limiter.limit("20/minute")
|
||
async def admin_provision_guest(request: Request):
|
||
"""Provision a GUEST account (the admin panel's invite action).
|
||
|
||
Same auth as /admin/provision (X-Admin-Token). Body: {"email": "..."} —
|
||
ONE field, the standard guest-invite form. No name is collected and none
|
||
is derived: the guest chooses their own display name and username during
|
||
Cloudron account setup, and the invitation email greets neutrally. The
|
||
inviter is always the co-op's ADMIN account
|
||
(ADMIN_INVITER_EMAIL, default info@inference.coop) — the panel has no
|
||
inviter input, and the ledger records the admin account.
|
||
|
||
Runs the guest pipeline: Cloudron user in the guests group, a $2 LiteLLM
|
||
team with NO monthly reset, the portal row marked as a guest with a
|
||
GUEST_DAYS expiry, the guest welcome email, the newsletter subscription,
|
||
and a guests-ledger row recording inviter → recipient. Admin invites
|
||
provision IMMEDIATELY (no pending state — the operator IS the approver)
|
||
and are NOT bound by the member's one-per-30-days cap; the recipient-side
|
||
guards still run (one guest per recipient ever, no existing portal row).
|
||
No Loomio/Matrix.
|
||
"""
|
||
_verify_admin_token(request)
|
||
|
||
body = await request.json()
|
||
email = (body.get("email") or "").strip().lower()
|
||
# No name is collected and none is derived: the guest chooses their own
|
||
# display name and username during Cloudron account setup, and the
|
||
# invitation email greets neutrally.
|
||
name = (body.get("name") or "").strip()
|
||
if not email or "@" not in email:
|
||
raise HTTPException(400, "A valid recipient email is required")
|
||
# The inviter is the admin account, never a caller-supplied member email:
|
||
# this endpoint speaks for the operator, and admins have no monthly cap.
|
||
inviter = ADMIN_INVITER_EMAIL
|
||
|
||
user_id = await provision_guest_member(
|
||
email, name, inviter, admin=True, inviter_display="The co-op"
|
||
)
|
||
return {
|
||
"status": "provisioned", "account_type": "guest", "email": email,
|
||
"inviter": inviter, "user_id": user_id, "budget": GUEST_BUDGET,
|
||
"expires_in_days": int(GUEST_DAYS),
|
||
}
|
||
|
||
|
||
@app.post("/admin/guest-convert")
|
||
@limiter.limit("20/minute")
|
||
async def admin_guest_convert(request: Request):
|
||
"""Convert a guest account to a member when OC cannot auto-match.
|
||
|
||
The automatic conversion matches the guest's OWN email against the live OC
|
||
backer list. When the membership was paid under a DIFFERENT Open
|
||
Collective email, the operator links it here: {"email": "<guest account
|
||
email>", "oc_email": "<the OC email the contribution is under>"}. The OC
|
||
member record (name, slug) is resolved from oc_email and stored on the row
|
||
(members.oc_email) so the sweep and the webhook keep matching renewals
|
||
under that address. Omitting oc_email converts on the account's own email
|
||
without an OC lookup (the operator asserts the membership exists; the row
|
||
is then matched by its own email as usual).
|
||
"""
|
||
_verify_admin_token(request)
|
||
|
||
body = await request.json()
|
||
email = (body.get("email") or "").strip().lower()
|
||
oc_email = (body.get("oc_email") or "").strip().lower()
|
||
if not email or "@" not in email:
|
||
raise HTTPException(400, "A valid guest account email is required")
|
||
if oc_email and "@" not in oc_email:
|
||
raise HTTPException(400, "oc_email must be a valid email")
|
||
if not is_guest_account(email):
|
||
raise HTTPException(409, f"{email} is not a guest account (already converted or a member row)")
|
||
|
||
name, slug = "", ""
|
||
if oc_email:
|
||
rec = next((m for m in await fetch_members()
|
||
if (m.get("email") or "").strip().lower() == oc_email), None)
|
||
if not rec:
|
||
raise HTTPException(404, f"No Open Collective member found under {oc_email}")
|
||
name = rec.get("name") or ""
|
||
slug = rec.get("slug") or ""
|
||
conn = get_db()
|
||
conn.execute("UPDATE members SET oc_email = ? WHERE email = ?", (oc_email, email))
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
user_id = await convert_guest_to_member(email, name or email.split("@")[0], slug)
|
||
return {
|
||
"status": "converted", "email": email, "oc_email": oc_email or None,
|
||
"user_id": user_id,
|
||
"note": ("account keeps its own email and login; group moved guests → members; "
|
||
"cap lifted to the member allowance with unspent guest credit carried; "
|
||
"anchor day (join day) kept"),
|
||
}
|
||
|
||
|
||
# TRANSITION ALIASES (guest rename, 2026-11-08): the deployed dashboard and
|
||
# panel builds predate the guest rename and still call the old paths. These
|
||
# answer identically (auth runs inside the real handler) until the renamed
|
||
# builds are deployed; REMOVE once no caller remains.
|
||
@app.post("/admin/provision-gift")
|
||
@limiter.limit("20/minute")
|
||
async def admin_provision_gift_alias(request: Request):
|
||
return await admin_provision_guest(request)
|
||
|
||
|
||
@app.post("/admin/gift-convert")
|
||
@limiter.limit("20/minute")
|
||
async def admin_gift_convert_alias(request: Request):
|
||
return await admin_guest_convert(request)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Guests ledger (the approval gate was removed on 2026-11-08: member
|
||
# invitations provision immediately, so there is nothing to approve).
|
||
# ---------------------------------------------------------------------------
|
||
|
||
@app.get("/admin/guest-ledger")
|
||
@limiter.limit("60/minute")
|
||
async def admin_guest_ledger(request: Request):
|
||
"""The guests ledger, for the panel's duplicate-send guard.
|
||
|
||
Authenticated by the X-Admin-Token header. Returns the guests ledger
|
||
(inviter → recipient, newest first) so the panel can refuse a second
|
||
guest for an address that already holds an ACTIVE account BEFORE calling
|
||
/admin/provision-guest. The portal's own recipient-side guards remain
|
||
the last authority behind it.
|
||
"""
|
||
_verify_admin_token(request)
|
||
conn = get_db()
|
||
ledger = conn.execute(
|
||
"SELECT id, inviter_email, recipient_email, created_at, status "
|
||
"FROM guests ORDER BY id DESC LIMIT 100"
|
||
).fetchall()
|
||
conn.close()
|
||
return {
|
||
"ledger": [
|
||
{"id": i, "inviter": inv, "recipient": rec, "created_at": created,
|
||
"status": status}
|
||
for i, inv, rec, created, status in ledger
|
||
],
|
||
}
|
||
|
||
|
||
@app.post("/admin/org-link")
|
||
@limiter.limit("20/minute")
|
||
async def admin_org_link(request: Request):
|
||
"""Link an OC organizational membership to a portal account.
|
||
|
||
OC Organizations never expose an email to our integration, so org-backed
|
||
memberships are linked manually: body {"org_slug": "...", "email": "...",
|
||
"name": "..."}. Creates (or reuses) the members row for the email with
|
||
org_slug set, and provisions the full pipeline — Cloudron user, LiteLLM
|
||
key, and the welcome email containing the Cloudron account-setup link, so
|
||
the organization's contact can complete signup. The nightly sweep keeps
|
||
the account active while the org holds the Membership tier and deactivates
|
||
it when the org's dues lapse (dues monitoring, automatic).
|
||
"""
|
||
_verify_admin_token(request)
|
||
|
||
body = await request.json()
|
||
org_slug = (body.get("org_slug") or "").strip().lower()
|
||
email = (body.get("email") or "").strip().lower()
|
||
name = (body.get("name") or org_slug).strip()
|
||
if not org_slug or not email or "@" not in email:
|
||
raise HTTPException(400, "org_slug and a valid email are required")
|
||
|
||
try:
|
||
org = (await fetch_org_backers()).get(org_slug)
|
||
except Exception as e:
|
||
# OC unreachable: link without provisioning — the sweep provisions and
|
||
# activates once OC is reachable again.
|
||
logger.warning("org-link: OC status fetch failed for %s (linking anyway): %s", org_slug, e)
|
||
org = None
|
||
|
||
conn = get_db()
|
||
existing = conn.execute("SELECT email FROM members WHERE org_slug = ?", (org_slug,)).fetchone()
|
||
# slug = the org slug (this row represents the org's contact, not an OC
|
||
# individual — provision_member() overwrites slug/cloudron_user_id/key for
|
||
# emails that resolve to a real OC account).
|
||
conn.execute(
|
||
"INSERT INTO members (email, slug, active, org_slug) VALUES (?, ?, 1, ?) "
|
||
"ON CONFLICT(email) DO UPDATE SET org_slug = excluded.org_slug, active = 1",
|
||
(email, org_slug, org_slug),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
provisioned = False
|
||
if org: # only provision while the org's membership is active
|
||
try:
|
||
await provision_member(email, name, org_slug)
|
||
provisioned = True
|
||
except Exception as e:
|
||
logger.warning("org-link: provision failed for %s: %s", email, e)
|
||
|
||
return {
|
||
"status": "linked",
|
||
"org_slug": org_slug,
|
||
"email": email,
|
||
"org_active": org is not None,
|
||
"tier": (org or {}).get("tier"),
|
||
"provisioned": provisioned,
|
||
"replaced_existing_link": bool(existing),
|
||
"note": "the nightly sweep deactivates this account automatically if the org's dues lapse",
|
||
}
|
||
|
||
|
||
@app.get("/admin/org-links")
|
||
@limiter.limit("30/minute")
|
||
async def admin_org_links(request: Request):
|
||
"""List organization-linked accounts with their live OC dues status."""
|
||
_verify_admin_token(request)
|
||
|
||
org_backers = await fetch_org_backers()
|
||
conn = get_db()
|
||
rows = conn.execute(
|
||
"SELECT email, org_slug, active FROM members WHERE org_slug IS NOT NULL"
|
||
).fetchall()
|
||
conn.close()
|
||
links = []
|
||
for email, org_slug, active in rows:
|
||
org = org_backers.get(org_slug)
|
||
links.append({
|
||
"org_slug": org_slug,
|
||
"org_name": (org or {}).get("name") or org_slug,
|
||
"email": email,
|
||
"account_active": bool(active),
|
||
"dues_active": org is not None,
|
||
"tier": (org or {}).get("tier"),
|
||
})
|
||
return {"links": links}
|
||
|
||
|
||
@app.post("/admin/set-balance")
|
||
@limiter.limit("20/minute")
|
||
async def admin_set_balance(request: Request):
|
||
"""Set (or add to) a member's credit balance and sync it to LiteLLM.
|
||
|
||
The flexible alternative to a fixed $15/month allowance. A payment (from OC
|
||
or another system) is recorded by adjusting the balance here, which pushes
|
||
the new cap to the member's team (every chat/API key under it follows).
|
||
|
||
Body: {"email": "...", "balance": 25.0} → set absolute balance
|
||
{"email": "...", "add": 10.0} → add to current balance
|
||
"""
|
||
_verify_admin_token(request)
|
||
|
||
body = await request.json()
|
||
email = (body.get("email") or "").strip().lower()
|
||
if not email:
|
||
raise HTTPException(400, "Missing email")
|
||
|
||
current = get_member_balance(email)
|
||
if "add" in body:
|
||
new_balance = current + float(body["add"])
|
||
elif "balance" in body:
|
||
new_balance = float(body["balance"])
|
||
else:
|
||
raise HTTPException(400, "Provide 'balance' or 'add'")
|
||
|
||
if new_balance < 0:
|
||
raise HTTPException(400, "Balance cannot be negative")
|
||
|
||
set_member_balance(email, new_balance)
|
||
applied = await sync_team_budget_from_balance(email)
|
||
return {"status": "updated", "email": email, "balance": new_balance, "team_budget": applied}
|
||
|
||
|
||
@app.post("/admin/set-credits")
|
||
@limiter.limit("20/minute")
|
||
async def admin_set_credits(request: Request):
|
||
"""Adjust a member's non-expiring credit balance (with ledger entry).
|
||
|
||
Body: {"email": "...", "add": 10.0, "reason": "goodwill"} → add/subtract
|
||
{"email": "...", "set": 0.0, "reason": "correction"} → set absolute
|
||
"""
|
||
_verify_admin_token(request)
|
||
|
||
body = await request.json()
|
||
email = (body.get("email") or "").strip().lower()
|
||
if not email:
|
||
raise HTTPException(400, "Missing email")
|
||
|
||
reason = (body.get("reason") or "admin adjustment").strip()
|
||
if "add" in body:
|
||
delta = float(body["add"])
|
||
new_balance = add_credits(email, delta, reason=reason)
|
||
elif "set" in body:
|
||
target = float(body["set"])
|
||
current = get_credit_balance(email)
|
||
new_balance = add_credits(email, target - current, reason=reason)
|
||
else:
|
||
raise HTTPException(400, "Provide 'add' or 'set'")
|
||
|
||
applied = await sync_team_budget_from_balance(email)
|
||
return {
|
||
"status": "updated", "email": email,
|
||
"credit_balance": new_balance, "team_budget": applied,
|
||
}
|
||
|
||
|
||
@app.get("/admin/credits/{email}")
|
||
@limiter.limit("30/minute")
|
||
async def admin_credit_ledger(email: str, request: Request):
|
||
"""Return a member's credit balance and full ledger (audit trail)."""
|
||
_verify_admin_token(request)
|
||
conn = get_db()
|
||
rows = conn.execute(
|
||
"SELECT delta, reason, reference, created_at FROM credit_ledger "
|
||
"WHERE email = ? ORDER BY created_at DESC",
|
||
(email,),
|
||
).fetchall()
|
||
conn.close()
|
||
return {
|
||
"email": email,
|
||
"credit_balance": get_credit_balance(email),
|
||
"ledger": [
|
||
{"delta": r[0], "reason": r[1], "reference": r[2], "created_at": r[3]}
|
||
for r in rows
|
||
],
|
||
}
|
||
|
||
|
||
@app.get("/admin/held-credits")
|
||
@limiter.limit("30/minute")
|
||
async def admin_held_credits(request: Request):
|
||
"""List held credit packs (non-member purchases awaiting refund or grant)."""
|
||
_verify_admin_token(request)
|
||
conn = get_db()
|
||
rows = conn.execute(
|
||
"SELECT id, slug, email, name, amount, oc_reference, status, created_at, resolved_at "
|
||
"FROM held_credits ORDER BY created_at DESC"
|
||
).fetchall()
|
||
conn.close()
|
||
return {
|
||
"held": [
|
||
{"id": r[0], "slug": r[1], "email": r[2], "name": r[3], "amount": r[4],
|
||
"reference": r[5], "status": r[6], "created_at": r[7], "resolved_at": r[8]}
|
||
for r in rows
|
||
]
|
||
}
|
||
|
||
|
||
@app.post("/admin/held-credits/{hold_id}/refund")
|
||
@limiter.limit("20/minute")
|
||
async def admin_held_refund(hold_id: int, request: Request):
|
||
"""Mark a held credit pack as refunded (the refund itself is done manually
|
||
in the Open Collective dashboard — this records it and updates the hold)."""
|
||
_verify_admin_token(request)
|
||
conn = get_db()
|
||
cur = conn.execute(
|
||
"UPDATE held_credits SET status = 'refunded', resolved_at = datetime('now') "
|
||
"WHERE id = ? AND status = 'held'",
|
||
(hold_id,),
|
||
)
|
||
conn.commit()
|
||
updated = cur.rowcount
|
||
conn.close()
|
||
if not updated:
|
||
raise HTTPException(404, "Held credit not found (or already resolved)")
|
||
return {"status": "refunded", "id": hold_id}
|
||
|
||
|
||
@app.post("/admin/held-credits/{hold_id}/grant")
|
||
@limiter.limit("20/minute")
|
||
async def admin_held_grant(hold_id: int, request: Request):
|
||
"""Apply a held credit pack to a specified member account.
|
||
|
||
Body: {"email": "member@..."} — the member account that should receive the
|
||
held credits. This covers purchases made by someone other than the member
|
||
(e.g. a credit pack bought on another account's behalf), which the
|
||
provisioning-time auto-grant cannot match because it keys on the buyer's
|
||
own email/slug.
|
||
|
||
Atomic and idempotent: the balance update and the ledger insert happen in
|
||
one transaction, and the hold is flipped `held` → `granted` in the same
|
||
transaction. The ledger reference is `held:<hold_id>` (the same shape the
|
||
provisioning-time auto-grant uses), so a re-run cannot double-apply.
|
||
"""
|
||
_verify_admin_token(request)
|
||
|
||
body = await request.json()
|
||
email = (body.get("email") or "").strip().lower()
|
||
if not email:
|
||
raise HTTPException(400, "Missing email")
|
||
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT id, slug, email, name, amount, oc_reference, status "
|
||
"FROM held_credits WHERE id = ?",
|
||
(hold_id,),
|
||
).fetchone()
|
||
if not row:
|
||
conn.close()
|
||
raise HTTPException(404, "Held credit not found")
|
||
if row[6] != "held":
|
||
conn.close()
|
||
raise HTTPException(409, f"Held credit already {row[6]}")
|
||
|
||
dest = conn.execute(
|
||
"SELECT active FROM members WHERE LOWER(email) = ?", (email,)
|
||
).fetchone()
|
||
if not dest:
|
||
conn.close()
|
||
raise HTTPException(404, f"No member account for {email}")
|
||
if not dest[0]:
|
||
conn.close()
|
||
raise HTTPException(409, f"Member {email} is not active; refusing")
|
||
|
||
amount = float(row[4] or 0.0)
|
||
reference = f"held:{hold_id}"
|
||
if credit_reference_exists(reference):
|
||
conn.close()
|
||
raise HTTPException(409, f"Credit already recorded for {reference}")
|
||
|
||
buyer = row[2] or row[1]
|
||
buyer_name = row[3] or row[1]
|
||
reason = (
|
||
f"credit pack (${amount:.2f}) bought by {buyer} ({buyer_name}); "
|
||
f"held credit transferred to this member account at the operator's "
|
||
f"instruction (hold #{hold_id}, {row[5] or 'no oc ref'})"
|
||
)
|
||
# One transaction: flip the hold, move the balance, write the audit row.
|
||
cur = conn.execute(
|
||
"UPDATE held_credits SET status = 'granted', resolved_at = datetime('now') "
|
||
"WHERE id = ? AND status = 'held'",
|
||
(hold_id,),
|
||
)
|
||
if cur.rowcount != 1:
|
||
conn.close()
|
||
raise HTTPException(409, "Held credit already resolved")
|
||
conn.execute(
|
||
"UPDATE members SET credit_balance = MAX(0, COALESCE(credit_balance, 0) + ?) "
|
||
"WHERE LOWER(email) = ?",
|
||
(amount, email),
|
||
)
|
||
conn.execute(
|
||
"INSERT INTO credit_ledger (email, delta, reason, reference) VALUES (?, ?, ?, ?)",
|
||
(email, amount, reason, reference),
|
||
)
|
||
new_balance = conn.execute(
|
||
"SELECT credit_balance FROM members WHERE LOWER(email) = ?", (email,)
|
||
).fetchone()[0]
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
applied = await sync_team_budget_from_balance(email)
|
||
logger.info(
|
||
"Held credit #%s ($%.2f) from %s granted to %s (balance %s)",
|
||
hold_id, amount, buyer, email, new_balance,
|
||
)
|
||
return {
|
||
"status": "granted",
|
||
"hold_id": hold_id,
|
||
"email": email,
|
||
"credits_added": amount,
|
||
"credit_balance": float(new_balance),
|
||
"reference": reference,
|
||
"oc_reference": row[5],
|
||
"team_budget": applied,
|
||
}
|
||
|
||
|
||
# Operator accounts — shown under a separate "admin" status in the panel and
|
||
# excluded from the member counts. They're infrastructure accounts (the SSO
|
||
# owner and the bot), not dues-paying members.
|
||
OPERATOR_EMAILS = {
|
||
e.strip().lower()
|
||
for e in os.environ.get("OPERATOR_EMAILS", "info@inference.coop,bot@inference.coop").split(",")
|
||
if e.strip()
|
||
}
|
||
|
||
|
||
@app.get("/admin/overview")
|
||
@limiter.limit("30/minute")
|
||
async def admin_overview(request: Request):
|
||
"""Return a co-op-wide overview for the admin dashboard.
|
||
|
||
Authenticated by the X-Admin-Token header. Returns every member (active +
|
||
inactive) with their balance, spend, remaining, and team reset date, plus
|
||
co-op aggregates (total members, total spend). Guest (guest) accounts come
|
||
back in their own `guests` list — never inside `members` — so no consumer
|
||
can count them as members by accident; their spend is broken out in the
|
||
totals rather than folded into the member figures.
|
||
"""
|
||
_verify_admin_token(request)
|
||
|
||
conn = get_db()
|
||
rows = conn.execute(
|
||
"SELECT email, active, balance, cloudron_user_id, org_slug, matrix_mxid, "
|
||
"account_type, oc_email, guest_expires_at FROM members ORDER BY email"
|
||
).fetchall()
|
||
# Guest inviter/expiry from the guests ledger (latest row per recipient) —
|
||
# the abuse-protection context the panel shows on the guest pill, plus the
|
||
# invite date for the invited (awaiting setup) reason line.
|
||
guest_info: dict[str, dict] = {}
|
||
try:
|
||
for r in conn.execute(
|
||
"SELECT recipient_email, inviter_email, guest_expires_at, status, created_at "
|
||
"FROM guests WHERE id IN (SELECT MAX(id) FROM guests GROUP BY recipient_email)"
|
||
).fetchall():
|
||
guest_info[r[0]] = {"inviter": r[1], "expires": r[2], "status": r[3],
|
||
"sent": r[4]}
|
||
except Exception as e:
|
||
logger.warning("admin overview: guests ledger read failed: %s", e)
|
||
conn.close()
|
||
|
||
# Fetch Cloudron users once to map user_id → username + inviteAccepted.
|
||
# username is set only after the member completes account setup (chooses a
|
||
# username + password via the setup link); inviteAccepted tracks that same
|
||
# step, so "activated" = inviteAccepted (username present).
|
||
usernames = {}
|
||
invite_accepted = {}
|
||
try:
|
||
async with httpx.AsyncClient(timeout=15.0) as client:
|
||
users = await cloudron_fetch_all_users(client)
|
||
if users is None:
|
||
raise RuntimeError("cloudron users fetch failed (page 1)")
|
||
for u in users:
|
||
uid = u.get("id")
|
||
if u.get("username"):
|
||
usernames[uid] = u["username"]
|
||
invite_accepted[uid] = bool(u.get("inviteAccepted"))
|
||
except Exception as e:
|
||
logger.warning("admin overview: cloudron users fetch failed: %s", e)
|
||
|
||
# Fetch team spend/budget from LiteLLM once.
|
||
teams = {}
|
||
try:
|
||
async with httpx.AsyncClient(timeout=15.0) as client:
|
||
r = await client.get(
|
||
f"{LITELLM_BASE}/team/list",
|
||
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
|
||
)
|
||
r.raise_for_status()
|
||
for t in r.json():
|
||
teams[t.get("team_alias")] = t
|
||
except Exception as e:
|
||
logger.warning("admin overview: team/list failed: %s", e)
|
||
|
||
# All-time spend per team, from SpendLogs (the team `spend` counter is
|
||
# cycle-only and resets at each team's budget_reset_at — the panel needs both
|
||
# to avoid the "month change wiped the numbers" confusion).
|
||
# Covers agent teams as well as members: an agent's cycle counter resets the
|
||
# same way, and its all-time spend is the only figure that stays meaningful.
|
||
all_time_spend: dict[str, float] = {}
|
||
try:
|
||
conn = _pg_connect()
|
||
try:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT t.team_alias, COALESCE(SUM(s.spend), 0)
|
||
FROM "LiteLLM_TeamTable" t
|
||
LEFT JOIN "LiteLLM_SpendLogs" s ON s."team_id" = t.team_id
|
||
WHERE t.team_alias LIKE 'member:%' OR t.team_alias LIKE 'agent:%'
|
||
GROUP BY 1
|
||
"""
|
||
)
|
||
all_time_spend = {alias: float(v or 0) for alias, v in cur.fetchall()}
|
||
finally:
|
||
conn.close()
|
||
except Exception as e:
|
||
logger.warning("admin overview: all-time spend query failed: %s", e)
|
||
|
||
members = []
|
||
guests = []
|
||
admins = []
|
||
total_spend = 0.0
|
||
total_all_time = 0.0
|
||
operator_spend = 0.0
|
||
operator_all_time = 0.0
|
||
guest_spend = 0.0
|
||
guest_all_time = 0.0
|
||
for email, active, balance, user_id, org_slug, matrix_mxid, acct_type, oc_email, guest_exp in rows:
|
||
team = teams.get(f"member:{email}", {})
|
||
spend = float(team.get("spend") or 0.0)
|
||
member_all_time = float(all_time_spend.get(f"member:{email}", 0.0))
|
||
# Normalize floating-point residue (e.g. -5.55e-17 after a cycle reset)
|
||
# so the panel/dashboards never render "-0.00".
|
||
if abs(spend) < 1e-9:
|
||
spend = 0.0
|
||
if abs(member_all_time) < 1e-9:
|
||
member_all_time = 0.0
|
||
credits = get_credit_balance(email)
|
||
eff = compute_max_budget(
|
||
float(balance) if balance is not None else MEMBER_BUDGET,
|
||
credits,
|
||
spend,
|
||
)
|
||
mstate, mstate_reason = member_membership_state(email, bool(active))
|
||
if ((acct_type or "member") == "guest" and active
|
||
and guest_invite_pending(user_id, usernames, invite_accepted)):
|
||
# A provisioned-but-NOT-set-up guest: the invitation is out but the
|
||
# guest has not accepted it (inviteAccepted false / no username), so
|
||
# they cannot log in yet. Reads as 'invited' — never 'active' —
|
||
# until acceptance flips it. Display/state semantics only: access
|
||
# rules, the $2 cap and the expiry box are unchanged (an unaccepted
|
||
# guest cannot reach anything anyway — the setup link is the only
|
||
# door in). This override is half of the SINGLE guest-state
|
||
# derivation (see the guest_status block below for the other half);
|
||
# the panel and dashboard consume `state` and derive nothing.
|
||
sent = (guest_info.get(email) or {}).get("sent")
|
||
mstate = "invited"
|
||
mstate_reason = (f"awaiting setup — invite sent {str(sent)[:10]}, "
|
||
"not yet accepted" if sent else
|
||
"awaiting setup — invitation not yet accepted")
|
||
entry = {
|
||
"email": email,
|
||
"all_time_spend": round(member_all_time, 2),
|
||
"username": usernames.get(user_id),
|
||
"activated": invite_accepted.get(user_id, False),
|
||
"active": bool(active),
|
||
"state": mstate,
|
||
"state_reason": mstate_reason,
|
||
"manual": member_source(email) == "manual",
|
||
"source": member_source(email),
|
||
"org": org_slug or None,
|
||
"matrix_mxid": matrix_mxid or None,
|
||
"balance": float(balance) if balance is not None else MEMBER_BUDGET,
|
||
"credit_balance": credits,
|
||
"credits_remaining": max(0.0, credits - eff["credits_used"]),
|
||
"allowance_used": eff["allowance_used"],
|
||
"credits_used": eff["credits_used"],
|
||
"spend": spend,
|
||
"remaining": max(eff["max_budget"] - spend, 0.0),
|
||
"reset_at": team.get("budget_reset_at"),
|
||
"cloudron_user_id": user_id,
|
||
"account_type": acct_type or "member",
|
||
"oc_email": oc_email or None,
|
||
"guest_expires_at": guest_exp or None,
|
||
}
|
||
if (acct_type or "member") == "guest":
|
||
gi = guest_info.get(email) or {}
|
||
entry["guest_inviter"] = gi.get("inviter")
|
||
entry["guest_status"] = gi.get("status")
|
||
# The other half of the SINGLE guest-state derivation (portal =
|
||
# source of truth; tested once in tests/test_guest_invited_state.py,
|
||
# consumed as-is by the panel and dashboard). Once the invite is
|
||
# accepted the guests-ledger status IS the display state — the
|
||
# record of record (active / converted / expired / depleted /
|
||
# cancelled by recipient) — so an ended guest reads its ledger
|
||
# status, not the bare active/inactive membership flag. A guest
|
||
# with no ledger row keeps the membership state as the fallback.
|
||
if entry["state"] != "invited" and gi.get("status"):
|
||
entry["state"] = gi["status"]
|
||
entry["state_reason"] = ""
|
||
if email.lower() in OPERATOR_EMAILS:
|
||
# Operator accounts: surfaced under a separate "admin" category in
|
||
# the panel, excluded from member counts and aggregates (their
|
||
# usage is a collective expense, not member activity).
|
||
entry["admin"] = True
|
||
admins.append(entry)
|
||
total_spend += spend
|
||
total_all_time += member_all_time
|
||
operator_spend += spend
|
||
operator_all_time += member_all_time
|
||
elif (acct_type or "member") == "guest":
|
||
# Guest (guest) accounts: their own list, never members. Excluded
|
||
# from the member count AND from the spend totals a reader reads
|
||
# as member activity — a guest is not a membership. Their spend is
|
||
# broken out separately in the totals so it stays visible (the
|
||
# same reasoning as the agents below).
|
||
guests.append(entry)
|
||
guest_spend += spend
|
||
guest_all_time += member_all_time
|
||
else:
|
||
members.append(entry)
|
||
total_spend += spend
|
||
total_all_time += member_all_time
|
||
|
||
# Bot and sandbox teams: real spend, but not members. Surfaced in their own
|
||
# category so the co-op's own usage is accountable without inflating member
|
||
# counts — the same reasoning as the operator accounts above.
|
||
agents = [
|
||
{
|
||
"alias": alias,
|
||
# Display name: the panel's section header already says "Agents", so the
|
||
# "agent:" prefix is noise there. `alias` keeps the full string.
|
||
"name": alias.split(":", 1)[1] if ":" in alias else alias,
|
||
"spend": float(t.get("spend") or 0.0),
|
||
# Cycle counter resets at budget_reset_at (the 1st, for the agents), so
|
||
# the panel shows all-time for these accounts instead of a number that
|
||
# reads as zeroed every month.
|
||
"all_time_spend": round(all_time_spend.get(alias, 0.0), 2),
|
||
"budget": t.get("max_budget"),
|
||
"team_members": len(t.get("members_with_roles") or []),
|
||
}
|
||
for alias, t in teams.items()
|
||
# member:* are members; sandbox:* are throwaway test accounts rather than
|
||
# agents of the co-op. Their spend stops being visible anywhere once
|
||
# filtered here — accepted knowingly.
|
||
if alias and not alias.startswith(("member:", "sandbox:"))
|
||
]
|
||
agents.sort(key=lambda a: a["spend"], reverse=True)
|
||
|
||
return {
|
||
"members": members,
|
||
"guests": guests,
|
||
"admins": admins,
|
||
"agents": agents,
|
||
"totals": {
|
||
"members": len(members),
|
||
"active": sum(1 for m in members if m["active"]),
|
||
# Guests are never members: their own count + spend breakout, so a
|
||
# consumer never has to guess whether a figure includes them.
|
||
"guests": len(guests),
|
||
"guests_active": sum(1 for g in guests if g["active"]),
|
||
"guest_spend": round(guest_spend, 2),
|
||
"guest_all_time_spend": round(guest_all_time, 2),
|
||
# total_spend spans members AND the co-op's own operator accounts.
|
||
# operator_spend is the breakout of that portion, so a consumer
|
||
# never has to guess whether a figure includes the co-op's own use.
|
||
"total_spend": round(total_spend, 2),
|
||
"total_all_time_spend": round(total_all_time, 2),
|
||
"operator_spend": round(operator_spend, 2),
|
||
"operator_all_time_spend": round(operator_all_time, 2),
|
||
# The co-op's spend, the ONE definition both apps show (see
|
||
# coop_spend_totals). spend_all_time spans members AND the co-op's
|
||
# own accounts; own_all_time_spend is the breakout.
|
||
**{
|
||
k: v
|
||
for k, v in _coop_spend_totals_safe().items()
|
||
if k in ("spend_all_time", "own_all_time_spend")
|
||
},
|
||
},
|
||
}
|
||
|
||
|
||
def _metrics_provenance(tokens_measured: int | None, tokens_estimated: int | None) -> str:
|
||
"""Classify a day's measured-vs-estimated split for the API.
|
||
|
||
"measured" — every token rested on a directly measured factor.
|
||
"estimated" — every token rested on a non-measured factor (borrowed
|
||
SAME_FAMILY or the flat proxy).
|
||
"mixed" — the day's total is a blend of both.
|
||
"none" — no provenance recorded: either the day has no output tokens
|
||
or the row predates these columns (both split columns NULL).
|
||
Deliberately NOT "measured" — an unknown split must never read
|
||
as a confident measurement.
|
||
"""
|
||
if tokens_measured is None and tokens_estimated is None:
|
||
return "none"
|
||
m = tokens_measured or 0
|
||
e = tokens_estimated or 0
|
||
if m and e:
|
||
return "mixed"
|
||
if m:
|
||
return "measured"
|
||
if e:
|
||
return "estimated"
|
||
return "none"
|
||
|
||
|
||
def metrics_series(days: int = 90) -> list[dict]:
|
||
"""Daily metrics rows, oldest first.
|
||
|
||
One query, two callers: the admin panel (via /admin/metrics/series) and the
|
||
member dashboard (via /broker/metrics). Columns are cumulative as of that
|
||
day except spend_usd/requests/out_tokens/energy_wh/co2e_g, which are that
|
||
day only; NULL means the source could not be read that day.
|
||
|
||
energy_wh and co2e_g are computed in metrics_snapshot.py from the day's
|
||
output tokens against a factor table: measured for GreenPT models (their API
|
||
returns an `impact` object), estimated for the rest. See co-op/support#26.
|
||
|
||
Each row also carries its provenance: `tokens_measured`/`tokens_estimated`
|
||
(the output-token split across measured vs non-measured factors),
|
||
`factor_revision` (a short stable hash of the factor table that produced the
|
||
row), and a derived `provenance` indicator — measured | mixed | estimated |
|
||
none. Rows written before these columns exist have NULL splits and report
|
||
`provenance: "none"`, never a confident "measured". See docs/energy-metrics.md.
|
||
"""
|
||
limit = max(1, min(int(days or 90), 730))
|
||
conn = get_db()
|
||
try:
|
||
rows = conn.execute(
|
||
"SELECT day, members_active, members_total, members_new, newsletter_subs, "
|
||
"matrix_space_members, spend_usd, operator_spend_usd, requests, out_tokens, energy_wh, co2e_g, "
|
||
"tokens_measured, tokens_estimated, factor_revision, recorded_at "
|
||
"FROM metrics_daily ORDER BY day DESC LIMIT ?",
|
||
(limit,),
|
||
).fetchall()
|
||
finally:
|
||
conn.close()
|
||
cols = [
|
||
"day",
|
||
"members_active",
|
||
"members_total",
|
||
"members_new",
|
||
"newsletter_subs",
|
||
"matrix_space_members",
|
||
"spend_usd",
|
||
"operator_spend_usd",
|
||
"requests",
|
||
"out_tokens",
|
||
"energy_wh",
|
||
"co2e_g",
|
||
"tokens_measured",
|
||
"tokens_estimated",
|
||
"factor_revision",
|
||
"recorded_at",
|
||
]
|
||
series = []
|
||
for row in reversed(rows):
|
||
rec = dict(zip(cols, row))
|
||
rec["provenance"] = _metrics_provenance(
|
||
rec["tokens_measured"], rec["tokens_estimated"]
|
||
)
|
||
series.append(rec)
|
||
return series
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Money series — the admin panel's Money chart: cumulative Open Collective
|
||
# revenue, token spend over time, and the OC balance.
|
||
#
|
||
# The three series have genuinely different provenance, and the endpoint says
|
||
# so rather than papering over it:
|
||
# revenue — OC contribution transactions (CREDIT / CONTRIBUTION / not a
|
||
# refund), each with a createdAt and an amount, summed
|
||
# chronologically. Monthly and one-time contributions both count;
|
||
# org-backed contributions count through the org account that
|
||
# paid. Exact from each contribution's start date (2026-09-06).
|
||
# spend — LiteLLM SpendLogs, summed per UTC day in ONE GROUP BY pass over
|
||
# the full history (134k+ rows), with no team filter — members,
|
||
# operators and agent teams all count, matching spend_by_day() in
|
||
# metrics_snapshot.py.
|
||
# balance — the OC API exposes only the collective's CURRENT balance; there
|
||
# is NO historical series. The current balance is returned live
|
||
# and a real history accrues from the daily metrics_snapshot
|
||
# snapshots into metrics_daily.oc_balance_usd. Days before that
|
||
# pipeline stay NULL — a gap in the chart, never a zero.
|
||
#
|
||
# Aggregate totals only: no member emails or names are selected or returned.
|
||
#
|
||
# Caching: the OC queries are slow (network + pagination), so the payload is
|
||
# computed at most once per MONEY_SERIES_TTL seconds (default 1h) and served
|
||
# from an in-memory cache between refreshes; a failed refresh serves the
|
||
# previous payload marked `stale: true` rather than breaking the chart. The
|
||
# daily snapshot owns history accrual, so no schedule entry is needed here.
|
||
# ---------------------------------------------------------------------------
|
||
|
||
MONEY_SERIES_TTL = float(os.environ.get("MONEY_SERIES_TTL", "3600"))
|
||
_MONEY_CACHE: dict = {"data": None, "ts": 0.0}
|
||
_money_lock = asyncio.Lock()
|
||
|
||
# Hard stop for the transactions pagination: 500 rows/page x 40 pages. The
|
||
# collective holds ~160 contribution transactions; the cap only exists so a
|
||
# surprising totalCount can never turn into an unbounded loop.
|
||
_MONEY_TX_PAGE = 500
|
||
_MONEY_TX_MAX_PAGES = 40
|
||
|
||
|
||
async def _oc_graphql(query: str, variables: dict) -> dict:
|
||
"""One authenticated OC GraphQL call (browser UA — Cloudflare blocks the
|
||
default Python UA with a 403/1010 bot signature). Raises on HTTP or
|
||
GraphQL errors so callers can fail per-source."""
|
||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||
r = await client.post(
|
||
OC_GRAPHQL_URL,
|
||
headers={"Content-Type": "application/json",
|
||
"Authorization": f"Bearer {OC_PERSONAL_TOKEN}",
|
||
"Api-Version": "2022-01-01",
|
||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"},
|
||
json={"query": query, "variables": variables},
|
||
)
|
||
if r.status_code != 200:
|
||
raise RuntimeError(f"OC GraphQL failed: HTTP {r.status_code}")
|
||
body = r.json()
|
||
if body.get("errors"):
|
||
raise RuntimeError(f"OC GraphQL errors: {str(body['errors'])[:200]}")
|
||
return body.get("data") or {}
|
||
|
||
|
||
async def oc_collective_balance() -> dict:
|
||
"""The collective's CURRENT balance (there is no historical endpoint)."""
|
||
if not OC_PERSONAL_TOKEN:
|
||
raise RuntimeError("OC_PERSONAL_TOKEN not set")
|
||
data = await _oc_graphql(
|
||
"query Bal($slug: String) { collective(slug: $slug) "
|
||
"{ stats { balance { valueInCents currency } } } }",
|
||
{"slug": OC_COLLECTIVE_SLUG},
|
||
)
|
||
bal = (
|
||
((data.get("collective") or {}).get("stats") or {}).get("balance") or {}
|
||
)
|
||
cents = bal.get("valueInCents")
|
||
if cents is None:
|
||
raise RuntimeError("OC balance field missing from response")
|
||
return {
|
||
"value": round(int(cents) / 100.0, 2),
|
||
"currency": bal.get("currency") or "USD",
|
||
"as_of": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||
}
|
||
|
||
|
||
async def oc_contribution_by_day() -> tuple[dict[str, float], int, float]:
|
||
"""{UTC day: contribution $} over every CREDIT CONTRIBUTION transaction.
|
||
|
||
Paginated (variables, never interpolation — the same injection rule as
|
||
get_order_amount). Returns (by_day, count, total). Payer identities are
|
||
deliberately NOT selected: this feed is aggregate-only.
|
||
"""
|
||
if not OC_PERSONAL_TOKEN:
|
||
raise RuntimeError("OC_PERSONAL_TOKEN not set")
|
||
by_day: dict[str, float] = {}
|
||
count = 0
|
||
total_cents = 0
|
||
offset = 0
|
||
for _page in range(_MONEY_TX_MAX_PAGES):
|
||
data = await _oc_graphql(
|
||
"query Contribs($slug: String, $limit: Int!, $offset: Int!) "
|
||
"{ collective(slug: $slug) { transactions(type: CREDIT, limit: $limit, "
|
||
"offset: $offset) { totalCount nodes { createdAt type kind isRefund "
|
||
"amount { valueInCents } } } } }",
|
||
{"slug": OC_COLLECTIVE_SLUG, "limit": _MONEY_TX_PAGE, "offset": offset},
|
||
)
|
||
coll = ((data.get("collective") or {}).get("transactions") or {})
|
||
total_count = int(coll.get("totalCount") or 0)
|
||
nodes = coll.get("nodes") or []
|
||
if not nodes:
|
||
break
|
||
for t in nodes:
|
||
if (t.get("type") or "").upper() != "CREDIT":
|
||
continue
|
||
if (t.get("kind") or "").upper() != "CONTRIBUTION":
|
||
continue
|
||
if t.get("isRefund"):
|
||
continue
|
||
dtv = _parse_oc_dt(t.get("createdAt"))
|
||
cents = ((t.get("amount") or {}).get("valueInCents")) or 0
|
||
if dtv is None:
|
||
continue
|
||
day = dtv.astimezone(timezone.utc).strftime("%Y-%m-%d")
|
||
by_day[day] = by_day.get(day, 0.0) + int(cents) / 100.0
|
||
count += 1
|
||
total_cents += int(cents)
|
||
offset += len(nodes)
|
||
if offset >= total_count or len(nodes) < _MONEY_TX_PAGE:
|
||
break
|
||
return by_day, count, round(total_cents / 100.0, 2)
|
||
|
||
|
||
def litellm_spend_by_day() -> tuple[dict[str, float], float, int]:
|
||
"""{UTC day: $}, total, and row count from LiteLLM's SpendLogs — one
|
||
GROUP BY pass over the whole table. No team filter: the co-op's total
|
||
spans members, operators and agent teams (same definition as
|
||
spend_by_day() in metrics_snapshot.py, so the two agree by construction)."""
|
||
if not litellm_db_ready():
|
||
raise RuntimeError("LiteLLM DB credentials not configured")
|
||
conn = _pg_connect()
|
||
try:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""SELECT to_char("startTime", 'YYYY-MM-DD') AS day,
|
||
COALESCE(SUM("spend"), 0) AS spend, COUNT(*) AS n
|
||
FROM "LiteLLM_SpendLogs"
|
||
GROUP BY 1 ORDER BY 1"""
|
||
)
|
||
rows = cur.fetchall()
|
||
finally:
|
||
conn.close()
|
||
by_day = {r[0]: round(float(r[1]), 6) for r in rows}
|
||
return by_day, round(sum(by_day.values()), 6), int(sum(int(r[2]) for r in rows))
|
||
|
||
|
||
def coop_spend_totals() -> dict:
|
||
"""The co-op's all-time spend, ONE definition, for every surface that shows it.
|
||
|
||
The admin panel's cards, the money payload and the member dashboard all read
|
||
THIS: the panel used its own per-team query and the dashboard summed the
|
||
chart series in the browser, so the same claim came out as $941.37 in one app
|
||
and $900.51 in the other (2026-11-13).
|
||
|
||
total = every SpendLogs row, whatever team it belongs to — the same universe
|
||
as litellm_spend_by_day(), so the cards and the money chart agree.
|
||
own = the co-op's own use: every team that is not a member team, plus rows
|
||
with no team at all (nothing here is a member's usage).
|
||
"""
|
||
if not litellm_db_ready():
|
||
raise RuntimeError("LiteLLM DB credentials not configured")
|
||
conn = _pg_connect()
|
||
try:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""SELECT COALESCE(t.team_alias, '') AS alias,
|
||
COALESCE(SUM(s."spend"), 0) AS spend
|
||
FROM "LiteLLM_SpendLogs" s
|
||
LEFT JOIN "LiteLLM_TeamTable" t ON t.team_id = s."team_id"
|
||
GROUP BY 1"""
|
||
)
|
||
rows = cur.fetchall()
|
||
finally:
|
||
conn.close()
|
||
member = own = 0.0
|
||
aliases = 0
|
||
for alias, spend in rows:
|
||
val = float(spend or 0.0)
|
||
aliases += 1
|
||
if str(alias or "").startswith("member:"):
|
||
member += val
|
||
else:
|
||
own += val
|
||
return {
|
||
"spend_all_time": round(member + own, 2),
|
||
"member_all_time_spend": round(member, 2),
|
||
"own_all_time_spend": round(own, 2),
|
||
"aliases": aliases,
|
||
}
|
||
|
||
|
||
def oc_balance_history() -> dict[str, float]:
|
||
"""metrics_daily.oc_balance_usd rows — the daily snapshots. Empty before
|
||
the pipeline deployed (earlier balances were never recorded)."""
|
||
conn = get_db()
|
||
try:
|
||
rows = conn.execute(
|
||
"SELECT day, oc_balance_usd FROM metrics_daily "
|
||
"WHERE oc_balance_usd IS NOT NULL ORDER BY day"
|
||
).fetchall()
|
||
finally:
|
||
conn.close()
|
||
return {r[0]: float(r[1]) for r in rows}
|
||
|
||
|
||
def _seed_balance_row(day: str, value: float) -> None:
|
||
"""Fold today's live OC balance into metrics_daily WITHOUT creating a row
|
||
and WITHOUT touching any other column. The daily snapshot owns the rest of
|
||
that row (a partial INSERT here would punch NULL gaps into the members
|
||
chart for the day). When the snapshot has not written today's row yet
|
||
there is nothing to seed and this is a no-op; its own 03:15 read then
|
||
writes the balance itself."""
|
||
if value is None:
|
||
return
|
||
conn = get_db()
|
||
try:
|
||
conn.execute(
|
||
"UPDATE metrics_daily SET oc_balance_usd = ? WHERE day = ?",
|
||
(float(value), day),
|
||
)
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def _build_money_series(rev_by_day: dict[str, float] | None,
|
||
spend_by_day: dict[str, float] | None,
|
||
bal_hist: dict[str, float],
|
||
today_key: str) -> list[dict]:
|
||
"""One row per UTC day across the full span, oldest first.
|
||
|
||
revenue_cum / spend_cum are running sums; days before a series' first
|
||
entry carry 0 — a genuine zero (the sources were readable and nothing had
|
||
arrived yet), unlike oc_balance, which is NULL before the first daily
|
||
snapshot and must render as a gap.
|
||
"""
|
||
day_keys = set(rev_by_day or {}) | set(spend_by_day or {}) | set(bal_hist or {})
|
||
day_keys.add(today_key)
|
||
cur = datetime.strptime(min(day_keys), "%Y-%m-%d").date()
|
||
end = datetime.strptime(today_key, "%Y-%m-%d").date()
|
||
rev_cum = 0.0
|
||
spend_cum = 0.0
|
||
out: list[dict] = []
|
||
while cur <= end:
|
||
key = cur.strftime("%Y-%m-%d")
|
||
rev_today = (rev_by_day or {}).get(key, 0.0)
|
||
spend_today = (spend_by_day or {}).get(key, 0.0)
|
||
rev_cum += rev_today
|
||
spend_cum += spend_today
|
||
out.append({
|
||
"day": key,
|
||
"revenue_cum": round(rev_cum, 2) if rev_by_day is not None else None,
|
||
"spend_day": round(spend_today, 6) if spend_by_day is not None else None,
|
||
"spend_cum": round(spend_cum, 2) if spend_by_day is not None else None,
|
||
"oc_balance": bal_hist.get(key),
|
||
})
|
||
cur += timedelta(days=1)
|
||
return out
|
||
|
||
|
||
async def _compute_money_payload() -> dict:
|
||
"""Fetch every source independently; a failed source yields NULL columns
|
||
plus an `errors` entry rather than taking the whole payload down. Only an
|
||
all-sources failure raises (the caller then serves stale or 503s)."""
|
||
errors: list[str] = []
|
||
today_key = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||
|
||
rev_by_day = None
|
||
contrib_count = None
|
||
revenue_total = None
|
||
try:
|
||
rev_by_day, contrib_count, revenue_total = await oc_contribution_by_day()
|
||
except Exception as e:
|
||
errors.append(f"revenue: {e}")
|
||
|
||
spend_by_day = None
|
||
spend_total = None
|
||
spend_rows = None
|
||
try:
|
||
spend_by_day, spend_total, spend_rows = litellm_spend_by_day()
|
||
except Exception as e:
|
||
errors.append(f"spend: {e}")
|
||
|
||
balance_current = None
|
||
try:
|
||
balance_current = await oc_collective_balance()
|
||
except Exception as e:
|
||
errors.append(f"balance: {e}")
|
||
|
||
if rev_by_day is None and spend_by_day is None and balance_current is None:
|
||
raise RuntimeError("all money sources failed: " + "; ".join(errors))
|
||
|
||
bal_hist = oc_balance_history()
|
||
# Today's balance is the LIVE value just fetched — fresher than the 03:15
|
||
# snapshot — so it overrides (or extends) the history at today's key, and
|
||
# is folded into the stored row when one already exists.
|
||
if balance_current is not None:
|
||
bal_hist = {**bal_hist, today_key: float(balance_current["value"])}
|
||
try:
|
||
_seed_balance_row(today_key, balance_current["value"])
|
||
except Exception as e:
|
||
errors.append(f"balance seed: {e}")
|
||
|
||
series = _build_money_series(rev_by_day, spend_by_day, bal_hist, today_key)
|
||
balance_history_start = min(bal_hist) if bal_hist else None
|
||
return {
|
||
"generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||
"currency": "USD",
|
||
"revenue_start": min(rev_by_day) if rev_by_day else None,
|
||
"spend_start": min(spend_by_day) if spend_by_day else None,
|
||
"balance_history_start": balance_history_start,
|
||
"balance_current": balance_current,
|
||
"totals": {
|
||
"revenue_all_time": revenue_total,
|
||
"contributions": contrib_count,
|
||
"spend_all_time": spend_total,
|
||
"spend_rows": spend_rows,
|
||
# The member-owned vs co-op's-own split of that same all-time spend,
|
||
# from the one shared definition.
|
||
"own_all_time_spend": (_coop_split := _coop_spend_totals_safe()).get("own_all_time_spend"),
|
||
"member_all_time_spend": _coop_split.get("member_all_time_spend"),
|
||
},
|
||
"sources": {
|
||
"revenue": "Open Collective contribution transactions (CREDIT, "
|
||
"CONTRIBUTION, non-refund) — monthly and one-time, "
|
||
"org-backed included via the org account",
|
||
"spend": "LiteLLM SpendLogs daily sums (UTC), all teams — members, "
|
||
"operators and agent teams included",
|
||
"balance": "Open Collective current balance + daily snapshots in "
|
||
"metrics_daily.oc_balance_usd",
|
||
},
|
||
"errors": errors,
|
||
"series": series,
|
||
}
|
||
|
||
|
||
async def _get_money_payload() -> dict:
|
||
"""Hourly-cached access to the money payload; serves the previous payload
|
||
(marked stale) when a refresh fails."""
|
||
now = datetime.now(timezone.utc).timestamp()
|
||
async with _money_lock:
|
||
cached = _MONEY_CACHE.get("data")
|
||
if cached and now - _MONEY_CACHE.get("ts", 0.0) < MONEY_SERIES_TTL:
|
||
out = dict(cached)
|
||
out["cached"] = True
|
||
out["cache_ttl_seconds"] = MONEY_SERIES_TTL
|
||
return out
|
||
try:
|
||
data = await _compute_money_payload()
|
||
except Exception as e:
|
||
if cached:
|
||
out = dict(cached)
|
||
out["cached"] = True
|
||
out["stale"] = True
|
||
out["errors"] = list(out.get("errors") or []) + [f"refresh failed, serving cached: {e}"]
|
||
return out
|
||
raise
|
||
_MONEY_CACHE["data"] = data
|
||
_MONEY_CACHE["ts"] = now
|
||
out = dict(data)
|
||
out["cached"] = False
|
||
out["cache_ttl_seconds"] = MONEY_SERIES_TTL
|
||
return out
|
||
|
||
|
||
@app.get("/admin/money-series")
|
||
@limiter.limit("30/minute")
|
||
async def admin_money_series(request: Request):
|
||
"""Revenue / token-spend / OC-balance series for the admin panel's Money
|
||
chart, hourly-cached (the OC queries are slow; never called per refresh).
|
||
|
||
Authenticated by the X-Admin-Token header, like the other /admin/* routes.
|
||
The full history is always returned (cumulative series need it); the panel
|
||
slices client-side so its period toggle never refetches. Aggregate totals
|
||
only — no member emails or names in this payload. See the block comment
|
||
above for each series' provenance and honest limits.
|
||
"""
|
||
_verify_admin_token(request)
|
||
return await _get_money_payload()
|
||
|
||
|
||
@app.get("/admin/metrics/series")
|
||
@limiter.limit("30/minute")
|
||
async def admin_metrics_series(request: Request, days: int = 90):
|
||
"""Daily metrics series for charts.
|
||
|
||
Authenticated by the X-Admin-Token header. Returns oldest-first rows from
|
||
`metrics_daily` for the last `days` days, for the admin panel and the member
|
||
dashboard to chart from one source of truth. A NULL column means that source
|
||
could not be read on that day: render a gap, not a zero.
|
||
|
||
Cumulative columns (members_*, newsletter_subs, matrix_space_members) are
|
||
counts as of that day; spend_usd/requests are that day's traffic only.
|
||
Backfilled days carry a LOWER BOUND for members_total (members provisioned
|
||
without a welcome email are absent from the history).
|
||
|
||
Each row carries its energy provenance — `provenance` (measured | mixed |
|
||
estimated | none) plus `tokens_measured`, `tokens_estimated` and
|
||
`factor_revision`. A row written before those columns existed reports
|
||
`provenance: "none"`, not "measured". See docs/energy-metrics.md.
|
||
"""
|
||
_verify_admin_token(request)
|
||
series = metrics_series(days)
|
||
return {"series": series, "count": len(series), "days": max(1, min(int(days or 90), 730))}
|
||
|
||
|
||
@app.get("/admin/model-usage")
|
||
@limiter.limit("30/minute")
|
||
async def admin_model_usage(request: Request, days: int = 0):
|
||
"""Co-op-wide per-model usage from LiteLLM spend logs.
|
||
|
||
Authenticated by the X-Admin-Token header. Aggregates spend-log rows
|
||
(across ALL members) by model: total spend, tokens, and the count of
|
||
successful requests (refused/failed attempts are excluded and reported
|
||
separately as `refused`), sorted by request count descending. Optional
|
||
?days=N limits to the last N days
|
||
(0 or omitted = all-time). No per-member breakdown — this is co-op-level
|
||
so admins can see which models the membership actually uses.
|
||
|
||
Reads go straight to LiteLLM's Postgres when configured (fast, bounded);
|
||
otherwise falls back to the HTTP API path.
|
||
"""
|
||
_verify_admin_token(request)
|
||
|
||
# Fast path: direct Postgres aggregation.
|
||
if litellm_db_ready():
|
||
try:
|
||
return aggregate_model_usage(days=days)
|
||
except Exception as e:
|
||
logger.warning("admin model-usage: DB path failed (%s); falling back to HTTP", e)
|
||
|
||
# Fallback path: LiteLLM HTTP API.
|
||
cutoff = None
|
||
if days and days > 0:
|
||
from datetime import datetime, timedelta, timezone
|
||
cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).strftime(
|
||
"%Y-%m-%dT%H:%M:%S"
|
||
)
|
||
|
||
models = {}
|
||
total_spend = 0.0
|
||
total_tokens = 0
|
||
total_requests = 0
|
||
total_refused = 0
|
||
try:
|
||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||
r = await client.get(
|
||
f"{LITELLM_BASE}/spend/logs",
|
||
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
|
||
)
|
||
r.raise_for_status()
|
||
rows = r.json()
|
||
if isinstance(rows, dict):
|
||
rows = rows.get("data", rows.get("logs", []))
|
||
for row in rows if isinstance(rows, list) else []:
|
||
# Prefer model_group: it carries our provider/model alias (e.g.
|
||
# greenpt/glm-5.3-flash, publicai/apertus-v1.5-8b) when present; the
|
||
# raw `model` field holds messy upstream/internal names. Older
|
||
# rows predating the alias convention have model_group empty,
|
||
# so fall back to `model` for those.
|
||
if cutoff:
|
||
started = str(row.get("startTime") or "")
|
||
if started and started < cutoff:
|
||
continue
|
||
model = row.get("model_group") or row.get("model") or "unknown"
|
||
m = models.setdefault(model, {"spend": 0.0, "tokens": 0, "requests": 0, "refused": 0})
|
||
m["spend"] += float(row.get("spend") or 0.0)
|
||
m["tokens"] += int(row.get("total_tokens") or 0)
|
||
# Refused/failed attempts are not served requests: keep them out
|
||
# of the request count (reported separately as `refused`).
|
||
if str(row.get("status") or "").lower() == "failure":
|
||
m["refused"] += 1
|
||
total_refused += 1
|
||
else:
|
||
m["requests"] += 1
|
||
total_requests += 1
|
||
total_spend += float(row.get("spend") or 0.0)
|
||
total_tokens += int(row.get("total_tokens") or 0)
|
||
except Exception as e:
|
||
logger.warning("admin model-usage: spend/logs fetch failed: %s", e)
|
||
|
||
model_list = [
|
||
{"model": name, **stats}
|
||
for name, stats in sorted(models.items(), key=lambda kv: kv[1]["spend"], reverse=True)
|
||
]
|
||
|
||
return {
|
||
"models": model_list,
|
||
"totals": {
|
||
"total_spend": total_spend,
|
||
"total_tokens": total_tokens,
|
||
"total_requests": total_requests,
|
||
"total_refused": total_refused,
|
||
},
|
||
}
|
||
|
||
|
||
# --- Broker HTTP endpoints (called by the member dashboard) ---
|
||
# The dashboard is Cloudron-SSO-gated; it authenticates the member and passes
|
||
# their email with a shared secret. The portal trusts the email only because it
|
||
# carries BROKER_SECRET. These endpoints expose ONLY the authenticated member's
|
||
# own keys — never another member's, and never the chat key.
|
||
|
||
|
||
# --- First-login service sync (activation hook) ---
|
||
# A newly provisioned member is deliberately skipped by the Loomio/Matrix syncs
|
||
# until their Cloudron account is activated (so a half-created account is never
|
||
# invited), and the 3x/day sweep was the only thing that picked them up AFTER
|
||
# they did activate — a new member could wait hours for forum/space access.
|
||
# The broker identity path is where the member first presents themselves after
|
||
# setting up (the dashboard calls it on first login), so it is the natural
|
||
# trigger: when a broker-authenticated, active member resolves AND their Loomio
|
||
# invite is still outstanding, confirm their Cloudron account is now activated
|
||
# and, only then, fire the same idempotent syncs the sweep runs. The syncs run
|
||
# as a background task, so no response ever waits on Loomio or Matrix.
|
||
|
||
_first_login_sync_inflight = False
|
||
|
||
# Guard for the scheduled service-sync (see /admin/service-sync). The pass does
|
||
# full-roster reconciles, so two overlapping runs would only duplicate work and
|
||
# burn Synapse's invite budget twice over.
|
||
_service_sync_inflight = False
|
||
|
||
|
||
def _member_slug(email: str) -> str:
|
||
"""Member slug for log lines (slugs survive log redaction; emails don't)."""
|
||
try:
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT slug FROM members WHERE email = ?", (email,)
|
||
).fetchone()
|
||
conn.close()
|
||
return (row[0] if row and row[0] else "") or "<slug-unset>"
|
||
except Exception:
|
||
return "<slug-unset>"
|
||
|
||
|
||
async def _member_cloudron_activated(email: str) -> bool:
|
||
"""True only when the member's Cloudron account is CONFIRMED activated.
|
||
|
||
Fail-CLOSED on purpose: if the Cloudron user list cannot be read (or the
|
||
member is not on it), report not-activated and let the 3x/day sweep try
|
||
later. The hook must never be the path that invites a half-created account,
|
||
so unlike the sweep's fail-open roster filter, an unverifiable activation
|
||
state means "do not fire".
|
||
"""
|
||
async with httpx.AsyncClient(timeout=15.0) as client:
|
||
users = await cloudron_fetch_all_users(client)
|
||
if not users:
|
||
return False
|
||
for u in users:
|
||
if (u.get("email") or "").lower() == email.lower():
|
||
return bool(u.get("inviteAccepted"))
|
||
return False
|
||
|
||
|
||
async def _first_login_service_sync(email: str) -> None:
|
||
"""Fire the Loomio + Matrix Space syncs for a newly-activated member.
|
||
|
||
Runs as a background task off the broker request (a Loomio outage must
|
||
never block a dashboard). The syncs are the SAME idempotent functions the
|
||
sweep and provisioning run — guarded by members.loomio_invited_at and the
|
||
live-roster pre-check — so a re-fire cannot double-invite. Best-effort:
|
||
any failure is logged and swallowed; the sweep remains the backstop.
|
||
"""
|
||
global _first_login_sync_inflight
|
||
slug = _member_slug(email)
|
||
try:
|
||
if not await _member_cloudron_activated(email):
|
||
logger.info(
|
||
"first-login sync: %s Cloudron account not yet activated; not "
|
||
"firing (the sweep covers it later)", slug,
|
||
)
|
||
return
|
||
# Each service independently best-effort: a Loomio outage must never
|
||
# stop the Matrix invite (or vice versa), and neither may raise out of
|
||
# this background task.
|
||
for label, sync_fn in (
|
||
("Loomio", sync_loomio_memberships),
|
||
("Matrix", sync_matrix_space),
|
||
):
|
||
try:
|
||
logger.info("first-login sync: firing %s sync for %s", label, slug)
|
||
await sync_fn()
|
||
except Exception:
|
||
logger.warning(
|
||
"first-login sync: %s sync for %s failed (non-fatal; the "
|
||
"sweep covers later)", label, slug, exc_info=True,
|
||
)
|
||
logger.info(
|
||
"first-login sync: done for %s (loomio_invited_at=%s)",
|
||
slug, get_loomio_invited_at(email),
|
||
)
|
||
except Exception:
|
||
logger.warning(
|
||
"first-login sync for %s failed (non-fatal; sweep covers later)",
|
||
slug, exc_info=True,
|
||
)
|
||
finally:
|
||
_first_login_sync_inflight = False
|
||
|
||
|
||
def maybe_first_login_service_sync(email: str) -> None:
|
||
"""Cheap gate on the broker hot path; never raises.
|
||
|
||
Established members (loomio_invited_at set) cost one indexed SELECT and
|
||
return. A never-invited member schedules the sync as a background task.
|
||
While one first-login sync is in flight, further gate hits do nothing: the
|
||
syncs are full-roster reconciles, so the running pass already covers every
|
||
pending member.
|
||
"""
|
||
global _first_login_sync_inflight
|
||
try:
|
||
if get_loomio_invited_at(email):
|
||
return # established member — zero further work
|
||
if is_guest_account(email):
|
||
return # guest accounts receive no Loomio/Matrix invites
|
||
if _first_login_sync_inflight:
|
||
return # a sync is already running; it covers every pending member
|
||
loop = asyncio.get_running_loop()
|
||
except Exception:
|
||
return # the gate must never break the broker response
|
||
_first_login_sync_inflight = True
|
||
try:
|
||
loop.create_task(_first_login_service_sync(email))
|
||
except Exception:
|
||
_first_login_sync_inflight = False
|
||
logger.warning(
|
||
"first-login sync could not be scheduled (non-fatal)", exc_info=True,
|
||
)
|
||
|
||
|
||
def _verify_broker_secret(request: Request) -> str:
|
||
"""Return the authenticated member's email, or 401.
|
||
|
||
The member dashboard sends X-Broker-Secret (shared secret) + X-Member-Email
|
||
(or X-Member-User, a Cloudron username). Both must be present and the secret
|
||
must match, or we refuse (fail closed).
|
||
|
||
Cloudron's proxyAuth injects the USERNAME (e.g. "ntnsndr"), not the email.
|
||
So when the dashboard sends a username (no "@"), we resolve it to an email
|
||
via the Cloudron user list (we already hold a Cloudron admin token).
|
||
"""
|
||
if not BROKER_SECRET:
|
||
raise HTTPException(503, "Broker secret not configured")
|
||
provided = request.headers.get("x-broker-secret", "")
|
||
if not secrets.compare_digest(provided, BROKER_SECRET):
|
||
raise HTTPException(401, "Invalid broker secret")
|
||
identity = (
|
||
request.headers.get("x-member-email")
|
||
or request.headers.get("x-member-user")
|
||
or ""
|
||
).strip()
|
||
if not identity:
|
||
raise HTTPException(401, "Missing member identity")
|
||
email = resolve_member_email(identity)
|
||
# The member must be active in our DB before they can manage keys.
|
||
if not get_member_key(email):
|
||
raise HTTPException(403, "No active membership")
|
||
# Activation hook: if this active member's service invites are still
|
||
# outstanding AND their Cloudron account is now activated, fire the Loomio
|
||
# + Matrix syncs (background task; established members cost one SELECT).
|
||
# Best-effort by construction — it must never affect this response.
|
||
maybe_first_login_service_sync(email)
|
||
return email
|
||
|
||
|
||
def resolve_member_email(identity: str) -> str:
|
||
"""Resolve a member identity (email, Cloudron username, or uid) to email.
|
||
|
||
Cloudron's OIDC `sub` claim is the user ID (uid-…), and email-invited users
|
||
have no username at all. So accept three identity shapes:
|
||
- email ("@") → return as-is
|
||
- uid-… → match on the user id
|
||
- username → match on the username field (legacy accounts)
|
||
"""
|
||
identity = identity.strip()
|
||
if "@" in identity:
|
||
return identity.lower()
|
||
# uid / username → email via the full (paginated) user list; one walk
|
||
# covers both matchers.
|
||
users = cloudron_fetch_all_users_sync()
|
||
for u in users:
|
||
if u.get("id") == identity:
|
||
return (u.get("email") or "").lower()
|
||
for u in users:
|
||
if u.get("username") == identity:
|
||
return (u.get("email") or "").lower()
|
||
raise HTTPException(403, f"No Cloudron user for identity {identity!r}")
|
||
|
||
|
||
def _verify_broker_guest_identity(request: Request) -> str:
|
||
"""Broker auth that also admits ENDED guest accounts.
|
||
|
||
Same shared-secret + identity contract as _verify_broker_secret, with one
|
||
widening scoped to guest accounts only: a guest is admitted in ANY state,
|
||
because the self-service cancel must stay a no-op after it takes effect
|
||
and the ended state must render on a later page load. Members (and only
|
||
members) still require an active row, so this never widens what a member
|
||
identity can reach — the routes on top enforce the guest-only rule again.
|
||
"""
|
||
if not BROKER_SECRET:
|
||
raise HTTPException(503, "Broker secret not configured")
|
||
provided = request.headers.get("x-broker-secret", "")
|
||
if not secrets.compare_digest(provided, BROKER_SECRET):
|
||
raise HTTPException(401, "Invalid broker secret")
|
||
identity = (
|
||
request.headers.get("x-member-email")
|
||
or request.headers.get("x-member-user")
|
||
or ""
|
||
).strip()
|
||
if not identity:
|
||
raise HTTPException(401, "Missing member identity")
|
||
email = resolve_member_email(identity)
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT account_type, active FROM members WHERE email = ?", (email,)
|
||
).fetchone()
|
||
conn.close()
|
||
if not row:
|
||
raise HTTPException(403, "No active membership")
|
||
account_type, active = (row[0] or "member"), row[1]
|
||
if account_type == "guest":
|
||
return email # any state — the cancel/guest-status route decides
|
||
if not active or not get_member_key(email):
|
||
raise HTTPException(403, "No active membership")
|
||
maybe_first_login_service_sync(email)
|
||
return email
|
||
|
||
|
||
@app.get("/broker/keys")
|
||
@limiter.limit("30/minute")
|
||
async def broker_list(request: Request):
|
||
email = _verify_broker_secret(request)
|
||
return {"keys": await broker_list_api_keys(email)}
|
||
|
||
|
||
@app.post("/broker/keys")
|
||
@limiter.limit("30/minute")
|
||
async def broker_create(request: Request):
|
||
email = _verify_broker_secret(request)
|
||
body = await request.json()
|
||
name = (body.get("name") or "").strip()
|
||
key = await broker_create_api_key(email, name)
|
||
return {"status": "created", **key}
|
||
|
||
|
||
@app.delete("/broker/keys/{name}")
|
||
@limiter.limit("30/minute")
|
||
async def broker_revoke(name: str, request: Request):
|
||
email = _verify_broker_secret(request)
|
||
await broker_revoke_api_key(email, name)
|
||
return {"status": "revoked", "name": name}
|
||
|
||
|
||
# Matrix MXID (self-serve federated identity). A member can attach ONE extra
|
||
# Matrix account (e.g. their existing @user:matrix.org) to be invited to the
|
||
# co-op Space. Empty string clears it. Validation is loose on purpose — Synapse
|
||
# will simply reject a malformed MXID on invite, and we log that.
|
||
_MXID_RE = re.compile(r"^@[^\s:]+:[^\s]+\.[^\s]+$")
|
||
|
||
|
||
@app.get("/broker/matrix-mxid")
|
||
@limiter.limit("30/minute")
|
||
async def broker_get_matrix_mxid(request: Request):
|
||
email = _verify_broker_secret(request)
|
||
if is_guest_account(email):
|
||
# Guests get no Matrix: their dashboard hides the section and the
|
||
# sync never invites them — the endpoint refuses too, in depth.
|
||
raise HTTPException(403, "Matrix chat is available to members")
|
||
return {"matrix_mxid": get_member_matrix_mxid(email), "local_mxid": local_mxid_for_email(email)}
|
||
|
||
|
||
def local_mxid_for_email(email: str) -> str:
|
||
"""Resolve a member's local Matrix ID (@username:inference.coop), or '' if unknown."""
|
||
try:
|
||
for u in cloudron_fetch_all_users_sync():
|
||
if (u.get("email") or "").lower() == email and u.get("username"):
|
||
return f"@{u['username']}:inference.coop"
|
||
except Exception as e:
|
||
logger.warning("local_mxid_for_email lookup failed: %s", e)
|
||
return ""
|
||
|
||
|
||
@app.post("/broker/matrix-mxid")
|
||
@limiter.limit("30/minute")
|
||
async def broker_set_matrix_mxid(request: Request):
|
||
email = _verify_broker_secret(request)
|
||
if is_guest_account(email):
|
||
# Setting an MXID would put a guest's account in the next Matrix sync
|
||
# invite pass — exactly what the feature excludes.
|
||
raise HTTPException(403, "Matrix chat is available to members")
|
||
body = await request.json()
|
||
mxid = (body.get("matrix_mxid") or "").strip()
|
||
if mxid and not _MXID_RE.match(mxid):
|
||
return JSONResponse(
|
||
{"error": "Enter a Matrix ID like @username:server.org"}, status_code=400
|
||
)
|
||
set_member_matrix_mxid(email, mxid)
|
||
# Invite the new account into the Space immediately (best-effort; the daily
|
||
# sync will also pick it up). Clear = no invite (just removes the mapping).
|
||
invite_sent = True
|
||
if mxid:
|
||
try:
|
||
await sync_matrix_space()
|
||
except Exception as e:
|
||
logger.warning("matrix-mxid: immediate invite failed: %s", e)
|
||
invite_sent = False
|
||
# invite_sent=False tells the dashboard the nightly sync will retry; the
|
||
# mapping is saved either way, so the member gets a confident confirmation.
|
||
return {"status": "saved", "matrix_mxid": mxid, "invite_sent": invite_sent}
|
||
|
||
|
||
def member_per_key_spend(email: str, team_id: str) -> list[dict]:
|
||
"""Per-key spend for a member, derived from LiteLLM spend LOGS.
|
||
|
||
The per-key `spend` counter LiteLLM exposes (`LiteLLM_VerificationToken.spend`,
|
||
`/spend/keys`) is CYCLE-ONLY: it resets at the team's `budget_reset_at`, so it
|
||
cannot answer "how much has this key spent?". We instead sum `spend` over the
|
||
raw rows in `LiteLLM_SpendLogs`, grouped by the key identifier in `api_key`
|
||
(the key's LiteLLM hash — the same value we store as `member_api_keys.hash`).
|
||
|
||
A row whose `api_key` matches one of the member's stored `member_api_keys`
|
||
rows is labelled with that key's name. Any row that matches no stored key
|
||
belongs to the portal-held CHAT key (alias `member:<email>`, minted at
|
||
provisioning, never in member_api_keys) and is grouped under the label
|
||
"chat" — so a member sees "chat: $X, <keyname>: $Y".
|
||
|
||
Returns [{"name", "is_chat", "spend", "tokens", "requests"}], API keys by
|
||
spend descending with the chat row last (it is secondary in the keys view).
|
||
Empty list when the DB is not configured — never raises for the caller.
|
||
"""
|
||
if not litellm_db_ready():
|
||
return []
|
||
|
||
conn = get_db()
|
||
try:
|
||
rows = conn.execute(
|
||
"SELECT name, hash FROM member_api_keys WHERE email = ?", (email,)
|
||
).fetchall()
|
||
finally:
|
||
conn.close()
|
||
by_hash = {r[1]: r[0] for r in rows if r[1]}
|
||
|
||
pg = _pg_connect()
|
||
try:
|
||
with pg.cursor() as cur:
|
||
cur.execute(
|
||
'SELECT api_key, COALESCE(SUM(spend), 0), '
|
||
'COALESCE(SUM(completion_tokens), 0), COUNT(*) '
|
||
'FROM "LiteLLM_SpendLogs" WHERE team_id = %s GROUP BY api_key',
|
||
(team_id,),
|
||
)
|
||
raw = cur.fetchall()
|
||
finally:
|
||
pg.close()
|
||
|
||
# Aggregate by a stable internal id. Rows matching no stored key are the chat
|
||
# key; the sentinel can't collide with a real name (key names are
|
||
# [A-Za-z0-9._-], so a member key literally named "chat" stays separate).
|
||
agg: dict = {}
|
||
for api_key, spend, tokens, reqs in raw:
|
||
name = by_hash.get(api_key)
|
||
cid = name if name else "__chat__"
|
||
a = agg.setdefault(
|
||
cid,
|
||
{"name": name or "chat", "is_chat": name is None,
|
||
"spend": 0.0, "tokens": 0, "requests": 0},
|
||
)
|
||
a["spend"] += float(spend or 0.0)
|
||
a["tokens"] += int(tokens or 0)
|
||
a["requests"] += int(reqs or 0)
|
||
|
||
out = list(agg.values())
|
||
# API keys by spend desc, then the chat key last.
|
||
out.sort(key=lambda x: (x["is_chat"], -x["spend"]))
|
||
for r in out:
|
||
r["spend"] = round(r["spend"], 2)
|
||
return out
|
||
|
||
|
||
async def broker_get_usage(email: str, days: int = 0) -> dict:
|
||
"""Return the member's spend vs. balance, plus per-model and token detail.
|
||
|
||
This is the read-only primitive the member dashboard uses (no master key on
|
||
the client). Reads go straight to LiteLLM's Postgres when configured (fast,
|
||
bounded); otherwise falls back to the HTTP API path. `days` bounds the
|
||
per-model breakdown window (0 = all time); balance/spend always come from
|
||
the team's live totals.
|
||
"""
|
||
team_id = await litellm_get_or_create_team(email, get_member_balance(email))
|
||
|
||
allowance = get_member_balance(email)
|
||
credits = get_credit_balance(email)
|
||
spend = 0.0
|
||
reset_at = None
|
||
|
||
# Fast path: direct Postgres reads.
|
||
if litellm_db_ready():
|
||
try:
|
||
team = get_team_budget_from_db(team_id)
|
||
if team:
|
||
if team["max_budget"] is not None:
|
||
balance = float(team["max_budget"])
|
||
spend = float(team["spend"] or 0.0)
|
||
reset_at = team["budget_reset_at"]
|
||
usage = aggregate_model_usage(days=days, team_id=team_id)
|
||
# All-time spend for this member: total of their spend-log rows
|
||
# (the team `spend` counter resets with their allowance period).
|
||
all_time_spend = 0.0
|
||
try:
|
||
conn = _pg_connect()
|
||
try:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
'SELECT COALESCE(SUM(spend), 0) FROM "LiteLLM_SpendLogs" WHERE team_id = %s',
|
||
(team_id,),
|
||
)
|
||
all_time_spend = float(cur.fetchone()[0] or 0.0)
|
||
finally:
|
||
conn.close()
|
||
except Exception as e:
|
||
logger.warning("broker_get_usage: all-time spend query failed: %s", e)
|
||
model_list = [
|
||
{"model": m["model"], "spend": m["spend"], "tokens": m["tokens"],
|
||
"calls": m["requests"]}
|
||
for m in usage["models"]
|
||
]
|
||
# Per-key spend (chat key shown separately), from the spend-log rows —
|
||
# never LiteLLM's cycle-only per-key counter. Best-effort: a failure
|
||
# must not blank the whole usage view.
|
||
try:
|
||
key_spend = member_per_key_spend(email, team_id)
|
||
except Exception as e:
|
||
logger.warning("broker_get_usage: per-key spend failed: %s", e)
|
||
key_spend = []
|
||
# Per-day series for the member's usage chart (30/90/all-time
|
||
# toggles re-render client-side from this). Best-effort like
|
||
# key_spend: a failure degrades to an empty series, never an error.
|
||
try:
|
||
daily = member_daily_usage(days=days, team_id=team_id)
|
||
except Exception as e:
|
||
logger.warning("broker_get_usage: daily usage query failed: %s", e)
|
||
daily = []
|
||
return {
|
||
"email": email,
|
||
"guest": is_guest_account(email),
|
||
"balance": balance,
|
||
"allowance": allowance,
|
||
"credits": credits,
|
||
"credits_remaining": max(0.0, credits - max(0.0, spend - allowance)),
|
||
"spend": spend,
|
||
"all_time_spend": round(all_time_spend, 2),
|
||
"remaining": max(balance - spend, 0.0),
|
||
"total_tokens": usage["totals"]["total_tokens"],
|
||
"reset_at": reset_at,
|
||
"models": model_list,
|
||
"key_spend": key_spend,
|
||
"daily": daily,
|
||
}
|
||
except Exception as e:
|
||
logger.warning("broker_get_usage: DB path failed (%s); falling back to HTTP", e)
|
||
|
||
# Fallback path: LiteLLM HTTP API.
|
||
# /team/list and /spend/logs have grown slow as usage accumulates; the
|
||
# httpx default 5s read timeout was too short and made /broker/usage 500.
|
||
client_timeout = httpx.Timeout(connect=15.0, read=60.0, write=15.0, pool=15.0)
|
||
async with httpx.AsyncClient(timeout=client_timeout) as client:
|
||
# 1. Team spend + budget
|
||
r = await client.get(
|
||
f"{LITELLM_BASE}/team/list",
|
||
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
|
||
)
|
||
r.raise_for_status()
|
||
for t in r.json():
|
||
if t.get("team_id") == team_id:
|
||
balance = float(t.get("max_budget") or balance)
|
||
spend = float(t.get("spend") or 0.0)
|
||
reset_at = t.get("budget_reset_at")
|
||
break
|
||
|
||
# 2. Per-model token/spend breakdown from spend logs (filtered by team).
|
||
# /spend/logs does not reliably filter by team_id server-side, so we
|
||
# filter client-side on the returned rows.
|
||
models = {}
|
||
total_tokens = 0
|
||
team_rows: list = []
|
||
try:
|
||
r2 = await client.get(
|
||
f"{LITELLM_BASE}/spend/logs",
|
||
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"},
|
||
)
|
||
r2.raise_for_status()
|
||
rows = r2.json()
|
||
if isinstance(rows, dict):
|
||
rows = rows.get("data", rows.get("logs", []))
|
||
for row in rows if isinstance(rows, list) else []:
|
||
if row.get("team_id") != team_id:
|
||
continue
|
||
team_rows.append(row)
|
||
# Prefer model_group: it carries our provider/model alias (e.g.
|
||
# greenpt/glm-5.3-flash) when present; the raw `model` field holds
|
||
# messy upstream/internal names. Older rows predating the alias
|
||
# convention have model_group empty, so fall back to `model`.
|
||
model = row.get("model_group") or row.get("model") or "unknown"
|
||
m = models.setdefault(model, {"spend": 0.0, "tokens": 0, "calls": 0})
|
||
m["spend"] += float(row.get("spend") or 0.0)
|
||
m["tokens"] += int(row.get("total_tokens") or 0)
|
||
# Refused/failed attempts are not served requests.
|
||
if str(row.get("status") or "").lower() != "failure":
|
||
m["calls"] += 1
|
||
total_tokens += int(row.get("total_tokens") or 0)
|
||
except Exception as e: # spend logs are best-effort; don't fail the whole view
|
||
logger.warning("spend/logs fetch failed: %s", e)
|
||
# Per-day series for the usage chart (same day convention as the DB
|
||
# path: the UTC day of startTime).
|
||
daily = daily_series_from_rows(team_rows)
|
||
|
||
# All-time spend from the same log rows (team counter is period-only).
|
||
all_time_spend = round(sum(m["spend"] for m in models.values()), 2)
|
||
|
||
# Sort models by spend descending (which models are draining the most).
|
||
model_list = [
|
||
{"model": name, **stats}
|
||
for name, stats in sorted(models.items(), key=lambda kv: kv[1]["spend"], reverse=True)
|
||
]
|
||
|
||
return {
|
||
"email": email,
|
||
"guest": is_guest_account(email),
|
||
"balance": balance,
|
||
"allowance": allowance,
|
||
"credits": credits,
|
||
"credits_remaining": max(0.0, credits - max(0.0, spend - allowance)),
|
||
"spend": spend,
|
||
"all_time_spend": all_time_spend,
|
||
"remaining": max(balance - spend, 0.0),
|
||
"total_tokens": total_tokens,
|
||
"reset_at": reset_at,
|
||
"models": model_list,
|
||
# Per-key spend needs the Postgres spend-log table (LiteLLM's exposed
|
||
# per-key counter is cycle-only); this HTTP fallback runs only when the
|
||
# DB is unconfigured, so there is nothing accurate to report here.
|
||
"key_spend": [],
|
||
"daily": daily,
|
||
}
|
||
|
||
|
||
@app.get("/broker/usage")
|
||
@limiter.limit("60/minute")
|
||
async def broker_usage(request: Request, days: int = 0):
|
||
email = _verify_broker_secret(request)
|
||
return await broker_get_usage(email, days=days)
|
||
|
||
|
||
@app.get("/broker/guest-status")
|
||
@limiter.limit("30/minute")
|
||
async def broker_guest_status(request: Request):
|
||
"""What became of the caller's guest account (self-service cancel support).
|
||
|
||
Served through the broker like every dashboard route. The one broker read
|
||
that answers for an ENDED guest too (see _verify_broker_guest_identity): a
|
||
cancelled or lapsed guest's dashboard loads this so it can say, plainly,
|
||
that access ended — the data is kept — and a membership is still one
|
||
contribution away. For an active guest it reports what the cancel button
|
||
needs (guest, active, ledger status); for a member it simply says
|
||
guest: false.
|
||
"""
|
||
email = _verify_broker_guest_identity(request)
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT active, balance, credit_balance FROM members WHERE email = ?",
|
||
(email,),
|
||
).fetchone()
|
||
conn.close()
|
||
if not row:
|
||
raise HTTPException(403, "No active membership")
|
||
active, balance, credits = row
|
||
if not is_guest_account(email):
|
||
return {"guest": False, "active": bool(active)}
|
||
guest_status = _guest_ledger_status(email) or ("active" if active else "inactive")
|
||
if active:
|
||
# Same remaining-credit figure the usage view shows.
|
||
spend = None
|
||
team_id = await litellm_team_id_for_email(email)
|
||
if team_id:
|
||
team = get_team_budget_from_db(team_id)
|
||
if team:
|
||
spend = team.get("spend")
|
||
eff = compute_max_budget(
|
||
float(balance) if balance is not None else 0.0,
|
||
float(credits or 0.0),
|
||
spend,
|
||
)
|
||
credits_remaining = round(eff["credits_remaining"], 2)
|
||
else:
|
||
credits_remaining = 0.0
|
||
return {
|
||
"guest": True,
|
||
"active": bool(active),
|
||
"ended": not active,
|
||
"guest_status": guest_status,
|
||
"cancelled": guest_status == "cancelled by recipient",
|
||
"credits_remaining": credits_remaining,
|
||
"credits_returned": _guest_returned_credits(email),
|
||
}
|
||
|
||
|
||
@app.post("/broker/cancel-guest")
|
||
@limiter.limit("10/minute")
|
||
async def broker_cancel_guest(request: Request):
|
||
"""End the caller's own guest account (self-service cancellation).
|
||
|
||
Authenticated with the member-dashboard broker identity, guest accounts
|
||
only (see cancel_guest_account for the semantics and the idempotence).
|
||
A paying member calling this gets a 403, never a cancellation.
|
||
"""
|
||
email = _verify_broker_guest_identity(request)
|
||
return await cancel_guest_account(email)
|
||
|
||
|
||
# TRANSITION ALIASES (guest rename, 2026-11-08): see the note above the admin
|
||
# aliases. The deployed dashboard build still calls these two paths.
|
||
@app.get("/broker/gift-status")
|
||
@limiter.limit("30/minute")
|
||
async def broker_gift_status_alias(request: Request):
|
||
return await broker_guest_status(request)
|
||
|
||
|
||
@app.post("/broker/cancel-gift")
|
||
@limiter.limit("10/minute")
|
||
async def broker_cancel_gift_alias(request: Request):
|
||
return await broker_cancel_guest(request)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Member self-serve guest invitations (provision immediately, 2026-11-08).
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# Member guest invites serialize on this lock (see POST /broker/guests):
|
||
# the one-per-30-days guard reads the guests ledger, and the row that closes
|
||
# the window is written only at the END of provisioning.
|
||
_GUEST_INVITE_LOCK = asyncio.Lock()
|
||
|
||
|
||
def _verify_broker_identity_any_state(request: Request) -> str:
|
||
"""Broker auth that admits a member row in ANY state, for STATUS reads only.
|
||
|
||
`_verify_broker_secret` refuses a lapsed member, which is right for usage,
|
||
keys and everything else — but it left a lapsed member's dashboard with nothing
|
||
to render. This verifier is scoped to one read-only route that says why you are
|
||
not active and what restores access; it never grants access to usage or keys.
|
||
"""
|
||
if not BROKER_SECRET:
|
||
raise HTTPException(503, "Broker secret not configured")
|
||
provided = request.headers.get("x-broker-secret", "")
|
||
if not secrets.compare_digest(provided, BROKER_SECRET):
|
||
raise HTTPException(401, "Invalid broker secret")
|
||
identity = (request.headers.get("x-member-email") or request.headers.get("x-member-user") or "").strip()
|
||
if not identity:
|
||
raise HTTPException(401, "Missing member identity")
|
||
email = resolve_member_email(identity)
|
||
conn = get_db()
|
||
row = conn.execute("SELECT email FROM members WHERE email = ?", (email,)).fetchone()
|
||
conn.close()
|
||
if not row:
|
||
raise HTTPException(403, "No membership on file")
|
||
return email
|
||
|
||
|
||
@app.get("/broker/membership-status")
|
||
@limiter.limit("30/minute")
|
||
async def broker_membership_status(request: Request):
|
||
"""Why is my dashboard empty? State, reason, minimum and what restores access.
|
||
|
||
Answers in ANY membership state, including lapsed and awaiting-contribution,
|
||
so the dashboard can explain rather than render a blank page.
|
||
"""
|
||
email = _verify_broker_identity_any_state(request)
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT active, slug, source, credit_balance, balance FROM members WHERE email = ?",
|
||
(email,)).fetchone()
|
||
decision = conn.execute(
|
||
"SELECT cycle, decision, evidence, decided_at FROM allowance_log "
|
||
"WHERE email = ? ORDER BY decided_at DESC LIMIT 1", (email,)).fetchone()
|
||
conn.close()
|
||
active, slug, source, credits, balance = row
|
||
return {
|
||
"active": bool(active),
|
||
"state": "active" if active else "inactive",
|
||
"source": source,
|
||
"minimum_contribution": MIN_CONTRIBUTION,
|
||
"contribute_url": credit_pack_url(),
|
||
"allowance": float(balance if balance is not None else 0.0),
|
||
"credits": float(credits or 0.0),
|
||
"last_decision": ({"cycle": decision[0], "decision": decision[1],
|
||
"evidence": decision[2], "decided_at": decision[3]}
|
||
if decision else None),
|
||
}
|
||
|
||
|
||
@app.get("/broker/guests")
|
||
@limiter.limit("30/minute")
|
||
async def broker_guest_list(request: Request):
|
||
"""The caller's own guest-access view: invitation allowance + their trail.
|
||
|
||
Served through the member broker identity (a guest identity fails it —
|
||
only members invite guests). Returns:
|
||
* eligible / next_eligible_at — the rolling one-guest-per-member-per-
|
||
GUEST_PER_INVITER_DAYS rule, counted from the guests ledger (a
|
||
legacy approved request carries its own ledger row);
|
||
* trail — every invitation this member has made, newest first: legacy
|
||
REJECTED requests from the trial-approval period (kept for history;
|
||
nothing new is ever decided) plus every row in the guests ledger
|
||
with what became of it (active, converted, expired, depleted,
|
||
cancelled by recipient). Each guest entry also carries `state` —
|
||
'invited' while the guest has not accepted their Cloudron invite
|
||
(provisioned but awaiting setup, cannot log in yet), else the
|
||
ledger status; consumers should render `state` and fall back to
|
||
`status` only when it is absent (older payloads).
|
||
"""
|
||
email = _verify_broker_secret(request)
|
||
conn = get_db()
|
||
requests = conn.execute(
|
||
"SELECT id, recipient_email, recipient_name, created_at, decided_at, status, note "
|
||
"FROM guest_requests WHERE inviter_email = ? ORDER BY id DESC",
|
||
(email,),
|
||
).fetchall()
|
||
sent = conn.execute(
|
||
"SELECT g.recipient_email, g.created_at, g.status, "
|
||
"m.cloudron_user_id, m.active, m.account_type "
|
||
"FROM guests g LEFT JOIN members m ON m.email = g.recipient_email "
|
||
"WHERE g.inviter_email = ? ORDER BY g.id DESC",
|
||
(email,),
|
||
).fetchall()
|
||
window = f"-{int(GUEST_PER_INVITER_DAYS)} days"
|
||
last_sent = conn.execute(
|
||
"SELECT MAX(created_at) FROM guests WHERE inviter_email = ? "
|
||
"AND created_at >= datetime('now', ?)",
|
||
(email, window),
|
||
).fetchone()[0]
|
||
conn.close()
|
||
|
||
# Invite acceptance per trail guest — the same Cloudron signal the admin
|
||
# overview uses (username + inviteAccepted). Fetched once, fail-soft: when
|
||
# the fetch fails the invited state is simply not claimed and the trail
|
||
# keeps showing the ledger status.
|
||
usernames: dict[str, str] = {}
|
||
invite_accepted: dict[str, bool] = {}
|
||
try:
|
||
async with httpx.AsyncClient(timeout=15.0) as client:
|
||
users = await cloudron_fetch_all_users(client)
|
||
if users is not None:
|
||
for u in users:
|
||
uid = u.get("id")
|
||
if u.get("username"):
|
||
usernames[uid] = u["username"]
|
||
invite_accepted[uid] = bool(u.get("inviteAccepted"))
|
||
except Exception as e:
|
||
logger.warning("broker guests: cloudron users fetch failed: %s", e)
|
||
|
||
eligible = last_sent is None
|
||
next_eligible_at = None
|
||
if not eligible:
|
||
try:
|
||
anchor = datetime.strptime(str(last_sent)[:19], "%Y-%m-%d %H:%M:%S")
|
||
next_eligible_at = (anchor + timedelta(days=int(GUEST_PER_INVITER_DAYS))
|
||
).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||
except ValueError:
|
||
pass
|
||
trail = []
|
||
for rid, recipient, rname, created, decided, status, note in requests:
|
||
if status == "rejected":
|
||
trail.append({
|
||
"kind": "request", "request_id": rid, "recipient": recipient,
|
||
"name": rname, "created_at": created, "decided_at": decided,
|
||
"status": status, "note": note,
|
||
})
|
||
for recipient, created, status, user_id, g_active, g_type in sent:
|
||
entry = {"kind": "guest", "recipient": recipient,
|
||
"created_at": created, "status": status}
|
||
if ((g_type or "member") == "guest" and g_active
|
||
and guest_invite_pending(user_id, usernames, invite_accepted)):
|
||
# Provisioned but the invite is not accepted yet — display as
|
||
# 'invited', never 'active' (the guest cannot log in until they
|
||
# accept; the ledger status stays the trail's record of record).
|
||
entry["state"] = "invited"
|
||
else:
|
||
entry["state"] = status
|
||
trail.append(entry)
|
||
trail.sort(key=lambda t: t.get("decided_at") or t["created_at"] or "", reverse=True)
|
||
return {
|
||
"eligible": eligible,
|
||
"per_inviter_days": int(GUEST_PER_INVITER_DAYS),
|
||
"next_eligible_at": next_eligible_at,
|
||
"trail": trail,
|
||
}
|
||
|
||
|
||
@app.post("/broker/guests")
|
||
@limiter.limit("10/minute")
|
||
async def broker_guest_invite(request: Request):
|
||
"""Invite a guest, from the member dashboard (self-serve guest access).
|
||
|
||
Authenticated with the standard member broker identity — a GUEST identity
|
||
fails it, so only members reach here. Body: {"email": "<recipient>",
|
||
"name": "<optional display name>"}. Every protection runs BEFORE anything
|
||
is created (guest_request_guard): active-member inviter, one invitation
|
||
per member per rolling GUEST_PER_INVITER_DAYS, recipient never an existing
|
||
member or a prior guest, and never a MANUAL_MEMBERS address. A valid
|
||
invitation PROVISIONS IMMEDIATELY (operator decision, 2026-11-08 — the
|
||
trial-era approval gate is gone): provision_guest_member is the same
|
||
pipeline the operator's /admin/provision-guest runs (Cloudron user in the
|
||
guests group, $2 non-expiring credit, GUEST_DAYS time box, invitation
|
||
email naming the inviter, newsletter subscribe, ledger row) and re-runs
|
||
every guard, so a moved-state refusal still refuses.
|
||
|
||
The whole guard-then-provision sequence runs under _GUEST_INVITE_LOCK:
|
||
the 30-day rule is a read of the guests ledger, provisioning takes
|
||
seconds of external calls, and the ledger row is written LAST — so two
|
||
PARALLEL direct calls could otherwise both pass the guard before either
|
||
row exists and slip two guests past the one-per-month rule. The app runs
|
||
a single uvicorn process, so this lock makes check-and-send atomic for
|
||
every member invite; the second caller re-runs the guard AFTER the first
|
||
caller's ledger row exists and is refused.
|
||
"""
|
||
email = _verify_broker_secret(request)
|
||
body = await request.json()
|
||
recipient = (body.get("email") or "").strip().lower()
|
||
name = (body.get("name") or "").strip()
|
||
if not recipient or "@" not in recipient:
|
||
raise HTTPException(400, "A valid guest email is required")
|
||
|
||
async with _GUEST_INVITE_LOCK:
|
||
conn = get_db()
|
||
refusal = guest_request_guard(conn, email, recipient)
|
||
conn.close()
|
||
if refusal:
|
||
raise HTTPException(429, refusal)
|
||
|
||
user_id = await provision_guest_member(recipient, name, email)
|
||
return {
|
||
"status": "provisioned", "account_type": "guest", "email": recipient,
|
||
"inviter": email, "user_id": user_id, "budget": GUEST_BUDGET,
|
||
"expires_in_days": int(GUEST_DAYS),
|
||
}
|
||
|
||
|
||
@app.get("/broker/metrics")
|
||
@limiter.limit("30/minute")
|
||
async def broker_metrics(request: Request, days: int = 90):
|
||
"""Co-op metrics over time, for the member dashboard.
|
||
|
||
The same series the admin panel reads. It comes through the broker because
|
||
it is AGGREGATE and non-personal — co-op size, requests served, spend — the
|
||
same class of social-proof data as /broker/model-leaderboard. No per-member
|
||
figure passes through here, and members asked to see how the co-op is doing.
|
||
|
||
Rows carry the energy provenance the admin series does: `provenance`
|
||
(measured | mixed | estimated | none) plus `tokens_measured`,
|
||
`tokens_estimated` and `factor_revision` — so a member-facing chart can say
|
||
how much of a day's figure rested on a measurement. NULL splits (older rows)
|
||
report `provenance: "none"`. See docs/energy-metrics.md.
|
||
"""
|
||
_verify_broker_secret(request)
|
||
series = metrics_series(days)
|
||
return {
|
||
"series": series,
|
||
"count": len(series),
|
||
"days": max(1, min(int(days or 90), 730)),
|
||
# The co-op's all-time spend, from the same helper the admin panel reads,
|
||
# so the two apps cannot report different totals for the same claim. The
|
||
# dashboard used to sum the series in the browser and got $900.51 where
|
||
# the panel said $941.37.
|
||
"totals": _coop_spend_totals_safe(),
|
||
}
|
||
|
||
|
||
@app.get("/broker/money-series")
|
||
@limiter.limit("30/minute")
|
||
async def broker_money_series(request: Request):
|
||
"""Revenue / token-spend / OC-balance series, for the member dashboard.
|
||
|
||
The same payload /admin/money-series serves the panel, through the broker for
|
||
the same reason /broker/metrics is: it is AGGREGATE and carries no member
|
||
identities — cumulative revenue, cumulative token spend, and the co-op's
|
||
Open Collective balance. The co-op's own finances are not private to
|
||
stewards here: the member dashboard charts them next to the other co-op
|
||
charts, from the shared chart definitions, so members and stewards see the
|
||
same picture. Hourly-cached (the OC queries are slow).
|
||
|
||
The balance series exists only from the day the snapshots began (the OC API
|
||
exposes just the CURRENT balance) — a chart must render that as a gap, never
|
||
as a zero, which is what the shared renderer does.
|
||
"""
|
||
_verify_broker_secret(request)
|
||
return await _get_money_payload()
|
||
|
||
|
||
@app.get("/broker/model-leaderboard")
|
||
@limiter.limit("30/minute")
|
||
async def broker_model_leaderboard(request: Request, days: int = 0):
|
||
"""Co-op-wide model popularity for the member dashboard.
|
||
|
||
Same aggregation as the admin model-usage view (spend/tokens/requests per
|
||
model, across all members) but served through the broker so members can
|
||
see which models the co-op uses most — social guidance for model choice.
|
||
No per-member data; aggregate totals only.
|
||
"""
|
||
_verify_broker_secret(request)
|
||
if litellm_db_ready():
|
||
try:
|
||
usage = aggregate_model_usage(days=days)
|
||
except Exception as e:
|
||
logger.warning("broker leaderboard: DB path failed (%s)", e)
|
||
usage = None
|
||
if usage:
|
||
return {
|
||
"models": [
|
||
{"model": m["model"], "spend": m["spend"],
|
||
"tokens": m["tokens"], "requests": m["requests"]}
|
||
for m in usage["models"][:10]
|
||
]
|
||
}
|
||
# Fallback: empty (rare; the DB path is the norm)
|
||
return {"models": []}
|
||
|
||
|
||
async def reconcile_memberships() -> dict:
|
||
"""Reconcile the portal against the live Open Collective member list.
|
||
|
||
- Provisions members who are BACKER/ADMIN on OC but not yet in our DB
|
||
(missed webhook, or contributed before the email step existed).
|
||
- Deactivates members who are in our DB but no longer BACKER/ADMIN
|
||
(lapsed or cancelled membership).
|
||
- Syncs Loomio to the resulting active-member set.
|
||
|
||
Idempotent and safe to run repeatedly. Returns a summary of actions taken.
|
||
"""
|
||
# fetch_members() raises when OC is unreachable or the query fails; treat
|
||
# that like an empty result so the documented fail-safe below actually
|
||
# applies, instead of letting the exception escape and 500 the sweep.
|
||
try:
|
||
active = await fetch_members()
|
||
except Exception as e:
|
||
logger.warning(
|
||
"Sweep: member fetch failed; skipping reconciliation (fail-safe): %s", e
|
||
)
|
||
return {
|
||
"status": "skipped",
|
||
"reason": "member_fetch_failed",
|
||
"provisioned": 0,
|
||
"deactivated": 0,
|
||
"reinvited": 0,
|
||
"newsletter_removed": 0,
|
||
}
|
||
active_emails = {m["email"] for m in active}
|
||
active_by_email = {m["email"]: m for m in active}
|
||
|
||
# FAIL-SAFE: if the OC fetch returned nothing (token expired, OC outage,
|
||
# or a query error), we must NOT deactivate everyone — that would be a
|
||
# catastrophic false-positive. Skip reconciliation entirely in that case.
|
||
if not active:
|
||
logger.warning("Sweep: fetch_members returned empty; skipping reconciliation (fail-safe)")
|
||
return {"status": "skipped", "reason": "empty_member_list", "provisioned": 0, "deactivated": 0}
|
||
|
||
# Currently provisioned accounts (from our DB), with their type — the guest
|
||
# marker drives both the conversion trigger and the deactivation skip.
|
||
conn = get_db()
|
||
rows = conn.execute(
|
||
"SELECT email, cloudron_user_id, account_type FROM members WHERE active = 1"
|
||
).fetchall()
|
||
conn.close()
|
||
provisioned = {
|
||
r[0]: {"user_id": r[1], "account_type": (r[2] or "member")}
|
||
for r in rows
|
||
}
|
||
|
||
# Members whose contribution arrives under a DIFFERENT OC email (set by
|
||
# POST /admin/guest-convert): their oc_email is an additional presence
|
||
# matcher — when it appears in the live OC list the account counts as
|
||
# active even though their own email does not.
|
||
conn = get_db()
|
||
oc_rows = conn.execute(
|
||
"SELECT email, oc_email FROM members WHERE oc_email IS NOT NULL AND oc_email != ''"
|
||
).fetchall()
|
||
conn.close()
|
||
for row_email, oc in oc_rows:
|
||
if (oc or "").strip().lower() in active_emails:
|
||
active_emails.add(row_email)
|
||
if row_email not in active_by_email:
|
||
active_by_email[row_email] = {
|
||
"email": row_email, "name": row_email.split("@")[0],
|
||
"slug": "", "role": "BACKER",
|
||
}
|
||
logger.info("Sweep: %s active via OC email %s", row_email, oc)
|
||
|
||
provisioned_count = 0
|
||
deactivated_count = 0
|
||
converted_count = 0
|
||
|
||
# 0. Organization-backed memberships: dues paid by an OC Organization.
|
||
# Linked accounts (members.org_slug) stay active while their org holds a
|
||
# BACKER/ADMIN role on the Membership tier; lapsed orgs deactivate like
|
||
# individuals. Fail-safe: if the org fetch fails, keep linked accounts
|
||
# active (never deactivate on our own fetch failure).
|
||
try:
|
||
org_backers = await fetch_org_backers()
|
||
org_fetch_ok = True
|
||
except Exception as e:
|
||
logger.warning("Sweep: org fetch failed (keeping linked accounts active): %s", e)
|
||
org_backers = {}
|
||
org_fetch_ok = False
|
||
if org_backers or not org_fetch_ok:
|
||
conn = get_db()
|
||
org_rows = conn.execute(
|
||
"SELECT email, org_slug FROM members WHERE org_slug IS NOT NULL"
|
||
).fetchall()
|
||
conn.close()
|
||
for org_email, org_slug in org_rows:
|
||
org = org_backers.get(org_slug)
|
||
if org_fetch_ok and not org:
|
||
continue # org lapsed → falls through to deactivation below
|
||
if org_email not in active_emails:
|
||
active.append({
|
||
"email": org_email,
|
||
"name": (org or {}).get("name") or org_slug,
|
||
"slug": org_slug,
|
||
"role": (org or {}).get("role", "BACKER"),
|
||
"tier": ((org or {}).get("tier", "") or "") + " (org)",
|
||
})
|
||
active_emails.add(org_email)
|
||
logger.info("Sweep: org-linked member %s active via org %s", org_email, org_slug)
|
||
|
||
# 1. Provision missing members — and CONVERT guest accounts whose owner has
|
||
# joined (their email now appears in the live OC backer list). The
|
||
# conversion itself lives in provision_member, so the webhook and
|
||
# /admin/provision trigger the same transition automatically.
|
||
for m in active:
|
||
email = m["email"]
|
||
entry = provisioned.get(email)
|
||
if entry is None:
|
||
try:
|
||
await provision_member(email, m["name"], m["slug"])
|
||
provisioned_count += 1
|
||
logger.info("Sweep: provisioned %s", email)
|
||
except Exception as e:
|
||
logger.warning("Sweep: failed to provision %s: %s", email, e)
|
||
elif entry["account_type"] == "guest":
|
||
try:
|
||
await provision_member(email, m["name"], m["slug"])
|
||
converted_count += 1
|
||
logger.info("Sweep: converted guest account %s to member", email)
|
||
except Exception as e:
|
||
logger.warning("Sweep: failed to convert guest %s: %s", email, e)
|
||
|
||
# 1b. Guest lifecycles: time-box expiry + spent-out guests. A guest is never
|
||
# deactivated for LACKING a contribution (next loop excludes them) —
|
||
# only by its own rules. Runs before the member pass so its results
|
||
# are independent of OC state.
|
||
try:
|
||
guest_counts = await reconcile_guest_lifecycles()
|
||
except Exception as e:
|
||
logger.warning("Sweep: guest lifecycle pass failed (non-fatal): %s", e)
|
||
guest_counts = {"expired": 0, "depleted": 0}
|
||
|
||
# 2. Deactivate lapsed members (but never manual/founder members, who have
|
||
# no OC membership to lapse, and never GUEST accounts — a guest has no OC
|
||
# membership by design; it lapses only via its own rules, handled above).
|
||
for email, info in provisioned.items():
|
||
if info["account_type"] == "guest":
|
||
continue
|
||
user_id = info["user_id"]
|
||
if email not in active_emails and not is_manual_member(email):
|
||
try:
|
||
if INACTIVE_GROUP_ID and user_id:
|
||
await cloudron_set_group(user_id, INACTIVE_GROUP_ID)
|
||
deactivate_member(email)
|
||
await revoke_member_litellm_access(email)
|
||
await listmonk_sync(email, subscribe=False)
|
||
deactivated_count += 1
|
||
logger.info("Sweep: deactivated %s", email)
|
||
except Exception as e:
|
||
logger.warning("Sweep: failed to deactivate %s: %s", email, e)
|
||
|
||
# 2b. Re-invite never-activated members (provisioned but no account setup).
|
||
# A member who was provisioned (e.g. webhook fired while the OC token was
|
||
# missing the `email` scope, or the invite email went to spam) may sit
|
||
# active-but-unactivated indefinitely. Re-send their welcome email, but
|
||
# throttle by REINVITE_AFTER_DAYS so we don't spam them nightly.
|
||
reinvited_count = 0
|
||
conn = get_db()
|
||
reinvite_rows = conn.execute(
|
||
"SELECT email, cloudron_user_id, welcome_sent_at, account_type FROM members WHERE active = 1"
|
||
).fetchall()
|
||
conn.close()
|
||
now = datetime.now(timezone.utc)
|
||
for email, user_id, sent_at, acct_type in reinvite_rows:
|
||
if not user_id:
|
||
continue
|
||
if (acct_type or "member") == "guest":
|
||
# Guest accounts are re-invited by re-running /admin/provision-guest
|
||
# (it re-sends the guest email idempotently). The MEMBER pipeline
|
||
# here would move the user into the members group.
|
||
continue
|
||
# Skip if already activated.
|
||
try:
|
||
if await cloudron_is_activated(user_id):
|
||
continue
|
||
except Exception as e:
|
||
logger.warning("Sweep: activation check failed for %s: %s", email, e)
|
||
continue
|
||
# Throttle: only re-invite if the last send was > REINVITE_AFTER_DAYS ago.
|
||
if sent_at:
|
||
try:
|
||
last = datetime.fromisoformat(sent_at)
|
||
# stored as UTC naive (datetime('now') is UTC); attach tz
|
||
if last.tzinfo is None:
|
||
last = last.replace(tzinfo=timezone.utc)
|
||
if (now - last).total_seconds() < REINVITE_AFTER_DAYS * 86400:
|
||
continue
|
||
except Exception:
|
||
pass
|
||
# Re-send the welcome email (reuses the stored key — no duplicate alias).
|
||
try:
|
||
name = active_by_email.get(email, {}).get("name") or email.split("@")[0]
|
||
await provision_member(email, name, "")
|
||
reinvited_count += 1
|
||
logger.info("Sweep: re-invited %s (never activated)", email)
|
||
except Exception as e:
|
||
logger.warning("Sweep: failed to re-invite %s: %s", email, e)
|
||
|
||
# 3. Sync Loomio.
|
||
await sync_loomio_memberships()
|
||
await sync_matrix_space()
|
||
|
||
# 4. Reconcile the newsletter list against active membership (removes
|
||
# deleted accounts, never-activated test accounts, lapsed members).
|
||
newsletter = await listmonk_reconcile()
|
||
|
||
return {
|
||
"status": "reconciled",
|
||
"provisioned": provisioned_count,
|
||
"deactivated": deactivated_count,
|
||
"reinvited": reinvited_count,
|
||
"converted": converted_count,
|
||
"guests_expired": guest_counts.get("expired", 0),
|
||
"guests_depleted": guest_counts.get("depleted", 0),
|
||
"guest_lapse_notices": guest_counts.get("lapse_notices", 0),
|
||
"newsletter_removed": newsletter.get("removed", 0),
|
||
}
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Allowance renewal — payment-driven, anchored to the member's JOIN day. #
|
||
# --------------------------------------------------------------------------- #
|
||
# Policy (operator decision 2026-11-04). Open Collective aligns every member's
|
||
# charge onto the collective's common billing day, so anchoring an allowance
|
||
# reset to the PAYMENT date lets a charge move the boundary and skip a cycle
|
||
# entirely: a member who joins mid-month can be charged on the common billing
|
||
# day within weeks, and the old reset rule re-anchored them to that new day
|
||
# with no reset in between. So the
|
||
# reset is anchored to the day of the month the member JOINED, and renewal is
|
||
# decided by asking Open Collective directly whether a recurring contribution is
|
||
# live — on our own schedule, not inferred from webhook delivery. A webhook that
|
||
# never arrives then costs a delayed renewal, never a lost month.
|
||
#
|
||
# Decisions (per member, per cycle):
|
||
# renewed — a recurring contribution is live; the allowance reset is pinned
|
||
# to the member's next join-day.
|
||
# reminded — no live recurring contribution; one reminder email is sent.
|
||
# deactivated — reminded on an earlier pass and still unpaid after the grace
|
||
# window; the member loses group access but keeps their account.
|
||
# would_* — report mode (and a send run while ALLOWANCE_DEACTIVATE is off)
|
||
# writes the `would_` form, including `would_deactivate` for a
|
||
# reminder older than the grace window, so the deactivation the
|
||
# policy WOULD apply is visible BEFORE the flag is enabled.
|
||
# Every decision is written to allowance_log with the Open Collective evidence
|
||
# (payment date, or its absence) so a dispute is answered from a record.
|
||
ALLOWANCE_POLICY = (os.environ.get("ALLOWANCE_POLICY", "report") or "report").strip().lower()
|
||
# Deactivation is gated SEPARATELY and defaults OFF, so the renewal/reminder path
|
||
# can be exercised before anyone loses access (operator directive 2026-11-04).
|
||
ALLOWANCE_DEACTIVATE = (os.environ.get("ALLOWANCE_DEACTIVATE", "off") or "off").strip().lower()
|
||
# Days to wait after the single reminder before deactivating a still-unpaid member.
|
||
PAYMENT_GRACE_DAYS = float(os.environ.get("PAYMENT_GRACE_DAYS", "7"))
|
||
|
||
|
||
def _ensure_allowance_tables(conn) -> None:
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS allowance_log ("
|
||
" id INTEGER PRIMARY KEY AUTOINCREMENT,"
|
||
" email TEXT NOT NULL, cycle TEXT NOT NULL, decision TEXT NOT NULL,"
|
||
" evidence TEXT, anchor_day INTEGER,"
|
||
" decided_at TEXT DEFAULT (datetime('now')))"
|
||
)
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS payment_reminders ("
|
||
" email TEXT NOT NULL, cycle TEXT NOT NULL, sent_at TEXT NOT NULL,"
|
||
" PRIMARY KEY (email, cycle))"
|
||
)
|
||
cols = {r[1] for r in conn.execute("PRAGMA table_info(members)")}
|
||
if "anchor_day" not in cols:
|
||
conn.execute("ALTER TABLE members ADD COLUMN anchor_day INTEGER")
|
||
conn.commit()
|
||
|
||
|
||
def member_membership_state(email: str, active: bool) -> tuple[str, str]:
|
||
"""Derived membership state for the admin panel — read-only, no side effects.
|
||
|
||
Mirrors what the allowance reconciliation already knows, so the panel reflects
|
||
the portal rather than inventing policy of its own:
|
||
|
||
inactive — membership deactivated (members.active = 0).
|
||
in_grace — a payment reminder went out and we are still inside
|
||
the PAYMENT_GRACE_DAYS window.
|
||
awaiting_contribution — the member is unpaid: either flagged with no
|
||
reminder yet, or the grace window has elapsed
|
||
(deactivation pending).
|
||
active — current.
|
||
|
||
THE REMINDER TABLE IS A HISTORY, NOT A STATE. A row stays after the member
|
||
pays, so "a reminder exists" is not "they are in grace". On 2026-11-13
|
||
mark@ofosos.org showed as in grace while his contribution was live: reminded
|
||
2026-11-04, grace elapsed 2026-11-11, and the reconciliation logged `renewed`
|
||
for him at 19:02 that afternoon. The old version returned grace from the
|
||
reminder row alone, forever.
|
||
|
||
So the LATEST RECONCILIATION DECISION decides, and a past reminder only
|
||
matters when nothing since says they paid. Ordered by decided_at, not id:
|
||
ids come from insertion order and a later-inserted row can carry an earlier
|
||
timestamp (the same member had id 2296 dated 11-11 next to id 2286 dated
|
||
11-13).
|
||
"""
|
||
if not active:
|
||
return "inactive", "deactivated"
|
||
try:
|
||
conn = get_db()
|
||
latest = conn.execute(
|
||
"SELECT decision, decided_at FROM allowance_log WHERE email = ? "
|
||
"ORDER BY decided_at DESC, id DESC LIMIT 1",
|
||
(email,),
|
||
).fetchone()
|
||
rem = conn.execute(
|
||
"SELECT sent_at FROM payment_reminders WHERE email = ? "
|
||
"ORDER BY sent_at DESC LIMIT 1",
|
||
(email,),
|
||
).fetchone()
|
||
conn.close()
|
||
except Exception: # tables absent / unreadable: fail safe
|
||
return "active", ""
|
||
decision = (latest[0] if latest else "") or ""
|
||
if decision in ("renewed", "would_renew"):
|
||
# They are paying. Any reminder belongs to an earlier cycle.
|
||
return "active", ""
|
||
if rem:
|
||
sent = _parse_oc_dt(str(rem[0])) if rem[0] else None
|
||
within = sent is not None and (datetime.now(timezone.utc) - sent) <= timedelta(days=PAYMENT_GRACE_DAYS)
|
||
if within:
|
||
return "in_grace", (f"awaiting contribution — reminder sent "
|
||
f"{str(rem[0])[:10]}, grace {int(PAYMENT_GRACE_DAYS)}d")
|
||
return "awaiting_contribution", (
|
||
f"unpaid — reminder sent {str(rem[0])[:10]}, grace of "
|
||
f"{int(PAYMENT_GRACE_DAYS)}d elapsed, deactivation pending")
|
||
if decision in ("reminded", "would_remind", "remind_failed"):
|
||
return "awaiting_contribution", "no current contribution at Open Collective"
|
||
if decision in ("would_deactivate", "deactivated"):
|
||
return "awaiting_contribution", f"grace elapsed ({decision})"
|
||
return "active", ""
|
||
|
||
|
||
def guest_invite_pending(user_id, usernames: dict, invite_accepted: dict) -> bool:
|
||
"""True when a guest's Cloudron invite is OUT but NOT yet accepted.
|
||
|
||
The provisioned-but-not-set-up state: the Cloudron account exists (the
|
||
invitation email with the setup link went out) but the guest has not
|
||
clicked it — inviteAccepted is false and no username is chosen, so they
|
||
cannot log in yet. Derived the same way the overview's `activated` flag
|
||
is (username and inviteAccepted move in lockstep for invite-created
|
||
accounts); NOT accepted = inviteAccepted false AND no username.
|
||
|
||
Only a SEEN user record can be pending: when the Cloudron users fetch
|
||
failed (or the user is unknown to it) the evidence is absent, and the
|
||
caller must fall back to the membership state rather than claim
|
||
'invited' — never assert a state the data does not show.
|
||
"""
|
||
if not user_id:
|
||
return False
|
||
if user_id not in invite_accepted and user_id not in usernames:
|
||
return False
|
||
return not (invite_accepted.get(user_id, False) or bool(usernames.get(user_id)))
|
||
|
||
|
||
|
||
def _parse_oc_dt(value):
|
||
"""Parse an Open Collective ISO timestamp ('…Z') into an aware datetime."""
|
||
if not value:
|
||
return None
|
||
try:
|
||
dt = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
||
return dt if dt.tzinfo else dt.replace(tzinfo=timezone.utc)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _clamp_anchor_day(anchor_day: int) -> int:
|
||
"""Clamp the anchor day to 28 so short months never skip a renewal."""
|
||
try:
|
||
d = int(anchor_day)
|
||
except Exception:
|
||
return 1
|
||
return min(max(d, 1), 28)
|
||
|
||
|
||
def next_reset_for_anchor_day(anchor_day: int, now: datetime | None = None) -> datetime:
|
||
"""The member's next allowance reset: the next occurrence of their join-day.
|
||
|
||
The day-of-month is the member's JOIN day (not their payment date), so an OC
|
||
charge landing early cannot move the boundary. Always strictly after `now`.
|
||
"""
|
||
now = now or datetime.now(timezone.utc)
|
||
day = _clamp_anchor_day(anchor_day)
|
||
y, m = now.year, now.month
|
||
cand = datetime(y, m, day, 0, 0, tzinfo=timezone.utc)
|
||
if cand <= now:
|
||
y, m = (y + 1, 1) if m == 12 else (y, m + 1)
|
||
cand = datetime(y, m, day, 0, 0, tzinfo=timezone.utc)
|
||
return cand
|
||
|
||
|
||
def cycle_start_for_anchor_day(anchor_day: int, now: datetime | None = None) -> datetime:
|
||
"""The most recent occurrence of the member's join-day at/before `now`."""
|
||
now = now or datetime.now(timezone.utc)
|
||
day = _clamp_anchor_day(anchor_day)
|
||
y, m = now.year, now.month
|
||
cand = datetime(y, m, day, 0, 0, tzinfo=timezone.utc)
|
||
if cand > now:
|
||
y, m = (y - 1, 12) if m == 1 else (y, m - 1)
|
||
cand = datetime(y, m, day, 0, 0, tzinfo=timezone.utc)
|
||
return cand
|
||
|
||
|
||
|
||
async def fetch_member_orders() -> dict[str, list[dict]]:
|
||
"""Every OC order for the collective, grouped by the payer's slug.
|
||
|
||
One query for the whole collective (orders(limit: 200)) rather than a call
|
||
per member. Raises on failure so the caller can skip rather than mis-decide.
|
||
"""
|
||
if not OC_PERSONAL_TOKEN:
|
||
raise RuntimeError("OC_PERSONAL_TOKEN not set; cannot fetch orders")
|
||
query = (
|
||
'{ collective(slug: "%s") { orders(limit: 200) { totalCount nodes { '
|
||
"frequency status createdAt lastChargedAt nextChargeDate "
|
||
"fromAccount { slug } totalAmount { value currency } } } } }"
|
||
% OC_COLLECTIVE_SLUG
|
||
)
|
||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||
r = await client.post(
|
||
OC_GRAPHQL_URL,
|
||
headers={"Content-Type": "application/json",
|
||
"Authorization": f"Bearer {OC_PERSONAL_TOKEN}",
|
||
"Api-Version": "2022-01-01",
|
||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"},
|
||
json={"query": query},
|
||
)
|
||
if r.status_code != 200:
|
||
raise RuntimeError(f"OC order fetch failed: HTTP {r.status_code}")
|
||
body = r.json()
|
||
if body.get("errors"):
|
||
raise RuntimeError(f"OC order fetch errors: {str(body['errors'])[:200]}")
|
||
orders = (((body.get("data") or {}).get("collective") or {}).get("orders") or {})
|
||
by_slug: dict[str, list[dict]] = {}
|
||
for o in orders.get("nodes") or []:
|
||
slug = ((o.get("fromAccount") or {}).get("slug") or "").strip()
|
||
if slug:
|
||
by_slug.setdefault(slug, []).append(o)
|
||
return by_slug
|
||
|
||
|
||
def _payment_verdict(orders: list[dict], cycle_start: datetime,
|
||
now: datetime | None = None) -> tuple[bool, str]:
|
||
"""Did this member's recurring contribution pay for the current cycle?
|
||
|
||
Frequency-aware. Only a RECURRING contribution (MONTHLY or YEARLY) can cover
|
||
an allowance cycle; a ONE-TIME payment is credit, not an allowance period, so
|
||
it is never treated as covering a month and never masks an unpaid member.
|
||
Returns (paid, evidence).
|
||
|
||
Rules, in order:
|
||
* PROCESSING — a payment is in flight; treated as PAID (a payment mid-flight
|
||
is not a failure), so a PROCESSING order never triggers a reminder.
|
||
* ACTIVE MONTHLY — a live subscription; treated as paid even if its next
|
||
charge is still ahead, so a member whose charge falls due tomorrow does
|
||
not read as unpaid on every sweep.
|
||
* ACTIVE YEARLY — paid only while its coverage window contains the current
|
||
cycle: a year from its last charge, or until its nextChargeDate. A yearly
|
||
contributor paid once and is current for twelve cycles; a flat monthly
|
||
test would read them as unpaid (and, with deactivation on, deactivate
|
||
them) for eleven of those months.
|
||
* otherwise — judge on the most recent RECURRING charge: within the cycle
|
||
(7-day slack) is paid, before it is the remind path.
|
||
|
||
DORMANT ON TODAY'S DATA: as of 2026-11-04 the collective has no yearly orders
|
||
(104 orders: 88 MONTHLY, 16 ONETIME, 0 YEARLY — read from the collective-level
|
||
order list, which is complete for our collective). The yearly branch is
|
||
defensive robustness for a tier we may add later; it is not a fix for any
|
||
current member. Attribution caution: one member's `account.orders` spans EVERY
|
||
collective they back, so an order's OWNING collective (`toAccount`) must be
|
||
checked before it is credited to us — a foreign YEARLY order (social.coop) was
|
||
once misread as a yearly payer here. `now` is injectable for tests.
|
||
"""
|
||
now = now or datetime.now(timezone.utc)
|
||
|
||
def _freq(o: dict) -> str:
|
||
return (o.get("frequency") or "").upper()
|
||
|
||
def _status(o: dict) -> str:
|
||
return (o.get("status") or "").upper()
|
||
|
||
recurring = [o for o in orders if _freq(o) in ("MONTHLY", "YEARLY")]
|
||
if not recurring:
|
||
return False, "no recurring (monthly) contribution on Open Collective"
|
||
|
||
live = [o for o in recurring if _status(o) in ("ACTIVE", "PROCESSING")]
|
||
# An ACTIVE monthly subscription, or any PROCESSING order, is a live
|
||
# contribution that covers the cycle (evidence string kept stable so a
|
||
# report is comparable run to run).
|
||
live_monthly = [o for o in live if _freq(o) == "MONTHLY" or _status(o) == "PROCESSING"]
|
||
if live_monthly:
|
||
charges = [c for c in (_parse_oc_dt(o.get("lastChargedAt")) for o in live_monthly) if c]
|
||
last = max(charges).date().isoformat() if charges else "unknown"
|
||
return True, f"live recurring contribution (last charged {last})"
|
||
# An ACTIVE yearly subscription covers the cycle only while its coverage
|
||
# window (last charge + 1 year, or up to nextChargeDate) still includes now.
|
||
for o in live:
|
||
if _freq(o) != "YEARLY":
|
||
continue
|
||
lc = _parse_oc_dt(o.get("lastChargedAt"))
|
||
nc = _parse_oc_dt(o.get("nextChargeDate"))
|
||
# Coverage runs to the later of (next charge date) and (last charge + 1yr).
|
||
covered_until = nc
|
||
if lc:
|
||
year_end = lc + timedelta(days=365)
|
||
if covered_until is None or year_end > covered_until:
|
||
covered_until = year_end
|
||
if covered_until and now < covered_until:
|
||
return True, (f"live yearly contribution covers this cycle "
|
||
f"(last charged {lc.date().isoformat() if lc else 'unknown'}, "
|
||
f"covers until {covered_until.date().isoformat()})")
|
||
charged = [(_parse_oc_dt(o.get("lastChargedAt")), _status(o)) for o in recurring]
|
||
charged = [(c, s) for c, s in charged if c]
|
||
if charged:
|
||
last_dt, _ = max(charged, key=lambda t: t[0])
|
||
if last_dt >= cycle_start - timedelta(days=7):
|
||
return True, f"charged {last_dt.date().isoformat()} within the current cycle"
|
||
return False, f"last charge {last_dt.date().isoformat()} is before this cycle"
|
||
statuses = ",".join(sorted({_status(o) or "?" for o in recurring}))
|
||
return False, f"no live recurring contribution (order status: {statuses})"
|
||
|
||
|
||
def payment_reminder_email(name: str, contribute_url: str) -> tuple[str, str, str]:
|
||
"""Build the one-per-cycle 'we haven't seen your contribution' reminder.
|
||
|
||
Same shell as welcome_email()/budget_notice_email(): cream background, white
|
||
card, forest-green button, inline pine-tree mark, table-based, no external
|
||
assets. Plain and direct — it names the action, not the plumbing.
|
||
"""
|
||
subject = "Your Inference Cooperative membership needs attention"
|
||
text = (
|
||
f"Hi {name},\n\n"
|
||
"Our records at Open Collective don't show a current monthly contribution "
|
||
"for your account, so your monthly allowance won't renew.\n\n"
|
||
"To keep your access, restart or update your contribution here:\n\n"
|
||
f"{contribute_url}\n\n"
|
||
"If your payment has already gone through, ignore this — your allowance "
|
||
"will renew on its own. If we don't see a contribution after a short grace "
|
||
"period, your account stays in place but loses access until your "
|
||
"contribution resumes.\n\n"
|
||
"Questions? Reply to this email or write to info@inference.coop.\n\n"
|
||
"— The Inference Cooperative\n"
|
||
)
|
||
html = f"""<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"></head>
|
||
<body style="margin:0;padding:0;background:#faf8f5;font-family:-apple-system,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;">
|
||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:#faf8f5;padding:40px 16px;">
|
||
<tr>
|
||
<td align="center">
|
||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="max-width:520px;">
|
||
<tr>
|
||
<td align="center" style="padding-bottom:24px;">
|
||
{PINE_LOGO_SVG}
|
||
</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="center" style="background:#ffffff;border:1px solid #e8e2d8;border-radius:16px;padding:40px 32px;">
|
||
<h1 style="margin:0 0 12px;font-size:24px;font-weight:700;color:#2d3327;">Your membership needs attention</h1>
|
||
<p style="margin:0 0 8px;font-size:16px;color:#6b7a62;line-height:1.5;">Our records at Open Collective don't show a current monthly contribution for your account, so your monthly allowance won't renew.</p>
|
||
<p style="margin:0 0 28px;font-size:15px;color:#6b7a62;line-height:1.5;">To keep your access, restart or update your contribution. If your payment has already gone through, ignore this.</p>
|
||
<a href="{contribute_url}" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:999px;font-size:16px;font-weight:600;">Update your contribution</a>
|
||
<p style="margin:28px 0 0;font-size:13px;color:#94a08c;line-height:1.5;">If we don't see a contribution after a short grace period, your account stays in place but loses access until your contribution resumes.</p>
|
||
</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="center" style="padding-top:24px;font-size:13px;color:#94a08c;line-height:1.6;">
|
||
Questions? Reply to this email or write to <a href="mailto:info@inference.coop" style="color:#6b7a62;">info@inference.coop</a>.<br>
|
||
<span style="color:#2d3327;">The Inference Cooperative</span> · inference.coop
|
||
</td>
|
||
</tr>
|
||
</table>
|
||
</td>
|
||
</tr>
|
||
</table>
|
||
</body>
|
||
</html>"""
|
||
return subject, text, html
|
||
|
||
|
||
async def fetch_account_by_slug(slug: str) -> dict | None:
|
||
"""The Open Collective account behind a payer slug, with its email if OC gives it.
|
||
|
||
Orders report their payer as `fromAccount { slug }`. For a FIRST-TIME
|
||
contributor that account is often not in the collective's members list yet
|
||
(OC reports the payer as `guest-…`/`incognito-…`), which is exactly the case
|
||
this exists for: it is the only way to learn who paid, and therefore the only
|
||
way to provision them.
|
||
|
||
OC exposes `email` on the Individual fragment — visible for an ordinary guest
|
||
checkout, null for an incognito profile (the contributor hid it on purpose).
|
||
Returns None when the account cannot be read; NEVER raises, because its
|
||
caller must be able to say "could not resolve" rather than die mid-webhook.
|
||
"""
|
||
slug = (slug or "").strip()
|
||
if not slug or not OC_PERSONAL_TOKEN:
|
||
return None
|
||
query = ("query($slug: String!) { account(slug: $slug) { __typename id name slug "
|
||
"... on Individual { email isIncognito } } }")
|
||
try:
|
||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||
# _oc_graphql_post returns the RESPONSE (it is a retrying wrapper
|
||
# around client.post), so the JSON is read here. Getting this wrong
|
||
# is invisible to a stubbed test and fatal in the container: the
|
||
# first live run raised 'Response object has no attribute get'.
|
||
resp = await _oc_graphql_post(
|
||
client,
|
||
{"query": query, "variables": {"slug": slug}},
|
||
# WITHOUT the personal token OC answers the unauthenticated view,
|
||
# which hides every email — the lookup then returns name='Conor W'
|
||
# and email=None while a hand query with the header returns
|
||
# conor.canaday@gmail.com. The header is the whole point.
|
||
headers={"Content-Type": "application/json",
|
||
"Personal-Token": OC_PERSONAL_TOKEN},
|
||
)
|
||
data = resp.json()
|
||
except Exception as e: # read failure: report, never raise
|
||
logger.warning("OC account lookup failed for slug %s: %s", slug, e)
|
||
return None
|
||
acct = ((data or {}).get("data") or {}).get("account") or None
|
||
if not acct:
|
||
return None
|
||
if acct.get("isIncognito"):
|
||
# Hidden on purpose: there is no email to provision from, and inventing
|
||
# one from the slug would be worse than saying so.
|
||
return {**acct, "email": None}
|
||
acct["email"] = (acct.get("email") or "").strip().lower() or None
|
||
return acct
|
||
|
||
|
||
async def fetch_member_index() -> dict[str, dict]:
|
||
"""email -> {slug, joined, role, tier} from the live OC member list.
|
||
|
||
Paginated (the members connection caps at 100/page and hides the rest).
|
||
The slug here is the member's CURRENT Open Collective slug — the members
|
||
table can hold a stale `guest-…` slug from first provisioning, so orders
|
||
must be matched on the slug resolved from the member's EMAIL, never on the
|
||
stored column (matching on the stored slug falsely read 18 current payers
|
||
as unpaid in the first report run).
|
||
"""
|
||
if not OC_PERSONAL_TOKEN:
|
||
raise RuntimeError("OC_PERSONAL_TOKEN not set; cannot fetch the member index")
|
||
out: dict[str, dict] = {}
|
||
offset = 0
|
||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||
for _ in range(20):
|
||
query = (
|
||
'{ collective(slug: "%s") { members(limit: 100, offset: %d) { totalCount '
|
||
"nodes { role createdAt tier { name } account { name slug type ... on Individual { email } } } } } }"
|
||
% (OC_COLLECTIVE_SLUG, offset)
|
||
)
|
||
r = await client.post(
|
||
OC_GRAPHQL_URL,
|
||
headers={"Content-Type": "application/json",
|
||
"Personal-Token": OC_PERSONAL_TOKEN,
|
||
"Api-Version": "2022-01-01",
|
||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"},
|
||
json={"query": query},
|
||
)
|
||
if r.status_code != 200:
|
||
raise RuntimeError(f"OC member index fetch failed: HTTP {r.status_code}")
|
||
members = (((r.json().get("data") or {}).get("collective") or {}).get("members") or {})
|
||
page = members.get("nodes") or []
|
||
for n in page:
|
||
a = n.get("account") or {}
|
||
e = (a.get("email") or "").strip().lower()
|
||
if e:
|
||
out[e] = {
|
||
"slug": a.get("slug") or "",
|
||
"joined": n.get("createdAt") or "",
|
||
"role": n.get("role") or "",
|
||
"tier": ((n.get("tier") or {}).get("name") or ""),
|
||
}
|
||
offset += len(page)
|
||
if not page or offset >= (members.get("totalCount") or 0):
|
||
break
|
||
return out
|
||
|
||
|
||
async def anchor_day_for_member(email: str, slug: str = "") -> int:
|
||
"""The member's allowance anchor day (the day of month they joined).
|
||
|
||
Stored on the members row; derived from the OC join date (resolved by email)
|
||
on first use and persisted so later decisions are stable. Clamped to 28.
|
||
"""
|
||
conn = get_db()
|
||
row = conn.execute("SELECT anchor_day FROM members WHERE email = ?", (email,)).fetchone()
|
||
conn.close()
|
||
if row and row[0]:
|
||
return _clamp_anchor_day(row[0])
|
||
try:
|
||
rec = (await fetch_member_index()).get(email.strip().lower())
|
||
if rec:
|
||
dt = _parse_oc_dt(rec.get("joined"))
|
||
if dt:
|
||
d = _clamp_anchor_day(dt.day)
|
||
conn = get_db()
|
||
conn.execute("UPDATE members SET anchor_day = ? WHERE email = ?", (d, email))
|
||
conn.commit()
|
||
conn.close()
|
||
return d
|
||
except Exception as e:
|
||
logger.warning("anchor_day_for_member: OC join-date lookup failed for %s: %s", email, e)
|
||
if slug:
|
||
# An oc-incognito member is absent from the member index (it drops anyone
|
||
# without an email), but their orders carry the slug we store. Their
|
||
# billing rhythm still anchors the cycle — falling back to "today" would
|
||
# silently move their renewal day.
|
||
try:
|
||
orders = (await fetch_member_orders()).get(slug) or []
|
||
joined = min((_parse_oc_dt(str(o.get("createdAt"))) for o in orders
|
||
if _parse_oc_dt(str(o.get("createdAt")))), default=None)
|
||
if joined:
|
||
d = _clamp_anchor_day(joined.day)
|
||
conn = get_db()
|
||
conn.execute("UPDATE members SET anchor_day = ? WHERE email = ?", (d, email))
|
||
conn.commit()
|
||
conn.close()
|
||
logger.info("anchor_day_for_member: %s anchored to %d from their OC order", email, d)
|
||
return d
|
||
except Exception as e:
|
||
logger.warning("anchor_day_for_member: order lookup failed for %s/%s: %s", email, slug, e)
|
||
return _clamp_anchor_day(datetime.now(timezone.utc).day)
|
||
|
||
|
||
def allowance_lapse_decision(reminder_sent_at, now, grace_days, acting):
|
||
"""What to do with a member who was reminded and is still unpaid (pure).
|
||
|
||
No I/O — `now`, the reminder time and the grace length are all arguments, so
|
||
the lapse logic can be unit-tested against a SYNTHETIC reminder time without
|
||
touching any member's rows (see tests/test_allowance_lapse.py). Returns
|
||
`(decision, deadline)`:
|
||
|
||
* ("in_grace", None) — the grace window has not elapsed yet.
|
||
* ("would_deactivate", deadline) — elapsed, but this caller may not act
|
||
(a report run, or ALLOWANCE_DEACTIVATE still off). NOTHING changes.
|
||
* ("deactivated", deadline) — elapsed AND the caller is authorised.
|
||
|
||
`acting` must be True only when BOTH gates hold — a verified send run
|
||
(ALLOWANCE_POLICY=on) AND ALLOWANCE_DEACTIVATE=on — so enabling the preview
|
||
never by itself removes anyone's access. `deadline` is the moment the grace
|
||
window elapses (reminder sent + grace_days), carried into the decision log.
|
||
"""
|
||
if reminder_sent_at is None:
|
||
return "in_grace", None
|
||
deadline = reminder_sent_at + timedelta(days=grace_days)
|
||
if now < deadline:
|
||
return "in_grace", None
|
||
return ("deactivated" if acting else "would_deactivate"), deadline
|
||
|
||
|
||
async def reconcile_allowances(send: bool = False) -> dict:
|
||
"""Decide every active member's allowance renewal from OC payment evidence.
|
||
|
||
`send=False` is a REPORT: it computes and records every decision (would
|
||
renew / would remind / would deactivate) and changes NOTHING — no resets,
|
||
no emails, no group changes. `send=True` performs renewals and sends the
|
||
single reminder. Deactivation still additionally requires
|
||
ALLOWANCE_DEACTIVATE=on.
|
||
"""
|
||
try:
|
||
by_slug = await fetch_member_orders()
|
||
except Exception as e:
|
||
logger.warning("Allowance reconcile: order fetch failed; skipping (fail-safe): %s", e)
|
||
return {"status": "skipped", "reason": "order_fetch_failed", "detail": str(e)[:200]}
|
||
try:
|
||
index = await fetch_member_index()
|
||
except Exception as e:
|
||
logger.warning("Allowance reconcile: member index fetch failed; skipping (fail-safe): %s", e)
|
||
return {"status": "skipped", "reason": "member_index_failed", "detail": str(e)[:200]}
|
||
|
||
conn = get_db()
|
||
_ensure_allowance_tables(conn)
|
||
cols = {r[1] for r in conn.execute("PRAGMA table_info(members)")}
|
||
extra = ", welcome_sent_at" if "welcome_sent_at" in cols else ""
|
||
rows = conn.execute(
|
||
f"SELECT email, slug, anchor_day, org_slug, account_type, oc_email{extra} "
|
||
"FROM members WHERE active = 1"
|
||
).fetchall()
|
||
|
||
now = datetime.now(timezone.utc)
|
||
contribute = credit_pack_url()
|
||
verified = "on" if (send and ALLOWANCE_POLICY == "on") else "report"
|
||
_APPLIED = {"renew": "renewed", "remind": "reminded", "deactivate": "deactivated"}
|
||
_WOULD = {"renew": "would_renew", "remind": "would_remind", "deactivate": "would_deactivate"}
|
||
label = (lambda a: _APPLIED[a]) if verified == "on" else (lambda a: _WOULD[a])
|
||
decisions: list[dict] = []
|
||
counts = {"renewed": 0, "reminded": 0, "deactivated": 0,
|
||
"would_renew": 0, "would_remind": 0, "would_deactivate": 0, "in_grace": 0,
|
||
"skipped": 0}
|
||
|
||
for row in rows:
|
||
email = row[0]
|
||
slug = row[1] or ""
|
||
anchor_day = row[2]
|
||
org_slug = row[3]
|
||
acct_type = row[4] or "member"
|
||
oc_email = (row[5] or "").strip().lower()
|
||
welcome_sent = row[6] if len(row) > 6 else None
|
||
|
||
if acct_type == "guest":
|
||
# Guest accounts have no OC membership by design — the $2 credit
|
||
# balance and the GUEST_DAYS time box govern them instead. No
|
||
# renewal decision, no reminder, ever.
|
||
counts["skipped"] += 1
|
||
continue
|
||
if member_source(email, conn) == "manual":
|
||
# Hand-provisioned: no OC contribution is expected, so there is
|
||
# nothing to reconcile. An 'oc-incognito' member is NOT skipped —
|
||
# their contribution is matched by the slug in row[1] below, which is
|
||
# the only key OC gives us for them.
|
||
counts["skipped"] += 1
|
||
continue
|
||
if org_slug:
|
||
counts["skipped"] += 1 # dues paid by an OC Organization
|
||
continue
|
||
rec = index.get(email.strip().lower())
|
||
if rec is None and oc_email:
|
||
# Contribution under a different OC email (guest-convert fallback).
|
||
rec = index.get(oc_email)
|
||
if rec and rec.get("role") == "ADMIN":
|
||
counts["skipped"] += 1 # the co-op's own account
|
||
continue
|
||
oc_slug = (rec or {}).get("slug") or ""
|
||
if not oc_slug and member_source(email, conn) == "oc-incognito":
|
||
# OC's member index DROPS anyone whose profile hides its email, so it
|
||
# can never supply the slug for these members — and without one the
|
||
# reconciler cannot see their orders at all and skips them as
|
||
# "not in the live Open Collective member list". Their own row holds
|
||
# the slug, and it is the same key fetch_member_orders() groups by,
|
||
# which is why matching works once we use it. (mmoya@mmoya.org,
|
||
# 2026-11-13: source oc-incognito, slug incognito-5ad60182, a live
|
||
# monthly contribution the member index could not see.)
|
||
oc_slug = slug
|
||
if oc_slug:
|
||
logger.info("Allowance reconcile: %s is oc-incognito; using their stored slug %s",
|
||
email, oc_slug)
|
||
if not oc_slug:
|
||
decisions.append({"email": email, "slug": slug, "anchor_day": anchor_day,
|
||
"cycle": "", "decision": "skipped",
|
||
"evidence": "not in the live Open Collective member list — cannot verify payment"})
|
||
counts["skipped"] += 1
|
||
continue
|
||
|
||
# Keep the stored slug in step with Open Collective (authoritative).
|
||
if send and oc_slug != slug:
|
||
conn.execute("UPDATE members SET slug = ? WHERE email = ?", (oc_slug, email))
|
||
conn.commit()
|
||
|
||
if not anchor_day:
|
||
dt = _parse_oc_dt(rec.get("joined")) or _parse_oc_dt(welcome_sent) or now
|
||
anchor_day = _clamp_anchor_day(dt.day)
|
||
if send:
|
||
conn.execute("UPDATE members SET anchor_day = ? WHERE email = ?", (anchor_day, email))
|
||
conn.commit()
|
||
anchor_day = _clamp_anchor_day(anchor_day)
|
||
cycle_start = cycle_start_for_anchor_day(anchor_day, now)
|
||
cycle_key = cycle_start.date().isoformat()
|
||
|
||
orders = by_slug.get(oc_slug, [])
|
||
paid, evidence = _payment_verdict(orders, cycle_start)
|
||
|
||
if paid:
|
||
next_reset = next_reset_for_anchor_day(anchor_day, now)
|
||
evidence = f"{evidence}; reset anchored to join-day {anchor_day} → next reset {next_reset.date().isoformat()}"
|
||
decision = label("renew")
|
||
counts[decision] += 1
|
||
if verified == "on":
|
||
try:
|
||
team_id = await litellm_team_id_for_email(email)
|
||
if team_id:
|
||
await litellm_pin_reset_date(team_id, next_reset)
|
||
except Exception as e:
|
||
logger.warning("Allowance renew failed for %s: %s", email, e)
|
||
else:
|
||
reminded = conn.execute(
|
||
"SELECT sent_at FROM payment_reminders WHERE email = ? AND cycle = ?",
|
||
(email, cycle_key),
|
||
).fetchone()
|
||
if not reminded:
|
||
decision = label("remind")
|
||
counts[decision] += 1
|
||
if verified == "on":
|
||
subject, text_body, html_body = payment_reminder_email(oc_slug or slug or "there", contribute)
|
||
if send_email(email, subject, text_body, html_body):
|
||
conn.execute(
|
||
"INSERT OR REPLACE INTO payment_reminders (email, cycle, sent_at)"
|
||
" VALUES (?, ?, datetime('now'))",
|
||
(email, cycle_key),
|
||
)
|
||
conn.commit()
|
||
else:
|
||
decision = "remind_failed"
|
||
else:
|
||
sent_at = _parse_oc_dt(str(reminded[0]))
|
||
# Lapse decision is a pure, testable function: past the grace
|
||
# window it reports `would_deactivate` in EVERY mode; the real
|
||
# action needs `acting` — a verified send run AND the separate
|
||
# ALLOWANCE_DEACTIVATE gate. So a report run (and a send run with
|
||
# deactivate still off) PREVIEWS the deactivation and changes
|
||
# nobody's access.
|
||
acting = verified == "on" and ALLOWANCE_DEACTIVATE == "on"
|
||
lapse, deadline = allowance_lapse_decision(
|
||
sent_at, now, PAYMENT_GRACE_DAYS, acting)
|
||
if lapse == "in_grace":
|
||
decision = "in_grace"
|
||
counts["in_grace"] += 1
|
||
evidence = (f"{evidence}; reminder sent "
|
||
f"{str(reminded[0])[:10] if reminded[0] else '?'} "
|
||
f"(grace {int(PAYMENT_GRACE_DAYS)}d)")
|
||
else:
|
||
decision = lapse # would_deactivate | deactivated
|
||
counts[decision] += 1
|
||
dl = deadline.date().isoformat() if deadline else "?"
|
||
evidence = (f"{evidence}; reminder sent {str(reminded[0])[:10]}, "
|
||
f"grace {int(PAYMENT_GRACE_DAYS)}d elapsed, "
|
||
f"deadline {dl}")
|
||
if decision == "deactivated":
|
||
urow = conn.execute(
|
||
"SELECT cloudron_user_id FROM members WHERE email = ?", (email,)
|
||
).fetchone()
|
||
if INACTIVE_GROUP_ID and urow and urow[0]:
|
||
await cloudron_set_group(urow[0], INACTIVE_GROUP_ID)
|
||
deactivate_member(email)
|
||
await revoke_member_litellm_access(email)
|
||
await listmonk_sync(email, subscribe=False)
|
||
|
||
# L4 (audit 2026-11-05): one row per (member, cycle, decision) — the
|
||
# same decision re-derived on a later run UPDATES that row's evidence
|
||
# and timestamp instead of appending a duplicate. The sweep runs 3x/day
|
||
# and report-mode runs also write here; without this the table grew
|
||
# ~500 rows/day (measured 2026-11-05: 1741 -> 2257). A decision CHANGE
|
||
# within a cycle still gets its own row, so the "latest decision per
|
||
# member" readers are unaffected and no history is rewritten.
|
||
cur = conn.execute(
|
||
"UPDATE allowance_log SET evidence = ?, anchor_day = ?, "
|
||
"decided_at = datetime('now') "
|
||
"WHERE id = (SELECT id FROM allowance_log WHERE email = ? AND cycle = ? "
|
||
"AND decision = ? ORDER BY id DESC LIMIT 1)",
|
||
(evidence, anchor_day, email, cycle_key, decision),
|
||
)
|
||
if cur.rowcount == 0:
|
||
conn.execute(
|
||
"INSERT INTO allowance_log (email, cycle, decision, evidence, anchor_day)"
|
||
" VALUES (?, ?, ?, ?, ?)",
|
||
(email, cycle_key, decision, evidence, anchor_day),
|
||
)
|
||
conn.commit()
|
||
decisions.append({"email": email, "slug": oc_slug, "anchor_day": anchor_day,
|
||
"cycle": cycle_key, "decision": decision, "evidence": evidence})
|
||
|
||
conn.close()
|
||
return {
|
||
"mode": ALLOWANCE_POLICY,
|
||
"verified": verified,
|
||
"deactivation_enabled": ALLOWANCE_DEACTIVATE == "on",
|
||
"considered": len(rows),
|
||
"counts": counts,
|
||
"decisions": decisions,
|
||
}
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Allowance shortfall — OC's billing-day alignment truncates a paid cycle. #
|
||
# --------------------------------------------------------------------------- #
|
||
# Open Collective aligns every recurring charge onto the collective's COMMON
|
||
# billing day, so a member who joins mid-month is charged again well before a
|
||
# full month has elapsed: they pay for a full month and receive less than one.
|
||
# Operator decision (2026-11-04): make the member whole by crediting the missing
|
||
# ALLOWANCE — the flat MEMBER_BUDGET pro-rated for the days lost — as
|
||
# non-expiring credits (the same balance that extends the allowance and is spent
|
||
# only after it). It is NOT a share of what they paid, and it is NOT rounded.
|
||
#
|
||
# Scope: EVERY monthly paid cycle shorter than a full month, for any member and
|
||
# any cycle (not only the first — OC can re-align an existing charge date too).
|
||
# Derived FROM DATA (each recurring order's real charge history); members are
|
||
# never asked to claim it. Idempotent: each credit is keyed to the specific OC
|
||
# charge that caused it, so a re-run credits nothing twice. A charge whose date
|
||
# cannot be determined is SKIPPED and logged — paying a credit that is not owed
|
||
# is worse than paying one late.
|
||
#
|
||
# Member notices go out automatically (the sweep does both in one pass), but they
|
||
# default OFF (ALLOWANCE_SHORTFALL_EMAILS=off) so the wording can be reviewed
|
||
# before any member is written to about billing. Setting it to `on` sends the
|
||
# staged notice on the next sweep; nothing is re-sent per charge.
|
||
FULL_PAID_MONTH_DAYS = 30 # the allowance is a flat monthly amount => "a month" = 30 days
|
||
ALLOWANCE_SHORTFALL_EMAILS = (os.environ.get("ALLOWANCE_SHORTFALL_EMAILS", "off") or "off").strip().lower()
|
||
|
||
|
||
def shortfall_reason(prev: str, cur: str) -> str:
|
||
"""Human-readable ledger reason — a member's question is answerable from it."""
|
||
return f"allowance shortfall, truncated paid cycle {prev}→{cur} (OC billing alignment)"
|
||
|
||
|
||
# Charges already made whole OUTSIDE this mechanism, keyed by the OC charge
|
||
# reference alone (the same idempotence key the shortfall path uses). The one
|
||
# listed charge received a manual allowance reset on 2026-11-04 for the cycle
|
||
# OC truncated; it is excluded so a re-run never credits it again. Later
|
||
# charges form new cycles and are credited normally. The member's identity and
|
||
# the full timeline behind this entry are recorded in the operations docs and
|
||
# the ledger — deliberately NOT in this public repository (audit H2,
|
||
# 2026-11-05: the exclusion used to embed a member's email and billing dates).
|
||
SHORTFALL_REMEDIATED: set[str] = {
|
||
"tx_Kr6n3rW577OSjr2bIt5IC",
|
||
}
|
||
|
||
|
||
def _usd(x: float) -> str:
|
||
"""Format a credit amount for a member without inventing cents (2 -> $2)."""
|
||
return f"${x:g}"
|
||
|
||
|
||
def _ensure_shortfall_tables(conn) -> None:
|
||
"""Records which charge-triggered notices have actually been sent.
|
||
|
||
Kept apart from the credit ledger on purpose: the ledger answers "what did we
|
||
credit, and why", this answers "what left the building". A notice is sent at
|
||
most once per charge reference.
|
||
"""
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS shortfall_notices ("
|
||
" reference TEXT PRIMARY KEY, email TEXT NOT NULL, amount REAL NOT NULL,"
|
||
" sent_at TEXT NOT NULL DEFAULT (datetime('now')))"
|
||
)
|
||
conn.commit()
|
||
|
||
|
||
def credit_reference_exists(reference: str) -> bool:
|
||
"""Has a credit already been recorded against this OC charge reference?"""
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT 1 FROM credit_ledger WHERE reference = ? LIMIT 1", (reference,)
|
||
).fetchone()
|
||
conn.close()
|
||
return bool(row)
|
||
|
||
|
||
def grant_held_credits(conn: sqlite3.Connection, email: str, slug: str = "") -> float:
|
||
"""Apply every HELD credit pack matched to this member, atomically (M5).
|
||
|
||
Per hold, in ONE transaction: flip the hold `held` → `granted`, move the
|
||
member's balance, and write the ledger row — mirroring admin_held_grant.
|
||
Committing the three together is the point: an exception can no longer
|
||
leave hold=granted with the member uncredited. The ledger reference is
|
||
`held:<hold_id>` (the same shape admin_held_grant uses), so re-running a
|
||
hold that was already applied is a no-op and the resolve-then-credit can
|
||
never double-apply. Returns the total amount granted.
|
||
"""
|
||
email = (email or "").strip().lower()
|
||
rows = conn.execute(
|
||
"SELECT id, amount FROM held_credits "
|
||
"WHERE status = 'held' AND (LOWER(email) = ? OR LOWER(slug) = ?)",
|
||
(email.lower(), slug.lower() if slug else ""),
|
||
).fetchall()
|
||
granted = 0.0
|
||
for r in rows:
|
||
hold_id, amount = r[0], float(r[1] or 0.0)
|
||
reference = f"held:{hold_id}"
|
||
if credit_reference_exists(reference):
|
||
# Already applied by an earlier run (e.g. an admin grant on this
|
||
# very hold): reconcile the status, move no money.
|
||
conn.execute(
|
||
"UPDATE held_credits SET status = 'granted', resolved_at = "
|
||
"COALESCE(resolved_at, datetime('now')) WHERE id = ? AND status = 'held'",
|
||
(hold_id,),
|
||
)
|
||
conn.commit()
|
||
continue
|
||
conn.execute(
|
||
"UPDATE held_credits SET status = 'granted', resolved_at = datetime('now') "
|
||
"WHERE id = ? AND status = 'held'",
|
||
(hold_id,),
|
||
)
|
||
conn.execute(
|
||
"UPDATE members SET credit_balance = "
|
||
"MAX(0, COALESCE(credit_balance, 0) + ?) WHERE LOWER(email) = ?",
|
||
(amount, email),
|
||
)
|
||
conn.execute(
|
||
"INSERT INTO credit_ledger (email, delta, reason, reference) "
|
||
"VALUES (?, ?, ?, ?)",
|
||
(email, amount, "held credit pack(s) granted on membership", reference),
|
||
)
|
||
conn.commit()
|
||
granted += amount
|
||
return granted
|
||
|
||
|
||
def shortfall_notice_sent(reference: str) -> bool:
|
||
conn = get_db()
|
||
row = conn.execute(
|
||
"SELECT 1 FROM shortfall_notices WHERE reference = ? LIMIT 1", (reference,)
|
||
).fetchone()
|
||
conn.close()
|
||
return bool(row)
|
||
|
||
|
||
def mark_shortfall_notice_sent(reference: str, email: str, amount: float) -> None:
|
||
conn = get_db()
|
||
_ensure_shortfall_tables(conn)
|
||
conn.execute(
|
||
"INSERT OR REPLACE INTO shortfall_notices (reference, email, amount, sent_at)"
|
||
" VALUES (?, ?, ?, datetime('now'))",
|
||
(reference, email, amount),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
async def fetch_oc_monthly_order_charges() -> list[dict]:
|
||
"""Every MONTHLY order's real contribution charges — one record per order.
|
||
|
||
A record is {"slug", "order_id", "charges": [(datetime, publicId), ...]} from
|
||
ONE order. Truncation is judged per order: merging the charges of two
|
||
DIFFERENT subscriptions a member happens to hold invents a short gap that is
|
||
not a truncated cycle (one member started a second monthly order six days
|
||
after the first; that is not OC re-aligning a charge). Raises on failure so
|
||
the caller skips rather than mis-decides. Charges are the CONTRIBUTION
|
||
amounts INTO the collective (a non-refund CREDIT), one per charge.
|
||
"""
|
||
if not OC_PERSONAL_TOKEN:
|
||
raise RuntimeError("OC_PERSONAL_TOKEN not set; cannot fetch order charges")
|
||
query = (
|
||
'{ collective(slug: "%s") { orders(limit: 200) { totalCount nodes { '
|
||
"id frequency fromAccount { slug } "
|
||
"transactions { publicId createdAt type kind isRefund } } } } }"
|
||
% OC_COLLECTIVE_SLUG
|
||
)
|
||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||
r = await client.post(
|
||
OC_GRAPHQL_URL,
|
||
headers={"Content-Type": "application/json",
|
||
"Authorization": f"Bearer {OC_PERSONAL_TOKEN}",
|
||
"Api-Version": "2022-01-01",
|
||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"},
|
||
json={"query": query},
|
||
)
|
||
if r.status_code != 200:
|
||
raise RuntimeError(f"OC order-charge fetch failed: HTTP {r.status_code}")
|
||
body = r.json()
|
||
if body.get("errors"):
|
||
raise RuntimeError(f"OC order-charge fetch errors: {str(body['errors'])[:200]}")
|
||
nodes = (((body.get("data") or {}).get("collective") or {}).get("orders") or {}).get("nodes") or []
|
||
out: list[dict] = []
|
||
for o in nodes:
|
||
if (o.get("frequency") or "").upper() != "MONTHLY":
|
||
continue
|
||
slug = ((o.get("fromAccount") or {}).get("slug") or "").strip()
|
||
if not slug:
|
||
continue
|
||
charges = []
|
||
for t in o.get("transactions") or []:
|
||
if t.get("kind") != "CONTRIBUTION" or t.get("isRefund"):
|
||
continue
|
||
if (t.get("type") or "").upper() != "CREDIT":
|
||
continue
|
||
dt = _parse_oc_dt(t.get("createdAt"))
|
||
if dt and t.get("publicId"):
|
||
charges.append((dt, t["publicId"]))
|
||
charges.sort()
|
||
if charges:
|
||
out.append({"slug": slug, "order_id": o.get("id"), "charges": charges})
|
||
return out
|
||
|
||
|
||
def derive_shortfalls(order_records: list[dict], budget: float | None = None,
|
||
full_month_days: int = FULL_PAID_MONTH_DAYS) -> list[dict]:
|
||
"""Pure: truncated paid cycles from per-order monthly charge ladders.
|
||
|
||
For each consecutive pair of charges on the SAME order, a gap shorter than a
|
||
full month means the member paid for a full month and received fewer days of
|
||
allowance. Credit = budget * days_lost / full_month_days (no rounding). One
|
||
record per truncated cycle; `reference` is the early charge that truncated it
|
||
(globally unique, so it is the idempotence key).
|
||
"""
|
||
budget = MEMBER_BUDGET if budget is None else budget
|
||
out: list[dict] = []
|
||
for rec in order_records:
|
||
charges = rec.get("charges") or []
|
||
for i in range(1, len(charges)):
|
||
prev_dt, _ = charges[i - 1]
|
||
cur_dt, cur_id = charges[i]
|
||
gap = (cur_dt - prev_dt).days
|
||
if gap >= full_month_days:
|
||
continue
|
||
lost = full_month_days - gap
|
||
out.append({
|
||
"slug": rec.get("slug"), "order_id": rec.get("order_id"),
|
||
"prev": prev_dt.date().isoformat(), "cur": cur_dt.date().isoformat(),
|
||
"gap_days": gap, "days_lost": lost,
|
||
"amount": budget * lost / full_month_days, "reference": cur_id,
|
||
})
|
||
return out
|
||
|
||
|
||
def shortfall_email(name: str, amount: float, prev: str, cur: str,
|
||
gap_days: int, days_lost: int, pack_url: str) -> tuple[str, str, str]:
|
||
"""Build the shortfall credit notice (subject, text, html).
|
||
|
||
Same shell as welcome_email()/budget_notice_email(): cream background, white
|
||
card, forest-green button, inline pine-tree mark, table-based, no external
|
||
assets. Plain and direct — says what happened, why, the amount, and how to get
|
||
more. No filler, no apology theatre.
|
||
"""
|
||
subject = "A credit for a short allowance cycle"
|
||
text = (
|
||
f"Hi {name},\n\n"
|
||
"Your Open Collective contribution is charged on the collective's billing "
|
||
"day, not on the day you joined. That can make a charge arrive less than a "
|
||
"month after the one before it, so a month you paid for turns out shorter "
|
||
"than a month of allowance.\n\n"
|
||
f"That happened on your account: charges on {prev} and {cur} were only "
|
||
f"{gap_days} days apart, so you paid for a full month and received "
|
||
f"{days_lost} days less than one.\n\n"
|
||
f"To make that up, we've added {_usd(amount)} to your account as credits. "
|
||
"Credits don't expire, and they're used after your monthly allowance.\n\n"
|
||
"If you'd like to add more, you can buy credit packs here:\n\n"
|
||
f"{pack_url}\n\n"
|
||
"Questions? Reply to this email or write to info@inference.coop.\n\n"
|
||
"— The Inference Cooperative\n"
|
||
)
|
||
html = f"""<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"></head>
|
||
<body style="margin:0;padding:0;background:#faf8f5;font-family:-apple-system,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;">
|
||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:#faf8f5;padding:40px 16px;">
|
||
<tr>
|
||
<td align="center">
|
||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="max-width:520px;">
|
||
<tr>
|
||
<td align="center" style="padding-bottom:24px;">
|
||
{PINE_LOGO_SVG}
|
||
</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="center" style="background:#ffffff;border:1px solid #e8e2d8;border-radius:16px;padding:40px 32px;">
|
||
<h1 style="margin:0 0 12px;font-size:24px;font-weight:700;color:#2d3327;">A credit for a short allowance cycle</h1>
|
||
<p style="margin:0 0 8px;font-size:16px;color:#6b7a62;line-height:1.5;">Your Open Collective contribution is charged on the collective's billing day, not on the day you joined. That can make a charge arrive less than a month after the one before it, so a month you paid for turns out shorter than a month of allowance.</p>
|
||
<p style="margin:0 0 20px;font-size:15px;color:#6b7a62;line-height:1.5;">That happened on your account: charges on {prev} and {cur} were only {gap_days} days apart, so you paid for a full month and received {days_lost} days less than one.</p>
|
||
<p style="margin:0 0 24px;font-size:15px;color:#2d3327;line-height:1.5;"><strong>We've added {_usd(amount)} to your account as credits.</strong> Credits don't expire, and they're used after your monthly allowance.</p>
|
||
<a href="{pack_url}" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:999px;font-size:16px;font-weight:600;">Buy credit packs</a>
|
||
<p style="margin:28px 0 0;font-size:13px;color:#94a08c;line-height:1.5;">If you'd like to add more, you can buy credit packs at the button above.</p>
|
||
</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="center" style="padding-top:24px;font-size:13px;color:#94a08c;line-height:1.6;">
|
||
Questions? Reply to this email or write to <a href="mailto:info@inference.coop" style="color:#6b7a62;">info@inference.coop</a>.<br>
|
||
<span style="color:#2d3327;">The Inference Cooperative</span> · inference.coop
|
||
</td>
|
||
</tr>
|
||
</table>
|
||
</td>
|
||
</tr>
|
||
</table>
|
||
</body>
|
||
</html>"""
|
||
return subject, text, html
|
||
|
||
|
||
# --- Credit-pack reconciliation (issue #7) --------------------------------- #
|
||
# Credits are granted on exactly ONE path: the OC `order.processed` webhook. If
|
||
# that delivery fails, or never arrives, the co-op holds a member's money and
|
||
# nothing on a schedule notices. 2026-11-13 produced two of those in one day — a
|
||
# $50 pack whose delivery 500'd on httpx's 5-second default, and a $10 pack with
|
||
# no delivery in the retained log at all, found only by reading OC's own order
|
||
# list against our books. This pass makes that routine.
|
||
#
|
||
# CREDIT_RECONCILE: report (default) | on | off
|
||
# report — find and LOG unaccounted packs; change nothing
|
||
# on — grant them, with the SAME per-order reference the webhook writes, so
|
||
# the webhook and this pass are idempotent against each other in both
|
||
# directions (a late delivery is a no-op; a pass after a delivery is a
|
||
# no-op too)
|
||
# off — skip the pass entirely
|
||
CREDIT_RECONCILE = (os.environ.get("CREDIT_RECONCILE", "report") or "report").strip().lower()
|
||
# How far back to look — DAYS off OC's own data, not "whatever our log still
|
||
# holds": the $10 gap was 11 hours old and the portal's log had already lost the
|
||
# whole morning, so a log-derived window cannot see the failures it exists for.
|
||
CREDIT_RECONCILE_DAYS = int(os.environ.get("CREDIT_RECONCILE_DAYS", "30") or "30")
|
||
CREDIT_RECONCILE_MAX_PAGES = int(os.environ.get("CREDIT_RECONCILE_MAX_PAGES", "3") or "3")
|
||
CREDIT_RECONCILE_MARKER = "credit_reconcile_fingerprint"
|
||
|
||
|
||
async def fetch_collective_orders(per_page: int = 100, max_pages: int = 3) -> list[dict]:
|
||
"""Recent orders for the collective, newest first. Raises on any failure.
|
||
|
||
Every failure raises rather than returning a short list: a caller that reads
|
||
a partial read as a complete one would conclude "nothing missing" from a
|
||
request that never answered. Needs the `orders` scope on OC_PERSONAL_TOKEN
|
||
(granted 2026-11-13; before that this answered `The Personal Token is not
|
||
allowed for operations in scope "orders"`).
|
||
"""
|
||
if not OC_PERSONAL_TOKEN:
|
||
raise RuntimeError("OC_PERSONAL_TOKEN not set; cannot fetch orders")
|
||
query = (
|
||
'{ collective(slug: "%s") { orders(limit: %d, offset: %d) { totalCount nodes { '
|
||
"id status createdAt processedAt frequency amount { value currency } "
|
||
"platformTipAmount { value } totalAmount { value } "
|
||
"tier { name slug } fromAccount { slug name type } } } } }"
|
||
)
|
||
out: list[dict] = []
|
||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||
for page in range(max(1, max_pages)):
|
||
r = await client.post(
|
||
OC_GRAPHQL_URL,
|
||
headers={"Content-Type": "application/json",
|
||
"Authorization": f"Bearer {OC_PERSONAL_TOKEN}",
|
||
"Api-Version": "2022-01-01",
|
||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64)"},
|
||
json={"query": query % (OC_COLLECTIVE_SLUG, per_page, page * per_page)},
|
||
)
|
||
if r.status_code != 200:
|
||
raise RuntimeError(f"OC order fetch failed: HTTP {r.status_code}")
|
||
body = r.json()
|
||
if body.get("errors"):
|
||
raise RuntimeError(f"OC order fetch errors: {str(body['errors'])[:200]}")
|
||
conn_ = ((body.get("data") or {}).get("collective") or {}).get("orders") or {}
|
||
nodes = conn_.get("nodes") or []
|
||
out.extend(nodes)
|
||
if len(nodes) < per_page or len(out) >= (conn_.get("totalCount") or 0):
|
||
break
|
||
return out
|
||
|
||
|
||
def _parse_ts(s) -> datetime | None:
|
||
"""OC sends ISO-8601 (…Z); our ledger rows are 'YYYY-MM-DD HH:MM:SS' (UTC)."""
|
||
if not s:
|
||
return None
|
||
t = str(s).strip()
|
||
if t.endswith("Z"):
|
||
t = t[:-1] + "+00:00"
|
||
if "T" not in t:
|
||
t = t.replace(" ", "T", 1)
|
||
try:
|
||
d = datetime.fromisoformat(t)
|
||
except ValueError:
|
||
return None
|
||
return d if d.tzinfo else d.replace(tzinfo=timezone.utc)
|
||
|
||
|
||
def _row_belongs_to(r: dict, slug: str, email: str) -> bool:
|
||
"""Does a ledger/held row identify this member — by slug in the reference, or by email?
|
||
|
||
Both routes matter: a manual grant often carries NO reference at all, and the
|
||
only link is the row's email. Requiring the slug to appear would drop those
|
||
rows (the slug and the email local part frequently differ), which is how a
|
||
hand-credited purchase comes to look like a gap.
|
||
"""
|
||
ref = (r.get("reference") or r.get("oc_reference") or "").lower()
|
||
em = (r.get("email") or "").lower()
|
||
slug_hit = bool(slug) and (slug.lower() in ref or slug.lower() in em)
|
||
email_hit = bool(email) and email.lower() in em
|
||
return slug_hit or email_hit
|
||
|
||
|
||
def credit_pack_accounted(order: dict, ledger_rows: list, held_rows: list,
|
||
member_email: str = "") -> str | None:
|
||
"""Evidence that this order is ALREADY in our books, else None.
|
||
|
||
The reference is NOT uniform across our own rows, and a bare
|
||
`oc:<slug>:<order.id>` match therefore reports credited purchases as missing
|
||
— it read 21 of them that way on 2026-11-13, all false:
|
||
|
||
* newer ledger rows `oc:nightrose:or_apShJowSoPPj1dlM3ExTH` (the v2 id)
|
||
* older ledger rows `oc:benlo_0:a47byg9n-xozdp84r-…` (a hyphenated id)
|
||
* held rows `oc:madrush:` (EMPTY suffix)
|
||
|
||
So match the exact reference when the row carries one, and otherwise demand a
|
||
SECOND signal — same member, same tip-free amount, within a few days — before
|
||
calling an order credited. A held row counts as accounted for: the money was
|
||
recorded on the hold path, which is what non-member purchases are supposed to
|
||
take.
|
||
"""
|
||
slug = ((order.get("fromAccount") or {}).get("slug") or "").strip()
|
||
oid = (order.get("id") or "").strip()
|
||
try:
|
||
amount = float((order.get("amount") or {}).get("value") or 0)
|
||
except (TypeError, ValueError):
|
||
amount = 0.0
|
||
when = _parse_ts(order.get("createdAt"))
|
||
|
||
if oid and slug:
|
||
exact = f"oc:{slug}:{oid}"
|
||
for r in ledger_rows:
|
||
if (r.get("reference") or "") == exact:
|
||
return f"ledger {exact}"
|
||
for h in held_rows:
|
||
if (h.get("oc_reference") or "") == exact:
|
||
return f"held #{h.get('id')} {exact}"
|
||
|
||
for rows, what in ((ledger_rows, "ledger"), (held_rows, "held")):
|
||
for r in rows:
|
||
ref = (r.get("reference") or r.get("oc_reference") or "")
|
||
if not _row_belongs_to(r, slug, member_email):
|
||
continue
|
||
try:
|
||
amt = float(r.get("delta") if what == "ledger" else r.get("amount") or 0)
|
||
except (TypeError, ValueError):
|
||
continue
|
||
if abs(amt - amount) > 0.005:
|
||
continue
|
||
rt = _parse_ts(r.get("created_at"))
|
||
if when and rt and abs((rt - when).total_seconds()) <= 5 * 86400:
|
||
return f"{what} {ref or '(no ref)'} — matched on member + amount + date"
|
||
return None
|
||
|
||
|
||
def _nearby_book_rows(slug: str, email: str, when: datetime | None, ledger_rows: list,
|
||
held_rows: list, days: int = 7, cap: int = 5) -> list[dict]:
|
||
"""Our own rows for this member NEAR this order — context for a human, never a decision.
|
||
|
||
WHY: a manual grant can be real and still carry no per-order reference.
|
||
ntnsndr's three launch-day $5 packs were credited on 2026-09-28 as +20.00 and
|
||
then corrected by -10.75, netting exactly 3 x $5.00 — with no reference on any
|
||
row. `credit_pack_accounted()` cannot see that (correctly: inferring
|
||
"accounted for" from a nearby amount would hide genuine gaps), so the order is
|
||
reported as unmatched. Without these neighbouring rows a reviewer cannot tell
|
||
that case from a real gap; with them, the manual grant is right there and the
|
||
candidate explains itself.
|
||
"""
|
||
out: list[dict] = []
|
||
if not when:
|
||
return out
|
||
for kind, rows in (("ledger", ledger_rows), ("held", held_rows)):
|
||
for r in rows:
|
||
ref = (r.get("reference") or r.get("oc_reference") or "")
|
||
if not _row_belongs_to(r, slug, email):
|
||
continue
|
||
rt = _parse_ts(r.get("created_at"))
|
||
if rt and abs((rt - when).total_seconds()) <= days * 86400:
|
||
out.append({
|
||
"kind": kind,
|
||
"delta": r.get("delta") if kind == "ledger" else r.get("amount"),
|
||
"reason": r.get("reason") or r.get("status") or "",
|
||
"reference": ref or None,
|
||
"created_at": r.get("created_at"),
|
||
})
|
||
out.sort(key=lambda x: str(x.get("created_at") or ""))
|
||
return out[:cap]
|
||
|
||
|
||
# --- Unmapped-backer check (was a script on the bot host) ------------------- #
|
||
# The failure mode (the mmoya case, 2026-10/11): a contributor pays the
|
||
# collective, but their OC account hides their email or belongs to an
|
||
# ORGANIZATION, so reconcile_memberships() cannot map them to a portal row and
|
||
# they are never provisioned — they wait until they ask. The sweep can only
|
||
# provision what fetch_members() can SEE (it drops nodes with no email); this
|
||
# reads the RAW member nodes, including the ones the sweep must skip, and
|
||
# compares them with members.db.
|
||
#
|
||
# It used to be a daily script on the bot's host, driving `cloudron exec` from
|
||
# outside. It belongs here: the portal holds both sides of the comparison, and a
|
||
# member waiting for an account must not depend on the bot being up.
|
||
CURRENT_ORDER_STATUS = {"ACTIVE", "PAID"} # an order that is paying right now
|
||
UNMAPPED_BACKERS_MARKER = "unmapped_backers_fingerprint"
|
||
|
||
|
||
def _oc_memberish(node: dict) -> bool:
|
||
"""Mirror fetch_members()'s member definition (ADMIN, or Membership/legacy tier)."""
|
||
role = node.get("role") or ""
|
||
if role not in ("BACKER", "ADMIN"):
|
||
return False
|
||
tier = (((node.get("tier") or {}) or {}).get("name") or "").strip()
|
||
return role == "ADMIN" or not tier or "membership" in tier.lower()
|
||
|
||
|
||
async def reconcile_unmapped_backers(notify: bool = True) -> dict:
|
||
"""Paid OC contributors with no portal row. Emails info@ only when there are findings.
|
||
|
||
Silent when everything is mapped, and — once a finding has been reported — quiet
|
||
while the SAME set persists, because this runs three times a day and a stuck
|
||
finding would otherwise mail every time. The fingerprint changes when a new
|
||
finding appears or an old one is fixed, and a recurrence after a clean run
|
||
emails again (the marker is cleared on a clean run).
|
||
|
||
A read that fails reports `skipped`, never a clean result: "could not look" is
|
||
not the same as "nothing wrong".
|
||
"""
|
||
try:
|
||
nodes = await fetch_all_member_nodes()
|
||
except Exception as e: # noqa: BLE001
|
||
logger.warning("unmapped-backer check: member fetch failed; skipping: %s", e)
|
||
return {"status": "skipped", "reason": "member_fetch_failed", "detail": str(e)[:200]}
|
||
try:
|
||
orders = await fetch_collective_orders()
|
||
except Exception as e: # noqa: BLE001
|
||
# A finding without dues context is still a finding; never lose it over this.
|
||
logger.warning("unmapped-backer check: order fetch failed (%s); findings omit dues", e)
|
||
orders = []
|
||
|
||
conn = get_db()
|
||
rows = [(r[0], r[1], r[2]) for r in
|
||
conn.execute("SELECT email, slug, org_slug FROM members")]
|
||
conn.close()
|
||
portal_emails = {str(e).strip().lower() for e, _s, _o in rows if e and "@" in str(e)}
|
||
portal_slugs = {str(s).strip() for _e, s, _o in rows if s}
|
||
portal_org_slugs = {str(o).strip() for _e, _s, o in rows if o}
|
||
|
||
def mapped(email, slug) -> bool:
|
||
if email and str(email).strip().lower() in portal_emails:
|
||
return True
|
||
return bool(slug) and (slug in portal_slugs or slug in portal_org_slugs)
|
||
|
||
def order_line(slug: str) -> str:
|
||
cands = [o for o in orders
|
||
if ((o.get("fromAccount") or {}).get("slug") or "") == slug]
|
||
if not cands:
|
||
return "no order found in the order list"
|
||
cands.sort(key=lambda o: str(o.get("createdAt") or ""))
|
||
ranked = ([o for o in cands if (o.get("status") or "") == "ACTIVE"]
|
||
or [o for o in cands if (o.get("status") or "") == "PAID"] or cands)
|
||
o = ranked[-1]
|
||
return (f"${(o.get('totalAmount') or {}).get('value')} {o.get('frequency')} "
|
||
f"{o.get('status')} (last order {str(o.get('createdAt') or '')[:10]})")
|
||
|
||
findings: list[dict] = []
|
||
seen: set[str] = set()
|
||
|
||
# Pass 1: account-bearing nodes (rich findings). Pass 2: null-account nodes,
|
||
# attributed through same-day orders.
|
||
for with_account in (True, False):
|
||
for n in nodes:
|
||
acct = n.get("account")
|
||
if with_account != bool(acct):
|
||
continue
|
||
if not _oc_memberish(n):
|
||
continue # e.g. a Credit-pack-only BACKER: the held-credits flow, not a member
|
||
if acct:
|
||
slug = (acct.get("slug") or "").strip()
|
||
email = (acct.get("email") or "").strip()
|
||
if mapped(email, slug):
|
||
continue
|
||
kind = (acct.get("type") or "INDIVIDUAL").upper()
|
||
name = acct.get("name") or slug or "(no name)"
|
||
if kind == "ORGANIZATION":
|
||
cls, vis = "org", "org email never visible (OC design, even for admins)"
|
||
action = ("org-backed membership — OC never exposes an org email. Link it: "
|
||
'POST /admin/org-link {"org_slug": "<slug>", "email": '
|
||
'"<preferred email>", "name": "<name>"} (X-Admin-Token)')
|
||
elif email:
|
||
cls, vis = "visible", f"VISIBLE to the admin view ({email})"
|
||
action = ("the email IS visible, so the sweep should have provisioned this "
|
||
"within 24h — check the sweep and the OC token "
|
||
"(POST /admin/provision re-provisions by hand)")
|
||
else:
|
||
cls, vis = "hidden", "HIDDEN from the admin view (null email)"
|
||
action = ("email hidden, so reconcile cannot map it. Get it out-of-band "
|
||
"(contribution page as collective admin) and provision with "
|
||
"POST /admin/provision on the operator's instruction")
|
||
key = slug or f"no-slug:{n.get('role')}"
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
findings.append({
|
||
"slug": slug, "name": name, "type": kind, "role": n.get("role"),
|
||
"tier": (((n.get("tier") or {}) or {}).get("name") or "none"),
|
||
"email_visibility": vis, "dues": order_line(slug), "action": action,
|
||
"class": cls,
|
||
"incognito_alias": slug.startswith("incognito-"),
|
||
})
|
||
else:
|
||
# No account object even in the ADMIN view (deleted, or
|
||
# incognito-hidden). Attribute via same-day current orders; stay
|
||
# silent when every same-day payer is already mapped.
|
||
day = str(n.get("createdAt") or "")[:10]
|
||
if not day:
|
||
# Without the node's date there is nothing to attribute it
|
||
# against, and guessing here is how a phantom finding appears:
|
||
# the first run of this check reported a null-account node as
|
||
# uncredited while the node's day was unknown to us (the
|
||
# members query did not select createdAt at all).
|
||
logger.warning("unmapped-backer check: a member node carries no "
|
||
"createdAt; cannot attribute it (role=%s tier=%s)",
|
||
n.get("role"), (((n.get("tier") or {}) or {}).get("name")))
|
||
continue
|
||
current = [o for o in orders
|
||
if str(o.get("createdAt") or "")[:10] == day
|
||
and (o.get("status") or "") in CURRENT_ORDER_STATUS]
|
||
unmapped = sorted({
|
||
((o.get("fromAccount") or {}).get("slug") or "")
|
||
for o in current
|
||
if not mapped(None, ((o.get("fromAccount") or {}).get("slug") or ""))})
|
||
if current and not unmapped:
|
||
continue
|
||
key = f"null-account:{day}"
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
findings.append({
|
||
"slug": "", "name": "(no account object in the admin view)",
|
||
"type": "UNKNOWN", "role": n.get("role"), "tier": "none",
|
||
"email_visibility": "no account object returned",
|
||
"dues": ", ".join(s for s in unmapped if s) or "no same-day order identified",
|
||
"action": ("a member node with no account object — attribute it from the "
|
||
"same-day orders above and provision by hand"),
|
||
"class": "null-account",
|
||
"incognito_alias": False,
|
||
})
|
||
|
||
# Keys, not a fingerprint: what has been TOLD is a set that grows, so a finding
|
||
# that stays put is never re-mailed while a genuinely new one always speaks.
|
||
keys = {f"{f['slug']}:{f['class']}" for f in findings}
|
||
conn = get_db()
|
||
reported_keys = {k for k in (_state_get(conn, UNMAPPED_BACKERS_MARKER, "") or "").split(",") if k}
|
||
conn.close()
|
||
new_keys = sorted(keys - reported_keys)
|
||
|
||
if findings:
|
||
logger.warning("unmapped backers: %d paid contributor(s) with no portal row "
|
||
"(%d new): %s", len(findings), len(new_keys),
|
||
"; ".join(f["slug"] or f["name"] for f in findings))
|
||
|
||
reported = False
|
||
if new_keys and notify:
|
||
lines = []
|
||
for f in findings:
|
||
if f"{f['slug']}:{f['class']}" not in new_keys:
|
||
continue
|
||
tag = " [incognito alias]" if f["incognito_alias"] else ""
|
||
lines.append(
|
||
f"- slug={f['slug'] or '?'}{tag} ({f['name']}) {f['type']} "
|
||
f"role={f['role']} tier={f['tier']}\n"
|
||
f" email: {f['email_visibility']}\n"
|
||
f" dues: {f['dues']}\n"
|
||
f" action: {f['action']}")
|
||
try:
|
||
send_email(
|
||
"info@inference.coop",
|
||
f"Inference Cooperative — {len(new_keys)} paid contributor(s) with no account",
|
||
"These Open Collective contributors look like members, but no portal row "
|
||
"matches them, so nothing has provisioned an account:\n\n"
|
||
+ "\n".join(lines)
|
||
+ "\n\nThis covers what is NEW since the last notice, not every sweep. A "
|
||
"contributor who has just joined may also be caught by the next membership "
|
||
"sweep (3x/day); if they are still listed after that, the sweep cannot see "
|
||
"them and someone has to act.\n",
|
||
)
|
||
reported = True
|
||
except Exception as e: # noqa: BLE001
|
||
logger.warning("unmapped-backer notice email failed: %s", e)
|
||
|
||
# Advance the marker only when someone was actually TOLD. A run with notify=False
|
||
# is a look, not a notice: advancing here would let a silent check swallow the
|
||
# only alert a finding ever gets. A failed send retries on the next sweep. A
|
||
# CLEAN run clears it, so a recurrence is news again.
|
||
if reported:
|
||
conn = get_db()
|
||
_state_set(conn, UNMAPPED_BACKERS_MARKER, ",".join(sorted(reported_keys | keys))[:400])
|
||
conn.commit()
|
||
conn.close()
|
||
elif not findings:
|
||
conn = get_db()
|
||
_state_set(conn, UNMAPPED_BACKERS_MARKER, "")
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
return {"status": "ok", "findings": len(findings), "new": len(new_keys), "classes":
|
||
sorted({f["class"] for f in findings}), "reported": reported,
|
||
"detail": findings}
|
||
|
||
|
||
# --- Spend-consistency check (was a script on the bot host) ----------------- #
|
||
# The failure it exists for (2026-11-05): the portal aggregates all-time spend
|
||
# from LiteLLM's Postgres using a COPY of the gateway's DB credential. When the
|
||
# gateway rotated that credential the copy went stale, the aggregation failed
|
||
# SOFT (empty -> $0), and the panel and dashboard showed $0 while every surface
|
||
# looked clean. Nobody would have noticed from inside the portal, because the
|
||
# portal WAS the thing that was wrong.
|
||
#
|
||
# The old check lived on the bot's host and read LiteLLM's own SpendLogs through
|
||
# the gateway's credential. This port keeps the two independent reads — one over
|
||
# the portal's DB copy, one over LiteLLM's HTTP API with the master key (a
|
||
# different transport AND a different credential) — and adds the guard that
|
||
# catches the silent-clean signature directly: a $0 (or empty) aggregate while
|
||
# LiteLLM's own API reports real spend.
|
||
#
|
||
# It does NOT compare against the figure /admin/overview computes inline: that
|
||
# aggregation is embedded in the handler, and the DB aggregate it is built from is
|
||
# what this checks. Extracting it into a shared helper is the next step if the two
|
||
# ever need to be compared directly.
|
||
SPEND_CHECK_TOLERANCE = float(os.environ.get("SPEND_CHECK_TOLERANCE", "5.00") or "5.00")
|
||
# LiteLLM's API reports spend per day per team rounded to cents, so a day can differ
|
||
# by a cent or two legitimately (the largest seen on live data was $0.022).
|
||
SPEND_CHECK_DAILY_TOLERANCE = float(os.environ.get("SPEND_CHECK_DAILY_TOLERANCE", "0.10") or "0.10")
|
||
SPEND_CHECK_MARKER = "spend_check_reported"
|
||
|
||
|
||
async def litellm_api_daily_spend() -> dict[str, float]:
|
||
"""{UTC date: $} from LiteLLM's OWN API (`/global/spend/teams`). Raises on failure.
|
||
|
||
WHY NOT `/global/spend/keys`: its per-key `total_spend` follows the key's own
|
||
budget cycle, so summing it compares the last cycle against our all-time figure
|
||
($827.89 vs $935.52 on 2026-11-13 — a $107 window mismatch that looks exactly
|
||
like drift). `/global/spend/report` would be all-time but is gated behind a
|
||
LiteLLM Enterprise licence (HTTP 400). The per-day teams view is the comparable
|
||
one, and comparing DAY BY DAY is sharper than comparing sums: it localises drift
|
||
instead of hiding it in an aggregate.
|
||
"""
|
||
if not LITELLM_MASTER_KEY:
|
||
raise RuntimeError("LITELLM_MASTER_KEY not set; cannot verify against LiteLLM")
|
||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||
r = await client.get(f"{LITELLM_BASE}/global/spend/teams",
|
||
headers={"Authorization": f"Bearer {LITELLM_MASTER_KEY}"})
|
||
if r.status_code != 200:
|
||
raise RuntimeError(f"LiteLLM spend API: HTTP {r.status_code}")
|
||
body = r.json()
|
||
if not isinstance(body, dict):
|
||
raise RuntimeError(f"LiteLLM spend API returned {type(body).__name__}, not an object")
|
||
out: dict[str, float] = {}
|
||
for day in body.get("daily_spend") or []:
|
||
day = day or {}
|
||
d = str(day.get("date") or "")[:10]
|
||
if not d:
|
||
continue
|
||
total = 0.0
|
||
for k, v in day.items():
|
||
if k == "date":
|
||
continue
|
||
try:
|
||
total += float(v or 0)
|
||
except (TypeError, ValueError):
|
||
continue
|
||
out[d] = round(total, 6)
|
||
return out
|
||
|
||
|
||
async def _spend_failure(notify: bool, reason: str, verdict: dict) -> dict:
|
||
"""Report a spend-check failure to info@ ONCE per failing episode.
|
||
|
||
The marker holds the reason a human was told, so a persistent failure does not
|
||
mail every sweep while a changed one always does. It is written only after a
|
||
notice actually went out: a silent (`notify=False`) run or a failed send must not
|
||
swallow the alert a failure gets.
|
||
"""
|
||
verdict = dict(verdict)
|
||
verdict["status"] = "failed"
|
||
verdict["reason"] = reason
|
||
verdict["tolerance"] = SPEND_CHECK_TOLERANCE
|
||
verdict["daily_tolerance"] = SPEND_CHECK_DAILY_TOLERANCE
|
||
verdict["reported"] = False
|
||
conn = get_db()
|
||
already = (_state_get(conn, SPEND_CHECK_MARKER, "") or "") == reason
|
||
conn.close()
|
||
if not notify or already:
|
||
return verdict
|
||
try:
|
||
send_email(
|
||
"info@inference.coop",
|
||
"Inference Cooperative — spend figures disagree (portal vs LiteLLM)",
|
||
"The portal's own spend aggregation and LiteLLM's API do not agree. This "
|
||
"check runs on every sweep and is SILENT when they match.\n\n"
|
||
f" window : {verdict.get('window')} ({verdict.get('days_compared')} day(s))\n"
|
||
f" portal : ${float(verdict.get('portal_window') or 0):.2f}\n"
|
||
f" LiteLLM : ${float(verdict.get('api_window') or 0):.2f}\n"
|
||
f" delta : ${float(verdict.get('delta') or 0):+.2f} "
|
||
f"(tolerance ${SPEND_CHECK_TOLERANCE:.2f}, per-day "
|
||
f"${SPEND_CHECK_DAILY_TOLERANCE:.2f})\n"
|
||
f" all-time portal: ${float(verdict.get('portal_all_time') or 0):.2f} over "
|
||
f"{verdict.get('portal_rows')} rows\n\n"
|
||
f"Reason: {reason}\n\n"
|
||
"Worst days, if any:\n"
|
||
+ ("\n".join(f" {d['date']}: portal ${d['portal']:.2f} vs LiteLLM "
|
||
f"${d['api']:.2f} (${d['delta']:+.2f})"
|
||
for d in (verdict.get("drifting_days") or []))
|
||
or " (none beyond the per-day tolerance)")
|
||
+ "\n\nThe likely cause is the portal's copy of the LiteLLM DB credential going "
|
||
"stale after a rotation: the aggregation then fails soft and every surface "
|
||
"shows $0 while looking clean. Check the credential in the portal's env "
|
||
"against the gateway app's own.\n",
|
||
)
|
||
verdict["reported"] = True
|
||
conn = get_db()
|
||
_state_set(conn, SPEND_CHECK_MARKER, reason[:400])
|
||
conn.commit()
|
||
conn.close()
|
||
except Exception as e: # noqa: BLE001
|
||
logger.warning("spend check: notice email failed: %s", e)
|
||
return verdict
|
||
|
||
|
||
async def check_spend_consistency(notify: bool = True) -> dict:
|
||
"""Compare the portal's SpendLogs aggregate with LiteLLM's own API. Emails only on FAIL.
|
||
|
||
Returns status="ok" with a verdict, or "failed" with the reason — and a read that
|
||
cannot be made is `skipped`, never a clean bill: "could not look" is an anomaly.
|
||
"""
|
||
try:
|
||
portal_by_day, portal_all_time, portal_rows = litellm_spend_by_day()
|
||
except Exception as e: # noqa: BLE001
|
||
logger.warning("spend check: portal SpendLogs read failed: %s", e)
|
||
return {"status": "skipped", "reason": "portal_read_failed", "detail": str(e)[:200]}
|
||
try:
|
||
api_by_day = await litellm_api_daily_spend()
|
||
except Exception as e: # noqa: BLE001
|
||
logger.warning("spend check: LiteLLM API read failed: %s", e)
|
||
return {"status": "skipped", "reason": "litellm_api_failed", "detail": str(e)[:200]}
|
||
|
||
# Degenerate reads come FIRST: in the failure this check exists for, the portal's
|
||
# aggregate comes back EMPTY, so there are no shared days to compare and the
|
||
# overlap logic would call it "cannot compare" instead of naming the real problem.
|
||
api_all_days_total = round(sum(api_by_day.values()), 2)
|
||
if portal_rows == 0 and api_all_days_total > 1:
|
||
reason = (f"the portal's SpendLogs aggregate is EMPTY ($0 over 0 rows) while LiteLLM's "
|
||
f"own API reports ${api_all_days_total:.2f} over {len(api_by_day)} day(s) — "
|
||
f"the silent-clean signature of a stale LiteLLM DB credential (the "
|
||
f"panel/dashboard $0 is the same failure)")
|
||
logger.error("spend check FAILED: %s", reason)
|
||
return await _spend_failure(
|
||
notify, reason,
|
||
{"portal_rows": 0, "portal_window": 0.0, "api_window": api_all_days_total,
|
||
"delta": -api_all_days_total, "window": "none (the portal read returned nothing)",
|
||
"days_compared": 0, "portal_all_time": 0.0, "drifting_days": []})
|
||
if not api_by_day and portal_rows > 0:
|
||
reason = ("LiteLLM's spend API returned no days while the portal read "
|
||
f"{portal_rows} SpendLogs rows — the comparison has no independent side")
|
||
logger.error("spend check FAILED: %s", reason)
|
||
return await _spend_failure(
|
||
notify, reason,
|
||
{"portal_rows": portal_rows, "portal_window": 0.0, "api_window": 0.0,
|
||
"delta": 0.0, "window": "none", "days_compared": 0,
|
||
"portal_all_time": portal_all_time, "drifting_days": []})
|
||
|
||
# Compare only the days BOTH sides cover. LiteLLM's per-day view reaches back a
|
||
# fixed number of days (31 on 2026-11-13) while our table holds everything, so a
|
||
# sum-vs-all-time comparison reports the missing window as drift — $107.63 of it.
|
||
overlap = sorted(set(api_by_day) & set(portal_by_day))
|
||
if not overlap:
|
||
return {"status": "skipped", "reason": "no_overlapping_window",
|
||
"detail": f"LiteLLM reports {len(api_by_day)} day(s), the portal "
|
||
f"{len(portal_by_day)}; they share none"}
|
||
|
||
per_day = [(d, portal_by_day[d], api_by_day[d],
|
||
round(api_by_day[d] - portal_by_day[d], 4)) for d in overlap]
|
||
portal_window = round(sum(p for _d, p, _a, _x in per_day), 2)
|
||
api_window = round(sum(a for _d, _p, a, _x in per_day), 2)
|
||
delta = round(api_window - portal_window, 2)
|
||
bad_days = sorted((x for x in per_day if abs(x[3]) > SPEND_CHECK_DAILY_TOLERANCE),
|
||
key=lambda x: -abs(x[3]))
|
||
verdict = {
|
||
"portal_window": portal_window,
|
||
"api_window": api_window,
|
||
"delta": delta,
|
||
"window": f"{overlap[0]}..{overlap[-1]}",
|
||
"days_compared": len(overlap),
|
||
"portal_all_time": round(portal_all_time, 2),
|
||
"portal_rows": portal_rows,
|
||
"drifting_days": [{"date": d, "portal": p, "api": a, "delta": x}
|
||
for d, p, a, x in bad_days[:5]],
|
||
"tolerance": SPEND_CHECK_TOLERANCE,
|
||
"daily_tolerance": SPEND_CHECK_DAILY_TOLERANCE,
|
||
}
|
||
|
||
reason = ""
|
||
if bad_days:
|
||
d, p, a, x = bad_days[0]
|
||
reason = (f"{len(bad_days)} day(s) differ by more than ${SPEND_CHECK_DAILY_TOLERANCE:.2f} "
|
||
f"across {verdict['window']} — worst {d}: portal ${p:.2f} vs LiteLLM "
|
||
f"${a:.2f} (${x:+.2f}); the two are not reading the same data")
|
||
elif abs(delta) > SPEND_CHECK_TOLERANCE:
|
||
reason = (f"window totals differ by ${delta:+.2f} (portal ${portal_window:.2f} vs "
|
||
f"LiteLLM ${api_window:.2f} over {len(overlap)} day(s)) — beyond the "
|
||
f"${SPEND_CHECK_TOLERANCE:.2f} tolerance")
|
||
|
||
if not reason:
|
||
verdict["status"] = "ok"
|
||
logger.info("spend check PASS: portal $%.2f vs LiteLLM $%.2f over %s (%d days), "
|
||
"delta $%+.2f within $%.2f; all-time portal $%.2f over %d rows",
|
||
portal_window, api_window, verdict["window"], len(overlap), delta,
|
||
SPEND_CHECK_TOLERANCE, portal_all_time, portal_rows)
|
||
# A clean result clears the marker, so a recurrence is reported again.
|
||
try:
|
||
conn = get_db()
|
||
_state_set(conn, SPEND_CHECK_MARKER, "")
|
||
conn.commit()
|
||
conn.close()
|
||
except Exception as e: # noqa: BLE001
|
||
logger.warning("spend check: could not clear the report marker: %s", e)
|
||
return verdict
|
||
|
||
logger.error("spend check FAILED: %s", reason)
|
||
return await _spend_failure(notify, reason, verdict)
|
||
|
||
|
||
async def reconcile_credit_packs(apply: bool = False, notify: bool = True) -> dict:
|
||
"""Credit PAID credit packs that never reached our books (issue #7).
|
||
|
||
`apply=False` (report) is the default and changes NOTHING; granting also
|
||
requires CREDIT_RECONCILE == "on", so the pass can run on every sweep without
|
||
moving money until it has been read once in production. Every grant goes
|
||
through `add_credits` with `credit_pack_reference()` — the same key the
|
||
webhook writes — never a hand-set balance, so a late OC delivery and this
|
||
pass cannot both credit the same purchase.
|
||
"""
|
||
if CREDIT_RECONCILE == "off":
|
||
return {"status": "skipped", "reason": "credit_reconcile_off"}
|
||
try:
|
||
orders = await fetch_collective_orders(max_pages=CREDIT_RECONCILE_MAX_PAGES)
|
||
except Exception as e: # noqa: BLE001
|
||
# Never "nothing missing": an unreadable order list has to read as UNKNOWN.
|
||
logger.warning("Credit reconcile: order fetch failed; skipping (fail-safe): %s", e)
|
||
return {"status": "skipped", "reason": "order_fetch_failed", "detail": str(e)[:200]}
|
||
try:
|
||
index = await fetch_member_index()
|
||
except Exception as e: # noqa: BLE001
|
||
logger.warning("Credit reconcile: member index fetch failed; skipping: %s", e)
|
||
return {"status": "skipped", "reason": "member_index_failed", "detail": str(e)[:200]}
|
||
|
||
# Orders are matched on the slug from the LIVE OC list, never on the stored
|
||
# `members.slug` column, which can hold a stale `guest-…` from provisioning.
|
||
slug2email: dict[str, str] = {}
|
||
for email, rec in index.items():
|
||
s = (rec.get("slug") or "").strip()
|
||
if s:
|
||
slug2email.setdefault(s, email)
|
||
|
||
conn = get_db()
|
||
# Positional rows: get_db() hands back a plain connection (no row_factory),
|
||
# which is how the rest of this module reads it too.
|
||
ledger_rows = [
|
||
{"email": r[0], "delta": r[1], "reference": r[2], "created_at": r[3], "reason": r[4]}
|
||
for r in conn.execute(
|
||
"SELECT email, delta, reference, created_at, reason FROM credit_ledger")]
|
||
held_rows = [
|
||
{"id": r[0], "slug": r[1], "email": r[2], "amount": r[3],
|
||
"oc_reference": r[4], "status": r[5], "created_at": r[6]}
|
||
for r in conn.execute(
|
||
"SELECT id, slug, email, amount, oc_reference, status, created_at "
|
||
"FROM held_credits")]
|
||
active = {r[0].strip().lower() for r in conn.execute("SELECT email FROM members WHERE active = 1")}
|
||
conn.close()
|
||
|
||
cutoff = datetime.now(timezone.utc) - timedelta(days=CREDIT_RECONCILE_DAYS)
|
||
packs = 0
|
||
unaccounted: list[dict] = []
|
||
for o in orders:
|
||
if (o.get("status") or "").upper() != "PAID":
|
||
continue
|
||
if (o.get("frequency") or "").upper() != "ONETIME":
|
||
continue
|
||
if "credit" not in (((o.get("tier") or {}).get("name") or "").lower()):
|
||
continue
|
||
when = _parse_ts(o.get("createdAt"))
|
||
if when and when < cutoff:
|
||
continue
|
||
packs += 1
|
||
slug = ((o.get("fromAccount") or {}).get("slug") or "").strip()
|
||
if not credit_pack_accounted(o, ledger_rows, held_rows,
|
||
member_email=slug2email.get(slug, "")):
|
||
unaccounted.append(o)
|
||
|
||
acting = bool(apply and CREDIT_RECONCILE == "on")
|
||
candidates: list[dict] = []
|
||
applied: list[dict] = []
|
||
needs_hold: list[dict] = []
|
||
for o in unaccounted:
|
||
slug = ((o.get("fromAccount") or {}).get("slug") or "").strip()
|
||
rec = {"slug": slug,
|
||
"order_id": o.get("id"),
|
||
"amount": float((o.get("amount") or {}).get("value") or 0),
|
||
"created_at": str(o.get("createdAt") or "")[:19],
|
||
"tier": (o.get("tier") or {}).get("name")}
|
||
email = slug2email.get(slug, "")
|
||
if not email or email.lower() not in active:
|
||
# A purchase by someone who is not an active member keeps the
|
||
# hold-and-email path. This pass must never mint credits for a buyer
|
||
# who has no membership to spend them under.
|
||
rec["reason"] = "no active member for this OC slug — hold path, not a grant"
|
||
needs_hold.append(rec)
|
||
continue
|
||
rec["email"] = email
|
||
ref = credit_pack_reference(slug, {"idV2": o.get("id")})
|
||
rec["reference"] = ref
|
||
nearby = _nearby_book_rows(slug, email, _parse_ts(o.get("createdAt")),
|
||
ledger_rows, held_rows)
|
||
if nearby:
|
||
# Printed for a HUMAN to judge: a manual grant can be real and carry
|
||
# no reference at all, and this is the only way to tell that from a gap.
|
||
rec["nearby_book_rows"] = nearby
|
||
candidates.append(rec)
|
||
if not acting or credit_reference_exists(ref):
|
||
continue
|
||
new_balance = add_credits(email, rec["amount"],
|
||
reason="credit pack purchase (Open Collective)",
|
||
reference=ref)
|
||
try:
|
||
await sync_team_budget_from_balance(email)
|
||
rec["result"] = f"granted (balance {new_balance})"
|
||
except Exception as e: # noqa: BLE001
|
||
# The ledger row is the record that matters; the budget push is
|
||
# retried by the next sweep rather than failing the grant.
|
||
logger.warning("Credit reconcile: budget sync failed for %s: %s", email, e)
|
||
rec["result"] = f"granted (balance {new_balance}); budget sync failed: {str(e)[:80]}"
|
||
applied.append(rec)
|
||
logger.warning("Credit reconcile: GRANTED $%.2f to %s for order %s (was missing)",
|
||
rec["amount"], email, rec["order_id"])
|
||
|
||
if candidates and not acting:
|
||
logger.warning(
|
||
"Credit reconcile: %d PAID credit pack(s) with no record in our books "
|
||
"(report mode — nothing changed): %s", len(candidates),
|
||
"; ".join(
|
||
f"${c['amount']:.2f} {c['slug']} {c['created_at']}"
|
||
+ (f" [nearby rows: "
|
||
+ ", ".join(f"{n['delta']} on {n['created_at']} ({n['reason']})"
|
||
for n in c["nearby_book_rows"]) + "]"
|
||
if c.get("nearby_book_rows") else "")
|
||
for c in candidates))
|
||
|
||
# What a human has already been TOLD is a SET of keys, not a fingerprint of the
|
||
# current set: a purchase drops out of the set once it is granted, and a
|
||
# fingerprint would make the remaining item look new and mail about it twice.
|
||
keys = {f"{c['order_id']}:grant" for c in candidates} | \
|
||
{f"{c['order_id']}:hold" for c in needs_hold}
|
||
conn = get_db()
|
||
reported_keys = {k for k in (_state_get(conn, CREDIT_RECONCILE_MARKER, "") or "").split(",") if k}
|
||
conn.close()
|
||
new_keys = keys - reported_keys
|
||
|
||
notified = False
|
||
if new_keys and notify:
|
||
granted = "\n".join(
|
||
f" granted ${c['amount']:.2f} to {c['slug']} for order {c['order_id']} "
|
||
f"({c['created_at']}) — {c.get('result', 'not applied (report mode)')}"
|
||
for c in candidates if f"{c['order_id']}:grant" in new_keys) or " (nothing granted)"
|
||
held = "\n".join(
|
||
f" NOT granted: ${c['amount']:.2f} bought by {c['slug']} for order "
|
||
f"{c['order_id']} ({c['created_at']}) — {c['reason']}" for c in needs_hold
|
||
if f"{c['order_id']}:hold" in new_keys)
|
||
try:
|
||
send_email(
|
||
"info@inference.coop",
|
||
f"Inference Cooperative — paid credit pack(s) missing from the ledger "
|
||
f"({len(new_keys)} new)",
|
||
"A credit pack that Open Collective recorded as PAID had no matching entry "
|
||
"in our books. Credits are normally granted by the Open Collective webhook; "
|
||
"this pass reads Open Collective's own order list every sweep, so a purchase "
|
||
"whose delivery failed is credited without anyone having to notice.\n\n"
|
||
+ granted
|
||
+ (("\n\n" + held + "\n\n(These buyers have no active membership, so they keep "
|
||
"the hold-and-email path — nothing was granted.)")
|
||
if held else "")
|
||
+ "\n\nThis notice covers what is NEW since the last one, not every sweep.\n",
|
||
)
|
||
notified = True
|
||
except Exception as e: # noqa: BLE001
|
||
logger.warning("Credit reconcile: notice email failed: %s", e)
|
||
|
||
# Same rule as the other checks: the marker records what someone was TOLD. A
|
||
# silent (notify=False) run must not swallow a real notice, and a clean pass
|
||
# clears it so the next occurrence is news again.
|
||
if notified:
|
||
conn = get_db()
|
||
_state_set(conn, CREDIT_RECONCILE_MARKER, ",".join(sorted(reported_keys | keys))[:400])
|
||
conn.commit()
|
||
conn.close()
|
||
elif not keys:
|
||
conn = get_db()
|
||
_state_set(conn, CREDIT_RECONCILE_MARKER, "")
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
return {
|
||
"status": "ok",
|
||
"dry_run": not acting,
|
||
"window_days": CREDIT_RECONCILE_DAYS,
|
||
"orders_read": len(orders),
|
||
"packs_in_window": packs,
|
||
"unaccounted": len(candidates) + len(needs_hold),
|
||
"candidates": candidates,
|
||
"needs_hold": needs_hold,
|
||
"applied": applied,
|
||
"notified": notified,
|
||
}
|
||
|
||
|
||
async def reconcile_allowance_shortfalls(send: bool = False) -> dict:
|
||
"""Credit members whose PAID cycle Open Collective's billing-day alignment truncated.
|
||
|
||
`send=False` (report) computes the set and changes NOTHING. `send=True`
|
||
(honoured only when ALLOWANCE_POLICY == "on") applies one credit per truncated
|
||
charge — idempotent on the charge reference — and, when
|
||
ALLOWANCE_SHORTFALL_EMAILS == "on", sends the member notice (once per charge).
|
||
Emails default OFF so the wording is reviewed before any member is written to.
|
||
"""
|
||
try:
|
||
records = await fetch_oc_monthly_order_charges()
|
||
except Exception as e:
|
||
logger.warning("Allowance shortfall: charge fetch failed; skipping (fail-safe): %s", e)
|
||
return {"status": "skipped", "reason": "charge_fetch_failed", "detail": str(e)[:200]}
|
||
try:
|
||
index = await fetch_member_index()
|
||
except Exception as e:
|
||
logger.warning("Allowance shortfall: member index fetch failed; skipping (fail-safe): %s", e)
|
||
return {"status": "skipped", "reason": "member_index_failed", "detail": str(e)[:200]}
|
||
|
||
cycles = derive_shortfalls(records)
|
||
|
||
# slug -> email from the LIVE OC member list; only provisioned, active members
|
||
# are credited. Orders are matched by the slug resolved from the member's email
|
||
# (the stored column can hold a stale guest-… slug from first provisioning).
|
||
slug2email: dict[str, str] = {}
|
||
for e, rec in index.items():
|
||
s = rec.get("slug") or ""
|
||
if s:
|
||
slug2email.setdefault(s, e)
|
||
|
||
conn = get_db()
|
||
_ensure_shortfall_tables(conn)
|
||
active = {r[0].strip().lower() for r in conn.execute("SELECT email FROM members WHERE active = 1")}
|
||
meta = {r[0].strip().lower(): {"slug": r[1], "org_slug": r[2]}
|
||
for r in conn.execute("SELECT email, slug, org_slug FROM members")}
|
||
conn.close()
|
||
|
||
acting = bool(send and ALLOWANCE_POLICY == "on")
|
||
mail_on = ALLOWANCE_SHORTFALL_EMAILS == "on"
|
||
pack_url = credit_pack_url()
|
||
credited, staged, skipped = [], [], []
|
||
seen: set[str] = set()
|
||
|
||
for c in cycles:
|
||
if c["reference"] in seen: # a charge truncates exactly one cycle
|
||
continue
|
||
email = slug2email.get(c["slug"])
|
||
if not email:
|
||
skipped.append({**c, "why": "no Open Collective account email for this charge's slug"})
|
||
continue
|
||
if email not in active:
|
||
skipped.append({**c, "email": email, "why": "not an active portal member"})
|
||
continue
|
||
m = meta.get(email) or {}
|
||
if is_manual_member(email) or m.get("org_slug"):
|
||
skipped.append({**c, "email": email, "why": "exempt member (manual / organization-backed)"})
|
||
continue
|
||
rec = index.get(email)
|
||
if rec and rec.get("role") == "ADMIN":
|
||
skipped.append({**c, "email": email, "why": "operator account"})
|
||
continue
|
||
if c["reference"] in SHORTFALL_REMEDIATED:
|
||
skipped.append({**c, "email": email,
|
||
"why": "already remediated by a manual allowance reset (2026-11-04)"})
|
||
continue
|
||
seen.add(c["reference"])
|
||
|
||
entry = {**c, "email": email, "name": (m.get("slug") or "there")}
|
||
already = credit_reference_exists(c["reference"])
|
||
entry["already_credited"] = already
|
||
if not already and acting:
|
||
add_credits(email, c["amount"],
|
||
reason=shortfall_reason(c["prev"], c["cur"]),
|
||
reference=c["reference"])
|
||
try:
|
||
await sync_team_budget_from_balance(email)
|
||
except Exception as e: # budget push is best-effort
|
||
logger.warning("Shortfall: team budget sync failed for %s: %s", email, e)
|
||
entry["credited"] = True
|
||
elif not already:
|
||
entry["credited"] = False # would-credit (report mode / policy off)
|
||
|
||
if mail_on and not shortfall_notice_sent(c["reference"]):
|
||
subject, text_body, html_body = shortfall_email(
|
||
entry["name"], c["amount"], c["prev"], c["cur"], c["gap_days"],
|
||
c["days_lost"], pack_url)
|
||
if send_email(email, subject, text_body, html_body):
|
||
mark_shortfall_notice_sent(c["reference"], email, c["amount"])
|
||
entry["notified"] = True
|
||
else:
|
||
entry["notified"] = False
|
||
elif not mail_on:
|
||
entry["notice"] = "staged (ALLOWANCE_SHORTFALL_EMAILS is off)"
|
||
|
||
(credited if entry.get("credited") or entry.get("already_credited") else staged).append(entry)
|
||
|
||
newly = sum(e["amount"] for e in credited if e.get("credited"))
|
||
total = sum(e["amount"] for e in credited)
|
||
return {
|
||
"status": "ok",
|
||
"mode": ALLOWANCE_POLICY,
|
||
"acting": acting,
|
||
"emails": ALLOWANCE_SHORTFALL_EMAILS,
|
||
"full_month_days": FULL_PAID_MONTH_DAYS,
|
||
"affected": len(credited),
|
||
"newly_credited": sum(1 for e in credited if e.get("credited")),
|
||
"already_credited": sum(1 for e in credited if e.get("already_credited")),
|
||
"new_credit_total": round(newly, 10),
|
||
"affected_credit_total": round(total, 10),
|
||
"credits": credited,
|
||
"would_credit": staged,
|
||
"skipped": skipped,
|
||
}
|
||
|
||
|
||
@app.post("/admin/shortfall-credits")
|
||
@limiter.limit("10/minute")
|
||
async def admin_shortfall_credits(request: Request, mode: str = "report"):
|
||
"""Preview (default) or apply allowance-shortfall credits.
|
||
|
||
`mode=report` computes the affected set and changes nothing. `mode=send`
|
||
applies the credits (requires ALLOWANCE_POLICY=on) — idempotent on the OC
|
||
charge reference, so a second call credits nothing again. Member notices are
|
||
sent only when ALLOWANCE_SHORTFALL_EMAILS=on.
|
||
"""
|
||
_verify_admin_token(request)
|
||
if mode == "send":
|
||
if ALLOWANCE_POLICY != "on":
|
||
return {"refused": f"ALLOWANCE_POLICY is '{ALLOWANCE_POLICY}', not 'on'"}
|
||
return await reconcile_allowance_shortfalls(send=True)
|
||
return await reconcile_allowance_shortfalls(send=False)
|
||
|
||
|
||
@app.post("/admin/allowance")
|
||
@limiter.limit("10/minute")
|
||
async def admin_allowance(request: Request, mode: str = "report", to: str | None = None):
|
||
"""Reconcile member allowances against Open Collective payments.
|
||
|
||
`mode=report` (default) changes nothing. `mode=send` performs renewals and
|
||
sends the single reminder, and requires ALLOWANCE_POLICY=on on the app.
|
||
Deactivation is separately gated by ALLOWANCE_DEACTIVATE=on.
|
||
`mode=test&to=<address>` emails one sample reminder so the copy and the
|
||
outbound path can be verified without touching the ledger.
|
||
"""
|
||
_verify_admin_token(request)
|
||
if mode == "test":
|
||
dest = (to or "").strip()
|
||
if not dest:
|
||
return {"refused": "mode=test needs &to=<address>"}
|
||
subject, text_body, html_body = payment_reminder_email("there", credit_pack_url())
|
||
ok = send_email(dest, subject, text_body, html_body)
|
||
return {"mode": "test", "to": dest, "sent": bool(ok), "subject": subject}
|
||
if mode == "send":
|
||
if ALLOWANCE_POLICY != "on":
|
||
return {"refused": f"ALLOWANCE_POLICY is '{ALLOWANCE_POLICY}', not 'on' — "
|
||
"nothing sent; set it on the portal app first"}
|
||
return await reconcile_allowances(send=True)
|
||
return await reconcile_allowances(send=False)
|
||
|
||
|
||
HELD_ALERT_HOURS = float(os.environ.get("HELD_ALERT_HOURS", "6") or 6)
|
||
|
||
|
||
async def settle_held_credits(send: bool = True) -> dict:
|
||
"""Grant held credit packs that belong to members who are now active.
|
||
|
||
A pack bought while the buyer's membership was inactive is held on purpose —
|
||
this pass must never mint credits for a non-member. The gap it closes is what
|
||
happens next: those holds used to stay until an operator noticed them by hand
|
||
(a $10 pack sat from 01:45 until the member wrote in). Now every sweep grants
|
||
a hold whose buyer resolves to an ACTIVE member, and reports holds older than
|
||
HELD_ALERT_HOURS that still have nobody to credit.
|
||
"""
|
||
conn = get_db()
|
||
granted: list[dict] = []
|
||
unresolved: list[dict] = []
|
||
stale: list[dict] = []
|
||
try:
|
||
held = conn.execute(
|
||
"SELECT id, slug, email, amount, created_at FROM held_credits "
|
||
"WHERE status = 'held'").fetchall()
|
||
active = {r[0].strip().lower(): (r[1] or "") for r in
|
||
conn.execute("SELECT email, slug FROM members WHERE active = 1")}
|
||
by_slug = {(sl or "").strip().lower(): em for em, sl in active.items() if sl}
|
||
now = datetime.now(timezone.utc)
|
||
for hold_id, slug, email, amount, created_at in held:
|
||
em = (email or "").strip().lower()
|
||
target = em if em in active else by_slug.get((slug or "").strip().lower(), "")
|
||
if target:
|
||
moved = grant_held_credits(conn, target, slug or "")
|
||
if moved:
|
||
granted.append({"hold": hold_id, "email": target,
|
||
"amount": float(amount or 0)})
|
||
continue
|
||
when = _parse_ts(created_at)
|
||
age_h = ((now - when).total_seconds() / 3600.0) if when else 0.0
|
||
row = {"hold": hold_id, "email": email, "slug": slug,
|
||
"amount": float(amount or 0), "age_h": round(age_h, 1)}
|
||
unresolved.append(row)
|
||
if age_h >= HELD_ALERT_HOURS:
|
||
stale.append(row)
|
||
finally:
|
||
conn.close()
|
||
notified = False
|
||
if stale and send:
|
||
lines = "\n".join(
|
||
f"- hold #{h['hold']}: ${h['amount']:.2f} from {h['email'] or h['slug']} ({h['age_h']}h old)"
|
||
for h in stale)
|
||
notified = bool(send_email(
|
||
ADMIN_INVITER_EMAIL,
|
||
f"{len(stale)} unapplied credit pack(s) waiting",
|
||
"These credit packs are held with no active member to credit:\n\n" + lines +
|
||
"\n\nGrant one at POST /admin/held-credits/<id>/grant {\"email\": \"...\"}, "
|
||
"or link the buyer to a member account.\n"))
|
||
return {"held": len(held), "granted": granted, "unresolved": unresolved,
|
||
"stale": stale, "notified": notified}
|
||
|
||
|
||
@app.post("/admin/sweep")
|
||
@limiter.limit("20/minute")
|
||
async def admin_sweep(request: Request):
|
||
"""Manually trigger a full membership reconciliation.
|
||
|
||
Authenticated by the X-Admin-Token header. Idempotent — safe to call
|
||
repeatedly. This is also what the Cloudron scheduler invokes on a cron
|
||
schedule (see the sweep script).
|
||
"""
|
||
_verify_admin_token(request)
|
||
|
||
result = await reconcile_memberships()
|
||
|
||
# Budget notices ride the same cron the Cloudron scheduler already hits. They are OFF
|
||
# unless BUDGET_NOTICES=on, and wrapped so a failure here can never break reconciliation.
|
||
try:
|
||
result["budget_notices"] = await budget_notice_sweep(send=(budget_notices_enabled() == "on"))
|
||
except Exception as exc: # noqa: BLE001
|
||
result["budget_notices"] = {"error": str(exc)[:200]}
|
||
|
||
# Payment-driven allowance renewal (join-day anchored). Rides the same sweep.
|
||
# REPORT-ONLY unless ALLOWANCE_POLICY=on; deactivation additionally needs
|
||
# ALLOWANCE_DEACTIVATE=on. Wrapped so it can never break reconciliation.
|
||
try:
|
||
result["allowances"] = await reconcile_allowances(send=(ALLOWANCE_POLICY == "on"))
|
||
except Exception as exc: # noqa: BLE001
|
||
result["allowances"] = {"error": str(exc)[:200]}
|
||
|
||
# Allowance-shortfall credits: OC's billing-day alignment truncates some paid
|
||
# cycles. Rides the same sweep so a new member is compensated automatically;
|
||
# credits are idempotent on the OC charge reference. Wrapped so it can never
|
||
# break reconciliation; notices only go out when ALLOWANCE_SHORTFALL_EMAILS=on.
|
||
try:
|
||
result["shortfall_credits"] = await reconcile_allowance_shortfalls(
|
||
send=(ALLOWANCE_POLICY == "on"))
|
||
except Exception as exc: # noqa: BLE001
|
||
result["shortfall_credits"] = {"error": str(exc)[:200]}
|
||
|
||
# Paid credit packs with no record in our books (issue #7). REPORT-ONLY
|
||
# unless CREDIT_RECONCILE=on; wrapped so it can never break reconciliation.
|
||
try:
|
||
result["credit_reconcile"] = await reconcile_credit_packs(
|
||
apply=(CREDIT_RECONCILE == "on"))
|
||
except Exception as exc: # noqa: BLE001
|
||
result["credit_reconcile"] = {"error": str(exc)[:200]}
|
||
|
||
# Held credit packs: grant any whose buyer is now an active member, and report
|
||
# the ones nobody can be credited for. Wrapped so it can never break the sweep.
|
||
try:
|
||
result["held_credits"] = await settle_held_credits(send=True)
|
||
except Exception as exc: # noqa: BLE001
|
||
result["held_credits"] = {"error": str(exc)[:200]}
|
||
|
||
# Paid Open Collective contributors with no portal row — the net for members
|
||
# the sweep cannot see (hidden email, org account). Emails info@ only when it
|
||
# finds something, and only when the set changes. Was a daily script on the
|
||
# bot's host.
|
||
try:
|
||
result["unmapped_backers"] = await reconcile_unmapped_backers()
|
||
except Exception as exc: # noqa: BLE001
|
||
result["unmapped_backers"] = {"error": str(exc)[:200]}
|
||
|
||
# Spend consistency: the portal's SpendLogs aggregate vs LiteLLM's own API.
|
||
# Silent when they agree; the reason it exists is that the portal's copy of the
|
||
# LiteLLM DB credential once went stale and every surface showed $0 while looking
|
||
# clean. Was a daily script on the bot's host.
|
||
try:
|
||
result["spend_check"] = await check_spend_consistency()
|
||
except Exception as exc: # noqa: BLE001
|
||
result["spend_check"] = {"error": str(exc)[:200]}
|
||
|
||
# Deliveries the portal refused. The portal records these itself (see
|
||
# record_webhook_failure) and reports them here ONCE per batch, so the alert
|
||
# for a failed delivery belongs to the system rather than to a log scraper on
|
||
# somebody's laptop. Wrapped so it can never break reconciliation.
|
||
try:
|
||
result["webhook_failures"] = webhook_failure_report()
|
||
except Exception as exc: # noqa: BLE001
|
||
result["webhook_failures"] = {"error": str(exc)[:200]}
|
||
|
||
return result
|
||
|
||
|
||
@app.post("/admin/credit-pack-reconcile")
|
||
@limiter.limit("20/minute")
|
||
async def admin_credit_pack_reconcile(request: Request, apply: int = 0, notify: int = 1):
|
||
"""Reconcile PAID credit packs against our books — DRY RUN by default.
|
||
|
||
Authenticated by the X-Admin-Token header, like the other /admin/* routes.
|
||
`?apply=1` asks for the grants to be applied, and is honoured only when
|
||
CREDIT_RECONCILE=on: the point of the default is that an operator can read
|
||
this pass's output in production before any member's money is touched. Every
|
||
grant uses the same per-order reference the webhook writes, so running it
|
||
twice — or running it after a late delivery — credits a purchase once.
|
||
|
||
`?notify=0` suppresses the info@ notice (for a check that should not mail).
|
||
"""
|
||
_verify_admin_token(request)
|
||
return await reconcile_credit_packs(apply=bool(apply), notify=bool(notify))
|
||
|
||
|
||
@app.post("/admin/unmapped-check")
|
||
@limiter.limit("20/minute")
|
||
async def admin_unmapped_check(request: Request, notify: int = 1):
|
||
"""Run the paid-but-unprovisioned check on demand (X-Admin-Token).
|
||
|
||
Emits info@ only when there are findings AND the set has changed, so this is
|
||
safe to call repeatedly; `?notify=0` suppresses the notice entirely.
|
||
"""
|
||
_verify_admin_token(request)
|
||
return await reconcile_unmapped_backers(notify=bool(notify))
|
||
|
||
|
||
@app.post("/admin/spend-check")
|
||
@limiter.limit("20/minute")
|
||
async def admin_spend_check(request: Request, notify: int = 1):
|
||
"""Compare the portal's spend aggregate with LiteLLM's own API (X-Admin-Token).
|
||
|
||
Silent when they agree; `?notify=0` runs it without any chance of a notice.
|
||
"""
|
||
_verify_admin_token(request)
|
||
return await check_spend_consistency(notify=bool(notify))
|
||
|
||
|
||
def _service_sync_pending() -> list[str]:
|
||
"""Active members still waiting on their Loomio invitation.
|
||
|
||
Cheap by design (one indexed SELECT, no network): it is what lets the
|
||
scheduled service-sync be a no-op in the steady state.
|
||
|
||
WHY LOOMIO IS THE CLOCK: a member is stamped `loomio_invited_at` only once
|
||
the invite POST has come back 200 — so this list is non-empty for EVERY
|
||
member who has been provisioned but not yet finished onboarding, which is
|
||
exactly the window in which their Matrix invites also have to fire. That
|
||
makes it the honest trigger for running the whole service sync, and it
|
||
covers the failure this endpoint exists for: an account activated between
|
||
two sweeps. GUEST accounts are excluded — they are never invited to Loomio
|
||
or Matrix.
|
||
"""
|
||
conn = get_db()
|
||
rows = conn.execute(
|
||
"SELECT email FROM members "
|
||
"WHERE active = 1 AND (account_type IS NULL OR account_type != 'guest') "
|
||
"AND loomio_invited_at IS NULL"
|
||
).fetchall()
|
||
conn.close()
|
||
return [r[0] for r in rows]
|
||
|
||
|
||
@app.post("/admin/service-sync")
|
||
@limiter.limit("20/minute")
|
||
async def admin_service_sync(request: Request):
|
||
"""Run the Loomio + Matrix invite syncs — the scheduler's frequent tick.
|
||
|
||
WHY THIS EXISTS: provisioning runs both syncs, but at provisioning time a
|
||
new member's Cloudron account is usually not ACTIVATED yet, and the syncs
|
||
deliberately skip an unactivated account (never invite a half-created one).
|
||
The activation happens later, when the member follows their setup link —
|
||
and at that moment nothing on our side is watching. Waiting for the 3x/day
|
||
sweep left a new member without forum or Space access for hours, which
|
||
reads as "the co-op is broken".
|
||
|
||
So this runs every few minutes: it fires the same idempotent syncs the
|
||
sweep runs, ONLY while some member is still awaiting their invites, and
|
||
costs one SELECT when nobody is. Together with the first-login hook on the
|
||
dashboard path, a member who has just set up their account waits minutes
|
||
at most rather than until the next sweep.
|
||
|
||
Authenticated by the X-Admin-Token header (same as /admin/sweep). Never
|
||
raises out to the caller on a sync failure: the syncs are individually
|
||
best-effort and the next tick retries.
|
||
"""
|
||
global _service_sync_inflight
|
||
_verify_admin_token(request)
|
||
|
||
if _service_sync_inflight:
|
||
return {"status": "busy"}
|
||
|
||
pending = _service_sync_pending()
|
||
if not pending:
|
||
return {"status": "idle", "pending": 0}
|
||
|
||
_service_sync_inflight = True
|
||
result: dict = {"status": "ran", "pending": len(pending)}
|
||
try:
|
||
try:
|
||
await sync_loomio_memberships()
|
||
result["loomio"] = "ok"
|
||
except Exception as e: # noqa: BLE001
|
||
result["loomio"] = str(e)[:200]
|
||
logger.warning("service-sync: Loomio sync failed: %s", e)
|
||
try:
|
||
result["matrix"] = await sync_matrix_space()
|
||
except Exception as e: # noqa: BLE001
|
||
result["matrix"] = str(e)[:200]
|
||
logger.warning("service-sync: Matrix sync failed: %s", e)
|
||
finally:
|
||
_service_sync_inflight = False
|
||
|
||
result["pending_after"] = len(_service_sync_pending())
|
||
return result
|
||
|
||
|
||
@app.post("/admin/newsletter/reconcile")
|
||
@limiter.limit("20/minute")
|
||
async def admin_newsletter_reconcile(request: Request):
|
||
"""Remove stale subscribers from the Members newsletter list.
|
||
|
||
The newsletter list must mirror ACTIVE members: deleted accounts,
|
||
never-activated test accounts, and lapsed members are removed (see
|
||
listmonk_reconcile for the exact rules). Also runs automatically as
|
||
step 4 of the nightly membership sweep.
|
||
|
||
Authenticated by the X-Admin-Token header.
|
||
"""
|
||
_verify_admin_token(request)
|
||
return await listmonk_reconcile()
|
||
|
||
|
||
def _verify_sync_token(request: Request) -> None:
|
||
"""Authenticate the LibreChat key-sync feed via X-Sync-Token.
|
||
|
||
Dedicated machine secret (least privilege): it only ever exposes the
|
||
members' own chat keys, never the admin API. The same secret is set on
|
||
the LibreChat sandbox app (LIBRECHAT_SYNC_TOKEN in /app/data/env).
|
||
"""
|
||
expected = os.environ.get("LIBRECHAT_SYNC_TOKEN", "")
|
||
if not expected:
|
||
raise HTTPException(503, "Sync token not configured")
|
||
provided = request.headers.get("x-sync-token", "")
|
||
if not secrets.compare_digest(provided, expected):
|
||
raise HTTPException(401, "Invalid sync token")
|
||
|
||
|
||
@app.get("/admin/librechat/keys")
|
||
@limiter.limit("30/minute")
|
||
async def admin_librechat_keys(request: Request):
|
||
"""Key feed for the LibreChat sandbox's per-member key auto-load.
|
||
|
||
Returns every ACTIVE member's portal-provisioned chat key so the sandbox
|
||
can seed LibreChat's user-key store and members never see an API-key
|
||
prompt (chat bills to the member's own LiteLLM team/budget). Auth is the
|
||
dedicated X-Sync-Token header — NOT the admin token.
|
||
|
||
DELIBERATE DESIGN (audit M2, 2026-11-05): one static token returns every
|
||
member's decrypted chat key — a total-member-key blast radius. Kept (not
|
||
per-member fetch) because the only consumer is the LibreChat sandbox's
|
||
in-app key-sync, and scoping the feed per member would require changing
|
||
the sandbox's sync identity model; a wrong change there stops key seeding
|
||
for every member at once. Recorded in ops-docs
|
||
(maintenance/librechat-key-feed.md). Monitoring: every fetch is logged
|
||
(count only, never key material) and the daily digest watches the rate —
|
||
the sandbox polls ~every 10 min, so anything far above that cadence (or a
|
||
401 burst on this route) is surfaced as an anomaly.
|
||
"""
|
||
_verify_sync_token(request)
|
||
|
||
conn = get_db()
|
||
rows = conn.execute(
|
||
"SELECT email, key_token FROM members WHERE active = 1 AND key_token != ''"
|
||
).fetchall()
|
||
conn.close()
|
||
logger.warning(
|
||
"librechat key feed served: %d member key(s) to %s (ua=%s)",
|
||
len(rows), request.client.host if request.client else "?",
|
||
(request.headers.get("user-agent") or "?")[:60],
|
||
)
|
||
return {
|
||
"members": [
|
||
{"email": email, "key": decrypt_key(key_token)}
|
||
for email, key_token in rows
|
||
]
|
||
}
|
||
|
||
|
||
@app.get("/")
|
||
async def index():
|
||
return {"service": "Inference Cooperative Member Portal", "version": "0.1.0"}
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Budget notices — email a member as they approach their monthly limit. #
|
||
# --------------------------------------------------------------------------- #
|
||
# OFF by default: BUDGET_NOTICES = off | dryrun | on. Nothing is ever emailed
|
||
# unless it is "on"; the admin endpoint below can preview at any time, including
|
||
# while off. One notice per level per budget cycle, so a member hears from us
|
||
# once when they cross 80% and once when they reach the cap — never on every
|
||
# sweep. Spend and budget are read from LiteLLM's own Postgres, the same source
|
||
# the dashboard and admin panel display, so the numbers cannot drift.
|
||
#
|
||
# CREDIT_PACK_URL is configuration because the website has no dedicated pack
|
||
# link today; the Open Collective contribution page is what actually credits a
|
||
# member (the webhook settles contributions into credits).
|
||
|
||
# Generic on purpose: the balance can be short because the monthly allowance is spent,
|
||
# because purchased credit is spent, or a mix — the letter says "balance" so it is true
|
||
# either way. The trigger is proportional — 90% of the balance used — so it means the same
|
||
# thing to a member on $15 and one on $35: the letter arrives when roughly a tenth is left,
|
||
# on whichever sweep follows, rather than at an arbitrary absolute amount.
|
||
BUDGET_NOTICE_AT = float(os.environ.get("BUDGET_NOTICE_AT", "0.90"))
|
||
|
||
PINE_LOGO_SVG = """<svg width="56" height="56" viewBox="0 0 130 130" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true">
|
||
<circle cx="65" cy="14" r="5" fill="#5b8c5a"/>
|
||
<circle cx="58" cy="30" r="6" fill="#7ab87a"/>
|
||
<circle cx="74" cy="30" r="4" fill="#6ba86b"/>
|
||
<circle cx="51.5" cy="46" r="6.5" fill="#5b8c5a"/>
|
||
<circle cx="65" cy="46" r="5" fill="#7ab87a"/>
|
||
<circle cx="81" cy="46" r="4" fill="#6ba86b"/>
|
||
<circle cx="44" cy="62" r="7" fill="#6ba86b"/>
|
||
<circle cx="61" cy="62" r="5.5" fill="#5b8c5a"/>
|
||
<circle cx="76" cy="62" r="4.5" fill="#7ab87a"/>
|
||
<circle cx="89.5" cy="62" r="3.5" fill="#6ba86b"/>
|
||
<circle cx="36.5" cy="78" r="7.5" fill="#5b8c5a"/>
|
||
<circle cx="54" cy="78" r="6" fill="#7ab87a"/>
|
||
<circle cx="70" cy="78" r="5" fill="#6ba86b"/>
|
||
<circle cx="84" cy="78" r="4" fill="#5b8c5a"/>
|
||
<circle cx="97.5" cy="78" r="3.5" fill="#7ab87a"/>
|
||
<circle cx="44" cy="94" r="6.5" fill="#6ba86b"/>
|
||
<circle cx="61" cy="94" r="5.5" fill="#5b8c5a"/>
|
||
<circle cx="76" cy="94" r="4.5" fill="#7ab87a"/>
|
||
<circle cx="89.5" cy="94" r="3.5" fill="#6ba86b"/>
|
||
<circle cx="51.5" cy="110" r="5.5" fill="#5b8c5a"/>
|
||
<circle cx="65" cy="110" r="4.5" fill="#7ab87a"/>
|
||
<circle cx="81" cy="110" r="3.5" fill="#6ba86b"/>
|
||
</svg>"""
|
||
|
||
|
||
def budget_notices_enabled() -> str:
|
||
"""BUDGET_NOTICES: off (default) | dryrun | on."""
|
||
return (os.environ.get("BUDGET_NOTICES", "off") or "off").strip().lower()
|
||
|
||
|
||
def credit_pack_url() -> str:
|
||
return os.environ.get(
|
||
"CREDIT_PACK_URL", "https://opencollective.com/inference-cooperative/contribute"
|
||
)
|
||
|
||
|
||
def _ensure_budget_notices_table(conn) -> None:
|
||
conn.execute(
|
||
"CREATE TABLE IF NOT EXISTS budget_notices ("
|
||
" email TEXT NOT NULL, level TEXT NOT NULL, cycle TEXT NOT NULL,"
|
||
" sent_at TEXT NOT NULL, detail TEXT,"
|
||
" PRIMARY KEY (email, level, cycle))"
|
||
)
|
||
conn.commit()
|
||
|
||
|
||
def budget_notice_email(
|
||
name: str, spend: float, budget: float, remaining: float, pack_url: str
|
||
) -> tuple[str, str, str]:
|
||
"""Build the low-balance notice (subject, text, html).
|
||
|
||
Deliberately generic about where the balance came from: a member may be short
|
||
because the monthly allowance is spent, because credit they bought is spent, or
|
||
a mix of both. The wording says "balance" so it is true either way, and it names
|
||
no mechanism a member would have to reconcile against their own dashboard.
|
||
|
||
Same shell as welcome_email(): cream background, white card, forest-green
|
||
button, inline pine-tree mark, table-based layout, no external assets.
|
||
"""
|
||
subject = "Your Inference Cooperative balance is running low"
|
||
heading = "Your balance is running low"
|
||
lede = (f"You have ${remaining:.2f} of your ${budget:.2f} balance left. When it "
|
||
"runs out, chat and API requests stop until you add more.")
|
||
|
||
money = f"${remaining:.2f} remaining"
|
||
text = (
|
||
f"Hi {name},\n\n"
|
||
f"{lede}\n\n"
|
||
f"{money}\n\n"
|
||
"If you'd like to keep going, you can buy credit packs here:\n\n"
|
||
f"{pack_url}\n\n"
|
||
"This is the only notice of its kind you'll get until your balance goes back "
|
||
"above the line and runs low again. Questions? Reply to this email or write to "
|
||
"info@inference.coop.\n\n"
|
||
"— The Inference Cooperative\n"
|
||
)
|
||
html = f"""<!DOCTYPE html>
|
||
<html lang="en">
|
||
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"></head>
|
||
<body style="margin:0;padding:0;background:#faf8f5;font-family:-apple-system,'Segoe UI',Roboto,Helvetica,Arial,sans-serif;">
|
||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="background:#faf8f5;padding:40px 16px;">
|
||
<tr>
|
||
<td align="center">
|
||
<table role="presentation" width="100%" cellpadding="0" cellspacing="0" style="max-width:520px;">
|
||
<tr>
|
||
<td align="center" style="padding-bottom:24px;">
|
||
{PINE_LOGO_SVG}
|
||
</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="center" style="background:#ffffff;border:1px solid #e8e2d8;border-radius:16px;padding:40px 32px;">
|
||
<h1 style="margin:0 0 12px;font-size:24px;font-weight:700;color:#2d3327;">{heading}</h1>
|
||
<p style="margin:0 0 8px;font-size:16px;color:#6b7a62;line-height:1.5;">{lede}</p>
|
||
<p style="margin:0 0 24px;font-size:15px;color:#2d3327;line-height:1.5;"><strong>{money}</strong></p>
|
||
<a href="{pack_url}" style="display:inline-block;background:#5b8c5a;color:#ffffff;text-decoration:none;padding:14px 36px;border-radius:999px;font-size:16px;font-weight:600;">Buy credit packs</a>
|
||
<p style="margin:28px 0 0;font-size:13px;color:#94a08c;line-height:1.5;">This is the only notice of its kind you'll get until your balance goes back above the line and runs low again.</p>
|
||
</td>
|
||
</tr>
|
||
<tr>
|
||
<td align="center" style="padding-top:24px;font-size:13px;color:#94a08c;line-height:1.6;">
|
||
Questions? Reply to this email or write to <a href="mailto:info@inference.coop" style="color:#6b7a62;">info@inference.coop</a>.<br>
|
||
<span style="color:#2d3327;">The Inference Cooperative</span> · inference.coop
|
||
</td>
|
||
</tr>
|
||
</table>
|
||
</td>
|
||
</tr>
|
||
</table>
|
||
</body>
|
||
</html>"""
|
||
return subject, text, html
|
||
|
||
|
||
async def budget_notice_sweep(send: bool = False) -> dict:
|
||
"""Find members whose balance has fallen below the low-water mark, and optionally email.
|
||
|
||
`send=False` is a preview: it computes exactly who would be written to, with the
|
||
numbers, and sends nothing. `send=True` emails and records the send — the caller
|
||
is responsible for requiring BUDGET_NOTICES=on first.
|
||
"""
|
||
mode = budget_notices_enabled()
|
||
if send and mode != "on":
|
||
return {"refused": f"BUDGET_NOTICES is '{mode}', not 'on' — nothing sent"}
|
||
|
||
conn = get_db()
|
||
_ensure_budget_notices_table(conn)
|
||
cols = {r[1] for r in conn.execute("PRAGMA table_info(members)")}
|
||
# There is no name column in `members` — `slug` is the username (e.g. ntnsndr). Falling
|
||
# back to the email local part produced "Hi n," for the operator, which is how this was
|
||
# caught. Prefer the username; say "there" rather than inventing a name.
|
||
ident = "slug" if "slug" in cols else None
|
||
sel = ("SELECT email" + (f", {ident}" if ident else "") + ", account_type"
|
||
" FROM members WHERE active = 1")
|
||
rows = conn.execute(sel).fetchall()
|
||
|
||
pack = credit_pack_url()
|
||
preview, sent, skipped = [], 0, 0
|
||
for row in rows:
|
||
email = row[0]
|
||
name = ((row[1] or "").strip() if len(row) > 1 else "") or "there"
|
||
acct_type = (row[2] or "member") if len(row) > 2 else "member"
|
||
if acct_type == "guest":
|
||
# A guest's exhaustion is handled by the guest lifecycle pass (the
|
||
# sweep deactivates it); member low-balance mail about buying
|
||
# credit packs is not for guests.
|
||
skipped += 1
|
||
continue
|
||
team_id = await litellm_team_id_for_email(email)
|
||
if not team_id:
|
||
skipped += 1
|
||
continue
|
||
stats = get_team_budget_from_db(team_id) or {}
|
||
budget = stats.get("max_budget")
|
||
spend = float(stats.get("spend") or 0.0)
|
||
if not budget or budget <= 0:
|
||
skipped += 1 # no cap configured: nothing to warn about
|
||
continue
|
||
|
||
remaining = float(budget) - spend
|
||
if spend < BUDGET_NOTICE_AT * float(budget):
|
||
# Above the line — they added funds, or the allowance reset. Clear the latch so the
|
||
# NEXT crossing sends a fresh notice. That is what makes it once each time, and it
|
||
# is why this is a latch rather than a key on the budget cycle.
|
||
conn.execute("DELETE FROM budget_notices WHERE email = ?", (email,))
|
||
conn.commit()
|
||
continue
|
||
if conn.execute("SELECT 1 FROM budget_notices WHERE email = ?", (email,)).fetchone():
|
||
skipped += 1 # already told them for this crossing
|
||
continue
|
||
|
||
subject, text_body, html_body = budget_notice_email(name, spend, budget, remaining, pack)
|
||
entry = {
|
||
"email": email, "remaining": round(remaining, 2),
|
||
"spend": round(spend, 2), "budget": round(budget, 2),
|
||
"resets": str(stats.get("budget_reset_at") or stats.get("reset_at") or "")[:19],
|
||
}
|
||
if send:
|
||
ok = send_email(email, subject, text_body, html_body)
|
||
entry["sent"] = bool(ok)
|
||
if ok:
|
||
conn.execute(
|
||
"INSERT OR REPLACE INTO budget_notices"
|
||
" (email, level, cycle, sent_at, detail) VALUES (?, 'low', ?, datetime('now'), ?)",
|
||
(email, entry["resets"] or "crossing", f"remaining={remaining:.4f}"),
|
||
)
|
||
conn.commit()
|
||
sent += 1
|
||
else:
|
||
entry["subject"] = subject
|
||
preview.append(entry)
|
||
conn.close()
|
||
if send:
|
||
return {"mode": mode, "sent": sent, "considered": len(rows),
|
||
"skipped": skipped, "notices": preview}
|
||
return {"mode": mode, "dry_run": True, "considered": len(rows), "skipped": skipped,
|
||
"would_notify": len(preview), "notices": preview}
|
||
|
||
|
||
async def budget_notice_test(dest: str) -> dict:
|
||
"""Email one sample low-balance letter to `dest`, using that member's real numbers.
|
||
|
||
Deliberately does NOT write to budget_notices: a test must not consume the real
|
||
once-per-crossing latch, or reviewing the copy would silence the actual notice.
|
||
Works while BUDGET_NOTICES is off, so copy can be reviewed before arming anything.
|
||
"""
|
||
conn = get_db()
|
||
row = conn.execute("SELECT slug FROM members WHERE email = ?", (dest,)).fetchone()
|
||
name = ((row[0] or "").strip() if row else "") or "there"
|
||
conn.close()
|
||
|
||
budget, spend = 15.0, 13.34 # sample numbers if they aren't a member
|
||
if row:
|
||
team_id = await litellm_team_id_for_email(dest)
|
||
stats = (get_team_budget_from_db(team_id) or {}) if team_id else {}
|
||
if stats.get("max_budget"):
|
||
budget = float(stats["max_budget"])
|
||
spend = float(stats.get("spend") or 0.0)
|
||
|
||
remaining = budget - spend
|
||
subject, text_body, html_body = budget_notice_email(
|
||
name, spend, budget, remaining, credit_pack_url()
|
||
)
|
||
ok = send_email(dest, subject, text_body, html_body)
|
||
return {
|
||
"mode": "test", "to": dest, "sent": bool(ok), "subject": subject,
|
||
"rendered": {"name": name, "spend": round(spend, 2), "budget": round(budget, 2),
|
||
"remaining": round(remaining, 2)},
|
||
"note": "no ledger row written — the real once-per-crossing latch is untouched",
|
||
}
|
||
|
||
|
||
@app.post("/admin/budget-notices")
|
||
@limiter.limit("10/minute")
|
||
async def admin_budget_notices(request: Request, mode: str = "dryrun", to: str | None = None):
|
||
"""Preview (default), send, or email a single sample.
|
||
|
||
Authenticated by X-Admin-Token. `mode=dryrun` never emails; `mode=send` requires
|
||
BUDGET_NOTICES=on and refuses otherwise; `mode=test&to=<address>` emails one sample
|
||
using that member's real numbers so the copy can be read in a mail client first.
|
||
A test works while the flag is off and leaves no trace in the ledger.
|
||
"""
|
||
_verify_admin_token(request)
|
||
if mode == "test":
|
||
dest = (to or "").strip()
|
||
if not dest:
|
||
return {"refused": "mode=test needs &to=<address>"}
|
||
return await budget_notice_test(dest)
|
||
if mode == "send":
|
||
if budget_notices_enabled() != "on":
|
||
return {"refused": f"BUDGET_NOTICES is '{budget_notices_enabled()}', not 'on'"
|
||
" — set it on the portal app first"}
|
||
return await budget_notice_sweep(send=True)
|
||
return await budget_notice_sweep(send=False)
|