From 43f8f8c3fe10044c5106119db47b3bcbf1e5932b Mon Sep 17 00:00:00 2001 From: inferencebot Date: Wed, 4 Nov 2026 14:36:57 +0000 Subject: [PATCH] operator-token-runbook: add the credential rotation policy and reminder Records the 12-month cadence and the trigger rules agreed in co-op/support#35, a last-rotated table (first entry: the Open Collective token, rotated 2026-10-27; other rows blank because nothing is recorded), and the account map behind the credentials. Adds scripts/rotation_due.py, which reads the table and prints only when a credential is within 30 days of its due date. --- operator-token-runbook.md | 83 +++++++++++++++++++- scripts/rotation_due.py | 156 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 237 insertions(+), 2 deletions(-) create mode 100755 scripts/rotation_due.py diff --git a/operator-token-runbook.md b/operator-token-runbook.md index 2ebf7a3..fa438b7 100644 --- a/operator-token-runbook.md +++ b/operator-token-runbook.md @@ -3,6 +3,85 @@ How the co-op's Cloudron credentials are organized, and how to rotate them without breaking the stack. +## Credential rotation policy + +**Cadence: 12 months for every credential.** One interval, no exceptions for now. +Twelve months is a starting point, not a principle — revisit it once the token +list is stable (co-op/support#35). + +**Rotate on a trigger immediately, off the calendar, when any of these happens:** + +- **Offboarding** — anyone who had access to the credential leaves or changes + role, or the device/account it lived on is lost. +- **Suspected exposure** — the value was seen somewhere it should not be: a log, + a chat, a commit, a screenshot. +- **After any incident, or any diagnostic session on the agent host in which a + secret could have been read.** Reading a live secret to debug with it is itself + the trigger; it does not have to be a confirmed breach. +- **After any restore** — the credential travels in the backup, so a restored + host re-materialises whatever values the snapshot held. + +**The GM controls all tokens for now** (operator, 2026-11-04), so this record +needs **no owner column**: every row is rotated by the GM. Provider keys are +minted on the provider account, but the value held in the co-op's environment is +still the GM's to rotate. + +### Last rotated + +One row per credential, or per credential group — a group rotates together, so it +dates together. A blank cell means **no rotation has been recorded**, not "not +rotated". Dates are UTC, `YYYY-MM-DD`. + +The reminder reads this exact table: `scripts/rotation_due.py` in this repository +(deployed on the agent host at `~/.hermes/scripts/rotation_due.py`) prints a line +only when a due date is within 30 days, and prints nothing otherwise. Keep the +table machine-legible — one `YYYY-MM-DD` date per row. + +| Credential or group | Last rotated (UTC) | +|---|---| +| Open Collective token (`OC_PERSONAL_TOKEN`) | 2026-10-27 | +| `LITELLM_MASTER_KEY` / `GW_MASTER_KEY` (one value, four holders) | | +| Cloudron API tokens — portal + agent (both on `inference-bot`) | | +| `REGISTRY_BUILD_TOKEN` (on `inference-admin`; moves to CI, co-op/support#30) | | +| Gitea token (`inference-bot`) | | +| `WEBHOOK_TOKEN` (portal env + Open Collective webhook URL) | | +| `ADMIN_TOKEN` (portal + admin panel + agent host) | | +| `BROKER_SECRET` and `PORTAL_SECRET` | | +| Matrix tokens — portal `MATRIX_BOT_TOKEN` + agent `MATRIX_ADMIN_TOKEN` (one account, `@inference-admin`) | | +| Matrix token — steward agent `@inferencebot` | | +| Loomio API keys (`LOOMIO_API_KEY`, `LOOMIO_B3_KEY`) | | +| Listmonk API (`LISTMONK_API_USER`, `LISTMONK_API_TOKEN`) | | +| Provider keys — Tinfoil, GreenPT, Regolo, and the GreenPT fallback | | +| Restic repository passphrase; Infomaniak Swiss Backup S3 keys | | + +### Accounts behind the credentials + +A credential belongs to an **account**, which is what a rotation is actually +performed against — ask the provider, do not read the variable name. Two +credentials on one account are a single hazard: rotating one and not the other is +the silent break. + +| Credential | Account | Basis | +|---|---|---| +| `LITELLM_MASTER_KEY` / `GW_MASTER_KEY` | self-issued by the gateway | one literal, four holders | +| Cloudron API tokens — portal and agent | `inference-bot` | verified (co-op/support#35) | +| `REGISTRY_BUILD_TOKEN` | `inference-admin` | verified (co-op/support#35) | +| `MATRIX_BOT_TOKEN` (portal) + `MATRIX_ADMIN_TOKEN` (agent) | `@inference-admin:inference.coop` — two tokens, one account | verified, homeserver `whoami` 2026-11-04 | +| Steward agent's `MATRIX_ACCESS_TOKEN` | `@inferencebot:inference.coop` | verified, homeserver `whoami` 2026-11-04 | +| `OC_PERSONAL_TOKEN` | `info@inference.coop` | from the 2026-10-27 rotation record (inference) | +| `TINFOIL_API_KEY` | `info@inference.coop` | operator-confirmed | +| `GREENPT_API_KEY` (gateway) and `GREENPT_FALLBACK_API_KEY` (agent) | the GreenPT account — the **same value** in both | verified: the "fallback" isolates nothing | +| `REGOLO_API_KEY` | Regolo account | not yet identified | +| Gitea token | `inference-bot` | inferred, not verified | +| `BROKER_SECRET`, `PORTAL_SECRET`, `WEBHOOK_TOKEN`, `ADMIN_TOKEN`, `LOOMIO_*`, `LISTMONK_*` | self-issued / service account | — | +| Restic passphrase; Infomaniak Swiss Backup S3 keys | self-issued; the Infomaniak account | — | + +The two couplings to watch: **one account with several credentials** (the +`@inference-admin` Matrix pair, the two Cloudron tokens, the four holders of the +master key) and **one value with several holders** (the GreenPT key, which is the +only such case found). Full account map, including the hash comparison behind the +GreenPT finding, is held with the security working notes. + ## The two-token split There are **two separate** Cloudron API tokens, deliberately decoupled so @@ -202,7 +281,7 @@ row that names the same credential: several are held by more than one app. | `WEBHOOK_TOKEN` | Secret in the Open Collective webhook URL | Portal app env, and the webhook URL on Open Collective | GM | Rotating means re-creating the webhook on OC as well | | `LITELLM_MASTER_KEY` | Create gateway teams and keys for members | Portal app env (copy of the gateway's) | GM | Three holders: portal, gateway, steward agent. Rotate all together | | `LITELLM_DB_*` | Read spend logs from the gateway database | Portal app env | Cloudron (addon) | Postgres credentials for the gateway's database | -| `MATRIX_BOT_TOKEN` | `@inference-bot:inference.coop` admin account: invites, space membership | Portal app env | GM | The account has no password. Paired with the steward agent's `MATRIX_LOCAL_BOT_TOKEN`: same account, separate token on a separate device (verified). Revoke or rotate both together | +| `MATRIX_BOT_TOKEN` | `@inference-admin:inference.coop` admin account: invites, space membership | Portal app env | GM | The account has no password. Paired with the steward agent's `MATRIX_ADMIN_TOKEN`: same account, separate token on a separate device (verified by homeserver `whoami`, 2026-11-04). Revoke or rotate both together | | `LOOMIO_API_KEY`, `LOOMIO_B3_KEY` | Forum (Loomio) membership sync | Portal app env | GM | | | `LISTMONK_API_USER`, `LISTMONK_API_TOKEN` | Newsletter list sync | Portal app env | GM | | | `OWUI_API_KEY`, `OWUI_JWT_SECRET` | Open WebUI knowledge refresh and SSO | Portal app env | GM | Chat moved to LibreChat in October; confirm whether still needed, and retire if not | @@ -236,7 +315,7 @@ granularity for the public half. Highest privilege first. | Credential | Purpose | Where it lives | Who rotates | Notes | |---|---|---|---|---| | Gateway master key (`GW_MASTER_KEY`) | Admin over every gateway team, key and spend record | Steward agent `.env` | GM | **Third holder**, with the portal and the gateway. The row most in need of coordinated rotation | -| `MATRIX_LOCAL_BOT_TOKEN` | The `@inference-bot:inference.coop` automation account: room administration in every co-op room | Steward agent `.env` | GM | Same account as the portal's `MATRIX_BOT_TOKEN`, but a separate token on a separate device (verified). Revoking one does not revoke the other; revoke or rotate both together | +| `MATRIX_ADMIN_TOKEN` | The `@inference-admin:inference.coop` automation account: room administration in every co-op room | Steward agent `.env` | GM | Same account as the portal's `MATRIX_BOT_TOKEN`, but a separate token on a separate device (verified by homeserver `whoami`, 2026-11-04). Revoking one does not revoke the other; revoke or rotate both together | | Cloudron agent token | Cloudron API | Steward agent `.env` | GM | See "The two-token split" above | | `REGISTRY_BUILD_TOKEN` (and its username) | Image build and push for the custom apps | Steward agent `.env` | GM | Moves to CI (co-op/support#30); the agent stops holding it then | | Gitea token | Commits, issues, PRs as `inference-bot` | Steward agent `.env` | GM | | diff --git a/scripts/rotation_due.py b/scripts/rotation_due.py new file mode 100755 index 0000000..1f5b53f --- /dev/null +++ b/scripts/rotation_due.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Rotation-due watchdog for the co-op credential rotation policy. + +Reads the `last rotated` table from the operator token runbook and prints a +reminder ONLY when a credential is due -- i.e. within 30 days of its due date +(last rotation + 12 months), or already past it. Prints NOTHING when nothing is +due, so it can drive a watchdog cron where empty output means "no message". + +Deterministic: the output depends only on the table and the current date. It +prints no timestamps of its own, so when nothing is due it produces no output at +all and stays silent across runs. + +Table source (first that applies): + 1. --file PATH + 2. $ROTATION_RUNBOOK + 3. the runbook fetched from Gitea's main branch (authoritative, always current) + 4. the local clone ~/repos/ops-docs/operator-token-runbook.md + +Exit codes: 0 ok (including "nothing due", no output), 2 could not read the +runbook or found no dated rows -- a loud failure so a watcher is not silently +blind. +""" +import argparse +import calendar +import datetime as dt +import os +import re +import sys +import urllib.request + +RUNBOOK_URL = "https://git.inference.coop/co-op/ops-docs/raw/branch/main/operator-token-runbook.md" +LOCAL_FALLBACK = os.path.expanduser("~/repos/ops-docs/operator-token-runbook.md") +DEFAULT_WITHIN_DAYS = 30 +SECTION_RE = re.compile(r"^#{1,6}\s+.*last\s+rotated.*$", re.IGNORECASE) +HEADING_RE = re.compile(r"^#{1,6}\s+") +SEP_RE = re.compile(r"[\s|:\-]+") +DATE_RE = re.compile(r"\b(\d{4})-(\d{2})-(\d{2})\b") + + +def fetch_text(args): + if args.file: + try: + with open(args.file, encoding="utf-8") as fh: + return fh.read() + except OSError as exc: + print(f"rotation_due: could not read --file {args.file} ({exc})", file=sys.stderr) + raise SystemExit(2) + env = os.environ.get("ROTATION_RUNBOOK") + if env and os.path.exists(env): + try: + with open(env, encoding="utf-8") as fh: + return fh.read() + except OSError as exc: + print(f"rotation_due: could not read $ROTATION_RUNBOOK {env} ({exc})", file=sys.stderr) + raise SystemExit(2) + try: + req = urllib.request.Request(RUNBOOK_URL, headers={"User-Agent": "rotation_due/1.0"}) + with urllib.request.urlopen(req, timeout=20) as resp: + return resp.read().decode("utf-8") + except Exception as exc: # noqa: BLE001 - fall back, then fail loudly + if os.path.exists(LOCAL_FALLBACK): + with open(LOCAL_FALLBACK, encoding="utf-8") as fh: + return fh.read() + print( + f"rotation_due: could not read the runbook from {RUNBOOK_URL} ({exc}) " + f"and no local copy at {LOCAL_FALLBACK}", + file=sys.stderr, + ) + raise SystemExit(2) + + +def extract_section(text): + """Return the lines of the first 'last rotated' section.""" + out, inside = [], False + for line in text.splitlines(): + if SECTION_RE.match(line): + inside = True + continue + if inside and HEADING_RE.match(line): + break + if inside: + out.append(line) + return out + + +def parse_rows(section_lines): + """Yield (name, last_rotated_date) for table rows that carry a date.""" + for line in section_lines: + if not line.lstrip().startswith("|"): + continue + stripped = line.strip().strip("|") + if not stripped or SEP_RE.fullmatch(line.strip()): + continue + cells = [c.strip() for c in stripped.split("|")] + if not cells: + continue + match = DATE_RE.search(line) + if not match: + continue + year, month, day = (int(g) for g in match.groups()) + try: + last = dt.date(year, month, day) + except ValueError: + continue + yield cells[0], last + + +def add_months(date, months): + month_index = date.month - 1 + months + year = date.year + month_index // 12 + month = month_index % 12 + 1 + day = min(date.day, calendar.monthrange(year, month)[1]) + return dt.date(year, month, day) + + +def main(): + ap = argparse.ArgumentParser(description="Remind when a credential rotation is due.") + ap.add_argument("--file", help="read the runbook from this path instead of fetching it") + ap.add_argument("--within-days", type=int, default=DEFAULT_WITHIN_DAYS, + help="days before the due date to start reminding (default 30)") + ap.add_argument("--as-of", help="evaluate as of this date (YYYY-MM-DD); default today UTC") + args = ap.parse_args() + + section = extract_section(fetch_text(args)) + if not section: + print("rotation_due: no 'last rotated' section found in the runbook", file=sys.stderr) + return 2 + + parsed = list(parse_rows(section)) + if not parsed: + print("rotation_due: the 'last rotated' section has no rows carrying a date", file=sys.stderr) + return 2 + + as_of = dt.date.fromisoformat(args.as_of) if args.as_of else dt.datetime.now(dt.timezone.utc).date() + + due = [] + for name, last in parsed: + due_date = add_months(last, 12) + days = (due_date - as_of).days + if days <= args.within_days: + due.append((days, name, due_date)) + + if not due: + return 0 # nothing due -> no output at all + + due.sort() + for days, name, due_date in due: + if days < 0: + print(f"credential rotation OVERDUE: {name} -- was due {due_date.isoformat()} ({-days} days ago)") + else: + print(f"credential rotation due soon: {name} -- due {due_date.isoformat()} (in {days} days)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) -- 2.53.0