diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f53a9e9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +# Generated by the documentation build (see publishing/build.sh) +.site-src/ +.site-build/ +.venv-docs/ diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..fdda977 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,118 @@ +# MkDocs configuration for the member-readable documentation site. +# +# The documentation source stays exactly where it is — Markdown files at the +# repository root (plus maintenance/ and incidents/). This is a *publishing* +# layer over that source, not an edit of it. docs_dir is the repo root, so no +# document has to move for the generator to find it. +# +# Build: see publishing/build.sh +# Deploy: see publishing/deploy.sh +# +# Constraints baked in here: +# * theme.font: false -> Material must NOT pull Google Fonts. +# * fonts are self-hosted (publishing/fonts/) and OFL-licensed (Figtree). +# * no CDN / third-party host appears anywhere in the output. + +site_name: Inference Cooperative +site_description: >- + Member-readable documentation for the Inference Cooperative: how to join and + use the service, the model catalogue, API access, governance and policies. +site_url: https://docs.inference.coop/ +# MkDocs requires the docs directory to be a *child* of the config's directory +# and refuses to read the repository root directly. So the build stages the +# Markdown into .site-src/ (a generated, gitignored mirror of the documents) +# and renders that. No document is moved or rewritten in the repository — the +# staging copy exists only during the build. +docs_dir: .site-src +# Build output. +site_dir: .site-build +use_directory_urls: true + +theme: + name: material + language: en + # Disable the theme's built-in Google Fonts loader. Without this, Material + # requests Roboto from fonts.googleapis.com / fonts.gstatic.com. + font: false + logo: publishing/logo.svg + favicon: publishing/logo.svg + palette: + # Light is listed first, so a fresh visitor gets the bright theme. + - media: "(prefers-color-scheme: light)" + scheme: default + primary: teal + accent: deep orange + toggle: + icon: material/weather-night + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + primary: teal + accent: deep orange + toggle: + icon: material/weather-sunny + name: Switch to light mode + features: + - navigation.tabs + - navigation.sections + - navigation.top + - navigation.indexes + - search.suggest + - search.highlight + - content.code.copy + - toc.integrate + +nav: + - Home: README.md + - Getting started: getting-started.md + - Models: models.md + - API access: api-access.md + - Energy & carbon: energy.md + - Infrastructure: infrastructure.md + - Background: background.md + - Governance: + - Charter: charter.md + - Code of conduct: code-of-conduct.md + - Open questions: open-questions.md + - Terms of Service: terms-of-service.md + - Privacy Policy: privacy-policy.md + - Operator runbook: operator-token-runbook.md + - Maintenance: + - Chat task models: maintenance/chat-task-models.md + - LibreChat sandbox: maintenance/librechat-sandbox.md + - Incidents: + - 2026-10-02 gateway CVE upgrade: incidents/2026-10-02-gateway-cve-upgrade.md + +plugins: + # Built-in, offline search — the index is a JSON file in the build output. + - search + +markdown_extensions: + - admonition + - attr_list + - md_in_html + - tables + - toc: + permalink: true + - pymdownx.details + - pymdownx.superfences + - pymdownx.tasklist: + custom_checkbox: true + +extra_css: + # Co-op brand: palette from the website + self-hosted Figtree (SIL OFL 1.1). + - publishing/brand.css + +# Keep build/tooling files out of the rendered site. +exclude_docs: | + .git/ + *.sh + publishing/README.md + publishing/requirements.txt + +# Links to git.inference.coop (e.g. code-of-conduct.md in charter.md) are +# intentional absolute URLs; do not warn about them. +validation: + links: + absolute_links: ignore + unrecognized_links: ignore diff --git a/publishing/README.md b/publishing/README.md new file mode 100644 index 0000000..131f5df --- /dev/null +++ b/publishing/README.md @@ -0,0 +1,69 @@ +# Publishing the documentation site + +The Markdown in this repository is the single source of truth. This directory +is the publishing layer that turns it into a member-readable static site and +puts it on the co-op's Surfer app. Documents are never edited here — a change +to a document is a normal commit or pull request against the Markdown at the +repository root. + +## What is built + +| File | Purpose | +|---|---| +| `../mkdocs.yml` | Site configuration (MkDocs + Material for MkDocs) | +| `brand.css` | Co-op palette + self-hosted Figtree webfont (SIL OFL 1.1) | +| `fonts/` | Figtree woff2 files and the OFL licence text | +| `logo.svg` | Co-op logo (shared with the website repo) | +| `build.sh` | Stage the Markdown into `.site-src/`, render to `.site-build/` | +| `deploy.sh` | Push the build into the Surfer app container | +| `webhook-receiver.py` | Gitea webhook → pull → build → deploy (agent host) | +| `webhook-receiver.service` | systemd unit for the receiver | +| `requirements.txt` | Pinned Python build toolchain | + +Nothing external is loaded by the site: no CDN, no third-party font service. +Fonts are served from this repository's own output. + +## Build + +```sh +publishing/build.sh # creates .venv-docs on first run +``` + +Output lands in `.site-build/` (gitignored). `mkdocs build --strict` is what +CI-style runs should use, so broken links fail the build. + +## Deploy + +```sh +CLOUDRON_TOKEN=... publishing/deploy.sh +``` + +Writes the build into the Surfer app (Cloudron app `inference.coop`) at: + +- `/app/data/public-docs/` — served at **https://docs.inference.coop/** +- `/app/data/public/docs/` — served at **https://inference.coop/docs/** + +Both are written so the site stays reachable either way; the hostname is the +canonical one. + +## Automatic rebuilds (push to main) + +A push to `main` on `co-op/docs` fires a Gitea webhook (repo settings → +webhooks) to the receiver on the agent host, which pulls, rebuilds and +redeploys. On the agent host: + +```sh +sudo cp publishing/webhook-receiver.service /etc/systemd/system/docs-deploy-webhook.service +sudo systemctl daemon-reload && sudo systemctl enable --now docs-deploy-webhook +tail -f /home/ubuntu/docs-deploy.log # what the receiver is doing +``` + +The receiver authenticates webhooks with an HMAC-SHA256 shared secret +(`DEPLOY_WEBHOOK_SECRET` in the agent host's env file) and only accepts pushes +to `main` of `co-op/docs` from the Cloudron box. Woodpecker CI (Cloudron +catalog) could do the same job inside Cloudron, at the cost of running another +app; the receiver was chosen because it adds no new Cloudron app. + +Each deploy writes `meta.json` into the site with the commit it was built +from, so a deploy can be verified from the served site itself: +https://docs.inference.coop/meta.json diff --git a/publishing/brand.css b/publishing/brand.css new file mode 100644 index 0000000..2776d0d --- /dev/null +++ b/publishing/brand.css @@ -0,0 +1,145 @@ +/* Inference Cooperative brand layer for the documentation site. + * + * Source of truth: the co-op's website (repos/website/index.html) — its CSS + * custom properties and its Figtree webfont. This file reuses those values so + * the documentation reads as the same organisation, not a default theme. + * + * Standing constraint: everything here is local. No third-party font service + * (the Google Fonts and gstatic hosts), no CDN, no other external host. + */ + +:root { + /* -- the co-op's palette (copied from repos/website/index.html) -- */ + --cream: #faf8f5; + --white: #ffffff; + --ink: #2d3327; + --muted: #6b7a62; + --light-muted: #94a08c; + --green-deep: #5b8c5a; + --green-mid: #6ba86b; + --green-light: #7ab87a; + --border: #e8e2d8; + --wash: #eef4ea; + --earth-red: #a04b3a; + --earth-red-wash: #f3e6e1; + --bronze: #a97a4a; + --bronze-wash: #f1e8da; +} + +/* ---------- Material for MkDocs: map the palette onto the theme ---------- */ + +:root, +[data-md-color-scheme="default"] { + --md-default-bg-color: var(--cream); + --md-primary-fg-color: var(--green-deep); + --md-primary-fg-color--light: var(--green-mid); + --md-primary-fg-color--dark: #4a7a49; + --md-primary-bg-color: var(--cream); + --md-accent-fg-color: var(--earth-red); + --md-typeset-a-color: var(--green-deep); + --md-typeset-color: var(--ink); + --md-footer-bg-color: var(--wash); + --md-footer-fg-color: var(--ink); + --md-footer-fg-color--light: var(--muted); + --md-code-bg-color: #f2efe9; + --md-code-fg-color: var(--ink); + --md-admonition-bg-color: var(--wash); +} + +/* Dark scheme keeps the same accents; light stays the default. */ +[data-md-color-scheme="slate"] { + --md-primary-fg-color: #3f6b3e; + --md-accent-fg-color: #d98a75; + --md-typeset-a-color: var(--green-light); + --md-code-bg-color: #2e3328; + --md-code-fg-color: #e8e2d8; +} + +/* ---------- Typography: Figtree, self-hosted (SIL OFL 1.1) ---------- */ + +@font-face { + font-family: "Figtree"; + font-style: normal; + font-weight: 400; + font-display: swap; + src: url("fonts/figtree-400.woff2") format("woff2"); +} +@font-face { + font-family: "Figtree"; + font-style: normal; + font-weight: 500; + font-display: swap; + src: url("fonts/figtree-500.woff2") format("woff2"); +} +@font-face { + font-family: "Figtree"; + font-style: normal; + font-weight: 600; + font-display: swap; + src: url("fonts/figtree-600.woff2") format("woff2"); +} +@font-face { + font-family: "Figtree"; + font-style: normal; + font-weight: 700; + font-display: swap; + src: url("fonts/figtree-700.woff2") format("woff2"); +} + +:root { + --md-text-font-family: "Figtree", system-ui, -apple-system, "Segoe UI", + Roboto, "Helvetica Neue", Arial, sans-serif; + --md-code-font-family: ui-monospace, "SFMono-Regular", "Cascadia Mono", + "Segoe UI Mono", Menlo, Consolas, monospace; +} + +/* ---------- Friendly details ---------- */ + +/* Links read like the website's: brand green, semibold, underline on hover. */ +.md-typeset a { + font-weight: 600; +} +.md-typeset a:hover, +.md-typeset a:focus { + text-decoration: underline; +} + +/* Rounder cards and tables; a wash instead of grey where the theme has one. */ +.md-typeset .grid.cards > ul > li, +.md-typeset .grid > .card, +.md-typeset table:not([class]) { + border-radius: 0.6rem; +} +.md-typeset table:not([class]) { + border: 1px solid var(--border); +} + +/* Header: co-op green with the cream wordmark. */ +.md-header { + box-shadow: none; + border-bottom: 1px solid rgba(255, 255, 255, 0.25); +} +.md-header .md-header__title { + font-weight: 600; +} + +/* Tabs and the search field pick up the wash. */ +.md-tabs { + background-color: var(--green-deep); +} +.md-search__input { + background-color: rgba(255, 255, 255, 0.92); +} + +/* Admonitions get the co-op's washes instead of stock greys. */ +.md-typeset .admonition, +.md-typeset details { + border-radius: 0.6rem; +} +.md-typeset .admonition.tip, +.md-typeset .admonition.note { + border-color: var(--green-mid); +} +.md-typeset .admonition.warning { + border-color: var(--earth-red); +} diff --git a/publishing/build.sh b/publishing/build.sh new file mode 100755 index 0000000..de2f8bd --- /dev/null +++ b/publishing/build.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Build the member-readable documentation site from this repository. +# +# The repository is the single source of truth. This script: +# 1. stages a mirror of the documents into .site-src/ (generated, gitignored), +# because MkDocs cannot read the repository root directly; +# 2. renders .site-src/ to .site-build/ with MkDocs + Material. +# +# No document is moved, renamed, or rewritten — the staging copy is a build +# artifact and disappears with `--clean`. +# +# Usage: +# publishing/build.sh # create/use .venv-docs and build +# publishing/build.sh /path/to/venv # use an existing venv +# +# In CI (Woodpecker), install the toolchain and run MkDocs directly: +# pip install -r publishing/requirements.txt && mkdocs build --strict + +set -euo pipefail + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_DIR" + +VENV="${1:-$REPO_DIR/.venv-docs}" +SITE_SRC="$REPO_DIR/.site-src" + +# 1. Stage the documents (Markdown at the root, maintenance/, incidents/) plus +# the publishing assets (fonts, CSS, icon). Exclude tooling and build output. +rm -rf "$SITE_SRC" +mkdir -p "$SITE_SRC" +tar cf - \ + --exclude='.git' \ + --exclude='.site-src' \ + --exclude='.site-build' \ + --exclude='.venv-docs' \ + --exclude='*.sh' \ + --exclude='publishing/webhook-receiver.py' \ + --exclude='publishing/webhook-receiver.service' \ + --exclude='mkdocs.yml' \ + --exclude='.gitignore' \ + --exclude='requirements.txt' \ + . | tar xf - -C "$SITE_SRC" + +# 2. Build. +if [ ! -x "$VENV/bin/mkdocs" ]; then + echo "Creating build venv at $VENV" + python3 -m venv "$VENV" + "$VENV/bin/pip" install --quiet --upgrade pip + "$VENV/bin/pip" install --quiet -r "$REPO_DIR/publishing/requirements.txt" +fi + +echo "Building site -> $REPO_DIR/.site-build" +"$VENV/bin/mkdocs" build --clean + +echo "Done: $REPO_DIR/.site-build" diff --git a/publishing/deploy.sh b/publishing/deploy.sh new file mode 100755 index 0000000..beb83db --- /dev/null +++ b/publishing/deploy.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Publish the built documentation site to the Cloudron Surfer app. +# +# Surfer serves static files from inside its app container. The build is +# written to two places: +# /app/data/public-docs/ the Surfer site bound to docs.inference.coop +# (served at the root of that hostname) +# /app/data/public/docs/ a subdirectory of the marketing site, so the docs +# stay viewable at https://inference.coop/docs/ +# +# Mechanism (evidence: Cloudron app "Surfer" io.cloudron.surfer 7.4.1, and +# repos/website/deploy.sh, which already deploys the marketing site this way): +# pipe a tar of the build output into the container via `cloudron exec`, which +# needs only the Cloudron API token — no WebDAV token, SFTP credentials or SSH. +# +# Usage: +# CLOUDRON_TOKEN=... publishing/deploy.sh +# +# Env: +# CLOUDRON_TOKEN (required) Cloudron API token +# CLOUDRON_SERVER default my.inference.coop +# SURFER_APP default inference.coop (the Surfer app's location) +# SITE_DIR default /.site-build + +set -euo pipefail + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SITE_DIR="${SITE_DIR:-$REPO_DIR/.site-build}" +SERVER="${CLOUDRON_SERVER:-my.inference.coop}" +APP="${SURFER_APP:-inference.coop}" +DEST_SITE="${DOCS_DEST_SITE:-/app/data/public-docs}" # docs.inference.coop +DEST_PATH="${DOCS_DEST_PATH:-/app/data/public/docs}" # inference.coop/docs/ + +CLOUDRON_BIN="${CLOUDRON_BIN:-$(command -v cloudron || true)}" +if [ -z "$CLOUDRON_BIN" ] && [ -x "$HOME/.local/bin/cloudron" ]; then + CLOUDRON_BIN="$HOME/.local/bin/cloudron" +fi +: "${CLOUDRON_BIN:?cloudron CLI not found (set CLOUDRON_BIN or add it to PATH)}" + +# The cloudron CLI is a node script (`#!/usr/bin/env node`); under systemd the +# PATH may not contain the host's node, so look in the usual places. +if ! command -v node >/dev/null 2>&1; then + for dir in "$HOME"/.hermes/tools/node*/bin /usr/local/bin; do + if [ -x "$dir/node" ]; then export PATH="$dir:$PATH"; break; fi + done +fi + +: "${CLOUDRON_TOKEN:?CLOUDRON_TOKEN is required (the Cloudron API token)}" + +if [ ! -f "$SITE_DIR/index.html" ]; then + echo "No build found at $SITE_DIR — run publishing/build.sh first." >&2 + exit 1 +fi + +# Build metadata: written into the output so a deploy can be verified from the +# served site itself (which push produced it, and when). +COMMIT="$(git -C "$REPO_DIR" rev-parse HEAD 2>/dev/null || echo unknown)" +COMMIT_SHORT="$(git -C "$REPO_DIR" rev-parse --short HEAD 2>/dev/null || echo unknown)" +BUILT_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +cat > "$SITE_DIR/meta.json" < $SERVER:$APP" +echo " $DEST_SITE (docs.inference.coop)" +echo " $DEST_PATH (inference.coop/docs/)" + +tar cf - -C "$SITE_DIR" . | \ + "$CLOUDRON_BIN" --server "$SERVER" --token "$CLOUDRON_TOKEN" exec --app "$APP" -- \ + sh -c "set -e + cat > /tmp/docs-site.tar + rm -rf '$DEST_SITE' && mkdir -p '$DEST_SITE' && tar xf /tmp/docs-site.tar -C '$DEST_SITE' + rm -rf '$DEST_PATH' && mkdir -p '$DEST_PATH' && tar xf /tmp/docs-site.tar -C '$DEST_PATH' + rm -f /tmp/docs-site.tar" + +echo "Done:" +echo " https://docs.inference.coop/" +echo " https://inference.coop/docs/" diff --git a/publishing/fonts/OFL.txt b/publishing/fonts/OFL.txt new file mode 100644 index 0000000..80b12bb --- /dev/null +++ b/publishing/fonts/OFL.txt @@ -0,0 +1,93 @@ +Copyright 2022 The Figtree Project Authors (https://github.com/erikdkennedy/figtree) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/publishing/fonts/figtree-400.woff2 b/publishing/fonts/figtree-400.woff2 new file mode 100644 index 0000000..ca6795e Binary files /dev/null and b/publishing/fonts/figtree-400.woff2 differ diff --git a/publishing/fonts/figtree-500.woff2 b/publishing/fonts/figtree-500.woff2 new file mode 100644 index 0000000..8601545 Binary files /dev/null and b/publishing/fonts/figtree-500.woff2 differ diff --git a/publishing/fonts/figtree-600.woff2 b/publishing/fonts/figtree-600.woff2 new file mode 100644 index 0000000..9b4ec79 Binary files /dev/null and b/publishing/fonts/figtree-600.woff2 differ diff --git a/publishing/fonts/figtree-700.woff2 b/publishing/fonts/figtree-700.woff2 new file mode 100644 index 0000000..3f75953 Binary files /dev/null and b/publishing/fonts/figtree-700.woff2 differ diff --git a/publishing/logo.svg b/publishing/logo.svg new file mode 100644 index 0000000..30ad0d0 --- /dev/null +++ b/publishing/logo.svg @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/publishing/requirements.txt b/publishing/requirements.txt new file mode 100644 index 0000000..a6f09c0 --- /dev/null +++ b/publishing/requirements.txt @@ -0,0 +1,3 @@ +# Pinned build toolchain for the documentation site. +# Reproducible: `pip install -r publishing/requirements.txt && mkdocs build` +mkdocs-material==9.8.0 diff --git a/publishing/webhook-receiver.py b/publishing/webhook-receiver.py new file mode 100644 index 0000000..cce134c --- /dev/null +++ b/publishing/webhook-receiver.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""Gitea webhook receiver that rebuilds and republishes the docs site. + +Flow: a push to main in co-op/docs fires a Gitea webhook -> this receiver +verifies it -> pulls the latest main -> runs publishing/build.sh -> +publishing/deploy.sh (which writes the build into the Surfer app). + +Runs on the agent host (no new Cloudron app). Standard library only. + +Security model: + * Only POSTs from the Cloudron box (git.inference.coop's host) are accepted. + * The payload must carry a valid HMAC-SHA256 signature (X-Gitea-Signature, + shared secret DEPLOY_WEBHOOK_SECRET). The secret itself is never sent. + * Only pushes to refs/heads/main of co-op/docs trigger a deploy. + * The receiver binds 0.0.0.0; an iptables rule installed by the systemd unit + narrows the port to the Cloudron box as well (defence in depth). + +Secrets are read from /home/ubuntu/.hermes/.env and are never logged. +""" + +import fcntl +import hashlib +import hmac +import ipaddress +import json +import logging +import os +import subprocess +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +REPO_DIR = os.environ.get("DOCS_REPO_DIR", "/home/ubuntu/repos/docs") +ENV_FILE = os.environ.get("DOCS_ENV_FILE", "/home/ubuntu/.hermes/.env") +LOG_FILE = os.environ.get("DOCS_DEPLOY_LOG", "/home/ubuntu/docs-deploy.log") +LOCK_FILE = os.environ.get("DOCS_DEPLOY_LOCK", "/home/ubuntu/.docs-deploy.lock") +PORT = int(os.environ.get("DOCS_WEBHOOK_PORT", "8787")) +EXPECT_REPO = "co-op/docs" +EXPECT_BRANCH = "refs/heads/main" +ALLOWED_SOURCES = [ + ipaddress.ip_network(n) + for n in os.environ.get( + "DOCS_WEBHOOK_SOURCES", + # The Cloudron box (hosts git.inference.coop) plus loopback for local tests. + "179.237.86.116/32,2001:1600:18:206::89/128,127.0.0.1/32,::1/128", + ).split(",") + if n.strip() +] + +log = logging.getLogger("docs-deploy") +deploy_lock_file = open(LOCK_FILE, "a") + + +def load_env_file(path): + """Return KEY=VALUE pairs from the agent host's env file (values stay local).""" + values = {} + try: + with open(path) as handle: + for line in handle: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + values[key.strip()] = value.strip().strip('"').strip("'") + except OSError: + pass + return values + + +_FILE_ENV = load_env_file(ENV_FILE) + + +def secret(name): + return os.environ.get(name) or _FILE_ENV.get(name) or "" + + +def source_allowed(address): + try: + ip = ipaddress.ip_address(address.split("%")[0]) + except ValueError: + return False + return any(ip in net for net in ALLOWED_SOURCES) + + +def run(cmd, timeout, cwd=None): + log.info("exec: %s", " ".join(cmd)) + # Hand the build/deploy scripts the credentials they need (from the env + # file, if not already set). Values are passed by environment only and are + # never logged. + env = dict(os.environ) + for key in ("CLOUDRON_TOKEN", "CLOUDRON_SERVER", "SURFER_APP", "GIT_ASKPASS"): + if not env.get(key) and secret(key): + env[key] = secret(key) + env.setdefault("GIT_ASKPASS", "/home/ubuntu/.hermes/scripts/git_askpass.sh") + env.setdefault("PATH", os.environ.get("PATH", "")) + result = subprocess.run( + cmd, cwd=cwd or REPO_DIR, capture_output=True, text=True, timeout=timeout, + env=env, + ) + if result.stdout.strip(): + log.info("stdout: %s", result.stdout.strip()[-2000:]) + if result.returncode != 0 and result.stderr.strip(): + log.error("stderr: %s", result.stderr.strip()[-2000:]) + return result + + +def deploy(commit_hint=""): + """Pull main, rebuild, republish. Returns (ok, log_tail).""" + try: + with open("/dev/null") as devnull: + fcntl.flock(deploy_lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return False, "another deploy is already running; skipped" + + tail = [] + try: + steps = [ + (["git", "pull", "--ff-only", "origin", "main"], 120), + (["bash", "publishing/build.sh"], 900), + (["bash", "publishing/deploy.sh"], 600), + ] + for cmd, timeout in steps: + result = run(cmd, timeout) + out = (result.stdout or "") + (result.stderr or "") + tail.append(f"$ {' '.join(cmd)} -> exit {result.returncode}") + tail.extend(out.strip().splitlines()[-12:]) + if result.returncode != 0: + log.error("step failed: %s", " ".join(cmd)) + return False, "\n".join(tail) + log.info("deploy complete%s", f" ({commit_hint})" if commit_hint else "") + return True, "\n".join(tail) + except Exception as exc: # noqa: BLE001 - report, never crash the receiver + log.exception("deploy failed") + tail.append(f"error: {exc}") + return False, "\n".join(tail) + + +class Handler(BaseHTTPRequestHandler): + server_version = "docs-deploy/1.0" + + def log_message(self, format, *args): # noqa: A002 - stdlib signature + log.info("%s - %s", self.client_address[0], format % args) + + def _json(self, code, payload): + body = json.dumps(payload).encode() + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + if self.path == "/healthz": + return self._json(200, {"status": "ok"}) + return self._json(404, {"error": "not found"}) + + def do_POST(self): + peer = self.client_address[0] + if not source_allowed(peer): + log.warning("rejected: source %s not allowed", peer) + return self._json(403, {"error": "forbidden"}) + + if self.headers.get("X-Gitea-Event") != "push": + return self._json(400, {"error": "not a push event"}) + + length = int(self.headers.get("Content-Length") or 0) + if length <= 0 or length > 1_000_000: + return self._json(400, {"error": "bad payload size"}) + body = self.rfile.read(length) + + signature = self.headers.get("X-Gitea-Signature") or "" + expected = hmac.new( + secret("DEPLOY_WEBHOOK_SECRET").encode(), body, hashlib.sha256 + ).hexdigest() + if not signature or not hmac.compare_digest(signature, expected): + log.warning("rejected: bad signature from %s", peer) + return self._json(403, {"error": "invalid signature"}) + + try: + payload = json.loads(body) + except json.JSONDecodeError: + return self._json(400, {"error": "invalid json"}) + + repo = (payload.get("repository") or {}).get("full_name", "") + ref = payload.get("ref", "") + commit = (payload.get("after") or "")[:12] + pusher = ((payload.get("pusher") or {}).get("login")) or "unknown" + if repo != EXPECT_REPO or ref != EXPECT_BRANCH: + log.info("ignored: push to %s %s (not %s %s)", repo, ref, EXPECT_REPO, EXPECT_BRANCH) + return self._json(200, {"status": "ignored", "reason": "not main on co-op/docs"}) + + log.info("push received: %s %s commit=%s pusher=%s", repo, ref, commit, pusher) + threading.Thread(target=self._run_deploy, args=(commit,), daemon=True).start() + return self._json(202, {"status": "accepted", "commit": commit}) + + def _run_deploy(self, commit): + ok, tail = deploy(commit) + log.info("deploy result for %s: %s", commit, "ok" if ok else "FAILED") + log.info("tail:\n%s", tail[-1500:]) + + +def main(): + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", + handlers=[ + logging.FileHandler(LOG_FILE), + logging.StreamHandler(sys.stdout), + ], + ) + if not secret("DEPLOY_WEBHOOK_SECRET"): + log.error("DEPLOY_WEBHOOK_SECRET is not set; refusing to start") + sys.exit(1) + if not secret("CLOUDRON_TOKEN"): + log.warning("CLOUDRON_TOKEN not set; deploys will fail") + server = ThreadingHTTPServer(("0.0.0.0", PORT), Handler) + log.info("listening on 0.0.0.0:%d, repo %s", PORT, REPO_DIR) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + + +if __name__ == "__main__": + main() diff --git a/publishing/webhook-receiver.service b/publishing/webhook-receiver.service new file mode 100644 index 0000000..4ce08b4 --- /dev/null +++ b/publishing/webhook-receiver.service @@ -0,0 +1,28 @@ +# systemd unit for the docs deploy webhook receiver (agent host). +# +# Install: +# sudo cp publishing/webhook-receiver.service /etc/systemd/system/ +# sudo systemctl daemon-reload && sudo systemctl enable --now docs-deploy-webhook +# +# The ExecStartPre lines narrow the webhook port to the Cloudron box (which +# hosts git.inference.coop); the receiver additionally checks the source address +# and the payload signature, so a scanner hitting the port gets nothing. + +[Unit] +Description=Inference docs: Gitea webhook -> build -> Surfer deploy +After=network-online.target +Wants=network-online.target + +[Service] +User=ubuntu +WorkingDirectory=/home/ubuntu/repos/docs +ExecStartPre=+/bin/sh -c '/usr/sbin/iptables -C INPUT -i lo -p tcp --dport 8787 -j ACCEPT 2>/dev/null || /usr/sbin/iptables -I INPUT 1 -i lo -p tcp --dport 8787 -j ACCEPT' +ExecStartPre=+/bin/sh -c '/usr/sbin/iptables -C INPUT -p tcp --dport 8787 -s 179.237.86.116 -j ACCEPT 2>/dev/null || /usr/sbin/iptables -I INPUT 2 -p tcp --dport 8787 -s 179.237.86.116 -j ACCEPT' +ExecStartPre=+/bin/sh -c '/usr/sbin/iptables -C INPUT -p tcp --dport 8787 ! -s 179.237.86.116 -j DROP 2>/dev/null || /usr/sbin/iptables -I INPUT 3 -p tcp --dport 8787 ! -s 179.237.86.116 -j DROP' +ExecStartPre=+/bin/sh -c '/usr/sbin/ip6tables -C INPUT -p tcp --dport 8787 -j DROP 2>/dev/null || /usr/sbin/ip6tables -I INPUT -p tcp --dport 8787 -j DROP' +ExecStart=/home/ubuntu/repos/docs/.venv-docs/bin/python /home/ubuntu/repos/docs/publishing/webhook-receiver.py +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target