From 20d384be8750e723d56b8dc4638ab6c48ed489f0 Mon Sep 17 00:00:00 2001 From: inference-bot Date: Thu, 12 Nov 2026 03:23:33 +0000 Subject: [PATCH] publishing: one canonical build venv; drift check; deploy-failure alerting The poller's private venv (.docs-publisher/.venv-docs) diverged from the canonical one twice, each time silently delaying a deploy. Consolidate: - poll-watcher.sh builds with the ONE canonical venv (~/repos/docs/.venv-docs) while keeping its dedicated checkout (uncommitted edits in a working clone must never block or leak into a deploy). - New drift check: publishing/ + mkdocs.yml are compared with the canonical checkout before every build; a mismatch fails the deploy loudly instead of shipping logic the canonical source cannot reproduce. - Pre-deploy gate: build.sh --check-only verifies the pinned toolchain and every configured plugin (self-heals missing pins, fails loudly naming an unpinned plugin). - Alerting: 3 consecutive deploy failures -> marker file + ONE Matrix message to the InferenceBot HQ room (hourly cap); counter resets on next success. - The watcher re-execs itself when a deploy changed poll-watcher.sh, so the running loop always matches the deployed one. - webhook receiver (dormant) repointed at the canonical venv too; unit files and README document the layout. --- publishing/README.md | 42 +++++++ publishing/docs-deploy-poll.service | 23 ++-- publishing/poll-watcher.sh | 166 ++++++++++++++++++++++++++-- publishing/webhook-receiver.py | 8 +- publishing/webhook-receiver.service | 4 +- 5 files changed, 222 insertions(+), 21 deletions(-) diff --git a/publishing/README.md b/publishing/README.md index 7468276..eb04385 100644 --- a/publishing/README.md +++ b/publishing/README.md @@ -105,3 +105,45 @@ app; the agent-host mechanisms were chosen because they add no new Cloudron app. Each deploy writes `meta.json` into the site with the commit it was built from, so a deploy can be verified from the served site itself: https://docs.inference.coop/meta.json + +## One build environment (2026-11-12) + +The deploy used to have TWO build environments — this repo's `.venv-docs` and +a second private venv in the poller's checkout at +`/home/ubuntu/.docs-publisher/.venv-docs`. They diverged twice, each time +silently delaying a deploy (the brand.css cache-busting confusion, then a +missing `mkdocs-redirects` that failed every build for ~15 minutes while the +poller retried without telling anyone). Consolidated: + +- **ONE venv.** Every trigger — the poller, the (dormant) webhook receiver, a + manual `publishing/build.sh`, CI — builds with the canonical checkout's + venv: `~/repos/docs/.venv-docs`. The poller's private venv was deleted. +- **The dedicated checkout stays, by design.** The poller still builds from + `/home/ubuntu/.docs-publisher`, its own clean clone: uncommitted edits in a + working clone must never block a deploy or leak into the published site. +- **Drift check.** Before every build the poller compares `publishing/` and + `mkdocs.yml` between its checkout and the canonical one; any mismatch fails + the deploy loudly (with a diff and the fix — `git -C ~/repos/docs pull + --ff-only`), and the failure counts toward the alert below. A canonical + checkout that lags behind main trips this too, on purpose: a deploy of + logic the canonical source cannot reproduce is the failure mode this exists + to prevent. A missing canonical checkout is fatal as well. +- **Plugin gate + self-heal.** `publishing/requirements.txt` pins every + package the build needs (`mkdocs`, `mkdocs-material`, `mkdocs-redirects`). + `build.sh` verifies the venv against it before every build and installs + anything missing or off-version (fast local no-op when current); it also + fails loudly if `mkdocs.yml` configures a plugin that `requirements.txt` + does not pin. The poller runs this as its pre-deploy check via + `build.sh --check-only`. +- **Failure alerting.** After 3 consecutive deploy failures the poller writes + a marker (`~/.docs-deploy.alert`) and sends ONE message to the InferenceBot + HQ Matrix room naming the failing commit — never more than one message per + hour (`~/.docs-deploy.last-alert`). The counter (`~/.docs-deploy.failcount`) + resets on the next successful deploy. A deploy that succeeds silently + resets everything; nothing announces routine successes. + +Files: `~/.docs-deploy.state` (last deployed commit), `~/.docs-deploy.lock` +(deploy lock), `~/.docs-deploy.failcount`, `~/.docs-deploy.alert`, +`~/.docs-deploy.last-alert`. Manual deploys should respect the lock: +`flock -n ~/.docs-deploy.lock bash publishing/build.sh && flock -n +~/.docs-deploy.lock bash publishing/deploy.sh`. diff --git a/publishing/docs-deploy-poll.service b/publishing/docs-deploy-poll.service index 42ab8d6..36f91f5 100644 --- a/publishing/docs-deploy-poll.service +++ b/publishing/docs-deploy-poll.service @@ -1,7 +1,4 @@ -# systemd units for the docs deploy automation (agent host). -# -# Two mechanisms, one job — keep https://docs.inference.coop/ in step with -# co-op/docs main: +# systemd unit for the docs auto-deploy poller (agent host). # # docs-deploy-poll.service ALWAYS ON. Polls Gitea every 60s; deploys on # any change to main. Works with the agent @@ -12,12 +9,21 @@ # panel allows inbound TCP to the agent host # (the Gitea webhook is already registered). # -# Install both: -# sudo cp publishing/poll-watcher.sh publishing/webhook-receiver.py /home/ubuntu/repos/docs/publishing/ -# sudo cp publishing/docs-deploy-poll.service publishing/webhook-receiver.service /etc/systemd/system/ +# Install: +# sudo cp publishing/docs-deploy-poll.service /etc/systemd/system/ # sudo systemctl daemon-reload # sudo systemctl enable --now docs-deploy-poll -# sudo systemctl enable --now docs-deploy-webhook # when the port is open +# +# poll-watcher.sh itself needs NO manual copy: the service runs the copy in +# the dedicated checkout (/home/ubuntu/.docs-publisher), which git pull keeps +# at main on every deploy, and the watcher re-execs itself when a deploy +# changed it. +# +# ONE BUILD ENVIRONMENT: the poller builds from its own dedicated checkout +# (uncommitted edits in a working clone must never block or leak into a +# deploy) but with the ONE canonical venv, DOCS_CANONICAL_DIR/.venv-docs, and +# drift-checks its publishing/ tree against the canonical checkout before +# every build. See publishing/README.md. [Unit] Description=Inference docs: poll Gitea main and deploy to Surfer @@ -28,6 +34,7 @@ Wants=network-online.target User=ubuntu WorkingDirectory=/home/ubuntu/.docs-publisher Environment=DOCS_REPO_DIR=/home/ubuntu/.docs-publisher +Environment=DOCS_CANONICAL_DIR=/home/ubuntu/repos/docs ExecStart=/bin/bash /home/ubuntu/.docs-publisher/publishing/poll-watcher.sh Restart=on-failure RestartSec=10 diff --git a/publishing/poll-watcher.sh b/publishing/poll-watcher.sh index 81af52a..d5f24e0 100755 --- a/publishing/poll-watcher.sh +++ b/publishing/poll-watcher.sh @@ -12,20 +12,157 @@ # Runs on the agent host as a systemd service (docs-deploy-poll.service). # Deploy steps are serialised with the same lock the webhook receiver uses, so # the two mechanisms can never deploy concurrently. +# +# ONE BUILD ENVIRONMENT (2026-11-12 consolidation): the watcher still builds +# from its own dedicated checkout ($DOCS_REPO_DIR) — uncommitted edits in a +# working clone must never block or leak into a deploy — but the build VENV is +# the single canonical one: $DOCS_CANONICAL_DIR/.venv-docs. The poller's old +# private venv (which silently diverged twice) is gone. Before every build: +# +# * DRIFT CHECK — publishing/ and mkdocs.yml in this checkout are compared +# with the canonical checkout; any mismatch FAILS the deploy loudly (a +# deploy of logic the canonical source cannot reproduce is how the site +# silently drifted before). +# * PRE-FLIGHT — `build.sh --check-only` verifies the pinned toolchain and +# every plugin mkdocs.yml configures, installing missing pins (self-heal) +# or failing loudly naming a plugin that was added without being pinned. +# +# ALERTING: after $DOCS_FAIL_THRESHOLD consecutive deploy failures the watcher +# writes a marker file and sends ONE message to the InferenceBot HQ Matrix +# room (never more than one per hour); the counter resets on the next success. +# A silent retry loop is how two commits once sat undeployed for ~15 minutes +# without anyone knowing. set -uo pipefail -REPO_DIR="${DOCS_REPO_DIR:-/home/ubuntu/repos/docs}" +REPO_DIR="${DOCS_REPO_DIR:-/home/ubuntu/.docs-publisher}" +CANONICAL_DIR="${DOCS_CANONICAL_DIR:-/home/ubuntu/repos/docs}" +CANONICAL_VENV="${DOCS_CANONICAL_VENV:-$CANONICAL_DIR/.venv-docs}" ENV_FILE="${DOCS_ENV_FILE:-/home/ubuntu/.hermes/.env}" LOG_FILE="${DOCS_DEPLOY_LOG:-/home/ubuntu/docs-deploy.log}" LOCK_FILE="${DOCS_DEPLOY_LOCK:-/home/ubuntu/.docs-deploy.lock}" STATE_FILE="${DOCS_DEPLOY_STATE:-/home/ubuntu/.docs-deploy.state}" +FAIL_COUNT_FILE="${DOCS_DEPLOY_FAILCOUNT:-/home/ubuntu/.docs-deploy.failcount}" +ALERT_MARKER="${DOCS_DEPLOY_ALERT_MARKER:-/home/ubuntu/.docs-deploy.alert}" +ALERT_TS_FILE="${DOCS_DEPLOY_LAST_ALERT:-/home/ubuntu/.docs-deploy.last-alert}" +# The InferenceBot HQ room (not a secret; overridable for tests). +HQ_ROOM_ID="${DOCS_HQ_ROOM_ID:-!cI_tvU4__A3Z4elsA39t5guQm6iY1YHOpWh6AXGuPic}" INTERVAL="${DOCS_POLL_INTERVAL:-60}" +FAIL_THRESHOLD="${DOCS_FAIL_THRESHOLD:-3}" +ALERT_EVERY_SECS="${DOCS_ALERT_EVERY_SECS:-3600}" cd "$REPO_DIR" log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) [poll] $*" | tee -a "$LOG_FILE" >&2; } +# --- drift check ------------------------------------------------------------ +# The dedicated checkout deploys; the canonical checkout is where people work +# and where the shared venv lives. If the two disagree on build logic or site +# config, deploying would publish logic the canonical source cannot reproduce +# — fail loudly instead. (A canonical checkout that simply lags behind main +# trips this too, by design: pull it, or commit+push its local edits.) +drift_check() { # $1 = target sha being deployed + local target="${1:-?}" + if [ ! -d "$CANONICAL_DIR/.git" ]; then + log "DRIFT-CHECK FATAL: canonical checkout $CANONICAL_DIR is missing." + log " The poller builds with that checkout's venv and compares its publishing/" + log " tree against it. Restore it: git clone https://git.inference.coop/co-op/docs.git $CANONICAL_DIR" + return 1 + fi + local detail mk + detail="$(diff -rq --exclude=__pycache__ "$REPO_DIR/publishing" "$CANONICAL_DIR/publishing" 2>&1 || true)" + mk="$(diff -q "$REPO_DIR/mkdocs.yml" "$CANONICAL_DIR/mkdocs.yml" 2>&1 || true)" + if [ -n "$detail" ] || [ -n "$mk" ]; then + log "DRIFT-CHECK FATAL: deploy checkout and canonical checkout ($CANONICAL_DIR) disagree." + log " deploy target: ${target:0:12}; canonical checkout HEAD: $(git -C "$CANONICAL_DIR" rev-parse HEAD 2>/dev/null || echo unknown)" + if [ -n "$detail" ]; then + echo "$detail" | head -15 | while IFS= read -r line; do log " $line"; done + fi + [ -n "$mk" ] && log " mkdocs.yml differs" + log " Deploying from this state could ship logic/config the canonical checkout does not have." + log " Fix: git -C $CANONICAL_DIR pull --ff-only (or commit+push its local edits) — the poller retries automatically." + return 1 + fi + log "drift check OK: publishing/ + mkdocs.yml match $CANONICAL_DIR" + return 0 +} + +# --- one-shot deploy, run under the deploy lock by the main loop ------------ +if [ "${1:-}" = "--deploy-once" ]; then + TARGET_SHA="${2:?usage: poll-watcher.sh --deploy-once }" + cd "$REPO_DIR" + set -e + git pull --ff-only origin main + drift_check "$TARGET_SHA" + # Pre-deploy gate: pinned toolchain present (self-heal installs missing + # pins) and every configured plugin pinned. A failure here fails the deploy + # loudly instead of leaving the venv silently behind mkdocs.yml. + bash publishing/build.sh --check-only "$CANONICAL_VENV" + bash publishing/build.sh "$CANONICAL_VENV" + bash publishing/deploy.sh + exit 0 +fi + +# --- HQ alerting ------------------------------------------------------------- +# One message per failure streak, never more than one per hour. State lives in +# files so a restart cannot reset a streak or re-fire an alert. +hq_alert() { # $1 = short sha, $2 = commit subject + local now last + now="$(date +%s)" + last=0 + [ -f "$ALERT_TS_FILE" ] && last="$(cat "$ALERT_TS_FILE" 2>/dev/null || echo 0)" + if [ "$((now - last))" -lt "$ALERT_EVERY_SECS" ]; then + log "HQ alert suppressed (last alert $(( (now - last) / 60 )) min ago; cap ${ALERT_EVERY_SECS}s) — streak continues" + return 0 + fi + local hs token url body json code + hs="$(sed -n 's/^MATRIX_HOMESERVER=//p' "$ENV_FILE" | head -1 | tr -d '"' | tr -d "'")" + hs="${hs%/}" + token="$(sed -n 's/^MATRIX_ACCESS_TOKEN=//p' "$ENV_FILE" | head -1 | tr -d '"' | tr -d "'")" + if [ -z "$hs" ] || [ -z "$token" ]; then + log "HQ alert FAILED: MATRIX_HOMESERVER / MATRIX_ACCESS_TOKEN missing in $ENV_FILE" + return 1 + fi + body="Docs deploy FAILED ${FAIL_THRESHOLD}x in a row — commit $1 ('$2') is NOT live on docs.inference.coop. Check journalctl -u docs-deploy-poll or $LOG_FILE on the agent host." + json="$(python3 -c 'import json,sys; print(json.dumps({"msgtype":"m.text","body":sys.argv[1]}))' "$body")" + url="$hs/_matrix/client/v3/rooms/${HQ_ROOM_ID/!/%21}/send/m.room.message/docs-deploy-$now" + code="$(curl -sS --max-time 20 -o /dev/null -w '%{http_code}' -X PUT \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $token" \ + --data "$json" "$url" 2>>"$LOG_FILE" || echo 000)" + if [ "$code" = "200" ]; then + printf '%s\n' "$now" > "$ALERT_TS_FILE" + touch "$ALERT_MARKER" + log "HQ alert sent to InferenceBot HQ room (Matrix 200)" + else + log "HQ alert FAILED: Matrix HTTP $code (will retry on the next failed deploy)" + fi +} + +record_failure() { # $1 = sha that failed to deploy + local n=1 + [ -f "$FAIL_COUNT_FILE" ] && n=$(( $(cat "$FAIL_COUNT_FILE" 2>/dev/null || echo 0) + 1 )) + printf '%s\n' "$n" > "$FAIL_COUNT_FILE" + log "deploy FAILED for ${1:0:12} (consecutive failures: $n)" + if [ "$n" -ge "$FAIL_THRESHOLD" ]; then + touch "$ALERT_MARKER" + local subject + subject="$(git log -1 --format=%s "$1" 2>/dev/null || echo 'unknown commit')" + hq_alert "${1:0:12}" "$subject" + fi +} + +reset_failures() { + local n=0 + [ -f "$FAIL_COUNT_FILE" ] && n="$(cat "$FAIL_COUNT_FILE" 2>/dev/null || echo 0)" + if [ "${n:-0}" -gt 0 ]; then + log "success: consecutive-failure counter reset (was $n); alert marker cleared" + fi + printf '0\n' > "$FAIL_COUNT_FILE" + rm -f "$ALERT_MARKER" +} + +# --- main loop (the watcher proper) ----------------------------------------- # The Cloudron token travels by environment only; it is read from the env file # and never echoed. CLOUDRON_TOKEN="$(sed -n 's/^CLOUDRON_TOKEN=//p' "$ENV_FILE" | head -1 | tr -d '"' | tr -d "'")" @@ -35,7 +172,10 @@ if [ -z "$CLOUDRON_TOKEN" ]; then exit 1 fi -log "watcher started: polling $REPO_DIR every ${INTERVAL}s" +# The one-shot deploy child re-runs this script; these must survive into it. +export REPO_DIR CANONICAL_DIR CANONICAL_VENV ENV_FILE LOG_FILE + +log "watcher started: polling $REPO_DIR every ${INTERVAL}s (build venv: $CANONICAL_VENV)" while true; do if ! git fetch --quiet origin main 2>>"$LOG_FILE"; then @@ -51,18 +191,22 @@ while true; do if [ "$LAST_DEPLOYED" != "$REMOTE" ]; then log "main moved: ${LAST_DEPLOYED:0:12} -> ${REMOTE:0:12}; deploying" - # flock -n shares the receiver's lock: one deploy at a time, ever. - if flock -n "$LOCK_FILE" bash -c ' - set -e - cd "$1" - git pull --ff-only origin main - bash publishing/build.sh - bash publishing/deploy.sh - ' _ "$REPO_DIR" >>"$LOG_FILE" 2>&1; then + # flock -n shares the receiver's lock: one deploy at a time, ever. The + # one-shot child re-executes this script in --deploy-once mode. + if flock -n "$LOCK_FILE" bash "$0" --deploy-once "$REMOTE" >>"$LOG_FILE" 2>&1; then printf '%s\n' "$REMOTE" > "$STATE_FILE" log "deploy finished for ${REMOTE:0:12}" + reset_failures + # A deploy that changed poll-watcher.sh leaves this loop running the old + # logic (bash parsed the whole loop before the file changed on disk, so + # the rewrite cannot corrupt it — but it also cannot take effect). + # Re-exec so the running watcher always matches the deployed one. + if [ "$LAST_DEPLOYED" != "none" ] && ! git diff --quiet "$LAST_DEPLOYED" "$REMOTE" -- publishing/poll-watcher.sh 2>/dev/null; then + log "poll-watcher.sh changed in ${REMOTE:0:12}; re-exec'ing to pick up the new watcher" + exec bash "$0" + fi else - log "deploy FAILED for ${REMOTE:0:12} (see log above)" + record_failure "$REMOTE" fi fi diff --git a/publishing/webhook-receiver.py b/publishing/webhook-receiver.py index 90c8ae3..73fcd59 100644 --- a/publishing/webhook-receiver.py +++ b/publishing/webhook-receiver.py @@ -31,6 +31,9 @@ import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer REPO_DIR = os.environ.get("DOCS_REPO_DIR", "/home/ubuntu/repos/docs") +# ONE build environment: the canonical checkout's venv, shared by every +# trigger (poller, webhook, manual builds) since the 2026-11-12 consolidation. +BUILD_VENV = os.environ.get("DOCS_BUILD_VENV", "/home/ubuntu/repos/docs/.venv-docs") ENV_FILE = os.environ.get("DOCS_ENV_FILE", "/home/ubuntu/.hermes/.env") LOG_FILE = os.environ.get("DOCS_DEPLOY_LOG", "/home/ubuntu/docs-deploy.log") LOCK_FILE = os.environ.get("DOCS_DEPLOY_LOCK", "/home/ubuntu/.docs-deploy.lock") @@ -115,7 +118,10 @@ def deploy(commit_hint=""): try: steps = [ (["git", "pull", "--ff-only", "origin", "main"], 120), - (["bash", "publishing/build.sh"], 900), + # ONE build environment: build with the canonical checkout's venv, + # not a per-checkout one (the poller's private venv diverged twice + # before the 2026-11-12 consolidation). + (["bash", "publishing/build.sh", BUILD_VENV], 900), (["bash", "publishing/deploy.sh"], 600), ] for cmd, timeout in steps: diff --git a/publishing/webhook-receiver.service b/publishing/webhook-receiver.service index a97a2c9..3b37fb4 100644 --- a/publishing/webhook-receiver.service +++ b/publishing/webhook-receiver.service @@ -21,7 +21,9 @@ ExecStartPre=+/bin/sh -c '/usr/sbin/iptables -C INPUT -i lo -p tcp --dport 8787 ExecStartPre=+/bin/sh -c '/usr/sbin/iptables -C INPUT -p tcp --dport 8787 -s 179.237.86.116 -j ACCEPT 2>/dev/null || /usr/sbin/iptables -I INPUT 2 -p tcp --dport 8787 -s 179.237.86.116 -j ACCEPT' ExecStartPre=+/bin/sh -c '/usr/sbin/iptables -C INPUT -p tcp --dport 8787 ! -s 179.237.86.116 -j DROP 2>/dev/null || /usr/sbin/iptables -I INPUT 3 -p tcp --dport 8787 ! -s 179.237.86.116 -j DROP' ExecStartPre=+/bin/sh -c '/usr/sbin/ip6tables -C INPUT -p tcp --dport 8787 -j DROP 2>/dev/null || /usr/sbin/ip6tables -I INPUT -p tcp --dport 8787 -j DROP' -ExecStart=/home/ubuntu/.docs-publisher/.venv-docs/bin/python /home/ubuntu/.docs-publisher/publishing/webhook-receiver.py +# The receiver runs on the ONE canonical build venv (the canonical checkout's +# .venv-docs), like every other trigger of this publishing layer. +ExecStart=/home/ubuntu/repos/docs/.venv-docs/bin/python /home/ubuntu/.docs-publisher/publishing/webhook-receiver.py Restart=on-failure RestartSec=5