#!/usr/bin/env bash # Poll Gitea for new commits on main and deploy the docs site when it moves. # # Why polling: the agent host's cloud firewall only admits inbound SSH, so a # Gitea webhook cannot reach the receiver (publishing/webhook-receiver.py) — # every port is silently dropped. Outbound to git.inference.coop works, so the # agent host polls instead: a push to main is picked up within ~a minute and # deployed with no human action. When the operator opens a TCP port to the # agent host in the Infomaniak panel, the webhook takes over (its unit is # already installed); until then this watcher is the automation. # # Runs on the agent host as a systemd service (docs-deploy-poll.service). # Deploy steps are serialised with the same lock the webhook receiver uses, so # the two mechanisms can never deploy concurrently. # # ONE BUILD ENVIRONMENT (2026-11-12 consolidation): the watcher still builds # from its own dedicated checkout ($DOCS_REPO_DIR) — uncommitted edits in a # working clone must never block or leak into a deploy — but the build VENV is # the single canonical one: $DOCS_CANONICAL_DIR/.venv-docs. The poller's old # private venv (which silently diverged twice) is gone. Before every build: # # * DRIFT CHECK — publishing/ and mkdocs.yml in this checkout are compared # with the canonical checkout; any mismatch FAILS the deploy loudly (a # deploy of logic the canonical source cannot reproduce is how the site # silently drifted before). # * PRE-FLIGHT — `build.sh --check-only` verifies the pinned toolchain and # every plugin mkdocs.yml configures, installing missing pins (self-heal) # or failing loudly naming a plugin that was added without being pinned. # # ALERTING: after $DOCS_FAIL_THRESHOLD consecutive deploy failures the watcher # writes a marker file and sends ONE message to the InferenceBot HQ Matrix # room (never more than one per hour); the counter resets on the next success. # A silent retry loop is how two commits once sat undeployed for ~15 minutes # without anyone knowing. set -uo pipefail REPO_DIR="${DOCS_REPO_DIR:-/home/ubuntu/.docs-publisher}" CANONICAL_DIR="${DOCS_CANONICAL_DIR:-/home/ubuntu/repos/docs}" CANONICAL_VENV="${DOCS_CANONICAL_VENV:-$CANONICAL_DIR/.venv-docs}" ENV_FILE="${DOCS_ENV_FILE:-/home/ubuntu/.hermes/.env}" LOG_FILE="${DOCS_DEPLOY_LOG:-/home/ubuntu/docs-deploy.log}" LOCK_FILE="${DOCS_DEPLOY_LOCK:-/home/ubuntu/.docs-deploy.lock}" STATE_FILE="${DOCS_DEPLOY_STATE:-/home/ubuntu/.docs-deploy.state}" FAIL_COUNT_FILE="${DOCS_DEPLOY_FAILCOUNT:-/home/ubuntu/.docs-deploy.failcount}" ALERT_MARKER="${DOCS_DEPLOY_ALERT_MARKER:-/home/ubuntu/.docs-deploy.alert}" ALERT_TS_FILE="${DOCS_DEPLOY_LAST_ALERT:-/home/ubuntu/.docs-deploy.last-alert}" # The InferenceBot HQ room (not a secret; overridable for tests). HQ_ROOM_ID="${DOCS_HQ_ROOM_ID:-!cI_tvU4__A3Z4elsA39t5guQm6iY1YHOpWh6AXGuPic}" INTERVAL="${DOCS_POLL_INTERVAL:-60}" FAIL_THRESHOLD="${DOCS_FAIL_THRESHOLD:-3}" ALERT_EVERY_SECS="${DOCS_ALERT_EVERY_SECS:-3600}" cd "$REPO_DIR" log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) [poll] $*" | tee -a "$LOG_FILE" >&2; } # --- drift check ------------------------------------------------------------ # The dedicated checkout deploys; the canonical checkout is where people work # and where the shared venv lives. If the two disagree on build logic or site # config, deploying would publish logic the canonical source cannot reproduce # — fail loudly instead. (A canonical checkout that simply lags behind main # trips this too, by design: pull it, or commit+push its local edits.) drift_check() { # $1 = target sha being deployed local target="${1:-?}" if [ ! -d "$CANONICAL_DIR/.git" ]; then log "DRIFT-CHECK FATAL: canonical checkout $CANONICAL_DIR is missing." log " The poller builds with that checkout's venv and compares its publishing/" log " tree against it. Restore it: git clone https://git.inference.coop/co-op/docs.git $CANONICAL_DIR" return 1 fi local detail mk detail="$(diff -rq --exclude=__pycache__ "$REPO_DIR/publishing" "$CANONICAL_DIR/publishing" 2>&1 || true)" mk="$(diff -q "$REPO_DIR/mkdocs.yml" "$CANONICAL_DIR/mkdocs.yml" 2>&1 || true)" if [ -n "$detail" ] || [ -n "$mk" ]; then log "DRIFT-CHECK FATAL: deploy checkout and canonical checkout ($CANONICAL_DIR) disagree." log " deploy target: ${target:0:12}; canonical checkout HEAD: $(git -C "$CANONICAL_DIR" rev-parse HEAD 2>/dev/null || echo unknown)" if [ -n "$detail" ]; then echo "$detail" | head -15 | while IFS= read -r line; do log " $line"; done fi [ -n "$mk" ] && log " mkdocs.yml differs" log " Deploying from this state could ship logic/config the canonical checkout does not have." log " Fix: git -C $CANONICAL_DIR pull --ff-only (or commit+push its local edits) — the poller retries automatically." return 1 fi log "drift check OK: publishing/ + mkdocs.yml match $CANONICAL_DIR" return 0 } # --- one-shot deploy, run under the deploy lock by the main loop ------------ if [ "${1:-}" = "--deploy-once" ]; then TARGET_SHA="${2:?usage: poll-watcher.sh --deploy-once }" cd "$REPO_DIR" set -e git pull --ff-only origin main drift_check "$TARGET_SHA" # Pre-deploy gate: pinned toolchain present (self-heal installs missing # pins) and every configured plugin pinned. A failure here fails the deploy # loudly instead of leaving the venv silently behind mkdocs.yml. bash publishing/build.sh --check-only "$CANONICAL_VENV" bash publishing/build.sh "$CANONICAL_VENV" bash publishing/deploy.sh exit 0 fi # --- HQ alerting ------------------------------------------------------------- # One message per failure streak, never more than one per hour. State lives in # files so a restart cannot reset a streak or re-fire an alert. hq_alert() { # $1 = short sha, $2 = commit subject local now last now="$(date +%s)" last=0 [ -f "$ALERT_TS_FILE" ] && last="$(cat "$ALERT_TS_FILE" 2>/dev/null || echo 0)" if [ "$((now - last))" -lt "$ALERT_EVERY_SECS" ]; then log "HQ alert suppressed (last alert $(( (now - last) / 60 )) min ago; cap ${ALERT_EVERY_SECS}s) — streak continues" return 0 fi local hs token url body json code hs="$(sed -n 's/^MATRIX_HOMESERVER=//p' "$ENV_FILE" | head -1 | tr -d '"' | tr -d "'")" hs="${hs%/}" token="$(sed -n 's/^MATRIX_ACCESS_TOKEN=//p' "$ENV_FILE" | head -1 | tr -d '"' | tr -d "'")" if [ -z "$hs" ] || [ -z "$token" ]; then log "HQ alert FAILED: MATRIX_HOMESERVER / MATRIX_ACCESS_TOKEN missing in $ENV_FILE" return 1 fi body="Docs deploy FAILED ${FAIL_THRESHOLD}x in a row — commit $1 ('$2') is NOT live on docs.inference.coop. Check journalctl -u docs-deploy-poll or $LOG_FILE on the agent host." json="$(python3 -c 'import json,sys; print(json.dumps({"msgtype":"m.text","body":sys.argv[1]}))' "$body")" url="$hs/_matrix/client/v3/rooms/${HQ_ROOM_ID/!/%21}/send/m.room.message/docs-deploy-$now" code="$(curl -sS --max-time 20 -o /dev/null -w '%{http_code}' -X PUT \ -H 'Content-Type: application/json' \ -H "Authorization: Bearer $token" \ --data "$json" "$url" 2>>"$LOG_FILE" || echo 000)" if [ "$code" = "200" ]; then printf '%s\n' "$now" > "$ALERT_TS_FILE" touch "$ALERT_MARKER" log "HQ alert sent to InferenceBot HQ room (Matrix 200)" else log "HQ alert FAILED: Matrix HTTP $code (will retry on the next failed deploy)" fi } record_failure() { # $1 = sha that failed to deploy local n=1 [ -f "$FAIL_COUNT_FILE" ] && n=$(( $(cat "$FAIL_COUNT_FILE" 2>/dev/null || echo 0) + 1 )) printf '%s\n' "$n" > "$FAIL_COUNT_FILE" log "deploy FAILED for ${1:0:12} (consecutive failures: $n)" if [ "$n" -ge "$FAIL_THRESHOLD" ]; then touch "$ALERT_MARKER" local subject subject="$(git log -1 --format=%s "$1" 2>/dev/null || echo 'unknown commit')" hq_alert "${1:0:12}" "$subject" fi } reset_failures() { local n=0 [ -f "$FAIL_COUNT_FILE" ] && n="$(cat "$FAIL_COUNT_FILE" 2>/dev/null || echo 0)" if [ "${n:-0}" -gt 0 ]; then log "success: consecutive-failure counter reset (was $n); alert marker cleared" fi printf '0\n' > "$FAIL_COUNT_FILE" rm -f "$ALERT_MARKER" } # --- main loop (the watcher proper) ----------------------------------------- # The Cloudron token travels by environment only; it is read from the env file # and never echoed. CLOUDRON_TOKEN="$(sed -n 's/^CLOUDRON_TOKEN=//p' "$ENV_FILE" | head -1 | tr -d '"' | tr -d "'")" export CLOUDRON_TOKEN if [ -z "$CLOUDRON_TOKEN" ]; then log "FATAL: CLOUDRON_TOKEN not found in $ENV_FILE" exit 1 fi # The one-shot deploy child re-runs this script; these must survive into it. export REPO_DIR CANONICAL_DIR CANONICAL_VENV ENV_FILE LOG_FILE log "watcher started: polling $REPO_DIR every ${INTERVAL}s (build venv: $CANONICAL_VENV)" while true; do if ! git fetch --quiet origin main 2>>"$LOG_FILE"; then log "git fetch failed; will retry" sleep "$INTERVAL" continue fi REMOTE="$(git rev-parse origin/main 2>/dev/null || echo none)" # Compare against the commit last DEPLOYED (not the clone's HEAD): a push # made from this very clone would otherwise look like "no change". LAST_DEPLOYED="$(cat "$STATE_FILE" 2>/dev/null || echo none)" if [ "$LAST_DEPLOYED" != "$REMOTE" ]; then log "main moved: ${LAST_DEPLOYED:0:12} -> ${REMOTE:0:12}; deploying" # flock -n shares the receiver's lock: one deploy at a time, ever. The # one-shot child re-executes this script in --deploy-once mode. if flock -n "$LOCK_FILE" bash "$0" --deploy-once "$REMOTE" >>"$LOG_FILE" 2>&1; then printf '%s\n' "$REMOTE" > "$STATE_FILE" log "deploy finished for ${REMOTE:0:12}" reset_failures # A deploy that changed poll-watcher.sh leaves this loop running the old # logic (bash parsed the whole loop before the file changed on disk, so # the rewrite cannot corrupt it — but it also cannot take effect). # Re-exec so the running watcher always matches the deployed one. if [ "$LAST_DEPLOYED" != "none" ] && ! git diff --quiet "$LAST_DEPLOYED" "$REMOTE" -- publishing/poll-watcher.sh 2>/dev/null; then log "poll-watcher.sh changed in ${REMOTE:0:12}; re-exec'ing to pick up the new watcher" exec bash "$0" fi else record_failure "$REMOTE" fi fi sleep "$INTERVAL" done