publishing: one canonical build venv; drift check; deploy-failure alerting
The poller's private venv (.docs-publisher/.venv-docs) diverged from the canonical one twice, each time silently delaying a deploy. Consolidate: - poll-watcher.sh builds with the ONE canonical venv (~/repos/docs/.venv-docs) while keeping its dedicated checkout (uncommitted edits in a working clone must never block or leak into a deploy). - New drift check: publishing/ + mkdocs.yml are compared with the canonical checkout before every build; a mismatch fails the deploy loudly instead of shipping logic the canonical source cannot reproduce. - Pre-deploy gate: build.sh --check-only verifies the pinned toolchain and every configured plugin (self-heals missing pins, fails loudly naming an unpinned plugin). - Alerting: 3 consecutive deploy failures -> marker file + ONE Matrix message to the InferenceBot HQ room (hourly cap); counter resets on next success. - The watcher re-execs itself when a deploy changed poll-watcher.sh, so the running loop always matches the deployed one. - webhook receiver (dormant) repointed at the canonical venv too; unit files and README document the layout.
This commit is contained in:
1 parent
1691f79621
commit
20d384be87
5 files changed
+222
-21
No files matched your search
+155
-11
@@ -12,20 +12,157 @@
|
||||
# Runs on the agent host as a systemd service (docs-deploy-poll.service).
|
||||
# Deploy steps are serialised with the same lock the webhook receiver uses, so
|
||||
# the two mechanisms can never deploy concurrently.
|
||||
#
|
||||
# ONE BUILD ENVIRONMENT (2026-11-12 consolidation): the watcher still builds
|
||||
# from its own dedicated checkout ($DOCS_REPO_DIR) — uncommitted edits in a
|
||||
# working clone must never block or leak into a deploy — but the build VENV is
|
||||
# the single canonical one: $DOCS_CANONICAL_DIR/.venv-docs. The poller's old
|
||||
# private venv (which silently diverged twice) is gone. Before every build:
|
||||
#
|
||||
# * DRIFT CHECK — publishing/ and mkdocs.yml in this checkout are compared
|
||||
# with the canonical checkout; any mismatch FAILS the deploy loudly (a
|
||||
# deploy of logic the canonical source cannot reproduce is how the site
|
||||
# silently drifted before).
|
||||
# * PRE-FLIGHT — `build.sh --check-only` verifies the pinned toolchain and
|
||||
# every plugin mkdocs.yml configures, installing missing pins (self-heal)
|
||||
# or failing loudly naming a plugin that was added without being pinned.
|
||||
#
|
||||
# ALERTING: after $DOCS_FAIL_THRESHOLD consecutive deploy failures the watcher
|
||||
# writes a marker file and sends ONE message to the InferenceBot HQ Matrix
|
||||
# room (never more than one per hour); the counter resets on the next success.
|
||||
# A silent retry loop is how two commits once sat undeployed for ~15 minutes
|
||||
# without anyone knowing.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
REPO_DIR="${DOCS_REPO_DIR:-/home/ubuntu/repos/docs}"
|
||||
REPO_DIR="${DOCS_REPO_DIR:-/home/ubuntu/.docs-publisher}"
|
||||
CANONICAL_DIR="${DOCS_CANONICAL_DIR:-/home/ubuntu/repos/docs}"
|
||||
CANONICAL_VENV="${DOCS_CANONICAL_VENV:-$CANONICAL_DIR/.venv-docs}"
|
||||
ENV_FILE="${DOCS_ENV_FILE:-/home/ubuntu/.hermes/.env}"
|
||||
LOG_FILE="${DOCS_DEPLOY_LOG:-/home/ubuntu/docs-deploy.log}"
|
||||
LOCK_FILE="${DOCS_DEPLOY_LOCK:-/home/ubuntu/.docs-deploy.lock}"
|
||||
STATE_FILE="${DOCS_DEPLOY_STATE:-/home/ubuntu/.docs-deploy.state}"
|
||||
FAIL_COUNT_FILE="${DOCS_DEPLOY_FAILCOUNT:-/home/ubuntu/.docs-deploy.failcount}"
|
||||
ALERT_MARKER="${DOCS_DEPLOY_ALERT_MARKER:-/home/ubuntu/.docs-deploy.alert}"
|
||||
ALERT_TS_FILE="${DOCS_DEPLOY_LAST_ALERT:-/home/ubuntu/.docs-deploy.last-alert}"
|
||||
# The InferenceBot HQ room (not a secret; overridable for tests).
|
||||
HQ_ROOM_ID="${DOCS_HQ_ROOM_ID:-!cI_tvU4__A3Z4elsA39t5guQm6iY1YHOpWh6AXGuPic}"
|
||||
INTERVAL="${DOCS_POLL_INTERVAL:-60}"
|
||||
FAIL_THRESHOLD="${DOCS_FAIL_THRESHOLD:-3}"
|
||||
ALERT_EVERY_SECS="${DOCS_ALERT_EVERY_SECS:-3600}"
|
||||
|
||||
cd "$REPO_DIR"
|
||||
|
||||
log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) [poll] $*" | tee -a "$LOG_FILE" >&2; }
|
||||
|
||||
# --- drift check ------------------------------------------------------------
|
||||
# The dedicated checkout deploys; the canonical checkout is where people work
|
||||
# and where the shared venv lives. If the two disagree on build logic or site
|
||||
# config, deploying would publish logic the canonical source cannot reproduce
|
||||
# — fail loudly instead. (A canonical checkout that simply lags behind main
|
||||
# trips this too, by design: pull it, or commit+push its local edits.)
|
||||
drift_check() { # $1 = target sha being deployed
|
||||
local target="${1:-?}"
|
||||
if [ ! -d "$CANONICAL_DIR/.git" ]; then
|
||||
log "DRIFT-CHECK FATAL: canonical checkout $CANONICAL_DIR is missing."
|
||||
log " The poller builds with that checkout's venv and compares its publishing/"
|
||||
log " tree against it. Restore it: git clone https://git.inference.coop/co-op/docs.git $CANONICAL_DIR"
|
||||
return 1
|
||||
fi
|
||||
local detail mk
|
||||
detail="$(diff -rq --exclude=__pycache__ "$REPO_DIR/publishing" "$CANONICAL_DIR/publishing" 2>&1 || true)"
|
||||
mk="$(diff -q "$REPO_DIR/mkdocs.yml" "$CANONICAL_DIR/mkdocs.yml" 2>&1 || true)"
|
||||
if [ -n "$detail" ] || [ -n "$mk" ]; then
|
||||
log "DRIFT-CHECK FATAL: deploy checkout and canonical checkout ($CANONICAL_DIR) disagree."
|
||||
log " deploy target: ${target:0:12}; canonical checkout HEAD: $(git -C "$CANONICAL_DIR" rev-parse HEAD 2>/dev/null || echo unknown)"
|
||||
if [ -n "$detail" ]; then
|
||||
echo "$detail" | head -15 | while IFS= read -r line; do log " $line"; done
|
||||
fi
|
||||
[ -n "$mk" ] && log " mkdocs.yml differs"
|
||||
log " Deploying from this state could ship logic/config the canonical checkout does not have."
|
||||
log " Fix: git -C $CANONICAL_DIR pull --ff-only (or commit+push its local edits) — the poller retries automatically."
|
||||
return 1
|
||||
fi
|
||||
log "drift check OK: publishing/ + mkdocs.yml match $CANONICAL_DIR"
|
||||
return 0
|
||||
}
|
||||
|
||||
# --- one-shot deploy, run under the deploy lock by the main loop ------------
|
||||
if [ "${1:-}" = "--deploy-once" ]; then
|
||||
TARGET_SHA="${2:?usage: poll-watcher.sh --deploy-once <sha>}"
|
||||
cd "$REPO_DIR"
|
||||
set -e
|
||||
git pull --ff-only origin main
|
||||
drift_check "$TARGET_SHA"
|
||||
# Pre-deploy gate: pinned toolchain present (self-heal installs missing
|
||||
# pins) and every configured plugin pinned. A failure here fails the deploy
|
||||
# loudly instead of leaving the venv silently behind mkdocs.yml.
|
||||
bash publishing/build.sh --check-only "$CANONICAL_VENV"
|
||||
bash publishing/build.sh "$CANONICAL_VENV"
|
||||
bash publishing/deploy.sh
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- HQ alerting -------------------------------------------------------------
|
||||
# One message per failure streak, never more than one per hour. State lives in
|
||||
# files so a restart cannot reset a streak or re-fire an alert.
|
||||
hq_alert() { # $1 = short sha, $2 = commit subject
|
||||
local now last
|
||||
now="$(date +%s)"
|
||||
last=0
|
||||
[ -f "$ALERT_TS_FILE" ] && last="$(cat "$ALERT_TS_FILE" 2>/dev/null || echo 0)"
|
||||
if [ "$((now - last))" -lt "$ALERT_EVERY_SECS" ]; then
|
||||
log "HQ alert suppressed (last alert $(( (now - last) / 60 )) min ago; cap ${ALERT_EVERY_SECS}s) — streak continues"
|
||||
return 0
|
||||
fi
|
||||
local hs token url body json code
|
||||
hs="$(sed -n 's/^MATRIX_HOMESERVER=//p' "$ENV_FILE" | head -1 | tr -d '"' | tr -d "'")"
|
||||
hs="${hs%/}"
|
||||
token="$(sed -n 's/^MATRIX_ACCESS_TOKEN=//p' "$ENV_FILE" | head -1 | tr -d '"' | tr -d "'")"
|
||||
if [ -z "$hs" ] || [ -z "$token" ]; then
|
||||
log "HQ alert FAILED: MATRIX_HOMESERVER / MATRIX_ACCESS_TOKEN missing in $ENV_FILE"
|
||||
return 1
|
||||
fi
|
||||
body="Docs deploy FAILED ${FAIL_THRESHOLD}x in a row — commit $1 ('$2') is NOT live on docs.inference.coop. Check journalctl -u docs-deploy-poll or $LOG_FILE on the agent host."
|
||||
json="$(python3 -c 'import json,sys; print(json.dumps({"msgtype":"m.text","body":sys.argv[1]}))' "$body")"
|
||||
url="$hs/_matrix/client/v3/rooms/${HQ_ROOM_ID/!/%21}/send/m.room.message/docs-deploy-$now"
|
||||
code="$(curl -sS --max-time 20 -o /dev/null -w '%{http_code}' -X PUT \
|
||||
-H 'Content-Type: application/json' \
|
||||
-H "Authorization: Bearer $token" \
|
||||
--data "$json" "$url" 2>>"$LOG_FILE" || echo 000)"
|
||||
if [ "$code" = "200" ]; then
|
||||
printf '%s\n' "$now" > "$ALERT_TS_FILE"
|
||||
touch "$ALERT_MARKER"
|
||||
log "HQ alert sent to InferenceBot HQ room (Matrix 200)"
|
||||
else
|
||||
log "HQ alert FAILED: Matrix HTTP $code (will retry on the next failed deploy)"
|
||||
fi
|
||||
}
|
||||
|
||||
record_failure() { # $1 = sha that failed to deploy
|
||||
local n=1
|
||||
[ -f "$FAIL_COUNT_FILE" ] && n=$(( $(cat "$FAIL_COUNT_FILE" 2>/dev/null || echo 0) + 1 ))
|
||||
printf '%s\n' "$n" > "$FAIL_COUNT_FILE"
|
||||
log "deploy FAILED for ${1:0:12} (consecutive failures: $n)"
|
||||
if [ "$n" -ge "$FAIL_THRESHOLD" ]; then
|
||||
touch "$ALERT_MARKER"
|
||||
local subject
|
||||
subject="$(git log -1 --format=%s "$1" 2>/dev/null || echo 'unknown commit')"
|
||||
hq_alert "${1:0:12}" "$subject"
|
||||
fi
|
||||
}
|
||||
|
||||
reset_failures() {
|
||||
local n=0
|
||||
[ -f "$FAIL_COUNT_FILE" ] && n="$(cat "$FAIL_COUNT_FILE" 2>/dev/null || echo 0)"
|
||||
if [ "${n:-0}" -gt 0 ]; then
|
||||
log "success: consecutive-failure counter reset (was $n); alert marker cleared"
|
||||
fi
|
||||
printf '0\n' > "$FAIL_COUNT_FILE"
|
||||
rm -f "$ALERT_MARKER"
|
||||
}
|
||||
|
||||
# --- main loop (the watcher proper) -----------------------------------------
|
||||
# The Cloudron token travels by environment only; it is read from the env file
|
||||
# and never echoed.
|
||||
CLOUDRON_TOKEN="$(sed -n 's/^CLOUDRON_TOKEN=//p' "$ENV_FILE" | head -1 | tr -d '"' | tr -d "'")"
|
||||
@@ -35,7 +172,10 @@ if [ -z "$CLOUDRON_TOKEN" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "watcher started: polling $REPO_DIR every ${INTERVAL}s"
|
||||
# The one-shot deploy child re-runs this script; these must survive into it.
|
||||
export REPO_DIR CANONICAL_DIR CANONICAL_VENV ENV_FILE LOG_FILE
|
||||
|
||||
log "watcher started: polling $REPO_DIR every ${INTERVAL}s (build venv: $CANONICAL_VENV)"
|
||||
|
||||
while true; do
|
||||
if ! git fetch --quiet origin main 2>>"$LOG_FILE"; then
|
||||
@@ -51,18 +191,22 @@ while true; do
|
||||
|
||||
if [ "$LAST_DEPLOYED" != "$REMOTE" ]; then
|
||||
log "main moved: ${LAST_DEPLOYED:0:12} -> ${REMOTE:0:12}; deploying"
|
||||
# flock -n shares the receiver's lock: one deploy at a time, ever.
|
||||
if flock -n "$LOCK_FILE" bash -c '
|
||||
set -e
|
||||
cd "$1"
|
||||
git pull --ff-only origin main
|
||||
bash publishing/build.sh
|
||||
bash publishing/deploy.sh
|
||||
' _ "$REPO_DIR" >>"$LOG_FILE" 2>&1; then
|
||||
# flock -n shares the receiver's lock: one deploy at a time, ever. The
|
||||
# one-shot child re-executes this script in --deploy-once mode.
|
||||
if flock -n "$LOCK_FILE" bash "$0" --deploy-once "$REMOTE" >>"$LOG_FILE" 2>&1; then
|
||||
printf '%s\n' "$REMOTE" > "$STATE_FILE"
|
||||
log "deploy finished for ${REMOTE:0:12}"
|
||||
reset_failures
|
||||
# A deploy that changed poll-watcher.sh leaves this loop running the old
|
||||
# logic (bash parsed the whole loop before the file changed on disk, so
|
||||
# the rewrite cannot corrupt it — but it also cannot take effect).
|
||||
# Re-exec so the running watcher always matches the deployed one.
|
||||
if [ "$LAST_DEPLOYED" != "none" ] && ! git diff --quiet "$LAST_DEPLOYED" "$REMOTE" -- publishing/poll-watcher.sh 2>/dev/null; then
|
||||
log "poll-watcher.sh changed in ${REMOTE:0:12}; re-exec'ing to pick up the new watcher"
|
||||
exec bash "$0"
|
||||
fi
|
||||
else
|
||||
log "deploy FAILED for ${REMOTE:0:12} (see log above)"
|
||||
record_failure "$REMOTE"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
Reference in new issue
Block a user