inference-bot
62158c16cc
Renders the repository's Markdown (which stays where it is) into a
member-readable static site with Material for MkDocs: co-op brand colours
and logo, self-hosted OFL Figtree, offline search, no external hosts.
- publishing/build.sh stage .site-src, render .site-build (mkdocs --strict)
- publishing/deploy.sh write the build into the Surfer app container
- publishing/webhook-receiver.py + .service Gitea push hook on main
triggers pull -> build -> deploy (agent host, no new
Cloudron app)
226 lines
8.2 KiB
Python
226 lines
8.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Gitea webhook receiver that rebuilds and republishes the docs site.
|
|
|
|
Flow: a push to main in co-op/docs fires a Gitea webhook -> this receiver
|
|
verifies it -> pulls the latest main -> runs publishing/build.sh ->
|
|
publishing/deploy.sh (which writes the build into the Surfer app).
|
|
|
|
Runs on the agent host (no new Cloudron app). Standard library only.
|
|
|
|
Security model:
|
|
* Only POSTs from the Cloudron box (git.inference.coop's host) are accepted.
|
|
* The payload must carry a valid HMAC-SHA256 signature (X-Gitea-Signature,
|
|
shared secret DEPLOY_WEBHOOK_SECRET). The secret itself is never sent.
|
|
* Only pushes to refs/heads/main of co-op/docs trigger a deploy.
|
|
* The receiver binds 0.0.0.0; an iptables rule installed by the systemd unit
|
|
narrows the port to the Cloudron box as well (defence in depth).
|
|
|
|
Secrets are read from /home/ubuntu/.hermes/.env and are never logged.
|
|
"""
|
|
|
|
import fcntl
|
|
import hashlib
|
|
import hmac
|
|
import ipaddress
|
|
import json
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
|
|
REPO_DIR = os.environ.get("DOCS_REPO_DIR", "/home/ubuntu/repos/docs")
|
|
ENV_FILE = os.environ.get("DOCS_ENV_FILE", "/home/ubuntu/.hermes/.env")
|
|
LOG_FILE = os.environ.get("DOCS_DEPLOY_LOG", "/home/ubuntu/docs-deploy.log")
|
|
LOCK_FILE = os.environ.get("DOCS_DEPLOY_LOCK", "/home/ubuntu/.docs-deploy.lock")
|
|
PORT = int(os.environ.get("DOCS_WEBHOOK_PORT", "8787"))
|
|
EXPECT_REPO = "co-op/docs"
|
|
EXPECT_BRANCH = "refs/heads/main"
|
|
ALLOWED_SOURCES = [
|
|
ipaddress.ip_network(n)
|
|
for n in os.environ.get(
|
|
"DOCS_WEBHOOK_SOURCES",
|
|
# The Cloudron box (hosts git.inference.coop) plus loopback for local tests.
|
|
"179.237.86.116/32,2001:1600:18:206::89/128,127.0.0.1/32,::1/128",
|
|
).split(",")
|
|
if n.strip()
|
|
]
|
|
|
|
log = logging.getLogger("docs-deploy")
|
|
deploy_lock_file = open(LOCK_FILE, "a")
|
|
|
|
|
|
def load_env_file(path):
|
|
"""Return KEY=VALUE pairs from the agent host's env file (values stay local)."""
|
|
values = {}
|
|
try:
|
|
with open(path) as handle:
|
|
for line in handle:
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, _, value = line.partition("=")
|
|
values[key.strip()] = value.strip().strip('"').strip("'")
|
|
except OSError:
|
|
pass
|
|
return values
|
|
|
|
|
|
_FILE_ENV = load_env_file(ENV_FILE)
|
|
|
|
|
|
def secret(name):
|
|
return os.environ.get(name) or _FILE_ENV.get(name) or ""
|
|
|
|
|
|
def source_allowed(address):
|
|
try:
|
|
ip = ipaddress.ip_address(address.split("%")[0])
|
|
except ValueError:
|
|
return False
|
|
return any(ip in net for net in ALLOWED_SOURCES)
|
|
|
|
|
|
def run(cmd, timeout, cwd=None):
|
|
log.info("exec: %s", " ".join(cmd))
|
|
# Hand the build/deploy scripts the credentials they need (from the env
|
|
# file, if not already set). Values are passed by environment only and are
|
|
# never logged.
|
|
env = dict(os.environ)
|
|
for key in ("CLOUDRON_TOKEN", "CLOUDRON_SERVER", "SURFER_APP", "GIT_ASKPASS"):
|
|
if not env.get(key) and secret(key):
|
|
env[key] = secret(key)
|
|
env.setdefault("GIT_ASKPASS", "/home/ubuntu/.hermes/scripts/git_askpass.sh")
|
|
env.setdefault("PATH", os.environ.get("PATH", ""))
|
|
result = subprocess.run(
|
|
cmd, cwd=cwd or REPO_DIR, capture_output=True, text=True, timeout=timeout,
|
|
env=env,
|
|
)
|
|
if result.stdout.strip():
|
|
log.info("stdout: %s", result.stdout.strip()[-2000:])
|
|
if result.returncode != 0 and result.stderr.strip():
|
|
log.error("stderr: %s", result.stderr.strip()[-2000:])
|
|
return result
|
|
|
|
|
|
def deploy(commit_hint=""):
|
|
"""Pull main, rebuild, republish. Returns (ok, log_tail)."""
|
|
try:
|
|
with open("/dev/null") as devnull:
|
|
fcntl.flock(deploy_lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
return False, "another deploy is already running; skipped"
|
|
|
|
tail = []
|
|
try:
|
|
steps = [
|
|
(["git", "pull", "--ff-only", "origin", "main"], 120),
|
|
(["bash", "publishing/build.sh"], 900),
|
|
(["bash", "publishing/deploy.sh"], 600),
|
|
]
|
|
for cmd, timeout in steps:
|
|
result = run(cmd, timeout)
|
|
out = (result.stdout or "") + (result.stderr or "")
|
|
tail.append(f"$ {' '.join(cmd)} -> exit {result.returncode}")
|
|
tail.extend(out.strip().splitlines()[-12:])
|
|
if result.returncode != 0:
|
|
log.error("step failed: %s", " ".join(cmd))
|
|
return False, "\n".join(tail)
|
|
log.info("deploy complete%s", f" ({commit_hint})" if commit_hint else "")
|
|
return True, "\n".join(tail)
|
|
except Exception as exc: # noqa: BLE001 - report, never crash the receiver
|
|
log.exception("deploy failed")
|
|
tail.append(f"error: {exc}")
|
|
return False, "\n".join(tail)
|
|
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
server_version = "docs-deploy/1.0"
|
|
|
|
def log_message(self, format, *args): # noqa: A002 - stdlib signature
|
|
log.info("%s - %s", self.client_address[0], format % args)
|
|
|
|
def _json(self, code, payload):
|
|
body = json.dumps(payload).encode()
|
|
self.send_response(code)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def do_GET(self):
|
|
if self.path == "/healthz":
|
|
return self._json(200, {"status": "ok"})
|
|
return self._json(404, {"error": "not found"})
|
|
|
|
def do_POST(self):
|
|
peer = self.client_address[0]
|
|
if not source_allowed(peer):
|
|
log.warning("rejected: source %s not allowed", peer)
|
|
return self._json(403, {"error": "forbidden"})
|
|
|
|
if self.headers.get("X-Gitea-Event") != "push":
|
|
return self._json(400, {"error": "not a push event"})
|
|
|
|
length = int(self.headers.get("Content-Length") or 0)
|
|
if length <= 0 or length > 1_000_000:
|
|
return self._json(400, {"error": "bad payload size"})
|
|
body = self.rfile.read(length)
|
|
|
|
signature = self.headers.get("X-Gitea-Signature") or ""
|
|
expected = hmac.new(
|
|
secret("DEPLOY_WEBHOOK_SECRET").encode(), body, hashlib.sha256
|
|
).hexdigest()
|
|
if not signature or not hmac.compare_digest(signature, expected):
|
|
log.warning("rejected: bad signature from %s", peer)
|
|
return self._json(403, {"error": "invalid signature"})
|
|
|
|
try:
|
|
payload = json.loads(body)
|
|
except json.JSONDecodeError:
|
|
return self._json(400, {"error": "invalid json"})
|
|
|
|
repo = (payload.get("repository") or {}).get("full_name", "")
|
|
ref = payload.get("ref", "")
|
|
commit = (payload.get("after") or "")[:12]
|
|
pusher = ((payload.get("pusher") or {}).get("login")) or "unknown"
|
|
if repo != EXPECT_REPO or ref != EXPECT_BRANCH:
|
|
log.info("ignored: push to %s %s (not %s %s)", repo, ref, EXPECT_REPO, EXPECT_BRANCH)
|
|
return self._json(200, {"status": "ignored", "reason": "not main on co-op/docs"})
|
|
|
|
log.info("push received: %s %s commit=%s pusher=%s", repo, ref, commit, pusher)
|
|
threading.Thread(target=self._run_deploy, args=(commit,), daemon=True).start()
|
|
return self._json(202, {"status": "accepted", "commit": commit})
|
|
|
|
def _run_deploy(self, commit):
|
|
ok, tail = deploy(commit)
|
|
log.info("deploy result for %s: %s", commit, "ok" if ok else "FAILED")
|
|
log.info("tail:\n%s", tail[-1500:])
|
|
|
|
|
|
def main():
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(message)s",
|
|
handlers=[
|
|
logging.FileHandler(LOG_FILE),
|
|
logging.StreamHandler(sys.stdout),
|
|
],
|
|
)
|
|
if not secret("DEPLOY_WEBHOOK_SECRET"):
|
|
log.error("DEPLOY_WEBHOOK_SECRET is not set; refusing to start")
|
|
sys.exit(1)
|
|
if not secret("CLOUDRON_TOKEN"):
|
|
log.warning("CLOUDRON_TOKEN not set; deploys will fail")
|
|
server = ThreadingHTTPServer(("0.0.0.0", PORT), Handler)
|
|
log.info("listening on 0.0.0.0:%d, repo %s", PORT, REPO_DIR)
|
|
try:
|
|
server.serve_forever()
|
|
except KeyboardInterrupt:
|
|
pass
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|