From fa1f38ced21df5d5d0c74a5a2b7da771a4906fb3 Mon Sep 17 00:00:00 2001
From: Inferencebot <bot@inference.coop>
Date: Tue, 3 Nov 2026 19:31:12 +0000
Subject: [PATCH] docs site: serve a robots.txt at the site root

---
 publishing/build.sh   | 3 +++
 publishing/robots.txt | 3 +++
 2 files changed, 6 insertions(+)
 create mode 100644 publishing/robots.txt

diff --git a/publishing/build.sh b/publishing/build.sh
index de2f8bd..0e49dfa 100755
--- a/publishing/build.sh
+++ b/publishing/build.sh
@@ -41,6 +41,9 @@ tar cf - \
   --exclude='requirements.txt' \
   . | tar xf - -C "$SITE_SRC"
 
+# A file that must sit at the site root (crawler rules), not under publishing/.
+cp publishing/robots.txt "$SITE_SRC/robots.txt"
+
 # 2. Build.
 if [ ! -x "$VENV/bin/mkdocs" ]; then
   echo "Creating build venv at $VENV"
diff --git a/publishing/robots.txt b/publishing/robots.txt
new file mode 100644
index 0000000..0840e54
--- /dev/null
+++ b/publishing/robots.txt
@@ -0,0 +1,3 @@
+# Documentation site crawler rules
+User-agent: *
+Allow: /