docs site: serve a robots.txt at the site root

publishing/build.sh

@@ -41,6 +41,9 @@ tar cf - \
   --exclude='requirements.txt' \
   . | tar xf - -C "$SITE_SRC"
 
+# A file that must sit at the site root (crawler rules), not under publishing/.
+cp publishing/robots.txt "$SITE_SRC/robots.txt"
+
 # 2. Build.
 if [ ! -x "$VENV/bin/mkdocs" ]; then
   echo "Creating build venv at $VENV"

publishing/robots.txt

@@ -0,0 +1,3 @@
+# Documentation site crawler rules
+User-agent: *
+Allow: /