docs site: serve a robots.txt at the site root

publishing/build.sh

@@ -41,6 +41,9 @@ tar cf - \
   --exclude='requirements.txt' \
   . | tar xf - -C "$SITE_SRC"
 
+# A file that must sit at the site root (crawler rules), not under publishing/.
+cp publishing/robots.txt "$SITE_SRC/robots.txt"
+
 # 2. Build.
 if [ ! -x "$VENV/bin/mkdocs" ]; then
   echo "Creating build venv at $VENV"