diff --git a/publishing/build.sh b/publishing/build.sh index de2f8bd..0e49dfa 100755 --- a/publishing/build.sh +++ b/publishing/build.sh @@ -41,6 +41,9 @@ tar cf - \ --exclude='requirements.txt' \ . | tar xf - -C "$SITE_SRC" +# A file that must sit at the site root (crawler rules), not under publishing/. +cp publishing/robots.txt "$SITE_SRC/robots.txt" + # 2. Build. if [ ! -x "$VENV/bin/mkdocs" ]; then echo "Creating build venv at $VENV" diff --git a/publishing/robots.txt b/publishing/robots.txt new file mode 100644 index 0000000..0840e54 --- /dev/null +++ b/publishing/robots.txt @@ -0,0 +1,3 @@ +# Documentation site crawler rules +User-agent: * +Allow: /