Add release.sh; make the verify block honest; print the seed script's own revision

Three drifts between this repo and reality, found while moving the per-member key
seed into the app:

- The 'verify after every image deploy' block asserted the CREAM palette
  (rgb(250,248,245)). The deployed palette has been the green wash
  (rgb(234,243,234) with white text areas) for some time, so a correct deploy
  would have read as a failure — and invited someone to 'fix' the CSS back.
- The block still verified against librechat-test.inference.coop; the swap to
  chat.inference.coop happened 2026-10-21 and that alias is not provisioned.
- Nothing checked that the scheduler task's script exists in the running image,
  which is exactly the failure mode a manifest repoint can introduce.

release.sh encodes the documented release flow (dirty-tree guard, always
--no-cache, pin-before-deploy) and its verify subcommand runs checks 1-5 plus the
seed task's log lines. sync-keys.js now prints its own md5 prefix on every run,
so a stale volume copy can never hide again — the repo copy's md5 is the
authority.

Verified: ./release.sh verify passes all six checks against the live app;
./release.sh volume sync-keys.js confirms md5 parity.

README.md

@@ -1,9 +1,11 @@
 # LibreChat sandbox — Inference Co-op customization
 
-Source of truth for the LibreChat evaluation instance at
+Source of truth for the co-op's chat instance at
-**librechat-test.inference.coop** (Cloudron app `4023ccbf-d57a-4500-97df-14de9cf3516e`,
+**chat.inference.coop** (Cloudron app `4023ccbf-d57a-4500-97df-14de9cf3516e`,
 appstore package `ai.librechat.cloudronapp@1.4.1`, which bundles **LibreChat
 v0.8.8** on image `cloudron/ai.librechat.cloudronapp:202610150224320000`).
+`librechat-test.inference.coop` was the pre-swap hostname (the swap to
+`chat.` happened 2026-10-21; the test alias is still not provisioned).
 
 The deployment has **two independent layers** — know which one you are changing:
 
@@ -32,6 +34,7 @@ md5sum librechat.yaml mcp/server.py mcp/image_server.py
 | Env vars (env-file wins!) | `/app/data/env` (volume, sourced LAST by start.sh) | yes | documented below |
 | Env vars (Cloudron-managed) | `cloudron env` (process env) | yes | documented below |
 | MCP servers | `/app/data/mcp/server.py` + `mcp/image_server.py` (volume) | yes | `mcp/` |
+| Per-member key seed | `/app/data/sync-keys.js` (volume; run by the manifest's `scheduler` task) | yes | `sync-keys.js` |
 | Brand assets + branding CSS/JS | baked into image overlay | no — rebuilt per image | `assets/` |
 | `CloudronManifest.json` | repo only (controls what `cloudron update` deploys) | n/a | official appstore manifest **except `dockerImage`** |
 
@@ -114,6 +117,13 @@ cloudron --server my.inference.coop --token "$CLOUDRON_TOKEN" update \
   --app 4023ccbf-d57a-4500-97df-14de9cf3516e --force
 ```
 
+`./release.sh release` runs this same sequence with both traps handled — it
+refuses to build from a dirty tree, always passes `--no-cache`, writes the tag
+into the manifest before deploying, then runs the verification block. `build`
+needs docker, so it runs on whoever's machine has it (deliberately not the agent
+VM); `verify`, `volume` and `pin`/`deploy` need only `CLOUDRON_TOKEN`.
+`./release.sh verify` is safe to run any time.
+
 Rollback: repoint `CloudronManifest.json.dockerImage` to the previous tag
 (every build is tagged `20261019-<git-sha>`; the base image
 `cloudron/ai.librechat.cloudronapp:<tag>` is the pre-overlay fallback) and
@@ -132,12 +142,24 @@ cloudron --server my.inference.coop --token "$CLOUDRON_TOKEN" exec --app $APP \
   -- md5sum /app/code/librechat/client/dist/assets/custom.css
 md5sum assets/custom.css
 # 3. Served HTML carries the injection
-curl -s https://librechat-test.inference.coop/login | grep -Eo 'assets/custom2?\.js|assets/custom\.css'
+curl -s https://chat.inference.coop/login | grep -Eo 'assets/custom2?\.js|assets/custom\.css'
-# 4. Rendered DOM shows it (browser devtools or a browser tool):
+# 4. Rendered DOM shows it (browser devtools or a browser tool). The DEPLOYED
-#    computed background of div.bg-surface-primary == rgb(250, 248, 245);
+#    palette is the GREEN WASH with white text areas — an earlier version of this
-#    the "Don't have an account?" <p> computes display: none.
+#    block asserted cream (rgb(250,248,245)), which would have led you to "fix"
+#    a correct deploy:
+#      computed background of div.bg-surface-primary == rgb(234, 243, 234);
+#      chat text areas / composer == rgb(255, 255, 255);
+#      dark scheme == rgb(34, 39, 31);
+#      the "Don't have an account?" <p> computes display: none.
+# 5. The scheduler task's script exists in THIS image — a manifest repoint
+#    deployed before its image lands leaves the key seed broken on schedule:
+cloudron --server my.inference.coop --token "$CLOUDRON_TOKEN" exec --app $APP \
+  -- test -f /app/data/sync-keys.js && echo "seed path present"
 ```
 
+`./release.sh verify` runs checks 1-5 plus the seed task's own log lines and the
+running script's revision hash, and exits non-zero on any failure.
+
 ### `CloudronManifest.json` rules
 
 - The **only** field you may change is `dockerImage` — it must point at the

release.sh

@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+# release.sh — build, pin, deploy and verify the LibreChat overlay image.
+#
+# Why this script exists: two traps on this app have each silently eaten a
+# deploy of this overlay, and both look like success.
+#   1. `cloudron update --image <tag>` is IGNORED when CloudronManifest.json is
+#      present in the cwd — the CLI deploys manifest.dockerImage instead, while
+#      still printing the --image value. So the built tag MUST be written into
+#      the manifest first (that is what `pin` does).
+#   2. The builder cache can serve stale COPY layers even when file content
+#      changed. Builds here always pass --no-cache.
+#
+# Usage (from the repo root):
+#   ./release.sh build              build + push the image, tag <utcdate>-<gitsha>
+#   ./release.sh pin <tag>          write <tag> into CloudronManifest.json, commit, push
+#   ./release.sh deploy             cloudron update --force, then verify
+#   ./release.sh release            build -> pin -> deploy -> verify   (the whole thing)
+#   ./release.sh verify             live checks only (safe, no changes)
+#   ./release.sh volume FILE...     push volume-resident files, assert md5 parity
+#
+# `build` needs docker + a registry login, so it runs on whoever's machine has
+# them — deliberately NOT on the agent VM, which holds no build capability.
+# `pin`, `deploy`, `verify` and `volume` only need CLOUDRON_TOKEN.
+#
+# Rollback: `./release.sh pin <previous-tag> && ./release.sh deploy`.
+set -euo pipefail
+
+APP=${APP:-4023ccbf-d57a-4500-97df-14de9cf3516e}
+REGISTRY_REPO=${REGISTRY_REPO:-registry.inference.coop/co-op/librechat-custom}
+SERVER=${SERVER:-my.inference.coop}
+BASE_IMAGE=${BASE_IMAGE:-cloudron/ai.librechat.cloudronapp:202610150224320000}
+MANIFEST=CloudronManifest.json
+
+die() { printf 'release.sh: %s\n' "$*" >&2; exit 1; }
+ok()   { printf '  \033[32mPASS\033[0m %s\n' "$*"; }
+bad()  { printf '  \033[31mFAIL\033[0m %s\n' "$*"; FAILED=1; }
+need_token() { : "${CLOUDRON_TOKEN:?export CLOUDRON_TOKEN first (set -a; . ~/.hermes/.env; set +a)}"; }
+cr() { cloudron --server "$SERVER" --token "$CLOUDRON_TOKEN" "$@"; }
+manifest_tag() { python3 -c "import json;print(json.load(open('$MANIFEST'))['dockerImage'].rsplit(':',1)[-1])"; }
+
+cmd_build() {
+  command -v docker >/dev/null || die "no docker on this host — builds run where docker + the registry login live, not on the agent VM"
+  local tag="${TAG:-$(date -u +%Y%m%d)-$(git rev-parse --short HEAD)}"
+  [ -z "$(git status --porcelain)" ] || die "working tree is dirty — commit first, the tag carries the git sha"
+  printf 'building %s:%s (base %s)\n' "$REGISTRY_REPO" "$tag" "$BASE_IMAGE"
+  cloudron builder build --no-cache --repository "$REGISTRY_REPO" --tag "$tag" || die "build failed"
+  printf '\nbuilt tag: %s\nnext: ./release.sh pin %s\n' "$tag" "$tag"
+}
+
+cmd_pin() {
+  local tag="${1:-}" ; [ -n "$tag" ] || die "usage: release.sh pin <tag>"
+  python3 - "$tag" <<'PY'
+import json, sys
+tag = sys.argv[1]
+m = json.load(open('CloudronManifest.json'))
+old = m['dockerImage']
+m['dockerImage'] = old.rsplit(':', 1)[0] + ':' + tag
+json.dump(m, open('CloudronManifest.json', 'w'), indent=4)
+open('CloudronManifest.json', 'a').write('\n')
+print(f'dockerImage: {old} -> {m["dockerImage"]}')
+PY
+  git add "$MANIFEST"
+  git commit -q -m "Pin manifest to $tag"
+  git push -q origin HEAD:master
+  ok "pinned + pushed $tag"
+}
+
+cmd_deploy() {
+  need_token
+  local want; want=$(manifest_tag)
+  printf 'deploying manifest (image tag %s)\n' "$want"
+  cr update --app "$APP" --force
+  cmd_verify
+}
+
+cmd_verify() {
+  need_token
+  FAILED=0
+  local api_tag repo_tag css_repo css_live
+  api_tag=$(curl -s -H "Authorization: Bearer $CLOUDRON_TOKEN" "https://$SERVER/api/v1/apps/$APP" \
+            | python3 -c "import json,sys;print(json.load(sys.stdin)['manifest']['dockerImage'].rsplit(':',1)[-1])")
+  repo_tag=$(manifest_tag)
+  [ "$api_tag" = "$repo_tag" ] && ok "running image tag == repo pin ($api_tag)" \
+    || bad "running image tag $api_tag != repo pin $repo_tag (did you pin before deploying?)"
+
+  css_repo=$(md5sum assets/custom.css | cut -d' ' -f1)
+  css_live=$(cr exec --app "$APP" -- md5sum /app/code/librechat/client/dist/assets/custom.css 2>/dev/null | cut -d' ' -f1)
+  [ "$css_repo" = "$css_live" ] && ok "container custom.css == repo (no stale build cache)" \
+    || bad "container custom.css $css_live != repo $css_repo (stale COPY layer — rebuild --no-cache)"
+
+  local html; html=$(curl -s "https://chat.inference.coop/login")
+  grep -qE 'assets/custom\.css' <<<"$html" && ok "served HTML links custom.css" || bad "served HTML missing custom.css link"
+  grep -qE 'assets/custom2\.js' <<<"$html" && ok "served HTML links custom2.js" || bad "served HTML missing custom2.js link"
+  curl -s "https://chat.inference.coop/" | grep -qE 'x-coop-build' && ok "x-coop-build marker present" \
+    || bad "no x-coop-build marker (stale service-worker clients will not self-heal)"
+
+  # The scheduler task must point at a file that exists in the RUNNING image.
+  # Catches a manifest repoint deployed before its image landed (2026-10-25).
+  local sched_cmd
+  sched_cmd=$(python3 -c "
+import json
+app=json.load(open('$MANIFEST'))
+t=app.get('addons',{}).get('scheduler',{}).get('librechat_key_sync')
+print(t['command'] if t else '')")
+  if [ -n "$sched_cmd" ]; then
+    cmd_path=$(grep -oE 'node [^ ]+' <<<"$sched_cmd" | awk '{print $2}')
+    if cr exec --app "$APP" -- test -f "$cmd_path" 2>/dev/null; then
+      ok "scheduler command path exists in the image ($cmd_path)"
+    else
+      bad "scheduler points at $cmd_path which is NOT in the running image — the seed will fail until the image lands"
+    fi
+  fi
+
+  printf '\nkey-sync task log lines from the app log:\n'
+  cr logs --app "$APP" --lines 4000 2>/dev/null | grep -E "sync-keys.js revision|creds check|portal feed|done:|FATAL" | tail -8 | sed 's/^/    /'
+  printf '  compare the revision hash above with: md5sum sync-keys.js -> %s\n' "$(md5sum sync-keys.js | cut -c1-8)"
+  [ "$FAILED" = 0 ] && printf '\nall checks passed\n' || { printf '\nCHECKS FAILED\n' >&2; exit 1; }
+}
+
+cmd_volume() {
+  need_token
+  [ "$#" -ge 1 ] || die "usage: release.sh volume FILE [FILE...]"
+  FAILED=0
+  for f in "$@"; do
+    [ -f "$f" ] || die "no such file: $f"
+    local base; base=$(basename "$f")
+    base64 -w0 "$f" | cr exec --app "$APP" -- sh -c "cat > /tmp/vpush && base64 -d /tmp/vpush > /app/data/$base && rm -f /tmp/vpush" 
+    local local_md5 live_md5
+    local_md5=$(md5sum "$f" | cut -d' ' -f1)
+    live_md5=$(cr exec --app "$APP" -- md5sum "/app/data/$base" | cut -d' ' -f1)
+    [ "$local_md5" = "$live_md5" ] && ok "volume $base == repo ($local_md5)" || bad "volume $base $live_md5 != repo $local_md5"
+  done
+  [ "$FAILED" = 0 ] || exit 1
+}
+
+case "${1:-}" in
+  build)  shift; cmd_build "$@" ;;
+  pin)    shift; cmd_pin "$@" ;;
+  deploy) shift; cmd_deploy "$@" ;;
+  verify) shift; cmd_verify "$@" ;;
+  volume) shift; cmd_volume "$@" ;;
+  release) shift; cmd_build "$@"; t=$(manifest_tag); cmd_deploy ;;
+  *) sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;;
+esac

sync-keys.js

@@ -53,6 +53,15 @@ const MONGO_URL = process.env.CLOUDRON_MONGODB_URL || process.env.MONGODB_URL;
 const PORTAL_URL = process.env.PORTAL_URL || "https://portal.inference.coop";
 const SYNC_TOKEN = process.env.LIBRECHAT_SYNC_TOKEN || "";
 const ENDPOINT_NAME = "Inference Co-op";
+
+// Which revision is actually running? The script lives on the persistent volume,
+// so it can be pushed out of band; printing its own hash on every run means a
+// stale copy can never hide — compare this with `md5sum sync-keys.js` in the repo.
+const SELF_HASH = (() => {
+  try {
+    return require("crypto").createHash("md5").update(fs.readFileSync(__filename)).digest("hex").slice(0, 8);
+  } catch (e) { return "unknown"; }
+})();
 const MCP_PLUGIN_KEY = "mcp_inference-image" // Constants.mcp_prefix is "mcp_" (underscore), not "mcp:";
 const MCP_AUTH_FIELD = "IC_API_KEY";
 
@@ -129,6 +138,7 @@ const mongoose = require("/app/code/librechat/node_modules/mongoose");
   console.log(`creds check: ${credsCheck} (${sample.length} existing row(s))`);
 
   let seeded = 0, skipped = 0;
+  console.log(`sync-keys.js revision ${SELF_HASH} (repo copy md5 starts with this)`);
   for (const m of members) {
     const email = String(m.email || "").toLowerCase();
     const sk = String(m.key || "");