From bb2503bb47ade03a3536eb505c79a293482ad7f6 Mon Sep 17 00:00:00 2001 From: Inferencebot Date: Mon, 26 Oct 2026 01:34:22 +0000 Subject: [PATCH] Add release.sh; make the verify block honest; print the seed script's own revision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three drifts between this repo and reality, found while moving the per-member key seed into the app: - The 'verify after every image deploy' block asserted the CREAM palette (rgb(250,248,245)). The deployed palette has been the green wash (rgb(234,243,234) with white text areas) for some time, so a correct deploy would have read as a failure — and invited someone to 'fix' the CSS back. - The block still verified against librechat-test.inference.coop; the swap to chat.inference.coop happened 2026-10-21 and that alias is not provisioned. - Nothing checked that the scheduler task's script exists in the running image, which is exactly the failure mode a manifest repoint can introduce. release.sh encodes the documented release flow (dirty-tree guard, always --no-cache, pin-before-deploy) and its verify subcommand runs checks 1-5 plus the seed task's log lines. sync-keys.js now prints its own md5 prefix on every run, so a stale volume copy can never hide again — the repo copy's md5 is the authority. Verified: ./release.sh verify passes all six checks against the live app; ./release.sh volume sync-keys.js confirms md5 parity. --- README.md | 34 +++++++++--- release.sh | 144 +++++++++++++++++++++++++++++++++++++++++++++++++++ sync-keys.js | 10 ++++ 3 files changed, 182 insertions(+), 6 deletions(-) create mode 100755 release.sh diff --git a/README.md b/README.md index 2c6ca3b..8c3c512 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ # LibreChat sandbox — Inference Co-op customization -Source of truth for the LibreChat evaluation instance at -**librechat-test.inference.coop** (Cloudron app `4023ccbf-d57a-4500-97df-14de9cf3516e`, +Source of truth for the co-op's chat instance at +**chat.inference.coop** (Cloudron app `4023ccbf-d57a-4500-97df-14de9cf3516e`, appstore package `ai.librechat.cloudronapp@1.4.1`, which bundles **LibreChat v0.8.8** on image `cloudron/ai.librechat.cloudronapp:202610150224320000`). +`librechat-test.inference.coop` was the pre-swap hostname (the swap to +`chat.` happened 2026-10-21; the test alias is still not provisioned). The deployment has **two independent layers** — know which one you are changing: @@ -32,6 +34,7 @@ md5sum librechat.yaml mcp/server.py mcp/image_server.py | Env vars (env-file wins!) | `/app/data/env` (volume, sourced LAST by start.sh) | yes | documented below | | Env vars (Cloudron-managed) | `cloudron env` (process env) | yes | documented below | | MCP servers | `/app/data/mcp/server.py` + `mcp/image_server.py` (volume) | yes | `mcp/` | +| Per-member key seed | `/app/data/sync-keys.js` (volume; run by the manifest's `scheduler` task) | yes | `sync-keys.js` | | Brand assets + branding CSS/JS | baked into image overlay | no — rebuilt per image | `assets/` | | `CloudronManifest.json` | repo only (controls what `cloudron update` deploys) | n/a | official appstore manifest **except `dockerImage`** | @@ -114,6 +117,13 @@ cloudron --server my.inference.coop --token "$CLOUDRON_TOKEN" update \ --app 4023ccbf-d57a-4500-97df-14de9cf3516e --force ``` +`./release.sh release` runs this same sequence with both traps handled — it +refuses to build from a dirty tree, always passes `--no-cache`, writes the tag +into the manifest before deploying, then runs the verification block. `build` +needs docker, so it runs on whoever's machine has it (deliberately not the agent +VM); `verify`, `volume` and `pin`/`deploy` need only `CLOUDRON_TOKEN`. +`./release.sh verify` is safe to run any time. + Rollback: repoint `CloudronManifest.json.dockerImage` to the previous tag (every build is tagged `20261019-`; the base image `cloudron/ai.librechat.cloudronapp:` is the pre-overlay fallback) and @@ -132,12 +142,24 @@ cloudron --server my.inference.coop --token "$CLOUDRON_TOKEN" exec --app $APP \ -- md5sum /app/code/librechat/client/dist/assets/custom.css md5sum assets/custom.css # 3. Served HTML carries the injection -curl -s https://librechat-test.inference.coop/login | grep -Eo 'assets/custom2?\.js|assets/custom\.css' -# 4. Rendered DOM shows it (browser devtools or a browser tool): -# computed background of div.bg-surface-primary == rgb(250, 248, 245); -# the "Don't have an account?"

computes display: none. +curl -s https://chat.inference.coop/login | grep -Eo 'assets/custom2?\.js|assets/custom\.css' +# 4. Rendered DOM shows it (browser devtools or a browser tool). The DEPLOYED +# palette is the GREEN WASH with white text areas — an earlier version of this +# block asserted cream (rgb(250,248,245)), which would have led you to "fix" +# a correct deploy: +# computed background of div.bg-surface-primary == rgb(234, 243, 234); +# chat text areas / composer == rgb(255, 255, 255); +# dark scheme == rgb(34, 39, 31); +# the "Don't have an account?"

computes display: none. +# 5. The scheduler task's script exists in THIS image — a manifest repoint +# deployed before its image lands leaves the key seed broken on schedule: +cloudron --server my.inference.coop --token "$CLOUDRON_TOKEN" exec --app $APP \ + -- test -f /app/data/sync-keys.js && echo "seed path present" ``` +`./release.sh verify` runs checks 1-5 plus the seed task's own log lines and the +running script's revision hash, and exits non-zero on any failure. + ### `CloudronManifest.json` rules - The **only** field you may change is `dockerImage` — it must point at the diff --git a/release.sh b/release.sh new file mode 100755 index 0000000..73c579d --- /dev/null +++ b/release.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# release.sh — build, pin, deploy and verify the LibreChat overlay image. +# +# Why this script exists: two traps on this app have each silently eaten a +# deploy of this overlay, and both look like success. +# 1. `cloudron update --image ` is IGNORED when CloudronManifest.json is +# present in the cwd — the CLI deploys manifest.dockerImage instead, while +# still printing the --image value. So the built tag MUST be written into +# the manifest first (that is what `pin` does). +# 2. The builder cache can serve stale COPY layers even when file content +# changed. Builds here always pass --no-cache. +# +# Usage (from the repo root): +# ./release.sh build build + push the image, tag - +# ./release.sh pin write into CloudronManifest.json, commit, push +# ./release.sh deploy cloudron update --force, then verify +# ./release.sh release build -> pin -> deploy -> verify (the whole thing) +# ./release.sh verify live checks only (safe, no changes) +# ./release.sh volume FILE... push volume-resident files, assert md5 parity +# +# `build` needs docker + a registry login, so it runs on whoever's machine has +# them — deliberately NOT on the agent VM, which holds no build capability. +# `pin`, `deploy`, `verify` and `volume` only need CLOUDRON_TOKEN. +# +# Rollback: `./release.sh pin && ./release.sh deploy`. +set -euo pipefail + +APP=${APP:-4023ccbf-d57a-4500-97df-14de9cf3516e} +REGISTRY_REPO=${REGISTRY_REPO:-registry.inference.coop/co-op/librechat-custom} +SERVER=${SERVER:-my.inference.coop} +BASE_IMAGE=${BASE_IMAGE:-cloudron/ai.librechat.cloudronapp:202610150224320000} +MANIFEST=CloudronManifest.json + +die() { printf 'release.sh: %s\n' "$*" >&2; exit 1; } +ok() { printf ' \033[32mPASS\033[0m %s\n' "$*"; } +bad() { printf ' \033[31mFAIL\033[0m %s\n' "$*"; FAILED=1; } +need_token() { : "${CLOUDRON_TOKEN:?export CLOUDRON_TOKEN first (set -a; . ~/.hermes/.env; set +a)}"; } +cr() { cloudron --server "$SERVER" --token "$CLOUDRON_TOKEN" "$@"; } +manifest_tag() { python3 -c "import json;print(json.load(open('$MANIFEST'))['dockerImage'].rsplit(':',1)[-1])"; } + +cmd_build() { + command -v docker >/dev/null || die "no docker on this host — builds run where docker + the registry login live, not on the agent VM" + local tag="${TAG:-$(date -u +%Y%m%d)-$(git rev-parse --short HEAD)}" + [ -z "$(git status --porcelain)" ] || die "working tree is dirty — commit first, the tag carries the git sha" + printf 'building %s:%s (base %s)\n' "$REGISTRY_REPO" "$tag" "$BASE_IMAGE" + cloudron builder build --no-cache --repository "$REGISTRY_REPO" --tag "$tag" || die "build failed" + printf '\nbuilt tag: %s\nnext: ./release.sh pin %s\n' "$tag" "$tag" +} + +cmd_pin() { + local tag="${1:-}" ; [ -n "$tag" ] || die "usage: release.sh pin " + python3 - "$tag" <<'PY' +import json, sys +tag = sys.argv[1] +m = json.load(open('CloudronManifest.json')) +old = m['dockerImage'] +m['dockerImage'] = old.rsplit(':', 1)[0] + ':' + tag +json.dump(m, open('CloudronManifest.json', 'w'), indent=4) +open('CloudronManifest.json', 'a').write('\n') +print(f'dockerImage: {old} -> {m["dockerImage"]}') +PY + git add "$MANIFEST" + git commit -q -m "Pin manifest to $tag" + git push -q origin HEAD:master + ok "pinned + pushed $tag" +} + +cmd_deploy() { + need_token + local want; want=$(manifest_tag) + printf 'deploying manifest (image tag %s)\n' "$want" + cr update --app "$APP" --force + cmd_verify +} + +cmd_verify() { + need_token + FAILED=0 + local api_tag repo_tag css_repo css_live + api_tag=$(curl -s -H "Authorization: Bearer $CLOUDRON_TOKEN" "https://$SERVER/api/v1/apps/$APP" \ + | python3 -c "import json,sys;print(json.load(sys.stdin)['manifest']['dockerImage'].rsplit(':',1)[-1])") + repo_tag=$(manifest_tag) + [ "$api_tag" = "$repo_tag" ] && ok "running image tag == repo pin ($api_tag)" \ + || bad "running image tag $api_tag != repo pin $repo_tag (did you pin before deploying?)" + + css_repo=$(md5sum assets/custom.css | cut -d' ' -f1) + css_live=$(cr exec --app "$APP" -- md5sum /app/code/librechat/client/dist/assets/custom.css 2>/dev/null | cut -d' ' -f1) + [ "$css_repo" = "$css_live" ] && ok "container custom.css == repo (no stale build cache)" \ + || bad "container custom.css $css_live != repo $css_repo (stale COPY layer — rebuild --no-cache)" + + local html; html=$(curl -s "https://chat.inference.coop/login") + grep -qE 'assets/custom\.css' <<<"$html" && ok "served HTML links custom.css" || bad "served HTML missing custom.css link" + grep -qE 'assets/custom2\.js' <<<"$html" && ok "served HTML links custom2.js" || bad "served HTML missing custom2.js link" + curl -s "https://chat.inference.coop/" | grep -qE 'x-coop-build' && ok "x-coop-build marker present" \ + || bad "no x-coop-build marker (stale service-worker clients will not self-heal)" + + # The scheduler task must point at a file that exists in the RUNNING image. + # Catches a manifest repoint deployed before its image landed (2026-10-25). + local sched_cmd + sched_cmd=$(python3 -c " +import json +app=json.load(open('$MANIFEST')) +t=app.get('addons',{}).get('scheduler',{}).get('librechat_key_sync') +print(t['command'] if t else '')") + if [ -n "$sched_cmd" ]; then + cmd_path=$(grep -oE 'node [^ ]+' <<<"$sched_cmd" | awk '{print $2}') + if cr exec --app "$APP" -- test -f "$cmd_path" 2>/dev/null; then + ok "scheduler command path exists in the image ($cmd_path)" + else + bad "scheduler points at $cmd_path which is NOT in the running image — the seed will fail until the image lands" + fi + fi + + printf '\nkey-sync task log lines from the app log:\n' + cr logs --app "$APP" --lines 4000 2>/dev/null | grep -E "sync-keys.js revision|creds check|portal feed|done:|FATAL" | tail -8 | sed 's/^/ /' + printf ' compare the revision hash above with: md5sum sync-keys.js -> %s\n' "$(md5sum sync-keys.js | cut -c1-8)" + [ "$FAILED" = 0 ] && printf '\nall checks passed\n' || { printf '\nCHECKS FAILED\n' >&2; exit 1; } +} + +cmd_volume() { + need_token + [ "$#" -ge 1 ] || die "usage: release.sh volume FILE [FILE...]" + FAILED=0 + for f in "$@"; do + [ -f "$f" ] || die "no such file: $f" + local base; base=$(basename "$f") + base64 -w0 "$f" | cr exec --app "$APP" -- sh -c "cat > /tmp/vpush && base64 -d /tmp/vpush > /app/data/$base && rm -f /tmp/vpush" + local local_md5 live_md5 + local_md5=$(md5sum "$f" | cut -d' ' -f1) + live_md5=$(cr exec --app "$APP" -- md5sum "/app/data/$base" | cut -d' ' -f1) + [ "$local_md5" = "$live_md5" ] && ok "volume $base == repo ($local_md5)" || bad "volume $base $live_md5 != repo $local_md5" + done + [ "$FAILED" = 0 ] || exit 1 +} + +case "${1:-}" in + build) shift; cmd_build "$@" ;; + pin) shift; cmd_pin "$@" ;; + deploy) shift; cmd_deploy "$@" ;; + verify) shift; cmd_verify "$@" ;; + volume) shift; cmd_volume "$@" ;; + release) shift; cmd_build "$@"; t=$(manifest_tag); cmd_deploy ;; + *) sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;; +esac diff --git a/sync-keys.js b/sync-keys.js index 96b54bb..46ae51f 100644 --- a/sync-keys.js +++ b/sync-keys.js @@ -53,6 +53,15 @@ const MONGO_URL = process.env.CLOUDRON_MONGODB_URL || process.env.MONGODB_URL; const PORTAL_URL = process.env.PORTAL_URL || "https://portal.inference.coop"; const SYNC_TOKEN = process.env.LIBRECHAT_SYNC_TOKEN || ""; const ENDPOINT_NAME = "Inference Co-op"; + +// Which revision is actually running? The script lives on the persistent volume, +// so it can be pushed out of band; printing its own hash on every run means a +// stale copy can never hide — compare this with `md5sum sync-keys.js` in the repo. +const SELF_HASH = (() => { + try { + return require("crypto").createHash("md5").update(fs.readFileSync(__filename)).digest("hex").slice(0, 8); + } catch (e) { return "unknown"; } +})(); const MCP_PLUGIN_KEY = "mcp_inference-image" // Constants.mcp_prefix is "mcp_" (underscore), not "mcp:"; const MCP_AUTH_FIELD = "IC_API_KEY"; @@ -129,6 +138,7 @@ const mongoose = require("/app/code/librechat/node_modules/mongoose"); console.log(`creds check: ${credsCheck} (${sample.length} existing row(s))`); let seeded = 0, skipped = 0; + console.log(`sync-keys.js revision ${SELF_HASH} (repo copy md5 starts with this)`); for (const m of members) { const email = String(m.email || "").toLowerCase(); const sk = String(m.key || "");