inference-bot
1b9b971c6e
The release chain read the tag back from CloudronManifest.json BEFORE pinning it, so t was always the previous tag: release built the new image, then deployed the old one and printed success — the exact silent no-op this script exists to prevent. Its usage text promises build -> pin -> deploy -> verify; it did build -> deploy. cmd_build now exports the tag it built and release pins THAT, then deploys and verifies. Not exercised end-to-end yet (the theme deploy that found this bug was run as separate build/pin/deploy steps and passed all checks); the edit is syntax-checked only.
193 lines
9.1 KiB
Bash
Executable File
193 lines
9.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# release.sh — build, pin, deploy and verify the LibreChat overlay image.
|
|
#
|
|
# Why this script exists: two traps on this app have each silently eaten a
|
|
# deploy of this overlay, and both look like success.
|
|
# 1. `cloudron update --image <tag>` is IGNORED when CloudronManifest.json is
|
|
# present in the cwd — the CLI deploys manifest.dockerImage instead, while
|
|
# still printing the --image value. So the built tag MUST be written into
|
|
# the manifest first (that is what `pin` does).
|
|
# 2. The builder cache can serve stale COPY layers even when file content
|
|
# changed. Builds here always pass --no-cache.
|
|
#
|
|
# Usage (from the repo root):
|
|
# ./release.sh build build + push the image, tag <utcdate>-<gitsha>
|
|
# ./release.sh pin <tag> write <tag> into CloudronManifest.json, commit, push
|
|
# ./release.sh deploy cloudron update --force, then verify
|
|
# ./release.sh release build -> pin -> deploy -> verify (the whole thing)
|
|
# ./release.sh verify live checks only (safe, no changes)
|
|
# ./release.sh volume FILE... push volume-resident files, assert md5 parity
|
|
#
|
|
# `build` needs docker + a registry login, so it runs on whoever's machine has
|
|
# them — deliberately NOT on the agent VM, which holds no build capability.
|
|
# `pin`, `deploy`, `verify` and `volume` only need CLOUDRON_TOKEN.
|
|
#
|
|
# Rollback: `./release.sh pin <previous-tag> && ./release.sh deploy`.
|
|
set -euo pipefail
|
|
|
|
APP=${APP:-4023ccbf-d57a-4500-97df-14de9cf3516e}
|
|
REGISTRY_REPO=${REGISTRY_REPO:-registry.inference.coop/co-op/librechat-custom}
|
|
SERVER=${SERVER:-my.inference.coop}
|
|
BASE_IMAGE=${BASE_IMAGE:-cloudron/ai.librechat.cloudronapp:202610150224320000}
|
|
MANIFEST=CloudronManifest.json
|
|
|
|
die() { printf 'release.sh: %s\n' "$*" >&2; exit 1; }
|
|
ok() { printf ' \033[32mPASS\033[0m %s\n' "$*"; }
|
|
bad() { printf ' \033[31mFAIL\033[0m %s\n' "$*"; FAILED=1; }
|
|
need_token() { : "${CLOUDRON_TOKEN:?export CLOUDRON_TOKEN first (set -a; . ~/.hermes/.env; set +a)}"; }
|
|
cr() { cloudron --server "$SERVER" --token "$CLOUDRON_TOKEN" "$@"; }
|
|
manifest_tag() { python3 -c "import json;print(json.load(open('$MANIFEST'))['dockerImage'].rsplit(':',1)[-1])"; }
|
|
|
|
# curl with retries: right after a deploy the container is mid-swap and a fetch
|
|
# can return empty — that false-negatives the served-HTML checks.
|
|
http_body() {
|
|
local url=$1 try out=""
|
|
for try in 1 2 3; do
|
|
out=$(curl -sf -m 20 "$url" 2>/dev/null) && { printf '%s' "$out"; return 0; }
|
|
sleep 3
|
|
done
|
|
return 1
|
|
}
|
|
|
|
cmd_build() {
|
|
# Builds are REMOTE: the CLI uploads the build context and the registry app's
|
|
# BuildKit does the work (no docker on the caller). What is required is a
|
|
# registry login for this user on this host — env token or builder config.
|
|
local tag="${TAG:-$(date -u +%Y%m%d)-$(git rev-parse --short HEAD)}"
|
|
OVERLAY_LAST_TAG="$tag" # for the `release` chain: pin THIS, not the manifest's stale tag
|
|
[ -z "$(git status --porcelain)" ] || die "working tree is dirty — commit first, the tag carries the git sha"
|
|
|
|
local logged_in=0
|
|
if ! cloudron builder info 2>/dev/null | grep -qi "not logged in"; then logged_in=1; fi
|
|
|
|
local authargs=()
|
|
if [ -n "${REGISTRY_BUILD_TOKEN:-}" ]; then
|
|
# Narrow path: a credential scoped to the registry app only, kept in the
|
|
# caller's env — nothing is written to disk here.
|
|
authargs=(--registry-url https://registry.inference.coop
|
|
--registry-username "${REGISTRY_BUILD_USERNAME:-registry-bot}"
|
|
--registry-password "$REGISTRY_BUILD_TOKEN")
|
|
elif [ "$logged_in" = 1 ]; then
|
|
authargs=() # builder config in ~/.cloudron.json is enough
|
|
else
|
|
cat >&2 <<'MSG'
|
|
release.sh: this host has no registry login, so it cannot start a remote build.
|
|
Bootstrap one of two ways:
|
|
a) narrow: mint an app password in the dashboard as `registry-bot`, scoped to
|
|
the registry app id (c257ba3c-6737-4375-ad51-ea19586bf3f0), then on this host
|
|
put REGISTRY_BUILD_TOKEN=<app password> into the environment
|
|
(e.g. append it to ~/.hermes/.env). Nothing else to configure.
|
|
b) broad: `cloudron login my.inference.coop` runs a device flow — a human with
|
|
dashboard access confirms a code — then `cloudron builder login`. This
|
|
grants a full CLI session, not just build rights.
|
|
MSG
|
|
exit 1
|
|
fi
|
|
|
|
printf 'building %s:%s (base %s)\n' "$REGISTRY_REPO" "$tag" "$BASE_IMAGE"
|
|
cloudron builder "${authargs[@]}" build --no-cache --repository "$REGISTRY_REPO" --tag "$tag" || die "build failed"
|
|
printf '\nbuilt tag: %s\nnext: ./release.sh pin %s\n' "$tag" "$tag"
|
|
}
|
|
|
|
cmd_pin() {
|
|
local tag="${1:-}" ; [ -n "$tag" ] || die "usage: release.sh pin <tag>"
|
|
local br; br=$(git rev-parse --abbrev-ref HEAD)
|
|
if [ "$br" != "master" ] && [ "${ALLOW_BRANCH:-0}" != "1" ]; then
|
|
die "on branch '$br' — the pin commit goes to master, so merge your change first (or set ALLOW_BRANCH=1 if you really mean it)"
|
|
fi
|
|
python3 - "$tag" <<'PY'
|
|
import json, sys
|
|
tag = sys.argv[1]
|
|
m = json.load(open('CloudronManifest.json'))
|
|
old = m['dockerImage']
|
|
m['dockerImage'] = old.rsplit(':', 1)[0] + ':' + tag
|
|
json.dump(m, open('CloudronManifest.json', 'w'), indent=4)
|
|
open('CloudronManifest.json', 'a').write('\n')
|
|
print(f'dockerImage: {old} -> {m["dockerImage"]}')
|
|
PY
|
|
git add "$MANIFEST"
|
|
git commit -q -m "Pin manifest to $tag"
|
|
git push -q
|
|
ok "pinned + pushed $tag"
|
|
}
|
|
|
|
cmd_deploy() {
|
|
need_token
|
|
local want; want=$(manifest_tag)
|
|
printf 'deploying manifest (image tag %s)\n' "$want"
|
|
cr update --app "$APP" --force
|
|
cmd_verify
|
|
}
|
|
|
|
cmd_verify() {
|
|
need_token
|
|
FAILED=0
|
|
local api_tag repo_tag css_repo css_live
|
|
api_tag=$(curl -s -H "Authorization: Bearer $CLOUDRON_TOKEN" "https://$SERVER/api/v1/apps/$APP" \
|
|
| python3 -c "import json,sys;print(json.load(sys.stdin)['manifest']['dockerImage'].rsplit(':',1)[-1])")
|
|
repo_tag=$(manifest_tag)
|
|
[ "$api_tag" = "$repo_tag" ] && ok "running image tag == repo pin ($api_tag)" \
|
|
|| bad "running image tag $api_tag != repo pin $repo_tag (did you pin before deploying?)"
|
|
|
|
css_repo=$(md5sum assets/custom.css | cut -d' ' -f1)
|
|
css_live=$(cr exec --app "$APP" -- md5sum /app/code/librechat/client/dist/assets/custom.css 2>/dev/null | cut -d' ' -f1)
|
|
[ "$css_repo" = "$css_live" ] && ok "container custom.css == repo (no stale build cache)" \
|
|
|| bad "container custom.css $css_live != repo $css_repo (stale COPY layer — rebuild --no-cache)"
|
|
|
|
local html; html=$(http_body "https://chat.inference.coop/login")
|
|
grep -qE 'assets/custom\.css' <<<"$html" && ok "served HTML links custom.css" || bad "served HTML missing custom.css link"
|
|
grep -qE 'assets/custom2\.js' <<<"$html" && ok "served HTML links custom2.js" || bad "served HTML missing custom2.js link"
|
|
# NOTE: right after `deploy` the container is mid-swap; a curl can come back
|
|
# empty and false-negative. The retry in http_body exists for that (seen 2026-10-26).
|
|
http_body "https://chat.inference.coop/" | grep -qE 'x-coop-build' && ok "x-coop-build marker present" \
|
|
|| bad "no x-coop-build marker (stale service-worker clients will not self-heal)"
|
|
|
|
# The scheduler task must point at a file that exists in the RUNNING image.
|
|
# Catches a manifest repoint deployed before its image landed (2026-10-25).
|
|
local sched_cmd
|
|
sched_cmd=$(python3 -c "
|
|
import json
|
|
app=json.load(open('$MANIFEST'))
|
|
t=app.get('addons',{}).get('scheduler',{}).get('librechat_key_sync')
|
|
print(t['command'] if t else '')")
|
|
if [ -n "$sched_cmd" ]; then
|
|
cmd_path=$(grep -oE 'node [^ ]+' <<<"$sched_cmd" | awk '{print $2}')
|
|
if cr exec --app "$APP" -- test -f "$cmd_path" 2>/dev/null; then
|
|
ok "scheduler command path exists in the image ($cmd_path)"
|
|
else
|
|
bad "scheduler points at $cmd_path which is NOT in the running image — the seed will fail until the image lands"
|
|
fi
|
|
fi
|
|
|
|
printf '\nkey-sync task log lines from the app log:\n'
|
|
cr logs --app "$APP" --lines 4000 2>/dev/null | grep -E "sync-keys.js revision|creds check|portal feed|done:|FATAL" | tail -8 | sed 's/^/ /'
|
|
printf ' compare the revision hash above with: md5sum sync-keys.js -> %s\n' "$(md5sum sync-keys.js | cut -c1-8)"
|
|
[ "$FAILED" = 0 ] && printf '\nall checks passed\n' || { printf '\nCHECKS FAILED\n' >&2; exit 1; }
|
|
}
|
|
|
|
cmd_volume() {
|
|
need_token
|
|
[ "$#" -ge 1 ] || die "usage: release.sh volume FILE [FILE...]"
|
|
FAILED=0
|
|
for f in "$@"; do
|
|
[ -f "$f" ] || die "no such file: $f"
|
|
local base; base=$(basename "$f")
|
|
base64 -w0 "$f" | cr exec --app "$APP" -- sh -c "cat > /tmp/vpush && base64 -d /tmp/vpush > /app/data/$base && rm -f /tmp/vpush"
|
|
local local_md5 live_md5
|
|
local_md5=$(md5sum "$f" | cut -d' ' -f1)
|
|
live_md5=$(cr exec --app "$APP" -- md5sum "/app/data/$base" | cut -d' ' -f1)
|
|
[ "$local_md5" = "$live_md5" ] && ok "volume $base == repo ($local_md5)" || bad "volume $base $live_md5 != repo $local_md5"
|
|
done
|
|
[ "$FAILED" = 0 ] || exit 1
|
|
}
|
|
|
|
case "${1:-}" in
|
|
build) shift; cmd_build "$@" ;;
|
|
pin) shift; cmd_pin "$@" ;;
|
|
deploy) shift; cmd_deploy "$@" ;;
|
|
verify) shift; cmd_verify "$@" ;;
|
|
volume) shift; cmd_volume "$@" ;;
|
|
release) shift; cmd_build "$@"; cmd_pin "$OVERLAY_LAST_TAG"; cmd_deploy; cmd_verify ;;
|
|
*) sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;;
|
|
esac
|