chat: fail-closed librechat.yaml validation at boot — insert configSchema.strict() check into /app/pkg/start.sh at build time (2026-10-31 customFooter outage guard)

This commit is contained in:
inference-admin committed 2026-11-11 03:55:18 +00:00
1 parent bb276dd7dd
commit 179fd7ce02
3 files changed
+341

No files matched your search

+10
View File
@@ -54,6 +54,16 @@ COPY sync-keys.js /app/code/sync-keys.js
COPY patch_ui_strings.py /app/code/patch_ui_strings.py COPY patch_ui_strings.py /app/code/patch_ui_strings.py
RUN python3 /app/code/patch_ui_strings.py RUN python3 /app/code/patch_ui_strings.py
# Fail-closed librechat.yaml validation, inserted into /app/pkg/start.sh at
# BUILD time (the runtime rootfs is read-only). The validator script's
# canonical copy lives in ops-docs/scripts/ — this byte-identical twin ships
# with the image. patch_start_validator.py is non-fatal at build (a missing
# anchor only skips the insert) and idempotent; an INVALID config is fatal at
# RUNTIME by design. Keep validate_config.py's md5 in step with the canonical.
COPY validate_config.py /app/code/validate_config.py
COPY patch_start_validator.py /app/code/patch_start_validator.py
RUN python3 /app/code/patch_start_validator.py
# Inject branding into built HTML (dist) before image is packaged. # Inject branding into built HTML (dist) before image is packaged.
# patch_index.py: title, custom.css/custom2.js links, and a per-build # patch_index.py: title, custom.css/custom2.js links, and a per-build
# x-coop-build meta (unique every build → service worker precache revision # x-coop-build meta (unique every build → service worker precache revision
+70
View File
@@ -0,0 +1,70 @@
"""Insert the fail-closed librechat.yaml validation into /app/pkg/start.sh.
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
if the anchor line is missing (upstream changed its start script) this prints a
loud note and exits 0 so a LibreChat version bump can never brick the build —
the RUNNING app then simply lacks the pre-boot check until the patch is fixed,
which the deploy verification catches by grepping the boot log.
Idempotent: the block carries a marker; re-insertion is skipped when present.
The block validates /app/data/librechat.yaml with the container's own
librechat-data-provider strict schema (the same check the API server runs at
boot, which exits 1 on an unrecognized top-level key — the 2026-10-31 outage).
Placement: after the first-run `touch /app/data/librechat.yaml` and before the
secrets/env assembly, so a bad config stops the boot BEFORE anything starts.
The -s guard skips the empty first-run file; the -f guard keeps a missing
validator non-fatal (patch did not apply). An INVALID config is fatal by
design — that is the point.
"""
import sys
START = "/app/pkg/start.sh"
ANCHOR = "if [[ ! -f /app/data/librechat.yaml ]]; then touch /app/data/librechat.yaml; fi"
MARKER = "co-op fail-closed librechat.yaml validation"
BLOCK = """
# --- {marker} (overlay, 2026-11-11) ---
# Invalid librechat.yaml must stop the boot LOUDLY, naming the offending key,
# before the API server can come half-up (SPA serving 200 while the API is
# down — the 2026-10-31 outage). The venv python carries pyyaml; the schema
# probe runs against the image's own data-provider build.
if [ -f /app/code/validate_config.py ] && [ -s /app/data/librechat.yaml ]; then
echo "Validating librechat.yaml (fail-closed)..."
if ! /app/code/venv/bin/python3 /app/code/validate_config.py librechat /app/data/librechat.yaml; then
echo "FATAL: librechat.yaml is invalid - refusing to start. Fix or remove the offending key named above." >&2
exit 1
fi
fi
""".format(marker=MARKER)
def main() -> int:
try:
with open(START, encoding="utf-8") as fh:
src = fh.read()
except OSError as exc:
print(f"WARN: cannot read {START}: {exc} — validation patch NOT applied",
flush=True)
return 0
if MARKER in src:
print("INFO: validation block already present in start.sh (idempotent skip)",
flush=True)
return 0
if ANCHOR not in src:
print("WARN: anchor line not found in /app/pkg/start.sh — validation "
"patch NOT applied; fix patch_start_validator.py after the "
"upstream change", flush=True)
return 0
patched = src.replace(ANCHOR, ANCHOR + "\n" + BLOCK.strip("\n") + "\n", 1)
with open(START, "w", encoding="utf-8") as fh:
fh.write(patched)
print(f"INFO: inserted fail-closed config validation into {START}", flush=True)
return 0
if __name__ == "__main__":
sys.exit(main())
+261
View File
@@ -0,0 +1,261 @@
#!/usr/bin/env python3
"""Fail-closed config validator for the co-op's gateway and chat apps.
ONE script, two modes, shared home in ops-docs/scripts/ and shipped
byte-identical into both app images (litellm repo root, librechat-custom repo
root). Each app's start path calls it before booting the service; a non-zero
exit refuses the boot with a ONE-line error naming the offending key.
Why this exists: the 2026-10-31 chat outage. A `customFooter:` top-level key in
librechat.yaml was rejected by LibreChat's strict config validator and the API
server refused to start, while the SPA kept being served with HTTP 200 — the
app looked up but every API route returned the HTML shell. Nothing had ever
checked the config before boot. This validator closes that gap: invalid config
fails LOUD and EARLY, at the top of the boot log, naming the key.
Modes
-----
litellm Structural check of the gateway's config.yaml:
- top-level keys within LiteLLM's documented config surface;
- model_list present and well-formed (model_name + litellm_params.model);
- the settings blocks are objects; pass_through_endpoints entries have paths.
When the gateway's own litellm package is importable it is additionally
used to resolve every model string to a provider (catches unknown
provider prefixes); the check is skipped with a note where litellm
itself is absent.
librechat Checks librechat.yaml against the DEPLOYED build's own strict schema:
it runs `configSchema.strict().safeParse()` from the container's
librechat-data-provider via node — the same code the API server runs
at boot (its failure mode is `process.exit(1)`, "Exiting due to
invalid configuration"). Unrecognized top-level keys are rejected,
named explicitly. Where node/the package cannot be found (e.g. this
script run from a checkout instead of the image) it falls back to a
static top-level allowlist recorded from the deployed build and says
so on the failure line.
Usage
-----
python3 validate_config.py litellm /app/data/config.yaml
python3 validate_config.py librechat /app/data/librechat.yaml
Exit codes: 0 valid, 1 invalid, 2 usage/environment error (file unreadable,
yaml unparseable). Never prints a value from the config — only key names and
counts, so it is safe to run against files that carry secrets.
"""
import json
import os
import subprocess
import sys
# LiteLLM's documented top-level config surface (proxy config). Unknown keys
# are rejected here — the gateway should never silently ignore a mistyped
# section (a "model-list:" typo would otherwise deploy an EMPTY catalog).
LITELLM_TOP_LEVEL = {
"model_list",
"router_settings",
"litellm_settings",
"general_settings",
"settings",
"environment_variables",
"guardrails",
"files",
}
# Fallback allowlist for librechat mode, recorded 2026-11-11 from the DEPLOYED
# build (chat.inference.coop, overlay 1.4.8) by reading
# configSchema.strict().shape keys. The live schema is always preferred; this
# exists so the validator still fails closed when run outside the container.
LIBRECHAT_TOP_LEVEL_FALLBACK = {
"actions", "balance", "cache", "cloudfront", "endpoints", "fileConfig",
"fileStrategies", "fileStrategy", "filteredTools", "filters",
"imageOutputType", "includedTools", "interface", "langfuse", "mcpServers",
"mcpSettings", "memory", "messageFilter", "modelSpecs", "ocr", "openapi",
"permissions", "rateLimits", "registration", "secureImageLinks",
"skillSync", "speech", "summarization", "transactions", "turnstile",
"version", "webSearch",
}
LC_DATA_PROVIDER_DIST = os.environ.get(
"LC_DATA_PROVIDER_DIST",
"/app/code/librechat/packages/data-provider/dist/index.js")
LC_JS_YAML = os.environ.get(
"LC_JS_YAML", "/app/code/librechat/node_modules/js-yaml")
def fail(key, reason):
"""ONE line, naming the offending key, then a non-zero exit."""
print(f"CONFIG VALIDATION FAILED: {key}: {reason}", flush=True)
sys.exit(1)
def load_yaml(path):
try:
import yaml
except ImportError:
print("CONFIG VALIDATION FAILED: pyyaml not available in this python",
flush=True)
sys.exit(2)
try:
with open(path) as fh:
data = yaml.safe_load(fh)
except FileNotFoundError:
print(f"CONFIG VALIDATION FAILED: config file not found: {path}",
flush=True)
sys.exit(2)
except PermissionError:
print(f"CONFIG VALIDATION FAILED: config file unreadable: {path}",
flush=True)
sys.exit(2)
except yaml.YAMLError as exc:
# Do not echo the raw yaml error: it can quote a secret-bearing line.
print("CONFIG VALIDATION FAILED: (root): yaml does not parse",
flush=True)
sys.exit(2)
if data is None:
data = {}
if not isinstance(data, dict):
fail("(root)", "top level must be a mapping")
return data
def validate_litellm(path):
cfg = load_yaml(path)
unknown = sorted(set(cfg) - LITELLM_TOP_LEVEL)
if unknown:
fail(", ".join(unknown), "unknown top-level key(s) for litellm "
f"(allowed: {', '.join(sorted(LITELLM_TOP_LEVEL))})")
model_list = cfg.get("model_list")
if not isinstance(model_list, list) or not model_list:
fail("model_list", "must be a non-empty list of model entries")
names = set()
for i, entry in enumerate(model_list):
if not isinstance(entry, dict):
fail(f"model_list[{i}]", "entry must be a mapping")
name = entry.get("model_name")
params = entry.get("litellm_params")
if not name or not isinstance(name, str):
fail(f"model_list[{i}].model_name", "missing or empty")
if name in names:
fail(f"model_list[{i}].model_name", f"duplicate model_name {name!r}")
names.add(name)
if not isinstance(params, dict) or not params.get("model"):
fail(f"model_list[{i}].litellm_params.model", "missing or empty")
for block in ("router_settings", "litellm_settings", "general_settings"):
if block in cfg and not isinstance(cfg[block], dict):
fail(block, "must be a mapping")
for ep in (cfg.get("general_settings") or {}).get("pass_through_endpoints") or []:
if not isinstance(ep, dict) or not ep.get("path"):
fail("general_settings.pass_through_endpoints", "each entry needs a path")
# litellm-assisted check (best effort): resolve each model string to a
# provider with the gateway's own package. Skipped where litellm itself
# is not importable (never fatal — the structural checks above ran).
try:
from litellm import get_llm_provider # noqa: WPS433
except Exception:
print("note: litellm not importable here; provider check skipped",
flush=True)
else:
for entry in model_list:
model = entry["litellm_params"]["model"]
try:
get_llm_provider(model=model, custom_llm_provider=None)
except Exception as exc:
short = str(exc).split("\n")[0][:120]
fail(f"model_list.{entry['model_name']}.litellm_params.model",
f"not resolvable to a provider: {short}")
print(f"config OK (litellm): {len(cfg)} top-level key(s), "
f"{len(model_list)} model(s) validated", flush=True)
sys.exit(0)
def validate_librechat(path):
cfg = load_yaml(path)
if not cfg:
print("config OK (librechat): empty config, nothing to validate",
flush=True)
sys.exit(0)
# Preferred: the DEPLOYED build's own strict schema, run through node —
# the exact code path the API server uses at boot (strict() rejects
# unrecognized top-level keys; upstream then exits 1).
node_code = r"""
const fs = require('fs');
let out = {mode: 'schema'};
try {
const yaml = require(JS_YAML_PATH);
const dp = require(DIST_PATH);
const cfg = yaml.load(fs.readFileSync(CONFIG_PATH, 'utf8')) || {};
const result = dp.configSchema.strict().safeParse(cfg);
if (result.success) {
out.ok = true;
out.keys = Object.keys(cfg).length;
} else {
out.ok = false;
out.errors = (result.error && result.error.issues || []).slice(0, 10)
.map(i => ({path: (i.path || []).join('.') || '(root)', msg: i.message}));
}
} catch (e) {
out.mode = 'schema-unavailable';
out.error = String(e && e.message || e).slice(0, 200);
}
console.log('@@' + JSON.stringify(out));
""".replace("JS_YAML_PATH", json.dumps(LC_JS_YAML)) \
.replace("DIST_PATH", json.dumps(LC_DATA_PROVIDER_DIST)) \
.replace("CONFIG_PATH", json.dumps(path))
if os.path.exists(LC_DATA_PROVIDER_DIST):
try:
proc = subprocess.run(["node", "-e", node_code], capture_output=True,
text=True, timeout=60)
for line in (proc.stdout or "").splitlines():
if line.startswith("@@"):
res = json.loads(line[2:])
if res.get("ok"):
print(f"config OK (librechat): {res['keys']} top-level "
"key(s) accepted by the deployed strict schema",
flush=True)
sys.exit(0)
errs = "; ".join(
f"{e['path']}: {e['msg']}" for e in res.get("errors", []))
if errs:
fail(errs, "rejected by the deployed build's strict "
"config schema (configSchema.strict())")
fail("(root)", "strict schema rejected the config")
# No @@ line: schema path unavailable — fall through to fallback.
print("note: deployed schema probe produced no result "
f"(stderr: {(proc.stderr or '')[:120]!r})", flush=True)
except (OSError, subprocess.TimeoutExpired) as exc:
print(f"note: could not run node against the deployed schema "
f"({type(exc).__name__})", flush=True)
else:
print("note: deployed data-provider dist not found; using the static "
"fallback allowlist recorded from the deployed build", flush=True)
# Fallback: static allowlist (still fail-closed).
unknown = sorted(set(cfg) - LIBRECHAT_TOP_LEVEL_FALLBACK)
if unknown:
fail(", ".join(unknown), "unknown top-level key(s) for librechat "
f"(fallback allowlist: {', '.join(sorted(LIBRECHAT_TOP_LEVEL_FALLBACK))})")
print(f"config OK (librechat, fallback allowlist): {len(cfg)} top-level "
"key(s)", flush=True)
sys.exit(0)
def main():
args = sys.argv[1:]
if len(args) != 2 or args[0] not in ("litellm", "librechat"):
print("usage: validate_config.py <litellm|librechat> <config.yaml>",
file=sys.stderr, flush=True)
sys.exit(2)
if args[0] == "litellm":
validate_litellm(args[1])
validate_librechat(args[1])
if __name__ == "__main__":
main()