diff --git a/Dockerfile b/Dockerfile index e0db542..5a3b4ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,6 +54,16 @@ COPY sync-keys.js /app/code/sync-keys.js COPY patch_ui_strings.py /app/code/patch_ui_strings.py RUN python3 /app/code/patch_ui_strings.py +# Fail-closed librechat.yaml validation, inserted into /app/pkg/start.sh at +# BUILD time (the runtime rootfs is read-only). The validator script's +# canonical copy lives in ops-docs/scripts/ — this byte-identical twin ships +# with the image. patch_start_validator.py is non-fatal at build (a missing +# anchor only skips the insert) and idempotent; an INVALID config is fatal at +# RUNTIME by design. Keep validate_config.py's md5 in step with the canonical. +COPY validate_config.py /app/code/validate_config.py +COPY patch_start_validator.py /app/code/patch_start_validator.py +RUN python3 /app/code/patch_start_validator.py + # Inject branding into built HTML (dist) before image is packaged. # patch_index.py: title, custom.css/custom2.js links, and a per-build # x-coop-build meta (unique every build → service worker precache revision diff --git a/patch_start_validator.py b/patch_start_validator.py new file mode 100644 index 0000000..ba4e0f8 --- /dev/null +++ b/patch_start_validator.py @@ -0,0 +1,70 @@ +"""Insert the fail-closed librechat.yaml validation into /app/pkg/start.sh. + +Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal: +if the anchor line is missing (upstream changed its start script) this prints a +loud note and exits 0 so a LibreChat version bump can never brick the build — +the RUNNING app then simply lacks the pre-boot check until the patch is fixed, +which the deploy verification catches by grepping the boot log. + +Idempotent: the block carries a marker; re-insertion is skipped when present. + +The block validates /app/data/librechat.yaml with the container's own +librechat-data-provider strict schema (the same check the API server runs at +boot, which exits 1 on an unrecognized top-level key — the 2026-10-31 outage). +Placement: after the first-run `touch /app/data/librechat.yaml` and before the +secrets/env assembly, so a bad config stops the boot BEFORE anything starts. +The -s guard skips the empty first-run file; the -f guard keeps a missing +validator non-fatal (patch did not apply). An INVALID config is fatal by +design — that is the point. +""" +import sys + +START = "/app/pkg/start.sh" +ANCHOR = "if [[ ! -f /app/data/librechat.yaml ]]; then touch /app/data/librechat.yaml; fi" +MARKER = "co-op fail-closed librechat.yaml validation" + +BLOCK = """ +# --- {marker} (overlay, 2026-11-11) --- +# Invalid librechat.yaml must stop the boot LOUDLY, naming the offending key, +# before the API server can come half-up (SPA serving 200 while the API is +# down — the 2026-10-31 outage). The venv python carries pyyaml; the schema +# probe runs against the image's own data-provider build. +if [ -f /app/code/validate_config.py ] && [ -s /app/data/librechat.yaml ]; then + echo "Validating librechat.yaml (fail-closed)..." + if ! /app/code/venv/bin/python3 /app/code/validate_config.py librechat /app/data/librechat.yaml; then + echo "FATAL: librechat.yaml is invalid - refusing to start. Fix or remove the offending key named above." >&2 + exit 1 + fi +fi +""".format(marker=MARKER) + + +def main() -> int: + try: + with open(START, encoding="utf-8") as fh: + src = fh.read() + except OSError as exc: + print(f"WARN: cannot read {START}: {exc} — validation patch NOT applied", + flush=True) + return 0 + + if MARKER in src: + print("INFO: validation block already present in start.sh (idempotent skip)", + flush=True) + return 0 + + if ANCHOR not in src: + print("WARN: anchor line not found in /app/pkg/start.sh — validation " + "patch NOT applied; fix patch_start_validator.py after the " + "upstream change", flush=True) + return 0 + + patched = src.replace(ANCHOR, ANCHOR + "\n" + BLOCK.strip("\n") + "\n", 1) + with open(START, "w", encoding="utf-8") as fh: + fh.write(patched) + print(f"INFO: inserted fail-closed config validation into {START}", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/validate_config.py b/validate_config.py new file mode 100644 index 0000000..2cb50c7 --- /dev/null +++ b/validate_config.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +"""Fail-closed config validator for the co-op's gateway and chat apps. + +ONE script, two modes, shared home in ops-docs/scripts/ and shipped +byte-identical into both app images (litellm repo root, librechat-custom repo +root). Each app's start path calls it before booting the service; a non-zero +exit refuses the boot with a ONE-line error naming the offending key. + +Why this exists: the 2026-10-31 chat outage. A `customFooter:` top-level key in +librechat.yaml was rejected by LibreChat's strict config validator and the API +server refused to start, while the SPA kept being served with HTTP 200 — the +app looked up but every API route returned the HTML shell. Nothing had ever +checked the config before boot. This validator closes that gap: invalid config +fails LOUD and EARLY, at the top of the boot log, naming the key. + +Modes +----- +litellm Structural check of the gateway's config.yaml: + - top-level keys within LiteLLM's documented config surface; + - model_list present and well-formed (model_name + litellm_params.model); + - the settings blocks are objects; pass_through_endpoints entries have paths. + When the gateway's own litellm package is importable it is additionally + used to resolve every model string to a provider (catches unknown + provider prefixes); the check is skipped with a note where litellm + itself is absent. + +librechat Checks librechat.yaml against the DEPLOYED build's own strict schema: + it runs `configSchema.strict().safeParse()` from the container's + librechat-data-provider via node — the same code the API server runs + at boot (its failure mode is `process.exit(1)`, "Exiting due to + invalid configuration"). Unrecognized top-level keys are rejected, + named explicitly. Where node/the package cannot be found (e.g. this + script run from a checkout instead of the image) it falls back to a + static top-level allowlist recorded from the deployed build and says + so on the failure line. + +Usage +----- + python3 validate_config.py litellm /app/data/config.yaml + python3 validate_config.py librechat /app/data/librechat.yaml + +Exit codes: 0 valid, 1 invalid, 2 usage/environment error (file unreadable, +yaml unparseable). Never prints a value from the config — only key names and +counts, so it is safe to run against files that carry secrets. +""" + +import json +import os +import subprocess +import sys + +# LiteLLM's documented top-level config surface (proxy config). Unknown keys +# are rejected here — the gateway should never silently ignore a mistyped +# section (a "model-list:" typo would otherwise deploy an EMPTY catalog). +LITELLM_TOP_LEVEL = { + "model_list", + "router_settings", + "litellm_settings", + "general_settings", + "settings", + "environment_variables", + "guardrails", + "files", +} + +# Fallback allowlist for librechat mode, recorded 2026-11-11 from the DEPLOYED +# build (chat.inference.coop, overlay 1.4.8) by reading +# configSchema.strict().shape keys. The live schema is always preferred; this +# exists so the validator still fails closed when run outside the container. +LIBRECHAT_TOP_LEVEL_FALLBACK = { + "actions", "balance", "cache", "cloudfront", "endpoints", "fileConfig", + "fileStrategies", "fileStrategy", "filteredTools", "filters", + "imageOutputType", "includedTools", "interface", "langfuse", "mcpServers", + "mcpSettings", "memory", "messageFilter", "modelSpecs", "ocr", "openapi", + "permissions", "rateLimits", "registration", "secureImageLinks", + "skillSync", "speech", "summarization", "transactions", "turnstile", + "version", "webSearch", +} + +LC_DATA_PROVIDER_DIST = os.environ.get( + "LC_DATA_PROVIDER_DIST", + "/app/code/librechat/packages/data-provider/dist/index.js") +LC_JS_YAML = os.environ.get( + "LC_JS_YAML", "/app/code/librechat/node_modules/js-yaml") + + +def fail(key, reason): + """ONE line, naming the offending key, then a non-zero exit.""" + print(f"CONFIG VALIDATION FAILED: {key}: {reason}", flush=True) + sys.exit(1) + + +def load_yaml(path): + try: + import yaml + except ImportError: + print("CONFIG VALIDATION FAILED: pyyaml not available in this python", + flush=True) + sys.exit(2) + try: + with open(path) as fh: + data = yaml.safe_load(fh) + except FileNotFoundError: + print(f"CONFIG VALIDATION FAILED: config file not found: {path}", + flush=True) + sys.exit(2) + except PermissionError: + print(f"CONFIG VALIDATION FAILED: config file unreadable: {path}", + flush=True) + sys.exit(2) + except yaml.YAMLError as exc: + # Do not echo the raw yaml error: it can quote a secret-bearing line. + print("CONFIG VALIDATION FAILED: (root): yaml does not parse", + flush=True) + sys.exit(2) + if data is None: + data = {} + if not isinstance(data, dict): + fail("(root)", "top level must be a mapping") + return data + + +def validate_litellm(path): + cfg = load_yaml(path) + unknown = sorted(set(cfg) - LITELLM_TOP_LEVEL) + if unknown: + fail(", ".join(unknown), "unknown top-level key(s) for litellm " + f"(allowed: {', '.join(sorted(LITELLM_TOP_LEVEL))})") + + model_list = cfg.get("model_list") + if not isinstance(model_list, list) or not model_list: + fail("model_list", "must be a non-empty list of model entries") + names = set() + for i, entry in enumerate(model_list): + if not isinstance(entry, dict): + fail(f"model_list[{i}]", "entry must be a mapping") + name = entry.get("model_name") + params = entry.get("litellm_params") + if not name or not isinstance(name, str): + fail(f"model_list[{i}].model_name", "missing or empty") + if name in names: + fail(f"model_list[{i}].model_name", f"duplicate model_name {name!r}") + names.add(name) + if not isinstance(params, dict) or not params.get("model"): + fail(f"model_list[{i}].litellm_params.model", "missing or empty") + for block in ("router_settings", "litellm_settings", "general_settings"): + if block in cfg and not isinstance(cfg[block], dict): + fail(block, "must be a mapping") + for ep in (cfg.get("general_settings") or {}).get("pass_through_endpoints") or []: + if not isinstance(ep, dict) or not ep.get("path"): + fail("general_settings.pass_through_endpoints", "each entry needs a path") + + # litellm-assisted check (best effort): resolve each model string to a + # provider with the gateway's own package. Skipped where litellm itself + # is not importable (never fatal — the structural checks above ran). + try: + from litellm import get_llm_provider # noqa: WPS433 + except Exception: + print("note: litellm not importable here; provider check skipped", + flush=True) + else: + for entry in model_list: + model = entry["litellm_params"]["model"] + try: + get_llm_provider(model=model, custom_llm_provider=None) + except Exception as exc: + short = str(exc).split("\n")[0][:120] + fail(f"model_list.{entry['model_name']}.litellm_params.model", + f"not resolvable to a provider: {short}") + + print(f"config OK (litellm): {len(cfg)} top-level key(s), " + f"{len(model_list)} model(s) validated", flush=True) + sys.exit(0) + + +def validate_librechat(path): + cfg = load_yaml(path) + if not cfg: + print("config OK (librechat): empty config, nothing to validate", + flush=True) + sys.exit(0) + + # Preferred: the DEPLOYED build's own strict schema, run through node — + # the exact code path the API server uses at boot (strict() rejects + # unrecognized top-level keys; upstream then exits 1). + node_code = r""" +const fs = require('fs'); +let out = {mode: 'schema'}; +try { + const yaml = require(JS_YAML_PATH); + const dp = require(DIST_PATH); + const cfg = yaml.load(fs.readFileSync(CONFIG_PATH, 'utf8')) || {}; + const result = dp.configSchema.strict().safeParse(cfg); + if (result.success) { + out.ok = true; + out.keys = Object.keys(cfg).length; + } else { + out.ok = false; + out.errors = (result.error && result.error.issues || []).slice(0, 10) + .map(i => ({path: (i.path || []).join('.') || '(root)', msg: i.message})); + } +} catch (e) { + out.mode = 'schema-unavailable'; + out.error = String(e && e.message || e).slice(0, 200); +} +console.log('@@' + JSON.stringify(out)); +""".replace("JS_YAML_PATH", json.dumps(LC_JS_YAML)) \ + .replace("DIST_PATH", json.dumps(LC_DATA_PROVIDER_DIST)) \ + .replace("CONFIG_PATH", json.dumps(path)) + + if os.path.exists(LC_DATA_PROVIDER_DIST): + try: + proc = subprocess.run(["node", "-e", node_code], capture_output=True, + text=True, timeout=60) + for line in (proc.stdout or "").splitlines(): + if line.startswith("@@"): + res = json.loads(line[2:]) + if res.get("ok"): + print(f"config OK (librechat): {res['keys']} top-level " + "key(s) accepted by the deployed strict schema", + flush=True) + sys.exit(0) + errs = "; ".join( + f"{e['path']}: {e['msg']}" for e in res.get("errors", [])) + if errs: + fail(errs, "rejected by the deployed build's strict " + "config schema (configSchema.strict())") + fail("(root)", "strict schema rejected the config") + # No @@ line: schema path unavailable — fall through to fallback. + print("note: deployed schema probe produced no result " + f"(stderr: {(proc.stderr or '')[:120]!r})", flush=True) + except (OSError, subprocess.TimeoutExpired) as exc: + print(f"note: could not run node against the deployed schema " + f"({type(exc).__name__})", flush=True) + else: + print("note: deployed data-provider dist not found; using the static " + "fallback allowlist recorded from the deployed build", flush=True) + + # Fallback: static allowlist (still fail-closed). + unknown = sorted(set(cfg) - LIBRECHAT_TOP_LEVEL_FALLBACK) + if unknown: + fail(", ".join(unknown), "unknown top-level key(s) for librechat " + f"(fallback allowlist: {', '.join(sorted(LIBRECHAT_TOP_LEVEL_FALLBACK))})") + print(f"config OK (librechat, fallback allowlist): {len(cfg)} top-level " + "key(s)", flush=True) + sys.exit(0) + + +def main(): + args = sys.argv[1:] + if len(args) != 2 or args[0] not in ("litellm", "librechat"): + print("usage: validate_config.py ", + file=sys.stderr, flush=True) + sys.exit(2) + if args[0] == "litellm": + validate_litellm(args[1]) + validate_librechat(args[1]) + + +if __name__ == "__main__": + main()