#!/usr/bin/env python3 """Fail-closed config validator for the co-op's gateway and chat apps. ONE script, two modes, shared home in ops-docs/scripts/ and shipped byte-identical into both app images (litellm repo root, librechat-custom repo root). Each app's start path calls it before booting the service; a non-zero exit refuses the boot with a ONE-line error naming the offending key. Why this exists: the 2026-10-31 chat outage. A `customFooter:` top-level key in librechat.yaml was rejected by LibreChat's strict config validator and the API server refused to start, while the SPA kept being served with HTTP 200 — the app looked up but every API route returned the HTML shell. Nothing had ever checked the config before boot. This validator closes that gap: invalid config fails LOUD and EARLY, at the top of the boot log, naming the key. Modes ----- litellm Structural check of the gateway's config.yaml: - top-level keys within LiteLLM's documented config surface; - model_list present and well-formed (model_name + litellm_params.model); - the settings blocks are objects; pass_through_endpoints entries have paths. When the gateway's own litellm package is importable it is additionally used to resolve every model string to a provider (catches unknown provider prefixes); the check is skipped with a note where litellm itself is absent. librechat Checks librechat.yaml against the DEPLOYED build's own strict schema: it runs `configSchema.strict().safeParse()` from the container's librechat-data-provider via node — the same code the API server runs at boot (its failure mode is `process.exit(1)`, "Exiting due to invalid configuration"). Unrecognized top-level keys are rejected, named explicitly. Where node/the package cannot be found (e.g. this script run from a checkout instead of the image) it falls back to a static top-level allowlist recorded from the deployed build and says so on the failure line. Usage ----- python3 validate_config.py litellm /app/data/config.yaml python3 validate_config.py librechat /app/data/librechat.yaml Exit codes: 0 valid, 1 invalid, 2 usage/environment error (file unreadable, yaml unparseable). Never prints a value from the config — only key names and counts, so it is safe to run against files that carry secrets. """ import json import os import subprocess import sys # LiteLLM's documented top-level config surface (proxy config). Unknown keys # are rejected here — the gateway should never silently ignore a mistyped # section (a "model-list:" typo would otherwise deploy an EMPTY catalog). LITELLM_TOP_LEVEL = { "model_list", "router_settings", "litellm_settings", "general_settings", "settings", "environment_variables", "guardrails", "files", } # Fallback allowlist for librechat mode, recorded 2026-11-11 from the DEPLOYED # build (chat.inference.coop, overlay 1.4.8) by reading # configSchema.strict().shape keys. The live schema is always preferred; this # exists so the validator still fails closed when run outside the container. LIBRECHAT_TOP_LEVEL_FALLBACK = { "actions", "balance", "cache", "cloudfront", "endpoints", "fileConfig", "fileStrategies", "fileStrategy", "filteredTools", "filters", "imageOutputType", "includedTools", "interface", "langfuse", "mcpServers", "mcpSettings", "memory", "messageFilter", "modelSpecs", "ocr", "openapi", "permissions", "rateLimits", "registration", "secureImageLinks", "skillSync", "speech", "summarization", "transactions", "turnstile", "version", "webSearch", } LC_DATA_PROVIDER_DIST = os.environ.get( "LC_DATA_PROVIDER_DIST", "/app/code/librechat/packages/data-provider/dist/index.js") LC_JS_YAML = os.environ.get( "LC_JS_YAML", "/app/code/librechat/node_modules/js-yaml") def fail(key, reason): """ONE line, naming the offending key, then a non-zero exit.""" print(f"CONFIG VALIDATION FAILED: {key}: {reason}", flush=True) sys.exit(1) def load_yaml(path): try: import yaml except ImportError: print("CONFIG VALIDATION FAILED: pyyaml not available in this python", flush=True) sys.exit(2) try: with open(path) as fh: data = yaml.safe_load(fh) except FileNotFoundError: print(f"CONFIG VALIDATION FAILED: config file not found: {path}", flush=True) sys.exit(2) except PermissionError: print(f"CONFIG VALIDATION FAILED: config file unreadable: {path}", flush=True) sys.exit(2) except yaml.YAMLError as exc: # Do not echo the raw yaml error: it can quote a secret-bearing line. print("CONFIG VALIDATION FAILED: (root): yaml does not parse", flush=True) sys.exit(2) if data is None: data = {} if not isinstance(data, dict): fail("(root)", "top level must be a mapping") return data def validate_litellm(path): cfg = load_yaml(path) unknown = sorted(set(cfg) - LITELLM_TOP_LEVEL) if unknown: fail(", ".join(unknown), "unknown top-level key(s) for litellm " f"(allowed: {', '.join(sorted(LITELLM_TOP_LEVEL))})") model_list = cfg.get("model_list") if not isinstance(model_list, list) or not model_list: fail("model_list", "must be a non-empty list of model entries") names = set() for i, entry in enumerate(model_list): if not isinstance(entry, dict): fail(f"model_list[{i}]", "entry must be a mapping") name = entry.get("model_name") params = entry.get("litellm_params") if not name or not isinstance(name, str): fail(f"model_list[{i}].model_name", "missing or empty") if name in names: fail(f"model_list[{i}].model_name", f"duplicate model_name {name!r}") names.add(name) if not isinstance(params, dict) or not params.get("model"): fail(f"model_list[{i}].litellm_params.model", "missing or empty") for block in ("router_settings", "litellm_settings", "general_settings"): if block in cfg and not isinstance(cfg[block], dict): fail(block, "must be a mapping") for ep in (cfg.get("general_settings") or {}).get("pass_through_endpoints") or []: if not isinstance(ep, dict) or not ep.get("path"): fail("general_settings.pass_through_endpoints", "each entry needs a path") # litellm-assisted check (best effort): resolve each model string to a # provider with the gateway's own package. Skipped where litellm itself # is not importable (never fatal — the structural checks above ran). try: from litellm import get_llm_provider # noqa: WPS433 except Exception: print("note: litellm not importable here; provider check skipped", flush=True) else: for entry in model_list: model = entry["litellm_params"]["model"] try: get_llm_provider(model=model, custom_llm_provider=None) except Exception as exc: short = str(exc).split("\n")[0][:120] fail(f"model_list.{entry['model_name']}.litellm_params.model", f"not resolvable to a provider: {short}") print(f"config OK (litellm): {len(cfg)} top-level key(s), " f"{len(model_list)} model(s) validated", flush=True) sys.exit(0) def validate_librechat(path): cfg = load_yaml(path) if not cfg: print("config OK (librechat): empty config, nothing to validate", flush=True) sys.exit(0) # Preferred: the DEPLOYED build's own strict schema, run through node — # the exact code path the API server uses at boot (strict() rejects # unrecognized top-level keys; upstream then exits 1). node_code = r""" const fs = require('fs'); let out = {mode: 'schema'}; try { const yaml = require(JS_YAML_PATH); const dp = require(DIST_PATH); const cfg = yaml.load(fs.readFileSync(CONFIG_PATH, 'utf8')) || {}; const result = dp.configSchema.strict().safeParse(cfg); if (result.success) { out.ok = true; out.keys = Object.keys(cfg).length; } else { out.ok = false; out.errors = (result.error && result.error.issues || []).slice(0, 10) .map(i => ({path: (i.path || []).join('.') || '(root)', msg: i.message})); } } catch (e) { out.mode = 'schema-unavailable'; out.error = String(e && e.message || e).slice(0, 200); } console.log('@@' + JSON.stringify(out)); """.replace("JS_YAML_PATH", json.dumps(LC_JS_YAML)) \ .replace("DIST_PATH", json.dumps(LC_DATA_PROVIDER_DIST)) \ .replace("CONFIG_PATH", json.dumps(path)) if os.path.exists(LC_DATA_PROVIDER_DIST): try: proc = subprocess.run(["node", "-e", node_code], capture_output=True, text=True, timeout=60) for line in (proc.stdout or "").splitlines(): if line.startswith("@@"): res = json.loads(line[2:]) if res.get("ok"): print(f"config OK (librechat): {res['keys']} top-level " "key(s) accepted by the deployed strict schema", flush=True) sys.exit(0) errs = "; ".join( f"{e['path']}: {e['msg']}" for e in res.get("errors", [])) if errs: fail(errs, "rejected by the deployed build's strict " "config schema (configSchema.strict())") fail("(root)", "strict schema rejected the config") # No @@ line: schema path unavailable — fall through to fallback. print("note: deployed schema probe produced no result " f"(stderr: {(proc.stderr or '')[:120]!r})", flush=True) except (OSError, subprocess.TimeoutExpired) as exc: print(f"note: could not run node against the deployed schema " f"({type(exc).__name__})", flush=True) else: print("note: deployed data-provider dist not found; using the static " "fallback allowlist recorded from the deployed build", flush=True) # Fallback: static allowlist (still fail-closed). unknown = sorted(set(cfg) - LIBRECHAT_TOP_LEVEL_FALLBACK) if unknown: fail(", ".join(unknown), "unknown top-level key(s) for librechat " f"(fallback allowlist: {', '.join(sorted(LIBRECHAT_TOP_LEVEL_FALLBACK))})") print(f"config OK (librechat, fallback allowlist): {len(cfg)} top-level " "key(s)", flush=True) sys.exit(0) def main(): args = sys.argv[1:] if len(args) != 2 or args[0] not in ("litellm", "librechat"): print("usage: validate_config.py ", file=sys.stderr, flush=True) sys.exit(2) if args[0] == "litellm": validate_litellm(args[1]) validate_librechat(args[1]) if __name__ == "__main__": main()