Compare commits
9
Commits
upgrade-v1.108.3
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
437761171c | ||
|
|
84a191851f | ||
|
|
1ec589cf8b | ||
|
|
9266f9bc3c | ||
|
|
02ff60bfbf | ||
|
|
5986ee7987 | ||
|
|
aa1531bb18 | ||
|
|
e21c24ea48 | ||
|
|
64d7e5afc5 |
No files matched your search
@@ -16,13 +16,7 @@
|
|||||||
"addons": {
|
"addons": {
|
||||||
"postgresql": {},
|
"postgresql": {},
|
||||||
"redis": {},
|
"redis": {},
|
||||||
"localstorage": {},
|
"localstorage": {}
|
||||||
"scheduler": {
|
|
||||||
"audio_keepalive": {
|
|
||||||
"schedule": "*/10 * * * *",
|
|
||||||
"command": "/app/code/audio_keepalive.py"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
"tags": [
|
"tags": [
|
||||||
"ai",
|
"ai",
|
||||||
@@ -31,5 +25,6 @@
|
|||||||
"api"
|
"api"
|
||||||
],
|
],
|
||||||
"mediaLinks": [],
|
"mediaLinks": [],
|
||||||
"changelog": "Initial package for inference.coop"
|
"changelog": "Initial package for inference.coop",
|
||||||
|
"dockerImage": "registry.inference.coop/code/litellm:20261114-84a1918"
|
||||||
}
|
}
|
||||||
+7
-2
@@ -23,6 +23,11 @@ COPY config.yaml /app/data/config.yaml
|
|||||||
# /app/data volume at boot, so config changes in the repo actually deploy.
|
# /app/data volume at boot, so config changes in the repo actually deploy.
|
||||||
COPY config.yaml /app/code/config.yaml
|
COPY config.yaml /app/code/config.yaml
|
||||||
|
|
||||||
|
# Fail-closed config validator (canonical copy lives in ops-docs/scripts/;
|
||||||
|
# this byte-identical twin ships with the image and is called from start.sh
|
||||||
|
# before the gateway boots). Mirrors validate_config.py's md5 — keep in step.
|
||||||
|
COPY validate_config.py /app/code/validate_config.py
|
||||||
|
|
||||||
# Append the co-op credit-pack hint to LiteLLM's hardcoded budget-exceeded
|
# Append the co-op credit-pack hint to LiteLLM's hardcoded budget-exceeded
|
||||||
# messages. Done at BUILD time — the runtime rootfs is read-only under
|
# messages. Done at BUILD time — the runtime rootfs is read-only under
|
||||||
# Cloudron, so a boot-time patch would crash the startup script (it did,
|
# Cloudron, so a boot-time patch would crash the startup script (it did,
|
||||||
@@ -87,8 +92,8 @@ RUN python3 /app/code/patch_passthrough_key_access.py
|
|||||||
COPY patch_passthrough_token_cost.py /app/code/patch_passthrough_token_cost.py
|
COPY patch_passthrough_token_cost.py /app/code/patch_passthrough_token_cost.py
|
||||||
RUN python3 /app/code/patch_passthrough_token_cost.py
|
RUN python3 /app/code/patch_passthrough_token_cost.py
|
||||||
|
|
||||||
# Audio keep-alive: scheduled pings keep the Tinfoil TEE audio enclave
|
# Audio keep-alive: the in-container loop in start.sh pings tinfoil/voxtral-tts
|
||||||
# warm so voice mode skips the ~24s cold-start (see audio_keepalive.py header).
|
# every 15 min so voice mode skips the ~24s cold-start (see audio_keepalive.py).
|
||||||
COPY audio_keepalive.py /app/code/audio_keepalive.py
|
COPY audio_keepalive.py /app/code/audio_keepalive.py
|
||||||
RUN chmod +x /app/code/audio_keepalive.py
|
RUN chmod +x /app/code/audio_keepalive.py
|
||||||
|
|
||||||
|
|||||||
+5
-3
@@ -2,10 +2,12 @@
|
|||||||
"""Audio keep-alive: ping the Tinfoil TEE audio models through localhost LiteLLM
|
"""Audio keep-alive: ping the Tinfoil TEE audio models through localhost LiteLLM
|
||||||
so voice mode doesn't pay the ~24s cold-start after idle.
|
so voice mode doesn't pay the ~24s cold-start after idle.
|
||||||
|
|
||||||
Runs INSIDE the gateway container via the Cloudron scheduler addon. Uses the
|
Runs INSIDE the gateway container, invoked by the keep-alive loop in
|
||||||
LITELLM_MASTER_KEY already in the container env. TTS-only (STT pings cost
|
start.sh (every 15 min; the Cloudron scheduler task was removed
|
||||||
|
2026-11-09 so the loop is the only warm path). Reads the master key from
|
||||||
|
env or /app/data/.master_key. TTS-only (STT pings cost
|
||||||
$0.01/request each; TTS $0.002 — see docs, keep-alive economics). Silent on
|
$0.01/request each; TTS $0.002 — see docs, keep-alive economics). Silent on
|
||||||
success; prints + exits non-zero on failure so scheduler logs capture it.
|
success; prints + exits non-zero on failure so keepalive.log captures it.
|
||||||
"""
|
"""
|
||||||
import json, os, urllib.request, urllib.error, sys
|
import json, os, urllib.request, urllib.error, sys
|
||||||
|
|
||||||
|
|||||||
+82
@@ -13,6 +13,8 @@
|
|||||||
# greenpt/qwen3-embedding-8b: $0.25 in / $0 out per 1M (embeddings, 32K ctx)
|
# greenpt/qwen3-embedding-8b: $0.25 in / $0 out per 1M (embeddings, 32K ctx)
|
||||||
# greenpt/green-embeddings: $0.20 in / $0 out per 1M (embeddings, 32K ctx)
|
# greenpt/green-embeddings: $0.20 in / $0 out per 1M (embeddings, 32K ctx)
|
||||||
# tinfoil/nomic-embed-text: $0.016 in / $0 out per 1M (embeddings, 8K ctx, TEE)
|
# tinfoil/nomic-embed-text: $0.016 in / $0 out per 1M (embeddings, 8K ctx, TEE)
|
||||||
|
# regolo/mistral-small-4-119b: $0.20 in / $0.70 out per 1M (60K ctx)
|
||||||
|
# regolo/gemma4-31b: $0.40 in / $0.60 out per 1M (100K ctx)
|
||||||
# tinfoil/voxtral-small-24b: $0.20 in / $0.60 out per 1M (token-priced audio;
|
# tinfoil/voxtral-small-24b: $0.20 in / $0.60 out per 1M (token-priced audio;
|
||||||
# +$0.00008/request on /v1/audio/transcriptions)
|
# +$0.00008/request on /v1/audio/transcriptions)
|
||||||
# Without these, LiteLLM can't price these custom models, so spend shows $0.00
|
# Without these, LiteLLM can't price these custom models, so spend shows $0.00
|
||||||
@@ -117,6 +119,10 @@ model_list:
|
|||||||
# qwen3.5-122b: large vision+reasoning MoE (262K ctx). REQUIRES
|
# qwen3.5-122b: large vision+reasoning MoE (262K ctx). REQUIRES
|
||||||
# chat_template_kwargs: {enable_thinking: false} — otherwise the reasoning
|
# chat_template_kwargs: {enable_thinking: false} — otherwise the reasoning
|
||||||
# phase consumes the entire token budget and content comes back empty.
|
# phase consumes the entire token budget and content comes back empty.
|
||||||
|
# mistral-small-4-119b + gemma4-31b: added 2026-11-13 (operator-approved).
|
||||||
|
# Rates read live from Regolo's own model catalog (regolo.ai/models):
|
||||||
|
# mistral $0.20/$0.70 per 1M, 60K in / 60K out; gemma4 $0.40/$0.60,
|
||||||
|
# 100K in / 100K out.
|
||||||
# Qwen-Image: image generation via /v1/images/generations, billed per pixel.
|
# Qwen-Image: image generation via /v1/images/generations, billed per pixel.
|
||||||
- model_name: regolo/apertus-70b
|
- model_name: regolo/apertus-70b
|
||||||
litellm_params:
|
litellm_params:
|
||||||
@@ -139,6 +145,30 @@ model_list:
|
|||||||
input_cost_per_token: 6.96e-7
|
input_cost_per_token: 6.96e-7
|
||||||
output_cost_per_token: 3.944e-6
|
output_cost_per_token: 3.944e-6
|
||||||
|
|
||||||
|
# Mistral Small 4 (119B) — hybrid instruct + reasoning model (three model
|
||||||
|
# families unified by Regolo). $0.20 in / $0.70 out per 1M, 60K ctx.
|
||||||
|
- model_name: regolo/mistral-small-4-119b
|
||||||
|
litellm_params:
|
||||||
|
model: openai/mistral-small-4-119b
|
||||||
|
api_base: https://api.regolo.ai/v1
|
||||||
|
api_key: os.environ/REGOLO_API_KEY
|
||||||
|
model_info:
|
||||||
|
mode: chat
|
||||||
|
input_cost_per_token: 2.0e-7
|
||||||
|
output_cost_per_token: 7.0e-7
|
||||||
|
|
||||||
|
# Gemma 4 (31B) — Google's dense multimodal Gemma 4. $0.40 in / $0.60 out
|
||||||
|
# per 1M, 100K ctx.
|
||||||
|
- model_name: regolo/gemma4-31b
|
||||||
|
litellm_params:
|
||||||
|
model: openai/gemma4-31b
|
||||||
|
api_base: https://api.regolo.ai/v1
|
||||||
|
api_key: os.environ/REGOLO_API_KEY
|
||||||
|
model_info:
|
||||||
|
mode: chat
|
||||||
|
input_cost_per_token: 4.0e-7
|
||||||
|
output_cost_per_token: 6.0e-7
|
||||||
|
|
||||||
- model_name: regolo/qwen-image
|
- model_name: regolo/qwen-image
|
||||||
litellm_params:
|
litellm_params:
|
||||||
model: openai/Qwen-Image
|
model: openai/Qwen-Image
|
||||||
@@ -299,6 +329,58 @@ model_list:
|
|||||||
# logged; and a retry_policy that retries transient connection errors (base
|
# logged; and a retry_policy that retries transient connection errors (base
|
||||||
# num_retries) but NEVER retries timeouts (TimeoutErrorRetries: 0) — a timed-out
|
# num_retries) but NEVER retries timeouts (TimeoutErrorRetries: 0) — a timed-out
|
||||||
# generation is already billed by Tinfoil, so re-hitting it re-bills.
|
# generation is already billed by Tinfoil, so re-hitting it re-bills.
|
||||||
|
|
||||||
|
# --- Stable aliases -------------------------------------------------------
|
||||||
|
# Version-free names members can pin once and keep using across transitions.
|
||||||
|
# When we move the underlying model, only the pointer here changes; member
|
||||||
|
# client configs and the docs do not. Pricing is stated AT the alias, so a
|
||||||
|
# transition that changes price is visible in the model list and in budgets
|
||||||
|
# (LiteLLM bills the alias's model_info, not the target's).
|
||||||
|
# Policy: `<family>-green` = the default route (renewable energy, policy-based
|
||||||
|
# privacy); `<family>-tee` = the same family served inside a TEE (attested
|
||||||
|
# hardware). The names say WHICH ROUTE, not how fresh a version is: we re-point
|
||||||
|
# them at transitions, so pinning one is what makes a config version-free.
|
||||||
|
- model_name: deepseek-flash-green
|
||||||
|
litellm_params:
|
||||||
|
model: openai/deepseek-v4.1-flash
|
||||||
|
api_base: https://api.greenpt.ai/v1
|
||||||
|
api_key: os.environ/GREENPT_API_KEY
|
||||||
|
model_info:
|
||||||
|
input_cost_per_token: 0.00000022
|
||||||
|
output_cost_per_token: 0.00000110
|
||||||
|
cache_read_input_token_cost: 0.000000011
|
||||||
|
|
||||||
|
- model_name: deepseek-flash-tee
|
||||||
|
litellm_params:
|
||||||
|
model: openai/deepseek-v4-1-flash
|
||||||
|
api_base: http://127.0.0.1:3301/v1
|
||||||
|
api_key: os.environ/TINFOIL_API_KEY
|
||||||
|
model_info:
|
||||||
|
input_cost_per_token: 0.00000065
|
||||||
|
output_cost_per_token: 0.00000145
|
||||||
|
cache_read_input_token_cost: 0.00000013
|
||||||
|
|
||||||
|
- model_name: glm-flash-green
|
||||||
|
litellm_params:
|
||||||
|
model: openai/glm-5.3-flash
|
||||||
|
api_base: https://api.greenpt.ai/v1
|
||||||
|
api_key: os.environ/GREENPT_API_KEY
|
||||||
|
model_info:
|
||||||
|
input_cost_per_token: 0.00000011
|
||||||
|
output_cost_per_token: 0.00000044
|
||||||
|
cache_read_input_token_cost: 0.000000022
|
||||||
|
|
||||||
|
# Tinfoil's flash GLM was deprecated 2026-10-09; the TEE route for this family
|
||||||
|
# is the flagship, so the alias is named glm-tee rather than glm-flash-tee.
|
||||||
|
- model_name: glm-tee
|
||||||
|
litellm_params:
|
||||||
|
model: openai/glm-5-3
|
||||||
|
api_base: http://127.0.0.1:3301/v1
|
||||||
|
api_key: os.environ/TINFOIL_API_KEY
|
||||||
|
model_info:
|
||||||
|
input_cost_per_token: 0.00000180
|
||||||
|
output_cost_per_token: 0.00000575
|
||||||
|
cache_read_input_token_cost: 0.00000045
|
||||||
router_settings:
|
router_settings:
|
||||||
num_retries: 2
|
num_retries: 2
|
||||||
timeout: 180
|
timeout: 180
|
||||||
|
|||||||
@@ -190,6 +190,20 @@ fi
|
|||||||
# cannot happen through this path, and the shipped /app/code/config.yaml is
|
# cannot happen through this path, and the shipped /app/code/config.yaml is
|
||||||
# the live catalog.
|
# the live catalog.
|
||||||
|
|
||||||
|
# --- Fail-closed config validation (2026-11-11) ---
|
||||||
|
# Validate the config that will actually be SERVED, before starting anything.
|
||||||
|
# A bad key must fail LOUD at the top of the boot log (the chat outage of
|
||||||
|
# 2026-10-31 showed what silent misconfiguration costs). The -f guard keeps a
|
||||||
|
# missing validator non-fatal (build patch did not apply); an INVALID config
|
||||||
|
# is fatal by design.
|
||||||
|
if [ -f /app/code/validate_config.py ]; then
|
||||||
|
echo "Validating gateway config (fail-closed)..."
|
||||||
|
if ! python3 /app/code/validate_config.py litellm "$CONFIG_FILE"; then
|
||||||
|
echo "FATAL: gateway config.yaml is invalid - refusing to start. Fix or remove the offending key named above." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# --- Start LiteLLM ---
|
# --- Start LiteLLM ---
|
||||||
echo "Starting LiteLLM on port 4000..."
|
echo "Starting LiteLLM on port 4000..."
|
||||||
|
|
||||||
@@ -239,20 +253,21 @@ if [ "$proxy_up" != "1" ]; then
|
|||||||
fi
|
fi
|
||||||
echo "Tinfoil proxy is up on 127.0.0.1:3301"
|
echo "Tinfoil proxy is up on 127.0.0.1:3301"
|
||||||
|
|
||||||
# --- In-container audio keep-alive loop ---
|
# --- In-container audio keep-alive loop (the ONLY warm path) ---
|
||||||
# The scheduler task (handled above in scheduler-task mode) runs in a SEPARATE
|
# A separate scheduler-task container was removed 2026-11-09: its container
|
||||||
# container whose ability to reach 127.0.0.1:4000 and read /app/data is not
|
# could not be proven to reach 127.0.0.1:4000 and /app/data, and two warm
|
||||||
# guaranteed, so the warm path is owned HERE — the app container, where the
|
# paths billed twice for the same ping. The warm path is owned HERE — the
|
||||||
# network namespace, the master key and the sidecar are all proven. Every 8
|
# app container, where the network namespace, the master key and the
|
||||||
# minutes ping tinfoil/voxtral-tts ($0.002/request) so voice mode skips the
|
# sidecar are all proven. Every 15 minutes ping tinfoil/voxtral-tts
|
||||||
# ~24s Tinfoil cold-start after idle. audio_keepalive.py is silent on success
|
# ($0.002/request) so voice mode skips the ~24s Tinfoil cold-start after
|
||||||
# and prints on failure, so /app/data/keepalive.log only ever grows on failure.
|
# idle. audio_keepalive.py is silent on success and prints on failure, so
|
||||||
|
# /app/data/keepalive.log only ever grows on failure.
|
||||||
(
|
(
|
||||||
while true; do
|
while true; do
|
||||||
python3 /app/code/audio_keepalive.py >> /app/data/keepalive.log 2>&1 || true
|
python3 /app/code/audio_keepalive.py >> /app/data/keepalive.log 2>&1 || true
|
||||||
sleep 480
|
sleep 900
|
||||||
done
|
done
|
||||||
) &
|
) &
|
||||||
echo "Audio keep-alive loop started (every 480s)"
|
echo "Audio keep-alive loop started (every 900s)"
|
||||||
|
|
||||||
exec litellm --config "$CONFIG_FILE" --port 4000 --host 0.0.0.0
|
exec litellm --config "$CONFIG_FILE" --port 4000 --host 0.0.0.0
|
||||||
@@ -0,0 +1,261 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Fail-closed config validator for the co-op's gateway and chat apps.
|
||||||
|
|
||||||
|
ONE script, two modes, shared home in ops-docs/scripts/ and shipped
|
||||||
|
byte-identical into both app images (litellm repo root, librechat-custom repo
|
||||||
|
root). Each app's start path calls it before booting the service; a non-zero
|
||||||
|
exit refuses the boot with a ONE-line error naming the offending key.
|
||||||
|
|
||||||
|
Why this exists: the 2026-10-31 chat outage. A `customFooter:` top-level key in
|
||||||
|
librechat.yaml was rejected by LibreChat's strict config validator and the API
|
||||||
|
server refused to start, while the SPA kept being served with HTTP 200 — the
|
||||||
|
app looked up but every API route returned the HTML shell. Nothing had ever
|
||||||
|
checked the config before boot. This validator closes that gap: invalid config
|
||||||
|
fails LOUD and EARLY, at the top of the boot log, naming the key.
|
||||||
|
|
||||||
|
Modes
|
||||||
|
-----
|
||||||
|
litellm Structural check of the gateway's config.yaml:
|
||||||
|
- top-level keys within LiteLLM's documented config surface;
|
||||||
|
- model_list present and well-formed (model_name + litellm_params.model);
|
||||||
|
- the settings blocks are objects; pass_through_endpoints entries have paths.
|
||||||
|
When the gateway's own litellm package is importable it is additionally
|
||||||
|
used to resolve every model string to a provider (catches unknown
|
||||||
|
provider prefixes); the check is skipped with a note where litellm
|
||||||
|
itself is absent.
|
||||||
|
|
||||||
|
librechat Checks librechat.yaml against the DEPLOYED build's own strict schema:
|
||||||
|
it runs `configSchema.strict().safeParse()` from the container's
|
||||||
|
librechat-data-provider via node — the same code the API server runs
|
||||||
|
at boot (its failure mode is `process.exit(1)`, "Exiting due to
|
||||||
|
invalid configuration"). Unrecognized top-level keys are rejected,
|
||||||
|
named explicitly. Where node/the package cannot be found (e.g. this
|
||||||
|
script run from a checkout instead of the image) it falls back to a
|
||||||
|
static top-level allowlist recorded from the deployed build and says
|
||||||
|
so on the failure line.
|
||||||
|
|
||||||
|
Usage
|
||||||
|
-----
|
||||||
|
python3 validate_config.py litellm /app/data/config.yaml
|
||||||
|
python3 validate_config.py librechat /app/data/librechat.yaml
|
||||||
|
|
||||||
|
Exit codes: 0 valid, 1 invalid, 2 usage/environment error (file unreadable,
|
||||||
|
yaml unparseable). Never prints a value from the config — only key names and
|
||||||
|
counts, so it is safe to run against files that carry secrets.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
# LiteLLM's documented top-level config surface (proxy config). Unknown keys
|
||||||
|
# are rejected here — the gateway should never silently ignore a mistyped
|
||||||
|
# section (a "model-list:" typo would otherwise deploy an EMPTY catalog).
|
||||||
|
LITELLM_TOP_LEVEL = {
|
||||||
|
"model_list",
|
||||||
|
"router_settings",
|
||||||
|
"litellm_settings",
|
||||||
|
"general_settings",
|
||||||
|
"settings",
|
||||||
|
"environment_variables",
|
||||||
|
"guardrails",
|
||||||
|
"files",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Fallback allowlist for librechat mode, recorded 2026-11-11 from the DEPLOYED
|
||||||
|
# build (chat.inference.coop, overlay 1.4.8) by reading
|
||||||
|
# configSchema.strict().shape keys. The live schema is always preferred; this
|
||||||
|
# exists so the validator still fails closed when run outside the container.
|
||||||
|
LIBRECHAT_TOP_LEVEL_FALLBACK = {
|
||||||
|
"actions", "balance", "cache", "cloudfront", "endpoints", "fileConfig",
|
||||||
|
"fileStrategies", "fileStrategy", "filteredTools", "filters",
|
||||||
|
"imageOutputType", "includedTools", "interface", "langfuse", "mcpServers",
|
||||||
|
"mcpSettings", "memory", "messageFilter", "modelSpecs", "ocr", "openapi",
|
||||||
|
"permissions", "rateLimits", "registration", "secureImageLinks",
|
||||||
|
"skillSync", "speech", "summarization", "transactions", "turnstile",
|
||||||
|
"version", "webSearch",
|
||||||
|
}
|
||||||
|
|
||||||
|
LC_DATA_PROVIDER_DIST = os.environ.get(
|
||||||
|
"LC_DATA_PROVIDER_DIST",
|
||||||
|
"/app/code/librechat/packages/data-provider/dist/index.js")
|
||||||
|
LC_JS_YAML = os.environ.get(
|
||||||
|
"LC_JS_YAML", "/app/code/librechat/node_modules/js-yaml")
|
||||||
|
|
||||||
|
|
||||||
|
def fail(key, reason):
|
||||||
|
"""ONE line, naming the offending key, then a non-zero exit."""
|
||||||
|
print(f"CONFIG VALIDATION FAILED: {key}: {reason}", flush=True)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def load_yaml(path):
|
||||||
|
try:
|
||||||
|
import yaml
|
||||||
|
except ImportError:
|
||||||
|
print("CONFIG VALIDATION FAILED: pyyaml not available in this python",
|
||||||
|
flush=True)
|
||||||
|
sys.exit(2)
|
||||||
|
try:
|
||||||
|
with open(path) as fh:
|
||||||
|
data = yaml.safe_load(fh)
|
||||||
|
except FileNotFoundError:
|
||||||
|
print(f"CONFIG VALIDATION FAILED: config file not found: {path}",
|
||||||
|
flush=True)
|
||||||
|
sys.exit(2)
|
||||||
|
except PermissionError:
|
||||||
|
print(f"CONFIG VALIDATION FAILED: config file unreadable: {path}",
|
||||||
|
flush=True)
|
||||||
|
sys.exit(2)
|
||||||
|
except yaml.YAMLError as exc:
|
||||||
|
# Do not echo the raw yaml error: it can quote a secret-bearing line.
|
||||||
|
print("CONFIG VALIDATION FAILED: (root): yaml does not parse",
|
||||||
|
flush=True)
|
||||||
|
sys.exit(2)
|
||||||
|
if data is None:
|
||||||
|
data = {}
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
fail("(root)", "top level must be a mapping")
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def validate_litellm(path):
|
||||||
|
cfg = load_yaml(path)
|
||||||
|
unknown = sorted(set(cfg) - LITELLM_TOP_LEVEL)
|
||||||
|
if unknown:
|
||||||
|
fail(", ".join(unknown), "unknown top-level key(s) for litellm "
|
||||||
|
f"(allowed: {', '.join(sorted(LITELLM_TOP_LEVEL))})")
|
||||||
|
|
||||||
|
model_list = cfg.get("model_list")
|
||||||
|
if not isinstance(model_list, list) or not model_list:
|
||||||
|
fail("model_list", "must be a non-empty list of model entries")
|
||||||
|
names = set()
|
||||||
|
for i, entry in enumerate(model_list):
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
fail(f"model_list[{i}]", "entry must be a mapping")
|
||||||
|
name = entry.get("model_name")
|
||||||
|
params = entry.get("litellm_params")
|
||||||
|
if not name or not isinstance(name, str):
|
||||||
|
fail(f"model_list[{i}].model_name", "missing or empty")
|
||||||
|
if name in names:
|
||||||
|
fail(f"model_list[{i}].model_name", f"duplicate model_name {name!r}")
|
||||||
|
names.add(name)
|
||||||
|
if not isinstance(params, dict) or not params.get("model"):
|
||||||
|
fail(f"model_list[{i}].litellm_params.model", "missing or empty")
|
||||||
|
for block in ("router_settings", "litellm_settings", "general_settings"):
|
||||||
|
if block in cfg and not isinstance(cfg[block], dict):
|
||||||
|
fail(block, "must be a mapping")
|
||||||
|
for ep in (cfg.get("general_settings") or {}).get("pass_through_endpoints") or []:
|
||||||
|
if not isinstance(ep, dict) or not ep.get("path"):
|
||||||
|
fail("general_settings.pass_through_endpoints", "each entry needs a path")
|
||||||
|
|
||||||
|
# litellm-assisted check (best effort): resolve each model string to a
|
||||||
|
# provider with the gateway's own package. Skipped where litellm itself
|
||||||
|
# is not importable (never fatal — the structural checks above ran).
|
||||||
|
try:
|
||||||
|
from litellm import get_llm_provider # noqa: WPS433
|
||||||
|
except Exception:
|
||||||
|
print("note: litellm not importable here; provider check skipped",
|
||||||
|
flush=True)
|
||||||
|
else:
|
||||||
|
for entry in model_list:
|
||||||
|
model = entry["litellm_params"]["model"]
|
||||||
|
try:
|
||||||
|
get_llm_provider(model=model, custom_llm_provider=None)
|
||||||
|
except Exception as exc:
|
||||||
|
short = str(exc).split("\n")[0][:120]
|
||||||
|
fail(f"model_list.{entry['model_name']}.litellm_params.model",
|
||||||
|
f"not resolvable to a provider: {short}")
|
||||||
|
|
||||||
|
print(f"config OK (litellm): {len(cfg)} top-level key(s), "
|
||||||
|
f"{len(model_list)} model(s) validated", flush=True)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_librechat(path):
|
||||||
|
cfg = load_yaml(path)
|
||||||
|
if not cfg:
|
||||||
|
print("config OK (librechat): empty config, nothing to validate",
|
||||||
|
flush=True)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
# Preferred: the DEPLOYED build's own strict schema, run through node —
|
||||||
|
# the exact code path the API server uses at boot (strict() rejects
|
||||||
|
# unrecognized top-level keys; upstream then exits 1).
|
||||||
|
node_code = r"""
|
||||||
|
const fs = require('fs');
|
||||||
|
let out = {mode: 'schema'};
|
||||||
|
try {
|
||||||
|
const yaml = require(JS_YAML_PATH);
|
||||||
|
const dp = require(DIST_PATH);
|
||||||
|
const cfg = yaml.load(fs.readFileSync(CONFIG_PATH, 'utf8')) || {};
|
||||||
|
const result = dp.configSchema.strict().safeParse(cfg);
|
||||||
|
if (result.success) {
|
||||||
|
out.ok = true;
|
||||||
|
out.keys = Object.keys(cfg).length;
|
||||||
|
} else {
|
||||||
|
out.ok = false;
|
||||||
|
out.errors = (result.error && result.error.issues || []).slice(0, 10)
|
||||||
|
.map(i => ({path: (i.path || []).join('.') || '(root)', msg: i.message}));
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
out.mode = 'schema-unavailable';
|
||||||
|
out.error = String(e && e.message || e).slice(0, 200);
|
||||||
|
}
|
||||||
|
console.log('@@' + JSON.stringify(out));
|
||||||
|
""".replace("JS_YAML_PATH", json.dumps(LC_JS_YAML)) \
|
||||||
|
.replace("DIST_PATH", json.dumps(LC_DATA_PROVIDER_DIST)) \
|
||||||
|
.replace("CONFIG_PATH", json.dumps(path))
|
||||||
|
|
||||||
|
if os.path.exists(LC_DATA_PROVIDER_DIST):
|
||||||
|
try:
|
||||||
|
proc = subprocess.run(["node", "-e", node_code], capture_output=True,
|
||||||
|
text=True, timeout=60)
|
||||||
|
for line in (proc.stdout or "").splitlines():
|
||||||
|
if line.startswith("@@"):
|
||||||
|
res = json.loads(line[2:])
|
||||||
|
if res.get("ok"):
|
||||||
|
print(f"config OK (librechat): {res['keys']} top-level "
|
||||||
|
"key(s) accepted by the deployed strict schema",
|
||||||
|
flush=True)
|
||||||
|
sys.exit(0)
|
||||||
|
errs = "; ".join(
|
||||||
|
f"{e['path']}: {e['msg']}" for e in res.get("errors", []))
|
||||||
|
if errs:
|
||||||
|
fail(errs, "rejected by the deployed build's strict "
|
||||||
|
"config schema (configSchema.strict())")
|
||||||
|
fail("(root)", "strict schema rejected the config")
|
||||||
|
# No @@ line: schema path unavailable — fall through to fallback.
|
||||||
|
print("note: deployed schema probe produced no result "
|
||||||
|
f"(stderr: {(proc.stderr or '')[:120]!r})", flush=True)
|
||||||
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||||
|
print(f"note: could not run node against the deployed schema "
|
||||||
|
f"({type(exc).__name__})", flush=True)
|
||||||
|
else:
|
||||||
|
print("note: deployed data-provider dist not found; using the static "
|
||||||
|
"fallback allowlist recorded from the deployed build", flush=True)
|
||||||
|
|
||||||
|
# Fallback: static allowlist (still fail-closed).
|
||||||
|
unknown = sorted(set(cfg) - LIBRECHAT_TOP_LEVEL_FALLBACK)
|
||||||
|
if unknown:
|
||||||
|
fail(", ".join(unknown), "unknown top-level key(s) for librechat "
|
||||||
|
f"(fallback allowlist: {', '.join(sorted(LIBRECHAT_TOP_LEVEL_FALLBACK))})")
|
||||||
|
print(f"config OK (librechat, fallback allowlist): {len(cfg)} top-level "
|
||||||
|
"key(s)", flush=True)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = sys.argv[1:]
|
||||||
|
if len(args) != 2 or args[0] not in ("litellm", "librechat"):
|
||||||
|
print("usage: validate_config.py <litellm|librechat> <config.yaml>",
|
||||||
|
file=sys.stderr, flush=True)
|
||||||
|
sys.exit(2)
|
||||||
|
if args[0] == "litellm":
|
||||||
|
validate_litellm(args[1])
|
||||||
|
validate_librechat(args[1])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in new issue
Block a user