Compare commits
41
Commits
upgrade-v1.108.0
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ac9b3b17af | ||
|
|
140b443980 | ||
|
|
fe48ca8e38 | ||
|
|
6fcc116199 | ||
|
|
c8816b58c1 | ||
|
|
718644c363 | ||
|
|
870b165321 | ||
|
|
aa9331cff3 | ||
|
|
6f6bb73d16 | ||
|
|
9c81df48b8 | ||
|
|
d1d342d8e6 | ||
|
|
0f7cf17cd8 | ||
|
|
278900684c | ||
|
|
df4e154128 | ||
|
|
4b6583c3c0 | ||
|
|
2483715ef3 | ||
|
|
f66bedfb55 | ||
|
|
0e8517b6be | ||
|
|
d35ee1760f | ||
|
|
faff547ff4 | ||
|
|
b81177c3ad | ||
|
|
d96df6fd5a | ||
|
|
7a167a8354 | ||
|
|
a1613401c3 | ||
|
|
77e38d81e0 | ||
|
|
d877bc8eee | ||
|
|
437761171c | ||
|
|
84a191851f | ||
|
|
1ec589cf8b | ||
|
|
9266f9bc3c | ||
|
|
02ff60bfbf | ||
|
|
5986ee7987 | ||
|
|
aa1531bb18 | ||
|
|
e21c24ea48 | ||
|
|
64d7e5afc5 | ||
|
|
c43d45c455 | ||
|
|
5d38403892 | ||
|
|
943701581c | ||
|
|
abaeb255b2 | ||
|
|
67df968a04 | ||
|
|
54925cae97 |
No files matched your search
@@ -0,0 +1,2 @@
|
|||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
+5
-10
@@ -5,10 +5,10 @@
|
|||||||
"description": "LiteLLM AI Gateway \u2014 OpenAI-compatible proxy with per-user API keys, usage tracking, rate limiting, and model routing. Packaged for Cloudron with SSO/OIDC and PostgreSQL.",
|
"description": "LiteLLM AI Gateway \u2014 OpenAI-compatible proxy with per-user API keys, usage tracking, rate limiting, and model routing. Packaged for Cloudron with SSO/OIDC and PostgreSQL.",
|
||||||
"tagline": "AI gateway for cooperative inference",
|
"tagline": "AI gateway for cooperative inference",
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"upstreamVersion": "1.103.2",
|
"upstreamVersion": "1.108.3",
|
||||||
"healthCheckPath": "/health/readiness",
|
"healthCheckPath": "/health/readiness",
|
||||||
"httpPort": 4000,
|
"httpPort": 4000,
|
||||||
"memoryLimit": 2147483648,
|
"memoryLimit": 3221225472,
|
||||||
"manifestVersion": 2,
|
"manifestVersion": 2,
|
||||||
"website": "https://litellm.ai",
|
"website": "https://litellm.ai",
|
||||||
"contactEmail": "info@inference.coop",
|
"contactEmail": "info@inference.coop",
|
||||||
@@ -16,13 +16,7 @@
|
|||||||
"addons": {
|
"addons": {
|
||||||
"postgresql": {},
|
"postgresql": {},
|
||||||
"redis": {},
|
"redis": {},
|
||||||
"localstorage": {},
|
"localstorage": {}
|
||||||
"scheduler": {
|
|
||||||
"audio_keepalive": {
|
|
||||||
"schedule": "*/10 * * * *",
|
|
||||||
"command": "/app/code/audio_keepalive.py"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
"tags": [
|
"tags": [
|
||||||
"ai",
|
"ai",
|
||||||
@@ -31,5 +25,6 @@
|
|||||||
"api"
|
"api"
|
||||||
],
|
],
|
||||||
"mediaLinks": [],
|
"mediaLinks": [],
|
||||||
"changelog": "Initial package for inference.coop"
|
"changelog": "Initial package for inference.coop",
|
||||||
|
"dockerImage": "registry.inference.coop/code/litellm:20261118-140b443"
|
||||||
}
|
}
|
||||||
+39
-3
@@ -1,4 +1,4 @@
|
|||||||
FROM ghcr.io/berriai/litellm:v1.103.2
|
FROM ghcr.io/berriai/litellm:v1.108.3
|
||||||
|
|
||||||
# v1.84.0 is the fix line for CVE-2026-35029 (auth bypass on
|
# v1.84.0 is the fix line for CVE-2026-35029 (auth bypass on
|
||||||
# /config/update, fixed 1.83.0) and CVE-2026-59822 (MCP session auth
|
# /config/update, fixed 1.83.0) and CVE-2026-59822 (MCP session auth
|
||||||
@@ -22,6 +22,13 @@ COPY config.yaml /app/data/config.yaml
|
|||||||
# A second copy inside the image: start.sh syncs it over the persistent
|
# A second copy inside the image: start.sh syncs it over the persistent
|
||||||
# /app/data volume at boot, so config changes in the repo actually deploy.
|
# /app/data volume at boot, so config changes in the repo actually deploy.
|
||||||
COPY config.yaml /app/code/config.yaml
|
COPY config.yaml /app/code/config.yaml
|
||||||
|
COPY fx_sync.py /app/code/fx_sync.py
|
||||||
|
COPY eur_prices.json /app/code/eur_prices.json
|
||||||
|
|
||||||
|
# Fail-closed config validator (canonical copy lives in ops-docs/scripts/;
|
||||||
|
# this byte-identical twin ships with the image and is called from start.sh
|
||||||
|
# before the gateway boots). Mirrors validate_config.py's md5 — keep in step.
|
||||||
|
COPY validate_config.py /app/code/validate_config.py
|
||||||
|
|
||||||
# Append the co-op credit-pack hint to LiteLLM's hardcoded budget-exceeded
|
# Append the co-op credit-pack hint to LiteLLM's hardcoded budget-exceeded
|
||||||
# messages. Done at BUILD time — the runtime rootfs is read-only under
|
# messages. Done at BUILD time — the runtime rootfs is read-only under
|
||||||
@@ -54,6 +61,18 @@ RUN python3 /app/code/patch_audio_request_cost.py
|
|||||||
COPY patch_audio_router_pricing.py /app/code/patch_audio_router_pricing.py
|
COPY patch_audio_router_pricing.py /app/code/patch_audio_router_pricing.py
|
||||||
RUN python3 /app/code/patch_audio_router_pricing.py
|
RUN python3 /app/code/patch_audio_router_pricing.py
|
||||||
|
|
||||||
|
# Audio per-request prices must live in the SHARED cost map, which is built from
|
||||||
|
# model_prices_and_context_window.json — the per-request helper reads that map, not
|
||||||
|
# model_info, so a router-only patch left every speech call at $0 (issue #5).
|
||||||
|
# This patch fails the build rather than warning.
|
||||||
|
COPY patch_audio_cost_map.py /app/code/patch_audio_cost_map.py
|
||||||
|
RUN python3 /app/code/patch_audio_cost_map.py
|
||||||
|
|
||||||
|
# Runtime registration: the build-time file edit above did not reach the deployed
|
||||||
|
# gateway's model_cost map (issue #5), so register the entries in-process instead.
|
||||||
|
# sitecustomize is imported before any application code.
|
||||||
|
COPY sitecustomize.py /app/.venv/lib/python3.13/site-packages/sitecustomize.py
|
||||||
|
|
||||||
# SECURITY (2026-10-27, member-disclosure finding 4): LiteLLM forwards every
|
# SECURITY (2026-10-27, member-disclosure finding 4): LiteLLM forwards every
|
||||||
# provider response header to callers as `llm_provider-*`, which leaked the
|
# provider response header to callers as `llm_provider-*`, which leaked the
|
||||||
# co-op's remaining provider balance (x-credits-remaining) to any member API key.
|
# co-op's remaining provider balance (x-credits-remaining) to any member API key.
|
||||||
@@ -87,11 +106,28 @@ RUN python3 /app/code/patch_passthrough_key_access.py
|
|||||||
COPY patch_passthrough_token_cost.py /app/code/patch_passthrough_token_cost.py
|
COPY patch_passthrough_token_cost.py /app/code/patch_passthrough_token_cost.py
|
||||||
RUN python3 /app/code/patch_passthrough_token_cost.py
|
RUN python3 /app/code/patch_passthrough_token_cost.py
|
||||||
|
|
||||||
# Audio keep-alive: scheduled pings keep the Tinfoil TEE audio enclave
|
# Audio keep-alive: the in-container loop in start.sh pings tinfoil/voxtral-tts
|
||||||
# warm so voice mode skips the ~24s cold-start (see audio_keepalive.py header).
|
# every 15 min so voice mode skips the ~24s cold-start (see audio_keepalive.py).
|
||||||
COPY audio_keepalive.py /app/code/audio_keepalive.py
|
COPY audio_keepalive.py /app/code/audio_keepalive.py
|
||||||
RUN chmod +x /app/code/audio_keepalive.py
|
RUN chmod +x /app/code/audio_keepalive.py
|
||||||
|
|
||||||
|
# Skip LiteLLM's forced verbose_json on transcription requests for opting-out
|
||||||
|
# models: Tinfoil's voxtral-small-24b rejects verbose_json with a 400, which
|
||||||
|
# broke every bare /v1/audio/transcriptions call to it (file + model — the
|
||||||
|
# shape members and dictation send). Its billing does not need the duration
|
||||||
|
# the injection exists for. Non-fatal; check the "[coop-patch] applied" line
|
||||||
|
# in the build log on every LiteLLM upgrade.
|
||||||
|
COPY patch_transcription_response_format.py /app/code/patch_transcription_response_format.py
|
||||||
|
RUN python3 /app/code/patch_transcription_response_format.py
|
||||||
|
|
||||||
|
# TTS default response format: LibreChat sends no response_format on its TTS
|
||||||
|
# request and Tinfoil's voxtral-tts then serves WAV (~8-17s warm measured) —
|
||||||
|
# while LibreChat labels the reply audio/mpeg regardless. Default to mp3
|
||||||
|
# (OpenAI's documented default, ~3s warm measured). Non-fatal; check the
|
||||||
|
# "[coop-patch] applied" line in the build log on every LiteLLM upgrade.
|
||||||
|
COPY patch_audio_default_mp3.py /app/code/patch_audio_default_mp3.py
|
||||||
|
RUN python3 /app/code/patch_audio_default_mp3.py
|
||||||
|
|
||||||
# Override the base image's ENTRYPOINT (which is `litellm`) so our
|
# Override the base image's ENTRYPOINT (which is `litellm`) so our
|
||||||
# startup script runs instead of being passed as an argument to litellm.
|
# startup script runs instead of being passed as an argument to litellm.
|
||||||
ENTRYPOINT ["/app/code/start.sh"]
|
ENTRYPOINT ["/app/code/start.sh"]
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ A Cloudron app package for [LiteLLM](https://litellm.ai/), the open-source AI ga
|
|||||||
|
|
||||||
LiteLLM is the single gateway between members and the inference backends. It handles per-member API keys, usage metering, rate limiting (via budget caps), and model routing — all behind one OpenAI-compatible endpoint.
|
LiteLLM is the single gateway between members and the inference backends. It handles per-member API keys, usage metering, rate limiting (via budget caps), and model routing — all behind one OpenAI-compatible endpoint.
|
||||||
|
|
||||||
**Current base image:** LiteLLM **v1.103.2**, pinned to the exact tag (fix line for CVE-2026-35029 / CVE-2026-59822, CISA KEV-listed; also carries the streaming-usage fix for prompt_tokens_details). See [Updating LiteLLM](#updating-litellm).
|
**Current base image:** LiteLLM **v1.108.3**, pinned to the exact tag (upgraded from v1.103.2 in the 2026-11-08/09 maintenance window; minimum supported line remains v1.84.0, the fix line for CVE-2026-35029 / CVE-2026-59822, CISA KEV-listed). See [Updating LiteLLM](#updating-litellm).
|
||||||
|
|
||||||
## What this provides
|
## What this provides
|
||||||
|
|
||||||
@@ -149,6 +149,19 @@ expect a 15–20 min maintenance window and **never panic-restart mid-migration*
|
|||||||
|
|
||||||
1. **Pick a target version** from the [LiteLLM changelog](https://github.com/BerriAI/litellm/releases). Stay at or above v1.84.0 (the fix line for CVE-2026-35029 and CVE-2026-59822, the latter on CISA's KEV list).
|
1. **Pick a target version** from the [LiteLLM changelog](https://github.com/BerriAI/litellm/releases). Stay at or above v1.84.0 (the fix line for CVE-2026-35029 and CVE-2026-59822, the latter on CISA's KEV list).
|
||||||
|
|
||||||
|
**Re-validate the build patches against the target's real sources first** — the
|
||||||
|
scripts fail soft (a moved anchor prints a WARN and exits 0), so a silently
|
||||||
|
skipped patch ships. Ten patches are in the Dockerfile today; each must print
|
||||||
|
its `applied` line in the build log. The cheapest rehearsal is to unpack the
|
||||||
|
target wheel into a scratch tree and run the scripts against it (2026-11-09,
|
||||||
|
v1.108.3: all ten apply — the CORS anchor needs the 1.108.x fallback already
|
||||||
|
carried by `patch_expose_cost_headers.py`). Two gotchas in that harness:
|
||||||
|
`patch_audio_request_cost.py` finds its target by `import litellm`, which fails
|
||||||
|
outside a full install — point `LITELLM_COST_CALCULATOR` at the scratch
|
||||||
|
`cost_calculator.py` instead, or it exits with "could not locate"; and running
|
||||||
|
the scripts by hand leaves files owned by your user, which is fine in scratch
|
||||||
|
and never in the image.
|
||||||
|
|
||||||
2. **Bump the pin** in `Dockerfile`:
|
2. **Bump the pin** in `Dockerfile`:
|
||||||
|
|
||||||
```dockerfile
|
```dockerfile
|
||||||
|
|||||||
Binary file not shown.
+5
-3
@@ -2,10 +2,12 @@
|
|||||||
"""Audio keep-alive: ping the Tinfoil TEE audio models through localhost LiteLLM
|
"""Audio keep-alive: ping the Tinfoil TEE audio models through localhost LiteLLM
|
||||||
so voice mode doesn't pay the ~24s cold-start after idle.
|
so voice mode doesn't pay the ~24s cold-start after idle.
|
||||||
|
|
||||||
Runs INSIDE the gateway container via the Cloudron scheduler addon. Uses the
|
Runs INSIDE the gateway container, invoked by the keep-alive loop in
|
||||||
LITELLM_MASTER_KEY already in the container env. TTS-only (STT pings cost
|
start.sh (every 15 min; the Cloudron scheduler task was removed
|
||||||
|
2026-11-09 so the loop is the only warm path). Reads the master key from
|
||||||
|
env or /app/data/.master_key. TTS-only (STT pings cost
|
||||||
$0.01/request each; TTS $0.002 — see docs, keep-alive economics). Silent on
|
$0.01/request each; TTS $0.002 — see docs, keep-alive economics). Silent on
|
||||||
success; prints + exits non-zero on failure so scheduler logs capture it.
|
success; prints + exits non-zero on failure so keepalive.log captures it.
|
||||||
"""
|
"""
|
||||||
import json, os, urllib.request, urllib.error, sys
|
import json, os, urllib.request, urllib.error, sys
|
||||||
|
|
||||||
|
|||||||
+201
-24
@@ -6,10 +6,20 @@
|
|||||||
# via environment variable TINFOIL_API_KEY.
|
# via environment variable TINFOIL_API_KEY.
|
||||||
#
|
#
|
||||||
# Pricing (model_info) is per-token, derived from provider per-1M rates:
|
# Pricing (model_info) is per-token, derived from provider per-1M rates:
|
||||||
# greenpt/deepseek-v4.1-flash: $0.22 in / $1.10 out / $0.011 cached per 1M
|
# EUR->USD FX = 1.16 (policy, set 2026-11-18). GreenPT and Regolo quote EUR;
|
||||||
|
# LiteLLM's cost fields are USD-shaped, so EUR list prices are converted here.
|
||||||
|
# Never paste EUR numerals into model_info — that under-bills members by the FX gap.
|
||||||
|
# greenpt/deepseek-v4.1-flash: $0.36 in / $1.45 out / $0.04 cached per 1M (1017 snapshot)
|
||||||
# tinfoil/deepseek-v4-1-flash: $0.65 in / $1.45 out / $0.13 cached per 1M
|
# tinfoil/deepseek-v4-1-flash: $0.65 in / $1.45 out / $0.13 cached per 1M
|
||||||
# tinfoil/gpt-oss-120b: $0.15 in / $0.60 out per 1M
|
# tinfoil/gpt-oss-120b: $0.15 in / $0.60 out per 1M
|
||||||
# tinfoil/glm-5.3: $1.80 in / $5.75 out / $0.45 cached per 1M
|
# tinfoil/glm-5.3: $1.80 in / $5.75 out / $0.45 cached per 1M
|
||||||
|
# greenpt/qwen3-embedding-8b: $0.25 in / $0 out per 1M (embeddings, 32K ctx)
|
||||||
|
# greenpt/green-embeddings: $0.20 in / $0 out per 1M (embeddings, 32K ctx)
|
||||||
|
# tinfoil/nomic-embed-text: $0.016 in / $0 out per 1M (embeddings, 8K ctx, TEE)
|
||||||
|
# regolo/mistral-small-4-119b: $0.20 in / $0.70 out per 1M (60K ctx)
|
||||||
|
# regolo/gemma4-31b: $0.40 in / $0.60 out per 1M (100K ctx)
|
||||||
|
# tinfoil/voxtral-small-24b: $0.20 in / $0.60 out per 1M (token-priced audio;
|
||||||
|
# +$0.00008/request on /v1/audio/transcriptions)
|
||||||
# Without these, LiteLLM can't price these custom models, so spend shows $0.00
|
# Without these, LiteLLM can't price these custom models, so spend shows $0.00
|
||||||
# and the team budget cap is not enforced.
|
# and the team budget cap is not enforced.
|
||||||
#
|
#
|
||||||
@@ -19,20 +29,25 @@
|
|||||||
# models.md is the member-facing mirror.
|
# models.md is the member-facing mirror.
|
||||||
|
|
||||||
model_list:
|
model_list:
|
||||||
# DeepSeek V4.1 Flash on GreenPT — DEFAULT model. Same model family as Tinfoil's
|
# DeepSeek V4.1 Flash on GreenPT — DEFAULT model. SERVES THE 1017 SNAPSHOT
|
||||||
|
# (17 Oct 2026) as of 2026-11-16, while KEEPING its version-free model_name: members
|
||||||
|
# pin this id in client configs we cannot see, and renaming would 400 every one of
|
||||||
|
# them (measured: 3,014 calls in 2 days on this name, 84 on the deepseek-flash-green
|
||||||
|
# alias). Repointing is the whole change — no member action.
|
||||||
|
# Same model family as Tinfoil's
|
||||||
# deepseek-v4-1-flash, but on renewable energy and NOT marked Experimental by the
|
# deepseek-v4-1-flash, but on renewable energy and NOT marked Experimental by the
|
||||||
# provider (Tinfoil's deepseek + glm-flash carry an "Experimental: may be deprecated
|
# provider (Tinfoil's deepseek + glm-flash carry an "Experimental: may be deprecated
|
||||||
# with minimal notice" flag). Policy-based privacy (GreenPT) vs TEE (Tinfoil).
|
# with minimal notice" flag). Policy-based privacy (GreenPT) vs TEE (Tinfoil).
|
||||||
# €0.22 in / €1.10 out / €0.011 cached per 1M.
|
# $0.36 in / $1.45 out / $0.04 cached per 1M (1017 snapshot; repointed 2026-11-16).
|
||||||
- model_name: greenpt/deepseek-v4.1-flash
|
- model_name: greenpt/deepseek-v4.1-flash
|
||||||
litellm_params:
|
litellm_params:
|
||||||
model: openai/deepseek-v4.1-flash
|
model: openai/deepseek-v4.1-flash-1017
|
||||||
api_base: https://api.greenpt.ai/v1
|
api_base: https://api.greenpt.ai/v1
|
||||||
api_key: os.environ/GREENPT_API_KEY
|
api_key: os.environ/GREENPT_API_KEY
|
||||||
model_info:
|
model_info:
|
||||||
input_cost_per_token: 0.00000022
|
input_cost_per_token: 4.176e-07
|
||||||
output_cost_per_token: 0.00000110
|
output_cost_per_token: 1.682e-06
|
||||||
cache_read_input_token_cost: 0.000000011
|
cache_read_input_token_cost: 4.64e-08
|
||||||
|
|
||||||
# DeepSeek V4.1 Flash on Tinfoil (TEE) — private counterpart to the default.
|
# DeepSeek V4.1 Flash on Tinfoil (TEE) — private counterpart to the default.
|
||||||
- model_name: tinfoil/deepseek-v4-1-flash
|
- model_name: tinfoil/deepseek-v4-1-flash
|
||||||
@@ -81,9 +96,9 @@ model_list:
|
|||||||
api_base: https://api.greenpt.ai/v1
|
api_base: https://api.greenpt.ai/v1
|
||||||
api_key: os.environ/GREENPT_API_KEY
|
api_key: os.environ/GREENPT_API_KEY
|
||||||
model_info:
|
model_info:
|
||||||
input_cost_per_token: 0.00000011
|
input_cost_per_token: 1.276e-07
|
||||||
output_cost_per_token: 0.00000044
|
output_cost_per_token: 5.104e-07
|
||||||
cache_read_input_token_cost: 0.000000022
|
cache_read_input_token_cost: 2.552e-08
|
||||||
|
|
||||||
# GLM-5.3 (full flagship) on GreenPT — strongest agentic model in the green
|
# GLM-5.3 (full flagship) on GreenPT — strongest agentic model in the green
|
||||||
# tier. $1.10 in / $4.40 out per 1M, 1M context.
|
# tier. $1.10 in / $4.40 out per 1M, 1M context.
|
||||||
@@ -93,17 +108,20 @@ model_list:
|
|||||||
api_base: https://api.greenpt.ai/v1
|
api_base: https://api.greenpt.ai/v1
|
||||||
api_key: os.environ/GREENPT_API_KEY
|
api_key: os.environ/GREENPT_API_KEY
|
||||||
model_info:
|
model_info:
|
||||||
input_cost_per_token: 0.0000011
|
input_cost_per_token: 1.276e-06
|
||||||
output_cost_per_token: 0.0000044
|
output_cost_per_token: 5.104e-06
|
||||||
cache_read_input_token_cost: 0.000000275
|
cache_read_input_token_cost: 3.19e-07
|
||||||
|
|
||||||
# --- Audio models (Tinfoil, inside the TEE) ---
|
# --- Audio models (Tinfoil, inside the TEE) ---
|
||||||
# NOT chat models: these power /v1/audio/* routes (STT for dictation, TTS for
|
# NOT chat models: these power /v1/audio/* routes (STT for dictation, TTS for
|
||||||
# voice mode). They stay VISIBLE in the API model list so developer members
|
# voice mode). They stay VISIBLE in the API model list so developer members
|
||||||
# can call them, but the portal filters them out of the chat-facing list
|
# can call them, but the portal filters them out of the chat-facing list
|
||||||
# (Open WebUI) by their `mode` field, so they never appear as pickable chat
|
# (Open WebUI) by their `mode` field, so they never appear as pickable chat
|
||||||
# options. Pricing per Tinfoil catalog: whisper $0.05/1M in; voxtral-tts
|
# options. Tinfoil bills audio per REQUEST, not per token: whisper
|
||||||
# listed $0/$0 (verify on invoice — may be beta-free).
|
# $0.01/request, voxtral-tts $0.002/request (both $0 per token). LiteLLM cannot
|
||||||
|
# price that natively, so each entry declares input_cost_per_request and the
|
||||||
|
# build-time patches make the calculator honour it (issue #5, closed 2026-11-17
|
||||||
|
# with a verified $0.002 spend row).
|
||||||
|
|
||||||
# --- Regolo (EU, zero data retention, 100% renewables) ---
|
# --- Regolo (EU, zero data retention, 100% renewables) ---
|
||||||
# Seeweb S.r.l. (Italy). Added 2026-10-16 following member suggestion (#22).
|
# Seeweb S.r.l. (Italy). Added 2026-10-16 following member suggestion (#22).
|
||||||
@@ -112,6 +130,10 @@ model_list:
|
|||||||
# qwen3.5-122b: large vision+reasoning MoE (262K ctx). REQUIRES
|
# qwen3.5-122b: large vision+reasoning MoE (262K ctx). REQUIRES
|
||||||
# chat_template_kwargs: {enable_thinking: false} — otherwise the reasoning
|
# chat_template_kwargs: {enable_thinking: false} — otherwise the reasoning
|
||||||
# phase consumes the entire token budget and content comes back empty.
|
# phase consumes the entire token budget and content comes back empty.
|
||||||
|
# mistral-small-4-119b + gemma4-31b: added 2026-11-13 (operator-approved).
|
||||||
|
# Rates read live from Regolo's own model catalog (regolo.ai/models):
|
||||||
|
# mistral $0.20/$0.70 per 1M, 60K in / 60K out; gemma4 $0.40/$0.60,
|
||||||
|
# 100K in / 100K out.
|
||||||
# Qwen-Image: image generation via /v1/images/generations, billed per pixel.
|
# Qwen-Image: image generation via /v1/images/generations, billed per pixel.
|
||||||
- model_name: regolo/apertus-70b
|
- model_name: regolo/apertus-70b
|
||||||
litellm_params:
|
litellm_params:
|
||||||
@@ -134,6 +156,30 @@ model_list:
|
|||||||
input_cost_per_token: 6.96e-7
|
input_cost_per_token: 6.96e-7
|
||||||
output_cost_per_token: 3.944e-6
|
output_cost_per_token: 3.944e-6
|
||||||
|
|
||||||
|
# Mistral Small 4 (119B) — hybrid instruct + reasoning model (three model
|
||||||
|
# families unified by Regolo). $0.20 in / $0.70 out per 1M, 60K ctx.
|
||||||
|
- model_name: regolo/mistral-small-4-119b
|
||||||
|
litellm_params:
|
||||||
|
model: openai/mistral-small-4-119b
|
||||||
|
api_base: https://api.regolo.ai/v1
|
||||||
|
api_key: os.environ/REGOLO_API_KEY
|
||||||
|
model_info:
|
||||||
|
mode: chat
|
||||||
|
input_cost_per_token: 2.32e-07
|
||||||
|
output_cost_per_token: 8.12e-07
|
||||||
|
|
||||||
|
# Gemma 4 (31B) — Google's dense multimodal Gemma 4. $0.40 in / $0.60 out
|
||||||
|
# per 1M, 100K ctx.
|
||||||
|
- model_name: regolo/gemma4-31b
|
||||||
|
litellm_params:
|
||||||
|
model: openai/gemma4-31b
|
||||||
|
api_base: https://api.regolo.ai/v1
|
||||||
|
api_key: os.environ/REGOLO_API_KEY
|
||||||
|
model_info:
|
||||||
|
mode: chat
|
||||||
|
input_cost_per_token: 4.64e-07
|
||||||
|
output_cost_per_token: 6.96e-07
|
||||||
|
|
||||||
- model_name: regolo/qwen-image
|
- model_name: regolo/qwen-image
|
||||||
litellm_params:
|
litellm_params:
|
||||||
model: openai/Qwen-Image
|
model: openai/Qwen-Image
|
||||||
@@ -150,6 +196,8 @@ model_list:
|
|||||||
# picker, but they remain fully callable via /v1/audio/* for API users.
|
# picker, but they remain fully callable via /v1/audio/* for API users.
|
||||||
# whisper-large-v3-turbo: $0/1M tokens + $0.01 per request.
|
# whisper-large-v3-turbo: $0/1M tokens + $0.01 per request.
|
||||||
# voxtral-tts: $0/1M tokens + $0.002 per request.
|
# voxtral-tts: $0/1M tokens + $0.002 per request.
|
||||||
|
# voxtral-small-24b: $0.20/1M in + $0.60/1M out tokens, $0 per request —
|
||||||
|
# the only TOKEN-PRICED audio model in their catalog (its block below).
|
||||||
# Tinfoil bills these per REQUEST (its model list carries a `requestPrice`);
|
# Tinfoil bills these per REQUEST (its model list carries a `requestPrice`);
|
||||||
# reading only the two token prices made TTS look free and left STT billed off
|
# reading only the two token prices made TTS look free and left STT billed off
|
||||||
# an older catalog. LiteLLM cannot price per request on the audio paths, so
|
# an older catalog. LiteLLM cannot price per request on the audio paths, so
|
||||||
@@ -171,6 +219,34 @@ model_list:
|
|||||||
input_cost_per_token: 0
|
input_cost_per_token: 0
|
||||||
input_cost_per_request: 0.01
|
input_cost_per_request: 0.01
|
||||||
|
|
||||||
|
# voxtral-small-24b — Tinfoil's TOKEN-PRICED STT model: the only audio entry
|
||||||
|
# in their catalog with real token prices ($0.20 in / $0.60 out per 1M, $0
|
||||||
|
# per request). The chat route (audio sent as input_audio) returns real
|
||||||
|
# token usage and bills exactly at those rates. The /v1/audio/transcriptions
|
||||||
|
# route returns NO token counts (duration only), so per-token billing cannot
|
||||||
|
# fire there; Tinfoil tokenizes audio in ~30s chunks (~375-380 tokens each —
|
||||||
|
# measured: a 1-second clip and a 6-second clip both cost ~380 prompt
|
||||||
|
# tokens), so for the short dictation requests this route serves the
|
||||||
|
# chunk-equivalent is ~$0.00008/request — declared as input_cost_per_request
|
||||||
|
# below, added by patch_audio_request_cost.py, which is scoped to the audio
|
||||||
|
# call types so chat calls to this model are NOT double-billed. Audio longer
|
||||||
|
# than ~30s should go through the chat route, which meters exactly.
|
||||||
|
- model_name: tinfoil/voxtral-small-24b
|
||||||
|
litellm_params:
|
||||||
|
model: openai/voxtral-small-24b
|
||||||
|
api_base: http://127.0.0.1:3301/v1
|
||||||
|
api_key: os.environ/TINFOIL_API_KEY
|
||||||
|
model_info:
|
||||||
|
mode: audio_transcription
|
||||||
|
# Opt out of LiteLLM's forced response_format=verbose_json injection on
|
||||||
|
# transcription calls (patch_transcription_response_format.py): Tinfoil
|
||||||
|
# rejects verbose_json for this model with a 400, and nothing in its
|
||||||
|
# billing needs the duration the injection exists to obtain.
|
||||||
|
coop_no_verbose_json: true
|
||||||
|
input_cost_per_token: 0.0000002
|
||||||
|
output_cost_per_token: 0.0000006
|
||||||
|
input_cost_per_request: 0.00008
|
||||||
|
|
||||||
- model_name: tinfoil/voxtral-tts
|
- model_name: tinfoil/voxtral-tts
|
||||||
litellm_params:
|
litellm_params:
|
||||||
model: openai/voxtral-tts
|
model: openai/voxtral-tts
|
||||||
@@ -184,19 +260,25 @@ model_list:
|
|||||||
# branch (cost_per_token, cost_metric == "cost_per_character") raises
|
# branch (cost_per_token, cost_metric == "cost_per_character") raises
|
||||||
# ValueError("cost for tts call is None") when there is no character price,
|
# ValueError("cost for tts call is None") when there is no character price,
|
||||||
# and that raise aborts the whole cost calculation — so the spend write is
|
# and that raise aborts the whole cost calculation — so the spend write is
|
||||||
# skipped and speech bills nothing, whatever input_cost_per_request says.
|
# skipped and speech billed nothing, whatever input_cost_per_request said
|
||||||
|
# (that was issue #5; the build patch now prices the base at 0.0 for a
|
||||||
|
# per-request model, so the declared charge is the whole bill).
|
||||||
# Declaring the zero lets the path complete; the patch then adds the
|
# Declaring the zero lets the path complete; the patch then adds the
|
||||||
# per-request charge.
|
# per-request charge.
|
||||||
input_cost_per_character: 0
|
input_cost_per_character: 0
|
||||||
input_cost_per_request: 0.002
|
input_cost_per_request: 0.002
|
||||||
|
|
||||||
# --- Embedding model (Tinfoil, in-TEE) ---
|
# --- Embedding models (Tinfoil, in-TEE; and GreenPT) ---
|
||||||
# nomic-embed-text — 768-dim text embeddings for RAG/semantic search.
|
# nomic-embed-text — 768-dim text embeddings for RAG/semantic search.
|
||||||
# NOT a chat model: it powers /v1/embeddings only. Scoped to the admin
|
# NOT a chat model: it powers /v1/embeddings only. MEMBER-FACING since
|
||||||
# account's team (model_info.team_id) so member keys cannot call it — it is
|
# 2026-11-06 — the team_id scoping (operator-account-only) was removed so
|
||||||
# an operator resource for the co-op's own docs-RAG, not a member-facing
|
# members get a TEE-protected embedding option. This is the same model the
|
||||||
# model. The portal also filters it out of the chat-facing /v1/models list by
|
# co-op's own docs-RAG runs on inside the TEE (Open WebUI's knowledge base);
|
||||||
# its `mode: embedding`. Pricing per Tinfoil catalog: $0.05/1M input, $0 out.
|
# that traffic is unaffected by members also being able to call it. The
|
||||||
|
# portal filters `mode: embedding` out of the chat-facing /v1/models list, so
|
||||||
|
# it stays API-only. Pricing per Tinfoil catalog (inference.tinfoil.sh
|
||||||
|
# /v1/models): $0.016/1M input, $0 out — corrected 2026-11-06 from a stale
|
||||||
|
# $0.05 declaration; 8K context declared so clients don't invent larger.
|
||||||
- model_name: tinfoil/nomic-embed-text
|
- model_name: tinfoil/nomic-embed-text
|
||||||
litellm_params:
|
litellm_params:
|
||||||
model: openai/nomic-embed-text
|
model: openai/nomic-embed-text
|
||||||
@@ -204,10 +286,53 @@ model_list:
|
|||||||
api_key: os.environ/TINFOIL_API_KEY
|
api_key: os.environ/TINFOIL_API_KEY
|
||||||
model_info:
|
model_info:
|
||||||
mode: embedding
|
mode: embedding
|
||||||
team_id: cdfdf422-633f-4432-8893-cffee82a8325
|
max_input_tokens: 8192
|
||||||
input_cost_per_token: 0.00000005
|
input_cost_per_token: 0.000000016
|
||||||
output_cost_per_token: 0.0
|
output_cost_per_token: 0.0
|
||||||
|
|
||||||
|
# --- Embedding model (GreenPT, member-facing) ---
|
||||||
|
# qwen3-embedding-8b — text embeddings via /v1/embeddings, callable by any
|
||||||
|
# member key. Added 2026-11-06 after a member asked for an embedding model,
|
||||||
|
# filling the member-facing gap on renewable EU hosting (Tinfoil's private
|
||||||
|
# embedding was opened to members the same day). The portal filters
|
||||||
|
# mode:embedding out of the chat picker, so it is API-only — same shape as
|
||||||
|
# the audio models above.
|
||||||
|
# Pricing per GreenPT /v1/pricing: $0.25/1M input, no output charge
|
||||||
|
# (embeddings return vectors; GreenPT lists maxOutputTokens null), 32K
|
||||||
|
# context. max_input_tokens declared from the provider's contextWindowTokens
|
||||||
|
# so clients don't invent a larger one (the null-caps defect).
|
||||||
|
- model_name: greenpt/qwen3-embedding-8b
|
||||||
|
litellm_params:
|
||||||
|
model: openai/qwen3-embedding-8b
|
||||||
|
api_base: https://api.greenpt.ai/v1
|
||||||
|
api_key: os.environ/GREENPT_API_KEY
|
||||||
|
model_info:
|
||||||
|
mode: embedding
|
||||||
|
input_cost_per_token: 2.9e-07
|
||||||
|
output_cost_per_token: 0
|
||||||
|
max_input_tokens: 32000
|
||||||
|
|
||||||
|
# --- Embedding model (GreenPT, member-facing) ---
|
||||||
|
# green-embeddings — GreenPT's own embedding model (2560-dim, measured live).
|
||||||
|
# Added 2026-11-06 after a catalog survey: the only genuinely distinct
|
||||||
|
# embedding model on GreenPT besides qwen3-embedding-8b. Their catalog also
|
||||||
|
# lists a `green-embedding` (singular) slug — verified the SAME model
|
||||||
|
# (identical vectors on the same input), so only green-embeddings is served.
|
||||||
|
# Cheaper per token than qwen3-embedding-8b; the reranker sibling
|
||||||
|
# (green-rerank) is a different capability and is NOT served.
|
||||||
|
# Same shape: /v1/embeddings only, filtered out of the chat picker.
|
||||||
|
# Pricing per GreenPT /v1/pricing: $0.20/1M input, no output charge, 32K ctx.
|
||||||
|
- model_name: greenpt/green-embeddings
|
||||||
|
litellm_params:
|
||||||
|
model: openai/green-embeddings
|
||||||
|
api_base: https://api.greenpt.ai/v1
|
||||||
|
api_key: os.environ/GREENPT_API_KEY
|
||||||
|
model_info:
|
||||||
|
mode: embedding
|
||||||
|
input_cost_per_token: 2.32e-07
|
||||||
|
output_cost_per_token: 0
|
||||||
|
max_input_tokens: 32000
|
||||||
|
|
||||||
# Fallback: if DeepSeek is down, use GPT-OSS
|
# Fallback: if DeepSeek is down, use GPT-OSS
|
||||||
# NOTE: DeepSeek V4.1 Flash is a reasoning model — generations routinely run
|
# NOTE: DeepSeek V4.1 Flash is a reasoning model — generations routinely run
|
||||||
# longer than 30s. A 30s timeout caused LiteLLM to abandon in-flight DeepSeek
|
# longer than 30s. A 30s timeout caused LiteLLM to abandon in-flight DeepSeek
|
||||||
@@ -217,6 +342,58 @@ model_list:
|
|||||||
# logged; and a retry_policy that retries transient connection errors (base
|
# logged; and a retry_policy that retries transient connection errors (base
|
||||||
# num_retries) but NEVER retries timeouts (TimeoutErrorRetries: 0) — a timed-out
|
# num_retries) but NEVER retries timeouts (TimeoutErrorRetries: 0) — a timed-out
|
||||||
# generation is already billed by Tinfoil, so re-hitting it re-bills.
|
# generation is already billed by Tinfoil, so re-hitting it re-bills.
|
||||||
|
|
||||||
|
# --- Stable aliases -------------------------------------------------------
|
||||||
|
# Version-free names members can pin once and keep using across transitions.
|
||||||
|
# When we move the underlying model, only the pointer here changes; member
|
||||||
|
# client configs and the docs do not. Pricing is stated AT the alias, so a
|
||||||
|
# transition that changes price is visible in the model list and in budgets
|
||||||
|
# (LiteLLM bills the alias's model_info, not the target's).
|
||||||
|
# Policy: `<family>-green` = the default route (renewable energy, policy-based
|
||||||
|
# privacy); `<family>-tee` = the same family served inside a TEE (attested
|
||||||
|
# hardware). The names say WHICH ROUTE, not how fresh a version is: we re-point
|
||||||
|
# them at transitions, so pinning one is what makes a config version-free.
|
||||||
|
- model_name: deepseek-flash-green
|
||||||
|
litellm_params:
|
||||||
|
model: openai/deepseek-v4.1-flash-1017
|
||||||
|
api_base: https://api.greenpt.ai/v1
|
||||||
|
api_key: os.environ/GREENPT_API_KEY
|
||||||
|
model_info:
|
||||||
|
input_cost_per_token: 4.176e-07
|
||||||
|
output_cost_per_token: 1.682e-06
|
||||||
|
cache_read_input_token_cost: 4.64e-08
|
||||||
|
|
||||||
|
- model_name: deepseek-flash-tee
|
||||||
|
litellm_params:
|
||||||
|
model: openai/deepseek-v4-1-flash
|
||||||
|
api_base: http://127.0.0.1:3301/v1
|
||||||
|
api_key: os.environ/TINFOIL_API_KEY
|
||||||
|
model_info:
|
||||||
|
input_cost_per_token: 0.00000065
|
||||||
|
output_cost_per_token: 0.00000145
|
||||||
|
cache_read_input_token_cost: 0.00000013
|
||||||
|
|
||||||
|
- model_name: glm-flash-green
|
||||||
|
litellm_params:
|
||||||
|
model: openai/glm-5.3-flash
|
||||||
|
api_base: https://api.greenpt.ai/v1
|
||||||
|
api_key: os.environ/GREENPT_API_KEY
|
||||||
|
model_info:
|
||||||
|
input_cost_per_token: 1.276e-07
|
||||||
|
output_cost_per_token: 5.104e-07
|
||||||
|
cache_read_input_token_cost: 2.552e-08
|
||||||
|
|
||||||
|
# Tinfoil's flash GLM was deprecated 2026-10-09; the TEE route for this family
|
||||||
|
# is the flagship, so the alias is named glm-tee rather than glm-flash-tee.
|
||||||
|
- model_name: glm-tee
|
||||||
|
litellm_params:
|
||||||
|
model: openai/glm-5-3
|
||||||
|
api_base: http://127.0.0.1:3301/v1
|
||||||
|
api_key: os.environ/TINFOIL_API_KEY
|
||||||
|
model_info:
|
||||||
|
input_cost_per_token: 0.00000180
|
||||||
|
output_cost_per_token: 0.00000575
|
||||||
|
cache_read_input_token_cost: 0.00000045
|
||||||
router_settings:
|
router_settings:
|
||||||
num_retries: 2
|
num_retries: 2
|
||||||
timeout: 180
|
timeout: 180
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
{
|
||||||
|
"_comment": "EUR list prices per 1M tokens (providers quoting EUR). fx_sync.py converts at the ECB daily rate at boot. USD values in config.yaml are fallbacks.",
|
||||||
|
"models": {
|
||||||
|
"greenpt/deepseek-v4.1-flash": {"input": 0.36, "output": 1.45, "cache": 0.04},
|
||||||
|
"greenpt/glm-5.3-flash": {"input": 0.11, "output": 0.44, "cache": 0.022},
|
||||||
|
"greenpt/glm-5.3": {"input": 1.10, "output": 4.40, "cache": 0.275},
|
||||||
|
"greenpt/green-embeddings": {"input": 0.20, "output": 0.0},
|
||||||
|
"greenpt/qwen3-embedding-8b": {"input": 0.25, "output": 0.0},
|
||||||
|
"deepseek-flash-green": {"input": 0.36, "output": 1.45, "cache": 0.04},
|
||||||
|
"glm-flash-green": {"input": 0.11, "output": 0.44, "cache": 0.022},
|
||||||
|
"regolo/apertus-70b": {"input": 1.00, "output": 3.40},
|
||||||
|
"regolo/qwen3.5-122b": {"input": 0.60, "output": 3.40},
|
||||||
|
"regolo/mistral-small-4-119b": {"input": 0.20, "output": 0.70},
|
||||||
|
"regolo/gemma4-31b": {"input": 0.40, "output": 0.60}
|
||||||
|
}
|
||||||
|
}
|
||||||
Executable
+85
@@ -0,0 +1,85 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Boot-time FX sync: rewrite USD model_info from the EUR table at the ECB rate.
|
||||||
|
|
||||||
|
Fail-open by design: if the feed is unreachable or unparseable, the config is left
|
||||||
|
untouched (the repo's fallback USD values stand) and the boot continues.
|
||||||
|
|
||||||
|
Run by start.sh before config validation, so the validator sees the final prices.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import urllib.request
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
CFG = "/app/data/config.yaml"
|
||||||
|
EUR_FILE = "/app/code/eur_prices.json"
|
||||||
|
STATE = "/app/data/fx_state.json"
|
||||||
|
ECB_URL = os.environ.get(
|
||||||
|
"ECB_FX_URL", "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml")
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_rate() -> tuple[float, str]:
|
||||||
|
with urllib.request.urlopen(ECB_URL, timeout=20) as r:
|
||||||
|
body = r.read().decode("utf-8", "ignore")
|
||||||
|
m = re.search(r"currency=['\"]USD['\"]\s+rate=['\"]([0-9.]+)['\"]", body)
|
||||||
|
if not m:
|
||||||
|
raise ValueError("USD rate not found in ECB feed")
|
||||||
|
d = re.search(r"time=['\"](\d{4}-\d{2}-\d{2})['\"]", body)
|
||||||
|
return float(m.group(1)), (d.group(1) if d else "")
|
||||||
|
|
||||||
|
|
||||||
|
def usd(eur: float, rate: float) -> float:
|
||||||
|
return 0.0 if eur == 0 else eur * rate / 1_000_000
|
||||||
|
|
||||||
|
|
||||||
|
def set_cost(text: str, name: str, key: str, value: float) -> tuple[str, bool]:
|
||||||
|
pat = re.compile(r"( - model_name: " + re.escape(name) +
|
||||||
|
r"\n)(.*?)(?=\n - model_name: |\n[a-zA-Z#]|\Z)", re.S)
|
||||||
|
m = pat.search(text)
|
||||||
|
if not m:
|
||||||
|
return text, False
|
||||||
|
block = m.group(2)
|
||||||
|
new = f"{value:.10g}"
|
||||||
|
block2, n = re.subn(key + r": \S+", key + ": " + new, block, count=1)
|
||||||
|
if n == 0:
|
||||||
|
return text, False
|
||||||
|
return text[:m.start(2)] + block2 + text[m.end(2):], True
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
try:
|
||||||
|
rate, feed_date = fetch_rate()
|
||||||
|
eur = json.load(open(EUR_FILE, encoding="utf-8"))["models"]
|
||||||
|
text = open(CFG, encoding="utf-8").read()
|
||||||
|
changed = []
|
||||||
|
for name, prices in eur.items():
|
||||||
|
for key, eur_val in (("input_cost_per_token", prices.get("input")),
|
||||||
|
("output_cost_per_token", prices.get("output"))):
|
||||||
|
if eur_val is None:
|
||||||
|
continue
|
||||||
|
text, ok = set_cost(text, name, key, usd(eur_val, rate))
|
||||||
|
if ok:
|
||||||
|
changed.append(f"{name}:{key}")
|
||||||
|
if prices.get("cache") is not None:
|
||||||
|
text, ok = set_cost(text, name, "cache_read_input_token_cost",
|
||||||
|
usd(prices["cache"], rate))
|
||||||
|
if ok:
|
||||||
|
changed.append(f"{name}:cache")
|
||||||
|
tmp = CFG + ".fx.tmp"
|
||||||
|
open(tmp, "w", encoding="utf-8").write(text)
|
||||||
|
os.replace(tmp, CFG)
|
||||||
|
json.dump({"rate": rate, "feed_date": feed_date,
|
||||||
|
"checked_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||||
|
"fields": len(changed)},
|
||||||
|
open(STATE, "w", encoding="utf-8"), indent=2)
|
||||||
|
print(f"[fx-sync] ECB USD/EUR={rate} ({feed_date}); updated {len(changed)} fields")
|
||||||
|
return 0
|
||||||
|
except Exception as exc: # fail-open
|
||||||
|
print(f"[fx-sync] FAIL-OPEN: {type(exc).__name__}: {str(exc)[:160]}; keeping existing prices")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
"""Put Tinfoil's per-request audio prices in the SHARED cost map, at BUILD time.
|
||||||
|
|
||||||
|
Why this exists (issue #5): the cost calculator's per-request helper reads
|
||||||
|
``litellm.model_cost`` — the map built from model_prices_and_context_window.json —
|
||||||
|
and our config's ``input_cost_per_request`` never reaches it. Patching the router
|
||||||
|
to keep per-deployment pricing on the shared model_info entry was not enough,
|
||||||
|
because the helper consults the JSON-backed map, not model_info. Result: every
|
||||||
|
successful voxtral-tts speech call logged cost 0.0 (~$0.22/day of unlogged spend).
|
||||||
|
|
||||||
|
This is the same mechanism as patch_image_cost.py: re-state the price where the
|
||||||
|
calculator actually looks. It FAILS THE BUILD rather than warning, because a silent
|
||||||
|
"billed at $0" is exactly the failure mode we are trying to close.
|
||||||
|
|
||||||
|
Prices from the Tinfoil catalogue (2026-11-17): voxtral-tts $0.002/request,
|
||||||
|
whisper-large-v3-turbo $0.01/request, both $0 per token.
|
||||||
|
"""
|
||||||
|
import glob
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
ENTRIES = {
|
||||||
|
"openai/voxtral-tts": {
|
||||||
|
"mode": "audio_speech",
|
||||||
|
"input_cost_per_request": 0.002,
|
||||||
|
"input_cost_per_token": 0.0,
|
||||||
|
"output_cost_per_token": 0.0,
|
||||||
|
},
|
||||||
|
"openai/whisper-large-v3-turbo": {
|
||||||
|
"mode": "audio_transcription",
|
||||||
|
"input_cost_per_request": 0.01,
|
||||||
|
"input_cost_per_token": 0.0,
|
||||||
|
"output_cost_per_token": 0.0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
# LiteLLM ships the local map as *_backup.json; cost.json is the download
|
||||||
|
# cache. Patch whichever exists, preferring the loadable backup.
|
||||||
|
files = (glob.glob("/app/.venv/lib/python*/site-packages/litellm/"
|
||||||
|
"model_prices_and_context_window_backup.json")
|
||||||
|
or glob.glob("/app/.venv/lib/python*/site-packages/litellm/"
|
||||||
|
"model_prices_and_context_window.json")
|
||||||
|
or glob.glob("/app/.venv/lib/python*/site-packages/litellm/cost.json"))
|
||||||
|
if not files:
|
||||||
|
print("[coop-patch] FAIL no local cost map found (backup / model_prices / cost.json)")
|
||||||
|
return 1
|
||||||
|
path = files[0]
|
||||||
|
with open(path, encoding="utf-8") as handle:
|
||||||
|
data = json.load(handle)
|
||||||
|
changed = []
|
||||||
|
for key, entry in ENTRIES.items():
|
||||||
|
current = data.get(key) or {}
|
||||||
|
if all(current.get(k) == v for k, v in entry.items()):
|
||||||
|
continue
|
||||||
|
merged = {**current, **entry}
|
||||||
|
data[key] = merged
|
||||||
|
changed.append(key)
|
||||||
|
if not changed:
|
||||||
|
print(f"[coop-patch] audio cost map already correct ({len(ENTRIES)} entries)")
|
||||||
|
return 0
|
||||||
|
with open(path, "w", encoding="utf-8") as handle:
|
||||||
|
json.dump(data, handle, indent=2)
|
||||||
|
print("[coop-patch] applied audio cost map: " + ", ".join(changed))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
"""Default /v1/audio/speech to mp3 when the caller omits response_format.
|
||||||
|
|
||||||
|
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
||||||
|
if the target file or the anchor is missing — e.g. after a version bump — it
|
||||||
|
prints a warning and exits 0, so the build and the gateway still succeed. Look
|
||||||
|
for the ``[coop-patch]`` line in the build log.
|
||||||
|
|
||||||
|
Why: LibreChat's speech client sends no ``response_format`` on its TTS request
|
||||||
|
(TTSService.openAIProvider builds {input, model, voice} — the field is not in
|
||||||
|
its schema), and Tinfoil's voxtral-tts then serves WAV. Measured 2026-11-07
|
||||||
|
through the gateway: WAV ~8-17s warm per request (145-300 KB), while the SAME
|
||||||
|
model with ``response_format: "mp3"`` answers in ~2.6-3.1s (28-39 KB) — the
|
||||||
|
provider streams/encodes mp3 progressively. LibreChat additionally labels the
|
||||||
|
reply ``Content-Type: audio/mpeg`` unconditionally, so WAV bytes were served
|
||||||
|
under an mp3 label and browsers depended on MIME sniffing to play them.
|
||||||
|
|
||||||
|
Patch: default ``response_format`` to ``mp3`` (OpenAI's own documented default
|
||||||
|
for /v1/audio/speech) when the request omits it. An explicit response_format
|
||||||
|
from any caller still wins, so API members who want WAV can keep requesting
|
||||||
|
it. Scope: only the aspeech route; transcriptions are untouched.
|
||||||
|
|
||||||
|
Upstream: LiteLLM 1.103.2 forwards response_format verbatim and leaves the
|
||||||
|
provider default in place when it is absent.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import glob
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
MARKER = "CO-OP PATCH (TTS default mp3, 2026-11-07)"
|
||||||
|
|
||||||
|
ANCHOR = ''' ### CALL HOOKS ### - modify incoming data / reject request before calling the model
|
||||||
|
data = await proxy_logging_obj.pre_call_hook(
|
||||||
|
user_api_key_dict=user_api_key_dict, data=data, call_type="aspeech"
|
||||||
|
)'''
|
||||||
|
|
||||||
|
INSERT = ANCHOR + '''
|
||||||
|
|
||||||
|
# CO-OP PATCH (TTS default mp3, 2026-11-07): LibreChat sends no
|
||||||
|
# response_format and Tinfoil's voxtral-tts then serves WAV — measured
|
||||||
|
# ~8-17s warm vs ~3s for mp3, which the provider streams. LibreChat
|
||||||
|
# labels the reply audio/mpeg regardless. Default to mp3 (OpenAI's
|
||||||
|
# documented default); an explicit response_format still wins.
|
||||||
|
if not data.get("response_format"):
|
||||||
|
data["response_format"] = "mp3"'''
|
||||||
|
|
||||||
|
|
||||||
|
def target_path() -> str:
|
||||||
|
if len(sys.argv) > 1:
|
||||||
|
return sys.argv[1]
|
||||||
|
env = os.environ.get("LITELLM_PROXY_SERVER")
|
||||||
|
if env:
|
||||||
|
return env
|
||||||
|
hits = []
|
||||||
|
try: # most reliable: ask the installed package where it lives
|
||||||
|
import litellm # noqa: PLC0415
|
||||||
|
_file = getattr(litellm, "__file__", None)
|
||||||
|
if _file:
|
||||||
|
hits.append(os.path.join(os.path.dirname(_file), "proxy", "proxy_server.py"))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/proxy/proxy_server.py")
|
||||||
|
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/proxy/proxy_server.py")
|
||||||
|
hits = [h for h in hits if os.path.exists(h)]
|
||||||
|
if not hits:
|
||||||
|
raise SystemExit("[coop-patch] could not locate litellm/proxy/proxy_server.py")
|
||||||
|
return sorted(hits)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
path = target_path()
|
||||||
|
try:
|
||||||
|
source = open(path).read()
|
||||||
|
except OSError as exc:
|
||||||
|
print(f"[coop-patch] cannot read {path}: {exc}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if MARKER in source:
|
||||||
|
print(f"[coop-patch] {path} already patched, nothing to do")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
count = source.count(ANCHOR)
|
||||||
|
if count != 1:
|
||||||
|
print(f"[coop-patch] WARNING aspeech pre_call_hook anchor found {count}x (need exactly 1) "
|
||||||
|
f"in {path}; TTS stays on the provider's default (WAV for Tinfoil)")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
patched = source.replace(ANCHOR, INSERT, 1)
|
||||||
|
compile(patched, path, "exec") # refuse to write something that will not import
|
||||||
|
|
||||||
|
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
||||||
|
print(f"[coop-patch] {path} is not writable; verified anchor and syntax only")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with open(path, "w") as handle:
|
||||||
|
handle.write(patched)
|
||||||
|
print(f"[coop-patch] applied TTS default mp3 to {path}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
+127
-5
@@ -27,7 +27,13 @@ from the per-deployment ``custom_pricing`` the logging layer builds out of our
|
|||||||
call metadata.
|
call metadata.
|
||||||
|
|
||||||
Scope: nothing changes unless a model declares ``input_cost_per_request``, so
|
Scope: nothing changes unless a model declares ``input_cost_per_request``, so
|
||||||
this is inert for every other model on the gateway.
|
this is inert for every other model on the gateway. Since 2026-11-06 the charge
|
||||||
|
is added only on the AUDIO call types (transcription/atranscription/speech/
|
||||||
|
aspeech): a model that also carries per-token pricing (``voxtral-small-24b``)
|
||||||
|
bills its chat calls exactly per token, and a per-request charge on top of
|
||||||
|
that would double-bill them. The audio endpoints are the paths that cannot
|
||||||
|
price per token (the provider returns no token counts there), so the flat
|
||||||
|
charge belongs to them alone.
|
||||||
|
|
||||||
Upstream: no issue filed; LiteLLM has no per-request audio pricing as of 1.103.2.
|
Upstream: no issue filed; LiteLLM has no per-request audio pricing as of 1.103.2.
|
||||||
"""
|
"""
|
||||||
@@ -58,7 +64,9 @@ HELPER = '''def _coop_per_request_cost(
|
|||||||
"""
|
"""
|
||||||
def _is_audio(name: Any) -> bool:
|
def _is_audio(name: Any) -> bool:
|
||||||
text = str(name or "").lower()
|
text = str(name or "").lower()
|
||||||
return "tts" in text or "whisper" in text
|
return (
|
||||||
|
"tts" in text or "whisper" in text or "voxtral" in text
|
||||||
|
)
|
||||||
|
|
||||||
cost_map = {}
|
cost_map = {}
|
||||||
try:
|
try:
|
||||||
@@ -127,12 +135,98 @@ HELPER = '''def _coop_per_request_cost(
|
|||||||
|
|
||||||
ANCHOR_DEF = "def response_cost_calculator("
|
ANCHOR_DEF = "def response_cost_calculator("
|
||||||
|
|
||||||
|
ANCHOR_EARLY = """ if provider_response_cost is not None and not custom_pricing:
|
||||||
|
return provider_response_cost"""
|
||||||
|
|
||||||
ANCHOR_RETURN = """ region_name=region_name,
|
ANCHOR_RETURN = """ region_name=region_name,
|
||||||
)
|
)
|
||||||
return response_cost
|
return response_cost
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise e"""
|
raise e"""
|
||||||
|
|
||||||
|
# The unit-price call. On speech/transcription LiteLLM prices by character,
|
||||||
|
# second or token and raises ValueError when the model declares none of those
|
||||||
|
# rates — which is exactly a per-request model. The declaration is then the
|
||||||
|
# whole bill, so price the base at 0 instead of letting the raise swallow the
|
||||||
|
# spend log write (issue #5: rows recorded spend 0.0 and the tail patch below
|
||||||
|
# never ran because the exception aborted the function first).
|
||||||
|
ANCHOR_COST = """ response_cost = completion_cost(
|
||||||
|
completion_response=response_object,
|
||||||
|
model=model,
|
||||||
|
call_type=call_type,
|
||||||
|
custom_llm_provider=custom_llm_provider,
|
||||||
|
optional_params=optional_params,
|
||||||
|
custom_pricing=custom_pricing,
|
||||||
|
base_model=base_model,
|
||||||
|
prompt=prompt,
|
||||||
|
standard_built_in_tools_params=standard_built_in_tools_params,
|
||||||
|
litellm_model_name=litellm_model_name,
|
||||||
|
router_model_id=router_model_id,
|
||||||
|
litellm_logging_obj=litellm_logging_obj,
|
||||||
|
service_tier=service_tier,
|
||||||
|
data_residency=data_residency,
|
||||||
|
vertex_location=vertex_location,
|
||||||
|
region_name=region_name,
|
||||||
|
)"""
|
||||||
|
|
||||||
|
PATCHED_COST = """ try:
|
||||||
|
response_cost = completion_cost(
|
||||||
|
completion_response=response_object,
|
||||||
|
model=model,
|
||||||
|
call_type=call_type,
|
||||||
|
custom_llm_provider=custom_llm_provider,
|
||||||
|
optional_params=optional_params,
|
||||||
|
custom_pricing=custom_pricing,
|
||||||
|
base_model=base_model,
|
||||||
|
prompt=prompt,
|
||||||
|
standard_built_in_tools_params=standard_built_in_tools_params,
|
||||||
|
litellm_model_name=litellm_model_name,
|
||||||
|
router_model_id=router_model_id,
|
||||||
|
litellm_logging_obj=litellm_logging_obj,
|
||||||
|
service_tier=service_tier,
|
||||||
|
data_residency=data_residency,
|
||||||
|
vertex_location=vertex_location,
|
||||||
|
region_name=region_name,
|
||||||
|
)
|
||||||
|
except Exception as _coop_unit_exc:
|
||||||
|
# CO-OP PATCH (audio per-request cost, 2026-11-17): a model billed
|
||||||
|
# per request declares no per-character/second/token rate, so
|
||||||
|
# LiteLLM's unit pricing raises. The per-request charge is the
|
||||||
|
# whole bill; let the tail patch below add it to a zero base.
|
||||||
|
_coop_declared_pr = (
|
||||||
|
_coop_per_request_cost(
|
||||||
|
model=model,
|
||||||
|
litellm_model_name=litellm_model_name,
|
||||||
|
custom_pricing=custom_pricing,
|
||||||
|
litellm_logging_obj=litellm_logging_obj,
|
||||||
|
)
|
||||||
|
if call_type in ("transcription", "atranscription", "speech", "aspeech")
|
||||||
|
else 0.0
|
||||||
|
)
|
||||||
|
if _coop_declared_pr > 0:
|
||||||
|
print(
|
||||||
|
f"[coop-audio] unit pricing unavailable ({type(_coop_unit_exc).__name__}); "
|
||||||
|
f"pricing at declared per-request={_coop_declared_pr!r}",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
response_cost = 0.0
|
||||||
|
else:
|
||||||
|
raise"""
|
||||||
|
|
||||||
|
PATCHED_EARLY = """ if provider_response_cost is not None and not custom_pricing:
|
||||||
|
# CO-OP PATCH (2026-11-16): a provider-reported cost is not the
|
||||||
|
# whole bill on the audio paths. Speech returns here and never
|
||||||
|
# reaches the tail patch, so add the declared per-request charge
|
||||||
|
# before returning. Guarded by the DECLARATION, not the call
|
||||||
|
# type: models without input_cost_per_request (voxtral-small-24b,
|
||||||
|
# token-priced) stay inert.
|
||||||
|
return provider_response_cost + _coop_per_request_cost(
|
||||||
|
model=model,
|
||||||
|
litellm_model_name=litellm_model_name,
|
||||||
|
custom_pricing=custom_pricing,
|
||||||
|
litellm_logging_obj=litellm_logging_obj,
|
||||||
|
)"""
|
||||||
|
|
||||||
PATCHED_RETURN = """ region_name=region_name,
|
PATCHED_RETURN = """ region_name=region_name,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -140,17 +234,34 @@ PATCHED_RETURN = """ region_name=region_name,
|
|||||||
# per-request charge for models LiteLLM can only price per unit
|
# per-request charge for models LiteLLM can only price per unit
|
||||||
# (Tinfoil's voxtral-tts, whisper-large-v3-turbo). Inert when the model
|
# (Tinfoil's voxtral-tts, whisper-large-v3-turbo). Inert when the model
|
||||||
# declares nothing; the early "provider reported the cost" return above
|
# declares nothing; the early "provider reported the cost" return above
|
||||||
# is unaffected.
|
# is unaffected. Scoped to the AUDIO call types (2026-11-06): a model
|
||||||
|
# that ALSO carries per-token pricing (voxtral-small-24b) bills its
|
||||||
|
# chat calls exactly per token and must not take the per-request
|
||||||
|
# charge on top of that.
|
||||||
try:
|
try:
|
||||||
_coop_pr = _coop_per_request_cost(
|
_coop_audio_call = call_type in (
|
||||||
|
"transcription",
|
||||||
|
"atranscription",
|
||||||
|
"speech",
|
||||||
|
"aspeech",
|
||||||
|
)
|
||||||
|
_coop_pr = (
|
||||||
|
_coop_per_request_cost(
|
||||||
model=model,
|
model=model,
|
||||||
litellm_model_name=litellm_model_name,
|
litellm_model_name=litellm_model_name,
|
||||||
custom_pricing=custom_pricing,
|
custom_pricing=custom_pricing,
|
||||||
litellm_logging_obj=litellm_logging_obj,
|
litellm_logging_obj=litellm_logging_obj,
|
||||||
)
|
)
|
||||||
|
if _coop_audio_call
|
||||||
|
else 0.0
|
||||||
|
)
|
||||||
response_cost = (response_cost or 0.0) + _coop_pr
|
response_cost = (response_cost or 0.0) + _coop_pr
|
||||||
_coop_name = str(model or "") + str(litellm_model_name or "")
|
_coop_name = str(model or "") + str(litellm_model_name or "")
|
||||||
if "tts" in _coop_name.lower() or "whisper" in _coop_name.lower():
|
if _coop_audio_call and (
|
||||||
|
"tts" in _coop_name.lower()
|
||||||
|
or "whisper" in _coop_name.lower()
|
||||||
|
or "voxtral" in _coop_name.lower()
|
||||||
|
):
|
||||||
print(
|
print(
|
||||||
f"[coop-audio] add per_request={_coop_pr!r} -> response_cost={response_cost!r}",
|
f"[coop-audio] add per_request={_coop_pr!r} -> response_cost={response_cost!r}",
|
||||||
flush=True,
|
flush=True,
|
||||||
@@ -207,6 +318,17 @@ def main() -> int:
|
|||||||
|
|
||||||
patched = source.replace(ANCHOR_DEF, HELPER + ANCHOR_DEF, 1)
|
patched = source.replace(ANCHOR_DEF, HELPER + ANCHOR_DEF, 1)
|
||||||
patched = patched.replace(ANCHOR_RETURN, PATCHED_RETURN, 1)
|
patched = patched.replace(ANCHOR_RETURN, PATCHED_RETURN, 1)
|
||||||
|
if ANCHOR_COST in patched:
|
||||||
|
patched = patched.replace(ANCHOR_COST, PATCHED_COST, 1)
|
||||||
|
print("[coop-patch] unit-price call made per-request tolerant (speech/transcription)")
|
||||||
|
else:
|
||||||
|
print("[coop-patch] WARNING unit-price anchor not found; per-request models "
|
||||||
|
"will still raise and bill $0")
|
||||||
|
if ANCHOR_EARLY in patched:
|
||||||
|
patched = patched.replace(ANCHOR_EARLY, PATCHED_EARLY, 1)
|
||||||
|
print("[coop-patch] early provider-cost return also patched (audio speech)")
|
||||||
|
else:
|
||||||
|
print("[coop-patch] WARNING early-return anchor not found; speech may stay unbilled")
|
||||||
compile(patched, path, "exec") # refuse to write something that will not import
|
compile(patched, path, "exec") # refuse to write something that will not import
|
||||||
|
|
||||||
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
||||||
|
|||||||
@@ -4,7 +4,9 @@ MEMBER REQUEST (co-op/support#39, 2026-10-31). Browser JS calling the gateway fr
|
|||||||
another origin cannot read a response header unless the server names it in
|
another origin cannot read a response header unless the server names it in
|
||||||
Access-Control-Expose-Headers. LiteLLM sets
|
Access-Control-Expose-Headers. LiteLLM sets
|
||||||
|
|
||||||
expose_headers=LITELLM_UI_ALLOW_HEADERS
|
expose_headers=LITELLM_UI_ALLOW_HEADERS # through 1.103.x
|
||||||
|
expose_headers=[*LITELLM_UI_ALLOW_HEADERS,
|
||||||
|
*trace_response_expose_headers()] # from 1.108.0
|
||||||
|
|
||||||
which covers seven names (semantic-filter, guardrail, cache-key, ...) and none of
|
which covers seven names (semantic-filter, guardrail, cache-key, ...) and none of
|
||||||
the cost ones — so `x-litellm-response-cost*` reads as null in JS even though the
|
the cost ones — so `x-litellm-response-cost*` reads as null in JS even though the
|
||||||
@@ -62,6 +64,20 @@ NEW = (
|
|||||||
+ " ],\n"
|
+ " ],\n"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# 1.108.0 refactored the middleware call (upstream added the trace-response
|
||||||
|
# header set to expose_headers). Same operation, new anchor; tried when the
|
||||||
|
# 1.103.2 shape is absent so the script works against either base image.
|
||||||
|
OLD_V1108 = (
|
||||||
|
" expose_headers=[*LITELLM_UI_ALLOW_HEADERS, *trace_response_expose_headers()],\n"
|
||||||
|
)
|
||||||
|
NEW_V1108 = (
|
||||||
|
" expose_headers=[\n"
|
||||||
|
" *LITELLM_UI_ALLOW_HEADERS,\n"
|
||||||
|
" *trace_response_expose_headers(),\n"
|
||||||
|
+ "".join(f' "{h}",\n' for h in EXPOSED_HEADERS)
|
||||||
|
+ " ],\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
# Glob rather than pin a Python version: the venv path moves when the base
|
# Glob rather than pin a Python version: the venv path moves when the base
|
||||||
@@ -76,13 +92,22 @@ def main() -> int:
|
|||||||
if "x-litellm-response-cost-input" in src:
|
if "x-litellm-response-cost-input" in src:
|
||||||
print("expose-cost-headers: already applied")
|
print("expose-cost-headers: already applied")
|
||||||
return 0
|
return 0
|
||||||
found = src.count(OLD)
|
for old, new, shape in ((OLD, NEW, "1.103.x"), (OLD_V1108, NEW_V1108, "1.108.x")):
|
||||||
if found != 1:
|
found = src.count(old)
|
||||||
print(f"WARN expose-cost-headers: expected exactly 1 CORS middleware call, found {found} — skipping")
|
if found == 1:
|
||||||
|
target.write_text(src.replace(old, new))
|
||||||
|
print(
|
||||||
|
f"expose-cost-headers: appended {len(EXPOSED_HEADERS)} cost header names "
|
||||||
|
f"to expose_headers (anchor {shape}) in {target}"
|
||||||
|
)
|
||||||
return 0
|
return 0
|
||||||
|
if found > 1:
|
||||||
target.write_text(src.replace(OLD, NEW))
|
print(f"WARN expose-cost-headers: expected exactly 1 {shape} CORS middleware call, found {found} — skipping")
|
||||||
print(f"expose-cost-headers: appended {len(EXPOSED_HEADERS)} cost header names to expose_headers in {target}")
|
return 0
|
||||||
|
print(
|
||||||
|
"WARN expose-cost-headers: no known expose_headers anchor found "
|
||||||
|
"(neither 1.103.x nor 1.108.x shape) — skipping"
|
||||||
|
)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
"""Skip LiteLLM's forced `verbose_json` on transcriptions for opting-out models.
|
||||||
|
|
||||||
|
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
||||||
|
if the target file or the anchor is missing — e.g. after a version bump — it
|
||||||
|
prints a warning and exits 0, so the build and the gateway still succeed. Look
|
||||||
|
for the ``[coop-patch]`` line in the build log.
|
||||||
|
|
||||||
|
Why: for every `openai/` transcription deployment, LiteLLM's transformation
|
||||||
|
injects ``response_format: verbose_json`` when the caller omits the field
|
||||||
|
("ensures 'duration' is received - used for cost calculation"). Tinfoil's
|
||||||
|
``voxtral-small-24b`` rejects verbose_json with a 400 ("Currently do not
|
||||||
|
support verbose_json for voxtral-small-24b"), so every bare
|
||||||
|
``/v1/audio/transcriptions`` call to it failed — including the shape members
|
||||||
|
actually use (file + model, no response_format), and Open WebUI dictation's
|
||||||
|
request shape. The injection exists to obtain a duration for per-SECOND
|
||||||
|
pricing; voxtral-small-24b is billed per token (chat route) / per request
|
||||||
|
(patch_audio_request_cost.py), so it does not need the duration at all.
|
||||||
|
|
||||||
|
Patch: honour a ``coop_no_verbose_json`` flag on the deployment's model_info
|
||||||
|
when it is reachable, and gate the model name explicitly — the flag's plumbing
|
||||||
|
into the transformation's litellm_params is not guaranteed on every path, the
|
||||||
|
name gate is. whisper-large-v3-turbo (which Tinfoil serves verbose_json for,
|
||||||
|
and whose dictation path depends on the injected format) is untouched.
|
||||||
|
|
||||||
|
Scope: only transcriptions that omit response_format AND match the opt-out;
|
||||||
|
every other request keeps upstream behavior exactly.
|
||||||
|
|
||||||
|
Upstream: LiteLLM has no opt-out for this injection as of 1.103.2.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import glob
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
MARKER = "CO-OP PATCH (skip verbose_json, 2026-11-06)"
|
||||||
|
|
||||||
|
ANCHOR = ''' if "response_format" not in data:
|
||||||
|
data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation'''
|
||||||
|
|
||||||
|
INSERT = ''' if "response_format" not in data:
|
||||||
|
# CO-OP PATCH (skip verbose_json, 2026-11-06): a deployment whose
|
||||||
|
# model_info declares coop_no_verbose_json opts out of the forced
|
||||||
|
# verbose_json injection — Tinfoil's voxtral-small-24b rejects
|
||||||
|
# verbose_json with a 400, and its per-token / per-request billing
|
||||||
|
# does not need the duration this injection exists to obtain. The
|
||||||
|
# model-name gate backs the flag up: the flag's plumbing into this
|
||||||
|
# method's litellm_params is not guaranteed on every call path.
|
||||||
|
_mi = (litellm_params or {}).get("model_info") or {}
|
||||||
|
_coop_skip = (isinstance(_mi, dict) and bool(_mi.get("coop_no_verbose_json"))) or (
|
||||||
|
"voxtral-small" in str(model or "")
|
||||||
|
)
|
||||||
|
if not _coop_skip:
|
||||||
|
data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation'''
|
||||||
|
|
||||||
|
|
||||||
|
def target_path() -> str:
|
||||||
|
if len(sys.argv) > 1:
|
||||||
|
return sys.argv[1]
|
||||||
|
env = os.environ.get("LITELLM_WHISPER_TRANSFORMATION")
|
||||||
|
if env:
|
||||||
|
return env
|
||||||
|
hits = []
|
||||||
|
try: # most reliable: ask the installed package where it lives
|
||||||
|
import litellm # noqa: PLC0415
|
||||||
|
_file = getattr(litellm, "__file__", None)
|
||||||
|
if _file:
|
||||||
|
hits.append(os.path.join(
|
||||||
|
os.path.dirname(_file), "llms", "openai", "transcriptions", "whisper_transformation.py"))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py")
|
||||||
|
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py")
|
||||||
|
hits = [h for h in hits if os.path.exists(h)]
|
||||||
|
if not hits:
|
||||||
|
raise SystemExit("[coop-patch] could not locate llms/openai/transcriptions/whisper_transformation.py")
|
||||||
|
return sorted(hits)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
path = target_path()
|
||||||
|
try:
|
||||||
|
source = open(path).read()
|
||||||
|
except OSError as exc:
|
||||||
|
print(f"[coop-patch] cannot read {path}: {exc}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if MARKER in source:
|
||||||
|
print(f"[coop-patch] {path} already patched, nothing to do")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if ANCHOR not in source:
|
||||||
|
print(f"[coop-patch] WARNING anchor not found in {path}; bare transcription calls to "
|
||||||
|
f"opt-out models (voxtral-small-24b) will fail with the provider's verbose_json 400")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
patched = source.replace(ANCHOR, INSERT, 1)
|
||||||
|
compile(patched, path, "exec") # refuse to write something that will not import
|
||||||
|
|
||||||
|
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
||||||
|
print(f"[coop-patch] {path} is not writable; verified anchor and syntax only")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with open(path, "w") as handle:
|
||||||
|
handle.write(patched)
|
||||||
|
print(f"[coop-patch] applied skip-verbose_json to {path}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
Executable
+48
@@ -0,0 +1,48 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Fail if a EUR-quoted provider's model_info is not EUR * FX.
|
||||||
|
|
||||||
|
GreenPT and Regolo quote EUR; LiteLLM costs are USD. Pasting EUR numerals into
|
||||||
|
model_info under-bills members by the FX gap (found 2026-11-18).
|
||||||
|
"""
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
FX = 1.16
|
||||||
|
EUR = {
|
||||||
|
"greenpt/deepseek-v4.1-flash": (0.36, 1.45, 0.04),
|
||||||
|
"greenpt/glm-5.3-flash": (0.11, 0.44, 0.022),
|
||||||
|
"greenpt/glm-5.3": (1.10, 4.40, 0.275),
|
||||||
|
"greenpt/green-embeddings": (0.20, 0.0, None),
|
||||||
|
"greenpt/qwen3-embedding-8b": (0.25, 0.0, None),
|
||||||
|
"regolo/mistral-small-4-119b": (0.20, 0.70, None),
|
||||||
|
"regolo/gemma4-31b": (0.40, 0.60, None),
|
||||||
|
# Route aliases carry their own model_info; members call these names, so a
|
||||||
|
# deployment-only conversion leaves billing stale (found 2026-11-18).
|
||||||
|
"deepseek-flash-green": (0.36, 1.45, 0.04),
|
||||||
|
"glm-flash-green": (0.11, 0.44, 0.022),
|
||||||
|
}
|
||||||
|
|
||||||
|
text = open("config.yaml", encoding="utf-8").read()
|
||||||
|
bad = []
|
||||||
|
for name, (i, o, c) in EUR.items():
|
||||||
|
m = re.search(r"- model_name: " + re.escape(name) + r"\n(.*?)(?=\n - model_name: |\n[a-zA-Z#])", text, re.S)
|
||||||
|
if not m:
|
||||||
|
bad.append(f"{name}: missing block")
|
||||||
|
continue
|
||||||
|
block = m.group(1)
|
||||||
|
def val(key):
|
||||||
|
mm = re.search(key + r": (\S+)", block)
|
||||||
|
return float(mm.group(1)) if mm else None
|
||||||
|
checks = [("input_cost_per_token", i), ("output_cost_per_token", o)]
|
||||||
|
if c is not None:
|
||||||
|
checks.append(("cache_read_input_token_cost", c))
|
||||||
|
for key, eur in checks:
|
||||||
|
want = eur * FX / 1_000_000
|
||||||
|
got = val(key)
|
||||||
|
if got is None or abs(got - want) > max(1e-12, want * 1e-6):
|
||||||
|
bad.append(f"{name}: {key} got {got}, want {want:.10g}")
|
||||||
|
print("checked", len(EUR), "EUR models at FX", FX)
|
||||||
|
if bad:
|
||||||
|
print("\n".join(bad))
|
||||||
|
sys.exit(1)
|
||||||
|
print("OK: EUR list prices converted to USD")
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
"""Register Tinfoil's per-request audio prices in litellm.model_cost at STARTUP.
|
||||||
|
|
||||||
|
Build-time edits of model_prices_and_context_window_backup.json did not reach the
|
||||||
|
runtime map (the deployed gateway still had no entry — issue #5). sitecustomize is
|
||||||
|
imported by CPython before any application import, so this runs inside the gateway
|
||||||
|
process itself, before LiteLLM prices a request. Idempotent and never fatal.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
|
||||||
|
ENTRIES = {
|
||||||
|
"openai/voxtral-tts": {
|
||||||
|
"mode": "audio_speech",
|
||||||
|
"input_cost_per_request": 0.002,
|
||||||
|
"input_cost_per_token": 0.0,
|
||||||
|
"output_cost_per_token": 0.0,
|
||||||
|
},
|
||||||
|
"openai/whisper-large-v3-turbo": {
|
||||||
|
"mode": "audio_transcription",
|
||||||
|
"input_cost_per_request": 0.01,
|
||||||
|
"input_cost_per_token": 0.0,
|
||||||
|
"output_cost_per_token": 0.0,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
import litellm
|
||||||
|
|
||||||
|
_cost = getattr(litellm, "model_cost", None)
|
||||||
|
if isinstance(_cost, dict):
|
||||||
|
for _k, _v in ENTRIES.items():
|
||||||
|
_cur = _cost.get(_k) or {}
|
||||||
|
if _cur.get("input_cost_per_request") != _v["input_cost_per_request"]:
|
||||||
|
_cost[_k] = {**_cur, **_v}
|
||||||
|
except Exception: # never break the interpreter for a pricing nicety
|
||||||
|
pass
|
||||||
@@ -6,6 +6,18 @@ set -eu
|
|||||||
# Configures LiteLLM with Cloudron's PostgreSQL and Redis addons
|
# Configures LiteLLM with Cloudron's PostgreSQL and Redis addons
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
|
||||||
|
# --- Cloudron scheduler tasks arrive as ARGUMENTS to this script ---
|
||||||
|
# The manifest's scheduler commands run as `docker run <image> <command>`, and
|
||||||
|
# this script is the image ENTRYPOINT — so the task command lands in "$@".
|
||||||
|
# Without this branch every scheduled task booted a second tinfoil-proxy and
|
||||||
|
# LiteLLM (whose ports are already bound in the app's network namespace) and
|
||||||
|
# died on the bind FATAL before doing its job — which looked exactly like a
|
||||||
|
# scheduler that never fired. Exec the task directly instead of booting.
|
||||||
|
if [ "$#" -gt 0 ]; then
|
||||||
|
echo "start.sh: scheduler-task mode — exec: $*"
|
||||||
|
exec "$@"
|
||||||
|
fi
|
||||||
|
|
||||||
echo "=== LiteLLM Cloudron Startup ==="
|
echo "=== LiteLLM Cloudron Startup ==="
|
||||||
|
|
||||||
# --- Cloudron PostgreSQL addon provides:
|
# --- Cloudron PostgreSQL addon provides:
|
||||||
@@ -178,6 +190,27 @@ fi
|
|||||||
# cannot happen through this path, and the shipped /app/code/config.yaml is
|
# cannot happen through this path, and the shipped /app/code/config.yaml is
|
||||||
# the live catalog.
|
# the live catalog.
|
||||||
|
|
||||||
|
# --- Daily FX sync (fail-open): convert EUR-priced providers at the ECB rate ---
|
||||||
|
# Prices are refreshed at boot; a nightly timer restarts the gateway at 04:30 UTC.
|
||||||
|
if [ -f /app/code/fx_sync.py ]; then
|
||||||
|
echo "Refreshing EUR->USD prices from the ECB feed (fail-open)..."
|
||||||
|
python3 /app/code/fx_sync.py || echo "fx_sync exited non-zero; continuing with existing prices"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# --- Fail-closed config validation (2026-11-11) ---
|
||||||
|
# Validate the config that will actually be SERVED, before starting anything.
|
||||||
|
# A bad key must fail LOUD at the top of the boot log (the chat outage of
|
||||||
|
# 2026-10-31 showed what silent misconfiguration costs). The -f guard keeps a
|
||||||
|
# missing validator non-fatal (build patch did not apply); an INVALID config
|
||||||
|
# is fatal by design.
|
||||||
|
if [ -f /app/code/validate_config.py ]; then
|
||||||
|
echo "Validating gateway config (fail-closed)..."
|
||||||
|
if ! python3 /app/code/validate_config.py litellm "$CONFIG_FILE"; then
|
||||||
|
echo "FATAL: gateway config.yaml is invalid - refusing to start. Fix or remove the offending key named above." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# --- Start LiteLLM ---
|
# --- Start LiteLLM ---
|
||||||
echo "Starting LiteLLM on port 4000..."
|
echo "Starting LiteLLM on port 4000..."
|
||||||
|
|
||||||
@@ -227,4 +260,21 @@ if [ "$proxy_up" != "1" ]; then
|
|||||||
fi
|
fi
|
||||||
echo "Tinfoil proxy is up on 127.0.0.1:3301"
|
echo "Tinfoil proxy is up on 127.0.0.1:3301"
|
||||||
|
|
||||||
|
# --- In-container audio keep-alive loop (the ONLY warm path) ---
|
||||||
|
# A separate scheduler-task container was removed 2026-11-09: its container
|
||||||
|
# could not be proven to reach 127.0.0.1:4000 and /app/data, and two warm
|
||||||
|
# paths billed twice for the same ping. The warm path is owned HERE — the
|
||||||
|
# app container, where the network namespace, the master key and the
|
||||||
|
# sidecar are all proven. Every 15 minutes ping tinfoil/voxtral-tts
|
||||||
|
# ($0.002/request) so voice mode skips the ~24s Tinfoil cold-start after
|
||||||
|
# idle. audio_keepalive.py is silent on success and prints on failure, so
|
||||||
|
# /app/data/keepalive.log only ever grows on failure.
|
||||||
|
(
|
||||||
|
while true; do
|
||||||
|
python3 /app/code/audio_keepalive.py >> /app/data/keepalive.log 2>&1 || true
|
||||||
|
sleep 900
|
||||||
|
done
|
||||||
|
) &
|
||||||
|
echo "Audio keep-alive loop started (every 900s)"
|
||||||
|
|
||||||
exec litellm --config "$CONFIG_FILE" --port 4000 --host 0.0.0.0
|
exec litellm --config "$CONFIG_FILE" --port 4000 --host 0.0.0.0
|
||||||
@@ -0,0 +1,261 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Fail-closed config validator for the co-op's gateway and chat apps.
|
||||||
|
|
||||||
|
ONE script, two modes, shared home in ops-docs/scripts/ and shipped
|
||||||
|
byte-identical into both app images (litellm repo root, librechat-custom repo
|
||||||
|
root). Each app's start path calls it before booting the service; a non-zero
|
||||||
|
exit refuses the boot with a ONE-line error naming the offending key.
|
||||||
|
|
||||||
|
Why this exists: the 2026-10-31 chat outage. A `customFooter:` top-level key in
|
||||||
|
librechat.yaml was rejected by LibreChat's strict config validator and the API
|
||||||
|
server refused to start, while the SPA kept being served with HTTP 200 — the
|
||||||
|
app looked up but every API route returned the HTML shell. Nothing had ever
|
||||||
|
checked the config before boot. This validator closes that gap: invalid config
|
||||||
|
fails LOUD and EARLY, at the top of the boot log, naming the key.
|
||||||
|
|
||||||
|
Modes
|
||||||
|
-----
|
||||||
|
litellm Structural check of the gateway's config.yaml:
|
||||||
|
- top-level keys within LiteLLM's documented config surface;
|
||||||
|
- model_list present and well-formed (model_name + litellm_params.model);
|
||||||
|
- the settings blocks are objects; pass_through_endpoints entries have paths.
|
||||||
|
When the gateway's own litellm package is importable it is additionally
|
||||||
|
used to resolve every model string to a provider (catches unknown
|
||||||
|
provider prefixes); the check is skipped with a note where litellm
|
||||||
|
itself is absent.
|
||||||
|
|
||||||
|
librechat Checks librechat.yaml against the DEPLOYED build's own strict schema:
|
||||||
|
it runs `configSchema.strict().safeParse()` from the container's
|
||||||
|
librechat-data-provider via node — the same code the API server runs
|
||||||
|
at boot (its failure mode is `process.exit(1)`, "Exiting due to
|
||||||
|
invalid configuration"). Unrecognized top-level keys are rejected,
|
||||||
|
named explicitly. Where node/the package cannot be found (e.g. this
|
||||||
|
script run from a checkout instead of the image) it falls back to a
|
||||||
|
static top-level allowlist recorded from the deployed build and says
|
||||||
|
so on the failure line.
|
||||||
|
|
||||||
|
Usage
|
||||||
|
-----
|
||||||
|
python3 validate_config.py litellm /app/data/config.yaml
|
||||||
|
python3 validate_config.py librechat /app/data/librechat.yaml
|
||||||
|
|
||||||
|
Exit codes: 0 valid, 1 invalid, 2 usage/environment error (file unreadable,
|
||||||
|
yaml unparseable). Never prints a value from the config — only key names and
|
||||||
|
counts, so it is safe to run against files that carry secrets.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
# LiteLLM's documented top-level config surface (proxy config). Unknown keys
|
||||||
|
# are rejected here — the gateway should never silently ignore a mistyped
|
||||||
|
# section (a "model-list:" typo would otherwise deploy an EMPTY catalog).
|
||||||
|
LITELLM_TOP_LEVEL = {
|
||||||
|
"model_list",
|
||||||
|
"router_settings",
|
||||||
|
"litellm_settings",
|
||||||
|
"general_settings",
|
||||||
|
"settings",
|
||||||
|
"environment_variables",
|
||||||
|
"guardrails",
|
||||||
|
"files",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Fallback allowlist for librechat mode, recorded 2026-11-11 from the DEPLOYED
|
||||||
|
# build (chat.inference.coop, overlay 1.4.8) by reading
|
||||||
|
# configSchema.strict().shape keys. The live schema is always preferred; this
|
||||||
|
# exists so the validator still fails closed when run outside the container.
|
||||||
|
LIBRECHAT_TOP_LEVEL_FALLBACK = {
|
||||||
|
"actions", "balance", "cache", "cloudfront", "endpoints", "fileConfig",
|
||||||
|
"fileStrategies", "fileStrategy", "filteredTools", "filters",
|
||||||
|
"imageOutputType", "includedTools", "interface", "langfuse", "mcpServers",
|
||||||
|
"mcpSettings", "memory", "messageFilter", "modelSpecs", "ocr", "openapi",
|
||||||
|
"permissions", "rateLimits", "registration", "secureImageLinks",
|
||||||
|
"skillSync", "speech", "summarization", "transactions", "turnstile",
|
||||||
|
"version", "webSearch",
|
||||||
|
}
|
||||||
|
|
||||||
|
LC_DATA_PROVIDER_DIST = os.environ.get(
|
||||||
|
"LC_DATA_PROVIDER_DIST",
|
||||||
|
"/app/code/librechat/packages/data-provider/dist/index.js")
|
||||||
|
LC_JS_YAML = os.environ.get(
|
||||||
|
"LC_JS_YAML", "/app/code/librechat/node_modules/js-yaml")
|
||||||
|
|
||||||
|
|
||||||
|
def fail(key, reason):
|
||||||
|
"""ONE line, naming the offending key, then a non-zero exit."""
|
||||||
|
print(f"CONFIG VALIDATION FAILED: {key}: {reason}", flush=True)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def load_yaml(path):
|
||||||
|
try:
|
||||||
|
import yaml
|
||||||
|
except ImportError:
|
||||||
|
print("CONFIG VALIDATION FAILED: pyyaml not available in this python",
|
||||||
|
flush=True)
|
||||||
|
sys.exit(2)
|
||||||
|
try:
|
||||||
|
with open(path) as fh:
|
||||||
|
data = yaml.safe_load(fh)
|
||||||
|
except FileNotFoundError:
|
||||||
|
print(f"CONFIG VALIDATION FAILED: config file not found: {path}",
|
||||||
|
flush=True)
|
||||||
|
sys.exit(2)
|
||||||
|
except PermissionError:
|
||||||
|
print(f"CONFIG VALIDATION FAILED: config file unreadable: {path}",
|
||||||
|
flush=True)
|
||||||
|
sys.exit(2)
|
||||||
|
except yaml.YAMLError as exc:
|
||||||
|
# Do not echo the raw yaml error: it can quote a secret-bearing line.
|
||||||
|
print("CONFIG VALIDATION FAILED: (root): yaml does not parse",
|
||||||
|
flush=True)
|
||||||
|
sys.exit(2)
|
||||||
|
if data is None:
|
||||||
|
data = {}
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
fail("(root)", "top level must be a mapping")
|
||||||
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def validate_litellm(path):
|
||||||
|
cfg = load_yaml(path)
|
||||||
|
unknown = sorted(set(cfg) - LITELLM_TOP_LEVEL)
|
||||||
|
if unknown:
|
||||||
|
fail(", ".join(unknown), "unknown top-level key(s) for litellm "
|
||||||
|
f"(allowed: {', '.join(sorted(LITELLM_TOP_LEVEL))})")
|
||||||
|
|
||||||
|
model_list = cfg.get("model_list")
|
||||||
|
if not isinstance(model_list, list) or not model_list:
|
||||||
|
fail("model_list", "must be a non-empty list of model entries")
|
||||||
|
names = set()
|
||||||
|
for i, entry in enumerate(model_list):
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
fail(f"model_list[{i}]", "entry must be a mapping")
|
||||||
|
name = entry.get("model_name")
|
||||||
|
params = entry.get("litellm_params")
|
||||||
|
if not name or not isinstance(name, str):
|
||||||
|
fail(f"model_list[{i}].model_name", "missing or empty")
|
||||||
|
if name in names:
|
||||||
|
fail(f"model_list[{i}].model_name", f"duplicate model_name {name!r}")
|
||||||
|
names.add(name)
|
||||||
|
if not isinstance(params, dict) or not params.get("model"):
|
||||||
|
fail(f"model_list[{i}].litellm_params.model", "missing or empty")
|
||||||
|
for block in ("router_settings", "litellm_settings", "general_settings"):
|
||||||
|
if block in cfg and not isinstance(cfg[block], dict):
|
||||||
|
fail(block, "must be a mapping")
|
||||||
|
for ep in (cfg.get("general_settings") or {}).get("pass_through_endpoints") or []:
|
||||||
|
if not isinstance(ep, dict) or not ep.get("path"):
|
||||||
|
fail("general_settings.pass_through_endpoints", "each entry needs a path")
|
||||||
|
|
||||||
|
# litellm-assisted check (best effort): resolve each model string to a
|
||||||
|
# provider with the gateway's own package. Skipped where litellm itself
|
||||||
|
# is not importable (never fatal — the structural checks above ran).
|
||||||
|
try:
|
||||||
|
from litellm import get_llm_provider # noqa: WPS433
|
||||||
|
except Exception:
|
||||||
|
print("note: litellm not importable here; provider check skipped",
|
||||||
|
flush=True)
|
||||||
|
else:
|
||||||
|
for entry in model_list:
|
||||||
|
model = entry["litellm_params"]["model"]
|
||||||
|
try:
|
||||||
|
get_llm_provider(model=model, custom_llm_provider=None)
|
||||||
|
except Exception as exc:
|
||||||
|
short = str(exc).split("\n")[0][:120]
|
||||||
|
fail(f"model_list.{entry['model_name']}.litellm_params.model",
|
||||||
|
f"not resolvable to a provider: {short}")
|
||||||
|
|
||||||
|
print(f"config OK (litellm): {len(cfg)} top-level key(s), "
|
||||||
|
f"{len(model_list)} model(s) validated", flush=True)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_librechat(path):
|
||||||
|
cfg = load_yaml(path)
|
||||||
|
if not cfg:
|
||||||
|
print("config OK (librechat): empty config, nothing to validate",
|
||||||
|
flush=True)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
# Preferred: the DEPLOYED build's own strict schema, run through node —
|
||||||
|
# the exact code path the API server uses at boot (strict() rejects
|
||||||
|
# unrecognized top-level keys; upstream then exits 1).
|
||||||
|
node_code = r"""
|
||||||
|
const fs = require('fs');
|
||||||
|
let out = {mode: 'schema'};
|
||||||
|
try {
|
||||||
|
const yaml = require(JS_YAML_PATH);
|
||||||
|
const dp = require(DIST_PATH);
|
||||||
|
const cfg = yaml.load(fs.readFileSync(CONFIG_PATH, 'utf8')) || {};
|
||||||
|
const result = dp.configSchema.strict().safeParse(cfg);
|
||||||
|
if (result.success) {
|
||||||
|
out.ok = true;
|
||||||
|
out.keys = Object.keys(cfg).length;
|
||||||
|
} else {
|
||||||
|
out.ok = false;
|
||||||
|
out.errors = (result.error && result.error.issues || []).slice(0, 10)
|
||||||
|
.map(i => ({path: (i.path || []).join('.') || '(root)', msg: i.message}));
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
out.mode = 'schema-unavailable';
|
||||||
|
out.error = String(e && e.message || e).slice(0, 200);
|
||||||
|
}
|
||||||
|
console.log('@@' + JSON.stringify(out));
|
||||||
|
""".replace("JS_YAML_PATH", json.dumps(LC_JS_YAML)) \
|
||||||
|
.replace("DIST_PATH", json.dumps(LC_DATA_PROVIDER_DIST)) \
|
||||||
|
.replace("CONFIG_PATH", json.dumps(path))
|
||||||
|
|
||||||
|
if os.path.exists(LC_DATA_PROVIDER_DIST):
|
||||||
|
try:
|
||||||
|
proc = subprocess.run(["node", "-e", node_code], capture_output=True,
|
||||||
|
text=True, timeout=60)
|
||||||
|
for line in (proc.stdout or "").splitlines():
|
||||||
|
if line.startswith("@@"):
|
||||||
|
res = json.loads(line[2:])
|
||||||
|
if res.get("ok"):
|
||||||
|
print(f"config OK (librechat): {res['keys']} top-level "
|
||||||
|
"key(s) accepted by the deployed strict schema",
|
||||||
|
flush=True)
|
||||||
|
sys.exit(0)
|
||||||
|
errs = "; ".join(
|
||||||
|
f"{e['path']}: {e['msg']}" for e in res.get("errors", []))
|
||||||
|
if errs:
|
||||||
|
fail(errs, "rejected by the deployed build's strict "
|
||||||
|
"config schema (configSchema.strict())")
|
||||||
|
fail("(root)", "strict schema rejected the config")
|
||||||
|
# No @@ line: schema path unavailable — fall through to fallback.
|
||||||
|
print("note: deployed schema probe produced no result "
|
||||||
|
f"(stderr: {(proc.stderr or '')[:120]!r})", flush=True)
|
||||||
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||||
|
print(f"note: could not run node against the deployed schema "
|
||||||
|
f"({type(exc).__name__})", flush=True)
|
||||||
|
else:
|
||||||
|
print("note: deployed data-provider dist not found; using the static "
|
||||||
|
"fallback allowlist recorded from the deployed build", flush=True)
|
||||||
|
|
||||||
|
# Fallback: static allowlist (still fail-closed).
|
||||||
|
unknown = sorted(set(cfg) - LIBRECHAT_TOP_LEVEL_FALLBACK)
|
||||||
|
if unknown:
|
||||||
|
fail(", ".join(unknown), "unknown top-level key(s) for librechat "
|
||||||
|
f"(fallback allowlist: {', '.join(sorted(LIBRECHAT_TOP_LEVEL_FALLBACK))})")
|
||||||
|
print(f"config OK (librechat, fallback allowlist): {len(cfg)} top-level "
|
||||||
|
"key(s)", flush=True)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = sys.argv[1:]
|
||||||
|
if len(args) != 2 or args[0] not in ("litellm", "librechat"):
|
||||||
|
print("usage: validate_config.py <litellm|librechat> <config.yaml>",
|
||||||
|
file=sys.stderr, flush=True)
|
||||||
|
sys.exit(2)
|
||||||
|
if args[0] == "litellm":
|
||||||
|
validate_litellm(args[1])
|
||||||
|
validate_librechat(args[1])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in new issue
Block a user