Author SHA1 Message Date
Ubuntu c877b61ef3 voxtral-tts: bill speech at the model's published rate ($16/1M characters)
Tinfoil's catalog lists voxtral-tts at $0/$0, so the entry declared no cost at
all and speech usage never consumed member allowance — voice mode was free
against the member budget while every other capability was metered. The OWUI
precedent was to bill TTS like anything else, so set the field.

input_cost_per_character: 0.000016 = $16/1M characters, from LiteLLM's bundled
map for the same weights (mistral/voxtral-mini-tts-*, sourced to Mistral's model
card) and the field OpenAI's TTS entries use on this adapter.

Revisit if the Tinfoil invoice shows a different number, and note the policy
question: Tinfoil may be serving this model beta-free, so members would be
consuming allowance for a resource the co-op is not paying for.
2026-10-26 20:17:06 +00:00
16 changed files with 63 additions and 1841 deletions

No files matched your search

-2
View File
@@ -1,2 +0,0 @@
__pycache__/
*.pyc
+11 -6
View File
@@ -5,18 +5,24 @@
"description": "LiteLLM AI Gateway \u2014 OpenAI-compatible proxy with per-user API keys, usage tracking, rate limiting, and model routing. Packaged for Cloudron with SSO/OIDC and PostgreSQL.",
"tagline": "AI gateway for cooperative inference",
"version": "1.0.0",
"upstreamVersion": "1.108.3",
"upstreamVersion": "1.103.2",
"healthCheckPath": "/health/readiness",
"httpPort": 4000,
"memoryLimit": 2147483648,
"manifestVersion": 2,
"website": "https://litellm.ai",
"contactEmail": "info@inference.coop",
"contactEmail": "botbot@hermes",
"icon": "file://logo.png",
"addons": {
"postgresql": {},
"redis": {},
"localstorage": {}
"localstorage": {},
"scheduler": {
"audio_keepalive": {
"schedule": "*/10 * * * *",
"command": "/app/code/audio_keepalive.py"
}
}
},
"tags": [
"ai",
@@ -25,6 +31,5 @@
"api"
],
"mediaLinks": [],
"changelog": "Initial package for inference.coop",
"dockerImage": "registry.inference.coop/code/litellm:20261114-84a1918"
}
"changelog": "Initial package for inference.coop"
}
+3 -77
View File
@@ -1,4 +1,4 @@
FROM ghcr.io/berriai/litellm:v1.108.3
FROM ghcr.io/berriai/litellm:v1.103.2
# v1.84.0 is the fix line for CVE-2026-35029 (auth bypass on
# /config/update, fixed 1.83.0) and CVE-2026-59822 (MCP session auth
@@ -19,14 +19,6 @@ RUN chmod +x /app/code/start.sh
# Default config — will be overridden by Cloudron env vars at runtime
COPY config.yaml /app/data/config.yaml
# A second copy inside the image: start.sh syncs it over the persistent
# /app/data volume at boot, so config changes in the repo actually deploy.
COPY config.yaml /app/code/config.yaml
# Fail-closed config validator (canonical copy lives in ops-docs/scripts/;
# this byte-identical twin ships with the image and is called from start.sh
# before the gateway boots). Mirrors validate_config.py's md5 — keep in step.
COPY validate_config.py /app/code/validate_config.py
# Append the co-op credit-pack hint to LiteLLM's hardcoded budget-exceeded
# messages. Done at BUILD time — the runtime rootfs is read-only under
@@ -43,77 +35,11 @@ RUN python3 /app/code/patch_budget_message.py
COPY patch_image_cost.py /app/code/patch_image_cost.py
RUN python3 /app/code/patch_image_cost.py
# Audio per-request pricing: Tinfoil bills voxtral-tts and whisper per request,
# which LiteLLM cannot express on the audio paths, so those two were billed at
# $0. The patch makes the cost calculator honour a declared
# input_cost_per_request. Check for the "[coop-patch] applied" line in the build
# log after upgrading LiteLLM: the script is non-fatal and warns instead.
COPY patch_audio_request_cost.py /app/code/patch_audio_request_cost.py
RUN python3 /app/code/patch_audio_request_cost.py
# The audio cost paths read pricing from the SHARED cost-map key, which the
# router strips per-deployment pricing from — so the lookup raised, the spend
# write was skipped, and audio produced no SpendLogs row at all. This re-adds the
# pricing (and input_cost_per_request) for the audio modes. Same mechanism as
# patch_image_cost.py above.
COPY patch_audio_router_pricing.py /app/code/patch_audio_router_pricing.py
RUN python3 /app/code/patch_audio_router_pricing.py
# SECURITY (2026-10-27, member-disclosure finding 4): LiteLLM forwards every
# provider response header to callers as `llm_provider-*`, which leaked the
# co-op's remaining provider balance (x-credits-remaining) to any member API key.
# Verify the "[coop-patch] applied: provider response headers" line in the build
# output on every version bump — the patch is non-fatal by design.
COPY patch_strip_provider_headers.py /app/code/patch_strip_provider_headers.py
RUN python3 /app/code/patch_strip_provider_headers.py
# Expose the per-request cost headers to cross-origin browser clients (co-op/support#39).
COPY patch_expose_cost_headers.py /app/code/patch_expose_cost_headers.py
RUN python3 /app/code/patch_expose_cost_headers.py
# Member access to config-declared pass-through routes: LiteLLM gates auth=true
# pass-through endpoints behind `allowed_passthrough_routes`, an Enterprise-only
# field that cannot be set on this community-edition proxy. This patch lets any
# authenticated key call routes the operator lists in
# general_settings.coop_passthrough_open_to_keys (unauthenticated access is
# unaffected — still rejected). Non-fatal; check the "[coop-patch] applied" line
# in the build log on every LiteLLM upgrade.
COPY patch_passthrough_key_access.py /app/code/patch_passthrough_key_access.py
RUN python3 /app/code/patch_passthrough_key_access.py
# Exact per-input-token pricing for generic pass-through routes. A route whose
# target matches none of vertex/anthropic/openai/gemini resolves to
# EndpointType.GENERIC: it IS logged, but LiteLLM has no model to price it from,
# so the only cost it knows is the flat cost_per_request. Metis (/v1/systemone)
# is billed at $0.12 per 1M input tokens and returns usage.input_tokens, so this
# patch prices a route listed in general_settings.coop_passthrough_cost_per_token
# from that count and writes the exact figure to member spend. Non-fatal; check
# the "[coop-patch] applied" line in the build log on every LiteLLM upgrade.
COPY patch_passthrough_token_cost.py /app/code/patch_passthrough_token_cost.py
RUN python3 /app/code/patch_passthrough_token_cost.py
# Audio keep-alive: the in-container loop in start.sh pings tinfoil/voxtral-tts
# every 15 min so voice mode skips the ~24s cold-start (see audio_keepalive.py).
# Audio keep-alive: scheduled pings keep the Tinfoil TEE audio enclave
# warm so voice mode skips the ~24s cold-start (see audio_keepalive.py header).
COPY audio_keepalive.py /app/code/audio_keepalive.py
RUN chmod +x /app/code/audio_keepalive.py
# Skip LiteLLM's forced verbose_json on transcription requests for opting-out
# models: Tinfoil's voxtral-small-24b rejects verbose_json with a 400, which
# broke every bare /v1/audio/transcriptions call to it (file + model — the
# shape members and dictation send). Its billing does not need the duration
# the injection exists for. Non-fatal; check the "[coop-patch] applied" line
# in the build log on every LiteLLM upgrade.
COPY patch_transcription_response_format.py /app/code/patch_transcription_response_format.py
RUN python3 /app/code/patch_transcription_response_format.py
# TTS default response format: LibreChat sends no response_format on its TTS
# request and Tinfoil's voxtral-tts then serves WAV (~8-17s warm measured) —
# while LibreChat labels the reply audio/mpeg regardless. Default to mp3
# (OpenAI's documented default, ~3s warm measured). Non-fatal; check the
# "[coop-patch] applied" line in the build log on every LiteLLM upgrade.
COPY patch_audio_default_mp3.py /app/code/patch_audio_default_mp3.py
RUN python3 /app/code/patch_audio_default_mp3.py
# Override the base image's ENTRYPOINT (which is `litellm`) so our
# startup script runs instead of being passed as an argument to litellm.
ENTRYPOINT ["/app/code/start.sh"]
+10 -54
View File
@@ -4,7 +4,7 @@ A Cloudron app package for [LiteLLM](https://litellm.ai/), the open-source AI ga
LiteLLM is the single gateway between members and the inference backends. It handles per-member API keys, usage metering, rate limiting (via budget caps), and model routing — all behind one OpenAI-compatible endpoint.
**Current base image:** LiteLLM **v1.108.3**, pinned to the exact tag (upgraded from v1.103.2 in the 2026-11-08/09 maintenance window; minimum supported line remains v1.84.0, the fix line for CVE-2026-35029 / CVE-2026-59822, CISA KEV-listed). See [Updating LiteLLM](#updating-litellm).
**Current base image:** LiteLLM **v1.103.2**, pinned to the exact tag (fix line for CVE-2026-35029 / CVE-2026-59822, CISA KEV-listed; also carries the streaming-usage fix for prompt_tokens_details). See [Updating LiteLLM](#updating-litellm).
## What this provides
@@ -16,16 +16,16 @@ LiteLLM is the single gateway between members and the inference backends. It han
- **Cloudron Redis** for rate limiting and caching
- **Automatic SSL, backups, and sandboxing** (Cloudron-managed)
- **Master key + salt key** auto-generated on first start, persisted in `/app/data`
- Hardened for the public internet: LiteLLM's built-in API docs surfaces are disabled (`NO_DOCS`/`NO_REDOC`/`NO_OPENAPI` → live 404s). The static `/ui/` admin-UI bundle **is served** on this version (see *Admin access*); logging into it still requires the master key. CORS is deliberately open (`LITELLM_CORS_ORIGINS=*`, co-op/support#38) — auth is the per-member bearer key, not origin checking.
- Hardened for the public internet: admin UI and API docs disabled at the proxy; CORS locked to the chat origin
## Architecture
```
Members → LibreChat (chat) / own tools (API) → member portal (key injection)
Members → Open WebUI (chat) / own tools (API) → member portal (key injection)
→ LiteLLM gateway (this app)
├→ Tinfoil (TEE enclaves — architectural privacy)
├→ GreenPT (100% renewable energy)
└→ Regolo (EU-hosted, renewable + zero data retention)
└→ PublicAI (publicly developed, sovereign models)
```
LiteLLM is the control plane (keys, metering, routing). It does **not** run models itself — it forwards to providers, each chosen by members for a different value. A [tinfoil-proxy](https://github.com/tinfoilsh/tinfoil-proxy) sidecar provides EHBP end-to-end encryption to Tinfoil's enclaves.
@@ -76,7 +76,7 @@ After install, set the provider API keys in Cloudron → app → Settings → En
- `TINFOIL_API_KEY` — Tinfoil (https://tinfoil.sh/)
- `GREENPT_API_KEY` — GreenPT (https://greenpt.ai/)
- `REGOLO_API_KEY` — Regolo (regolo.ai; replaced PublicAI, removed 2026-10-16)
- `PUBLICAI_API_KEY` — PublicAI (https://publicai.co/)
The following are auto-generated and should **not** be set manually:
@@ -85,36 +85,9 @@ The following are auto-generated and should **not** be set manually:
- `DATABASE_URL` — from Cloudron's PostgreSQL addon
- `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD` — from Cloudron's Redis addon
### Config deployment
`config.yaml` ships **with the image** and `start.sh` syncs it over the persistent
`/app/data/config.yaml` at boot whenever they differ. The volume copy would
otherwise shadow the image's file forever — which is exactly how a config change
can be merged, built and "deployed" while the gateway keeps serving the old
configuration. Change the repo and redeploy; never edit the volume copy directly.
### Admin access
The state of the admin surfaces on the pinned v1.103.2, wire-verified 2026-11-05:
- The built-in **API docs are disabled** (`NO_DOCS`/`NO_REDOC`/`NO_OPENAPI` in
`start.sh`) — `/docs`, `/redoc` and `/openapi.json` return 404.
- The **static admin UI bundle is still served at `/ui/`** — upstream's
`DISABLE_ADMIN_UI` env (which we set) only disables the UI's *SSO login
redirect* routes in this version; nothing on v1.103.2 stops the unconditional
`app.mount("/ui", …)`. So `GET /ui/` serves the login page. **Logging in
still requires the master key** (auto-generated here, never published), and
administrative operations normally run through the
[member portal](https://git.inference.coop/code/member-portal), which holds
the master key server-side.
- If the login surface itself should disappear, it needs a build-time patch
(strip the `/ui` mount from `proxy_server` in the Dockerfile, the same
non-fatal patch pattern the other 8 build patches use) — there is no env/flag
for it on this version. Until then treat `/ui/` as a master-key-only
endpoint, like any other control-plane route.
If direct LiteLLM admin access is ever needed, log in at `/ui/` with the
master key from `/app/data/.master_key` and lock it away again afterwards.
The LiteLLM admin UI and API docs are **disabled** on the public gateway (hardened by default; see `start.sh`). Administrative operations run through the [member portal](https://git.inference.coop/code/member-portal), which holds the master key server-side. If direct LiteLLM admin access is ever needed, re-enable the UI in `start.sh`, redeploy, and use the master key from `/app/data/.master_key` — then disable it again.
---
@@ -149,19 +122,6 @@ expect a 15–20 min maintenance window and **never panic-restart mid-migration*
1. **Pick a target version** from the [LiteLLM changelog](https://github.com/BerriAI/litellm/releases). Stay at or above v1.84.0 (the fix line for CVE-2026-35029 and CVE-2026-59822, the latter on CISA's KEV list).
**Re-validate the build patches against the target's real sources first** — the
scripts fail soft (a moved anchor prints a WARN and exits 0), so a silently
skipped patch ships. Ten patches are in the Dockerfile today; each must print
its `applied` line in the build log. The cheapest rehearsal is to unpack the
target wheel into a scratch tree and run the scripts against it (2026-11-09,
v1.108.3: all ten apply — the CORS anchor needs the 1.108.x fallback already
carried by `patch_expose_cost_headers.py`). Two gotchas in that harness:
`patch_audio_request_cost.py` finds its target by `import litellm`, which fails
outside a full install — point `LITELLM_COST_CALCULATOR` at the scratch
`cost_calculator.py` instead, or it exits with "could not locate"; and running
the scripts by hand leaves files owned by your user, which is fine in scratch
and never in the image.
2. **Bump the pin** in `Dockerfile`:
```dockerfile
@@ -191,17 +151,13 @@ expect a 15–20 min maintenance window and **never panic-restart mid-migration*
2. Generate a temp key with the master key → if `/key/generate` 500s, the schema is incomplete; fix the migration before anything else
3. Use the **temp key** (not master) for `GET /v1/models` → full catalog
4. Use the **temp key** for a small `POST /v1/chat/completions` → content + usage
5. OPTIONS preflight: `LITELLM_CORS_ORIGINS` is set at the app level (`*` by co-op decision, co-op/support#38) — a preflight from any origin answers with the matching `access-control-allow-origin` header rather than a 400. Auth is the bearer key, not the origin.
5. OPTIONS preflight from a foreign origin → 400; from `chat.inference.coop` → allowed
6. Cloudron app health = healthy, then delete the temp key
### Before updating
- Check the [LiteLLM changelog](https://github.com/BerriAI/litellm/releases) for breaking changes
- The `start.sh` and `config.yaml` may need adjustment if LiteLLM changes its env-var or config schema
- **Build-time patches ride on LiteLLM internals — verify each one after an upgrade.** `patch_budget_message.py`, `patch_image_cost.py` (keeps image-gen pricing on the shared cost-map key) and `patch_audio_request_cost.py` (adds the declared per-request charge on the audio paths: Tinfoil bills `voxtral-tts` $0.002 and `whisper-large-v3-turbo` $0.01 per request, which the cost map cannot express). Each prints `[coop-patch]` in the build log. They are non-fatal by design: a missing anchor warns and the build still succeeds, so the behaviour can vanish silently on a version bump. Related: code/litellm#5.
- **`patch_passthrough_token_cost.py`** prices a generic pass-through route from the token count it reports in its own response body. A `pass_through_endpoints` target whose host matches none of vertex/anthropic/openai/gemini resolves to `EndpointType.GENERIC`: it is logged, but with no model on the route LiteLLM only knows the flat `cost_per_request`, which is a budget estimate, not an invoice. The patch bills any route listed in `general_settings.coop_passthrough_cost_per_token` (USD per input token, keyed by path) as `usage.input_tokens × rate` — Metis (`/v1/systemone`) at $0.12 per 1M input tokens — and records the same count on the row (see below). Same rules: non-fatal, `[coop-patch]` in the build log, verify after every upgrade. `cost_per_request` stays as the fallback for a listed route whose response carries no token count. The block prints `[coop-passthrough-cost] input_tokens=… rate=… -> response_cost=…` on each priced call, so the log is the field check that it fired.
- **Pass-through spend rows also carry the token counts (Metis defect fix).** Pricing the row from the body was not enough: the row recorded the correct `spend` but `prompt_tokens`/`completion_tokens`/`total_tokens` stayed `0`, because the pass-through success handler hands the spend writer a `StandardPassThroughResponseObject` whose `usage` is empty and the generic branch builds no `standard_logging_object` usage. Anything summing tokens for the route (panel, dashboard, energy reporting) then under-reported. The same patch now sets `combined_usage_object` (a `litellm.Usage`) on the logging object — the override the streaming handler already uses for recovered partial usage — which the spend writer reads before the response object (`spend_tracking_utils.get_logging_payload`, `StandardLoggingPayloadSetup.get_usage_as_dict`). The row now records `prompt_tokens = usage.input_tokens`, `completion_tokens = 0`, `total_tokens = usage.input_tokens`, with spend unchanged.
- **A zero-spend row with `call_type = /v1/<path>` is NOT a duplicate of the priced row — it is the 401 refusal.** Every *rejected* pass-through request (missing/invalid key) is logged by upstream LiteLLM as a failure row whose `call_type` is the route path and whose `spend` is `0`, via `ProxyLogging._handle_logging_proxy_only_error` (`litellm/proxy/utils.py`, `original_function=route`). That is upstream behaviour on the pristine image, not something our patches add (neither `patch_passthrough_key_access.py` nor `patch_passthrough_token_cost.py` writes a row). A single *authenticated* call produces exactly **one** row (`call_type=pass_through_endpoint`, the real spend). Measured: one 200 → one row; one unauthenticated 401 → one failure row. Count requests for the route by `call_type = pass_through_endpoint`, not by summing all rows carrying the model name.
- The CORS env var is `LITELLM_CORS_ORIGINS` (upstream-native since 1.84.0); the old `LITELLM_CORS_ALLOWED_ORIGINS` + sed patch are dead — don't re-add them
- Test on a staging instance if the version jump is large
@@ -213,13 +169,13 @@ expect a 15–20 min maintenance window and **never panic-restart mid-migration*
|----------|----------|---------|
| `TINFOIL_API_KEY` | Yes | Tinfoil provider (TEE inference) |
| `GREENPT_API_KEY` | Yes | GreenPT provider (renewable energy) |
| `REGOLO_API_KEY` | Yes | Regolo provider (EU, renewable + ZDR) |
| `LITELLM_CORS_ORIGINS` | Set at app level | Currently `*` by co-op decision (co-op/support#38); `start.sh` falls back to the chat origin only if the app env omits it |
| `PUBLICAI_API_KEY` | Yes | PublicAI provider (public models) |
| `LITELLM_CORS_ORIGINS` | Set by start.sh | CORS allowlist (chat origin only) |
| `LITELLM_MIGRATION_DIR` | Set by start.sh | Persistent Prisma migrations ledger |
## Connecting chat clients
LibreChat (chat) does **not** talk to this gateway directly — it routes through the [member portal](https://git.inference.coop/code/member-portal), which authenticates the member and injects their personal LiteLLM key per request. Members building their own tools create API keys on the [member dashboard](https://dashboard.inference.coop) and use:
Open WebUI does **not** talk to this gateway directly — it routes through the [member portal](https://git.inference.coop/code/member-portal), which authenticates the member and injects their personal LiteLLM key per request. Members building their own tools create API keys on the [member dashboard](https://dashboard.inference.coop) and use:
- **API Base URL:** `https://gateway.inference.coop/v1` (or the portal's proxy endpoint for chat-integrated tools)
- **API Key:** their own virtual key from the dashboard
+3 -5
View File
@@ -2,12 +2,10 @@
"""Audio keep-alive: ping the Tinfoil TEE audio models through localhost LiteLLM
so voice mode doesn't pay the ~24s cold-start after idle.
Runs INSIDE the gateway container, invoked by the keep-alive loop in
start.sh (every 15 min; the Cloudron scheduler task was removed
2026-11-09 so the loop is the only warm path). Reads the master key from
env or /app/data/.master_key. TTS-only (STT pings cost
Runs INSIDE the gateway container via the Cloudron scheduler addon. Uses the
LITELLM_MASTER_KEY already in the container env. TTS-only (STT pings cost
$0.01/request each; TTS $0.002 — see docs, keep-alive economics). Silent on
success; prints + exits non-zero on failure so keepalive.log captures it.
success; prints + exits non-zero on failure so scheduler logs capture it.
"""
import json, os, urllib.request, urllib.error, sys
+25 -254
View File
@@ -10,13 +10,6 @@
# tinfoil/deepseek-v4-1-flash: $0.65 in / $1.45 out / $0.13 cached per 1M
# tinfoil/gpt-oss-120b: $0.15 in / $0.60 out per 1M
# tinfoil/glm-5.3: $1.80 in / $5.75 out / $0.45 cached per 1M
# greenpt/qwen3-embedding-8b: $0.25 in / $0 out per 1M (embeddings, 32K ctx)
# greenpt/green-embeddings: $0.20 in / $0 out per 1M (embeddings, 32K ctx)
# tinfoil/nomic-embed-text: $0.016 in / $0 out per 1M (embeddings, 8K ctx, TEE)
# regolo/mistral-small-4-119b: $0.20 in / $0.70 out per 1M (60K ctx)
# regolo/gemma4-31b: $0.40 in / $0.60 out per 1M (100K ctx)
# tinfoil/voxtral-small-24b: $0.20 in / $0.60 out per 1M (token-priced audio;
# +$0.00008/request on /v1/audio/transcriptions)
# Without these, LiteLLM can't price these custom models, so spend shows $0.00
# and the team budget cap is not enforced.
#
@@ -119,10 +112,6 @@ model_list:
# qwen3.5-122b: large vision+reasoning MoE (262K ctx). REQUIRES
# chat_template_kwargs: {enable_thinking: false} — otherwise the reasoning
# phase consumes the entire token budget and content comes back empty.
# mistral-small-4-119b + gemma4-31b: added 2026-11-13 (operator-approved).
# Rates read live from Regolo's own model catalog (regolo.ai/models):
# mistral $0.20/$0.70 per 1M, 60K in / 60K out; gemma4 $0.40/$0.60,
# 100K in / 100K out.
# Qwen-Image: image generation via /v1/images/generations, billed per pixel.
- model_name: regolo/apertus-70b
litellm_params:
@@ -131,8 +120,8 @@ model_list:
api_key: os.environ/REGOLO_API_KEY
model_info:
mode: chat
input_cost_per_token: 1.16e-6
output_cost_per_token: 3.944e-6
input_cost_per_token: 4.6e-7
output_cost_per_token: 2.44e-6
- model_name: regolo/qwen3.5-122b
litellm_params:
@@ -142,32 +131,8 @@ model_list:
chat_template_kwargs: {"enable_thinking": false}
model_info:
mode: chat
input_cost_per_token: 6.96e-7
output_cost_per_token: 3.944e-6
# Mistral Small 4 (119B) — hybrid instruct + reasoning model (three model
# families unified by Regolo). $0.20 in / $0.70 out per 1M, 60K ctx.
- model_name: regolo/mistral-small-4-119b
litellm_params:
model: openai/mistral-small-4-119b
api_base: https://api.regolo.ai/v1
api_key: os.environ/REGOLO_API_KEY
model_info:
mode: chat
input_cost_per_token: 2.0e-7
output_cost_per_token: 7.0e-7
# Gemma 4 (31B) — Google's dense multimodal Gemma 4. $0.40 in / $0.60 out
# per 1M, 100K ctx.
- model_name: regolo/gemma4-31b
litellm_params:
model: openai/gemma4-31b
api_base: https://api.regolo.ai/v1
api_key: os.environ/REGOLO_API_KEY
model_info:
mode: chat
input_cost_per_token: 4.0e-7
output_cost_per_token: 6.0e-7
input_cost_per_token: 1.16e-6
output_cost_per_token: 4.87e-6
- model_name: regolo/qwen-image
litellm_params:
@@ -176,23 +141,20 @@ model_list:
api_key: os.environ/REGOLO_API_KEY
model_info:
mode: image_generation
input_cost_per_pixel: 1.74e-8
input_cost_per_pixel: 1.5e-8
# --- Audio models (Tinfoil, in-TEE) ---
# Used for chat dictation (STT) and voice mode (TTS). These are NOT chat
# models: the member portal's /v1/models filters them out of the chat model
# list (see AUDIO_MODEL_PREFIXES there) so members don't see them in the
# picker, but they remain fully callable via /v1/audio/* for API users.
# whisper-large-v3-turbo: $0/1M tokens + $0.01 per request.
# voxtral-tts: $0/1M tokens + $0.002 per request.
# voxtral-small-24b: $0.20/1M in + $0.60/1M out tokens, $0 per request —
# the only TOKEN-PRICED audio model in their catalog (its block below).
# Tinfoil bills these per REQUEST (its model list carries a `requestPrice`);
# reading only the two token prices made TTS look free and left STT billed off
# an older catalog. LiteLLM cannot price per request on the audio paths, so
# patch_audio_request_cost.py (built in via the Dockerfile) makes the cost
# calculator honour `input_cost_per_request` below. Re-check that build log
# line after every LiteLLM upgrade.
# whisper-large-v3-turbo: $0.05/1M input tokens. voxtral-tts: Tinfoil's
# catalog lists it $0/$0 and every cost field was left NULL, so speech usage
# never consumed member allowance. Billed at the model's published rate
# instead: $16/1M characters, i.e. LiteLLM's own bundled price for the same
# weights (mistral/voxtral-mini-tts-*, sourced to Mistral's model card) and
# the rate OpenAI's TTS entries use for the same field. Revisit if the
# Tinfoil invoice shows a different number.
# NOTE: LiteLLM routes audio by the openai/ provider + the endpoint called
# (/v1/audio/transcriptions or /v1/audio/speech) — there is no
# "audio_transcription/" provider prefix.
@@ -203,38 +165,7 @@ model_list:
api_key: os.environ/TINFOIL_API_KEY
model_info:
mode: audio_transcription
# Tinfoil: $0 per token, $0.01 per request. The per-request charge is the
# whole cost, and the patch is what makes it billable.
input_cost_per_token: 0
input_cost_per_request: 0.01
# voxtral-small-24b — Tinfoil's TOKEN-PRICED STT model: the only audio entry
# in their catalog with real token prices ($0.20 in / $0.60 out per 1M, $0
# per request). The chat route (audio sent as input_audio) returns real
# token usage and bills exactly at those rates. The /v1/audio/transcriptions
# route returns NO token counts (duration only), so per-token billing cannot
# fire there; Tinfoil tokenizes audio in ~30s chunks (~375-380 tokens each —
# measured: a 1-second clip and a 6-second clip both cost ~380 prompt
# tokens), so for the short dictation requests this route serves the
# chunk-equivalent is ~$0.00008/request — declared as input_cost_per_request
# below, added by patch_audio_request_cost.py, which is scoped to the audio
# call types so chat calls to this model are NOT double-billed. Audio longer
# than ~30s should go through the chat route, which meters exactly.
- model_name: tinfoil/voxtral-small-24b
litellm_params:
model: openai/voxtral-small-24b
api_base: http://127.0.0.1:3301/v1
api_key: os.environ/TINFOIL_API_KEY
model_info:
mode: audio_transcription
# Opt out of LiteLLM's forced response_format=verbose_json injection on
# transcription calls (patch_transcription_response_format.py): Tinfoil
# rejects verbose_json for this model with a 400, and nothing in its
# billing needs the duration the injection exists to obtain.
coop_no_verbose_json: true
input_cost_per_token: 0.0000002
output_cost_per_token: 0.0000006
input_cost_per_request: 0.00008
input_cost_per_token: 0.00000005
- model_name: tinfoil/voxtral-tts
litellm_params:
@@ -243,29 +174,18 @@ model_list:
api_key: os.environ/TINFOIL_API_KEY
model_info:
mode: audio_speech
# Tinfoil: $0 per token, $0.002 per request — see the audio section note.
#
# input_cost_per_character: 0 is load-bearing, not decoration. The TTS cost
# branch (cost_per_token, cost_metric == "cost_per_character") raises
# ValueError("cost for tts call is None") when there is no character price,
# and that raise aborts the whole cost calculation — so the spend write is
# skipped and speech bills nothing, whatever input_cost_per_request says.
# Declaring the zero lets the path complete; the patch then adds the
# per-request charge.
input_cost_per_character: 0
input_cost_per_request: 0.002
# $16 per 1M characters. This is the field LiteLLM reads for speech on the
# openai adapter (same key OpenAI's tts-1 uses upstream), so member spend
# and budget caps now account for voice mode.
input_cost_per_character: 0.000016
# --- Embedding models (Tinfoil, in-TEE; and GreenPT) ---
# --- Embedding model (Tinfoil, in-TEE) ---
# nomic-embed-text — 768-dim text embeddings for RAG/semantic search.
# NOT a chat model: it powers /v1/embeddings only. MEMBER-FACING since
# 2026-11-06 — the team_id scoping (operator-account-only) was removed so
# members get a TEE-protected embedding option. This is the same model the
# co-op's own docs-RAG runs on inside the TEE (Open WebUI's knowledge base);
# that traffic is unaffected by members also being able to call it. The
# portal filters `mode: embedding` out of the chat-facing /v1/models list, so
# it stays API-only. Pricing per Tinfoil catalog (inference.tinfoil.sh
# /v1/models): $0.016/1M input, $0 out — corrected 2026-11-06 from a stale
# $0.05 declaration; 8K context declared so clients don't invent larger.
# NOT a chat model: it powers /v1/embeddings only. Scoped to the admin
# account's team (model_info.team_id) so member keys cannot call it — it is
# an operator resource for the co-op's own docs-RAG, not a member-facing
# model. The portal also filters it out of the chat-facing /v1/models list by
# its `mode: embedding`. Pricing per Tinfoil catalog: $0.05/1M input, $0 out.
- model_name: tinfoil/nomic-embed-text
litellm_params:
model: openai/nomic-embed-text
@@ -273,53 +193,10 @@ model_list:
api_key: os.environ/TINFOIL_API_KEY
model_info:
mode: embedding
max_input_tokens: 8192
input_cost_per_token: 0.000000016
team_id: cdfdf422-633f-4432-8893-cffee82a8325
input_cost_per_token: 0.00000005
output_cost_per_token: 0.0
# --- Embedding model (GreenPT, member-facing) ---
# qwen3-embedding-8b — text embeddings via /v1/embeddings, callable by any
# member key. Added 2026-11-06 after a member asked for an embedding model,
# filling the member-facing gap on renewable EU hosting (Tinfoil's private
# embedding was opened to members the same day). The portal filters
# mode:embedding out of the chat picker, so it is API-only — same shape as
# the audio models above.
# Pricing per GreenPT /v1/pricing: $0.25/1M input, no output charge
# (embeddings return vectors; GreenPT lists maxOutputTokens null), 32K
# context. max_input_tokens declared from the provider's contextWindowTokens
# so clients don't invent a larger one (the null-caps defect).
- model_name: greenpt/qwen3-embedding-8b
litellm_params:
model: openai/qwen3-embedding-8b
api_base: https://api.greenpt.ai/v1
api_key: os.environ/GREENPT_API_KEY
model_info:
mode: embedding
input_cost_per_token: 0.00000025
output_cost_per_token: 0.0
max_input_tokens: 32000
# --- Embedding model (GreenPT, member-facing) ---
# green-embeddings — GreenPT's own embedding model (2560-dim, measured live).
# Added 2026-11-06 after a catalog survey: the only genuinely distinct
# embedding model on GreenPT besides qwen3-embedding-8b. Their catalog also
# lists a `green-embedding` (singular) slug — verified the SAME model
# (identical vectors on the same input), so only green-embeddings is served.
# Cheaper per token than qwen3-embedding-8b; the reranker sibling
# (green-rerank) is a different capability and is NOT served.
# Same shape: /v1/embeddings only, filtered out of the chat picker.
# Pricing per GreenPT /v1/pricing: $0.20/1M input, no output charge, 32K ctx.
- model_name: greenpt/green-embeddings
litellm_params:
model: openai/green-embeddings
api_base: https://api.greenpt.ai/v1
api_key: os.environ/GREENPT_API_KEY
model_info:
mode: embedding
input_cost_per_token: 0.0000002
output_cost_per_token: 0.0
max_input_tokens: 32000
# Fallback: if DeepSeek is down, use GPT-OSS
# NOTE: DeepSeek V4.1 Flash is a reasoning model — generations routinely run
# longer than 30s. A 30s timeout caused LiteLLM to abandon in-flight DeepSeek
@@ -329,58 +206,6 @@ model_list:
# logged; and a retry_policy that retries transient connection errors (base
# num_retries) but NEVER retries timeouts (TimeoutErrorRetries: 0) — a timed-out
# generation is already billed by Tinfoil, so re-hitting it re-bills.
# --- Stable aliases -------------------------------------------------------
# Version-free names members can pin once and keep using across transitions.
# When we move the underlying model, only the pointer here changes; member
# client configs and the docs do not. Pricing is stated AT the alias, so a
# transition that changes price is visible in the model list and in budgets
# (LiteLLM bills the alias's model_info, not the target's).
# Policy: `<family>-green` = the default route (renewable energy, policy-based
# privacy); `<family>-tee` = the same family served inside a TEE (attested
# hardware). The names say WHICH ROUTE, not how fresh a version is: we re-point
# them at transitions, so pinning one is what makes a config version-free.
- model_name: deepseek-flash-green
litellm_params:
model: openai/deepseek-v4.1-flash
api_base: https://api.greenpt.ai/v1
api_key: os.environ/GREENPT_API_KEY
model_info:
input_cost_per_token: 0.00000022
output_cost_per_token: 0.00000110
cache_read_input_token_cost: 0.000000011
- model_name: deepseek-flash-tee
litellm_params:
model: openai/deepseek-v4-1-flash
api_base: http://127.0.0.1:3301/v1
api_key: os.environ/TINFOIL_API_KEY
model_info:
input_cost_per_token: 0.00000065
output_cost_per_token: 0.00000145
cache_read_input_token_cost: 0.00000013
- model_name: glm-flash-green
litellm_params:
model: openai/glm-5.3-flash
api_base: https://api.greenpt.ai/v1
api_key: os.environ/GREENPT_API_KEY
model_info:
input_cost_per_token: 0.00000011
output_cost_per_token: 0.00000044
cache_read_input_token_cost: 0.000000022
# Tinfoil's flash GLM was deprecated 2026-10-09; the TEE route for this family
# is the flagship, so the alias is named glm-tee rather than glm-flash-tee.
- model_name: glm-tee
litellm_params:
model: openai/glm-5-3
api_base: http://127.0.0.1:3301/v1
api_key: os.environ/TINFOIL_API_KEY
model_info:
input_cost_per_token: 0.00000180
output_cost_per_token: 0.00000575
cache_read_input_token_cost: 0.00000045
router_settings:
num_retries: 2
timeout: 180
@@ -397,60 +222,6 @@ general_settings:
database_url: os.environ/DATABASE_URL
store_model_in_db: true
# --- Metis (GreenPT) — decision model, not a chat model ---
# Metis is a *decision* endpoint, not a chat completion: it takes a `state`
# plus up to 32 typed questions and returns PROBABILITIES / choices / scores.
# It generates no text (output_tokens is always 0), so it cannot be a
# model_list entry — LiteLLM's chat pipeline has nothing to parse. It is
# exposed as a pass-through route instead, and members call it with their
# normal co-op API key (auth: true), never the provider key.
#
# GreenPT-hosted (policy-based privacy, EU, 100% renewable), same
# GREENPT_API_KEY the green chat models already use.
# Price: $0.12 per 1M *input* tokens; output is free.
#
# METERING: exact, from the token count GreenPT reports back. Metis is not a
# model_list entry (LiteLLM's chat pipeline has nothing to parse), so it is
# priced through the coop_passthrough_cost_per_token map below, keyed by this
# route's path: a call's cost is usage.input_tokens × the rate. patch_
# passthrough_token_cost.py (built in via the Dockerfile) reads
# usage.input_tokens from the Metis response body at logging time and writes
# the exact figure to member spend. cost_per_request on the route is now only
# a fallback: it applies if a response ever arrives without a usable token
# count, so a call is never silently free. Keep the two in sync.
#
# Routes here (matched by prefix) may be called by ANY authenticated member
# key. LiteLLM's own gate for auth=true pass-through routes
# (`allowed_passthrough_routes`) is an Enterprise-only field, so on this
# community-edition proxy it can never be granted to member keys — the only
# other option, auth:false, would expose the route publicly and forward the
# co-op's GreenPT key to anonymous callers. This list is consumed at build
# time by patch_passthrough_key_access.py; authentication is unchanged (the
# endpoint still requires a valid key).
coop_passthrough_open_to_keys:
- "/v1/systemone"
# Exact per-input-token pricing (USD per token) for pass-through routes that
# report their own usage in the RESPONSE BODY — the shape the x-litellm-*
# response-header contract does not cover. Keyed by the gateway route path
# (an upstream target URL also matches). Any route listed here is billed from
# usage.input_tokens × the rate instead of its flat cost_per_request; a route
# not listed is unchanged. Consumed at build time by
# patch_passthrough_token_cost.py.
coop_passthrough_cost_per_token:
# Metis (GreenPT): $0.12 per 1M input tokens, output always free.
"/v1/systemone": 0.00000012
pass_through_endpoints:
- path: "/v1/systemone"
target: "https://api.greenpt.ai/v1/systemone"
headers:
Authorization: "Bearer os.environ/GREENPT_API_KEY"
Content-Type: "application/json"
auth: true
methods: ["POST"]
timeout: 60
# Fallback only — the exact token cost above is what bills in practice.
cost_per_request: 0.0001
litellm_settings:
salt_key: os.environ/LITELLM_SALT_KEY
drop_params: true
-103
View File
@@ -1,103 +0,0 @@
"""Default /v1/audio/speech to mp3 when the caller omits response_format.
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
if the target file or the anchor is missing — e.g. after a version bump — it
prints a warning and exits 0, so the build and the gateway still succeed. Look
for the ``[coop-patch]`` line in the build log.
Why: LibreChat's speech client sends no ``response_format`` on its TTS request
(TTSService.openAIProvider builds {input, model, voice} — the field is not in
its schema), and Tinfoil's voxtral-tts then serves WAV. Measured 2026-11-07
through the gateway: WAV ~8-17s warm per request (145-300 KB), while the SAME
model with ``response_format: "mp3"`` answers in ~2.6-3.1s (28-39 KB) — the
provider streams/encodes mp3 progressively. LibreChat additionally labels the
reply ``Content-Type: audio/mpeg`` unconditionally, so WAV bytes were served
under an mp3 label and browsers depended on MIME sniffing to play them.
Patch: default ``response_format`` to ``mp3`` (OpenAI's own documented default
for /v1/audio/speech) when the request omits it. An explicit response_format
from any caller still wins, so API members who want WAV can keep requesting
it. Scope: only the aspeech route; transcriptions are untouched.
Upstream: LiteLLM 1.103.2 forwards response_format verbatim and leaves the
provider default in place when it is absent.
"""
from __future__ import annotations
import glob
import os
import sys
MARKER = "CO-OP PATCH (TTS default mp3, 2026-11-07)"
ANCHOR = ''' ### CALL HOOKS ### - modify incoming data / reject request before calling the model
data = await proxy_logging_obj.pre_call_hook(
user_api_key_dict=user_api_key_dict, data=data, call_type="aspeech"
)'''
INSERT = ANCHOR + '''
# CO-OP PATCH (TTS default mp3, 2026-11-07): LibreChat sends no
# response_format and Tinfoil's voxtral-tts then serves WAV — measured
# ~8-17s warm vs ~3s for mp3, which the provider streams. LibreChat
# labels the reply audio/mpeg regardless. Default to mp3 (OpenAI's
# documented default); an explicit response_format still wins.
if not data.get("response_format"):
data["response_format"] = "mp3"'''
def target_path() -> str:
if len(sys.argv) > 1:
return sys.argv[1]
env = os.environ.get("LITELLM_PROXY_SERVER")
if env:
return env
hits = []
try: # most reliable: ask the installed package where it lives
import litellm # noqa: PLC0415
_file = getattr(litellm, "__file__", None)
if _file:
hits.append(os.path.join(os.path.dirname(_file), "proxy", "proxy_server.py"))
except Exception:
pass
hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/proxy/proxy_server.py")
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/proxy/proxy_server.py")
hits = [h for h in hits if os.path.exists(h)]
if not hits:
raise SystemExit("[coop-patch] could not locate litellm/proxy/proxy_server.py")
return sorted(hits)[0]
def main() -> int:
path = target_path()
try:
source = open(path).read()
except OSError as exc:
print(f"[coop-patch] cannot read {path}: {exc}")
return 0
if MARKER in source:
print(f"[coop-patch] {path} already patched, nothing to do")
return 0
count = source.count(ANCHOR)
if count != 1:
print(f"[coop-patch] WARNING aspeech pre_call_hook anchor found {count}x (need exactly 1) "
f"in {path}; TTS stays on the provider's default (WAV for Tinfoil)")
return 0
patched = source.replace(ANCHOR, INSERT, 1)
compile(patched, path, "exec") # refuse to write something that will not import
if not os.access(path, os.W_OK): # practice run against a read-only copy
print(f"[coop-patch] {path} is not writable; verified anchor and syntax only")
return 0
with open(path, "w") as handle:
handle.write(patched)
print(f"[coop-patch] applied TTS default mp3 to {path}")
return 0
if __name__ == "__main__":
sys.exit(main())
-248
View File
@@ -1,248 +0,0 @@
"""Bill a declared per-request charge on requests LiteLLM prices only per unit.
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
if the target file or the anchor strings are missing — e.g. after an upstream
version bump — it prints a warning and exits 0, so the build and the gateway
still succeed. The build log line ``[coop-patch] applied`` is what confirms it
took effect; check it on every LiteLLM upgrade (see README, upgrade checklist).
Why: Tinfoil charges some models a flat amount PER REQUEST on top of (or instead
of) per-token pricing — ``voxtral-tts`` $0.002/request and
``whisper-large-v3-turbo`` $0.01/request, both with $0 per-token. LiteLLM can
only express per-request pricing on the search / rerank / MCP paths, via
``input_cost_per_query`` (see ``litellm/search/cost_calculator.py``, and
``litellm/types/utils.py``: "per-request pricing: rerank, search, and Bedrock
Marengo embed"). The audio paths price by character, second or token, and
``input_cost_per_request`` exists only in the proxy's cost-tracking settings —
the model cost calculator never reads it. So speech and transcription usage is
billed at $0 however the config is written.
Patch: make the model cost calculator honour a declared
``input_cost_per_request``. ``response_cost_calculator`` is the single funnel for
every call type (the proxy's audio endpoints dispatch as ``aspeech`` and
``transcription`` and reach it through ``Logging._response_cost_calculator``), so
adding the charge at its final return covers both. The declared value is resolved
from the per-deployment ``custom_pricing`` the logging layer builds out of our
``config.yaml`` ``model_info``, falling back to the shared cost map and to the
call metadata.
Scope: nothing changes unless a model declares ``input_cost_per_request``, so
this is inert for every other model on the gateway. Since 2026-11-06 the charge
is added only on the AUDIO call types (transcription/atranscription/speech/
aspeech): a model that also carries per-token pricing (``voxtral-small-24b``)
bills its chat calls exactly per token, and a per-request charge on top of
that would double-bill them. The audio endpoints are the paths that cannot
price per token (the provider returns no token counts there), so the flat
charge belongs to them alone.
Upstream: no issue filed; LiteLLM has no per-request audio pricing as of 1.103.2.
"""
from __future__ import annotations
import glob
import os
import sys
MARKER = "CO-OP PATCH (audio per-request cost, 2026-10-26)"
HELPER = '''def _coop_per_request_cost(
*,
model: str | None = None,
litellm_model_name: str | None = None,
custom_pricing: Any | None = None,
litellm_logging_obj: Any | None = None,
) -> float:
"""CO-OP PATCH (audio per-request cost, 2026-10-26)
Return the model's declared ``input_cost_per_request``, else 0.0.
LiteLLM cannot price a flat per-request charge on the audio paths, so
Tinfoil's per-request models (voxtral-tts, whisper-large-v3-turbo) were
billed at $0. Resolution order: the per-deployment custom pricing the
logging layer builds from our config.yaml model_info, then the shared cost
map, then the call metadata. Inert unless a model declares the field.
"""
def _is_audio(name: Any) -> bool:
text = str(name or "").lower()
return (
"tts" in text or "whisper" in text or "voxtral" in text
)
cost_map = {}
try:
import litellm as _litellm # local import: not every caller binds it globally
cost_map = getattr(_litellm, "model_cost", {}) or {}
except Exception:
cost_map = {}
candidates = []
if isinstance(custom_pricing, dict):
candidates.append(custom_pricing)
map_hits = []
for key in (litellm_model_name, model):
entry = cost_map.get(key) if key else None
if isinstance(entry, dict):
candidates.append(entry)
map_hits.append((key, entry.get("input_cost_per_request")))
try:
details = getattr(litellm_logging_obj, "model_call_details", None) or {}
info = (details.get("metadata") or {}).get("model_info")
if isinstance(info, dict):
candidates.append(info)
except Exception:
pass
# Diagnostic (audio models only): did this run at all, and what could it see?
# If this line never appears for a speech call, the request never reaches the
# model cost calculator and the loss is upstream of this patch.
if _is_audio(model) or _is_audio(litellm_model_name):
try:
print(
"[coop-audio] probe"
f" model={model!r} litellm={litellm_model_name!r}"
f" custom_pricing={sorted(custom_pricing) if isinstance(custom_pricing, dict) else type(custom_pricing).__name__}"
f" map_hits={map_hits!r} candidates={len(candidates)}",
flush=True,
)
except Exception:
pass
for candidate in candidates:
try:
value = candidate.get("input_cost_per_request")
except Exception:
continue
if value:
try:
resolved = float(value)
except (TypeError, ValueError):
continue
if _is_audio(model) or _is_audio(litellm_model_name):
try:
print(f"[coop-audio] resolved per_request={resolved}", flush=True)
except Exception:
pass
return resolved
if _is_audio(model) or _is_audio(litellm_model_name):
try:
print("[coop-audio] resolved per_request=0.0 (nothing declared)", flush=True)
except Exception:
pass
return 0.0
'''
ANCHOR_DEF = "def response_cost_calculator("
ANCHOR_RETURN = """ region_name=region_name,
)
return response_cost
except Exception as e:
raise e"""
PATCHED_RETURN = """ region_name=region_name,
)
# CO-OP PATCH (audio per-request cost, 2026-10-26): add the declared
# per-request charge for models LiteLLM can only price per unit
# (Tinfoil's voxtral-tts, whisper-large-v3-turbo). Inert when the model
# declares nothing; the early "provider reported the cost" return above
# is unaffected. Scoped to the AUDIO call types (2026-11-06): a model
# that ALSO carries per-token pricing (voxtral-small-24b) bills its
# chat calls exactly per token and must not take the per-request
# charge on top of that.
try:
_coop_audio_call = call_type in (
"transcription",
"atranscription",
"speech",
"aspeech",
)
_coop_pr = (
_coop_per_request_cost(
model=model,
litellm_model_name=litellm_model_name,
custom_pricing=custom_pricing,
litellm_logging_obj=litellm_logging_obj,
)
if _coop_audio_call
else 0.0
)
response_cost = (response_cost or 0.0) + _coop_pr
_coop_name = str(model or "") + str(litellm_model_name or "")
if _coop_audio_call and (
"tts" in _coop_name.lower()
or "whisper" in _coop_name.lower()
or "voxtral" in _coop_name.lower()
):
print(
f"[coop-audio] add per_request={_coop_pr!r} -> response_cost={response_cost!r}",
flush=True,
)
except Exception as _coop_exc:
print(f"[coop-audio] add failed: {type(_coop_exc).__name__}: {_coop_exc}", flush=True)
return response_cost
except Exception as e:
raise e"""
def target_path() -> str:
"""The installed litellm cost calculator (or an explicit path for testing)."""
if len(sys.argv) > 1:
return sys.argv[1]
env = os.environ.get("LITELLM_COST_CALCULATOR")
if env:
return env
hits = []
try: # most reliable: ask the installed package where it lives
import litellm # noqa: PLC0415
_file = getattr(litellm, "__file__", None)
if _file:
hits.append(os.path.join(os.path.dirname(_file), "cost_calculator.py"))
except Exception:
pass
hits += glob.glob("/usr/lib/python3*/site-packages/litellm/cost_calculator.py")
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/cost_calculator.py")
hits += glob.glob("/usr/local/lib/python3*/dist-packages/litellm/cost_calculator.py")
hits += glob.glob("/opt/**/litellm/cost_calculator.py", recursive=True)
hits = [h for h in hits if os.path.exists(h)]
if not hits:
raise SystemExit("[coop-patch] could not locate litellm/cost_calculator.py")
return sorted(hits)[0]
def main() -> int:
path = target_path()
try:
source = open(path).read()
except OSError as exc:
print(f"[coop-patch] cannot read {path}: {exc}")
return 0
if MARKER in source:
print(f"[coop-patch] {path} already patched, nothing to do")
return 0
if ANCHOR_DEF not in source or ANCHOR_RETURN not in source:
print(f"[coop-patch] WARNING anchors not found in {path}; audio per-request "
f"cost will NOT be billed (check this after a LiteLLM upgrade)")
return 0
patched = source.replace(ANCHOR_DEF, HELPER + ANCHOR_DEF, 1)
patched = patched.replace(ANCHOR_RETURN, PATCHED_RETURN, 1)
compile(patched, path, "exec") # refuse to write something that will not import
if not os.access(path, os.W_OK): # practice run against a read-only copy
print(f"[coop-patch] {path} is not writable; verified anchors and syntax only")
return 0
with open(path, "w") as handle:
handle.write(patched)
print(f"[coop-patch] applied to {path}")
return 0
if __name__ == "__main__":
sys.exit(main())
-112
View File
@@ -1,112 +0,0 @@
"""Keep audio-model pricing on the shared cost-map key (speech + transcription).
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
if the target file or the anchor is missing — e.g. after a version bump — it
prints a warning and exits 0, so the build and the gateway still succeed. Look
for the ``[coop-patch]`` line in the build log.
Why this is needed on top of patch_audio_request_cost.py: the audio cost paths
(speech, transcription) resolve pricing from the **shared** cost-map key, and the
router deliberately strips per-deployment pricing from that key ("so one
deployment's overrides don't pollute sibling deployments"). For a deployment on a
custom backend — ours are all `openai/<model>` against a local Tinfoil proxy —
the cost lookup then raises, the spend-log write is skipped, and **no SpendLogs
row appears at all**. That is the symptom we measured: `/v1/audio/speech` and
`/v1/audio/transcriptions` returned 200 with real audio, and the member key's
spend stayed exactly $0.000000 with no spend row anywhere. It is the same
mechanism as the image-generation case that patch_image_cost.py already fixes
(upstream BerriAI/litellm#10023/#12471).
The cost-calculator patch alone cannot fix it: it only adds a per-request charge
*on top of* a cost the normal path computed, and that path never got that far.
`input_cost_per_request` is deliberately allowed through even though it is **not**
in `CustomPricingLiteLLMParams.model_fields` — LiteLLM does not list it there, and
patch_audio_request_cost.py is what makes the cost calculator read it.
Safe for us: the co-op gateway has one deployment per audio backend, so there are
no sibling deployments for the shared key to pollute.
"""
from __future__ import annotations
import glob
import os
import sys
MARKER = "CO-OP PATCH (audio shared pricing, 2026-10-26)"
ANCHOR = " # Always register the (possibly mode-preserved) shared backend info."
INSERT = ''' # CO-OP PATCH (audio shared pricing, 2026-10-26): the audio cost paths
# (speech, transcription) read pricing from the shared cost-map key, and
# this router strips per-deployment pricing from it — so a custom-backend
# audio deployment raises in the cost lookup, the spend write is skipped,
# and no SpendLogs row appears at all (same mechanism as the image case
# this file already patches). Re-add the pricing for audio modes,
# including input_cost_per_request, which LiteLLM does not list among its
# CustomPricing fields.
if deployment_mode in ("audio_speech", "audio_transcription"):
for _ak, _av in model_info.items():
if (
_ak in CustomPricingLiteLLMParams.model_fields
or _ak == "input_cost_per_request"
) and _av is not None:
shared_model_info[_ak] = _av
'''
def target_path() -> str:
if len(sys.argv) > 1:
return sys.argv[1]
env = os.environ.get("LITELLM_ROUTER")
if env:
return env
hits = []
try: # most reliable: ask the installed package where it lives
import litellm # noqa: PLC0415
_file = getattr(litellm, "__file__", None)
if _file:
hits.append(os.path.join(os.path.dirname(_file), "router.py"))
except Exception:
pass
hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/router.py")
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/router.py")
hits = [h for h in hits if os.path.exists(h)]
if not hits:
raise SystemExit("[coop-patch] could not locate litellm/router.py")
return sorted(hits)[0]
def main() -> int:
path = target_path()
try:
source = open(path).read()
except OSError as exc:
print(f"[coop-patch] cannot read {path}: {exc}")
return 0
if MARKER in source:
print(f"[coop-patch] {path} already patched, nothing to do")
return 0
if ANCHOR not in source:
print(f"[coop-patch] WARNING anchor not found in {path}; audio pricing will "
f"NOT reach the shared cost map and audio spend will stay invisible")
return 0
patched = source.replace(ANCHOR, INSERT + ANCHOR, 1)
compile(patched, path, "exec") # refuse to write something that will not import
if not os.access(path, os.W_OK): # practice run against a read-only copy
print(f"[coop-patch] {path} is not writable; verified anchor and syntax only")
return 0
with open(path, "w") as handle:
handle.write(patched)
print(f"[coop-patch] applied audio shared pricing to {path}")
return 0
if __name__ == "__main__":
sys.exit(main())
-115
View File
@@ -1,115 +0,0 @@
"""Expose LiteLLM's per-request cost headers to cross-origin browser clients.
MEMBER REQUEST (co-op/support#39, 2026-10-31). Browser JS calling the gateway from
another origin cannot read a response header unless the server names it in
Access-Control-Expose-Headers. LiteLLM sets
expose_headers=LITELLM_UI_ALLOW_HEADERS # through 1.103.x
expose_headers=[*LITELLM_UI_ALLOW_HEADERS,
*trace_response_expose_headers()] # from 1.108.0
which covers seven names (semantic-filter, guardrail, cache-key, ...) and none of
the cost ones — so `x-litellm-response-cost*` reads as null in JS even though the
proxy sends those headers on the response. Upstream closed the request
(BerriAI/litellm#33123) as stale, so it has to be handled here at the app layer,
exactly like the other patches in this directory.
Additive only. It lets a caller read per-request pricing for requests they made
with their own API key — numbers already present in the response body for
non-streaming calls, and already visible in their dashboard. It exposes no prompt,
key, or account data, and no other member's usage.
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal, per
house rule: if upstream moves or rewrites this call, WARN and exit 0 — a missing
convenience header must never keep the gateway from booting.
Regression check: after a deploy, a browser request from an unrelated origin should
see the cost headers listed in Access-Control-Expose-Headers on the response.
"""
from pathlib import Path
import sys
# The header names the proxy can set, taken from the running image rather than
# guessed: grep -rhoE 'x-litellm-[a-z0-9-]*cost[a-z0-9-]*' over the litellm package.
EXPOSED_HEADERS = [
"x-litellm-response-cost",
"x-litellm-response-cost-input",
"x-litellm-response-cost-output",
"x-litellm-response-cost-original",
"x-litellm-response-cost-discount-amount",
"x-litellm-response-cost-margin-amount",
"x-litellm-response-cost-margin-percent",
"x-litellm-response-cost-cache-creation",
"x-litellm-response-cost-cache-read",
"x-litellm-response-cost-reasoning",
"x-litellm-response-cost-tool-usage",
"x-litellm-classifier-cost",
# Telemetry names, requested in co-op/support#39 (2026-10-31): model identity,
# call id, the caller's own key spend, timings and retry/fallback counts.
"x-litellm-model-name",
"x-litellm-call-id",
"x-litellm-key-spend",
"x-litellm-response-duration-ms",
"x-litellm-overhead-duration-ms",
"x-litellm-callback-duration-ms",
"x-litellm-attempted-retries",
"x-litellm-attempted-fallbacks",
]
OLD = " expose_headers=LITELLM_UI_ALLOW_HEADERS,\n"
NEW = (
" expose_headers=LITELLM_UI_ALLOW_HEADERS\n"
" + [\n"
+ "".join(f' "{h}",\n' for h in EXPOSED_HEADERS)
+ " ],\n"
)
# 1.108.0 refactored the middleware call (upstream added the trace-response
# header set to expose_headers). Same operation, new anchor; tried when the
# 1.103.2 shape is absent so the script works against either base image.
OLD_V1108 = (
" expose_headers=[*LITELLM_UI_ALLOW_HEADERS, *trace_response_expose_headers()],\n"
)
NEW_V1108 = (
" expose_headers=[\n"
" *LITELLM_UI_ALLOW_HEADERS,\n"
" *trace_response_expose_headers(),\n"
+ "".join(f' "{h}",\n' for h in EXPOSED_HEADERS)
+ " ],\n"
)
def main() -> int:
# Glob rather than pin a Python version: the venv path moves when the base
# image bumps its interpreter, and a stale path would silently skip the patch.
candidates = sorted(Path("/app/.venv/lib").glob("python3*/site-packages/litellm/proxy/proxy_server.py"))
if not candidates:
print("WARN expose-cost-headers: proxy_server.py not found under /app/.venv/lib — skipping")
return 0
target = candidates[-1]
src = target.read_text()
if "x-litellm-response-cost-input" in src:
print("expose-cost-headers: already applied")
return 0
for old, new, shape in ((OLD, NEW, "1.103.x"), (OLD_V1108, NEW_V1108, "1.108.x")):
found = src.count(old)
if found == 1:
target.write_text(src.replace(old, new))
print(
f"expose-cost-headers: appended {len(EXPOSED_HEADERS)} cost header names "
f"to expose_headers (anchor {shape}) in {target}"
)
return 0
if found > 1:
print(f"WARN expose-cost-headers: expected exactly 1 {shape} CORS middleware call, found {found} — skipping")
return 0
print(
"WARN expose-cost-headers: no known expose_headers anchor found "
"(neither 1.103.x nor 1.108.x shape) — skipping"
)
return 0
if __name__ == "__main__":
sys.exit(main())
-107
View File
@@ -1,107 +0,0 @@
"""Open explicitly-listed pass-through routes to any authenticated key.
WHY. Config-declared pass-through endpoints registered with ``auth: true`` are
gated at request time by ``allowed_passthrough_routes`` on the key or team
(``RouteChecks.check_passthrough_route_access``). That field is a **LiteLLM
Enterprise** feature: on a community-edition proxy the management API refuses to
set it ("This feature is only available for LiteLLM Enterprise users"), so a
member key can never be granted access by any supported route. The only other
lever is ``auth: false``, which forwards **unauthenticated** requests — a public
relay of the co-op's GreenPT key. Neither is acceptable.
This patch lets the operator name such routes in the proxy config:
general_settings:
coop_passthrough_open_to_keys: ["/v1/systemone"]
Any route listed there is allowed for any key that has already authenticated.
Authentication itself is untouched: ``auth: true`` still installs
``Depends(user_api_key_auth)``, so an unauthenticated request is rejected earlier
and never reaches this function. The patch only relaxes the *second* gate
(per-key allow-list), and only for routes the operator listed by hand.
House style: run at BUILD time (the runtime rootfs is read-only under Cloudron);
non-fatal — if upstream renames or restructures this function, WARN and exit 0;
idempotent — detect the marker on a rebuild and skip; and compile the patched
source in memory before writing, so a patch that cannot import never ships.
Check the ``[coop-patch] applied`` line in the build log on every LiteLLM bump.
"""
from pathlib import Path
import os
import sys
MARKER = "coop_passthrough_open_to_keys"
# Unique multi-line anchor: the def + docstring + the first statement of the body.
OLD = (
" def check_passthrough_route_access(route: str, user_api_key_dict: UserAPIKeyAuth) -> bool:\n"
" \"\"\"\n"
" Check if route is a passthrough route.\n"
" Supports both exact match and prefix match.\n"
" \"\"\"\n"
" metadata: Final = user_api_key_dict.metadata\n"
)
NEW = (
" def check_passthrough_route_access(route: str, user_api_key_dict: UserAPIKeyAuth) -> bool:\n"
" \"\"\"\n"
" Check if route is a passthrough route.\n"
" Supports both exact match and prefix match.\n"
" \"\"\"\n"
" # [coop-patch] Routes the operator lists in\n"
" # general_settings.coop_passthrough_open_to_keys are callable by ANY\n"
" # authenticated key. LiteLLM's own gate (allowed_passthrough_routes)\n"
" # is Enterprise-only, so it can never be granted on this deployment.\n"
" # Authentication is unaffected: auth=True on the endpoint still\n"
" # requires a valid key before this function runs.\n"
" try:\n"
" from litellm.proxy.proxy_server import general_settings as _coop_general_settings\n"
"\n"
" _coop_open_to_keys = (_coop_general_settings or {}).get(\"coop_passthrough_open_to_keys\") or []\n"
" for _coop_route in _coop_open_to_keys:\n"
" if RouteChecks._route_matches_allowed_route(route=route, allowed_route=_coop_route):\n"
" return True\n"
" except Exception:\n"
" pass\n"
"\n"
" metadata: Final = user_api_key_dict.metadata\n"
)
def main() -> int:
# ROOT override (house style) lets the same script run against a pulled copy
# locally: COOP_PATCH_ROOT=<dir containing .venv/lib>.
root = Path(os.environ.get("COOP_PATCH_ROOT", "/app"))
candidates = sorted(
(root / ".venv/lib").glob("python3*/site-packages/litellm/proxy/auth/route_checks.py")
)
if not candidates:
print("WARN passthrough-key-access: route_checks.py not found under /app/.venv/lib — skipping")
return 0
target = candidates[-1]
src = target.read_text()
if MARKER in src:
print("passthrough-key-access: already applied")
return 0
found = src.count(OLD)
if found != 1:
print(f"WARN passthrough-key-access: expected exactly 1 anchor, found {found} — skipping")
return 0
patched = src.replace(OLD, NEW)
try:
compile(patched, str(target), "exec")
except SyntaxError as e:
print(f"WARN passthrough-key-access: patched source does not compile ({e}) — skipping")
return 0
target.write_text(patched)
print(f"[coop-patch] applied: passthrough-key-access -> {target}")
return 0
if __name__ == "__main__":
sys.exit(main())
-229
View File
@@ -1,229 +0,0 @@
"""Price a generic pass-through route from the usage it reports in its OWN body.
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
if the target file or the anchor is missing — e.g. after an upstream version bump
— it prints a warning and exits 0, so the build and the gateway still succeed.
The build log line ``[coop-patch] applied`` is what confirms it took effect;
check it on every LiteLLM upgrade (see README, upgrade checklist).
Why: config-declared ``pass_through_endpoints`` with ``auth: true`` are logged
through ``PassThroughEndpointLogging``. A target whose hostname matches none of
vertex / anthropic / openai / gemini resolves to ``EndpointType.GENERIC`` and
takes the generic branch of ``pass_through_async_success_handler`` — it is
logged, but LiteLLM has no model on the route to price, so the only cost it
knows is the flat ``cost_per_request`` from the config entry. That is a whole-
request estimate used for budget accounting, not an invoice: Metis is billed at
$0.12 per 1M *input* tokens and returns the true count in ``usage.input_tokens``,
so the flat figure is off by whatever factor the call differed from the estimate.
LiteLLM's own supported mechanism for an unpriced pass-through is the
``x-litellm-response-cost`` / ``x-litellm-total-tokens`` response-header
contract (``litellm/proxy/pass_through_endpoints/upstream_usage_headers.py``),
but that is consumed inside this SAME logging path, so it only helps where the
logging path already runs — and GreenPT does not send those headers. Reading the
body, which is already in hand here, needs no extra process in front of the
provider (a sidecar would be a single point of failure for the route itself, not
just for its metering).
Patch: make ``PassThroughEndpointLogging._set_cost_per_request`` price a route
listed in ``general_settings.coop_passthrough_cost_per_token`` (USD per input
token, keyed by the gateway route path, or by the upstream target) from the
``usage.input_tokens`` in the response body:
cost = input_tokens * coop_passthrough_cost_per_token[route]
The rate lives in ``general_settings`` rather than the endpoint dict because the
endpoint model (``PassThroughGenericEndpoint``) drops unknown fields and the
exact-route registration copies only ``cost_per_request`` into the logging
payload — the ``general_settings`` dict is read straight from the proxy module,
the same way ``patch_passthrough_key_access.py`` reads
``coop_passthrough_open_to_keys``.
TOKEN COUNTS. Pricing the row from the body is not enough: the row still
recorded ``prompt_tokens=0``, ``completion_tokens=0``, ``total_tokens=0``,
because the pass-through success handler hands the spend writer a
``StandardPassThroughResponseObject`` whose text carries the body but whose
``usage`` is empty, and the generic pass-through branch builds no
``standard_logging_object`` usage either. Any per-token reporting (panel,
dashboard, energy) that sums tokens for this route then under-reports. So the
patch also carries the count onto the logging object as
``combined_usage_object`` — the same ``litellm.Usage`` override the streaming
handler uses to recover partial usage. The spend writer reads
``combined_usage_object`` (``spend_tracking_utils.get_logging_payload`` and
``StandardLoggingPayloadSetup.get_usage_as_dict``) before it falls back to the
response object, so the row records ``prompt_tokens=input_tokens``,
``completion_tokens=0``, ``total_tokens=input_tokens`` while spend stays exactly
``input_tokens × rate``.
The exact figure wins; ``cost_per_request`` stays as the fallback for a listed
route whose body carries no usable input-token count, and for every route not
listed. An upstream that reports its own totals via the x-litellm-* headers still
wins outright (the early return is left intact). Inert unless an operator lists
a route, so it changes nothing for the chat models or the audio paths.
Upstream: no issue filed; LiteLLM has no per-token pricing for pass-through
routes whose body is not an OpenAI/Anthropic/Gemini shape (as of 1.103.2).
"""
from pathlib import Path
import os
import sys
MARKER = "coop_passthrough_cost_per_token"
OLD = (
" #########################################################\n"
" # Check if cost per request is set\n"
" #########################################################\n"
" if has_upstream_reported_usage(logging_obj):\n"
" return kwargs\n"
"\n"
" if passthrough_logging_payload.get(\"cost_per_request\") is not None:\n"
" kwargs[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n"
" logging_obj.model_call_details[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n"
"\n"
" return kwargs\n"
)
NEW = (
" #########################################################\n"
" # Check if cost per request is set\n"
" #########################################################\n"
" if has_upstream_reported_usage(logging_obj):\n"
" return kwargs\n"
"\n"
" # [coop-patch] Exact per-input-token pricing for a generic\n"
" # pass-through route. A route the operator lists in\n"
" # general_settings.coop_passthrough_cost_per_token (USD per input\n"
" # token, keyed by gateway path or upstream target) is priced from the\n"
" # usage it reports in its OWN response body, so spend is the real\n"
" # token count rather than cost_per_request's flat estimate. GreenPT's\n"
" # Metis decision route (/v1/systemone) is the first such route: it\n"
" # returns usage.input_tokens and always 0 output tokens, billed at\n"
" # $0.12 per 1M input tokens and recorded on the spend row with the\n"
" # same count. Inert unless a route is listed; a listed\n"
" # route whose body carries no token count falls through to\n"
" # cost_per_request below, and every unlisted route is unchanged.\n"
" try:\n"
" from litellm.proxy.proxy_server import general_settings as _coop_general_settings\n"
"\n"
" _coop_rates = (_coop_general_settings or {}).get(\"coop_passthrough_cost_per_token\") or {}\n"
" if _coop_rates:\n"
" _coop_candidates = set()\n"
" _coop_psr = ((kwargs.get(\"litellm_params\") or {}).get(\"proxy_server_request\") or {})\n"
" _coop_req_url = _coop_psr.get(\"url\")\n"
" if _coop_req_url:\n"
" _coop_candidates.add(str(_coop_req_url))\n"
" _coop_candidates.add(urlparse(str(_coop_req_url)).path)\n"
" _coop_upstream_url = passthrough_logging_payload.get(\"url\")\n"
" if _coop_upstream_url:\n"
" _coop_candidates.add(str(_coop_upstream_url))\n"
" _coop_candidates.add(urlparse(str(_coop_upstream_url)).path)\n"
"\n"
" _coop_rate = None\n"
" for _coop_route in _coop_candidates:\n"
" if _coop_route in _coop_rates:\n"
" _coop_rate = _coop_rates[_coop_route]\n"
" break\n"
" if _coop_rate is None:\n"
" for _coop_key, _coop_val in _coop_rates.items():\n"
" _coop_prefix = str(_coop_key).rstrip(\"/\")\n"
" if any(\n"
" c == _coop_key or (c and c.startswith(_coop_prefix + \"/\"))\n"
" for c in _coop_candidates\n"
" ):\n"
" _coop_rate = _coop_val\n"
" break\n"
"\n"
" _coop_tokens = 0\n"
" _coop_body = passthrough_logging_payload.get(\"response_body\")\n"
" if isinstance(_coop_body, dict):\n"
" _coop_usage = _coop_body.get(\"usage\")\n"
" if isinstance(_coop_usage, dict):\n"
" _coop_tokens = _coop_usage.get(\"input_tokens\") or 0\n"
" if not _coop_tokens:\n"
" _coop_tokens = _coop_body.get(\"input_tokens\") or 0\n"
"\n"
" if _coop_rate is not None and _coop_tokens:\n"
" _coop_cost = float(_coop_tokens) * float(_coop_rate)\n"
" kwargs[\"response_cost\"] = _coop_cost\n"
" logging_obj.model_call_details[\"response_cost\"] = _coop_cost\n"
" # [coop-patch] Record the true token counts on the row too.\n"
" # The spend writer reads usage from combined_usage_object\n"
" # (the override the streaming handler already uses for\n"
" # recovered partial usage); without it the generic\n"
" # pass-through response object carries no usage and the\n"
" # row's prompt/completion/total tokens stay 0.\n"
" _coop_usage = None\n"
" try:\n"
" from litellm.types.utils import Usage as _CoopUsage\n"
"\n"
" _coop_usage = _CoopUsage(\n"
" prompt_tokens=int(_coop_tokens),\n"
" completion_tokens=0,\n"
" total_tokens=int(_coop_tokens),\n"
" )\n"
" except Exception as _coop_usage_exc:\n"
" print(\n"
" \"[coop-passthrough-cost] usage object failed: %s: %s\"\n"
" % (type(_coop_usage_exc).__name__, _coop_usage_exc),\n"
" flush=True,\n"
" )\n"
" if _coop_usage is not None:\n"
" kwargs[\"combined_usage_object\"] = _coop_usage\n"
" logging_obj.model_call_details[\"combined_usage_object\"] = _coop_usage\n"
" print(\n"
" \"[coop-passthrough-cost] input_tokens=%s rate=%s -> response_cost=%s\"\n"
" % (_coop_tokens, _coop_rate, _coop_cost),\n"
" flush=True,\n"
" )\n"
" return kwargs\n"
" except Exception as _coop_exc:\n"
" print(\n"
" \"[coop-passthrough-cost] exact cost failed: %s: %s\"\n"
" % (type(_coop_exc).__name__, _coop_exc),\n"
" flush=True,\n"
" )\n"
"\n"
" if passthrough_logging_payload.get(\"cost_per_request\") is not None:\n"
" kwargs[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n"
" logging_obj.model_call_details[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n"
"\n"
" return kwargs\n"
)
def main() -> int:
# ROOT override (house style) lets the same script run against a pulled copy
# locally: COOP_PATCH_ROOT=<dir containing .venv/lib>.
root = Path(os.environ.get("COOP_PATCH_ROOT", "/app"))
candidates = sorted(
(root / ".venv/lib").glob("python3*/site-packages/litellm/proxy/pass_through_endpoints/success_handler.py")
)
if not candidates:
print("WARN passthrough-token-cost: success_handler.py not found under /app/.venv/lib — skipping")
return 0
target = candidates[-1]
src = target.read_text()
if MARKER in src:
print("passthrough-token-cost: already applied")
return 0
if src.count(OLD) != 1:
print(f"WARN passthrough-token-cost: expected exactly 1 anchor, found {src.count(OLD)} — skipping")
return 0
patched = src.replace(OLD, NEW, 1)
try:
compile(patched, str(target), "exec")
except SyntaxError as e:
print(f"WARN passthrough-token-cost: patched source does not compile ({e}) — skipping")
return 0
target.write_text(patched)
print(f"[coop-patch] applied: passthrough-token-cost -> {target}")
return 0
if __name__ == "__main__":
sys.exit(main())
-75
View File
@@ -1,75 +0,0 @@
"""Stop LiteLLM from forwarding the provider's response headers to callers.
SECURITY (2026-10-27, member-disclosure finding 4): LiteLLM's proxy copies every
upstream response header onto its own response as `llm_provider-<name>` — see
litellm_core_utils/llm_response_utils/get_headers.py::_get_llm_provider_headers,
whose docstring reads "Forward all headers as llm_provider-{header}". Verified
live on the co-op gateway: one chat completion returned 27 such headers, among
them `llm_provider-x-credits-remaining: 16.52` — the co-op's remaining balance
with the provider, readable by any member API key, and (read across calls) its
burn rate.
This patch makes _get_llm_provider_headers() return {} so no provider header is
copied. get_response_headers() still forwards the four OpenAI-style x-ratelimit-*
headers WHEN a provider sends them; those describe the caller's own request
rather than the co-op's account, and no upstream in our config sends them today.
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal, per
house rule: if upstream moves or rewrites the function, WARN and exit 0 — a
cosmetic leak must never keep the gateway from booting. Regression check: the
daily litellm watcher asserts the gateway returns no `llm_provider-*` header.
"""
import glob
import os
import sys
ROOT = os.environ.get("COOP_PATCH_ROOT", "/app")
TARGET = "/litellm/litellm_core_utils/llm_response_utils/get_headers.py"
ANCHOR = ''' for k, v in response_headers.items():
if "llm_provider" not in k:
_key = "{}-{}".format("llm_provider", k)
llm_provider_headers[_key] = v
else:
llm_provider_headers[k] = v
return llm_provider_headers'''
REPLACEMENT = ''' # CO-OP PATCH (2026-10-27): do NOT forward provider headers to callers.
# Upstream prefixes EVERY response header with "llm_provider-", which leaked
# the co-op's remaining provider balance (llm_provider-x-credits-remaining)
# to any member API key. Delete this patch from the Dockerfile to revert.
return {}'''
candidates = glob.glob(f"{ROOT}/.venv/lib/python*/site-packages{TARGET}")
if not candidates:
candidates = glob.glob(f"{ROOT}/**{TARGET}", recursive=True)
patched_any = False
for p in candidates:
src = open(p).read()
if "CO-OP PATCH (2026-10-27)" in src:
print(f"INFO: provider-header strip already present in {p}")
patched_any = True
continue
if ANCHOR not in src:
print(f"WARN: anchor not found in {p} (upstream changed it); "
"provider headers are still forwarded (non-fatal)", file=sys.stderr)
continue
new = src.replace(ANCHOR, REPLACEMENT, 1)
try:
compile(new, p, "exec")
except SyntaxError as e:
print(f"WARN: patched source would not compile ({e}); not written "
"(non-fatal)", file=sys.stderr)
continue
with open(p, "w") as f:
f.write(new)
print("[coop-patch] applied: provider response headers are no longer "
f"forwarded to callers ({p})")
patched_any = True
if not candidates:
print("WARN: get_headers.py not found; provider headers are still forwarded "
"(non-fatal)", file=sys.stderr)
elif not patched_any:
print("WARN: no copy of get_headers.py patched (non-fatal)", file=sys.stderr)
sys.exit(0)
-111
View File
@@ -1,111 +0,0 @@
"""Skip LiteLLM's forced `verbose_json` on transcriptions for opting-out models.
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
if the target file or the anchor is missing — e.g. after a version bump — it
prints a warning and exits 0, so the build and the gateway still succeed. Look
for the ``[coop-patch]`` line in the build log.
Why: for every `openai/` transcription deployment, LiteLLM's transformation
injects ``response_format: verbose_json`` when the caller omits the field
("ensures 'duration' is received - used for cost calculation"). Tinfoil's
``voxtral-small-24b`` rejects verbose_json with a 400 ("Currently do not
support verbose_json for voxtral-small-24b"), so every bare
``/v1/audio/transcriptions`` call to it failed — including the shape members
actually use (file + model, no response_format), and Open WebUI dictation's
request shape. The injection exists to obtain a duration for per-SECOND
pricing; voxtral-small-24b is billed per token (chat route) / per request
(patch_audio_request_cost.py), so it does not need the duration at all.
Patch: honour a ``coop_no_verbose_json`` flag on the deployment's model_info
when it is reachable, and gate the model name explicitly — the flag's plumbing
into the transformation's litellm_params is not guaranteed on every path, the
name gate is. whisper-large-v3-turbo (which Tinfoil serves verbose_json for,
and whose dictation path depends on the injected format) is untouched.
Scope: only transcriptions that omit response_format AND match the opt-out;
every other request keeps upstream behavior exactly.
Upstream: LiteLLM has no opt-out for this injection as of 1.103.2.
"""
from __future__ import annotations
import glob
import os
import sys
MARKER = "CO-OP PATCH (skip verbose_json, 2026-11-06)"
ANCHOR = ''' if "response_format" not in data:
data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation'''
INSERT = ''' if "response_format" not in data:
# CO-OP PATCH (skip verbose_json, 2026-11-06): a deployment whose
# model_info declares coop_no_verbose_json opts out of the forced
# verbose_json injection — Tinfoil's voxtral-small-24b rejects
# verbose_json with a 400, and its per-token / per-request billing
# does not need the duration this injection exists to obtain. The
# model-name gate backs the flag up: the flag's plumbing into this
# method's litellm_params is not guaranteed on every call path.
_mi = (litellm_params or {}).get("model_info") or {}
_coop_skip = (isinstance(_mi, dict) and bool(_mi.get("coop_no_verbose_json"))) or (
"voxtral-small" in str(model or "")
)
if not _coop_skip:
data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation'''
def target_path() -> str:
if len(sys.argv) > 1:
return sys.argv[1]
env = os.environ.get("LITELLM_WHISPER_TRANSFORMATION")
if env:
return env
hits = []
try: # most reliable: ask the installed package where it lives
import litellm # noqa: PLC0415
_file = getattr(litellm, "__file__", None)
if _file:
hits.append(os.path.join(
os.path.dirname(_file), "llms", "openai", "transcriptions", "whisper_transformation.py"))
except Exception:
pass
hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py")
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py")
hits = [h for h in hits if os.path.exists(h)]
if not hits:
raise SystemExit("[coop-patch] could not locate llms/openai/transcriptions/whisper_transformation.py")
return sorted(hits)[0]
def main() -> int:
path = target_path()
try:
source = open(path).read()
except OSError as exc:
print(f"[coop-patch] cannot read {path}: {exc}")
return 0
if MARKER in source:
print(f"[coop-patch] {path} already patched, nothing to do")
return 0
if ANCHOR not in source:
print(f"[coop-patch] WARNING anchor not found in {path}; bare transcription calls to "
f"opt-out models (voxtral-small-24b) will fail with the provider's verbose_json 400")
return 0
patched = source.replace(ANCHOR, INSERT, 1)
compile(patched, path, "exec") # refuse to write something that will not import
if not os.access(path, os.W_OK): # practice run against a read-only copy
print(f"[coop-patch] {path} is not writable; verified anchor and syntax only")
return 0
with open(path, "w") as handle:
handle.write(patched)
print(f"[coop-patch] applied skip-verbose_json to {path}")
return 0
if __name__ == "__main__":
sys.exit(main())
+11 -82
View File
@@ -6,18 +6,6 @@ set -eu
# Configures LiteLLM with Cloudron's PostgreSQL and Redis addons
# ============================================================================
# --- Cloudron scheduler tasks arrive as ARGUMENTS to this script ---
# The manifest's scheduler commands run as `docker run <image> <command>`, and
# this script is the image ENTRYPOINT — so the task command lands in "$@".
# Without this branch every scheduled task booted a second tinfoil-proxy and
# LiteLLM (whose ports are already bound in the app's network namespace) and
# died on the bind FATAL before doing its job — which looked exactly like a
# scheduler that never fired. Exec the task directly instead of booting.
if [ "$#" -gt 0 ]; then
echo "start.sh: scheduler-task mode — exec: $*"
exec "$@"
fi
echo "=== LiteLLM Cloudron Startup ==="
# --- Cloudron PostgreSQL addon provides:
@@ -89,13 +77,10 @@ export LITELLM_MIGRATION_DIR="/app/data/migrations"
export UI_USERNAME="admin"
export UI_PASSWORD="${LITELLM_MASTER_KEY}"
# Nothing disables the static /ui admin bundle on the pinned v1.103.2 (there
# is no flag for it): DISABLE_ADMIN_UI="True" only turns off the UI's SSO
# login-redirect routes upstream — GET /ui/ still serves the static login
# page. Logging into it still requires the master key, and the docs surfaces
# below ARE fully disabled (NO_DOCS/NO_REDOC/NO_OPENAPI → 404s). See README
# "Admin access" for the wire-verified state and the patch needed to remove
# the bundle itself. Kept set because the SSO-redirect disable is still real.
# DISABLE the admin UI and API docs. The gateway must stay publicly reachable
# for the /v1 API (member keys), but we don't want to advertise a login-gated
# admin console (/ui) or LiteLLM's Swagger/ReDoc/openapi surfaces. Admin
# visibility comes from the admin dashboard / the API directly.
export DISABLE_ADMIN_UI="True"
export NO_DOCS="True"
export NO_REDOC="True"
@@ -104,19 +89,13 @@ export NO_OPENAPI="True"
# Proxy settings for Cloudron's reverse proxy
export LITELLM_PROXY_BASE_URL="https://${CLOUDRON_APP_DOMAIN}"
# CORS: LiteLLM reads LITELLM_CORS_ORIGINS natively (upstream replaced the
# hardcoded `origins = ["*"]` with this env var in v1.84.0). The value is
# deliberately OVERRIDABLE — a Cloudron app env now wins, where before this
# export silently overrode it and made the app env look effective when it was not.
#
# The co-op decided to allow all origins (co-op/support#38, agreed on Loomio and
# by a member): CORS is not an auth boundary here. Every request carries a
# per-member bearer key, which the browser never attaches implicitly, so an origin
# list buys nothing against a party that already holds a key — and a leaked key is
# curl-able regardless. What actually constrains abuse is the per-key budget,
# model scoping and revocation. The fallback keeps the old chat-only lock if the
# env var is ever unset.
export LITELLM_CORS_ORIGINS="${LITELLM_CORS_ORIGINS:-https://chat.inference.coop}"
# CORS: LiteLLM v1.84.0 reads LITELLM_CORS_ORIGINS natively (upstream added
# the env var in place of the old hardcoded `origins = ["*"]`). Lock browser
# access to the co-op's own chat origin. The gateway is called
# server-to-server (chat → portal → gateway) and by member API keys
# (curl/SDKs, not browsers), so no wildcard is needed. A `*` here would let
# any website's JS hit the gateway.
export LITELLM_CORS_ORIGINS="https://chat.inference.coop"
echo "PostgreSQL: ${CLOUDRON_POSTGRESQL_HOST}:${CLOUDRON_POSTGRESQL_PORT}"
echo "Redis: ${CLOUDRON_REDIS_HOST}:${CLOUDRON_REDIS_PORT}"
@@ -171,39 +150,6 @@ YAML
echo "Config written to $CONFIG_FILE"
fi
# --- Ship config updates with the image ---
# /app/data is a persistent volume, so the copy the Dockerfile bakes into
# /app/data/config.yaml is shadowed by the volume's file forever — config
# changes in the repo would never reach a running gateway. The repo is the
# source of truth (README: repos mirror production), so sync the shipped copy
# over the volume whenever they differ.
if [ -f /app/code/config.yaml ] && ! cmp -s /app/code/config.yaml "$CONFIG_FILE"; then
echo "Updating /app/data/config.yaml from the image (repo is the source of truth)"
cp /app/code/config.yaml "$CONFIG_FILE"
fi
# NOTE on the fallback config written above: it can never be SERVED. The very
# same boot's sync above overwrites it with the image's (repo) config whenever
# they differ — including the boot right after the fallback ran. It is dead
# by construction; it exists only so the container is never left with an
# empty config file if the volume copy vanishes. Historically this block
# carried stale api.tinfoil.sh models that could have resurrected — that
# cannot happen through this path, and the shipped /app/code/config.yaml is
# the live catalog.
# --- Fail-closed config validation (2026-11-11) ---
# Validate the config that will actually be SERVED, before starting anything.
# A bad key must fail LOUD at the top of the boot log (the chat outage of
# 2026-10-31 showed what silent misconfiguration costs). The -f guard keeps a
# missing validator non-fatal (build patch did not apply); an INVALID config
# is fatal by design.
if [ -f /app/code/validate_config.py ]; then
echo "Validating gateway config (fail-closed)..."
if ! python3 /app/code/validate_config.py litellm "$CONFIG_FILE"; then
echo "FATAL: gateway config.yaml is invalid - refusing to start. Fix or remove the offending key named above." >&2
exit 1
fi
fi
# --- Start LiteLLM ---
echo "Starting LiteLLM on port 4000..."
@@ -253,21 +199,4 @@ if [ "$proxy_up" != "1" ]; then
fi
echo "Tinfoil proxy is up on 127.0.0.1:3301"
# --- In-container audio keep-alive loop (the ONLY warm path) ---
# A separate scheduler-task container was removed 2026-11-09: its container
# could not be proven to reach 127.0.0.1:4000 and /app/data, and two warm
# paths billed twice for the same ping. The warm path is owned HERE — the
# app container, where the network namespace, the master key and the
# sidecar are all proven. Every 15 minutes ping tinfoil/voxtral-tts
# ($0.002/request) so voice mode skips the ~24s Tinfoil cold-start after
# idle. audio_keepalive.py is silent on success and prints on failure, so
# /app/data/keepalive.log only ever grows on failure.
(
while true; do
python3 /app/code/audio_keepalive.py >> /app/data/keepalive.log 2>&1 || true
sleep 900
done
) &
echo "Audio keep-alive loop started (every 900s)"
exec litellm --config "$CONFIG_FILE" --port 4000 --host 0.0.0.0
-261
View File
@@ -1,261 +0,0 @@
#!/usr/bin/env python3
"""Fail-closed config validator for the co-op's gateway and chat apps.
ONE script, two modes, shared home in ops-docs/scripts/ and shipped
byte-identical into both app images (litellm repo root, librechat-custom repo
root). Each app's start path calls it before booting the service; a non-zero
exit refuses the boot with a ONE-line error naming the offending key.
Why this exists: the 2026-10-31 chat outage. A `customFooter:` top-level key in
librechat.yaml was rejected by LibreChat's strict config validator and the API
server refused to start, while the SPA kept being served with HTTP 200 — the
app looked up but every API route returned the HTML shell. Nothing had ever
checked the config before boot. This validator closes that gap: invalid config
fails LOUD and EARLY, at the top of the boot log, naming the key.
Modes
-----
litellm Structural check of the gateway's config.yaml:
- top-level keys within LiteLLM's documented config surface;
- model_list present and well-formed (model_name + litellm_params.model);
- the settings blocks are objects; pass_through_endpoints entries have paths.
When the gateway's own litellm package is importable it is additionally
used to resolve every model string to a provider (catches unknown
provider prefixes); the check is skipped with a note where litellm
itself is absent.
librechat Checks librechat.yaml against the DEPLOYED build's own strict schema:
it runs `configSchema.strict().safeParse()` from the container's
librechat-data-provider via node — the same code the API server runs
at boot (its failure mode is `process.exit(1)`, "Exiting due to
invalid configuration"). Unrecognized top-level keys are rejected,
named explicitly. Where node/the package cannot be found (e.g. this
script run from a checkout instead of the image) it falls back to a
static top-level allowlist recorded from the deployed build and says
so on the failure line.
Usage
-----
python3 validate_config.py litellm /app/data/config.yaml
python3 validate_config.py librechat /app/data/librechat.yaml
Exit codes: 0 valid, 1 invalid, 2 usage/environment error (file unreadable,
yaml unparseable). Never prints a value from the config — only key names and
counts, so it is safe to run against files that carry secrets.
"""
import json
import os
import subprocess
import sys
# LiteLLM's documented top-level config surface (proxy config). Unknown keys
# are rejected here — the gateway should never silently ignore a mistyped
# section (a "model-list:" typo would otherwise deploy an EMPTY catalog).
LITELLM_TOP_LEVEL = {
"model_list",
"router_settings",
"litellm_settings",
"general_settings",
"settings",
"environment_variables",
"guardrails",
"files",
}
# Fallback allowlist for librechat mode, recorded 2026-11-11 from the DEPLOYED
# build (chat.inference.coop, overlay 1.4.8) by reading
# configSchema.strict().shape keys. The live schema is always preferred; this
# exists so the validator still fails closed when run outside the container.
LIBRECHAT_TOP_LEVEL_FALLBACK = {
"actions", "balance", "cache", "cloudfront", "endpoints", "fileConfig",
"fileStrategies", "fileStrategy", "filteredTools", "filters",
"imageOutputType", "includedTools", "interface", "langfuse", "mcpServers",
"mcpSettings", "memory", "messageFilter", "modelSpecs", "ocr", "openapi",
"permissions", "rateLimits", "registration", "secureImageLinks",
"skillSync", "speech", "summarization", "transactions", "turnstile",
"version", "webSearch",
}
LC_DATA_PROVIDER_DIST = os.environ.get(
"LC_DATA_PROVIDER_DIST",
"/app/code/librechat/packages/data-provider/dist/index.js")
LC_JS_YAML = os.environ.get(
"LC_JS_YAML", "/app/code/librechat/node_modules/js-yaml")
def fail(key, reason):
"""ONE line, naming the offending key, then a non-zero exit."""
print(f"CONFIG VALIDATION FAILED: {key}: {reason}", flush=True)
sys.exit(1)
def load_yaml(path):
try:
import yaml
except ImportError:
print("CONFIG VALIDATION FAILED: pyyaml not available in this python",
flush=True)
sys.exit(2)
try:
with open(path) as fh:
data = yaml.safe_load(fh)
except FileNotFoundError:
print(f"CONFIG VALIDATION FAILED: config file not found: {path}",
flush=True)
sys.exit(2)
except PermissionError:
print(f"CONFIG VALIDATION FAILED: config file unreadable: {path}",
flush=True)
sys.exit(2)
except yaml.YAMLError as exc:
# Do not echo the raw yaml error: it can quote a secret-bearing line.
print("CONFIG VALIDATION FAILED: (root): yaml does not parse",
flush=True)
sys.exit(2)
if data is None:
data = {}
if not isinstance(data, dict):
fail("(root)", "top level must be a mapping")
return data
def validate_litellm(path):
cfg = load_yaml(path)
unknown = sorted(set(cfg) - LITELLM_TOP_LEVEL)
if unknown:
fail(", ".join(unknown), "unknown top-level key(s) for litellm "
f"(allowed: {', '.join(sorted(LITELLM_TOP_LEVEL))})")
model_list = cfg.get("model_list")
if not isinstance(model_list, list) or not model_list:
fail("model_list", "must be a non-empty list of model entries")
names = set()
for i, entry in enumerate(model_list):
if not isinstance(entry, dict):
fail(f"model_list[{i}]", "entry must be a mapping")
name = entry.get("model_name")
params = entry.get("litellm_params")
if not name or not isinstance(name, str):
fail(f"model_list[{i}].model_name", "missing or empty")
if name in names:
fail(f"model_list[{i}].model_name", f"duplicate model_name {name!r}")
names.add(name)
if not isinstance(params, dict) or not params.get("model"):
fail(f"model_list[{i}].litellm_params.model", "missing or empty")
for block in ("router_settings", "litellm_settings", "general_settings"):
if block in cfg and not isinstance(cfg[block], dict):
fail(block, "must be a mapping")
for ep in (cfg.get("general_settings") or {}).get("pass_through_endpoints") or []:
if not isinstance(ep, dict) or not ep.get("path"):
fail("general_settings.pass_through_endpoints", "each entry needs a path")
# litellm-assisted check (best effort): resolve each model string to a
# provider with the gateway's own package. Skipped where litellm itself
# is not importable (never fatal — the structural checks above ran).
try:
from litellm import get_llm_provider # noqa: WPS433
except Exception:
print("note: litellm not importable here; provider check skipped",
flush=True)
else:
for entry in model_list:
model = entry["litellm_params"]["model"]
try:
get_llm_provider(model=model, custom_llm_provider=None)
except Exception as exc:
short = str(exc).split("\n")[0][:120]
fail(f"model_list.{entry['model_name']}.litellm_params.model",
f"not resolvable to a provider: {short}")
print(f"config OK (litellm): {len(cfg)} top-level key(s), "
f"{len(model_list)} model(s) validated", flush=True)
sys.exit(0)
def validate_librechat(path):
cfg = load_yaml(path)
if not cfg:
print("config OK (librechat): empty config, nothing to validate",
flush=True)
sys.exit(0)
# Preferred: the DEPLOYED build's own strict schema, run through node —
# the exact code path the API server uses at boot (strict() rejects
# unrecognized top-level keys; upstream then exits 1).
node_code = r"""
const fs = require('fs');
let out = {mode: 'schema'};
try {
const yaml = require(JS_YAML_PATH);
const dp = require(DIST_PATH);
const cfg = yaml.load(fs.readFileSync(CONFIG_PATH, 'utf8')) || {};
const result = dp.configSchema.strict().safeParse(cfg);
if (result.success) {
out.ok = true;
out.keys = Object.keys(cfg).length;
} else {
out.ok = false;
out.errors = (result.error && result.error.issues || []).slice(0, 10)
.map(i => ({path: (i.path || []).join('.') || '(root)', msg: i.message}));
}
} catch (e) {
out.mode = 'schema-unavailable';
out.error = String(e && e.message || e).slice(0, 200);
}
console.log('@@' + JSON.stringify(out));
""".replace("JS_YAML_PATH", json.dumps(LC_JS_YAML)) \
.replace("DIST_PATH", json.dumps(LC_DATA_PROVIDER_DIST)) \
.replace("CONFIG_PATH", json.dumps(path))
if os.path.exists(LC_DATA_PROVIDER_DIST):
try:
proc = subprocess.run(["node", "-e", node_code], capture_output=True,
text=True, timeout=60)
for line in (proc.stdout or "").splitlines():
if line.startswith("@@"):
res = json.loads(line[2:])
if res.get("ok"):
print(f"config OK (librechat): {res['keys']} top-level "
"key(s) accepted by the deployed strict schema",
flush=True)
sys.exit(0)
errs = "; ".join(
f"{e['path']}: {e['msg']}" for e in res.get("errors", []))
if errs:
fail(errs, "rejected by the deployed build's strict "
"config schema (configSchema.strict())")
fail("(root)", "strict schema rejected the config")
# No @@ line: schema path unavailable — fall through to fallback.
print("note: deployed schema probe produced no result "
f"(stderr: {(proc.stderr or '')[:120]!r})", flush=True)
except (OSError, subprocess.TimeoutExpired) as exc:
print(f"note: could not run node against the deployed schema "
f"({type(exc).__name__})", flush=True)
else:
print("note: deployed data-provider dist not found; using the static "
"fallback allowlist recorded from the deployed build", flush=True)
# Fallback: static allowlist (still fail-closed).
unknown = sorted(set(cfg) - LIBRECHAT_TOP_LEVEL_FALLBACK)
if unknown:
fail(", ".join(unknown), "unknown top-level key(s) for librechat "
f"(fallback allowlist: {', '.join(sorted(LIBRECHAT_TOP_LEVEL_FALLBACK))})")
print(f"config OK (librechat, fallback allowlist): {len(cfg)} top-level "
"key(s)", flush=True)
sys.exit(0)
def main():
args = sys.argv[1:]
if len(args) != 2 or args[0] not in ("litellm", "librechat"):
print("usage: validate_config.py <litellm|librechat> <config.yaml>",
file=sys.stderr, flush=True)
sys.exit(2)
if args[0] == "litellm":
validate_litellm(args[1])
validate_librechat(args[1])
if __name__ == "__main__":
main()