Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c877b61ef3 |
No files matched your search
@@ -1,2 +0,0 @@
|
|||||||
__pycache__/
|
|
||||||
*.pyc
|
|
||||||
@@ -5,13 +5,13 @@
|
|||||||
"description": "LiteLLM AI Gateway \u2014 OpenAI-compatible proxy with per-user API keys, usage tracking, rate limiting, and model routing. Packaged for Cloudron with SSO/OIDC and PostgreSQL.",
|
"description": "LiteLLM AI Gateway \u2014 OpenAI-compatible proxy with per-user API keys, usage tracking, rate limiting, and model routing. Packaged for Cloudron with SSO/OIDC and PostgreSQL.",
|
||||||
"tagline": "AI gateway for cooperative inference",
|
"tagline": "AI gateway for cooperative inference",
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"upstreamVersion": "1.108.3",
|
"upstreamVersion": "1.103.2",
|
||||||
"healthCheckPath": "/health/readiness",
|
"healthCheckPath": "/health/readiness",
|
||||||
"httpPort": 4000,
|
"httpPort": 4000,
|
||||||
"memoryLimit": 2147483648,
|
"memoryLimit": 2147483648,
|
||||||
"manifestVersion": 2,
|
"manifestVersion": 2,
|
||||||
"website": "https://litellm.ai",
|
"website": "https://litellm.ai",
|
||||||
"contactEmail": "info@inference.coop",
|
"contactEmail": "botbot@hermes",
|
||||||
"icon": "file://logo.png",
|
"icon": "file://logo.png",
|
||||||
"addons": {
|
"addons": {
|
||||||
"postgresql": {},
|
"postgresql": {},
|
||||||
|
|||||||
+1
-70
@@ -1,4 +1,4 @@
|
|||||||
FROM ghcr.io/berriai/litellm:v1.108.3
|
FROM ghcr.io/berriai/litellm:v1.103.2
|
||||||
|
|
||||||
# v1.84.0 is the fix line for CVE-2026-35029 (auth bypass on
|
# v1.84.0 is the fix line for CVE-2026-35029 (auth bypass on
|
||||||
# /config/update, fixed 1.83.0) and CVE-2026-59822 (MCP session auth
|
# /config/update, fixed 1.83.0) and CVE-2026-59822 (MCP session auth
|
||||||
@@ -19,9 +19,6 @@ RUN chmod +x /app/code/start.sh
|
|||||||
|
|
||||||
# Default config — will be overridden by Cloudron env vars at runtime
|
# Default config — will be overridden by Cloudron env vars at runtime
|
||||||
COPY config.yaml /app/data/config.yaml
|
COPY config.yaml /app/data/config.yaml
|
||||||
# A second copy inside the image: start.sh syncs it over the persistent
|
|
||||||
# /app/data volume at boot, so config changes in the repo actually deploy.
|
|
||||||
COPY config.yaml /app/code/config.yaml
|
|
||||||
|
|
||||||
# Append the co-op credit-pack hint to LiteLLM's hardcoded budget-exceeded
|
# Append the co-op credit-pack hint to LiteLLM's hardcoded budget-exceeded
|
||||||
# messages. Done at BUILD time — the runtime rootfs is read-only under
|
# messages. Done at BUILD time — the runtime rootfs is read-only under
|
||||||
@@ -38,77 +35,11 @@ RUN python3 /app/code/patch_budget_message.py
|
|||||||
COPY patch_image_cost.py /app/code/patch_image_cost.py
|
COPY patch_image_cost.py /app/code/patch_image_cost.py
|
||||||
RUN python3 /app/code/patch_image_cost.py
|
RUN python3 /app/code/patch_image_cost.py
|
||||||
|
|
||||||
# Audio per-request pricing: Tinfoil bills voxtral-tts and whisper per request,
|
|
||||||
# which LiteLLM cannot express on the audio paths, so those two were billed at
|
|
||||||
# $0. The patch makes the cost calculator honour a declared
|
|
||||||
# input_cost_per_request. Check for the "[coop-patch] applied" line in the build
|
|
||||||
# log after upgrading LiteLLM: the script is non-fatal and warns instead.
|
|
||||||
COPY patch_audio_request_cost.py /app/code/patch_audio_request_cost.py
|
|
||||||
RUN python3 /app/code/patch_audio_request_cost.py
|
|
||||||
|
|
||||||
# The audio cost paths read pricing from the SHARED cost-map key, which the
|
|
||||||
# router strips per-deployment pricing from — so the lookup raised, the spend
|
|
||||||
# write was skipped, and audio produced no SpendLogs row at all. This re-adds the
|
|
||||||
# pricing (and input_cost_per_request) for the audio modes. Same mechanism as
|
|
||||||
# patch_image_cost.py above.
|
|
||||||
COPY patch_audio_router_pricing.py /app/code/patch_audio_router_pricing.py
|
|
||||||
RUN python3 /app/code/patch_audio_router_pricing.py
|
|
||||||
|
|
||||||
# SECURITY (2026-10-27, member-disclosure finding 4): LiteLLM forwards every
|
|
||||||
# provider response header to callers as `llm_provider-*`, which leaked the
|
|
||||||
# co-op's remaining provider balance (x-credits-remaining) to any member API key.
|
|
||||||
# Verify the "[coop-patch] applied: provider response headers" line in the build
|
|
||||||
# output on every version bump — the patch is non-fatal by design.
|
|
||||||
COPY patch_strip_provider_headers.py /app/code/patch_strip_provider_headers.py
|
|
||||||
RUN python3 /app/code/patch_strip_provider_headers.py
|
|
||||||
|
|
||||||
# Expose the per-request cost headers to cross-origin browser clients (co-op/support#39).
|
|
||||||
COPY patch_expose_cost_headers.py /app/code/patch_expose_cost_headers.py
|
|
||||||
RUN python3 /app/code/patch_expose_cost_headers.py
|
|
||||||
|
|
||||||
# Member access to config-declared pass-through routes: LiteLLM gates auth=true
|
|
||||||
# pass-through endpoints behind `allowed_passthrough_routes`, an Enterprise-only
|
|
||||||
# field that cannot be set on this community-edition proxy. This patch lets any
|
|
||||||
# authenticated key call routes the operator lists in
|
|
||||||
# general_settings.coop_passthrough_open_to_keys (unauthenticated access is
|
|
||||||
# unaffected — still rejected). Non-fatal; check the "[coop-patch] applied" line
|
|
||||||
# in the build log on every LiteLLM upgrade.
|
|
||||||
COPY patch_passthrough_key_access.py /app/code/patch_passthrough_key_access.py
|
|
||||||
RUN python3 /app/code/patch_passthrough_key_access.py
|
|
||||||
|
|
||||||
# Exact per-input-token pricing for generic pass-through routes. A route whose
|
|
||||||
# target matches none of vertex/anthropic/openai/gemini resolves to
|
|
||||||
# EndpointType.GENERIC: it IS logged, but LiteLLM has no model to price it from,
|
|
||||||
# so the only cost it knows is the flat cost_per_request. Metis (/v1/systemone)
|
|
||||||
# is billed at $0.12 per 1M input tokens and returns usage.input_tokens, so this
|
|
||||||
# patch prices a route listed in general_settings.coop_passthrough_cost_per_token
|
|
||||||
# from that count and writes the exact figure to member spend. Non-fatal; check
|
|
||||||
# the "[coop-patch] applied" line in the build log on every LiteLLM upgrade.
|
|
||||||
COPY patch_passthrough_token_cost.py /app/code/patch_passthrough_token_cost.py
|
|
||||||
RUN python3 /app/code/patch_passthrough_token_cost.py
|
|
||||||
|
|
||||||
# Audio keep-alive: scheduled pings keep the Tinfoil TEE audio enclave
|
# Audio keep-alive: scheduled pings keep the Tinfoil TEE audio enclave
|
||||||
# warm so voice mode skips the ~24s cold-start (see audio_keepalive.py header).
|
# warm so voice mode skips the ~24s cold-start (see audio_keepalive.py header).
|
||||||
COPY audio_keepalive.py /app/code/audio_keepalive.py
|
COPY audio_keepalive.py /app/code/audio_keepalive.py
|
||||||
RUN chmod +x /app/code/audio_keepalive.py
|
RUN chmod +x /app/code/audio_keepalive.py
|
||||||
|
|
||||||
# Skip LiteLLM's forced verbose_json on transcription requests for opting-out
|
|
||||||
# models: Tinfoil's voxtral-small-24b rejects verbose_json with a 400, which
|
|
||||||
# broke every bare /v1/audio/transcriptions call to it (file + model — the
|
|
||||||
# shape members and dictation send). Its billing does not need the duration
|
|
||||||
# the injection exists for. Non-fatal; check the "[coop-patch] applied" line
|
|
||||||
# in the build log on every LiteLLM upgrade.
|
|
||||||
COPY patch_transcription_response_format.py /app/code/patch_transcription_response_format.py
|
|
||||||
RUN python3 /app/code/patch_transcription_response_format.py
|
|
||||||
|
|
||||||
# TTS default response format: LibreChat sends no response_format on its TTS
|
|
||||||
# request and Tinfoil's voxtral-tts then serves WAV (~8-17s warm measured) —
|
|
||||||
# while LibreChat labels the reply audio/mpeg regardless. Default to mp3
|
|
||||||
# (OpenAI's documented default, ~3s warm measured). Non-fatal; check the
|
|
||||||
# "[coop-patch] applied" line in the build log on every LiteLLM upgrade.
|
|
||||||
COPY patch_audio_default_mp3.py /app/code/patch_audio_default_mp3.py
|
|
||||||
RUN python3 /app/code/patch_audio_default_mp3.py
|
|
||||||
|
|
||||||
# Override the base image's ENTRYPOINT (which is `litellm`) so our
|
# Override the base image's ENTRYPOINT (which is `litellm`) so our
|
||||||
# startup script runs instead of being passed as an argument to litellm.
|
# startup script runs instead of being passed as an argument to litellm.
|
||||||
ENTRYPOINT ["/app/code/start.sh"]
|
ENTRYPOINT ["/app/code/start.sh"]
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ A Cloudron app package for [LiteLLM](https://litellm.ai/), the open-source AI ga
|
|||||||
|
|
||||||
LiteLLM is the single gateway between members and the inference backends. It handles per-member API keys, usage metering, rate limiting (via budget caps), and model routing — all behind one OpenAI-compatible endpoint.
|
LiteLLM is the single gateway between members and the inference backends. It handles per-member API keys, usage metering, rate limiting (via budget caps), and model routing — all behind one OpenAI-compatible endpoint.
|
||||||
|
|
||||||
**Current base image:** LiteLLM **v1.108.3**, pinned to the exact tag (upgraded from v1.103.2 in the 2026-11-08/09 maintenance window; minimum supported line remains v1.84.0, the fix line for CVE-2026-35029 / CVE-2026-59822, CISA KEV-listed). See [Updating LiteLLM](#updating-litellm).
|
**Current base image:** LiteLLM **v1.103.2**, pinned to the exact tag (fix line for CVE-2026-35029 / CVE-2026-59822, CISA KEV-listed; also carries the streaming-usage fix for prompt_tokens_details). See [Updating LiteLLM](#updating-litellm).
|
||||||
|
|
||||||
## What this provides
|
## What this provides
|
||||||
|
|
||||||
@@ -16,16 +16,16 @@ LiteLLM is the single gateway between members and the inference backends. It han
|
|||||||
- **Cloudron Redis** for rate limiting and caching
|
- **Cloudron Redis** for rate limiting and caching
|
||||||
- **Automatic SSL, backups, and sandboxing** (Cloudron-managed)
|
- **Automatic SSL, backups, and sandboxing** (Cloudron-managed)
|
||||||
- **Master key + salt key** auto-generated on first start, persisted in `/app/data`
|
- **Master key + salt key** auto-generated on first start, persisted in `/app/data`
|
||||||
- Hardened for the public internet: LiteLLM's built-in API docs surfaces are disabled (`NO_DOCS`/`NO_REDOC`/`NO_OPENAPI` → live 404s). The static `/ui/` admin-UI bundle **is served** on this version (see *Admin access*); logging into it still requires the master key. CORS is deliberately open (`LITELLM_CORS_ORIGINS=*`, co-op/support#38) — auth is the per-member bearer key, not origin checking.
|
- Hardened for the public internet: admin UI and API docs disabled at the proxy; CORS locked to the chat origin
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
```
|
```
|
||||||
Members → LibreChat (chat) / own tools (API) → member portal (key injection)
|
Members → Open WebUI (chat) / own tools (API) → member portal (key injection)
|
||||||
→ LiteLLM gateway (this app)
|
→ LiteLLM gateway (this app)
|
||||||
├→ Tinfoil (TEE enclaves — architectural privacy)
|
├→ Tinfoil (TEE enclaves — architectural privacy)
|
||||||
├→ GreenPT (100% renewable energy)
|
├→ GreenPT (100% renewable energy)
|
||||||
└→ Regolo (EU-hosted, renewable + zero data retention)
|
└→ PublicAI (publicly developed, sovereign models)
|
||||||
```
|
```
|
||||||
|
|
||||||
LiteLLM is the control plane (keys, metering, routing). It does **not** run models itself — it forwards to providers, each chosen by members for a different value. A [tinfoil-proxy](https://github.com/tinfoilsh/tinfoil-proxy) sidecar provides EHBP end-to-end encryption to Tinfoil's enclaves.
|
LiteLLM is the control plane (keys, metering, routing). It does **not** run models itself — it forwards to providers, each chosen by members for a different value. A [tinfoil-proxy](https://github.com/tinfoilsh/tinfoil-proxy) sidecar provides EHBP end-to-end encryption to Tinfoil's enclaves.
|
||||||
@@ -76,7 +76,7 @@ After install, set the provider API keys in Cloudron → app → Settings → En
|
|||||||
|
|
||||||
- `TINFOIL_API_KEY` — Tinfoil (https://tinfoil.sh/)
|
- `TINFOIL_API_KEY` — Tinfoil (https://tinfoil.sh/)
|
||||||
- `GREENPT_API_KEY` — GreenPT (https://greenpt.ai/)
|
- `GREENPT_API_KEY` — GreenPT (https://greenpt.ai/)
|
||||||
- `REGOLO_API_KEY` — Regolo (regolo.ai; replaced PublicAI, removed 2026-10-16)
|
- `PUBLICAI_API_KEY` — PublicAI (https://publicai.co/)
|
||||||
|
|
||||||
The following are auto-generated and should **not** be set manually:
|
The following are auto-generated and should **not** be set manually:
|
||||||
|
|
||||||
@@ -85,36 +85,9 @@ The following are auto-generated and should **not** be set manually:
|
|||||||
- `DATABASE_URL` — from Cloudron's PostgreSQL addon
|
- `DATABASE_URL` — from Cloudron's PostgreSQL addon
|
||||||
- `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD` — from Cloudron's Redis addon
|
- `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD` — from Cloudron's Redis addon
|
||||||
|
|
||||||
### Config deployment
|
|
||||||
|
|
||||||
`config.yaml` ships **with the image** and `start.sh` syncs it over the persistent
|
|
||||||
`/app/data/config.yaml` at boot whenever they differ. The volume copy would
|
|
||||||
otherwise shadow the image's file forever — which is exactly how a config change
|
|
||||||
can be merged, built and "deployed" while the gateway keeps serving the old
|
|
||||||
configuration. Change the repo and redeploy; never edit the volume copy directly.
|
|
||||||
|
|
||||||
### Admin access
|
### Admin access
|
||||||
|
|
||||||
The state of the admin surfaces on the pinned v1.103.2, wire-verified 2026-11-05:
|
The LiteLLM admin UI and API docs are **disabled** on the public gateway (hardened by default; see `start.sh`). Administrative operations run through the [member portal](https://git.inference.coop/code/member-portal), which holds the master key server-side. If direct LiteLLM admin access is ever needed, re-enable the UI in `start.sh`, redeploy, and use the master key from `/app/data/.master_key` — then disable it again.
|
||||||
|
|
||||||
- The built-in **API docs are disabled** (`NO_DOCS`/`NO_REDOC`/`NO_OPENAPI` in
|
|
||||||
`start.sh`) — `/docs`, `/redoc` and `/openapi.json` return 404.
|
|
||||||
- The **static admin UI bundle is still served at `/ui/`** — upstream's
|
|
||||||
`DISABLE_ADMIN_UI` env (which we set) only disables the UI's *SSO login
|
|
||||||
redirect* routes in this version; nothing on v1.103.2 stops the unconditional
|
|
||||||
`app.mount("/ui", …)`. So `GET /ui/` serves the login page. **Logging in
|
|
||||||
still requires the master key** (auto-generated here, never published), and
|
|
||||||
administrative operations normally run through the
|
|
||||||
[member portal](https://git.inference.coop/code/member-portal), which holds
|
|
||||||
the master key server-side.
|
|
||||||
- If the login surface itself should disappear, it needs a build-time patch
|
|
||||||
(strip the `/ui` mount from `proxy_server` in the Dockerfile, the same
|
|
||||||
non-fatal patch pattern the other 8 build patches use) — there is no env/flag
|
|
||||||
for it on this version. Until then treat `/ui/` as a master-key-only
|
|
||||||
endpoint, like any other control-plane route.
|
|
||||||
|
|
||||||
If direct LiteLLM admin access is ever needed, log in at `/ui/` with the
|
|
||||||
master key from `/app/data/.master_key` and lock it away again afterwards.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -149,19 +122,6 @@ expect a 15–20 min maintenance window and **never panic-restart mid-migration*
|
|||||||
|
|
||||||
1. **Pick a target version** from the [LiteLLM changelog](https://github.com/BerriAI/litellm/releases). Stay at or above v1.84.0 (the fix line for CVE-2026-35029 and CVE-2026-59822, the latter on CISA's KEV list).
|
1. **Pick a target version** from the [LiteLLM changelog](https://github.com/BerriAI/litellm/releases). Stay at or above v1.84.0 (the fix line for CVE-2026-35029 and CVE-2026-59822, the latter on CISA's KEV list).
|
||||||
|
|
||||||
**Re-validate the build patches against the target's real sources first** — the
|
|
||||||
scripts fail soft (a moved anchor prints a WARN and exits 0), so a silently
|
|
||||||
skipped patch ships. Ten patches are in the Dockerfile today; each must print
|
|
||||||
its `applied` line in the build log. The cheapest rehearsal is to unpack the
|
|
||||||
target wheel into a scratch tree and run the scripts against it (2026-11-09,
|
|
||||||
v1.108.3: all ten apply — the CORS anchor needs the 1.108.x fallback already
|
|
||||||
carried by `patch_expose_cost_headers.py`). Two gotchas in that harness:
|
|
||||||
`patch_audio_request_cost.py` finds its target by `import litellm`, which fails
|
|
||||||
outside a full install — point `LITELLM_COST_CALCULATOR` at the scratch
|
|
||||||
`cost_calculator.py` instead, or it exits with "could not locate"; and running
|
|
||||||
the scripts by hand leaves files owned by your user, which is fine in scratch
|
|
||||||
and never in the image.
|
|
||||||
|
|
||||||
2. **Bump the pin** in `Dockerfile`:
|
2. **Bump the pin** in `Dockerfile`:
|
||||||
|
|
||||||
```dockerfile
|
```dockerfile
|
||||||
@@ -191,17 +151,13 @@ expect a 15–20 min maintenance window and **never panic-restart mid-migration*
|
|||||||
2. Generate a temp key with the master key → if `/key/generate` 500s, the schema is incomplete; fix the migration before anything else
|
2. Generate a temp key with the master key → if `/key/generate` 500s, the schema is incomplete; fix the migration before anything else
|
||||||
3. Use the **temp key** (not master) for `GET /v1/models` → full catalog
|
3. Use the **temp key** (not master) for `GET /v1/models` → full catalog
|
||||||
4. Use the **temp key** for a small `POST /v1/chat/completions` → content + usage
|
4. Use the **temp key** for a small `POST /v1/chat/completions` → content + usage
|
||||||
5. OPTIONS preflight: `LITELLM_CORS_ORIGINS` is set at the app level (`*` by co-op decision, co-op/support#38) — a preflight from any origin answers with the matching `access-control-allow-origin` header rather than a 400. Auth is the bearer key, not the origin.
|
5. OPTIONS preflight from a foreign origin → 400; from `chat.inference.coop` → allowed
|
||||||
6. Cloudron app health = healthy, then delete the temp key
|
6. Cloudron app health = healthy, then delete the temp key
|
||||||
|
|
||||||
### Before updating
|
### Before updating
|
||||||
|
|
||||||
- Check the [LiteLLM changelog](https://github.com/BerriAI/litellm/releases) for breaking changes
|
- Check the [LiteLLM changelog](https://github.com/BerriAI/litellm/releases) for breaking changes
|
||||||
- The `start.sh` and `config.yaml` may need adjustment if LiteLLM changes its env-var or config schema
|
- The `start.sh` and `config.yaml` may need adjustment if LiteLLM changes its env-var or config schema
|
||||||
- **Build-time patches ride on LiteLLM internals — verify each one after an upgrade.** `patch_budget_message.py`, `patch_image_cost.py` (keeps image-gen pricing on the shared cost-map key) and `patch_audio_request_cost.py` (adds the declared per-request charge on the audio paths: Tinfoil bills `voxtral-tts` $0.002 and `whisper-large-v3-turbo` $0.01 per request, which the cost map cannot express). Each prints `[coop-patch]` in the build log. They are non-fatal by design: a missing anchor warns and the build still succeeds, so the behaviour can vanish silently on a version bump. Related: code/litellm#5.
|
|
||||||
- **`patch_passthrough_token_cost.py`** prices a generic pass-through route from the token count it reports in its own response body. A `pass_through_endpoints` target whose host matches none of vertex/anthropic/openai/gemini resolves to `EndpointType.GENERIC`: it is logged, but with no model on the route LiteLLM only knows the flat `cost_per_request`, which is a budget estimate, not an invoice. The patch bills any route listed in `general_settings.coop_passthrough_cost_per_token` (USD per input token, keyed by path) as `usage.input_tokens × rate` — Metis (`/v1/systemone`) at $0.12 per 1M input tokens — and records the same count on the row (see below). Same rules: non-fatal, `[coop-patch]` in the build log, verify after every upgrade. `cost_per_request` stays as the fallback for a listed route whose response carries no token count. The block prints `[coop-passthrough-cost] input_tokens=… rate=… -> response_cost=…` on each priced call, so the log is the field check that it fired.
|
|
||||||
- **Pass-through spend rows also carry the token counts (Metis defect fix).** Pricing the row from the body was not enough: the row recorded the correct `spend` but `prompt_tokens`/`completion_tokens`/`total_tokens` stayed `0`, because the pass-through success handler hands the spend writer a `StandardPassThroughResponseObject` whose `usage` is empty and the generic branch builds no `standard_logging_object` usage. Anything summing tokens for the route (panel, dashboard, energy reporting) then under-reported. The same patch now sets `combined_usage_object` (a `litellm.Usage`) on the logging object — the override the streaming handler already uses for recovered partial usage — which the spend writer reads before the response object (`spend_tracking_utils.get_logging_payload`, `StandardLoggingPayloadSetup.get_usage_as_dict`). The row now records `prompt_tokens = usage.input_tokens`, `completion_tokens = 0`, `total_tokens = usage.input_tokens`, with spend unchanged.
|
|
||||||
- **A zero-spend row with `call_type = /v1/<path>` is NOT a duplicate of the priced row — it is the 401 refusal.** Every *rejected* pass-through request (missing/invalid key) is logged by upstream LiteLLM as a failure row whose `call_type` is the route path and whose `spend` is `0`, via `ProxyLogging._handle_logging_proxy_only_error` (`litellm/proxy/utils.py`, `original_function=route`). That is upstream behaviour on the pristine image, not something our patches add (neither `patch_passthrough_key_access.py` nor `patch_passthrough_token_cost.py` writes a row). A single *authenticated* call produces exactly **one** row (`call_type=pass_through_endpoint`, the real spend). Measured: one 200 → one row; one unauthenticated 401 → one failure row. Count requests for the route by `call_type = pass_through_endpoint`, not by summing all rows carrying the model name.
|
|
||||||
- The CORS env var is `LITELLM_CORS_ORIGINS` (upstream-native since 1.84.0); the old `LITELLM_CORS_ALLOWED_ORIGINS` + sed patch are dead — don't re-add them
|
- The CORS env var is `LITELLM_CORS_ORIGINS` (upstream-native since 1.84.0); the old `LITELLM_CORS_ALLOWED_ORIGINS` + sed patch are dead — don't re-add them
|
||||||
- Test on a staging instance if the version jump is large
|
- Test on a staging instance if the version jump is large
|
||||||
|
|
||||||
@@ -213,13 +169,13 @@ expect a 15–20 min maintenance window and **never panic-restart mid-migration*
|
|||||||
|----------|----------|---------|
|
|----------|----------|---------|
|
||||||
| `TINFOIL_API_KEY` | Yes | Tinfoil provider (TEE inference) |
|
| `TINFOIL_API_KEY` | Yes | Tinfoil provider (TEE inference) |
|
||||||
| `GREENPT_API_KEY` | Yes | GreenPT provider (renewable energy) |
|
| `GREENPT_API_KEY` | Yes | GreenPT provider (renewable energy) |
|
||||||
| `REGOLO_API_KEY` | Yes | Regolo provider (EU, renewable + ZDR) |
|
| `PUBLICAI_API_KEY` | Yes | PublicAI provider (public models) |
|
||||||
| `LITELLM_CORS_ORIGINS` | Set at app level | Currently `*` by co-op decision (co-op/support#38); `start.sh` falls back to the chat origin only if the app env omits it |
|
| `LITELLM_CORS_ORIGINS` | Set by start.sh | CORS allowlist (chat origin only) |
|
||||||
| `LITELLM_MIGRATION_DIR` | Set by start.sh | Persistent Prisma migrations ledger |
|
| `LITELLM_MIGRATION_DIR` | Set by start.sh | Persistent Prisma migrations ledger |
|
||||||
|
|
||||||
## Connecting chat clients
|
## Connecting chat clients
|
||||||
|
|
||||||
LibreChat (chat) does **not** talk to this gateway directly — it routes through the [member portal](https://git.inference.coop/code/member-portal), which authenticates the member and injects their personal LiteLLM key per request. Members building their own tools create API keys on the [member dashboard](https://dashboard.inference.coop) and use:
|
Open WebUI does **not** talk to this gateway directly — it routes through the [member portal](https://git.inference.coop/code/member-portal), which authenticates the member and injects their personal LiteLLM key per request. Members building their own tools create API keys on the [member dashboard](https://dashboard.inference.coop) and use:
|
||||||
|
|
||||||
- **API Base URL:** `https://gateway.inference.coop/v1` (or the portal's proxy endpoint for chat-integrated tools)
|
- **API Base URL:** `https://gateway.inference.coop/v1` (or the portal's proxy endpoint for chat-integrated tools)
|
||||||
- **API Key:** their own virtual key from the dashboard
|
- **API Key:** their own virtual key from the dashboard
|
||||||
|
|||||||
+25
-172
@@ -10,11 +10,6 @@
|
|||||||
# tinfoil/deepseek-v4-1-flash: $0.65 in / $1.45 out / $0.13 cached per 1M
|
# tinfoil/deepseek-v4-1-flash: $0.65 in / $1.45 out / $0.13 cached per 1M
|
||||||
# tinfoil/gpt-oss-120b: $0.15 in / $0.60 out per 1M
|
# tinfoil/gpt-oss-120b: $0.15 in / $0.60 out per 1M
|
||||||
# tinfoil/glm-5.3: $1.80 in / $5.75 out / $0.45 cached per 1M
|
# tinfoil/glm-5.3: $1.80 in / $5.75 out / $0.45 cached per 1M
|
||||||
# greenpt/qwen3-embedding-8b: $0.25 in / $0 out per 1M (embeddings, 32K ctx)
|
|
||||||
# greenpt/green-embeddings: $0.20 in / $0 out per 1M (embeddings, 32K ctx)
|
|
||||||
# tinfoil/nomic-embed-text: $0.016 in / $0 out per 1M (embeddings, 8K ctx, TEE)
|
|
||||||
# tinfoil/voxtral-small-24b: $0.20 in / $0.60 out per 1M (token-priced audio;
|
|
||||||
# +$0.00008/request on /v1/audio/transcriptions)
|
|
||||||
# Without these, LiteLLM can't price these custom models, so spend shows $0.00
|
# Without these, LiteLLM can't price these custom models, so spend shows $0.00
|
||||||
# and the team budget cap is not enforced.
|
# and the team budget cap is not enforced.
|
||||||
#
|
#
|
||||||
@@ -125,8 +120,8 @@ model_list:
|
|||||||
api_key: os.environ/REGOLO_API_KEY
|
api_key: os.environ/REGOLO_API_KEY
|
||||||
model_info:
|
model_info:
|
||||||
mode: chat
|
mode: chat
|
||||||
input_cost_per_token: 1.16e-6
|
input_cost_per_token: 4.6e-7
|
||||||
output_cost_per_token: 3.944e-6
|
output_cost_per_token: 2.44e-6
|
||||||
|
|
||||||
- model_name: regolo/qwen3.5-122b
|
- model_name: regolo/qwen3.5-122b
|
||||||
litellm_params:
|
litellm_params:
|
||||||
@@ -136,8 +131,8 @@ model_list:
|
|||||||
chat_template_kwargs: {"enable_thinking": false}
|
chat_template_kwargs: {"enable_thinking": false}
|
||||||
model_info:
|
model_info:
|
||||||
mode: chat
|
mode: chat
|
||||||
input_cost_per_token: 6.96e-7
|
input_cost_per_token: 1.16e-6
|
||||||
output_cost_per_token: 3.944e-6
|
output_cost_per_token: 4.87e-6
|
||||||
|
|
||||||
- model_name: regolo/qwen-image
|
- model_name: regolo/qwen-image
|
||||||
litellm_params:
|
litellm_params:
|
||||||
@@ -146,23 +141,20 @@ model_list:
|
|||||||
api_key: os.environ/REGOLO_API_KEY
|
api_key: os.environ/REGOLO_API_KEY
|
||||||
model_info:
|
model_info:
|
||||||
mode: image_generation
|
mode: image_generation
|
||||||
input_cost_per_pixel: 1.74e-8
|
input_cost_per_pixel: 1.5e-8
|
||||||
|
|
||||||
# --- Audio models (Tinfoil, in-TEE) ---
|
# --- Audio models (Tinfoil, in-TEE) ---
|
||||||
# Used for chat dictation (STT) and voice mode (TTS). These are NOT chat
|
# Used for chat dictation (STT) and voice mode (TTS). These are NOT chat
|
||||||
# models: the member portal's /v1/models filters them out of the chat model
|
# models: the member portal's /v1/models filters them out of the chat model
|
||||||
# list (see AUDIO_MODEL_PREFIXES there) so members don't see them in the
|
# list (see AUDIO_MODEL_PREFIXES there) so members don't see them in the
|
||||||
# picker, but they remain fully callable via /v1/audio/* for API users.
|
# picker, but they remain fully callable via /v1/audio/* for API users.
|
||||||
# whisper-large-v3-turbo: $0/1M tokens + $0.01 per request.
|
# whisper-large-v3-turbo: $0.05/1M input tokens. voxtral-tts: Tinfoil's
|
||||||
# voxtral-tts: $0/1M tokens + $0.002 per request.
|
# catalog lists it $0/$0 and every cost field was left NULL, so speech usage
|
||||||
# voxtral-small-24b: $0.20/1M in + $0.60/1M out tokens, $0 per request —
|
# never consumed member allowance. Billed at the model's published rate
|
||||||
# the only TOKEN-PRICED audio model in their catalog (its block below).
|
# instead: $16/1M characters, i.e. LiteLLM's own bundled price for the same
|
||||||
# Tinfoil bills these per REQUEST (its model list carries a `requestPrice`);
|
# weights (mistral/voxtral-mini-tts-*, sourced to Mistral's model card) and
|
||||||
# reading only the two token prices made TTS look free and left STT billed off
|
# the rate OpenAI's TTS entries use for the same field. Revisit if the
|
||||||
# an older catalog. LiteLLM cannot price per request on the audio paths, so
|
# Tinfoil invoice shows a different number.
|
||||||
# patch_audio_request_cost.py (built in via the Dockerfile) makes the cost
|
|
||||||
# calculator honour `input_cost_per_request` below. Re-check that build log
|
|
||||||
# line after every LiteLLM upgrade.
|
|
||||||
# NOTE: LiteLLM routes audio by the openai/ provider + the endpoint called
|
# NOTE: LiteLLM routes audio by the openai/ provider + the endpoint called
|
||||||
# (/v1/audio/transcriptions or /v1/audio/speech) — there is no
|
# (/v1/audio/transcriptions or /v1/audio/speech) — there is no
|
||||||
# "audio_transcription/" provider prefix.
|
# "audio_transcription/" provider prefix.
|
||||||
@@ -173,38 +165,7 @@ model_list:
|
|||||||
api_key: os.environ/TINFOIL_API_KEY
|
api_key: os.environ/TINFOIL_API_KEY
|
||||||
model_info:
|
model_info:
|
||||||
mode: audio_transcription
|
mode: audio_transcription
|
||||||
# Tinfoil: $0 per token, $0.01 per request. The per-request charge is the
|
input_cost_per_token: 0.00000005
|
||||||
# whole cost, and the patch is what makes it billable.
|
|
||||||
input_cost_per_token: 0
|
|
||||||
input_cost_per_request: 0.01
|
|
||||||
|
|
||||||
# voxtral-small-24b — Tinfoil's TOKEN-PRICED STT model: the only audio entry
|
|
||||||
# in their catalog with real token prices ($0.20 in / $0.60 out per 1M, $0
|
|
||||||
# per request). The chat route (audio sent as input_audio) returns real
|
|
||||||
# token usage and bills exactly at those rates. The /v1/audio/transcriptions
|
|
||||||
# route returns NO token counts (duration only), so per-token billing cannot
|
|
||||||
# fire there; Tinfoil tokenizes audio in ~30s chunks (~375-380 tokens each —
|
|
||||||
# measured: a 1-second clip and a 6-second clip both cost ~380 prompt
|
|
||||||
# tokens), so for the short dictation requests this route serves the
|
|
||||||
# chunk-equivalent is ~$0.00008/request — declared as input_cost_per_request
|
|
||||||
# below, added by patch_audio_request_cost.py, which is scoped to the audio
|
|
||||||
# call types so chat calls to this model are NOT double-billed. Audio longer
|
|
||||||
# than ~30s should go through the chat route, which meters exactly.
|
|
||||||
- model_name: tinfoil/voxtral-small-24b
|
|
||||||
litellm_params:
|
|
||||||
model: openai/voxtral-small-24b
|
|
||||||
api_base: http://127.0.0.1:3301/v1
|
|
||||||
api_key: os.environ/TINFOIL_API_KEY
|
|
||||||
model_info:
|
|
||||||
mode: audio_transcription
|
|
||||||
# Opt out of LiteLLM's forced response_format=verbose_json injection on
|
|
||||||
# transcription calls (patch_transcription_response_format.py): Tinfoil
|
|
||||||
# rejects verbose_json for this model with a 400, and nothing in its
|
|
||||||
# billing needs the duration the injection exists to obtain.
|
|
||||||
coop_no_verbose_json: true
|
|
||||||
input_cost_per_token: 0.0000002
|
|
||||||
output_cost_per_token: 0.0000006
|
|
||||||
input_cost_per_request: 0.00008
|
|
||||||
|
|
||||||
- model_name: tinfoil/voxtral-tts
|
- model_name: tinfoil/voxtral-tts
|
||||||
litellm_params:
|
litellm_params:
|
||||||
@@ -213,29 +174,18 @@ model_list:
|
|||||||
api_key: os.environ/TINFOIL_API_KEY
|
api_key: os.environ/TINFOIL_API_KEY
|
||||||
model_info:
|
model_info:
|
||||||
mode: audio_speech
|
mode: audio_speech
|
||||||
# Tinfoil: $0 per token, $0.002 per request — see the audio section note.
|
# $16 per 1M characters. This is the field LiteLLM reads for speech on the
|
||||||
#
|
# openai adapter (same key OpenAI's tts-1 uses upstream), so member spend
|
||||||
# input_cost_per_character: 0 is load-bearing, not decoration. The TTS cost
|
# and budget caps now account for voice mode.
|
||||||
# branch (cost_per_token, cost_metric == "cost_per_character") raises
|
input_cost_per_character: 0.000016
|
||||||
# ValueError("cost for tts call is None") when there is no character price,
|
|
||||||
# and that raise aborts the whole cost calculation — so the spend write is
|
|
||||||
# skipped and speech bills nothing, whatever input_cost_per_request says.
|
|
||||||
# Declaring the zero lets the path complete; the patch then adds the
|
|
||||||
# per-request charge.
|
|
||||||
input_cost_per_character: 0
|
|
||||||
input_cost_per_request: 0.002
|
|
||||||
|
|
||||||
# --- Embedding models (Tinfoil, in-TEE; and GreenPT) ---
|
# --- Embedding model (Tinfoil, in-TEE) ---
|
||||||
# nomic-embed-text — 768-dim text embeddings for RAG/semantic search.
|
# nomic-embed-text — 768-dim text embeddings for RAG/semantic search.
|
||||||
# NOT a chat model: it powers /v1/embeddings only. MEMBER-FACING since
|
# NOT a chat model: it powers /v1/embeddings only. Scoped to the admin
|
||||||
# 2026-11-06 — the team_id scoping (operator-account-only) was removed so
|
# account's team (model_info.team_id) so member keys cannot call it — it is
|
||||||
# members get a TEE-protected embedding option. This is the same model the
|
# an operator resource for the co-op's own docs-RAG, not a member-facing
|
||||||
# co-op's own docs-RAG runs on inside the TEE (Open WebUI's knowledge base);
|
# model. The portal also filters it out of the chat-facing /v1/models list by
|
||||||
# that traffic is unaffected by members also being able to call it. The
|
# its `mode: embedding`. Pricing per Tinfoil catalog: $0.05/1M input, $0 out.
|
||||||
# portal filters `mode: embedding` out of the chat-facing /v1/models list, so
|
|
||||||
# it stays API-only. Pricing per Tinfoil catalog (inference.tinfoil.sh
|
|
||||||
# /v1/models): $0.016/1M input, $0 out — corrected 2026-11-06 from a stale
|
|
||||||
# $0.05 declaration; 8K context declared so clients don't invent larger.
|
|
||||||
- model_name: tinfoil/nomic-embed-text
|
- model_name: tinfoil/nomic-embed-text
|
||||||
litellm_params:
|
litellm_params:
|
||||||
model: openai/nomic-embed-text
|
model: openai/nomic-embed-text
|
||||||
@@ -243,53 +193,10 @@ model_list:
|
|||||||
api_key: os.environ/TINFOIL_API_KEY
|
api_key: os.environ/TINFOIL_API_KEY
|
||||||
model_info:
|
model_info:
|
||||||
mode: embedding
|
mode: embedding
|
||||||
max_input_tokens: 8192
|
team_id: cdfdf422-633f-4432-8893-cffee82a8325
|
||||||
input_cost_per_token: 0.000000016
|
input_cost_per_token: 0.00000005
|
||||||
output_cost_per_token: 0.0
|
output_cost_per_token: 0.0
|
||||||
|
|
||||||
# --- Embedding model (GreenPT, member-facing) ---
|
|
||||||
# qwen3-embedding-8b — text embeddings via /v1/embeddings, callable by any
|
|
||||||
# member key. Added 2026-11-06 after a member asked for an embedding model,
|
|
||||||
# filling the member-facing gap on renewable EU hosting (Tinfoil's private
|
|
||||||
# embedding was opened to members the same day). The portal filters
|
|
||||||
# mode:embedding out of the chat picker, so it is API-only — same shape as
|
|
||||||
# the audio models above.
|
|
||||||
# Pricing per GreenPT /v1/pricing: $0.25/1M input, no output charge
|
|
||||||
# (embeddings return vectors; GreenPT lists maxOutputTokens null), 32K
|
|
||||||
# context. max_input_tokens declared from the provider's contextWindowTokens
|
|
||||||
# so clients don't invent a larger one (the null-caps defect).
|
|
||||||
- model_name: greenpt/qwen3-embedding-8b
|
|
||||||
litellm_params:
|
|
||||||
model: openai/qwen3-embedding-8b
|
|
||||||
api_base: https://api.greenpt.ai/v1
|
|
||||||
api_key: os.environ/GREENPT_API_KEY
|
|
||||||
model_info:
|
|
||||||
mode: embedding
|
|
||||||
input_cost_per_token: 0.00000025
|
|
||||||
output_cost_per_token: 0.0
|
|
||||||
max_input_tokens: 32000
|
|
||||||
|
|
||||||
# --- Embedding model (GreenPT, member-facing) ---
|
|
||||||
# green-embeddings — GreenPT's own embedding model (2560-dim, measured live).
|
|
||||||
# Added 2026-11-06 after a catalog survey: the only genuinely distinct
|
|
||||||
# embedding model on GreenPT besides qwen3-embedding-8b. Their catalog also
|
|
||||||
# lists a `green-embedding` (singular) slug — verified the SAME model
|
|
||||||
# (identical vectors on the same input), so only green-embeddings is served.
|
|
||||||
# Cheaper per token than qwen3-embedding-8b; the reranker sibling
|
|
||||||
# (green-rerank) is a different capability and is NOT served.
|
|
||||||
# Same shape: /v1/embeddings only, filtered out of the chat picker.
|
|
||||||
# Pricing per GreenPT /v1/pricing: $0.20/1M input, no output charge, 32K ctx.
|
|
||||||
- model_name: greenpt/green-embeddings
|
|
||||||
litellm_params:
|
|
||||||
model: openai/green-embeddings
|
|
||||||
api_base: https://api.greenpt.ai/v1
|
|
||||||
api_key: os.environ/GREENPT_API_KEY
|
|
||||||
model_info:
|
|
||||||
mode: embedding
|
|
||||||
input_cost_per_token: 0.0000002
|
|
||||||
output_cost_per_token: 0.0
|
|
||||||
max_input_tokens: 32000
|
|
||||||
|
|
||||||
# Fallback: if DeepSeek is down, use GPT-OSS
|
# Fallback: if DeepSeek is down, use GPT-OSS
|
||||||
# NOTE: DeepSeek V4.1 Flash is a reasoning model — generations routinely run
|
# NOTE: DeepSeek V4.1 Flash is a reasoning model — generations routinely run
|
||||||
# longer than 30s. A 30s timeout caused LiteLLM to abandon in-flight DeepSeek
|
# longer than 30s. A 30s timeout caused LiteLLM to abandon in-flight DeepSeek
|
||||||
@@ -315,60 +222,6 @@ general_settings:
|
|||||||
database_url: os.environ/DATABASE_URL
|
database_url: os.environ/DATABASE_URL
|
||||||
store_model_in_db: true
|
store_model_in_db: true
|
||||||
|
|
||||||
# --- Metis (GreenPT) — decision model, not a chat model ---
|
|
||||||
# Metis is a *decision* endpoint, not a chat completion: it takes a `state`
|
|
||||||
# plus up to 32 typed questions and returns PROBABILITIES / choices / scores.
|
|
||||||
# It generates no text (output_tokens is always 0), so it cannot be a
|
|
||||||
# model_list entry — LiteLLM's chat pipeline has nothing to parse. It is
|
|
||||||
# exposed as a pass-through route instead, and members call it with their
|
|
||||||
# normal co-op API key (auth: true), never the provider key.
|
|
||||||
#
|
|
||||||
# GreenPT-hosted (policy-based privacy, EU, 100% renewable), same
|
|
||||||
# GREENPT_API_KEY the green chat models already use.
|
|
||||||
# Price: $0.12 per 1M *input* tokens; output is free.
|
|
||||||
#
|
|
||||||
# METERING: exact, from the token count GreenPT reports back. Metis is not a
|
|
||||||
# model_list entry (LiteLLM's chat pipeline has nothing to parse), so it is
|
|
||||||
# priced through the coop_passthrough_cost_per_token map below, keyed by this
|
|
||||||
# route's path: a call's cost is usage.input_tokens × the rate. patch_
|
|
||||||
# passthrough_token_cost.py (built in via the Dockerfile) reads
|
|
||||||
# usage.input_tokens from the Metis response body at logging time and writes
|
|
||||||
# the exact figure to member spend. cost_per_request on the route is now only
|
|
||||||
# a fallback: it applies if a response ever arrives without a usable token
|
|
||||||
# count, so a call is never silently free. Keep the two in sync.
|
|
||||||
#
|
|
||||||
# Routes here (matched by prefix) may be called by ANY authenticated member
|
|
||||||
# key. LiteLLM's own gate for auth=true pass-through routes
|
|
||||||
# (`allowed_passthrough_routes`) is an Enterprise-only field, so on this
|
|
||||||
# community-edition proxy it can never be granted to member keys — the only
|
|
||||||
# other option, auth:false, would expose the route publicly and forward the
|
|
||||||
# co-op's GreenPT key to anonymous callers. This list is consumed at build
|
|
||||||
# time by patch_passthrough_key_access.py; authentication is unchanged (the
|
|
||||||
# endpoint still requires a valid key).
|
|
||||||
coop_passthrough_open_to_keys:
|
|
||||||
- "/v1/systemone"
|
|
||||||
# Exact per-input-token pricing (USD per token) for pass-through routes that
|
|
||||||
# report their own usage in the RESPONSE BODY — the shape the x-litellm-*
|
|
||||||
# response-header contract does not cover. Keyed by the gateway route path
|
|
||||||
# (an upstream target URL also matches). Any route listed here is billed from
|
|
||||||
# usage.input_tokens × the rate instead of its flat cost_per_request; a route
|
|
||||||
# not listed is unchanged. Consumed at build time by
|
|
||||||
# patch_passthrough_token_cost.py.
|
|
||||||
coop_passthrough_cost_per_token:
|
|
||||||
# Metis (GreenPT): $0.12 per 1M input tokens, output always free.
|
|
||||||
"/v1/systemone": 0.00000012
|
|
||||||
pass_through_endpoints:
|
|
||||||
- path: "/v1/systemone"
|
|
||||||
target: "https://api.greenpt.ai/v1/systemone"
|
|
||||||
headers:
|
|
||||||
Authorization: "Bearer os.environ/GREENPT_API_KEY"
|
|
||||||
Content-Type: "application/json"
|
|
||||||
auth: true
|
|
||||||
methods: ["POST"]
|
|
||||||
timeout: 60
|
|
||||||
# Fallback only — the exact token cost above is what bills in practice.
|
|
||||||
cost_per_request: 0.0001
|
|
||||||
|
|
||||||
litellm_settings:
|
litellm_settings:
|
||||||
salt_key: os.environ/LITELLM_SALT_KEY
|
salt_key: os.environ/LITELLM_SALT_KEY
|
||||||
drop_params: true
|
drop_params: true
|
||||||
|
|||||||
@@ -1,103 +0,0 @@
|
|||||||
"""Default /v1/audio/speech to mp3 when the caller omits response_format.
|
|
||||||
|
|
||||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
|
||||||
if the target file or the anchor is missing — e.g. after a version bump — it
|
|
||||||
prints a warning and exits 0, so the build and the gateway still succeed. Look
|
|
||||||
for the ``[coop-patch]`` line in the build log.
|
|
||||||
|
|
||||||
Why: LibreChat's speech client sends no ``response_format`` on its TTS request
|
|
||||||
(TTSService.openAIProvider builds {input, model, voice} — the field is not in
|
|
||||||
its schema), and Tinfoil's voxtral-tts then serves WAV. Measured 2026-11-07
|
|
||||||
through the gateway: WAV ~8-17s warm per request (145-300 KB), while the SAME
|
|
||||||
model with ``response_format: "mp3"`` answers in ~2.6-3.1s (28-39 KB) — the
|
|
||||||
provider streams/encodes mp3 progressively. LibreChat additionally labels the
|
|
||||||
reply ``Content-Type: audio/mpeg`` unconditionally, so WAV bytes were served
|
|
||||||
under an mp3 label and browsers depended on MIME sniffing to play them.
|
|
||||||
|
|
||||||
Patch: default ``response_format`` to ``mp3`` (OpenAI's own documented default
|
|
||||||
for /v1/audio/speech) when the request omits it. An explicit response_format
|
|
||||||
from any caller still wins, so API members who want WAV can keep requesting
|
|
||||||
it. Scope: only the aspeech route; transcriptions are untouched.
|
|
||||||
|
|
||||||
Upstream: LiteLLM 1.103.2 forwards response_format verbatim and leaves the
|
|
||||||
provider default in place when it is absent.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import glob
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
MARKER = "CO-OP PATCH (TTS default mp3, 2026-11-07)"
|
|
||||||
|
|
||||||
ANCHOR = ''' ### CALL HOOKS ### - modify incoming data / reject request before calling the model
|
|
||||||
data = await proxy_logging_obj.pre_call_hook(
|
|
||||||
user_api_key_dict=user_api_key_dict, data=data, call_type="aspeech"
|
|
||||||
)'''
|
|
||||||
|
|
||||||
INSERT = ANCHOR + '''
|
|
||||||
|
|
||||||
# CO-OP PATCH (TTS default mp3, 2026-11-07): LibreChat sends no
|
|
||||||
# response_format and Tinfoil's voxtral-tts then serves WAV — measured
|
|
||||||
# ~8-17s warm vs ~3s for mp3, which the provider streams. LibreChat
|
|
||||||
# labels the reply audio/mpeg regardless. Default to mp3 (OpenAI's
|
|
||||||
# documented default); an explicit response_format still wins.
|
|
||||||
if not data.get("response_format"):
|
|
||||||
data["response_format"] = "mp3"'''
|
|
||||||
|
|
||||||
|
|
||||||
def target_path() -> str:
|
|
||||||
if len(sys.argv) > 1:
|
|
||||||
return sys.argv[1]
|
|
||||||
env = os.environ.get("LITELLM_PROXY_SERVER")
|
|
||||||
if env:
|
|
||||||
return env
|
|
||||||
hits = []
|
|
||||||
try: # most reliable: ask the installed package where it lives
|
|
||||||
import litellm # noqa: PLC0415
|
|
||||||
_file = getattr(litellm, "__file__", None)
|
|
||||||
if _file:
|
|
||||||
hits.append(os.path.join(os.path.dirname(_file), "proxy", "proxy_server.py"))
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/proxy/proxy_server.py")
|
|
||||||
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/proxy/proxy_server.py")
|
|
||||||
hits = [h for h in hits if os.path.exists(h)]
|
|
||||||
if not hits:
|
|
||||||
raise SystemExit("[coop-patch] could not locate litellm/proxy/proxy_server.py")
|
|
||||||
return sorted(hits)[0]
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
path = target_path()
|
|
||||||
try:
|
|
||||||
source = open(path).read()
|
|
||||||
except OSError as exc:
|
|
||||||
print(f"[coop-patch] cannot read {path}: {exc}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
if MARKER in source:
|
|
||||||
print(f"[coop-patch] {path} already patched, nothing to do")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
count = source.count(ANCHOR)
|
|
||||||
if count != 1:
|
|
||||||
print(f"[coop-patch] WARNING aspeech pre_call_hook anchor found {count}x (need exactly 1) "
|
|
||||||
f"in {path}; TTS stays on the provider's default (WAV for Tinfoil)")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
patched = source.replace(ANCHOR, INSERT, 1)
|
|
||||||
compile(patched, path, "exec") # refuse to write something that will not import
|
|
||||||
|
|
||||||
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
|
||||||
print(f"[coop-patch] {path} is not writable; verified anchor and syntax only")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
with open(path, "w") as handle:
|
|
||||||
handle.write(patched)
|
|
||||||
print(f"[coop-patch] applied TTS default mp3 to {path}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,248 +0,0 @@
|
|||||||
"""Bill a declared per-request charge on requests LiteLLM prices only per unit.
|
|
||||||
|
|
||||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
|
||||||
if the target file or the anchor strings are missing — e.g. after an upstream
|
|
||||||
version bump — it prints a warning and exits 0, so the build and the gateway
|
|
||||||
still succeed. The build log line ``[coop-patch] applied`` is what confirms it
|
|
||||||
took effect; check it on every LiteLLM upgrade (see README, upgrade checklist).
|
|
||||||
|
|
||||||
Why: Tinfoil charges some models a flat amount PER REQUEST on top of (or instead
|
|
||||||
of) per-token pricing — ``voxtral-tts`` $0.002/request and
|
|
||||||
``whisper-large-v3-turbo`` $0.01/request, both with $0 per-token. LiteLLM can
|
|
||||||
only express per-request pricing on the search / rerank / MCP paths, via
|
|
||||||
``input_cost_per_query`` (see ``litellm/search/cost_calculator.py``, and
|
|
||||||
``litellm/types/utils.py``: "per-request pricing: rerank, search, and Bedrock
|
|
||||||
Marengo embed"). The audio paths price by character, second or token, and
|
|
||||||
``input_cost_per_request`` exists only in the proxy's cost-tracking settings —
|
|
||||||
the model cost calculator never reads it. So speech and transcription usage is
|
|
||||||
billed at $0 however the config is written.
|
|
||||||
|
|
||||||
Patch: make the model cost calculator honour a declared
|
|
||||||
``input_cost_per_request``. ``response_cost_calculator`` is the single funnel for
|
|
||||||
every call type (the proxy's audio endpoints dispatch as ``aspeech`` and
|
|
||||||
``transcription`` and reach it through ``Logging._response_cost_calculator``), so
|
|
||||||
adding the charge at its final return covers both. The declared value is resolved
|
|
||||||
from the per-deployment ``custom_pricing`` the logging layer builds out of our
|
|
||||||
``config.yaml`` ``model_info``, falling back to the shared cost map and to the
|
|
||||||
call metadata.
|
|
||||||
|
|
||||||
Scope: nothing changes unless a model declares ``input_cost_per_request``, so
|
|
||||||
this is inert for every other model on the gateway. Since 2026-11-06 the charge
|
|
||||||
is added only on the AUDIO call types (transcription/atranscription/speech/
|
|
||||||
aspeech): a model that also carries per-token pricing (``voxtral-small-24b``)
|
|
||||||
bills its chat calls exactly per token, and a per-request charge on top of
|
|
||||||
that would double-bill them. The audio endpoints are the paths that cannot
|
|
||||||
price per token (the provider returns no token counts there), so the flat
|
|
||||||
charge belongs to them alone.
|
|
||||||
|
|
||||||
Upstream: no issue filed; LiteLLM has no per-request audio pricing as of 1.103.2.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import glob
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
MARKER = "CO-OP PATCH (audio per-request cost, 2026-10-26)"
|
|
||||||
|
|
||||||
HELPER = '''def _coop_per_request_cost(
|
|
||||||
*,
|
|
||||||
model: str | None = None,
|
|
||||||
litellm_model_name: str | None = None,
|
|
||||||
custom_pricing: Any | None = None,
|
|
||||||
litellm_logging_obj: Any | None = None,
|
|
||||||
) -> float:
|
|
||||||
"""CO-OP PATCH (audio per-request cost, 2026-10-26)
|
|
||||||
|
|
||||||
Return the model's declared ``input_cost_per_request``, else 0.0.
|
|
||||||
|
|
||||||
LiteLLM cannot price a flat per-request charge on the audio paths, so
|
|
||||||
Tinfoil's per-request models (voxtral-tts, whisper-large-v3-turbo) were
|
|
||||||
billed at $0. Resolution order: the per-deployment custom pricing the
|
|
||||||
logging layer builds from our config.yaml model_info, then the shared cost
|
|
||||||
map, then the call metadata. Inert unless a model declares the field.
|
|
||||||
"""
|
|
||||||
def _is_audio(name: Any) -> bool:
|
|
||||||
text = str(name or "").lower()
|
|
||||||
return (
|
|
||||||
"tts" in text or "whisper" in text or "voxtral" in text
|
|
||||||
)
|
|
||||||
|
|
||||||
cost_map = {}
|
|
||||||
try:
|
|
||||||
import litellm as _litellm # local import: not every caller binds it globally
|
|
||||||
cost_map = getattr(_litellm, "model_cost", {}) or {}
|
|
||||||
except Exception:
|
|
||||||
cost_map = {}
|
|
||||||
|
|
||||||
candidates = []
|
|
||||||
if isinstance(custom_pricing, dict):
|
|
||||||
candidates.append(custom_pricing)
|
|
||||||
map_hits = []
|
|
||||||
for key in (litellm_model_name, model):
|
|
||||||
entry = cost_map.get(key) if key else None
|
|
||||||
if isinstance(entry, dict):
|
|
||||||
candidates.append(entry)
|
|
||||||
map_hits.append((key, entry.get("input_cost_per_request")))
|
|
||||||
try:
|
|
||||||
details = getattr(litellm_logging_obj, "model_call_details", None) or {}
|
|
||||||
info = (details.get("metadata") or {}).get("model_info")
|
|
||||||
if isinstance(info, dict):
|
|
||||||
candidates.append(info)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Diagnostic (audio models only): did this run at all, and what could it see?
|
|
||||||
# If this line never appears for a speech call, the request never reaches the
|
|
||||||
# model cost calculator and the loss is upstream of this patch.
|
|
||||||
if _is_audio(model) or _is_audio(litellm_model_name):
|
|
||||||
try:
|
|
||||||
print(
|
|
||||||
"[coop-audio] probe"
|
|
||||||
f" model={model!r} litellm={litellm_model_name!r}"
|
|
||||||
f" custom_pricing={sorted(custom_pricing) if isinstance(custom_pricing, dict) else type(custom_pricing).__name__}"
|
|
||||||
f" map_hits={map_hits!r} candidates={len(candidates)}",
|
|
||||||
flush=True,
|
|
||||||
)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
for candidate in candidates:
|
|
||||||
try:
|
|
||||||
value = candidate.get("input_cost_per_request")
|
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
if value:
|
|
||||||
try:
|
|
||||||
resolved = float(value)
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
continue
|
|
||||||
if _is_audio(model) or _is_audio(litellm_model_name):
|
|
||||||
try:
|
|
||||||
print(f"[coop-audio] resolved per_request={resolved}", flush=True)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
return resolved
|
|
||||||
if _is_audio(model) or _is_audio(litellm_model_name):
|
|
||||||
try:
|
|
||||||
print("[coop-audio] resolved per_request=0.0 (nothing declared)", flush=True)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
return 0.0
|
|
||||||
|
|
||||||
|
|
||||||
'''
|
|
||||||
|
|
||||||
ANCHOR_DEF = "def response_cost_calculator("
|
|
||||||
|
|
||||||
ANCHOR_RETURN = """ region_name=region_name,
|
|
||||||
)
|
|
||||||
return response_cost
|
|
||||||
except Exception as e:
|
|
||||||
raise e"""
|
|
||||||
|
|
||||||
PATCHED_RETURN = """ region_name=region_name,
|
|
||||||
)
|
|
||||||
|
|
||||||
# CO-OP PATCH (audio per-request cost, 2026-10-26): add the declared
|
|
||||||
# per-request charge for models LiteLLM can only price per unit
|
|
||||||
# (Tinfoil's voxtral-tts, whisper-large-v3-turbo). Inert when the model
|
|
||||||
# declares nothing; the early "provider reported the cost" return above
|
|
||||||
# is unaffected. Scoped to the AUDIO call types (2026-11-06): a model
|
|
||||||
# that ALSO carries per-token pricing (voxtral-small-24b) bills its
|
|
||||||
# chat calls exactly per token and must not take the per-request
|
|
||||||
# charge on top of that.
|
|
||||||
try:
|
|
||||||
_coop_audio_call = call_type in (
|
|
||||||
"transcription",
|
|
||||||
"atranscription",
|
|
||||||
"speech",
|
|
||||||
"aspeech",
|
|
||||||
)
|
|
||||||
_coop_pr = (
|
|
||||||
_coop_per_request_cost(
|
|
||||||
model=model,
|
|
||||||
litellm_model_name=litellm_model_name,
|
|
||||||
custom_pricing=custom_pricing,
|
|
||||||
litellm_logging_obj=litellm_logging_obj,
|
|
||||||
)
|
|
||||||
if _coop_audio_call
|
|
||||||
else 0.0
|
|
||||||
)
|
|
||||||
response_cost = (response_cost or 0.0) + _coop_pr
|
|
||||||
_coop_name = str(model or "") + str(litellm_model_name or "")
|
|
||||||
if _coop_audio_call and (
|
|
||||||
"tts" in _coop_name.lower()
|
|
||||||
or "whisper" in _coop_name.lower()
|
|
||||||
or "voxtral" in _coop_name.lower()
|
|
||||||
):
|
|
||||||
print(
|
|
||||||
f"[coop-audio] add per_request={_coop_pr!r} -> response_cost={response_cost!r}",
|
|
||||||
flush=True,
|
|
||||||
)
|
|
||||||
except Exception as _coop_exc:
|
|
||||||
print(f"[coop-audio] add failed: {type(_coop_exc).__name__}: {_coop_exc}", flush=True)
|
|
||||||
|
|
||||||
return response_cost
|
|
||||||
except Exception as e:
|
|
||||||
raise e"""
|
|
||||||
|
|
||||||
|
|
||||||
def target_path() -> str:
|
|
||||||
"""The installed litellm cost calculator (or an explicit path for testing)."""
|
|
||||||
if len(sys.argv) > 1:
|
|
||||||
return sys.argv[1]
|
|
||||||
env = os.environ.get("LITELLM_COST_CALCULATOR")
|
|
||||||
if env:
|
|
||||||
return env
|
|
||||||
hits = []
|
|
||||||
try: # most reliable: ask the installed package where it lives
|
|
||||||
import litellm # noqa: PLC0415
|
|
||||||
_file = getattr(litellm, "__file__", None)
|
|
||||||
if _file:
|
|
||||||
hits.append(os.path.join(os.path.dirname(_file), "cost_calculator.py"))
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
hits += glob.glob("/usr/lib/python3*/site-packages/litellm/cost_calculator.py")
|
|
||||||
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/cost_calculator.py")
|
|
||||||
hits += glob.glob("/usr/local/lib/python3*/dist-packages/litellm/cost_calculator.py")
|
|
||||||
hits += glob.glob("/opt/**/litellm/cost_calculator.py", recursive=True)
|
|
||||||
hits = [h for h in hits if os.path.exists(h)]
|
|
||||||
if not hits:
|
|
||||||
raise SystemExit("[coop-patch] could not locate litellm/cost_calculator.py")
|
|
||||||
return sorted(hits)[0]
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
path = target_path()
|
|
||||||
try:
|
|
||||||
source = open(path).read()
|
|
||||||
except OSError as exc:
|
|
||||||
print(f"[coop-patch] cannot read {path}: {exc}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
if MARKER in source:
|
|
||||||
print(f"[coop-patch] {path} already patched, nothing to do")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
if ANCHOR_DEF not in source or ANCHOR_RETURN not in source:
|
|
||||||
print(f"[coop-patch] WARNING anchors not found in {path}; audio per-request "
|
|
||||||
f"cost will NOT be billed (check this after a LiteLLM upgrade)")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
patched = source.replace(ANCHOR_DEF, HELPER + ANCHOR_DEF, 1)
|
|
||||||
patched = patched.replace(ANCHOR_RETURN, PATCHED_RETURN, 1)
|
|
||||||
compile(patched, path, "exec") # refuse to write something that will not import
|
|
||||||
|
|
||||||
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
|
||||||
print(f"[coop-patch] {path} is not writable; verified anchors and syntax only")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
with open(path, "w") as handle:
|
|
||||||
handle.write(patched)
|
|
||||||
print(f"[coop-patch] applied to {path}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,112 +0,0 @@
|
|||||||
"""Keep audio-model pricing on the shared cost-map key (speech + transcription).
|
|
||||||
|
|
||||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
|
||||||
if the target file or the anchor is missing — e.g. after a version bump — it
|
|
||||||
prints a warning and exits 0, so the build and the gateway still succeed. Look
|
|
||||||
for the ``[coop-patch]`` line in the build log.
|
|
||||||
|
|
||||||
Why this is needed on top of patch_audio_request_cost.py: the audio cost paths
|
|
||||||
(speech, transcription) resolve pricing from the **shared** cost-map key, and the
|
|
||||||
router deliberately strips per-deployment pricing from that key ("so one
|
|
||||||
deployment's overrides don't pollute sibling deployments"). For a deployment on a
|
|
||||||
custom backend — ours are all `openai/<model>` against a local Tinfoil proxy —
|
|
||||||
the cost lookup then raises, the spend-log write is skipped, and **no SpendLogs
|
|
||||||
row appears at all**. That is the symptom we measured: `/v1/audio/speech` and
|
|
||||||
`/v1/audio/transcriptions` returned 200 with real audio, and the member key's
|
|
||||||
spend stayed exactly $0.000000 with no spend row anywhere. It is the same
|
|
||||||
mechanism as the image-generation case that patch_image_cost.py already fixes
|
|
||||||
(upstream BerriAI/litellm#10023/#12471).
|
|
||||||
|
|
||||||
The cost-calculator patch alone cannot fix it: it only adds a per-request charge
|
|
||||||
*on top of* a cost the normal path computed, and that path never got that far.
|
|
||||||
|
|
||||||
`input_cost_per_request` is deliberately allowed through even though it is **not**
|
|
||||||
in `CustomPricingLiteLLMParams.model_fields` — LiteLLM does not list it there, and
|
|
||||||
patch_audio_request_cost.py is what makes the cost calculator read it.
|
|
||||||
|
|
||||||
Safe for us: the co-op gateway has one deployment per audio backend, so there are
|
|
||||||
no sibling deployments for the shared key to pollute.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import glob
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
MARKER = "CO-OP PATCH (audio shared pricing, 2026-10-26)"
|
|
||||||
|
|
||||||
ANCHOR = " # Always register the (possibly mode-preserved) shared backend info."
|
|
||||||
|
|
||||||
INSERT = ''' # CO-OP PATCH (audio shared pricing, 2026-10-26): the audio cost paths
|
|
||||||
# (speech, transcription) read pricing from the shared cost-map key, and
|
|
||||||
# this router strips per-deployment pricing from it — so a custom-backend
|
|
||||||
# audio deployment raises in the cost lookup, the spend write is skipped,
|
|
||||||
# and no SpendLogs row appears at all (same mechanism as the image case
|
|
||||||
# this file already patches). Re-add the pricing for audio modes,
|
|
||||||
# including input_cost_per_request, which LiteLLM does not list among its
|
|
||||||
# CustomPricing fields.
|
|
||||||
if deployment_mode in ("audio_speech", "audio_transcription"):
|
|
||||||
for _ak, _av in model_info.items():
|
|
||||||
if (
|
|
||||||
_ak in CustomPricingLiteLLMParams.model_fields
|
|
||||||
or _ak == "input_cost_per_request"
|
|
||||||
) and _av is not None:
|
|
||||||
shared_model_info[_ak] = _av
|
|
||||||
|
|
||||||
'''
|
|
||||||
|
|
||||||
|
|
||||||
def target_path() -> str:
|
|
||||||
if len(sys.argv) > 1:
|
|
||||||
return sys.argv[1]
|
|
||||||
env = os.environ.get("LITELLM_ROUTER")
|
|
||||||
if env:
|
|
||||||
return env
|
|
||||||
hits = []
|
|
||||||
try: # most reliable: ask the installed package where it lives
|
|
||||||
import litellm # noqa: PLC0415
|
|
||||||
_file = getattr(litellm, "__file__", None)
|
|
||||||
if _file:
|
|
||||||
hits.append(os.path.join(os.path.dirname(_file), "router.py"))
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/router.py")
|
|
||||||
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/router.py")
|
|
||||||
hits = [h for h in hits if os.path.exists(h)]
|
|
||||||
if not hits:
|
|
||||||
raise SystemExit("[coop-patch] could not locate litellm/router.py")
|
|
||||||
return sorted(hits)[0]
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
path = target_path()
|
|
||||||
try:
|
|
||||||
source = open(path).read()
|
|
||||||
except OSError as exc:
|
|
||||||
print(f"[coop-patch] cannot read {path}: {exc}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
if MARKER in source:
|
|
||||||
print(f"[coop-patch] {path} already patched, nothing to do")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
if ANCHOR not in source:
|
|
||||||
print(f"[coop-patch] WARNING anchor not found in {path}; audio pricing will "
|
|
||||||
f"NOT reach the shared cost map and audio spend will stay invisible")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
patched = source.replace(ANCHOR, INSERT + ANCHOR, 1)
|
|
||||||
compile(patched, path, "exec") # refuse to write something that will not import
|
|
||||||
|
|
||||||
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
|
||||||
print(f"[coop-patch] {path} is not writable; verified anchor and syntax only")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
with open(path, "w") as handle:
|
|
||||||
handle.write(patched)
|
|
||||||
print(f"[coop-patch] applied audio shared pricing to {path}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,115 +0,0 @@
|
|||||||
"""Expose LiteLLM's per-request cost headers to cross-origin browser clients.
|
|
||||||
|
|
||||||
MEMBER REQUEST (co-op/support#39, 2026-10-31). Browser JS calling the gateway from
|
|
||||||
another origin cannot read a response header unless the server names it in
|
|
||||||
Access-Control-Expose-Headers. LiteLLM sets
|
|
||||||
|
|
||||||
expose_headers=LITELLM_UI_ALLOW_HEADERS # through 1.103.x
|
|
||||||
expose_headers=[*LITELLM_UI_ALLOW_HEADERS,
|
|
||||||
*trace_response_expose_headers()] # from 1.108.0
|
|
||||||
|
|
||||||
which covers seven names (semantic-filter, guardrail, cache-key, ...) and none of
|
|
||||||
the cost ones — so `x-litellm-response-cost*` reads as null in JS even though the
|
|
||||||
proxy sends those headers on the response. Upstream closed the request
|
|
||||||
(BerriAI/litellm#33123) as stale, so it has to be handled here at the app layer,
|
|
||||||
exactly like the other patches in this directory.
|
|
||||||
|
|
||||||
Additive only. It lets a caller read per-request pricing for requests they made
|
|
||||||
with their own API key — numbers already present in the response body for
|
|
||||||
non-streaming calls, and already visible in their dashboard. It exposes no prompt,
|
|
||||||
key, or account data, and no other member's usage.
|
|
||||||
|
|
||||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal, per
|
|
||||||
house rule: if upstream moves or rewrites this call, WARN and exit 0 — a missing
|
|
||||||
convenience header must never keep the gateway from booting.
|
|
||||||
|
|
||||||
Regression check: after a deploy, a browser request from an unrelated origin should
|
|
||||||
see the cost headers listed in Access-Control-Expose-Headers on the response.
|
|
||||||
"""
|
|
||||||
from pathlib import Path
|
|
||||||
import sys
|
|
||||||
|
|
||||||
# The header names the proxy can set, taken from the running image rather than
|
|
||||||
# guessed: grep -rhoE 'x-litellm-[a-z0-9-]*cost[a-z0-9-]*' over the litellm package.
|
|
||||||
EXPOSED_HEADERS = [
|
|
||||||
"x-litellm-response-cost",
|
|
||||||
"x-litellm-response-cost-input",
|
|
||||||
"x-litellm-response-cost-output",
|
|
||||||
"x-litellm-response-cost-original",
|
|
||||||
"x-litellm-response-cost-discount-amount",
|
|
||||||
"x-litellm-response-cost-margin-amount",
|
|
||||||
"x-litellm-response-cost-margin-percent",
|
|
||||||
"x-litellm-response-cost-cache-creation",
|
|
||||||
"x-litellm-response-cost-cache-read",
|
|
||||||
"x-litellm-response-cost-reasoning",
|
|
||||||
"x-litellm-response-cost-tool-usage",
|
|
||||||
"x-litellm-classifier-cost",
|
|
||||||
# Telemetry names, requested in co-op/support#39 (2026-10-31): model identity,
|
|
||||||
# call id, the caller's own key spend, timings and retry/fallback counts.
|
|
||||||
"x-litellm-model-name",
|
|
||||||
"x-litellm-call-id",
|
|
||||||
"x-litellm-key-spend",
|
|
||||||
"x-litellm-response-duration-ms",
|
|
||||||
"x-litellm-overhead-duration-ms",
|
|
||||||
"x-litellm-callback-duration-ms",
|
|
||||||
"x-litellm-attempted-retries",
|
|
||||||
"x-litellm-attempted-fallbacks",
|
|
||||||
]
|
|
||||||
|
|
||||||
OLD = " expose_headers=LITELLM_UI_ALLOW_HEADERS,\n"
|
|
||||||
NEW = (
|
|
||||||
" expose_headers=LITELLM_UI_ALLOW_HEADERS\n"
|
|
||||||
" + [\n"
|
|
||||||
+ "".join(f' "{h}",\n' for h in EXPOSED_HEADERS)
|
|
||||||
+ " ],\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
# 1.108.0 refactored the middleware call (upstream added the trace-response
|
|
||||||
# header set to expose_headers). Same operation, new anchor; tried when the
|
|
||||||
# 1.103.2 shape is absent so the script works against either base image.
|
|
||||||
OLD_V1108 = (
|
|
||||||
" expose_headers=[*LITELLM_UI_ALLOW_HEADERS, *trace_response_expose_headers()],\n"
|
|
||||||
)
|
|
||||||
NEW_V1108 = (
|
|
||||||
" expose_headers=[\n"
|
|
||||||
" *LITELLM_UI_ALLOW_HEADERS,\n"
|
|
||||||
" *trace_response_expose_headers(),\n"
|
|
||||||
+ "".join(f' "{h}",\n' for h in EXPOSED_HEADERS)
|
|
||||||
+ " ],\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
# Glob rather than pin a Python version: the venv path moves when the base
|
|
||||||
# image bumps its interpreter, and a stale path would silently skip the patch.
|
|
||||||
candidates = sorted(Path("/app/.venv/lib").glob("python3*/site-packages/litellm/proxy/proxy_server.py"))
|
|
||||||
if not candidates:
|
|
||||||
print("WARN expose-cost-headers: proxy_server.py not found under /app/.venv/lib — skipping")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
target = candidates[-1]
|
|
||||||
src = target.read_text()
|
|
||||||
if "x-litellm-response-cost-input" in src:
|
|
||||||
print("expose-cost-headers: already applied")
|
|
||||||
return 0
|
|
||||||
for old, new, shape in ((OLD, NEW, "1.103.x"), (OLD_V1108, NEW_V1108, "1.108.x")):
|
|
||||||
found = src.count(old)
|
|
||||||
if found == 1:
|
|
||||||
target.write_text(src.replace(old, new))
|
|
||||||
print(
|
|
||||||
f"expose-cost-headers: appended {len(EXPOSED_HEADERS)} cost header names "
|
|
||||||
f"to expose_headers (anchor {shape}) in {target}"
|
|
||||||
)
|
|
||||||
return 0
|
|
||||||
if found > 1:
|
|
||||||
print(f"WARN expose-cost-headers: expected exactly 1 {shape} CORS middleware call, found {found} — skipping")
|
|
||||||
return 0
|
|
||||||
print(
|
|
||||||
"WARN expose-cost-headers: no known expose_headers anchor found "
|
|
||||||
"(neither 1.103.x nor 1.108.x shape) — skipping"
|
|
||||||
)
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,107 +0,0 @@
|
|||||||
"""Open explicitly-listed pass-through routes to any authenticated key.
|
|
||||||
|
|
||||||
WHY. Config-declared pass-through endpoints registered with ``auth: true`` are
|
|
||||||
gated at request time by ``allowed_passthrough_routes`` on the key or team
|
|
||||||
(``RouteChecks.check_passthrough_route_access``). That field is a **LiteLLM
|
|
||||||
Enterprise** feature: on a community-edition proxy the management API refuses to
|
|
||||||
set it ("This feature is only available for LiteLLM Enterprise users"), so a
|
|
||||||
member key can never be granted access by any supported route. The only other
|
|
||||||
lever is ``auth: false``, which forwards **unauthenticated** requests — a public
|
|
||||||
relay of the co-op's GreenPT key. Neither is acceptable.
|
|
||||||
|
|
||||||
This patch lets the operator name such routes in the proxy config:
|
|
||||||
|
|
||||||
general_settings:
|
|
||||||
coop_passthrough_open_to_keys: ["/v1/systemone"]
|
|
||||||
|
|
||||||
Any route listed there is allowed for any key that has already authenticated.
|
|
||||||
Authentication itself is untouched: ``auth: true`` still installs
|
|
||||||
``Depends(user_api_key_auth)``, so an unauthenticated request is rejected earlier
|
|
||||||
and never reaches this function. The patch only relaxes the *second* gate
|
|
||||||
(per-key allow-list), and only for routes the operator listed by hand.
|
|
||||||
|
|
||||||
House style: run at BUILD time (the runtime rootfs is read-only under Cloudron);
|
|
||||||
non-fatal — if upstream renames or restructures this function, WARN and exit 0;
|
|
||||||
idempotent — detect the marker on a rebuild and skip; and compile the patched
|
|
||||||
source in memory before writing, so a patch that cannot import never ships.
|
|
||||||
|
|
||||||
Check the ``[coop-patch] applied`` line in the build log on every LiteLLM bump.
|
|
||||||
"""
|
|
||||||
from pathlib import Path
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
MARKER = "coop_passthrough_open_to_keys"
|
|
||||||
|
|
||||||
# Unique multi-line anchor: the def + docstring + the first statement of the body.
|
|
||||||
OLD = (
|
|
||||||
" def check_passthrough_route_access(route: str, user_api_key_dict: UserAPIKeyAuth) -> bool:\n"
|
|
||||||
" \"\"\"\n"
|
|
||||||
" Check if route is a passthrough route.\n"
|
|
||||||
" Supports both exact match and prefix match.\n"
|
|
||||||
" \"\"\"\n"
|
|
||||||
" metadata: Final = user_api_key_dict.metadata\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
NEW = (
|
|
||||||
" def check_passthrough_route_access(route: str, user_api_key_dict: UserAPIKeyAuth) -> bool:\n"
|
|
||||||
" \"\"\"\n"
|
|
||||||
" Check if route is a passthrough route.\n"
|
|
||||||
" Supports both exact match and prefix match.\n"
|
|
||||||
" \"\"\"\n"
|
|
||||||
" # [coop-patch] Routes the operator lists in\n"
|
|
||||||
" # general_settings.coop_passthrough_open_to_keys are callable by ANY\n"
|
|
||||||
" # authenticated key. LiteLLM's own gate (allowed_passthrough_routes)\n"
|
|
||||||
" # is Enterprise-only, so it can never be granted on this deployment.\n"
|
|
||||||
" # Authentication is unaffected: auth=True on the endpoint still\n"
|
|
||||||
" # requires a valid key before this function runs.\n"
|
|
||||||
" try:\n"
|
|
||||||
" from litellm.proxy.proxy_server import general_settings as _coop_general_settings\n"
|
|
||||||
"\n"
|
|
||||||
" _coop_open_to_keys = (_coop_general_settings or {}).get(\"coop_passthrough_open_to_keys\") or []\n"
|
|
||||||
" for _coop_route in _coop_open_to_keys:\n"
|
|
||||||
" if RouteChecks._route_matches_allowed_route(route=route, allowed_route=_coop_route):\n"
|
|
||||||
" return True\n"
|
|
||||||
" except Exception:\n"
|
|
||||||
" pass\n"
|
|
||||||
"\n"
|
|
||||||
" metadata: Final = user_api_key_dict.metadata\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
# ROOT override (house style) lets the same script run against a pulled copy
|
|
||||||
# locally: COOP_PATCH_ROOT=<dir containing .venv/lib>.
|
|
||||||
root = Path(os.environ.get("COOP_PATCH_ROOT", "/app"))
|
|
||||||
candidates = sorted(
|
|
||||||
(root / ".venv/lib").glob("python3*/site-packages/litellm/proxy/auth/route_checks.py")
|
|
||||||
)
|
|
||||||
if not candidates:
|
|
||||||
print("WARN passthrough-key-access: route_checks.py not found under /app/.venv/lib — skipping")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
target = candidates[-1]
|
|
||||||
src = target.read_text()
|
|
||||||
if MARKER in src:
|
|
||||||
print("passthrough-key-access: already applied")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
found = src.count(OLD)
|
|
||||||
if found != 1:
|
|
||||||
print(f"WARN passthrough-key-access: expected exactly 1 anchor, found {found} — skipping")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
patched = src.replace(OLD, NEW)
|
|
||||||
try:
|
|
||||||
compile(patched, str(target), "exec")
|
|
||||||
except SyntaxError as e:
|
|
||||||
print(f"WARN passthrough-key-access: patched source does not compile ({e}) — skipping")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
target.write_text(patched)
|
|
||||||
print(f"[coop-patch] applied: passthrough-key-access -> {target}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,229 +0,0 @@
|
|||||||
"""Price a generic pass-through route from the usage it reports in its OWN body.
|
|
||||||
|
|
||||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
|
||||||
if the target file or the anchor is missing — e.g. after an upstream version bump
|
|
||||||
— it prints a warning and exits 0, so the build and the gateway still succeed.
|
|
||||||
The build log line ``[coop-patch] applied`` is what confirms it took effect;
|
|
||||||
check it on every LiteLLM upgrade (see README, upgrade checklist).
|
|
||||||
|
|
||||||
Why: config-declared ``pass_through_endpoints`` with ``auth: true`` are logged
|
|
||||||
through ``PassThroughEndpointLogging``. A target whose hostname matches none of
|
|
||||||
vertex / anthropic / openai / gemini resolves to ``EndpointType.GENERIC`` and
|
|
||||||
takes the generic branch of ``pass_through_async_success_handler`` — it is
|
|
||||||
logged, but LiteLLM has no model on the route to price, so the only cost it
|
|
||||||
knows is the flat ``cost_per_request`` from the config entry. That is a whole-
|
|
||||||
request estimate used for budget accounting, not an invoice: Metis is billed at
|
|
||||||
$0.12 per 1M *input* tokens and returns the true count in ``usage.input_tokens``,
|
|
||||||
so the flat figure is off by whatever factor the call differed from the estimate.
|
|
||||||
|
|
||||||
LiteLLM's own supported mechanism for an unpriced pass-through is the
|
|
||||||
``x-litellm-response-cost`` / ``x-litellm-total-tokens`` response-header
|
|
||||||
contract (``litellm/proxy/pass_through_endpoints/upstream_usage_headers.py``),
|
|
||||||
but that is consumed inside this SAME logging path, so it only helps where the
|
|
||||||
logging path already runs — and GreenPT does not send those headers. Reading the
|
|
||||||
body, which is already in hand here, needs no extra process in front of the
|
|
||||||
provider (a sidecar would be a single point of failure for the route itself, not
|
|
||||||
just for its metering).
|
|
||||||
|
|
||||||
Patch: make ``PassThroughEndpointLogging._set_cost_per_request`` price a route
|
|
||||||
listed in ``general_settings.coop_passthrough_cost_per_token`` (USD per input
|
|
||||||
token, keyed by the gateway route path, or by the upstream target) from the
|
|
||||||
``usage.input_tokens`` in the response body:
|
|
||||||
|
|
||||||
cost = input_tokens * coop_passthrough_cost_per_token[route]
|
|
||||||
|
|
||||||
The rate lives in ``general_settings`` rather than the endpoint dict because the
|
|
||||||
endpoint model (``PassThroughGenericEndpoint``) drops unknown fields and the
|
|
||||||
exact-route registration copies only ``cost_per_request`` into the logging
|
|
||||||
payload — the ``general_settings`` dict is read straight from the proxy module,
|
|
||||||
the same way ``patch_passthrough_key_access.py`` reads
|
|
||||||
``coop_passthrough_open_to_keys``.
|
|
||||||
|
|
||||||
TOKEN COUNTS. Pricing the row from the body is not enough: the row still
|
|
||||||
recorded ``prompt_tokens=0``, ``completion_tokens=0``, ``total_tokens=0``,
|
|
||||||
because the pass-through success handler hands the spend writer a
|
|
||||||
``StandardPassThroughResponseObject`` whose text carries the body but whose
|
|
||||||
``usage`` is empty, and the generic pass-through branch builds no
|
|
||||||
``standard_logging_object`` usage either. Any per-token reporting (panel,
|
|
||||||
dashboard, energy) that sums tokens for this route then under-reports. So the
|
|
||||||
patch also carries the count onto the logging object as
|
|
||||||
``combined_usage_object`` — the same ``litellm.Usage`` override the streaming
|
|
||||||
handler uses to recover partial usage. The spend writer reads
|
|
||||||
``combined_usage_object`` (``spend_tracking_utils.get_logging_payload`` and
|
|
||||||
``StandardLoggingPayloadSetup.get_usage_as_dict``) before it falls back to the
|
|
||||||
response object, so the row records ``prompt_tokens=input_tokens``,
|
|
||||||
``completion_tokens=0``, ``total_tokens=input_tokens`` while spend stays exactly
|
|
||||||
``input_tokens × rate``.
|
|
||||||
|
|
||||||
The exact figure wins; ``cost_per_request`` stays as the fallback for a listed
|
|
||||||
route whose body carries no usable input-token count, and for every route not
|
|
||||||
listed. An upstream that reports its own totals via the x-litellm-* headers still
|
|
||||||
wins outright (the early return is left intact). Inert unless an operator lists
|
|
||||||
a route, so it changes nothing for the chat models or the audio paths.
|
|
||||||
|
|
||||||
Upstream: no issue filed; LiteLLM has no per-token pricing for pass-through
|
|
||||||
routes whose body is not an OpenAI/Anthropic/Gemini shape (as of 1.103.2).
|
|
||||||
"""
|
|
||||||
from pathlib import Path
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
MARKER = "coop_passthrough_cost_per_token"
|
|
||||||
|
|
||||||
OLD = (
|
|
||||||
" #########################################################\n"
|
|
||||||
" # Check if cost per request is set\n"
|
|
||||||
" #########################################################\n"
|
|
||||||
" if has_upstream_reported_usage(logging_obj):\n"
|
|
||||||
" return kwargs\n"
|
|
||||||
"\n"
|
|
||||||
" if passthrough_logging_payload.get(\"cost_per_request\") is not None:\n"
|
|
||||||
" kwargs[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n"
|
|
||||||
" logging_obj.model_call_details[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n"
|
|
||||||
"\n"
|
|
||||||
" return kwargs\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
NEW = (
|
|
||||||
" #########################################################\n"
|
|
||||||
" # Check if cost per request is set\n"
|
|
||||||
" #########################################################\n"
|
|
||||||
" if has_upstream_reported_usage(logging_obj):\n"
|
|
||||||
" return kwargs\n"
|
|
||||||
"\n"
|
|
||||||
" # [coop-patch] Exact per-input-token pricing for a generic\n"
|
|
||||||
" # pass-through route. A route the operator lists in\n"
|
|
||||||
" # general_settings.coop_passthrough_cost_per_token (USD per input\n"
|
|
||||||
" # token, keyed by gateway path or upstream target) is priced from the\n"
|
|
||||||
" # usage it reports in its OWN response body, so spend is the real\n"
|
|
||||||
" # token count rather than cost_per_request's flat estimate. GreenPT's\n"
|
|
||||||
" # Metis decision route (/v1/systemone) is the first such route: it\n"
|
|
||||||
" # returns usage.input_tokens and always 0 output tokens, billed at\n"
|
|
||||||
" # $0.12 per 1M input tokens and recorded on the spend row with the\n"
|
|
||||||
" # same count. Inert unless a route is listed; a listed\n"
|
|
||||||
" # route whose body carries no token count falls through to\n"
|
|
||||||
" # cost_per_request below, and every unlisted route is unchanged.\n"
|
|
||||||
" try:\n"
|
|
||||||
" from litellm.proxy.proxy_server import general_settings as _coop_general_settings\n"
|
|
||||||
"\n"
|
|
||||||
" _coop_rates = (_coop_general_settings or {}).get(\"coop_passthrough_cost_per_token\") or {}\n"
|
|
||||||
" if _coop_rates:\n"
|
|
||||||
" _coop_candidates = set()\n"
|
|
||||||
" _coop_psr = ((kwargs.get(\"litellm_params\") or {}).get(\"proxy_server_request\") or {})\n"
|
|
||||||
" _coop_req_url = _coop_psr.get(\"url\")\n"
|
|
||||||
" if _coop_req_url:\n"
|
|
||||||
" _coop_candidates.add(str(_coop_req_url))\n"
|
|
||||||
" _coop_candidates.add(urlparse(str(_coop_req_url)).path)\n"
|
|
||||||
" _coop_upstream_url = passthrough_logging_payload.get(\"url\")\n"
|
|
||||||
" if _coop_upstream_url:\n"
|
|
||||||
" _coop_candidates.add(str(_coop_upstream_url))\n"
|
|
||||||
" _coop_candidates.add(urlparse(str(_coop_upstream_url)).path)\n"
|
|
||||||
"\n"
|
|
||||||
" _coop_rate = None\n"
|
|
||||||
" for _coop_route in _coop_candidates:\n"
|
|
||||||
" if _coop_route in _coop_rates:\n"
|
|
||||||
" _coop_rate = _coop_rates[_coop_route]\n"
|
|
||||||
" break\n"
|
|
||||||
" if _coop_rate is None:\n"
|
|
||||||
" for _coop_key, _coop_val in _coop_rates.items():\n"
|
|
||||||
" _coop_prefix = str(_coop_key).rstrip(\"/\")\n"
|
|
||||||
" if any(\n"
|
|
||||||
" c == _coop_key or (c and c.startswith(_coop_prefix + \"/\"))\n"
|
|
||||||
" for c in _coop_candidates\n"
|
|
||||||
" ):\n"
|
|
||||||
" _coop_rate = _coop_val\n"
|
|
||||||
" break\n"
|
|
||||||
"\n"
|
|
||||||
" _coop_tokens = 0\n"
|
|
||||||
" _coop_body = passthrough_logging_payload.get(\"response_body\")\n"
|
|
||||||
" if isinstance(_coop_body, dict):\n"
|
|
||||||
" _coop_usage = _coop_body.get(\"usage\")\n"
|
|
||||||
" if isinstance(_coop_usage, dict):\n"
|
|
||||||
" _coop_tokens = _coop_usage.get(\"input_tokens\") or 0\n"
|
|
||||||
" if not _coop_tokens:\n"
|
|
||||||
" _coop_tokens = _coop_body.get(\"input_tokens\") or 0\n"
|
|
||||||
"\n"
|
|
||||||
" if _coop_rate is not None and _coop_tokens:\n"
|
|
||||||
" _coop_cost = float(_coop_tokens) * float(_coop_rate)\n"
|
|
||||||
" kwargs[\"response_cost\"] = _coop_cost\n"
|
|
||||||
" logging_obj.model_call_details[\"response_cost\"] = _coop_cost\n"
|
|
||||||
" # [coop-patch] Record the true token counts on the row too.\n"
|
|
||||||
" # The spend writer reads usage from combined_usage_object\n"
|
|
||||||
" # (the override the streaming handler already uses for\n"
|
|
||||||
" # recovered partial usage); without it the generic\n"
|
|
||||||
" # pass-through response object carries no usage and the\n"
|
|
||||||
" # row's prompt/completion/total tokens stay 0.\n"
|
|
||||||
" _coop_usage = None\n"
|
|
||||||
" try:\n"
|
|
||||||
" from litellm.types.utils import Usage as _CoopUsage\n"
|
|
||||||
"\n"
|
|
||||||
" _coop_usage = _CoopUsage(\n"
|
|
||||||
" prompt_tokens=int(_coop_tokens),\n"
|
|
||||||
" completion_tokens=0,\n"
|
|
||||||
" total_tokens=int(_coop_tokens),\n"
|
|
||||||
" )\n"
|
|
||||||
" except Exception as _coop_usage_exc:\n"
|
|
||||||
" print(\n"
|
|
||||||
" \"[coop-passthrough-cost] usage object failed: %s: %s\"\n"
|
|
||||||
" % (type(_coop_usage_exc).__name__, _coop_usage_exc),\n"
|
|
||||||
" flush=True,\n"
|
|
||||||
" )\n"
|
|
||||||
" if _coop_usage is not None:\n"
|
|
||||||
" kwargs[\"combined_usage_object\"] = _coop_usage\n"
|
|
||||||
" logging_obj.model_call_details[\"combined_usage_object\"] = _coop_usage\n"
|
|
||||||
" print(\n"
|
|
||||||
" \"[coop-passthrough-cost] input_tokens=%s rate=%s -> response_cost=%s\"\n"
|
|
||||||
" % (_coop_tokens, _coop_rate, _coop_cost),\n"
|
|
||||||
" flush=True,\n"
|
|
||||||
" )\n"
|
|
||||||
" return kwargs\n"
|
|
||||||
" except Exception as _coop_exc:\n"
|
|
||||||
" print(\n"
|
|
||||||
" \"[coop-passthrough-cost] exact cost failed: %s: %s\"\n"
|
|
||||||
" % (type(_coop_exc).__name__, _coop_exc),\n"
|
|
||||||
" flush=True,\n"
|
|
||||||
" )\n"
|
|
||||||
"\n"
|
|
||||||
" if passthrough_logging_payload.get(\"cost_per_request\") is not None:\n"
|
|
||||||
" kwargs[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n"
|
|
||||||
" logging_obj.model_call_details[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n"
|
|
||||||
"\n"
|
|
||||||
" return kwargs\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
# ROOT override (house style) lets the same script run against a pulled copy
|
|
||||||
# locally: COOP_PATCH_ROOT=<dir containing .venv/lib>.
|
|
||||||
root = Path(os.environ.get("COOP_PATCH_ROOT", "/app"))
|
|
||||||
candidates = sorted(
|
|
||||||
(root / ".venv/lib").glob("python3*/site-packages/litellm/proxy/pass_through_endpoints/success_handler.py")
|
|
||||||
)
|
|
||||||
if not candidates:
|
|
||||||
print("WARN passthrough-token-cost: success_handler.py not found under /app/.venv/lib — skipping")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
target = candidates[-1]
|
|
||||||
src = target.read_text()
|
|
||||||
if MARKER in src:
|
|
||||||
print("passthrough-token-cost: already applied")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
if src.count(OLD) != 1:
|
|
||||||
print(f"WARN passthrough-token-cost: expected exactly 1 anchor, found {src.count(OLD)} — skipping")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
patched = src.replace(OLD, NEW, 1)
|
|
||||||
try:
|
|
||||||
compile(patched, str(target), "exec")
|
|
||||||
except SyntaxError as e:
|
|
||||||
print(f"WARN passthrough-token-cost: patched source does not compile ({e}) — skipping")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
target.write_text(patched)
|
|
||||||
print(f"[coop-patch] applied: passthrough-token-cost -> {target}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -1,75 +0,0 @@
|
|||||||
"""Stop LiteLLM from forwarding the provider's response headers to callers.
|
|
||||||
|
|
||||||
SECURITY (2026-10-27, member-disclosure finding 4): LiteLLM's proxy copies every
|
|
||||||
upstream response header onto its own response as `llm_provider-<name>` — see
|
|
||||||
litellm_core_utils/llm_response_utils/get_headers.py::_get_llm_provider_headers,
|
|
||||||
whose docstring reads "Forward all headers as llm_provider-{header}". Verified
|
|
||||||
live on the co-op gateway: one chat completion returned 27 such headers, among
|
|
||||||
them `llm_provider-x-credits-remaining: 16.52` — the co-op's remaining balance
|
|
||||||
with the provider, readable by any member API key, and (read across calls) its
|
|
||||||
burn rate.
|
|
||||||
|
|
||||||
This patch makes _get_llm_provider_headers() return {} so no provider header is
|
|
||||||
copied. get_response_headers() still forwards the four OpenAI-style x-ratelimit-*
|
|
||||||
headers WHEN a provider sends them; those describe the caller's own request
|
|
||||||
rather than the co-op's account, and no upstream in our config sends them today.
|
|
||||||
|
|
||||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal, per
|
|
||||||
house rule: if upstream moves or rewrites the function, WARN and exit 0 — a
|
|
||||||
cosmetic leak must never keep the gateway from booting. Regression check: the
|
|
||||||
daily litellm watcher asserts the gateway returns no `llm_provider-*` header.
|
|
||||||
"""
|
|
||||||
import glob
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
ROOT = os.environ.get("COOP_PATCH_ROOT", "/app")
|
|
||||||
TARGET = "/litellm/litellm_core_utils/llm_response_utils/get_headers.py"
|
|
||||||
ANCHOR = ''' for k, v in response_headers.items():
|
|
||||||
if "llm_provider" not in k:
|
|
||||||
_key = "{}-{}".format("llm_provider", k)
|
|
||||||
llm_provider_headers[_key] = v
|
|
||||||
else:
|
|
||||||
llm_provider_headers[k] = v
|
|
||||||
return llm_provider_headers'''
|
|
||||||
REPLACEMENT = ''' # CO-OP PATCH (2026-10-27): do NOT forward provider headers to callers.
|
|
||||||
# Upstream prefixes EVERY response header with "llm_provider-", which leaked
|
|
||||||
# the co-op's remaining provider balance (llm_provider-x-credits-remaining)
|
|
||||||
# to any member API key. Delete this patch from the Dockerfile to revert.
|
|
||||||
return {}'''
|
|
||||||
|
|
||||||
candidates = glob.glob(f"{ROOT}/.venv/lib/python*/site-packages{TARGET}")
|
|
||||||
if not candidates:
|
|
||||||
candidates = glob.glob(f"{ROOT}/**{TARGET}", recursive=True)
|
|
||||||
|
|
||||||
patched_any = False
|
|
||||||
for p in candidates:
|
|
||||||
src = open(p).read()
|
|
||||||
if "CO-OP PATCH (2026-10-27)" in src:
|
|
||||||
print(f"INFO: provider-header strip already present in {p}")
|
|
||||||
patched_any = True
|
|
||||||
continue
|
|
||||||
if ANCHOR not in src:
|
|
||||||
print(f"WARN: anchor not found in {p} (upstream changed it); "
|
|
||||||
"provider headers are still forwarded (non-fatal)", file=sys.stderr)
|
|
||||||
continue
|
|
||||||
new = src.replace(ANCHOR, REPLACEMENT, 1)
|
|
||||||
try:
|
|
||||||
compile(new, p, "exec")
|
|
||||||
except SyntaxError as e:
|
|
||||||
print(f"WARN: patched source would not compile ({e}); not written "
|
|
||||||
"(non-fatal)", file=sys.stderr)
|
|
||||||
continue
|
|
||||||
with open(p, "w") as f:
|
|
||||||
f.write(new)
|
|
||||||
print("[coop-patch] applied: provider response headers are no longer "
|
|
||||||
f"forwarded to callers ({p})")
|
|
||||||
patched_any = True
|
|
||||||
|
|
||||||
if not candidates:
|
|
||||||
print("WARN: get_headers.py not found; provider headers are still forwarded "
|
|
||||||
"(non-fatal)", file=sys.stderr)
|
|
||||||
elif not patched_any:
|
|
||||||
print("WARN: no copy of get_headers.py patched (non-fatal)", file=sys.stderr)
|
|
||||||
|
|
||||||
sys.exit(0)
|
|
||||||
@@ -1,111 +0,0 @@
|
|||||||
"""Skip LiteLLM's forced `verbose_json` on transcriptions for opting-out models.
|
|
||||||
|
|
||||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
|
||||||
if the target file or the anchor is missing — e.g. after a version bump — it
|
|
||||||
prints a warning and exits 0, so the build and the gateway still succeed. Look
|
|
||||||
for the ``[coop-patch]`` line in the build log.
|
|
||||||
|
|
||||||
Why: for every `openai/` transcription deployment, LiteLLM's transformation
|
|
||||||
injects ``response_format: verbose_json`` when the caller omits the field
|
|
||||||
("ensures 'duration' is received - used for cost calculation"). Tinfoil's
|
|
||||||
``voxtral-small-24b`` rejects verbose_json with a 400 ("Currently do not
|
|
||||||
support verbose_json for voxtral-small-24b"), so every bare
|
|
||||||
``/v1/audio/transcriptions`` call to it failed — including the shape members
|
|
||||||
actually use (file + model, no response_format), and Open WebUI dictation's
|
|
||||||
request shape. The injection exists to obtain a duration for per-SECOND
|
|
||||||
pricing; voxtral-small-24b is billed per token (chat route) / per request
|
|
||||||
(patch_audio_request_cost.py), so it does not need the duration at all.
|
|
||||||
|
|
||||||
Patch: honour a ``coop_no_verbose_json`` flag on the deployment's model_info
|
|
||||||
when it is reachable, and gate the model name explicitly — the flag's plumbing
|
|
||||||
into the transformation's litellm_params is not guaranteed on every path, the
|
|
||||||
name gate is. whisper-large-v3-turbo (which Tinfoil serves verbose_json for,
|
|
||||||
and whose dictation path depends on the injected format) is untouched.
|
|
||||||
|
|
||||||
Scope: only transcriptions that omit response_format AND match the opt-out;
|
|
||||||
every other request keeps upstream behavior exactly.
|
|
||||||
|
|
||||||
Upstream: LiteLLM has no opt-out for this injection as of 1.103.2.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import glob
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
|
|
||||||
MARKER = "CO-OP PATCH (skip verbose_json, 2026-11-06)"
|
|
||||||
|
|
||||||
ANCHOR = ''' if "response_format" not in data:
|
|
||||||
data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation'''
|
|
||||||
|
|
||||||
INSERT = ''' if "response_format" not in data:
|
|
||||||
# CO-OP PATCH (skip verbose_json, 2026-11-06): a deployment whose
|
|
||||||
# model_info declares coop_no_verbose_json opts out of the forced
|
|
||||||
# verbose_json injection — Tinfoil's voxtral-small-24b rejects
|
|
||||||
# verbose_json with a 400, and its per-token / per-request billing
|
|
||||||
# does not need the duration this injection exists to obtain. The
|
|
||||||
# model-name gate backs the flag up: the flag's plumbing into this
|
|
||||||
# method's litellm_params is not guaranteed on every call path.
|
|
||||||
_mi = (litellm_params or {}).get("model_info") or {}
|
|
||||||
_coop_skip = (isinstance(_mi, dict) and bool(_mi.get("coop_no_verbose_json"))) or (
|
|
||||||
"voxtral-small" in str(model or "")
|
|
||||||
)
|
|
||||||
if not _coop_skip:
|
|
||||||
data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation'''
|
|
||||||
|
|
||||||
|
|
||||||
def target_path() -> str:
|
|
||||||
if len(sys.argv) > 1:
|
|
||||||
return sys.argv[1]
|
|
||||||
env = os.environ.get("LITELLM_WHISPER_TRANSFORMATION")
|
|
||||||
if env:
|
|
||||||
return env
|
|
||||||
hits = []
|
|
||||||
try: # most reliable: ask the installed package where it lives
|
|
||||||
import litellm # noqa: PLC0415
|
|
||||||
_file = getattr(litellm, "__file__", None)
|
|
||||||
if _file:
|
|
||||||
hits.append(os.path.join(
|
|
||||||
os.path.dirname(_file), "llms", "openai", "transcriptions", "whisper_transformation.py"))
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py")
|
|
||||||
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py")
|
|
||||||
hits = [h for h in hits if os.path.exists(h)]
|
|
||||||
if not hits:
|
|
||||||
raise SystemExit("[coop-patch] could not locate llms/openai/transcriptions/whisper_transformation.py")
|
|
||||||
return sorted(hits)[0]
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
path = target_path()
|
|
||||||
try:
|
|
||||||
source = open(path).read()
|
|
||||||
except OSError as exc:
|
|
||||||
print(f"[coop-patch] cannot read {path}: {exc}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
if MARKER in source:
|
|
||||||
print(f"[coop-patch] {path} already patched, nothing to do")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
if ANCHOR not in source:
|
|
||||||
print(f"[coop-patch] WARNING anchor not found in {path}; bare transcription calls to "
|
|
||||||
f"opt-out models (voxtral-small-24b) will fail with the provider's verbose_json 400")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
patched = source.replace(ANCHOR, INSERT, 1)
|
|
||||||
compile(patched, path, "exec") # refuse to write something that will not import
|
|
||||||
|
|
||||||
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
|
||||||
print(f"[coop-patch] {path} is not writable; verified anchor and syntax only")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
with open(path, "w") as handle:
|
|
||||||
handle.write(patched)
|
|
||||||
print(f"[coop-patch] applied skip-verbose_json to {path}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -6,18 +6,6 @@ set -eu
|
|||||||
# Configures LiteLLM with Cloudron's PostgreSQL and Redis addons
|
# Configures LiteLLM with Cloudron's PostgreSQL and Redis addons
|
||||||
# ============================================================================
|
# ============================================================================
|
||||||
|
|
||||||
# --- Cloudron scheduler tasks arrive as ARGUMENTS to this script ---
|
|
||||||
# The manifest's scheduler commands run as `docker run <image> <command>`, and
|
|
||||||
# this script is the image ENTRYPOINT — so the task command lands in "$@".
|
|
||||||
# Without this branch every scheduled task booted a second tinfoil-proxy and
|
|
||||||
# LiteLLM (whose ports are already bound in the app's network namespace) and
|
|
||||||
# died on the bind FATAL before doing its job — which looked exactly like a
|
|
||||||
# scheduler that never fired. Exec the task directly instead of booting.
|
|
||||||
if [ "$#" -gt 0 ]; then
|
|
||||||
echo "start.sh: scheduler-task mode — exec: $*"
|
|
||||||
exec "$@"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "=== LiteLLM Cloudron Startup ==="
|
echo "=== LiteLLM Cloudron Startup ==="
|
||||||
|
|
||||||
# --- Cloudron PostgreSQL addon provides:
|
# --- Cloudron PostgreSQL addon provides:
|
||||||
@@ -89,13 +77,10 @@ export LITELLM_MIGRATION_DIR="/app/data/migrations"
|
|||||||
export UI_USERNAME="admin"
|
export UI_USERNAME="admin"
|
||||||
export UI_PASSWORD="${LITELLM_MASTER_KEY}"
|
export UI_PASSWORD="${LITELLM_MASTER_KEY}"
|
||||||
|
|
||||||
# Nothing disables the static /ui admin bundle on the pinned v1.103.2 (there
|
# DISABLE the admin UI and API docs. The gateway must stay publicly reachable
|
||||||
# is no flag for it): DISABLE_ADMIN_UI="True" only turns off the UI's SSO
|
# for the /v1 API (member keys), but we don't want to advertise a login-gated
|
||||||
# login-redirect routes upstream — GET /ui/ still serves the static login
|
# admin console (/ui) or LiteLLM's Swagger/ReDoc/openapi surfaces. Admin
|
||||||
# page. Logging into it still requires the master key, and the docs surfaces
|
# visibility comes from the admin dashboard / the API directly.
|
||||||
# below ARE fully disabled (NO_DOCS/NO_REDOC/NO_OPENAPI → 404s). See README
|
|
||||||
# "Admin access" for the wire-verified state and the patch needed to remove
|
|
||||||
# the bundle itself. Kept set because the SSO-redirect disable is still real.
|
|
||||||
export DISABLE_ADMIN_UI="True"
|
export DISABLE_ADMIN_UI="True"
|
||||||
export NO_DOCS="True"
|
export NO_DOCS="True"
|
||||||
export NO_REDOC="True"
|
export NO_REDOC="True"
|
||||||
@@ -104,19 +89,13 @@ export NO_OPENAPI="True"
|
|||||||
# Proxy settings for Cloudron's reverse proxy
|
# Proxy settings for Cloudron's reverse proxy
|
||||||
export LITELLM_PROXY_BASE_URL="https://${CLOUDRON_APP_DOMAIN}"
|
export LITELLM_PROXY_BASE_URL="https://${CLOUDRON_APP_DOMAIN}"
|
||||||
|
|
||||||
# CORS: LiteLLM reads LITELLM_CORS_ORIGINS natively (upstream replaced the
|
# CORS: LiteLLM v1.84.0 reads LITELLM_CORS_ORIGINS natively (upstream added
|
||||||
# hardcoded `origins = ["*"]` with this env var in v1.84.0). The value is
|
# the env var in place of the old hardcoded `origins = ["*"]`). Lock browser
|
||||||
# deliberately OVERRIDABLE — a Cloudron app env now wins, where before this
|
# access to the co-op's own chat origin. The gateway is called
|
||||||
# export silently overrode it and made the app env look effective when it was not.
|
# server-to-server (chat → portal → gateway) and by member API keys
|
||||||
#
|
# (curl/SDKs, not browsers), so no wildcard is needed. A `*` here would let
|
||||||
# The co-op decided to allow all origins (co-op/support#38, agreed on Loomio and
|
# any website's JS hit the gateway.
|
||||||
# by a member): CORS is not an auth boundary here. Every request carries a
|
export LITELLM_CORS_ORIGINS="https://chat.inference.coop"
|
||||||
# per-member bearer key, which the browser never attaches implicitly, so an origin
|
|
||||||
# list buys nothing against a party that already holds a key — and a leaked key is
|
|
||||||
# curl-able regardless. What actually constrains abuse is the per-key budget,
|
|
||||||
# model scoping and revocation. The fallback keeps the old chat-only lock if the
|
|
||||||
# env var is ever unset.
|
|
||||||
export LITELLM_CORS_ORIGINS="${LITELLM_CORS_ORIGINS:-https://chat.inference.coop}"
|
|
||||||
|
|
||||||
echo "PostgreSQL: ${CLOUDRON_POSTGRESQL_HOST}:${CLOUDRON_POSTGRESQL_PORT}"
|
echo "PostgreSQL: ${CLOUDRON_POSTGRESQL_HOST}:${CLOUDRON_POSTGRESQL_PORT}"
|
||||||
echo "Redis: ${CLOUDRON_REDIS_HOST}:${CLOUDRON_REDIS_PORT}"
|
echo "Redis: ${CLOUDRON_REDIS_HOST}:${CLOUDRON_REDIS_PORT}"
|
||||||
@@ -171,25 +150,6 @@ YAML
|
|||||||
echo "Config written to $CONFIG_FILE"
|
echo "Config written to $CONFIG_FILE"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# --- Ship config updates with the image ---
|
|
||||||
# /app/data is a persistent volume, so the copy the Dockerfile bakes into
|
|
||||||
# /app/data/config.yaml is shadowed by the volume's file forever — config
|
|
||||||
# changes in the repo would never reach a running gateway. The repo is the
|
|
||||||
# source of truth (README: repos mirror production), so sync the shipped copy
|
|
||||||
# over the volume whenever they differ.
|
|
||||||
if [ -f /app/code/config.yaml ] && ! cmp -s /app/code/config.yaml "$CONFIG_FILE"; then
|
|
||||||
echo "Updating /app/data/config.yaml from the image (repo is the source of truth)"
|
|
||||||
cp /app/code/config.yaml "$CONFIG_FILE"
|
|
||||||
fi
|
|
||||||
# NOTE on the fallback config written above: it can never be SERVED. The very
|
|
||||||
# same boot's sync above overwrites it with the image's (repo) config whenever
|
|
||||||
# they differ — including the boot right after the fallback ran. It is dead
|
|
||||||
# by construction; it exists only so the container is never left with an
|
|
||||||
# empty config file if the volume copy vanishes. Historically this block
|
|
||||||
# carried stale api.tinfoil.sh models that could have resurrected — that
|
|
||||||
# cannot happen through this path, and the shipped /app/code/config.yaml is
|
|
||||||
# the live catalog.
|
|
||||||
|
|
||||||
# --- Start LiteLLM ---
|
# --- Start LiteLLM ---
|
||||||
echo "Starting LiteLLM on port 4000..."
|
echo "Starting LiteLLM on port 4000..."
|
||||||
|
|
||||||
@@ -239,20 +199,4 @@ if [ "$proxy_up" != "1" ]; then
|
|||||||
fi
|
fi
|
||||||
echo "Tinfoil proxy is up on 127.0.0.1:3301"
|
echo "Tinfoil proxy is up on 127.0.0.1:3301"
|
||||||
|
|
||||||
# --- In-container audio keep-alive loop ---
|
|
||||||
# The scheduler task (handled above in scheduler-task mode) runs in a SEPARATE
|
|
||||||
# container whose ability to reach 127.0.0.1:4000 and read /app/data is not
|
|
||||||
# guaranteed, so the warm path is owned HERE — the app container, where the
|
|
||||||
# network namespace, the master key and the sidecar are all proven. Every 8
|
|
||||||
# minutes ping tinfoil/voxtral-tts ($0.002/request) so voice mode skips the
|
|
||||||
# ~24s Tinfoil cold-start after idle. audio_keepalive.py is silent on success
|
|
||||||
# and prints on failure, so /app/data/keepalive.log only ever grows on failure.
|
|
||||||
(
|
|
||||||
while true; do
|
|
||||||
python3 /app/code/audio_keepalive.py >> /app/data/keepalive.log 2>&1 || true
|
|
||||||
sleep 480
|
|
||||||
done
|
|
||||||
) &
|
|
||||||
echo "Audio keep-alive loop started (every 480s)"
|
|
||||||
|
|
||||||
exec litellm --config "$CONFIG_FILE" --port 4000 --host 0.0.0.0
|
exec litellm --config "$CONFIG_FILE" --port 4000 --host 0.0.0.0
|
||||||
Reference in new issue
Block a user