Compare commits
55
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ac9b3b17af | ||
|
|
140b443980 | ||
|
|
fe48ca8e38 | ||
|
|
6fcc116199 | ||
|
|
c8816b58c1 | ||
|
|
718644c363 | ||
|
|
870b165321 | ||
|
|
aa9331cff3 | ||
|
|
6f6bb73d16 | ||
|
|
9c81df48b8 | ||
|
|
d1d342d8e6 | ||
|
|
0f7cf17cd8 | ||
|
|
278900684c | ||
|
|
df4e154128 | ||
|
|
4b6583c3c0 | ||
|
|
2483715ef3 | ||
|
|
f66bedfb55 | ||
|
|
0e8517b6be | ||
|
|
d35ee1760f | ||
|
|
faff547ff4 | ||
|
|
b81177c3ad | ||
|
|
d96df6fd5a | ||
|
|
7a167a8354 | ||
|
|
a1613401c3 | ||
|
|
77e38d81e0 | ||
|
|
d877bc8eee | ||
|
|
437761171c | ||
|
|
84a191851f | ||
|
|
1ec589cf8b | ||
|
|
9266f9bc3c | ||
|
|
02ff60bfbf | ||
|
|
5986ee7987 | ||
|
|
aa1531bb18 | ||
|
|
e21c24ea48 | ||
|
|
64d7e5afc5 | ||
|
|
c43d45c455 | ||
|
|
5d38403892 | ||
|
|
943701581c | ||
|
|
abaeb255b2 | ||
|
|
67df968a04 | ||
|
|
54925cae97 | ||
|
|
e161db9ec8 | ||
|
|
d6cfe78c77 | ||
|
|
da02f8eb32 | ||
|
|
557999a16f | ||
|
|
e4e70962ec | ||
|
|
e8378d941a | ||
|
|
83c543a0ba | ||
|
|
197abce9e6 | ||
|
|
8ddd023786 | ||
|
|
a3acce2107 | ||
|
|
a6098741d0 | ||
|
|
c03abaa330 | ||
|
|
facf01d578 | ||
|
|
425d738625 |
No files matched your search
@@ -0,0 +1,2 @@
|
||||
__pycache__/
|
||||
*.pyc
|
||||
+7
-12
@@ -5,24 +5,18 @@
|
||||
"description": "LiteLLM AI Gateway \u2014 OpenAI-compatible proxy with per-user API keys, usage tracking, rate limiting, and model routing. Packaged for Cloudron with SSO/OIDC and PostgreSQL.",
|
||||
"tagline": "AI gateway for cooperative inference",
|
||||
"version": "1.0.0",
|
||||
"upstreamVersion": "1.103.2",
|
||||
"upstreamVersion": "1.108.3",
|
||||
"healthCheckPath": "/health/readiness",
|
||||
"httpPort": 4000,
|
||||
"memoryLimit": 2147483648,
|
||||
"memoryLimit": 3221225472,
|
||||
"manifestVersion": 2,
|
||||
"website": "https://litellm.ai",
|
||||
"contactEmail": "botbot@hermes",
|
||||
"contactEmail": "info@inference.coop",
|
||||
"icon": "file://logo.png",
|
||||
"addons": {
|
||||
"postgresql": {},
|
||||
"redis": {},
|
||||
"localstorage": {},
|
||||
"scheduler": {
|
||||
"audio_keepalive": {
|
||||
"schedule": "*/10 * * * *",
|
||||
"command": "/app/code/audio_keepalive.py"
|
||||
}
|
||||
}
|
||||
"localstorage": {}
|
||||
},
|
||||
"tags": [
|
||||
"ai",
|
||||
@@ -31,5 +25,6 @@
|
||||
"api"
|
||||
],
|
||||
"mediaLinks": [],
|
||||
"changelog": "Initial package for inference.coop"
|
||||
}
|
||||
"changelog": "Initial package for inference.coop",
|
||||
"dockerImage": "registry.inference.coop/code/litellm:20261118-140b443"
|
||||
}
|
||||
+91
-3
@@ -1,4 +1,4 @@
|
||||
FROM ghcr.io/berriai/litellm:v1.103.2
|
||||
FROM ghcr.io/berriai/litellm:v1.108.3
|
||||
|
||||
# v1.84.0 is the fix line for CVE-2026-35029 (auth bypass on
|
||||
# /config/update, fixed 1.83.0) and CVE-2026-59822 (MCP session auth
|
||||
@@ -19,6 +19,16 @@ RUN chmod +x /app/code/start.sh
|
||||
|
||||
# Default config — will be overridden by Cloudron env vars at runtime
|
||||
COPY config.yaml /app/data/config.yaml
|
||||
# A second copy inside the image: start.sh syncs it over the persistent
|
||||
# /app/data volume at boot, so config changes in the repo actually deploy.
|
||||
COPY config.yaml /app/code/config.yaml
|
||||
COPY fx_sync.py /app/code/fx_sync.py
|
||||
COPY eur_prices.json /app/code/eur_prices.json
|
||||
|
||||
# Fail-closed config validator (canonical copy lives in ops-docs/scripts/;
|
||||
# this byte-identical twin ships with the image and is called from start.sh
|
||||
# before the gateway boots). Mirrors validate_config.py's md5 — keep in step.
|
||||
COPY validate_config.py /app/code/validate_config.py
|
||||
|
||||
# Append the co-op credit-pack hint to LiteLLM's hardcoded budget-exceeded
|
||||
# messages. Done at BUILD time — the runtime rootfs is read-only under
|
||||
@@ -35,11 +45,89 @@ RUN python3 /app/code/patch_budget_message.py
|
||||
COPY patch_image_cost.py /app/code/patch_image_cost.py
|
||||
RUN python3 /app/code/patch_image_cost.py
|
||||
|
||||
# Audio keep-alive: scheduled pings keep the Tinfoil TEE audio enclave
|
||||
# warm so voice mode skips the ~24s cold-start (see audio_keepalive.py header).
|
||||
# Audio per-request pricing: Tinfoil bills voxtral-tts and whisper per request,
|
||||
# which LiteLLM cannot express on the audio paths, so those two were billed at
|
||||
# $0. The patch makes the cost calculator honour a declared
|
||||
# input_cost_per_request. Check for the "[coop-patch] applied" line in the build
|
||||
# log after upgrading LiteLLM: the script is non-fatal and warns instead.
|
||||
COPY patch_audio_request_cost.py /app/code/patch_audio_request_cost.py
|
||||
RUN python3 /app/code/patch_audio_request_cost.py
|
||||
|
||||
# The audio cost paths read pricing from the SHARED cost-map key, which the
|
||||
# router strips per-deployment pricing from — so the lookup raised, the spend
|
||||
# write was skipped, and audio produced no SpendLogs row at all. This re-adds the
|
||||
# pricing (and input_cost_per_request) for the audio modes. Same mechanism as
|
||||
# patch_image_cost.py above.
|
||||
COPY patch_audio_router_pricing.py /app/code/patch_audio_router_pricing.py
|
||||
RUN python3 /app/code/patch_audio_router_pricing.py
|
||||
|
||||
# Audio per-request prices must live in the SHARED cost map, which is built from
|
||||
# model_prices_and_context_window.json — the per-request helper reads that map, not
|
||||
# model_info, so a router-only patch left every speech call at $0 (issue #5).
|
||||
# This patch fails the build rather than warning.
|
||||
COPY patch_audio_cost_map.py /app/code/patch_audio_cost_map.py
|
||||
RUN python3 /app/code/patch_audio_cost_map.py
|
||||
|
||||
# Runtime registration: the build-time file edit above did not reach the deployed
|
||||
# gateway's model_cost map (issue #5), so register the entries in-process instead.
|
||||
# sitecustomize is imported before any application code.
|
||||
COPY sitecustomize.py /app/.venv/lib/python3.13/site-packages/sitecustomize.py
|
||||
|
||||
# SECURITY (2026-10-27, member-disclosure finding 4): LiteLLM forwards every
|
||||
# provider response header to callers as `llm_provider-*`, which leaked the
|
||||
# co-op's remaining provider balance (x-credits-remaining) to any member API key.
|
||||
# Verify the "[coop-patch] applied: provider response headers" line in the build
|
||||
# output on every version bump — the patch is non-fatal by design.
|
||||
COPY patch_strip_provider_headers.py /app/code/patch_strip_provider_headers.py
|
||||
RUN python3 /app/code/patch_strip_provider_headers.py
|
||||
|
||||
# Expose the per-request cost headers to cross-origin browser clients (co-op/support#39).
|
||||
COPY patch_expose_cost_headers.py /app/code/patch_expose_cost_headers.py
|
||||
RUN python3 /app/code/patch_expose_cost_headers.py
|
||||
|
||||
# Member access to config-declared pass-through routes: LiteLLM gates auth=true
|
||||
# pass-through endpoints behind `allowed_passthrough_routes`, an Enterprise-only
|
||||
# field that cannot be set on this community-edition proxy. This patch lets any
|
||||
# authenticated key call routes the operator lists in
|
||||
# general_settings.coop_passthrough_open_to_keys (unauthenticated access is
|
||||
# unaffected — still rejected). Non-fatal; check the "[coop-patch] applied" line
|
||||
# in the build log on every LiteLLM upgrade.
|
||||
COPY patch_passthrough_key_access.py /app/code/patch_passthrough_key_access.py
|
||||
RUN python3 /app/code/patch_passthrough_key_access.py
|
||||
|
||||
# Exact per-input-token pricing for generic pass-through routes. A route whose
|
||||
# target matches none of vertex/anthropic/openai/gemini resolves to
|
||||
# EndpointType.GENERIC: it IS logged, but LiteLLM has no model to price it from,
|
||||
# so the only cost it knows is the flat cost_per_request. Metis (/v1/systemone)
|
||||
# is billed at $0.12 per 1M input tokens and returns usage.input_tokens, so this
|
||||
# patch prices a route listed in general_settings.coop_passthrough_cost_per_token
|
||||
# from that count and writes the exact figure to member spend. Non-fatal; check
|
||||
# the "[coop-patch] applied" line in the build log on every LiteLLM upgrade.
|
||||
COPY patch_passthrough_token_cost.py /app/code/patch_passthrough_token_cost.py
|
||||
RUN python3 /app/code/patch_passthrough_token_cost.py
|
||||
|
||||
# Audio keep-alive: the in-container loop in start.sh pings tinfoil/voxtral-tts
|
||||
# every 15 min so voice mode skips the ~24s cold-start (see audio_keepalive.py).
|
||||
COPY audio_keepalive.py /app/code/audio_keepalive.py
|
||||
RUN chmod +x /app/code/audio_keepalive.py
|
||||
|
||||
# Skip LiteLLM's forced verbose_json on transcription requests for opting-out
|
||||
# models: Tinfoil's voxtral-small-24b rejects verbose_json with a 400, which
|
||||
# broke every bare /v1/audio/transcriptions call to it (file + model — the
|
||||
# shape members and dictation send). Its billing does not need the duration
|
||||
# the injection exists for. Non-fatal; check the "[coop-patch] applied" line
|
||||
# in the build log on every LiteLLM upgrade.
|
||||
COPY patch_transcription_response_format.py /app/code/patch_transcription_response_format.py
|
||||
RUN python3 /app/code/patch_transcription_response_format.py
|
||||
|
||||
# TTS default response format: LibreChat sends no response_format on its TTS
|
||||
# request and Tinfoil's voxtral-tts then serves WAV (~8-17s warm measured) —
|
||||
# while LibreChat labels the reply audio/mpeg regardless. Default to mp3
|
||||
# (OpenAI's documented default, ~3s warm measured). Non-fatal; check the
|
||||
# "[coop-patch] applied" line in the build log on every LiteLLM upgrade.
|
||||
COPY patch_audio_default_mp3.py /app/code/patch_audio_default_mp3.py
|
||||
RUN python3 /app/code/patch_audio_default_mp3.py
|
||||
|
||||
# Override the base image's ENTRYPOINT (which is `litellm`) so our
|
||||
# startup script runs instead of being passed as an argument to litellm.
|
||||
ENTRYPOINT ["/app/code/start.sh"]
|
||||
|
||||
@@ -4,7 +4,7 @@ A Cloudron app package for [LiteLLM](https://litellm.ai/), the open-source AI ga
|
||||
|
||||
LiteLLM is the single gateway between members and the inference backends. It handles per-member API keys, usage metering, rate limiting (via budget caps), and model routing — all behind one OpenAI-compatible endpoint.
|
||||
|
||||
**Current base image:** LiteLLM **v1.103.2**, pinned to the exact tag (fix line for CVE-2026-35029 / CVE-2026-59822, CISA KEV-listed; also carries the streaming-usage fix for prompt_tokens_details). See [Updating LiteLLM](#updating-litellm).
|
||||
**Current base image:** LiteLLM **v1.108.3**, pinned to the exact tag (upgraded from v1.103.2 in the 2026-11-08/09 maintenance window; minimum supported line remains v1.84.0, the fix line for CVE-2026-35029 / CVE-2026-59822, CISA KEV-listed). See [Updating LiteLLM](#updating-litellm).
|
||||
|
||||
## What this provides
|
||||
|
||||
@@ -16,16 +16,16 @@ LiteLLM is the single gateway between members and the inference backends. It han
|
||||
- **Cloudron Redis** for rate limiting and caching
|
||||
- **Automatic SSL, backups, and sandboxing** (Cloudron-managed)
|
||||
- **Master key + salt key** auto-generated on first start, persisted in `/app/data`
|
||||
- Hardened for the public internet: admin UI and API docs disabled at the proxy; CORS locked to the chat origin
|
||||
- Hardened for the public internet: LiteLLM's built-in API docs surfaces are disabled (`NO_DOCS`/`NO_REDOC`/`NO_OPENAPI` → live 404s). The static `/ui/` admin-UI bundle **is served** on this version (see *Admin access*); logging into it still requires the master key. CORS is deliberately open (`LITELLM_CORS_ORIGINS=*`, co-op/support#38) — auth is the per-member bearer key, not origin checking.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Members → Open WebUI (chat) / own tools (API) → member portal (key injection)
|
||||
Members → LibreChat (chat) / own tools (API) → member portal (key injection)
|
||||
→ LiteLLM gateway (this app)
|
||||
├→ Tinfoil (TEE enclaves — architectural privacy)
|
||||
├→ GreenPT (100% renewable energy)
|
||||
└→ PublicAI (publicly developed, sovereign models)
|
||||
└→ Regolo (EU-hosted, renewable + zero data retention)
|
||||
```
|
||||
|
||||
LiteLLM is the control plane (keys, metering, routing). It does **not** run models itself — it forwards to providers, each chosen by members for a different value. A [tinfoil-proxy](https://github.com/tinfoilsh/tinfoil-proxy) sidecar provides EHBP end-to-end encryption to Tinfoil's enclaves.
|
||||
@@ -76,7 +76,7 @@ After install, set the provider API keys in Cloudron → app → Settings → En
|
||||
|
||||
- `TINFOIL_API_KEY` — Tinfoil (https://tinfoil.sh/)
|
||||
- `GREENPT_API_KEY` — GreenPT (https://greenpt.ai/)
|
||||
- `PUBLICAI_API_KEY` — PublicAI (https://publicai.co/)
|
||||
- `REGOLO_API_KEY` — Regolo (regolo.ai; replaced PublicAI, removed 2026-10-16)
|
||||
|
||||
The following are auto-generated and should **not** be set manually:
|
||||
|
||||
@@ -85,9 +85,36 @@ The following are auto-generated and should **not** be set manually:
|
||||
- `DATABASE_URL` — from Cloudron's PostgreSQL addon
|
||||
- `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD` — from Cloudron's Redis addon
|
||||
|
||||
### Config deployment
|
||||
|
||||
`config.yaml` ships **with the image** and `start.sh` syncs it over the persistent
|
||||
`/app/data/config.yaml` at boot whenever they differ. The volume copy would
|
||||
otherwise shadow the image's file forever — which is exactly how a config change
|
||||
can be merged, built and "deployed" while the gateway keeps serving the old
|
||||
configuration. Change the repo and redeploy; never edit the volume copy directly.
|
||||
|
||||
### Admin access
|
||||
|
||||
The LiteLLM admin UI and API docs are **disabled** on the public gateway (hardened by default; see `start.sh`). Administrative operations run through the [member portal](https://git.inference.coop/code/member-portal), which holds the master key server-side. If direct LiteLLM admin access is ever needed, re-enable the UI in `start.sh`, redeploy, and use the master key from `/app/data/.master_key` — then disable it again.
|
||||
The state of the admin surfaces on the pinned v1.103.2, wire-verified 2026-11-05:
|
||||
|
||||
- The built-in **API docs are disabled** (`NO_DOCS`/`NO_REDOC`/`NO_OPENAPI` in
|
||||
`start.sh`) — `/docs`, `/redoc` and `/openapi.json` return 404.
|
||||
- The **static admin UI bundle is still served at `/ui/`** — upstream's
|
||||
`DISABLE_ADMIN_UI` env (which we set) only disables the UI's *SSO login
|
||||
redirect* routes in this version; nothing on v1.103.2 stops the unconditional
|
||||
`app.mount("/ui", …)`. So `GET /ui/` serves the login page. **Logging in
|
||||
still requires the master key** (auto-generated here, never published), and
|
||||
administrative operations normally run through the
|
||||
[member portal](https://git.inference.coop/code/member-portal), which holds
|
||||
the master key server-side.
|
||||
- If the login surface itself should disappear, it needs a build-time patch
|
||||
(strip the `/ui` mount from `proxy_server` in the Dockerfile, the same
|
||||
non-fatal patch pattern the other 8 build patches use) — there is no env/flag
|
||||
for it on this version. Until then treat `/ui/` as a master-key-only
|
||||
endpoint, like any other control-plane route.
|
||||
|
||||
If direct LiteLLM admin access is ever needed, log in at `/ui/` with the
|
||||
master key from `/app/data/.master_key` and lock it away again afterwards.
|
||||
|
||||
---
|
||||
|
||||
@@ -122,6 +149,19 @@ expect a 15–20 min maintenance window and **never panic-restart mid-migration*
|
||||
|
||||
1. **Pick a target version** from the [LiteLLM changelog](https://github.com/BerriAI/litellm/releases). Stay at or above v1.84.0 (the fix line for CVE-2026-35029 and CVE-2026-59822, the latter on CISA's KEV list).
|
||||
|
||||
**Re-validate the build patches against the target's real sources first** — the
|
||||
scripts fail soft (a moved anchor prints a WARN and exits 0), so a silently
|
||||
skipped patch ships. Ten patches are in the Dockerfile today; each must print
|
||||
its `applied` line in the build log. The cheapest rehearsal is to unpack the
|
||||
target wheel into a scratch tree and run the scripts against it (2026-11-09,
|
||||
v1.108.3: all ten apply — the CORS anchor needs the 1.108.x fallback already
|
||||
carried by `patch_expose_cost_headers.py`). Two gotchas in that harness:
|
||||
`patch_audio_request_cost.py` finds its target by `import litellm`, which fails
|
||||
outside a full install — point `LITELLM_COST_CALCULATOR` at the scratch
|
||||
`cost_calculator.py` instead, or it exits with "could not locate"; and running
|
||||
the scripts by hand leaves files owned by your user, which is fine in scratch
|
||||
and never in the image.
|
||||
|
||||
2. **Bump the pin** in `Dockerfile`:
|
||||
|
||||
```dockerfile
|
||||
@@ -151,13 +191,17 @@ expect a 15–20 min maintenance window and **never panic-restart mid-migration*
|
||||
2. Generate a temp key with the master key → if `/key/generate` 500s, the schema is incomplete; fix the migration before anything else
|
||||
3. Use the **temp key** (not master) for `GET /v1/models` → full catalog
|
||||
4. Use the **temp key** for a small `POST /v1/chat/completions` → content + usage
|
||||
5. OPTIONS preflight from a foreign origin → 400; from `chat.inference.coop` → allowed
|
||||
5. OPTIONS preflight: `LITELLM_CORS_ORIGINS` is set at the app level (`*` by co-op decision, co-op/support#38) — a preflight from any origin answers with the matching `access-control-allow-origin` header rather than a 400. Auth is the bearer key, not the origin.
|
||||
6. Cloudron app health = healthy, then delete the temp key
|
||||
|
||||
### Before updating
|
||||
|
||||
- Check the [LiteLLM changelog](https://github.com/BerriAI/litellm/releases) for breaking changes
|
||||
- The `start.sh` and `config.yaml` may need adjustment if LiteLLM changes its env-var or config schema
|
||||
- **Build-time patches ride on LiteLLM internals — verify each one after an upgrade.** `patch_budget_message.py`, `patch_image_cost.py` (keeps image-gen pricing on the shared cost-map key) and `patch_audio_request_cost.py` (adds the declared per-request charge on the audio paths: Tinfoil bills `voxtral-tts` $0.002 and `whisper-large-v3-turbo` $0.01 per request, which the cost map cannot express). Each prints `[coop-patch]` in the build log. They are non-fatal by design: a missing anchor warns and the build still succeeds, so the behaviour can vanish silently on a version bump. Related: code/litellm#5.
|
||||
- **`patch_passthrough_token_cost.py`** prices a generic pass-through route from the token count it reports in its own response body. A `pass_through_endpoints` target whose host matches none of vertex/anthropic/openai/gemini resolves to `EndpointType.GENERIC`: it is logged, but with no model on the route LiteLLM only knows the flat `cost_per_request`, which is a budget estimate, not an invoice. The patch bills any route listed in `general_settings.coop_passthrough_cost_per_token` (USD per input token, keyed by path) as `usage.input_tokens × rate` — Metis (`/v1/systemone`) at $0.12 per 1M input tokens — and records the same count on the row (see below). Same rules: non-fatal, `[coop-patch]` in the build log, verify after every upgrade. `cost_per_request` stays as the fallback for a listed route whose response carries no token count. The block prints `[coop-passthrough-cost] input_tokens=… rate=… -> response_cost=…` on each priced call, so the log is the field check that it fired.
|
||||
- **Pass-through spend rows also carry the token counts (Metis defect fix).** Pricing the row from the body was not enough: the row recorded the correct `spend` but `prompt_tokens`/`completion_tokens`/`total_tokens` stayed `0`, because the pass-through success handler hands the spend writer a `StandardPassThroughResponseObject` whose `usage` is empty and the generic branch builds no `standard_logging_object` usage. Anything summing tokens for the route (panel, dashboard, energy reporting) then under-reported. The same patch now sets `combined_usage_object` (a `litellm.Usage`) on the logging object — the override the streaming handler already uses for recovered partial usage — which the spend writer reads before the response object (`spend_tracking_utils.get_logging_payload`, `StandardLoggingPayloadSetup.get_usage_as_dict`). The row now records `prompt_tokens = usage.input_tokens`, `completion_tokens = 0`, `total_tokens = usage.input_tokens`, with spend unchanged.
|
||||
- **A zero-spend row with `call_type = /v1/<path>` is NOT a duplicate of the priced row — it is the 401 refusal.** Every *rejected* pass-through request (missing/invalid key) is logged by upstream LiteLLM as a failure row whose `call_type` is the route path and whose `spend` is `0`, via `ProxyLogging._handle_logging_proxy_only_error` (`litellm/proxy/utils.py`, `original_function=route`). That is upstream behaviour on the pristine image, not something our patches add (neither `patch_passthrough_key_access.py` nor `patch_passthrough_token_cost.py` writes a row). A single *authenticated* call produces exactly **one** row (`call_type=pass_through_endpoint`, the real spend). Measured: one 200 → one row; one unauthenticated 401 → one failure row. Count requests for the route by `call_type = pass_through_endpoint`, not by summing all rows carrying the model name.
|
||||
- The CORS env var is `LITELLM_CORS_ORIGINS` (upstream-native since 1.84.0); the old `LITELLM_CORS_ALLOWED_ORIGINS` + sed patch are dead — don't re-add them
|
||||
- Test on a staging instance if the version jump is large
|
||||
|
||||
@@ -169,13 +213,13 @@ expect a 15–20 min maintenance window and **never panic-restart mid-migration*
|
||||
|----------|----------|---------|
|
||||
| `TINFOIL_API_KEY` | Yes | Tinfoil provider (TEE inference) |
|
||||
| `GREENPT_API_KEY` | Yes | GreenPT provider (renewable energy) |
|
||||
| `PUBLICAI_API_KEY` | Yes | PublicAI provider (public models) |
|
||||
| `LITELLM_CORS_ORIGINS` | Set by start.sh | CORS allowlist (chat origin only) |
|
||||
| `REGOLO_API_KEY` | Yes | Regolo provider (EU, renewable + ZDR) |
|
||||
| `LITELLM_CORS_ORIGINS` | Set at app level | Currently `*` by co-op decision (co-op/support#38); `start.sh` falls back to the chat origin only if the app env omits it |
|
||||
| `LITELLM_MIGRATION_DIR` | Set by start.sh | Persistent Prisma migrations ledger |
|
||||
|
||||
## Connecting chat clients
|
||||
|
||||
Open WebUI does **not** talk to this gateway directly — it routes through the [member portal](https://git.inference.coop/code/member-portal), which authenticates the member and injects their personal LiteLLM key per request. Members building their own tools create API keys on the [member dashboard](https://dashboard.inference.coop) and use:
|
||||
LibreChat (chat) does **not** talk to this gateway directly — it routes through the [member portal](https://git.inference.coop/code/member-portal), which authenticates the member and injects their personal LiteLLM key per request. Members building their own tools create API keys on the [member dashboard](https://dashboard.inference.coop) and use:
|
||||
|
||||
- **API Base URL:** `https://gateway.inference.coop/v1` (or the portal's proxy endpoint for chat-integrated tools)
|
||||
- **API Key:** their own virtual key from the dashboard
|
||||
|
||||
+5
-3
@@ -2,10 +2,12 @@
|
||||
"""Audio keep-alive: ping the Tinfoil TEE audio models through localhost LiteLLM
|
||||
so voice mode doesn't pay the ~24s cold-start after idle.
|
||||
|
||||
Runs INSIDE the gateway container via the Cloudron scheduler addon. Uses the
|
||||
LITELLM_MASTER_KEY already in the container env. TTS-only (STT pings cost
|
||||
Runs INSIDE the gateway container, invoked by the keep-alive loop in
|
||||
start.sh (every 15 min; the Cloudron scheduler task was removed
|
||||
2026-11-09 so the loop is the only warm path). Reads the master key from
|
||||
env or /app/data/.master_key. TTS-only (STT pings cost
|
||||
$0.01/request each; TTS $0.002 — see docs, keep-alive economics). Silent on
|
||||
success; prints + exits non-zero on failure so scheduler logs capture it.
|
||||
success; prints + exits non-zero on failure so keepalive.log captures it.
|
||||
"""
|
||||
import json, os, urllib.request, urllib.error, sys
|
||||
|
||||
|
||||
+282
-40
@@ -6,10 +6,20 @@
|
||||
# via environment variable TINFOIL_API_KEY.
|
||||
#
|
||||
# Pricing (model_info) is per-token, derived from provider per-1M rates:
|
||||
# greenpt/deepseek-v4.1-flash: $0.22 in / $1.10 out / $0.011 cached per 1M
|
||||
# EUR->USD FX = 1.16 (policy, set 2026-11-18). GreenPT and Regolo quote EUR;
|
||||
# LiteLLM's cost fields are USD-shaped, so EUR list prices are converted here.
|
||||
# Never paste EUR numerals into model_info — that under-bills members by the FX gap.
|
||||
# greenpt/deepseek-v4.1-flash: $0.36 in / $1.45 out / $0.04 cached per 1M (1017 snapshot)
|
||||
# tinfoil/deepseek-v4-1-flash: $0.65 in / $1.45 out / $0.13 cached per 1M
|
||||
# tinfoil/gpt-oss-120b: $0.15 in / $0.60 out per 1M
|
||||
# tinfoil/glm-5.3: $1.80 in / $5.75 out / $0.45 cached per 1M
|
||||
# greenpt/qwen3-embedding-8b: $0.25 in / $0 out per 1M (embeddings, 32K ctx)
|
||||
# greenpt/green-embeddings: $0.20 in / $0 out per 1M (embeddings, 32K ctx)
|
||||
# tinfoil/nomic-embed-text: $0.016 in / $0 out per 1M (embeddings, 8K ctx, TEE)
|
||||
# regolo/mistral-small-4-119b: $0.20 in / $0.70 out per 1M (60K ctx)
|
||||
# regolo/gemma4-31b: $0.40 in / $0.60 out per 1M (100K ctx)
|
||||
# tinfoil/voxtral-small-24b: $0.20 in / $0.60 out per 1M (token-priced audio;
|
||||
# +$0.00008/request on /v1/audio/transcriptions)
|
||||
# Without these, LiteLLM can't price these custom models, so spend shows $0.00
|
||||
# and the team budget cap is not enforced.
|
||||
#
|
||||
@@ -19,20 +29,25 @@
|
||||
# models.md is the member-facing mirror.
|
||||
|
||||
model_list:
|
||||
# DeepSeek V4.1 Flash on GreenPT — DEFAULT model. Same model family as Tinfoil's
|
||||
# DeepSeek V4.1 Flash on GreenPT — DEFAULT model. SERVES THE 1017 SNAPSHOT
|
||||
# (17 Oct 2026) as of 2026-11-16, while KEEPING its version-free model_name: members
|
||||
# pin this id in client configs we cannot see, and renaming would 400 every one of
|
||||
# them (measured: 3,014 calls in 2 days on this name, 84 on the deepseek-flash-green
|
||||
# alias). Repointing is the whole change — no member action.
|
||||
# Same model family as Tinfoil's
|
||||
# deepseek-v4-1-flash, but on renewable energy and NOT marked Experimental by the
|
||||
# provider (Tinfoil's deepseek + glm-flash carry an "Experimental: may be deprecated
|
||||
# with minimal notice" flag). Policy-based privacy (GreenPT) vs TEE (Tinfoil).
|
||||
# €0.22 in / €1.10 out / €0.011 cached per 1M.
|
||||
# $0.36 in / $1.45 out / $0.04 cached per 1M (1017 snapshot; repointed 2026-11-16).
|
||||
- model_name: greenpt/deepseek-v4.1-flash
|
||||
litellm_params:
|
||||
model: openai/deepseek-v4.1-flash
|
||||
model: openai/deepseek-v4.1-flash-1017
|
||||
api_base: https://api.greenpt.ai/v1
|
||||
api_key: os.environ/GREENPT_API_KEY
|
||||
model_info:
|
||||
input_cost_per_token: 0.00000022
|
||||
output_cost_per_token: 0.00000110
|
||||
cache_read_input_token_cost: 0.000000011
|
||||
input_cost_per_token: 4.176e-07
|
||||
output_cost_per_token: 1.682e-06
|
||||
cache_read_input_token_cost: 4.64e-08
|
||||
|
||||
# DeepSeek V4.1 Flash on Tinfoil (TEE) — private counterpart to the default.
|
||||
- model_name: tinfoil/deepseek-v4-1-flash
|
||||
@@ -81,9 +96,9 @@ model_list:
|
||||
api_base: https://api.greenpt.ai/v1
|
||||
api_key: os.environ/GREENPT_API_KEY
|
||||
model_info:
|
||||
input_cost_per_token: 0.00000011
|
||||
output_cost_per_token: 0.00000044
|
||||
cache_read_input_token_cost: 0.000000022
|
||||
input_cost_per_token: 1.276e-07
|
||||
output_cost_per_token: 5.104e-07
|
||||
cache_read_input_token_cost: 2.552e-08
|
||||
|
||||
# GLM-5.3 (full flagship) on GreenPT — strongest agentic model in the green
|
||||
# tier. $1.10 in / $4.40 out per 1M, 1M context.
|
||||
@@ -93,17 +108,20 @@ model_list:
|
||||
api_base: https://api.greenpt.ai/v1
|
||||
api_key: os.environ/GREENPT_API_KEY
|
||||
model_info:
|
||||
input_cost_per_token: 0.0000011
|
||||
output_cost_per_token: 0.0000044
|
||||
cache_read_input_token_cost: 0.000000275
|
||||
input_cost_per_token: 1.276e-06
|
||||
output_cost_per_token: 5.104e-06
|
||||
cache_read_input_token_cost: 3.19e-07
|
||||
|
||||
# --- Audio models (Tinfoil, inside the TEE) ---
|
||||
# NOT chat models: these power /v1/audio/* routes (STT for dictation, TTS for
|
||||
# voice mode). They stay VISIBLE in the API model list so developer members
|
||||
# can call them, but the portal filters them out of the chat-facing list
|
||||
# (Open WebUI) by their `mode` field, so they never appear as pickable chat
|
||||
# options. Pricing per Tinfoil catalog: whisper $0.05/1M in; voxtral-tts
|
||||
# listed $0/$0 (verify on invoice — may be beta-free).
|
||||
# options. Tinfoil bills audio per REQUEST, not per token: whisper
|
||||
# $0.01/request, voxtral-tts $0.002/request (both $0 per token). LiteLLM cannot
|
||||
# price that natively, so each entry declares input_cost_per_request and the
|
||||
# build-time patches make the calculator honour it (issue #5, closed 2026-11-17
|
||||
# with a verified $0.002 spend row).
|
||||
|
||||
# --- Regolo (EU, zero data retention, 100% renewables) ---
|
||||
# Seeweb S.r.l. (Italy). Added 2026-10-16 following member suggestion (#22).
|
||||
@@ -112,6 +130,10 @@ model_list:
|
||||
# qwen3.5-122b: large vision+reasoning MoE (262K ctx). REQUIRES
|
||||
# chat_template_kwargs: {enable_thinking: false} — otherwise the reasoning
|
||||
# phase consumes the entire token budget and content comes back empty.
|
||||
# mistral-small-4-119b + gemma4-31b: added 2026-11-13 (operator-approved).
|
||||
# Rates read live from Regolo's own model catalog (regolo.ai/models):
|
||||
# mistral $0.20/$0.70 per 1M, 60K in / 60K out; gemma4 $0.40/$0.60,
|
||||
# 100K in / 100K out.
|
||||
# Qwen-Image: image generation via /v1/images/generations, billed per pixel.
|
||||
- model_name: regolo/apertus-70b
|
||||
litellm_params:
|
||||
@@ -120,8 +142,8 @@ model_list:
|
||||
api_key: os.environ/REGOLO_API_KEY
|
||||
model_info:
|
||||
mode: chat
|
||||
input_cost_per_token: 4.6e-7
|
||||
output_cost_per_token: 2.44e-6
|
||||
input_cost_per_token: 1.16e-6
|
||||
output_cost_per_token: 3.944e-6
|
||||
|
||||
- model_name: regolo/qwen3.5-122b
|
||||
litellm_params:
|
||||
@@ -131,8 +153,32 @@ model_list:
|
||||
chat_template_kwargs: {"enable_thinking": false}
|
||||
model_info:
|
||||
mode: chat
|
||||
input_cost_per_token: 1.16e-6
|
||||
output_cost_per_token: 4.87e-6
|
||||
input_cost_per_token: 6.96e-7
|
||||
output_cost_per_token: 3.944e-6
|
||||
|
||||
# Mistral Small 4 (119B) — hybrid instruct + reasoning model (three model
|
||||
# families unified by Regolo). $0.20 in / $0.70 out per 1M, 60K ctx.
|
||||
- model_name: regolo/mistral-small-4-119b
|
||||
litellm_params:
|
||||
model: openai/mistral-small-4-119b
|
||||
api_base: https://api.regolo.ai/v1
|
||||
api_key: os.environ/REGOLO_API_KEY
|
||||
model_info:
|
||||
mode: chat
|
||||
input_cost_per_token: 2.32e-07
|
||||
output_cost_per_token: 8.12e-07
|
||||
|
||||
# Gemma 4 (31B) — Google's dense multimodal Gemma 4. $0.40 in / $0.60 out
|
||||
# per 1M, 100K ctx.
|
||||
- model_name: regolo/gemma4-31b
|
||||
litellm_params:
|
||||
model: openai/gemma4-31b
|
||||
api_base: https://api.regolo.ai/v1
|
||||
api_key: os.environ/REGOLO_API_KEY
|
||||
model_info:
|
||||
mode: chat
|
||||
input_cost_per_token: 4.64e-07
|
||||
output_cost_per_token: 6.96e-07
|
||||
|
||||
- model_name: regolo/qwen-image
|
||||
litellm_params:
|
||||
@@ -141,20 +187,23 @@ model_list:
|
||||
api_key: os.environ/REGOLO_API_KEY
|
||||
model_info:
|
||||
mode: image_generation
|
||||
input_cost_per_pixel: 1.5e-8
|
||||
input_cost_per_pixel: 1.74e-8
|
||||
|
||||
# --- Audio models (Tinfoil, in-TEE) ---
|
||||
# Used for chat dictation (STT) and voice mode (TTS). These are NOT chat
|
||||
# models: the member portal's /v1/models filters them out of the chat model
|
||||
# list (see AUDIO_MODEL_PREFIXES there) so members don't see them in the
|
||||
# picker, but they remain fully callable via /v1/audio/* for API users.
|
||||
# whisper-large-v3-turbo: $0.05/1M input tokens. voxtral-tts: Tinfoil's
|
||||
# catalog lists it $0/$0 and every cost field was left NULL, so speech usage
|
||||
# never consumed member allowance. Billed at the model's published rate
|
||||
# instead: $16/1M characters, i.e. LiteLLM's own bundled price for the same
|
||||
# weights (mistral/voxtral-mini-tts-*, sourced to Mistral's model card) and
|
||||
# the rate OpenAI's TTS entries use for the same field. Revisit if the
|
||||
# Tinfoil invoice shows a different number.
|
||||
# whisper-large-v3-turbo: $0/1M tokens + $0.01 per request.
|
||||
# voxtral-tts: $0/1M tokens + $0.002 per request.
|
||||
# voxtral-small-24b: $0.20/1M in + $0.60/1M out tokens, $0 per request —
|
||||
# the only TOKEN-PRICED audio model in their catalog (its block below).
|
||||
# Tinfoil bills these per REQUEST (its model list carries a `requestPrice`);
|
||||
# reading only the two token prices made TTS look free and left STT billed off
|
||||
# an older catalog. LiteLLM cannot price per request on the audio paths, so
|
||||
# patch_audio_request_cost.py (built in via the Dockerfile) makes the cost
|
||||
# calculator honour `input_cost_per_request` below. Re-check that build log
|
||||
# line after every LiteLLM upgrade.
|
||||
# NOTE: LiteLLM routes audio by the openai/ provider + the endpoint called
|
||||
# (/v1/audio/transcriptions or /v1/audio/speech) — there is no
|
||||
# "audio_transcription/" provider prefix.
|
||||
@@ -165,7 +214,38 @@ model_list:
|
||||
api_key: os.environ/TINFOIL_API_KEY
|
||||
model_info:
|
||||
mode: audio_transcription
|
||||
input_cost_per_token: 0.00000005
|
||||
# Tinfoil: $0 per token, $0.01 per request. The per-request charge is the
|
||||
# whole cost, and the patch is what makes it billable.
|
||||
input_cost_per_token: 0
|
||||
input_cost_per_request: 0.01
|
||||
|
||||
# voxtral-small-24b — Tinfoil's TOKEN-PRICED STT model: the only audio entry
|
||||
# in their catalog with real token prices ($0.20 in / $0.60 out per 1M, $0
|
||||
# per request). The chat route (audio sent as input_audio) returns real
|
||||
# token usage and bills exactly at those rates. The /v1/audio/transcriptions
|
||||
# route returns NO token counts (duration only), so per-token billing cannot
|
||||
# fire there; Tinfoil tokenizes audio in ~30s chunks (~375-380 tokens each —
|
||||
# measured: a 1-second clip and a 6-second clip both cost ~380 prompt
|
||||
# tokens), so for the short dictation requests this route serves the
|
||||
# chunk-equivalent is ~$0.00008/request — declared as input_cost_per_request
|
||||
# below, added by patch_audio_request_cost.py, which is scoped to the audio
|
||||
# call types so chat calls to this model are NOT double-billed. Audio longer
|
||||
# than ~30s should go through the chat route, which meters exactly.
|
||||
- model_name: tinfoil/voxtral-small-24b
|
||||
litellm_params:
|
||||
model: openai/voxtral-small-24b
|
||||
api_base: http://127.0.0.1:3301/v1
|
||||
api_key: os.environ/TINFOIL_API_KEY
|
||||
model_info:
|
||||
mode: audio_transcription
|
||||
# Opt out of LiteLLM's forced response_format=verbose_json injection on
|
||||
# transcription calls (patch_transcription_response_format.py): Tinfoil
|
||||
# rejects verbose_json for this model with a 400, and nothing in its
|
||||
# billing needs the duration the injection exists to obtain.
|
||||
coop_no_verbose_json: true
|
||||
input_cost_per_token: 0.0000002
|
||||
output_cost_per_token: 0.0000006
|
||||
input_cost_per_request: 0.00008
|
||||
|
||||
- model_name: tinfoil/voxtral-tts
|
||||
litellm_params:
|
||||
@@ -174,18 +254,31 @@ model_list:
|
||||
api_key: os.environ/TINFOIL_API_KEY
|
||||
model_info:
|
||||
mode: audio_speech
|
||||
# $16 per 1M characters. This is the field LiteLLM reads for speech on the
|
||||
# openai adapter (same key OpenAI's tts-1 uses upstream), so member spend
|
||||
# and budget caps now account for voice mode.
|
||||
input_cost_per_character: 0.000016
|
||||
# Tinfoil: $0 per token, $0.002 per request — see the audio section note.
|
||||
#
|
||||
# input_cost_per_character: 0 is load-bearing, not decoration. The TTS cost
|
||||
# branch (cost_per_token, cost_metric == "cost_per_character") raises
|
||||
# ValueError("cost for tts call is None") when there is no character price,
|
||||
# and that raise aborts the whole cost calculation — so the spend write is
|
||||
# skipped and speech billed nothing, whatever input_cost_per_request said
|
||||
# (that was issue #5; the build patch now prices the base at 0.0 for a
|
||||
# per-request model, so the declared charge is the whole bill).
|
||||
# Declaring the zero lets the path complete; the patch then adds the
|
||||
# per-request charge.
|
||||
input_cost_per_character: 0
|
||||
input_cost_per_request: 0.002
|
||||
|
||||
# --- Embedding model (Tinfoil, in-TEE) ---
|
||||
# --- Embedding models (Tinfoil, in-TEE; and GreenPT) ---
|
||||
# nomic-embed-text — 768-dim text embeddings for RAG/semantic search.
|
||||
# NOT a chat model: it powers /v1/embeddings only. Scoped to the admin
|
||||
# account's team (model_info.team_id) so member keys cannot call it — it is
|
||||
# an operator resource for the co-op's own docs-RAG, not a member-facing
|
||||
# model. The portal also filters it out of the chat-facing /v1/models list by
|
||||
# its `mode: embedding`. Pricing per Tinfoil catalog: $0.05/1M input, $0 out.
|
||||
# NOT a chat model: it powers /v1/embeddings only. MEMBER-FACING since
|
||||
# 2026-11-06 — the team_id scoping (operator-account-only) was removed so
|
||||
# members get a TEE-protected embedding option. This is the same model the
|
||||
# co-op's own docs-RAG runs on inside the TEE (Open WebUI's knowledge base);
|
||||
# that traffic is unaffected by members also being able to call it. The
|
||||
# portal filters `mode: embedding` out of the chat-facing /v1/models list, so
|
||||
# it stays API-only. Pricing per Tinfoil catalog (inference.tinfoil.sh
|
||||
# /v1/models): $0.016/1M input, $0 out — corrected 2026-11-06 from a stale
|
||||
# $0.05 declaration; 8K context declared so clients don't invent larger.
|
||||
- model_name: tinfoil/nomic-embed-text
|
||||
litellm_params:
|
||||
model: openai/nomic-embed-text
|
||||
@@ -193,10 +286,53 @@ model_list:
|
||||
api_key: os.environ/TINFOIL_API_KEY
|
||||
model_info:
|
||||
mode: embedding
|
||||
team_id: cdfdf422-633f-4432-8893-cffee82a8325
|
||||
input_cost_per_token: 0.00000005
|
||||
max_input_tokens: 8192
|
||||
input_cost_per_token: 0.000000016
|
||||
output_cost_per_token: 0.0
|
||||
|
||||
# --- Embedding model (GreenPT, member-facing) ---
|
||||
# qwen3-embedding-8b — text embeddings via /v1/embeddings, callable by any
|
||||
# member key. Added 2026-11-06 after a member asked for an embedding model,
|
||||
# filling the member-facing gap on renewable EU hosting (Tinfoil's private
|
||||
# embedding was opened to members the same day). The portal filters
|
||||
# mode:embedding out of the chat picker, so it is API-only — same shape as
|
||||
# the audio models above.
|
||||
# Pricing per GreenPT /v1/pricing: $0.25/1M input, no output charge
|
||||
# (embeddings return vectors; GreenPT lists maxOutputTokens null), 32K
|
||||
# context. max_input_tokens declared from the provider's contextWindowTokens
|
||||
# so clients don't invent a larger one (the null-caps defect).
|
||||
- model_name: greenpt/qwen3-embedding-8b
|
||||
litellm_params:
|
||||
model: openai/qwen3-embedding-8b
|
||||
api_base: https://api.greenpt.ai/v1
|
||||
api_key: os.environ/GREENPT_API_KEY
|
||||
model_info:
|
||||
mode: embedding
|
||||
input_cost_per_token: 2.9e-07
|
||||
output_cost_per_token: 0
|
||||
max_input_tokens: 32000
|
||||
|
||||
# --- Embedding model (GreenPT, member-facing) ---
|
||||
# green-embeddings — GreenPT's own embedding model (2560-dim, measured live).
|
||||
# Added 2026-11-06 after a catalog survey: the only genuinely distinct
|
||||
# embedding model on GreenPT besides qwen3-embedding-8b. Their catalog also
|
||||
# lists a `green-embedding` (singular) slug — verified the SAME model
|
||||
# (identical vectors on the same input), so only green-embeddings is served.
|
||||
# Cheaper per token than qwen3-embedding-8b; the reranker sibling
|
||||
# (green-rerank) is a different capability and is NOT served.
|
||||
# Same shape: /v1/embeddings only, filtered out of the chat picker.
|
||||
# Pricing per GreenPT /v1/pricing: $0.20/1M input, no output charge, 32K ctx.
|
||||
- model_name: greenpt/green-embeddings
|
||||
litellm_params:
|
||||
model: openai/green-embeddings
|
||||
api_base: https://api.greenpt.ai/v1
|
||||
api_key: os.environ/GREENPT_API_KEY
|
||||
model_info:
|
||||
mode: embedding
|
||||
input_cost_per_token: 2.32e-07
|
||||
output_cost_per_token: 0
|
||||
max_input_tokens: 32000
|
||||
|
||||
# Fallback: if DeepSeek is down, use GPT-OSS
|
||||
# NOTE: DeepSeek V4.1 Flash is a reasoning model — generations routinely run
|
||||
# longer than 30s. A 30s timeout caused LiteLLM to abandon in-flight DeepSeek
|
||||
@@ -206,6 +342,58 @@ model_list:
|
||||
# logged; and a retry_policy that retries transient connection errors (base
|
||||
# num_retries) but NEVER retries timeouts (TimeoutErrorRetries: 0) — a timed-out
|
||||
# generation is already billed by Tinfoil, so re-hitting it re-bills.
|
||||
|
||||
# --- Stable aliases -------------------------------------------------------
|
||||
# Version-free names members can pin once and keep using across transitions.
|
||||
# When we move the underlying model, only the pointer here changes; member
|
||||
# client configs and the docs do not. Pricing is stated AT the alias, so a
|
||||
# transition that changes price is visible in the model list and in budgets
|
||||
# (LiteLLM bills the alias's model_info, not the target's).
|
||||
# Policy: `<family>-green` = the default route (renewable energy, policy-based
|
||||
# privacy); `<family>-tee` = the same family served inside a TEE (attested
|
||||
# hardware). The names say WHICH ROUTE, not how fresh a version is: we re-point
|
||||
# them at transitions, so pinning one is what makes a config version-free.
|
||||
- model_name: deepseek-flash-green
|
||||
litellm_params:
|
||||
model: openai/deepseek-v4.1-flash-1017
|
||||
api_base: https://api.greenpt.ai/v1
|
||||
api_key: os.environ/GREENPT_API_KEY
|
||||
model_info:
|
||||
input_cost_per_token: 4.176e-07
|
||||
output_cost_per_token: 1.682e-06
|
||||
cache_read_input_token_cost: 4.64e-08
|
||||
|
||||
- model_name: deepseek-flash-tee
|
||||
litellm_params:
|
||||
model: openai/deepseek-v4-1-flash
|
||||
api_base: http://127.0.0.1:3301/v1
|
||||
api_key: os.environ/TINFOIL_API_KEY
|
||||
model_info:
|
||||
input_cost_per_token: 0.00000065
|
||||
output_cost_per_token: 0.00000145
|
||||
cache_read_input_token_cost: 0.00000013
|
||||
|
||||
- model_name: glm-flash-green
|
||||
litellm_params:
|
||||
model: openai/glm-5.3-flash
|
||||
api_base: https://api.greenpt.ai/v1
|
||||
api_key: os.environ/GREENPT_API_KEY
|
||||
model_info:
|
||||
input_cost_per_token: 1.276e-07
|
||||
output_cost_per_token: 5.104e-07
|
||||
cache_read_input_token_cost: 2.552e-08
|
||||
|
||||
# Tinfoil's flash GLM was deprecated 2026-10-09; the TEE route for this family
|
||||
# is the flagship, so the alias is named glm-tee rather than glm-flash-tee.
|
||||
- model_name: glm-tee
|
||||
litellm_params:
|
||||
model: openai/glm-5-3
|
||||
api_base: http://127.0.0.1:3301/v1
|
||||
api_key: os.environ/TINFOIL_API_KEY
|
||||
model_info:
|
||||
input_cost_per_token: 0.00000180
|
||||
output_cost_per_token: 0.00000575
|
||||
cache_read_input_token_cost: 0.00000045
|
||||
router_settings:
|
||||
num_retries: 2
|
||||
timeout: 180
|
||||
@@ -222,6 +410,60 @@ general_settings:
|
||||
database_url: os.environ/DATABASE_URL
|
||||
store_model_in_db: true
|
||||
|
||||
# --- Metis (GreenPT) — decision model, not a chat model ---
|
||||
# Metis is a *decision* endpoint, not a chat completion: it takes a `state`
|
||||
# plus up to 32 typed questions and returns PROBABILITIES / choices / scores.
|
||||
# It generates no text (output_tokens is always 0), so it cannot be a
|
||||
# model_list entry — LiteLLM's chat pipeline has nothing to parse. It is
|
||||
# exposed as a pass-through route instead, and members call it with their
|
||||
# normal co-op API key (auth: true), never the provider key.
|
||||
#
|
||||
# GreenPT-hosted (policy-based privacy, EU, 100% renewable), same
|
||||
# GREENPT_API_KEY the green chat models already use.
|
||||
# Price: $0.12 per 1M *input* tokens; output is free.
|
||||
#
|
||||
# METERING: exact, from the token count GreenPT reports back. Metis is not a
|
||||
# model_list entry (LiteLLM's chat pipeline has nothing to parse), so it is
|
||||
# priced through the coop_passthrough_cost_per_token map below, keyed by this
|
||||
# route's path: a call's cost is usage.input_tokens × the rate. patch_
|
||||
# passthrough_token_cost.py (built in via the Dockerfile) reads
|
||||
# usage.input_tokens from the Metis response body at logging time and writes
|
||||
# the exact figure to member spend. cost_per_request on the route is now only
|
||||
# a fallback: it applies if a response ever arrives without a usable token
|
||||
# count, so a call is never silently free. Keep the two in sync.
|
||||
#
|
||||
# Routes here (matched by prefix) may be called by ANY authenticated member
|
||||
# key. LiteLLM's own gate for auth=true pass-through routes
|
||||
# (`allowed_passthrough_routes`) is an Enterprise-only field, so on this
|
||||
# community-edition proxy it can never be granted to member keys — the only
|
||||
# other option, auth:false, would expose the route publicly and forward the
|
||||
# co-op's GreenPT key to anonymous callers. This list is consumed at build
|
||||
# time by patch_passthrough_key_access.py; authentication is unchanged (the
|
||||
# endpoint still requires a valid key).
|
||||
coop_passthrough_open_to_keys:
|
||||
- "/v1/systemone"
|
||||
# Exact per-input-token pricing (USD per token) for pass-through routes that
|
||||
# report their own usage in the RESPONSE BODY — the shape the x-litellm-*
|
||||
# response-header contract does not cover. Keyed by the gateway route path
|
||||
# (an upstream target URL also matches). Any route listed here is billed from
|
||||
# usage.input_tokens × the rate instead of its flat cost_per_request; a route
|
||||
# not listed is unchanged. Consumed at build time by
|
||||
# patch_passthrough_token_cost.py.
|
||||
coop_passthrough_cost_per_token:
|
||||
# Metis (GreenPT): $0.12 per 1M input tokens, output always free.
|
||||
"/v1/systemone": 0.00000012
|
||||
pass_through_endpoints:
|
||||
- path: "/v1/systemone"
|
||||
target: "https://api.greenpt.ai/v1/systemone"
|
||||
headers:
|
||||
Authorization: "Bearer os.environ/GREENPT_API_KEY"
|
||||
Content-Type: "application/json"
|
||||
auth: true
|
||||
methods: ["POST"]
|
||||
timeout: 60
|
||||
# Fallback only — the exact token cost above is what bills in practice.
|
||||
cost_per_request: 0.0001
|
||||
|
||||
litellm_settings:
|
||||
salt_key: os.environ/LITELLM_SALT_KEY
|
||||
drop_params: true
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"_comment": "EUR list prices per 1M tokens (providers quoting EUR). fx_sync.py converts at the ECB daily rate at boot. USD values in config.yaml are fallbacks.",
|
||||
"models": {
|
||||
"greenpt/deepseek-v4.1-flash": {"input": 0.36, "output": 1.45, "cache": 0.04},
|
||||
"greenpt/glm-5.3-flash": {"input": 0.11, "output": 0.44, "cache": 0.022},
|
||||
"greenpt/glm-5.3": {"input": 1.10, "output": 4.40, "cache": 0.275},
|
||||
"greenpt/green-embeddings": {"input": 0.20, "output": 0.0},
|
||||
"greenpt/qwen3-embedding-8b": {"input": 0.25, "output": 0.0},
|
||||
"deepseek-flash-green": {"input": 0.36, "output": 1.45, "cache": 0.04},
|
||||
"glm-flash-green": {"input": 0.11, "output": 0.44, "cache": 0.022},
|
||||
"regolo/apertus-70b": {"input": 1.00, "output": 3.40},
|
||||
"regolo/qwen3.5-122b": {"input": 0.60, "output": 3.40},
|
||||
"regolo/mistral-small-4-119b": {"input": 0.20, "output": 0.70},
|
||||
"regolo/gemma4-31b": {"input": 0.40, "output": 0.60}
|
||||
}
|
||||
}
|
||||
Executable
+85
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Boot-time FX sync: rewrite USD model_info from the EUR table at the ECB rate.
|
||||
|
||||
Fail-open by design: if the feed is unreachable or unparseable, the config is left
|
||||
untouched (the repo's fallback USD values stand) and the boot continues.
|
||||
|
||||
Run by start.sh before config validation, so the validator sees the final prices.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
|
||||
CFG = "/app/data/config.yaml"
|
||||
EUR_FILE = "/app/code/eur_prices.json"
|
||||
STATE = "/app/data/fx_state.json"
|
||||
ECB_URL = os.environ.get(
|
||||
"ECB_FX_URL", "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml")
|
||||
|
||||
|
||||
def fetch_rate() -> tuple[float, str]:
|
||||
with urllib.request.urlopen(ECB_URL, timeout=20) as r:
|
||||
body = r.read().decode("utf-8", "ignore")
|
||||
m = re.search(r"currency=['\"]USD['\"]\s+rate=['\"]([0-9.]+)['\"]", body)
|
||||
if not m:
|
||||
raise ValueError("USD rate not found in ECB feed")
|
||||
d = re.search(r"time=['\"](\d{4}-\d{2}-\d{2})['\"]", body)
|
||||
return float(m.group(1)), (d.group(1) if d else "")
|
||||
|
||||
|
||||
def usd(eur: float, rate: float) -> float:
|
||||
return 0.0 if eur == 0 else eur * rate / 1_000_000
|
||||
|
||||
|
||||
def set_cost(text: str, name: str, key: str, value: float) -> tuple[str, bool]:
|
||||
pat = re.compile(r"( - model_name: " + re.escape(name) +
|
||||
r"\n)(.*?)(?=\n - model_name: |\n[a-zA-Z#]|\Z)", re.S)
|
||||
m = pat.search(text)
|
||||
if not m:
|
||||
return text, False
|
||||
block = m.group(2)
|
||||
new = f"{value:.10g}"
|
||||
block2, n = re.subn(key + r": \S+", key + ": " + new, block, count=1)
|
||||
if n == 0:
|
||||
return text, False
|
||||
return text[:m.start(2)] + block2 + text[m.end(2):], True
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
rate, feed_date = fetch_rate()
|
||||
eur = json.load(open(EUR_FILE, encoding="utf-8"))["models"]
|
||||
text = open(CFG, encoding="utf-8").read()
|
||||
changed = []
|
||||
for name, prices in eur.items():
|
||||
for key, eur_val in (("input_cost_per_token", prices.get("input")),
|
||||
("output_cost_per_token", prices.get("output"))):
|
||||
if eur_val is None:
|
||||
continue
|
||||
text, ok = set_cost(text, name, key, usd(eur_val, rate))
|
||||
if ok:
|
||||
changed.append(f"{name}:{key}")
|
||||
if prices.get("cache") is not None:
|
||||
text, ok = set_cost(text, name, "cache_read_input_token_cost",
|
||||
usd(prices["cache"], rate))
|
||||
if ok:
|
||||
changed.append(f"{name}:cache")
|
||||
tmp = CFG + ".fx.tmp"
|
||||
open(tmp, "w", encoding="utf-8").write(text)
|
||||
os.replace(tmp, CFG)
|
||||
json.dump({"rate": rate, "feed_date": feed_date,
|
||||
"checked_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||
"fields": len(changed)},
|
||||
open(STATE, "w", encoding="utf-8"), indent=2)
|
||||
print(f"[fx-sync] ECB USD/EUR={rate} ({feed_date}); updated {len(changed)} fields")
|
||||
return 0
|
||||
except Exception as exc: # fail-open
|
||||
print(f"[fx-sync] FAIL-OPEN: {type(exc).__name__}: {str(exc)[:160]}; keeping existing prices")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Put Tinfoil's per-request audio prices in the SHARED cost map, at BUILD time.
|
||||
|
||||
Why this exists (issue #5): the cost calculator's per-request helper reads
|
||||
``litellm.model_cost`` — the map built from model_prices_and_context_window.json —
|
||||
and our config's ``input_cost_per_request`` never reaches it. Patching the router
|
||||
to keep per-deployment pricing on the shared model_info entry was not enough,
|
||||
because the helper consults the JSON-backed map, not model_info. Result: every
|
||||
successful voxtral-tts speech call logged cost 0.0 (~$0.22/day of unlogged spend).
|
||||
|
||||
This is the same mechanism as patch_image_cost.py: re-state the price where the
|
||||
calculator actually looks. It FAILS THE BUILD rather than warning, because a silent
|
||||
"billed at $0" is exactly the failure mode we are trying to close.
|
||||
|
||||
Prices from the Tinfoil catalogue (2026-11-17): voxtral-tts $0.002/request,
|
||||
whisper-large-v3-turbo $0.01/request, both $0 per token.
|
||||
"""
|
||||
import glob
|
||||
import json
|
||||
import sys
|
||||
|
||||
ENTRIES = {
|
||||
"openai/voxtral-tts": {
|
||||
"mode": "audio_speech",
|
||||
"input_cost_per_request": 0.002,
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
},
|
||||
"openai/whisper-large-v3-turbo": {
|
||||
"mode": "audio_transcription",
|
||||
"input_cost_per_request": 0.01,
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# LiteLLM ships the local map as *_backup.json; cost.json is the download
|
||||
# cache. Patch whichever exists, preferring the loadable backup.
|
||||
files = (glob.glob("/app/.venv/lib/python*/site-packages/litellm/"
|
||||
"model_prices_and_context_window_backup.json")
|
||||
or glob.glob("/app/.venv/lib/python*/site-packages/litellm/"
|
||||
"model_prices_and_context_window.json")
|
||||
or glob.glob("/app/.venv/lib/python*/site-packages/litellm/cost.json"))
|
||||
if not files:
|
||||
print("[coop-patch] FAIL no local cost map found (backup / model_prices / cost.json)")
|
||||
return 1
|
||||
path = files[0]
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
data = json.load(handle)
|
||||
changed = []
|
||||
for key, entry in ENTRIES.items():
|
||||
current = data.get(key) or {}
|
||||
if all(current.get(k) == v for k, v in entry.items()):
|
||||
continue
|
||||
merged = {**current, **entry}
|
||||
data[key] = merged
|
||||
changed.append(key)
|
||||
if not changed:
|
||||
print(f"[coop-patch] audio cost map already correct ({len(ENTRIES)} entries)")
|
||||
return 0
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
json.dump(data, handle, indent=2)
|
||||
print("[coop-patch] applied audio cost map: " + ", ".join(changed))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Default /v1/audio/speech to mp3 when the caller omits response_format.
|
||||
|
||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
||||
if the target file or the anchor is missing — e.g. after a version bump — it
|
||||
prints a warning and exits 0, so the build and the gateway still succeed. Look
|
||||
for the ``[coop-patch]`` line in the build log.
|
||||
|
||||
Why: LibreChat's speech client sends no ``response_format`` on its TTS request
|
||||
(TTSService.openAIProvider builds {input, model, voice} — the field is not in
|
||||
its schema), and Tinfoil's voxtral-tts then serves WAV. Measured 2026-11-07
|
||||
through the gateway: WAV ~8-17s warm per request (145-300 KB), while the SAME
|
||||
model with ``response_format: "mp3"`` answers in ~2.6-3.1s (28-39 KB) — the
|
||||
provider streams/encodes mp3 progressively. LibreChat additionally labels the
|
||||
reply ``Content-Type: audio/mpeg`` unconditionally, so WAV bytes were served
|
||||
under an mp3 label and browsers depended on MIME sniffing to play them.
|
||||
|
||||
Patch: default ``response_format`` to ``mp3`` (OpenAI's own documented default
|
||||
for /v1/audio/speech) when the request omits it. An explicit response_format
|
||||
from any caller still wins, so API members who want WAV can keep requesting
|
||||
it. Scope: only the aspeech route; transcriptions are untouched.
|
||||
|
||||
Upstream: LiteLLM 1.103.2 forwards response_format verbatim and leaves the
|
||||
provider default in place when it is absent.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
MARKER = "CO-OP PATCH (TTS default mp3, 2026-11-07)"
|
||||
|
||||
ANCHOR = ''' ### CALL HOOKS ### - modify incoming data / reject request before calling the model
|
||||
data = await proxy_logging_obj.pre_call_hook(
|
||||
user_api_key_dict=user_api_key_dict, data=data, call_type="aspeech"
|
||||
)'''
|
||||
|
||||
INSERT = ANCHOR + '''
|
||||
|
||||
# CO-OP PATCH (TTS default mp3, 2026-11-07): LibreChat sends no
|
||||
# response_format and Tinfoil's voxtral-tts then serves WAV — measured
|
||||
# ~8-17s warm vs ~3s for mp3, which the provider streams. LibreChat
|
||||
# labels the reply audio/mpeg regardless. Default to mp3 (OpenAI's
|
||||
# documented default); an explicit response_format still wins.
|
||||
if not data.get("response_format"):
|
||||
data["response_format"] = "mp3"'''
|
||||
|
||||
|
||||
def target_path() -> str:
|
||||
if len(sys.argv) > 1:
|
||||
return sys.argv[1]
|
||||
env = os.environ.get("LITELLM_PROXY_SERVER")
|
||||
if env:
|
||||
return env
|
||||
hits = []
|
||||
try: # most reliable: ask the installed package where it lives
|
||||
import litellm # noqa: PLC0415
|
||||
_file = getattr(litellm, "__file__", None)
|
||||
if _file:
|
||||
hits.append(os.path.join(os.path.dirname(_file), "proxy", "proxy_server.py"))
|
||||
except Exception:
|
||||
pass
|
||||
hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/proxy/proxy_server.py")
|
||||
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/proxy/proxy_server.py")
|
||||
hits = [h for h in hits if os.path.exists(h)]
|
||||
if not hits:
|
||||
raise SystemExit("[coop-patch] could not locate litellm/proxy/proxy_server.py")
|
||||
return sorted(hits)[0]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
path = target_path()
|
||||
try:
|
||||
source = open(path).read()
|
||||
except OSError as exc:
|
||||
print(f"[coop-patch] cannot read {path}: {exc}")
|
||||
return 0
|
||||
|
||||
if MARKER in source:
|
||||
print(f"[coop-patch] {path} already patched, nothing to do")
|
||||
return 0
|
||||
|
||||
count = source.count(ANCHOR)
|
||||
if count != 1:
|
||||
print(f"[coop-patch] WARNING aspeech pre_call_hook anchor found {count}x (need exactly 1) "
|
||||
f"in {path}; TTS stays on the provider's default (WAV for Tinfoil)")
|
||||
return 0
|
||||
|
||||
patched = source.replace(ANCHOR, INSERT, 1)
|
||||
compile(patched, path, "exec") # refuse to write something that will not import
|
||||
|
||||
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
||||
print(f"[coop-patch] {path} is not writable; verified anchor and syntax only")
|
||||
return 0
|
||||
|
||||
with open(path, "w") as handle:
|
||||
handle.write(patched)
|
||||
print(f"[coop-patch] applied TTS default mp3 to {path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,345 @@
|
||||
"""Bill a declared per-request charge on requests LiteLLM prices only per unit.
|
||||
|
||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
||||
if the target file or the anchor strings are missing — e.g. after an upstream
|
||||
version bump — it prints a warning and exits 0, so the build and the gateway
|
||||
still succeed. The build log line ``[coop-patch] applied`` is what confirms it
|
||||
took effect; check it on every LiteLLM upgrade (see README, upgrade checklist).
|
||||
|
||||
Why: Tinfoil charges some models a flat amount PER REQUEST on top of (or instead
|
||||
of) per-token pricing — ``voxtral-tts`` $0.002/request and
|
||||
``whisper-large-v3-turbo`` $0.01/request, both with $0 per-token. LiteLLM can
|
||||
only express per-request pricing on the search / rerank / MCP paths, via
|
||||
``input_cost_per_query`` (see ``litellm/search/cost_calculator.py``, and
|
||||
``litellm/types/utils.py``: "per-request pricing: rerank, search, and Bedrock
|
||||
Marengo embed"). The audio paths price by character, second or token, and
|
||||
``input_cost_per_request`` exists only in the proxy's cost-tracking settings —
|
||||
the model cost calculator never reads it. So speech and transcription usage is
|
||||
billed at $0 however the config is written.
|
||||
|
||||
Patch: make the model cost calculator honour a declared
|
||||
``input_cost_per_request``. ``response_cost_calculator`` is the single funnel for
|
||||
every call type (the proxy's audio endpoints dispatch as ``aspeech`` and
|
||||
``transcription`` and reach it through ``Logging._response_cost_calculator``), so
|
||||
adding the charge at its final return covers both. The declared value is resolved
|
||||
from the per-deployment ``custom_pricing`` the logging layer builds out of our
|
||||
``config.yaml`` ``model_info``, falling back to the shared cost map and to the
|
||||
call metadata.
|
||||
|
||||
Scope: nothing changes unless a model declares ``input_cost_per_request``, so
|
||||
this is inert for every other model on the gateway. Since 2026-11-06 the charge
|
||||
is added only on the AUDIO call types (transcription/atranscription/speech/
|
||||
aspeech): a model that also carries per-token pricing (``voxtral-small-24b``)
|
||||
bills its chat calls exactly per token, and a per-request charge on top of
|
||||
that would double-bill them. The audio endpoints are the paths that cannot
|
||||
price per token (the provider returns no token counts there), so the flat
|
||||
charge belongs to them alone.
|
||||
|
||||
Upstream: no issue filed; LiteLLM has no per-request audio pricing as of 1.103.2.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
MARKER = "CO-OP PATCH (audio per-request cost, 2026-10-26)"
|
||||
|
||||
HELPER = '''def _coop_per_request_cost(
|
||||
*,
|
||||
model: str | None = None,
|
||||
litellm_model_name: str | None = None,
|
||||
custom_pricing: Any | None = None,
|
||||
litellm_logging_obj: Any | None = None,
|
||||
) -> float:
|
||||
"""CO-OP PATCH (audio per-request cost, 2026-10-26)
|
||||
|
||||
Return the model's declared ``input_cost_per_request``, else 0.0.
|
||||
|
||||
LiteLLM cannot price a flat per-request charge on the audio paths, so
|
||||
Tinfoil's per-request models (voxtral-tts, whisper-large-v3-turbo) were
|
||||
billed at $0. Resolution order: the per-deployment custom pricing the
|
||||
logging layer builds from our config.yaml model_info, then the shared cost
|
||||
map, then the call metadata. Inert unless a model declares the field.
|
||||
"""
|
||||
def _is_audio(name: Any) -> bool:
|
||||
text = str(name or "").lower()
|
||||
return (
|
||||
"tts" in text or "whisper" in text or "voxtral" in text
|
||||
)
|
||||
|
||||
cost_map = {}
|
||||
try:
|
||||
import litellm as _litellm # local import: not every caller binds it globally
|
||||
cost_map = getattr(_litellm, "model_cost", {}) or {}
|
||||
except Exception:
|
||||
cost_map = {}
|
||||
|
||||
candidates = []
|
||||
if isinstance(custom_pricing, dict):
|
||||
candidates.append(custom_pricing)
|
||||
map_hits = []
|
||||
for key in (litellm_model_name, model):
|
||||
entry = cost_map.get(key) if key else None
|
||||
if isinstance(entry, dict):
|
||||
candidates.append(entry)
|
||||
map_hits.append((key, entry.get("input_cost_per_request")))
|
||||
try:
|
||||
details = getattr(litellm_logging_obj, "model_call_details", None) or {}
|
||||
info = (details.get("metadata") or {}).get("model_info")
|
||||
if isinstance(info, dict):
|
||||
candidates.append(info)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Diagnostic (audio models only): did this run at all, and what could it see?
|
||||
# If this line never appears for a speech call, the request never reaches the
|
||||
# model cost calculator and the loss is upstream of this patch.
|
||||
if _is_audio(model) or _is_audio(litellm_model_name):
|
||||
try:
|
||||
print(
|
||||
"[coop-audio] probe"
|
||||
f" model={model!r} litellm={litellm_model_name!r}"
|
||||
f" custom_pricing={sorted(custom_pricing) if isinstance(custom_pricing, dict) else type(custom_pricing).__name__}"
|
||||
f" map_hits={map_hits!r} candidates={len(candidates)}",
|
||||
flush=True,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for candidate in candidates:
|
||||
try:
|
||||
value = candidate.get("input_cost_per_request")
|
||||
except Exception:
|
||||
continue
|
||||
if value:
|
||||
try:
|
||||
resolved = float(value)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if _is_audio(model) or _is_audio(litellm_model_name):
|
||||
try:
|
||||
print(f"[coop-audio] resolved per_request={resolved}", flush=True)
|
||||
except Exception:
|
||||
pass
|
||||
return resolved
|
||||
if _is_audio(model) or _is_audio(litellm_model_name):
|
||||
try:
|
||||
print("[coop-audio] resolved per_request=0.0 (nothing declared)", flush=True)
|
||||
except Exception:
|
||||
pass
|
||||
return 0.0
|
||||
|
||||
|
||||
'''
|
||||
|
||||
ANCHOR_DEF = "def response_cost_calculator("
|
||||
|
||||
ANCHOR_EARLY = """ if provider_response_cost is not None and not custom_pricing:
|
||||
return provider_response_cost"""
|
||||
|
||||
ANCHOR_RETURN = """ region_name=region_name,
|
||||
)
|
||||
return response_cost
|
||||
except Exception as e:
|
||||
raise e"""
|
||||
|
||||
# The unit-price call. On speech/transcription LiteLLM prices by character,
|
||||
# second or token and raises ValueError when the model declares none of those
|
||||
# rates — which is exactly a per-request model. The declaration is then the
|
||||
# whole bill, so price the base at 0 instead of letting the raise swallow the
|
||||
# spend log write (issue #5: rows recorded spend 0.0 and the tail patch below
|
||||
# never ran because the exception aborted the function first).
|
||||
ANCHOR_COST = """ response_cost = completion_cost(
|
||||
completion_response=response_object,
|
||||
model=model,
|
||||
call_type=call_type,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
optional_params=optional_params,
|
||||
custom_pricing=custom_pricing,
|
||||
base_model=base_model,
|
||||
prompt=prompt,
|
||||
standard_built_in_tools_params=standard_built_in_tools_params,
|
||||
litellm_model_name=litellm_model_name,
|
||||
router_model_id=router_model_id,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
region_name=region_name,
|
||||
)"""
|
||||
|
||||
PATCHED_COST = """ try:
|
||||
response_cost = completion_cost(
|
||||
completion_response=response_object,
|
||||
model=model,
|
||||
call_type=call_type,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
optional_params=optional_params,
|
||||
custom_pricing=custom_pricing,
|
||||
base_model=base_model,
|
||||
prompt=prompt,
|
||||
standard_built_in_tools_params=standard_built_in_tools_params,
|
||||
litellm_model_name=litellm_model_name,
|
||||
router_model_id=router_model_id,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
region_name=region_name,
|
||||
)
|
||||
except Exception as _coop_unit_exc:
|
||||
# CO-OP PATCH (audio per-request cost, 2026-11-17): a model billed
|
||||
# per request declares no per-character/second/token rate, so
|
||||
# LiteLLM's unit pricing raises. The per-request charge is the
|
||||
# whole bill; let the tail patch below add it to a zero base.
|
||||
_coop_declared_pr = (
|
||||
_coop_per_request_cost(
|
||||
model=model,
|
||||
litellm_model_name=litellm_model_name,
|
||||
custom_pricing=custom_pricing,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
)
|
||||
if call_type in ("transcription", "atranscription", "speech", "aspeech")
|
||||
else 0.0
|
||||
)
|
||||
if _coop_declared_pr > 0:
|
||||
print(
|
||||
f"[coop-audio] unit pricing unavailable ({type(_coop_unit_exc).__name__}); "
|
||||
f"pricing at declared per-request={_coop_declared_pr!r}",
|
||||
flush=True,
|
||||
)
|
||||
response_cost = 0.0
|
||||
else:
|
||||
raise"""
|
||||
|
||||
PATCHED_EARLY = """ if provider_response_cost is not None and not custom_pricing:
|
||||
# CO-OP PATCH (2026-11-16): a provider-reported cost is not the
|
||||
# whole bill on the audio paths. Speech returns here and never
|
||||
# reaches the tail patch, so add the declared per-request charge
|
||||
# before returning. Guarded by the DECLARATION, not the call
|
||||
# type: models without input_cost_per_request (voxtral-small-24b,
|
||||
# token-priced) stay inert.
|
||||
return provider_response_cost + _coop_per_request_cost(
|
||||
model=model,
|
||||
litellm_model_name=litellm_model_name,
|
||||
custom_pricing=custom_pricing,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
)"""
|
||||
|
||||
PATCHED_RETURN = """ region_name=region_name,
|
||||
)
|
||||
|
||||
# CO-OP PATCH (audio per-request cost, 2026-10-26): add the declared
|
||||
# per-request charge for models LiteLLM can only price per unit
|
||||
# (Tinfoil's voxtral-tts, whisper-large-v3-turbo). Inert when the model
|
||||
# declares nothing; the early "provider reported the cost" return above
|
||||
# is unaffected. Scoped to the AUDIO call types (2026-11-06): a model
|
||||
# that ALSO carries per-token pricing (voxtral-small-24b) bills its
|
||||
# chat calls exactly per token and must not take the per-request
|
||||
# charge on top of that.
|
||||
try:
|
||||
_coop_audio_call = call_type in (
|
||||
"transcription",
|
||||
"atranscription",
|
||||
"speech",
|
||||
"aspeech",
|
||||
)
|
||||
_coop_pr = (
|
||||
_coop_per_request_cost(
|
||||
model=model,
|
||||
litellm_model_name=litellm_model_name,
|
||||
custom_pricing=custom_pricing,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
)
|
||||
if _coop_audio_call
|
||||
else 0.0
|
||||
)
|
||||
response_cost = (response_cost or 0.0) + _coop_pr
|
||||
_coop_name = str(model or "") + str(litellm_model_name or "")
|
||||
if _coop_audio_call and (
|
||||
"tts" in _coop_name.lower()
|
||||
or "whisper" in _coop_name.lower()
|
||||
or "voxtral" in _coop_name.lower()
|
||||
):
|
||||
print(
|
||||
f"[coop-audio] add per_request={_coop_pr!r} -> response_cost={response_cost!r}",
|
||||
flush=True,
|
||||
)
|
||||
except Exception as _coop_exc:
|
||||
print(f"[coop-audio] add failed: {type(_coop_exc).__name__}: {_coop_exc}", flush=True)
|
||||
|
||||
return response_cost
|
||||
except Exception as e:
|
||||
raise e"""
|
||||
|
||||
|
||||
def target_path() -> str:
|
||||
"""The installed litellm cost calculator (or an explicit path for testing)."""
|
||||
if len(sys.argv) > 1:
|
||||
return sys.argv[1]
|
||||
env = os.environ.get("LITELLM_COST_CALCULATOR")
|
||||
if env:
|
||||
return env
|
||||
hits = []
|
||||
try: # most reliable: ask the installed package where it lives
|
||||
import litellm # noqa: PLC0415
|
||||
_file = getattr(litellm, "__file__", None)
|
||||
if _file:
|
||||
hits.append(os.path.join(os.path.dirname(_file), "cost_calculator.py"))
|
||||
except Exception:
|
||||
pass
|
||||
hits += glob.glob("/usr/lib/python3*/site-packages/litellm/cost_calculator.py")
|
||||
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/cost_calculator.py")
|
||||
hits += glob.glob("/usr/local/lib/python3*/dist-packages/litellm/cost_calculator.py")
|
||||
hits += glob.glob("/opt/**/litellm/cost_calculator.py", recursive=True)
|
||||
hits = [h for h in hits if os.path.exists(h)]
|
||||
if not hits:
|
||||
raise SystemExit("[coop-patch] could not locate litellm/cost_calculator.py")
|
||||
return sorted(hits)[0]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
path = target_path()
|
||||
try:
|
||||
source = open(path).read()
|
||||
except OSError as exc:
|
||||
print(f"[coop-patch] cannot read {path}: {exc}")
|
||||
return 0
|
||||
|
||||
if MARKER in source:
|
||||
print(f"[coop-patch] {path} already patched, nothing to do")
|
||||
return 0
|
||||
|
||||
if ANCHOR_DEF not in source or ANCHOR_RETURN not in source:
|
||||
print(f"[coop-patch] WARNING anchors not found in {path}; audio per-request "
|
||||
f"cost will NOT be billed (check this after a LiteLLM upgrade)")
|
||||
return 0
|
||||
|
||||
patched = source.replace(ANCHOR_DEF, HELPER + ANCHOR_DEF, 1)
|
||||
patched = patched.replace(ANCHOR_RETURN, PATCHED_RETURN, 1)
|
||||
if ANCHOR_COST in patched:
|
||||
patched = patched.replace(ANCHOR_COST, PATCHED_COST, 1)
|
||||
print("[coop-patch] unit-price call made per-request tolerant (speech/transcription)")
|
||||
else:
|
||||
print("[coop-patch] WARNING unit-price anchor not found; per-request models "
|
||||
"will still raise and bill $0")
|
||||
if ANCHOR_EARLY in patched:
|
||||
patched = patched.replace(ANCHOR_EARLY, PATCHED_EARLY, 1)
|
||||
print("[coop-patch] early provider-cost return also patched (audio speech)")
|
||||
else:
|
||||
print("[coop-patch] WARNING early-return anchor not found; speech may stay unbilled")
|
||||
compile(patched, path, "exec") # refuse to write something that will not import
|
||||
|
||||
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
||||
print(f"[coop-patch] {path} is not writable; verified anchors and syntax only")
|
||||
return 0
|
||||
|
||||
with open(path, "w") as handle:
|
||||
handle.write(patched)
|
||||
print(f"[coop-patch] applied to {path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Keep audio-model pricing on the shared cost-map key (speech + transcription).
|
||||
|
||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
||||
if the target file or the anchor is missing — e.g. after a version bump — it
|
||||
prints a warning and exits 0, so the build and the gateway still succeed. Look
|
||||
for the ``[coop-patch]`` line in the build log.
|
||||
|
||||
Why this is needed on top of patch_audio_request_cost.py: the audio cost paths
|
||||
(speech, transcription) resolve pricing from the **shared** cost-map key, and the
|
||||
router deliberately strips per-deployment pricing from that key ("so one
|
||||
deployment's overrides don't pollute sibling deployments"). For a deployment on a
|
||||
custom backend — ours are all `openai/<model>` against a local Tinfoil proxy —
|
||||
the cost lookup then raises, the spend-log write is skipped, and **no SpendLogs
|
||||
row appears at all**. That is the symptom we measured: `/v1/audio/speech` and
|
||||
`/v1/audio/transcriptions` returned 200 with real audio, and the member key's
|
||||
spend stayed exactly $0.000000 with no spend row anywhere. It is the same
|
||||
mechanism as the image-generation case that patch_image_cost.py already fixes
|
||||
(upstream BerriAI/litellm#10023/#12471).
|
||||
|
||||
The cost-calculator patch alone cannot fix it: it only adds a per-request charge
|
||||
*on top of* a cost the normal path computed, and that path never got that far.
|
||||
|
||||
`input_cost_per_request` is deliberately allowed through even though it is **not**
|
||||
in `CustomPricingLiteLLMParams.model_fields` — LiteLLM does not list it there, and
|
||||
patch_audio_request_cost.py is what makes the cost calculator read it.
|
||||
|
||||
Safe for us: the co-op gateway has one deployment per audio backend, so there are
|
||||
no sibling deployments for the shared key to pollute.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
MARKER = "CO-OP PATCH (audio shared pricing, 2026-10-26)"
|
||||
|
||||
ANCHOR = " # Always register the (possibly mode-preserved) shared backend info."
|
||||
|
||||
INSERT = ''' # CO-OP PATCH (audio shared pricing, 2026-10-26): the audio cost paths
|
||||
# (speech, transcription) read pricing from the shared cost-map key, and
|
||||
# this router strips per-deployment pricing from it — so a custom-backend
|
||||
# audio deployment raises in the cost lookup, the spend write is skipped,
|
||||
# and no SpendLogs row appears at all (same mechanism as the image case
|
||||
# this file already patches). Re-add the pricing for audio modes,
|
||||
# including input_cost_per_request, which LiteLLM does not list among its
|
||||
# CustomPricing fields.
|
||||
if deployment_mode in ("audio_speech", "audio_transcription"):
|
||||
for _ak, _av in model_info.items():
|
||||
if (
|
||||
_ak in CustomPricingLiteLLMParams.model_fields
|
||||
or _ak == "input_cost_per_request"
|
||||
) and _av is not None:
|
||||
shared_model_info[_ak] = _av
|
||||
|
||||
'''
|
||||
|
||||
|
||||
def target_path() -> str:
|
||||
if len(sys.argv) > 1:
|
||||
return sys.argv[1]
|
||||
env = os.environ.get("LITELLM_ROUTER")
|
||||
if env:
|
||||
return env
|
||||
hits = []
|
||||
try: # most reliable: ask the installed package where it lives
|
||||
import litellm # noqa: PLC0415
|
||||
_file = getattr(litellm, "__file__", None)
|
||||
if _file:
|
||||
hits.append(os.path.join(os.path.dirname(_file), "router.py"))
|
||||
except Exception:
|
||||
pass
|
||||
hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/router.py")
|
||||
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/router.py")
|
||||
hits = [h for h in hits if os.path.exists(h)]
|
||||
if not hits:
|
||||
raise SystemExit("[coop-patch] could not locate litellm/router.py")
|
||||
return sorted(hits)[0]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
path = target_path()
|
||||
try:
|
||||
source = open(path).read()
|
||||
except OSError as exc:
|
||||
print(f"[coop-patch] cannot read {path}: {exc}")
|
||||
return 0
|
||||
|
||||
if MARKER in source:
|
||||
print(f"[coop-patch] {path} already patched, nothing to do")
|
||||
return 0
|
||||
|
||||
if ANCHOR not in source:
|
||||
print(f"[coop-patch] WARNING anchor not found in {path}; audio pricing will "
|
||||
f"NOT reach the shared cost map and audio spend will stay invisible")
|
||||
return 0
|
||||
|
||||
patched = source.replace(ANCHOR, INSERT + ANCHOR, 1)
|
||||
compile(patched, path, "exec") # refuse to write something that will not import
|
||||
|
||||
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
||||
print(f"[coop-patch] {path} is not writable; verified anchor and syntax only")
|
||||
return 0
|
||||
|
||||
with open(path, "w") as handle:
|
||||
handle.write(patched)
|
||||
print(f"[coop-patch] applied audio shared pricing to {path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Expose LiteLLM's per-request cost headers to cross-origin browser clients.
|
||||
|
||||
MEMBER REQUEST (co-op/support#39, 2026-10-31). Browser JS calling the gateway from
|
||||
another origin cannot read a response header unless the server names it in
|
||||
Access-Control-Expose-Headers. LiteLLM sets
|
||||
|
||||
expose_headers=LITELLM_UI_ALLOW_HEADERS # through 1.103.x
|
||||
expose_headers=[*LITELLM_UI_ALLOW_HEADERS,
|
||||
*trace_response_expose_headers()] # from 1.108.0
|
||||
|
||||
which covers seven names (semantic-filter, guardrail, cache-key, ...) and none of
|
||||
the cost ones — so `x-litellm-response-cost*` reads as null in JS even though the
|
||||
proxy sends those headers on the response. Upstream closed the request
|
||||
(BerriAI/litellm#33123) as stale, so it has to be handled here at the app layer,
|
||||
exactly like the other patches in this directory.
|
||||
|
||||
Additive only. It lets a caller read per-request pricing for requests they made
|
||||
with their own API key — numbers already present in the response body for
|
||||
non-streaming calls, and already visible in their dashboard. It exposes no prompt,
|
||||
key, or account data, and no other member's usage.
|
||||
|
||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal, per
|
||||
house rule: if upstream moves or rewrites this call, WARN and exit 0 — a missing
|
||||
convenience header must never keep the gateway from booting.
|
||||
|
||||
Regression check: after a deploy, a browser request from an unrelated origin should
|
||||
see the cost headers listed in Access-Control-Expose-Headers on the response.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
# The header names the proxy can set, taken from the running image rather than
|
||||
# guessed: grep -rhoE 'x-litellm-[a-z0-9-]*cost[a-z0-9-]*' over the litellm package.
|
||||
EXPOSED_HEADERS = [
|
||||
"x-litellm-response-cost",
|
||||
"x-litellm-response-cost-input",
|
||||
"x-litellm-response-cost-output",
|
||||
"x-litellm-response-cost-original",
|
||||
"x-litellm-response-cost-discount-amount",
|
||||
"x-litellm-response-cost-margin-amount",
|
||||
"x-litellm-response-cost-margin-percent",
|
||||
"x-litellm-response-cost-cache-creation",
|
||||
"x-litellm-response-cost-cache-read",
|
||||
"x-litellm-response-cost-reasoning",
|
||||
"x-litellm-response-cost-tool-usage",
|
||||
"x-litellm-classifier-cost",
|
||||
# Telemetry names, requested in co-op/support#39 (2026-10-31): model identity,
|
||||
# call id, the caller's own key spend, timings and retry/fallback counts.
|
||||
"x-litellm-model-name",
|
||||
"x-litellm-call-id",
|
||||
"x-litellm-key-spend",
|
||||
"x-litellm-response-duration-ms",
|
||||
"x-litellm-overhead-duration-ms",
|
||||
"x-litellm-callback-duration-ms",
|
||||
"x-litellm-attempted-retries",
|
||||
"x-litellm-attempted-fallbacks",
|
||||
]
|
||||
|
||||
OLD = " expose_headers=LITELLM_UI_ALLOW_HEADERS,\n"
|
||||
NEW = (
|
||||
" expose_headers=LITELLM_UI_ALLOW_HEADERS\n"
|
||||
" + [\n"
|
||||
+ "".join(f' "{h}",\n' for h in EXPOSED_HEADERS)
|
||||
+ " ],\n"
|
||||
)
|
||||
|
||||
# 1.108.0 refactored the middleware call (upstream added the trace-response
|
||||
# header set to expose_headers). Same operation, new anchor; tried when the
|
||||
# 1.103.2 shape is absent so the script works against either base image.
|
||||
OLD_V1108 = (
|
||||
" expose_headers=[*LITELLM_UI_ALLOW_HEADERS, *trace_response_expose_headers()],\n"
|
||||
)
|
||||
NEW_V1108 = (
|
||||
" expose_headers=[\n"
|
||||
" *LITELLM_UI_ALLOW_HEADERS,\n"
|
||||
" *trace_response_expose_headers(),\n"
|
||||
+ "".join(f' "{h}",\n' for h in EXPOSED_HEADERS)
|
||||
+ " ],\n"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# Glob rather than pin a Python version: the venv path moves when the base
|
||||
# image bumps its interpreter, and a stale path would silently skip the patch.
|
||||
candidates = sorted(Path("/app/.venv/lib").glob("python3*/site-packages/litellm/proxy/proxy_server.py"))
|
||||
if not candidates:
|
||||
print("WARN expose-cost-headers: proxy_server.py not found under /app/.venv/lib — skipping")
|
||||
return 0
|
||||
|
||||
target = candidates[-1]
|
||||
src = target.read_text()
|
||||
if "x-litellm-response-cost-input" in src:
|
||||
print("expose-cost-headers: already applied")
|
||||
return 0
|
||||
for old, new, shape in ((OLD, NEW, "1.103.x"), (OLD_V1108, NEW_V1108, "1.108.x")):
|
||||
found = src.count(old)
|
||||
if found == 1:
|
||||
target.write_text(src.replace(old, new))
|
||||
print(
|
||||
f"expose-cost-headers: appended {len(EXPOSED_HEADERS)} cost header names "
|
||||
f"to expose_headers (anchor {shape}) in {target}"
|
||||
)
|
||||
return 0
|
||||
if found > 1:
|
||||
print(f"WARN expose-cost-headers: expected exactly 1 {shape} CORS middleware call, found {found} — skipping")
|
||||
return 0
|
||||
print(
|
||||
"WARN expose-cost-headers: no known expose_headers anchor found "
|
||||
"(neither 1.103.x nor 1.108.x shape) — skipping"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Open explicitly-listed pass-through routes to any authenticated key.
|
||||
|
||||
WHY. Config-declared pass-through endpoints registered with ``auth: true`` are
|
||||
gated at request time by ``allowed_passthrough_routes`` on the key or team
|
||||
(``RouteChecks.check_passthrough_route_access``). That field is a **LiteLLM
|
||||
Enterprise** feature: on a community-edition proxy the management API refuses to
|
||||
set it ("This feature is only available for LiteLLM Enterprise users"), so a
|
||||
member key can never be granted access by any supported route. The only other
|
||||
lever is ``auth: false``, which forwards **unauthenticated** requests — a public
|
||||
relay of the co-op's GreenPT key. Neither is acceptable.
|
||||
|
||||
This patch lets the operator name such routes in the proxy config:
|
||||
|
||||
general_settings:
|
||||
coop_passthrough_open_to_keys: ["/v1/systemone"]
|
||||
|
||||
Any route listed there is allowed for any key that has already authenticated.
|
||||
Authentication itself is untouched: ``auth: true`` still installs
|
||||
``Depends(user_api_key_auth)``, so an unauthenticated request is rejected earlier
|
||||
and never reaches this function. The patch only relaxes the *second* gate
|
||||
(per-key allow-list), and only for routes the operator listed by hand.
|
||||
|
||||
House style: run at BUILD time (the runtime rootfs is read-only under Cloudron);
|
||||
non-fatal — if upstream renames or restructures this function, WARN and exit 0;
|
||||
idempotent — detect the marker on a rebuild and skip; and compile the patched
|
||||
source in memory before writing, so a patch that cannot import never ships.
|
||||
|
||||
Check the ``[coop-patch] applied`` line in the build log on every LiteLLM bump.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import os
|
||||
import sys
|
||||
|
||||
MARKER = "coop_passthrough_open_to_keys"
|
||||
|
||||
# Unique multi-line anchor: the def + docstring + the first statement of the body.
|
||||
OLD = (
|
||||
" def check_passthrough_route_access(route: str, user_api_key_dict: UserAPIKeyAuth) -> bool:\n"
|
||||
" \"\"\"\n"
|
||||
" Check if route is a passthrough route.\n"
|
||||
" Supports both exact match and prefix match.\n"
|
||||
" \"\"\"\n"
|
||||
" metadata: Final = user_api_key_dict.metadata\n"
|
||||
)
|
||||
|
||||
NEW = (
|
||||
" def check_passthrough_route_access(route: str, user_api_key_dict: UserAPIKeyAuth) -> bool:\n"
|
||||
" \"\"\"\n"
|
||||
" Check if route is a passthrough route.\n"
|
||||
" Supports both exact match and prefix match.\n"
|
||||
" \"\"\"\n"
|
||||
" # [coop-patch] Routes the operator lists in\n"
|
||||
" # general_settings.coop_passthrough_open_to_keys are callable by ANY\n"
|
||||
" # authenticated key. LiteLLM's own gate (allowed_passthrough_routes)\n"
|
||||
" # is Enterprise-only, so it can never be granted on this deployment.\n"
|
||||
" # Authentication is unaffected: auth=True on the endpoint still\n"
|
||||
" # requires a valid key before this function runs.\n"
|
||||
" try:\n"
|
||||
" from litellm.proxy.proxy_server import general_settings as _coop_general_settings\n"
|
||||
"\n"
|
||||
" _coop_open_to_keys = (_coop_general_settings or {}).get(\"coop_passthrough_open_to_keys\") or []\n"
|
||||
" for _coop_route in _coop_open_to_keys:\n"
|
||||
" if RouteChecks._route_matches_allowed_route(route=route, allowed_route=_coop_route):\n"
|
||||
" return True\n"
|
||||
" except Exception:\n"
|
||||
" pass\n"
|
||||
"\n"
|
||||
" metadata: Final = user_api_key_dict.metadata\n"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# ROOT override (house style) lets the same script run against a pulled copy
|
||||
# locally: COOP_PATCH_ROOT=<dir containing .venv/lib>.
|
||||
root = Path(os.environ.get("COOP_PATCH_ROOT", "/app"))
|
||||
candidates = sorted(
|
||||
(root / ".venv/lib").glob("python3*/site-packages/litellm/proxy/auth/route_checks.py")
|
||||
)
|
||||
if not candidates:
|
||||
print("WARN passthrough-key-access: route_checks.py not found under /app/.venv/lib — skipping")
|
||||
return 0
|
||||
|
||||
target = candidates[-1]
|
||||
src = target.read_text()
|
||||
if MARKER in src:
|
||||
print("passthrough-key-access: already applied")
|
||||
return 0
|
||||
|
||||
found = src.count(OLD)
|
||||
if found != 1:
|
||||
print(f"WARN passthrough-key-access: expected exactly 1 anchor, found {found} — skipping")
|
||||
return 0
|
||||
|
||||
patched = src.replace(OLD, NEW)
|
||||
try:
|
||||
compile(patched, str(target), "exec")
|
||||
except SyntaxError as e:
|
||||
print(f"WARN passthrough-key-access: patched source does not compile ({e}) — skipping")
|
||||
return 0
|
||||
|
||||
target.write_text(patched)
|
||||
print(f"[coop-patch] applied: passthrough-key-access -> {target}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Price a generic pass-through route from the usage it reports in its OWN body.
|
||||
|
||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
||||
if the target file or the anchor is missing — e.g. after an upstream version bump
|
||||
— it prints a warning and exits 0, so the build and the gateway still succeed.
|
||||
The build log line ``[coop-patch] applied`` is what confirms it took effect;
|
||||
check it on every LiteLLM upgrade (see README, upgrade checklist).
|
||||
|
||||
Why: config-declared ``pass_through_endpoints`` with ``auth: true`` are logged
|
||||
through ``PassThroughEndpointLogging``. A target whose hostname matches none of
|
||||
vertex / anthropic / openai / gemini resolves to ``EndpointType.GENERIC`` and
|
||||
takes the generic branch of ``pass_through_async_success_handler`` — it is
|
||||
logged, but LiteLLM has no model on the route to price, so the only cost it
|
||||
knows is the flat ``cost_per_request`` from the config entry. That is a whole-
|
||||
request estimate used for budget accounting, not an invoice: Metis is billed at
|
||||
$0.12 per 1M *input* tokens and returns the true count in ``usage.input_tokens``,
|
||||
so the flat figure is off by whatever factor the call differed from the estimate.
|
||||
|
||||
LiteLLM's own supported mechanism for an unpriced pass-through is the
|
||||
``x-litellm-response-cost`` / ``x-litellm-total-tokens`` response-header
|
||||
contract (``litellm/proxy/pass_through_endpoints/upstream_usage_headers.py``),
|
||||
but that is consumed inside this SAME logging path, so it only helps where the
|
||||
logging path already runs — and GreenPT does not send those headers. Reading the
|
||||
body, which is already in hand here, needs no extra process in front of the
|
||||
provider (a sidecar would be a single point of failure for the route itself, not
|
||||
just for its metering).
|
||||
|
||||
Patch: make ``PassThroughEndpointLogging._set_cost_per_request`` price a route
|
||||
listed in ``general_settings.coop_passthrough_cost_per_token`` (USD per input
|
||||
token, keyed by the gateway route path, or by the upstream target) from the
|
||||
``usage.input_tokens`` in the response body:
|
||||
|
||||
cost = input_tokens * coop_passthrough_cost_per_token[route]
|
||||
|
||||
The rate lives in ``general_settings`` rather than the endpoint dict because the
|
||||
endpoint model (``PassThroughGenericEndpoint``) drops unknown fields and the
|
||||
exact-route registration copies only ``cost_per_request`` into the logging
|
||||
payload — the ``general_settings`` dict is read straight from the proxy module,
|
||||
the same way ``patch_passthrough_key_access.py`` reads
|
||||
``coop_passthrough_open_to_keys``.
|
||||
|
||||
TOKEN COUNTS. Pricing the row from the body is not enough: the row still
|
||||
recorded ``prompt_tokens=0``, ``completion_tokens=0``, ``total_tokens=0``,
|
||||
because the pass-through success handler hands the spend writer a
|
||||
``StandardPassThroughResponseObject`` whose text carries the body but whose
|
||||
``usage`` is empty, and the generic pass-through branch builds no
|
||||
``standard_logging_object`` usage either. Any per-token reporting (panel,
|
||||
dashboard, energy) that sums tokens for this route then under-reports. So the
|
||||
patch also carries the count onto the logging object as
|
||||
``combined_usage_object`` — the same ``litellm.Usage`` override the streaming
|
||||
handler uses to recover partial usage. The spend writer reads
|
||||
``combined_usage_object`` (``spend_tracking_utils.get_logging_payload`` and
|
||||
``StandardLoggingPayloadSetup.get_usage_as_dict``) before it falls back to the
|
||||
response object, so the row records ``prompt_tokens=input_tokens``,
|
||||
``completion_tokens=0``, ``total_tokens=input_tokens`` while spend stays exactly
|
||||
``input_tokens × rate``.
|
||||
|
||||
The exact figure wins; ``cost_per_request`` stays as the fallback for a listed
|
||||
route whose body carries no usable input-token count, and for every route not
|
||||
listed. An upstream that reports its own totals via the x-litellm-* headers still
|
||||
wins outright (the early return is left intact). Inert unless an operator lists
|
||||
a route, so it changes nothing for the chat models or the audio paths.
|
||||
|
||||
Upstream: no issue filed; LiteLLM has no per-token pricing for pass-through
|
||||
routes whose body is not an OpenAI/Anthropic/Gemini shape (as of 1.103.2).
|
||||
"""
|
||||
from pathlib import Path
|
||||
import os
|
||||
import sys
|
||||
|
||||
MARKER = "coop_passthrough_cost_per_token"
|
||||
|
||||
OLD = (
|
||||
" #########################################################\n"
|
||||
" # Check if cost per request is set\n"
|
||||
" #########################################################\n"
|
||||
" if has_upstream_reported_usage(logging_obj):\n"
|
||||
" return kwargs\n"
|
||||
"\n"
|
||||
" if passthrough_logging_payload.get(\"cost_per_request\") is not None:\n"
|
||||
" kwargs[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n"
|
||||
" logging_obj.model_call_details[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n"
|
||||
"\n"
|
||||
" return kwargs\n"
|
||||
)
|
||||
|
||||
NEW = (
|
||||
" #########################################################\n"
|
||||
" # Check if cost per request is set\n"
|
||||
" #########################################################\n"
|
||||
" if has_upstream_reported_usage(logging_obj):\n"
|
||||
" return kwargs\n"
|
||||
"\n"
|
||||
" # [coop-patch] Exact per-input-token pricing for a generic\n"
|
||||
" # pass-through route. A route the operator lists in\n"
|
||||
" # general_settings.coop_passthrough_cost_per_token (USD per input\n"
|
||||
" # token, keyed by gateway path or upstream target) is priced from the\n"
|
||||
" # usage it reports in its OWN response body, so spend is the real\n"
|
||||
" # token count rather than cost_per_request's flat estimate. GreenPT's\n"
|
||||
" # Metis decision route (/v1/systemone) is the first such route: it\n"
|
||||
" # returns usage.input_tokens and always 0 output tokens, billed at\n"
|
||||
" # $0.12 per 1M input tokens and recorded on the spend row with the\n"
|
||||
" # same count. Inert unless a route is listed; a listed\n"
|
||||
" # route whose body carries no token count falls through to\n"
|
||||
" # cost_per_request below, and every unlisted route is unchanged.\n"
|
||||
" try:\n"
|
||||
" from litellm.proxy.proxy_server import general_settings as _coop_general_settings\n"
|
||||
"\n"
|
||||
" _coop_rates = (_coop_general_settings or {}).get(\"coop_passthrough_cost_per_token\") or {}\n"
|
||||
" if _coop_rates:\n"
|
||||
" _coop_candidates = set()\n"
|
||||
" _coop_psr = ((kwargs.get(\"litellm_params\") or {}).get(\"proxy_server_request\") or {})\n"
|
||||
" _coop_req_url = _coop_psr.get(\"url\")\n"
|
||||
" if _coop_req_url:\n"
|
||||
" _coop_candidates.add(str(_coop_req_url))\n"
|
||||
" _coop_candidates.add(urlparse(str(_coop_req_url)).path)\n"
|
||||
" _coop_upstream_url = passthrough_logging_payload.get(\"url\")\n"
|
||||
" if _coop_upstream_url:\n"
|
||||
" _coop_candidates.add(str(_coop_upstream_url))\n"
|
||||
" _coop_candidates.add(urlparse(str(_coop_upstream_url)).path)\n"
|
||||
"\n"
|
||||
" _coop_rate = None\n"
|
||||
" for _coop_route in _coop_candidates:\n"
|
||||
" if _coop_route in _coop_rates:\n"
|
||||
" _coop_rate = _coop_rates[_coop_route]\n"
|
||||
" break\n"
|
||||
" if _coop_rate is None:\n"
|
||||
" for _coop_key, _coop_val in _coop_rates.items():\n"
|
||||
" _coop_prefix = str(_coop_key).rstrip(\"/\")\n"
|
||||
" if any(\n"
|
||||
" c == _coop_key or (c and c.startswith(_coop_prefix + \"/\"))\n"
|
||||
" for c in _coop_candidates\n"
|
||||
" ):\n"
|
||||
" _coop_rate = _coop_val\n"
|
||||
" break\n"
|
||||
"\n"
|
||||
" _coop_tokens = 0\n"
|
||||
" _coop_body = passthrough_logging_payload.get(\"response_body\")\n"
|
||||
" if isinstance(_coop_body, dict):\n"
|
||||
" _coop_usage = _coop_body.get(\"usage\")\n"
|
||||
" if isinstance(_coop_usage, dict):\n"
|
||||
" _coop_tokens = _coop_usage.get(\"input_tokens\") or 0\n"
|
||||
" if not _coop_tokens:\n"
|
||||
" _coop_tokens = _coop_body.get(\"input_tokens\") or 0\n"
|
||||
"\n"
|
||||
" if _coop_rate is not None and _coop_tokens:\n"
|
||||
" _coop_cost = float(_coop_tokens) * float(_coop_rate)\n"
|
||||
" kwargs[\"response_cost\"] = _coop_cost\n"
|
||||
" logging_obj.model_call_details[\"response_cost\"] = _coop_cost\n"
|
||||
" # [coop-patch] Record the true token counts on the row too.\n"
|
||||
" # The spend writer reads usage from combined_usage_object\n"
|
||||
" # (the override the streaming handler already uses for\n"
|
||||
" # recovered partial usage); without it the generic\n"
|
||||
" # pass-through response object carries no usage and the\n"
|
||||
" # row's prompt/completion/total tokens stay 0.\n"
|
||||
" _coop_usage = None\n"
|
||||
" try:\n"
|
||||
" from litellm.types.utils import Usage as _CoopUsage\n"
|
||||
"\n"
|
||||
" _coop_usage = _CoopUsage(\n"
|
||||
" prompt_tokens=int(_coop_tokens),\n"
|
||||
" completion_tokens=0,\n"
|
||||
" total_tokens=int(_coop_tokens),\n"
|
||||
" )\n"
|
||||
" except Exception as _coop_usage_exc:\n"
|
||||
" print(\n"
|
||||
" \"[coop-passthrough-cost] usage object failed: %s: %s\"\n"
|
||||
" % (type(_coop_usage_exc).__name__, _coop_usage_exc),\n"
|
||||
" flush=True,\n"
|
||||
" )\n"
|
||||
" if _coop_usage is not None:\n"
|
||||
" kwargs[\"combined_usage_object\"] = _coop_usage\n"
|
||||
" logging_obj.model_call_details[\"combined_usage_object\"] = _coop_usage\n"
|
||||
" print(\n"
|
||||
" \"[coop-passthrough-cost] input_tokens=%s rate=%s -> response_cost=%s\"\n"
|
||||
" % (_coop_tokens, _coop_rate, _coop_cost),\n"
|
||||
" flush=True,\n"
|
||||
" )\n"
|
||||
" return kwargs\n"
|
||||
" except Exception as _coop_exc:\n"
|
||||
" print(\n"
|
||||
" \"[coop-passthrough-cost] exact cost failed: %s: %s\"\n"
|
||||
" % (type(_coop_exc).__name__, _coop_exc),\n"
|
||||
" flush=True,\n"
|
||||
" )\n"
|
||||
"\n"
|
||||
" if passthrough_logging_payload.get(\"cost_per_request\") is not None:\n"
|
||||
" kwargs[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n"
|
||||
" logging_obj.model_call_details[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n"
|
||||
"\n"
|
||||
" return kwargs\n"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# ROOT override (house style) lets the same script run against a pulled copy
|
||||
# locally: COOP_PATCH_ROOT=<dir containing .venv/lib>.
|
||||
root = Path(os.environ.get("COOP_PATCH_ROOT", "/app"))
|
||||
candidates = sorted(
|
||||
(root / ".venv/lib").glob("python3*/site-packages/litellm/proxy/pass_through_endpoints/success_handler.py")
|
||||
)
|
||||
if not candidates:
|
||||
print("WARN passthrough-token-cost: success_handler.py not found under /app/.venv/lib — skipping")
|
||||
return 0
|
||||
|
||||
target = candidates[-1]
|
||||
src = target.read_text()
|
||||
if MARKER in src:
|
||||
print("passthrough-token-cost: already applied")
|
||||
return 0
|
||||
|
||||
if src.count(OLD) != 1:
|
||||
print(f"WARN passthrough-token-cost: expected exactly 1 anchor, found {src.count(OLD)} — skipping")
|
||||
return 0
|
||||
|
||||
patched = src.replace(OLD, NEW, 1)
|
||||
try:
|
||||
compile(patched, str(target), "exec")
|
||||
except SyntaxError as e:
|
||||
print(f"WARN passthrough-token-cost: patched source does not compile ({e}) — skipping")
|
||||
return 0
|
||||
|
||||
target.write_text(patched)
|
||||
print(f"[coop-patch] applied: passthrough-token-cost -> {target}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Stop LiteLLM from forwarding the provider's response headers to callers.
|
||||
|
||||
SECURITY (2026-10-27, member-disclosure finding 4): LiteLLM's proxy copies every
|
||||
upstream response header onto its own response as `llm_provider-<name>` — see
|
||||
litellm_core_utils/llm_response_utils/get_headers.py::_get_llm_provider_headers,
|
||||
whose docstring reads "Forward all headers as llm_provider-{header}". Verified
|
||||
live on the co-op gateway: one chat completion returned 27 such headers, among
|
||||
them `llm_provider-x-credits-remaining: 16.52` — the co-op's remaining balance
|
||||
with the provider, readable by any member API key, and (read across calls) its
|
||||
burn rate.
|
||||
|
||||
This patch makes _get_llm_provider_headers() return {} so no provider header is
|
||||
copied. get_response_headers() still forwards the four OpenAI-style x-ratelimit-*
|
||||
headers WHEN a provider sends them; those describe the caller's own request
|
||||
rather than the co-op's account, and no upstream in our config sends them today.
|
||||
|
||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal, per
|
||||
house rule: if upstream moves or rewrites the function, WARN and exit 0 — a
|
||||
cosmetic leak must never keep the gateway from booting. Regression check: the
|
||||
daily litellm watcher asserts the gateway returns no `llm_provider-*` header.
|
||||
"""
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
ROOT = os.environ.get("COOP_PATCH_ROOT", "/app")
|
||||
TARGET = "/litellm/litellm_core_utils/llm_response_utils/get_headers.py"
|
||||
ANCHOR = ''' for k, v in response_headers.items():
|
||||
if "llm_provider" not in k:
|
||||
_key = "{}-{}".format("llm_provider", k)
|
||||
llm_provider_headers[_key] = v
|
||||
else:
|
||||
llm_provider_headers[k] = v
|
||||
return llm_provider_headers'''
|
||||
REPLACEMENT = ''' # CO-OP PATCH (2026-10-27): do NOT forward provider headers to callers.
|
||||
# Upstream prefixes EVERY response header with "llm_provider-", which leaked
|
||||
# the co-op's remaining provider balance (llm_provider-x-credits-remaining)
|
||||
# to any member API key. Delete this patch from the Dockerfile to revert.
|
||||
return {}'''
|
||||
|
||||
candidates = glob.glob(f"{ROOT}/.venv/lib/python*/site-packages{TARGET}")
|
||||
if not candidates:
|
||||
candidates = glob.glob(f"{ROOT}/**{TARGET}", recursive=True)
|
||||
|
||||
patched_any = False
|
||||
for p in candidates:
|
||||
src = open(p).read()
|
||||
if "CO-OP PATCH (2026-10-27)" in src:
|
||||
print(f"INFO: provider-header strip already present in {p}")
|
||||
patched_any = True
|
||||
continue
|
||||
if ANCHOR not in src:
|
||||
print(f"WARN: anchor not found in {p} (upstream changed it); "
|
||||
"provider headers are still forwarded (non-fatal)", file=sys.stderr)
|
||||
continue
|
||||
new = src.replace(ANCHOR, REPLACEMENT, 1)
|
||||
try:
|
||||
compile(new, p, "exec")
|
||||
except SyntaxError as e:
|
||||
print(f"WARN: patched source would not compile ({e}); not written "
|
||||
"(non-fatal)", file=sys.stderr)
|
||||
continue
|
||||
with open(p, "w") as f:
|
||||
f.write(new)
|
||||
print("[coop-patch] applied: provider response headers are no longer "
|
||||
f"forwarded to callers ({p})")
|
||||
patched_any = True
|
||||
|
||||
if not candidates:
|
||||
print("WARN: get_headers.py not found; provider headers are still forwarded "
|
||||
"(non-fatal)", file=sys.stderr)
|
||||
elif not patched_any:
|
||||
print("WARN: no copy of get_headers.py patched (non-fatal)", file=sys.stderr)
|
||||
|
||||
sys.exit(0)
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Skip LiteLLM's forced `verbose_json` on transcriptions for opting-out models.
|
||||
|
||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
||||
if the target file or the anchor is missing — e.g. after a version bump — it
|
||||
prints a warning and exits 0, so the build and the gateway still succeed. Look
|
||||
for the ``[coop-patch]`` line in the build log.
|
||||
|
||||
Why: for every `openai/` transcription deployment, LiteLLM's transformation
|
||||
injects ``response_format: verbose_json`` when the caller omits the field
|
||||
("ensures 'duration' is received - used for cost calculation"). Tinfoil's
|
||||
``voxtral-small-24b`` rejects verbose_json with a 400 ("Currently do not
|
||||
support verbose_json for voxtral-small-24b"), so every bare
|
||||
``/v1/audio/transcriptions`` call to it failed — including the shape members
|
||||
actually use (file + model, no response_format), and Open WebUI dictation's
|
||||
request shape. The injection exists to obtain a duration for per-SECOND
|
||||
pricing; voxtral-small-24b is billed per token (chat route) / per request
|
||||
(patch_audio_request_cost.py), so it does not need the duration at all.
|
||||
|
||||
Patch: honour a ``coop_no_verbose_json`` flag on the deployment's model_info
|
||||
when it is reachable, and gate the model name explicitly — the flag's plumbing
|
||||
into the transformation's litellm_params is not guaranteed on every path, the
|
||||
name gate is. whisper-large-v3-turbo (which Tinfoil serves verbose_json for,
|
||||
and whose dictation path depends on the injected format) is untouched.
|
||||
|
||||
Scope: only transcriptions that omit response_format AND match the opt-out;
|
||||
every other request keeps upstream behavior exactly.
|
||||
|
||||
Upstream: LiteLLM has no opt-out for this injection as of 1.103.2.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
MARKER = "CO-OP PATCH (skip verbose_json, 2026-11-06)"
|
||||
|
||||
ANCHOR = ''' if "response_format" not in data:
|
||||
data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation'''
|
||||
|
||||
INSERT = ''' if "response_format" not in data:
|
||||
# CO-OP PATCH (skip verbose_json, 2026-11-06): a deployment whose
|
||||
# model_info declares coop_no_verbose_json opts out of the forced
|
||||
# verbose_json injection — Tinfoil's voxtral-small-24b rejects
|
||||
# verbose_json with a 400, and its per-token / per-request billing
|
||||
# does not need the duration this injection exists to obtain. The
|
||||
# model-name gate backs the flag up: the flag's plumbing into this
|
||||
# method's litellm_params is not guaranteed on every call path.
|
||||
_mi = (litellm_params or {}).get("model_info") or {}
|
||||
_coop_skip = (isinstance(_mi, dict) and bool(_mi.get("coop_no_verbose_json"))) or (
|
||||
"voxtral-small" in str(model or "")
|
||||
)
|
||||
if not _coop_skip:
|
||||
data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation'''
|
||||
|
||||
|
||||
def target_path() -> str:
|
||||
if len(sys.argv) > 1:
|
||||
return sys.argv[1]
|
||||
env = os.environ.get("LITELLM_WHISPER_TRANSFORMATION")
|
||||
if env:
|
||||
return env
|
||||
hits = []
|
||||
try: # most reliable: ask the installed package where it lives
|
||||
import litellm # noqa: PLC0415
|
||||
_file = getattr(litellm, "__file__", None)
|
||||
if _file:
|
||||
hits.append(os.path.join(
|
||||
os.path.dirname(_file), "llms", "openai", "transcriptions", "whisper_transformation.py"))
|
||||
except Exception:
|
||||
pass
|
||||
hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py")
|
||||
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py")
|
||||
hits = [h for h in hits if os.path.exists(h)]
|
||||
if not hits:
|
||||
raise SystemExit("[coop-patch] could not locate llms/openai/transcriptions/whisper_transformation.py")
|
||||
return sorted(hits)[0]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
path = target_path()
|
||||
try:
|
||||
source = open(path).read()
|
||||
except OSError as exc:
|
||||
print(f"[coop-patch] cannot read {path}: {exc}")
|
||||
return 0
|
||||
|
||||
if MARKER in source:
|
||||
print(f"[coop-patch] {path} already patched, nothing to do")
|
||||
return 0
|
||||
|
||||
if ANCHOR not in source:
|
||||
print(f"[coop-patch] WARNING anchor not found in {path}; bare transcription calls to "
|
||||
f"opt-out models (voxtral-small-24b) will fail with the provider's verbose_json 400")
|
||||
return 0
|
||||
|
||||
patched = source.replace(ANCHOR, INSERT, 1)
|
||||
compile(patched, path, "exec") # refuse to write something that will not import
|
||||
|
||||
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
||||
print(f"[coop-patch] {path} is not writable; verified anchor and syntax only")
|
||||
return 0
|
||||
|
||||
with open(path, "w") as handle:
|
||||
handle.write(patched)
|
||||
print(f"[coop-patch] applied skip-verbose_json to {path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail if a EUR-quoted provider's model_info is not EUR * FX.
|
||||
|
||||
GreenPT and Regolo quote EUR; LiteLLM costs are USD. Pasting EUR numerals into
|
||||
model_info under-bills members by the FX gap (found 2026-11-18).
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
|
||||
FX = 1.16
|
||||
EUR = {
|
||||
"greenpt/deepseek-v4.1-flash": (0.36, 1.45, 0.04),
|
||||
"greenpt/glm-5.3-flash": (0.11, 0.44, 0.022),
|
||||
"greenpt/glm-5.3": (1.10, 4.40, 0.275),
|
||||
"greenpt/green-embeddings": (0.20, 0.0, None),
|
||||
"greenpt/qwen3-embedding-8b": (0.25, 0.0, None),
|
||||
"regolo/mistral-small-4-119b": (0.20, 0.70, None),
|
||||
"regolo/gemma4-31b": (0.40, 0.60, None),
|
||||
# Route aliases carry their own model_info; members call these names, so a
|
||||
# deployment-only conversion leaves billing stale (found 2026-11-18).
|
||||
"deepseek-flash-green": (0.36, 1.45, 0.04),
|
||||
"glm-flash-green": (0.11, 0.44, 0.022),
|
||||
}
|
||||
|
||||
text = open("config.yaml", encoding="utf-8").read()
|
||||
bad = []
|
||||
for name, (i, o, c) in EUR.items():
|
||||
m = re.search(r"- model_name: " + re.escape(name) + r"\n(.*?)(?=\n - model_name: |\n[a-zA-Z#])", text, re.S)
|
||||
if not m:
|
||||
bad.append(f"{name}: missing block")
|
||||
continue
|
||||
block = m.group(1)
|
||||
def val(key):
|
||||
mm = re.search(key + r": (\S+)", block)
|
||||
return float(mm.group(1)) if mm else None
|
||||
checks = [("input_cost_per_token", i), ("output_cost_per_token", o)]
|
||||
if c is not None:
|
||||
checks.append(("cache_read_input_token_cost", c))
|
||||
for key, eur in checks:
|
||||
want = eur * FX / 1_000_000
|
||||
got = val(key)
|
||||
if got is None or abs(got - want) > max(1e-12, want * 1e-6):
|
||||
bad.append(f"{name}: {key} got {got}, want {want:.10g}")
|
||||
print("checked", len(EUR), "EUR models at FX", FX)
|
||||
if bad:
|
||||
print("\n".join(bad))
|
||||
sys.exit(1)
|
||||
print("OK: EUR list prices converted to USD")
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Register Tinfoil's per-request audio prices in litellm.model_cost at STARTUP.
|
||||
|
||||
Build-time edits of model_prices_and_context_window_backup.json did not reach the
|
||||
runtime map (the deployed gateway still had no entry — issue #5). sitecustomize is
|
||||
imported by CPython before any application import, so this runs inside the gateway
|
||||
process itself, before LiteLLM prices a request. Idempotent and never fatal.
|
||||
"""
|
||||
import os
|
||||
|
||||
ENTRIES = {
|
||||
"openai/voxtral-tts": {
|
||||
"mode": "audio_speech",
|
||||
"input_cost_per_request": 0.002,
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
},
|
||||
"openai/whisper-large-v3-turbo": {
|
||||
"mode": "audio_transcription",
|
||||
"input_cost_per_request": 0.01,
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
},
|
||||
}
|
||||
|
||||
try:
|
||||
import litellm
|
||||
|
||||
_cost = getattr(litellm, "model_cost", None)
|
||||
if isinstance(_cost, dict):
|
||||
for _k, _v in ENTRIES.items():
|
||||
_cur = _cost.get(_k) or {}
|
||||
if _cur.get("input_cost_per_request") != _v["input_cost_per_request"]:
|
||||
_cost[_k] = {**_cur, **_v}
|
||||
except Exception: # never break the interpreter for a pricing nicety
|
||||
pass
|
||||
@@ -6,6 +6,18 @@ set -eu
|
||||
# Configures LiteLLM with Cloudron's PostgreSQL and Redis addons
|
||||
# ============================================================================
|
||||
|
||||
# --- Cloudron scheduler tasks arrive as ARGUMENTS to this script ---
|
||||
# The manifest's scheduler commands run as `docker run <image> <command>`, and
|
||||
# this script is the image ENTRYPOINT — so the task command lands in "$@".
|
||||
# Without this branch every scheduled task booted a second tinfoil-proxy and
|
||||
# LiteLLM (whose ports are already bound in the app's network namespace) and
|
||||
# died on the bind FATAL before doing its job — which looked exactly like a
|
||||
# scheduler that never fired. Exec the task directly instead of booting.
|
||||
if [ "$#" -gt 0 ]; then
|
||||
echo "start.sh: scheduler-task mode — exec: $*"
|
||||
exec "$@"
|
||||
fi
|
||||
|
||||
echo "=== LiteLLM Cloudron Startup ==="
|
||||
|
||||
# --- Cloudron PostgreSQL addon provides:
|
||||
@@ -77,10 +89,13 @@ export LITELLM_MIGRATION_DIR="/app/data/migrations"
|
||||
export UI_USERNAME="admin"
|
||||
export UI_PASSWORD="${LITELLM_MASTER_KEY}"
|
||||
|
||||
# DISABLE the admin UI and API docs. The gateway must stay publicly reachable
|
||||
# for the /v1 API (member keys), but we don't want to advertise a login-gated
|
||||
# admin console (/ui) or LiteLLM's Swagger/ReDoc/openapi surfaces. Admin
|
||||
# visibility comes from the admin dashboard / the API directly.
|
||||
# Nothing disables the static /ui admin bundle on the pinned v1.103.2 (there
|
||||
# is no flag for it): DISABLE_ADMIN_UI="True" only turns off the UI's SSO
|
||||
# login-redirect routes upstream — GET /ui/ still serves the static login
|
||||
# page. Logging into it still requires the master key, and the docs surfaces
|
||||
# below ARE fully disabled (NO_DOCS/NO_REDOC/NO_OPENAPI → 404s). See README
|
||||
# "Admin access" for the wire-verified state and the patch needed to remove
|
||||
# the bundle itself. Kept set because the SSO-redirect disable is still real.
|
||||
export DISABLE_ADMIN_UI="True"
|
||||
export NO_DOCS="True"
|
||||
export NO_REDOC="True"
|
||||
@@ -89,13 +104,19 @@ export NO_OPENAPI="True"
|
||||
# Proxy settings for Cloudron's reverse proxy
|
||||
export LITELLM_PROXY_BASE_URL="https://${CLOUDRON_APP_DOMAIN}"
|
||||
|
||||
# CORS: LiteLLM v1.84.0 reads LITELLM_CORS_ORIGINS natively (upstream added
|
||||
# the env var in place of the old hardcoded `origins = ["*"]`). Lock browser
|
||||
# access to the co-op's own chat origin. The gateway is called
|
||||
# server-to-server (chat → portal → gateway) and by member API keys
|
||||
# (curl/SDKs, not browsers), so no wildcard is needed. A `*` here would let
|
||||
# any website's JS hit the gateway.
|
||||
export LITELLM_CORS_ORIGINS="https://chat.inference.coop"
|
||||
# CORS: LiteLLM reads LITELLM_CORS_ORIGINS natively (upstream replaced the
|
||||
# hardcoded `origins = ["*"]` with this env var in v1.84.0). The value is
|
||||
# deliberately OVERRIDABLE — a Cloudron app env now wins, where before this
|
||||
# export silently overrode it and made the app env look effective when it was not.
|
||||
#
|
||||
# The co-op decided to allow all origins (co-op/support#38, agreed on Loomio and
|
||||
# by a member): CORS is not an auth boundary here. Every request carries a
|
||||
# per-member bearer key, which the browser never attaches implicitly, so an origin
|
||||
# list buys nothing against a party that already holds a key — and a leaked key is
|
||||
# curl-able regardless. What actually constrains abuse is the per-key budget,
|
||||
# model scoping and revocation. The fallback keeps the old chat-only lock if the
|
||||
# env var is ever unset.
|
||||
export LITELLM_CORS_ORIGINS="${LITELLM_CORS_ORIGINS:-https://chat.inference.coop}"
|
||||
|
||||
echo "PostgreSQL: ${CLOUDRON_POSTGRESQL_HOST}:${CLOUDRON_POSTGRESQL_PORT}"
|
||||
echo "Redis: ${CLOUDRON_REDIS_HOST}:${CLOUDRON_REDIS_PORT}"
|
||||
@@ -150,6 +171,46 @@ YAML
|
||||
echo "Config written to $CONFIG_FILE"
|
||||
fi
|
||||
|
||||
# --- Ship config updates with the image ---
|
||||
# /app/data is a persistent volume, so the copy the Dockerfile bakes into
|
||||
# /app/data/config.yaml is shadowed by the volume's file forever — config
|
||||
# changes in the repo would never reach a running gateway. The repo is the
|
||||
# source of truth (README: repos mirror production), so sync the shipped copy
|
||||
# over the volume whenever they differ.
|
||||
if [ -f /app/code/config.yaml ] && ! cmp -s /app/code/config.yaml "$CONFIG_FILE"; then
|
||||
echo "Updating /app/data/config.yaml from the image (repo is the source of truth)"
|
||||
cp /app/code/config.yaml "$CONFIG_FILE"
|
||||
fi
|
||||
# NOTE on the fallback config written above: it can never be SERVED. The very
|
||||
# same boot's sync above overwrites it with the image's (repo) config whenever
|
||||
# they differ — including the boot right after the fallback ran. It is dead
|
||||
# by construction; it exists only so the container is never left with an
|
||||
# empty config file if the volume copy vanishes. Historically this block
|
||||
# carried stale api.tinfoil.sh models that could have resurrected — that
|
||||
# cannot happen through this path, and the shipped /app/code/config.yaml is
|
||||
# the live catalog.
|
||||
|
||||
# --- Daily FX sync (fail-open): convert EUR-priced providers at the ECB rate ---
|
||||
# Prices are refreshed at boot; a nightly timer restarts the gateway at 04:30 UTC.
|
||||
if [ -f /app/code/fx_sync.py ]; then
|
||||
echo "Refreshing EUR->USD prices from the ECB feed (fail-open)..."
|
||||
python3 /app/code/fx_sync.py || echo "fx_sync exited non-zero; continuing with existing prices"
|
||||
fi
|
||||
|
||||
# --- Fail-closed config validation (2026-11-11) ---
|
||||
# Validate the config that will actually be SERVED, before starting anything.
|
||||
# A bad key must fail LOUD at the top of the boot log (the chat outage of
|
||||
# 2026-10-31 showed what silent misconfiguration costs). The -f guard keeps a
|
||||
# missing validator non-fatal (build patch did not apply); an INVALID config
|
||||
# is fatal by design.
|
||||
if [ -f /app/code/validate_config.py ]; then
|
||||
echo "Validating gateway config (fail-closed)..."
|
||||
if ! python3 /app/code/validate_config.py litellm "$CONFIG_FILE"; then
|
||||
echo "FATAL: gateway config.yaml is invalid - refusing to start. Fix or remove the offending key named above." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- Start LiteLLM ---
|
||||
echo "Starting LiteLLM on port 4000..."
|
||||
|
||||
@@ -199,4 +260,21 @@ if [ "$proxy_up" != "1" ]; then
|
||||
fi
|
||||
echo "Tinfoil proxy is up on 127.0.0.1:3301"
|
||||
|
||||
# --- In-container audio keep-alive loop (the ONLY warm path) ---
|
||||
# A separate scheduler-task container was removed 2026-11-09: its container
|
||||
# could not be proven to reach 127.0.0.1:4000 and /app/data, and two warm
|
||||
# paths billed twice for the same ping. The warm path is owned HERE — the
|
||||
# app container, where the network namespace, the master key and the
|
||||
# sidecar are all proven. Every 15 minutes ping tinfoil/voxtral-tts
|
||||
# ($0.002/request) so voice mode skips the ~24s Tinfoil cold-start after
|
||||
# idle. audio_keepalive.py is silent on success and prints on failure, so
|
||||
# /app/data/keepalive.log only ever grows on failure.
|
||||
(
|
||||
while true; do
|
||||
python3 /app/code/audio_keepalive.py >> /app/data/keepalive.log 2>&1 || true
|
||||
sleep 900
|
||||
done
|
||||
) &
|
||||
echo "Audio keep-alive loop started (every 900s)"
|
||||
|
||||
exec litellm --config "$CONFIG_FILE" --port 4000 --host 0.0.0.0
|
||||
@@ -0,0 +1,261 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail-closed config validator for the co-op's gateway and chat apps.
|
||||
|
||||
ONE script, two modes, shared home in ops-docs/scripts/ and shipped
|
||||
byte-identical into both app images (litellm repo root, librechat-custom repo
|
||||
root). Each app's start path calls it before booting the service; a non-zero
|
||||
exit refuses the boot with a ONE-line error naming the offending key.
|
||||
|
||||
Why this exists: the 2026-10-31 chat outage. A `customFooter:` top-level key in
|
||||
librechat.yaml was rejected by LibreChat's strict config validator and the API
|
||||
server refused to start, while the SPA kept being served with HTTP 200 — the
|
||||
app looked up but every API route returned the HTML shell. Nothing had ever
|
||||
checked the config before boot. This validator closes that gap: invalid config
|
||||
fails LOUD and EARLY, at the top of the boot log, naming the key.
|
||||
|
||||
Modes
|
||||
-----
|
||||
litellm Structural check of the gateway's config.yaml:
|
||||
- top-level keys within LiteLLM's documented config surface;
|
||||
- model_list present and well-formed (model_name + litellm_params.model);
|
||||
- the settings blocks are objects; pass_through_endpoints entries have paths.
|
||||
When the gateway's own litellm package is importable it is additionally
|
||||
used to resolve every model string to a provider (catches unknown
|
||||
provider prefixes); the check is skipped with a note where litellm
|
||||
itself is absent.
|
||||
|
||||
librechat Checks librechat.yaml against the DEPLOYED build's own strict schema:
|
||||
it runs `configSchema.strict().safeParse()` from the container's
|
||||
librechat-data-provider via node — the same code the API server runs
|
||||
at boot (its failure mode is `process.exit(1)`, "Exiting due to
|
||||
invalid configuration"). Unrecognized top-level keys are rejected,
|
||||
named explicitly. Where node/the package cannot be found (e.g. this
|
||||
script run from a checkout instead of the image) it falls back to a
|
||||
static top-level allowlist recorded from the deployed build and says
|
||||
so on the failure line.
|
||||
|
||||
Usage
|
||||
-----
|
||||
python3 validate_config.py litellm /app/data/config.yaml
|
||||
python3 validate_config.py librechat /app/data/librechat.yaml
|
||||
|
||||
Exit codes: 0 valid, 1 invalid, 2 usage/environment error (file unreadable,
|
||||
yaml unparseable). Never prints a value from the config — only key names and
|
||||
counts, so it is safe to run against files that carry secrets.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
# LiteLLM's documented top-level config surface (proxy config). Unknown keys
|
||||
# are rejected here — the gateway should never silently ignore a mistyped
|
||||
# section (a "model-list:" typo would otherwise deploy an EMPTY catalog).
|
||||
LITELLM_TOP_LEVEL = {
|
||||
"model_list",
|
||||
"router_settings",
|
||||
"litellm_settings",
|
||||
"general_settings",
|
||||
"settings",
|
||||
"environment_variables",
|
||||
"guardrails",
|
||||
"files",
|
||||
}
|
||||
|
||||
# Fallback allowlist for librechat mode, recorded 2026-11-11 from the DEPLOYED
|
||||
# build (chat.inference.coop, overlay 1.4.8) by reading
|
||||
# configSchema.strict().shape keys. The live schema is always preferred; this
|
||||
# exists so the validator still fails closed when run outside the container.
|
||||
LIBRECHAT_TOP_LEVEL_FALLBACK = {
|
||||
"actions", "balance", "cache", "cloudfront", "endpoints", "fileConfig",
|
||||
"fileStrategies", "fileStrategy", "filteredTools", "filters",
|
||||
"imageOutputType", "includedTools", "interface", "langfuse", "mcpServers",
|
||||
"mcpSettings", "memory", "messageFilter", "modelSpecs", "ocr", "openapi",
|
||||
"permissions", "rateLimits", "registration", "secureImageLinks",
|
||||
"skillSync", "speech", "summarization", "transactions", "turnstile",
|
||||
"version", "webSearch",
|
||||
}
|
||||
|
||||
LC_DATA_PROVIDER_DIST = os.environ.get(
|
||||
"LC_DATA_PROVIDER_DIST",
|
||||
"/app/code/librechat/packages/data-provider/dist/index.js")
|
||||
LC_JS_YAML = os.environ.get(
|
||||
"LC_JS_YAML", "/app/code/librechat/node_modules/js-yaml")
|
||||
|
||||
|
||||
def fail(key, reason):
|
||||
"""ONE line, naming the offending key, then a non-zero exit."""
|
||||
print(f"CONFIG VALIDATION FAILED: {key}: {reason}", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def load_yaml(path):
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
print("CONFIG VALIDATION FAILED: pyyaml not available in this python",
|
||||
flush=True)
|
||||
sys.exit(2)
|
||||
try:
|
||||
with open(path) as fh:
|
||||
data = yaml.safe_load(fh)
|
||||
except FileNotFoundError:
|
||||
print(f"CONFIG VALIDATION FAILED: config file not found: {path}",
|
||||
flush=True)
|
||||
sys.exit(2)
|
||||
except PermissionError:
|
||||
print(f"CONFIG VALIDATION FAILED: config file unreadable: {path}",
|
||||
flush=True)
|
||||
sys.exit(2)
|
||||
except yaml.YAMLError as exc:
|
||||
# Do not echo the raw yaml error: it can quote a secret-bearing line.
|
||||
print("CONFIG VALIDATION FAILED: (root): yaml does not parse",
|
||||
flush=True)
|
||||
sys.exit(2)
|
||||
if data is None:
|
||||
data = {}
|
||||
if not isinstance(data, dict):
|
||||
fail("(root)", "top level must be a mapping")
|
||||
return data
|
||||
|
||||
|
||||
def validate_litellm(path):
|
||||
cfg = load_yaml(path)
|
||||
unknown = sorted(set(cfg) - LITELLM_TOP_LEVEL)
|
||||
if unknown:
|
||||
fail(", ".join(unknown), "unknown top-level key(s) for litellm "
|
||||
f"(allowed: {', '.join(sorted(LITELLM_TOP_LEVEL))})")
|
||||
|
||||
model_list = cfg.get("model_list")
|
||||
if not isinstance(model_list, list) or not model_list:
|
||||
fail("model_list", "must be a non-empty list of model entries")
|
||||
names = set()
|
||||
for i, entry in enumerate(model_list):
|
||||
if not isinstance(entry, dict):
|
||||
fail(f"model_list[{i}]", "entry must be a mapping")
|
||||
name = entry.get("model_name")
|
||||
params = entry.get("litellm_params")
|
||||
if not name or not isinstance(name, str):
|
||||
fail(f"model_list[{i}].model_name", "missing or empty")
|
||||
if name in names:
|
||||
fail(f"model_list[{i}].model_name", f"duplicate model_name {name!r}")
|
||||
names.add(name)
|
||||
if not isinstance(params, dict) or not params.get("model"):
|
||||
fail(f"model_list[{i}].litellm_params.model", "missing or empty")
|
||||
for block in ("router_settings", "litellm_settings", "general_settings"):
|
||||
if block in cfg and not isinstance(cfg[block], dict):
|
||||
fail(block, "must be a mapping")
|
||||
for ep in (cfg.get("general_settings") or {}).get("pass_through_endpoints") or []:
|
||||
if not isinstance(ep, dict) or not ep.get("path"):
|
||||
fail("general_settings.pass_through_endpoints", "each entry needs a path")
|
||||
|
||||
# litellm-assisted check (best effort): resolve each model string to a
|
||||
# provider with the gateway's own package. Skipped where litellm itself
|
||||
# is not importable (never fatal — the structural checks above ran).
|
||||
try:
|
||||
from litellm import get_llm_provider # noqa: WPS433
|
||||
except Exception:
|
||||
print("note: litellm not importable here; provider check skipped",
|
||||
flush=True)
|
||||
else:
|
||||
for entry in model_list:
|
||||
model = entry["litellm_params"]["model"]
|
||||
try:
|
||||
get_llm_provider(model=model, custom_llm_provider=None)
|
||||
except Exception as exc:
|
||||
short = str(exc).split("\n")[0][:120]
|
||||
fail(f"model_list.{entry['model_name']}.litellm_params.model",
|
||||
f"not resolvable to a provider: {short}")
|
||||
|
||||
print(f"config OK (litellm): {len(cfg)} top-level key(s), "
|
||||
f"{len(model_list)} model(s) validated", flush=True)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def validate_librechat(path):
|
||||
cfg = load_yaml(path)
|
||||
if not cfg:
|
||||
print("config OK (librechat): empty config, nothing to validate",
|
||||
flush=True)
|
||||
sys.exit(0)
|
||||
|
||||
# Preferred: the DEPLOYED build's own strict schema, run through node —
|
||||
# the exact code path the API server uses at boot (strict() rejects
|
||||
# unrecognized top-level keys; upstream then exits 1).
|
||||
node_code = r"""
|
||||
const fs = require('fs');
|
||||
let out = {mode: 'schema'};
|
||||
try {
|
||||
const yaml = require(JS_YAML_PATH);
|
||||
const dp = require(DIST_PATH);
|
||||
const cfg = yaml.load(fs.readFileSync(CONFIG_PATH, 'utf8')) || {};
|
||||
const result = dp.configSchema.strict().safeParse(cfg);
|
||||
if (result.success) {
|
||||
out.ok = true;
|
||||
out.keys = Object.keys(cfg).length;
|
||||
} else {
|
||||
out.ok = false;
|
||||
out.errors = (result.error && result.error.issues || []).slice(0, 10)
|
||||
.map(i => ({path: (i.path || []).join('.') || '(root)', msg: i.message}));
|
||||
}
|
||||
} catch (e) {
|
||||
out.mode = 'schema-unavailable';
|
||||
out.error = String(e && e.message || e).slice(0, 200);
|
||||
}
|
||||
console.log('@@' + JSON.stringify(out));
|
||||
""".replace("JS_YAML_PATH", json.dumps(LC_JS_YAML)) \
|
||||
.replace("DIST_PATH", json.dumps(LC_DATA_PROVIDER_DIST)) \
|
||||
.replace("CONFIG_PATH", json.dumps(path))
|
||||
|
||||
if os.path.exists(LC_DATA_PROVIDER_DIST):
|
||||
try:
|
||||
proc = subprocess.run(["node", "-e", node_code], capture_output=True,
|
||||
text=True, timeout=60)
|
||||
for line in (proc.stdout or "").splitlines():
|
||||
if line.startswith("@@"):
|
||||
res = json.loads(line[2:])
|
||||
if res.get("ok"):
|
||||
print(f"config OK (librechat): {res['keys']} top-level "
|
||||
"key(s) accepted by the deployed strict schema",
|
||||
flush=True)
|
||||
sys.exit(0)
|
||||
errs = "; ".join(
|
||||
f"{e['path']}: {e['msg']}" for e in res.get("errors", []))
|
||||
if errs:
|
||||
fail(errs, "rejected by the deployed build's strict "
|
||||
"config schema (configSchema.strict())")
|
||||
fail("(root)", "strict schema rejected the config")
|
||||
# No @@ line: schema path unavailable — fall through to fallback.
|
||||
print("note: deployed schema probe produced no result "
|
||||
f"(stderr: {(proc.stderr or '')[:120]!r})", flush=True)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
print(f"note: could not run node against the deployed schema "
|
||||
f"({type(exc).__name__})", flush=True)
|
||||
else:
|
||||
print("note: deployed data-provider dist not found; using the static "
|
||||
"fallback allowlist recorded from the deployed build", flush=True)
|
||||
|
||||
# Fallback: static allowlist (still fail-closed).
|
||||
unknown = sorted(set(cfg) - LIBRECHAT_TOP_LEVEL_FALLBACK)
|
||||
if unknown:
|
||||
fail(", ".join(unknown), "unknown top-level key(s) for librechat "
|
||||
f"(fallback allowlist: {', '.join(sorted(LIBRECHAT_TOP_LEVEL_FALLBACK))})")
|
||||
print(f"config OK (librechat, fallback allowlist): {len(cfg)} top-level "
|
||||
"key(s)", flush=True)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def main():
|
||||
args = sys.argv[1:]
|
||||
if len(args) != 2 or args[0] not in ("litellm", "librechat"):
|
||||
print("usage: validate_config.py <litellm|librechat> <config.yaml>",
|
||||
file=sys.stderr, flush=True)
|
||||
sys.exit(2)
|
||||
if args[0] == "litellm":
|
||||
validate_litellm(args[1])
|
||||
validate_librechat(args[1])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user