Compare commits
19
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
425d738625 | ||
|
|
f75851d6cf | ||
|
|
79f0447089 | ||
|
|
9d95527e93 | ||
|
|
ad34da72a4 | ||
|
|
d082957167 | ||
|
|
0d73c80562 | ||
|
|
acceb43995 | ||
|
|
3397d07ec1 | ||
|
|
cc8f7d237a | ||
|
|
5389d9acfc | ||
|
|
91aa0e0b1d | ||
|
|
1f2c330762 | ||
|
|
a0ae99329f | ||
|
|
516167aaf9 | ||
|
|
b6e8542146 | ||
|
|
b9af62a614 | ||
|
|
a2864ce34a | ||
|
|
23f73cf6ae |
No files matched your search
+15
-4
@@ -2,10 +2,10 @@
|
|||||||
"id": "ai.coop.litellm",
|
"id": "ai.coop.litellm",
|
||||||
"title": "LiteLLM Gateway",
|
"title": "LiteLLM Gateway",
|
||||||
"author": "inference.coop",
|
"author": "inference.coop",
|
||||||
"description": "LiteLLM AI Gateway — OpenAI-compatible proxy with per-user API keys, usage tracking, rate limiting, and model routing. Packaged for Cloudron with SSO/OIDC and PostgreSQL.",
|
"description": "LiteLLM AI Gateway \u2014 OpenAI-compatible proxy with per-user API keys, usage tracking, rate limiting, and model routing. Packaged for Cloudron with SSO/OIDC and PostgreSQL.",
|
||||||
"tagline": "AI gateway for cooperative inference",
|
"tagline": "AI gateway for cooperative inference",
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"upstreamVersion": "1.84.0",
|
"upstreamVersion": "1.103.2",
|
||||||
"healthCheckPath": "/health/readiness",
|
"healthCheckPath": "/health/readiness",
|
||||||
"httpPort": 4000,
|
"httpPort": 4000,
|
||||||
"memoryLimit": 2147483648,
|
"memoryLimit": 2147483648,
|
||||||
@@ -16,9 +16,20 @@
|
|||||||
"addons": {
|
"addons": {
|
||||||
"postgresql": {},
|
"postgresql": {},
|
||||||
"redis": {},
|
"redis": {},
|
||||||
"localstorage": {}
|
"localstorage": {},
|
||||||
|
"scheduler": {
|
||||||
|
"audio_keepalive": {
|
||||||
|
"schedule": "*/10 * * * *",
|
||||||
|
"command": "/app/code/audio_keepalive.py"
|
||||||
|
}
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"tags": ["ai", "gateway", "proxy", "api"],
|
"tags": [
|
||||||
|
"ai",
|
||||||
|
"gateway",
|
||||||
|
"proxy",
|
||||||
|
"api"
|
||||||
|
],
|
||||||
"mediaLinks": [],
|
"mediaLinks": [],
|
||||||
"changelog": "Initial package for inference.coop"
|
"changelog": "Initial package for inference.coop"
|
||||||
}
|
}
|
||||||
+29
-1
@@ -1,4 +1,4 @@
|
|||||||
FROM ghcr.io/berriai/litellm:v1.84.0
|
FROM ghcr.io/berriai/litellm:v1.103.2
|
||||||
|
|
||||||
# v1.84.0 is the fix line for CVE-2026-35029 (auth bypass on
|
# v1.84.0 is the fix line for CVE-2026-35029 (auth bypass on
|
||||||
# /config/update, fixed 1.83.0) and CVE-2026-59822 (MCP session auth
|
# /config/update, fixed 1.83.0) and CVE-2026-59822 (MCP session auth
|
||||||
@@ -20,6 +20,34 @@ RUN chmod +x /app/code/start.sh
|
|||||||
# Default config — will be overridden by Cloudron env vars at runtime
|
# Default config — will be overridden by Cloudron env vars at runtime
|
||||||
COPY config.yaml /app/data/config.yaml
|
COPY config.yaml /app/data/config.yaml
|
||||||
|
|
||||||
|
# Append the co-op credit-pack hint to LiteLLM's hardcoded budget-exceeded
|
||||||
|
# messages. Done at BUILD time — the runtime rootfs is read-only under
|
||||||
|
# Cloudron, so a boot-time patch would crash the startup script (it did,
|
||||||
|
# 2026-10-06). patch_budget_message.py is non-fatal: it exits 0 even when the
|
||||||
|
# file/message can't be found, so a future LiteLLM version bump can't take the
|
||||||
|
# gateway down.
|
||||||
|
COPY patch_budget_message.py /app/code/patch_budget_message.py
|
||||||
|
RUN python3 /app/code/patch_budget_message.py
|
||||||
|
|
||||||
|
# Image-cost tracking: keep image-generation pricing on the shared cost-map
|
||||||
|
# key so successful image generations appear in SpendLogs (upstream bug
|
||||||
|
# BerriAI/litellm#10023/#12471 — closed not-planned). Non-fatal at build.
|
||||||
|
COPY patch_image_cost.py /app/code/patch_image_cost.py
|
||||||
|
RUN python3 /app/code/patch_image_cost.py
|
||||||
|
|
||||||
|
# Audio per-request pricing: Tinfoil bills voxtral-tts and whisper per request,
|
||||||
|
# which LiteLLM cannot express on the audio paths, so those two were billed at
|
||||||
|
# $0. The patch makes the cost calculator honour a declared
|
||||||
|
# input_cost_per_request. Check for the "[coop-patch] applied" line in the build
|
||||||
|
# log after upgrading LiteLLM: the script is non-fatal and warns instead.
|
||||||
|
COPY patch_audio_request_cost.py /app/code/patch_audio_request_cost.py
|
||||||
|
RUN python3 /app/code/patch_audio_request_cost.py
|
||||||
|
|
||||||
|
# Audio keep-alive: scheduled pings keep the Tinfoil TEE audio enclave
|
||||||
|
# warm so voice mode skips the ~24s cold-start (see audio_keepalive.py header).
|
||||||
|
COPY audio_keepalive.py /app/code/audio_keepalive.py
|
||||||
|
RUN chmod +x /app/code/audio_keepalive.py
|
||||||
|
|
||||||
# Override the base image's ENTRYPOINT (which is `litellm`) so our
|
# Override the base image's ENTRYPOINT (which is `litellm`) so our
|
||||||
# startup script runs instead of being passed as an argument to litellm.
|
# startup script runs instead of being passed as an argument to litellm.
|
||||||
ENTRYPOINT ["/app/code/start.sh"]
|
ENTRYPOINT ["/app/code/start.sh"]
|
||||||
|
|||||||
@@ -4,43 +4,45 @@ A Cloudron app package for [LiteLLM](https://litellm.ai/), the open-source AI ga
|
|||||||
|
|
||||||
LiteLLM is the single gateway between members and the inference backends. It handles per-member API keys, usage metering, rate limiting (via budget caps), and model routing — all behind one OpenAI-compatible endpoint.
|
LiteLLM is the single gateway between members and the inference backends. It handles per-member API keys, usage metering, rate limiting (via budget caps), and model routing — all behind one OpenAI-compatible endpoint.
|
||||||
|
|
||||||
|
**Current base image:** LiteLLM **v1.103.2**, pinned to the exact tag (fix line for CVE-2026-35029 / CVE-2026-59822, CISA KEV-listed; also carries the streaming-usage fix for prompt_tokens_details). See [Updating LiteLLM](#updating-litellm).
|
||||||
|
|
||||||
## What this provides
|
## What this provides
|
||||||
|
|
||||||
- **LiteLLM proxy** with an OpenAI-compatible API on port 4000
|
- **LiteLLM proxy** with an OpenAI-compatible API on port 4000
|
||||||
- **Per-member virtual API keys** with budget caps and rate limits
|
- **Per-member virtual API keys** with budget caps and rate limits
|
||||||
- **Usage / spend tracking** per key, per model
|
- **Usage / spend tracking** per key, per model
|
||||||
- **Model routing** to Tinfoil (TEE-protected inference)
|
- **Model routing** to multiple providers, chosen by member values (private / green / public)
|
||||||
- **Cloudron PostgreSQL** for keys, teams, and spend logs
|
- **Cloudron PostgreSQL** for keys, teams, and spend logs
|
||||||
- **Cloudron Redis** for rate limiting and caching
|
- **Cloudron Redis** for rate limiting and caching
|
||||||
- **Automatic SSL, backups, and sandboxing** (Cloudron-managed)
|
- **Automatic SSL, backups, and sandboxing** (Cloudron-managed)
|
||||||
- **Master key + salt key** auto-generated on first start, persisted in `/app/data`
|
- **Master key + salt key** auto-generated on first start, persisted in `/app/data`
|
||||||
|
- Hardened for the public internet: admin UI and API docs disabled at the proxy; CORS locked to the chat origin
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
```
|
```
|
||||||
Members → Open WebUI / LobeChat → LiteLLM (this app) → Tinfoil (TEE)
|
Members → Open WebUI (chat) / own tools (API) → member portal (key injection)
|
||||||
|
→ LiteLLM gateway (this app)
|
||||||
|
├→ Tinfoil (TEE enclaves — architectural privacy)
|
||||||
|
├→ GreenPT (100% renewable energy)
|
||||||
|
└→ PublicAI (publicly developed, sovereign models)
|
||||||
```
|
```
|
||||||
|
|
||||||
LiteLLM is the control plane (keys, metering, routing). It does **not** run models itself — it forwards to Tinfoil, which runs open models inside trusted execution environments (TEEs).
|
LiteLLM is the control plane (keys, metering, routing). It does **not** run models itself — it forwards to providers, each chosen by members for a different value. A [tinfoil-proxy](https://github.com/tinfoilsh/tinfoil-proxy) sidecar provides EHBP end-to-end encryption to Tinfoil's enclaves.
|
||||||
|
|
||||||
## Files
|
## Files
|
||||||
|
|
||||||
- `CloudronManifest.json` — Cloudron app manifest (addons, ports, metadata)
|
- `CloudronManifest.json` — Cloudron app manifest (addons, ports, memory limit, metadata)
|
||||||
- `Dockerfile` — wraps LiteLLM's official image for Cloudron's environment
|
- `Dockerfile` — wraps LiteLLM's official image for Cloudron's environment (pinned to an exact tag)
|
||||||
- `start.sh` — startup script that wires up Cloudron's Postgres/Redis and writes the default config
|
- `start.sh` — startup script: wires Cloudron's Postgres/Redis, downloads the tinfoil-proxy sidecar, exports `LITELLM_MIGRATION_DIR`
|
||||||
- `config.yaml` — default LiteLLM config (Tinfoil-only, two models)
|
- `config.yaml` — LiteLLM config: model catalog, provider routing, retry policy, custom per-token pricing
|
||||||
- `logo.png` — app icon
|
- `logo.png` — app icon
|
||||||
|
|
||||||
## Models (MVP)
|
## Models
|
||||||
|
|
||||||
Two models, both TEE-protected via Tinfoil:
|
The catalog is defined in `config.yaml` and documented for members in [co-op/docs models](https://git.inference.coop/co-op/docs/src/branch/main/models.md) — that page is the single source of truth; this repo's `config.yaml` mirrors it. Models follow the `provider/model-name` convention so members can choose not just which model but which provider values it embodies (privacy, renewable energy, public development).
|
||||||
|
|
||||||
| Model | Role | Price (in/out per M tokens) |
|
The catalog changes by member decision — expect it to evolve.
|
||||||
|-------|------|------------------------------|
|
|
||||||
| `deepseek-v4-flash` | Default — best agent performance, 1M context | $0.30 / $0.70 |
|
|
||||||
| `gpt-oss-120b` | Fallback — cheapest, built for agentic workflows | $0.15 / $0.60 |
|
|
||||||
|
|
||||||
If DeepSeek is down, LiteLLM automatically falls back to GPT-OSS.
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -70,9 +72,11 @@ cloudron install --location gateway
|
|||||||
|
|
||||||
### Configure
|
### Configure
|
||||||
|
|
||||||
After install, set the Tinfoil API key in Cloudron → app → Settings → Environment Variables:
|
After install, set the provider API keys in Cloudron → app → Settings → Environment Variables:
|
||||||
|
|
||||||
- `TINFOIL_API_KEY` — your Tinfoil API key (from https://tinfoil.sh/)
|
- `TINFOIL_API_KEY` — Tinfoil (https://tinfoil.sh/)
|
||||||
|
- `GREENPT_API_KEY` — GreenPT (https://greenpt.ai/)
|
||||||
|
- `PUBLICAI_API_KEY` — PublicAI (https://publicai.co/)
|
||||||
|
|
||||||
The following are auto-generated and should **not** be set manually:
|
The following are auto-generated and should **not** be set manually:
|
||||||
|
|
||||||
@@ -81,11 +85,9 @@ The following are auto-generated and should **not** be set manually:
|
|||||||
- `DATABASE_URL` — from Cloudron's PostgreSQL addon
|
- `DATABASE_URL` — from Cloudron's PostgreSQL addon
|
||||||
- `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD` — from Cloudron's Redis addon
|
- `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD` — from Cloudron's Redis addon
|
||||||
|
|
||||||
### Access the admin UI
|
### Admin access
|
||||||
|
|
||||||
1. Open `https://gateway.inference.coop/ui`
|
The LiteLLM admin UI and API docs are **disabled** on the public gateway (hardened by default; see `start.sh`). Administrative operations run through the [member portal](https://git.inference.coop/code/member-portal), which holds the master key server-side. If direct LiteLLM admin access is ever needed, re-enable the UI in `start.sh`, redeploy, and use the master key from `/app/data/.master_key` — then disable it again.
|
||||||
2. Log in with username `admin` and the master key as password
|
|
||||||
3. Find the master key: `cloudron exec --app <app-id> cat /app/data/.master_key`
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -156,6 +158,7 @@ expect a 15–20 min maintenance window and **never panic-restart mid-migration*
|
|||||||
|
|
||||||
- Check the [LiteLLM changelog](https://github.com/BerriAI/litellm/releases) for breaking changes
|
- Check the [LiteLLM changelog](https://github.com/BerriAI/litellm/releases) for breaking changes
|
||||||
- The `start.sh` and `config.yaml` may need adjustment if LiteLLM changes its env-var or config schema
|
- The `start.sh` and `config.yaml` may need adjustment if LiteLLM changes its env-var or config schema
|
||||||
|
- **Build-time patches ride on LiteLLM internals — verify each one after an upgrade.** `patch_budget_message.py`, `patch_image_cost.py` (keeps image-gen pricing on the shared cost-map key) and `patch_audio_request_cost.py` (adds the declared per-request charge on the audio paths: Tinfoil bills `voxtral-tts` $0.002 and `whisper-large-v3-turbo` $0.01 per request, which the cost map cannot express). Each prints `[coop-patch]` in the build log. They are non-fatal by design: a missing anchor warns and the build still succeeds, so the behaviour can vanish silently on a version bump. Related: code/litellm#5.
|
||||||
- The CORS env var is `LITELLM_CORS_ORIGINS` (upstream-native since 1.84.0); the old `LITELLM_CORS_ALLOWED_ORIGINS` + sed patch are dead — don't re-add them
|
- The CORS env var is `LITELLM_CORS_ORIGINS` (upstream-native since 1.84.0); the old `LITELLM_CORS_ALLOWED_ORIGINS` + sed patch are dead — don't re-add them
|
||||||
- Test on a staging instance if the version jump is large
|
- Test on a staging instance if the version jump is large
|
||||||
|
|
||||||
@@ -165,21 +168,37 @@ expect a 15–20 min maintenance window and **never panic-restart mid-migration*
|
|||||||
|
|
||||||
| Variable | Required | Purpose |
|
| Variable | Required | Purpose |
|
||||||
|----------|----------|---------|
|
|----------|----------|---------|
|
||||||
| `TINFOIL_API_KEY` | Yes | Tinfoil API key for TEE inference |
|
| `TINFOIL_API_KEY` | Yes | Tinfoil provider (TEE inference) |
|
||||||
|
| `GREENPT_API_KEY` | Yes | GreenPT provider (renewable energy) |
|
||||||
|
| `PUBLICAI_API_KEY` | Yes | PublicAI provider (public models) |
|
||||||
|
| `LITELLM_CORS_ORIGINS` | Set by start.sh | CORS allowlist (chat origin only) |
|
||||||
|
| `LITELLM_MIGRATION_DIR` | Set by start.sh | Persistent Prisma migrations ledger |
|
||||||
|
|
||||||
## Connecting Open WebUI
|
## Connecting chat clients
|
||||||
|
|
||||||
In Open WebUI settings:
|
Open WebUI does **not** talk to this gateway directly — it routes through the [member portal](https://git.inference.coop/code/member-portal), which authenticates the member and injects their personal LiteLLM key per request. Members building their own tools create API keys on the [member dashboard](https://dashboard.inference.coop) and use:
|
||||||
|
|
||||||
- **API Base URL:** `https://gateway.inference.coop/v1`
|
- **API Base URL:** `https://gateway.inference.coop/v1` (or the portal's proxy endpoint for chat-integrated tools)
|
||||||
- **API Key:** a virtual key created in LiteLLM's Admin UI
|
- **API Key:** their own virtual key from the dashboard
|
||||||
|
|
||||||
## OIDC / SSO (future)
|
|
||||||
|
|
||||||
The current package uses LiteLLM's built-in admin/master-key auth, which is sufficient for the MVP. Full OIDC SSO (admin UI behind Cloudron login) is a TODO — add the `"oidc": {}` addon to the manifest and configure LiteLLM's SSO hooks per https://docs.litellm.ai/docs/proxy/custom_sso.
|
|
||||||
|
|
||||||
## Related repos
|
## Related repos
|
||||||
|
|
||||||
- `co-op/org-dev` — the full inference.coop design doc and deploy scripts
|
- `code/member-portal` — membership middleware: provisioning, billing webhooks, key injection
|
||||||
|
- `code/member-dashboard` — member-facing usage and API-key management
|
||||||
|
- `code/admin-panel` — admin overview
|
||||||
- `co-op/website` — the public landing page
|
- `co-op/website` — the public landing page
|
||||||
|
- `co-op/docs` — documentation, including the member-facing model list
|
||||||
- `co-op/design-assets` — logos, fonts, brand materials
|
- `co-op/design-assets` — logos, fonts, brand materials
|
||||||
|
|
||||||
|
## Scheduled tasks (scheduler addon)
|
||||||
|
|
||||||
|
The gateway pings the Tinfoil audio models every 10 minutes (`audio_keepalive.py`,
|
||||||
|
declared in `CloudronManifest.json` → `addons.scheduler`) to keep the TEE audio
|
||||||
|
enclave warm — voice mode then skips the ~24s cold-start after idle. TTS-only:
|
||||||
|
an STT ping would cost $0.01/request (~$52/mo at 6/hr) versus $0.002 for TTS.
|
||||||
|
|
||||||
|
The script reads the master key from `/app/data/.master_key` (written by
|
||||||
|
`start.sh`); the `LITELLM_MASTER_KEY` env var is only exported into the litellm
|
||||||
|
process, not scheduler tasks. Verify after deploys: `cloudron exec --app <id>
|
||||||
|
-- python3 /app/code/audio_keepalive.py` should exit 0 silently, and the deploy
|
||||||
|
log must show `setupAddons: ... scheduler ... audio_keepalive`.
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Audio keep-alive: ping the Tinfoil TEE audio models through localhost LiteLLM
|
||||||
|
so voice mode doesn't pay the ~24s cold-start after idle.
|
||||||
|
|
||||||
|
Runs INSIDE the gateway container via the Cloudron scheduler addon. Uses the
|
||||||
|
LITELLM_MASTER_KEY already in the container env. TTS-only (STT pings cost
|
||||||
|
$0.01/request each; TTS $0.002 — see docs, keep-alive economics). Silent on
|
||||||
|
success; prints + exits non-zero on failure so scheduler logs capture it.
|
||||||
|
"""
|
||||||
|
import json, os, urllib.request, urllib.error, sys
|
||||||
|
|
||||||
|
PORT = os.environ.get("PORT", "4000")
|
||||||
|
|
||||||
|
def master_key():
|
||||||
|
"""The gateway's start.sh generates the master key into /app/data/.master_key
|
||||||
|
and exports LITELLM_MASTER_KEY for the litellm process — scheduler tasks run
|
||||||
|
in a separate process tree, so read the key file directly."""
|
||||||
|
key = os.environ.get("LITELLM_MASTER_KEY", "")
|
||||||
|
if key:
|
||||||
|
return key
|
||||||
|
try:
|
||||||
|
with open("/app/data/.master_key") as f:
|
||||||
|
return f.read().strip()
|
||||||
|
except OSError:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
KEY = master_key()
|
||||||
|
if not KEY:
|
||||||
|
print("keepalive: no master key (env or /app/data/.master_key)", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
fail = 0
|
||||||
|
|
||||||
|
# TTS warm (voxtral-tts) — $0.002/request (Tinfoil request-priced)
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"http://127.0.0.1:{PORT}/v1/audio/speech",
|
||||||
|
data=json.dumps({"model": "tinfoil/voxtral-tts", "input": "ok",
|
||||||
|
"voice": "casual_female"}).encode(),
|
||||||
|
method="POST",
|
||||||
|
headers={"Authorization": f"Bearer {KEY}", "Content-Type": "application/json"})
|
||||||
|
try:
|
||||||
|
urllib.request.urlopen(req, timeout=30)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"keepalive: TTS ping failed: {e}", file=sys.stderr)
|
||||||
|
fail = 1
|
||||||
|
|
||||||
|
# STT warm disabled 2026-10-11: $0.01/STT request × 6/hr ≈ $52/mo. Revisit only
|
||||||
|
# with a cheaper warm strategy.
|
||||||
|
sys.exit(fail)
|
||||||
+93
-64
@@ -5,10 +5,11 @@
|
|||||||
# with EHBP (HPKE) before forwarding to the Tinfoil enclave. Provider key set
|
# with EHBP (HPKE) before forwarding to the Tinfoil enclave. Provider key set
|
||||||
# via environment variable TINFOIL_API_KEY.
|
# via environment variable TINFOIL_API_KEY.
|
||||||
#
|
#
|
||||||
# Pricing (model_info) is per-token, derived from Tinfoil's per-1M rates:
|
# Pricing (model_info) is per-token, derived from provider per-1M rates:
|
||||||
|
# greenpt/deepseek-v4.1-flash: $0.22 in / $1.10 out / $0.011 cached per 1M
|
||||||
# tinfoil/deepseek-v4-1-flash: $0.65 in / $1.45 out / $0.13 cached per 1M
|
# tinfoil/deepseek-v4-1-flash: $0.65 in / $1.45 out / $0.13 cached per 1M
|
||||||
# tinfoil/gpt-oss-120b: $0.15 in / $0.60 out per 1M
|
# tinfoil/gpt-oss-120b: $0.15 in / $0.60 out per 1M
|
||||||
# tinfoil/glm-5-3-flash: $0.40 in / $1.25 out / $0.10 cached per 1M
|
# tinfoil/glm-5.3: $1.80 in / $5.75 out / $0.45 cached per 1M
|
||||||
# Without these, LiteLLM can't price these custom models, so spend shows $0.00
|
# Without these, LiteLLM can't price these custom models, so spend shows $0.00
|
||||||
# and the team budget cap is not enforced.
|
# and the team budget cap is not enforced.
|
||||||
#
|
#
|
||||||
@@ -18,7 +19,22 @@
|
|||||||
# models.md is the member-facing mirror.
|
# models.md is the member-facing mirror.
|
||||||
|
|
||||||
model_list:
|
model_list:
|
||||||
# DeepSeek V4.1 Flash — default model (replaces V4 Flash, deprecated 2026-09-15)
|
# DeepSeek V4.1 Flash on GreenPT — DEFAULT model. Same model family as Tinfoil's
|
||||||
|
# deepseek-v4-1-flash, but on renewable energy and NOT marked Experimental by the
|
||||||
|
# provider (Tinfoil's deepseek + glm-flash carry an "Experimental: may be deprecated
|
||||||
|
# with minimal notice" flag). Policy-based privacy (GreenPT) vs TEE (Tinfoil).
|
||||||
|
# €0.22 in / €1.10 out / €0.011 cached per 1M.
|
||||||
|
- model_name: greenpt/deepseek-v4.1-flash
|
||||||
|
litellm_params:
|
||||||
|
model: openai/deepseek-v4.1-flash
|
||||||
|
api_base: https://api.greenpt.ai/v1
|
||||||
|
api_key: os.environ/GREENPT_API_KEY
|
||||||
|
model_info:
|
||||||
|
input_cost_per_token: 0.00000022
|
||||||
|
output_cost_per_token: 0.00000110
|
||||||
|
cache_read_input_token_cost: 0.000000011
|
||||||
|
|
||||||
|
# DeepSeek V4.1 Flash on Tinfoil (TEE) — private counterpart to the default.
|
||||||
- model_name: tinfoil/deepseek-v4-1-flash
|
- model_name: tinfoil/deepseek-v4-1-flash
|
||||||
litellm_params:
|
litellm_params:
|
||||||
model: openai/deepseek-v4-1-flash
|
model: openai/deepseek-v4-1-flash
|
||||||
@@ -39,39 +55,23 @@ model_list:
|
|||||||
input_cost_per_token: 0.00000015
|
input_cost_per_token: 0.00000015
|
||||||
output_cost_per_token: 0.00000060
|
output_cost_per_token: 0.00000060
|
||||||
|
|
||||||
# GLM-5.3 Flash — fast, efficient MoE model
|
# GLM-5.3 (full flagship) on Tinfoil (TEE) — replaced tinfoil/glm-5-3-flash
|
||||||
- model_name: tinfoil/glm-5-3-flash
|
# (Tinfoil deprecated the flash variant 2026-10-09). $1.80 in / $5.75 out /
|
||||||
|
# $0.45 cached per 1M. Text-only (no vision), reasoning + tool calling.
|
||||||
|
- model_name: tinfoil/glm-5.3
|
||||||
litellm_params:
|
litellm_params:
|
||||||
model: openai/glm-5-3-flash
|
model: openai/glm-5-3
|
||||||
api_base: http://127.0.0.1:3301/v1
|
api_base: http://127.0.0.1:3301/v1
|
||||||
api_key: os.environ/TINFOIL_API_KEY
|
api_key: os.environ/TINFOIL_API_KEY
|
||||||
model_info:
|
model_info:
|
||||||
input_cost_per_token: 0.00000040
|
input_cost_per_token: 0.0000018
|
||||||
output_cost_per_token: 0.00000125
|
output_cost_per_token: 0.00000575
|
||||||
cache_read_input_token_cost: 0.00000010
|
cache_read_input_token_cost: 0.00000045
|
||||||
|
|
||||||
# --- GreenPT (renewable energy) ---
|
# --- GreenPT (renewable energy) ---
|
||||||
# OpenAI-compatible, EU-hosted on 100% renewable energy. No TEE proxy needed —
|
# OpenAI-compatible, EU-hosted on 100% renewable energy. No TEE proxy needed —
|
||||||
# LiteLLM talks to GreenPT directly. Pricing from GreenPT /v1/pricing endpoint.
|
# LiteLLM talks to GreenPT directly.
|
||||||
# green-r: $0.35 in / $0.95 out per 1M (128k ctx)
|
# (Removed 2026-10: green-l/green-r — GreenPT now marks them "not recommended".)
|
||||||
# green-l: $0.25 in / $0.80 out per 1M (128k ctx)
|
|
||||||
- model_name: greenpt/green-r
|
|
||||||
litellm_params:
|
|
||||||
model: openai/green-r
|
|
||||||
api_base: https://api.greenpt.ai/v1
|
|
||||||
api_key: os.environ/GREENPT_API_KEY
|
|
||||||
model_info:
|
|
||||||
input_cost_per_token: 0.00000035
|
|
||||||
output_cost_per_token: 0.00000095
|
|
||||||
|
|
||||||
- model_name: greenpt/green-l
|
|
||||||
litellm_params:
|
|
||||||
model: openai/green-l
|
|
||||||
api_base: https://api.greenpt.ai/v1
|
|
||||||
api_key: os.environ/GREENPT_API_KEY
|
|
||||||
model_info:
|
|
||||||
input_cost_per_token: 0.00000025
|
|
||||||
output_cost_per_token: 0.00000080
|
|
||||||
|
|
||||||
# GLM-5.3 Flash on GreenPT (note the dotted slug — distinct from Tinfoil's
|
# GLM-5.3 Flash on GreenPT (note the dotted slug — distinct from Tinfoil's
|
||||||
# glm-5-3-flash). $0.11 in / $0.44 out / $0.022 cached per 1M.
|
# glm-5-3-flash). $0.11 in / $0.44 out / $0.022 cached per 1M.
|
||||||
@@ -95,6 +95,7 @@ model_list:
|
|||||||
model_info:
|
model_info:
|
||||||
input_cost_per_token: 0.0000011
|
input_cost_per_token: 0.0000011
|
||||||
output_cost_per_token: 0.0000044
|
output_cost_per_token: 0.0000044
|
||||||
|
cache_read_input_token_cost: 0.000000275
|
||||||
|
|
||||||
# --- Audio models (Tinfoil, inside the TEE) ---
|
# --- Audio models (Tinfoil, inside the TEE) ---
|
||||||
# NOT chat models: these power /v1/audio/* routes (STT for dictation, TTS for
|
# NOT chat models: these power /v1/audio/* routes (STT for dictation, TTS for
|
||||||
@@ -103,55 +104,58 @@ model_list:
|
|||||||
# (Open WebUI) by their `mode` field, so they never appear as pickable chat
|
# (Open WebUI) by their `mode` field, so they never appear as pickable chat
|
||||||
# options. Pricing per Tinfoil catalog: whisper $0.05/1M in; voxtral-tts
|
# options. Pricing per Tinfoil catalog: whisper $0.05/1M in; voxtral-tts
|
||||||
# listed $0/$0 (verify on invoice — may be beta-free).
|
# listed $0/$0 (verify on invoice — may be beta-free).
|
||||||
- model_name: tinfoil/whisper-large-v3-turbo
|
|
||||||
litellm_params:
|
|
||||||
model: audio_transcription/whisper-large-v3-turbo
|
|
||||||
api_base: http://127.0.0.1:3301/v1
|
|
||||||
api_key: os.environ/TINFOIL_API_KEY
|
|
||||||
model_info:
|
|
||||||
mode: audio_transcription
|
|
||||||
input_cost_per_token: 0.00000005
|
|
||||||
|
|
||||||
- model_name: tinfoil/voxtral-tts
|
# --- Regolo (EU, zero data retention, 100% renewables) ---
|
||||||
|
# Seeweb S.r.l. (Italy). Added 2026-10-16 following member suggestion (#22).
|
||||||
|
# apertus-70b: replaced PublicAI (removed 2026-10-16) as our Apertus provider —
|
||||||
|
# same open-weights model, ~45% cheaper input, EU ZDR + renewables.
|
||||||
|
# qwen3.5-122b: large vision+reasoning MoE (262K ctx). REQUIRES
|
||||||
|
# chat_template_kwargs: {enable_thinking: false} — otherwise the reasoning
|
||||||
|
# phase consumes the entire token budget and content comes back empty.
|
||||||
|
# Qwen-Image: image generation via /v1/images/generations, billed per pixel.
|
||||||
|
- model_name: regolo/apertus-70b
|
||||||
litellm_params:
|
litellm_params:
|
||||||
model: audio_speech/voxtral-tts
|
model: openai/apertus-70b
|
||||||
api_base: http://127.0.0.1:3301/v1
|
api_base: https://api.regolo.ai/v1
|
||||||
api_key: os.environ/TINFOIL_API_KEY
|
api_key: os.environ/REGOLO_API_KEY
|
||||||
model_info:
|
model_info:
|
||||||
mode: audio_speech
|
mode: chat
|
||||||
input_cost_per_token: 0.0
|
input_cost_per_token: 4.6e-7
|
||||||
output_cost_per_token: 0.0
|
output_cost_per_token: 2.44e-6
|
||||||
|
|
||||||
# --- PublicAI (publicly developed / sovereign models) ---
|
- model_name: regolo/qwen3.5-122b
|
||||||
# OpenAI-compatible gateway for public open models. Apertus is the Swiss AI
|
|
||||||
# Initiative's fully-open model (Apache-2.0: weights, code, and training data).
|
|
||||||
# Pricing from PublicAI /v1/models (per 1M): 8b $0.10/$0.20, 70b $0.82/$2.92.
|
|
||||||
# Member-facing name drops the upstream `swiss-ai/` owner prefix for brevity;
|
|
||||||
# the litellm_params.model keeps the full PublicAI id.
|
|
||||||
- model_name: publicai/apertus-v1.5-8b
|
|
||||||
litellm_params:
|
litellm_params:
|
||||||
model: openai/swiss-ai/apertus-v1.5-8b
|
model: openai/qwen3.5-122b
|
||||||
api_base: https://api.publicai.co/v1
|
api_base: https://api.regolo.ai/v1
|
||||||
api_key: os.environ/PUBLICAI_API_KEY
|
api_key: os.environ/REGOLO_API_KEY
|
||||||
|
chat_template_kwargs: {"enable_thinking": false}
|
||||||
model_info:
|
model_info:
|
||||||
input_cost_per_token: 0.00000010
|
mode: chat
|
||||||
output_cost_per_token: 0.00000020
|
input_cost_per_token: 1.16e-6
|
||||||
|
output_cost_per_token: 4.87e-6
|
||||||
|
|
||||||
- model_name: publicai/apertus-v1.5-70b
|
- model_name: regolo/qwen-image
|
||||||
litellm_params:
|
litellm_params:
|
||||||
model: openai/swiss-ai/apertus-v1.5-70b
|
model: openai/Qwen-Image
|
||||||
api_base: https://api.publicai.co/v1
|
api_base: https://api.regolo.ai/v1
|
||||||
api_key: os.environ/PUBLICAI_API_KEY
|
api_key: os.environ/REGOLO_API_KEY
|
||||||
model_info:
|
model_info:
|
||||||
input_cost_per_token: 0.00000082
|
mode: image_generation
|
||||||
output_cost_per_token: 0.00000292
|
input_cost_per_pixel: 1.5e-8
|
||||||
|
|
||||||
# --- Audio models (Tinfoil, in-TEE) ---
|
# --- Audio models (Tinfoil, in-TEE) ---
|
||||||
# Used for chat dictation (STT) and voice mode (TTS). These are NOT chat
|
# Used for chat dictation (STT) and voice mode (TTS). These are NOT chat
|
||||||
# models: the member portal's /v1/models filters them out of the chat model
|
# models: the member portal's /v1/models filters them out of the chat model
|
||||||
# list (see AUDIO_MODEL_PREFIXES there) so members don't see them in the
|
# list (see AUDIO_MODEL_PREFIXES there) so members don't see them in the
|
||||||
# picker, but they remain fully callable via /v1/audio/* for API users.
|
# picker, but they remain fully callable via /v1/audio/* for API users.
|
||||||
# whisper-large-v3-turbo: $0.05/1M input tokens. voxtral-tts: listed $0/$0.
|
# whisper-large-v3-turbo: $0/1M tokens + $0.01 per request.
|
||||||
|
# voxtral-tts: $0/1M tokens + $0.002 per request.
|
||||||
|
# Tinfoil bills these per REQUEST (its model list carries a `requestPrice`);
|
||||||
|
# reading only the two token prices made TTS look free and left STT billed off
|
||||||
|
# an older catalog. LiteLLM cannot price per request on the audio paths, so
|
||||||
|
# patch_audio_request_cost.py (built in via the Dockerfile) makes the cost
|
||||||
|
# calculator honour `input_cost_per_request` below. Re-check that build log
|
||||||
|
# line after every LiteLLM upgrade.
|
||||||
# NOTE: LiteLLM routes audio by the openai/ provider + the endpoint called
|
# NOTE: LiteLLM routes audio by the openai/ provider + the endpoint called
|
||||||
# (/v1/audio/transcriptions or /v1/audio/speech) — there is no
|
# (/v1/audio/transcriptions or /v1/audio/speech) — there is no
|
||||||
# "audio_transcription/" provider prefix.
|
# "audio_transcription/" provider prefix.
|
||||||
@@ -162,7 +166,10 @@ model_list:
|
|||||||
api_key: os.environ/TINFOIL_API_KEY
|
api_key: os.environ/TINFOIL_API_KEY
|
||||||
model_info:
|
model_info:
|
||||||
mode: audio_transcription
|
mode: audio_transcription
|
||||||
input_cost_per_token: 0.00000005
|
# Tinfoil: $0 per token, $0.01 per request. The per-request charge is the
|
||||||
|
# whole cost, and the patch is what makes it billable.
|
||||||
|
input_cost_per_token: 0
|
||||||
|
input_cost_per_request: 0.01
|
||||||
|
|
||||||
- model_name: tinfoil/voxtral-tts
|
- model_name: tinfoil/voxtral-tts
|
||||||
litellm_params:
|
litellm_params:
|
||||||
@@ -171,6 +178,26 @@ model_list:
|
|||||||
api_key: os.environ/TINFOIL_API_KEY
|
api_key: os.environ/TINFOIL_API_KEY
|
||||||
model_info:
|
model_info:
|
||||||
mode: audio_speech
|
mode: audio_speech
|
||||||
|
# Tinfoil: $0 per token, $0.002 per request — see the audio section note.
|
||||||
|
input_cost_per_request: 0.002
|
||||||
|
|
||||||
|
# --- Embedding model (Tinfoil, in-TEE) ---
|
||||||
|
# nomic-embed-text — 768-dim text embeddings for RAG/semantic search.
|
||||||
|
# NOT a chat model: it powers /v1/embeddings only. Scoped to the admin
|
||||||
|
# account's team (model_info.team_id) so member keys cannot call it — it is
|
||||||
|
# an operator resource for the co-op's own docs-RAG, not a member-facing
|
||||||
|
# model. The portal also filters it out of the chat-facing /v1/models list by
|
||||||
|
# its `mode: embedding`. Pricing per Tinfoil catalog: $0.05/1M input, $0 out.
|
||||||
|
- model_name: tinfoil/nomic-embed-text
|
||||||
|
litellm_params:
|
||||||
|
model: openai/nomic-embed-text
|
||||||
|
api_base: http://127.0.0.1:3301/v1
|
||||||
|
api_key: os.environ/TINFOIL_API_KEY
|
||||||
|
model_info:
|
||||||
|
mode: embedding
|
||||||
|
team_id: cdfdf422-633f-4432-8893-cffee82a8325
|
||||||
|
input_cost_per_token: 0.00000005
|
||||||
|
output_cost_per_token: 0.0
|
||||||
|
|
||||||
# Fallback: if DeepSeek is down, use GPT-OSS
|
# Fallback: if DeepSeek is down, use GPT-OSS
|
||||||
# NOTE: DeepSeek V4.1 Flash is a reasoning model — generations routinely run
|
# NOTE: DeepSeek V4.1 Flash is a reasoning model — generations routinely run
|
||||||
@@ -187,6 +214,8 @@ router_settings:
|
|||||||
retry_policy:
|
retry_policy:
|
||||||
TimeoutErrorRetries: 0
|
TimeoutErrorRetries: 0
|
||||||
fallbacks:
|
fallbacks:
|
||||||
|
- greenpt/deepseek-v4.1-flash:
|
||||||
|
- tinfoil/deepseek-v4-1-flash
|
||||||
- tinfoil/deepseek-v4-1-flash:
|
- tinfoil/deepseek-v4-1-flash:
|
||||||
- tinfoil/gpt-oss-120b
|
- tinfoil/gpt-oss-120b
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,181 @@
|
|||||||
|
"""Bill a declared per-request charge on requests LiteLLM prices only per unit.
|
||||||
|
|
||||||
|
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
||||||
|
if the target file or the anchor strings are missing — e.g. after an upstream
|
||||||
|
version bump — it prints a warning and exits 0, so the build and the gateway
|
||||||
|
still succeed. The build log line ``[coop-patch] applied`` is what confirms it
|
||||||
|
took effect; check it on every LiteLLM upgrade (see README, upgrade checklist).
|
||||||
|
|
||||||
|
Why: Tinfoil charges some models a flat amount PER REQUEST on top of (or instead
|
||||||
|
of) per-token pricing — ``voxtral-tts`` $0.002/request and
|
||||||
|
``whisper-large-v3-turbo`` $0.01/request, both with $0 per-token. LiteLLM can
|
||||||
|
only express per-request pricing on the search / rerank / MCP paths, via
|
||||||
|
``input_cost_per_query`` (see ``litellm/search/cost_calculator.py``, and
|
||||||
|
``litellm/types/utils.py``: "per-request pricing: rerank, search, and Bedrock
|
||||||
|
Marengo embed"). The audio paths price by character, second or token, and
|
||||||
|
``input_cost_per_request`` exists only in the proxy's cost-tracking settings —
|
||||||
|
the model cost calculator never reads it. So speech and transcription usage is
|
||||||
|
billed at $0 however the config is written.
|
||||||
|
|
||||||
|
Patch: make the model cost calculator honour a declared
|
||||||
|
``input_cost_per_request``. ``response_cost_calculator`` is the single funnel for
|
||||||
|
every call type (the proxy's audio endpoints dispatch as ``aspeech`` and
|
||||||
|
``transcription`` and reach it through ``Logging._response_cost_calculator``), so
|
||||||
|
adding the charge at its final return covers both. The declared value is resolved
|
||||||
|
from the per-deployment ``custom_pricing`` the logging layer builds out of our
|
||||||
|
``config.yaml`` ``model_info``, falling back to the shared cost map and to the
|
||||||
|
call metadata.
|
||||||
|
|
||||||
|
Scope: nothing changes unless a model declares ``input_cost_per_request``, so
|
||||||
|
this is inert for every other model on the gateway.
|
||||||
|
|
||||||
|
Upstream: no issue filed; LiteLLM has no per-request audio pricing as of 1.103.2.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import glob
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
MARKER = "CO-OP PATCH (audio per-request cost, 2026-10-26)"
|
||||||
|
|
||||||
|
HELPER = '''def _coop_per_request_cost(
|
||||||
|
*,
|
||||||
|
model: str | None = None,
|
||||||
|
litellm_model_name: str | None = None,
|
||||||
|
custom_pricing: Any | None = None,
|
||||||
|
litellm_logging_obj: Any | None = None,
|
||||||
|
) -> float:
|
||||||
|
"""CO-OP PATCH (audio per-request cost, 2026-10-26)
|
||||||
|
|
||||||
|
Return the model's declared ``input_cost_per_request``, else 0.0.
|
||||||
|
|
||||||
|
LiteLLM cannot price a flat per-request charge on the audio paths, so
|
||||||
|
Tinfoil's per-request models (voxtral-tts, whisper-large-v3-turbo) were
|
||||||
|
billed at $0. Resolution order: the per-deployment custom pricing the
|
||||||
|
logging layer builds from our config.yaml model_info, then the shared cost
|
||||||
|
map, then the call metadata. Inert unless a model declares the field.
|
||||||
|
"""
|
||||||
|
candidates = []
|
||||||
|
if isinstance(custom_pricing, dict):
|
||||||
|
candidates.append(custom_pricing)
|
||||||
|
try:
|
||||||
|
import litellm as _litellm # local import: not every caller binds it globally
|
||||||
|
cost_map = _litellm.model_cost
|
||||||
|
for key in (litellm_model_name, model):
|
||||||
|
entry = cost_map.get(key) if key else None
|
||||||
|
if isinstance(entry, dict):
|
||||||
|
candidates.append(entry)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
details = getattr(litellm_logging_obj, "model_call_details", None) or {}
|
||||||
|
info = (details.get("metadata") or {}).get("model_info")
|
||||||
|
if isinstance(info, dict):
|
||||||
|
candidates.append(info)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
for candidate in candidates:
|
||||||
|
try:
|
||||||
|
value = candidate.get("input_cost_per_request")
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
if value:
|
||||||
|
try:
|
||||||
|
return float(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
'''
|
||||||
|
|
||||||
|
ANCHOR_DEF = "def response_cost_calculator("
|
||||||
|
|
||||||
|
ANCHOR_RETURN = """ region_name=region_name,
|
||||||
|
)
|
||||||
|
return response_cost
|
||||||
|
except Exception as e:
|
||||||
|
raise e"""
|
||||||
|
|
||||||
|
PATCHED_RETURN = """ region_name=region_name,
|
||||||
|
)
|
||||||
|
|
||||||
|
# CO-OP PATCH (audio per-request cost, 2026-10-26): add the declared
|
||||||
|
# per-request charge for models LiteLLM can only price per unit
|
||||||
|
# (Tinfoil's voxtral-tts, whisper-large-v3-turbo). Inert when the model
|
||||||
|
# declares nothing; the early "provider reported the cost" return above
|
||||||
|
# is unaffected.
|
||||||
|
try:
|
||||||
|
response_cost = response_cost + _coop_per_request_cost(
|
||||||
|
model=model,
|
||||||
|
litellm_model_name=litellm_model_name,
|
||||||
|
custom_pricing=custom_pricing,
|
||||||
|
litellm_logging_obj=litellm_logging_obj,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return response_cost
|
||||||
|
except Exception as e:
|
||||||
|
raise e"""
|
||||||
|
|
||||||
|
|
||||||
|
def target_path() -> str:
|
||||||
|
"""The installed litellm cost calculator (or an explicit path for testing)."""
|
||||||
|
if len(sys.argv) > 1:
|
||||||
|
return sys.argv[1]
|
||||||
|
env = os.environ.get("LITELLM_COST_CALCULATOR")
|
||||||
|
if env:
|
||||||
|
return env
|
||||||
|
hits = []
|
||||||
|
try: # most reliable: ask the installed package where it lives
|
||||||
|
import litellm # noqa: PLC0415
|
||||||
|
_file = getattr(litellm, "__file__", None)
|
||||||
|
if _file:
|
||||||
|
hits.append(os.path.join(os.path.dirname(_file), "cost_calculator.py"))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
hits += glob.glob("/usr/lib/python3*/site-packages/litellm/cost_calculator.py")
|
||||||
|
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/cost_calculator.py")
|
||||||
|
hits += glob.glob("/usr/local/lib/python3*/dist-packages/litellm/cost_calculator.py")
|
||||||
|
hits += glob.glob("/opt/**/litellm/cost_calculator.py", recursive=True)
|
||||||
|
hits = [h for h in hits if os.path.exists(h)]
|
||||||
|
if not hits:
|
||||||
|
raise SystemExit("[coop-patch] could not locate litellm/cost_calculator.py")
|
||||||
|
return sorted(hits)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
path = target_path()
|
||||||
|
try:
|
||||||
|
source = open(path).read()
|
||||||
|
except OSError as exc:
|
||||||
|
print(f"[coop-patch] cannot read {path}: {exc}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if MARKER in source:
|
||||||
|
print(f"[coop-patch] {path} already patched, nothing to do")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if ANCHOR_DEF not in source or ANCHOR_RETURN not in source:
|
||||||
|
print(f"[coop-patch] WARNING anchors not found in {path}; audio per-request "
|
||||||
|
f"cost will NOT be billed (check this after a LiteLLM upgrade)")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
patched = source.replace(ANCHOR_DEF, HELPER + ANCHOR_DEF, 1)
|
||||||
|
patched = patched.replace(ANCHOR_RETURN, PATCHED_RETURN, 1)
|
||||||
|
compile(patched, path, "exec") # refuse to write something that will not import
|
||||||
|
|
||||||
|
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
||||||
|
print(f"[coop-patch] {path} is not writable; verified anchors and syntax only")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with open(path, "w") as handle:
|
||||||
|
handle.write(patched)
|
||||||
|
print(f"[coop-patch] applied to {path}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
"""Append the co-op credit-pack hint to LiteLLM's hardcoded budget-exceeded messages.
|
||||||
|
|
||||||
|
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
||||||
|
if the target file or message strings are missing (e.g. after an upstream
|
||||||
|
version bump moves or rewrites them), it prints a warning and exits 0 so the
|
||||||
|
build — and the gateway — still succeeds. The hint is a nice-to-have, never a
|
||||||
|
reason to take the gateway down.
|
||||||
|
"""
|
||||||
|
import sys
|
||||||
|
import glob
|
||||||
|
|
||||||
|
HINT = '. You can add credit packs at https://opencollective.com/inference-cooperative'
|
||||||
|
|
||||||
|
TEAM_MSG = (
|
||||||
|
'message=f"Budget has been exceeded! Team={team_object.team_id} '
|
||||||
|
'Current cost: {spend}, Max budget: {team_object.max_budget}"'
|
||||||
|
)
|
||||||
|
KEY_MSG = (
|
||||||
|
'message=f"Budget has been exceeded! Key={key_descriptor} '
|
||||||
|
'Current cost: {spend}, Max budget: {valid_token.max_budget}"'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Find the proxy auth_checks.py wherever it lives in this image (venv path
|
||||||
|
# varies by Python minor version; the running CLI imports from the image's
|
||||||
|
# venv). Search the site-packages copies, not the type-stub one under
|
||||||
|
# litellm/types/.
|
||||||
|
candidates = glob.glob(
|
||||||
|
"/app/.venv/lib/python*/site-packages/litellm/proxy/auth/auth_checks.py"
|
||||||
|
)
|
||||||
|
|
||||||
|
patched_any = False
|
||||||
|
for p in candidates:
|
||||||
|
with open(p) as f:
|
||||||
|
src = f.read()
|
||||||
|
|
||||||
|
if "opencollective.com/inference-cooperative" in src:
|
||||||
|
print(f"INFO: credit-pack hint already present in {p}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
new = src.replace(TEAM_MSG, TEAM_MSG + ' + "' + HINT + '"')
|
||||||
|
new = new.replace(KEY_MSG, KEY_MSG + ' + "' + HINT + '"')
|
||||||
|
|
||||||
|
if new == src:
|
||||||
|
print(f"WARN: budget-exceeded message format changed in {p}; "
|
||||||
|
"hint not applied (non-fatal)", file=sys.stderr)
|
||||||
|
continue
|
||||||
|
|
||||||
|
with open(p, "w") as f:
|
||||||
|
f.write(new)
|
||||||
|
print(f"INFO: patched budget-exceeded messages in {p}")
|
||||||
|
patched_any = True
|
||||||
|
|
||||||
|
if not candidates:
|
||||||
|
print("WARN: auth_checks.py not found; hint not applied (non-fatal)",
|
||||||
|
file=sys.stderr)
|
||||||
|
elif not patched_any:
|
||||||
|
print("WARN: no budget-exceeded messages patched (non-fatal)",
|
||||||
|
file=sys.stderr)
|
||||||
|
|
||||||
|
sys.exit(0)
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
"""Keep image-generation pricing on the shared cost-map key.
|
||||||
|
|
||||||
|
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
||||||
|
if the target file or anchor strings are missing (e.g. after an upstream
|
||||||
|
version bump), it prints a warning and exits 0 so the build — and the
|
||||||
|
gateway — still succeeds.
|
||||||
|
|
||||||
|
Why: LiteLLM's image cost calculator (default_image_cost_calculator) looks
|
||||||
|
pricing up ONLY in the shared ``{provider}/{model}`` key of
|
||||||
|
``litellm.model_cost`` — it never consults the per-deployment entry. But the
|
||||||
|
router deliberately strips per-deployment pricing from that shared key
|
||||||
|
("so one deployment's overrides don't pollute sibling deployments"), which
|
||||||
|
makes every successful image generation on a custom backend invisible:
|
||||||
|
the cost lookup raises, the spend-log write is skipped, and SpendLogs /
|
||||||
|
the dashboards show nothing. Upstream: BerriAI/litellm#10023 and #12471,
|
||||||
|
closed not-planned.
|
||||||
|
|
||||||
|
Patch: for image_generation deployments, re-add the custom pricing fields to
|
||||||
|
the shared-key entry. Safe for us: the co-op gateway has a single deployment
|
||||||
|
per image backend, so there are no sibling deployments to pollute.
|
||||||
|
"""
|
||||||
|
import sys
|
||||||
|
import glob
|
||||||
|
|
||||||
|
ANCHOR = " # Always register the (possibly mode-preserved) shared backend info."
|
||||||
|
INSERT = ''' # CO-OP PATCH (2026-10-19): image-generation deployments must keep
|
||||||
|
# their pricing on the shared cost-map key — the image cost calculator
|
||||||
|
# (default_image_cost_calculator) reads ONLY this key and never the
|
||||||
|
# per-deployment model_id entry, so stripping pricing here makes every
|
||||||
|
# successful image generation invisible in SpendLogs (upstream:
|
||||||
|
# BerriAI/litellm#10023, #12471, closed not-planned). Safe: the co-op
|
||||||
|
# gateway has a single deployment per image backend — no siblings.
|
||||||
|
if deployment_mode == "image_generation":
|
||||||
|
for _pk, _pv in model_info.items():
|
||||||
|
if _pk in CustomPricingLiteLLMParams.model_fields and _pv is not None:
|
||||||
|
shared_model_info[_pk] = _pv
|
||||||
|
|
||||||
|
'''
|
||||||
|
|
||||||
|
candidates = glob.glob(
|
||||||
|
"/app/.venv/lib/python*/site-packages/litellm/router.py"
|
||||||
|
)
|
||||||
|
|
||||||
|
patched_any = False
|
||||||
|
for p in candidates:
|
||||||
|
with open(p) as f:
|
||||||
|
src = f.read()
|
||||||
|
|
||||||
|
if "CO-OP PATCH (2026-10-19)" in src:
|
||||||
|
print(f"INFO: image-cost patch already present in {p}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
if ANCHOR not in src:
|
||||||
|
print(f"WARN: anchor not found in {p}; patch not applied (non-fatal)",
|
||||||
|
file=sys.stderr)
|
||||||
|
continue
|
||||||
|
|
||||||
|
new = src.replace(ANCHOR, INSERT + ANCHOR, 1)
|
||||||
|
|
||||||
|
if new == src:
|
||||||
|
print(f"WARN: insertion failed in {p} (non-fatal)", file=sys.stderr)
|
||||||
|
continue
|
||||||
|
|
||||||
|
with open(p, "w") as f:
|
||||||
|
f.write(new)
|
||||||
|
print(f"INFO: patched image-cost registration in {p}")
|
||||||
|
patched_any = True
|
||||||
|
|
||||||
|
if not candidates:
|
||||||
|
print("WARN: router.py not found; patch not applied (non-fatal)",
|
||||||
|
file=sys.stderr)
|
||||||
|
elif not patched_any:
|
||||||
|
print("WARN: no files patched (non-fatal)", file=sys.stderr)
|
||||||
|
|
||||||
|
sys.exit(0)
|
||||||
@@ -37,6 +37,11 @@ if [ ! -f "$MASTER_KEY_FILE" ]; then
|
|||||||
python3 -c "import secrets; print('sk-' + secrets.token_urlsafe(32))" > "$MASTER_KEY_FILE"
|
python3 -c "import secrets; print('sk-' + secrets.token_urlsafe(32))" > "$MASTER_KEY_FILE"
|
||||||
fi
|
fi
|
||||||
export LITELLM_MASTER_KEY=$(cat "$MASTER_KEY_FILE")
|
export LITELLM_MASTER_KEY=$(cat "$MASTER_KEY_FILE")
|
||||||
|
# Regolo API key (member-suggested provider, 2026-10-16): stored in /app/data by the operator
|
||||||
|
# via the Cloudron file manager — kept out of app env config and git.
|
||||||
|
if [ -f "/app/data/regolo_key.txt" ]; then
|
||||||
|
export REGOLO_API_KEY=$(cat /app/data/regolo_key.txt | tr -d '[:space:]')
|
||||||
|
fi
|
||||||
|
|
||||||
# Generate a salt key if not already in /app/data
|
# Generate a salt key if not already in /app/data
|
||||||
SALT_KEY_FILE="/app/data/.salt_key"
|
SALT_KEY_FILE="/app/data/.salt_key"
|
||||||
@@ -165,11 +170,33 @@ if [ ! -f "$TINFOIL_PROXY_BIN" ]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
echo "Starting Tinfoil proxy on 127.0.0.1:3301..."
|
echo "Starting Tinfoil proxy on 127.0.0.1:3301..."
|
||||||
|
# Kill any stale proxy from a previous start attempt first — on process-level
|
||||||
|
# restarts (container NOT recreated) the old proxy still holds 127.0.0.1:3301
|
||||||
|
# and the new one dies with "bind: address already in use", leaving an
|
||||||
|
# unmanaged listener running pre-update code/flags.
|
||||||
|
pkill -f "tinfoil-proxy" 2>/dev/null || true
|
||||||
|
sleep 1
|
||||||
"$TINFOIL_PROXY_BIN" -b 127.0.0.1 -p 3301 &
|
"$TINFOIL_PROXY_BIN" -b 127.0.0.1 -p 3301 &
|
||||||
TINFOIL_PROXY_PID=$!
|
TINFOIL_PROXY_PID=$!
|
||||||
echo "Tinfoil proxy PID: $TINFOIL_PROXY_PID"
|
echo "Tinfoil proxy PID: $TINFOIL_PROXY_PID"
|
||||||
|
|
||||||
# Give the proxy a moment to verify the enclave attestation
|
# Give the proxy a moment to verify the enclave attestation, and FAIL HARD if
|
||||||
sleep 3
|
# it didn't come up — LiteLLM routing to a dead 3301 breaks every tinfoil
|
||||||
|
# request, and a silent continue makes that look like a provider outage.
|
||||||
|
proxy_up=0
|
||||||
|
for _ in $(seq 1 10); do
|
||||||
|
if kill -0 "$TINFOIL_PROXY_PID" 2>/dev/null && \
|
||||||
|
(exec 3<>"/dev/tcp/127.0.0.1/3301") 2>/dev/null; then
|
||||||
|
exec 3>&- 3<&- || true
|
||||||
|
proxy_up=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
if [ "$proxy_up" != "1" ]; then
|
||||||
|
echo "FATAL: tinfoil-proxy did not bind 127.0.0.1:3301" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Tinfoil proxy is up on 127.0.0.1:3301"
|
||||||
|
|
||||||
exec litellm --config "$CONFIG_FILE" --port 4000 --host 0.0.0.0
|
exec litellm --config "$CONFIG_FILE" --port 4000 --host 0.0.0.0
|
||||||
Reference in new issue
Block a user