Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c43d45c455 | ||
|
|
5d38403892 | ||
|
|
943701581c | ||
|
|
abaeb255b2 | ||
|
|
67df968a04 | ||
|
|
54925cae97 |
No files matched your search
@@ -0,0 +1,2 @@
|
||||
__pycache__/
|
||||
*.pyc
|
||||
@@ -5,7 +5,7 @@
|
||||
"description": "LiteLLM AI Gateway \u2014 OpenAI-compatible proxy with per-user API keys, usage tracking, rate limiting, and model routing. Packaged for Cloudron with SSO/OIDC and PostgreSQL.",
|
||||
"tagline": "AI gateway for cooperative inference",
|
||||
"version": "1.0.0",
|
||||
"upstreamVersion": "1.103.2",
|
||||
"upstreamVersion": "1.108.3",
|
||||
"healthCheckPath": "/health/readiness",
|
||||
"httpPort": 4000,
|
||||
"memoryLimit": 2147483648,
|
||||
|
||||
+18
-1
@@ -1,4 +1,4 @@
|
||||
FROM ghcr.io/berriai/litellm:v1.103.2
|
||||
FROM ghcr.io/berriai/litellm:v1.108.3
|
||||
|
||||
# v1.84.0 is the fix line for CVE-2026-35029 (auth bypass on
|
||||
# /config/update, fixed 1.83.0) and CVE-2026-59822 (MCP session auth
|
||||
@@ -92,6 +92,23 @@ RUN python3 /app/code/patch_passthrough_token_cost.py
|
||||
COPY audio_keepalive.py /app/code/audio_keepalive.py
|
||||
RUN chmod +x /app/code/audio_keepalive.py
|
||||
|
||||
# Skip LiteLLM's forced verbose_json on transcription requests for opting-out
|
||||
# models: Tinfoil's voxtral-small-24b rejects verbose_json with a 400, which
|
||||
# broke every bare /v1/audio/transcriptions call to it (file + model — the
|
||||
# shape members and dictation send). Its billing does not need the duration
|
||||
# the injection exists for. Non-fatal; check the "[coop-patch] applied" line
|
||||
# in the build log on every LiteLLM upgrade.
|
||||
COPY patch_transcription_response_format.py /app/code/patch_transcription_response_format.py
|
||||
RUN python3 /app/code/patch_transcription_response_format.py
|
||||
|
||||
# TTS default response format: LibreChat sends no response_format on its TTS
|
||||
# request and Tinfoil's voxtral-tts then serves WAV (~8-17s warm measured) —
|
||||
# while LibreChat labels the reply audio/mpeg regardless. Default to mp3
|
||||
# (OpenAI's documented default, ~3s warm measured). Non-fatal; check the
|
||||
# "[coop-patch] applied" line in the build log on every LiteLLM upgrade.
|
||||
COPY patch_audio_default_mp3.py /app/code/patch_audio_default_mp3.py
|
||||
RUN python3 /app/code/patch_audio_default_mp3.py
|
||||
|
||||
# Override the base image's ENTRYPOINT (which is `litellm`) so our
|
||||
# startup script runs instead of being passed as an argument to litellm.
|
||||
ENTRYPOINT ["/app/code/start.sh"]
|
||||
|
||||
@@ -4,7 +4,7 @@ A Cloudron app package for [LiteLLM](https://litellm.ai/), the open-source AI ga
|
||||
|
||||
LiteLLM is the single gateway between members and the inference backends. It handles per-member API keys, usage metering, rate limiting (via budget caps), and model routing — all behind one OpenAI-compatible endpoint.
|
||||
|
||||
**Current base image:** LiteLLM **v1.103.2**, pinned to the exact tag (fix line for CVE-2026-35029 / CVE-2026-59822, CISA KEV-listed; also carries the streaming-usage fix for prompt_tokens_details). See [Updating LiteLLM](#updating-litellm).
|
||||
**Current base image:** LiteLLM **v1.108.3**, pinned to the exact tag (upgraded from v1.103.2 in the 2026-11-08/09 maintenance window; minimum supported line remains v1.84.0, the fix line for CVE-2026-35029 / CVE-2026-59822, CISA KEV-listed). See [Updating LiteLLM](#updating-litellm).
|
||||
|
||||
## What this provides
|
||||
|
||||
@@ -149,6 +149,19 @@ expect a 15–20 min maintenance window and **never panic-restart mid-migration*
|
||||
|
||||
1. **Pick a target version** from the [LiteLLM changelog](https://github.com/BerriAI/litellm/releases). Stay at or above v1.84.0 (the fix line for CVE-2026-35029 and CVE-2026-59822, the latter on CISA's KEV list).
|
||||
|
||||
**Re-validate the build patches against the target's real sources first** — the
|
||||
scripts fail soft (a moved anchor prints a WARN and exits 0), so a silently
|
||||
skipped patch ships. Ten patches are in the Dockerfile today; each must print
|
||||
its `applied` line in the build log. The cheapest rehearsal is to unpack the
|
||||
target wheel into a scratch tree and run the scripts against it (2026-11-09,
|
||||
v1.108.3: all ten apply — the CORS anchor needs the 1.108.x fallback already
|
||||
carried by `patch_expose_cost_headers.py`). Two gotchas in that harness:
|
||||
`patch_audio_request_cost.py` finds its target by `import litellm`, which fails
|
||||
outside a full install — point `LITELLM_COST_CALCULATOR` at the scratch
|
||||
`cost_calculator.py` instead, or it exits with "could not locate"; and running
|
||||
the scripts by hand leaves files owned by your user, which is fine in scratch
|
||||
and never in the image.
|
||||
|
||||
2. **Bump the pin** in `Dockerfile`:
|
||||
|
||||
```dockerfile
|
||||
|
||||
Binary file not shown.
+90
-8
@@ -10,6 +10,11 @@
|
||||
# tinfoil/deepseek-v4-1-flash: $0.65 in / $1.45 out / $0.13 cached per 1M
|
||||
# tinfoil/gpt-oss-120b: $0.15 in / $0.60 out per 1M
|
||||
# tinfoil/glm-5.3: $1.80 in / $5.75 out / $0.45 cached per 1M
|
||||
# greenpt/qwen3-embedding-8b: $0.25 in / $0 out per 1M (embeddings, 32K ctx)
|
||||
# greenpt/green-embeddings: $0.20 in / $0 out per 1M (embeddings, 32K ctx)
|
||||
# tinfoil/nomic-embed-text: $0.016 in / $0 out per 1M (embeddings, 8K ctx, TEE)
|
||||
# tinfoil/voxtral-small-24b: $0.20 in / $0.60 out per 1M (token-priced audio;
|
||||
# +$0.00008/request on /v1/audio/transcriptions)
|
||||
# Without these, LiteLLM can't price these custom models, so spend shows $0.00
|
||||
# and the team budget cap is not enforced.
|
||||
#
|
||||
@@ -150,6 +155,8 @@ model_list:
|
||||
# picker, but they remain fully callable via /v1/audio/* for API users.
|
||||
# whisper-large-v3-turbo: $0/1M tokens + $0.01 per request.
|
||||
# voxtral-tts: $0/1M tokens + $0.002 per request.
|
||||
# voxtral-small-24b: $0.20/1M in + $0.60/1M out tokens, $0 per request —
|
||||
# the only TOKEN-PRICED audio model in their catalog (its block below).
|
||||
# Tinfoil bills these per REQUEST (its model list carries a `requestPrice`);
|
||||
# reading only the two token prices made TTS look free and left STT billed off
|
||||
# an older catalog. LiteLLM cannot price per request on the audio paths, so
|
||||
@@ -171,6 +178,34 @@ model_list:
|
||||
input_cost_per_token: 0
|
||||
input_cost_per_request: 0.01
|
||||
|
||||
# voxtral-small-24b — Tinfoil's TOKEN-PRICED STT model: the only audio entry
|
||||
# in their catalog with real token prices ($0.20 in / $0.60 out per 1M, $0
|
||||
# per request). The chat route (audio sent as input_audio) returns real
|
||||
# token usage and bills exactly at those rates. The /v1/audio/transcriptions
|
||||
# route returns NO token counts (duration only), so per-token billing cannot
|
||||
# fire there; Tinfoil tokenizes audio in ~30s chunks (~375-380 tokens each —
|
||||
# measured: a 1-second clip and a 6-second clip both cost ~380 prompt
|
||||
# tokens), so for the short dictation requests this route serves the
|
||||
# chunk-equivalent is ~$0.00008/request — declared as input_cost_per_request
|
||||
# below, added by patch_audio_request_cost.py, which is scoped to the audio
|
||||
# call types so chat calls to this model are NOT double-billed. Audio longer
|
||||
# than ~30s should go through the chat route, which meters exactly.
|
||||
- model_name: tinfoil/voxtral-small-24b
|
||||
litellm_params:
|
||||
model: openai/voxtral-small-24b
|
||||
api_base: http://127.0.0.1:3301/v1
|
||||
api_key: os.environ/TINFOIL_API_KEY
|
||||
model_info:
|
||||
mode: audio_transcription
|
||||
# Opt out of LiteLLM's forced response_format=verbose_json injection on
|
||||
# transcription calls (patch_transcription_response_format.py): Tinfoil
|
||||
# rejects verbose_json for this model with a 400, and nothing in its
|
||||
# billing needs the duration the injection exists to obtain.
|
||||
coop_no_verbose_json: true
|
||||
input_cost_per_token: 0.0000002
|
||||
output_cost_per_token: 0.0000006
|
||||
input_cost_per_request: 0.00008
|
||||
|
||||
- model_name: tinfoil/voxtral-tts
|
||||
litellm_params:
|
||||
model: openai/voxtral-tts
|
||||
@@ -190,13 +225,17 @@ model_list:
|
||||
input_cost_per_character: 0
|
||||
input_cost_per_request: 0.002
|
||||
|
||||
# --- Embedding model (Tinfoil, in-TEE) ---
|
||||
# --- Embedding models (Tinfoil, in-TEE; and GreenPT) ---
|
||||
# nomic-embed-text — 768-dim text embeddings for RAG/semantic search.
|
||||
# NOT a chat model: it powers /v1/embeddings only. Scoped to the admin
|
||||
# account's team (model_info.team_id) so member keys cannot call it — it is
|
||||
# an operator resource for the co-op's own docs-RAG, not a member-facing
|
||||
# model. The portal also filters it out of the chat-facing /v1/models list by
|
||||
# its `mode: embedding`. Pricing per Tinfoil catalog: $0.05/1M input, $0 out.
|
||||
# NOT a chat model: it powers /v1/embeddings only. MEMBER-FACING since
|
||||
# 2026-11-06 — the team_id scoping (operator-account-only) was removed so
|
||||
# members get a TEE-protected embedding option. This is the same model the
|
||||
# co-op's own docs-RAG runs on inside the TEE (Open WebUI's knowledge base);
|
||||
# that traffic is unaffected by members also being able to call it. The
|
||||
# portal filters `mode: embedding` out of the chat-facing /v1/models list, so
|
||||
# it stays API-only. Pricing per Tinfoil catalog (inference.tinfoil.sh
|
||||
# /v1/models): $0.016/1M input, $0 out — corrected 2026-11-06 from a stale
|
||||
# $0.05 declaration; 8K context declared so clients don't invent larger.
|
||||
- model_name: tinfoil/nomic-embed-text
|
||||
litellm_params:
|
||||
model: openai/nomic-embed-text
|
||||
@@ -204,10 +243,53 @@ model_list:
|
||||
api_key: os.environ/TINFOIL_API_KEY
|
||||
model_info:
|
||||
mode: embedding
|
||||
team_id: cdfdf422-633f-4432-8893-cffee82a8325
|
||||
input_cost_per_token: 0.00000005
|
||||
max_input_tokens: 8192
|
||||
input_cost_per_token: 0.000000016
|
||||
output_cost_per_token: 0.0
|
||||
|
||||
# --- Embedding model (GreenPT, member-facing) ---
|
||||
# qwen3-embedding-8b — text embeddings via /v1/embeddings, callable by any
|
||||
# member key. Added 2026-11-06 after a member asked for an embedding model,
|
||||
# filling the member-facing gap on renewable EU hosting (Tinfoil's private
|
||||
# embedding was opened to members the same day). The portal filters
|
||||
# mode:embedding out of the chat picker, so it is API-only — same shape as
|
||||
# the audio models above.
|
||||
# Pricing per GreenPT /v1/pricing: $0.25/1M input, no output charge
|
||||
# (embeddings return vectors; GreenPT lists maxOutputTokens null), 32K
|
||||
# context. max_input_tokens declared from the provider's contextWindowTokens
|
||||
# so clients don't invent a larger one (the null-caps defect).
|
||||
- model_name: greenpt/qwen3-embedding-8b
|
||||
litellm_params:
|
||||
model: openai/qwen3-embedding-8b
|
||||
api_base: https://api.greenpt.ai/v1
|
||||
api_key: os.environ/GREENPT_API_KEY
|
||||
model_info:
|
||||
mode: embedding
|
||||
input_cost_per_token: 0.00000025
|
||||
output_cost_per_token: 0.0
|
||||
max_input_tokens: 32000
|
||||
|
||||
# --- Embedding model (GreenPT, member-facing) ---
|
||||
# green-embeddings — GreenPT's own embedding model (2560-dim, measured live).
|
||||
# Added 2026-11-06 after a catalog survey: the only genuinely distinct
|
||||
# embedding model on GreenPT besides qwen3-embedding-8b. Their catalog also
|
||||
# lists a `green-embedding` (singular) slug — verified the SAME model
|
||||
# (identical vectors on the same input), so only green-embeddings is served.
|
||||
# Cheaper per token than qwen3-embedding-8b; the reranker sibling
|
||||
# (green-rerank) is a different capability and is NOT served.
|
||||
# Same shape: /v1/embeddings only, filtered out of the chat picker.
|
||||
# Pricing per GreenPT /v1/pricing: $0.20/1M input, no output charge, 32K ctx.
|
||||
- model_name: greenpt/green-embeddings
|
||||
litellm_params:
|
||||
model: openai/green-embeddings
|
||||
api_base: https://api.greenpt.ai/v1
|
||||
api_key: os.environ/GREENPT_API_KEY
|
||||
model_info:
|
||||
mode: embedding
|
||||
input_cost_per_token: 0.0000002
|
||||
output_cost_per_token: 0.0
|
||||
max_input_tokens: 32000
|
||||
|
||||
# Fallback: if DeepSeek is down, use GPT-OSS
|
||||
# NOTE: DeepSeek V4.1 Flash is a reasoning model — generations routinely run
|
||||
# longer than 30s. A 30s timeout caused LiteLLM to abandon in-flight DeepSeek
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Default /v1/audio/speech to mp3 when the caller omits response_format.
|
||||
|
||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
||||
if the target file or the anchor is missing — e.g. after a version bump — it
|
||||
prints a warning and exits 0, so the build and the gateway still succeed. Look
|
||||
for the ``[coop-patch]`` line in the build log.
|
||||
|
||||
Why: LibreChat's speech client sends no ``response_format`` on its TTS request
|
||||
(TTSService.openAIProvider builds {input, model, voice} — the field is not in
|
||||
its schema), and Tinfoil's voxtral-tts then serves WAV. Measured 2026-11-07
|
||||
through the gateway: WAV ~8-17s warm per request (145-300 KB), while the SAME
|
||||
model with ``response_format: "mp3"`` answers in ~2.6-3.1s (28-39 KB) — the
|
||||
provider streams/encodes mp3 progressively. LibreChat additionally labels the
|
||||
reply ``Content-Type: audio/mpeg`` unconditionally, so WAV bytes were served
|
||||
under an mp3 label and browsers depended on MIME sniffing to play them.
|
||||
|
||||
Patch: default ``response_format`` to ``mp3`` (OpenAI's own documented default
|
||||
for /v1/audio/speech) when the request omits it. An explicit response_format
|
||||
from any caller still wins, so API members who want WAV can keep requesting
|
||||
it. Scope: only the aspeech route; transcriptions are untouched.
|
||||
|
||||
Upstream: LiteLLM 1.103.2 forwards response_format verbatim and leaves the
|
||||
provider default in place when it is absent.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
MARKER = "CO-OP PATCH (TTS default mp3, 2026-11-07)"
|
||||
|
||||
ANCHOR = ''' ### CALL HOOKS ### - modify incoming data / reject request before calling the model
|
||||
data = await proxy_logging_obj.pre_call_hook(
|
||||
user_api_key_dict=user_api_key_dict, data=data, call_type="aspeech"
|
||||
)'''
|
||||
|
||||
INSERT = ANCHOR + '''
|
||||
|
||||
# CO-OP PATCH (TTS default mp3, 2026-11-07): LibreChat sends no
|
||||
# response_format and Tinfoil's voxtral-tts then serves WAV — measured
|
||||
# ~8-17s warm vs ~3s for mp3, which the provider streams. LibreChat
|
||||
# labels the reply audio/mpeg regardless. Default to mp3 (OpenAI's
|
||||
# documented default); an explicit response_format still wins.
|
||||
if not data.get("response_format"):
|
||||
data["response_format"] = "mp3"'''
|
||||
|
||||
|
||||
def target_path() -> str:
|
||||
if len(sys.argv) > 1:
|
||||
return sys.argv[1]
|
||||
env = os.environ.get("LITELLM_PROXY_SERVER")
|
||||
if env:
|
||||
return env
|
||||
hits = []
|
||||
try: # most reliable: ask the installed package where it lives
|
||||
import litellm # noqa: PLC0415
|
||||
_file = getattr(litellm, "__file__", None)
|
||||
if _file:
|
||||
hits.append(os.path.join(os.path.dirname(_file), "proxy", "proxy_server.py"))
|
||||
except Exception:
|
||||
pass
|
||||
hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/proxy/proxy_server.py")
|
||||
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/proxy/proxy_server.py")
|
||||
hits = [h for h in hits if os.path.exists(h)]
|
||||
if not hits:
|
||||
raise SystemExit("[coop-patch] could not locate litellm/proxy/proxy_server.py")
|
||||
return sorted(hits)[0]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
path = target_path()
|
||||
try:
|
||||
source = open(path).read()
|
||||
except OSError as exc:
|
||||
print(f"[coop-patch] cannot read {path}: {exc}")
|
||||
return 0
|
||||
|
||||
if MARKER in source:
|
||||
print(f"[coop-patch] {path} already patched, nothing to do")
|
||||
return 0
|
||||
|
||||
count = source.count(ANCHOR)
|
||||
if count != 1:
|
||||
print(f"[coop-patch] WARNING aspeech pre_call_hook anchor found {count}x (need exactly 1) "
|
||||
f"in {path}; TTS stays on the provider's default (WAV for Tinfoil)")
|
||||
return 0
|
||||
|
||||
patched = source.replace(ANCHOR, INSERT, 1)
|
||||
compile(patched, path, "exec") # refuse to write something that will not import
|
||||
|
||||
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
||||
print(f"[coop-patch] {path} is not writable; verified anchor and syntax only")
|
||||
return 0
|
||||
|
||||
with open(path, "w") as handle:
|
||||
handle.write(patched)
|
||||
print(f"[coop-patch] applied TTS default mp3 to {path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -27,7 +27,13 @@ from the per-deployment ``custom_pricing`` the logging layer builds out of our
|
||||
call metadata.
|
||||
|
||||
Scope: nothing changes unless a model declares ``input_cost_per_request``, so
|
||||
this is inert for every other model on the gateway.
|
||||
this is inert for every other model on the gateway. Since 2026-11-06 the charge
|
||||
is added only on the AUDIO call types (transcription/atranscription/speech/
|
||||
aspeech): a model that also carries per-token pricing (``voxtral-small-24b``)
|
||||
bills its chat calls exactly per token, and a per-request charge on top of
|
||||
that would double-bill them. The audio endpoints are the paths that cannot
|
||||
price per token (the provider returns no token counts there), so the flat
|
||||
charge belongs to them alone.
|
||||
|
||||
Upstream: no issue filed; LiteLLM has no per-request audio pricing as of 1.103.2.
|
||||
"""
|
||||
@@ -58,7 +64,9 @@ HELPER = '''def _coop_per_request_cost(
|
||||
"""
|
||||
def _is_audio(name: Any) -> bool:
|
||||
text = str(name or "").lower()
|
||||
return "tts" in text or "whisper" in text
|
||||
return (
|
||||
"tts" in text or "whisper" in text or "voxtral" in text
|
||||
)
|
||||
|
||||
cost_map = {}
|
||||
try:
|
||||
@@ -140,17 +148,34 @@ PATCHED_RETURN = """ region_name=region_name,
|
||||
# per-request charge for models LiteLLM can only price per unit
|
||||
# (Tinfoil's voxtral-tts, whisper-large-v3-turbo). Inert when the model
|
||||
# declares nothing; the early "provider reported the cost" return above
|
||||
# is unaffected.
|
||||
# is unaffected. Scoped to the AUDIO call types (2026-11-06): a model
|
||||
# that ALSO carries per-token pricing (voxtral-small-24b) bills its
|
||||
# chat calls exactly per token and must not take the per-request
|
||||
# charge on top of that.
|
||||
try:
|
||||
_coop_pr = _coop_per_request_cost(
|
||||
model=model,
|
||||
litellm_model_name=litellm_model_name,
|
||||
custom_pricing=custom_pricing,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
_coop_audio_call = call_type in (
|
||||
"transcription",
|
||||
"atranscription",
|
||||
"speech",
|
||||
"aspeech",
|
||||
)
|
||||
_coop_pr = (
|
||||
_coop_per_request_cost(
|
||||
model=model,
|
||||
litellm_model_name=litellm_model_name,
|
||||
custom_pricing=custom_pricing,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
)
|
||||
if _coop_audio_call
|
||||
else 0.0
|
||||
)
|
||||
response_cost = (response_cost or 0.0) + _coop_pr
|
||||
_coop_name = str(model or "") + str(litellm_model_name or "")
|
||||
if "tts" in _coop_name.lower() or "whisper" in _coop_name.lower():
|
||||
if _coop_audio_call and (
|
||||
"tts" in _coop_name.lower()
|
||||
or "whisper" in _coop_name.lower()
|
||||
or "voxtral" in _coop_name.lower()
|
||||
):
|
||||
print(
|
||||
f"[coop-audio] add per_request={_coop_pr!r} -> response_cost={response_cost!r}",
|
||||
flush=True,
|
||||
|
||||
@@ -4,7 +4,9 @@ MEMBER REQUEST (co-op/support#39, 2026-10-31). Browser JS calling the gateway fr
|
||||
another origin cannot read a response header unless the server names it in
|
||||
Access-Control-Expose-Headers. LiteLLM sets
|
||||
|
||||
expose_headers=LITELLM_UI_ALLOW_HEADERS
|
||||
expose_headers=LITELLM_UI_ALLOW_HEADERS # through 1.103.x
|
||||
expose_headers=[*LITELLM_UI_ALLOW_HEADERS,
|
||||
*trace_response_expose_headers()] # from 1.108.0
|
||||
|
||||
which covers seven names (semantic-filter, guardrail, cache-key, ...) and none of
|
||||
the cost ones — so `x-litellm-response-cost*` reads as null in JS even though the
|
||||
@@ -62,6 +64,20 @@ NEW = (
|
||||
+ " ],\n"
|
||||
)
|
||||
|
||||
# 1.108.0 refactored the middleware call (upstream added the trace-response
|
||||
# header set to expose_headers). Same operation, new anchor; tried when the
|
||||
# 1.103.2 shape is absent so the script works against either base image.
|
||||
OLD_V1108 = (
|
||||
" expose_headers=[*LITELLM_UI_ALLOW_HEADERS, *trace_response_expose_headers()],\n"
|
||||
)
|
||||
NEW_V1108 = (
|
||||
" expose_headers=[\n"
|
||||
" *LITELLM_UI_ALLOW_HEADERS,\n"
|
||||
" *trace_response_expose_headers(),\n"
|
||||
+ "".join(f' "{h}",\n' for h in EXPOSED_HEADERS)
|
||||
+ " ],\n"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# Glob rather than pin a Python version: the venv path moves when the base
|
||||
@@ -76,13 +92,22 @@ def main() -> int:
|
||||
if "x-litellm-response-cost-input" in src:
|
||||
print("expose-cost-headers: already applied")
|
||||
return 0
|
||||
found = src.count(OLD)
|
||||
if found != 1:
|
||||
print(f"WARN expose-cost-headers: expected exactly 1 CORS middleware call, found {found} — skipping")
|
||||
return 0
|
||||
|
||||
target.write_text(src.replace(OLD, NEW))
|
||||
print(f"expose-cost-headers: appended {len(EXPOSED_HEADERS)} cost header names to expose_headers in {target}")
|
||||
for old, new, shape in ((OLD, NEW, "1.103.x"), (OLD_V1108, NEW_V1108, "1.108.x")):
|
||||
found = src.count(old)
|
||||
if found == 1:
|
||||
target.write_text(src.replace(old, new))
|
||||
print(
|
||||
f"expose-cost-headers: appended {len(EXPOSED_HEADERS)} cost header names "
|
||||
f"to expose_headers (anchor {shape}) in {target}"
|
||||
)
|
||||
return 0
|
||||
if found > 1:
|
||||
print(f"WARN expose-cost-headers: expected exactly 1 {shape} CORS middleware call, found {found} — skipping")
|
||||
return 0
|
||||
print(
|
||||
"WARN expose-cost-headers: no known expose_headers anchor found "
|
||||
"(neither 1.103.x nor 1.108.x shape) — skipping"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Skip LiteLLM's forced `verbose_json` on transcriptions for opting-out models.
|
||||
|
||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
||||
if the target file or the anchor is missing — e.g. after a version bump — it
|
||||
prints a warning and exits 0, so the build and the gateway still succeed. Look
|
||||
for the ``[coop-patch]`` line in the build log.
|
||||
|
||||
Why: for every `openai/` transcription deployment, LiteLLM's transformation
|
||||
injects ``response_format: verbose_json`` when the caller omits the field
|
||||
("ensures 'duration' is received - used for cost calculation"). Tinfoil's
|
||||
``voxtral-small-24b`` rejects verbose_json with a 400 ("Currently do not
|
||||
support verbose_json for voxtral-small-24b"), so every bare
|
||||
``/v1/audio/transcriptions`` call to it failed — including the shape members
|
||||
actually use (file + model, no response_format), and Open WebUI dictation's
|
||||
request shape. The injection exists to obtain a duration for per-SECOND
|
||||
pricing; voxtral-small-24b is billed per token (chat route) / per request
|
||||
(patch_audio_request_cost.py), so it does not need the duration at all.
|
||||
|
||||
Patch: honour a ``coop_no_verbose_json`` flag on the deployment's model_info
|
||||
when it is reachable, and gate the model name explicitly — the flag's plumbing
|
||||
into the transformation's litellm_params is not guaranteed on every path, the
|
||||
name gate is. whisper-large-v3-turbo (which Tinfoil serves verbose_json for,
|
||||
and whose dictation path depends on the injected format) is untouched.
|
||||
|
||||
Scope: only transcriptions that omit response_format AND match the opt-out;
|
||||
every other request keeps upstream behavior exactly.
|
||||
|
||||
Upstream: LiteLLM has no opt-out for this injection as of 1.103.2.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
MARKER = "CO-OP PATCH (skip verbose_json, 2026-11-06)"
|
||||
|
||||
ANCHOR = ''' if "response_format" not in data:
|
||||
data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation'''
|
||||
|
||||
INSERT = ''' if "response_format" not in data:
|
||||
# CO-OP PATCH (skip verbose_json, 2026-11-06): a deployment whose
|
||||
# model_info declares coop_no_verbose_json opts out of the forced
|
||||
# verbose_json injection — Tinfoil's voxtral-small-24b rejects
|
||||
# verbose_json with a 400, and its per-token / per-request billing
|
||||
# does not need the duration this injection exists to obtain. The
|
||||
# model-name gate backs the flag up: the flag's plumbing into this
|
||||
# method's litellm_params is not guaranteed on every call path.
|
||||
_mi = (litellm_params or {}).get("model_info") or {}
|
||||
_coop_skip = (isinstance(_mi, dict) and bool(_mi.get("coop_no_verbose_json"))) or (
|
||||
"voxtral-small" in str(model or "")
|
||||
)
|
||||
if not _coop_skip:
|
||||
data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation'''
|
||||
|
||||
|
||||
def target_path() -> str:
|
||||
if len(sys.argv) > 1:
|
||||
return sys.argv[1]
|
||||
env = os.environ.get("LITELLM_WHISPER_TRANSFORMATION")
|
||||
if env:
|
||||
return env
|
||||
hits = []
|
||||
try: # most reliable: ask the installed package where it lives
|
||||
import litellm # noqa: PLC0415
|
||||
_file = getattr(litellm, "__file__", None)
|
||||
if _file:
|
||||
hits.append(os.path.join(
|
||||
os.path.dirname(_file), "llms", "openai", "transcriptions", "whisper_transformation.py"))
|
||||
except Exception:
|
||||
pass
|
||||
hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py")
|
||||
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py")
|
||||
hits = [h for h in hits if os.path.exists(h)]
|
||||
if not hits:
|
||||
raise SystemExit("[coop-patch] could not locate llms/openai/transcriptions/whisper_transformation.py")
|
||||
return sorted(hits)[0]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
path = target_path()
|
||||
try:
|
||||
source = open(path).read()
|
||||
except OSError as exc:
|
||||
print(f"[coop-patch] cannot read {path}: {exc}")
|
||||
return 0
|
||||
|
||||
if MARKER in source:
|
||||
print(f"[coop-patch] {path} already patched, nothing to do")
|
||||
return 0
|
||||
|
||||
if ANCHOR not in source:
|
||||
print(f"[coop-patch] WARNING anchor not found in {path}; bare transcription calls to "
|
||||
f"opt-out models (voxtral-small-24b) will fail with the provider's verbose_json 400")
|
||||
return 0
|
||||
|
||||
patched = source.replace(ANCHOR, INSERT, 1)
|
||||
compile(patched, path, "exec") # refuse to write something that will not import
|
||||
|
||||
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
||||
print(f"[coop-patch] {path} is not writable; verified anchor and syntax only")
|
||||
return 0
|
||||
|
||||
with open(path, "w") as handle:
|
||||
handle.write(patched)
|
||||
print(f"[coop-patch] applied skip-verbose_json to {path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -6,6 +6,18 @@ set -eu
|
||||
# Configures LiteLLM with Cloudron's PostgreSQL and Redis addons
|
||||
# ============================================================================
|
||||
|
||||
# --- Cloudron scheduler tasks arrive as ARGUMENTS to this script ---
|
||||
# The manifest's scheduler commands run as `docker run <image> <command>`, and
|
||||
# this script is the image ENTRYPOINT — so the task command lands in "$@".
|
||||
# Without this branch every scheduled task booted a second tinfoil-proxy and
|
||||
# LiteLLM (whose ports are already bound in the app's network namespace) and
|
||||
# died on the bind FATAL before doing its job — which looked exactly like a
|
||||
# scheduler that never fired. Exec the task directly instead of booting.
|
||||
if [ "$#" -gt 0 ]; then
|
||||
echo "start.sh: scheduler-task mode — exec: $*"
|
||||
exec "$@"
|
||||
fi
|
||||
|
||||
echo "=== LiteLLM Cloudron Startup ==="
|
||||
|
||||
# --- Cloudron PostgreSQL addon provides:
|
||||
@@ -227,4 +239,20 @@ if [ "$proxy_up" != "1" ]; then
|
||||
fi
|
||||
echo "Tinfoil proxy is up on 127.0.0.1:3301"
|
||||
|
||||
# --- In-container audio keep-alive loop ---
|
||||
# The scheduler task (handled above in scheduler-task mode) runs in a SEPARATE
|
||||
# container whose ability to reach 127.0.0.1:4000 and read /app/data is not
|
||||
# guaranteed, so the warm path is owned HERE — the app container, where the
|
||||
# network namespace, the master key and the sidecar are all proven. Every 8
|
||||
# minutes ping tinfoil/voxtral-tts ($0.002/request) so voice mode skips the
|
||||
# ~24s Tinfoil cold-start after idle. audio_keepalive.py is silent on success
|
||||
# and prints on failure, so /app/data/keepalive.log only ever grows on failure.
|
||||
(
|
||||
while true; do
|
||||
python3 /app/code/audio_keepalive.py >> /app/data/keepalive.log 2>&1 || true
|
||||
sleep 480
|
||||
done
|
||||
) &
|
||||
echo "Audio keep-alive loop started (every 480s)"
|
||||
|
||||
exec litellm --config "$CONFIG_FILE" --port 4000 --host 0.0.0.0
|
||||
Reference in new issue
Block a user