gateway: make /v1/audio/transcriptions work bare for voxtral-small-24b
LiteLLM's openai transcription transformation force-injects response_format=verbose_json when the caller omits it (to obtain a duration for per-second pricing). Tinfoil rejects verbose_json for voxtral-small-24b with a 400, so every bare transcription call failed — including the file+model shape members and Open WebUI dictation send. patch_transcription_response_format.py (new build-time patch, non-fatal): skip the injection for deployments whose model_info declares coop_no_verbose_json, with an explicit model-name gate as the reliable mechanism. Its billing (per-token via chat, per-request via the audio patch) does not need the duration. whisper-large-v3-turbo is untouched: Tinfoil serves verbose_json for it and dictation depends on the injection.
This commit is contained in:
1 parent
abaeb255b2
commit
943701581c
3 files changed
+125
No files matched your search
@@ -92,6 +92,15 @@ RUN python3 /app/code/patch_passthrough_token_cost.py
|
||||
COPY audio_keepalive.py /app/code/audio_keepalive.py
|
||||
RUN chmod +x /app/code/audio_keepalive.py
|
||||
|
||||
# Skip LiteLLM's forced verbose_json on transcription requests for opting-out
|
||||
# models: Tinfoil's voxtral-small-24b rejects verbose_json with a 400, which
|
||||
# broke every bare /v1/audio/transcriptions call to it (file + model — the
|
||||
# shape members and dictation send). Its billing does not need the duration
|
||||
# the injection exists for. Non-fatal; check the "[coop-patch] applied" line
|
||||
# in the build log on every LiteLLM upgrade.
|
||||
COPY patch_transcription_response_format.py /app/code/patch_transcription_response_format.py
|
||||
RUN python3 /app/code/patch_transcription_response_format.py
|
||||
|
||||
# Override the base image's ENTRYPOINT (which is `litellm`) so our
|
||||
# startup script runs instead of being passed as an argument to litellm.
|
||||
ENTRYPOINT ["/app/code/start.sh"]
|
||||
|
||||
@@ -197,6 +197,11 @@ model_list:
|
||||
api_key: os.environ/TINFOIL_API_KEY
|
||||
model_info:
|
||||
mode: audio_transcription
|
||||
# Opt out of LiteLLM's forced response_format=verbose_json injection on
|
||||
# transcription calls (patch_transcription_response_format.py): Tinfoil
|
||||
# rejects verbose_json for this model with a 400, and nothing in its
|
||||
# billing needs the duration the injection exists to obtain.
|
||||
coop_no_verbose_json: true
|
||||
input_cost_per_token: 0.0000002
|
||||
output_cost_per_token: 0.0000006
|
||||
input_cost_per_request: 0.00008
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Skip LiteLLM's forced `verbose_json` on transcriptions for opting-out models.
|
||||
|
||||
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
||||
if the target file or the anchor is missing — e.g. after a version bump — it
|
||||
prints a warning and exits 0, so the build and the gateway still succeed. Look
|
||||
for the ``[coop-patch]`` line in the build log.
|
||||
|
||||
Why: for every `openai/` transcription deployment, LiteLLM's transformation
|
||||
injects ``response_format: verbose_json`` when the caller omits the field
|
||||
("ensures 'duration' is received - used for cost calculation"). Tinfoil's
|
||||
``voxtral-small-24b`` rejects verbose_json with a 400 ("Currently do not
|
||||
support verbose_json for voxtral-small-24b"), so every bare
|
||||
``/v1/audio/transcriptions`` call to it failed — including the shape members
|
||||
actually use (file + model, no response_format), and Open WebUI dictation's
|
||||
request shape. The injection exists to obtain a duration for per-SECOND
|
||||
pricing; voxtral-small-24b is billed per token (chat route) / per request
|
||||
(patch_audio_request_cost.py), so it does not need the duration at all.
|
||||
|
||||
Patch: honour a ``coop_no_verbose_json`` flag on the deployment's model_info
|
||||
when it is reachable, and gate the model name explicitly — the flag's plumbing
|
||||
into the transformation's litellm_params is not guaranteed on every path, the
|
||||
name gate is. whisper-large-v3-turbo (which Tinfoil serves verbose_json for,
|
||||
and whose dictation path depends on the injected format) is untouched.
|
||||
|
||||
Scope: only transcriptions that omit response_format AND match the opt-out;
|
||||
every other request keeps upstream behavior exactly.
|
||||
|
||||
Upstream: LiteLLM has no opt-out for this injection as of 1.103.2.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
|
||||
MARKER = "CO-OP PATCH (skip verbose_json, 2026-11-06)"
|
||||
|
||||
ANCHOR = ''' if "response_format" not in data:
|
||||
data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation'''
|
||||
|
||||
INSERT = ''' if "response_format" not in data:
|
||||
# CO-OP PATCH (skip verbose_json, 2026-11-06): a deployment whose
|
||||
# model_info declares coop_no_verbose_json opts out of the forced
|
||||
# verbose_json injection — Tinfoil's voxtral-small-24b rejects
|
||||
# verbose_json with a 400, and its per-token / per-request billing
|
||||
# does not need the duration this injection exists to obtain. The
|
||||
# model-name gate backs the flag up: the flag's plumbing into this
|
||||
# method's litellm_params is not guaranteed on every call path.
|
||||
_mi = (litellm_params or {}).get("model_info") or {}
|
||||
_coop_skip = (isinstance(_mi, dict) and bool(_mi.get("coop_no_verbose_json"))) or (
|
||||
"voxtral-small" in str(model or "")
|
||||
)
|
||||
if not _coop_skip:
|
||||
data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation'''
|
||||
|
||||
|
||||
def target_path() -> str:
|
||||
if len(sys.argv) > 1:
|
||||
return sys.argv[1]
|
||||
env = os.environ.get("LITELLM_WHISPER_TRANSFORMATION")
|
||||
if env:
|
||||
return env
|
||||
hits = []
|
||||
try: # most reliable: ask the installed package where it lives
|
||||
import litellm # noqa: PLC0415
|
||||
_file = getattr(litellm, "__file__", None)
|
||||
if _file:
|
||||
hits.append(os.path.join(
|
||||
os.path.dirname(_file), "llms", "openai", "transcriptions", "whisper_transformation.py"))
|
||||
except Exception:
|
||||
pass
|
||||
hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py")
|
||||
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py")
|
||||
hits = [h for h in hits if os.path.exists(h)]
|
||||
if not hits:
|
||||
raise SystemExit("[coop-patch] could not locate llms/openai/transcriptions/whisper_transformation.py")
|
||||
return sorted(hits)[0]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
path = target_path()
|
||||
try:
|
||||
source = open(path).read()
|
||||
except OSError as exc:
|
||||
print(f"[coop-patch] cannot read {path}: {exc}")
|
||||
return 0
|
||||
|
||||
if MARKER in source:
|
||||
print(f"[coop-patch] {path} already patched, nothing to do")
|
||||
return 0
|
||||
|
||||
if ANCHOR not in source:
|
||||
print(f"[coop-patch] WARNING anchor not found in {path}; bare transcription calls to "
|
||||
f"opt-out models (voxtral-small-24b) will fail with the provider's verbose_json 400")
|
||||
return 0
|
||||
|
||||
patched = source.replace(ANCHOR, INSERT, 1)
|
||||
compile(patched, path, "exec") # refuse to write something that will not import
|
||||
|
||||
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
||||
print(f"[coop-patch] {path} is not writable; verified anchor and syntax only")
|
||||
return 0
|
||||
|
||||
with open(path, "w") as handle:
|
||||
handle.write(patched)
|
||||
print(f"[coop-patch] applied skip-verbose_json to {path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user