gateway: make /v1/audio/transcriptions work bare for voxtral-small-24b
LiteLLM's openai transcription transformation force-injects response_format=verbose_json when the caller omits it (to obtain a duration for per-second pricing). Tinfoil rejects verbose_json for voxtral-small-24b with a 400, so every bare transcription call failed — including the file+model shape members and Open WebUI dictation send. patch_transcription_response_format.py (new build-time patch, non-fatal): skip the injection for deployments whose model_info declares coop_no_verbose_json, with an explicit model-name gate as the reliable mechanism. Its billing (per-token via chat, per-request via the audio patch) does not need the duration. whisper-large-v3-turbo is untouched: Tinfoil serves verbose_json for it and dictation depends on the injection.
This commit is contained in:
1 parent
abaeb255b2
commit
943701581c
3 files changed
+125
No files matched your search
@@ -92,6 +92,15 @@ RUN python3 /app/code/patch_passthrough_token_cost.py
|
|||||||
COPY audio_keepalive.py /app/code/audio_keepalive.py
|
COPY audio_keepalive.py /app/code/audio_keepalive.py
|
||||||
RUN chmod +x /app/code/audio_keepalive.py
|
RUN chmod +x /app/code/audio_keepalive.py
|
||||||
|
|
||||||
|
# Skip LiteLLM's forced verbose_json on transcription requests for opting-out
|
||||||
|
# models: Tinfoil's voxtral-small-24b rejects verbose_json with a 400, which
|
||||||
|
# broke every bare /v1/audio/transcriptions call to it (file + model — the
|
||||||
|
# shape members and dictation send). Its billing does not need the duration
|
||||||
|
# the injection exists for. Non-fatal; check the "[coop-patch] applied" line
|
||||||
|
# in the build log on every LiteLLM upgrade.
|
||||||
|
COPY patch_transcription_response_format.py /app/code/patch_transcription_response_format.py
|
||||||
|
RUN python3 /app/code/patch_transcription_response_format.py
|
||||||
|
|
||||||
# Override the base image's ENTRYPOINT (which is `litellm`) so our
|
# Override the base image's ENTRYPOINT (which is `litellm`) so our
|
||||||
# startup script runs instead of being passed as an argument to litellm.
|
# startup script runs instead of being passed as an argument to litellm.
|
||||||
ENTRYPOINT ["/app/code/start.sh"]
|
ENTRYPOINT ["/app/code/start.sh"]
|
||||||
|
|||||||
@@ -197,6 +197,11 @@ model_list:
|
|||||||
api_key: os.environ/TINFOIL_API_KEY
|
api_key: os.environ/TINFOIL_API_KEY
|
||||||
model_info:
|
model_info:
|
||||||
mode: audio_transcription
|
mode: audio_transcription
|
||||||
|
# Opt out of LiteLLM's forced response_format=verbose_json injection on
|
||||||
|
# transcription calls (patch_transcription_response_format.py): Tinfoil
|
||||||
|
# rejects verbose_json for this model with a 400, and nothing in its
|
||||||
|
# billing needs the duration the injection exists to obtain.
|
||||||
|
coop_no_verbose_json: true
|
||||||
input_cost_per_token: 0.0000002
|
input_cost_per_token: 0.0000002
|
||||||
output_cost_per_token: 0.0000006
|
output_cost_per_token: 0.0000006
|
||||||
input_cost_per_request: 0.00008
|
input_cost_per_request: 0.00008
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
"""Skip LiteLLM's forced `verbose_json` on transcriptions for opting-out models.
|
||||||
|
|
||||||
|
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
||||||
|
if the target file or the anchor is missing — e.g. after a version bump — it
|
||||||
|
prints a warning and exits 0, so the build and the gateway still succeed. Look
|
||||||
|
for the ``[coop-patch]`` line in the build log.
|
||||||
|
|
||||||
|
Why: for every `openai/` transcription deployment, LiteLLM's transformation
|
||||||
|
injects ``response_format: verbose_json`` when the caller omits the field
|
||||||
|
("ensures 'duration' is received - used for cost calculation"). Tinfoil's
|
||||||
|
``voxtral-small-24b`` rejects verbose_json with a 400 ("Currently do not
|
||||||
|
support verbose_json for voxtral-small-24b"), so every bare
|
||||||
|
``/v1/audio/transcriptions`` call to it failed — including the shape members
|
||||||
|
actually use (file + model, no response_format), and Open WebUI dictation's
|
||||||
|
request shape. The injection exists to obtain a duration for per-SECOND
|
||||||
|
pricing; voxtral-small-24b is billed per token (chat route) / per request
|
||||||
|
(patch_audio_request_cost.py), so it does not need the duration at all.
|
||||||
|
|
||||||
|
Patch: honour a ``coop_no_verbose_json`` flag on the deployment's model_info
|
||||||
|
when it is reachable, and gate the model name explicitly — the flag's plumbing
|
||||||
|
into the transformation's litellm_params is not guaranteed on every path, the
|
||||||
|
name gate is. whisper-large-v3-turbo (which Tinfoil serves verbose_json for,
|
||||||
|
and whose dictation path depends on the injected format) is untouched.
|
||||||
|
|
||||||
|
Scope: only transcriptions that omit response_format AND match the opt-out;
|
||||||
|
every other request keeps upstream behavior exactly.
|
||||||
|
|
||||||
|
Upstream: LiteLLM has no opt-out for this injection as of 1.103.2.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import glob
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
MARKER = "CO-OP PATCH (skip verbose_json, 2026-11-06)"
|
||||||
|
|
||||||
|
ANCHOR = ''' if "response_format" not in data:
|
||||||
|
data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation'''
|
||||||
|
|
||||||
|
INSERT = ''' if "response_format" not in data:
|
||||||
|
# CO-OP PATCH (skip verbose_json, 2026-11-06): a deployment whose
|
||||||
|
# model_info declares coop_no_verbose_json opts out of the forced
|
||||||
|
# verbose_json injection — Tinfoil's voxtral-small-24b rejects
|
||||||
|
# verbose_json with a 400, and its per-token / per-request billing
|
||||||
|
# does not need the duration this injection exists to obtain. The
|
||||||
|
# model-name gate backs the flag up: the flag's plumbing into this
|
||||||
|
# method's litellm_params is not guaranteed on every call path.
|
||||||
|
_mi = (litellm_params or {}).get("model_info") or {}
|
||||||
|
_coop_skip = (isinstance(_mi, dict) and bool(_mi.get("coop_no_verbose_json"))) or (
|
||||||
|
"voxtral-small" in str(model or "")
|
||||||
|
)
|
||||||
|
if not _coop_skip:
|
||||||
|
data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation'''
|
||||||
|
|
||||||
|
|
||||||
|
def target_path() -> str:
|
||||||
|
if len(sys.argv) > 1:
|
||||||
|
return sys.argv[1]
|
||||||
|
env = os.environ.get("LITELLM_WHISPER_TRANSFORMATION")
|
||||||
|
if env:
|
||||||
|
return env
|
||||||
|
hits = []
|
||||||
|
try: # most reliable: ask the installed package where it lives
|
||||||
|
import litellm # noqa: PLC0415
|
||||||
|
_file = getattr(litellm, "__file__", None)
|
||||||
|
if _file:
|
||||||
|
hits.append(os.path.join(
|
||||||
|
os.path.dirname(_file), "llms", "openai", "transcriptions", "whisper_transformation.py"))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py")
|
||||||
|
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py")
|
||||||
|
hits = [h for h in hits if os.path.exists(h)]
|
||||||
|
if not hits:
|
||||||
|
raise SystemExit("[coop-patch] could not locate llms/openai/transcriptions/whisper_transformation.py")
|
||||||
|
return sorted(hits)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
path = target_path()
|
||||||
|
try:
|
||||||
|
source = open(path).read()
|
||||||
|
except OSError as exc:
|
||||||
|
print(f"[coop-patch] cannot read {path}: {exc}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if MARKER in source:
|
||||||
|
print(f"[coop-patch] {path} already patched, nothing to do")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if ANCHOR not in source:
|
||||||
|
print(f"[coop-patch] WARNING anchor not found in {path}; bare transcription calls to "
|
||||||
|
f"opt-out models (voxtral-small-24b) will fail with the provider's verbose_json 400")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
patched = source.replace(ANCHOR, INSERT, 1)
|
||||||
|
compile(patched, path, "exec") # refuse to write something that will not import
|
||||||
|
|
||||||
|
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
||||||
|
print(f"[coop-patch] {path} is not writable; verified anchor and syntax only")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with open(path, "w") as handle:
|
||||||
|
handle.write(patched)
|
||||||
|
print(f"[coop-patch] applied skip-verbose_json to {path}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
Reference in new issue
Block a user