LiteLLM's openai transcription transformation force-injects response_format=verbose_json when the caller omits it (to obtain a duration for per-second pricing). Tinfoil rejects verbose_json for voxtral-small-24b with a 400, so every bare transcription call failed — including the file+model shape members and Open WebUI dictation send. patch_transcription_response_format.py (new build-time patch, non-fatal): skip the injection for deployments whose model_info declares coop_no_verbose_json, with an explicit model-name gate as the reliable mechanism. Its billing (per-token via chat, per-request via the audio patch) does not need the duration. whisper-large-v3-turbo is untouched: Tinfoil serves verbose_json for it and dictation depends on the injection.
112 lines
4.9 KiB
Python
112 lines
4.9 KiB
Python
"""Skip LiteLLM's forced `verbose_json` on transcriptions for opting-out models.
|
|
|
|
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
|
if the target file or the anchor is missing — e.g. after a version bump — it
|
|
prints a warning and exits 0, so the build and the gateway still succeed. Look
|
|
for the ``[coop-patch]`` line in the build log.
|
|
|
|
Why: for every `openai/` transcription deployment, LiteLLM's transformation
|
|
injects ``response_format: verbose_json`` when the caller omits the field
|
|
("ensures 'duration' is received - used for cost calculation"). Tinfoil's
|
|
``voxtral-small-24b`` rejects verbose_json with a 400 ("Currently do not
|
|
support verbose_json for voxtral-small-24b"), so every bare
|
|
``/v1/audio/transcriptions`` call to it failed — including the shape members
|
|
actually use (file + model, no response_format), and Open WebUI dictation's
|
|
request shape. The injection exists to obtain a duration for per-SECOND
|
|
pricing; voxtral-small-24b is billed per token (chat route) / per request
|
|
(patch_audio_request_cost.py), so it does not need the duration at all.
|
|
|
|
Patch: honour a ``coop_no_verbose_json`` flag on the deployment's model_info
|
|
when it is reachable, and gate the model name explicitly — the flag's plumbing
|
|
into the transformation's litellm_params is not guaranteed on every path, the
|
|
name gate is. whisper-large-v3-turbo (which Tinfoil serves verbose_json for,
|
|
and whose dictation path depends on the injected format) is untouched.
|
|
|
|
Scope: only transcriptions that omit response_format AND match the opt-out;
|
|
every other request keeps upstream behavior exactly.
|
|
|
|
Upstream: LiteLLM has no opt-out for this injection as of 1.103.2.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import glob
|
|
import os
|
|
import sys
|
|
|
|
MARKER = "CO-OP PATCH (skip verbose_json, 2026-11-06)"
|
|
|
|
ANCHOR = ''' if "response_format" not in data:
|
|
data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation'''
|
|
|
|
INSERT = ''' if "response_format" not in data:
|
|
# CO-OP PATCH (skip verbose_json, 2026-11-06): a deployment whose
|
|
# model_info declares coop_no_verbose_json opts out of the forced
|
|
# verbose_json injection — Tinfoil's voxtral-small-24b rejects
|
|
# verbose_json with a 400, and its per-token / per-request billing
|
|
# does not need the duration this injection exists to obtain. The
|
|
# model-name gate backs the flag up: the flag's plumbing into this
|
|
# method's litellm_params is not guaranteed on every call path.
|
|
_mi = (litellm_params or {}).get("model_info") or {}
|
|
_coop_skip = (isinstance(_mi, dict) and bool(_mi.get("coop_no_verbose_json"))) or (
|
|
"voxtral-small" in str(model or "")
|
|
)
|
|
if not _coop_skip:
|
|
data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation'''
|
|
|
|
|
|
def target_path() -> str:
|
|
if len(sys.argv) > 1:
|
|
return sys.argv[1]
|
|
env = os.environ.get("LITELLM_WHISPER_TRANSFORMATION")
|
|
if env:
|
|
return env
|
|
hits = []
|
|
try: # most reliable: ask the installed package where it lives
|
|
import litellm # noqa: PLC0415
|
|
_file = getattr(litellm, "__file__", None)
|
|
if _file:
|
|
hits.append(os.path.join(
|
|
os.path.dirname(_file), "llms", "openai", "transcriptions", "whisper_transformation.py"))
|
|
except Exception:
|
|
pass
|
|
hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py")
|
|
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py")
|
|
hits = [h for h in hits if os.path.exists(h)]
|
|
if not hits:
|
|
raise SystemExit("[coop-patch] could not locate llms/openai/transcriptions/whisper_transformation.py")
|
|
return sorted(hits)[0]
|
|
|
|
|
|
def main() -> int:
|
|
path = target_path()
|
|
try:
|
|
source = open(path).read()
|
|
except OSError as exc:
|
|
print(f"[coop-patch] cannot read {path}: {exc}")
|
|
return 0
|
|
|
|
if MARKER in source:
|
|
print(f"[coop-patch] {path} already patched, nothing to do")
|
|
return 0
|
|
|
|
if ANCHOR not in source:
|
|
print(f"[coop-patch] WARNING anchor not found in {path}; bare transcription calls to "
|
|
f"opt-out models (voxtral-small-24b) will fail with the provider's verbose_json 400")
|
|
return 0
|
|
|
|
patched = source.replace(ANCHOR, INSERT, 1)
|
|
compile(patched, path, "exec") # refuse to write something that will not import
|
|
|
|
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
|
print(f"[coop-patch] {path} is not writable; verified anchor and syntax only")
|
|
return 0
|
|
|
|
with open(path, "w") as handle:
|
|
handle.write(patched)
|
|
print(f"[coop-patch] applied skip-verbose_json to {path}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|