From 943701581c6e7683e29e5a35d56697547e5e59b5 Mon Sep 17 00:00:00 2001 From: Inferencebot Date: Fri, 6 Nov 2026 22:40:53 +0000 Subject: [PATCH] gateway: make /v1/audio/transcriptions work bare for voxtral-small-24b MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LiteLLM's openai transcription transformation force-injects response_format=verbose_json when the caller omits it (to obtain a duration for per-second pricing). Tinfoil rejects verbose_json for voxtral-small-24b with a 400, so every bare transcription call failed — including the file+model shape members and Open WebUI dictation send. patch_transcription_response_format.py (new build-time patch, non-fatal): skip the injection for deployments whose model_info declares coop_no_verbose_json, with an explicit model-name gate as the reliable mechanism. Its billing (per-token via chat, per-request via the audio patch) does not need the duration. whisper-large-v3-turbo is untouched: Tinfoil serves verbose_json for it and dictation depends on the injection. --- Dockerfile | 9 ++ config.yaml | 5 ++ patch_transcription_response_format.py | 111 +++++++++++++++++++++++++ 3 files changed, 125 insertions(+) create mode 100644 patch_transcription_response_format.py diff --git a/Dockerfile b/Dockerfile index 6c264d1..a9fd400 100644 --- a/Dockerfile +++ b/Dockerfile @@ -92,6 +92,15 @@ RUN python3 /app/code/patch_passthrough_token_cost.py COPY audio_keepalive.py /app/code/audio_keepalive.py RUN chmod +x /app/code/audio_keepalive.py +# Skip LiteLLM's forced verbose_json on transcription requests for opting-out +# models: Tinfoil's voxtral-small-24b rejects verbose_json with a 400, which +# broke every bare /v1/audio/transcriptions call to it (file + model — the +# shape members and dictation send). Its billing does not need the duration +# the injection exists for. Non-fatal; check the "[coop-patch] applied" line +# in the build log on every LiteLLM upgrade. +COPY patch_transcription_response_format.py /app/code/patch_transcription_response_format.py +RUN python3 /app/code/patch_transcription_response_format.py + # Override the base image's ENTRYPOINT (which is `litellm`) so our # startup script runs instead of being passed as an argument to litellm. ENTRYPOINT ["/app/code/start.sh"] diff --git a/config.yaml b/config.yaml index b2f8170..9341e3c 100644 --- a/config.yaml +++ b/config.yaml @@ -197,6 +197,11 @@ model_list: api_key: os.environ/TINFOIL_API_KEY model_info: mode: audio_transcription + # Opt out of LiteLLM's forced response_format=verbose_json injection on + # transcription calls (patch_transcription_response_format.py): Tinfoil + # rejects verbose_json for this model with a 400, and nothing in its + # billing needs the duration the injection exists to obtain. + coop_no_verbose_json: true input_cost_per_token: 0.0000002 output_cost_per_token: 0.0000006 input_cost_per_request: 0.00008 diff --git a/patch_transcription_response_format.py b/patch_transcription_response_format.py new file mode 100644 index 0000000..c8b210e --- /dev/null +++ b/patch_transcription_response_format.py @@ -0,0 +1,111 @@ +"""Skip LiteLLM's forced `verbose_json` on transcriptions for opting-out models. + +Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal: +if the target file or the anchor is missing — e.g. after a version bump — it +prints a warning and exits 0, so the build and the gateway still succeed. Look +for the ``[coop-patch]`` line in the build log. + +Why: for every `openai/` transcription deployment, LiteLLM's transformation +injects ``response_format: verbose_json`` when the caller omits the field +("ensures 'duration' is received - used for cost calculation"). Tinfoil's +``voxtral-small-24b`` rejects verbose_json with a 400 ("Currently do not +support verbose_json for voxtral-small-24b"), so every bare +``/v1/audio/transcriptions`` call to it failed — including the shape members +actually use (file + model, no response_format), and Open WebUI dictation's +request shape. The injection exists to obtain a duration for per-SECOND +pricing; voxtral-small-24b is billed per token (chat route) / per request +(patch_audio_request_cost.py), so it does not need the duration at all. + +Patch: honour a ``coop_no_verbose_json`` flag on the deployment's model_info +when it is reachable, and gate the model name explicitly — the flag's plumbing +into the transformation's litellm_params is not guaranteed on every path, the +name gate is. whisper-large-v3-turbo (which Tinfoil serves verbose_json for, +and whose dictation path depends on the injected format) is untouched. + +Scope: only transcriptions that omit response_format AND match the opt-out; +every other request keeps upstream behavior exactly. + +Upstream: LiteLLM has no opt-out for this injection as of 1.103.2. +""" +from __future__ import annotations + +import glob +import os +import sys + +MARKER = "CO-OP PATCH (skip verbose_json, 2026-11-06)" + +ANCHOR = ''' if "response_format" not in data: + data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation''' + +INSERT = ''' if "response_format" not in data: + # CO-OP PATCH (skip verbose_json, 2026-11-06): a deployment whose + # model_info declares coop_no_verbose_json opts out of the forced + # verbose_json injection — Tinfoil's voxtral-small-24b rejects + # verbose_json with a 400, and its per-token / per-request billing + # does not need the duration this injection exists to obtain. The + # model-name gate backs the flag up: the flag's plumbing into this + # method's litellm_params is not guaranteed on every call path. + _mi = (litellm_params or {}).get("model_info") or {} + _coop_skip = (isinstance(_mi, dict) and bool(_mi.get("coop_no_verbose_json"))) or ( + "voxtral-small" in str(model or "") + ) + if not _coop_skip: + data["response_format"] = "verbose_json" # ensures 'duration' is received - used for cost calculation''' + + +def target_path() -> str: + if len(sys.argv) > 1: + return sys.argv[1] + env = os.environ.get("LITELLM_WHISPER_TRANSFORMATION") + if env: + return env + hits = [] + try: # most reliable: ask the installed package where it lives + import litellm # noqa: PLC0415 + _file = getattr(litellm, "__file__", None) + if _file: + hits.append(os.path.join( + os.path.dirname(_file), "llms", "openai", "transcriptions", "whisper_transformation.py")) + except Exception: + pass + hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py") + hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/llms/openai/transcriptions/whisper_transformation.py") + hits = [h for h in hits if os.path.exists(h)] + if not hits: + raise SystemExit("[coop-patch] could not locate llms/openai/transcriptions/whisper_transformation.py") + return sorted(hits)[0] + + +def main() -> int: + path = target_path() + try: + source = open(path).read() + except OSError as exc: + print(f"[coop-patch] cannot read {path}: {exc}") + return 0 + + if MARKER in source: + print(f"[coop-patch] {path} already patched, nothing to do") + return 0 + + if ANCHOR not in source: + print(f"[coop-patch] WARNING anchor not found in {path}; bare transcription calls to " + f"opt-out models (voxtral-small-24b) will fail with the provider's verbose_json 400") + return 0 + + patched = source.replace(ANCHOR, INSERT, 1) + compile(patched, path, "exec") # refuse to write something that will not import + + if not os.access(path, os.W_OK): # practice run against a read-only copy + print(f"[coop-patch] {path} is not writable; verified anchor and syntax only") + return 0 + + with open(path, "w") as handle: + handle.write(patched) + print(f"[coop-patch] applied skip-verbose_json to {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())