diff --git a/Dockerfile b/Dockerfile index a9fd400..3c0f448 100644 --- a/Dockerfile +++ b/Dockerfile @@ -101,6 +101,14 @@ RUN chmod +x /app/code/audio_keepalive.py COPY patch_transcription_response_format.py /app/code/patch_transcription_response_format.py RUN python3 /app/code/patch_transcription_response_format.py +# TTS default response format: LibreChat sends no response_format on its TTS +# request and Tinfoil's voxtral-tts then serves WAV (~8-17s warm measured) — +# while LibreChat labels the reply audio/mpeg regardless. Default to mp3 +# (OpenAI's documented default, ~3s warm measured). Non-fatal; check the +# "[coop-patch] applied" line in the build log on every LiteLLM upgrade. +COPY patch_audio_default_mp3.py /app/code/patch_audio_default_mp3.py +RUN python3 /app/code/patch_audio_default_mp3.py + # Override the base image's ENTRYPOINT (which is `litellm`) so our # startup script runs instead of being passed as an argument to litellm. ENTRYPOINT ["/app/code/start.sh"] diff --git a/patch_audio_default_mp3.py b/patch_audio_default_mp3.py new file mode 100644 index 0000000..baba325 --- /dev/null +++ b/patch_audio_default_mp3.py @@ -0,0 +1,103 @@ +"""Default /v1/audio/speech to mp3 when the caller omits response_format. + +Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal: +if the target file or the anchor is missing — e.g. after a version bump — it +prints a warning and exits 0, so the build and the gateway still succeed. Look +for the ``[coop-patch]`` line in the build log. + +Why: LibreChat's speech client sends no ``response_format`` on its TTS request +(TTSService.openAIProvider builds {input, model, voice} — the field is not in +its schema), and Tinfoil's voxtral-tts then serves WAV. Measured 2026-11-07 +through the gateway: WAV ~8-17s warm per request (145-300 KB), while the SAME +model with ``response_format: "mp3"`` answers in ~2.6-3.1s (28-39 KB) — the +provider streams/encodes mp3 progressively. LibreChat additionally labels the +reply ``Content-Type: audio/mpeg`` unconditionally, so WAV bytes were served +under an mp3 label and browsers depended on MIME sniffing to play them. + +Patch: default ``response_format`` to ``mp3`` (OpenAI's own documented default +for /v1/audio/speech) when the request omits it. An explicit response_format +from any caller still wins, so API members who want WAV can keep requesting +it. Scope: only the aspeech route; transcriptions are untouched. + +Upstream: LiteLLM 1.103.2 forwards response_format verbatim and leaves the +provider default in place when it is absent. +""" +from __future__ import annotations + +import glob +import os +import sys + +MARKER = "CO-OP PATCH (TTS default mp3, 2026-11-07)" + +ANCHOR = ''' ### CALL HOOKS ### - modify incoming data / reject request before calling the model + data = await proxy_logging_obj.pre_call_hook( + user_api_key_dict=user_api_key_dict, data=data, call_type="aspeech" + )''' + +INSERT = ANCHOR + ''' + + # CO-OP PATCH (TTS default mp3, 2026-11-07): LibreChat sends no + # response_format and Tinfoil's voxtral-tts then serves WAV — measured + # ~8-17s warm vs ~3s for mp3, which the provider streams. LibreChat + # labels the reply audio/mpeg regardless. Default to mp3 (OpenAI's + # documented default); an explicit response_format still wins. + if not data.get("response_format"): + data["response_format"] = "mp3"''' + + +def target_path() -> str: + if len(sys.argv) > 1: + return sys.argv[1] + env = os.environ.get("LITELLM_PROXY_SERVER") + if env: + return env + hits = [] + try: # most reliable: ask the installed package where it lives + import litellm # noqa: PLC0415 + _file = getattr(litellm, "__file__", None) + if _file: + hits.append(os.path.join(os.path.dirname(_file), "proxy", "proxy_server.py")) + except Exception: + pass + hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/proxy/proxy_server.py") + hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/proxy/proxy_server.py") + hits = [h for h in hits if os.path.exists(h)] + if not hits: + raise SystemExit("[coop-patch] could not locate litellm/proxy/proxy_server.py") + return sorted(hits)[0] + + +def main() -> int: + path = target_path() + try: + source = open(path).read() + except OSError as exc: + print(f"[coop-patch] cannot read {path}: {exc}") + return 0 + + if MARKER in source: + print(f"[coop-patch] {path} already patched, nothing to do") + return 0 + + count = source.count(ANCHOR) + if count != 1: + print(f"[coop-patch] WARNING aspeech pre_call_hook anchor found {count}x (need exactly 1) " + f"in {path}; TTS stays on the provider's default (WAV for Tinfoil)") + return 0 + + patched = source.replace(ANCHOR, INSERT, 1) + compile(patched, path, "exec") # refuse to write something that will not import + + if not os.access(path, os.W_OK): # practice run against a read-only copy + print(f"[coop-patch] {path} is not writable; verified anchor and syntax only") + return 0 + + with open(path, "w") as handle: + handle.write(patched) + print(f"[coop-patch] applied TTS default mp3 to {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/start.sh b/start.sh index be94550..a80ae70 100644 --- a/start.sh +++ b/start.sh @@ -6,6 +6,18 @@ set -eu # Configures LiteLLM with Cloudron's PostgreSQL and Redis addons # ============================================================================ +# --- Cloudron scheduler tasks arrive as ARGUMENTS to this script --- +# The manifest's scheduler commands run as `docker run `, and +# this script is the image ENTRYPOINT — so the task command lands in "$@". +# Without this branch every scheduled task booted a second tinfoil-proxy and +# LiteLLM (whose ports are already bound in the app's network namespace) and +# died on the bind FATAL before doing its job — which looked exactly like a +# scheduler that never fired. Exec the task directly instead of booting. +if [ "$#" -gt 0 ]; then + echo "start.sh: scheduler-task mode — exec: $*" + exec "$@" +fi + echo "=== LiteLLM Cloudron Startup ===" # --- Cloudron PostgreSQL addon provides: @@ -227,4 +239,20 @@ if [ "$proxy_up" != "1" ]; then fi echo "Tinfoil proxy is up on 127.0.0.1:3301" +# --- In-container audio keep-alive loop --- +# The scheduler task (handled above in scheduler-task mode) runs in a SEPARATE +# container whose ability to reach 127.0.0.1:4000 and read /app/data is not +# guaranteed, so the warm path is owned HERE — the app container, where the +# network namespace, the master key and the sidecar are all proven. Every 8 +# minutes ping tinfoil/voxtral-tts ($0.002/request) so voice mode skips the +# ~24s Tinfoil cold-start after idle. audio_keepalive.py is silent on success +# and prints on failure, so /app/data/keepalive.log only ever grows on failure. +( + while true; do + python3 /app/code/audio_keepalive.py >> /app/data/keepalive.log 2>&1 || true + sleep 480 + done +) & +echo "Audio keep-alive loop started (every 480s)" + exec litellm --config "$CONFIG_FILE" --port 4000 --host 0.0.0.0 \ No newline at end of file