From 425d738625bd3f1177815a1de38971a8225073de Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 26 Oct 2026 20:45:06 +0000 Subject: [PATCH] gateway: bill Tinfoil's per-request charges on the audio paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tinfoil bills voxtral-tts $0.002 and whisper-large-v3-turbo $0.01 per REQUEST, with $0 per token for both. Reading only the two token prices in its model list made TTS look free and left STT billed at an older catalog's $0.05/1M tokens, so neither matched what the co-op actually pays. LiteLLM cannot express a per-request charge on the audio paths: per-request pricing exists only on the search/rerank/MCP paths via input_cost_per_query, and input_cost_per_request is read only by the proxy's cost-tracking settings, never by the model cost calculator. So patch_audio_request_cost.py, a build-time patch in the same shape as patch_image_cost.py, makes response_cost_calculator honour a declared input_cost_per_request. That function is the single funnel for every call type — the proxy dispatches audio as `aspeech` and `transcription` and reaches it through Logging._response_cost_calculator — so one insertion covers both. The charge resolves from the per-deployment custom_pricing the logging layer builds from our model_info, then the shared cost map, then call metadata, and the patch is inert for any model that declares nothing. config.yaml declares the real charges: $0.002/request for voxtral-tts, and $0.01/request for whisper with its token price corrected to $0. Verified before committing: the patch applies to the installed 1.103.2 anchors and compiles; the helper's resolution order passes unit checks (all branches, plus inert and garbage cases); the resulting config validates as YAML with all 12 entries intact. End-to-end spend is proven after deploy by a real TTS/STT call. Refs code/litellm#5 --- Dockerfile | 8 ++ README.md | 1 + config.yaml | 16 +++- patch_audio_request_cost.py | 181 ++++++++++++++++++++++++++++++++++++ 4 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 patch_audio_request_cost.py diff --git a/Dockerfile b/Dockerfile index 1f4a7f0..e3a2877 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,14 @@ RUN python3 /app/code/patch_budget_message.py COPY patch_image_cost.py /app/code/patch_image_cost.py RUN python3 /app/code/patch_image_cost.py +# Audio per-request pricing: Tinfoil bills voxtral-tts and whisper per request, +# which LiteLLM cannot express on the audio paths, so those two were billed at +# $0. The patch makes the cost calculator honour a declared +# input_cost_per_request. Check for the "[coop-patch] applied" line in the build +# log after upgrading LiteLLM: the script is non-fatal and warns instead. +COPY patch_audio_request_cost.py /app/code/patch_audio_request_cost.py +RUN python3 /app/code/patch_audio_request_cost.py + # Audio keep-alive: scheduled pings keep the Tinfoil TEE audio enclave # warm so voice mode skips the ~24s cold-start (see audio_keepalive.py header). COPY audio_keepalive.py /app/code/audio_keepalive.py diff --git a/README.md b/README.md index 7391afa..96bcb2b 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,7 @@ expect a 15–20 min maintenance window and **never panic-restart mid-migration* - Check the [LiteLLM changelog](https://github.com/BerriAI/litellm/releases) for breaking changes - The `start.sh` and `config.yaml` may need adjustment if LiteLLM changes its env-var or config schema +- **Build-time patches ride on LiteLLM internals — verify each one after an upgrade.** `patch_budget_message.py`, `patch_image_cost.py` (keeps image-gen pricing on the shared cost-map key) and `patch_audio_request_cost.py` (adds the declared per-request charge on the audio paths: Tinfoil bills `voxtral-tts` $0.002 and `whisper-large-v3-turbo` $0.01 per request, which the cost map cannot express). Each prints `[coop-patch]` in the build log. They are non-fatal by design: a missing anchor warns and the build still succeeds, so the behaviour can vanish silently on a version bump. Related: code/litellm#5. - The CORS env var is `LITELLM_CORS_ORIGINS` (upstream-native since 1.84.0); the old `LITELLM_CORS_ALLOWED_ORIGINS` + sed patch are dead — don't re-add them - Test on a staging instance if the version jump is large diff --git a/config.yaml b/config.yaml index 51cdfab..e735155 100644 --- a/config.yaml +++ b/config.yaml @@ -148,7 +148,14 @@ model_list: # models: the member portal's /v1/models filters them out of the chat model # list (see AUDIO_MODEL_PREFIXES there) so members don't see them in the # picker, but they remain fully callable via /v1/audio/* for API users. - # whisper-large-v3-turbo: $0.05/1M input tokens. voxtral-tts: listed $0/$0. + # whisper-large-v3-turbo: $0/1M tokens + $0.01 per request. + # voxtral-tts: $0/1M tokens + $0.002 per request. + # Tinfoil bills these per REQUEST (its model list carries a `requestPrice`); + # reading only the two token prices made TTS look free and left STT billed off + # an older catalog. LiteLLM cannot price per request on the audio paths, so + # patch_audio_request_cost.py (built in via the Dockerfile) makes the cost + # calculator honour `input_cost_per_request` below. Re-check that build log + # line after every LiteLLM upgrade. # NOTE: LiteLLM routes audio by the openai/ provider + the endpoint called # (/v1/audio/transcriptions or /v1/audio/speech) — there is no # "audio_transcription/" provider prefix. @@ -159,7 +166,10 @@ model_list: api_key: os.environ/TINFOIL_API_KEY model_info: mode: audio_transcription - input_cost_per_token: 0.00000005 + # Tinfoil: $0 per token, $0.01 per request. The per-request charge is the + # whole cost, and the patch is what makes it billable. + input_cost_per_token: 0 + input_cost_per_request: 0.01 - model_name: tinfoil/voxtral-tts litellm_params: @@ -168,6 +178,8 @@ model_list: api_key: os.environ/TINFOIL_API_KEY model_info: mode: audio_speech + # Tinfoil: $0 per token, $0.002 per request — see the audio section note. + input_cost_per_request: 0.002 # --- Embedding model (Tinfoil, in-TEE) --- # nomic-embed-text — 768-dim text embeddings for RAG/semantic search. diff --git a/patch_audio_request_cost.py b/patch_audio_request_cost.py new file mode 100644 index 0000000..f07f5e7 --- /dev/null +++ b/patch_audio_request_cost.py @@ -0,0 +1,181 @@ +"""Bill a declared per-request charge on requests LiteLLM prices only per unit. + +Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal: +if the target file or the anchor strings are missing — e.g. after an upstream +version bump — it prints a warning and exits 0, so the build and the gateway +still succeed. The build log line ``[coop-patch] applied`` is what confirms it +took effect; check it on every LiteLLM upgrade (see README, upgrade checklist). + +Why: Tinfoil charges some models a flat amount PER REQUEST on top of (or instead +of) per-token pricing — ``voxtral-tts`` $0.002/request and +``whisper-large-v3-turbo`` $0.01/request, both with $0 per-token. LiteLLM can +only express per-request pricing on the search / rerank / MCP paths, via +``input_cost_per_query`` (see ``litellm/search/cost_calculator.py``, and +``litellm/types/utils.py``: "per-request pricing: rerank, search, and Bedrock +Marengo embed"). The audio paths price by character, second or token, and +``input_cost_per_request`` exists only in the proxy's cost-tracking settings — +the model cost calculator never reads it. So speech and transcription usage is +billed at $0 however the config is written. + +Patch: make the model cost calculator honour a declared +``input_cost_per_request``. ``response_cost_calculator`` is the single funnel for +every call type (the proxy's audio endpoints dispatch as ``aspeech`` and +``transcription`` and reach it through ``Logging._response_cost_calculator``), so +adding the charge at its final return covers both. The declared value is resolved +from the per-deployment ``custom_pricing`` the logging layer builds out of our +``config.yaml`` ``model_info``, falling back to the shared cost map and to the +call metadata. + +Scope: nothing changes unless a model declares ``input_cost_per_request``, so +this is inert for every other model on the gateway. + +Upstream: no issue filed; LiteLLM has no per-request audio pricing as of 1.103.2. +""" +from __future__ import annotations + +import glob +import os +import sys + +MARKER = "CO-OP PATCH (audio per-request cost, 2026-10-26)" + +HELPER = '''def _coop_per_request_cost( + *, + model: str | None = None, + litellm_model_name: str | None = None, + custom_pricing: Any | None = None, + litellm_logging_obj: Any | None = None, +) -> float: + """CO-OP PATCH (audio per-request cost, 2026-10-26) + + Return the model's declared ``input_cost_per_request``, else 0.0. + + LiteLLM cannot price a flat per-request charge on the audio paths, so + Tinfoil's per-request models (voxtral-tts, whisper-large-v3-turbo) were + billed at $0. Resolution order: the per-deployment custom pricing the + logging layer builds from our config.yaml model_info, then the shared cost + map, then the call metadata. Inert unless a model declares the field. + """ + candidates = [] + if isinstance(custom_pricing, dict): + candidates.append(custom_pricing) + try: + import litellm as _litellm # local import: not every caller binds it globally + cost_map = _litellm.model_cost + for key in (litellm_model_name, model): + entry = cost_map.get(key) if key else None + if isinstance(entry, dict): + candidates.append(entry) + except Exception: + pass + try: + details = getattr(litellm_logging_obj, "model_call_details", None) or {} + info = (details.get("metadata") or {}).get("model_info") + if isinstance(info, dict): + candidates.append(info) + except Exception: + pass + for candidate in candidates: + try: + value = candidate.get("input_cost_per_request") + except Exception: + continue + if value: + try: + return float(value) + except (TypeError, ValueError): + continue + return 0.0 + + +''' + +ANCHOR_DEF = "def response_cost_calculator(" + +ANCHOR_RETURN = """ region_name=region_name, + ) + return response_cost + except Exception as e: + raise e""" + +PATCHED_RETURN = """ region_name=region_name, + ) + + # CO-OP PATCH (audio per-request cost, 2026-10-26): add the declared + # per-request charge for models LiteLLM can only price per unit + # (Tinfoil's voxtral-tts, whisper-large-v3-turbo). Inert when the model + # declares nothing; the early "provider reported the cost" return above + # is unaffected. + try: + response_cost = response_cost + _coop_per_request_cost( + model=model, + litellm_model_name=litellm_model_name, + custom_pricing=custom_pricing, + litellm_logging_obj=litellm_logging_obj, + ) + except Exception: + pass + + return response_cost + except Exception as e: + raise e""" + + +def target_path() -> str: + """The installed litellm cost calculator (or an explicit path for testing).""" + if len(sys.argv) > 1: + return sys.argv[1] + env = os.environ.get("LITELLM_COST_CALCULATOR") + if env: + return env + hits = [] + try: # most reliable: ask the installed package where it lives + import litellm # noqa: PLC0415 + _file = getattr(litellm, "__file__", None) + if _file: + hits.append(os.path.join(os.path.dirname(_file), "cost_calculator.py")) + except Exception: + pass + hits += glob.glob("/usr/lib/python3*/site-packages/litellm/cost_calculator.py") + hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/cost_calculator.py") + hits += glob.glob("/usr/local/lib/python3*/dist-packages/litellm/cost_calculator.py") + hits += glob.glob("/opt/**/litellm/cost_calculator.py", recursive=True) + hits = [h for h in hits if os.path.exists(h)] + if not hits: + raise SystemExit("[coop-patch] could not locate litellm/cost_calculator.py") + return sorted(hits)[0] + + +def main() -> int: + path = target_path() + try: + source = open(path).read() + except OSError as exc: + print(f"[coop-patch] cannot read {path}: {exc}") + return 0 + + if MARKER in source: + print(f"[coop-patch] {path} already patched, nothing to do") + return 0 + + if ANCHOR_DEF not in source or ANCHOR_RETURN not in source: + print(f"[coop-patch] WARNING anchors not found in {path}; audio per-request " + f"cost will NOT be billed (check this after a LiteLLM upgrade)") + return 0 + + patched = source.replace(ANCHOR_DEF, HELPER + ANCHOR_DEF, 1) + patched = patched.replace(ANCHOR_RETURN, PATCHED_RETURN, 1) + compile(patched, path, "exec") # refuse to write something that will not import + + if not os.access(path, os.W_OK): # practice run against a read-only copy + print(f"[coop-patch] {path} is not writable; verified anchors and syntax only") + return 0 + + with open(path, "w") as handle: + handle.write(patched) + print(f"[coop-patch] applied to {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())