From c03abaa330f8c41ca7920065b4bb8f1330b3fd31 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 26 Oct 2026 21:24:42 +0000 Subject: [PATCH] gateway: put audio pricing on the shared cost-map key (no SpendLogs row at all) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first audio deploy proved the config and the cost-calculator patch were both live — /model/info served input_cost_per_request, and the build log confirmed '[coop-patch] applied' — yet a real /v1/audio/speech and /v1/audio/transcriptions pair still billed exactly $0.000000, with no spend row anywhere. Diagnosis: the audio cost paths resolve pricing from the SHARED cost-map key, and the router strips per-deployment pricing from that key, so the lookup raises and the spend write is skipped. Same mechanism as the image-generation case patch_image_cost.py already fixes. The cost-calculator patch cannot cover this on its own: it adds a per-request charge on top of a cost the normal path computes, and that path never got there. This patch re-adds the pricing for audio_speech and audio_transcription deployments, explicitly allowing input_cost_per_request through — LiteLLM does not list it among its CustomPricing fields, which is precisely why the cost-calculator patch has to read it. --- Dockerfile | 8 +++ patch_audio_router_pricing.py | 112 ++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 patch_audio_router_pricing.py diff --git a/Dockerfile b/Dockerfile index d74fef7..5d94236 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,6 +46,14 @@ RUN python3 /app/code/patch_image_cost.py COPY patch_audio_request_cost.py /app/code/patch_audio_request_cost.py RUN python3 /app/code/patch_audio_request_cost.py +# The audio cost paths read pricing from the SHARED cost-map key, which the +# router strips per-deployment pricing from — so the lookup raised, the spend +# write was skipped, and audio produced no SpendLogs row at all. This re-adds the +# pricing (and input_cost_per_request) for the audio modes. Same mechanism as +# patch_image_cost.py above. +COPY patch_audio_router_pricing.py /app/code/patch_audio_router_pricing.py +RUN python3 /app/code/patch_audio_router_pricing.py + # Audio keep-alive: scheduled pings keep the Tinfoil TEE audio enclave # warm so voice mode skips the ~24s cold-start (see audio_keepalive.py header). COPY audio_keepalive.py /app/code/audio_keepalive.py diff --git a/patch_audio_router_pricing.py b/patch_audio_router_pricing.py new file mode 100644 index 0000000..1b18656 --- /dev/null +++ b/patch_audio_router_pricing.py @@ -0,0 +1,112 @@ +"""Keep audio-model pricing on the shared cost-map key (speech + transcription). + +Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal: +if the target file or the anchor is missing — e.g. after a version bump — it +prints a warning and exits 0, so the build and the gateway still succeed. Look +for the ``[coop-patch]`` line in the build log. + +Why this is needed on top of patch_audio_request_cost.py: the audio cost paths +(speech, transcription) resolve pricing from the **shared** cost-map key, and the +router deliberately strips per-deployment pricing from that key ("so one +deployment's overrides don't pollute sibling deployments"). For a deployment on a +custom backend — ours are all `openai/` against a local Tinfoil proxy — +the cost lookup then raises, the spend-log write is skipped, and **no SpendLogs +row appears at all**. That is the symptom we measured: `/v1/audio/speech` and +`/v1/audio/transcriptions` returned 200 with real audio, and the member key's +spend stayed exactly $0.000000 with no spend row anywhere. It is the same +mechanism as the image-generation case that patch_image_cost.py already fixes +(upstream BerriAI/litellm#10023/#12471). + +The cost-calculator patch alone cannot fix it: it only adds a per-request charge +*on top of* a cost the normal path computed, and that path never got that far. + +`input_cost_per_request` is deliberately allowed through even though it is **not** +in `CustomPricingLiteLLMParams.model_fields` — LiteLLM does not list it there, and +patch_audio_request_cost.py is what makes the cost calculator read it. + +Safe for us: the co-op gateway has one deployment per audio backend, so there are +no sibling deployments for the shared key to pollute. +""" +from __future__ import annotations + +import glob +import os +import sys + +MARKER = "CO-OP PATCH (audio shared pricing, 2026-10-26)" + +ANCHOR = " # Always register the (possibly mode-preserved) shared backend info." + +INSERT = ''' # CO-OP PATCH (audio shared pricing, 2026-10-26): the audio cost paths + # (speech, transcription) read pricing from the shared cost-map key, and + # this router strips per-deployment pricing from it — so a custom-backend + # audio deployment raises in the cost lookup, the spend write is skipped, + # and no SpendLogs row appears at all (same mechanism as the image case + # this file already patches). Re-add the pricing for audio modes, + # including input_cost_per_request, which LiteLLM does not list among its + # CustomPricing fields. + if deployment_mode in ("audio_speech", "audio_transcription"): + for _ak, _av in model_info.items(): + if ( + _ak in CustomPricingLiteLLMParams.model_fields + or _ak == "input_cost_per_request" + ) and _av is not None: + shared_model_info[_ak] = _av + +''' + + +def target_path() -> str: + if len(sys.argv) > 1: + return sys.argv[1] + env = os.environ.get("LITELLM_ROUTER") + if env: + return env + hits = [] + try: # most reliable: ask the installed package where it lives + import litellm # noqa: PLC0415 + _file = getattr(litellm, "__file__", None) + if _file: + hits.append(os.path.join(os.path.dirname(_file), "router.py")) + except Exception: + pass + hits += glob.glob("/app/.venv/lib/python*/site-packages/litellm/router.py") + hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/router.py") + hits = [h for h in hits if os.path.exists(h)] + if not hits: + raise SystemExit("[coop-patch] could not locate litellm/router.py") + return sorted(hits)[0] + + +def main() -> int: + path = target_path() + try: + source = open(path).read() + except OSError as exc: + print(f"[coop-patch] cannot read {path}: {exc}") + return 0 + + if MARKER in source: + print(f"[coop-patch] {path} already patched, nothing to do") + return 0 + + if ANCHOR not in source: + print(f"[coop-patch] WARNING anchor not found in {path}; audio pricing will " + f"NOT reach the shared cost map and audio spend will stay invisible") + return 0 + + patched = source.replace(ANCHOR, INSERT + ANCHOR, 1) + compile(patched, path, "exec") # refuse to write something that will not import + + if not os.access(path, os.W_OK): # practice run against a read-only copy + print(f"[coop-patch] {path} is not writable; verified anchor and syntax only") + return 0 + + with open(path, "w") as handle: + handle.write(patched) + print(f"[coop-patch] applied audio shared pricing to {path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())