Files
litellm/patch_audio_request_cost.py
T
inference-bot df4e154128 audio patch: price per-request models when unit pricing raises (#5)
Speech and transcription price per character/second/token and raise ValueError
when the model declares none of those rates — every Tinfoil per-request model.
The raise aborted cost calculation before the per-request charge was added, so
the gateway wrote spend 0.0 rows. Catch that case for declared per-request
audio models and price the base at 0, leaving the per-request charge as the
whole bill.
2026-11-17 09:04:28 +00:00

346 lines
15 KiB
Python

"""Bill a declared per-request charge on requests LiteLLM prices only per unit.
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
if the target file or the anchor strings are missing — e.g. after an upstream
version bump — it prints a warning and exits 0, so the build and the gateway
still succeed. The build log line ``[coop-patch] applied`` is what confirms it
took effect; check it on every LiteLLM upgrade (see README, upgrade checklist).
Why: Tinfoil charges some models a flat amount PER REQUEST on top of (or instead
of) per-token pricing — ``voxtral-tts`` $0.002/request and
``whisper-large-v3-turbo`` $0.01/request, both with $0 per-token. LiteLLM can
only express per-request pricing on the search / rerank / MCP paths, via
``input_cost_per_query`` (see ``litellm/search/cost_calculator.py``, and
``litellm/types/utils.py``: "per-request pricing: rerank, search, and Bedrock
Marengo embed"). The audio paths price by character, second or token, and
``input_cost_per_request`` exists only in the proxy's cost-tracking settings —
the model cost calculator never reads it. So speech and transcription usage is
billed at $0 however the config is written.
Patch: make the model cost calculator honour a declared
``input_cost_per_request``. ``response_cost_calculator`` is the single funnel for
every call type (the proxy's audio endpoints dispatch as ``aspeech`` and
``transcription`` and reach it through ``Logging._response_cost_calculator``), so
adding the charge at its final return covers both. The declared value is resolved
from the per-deployment ``custom_pricing`` the logging layer builds out of our
``config.yaml`` ``model_info``, falling back to the shared cost map and to the
call metadata.
Scope: nothing changes unless a model declares ``input_cost_per_request``, so
this is inert for every other model on the gateway. Since 2026-11-06 the charge
is added only on the AUDIO call types (transcription/atranscription/speech/
aspeech): a model that also carries per-token pricing (``voxtral-small-24b``)
bills its chat calls exactly per token, and a per-request charge on top of
that would double-bill them. The audio endpoints are the paths that cannot
price per token (the provider returns no token counts there), so the flat
charge belongs to them alone.
Upstream: no issue filed; LiteLLM has no per-request audio pricing as of 1.103.2.
"""
from __future__ import annotations
import glob
import os
import sys
MARKER = "CO-OP PATCH (audio per-request cost, 2026-10-26)"
HELPER = '''def _coop_per_request_cost(
*,
model: str | None = None,
litellm_model_name: str | None = None,
custom_pricing: Any | None = None,
litellm_logging_obj: Any | None = None,
) -> float:
"""CO-OP PATCH (audio per-request cost, 2026-10-26)
Return the model's declared ``input_cost_per_request``, else 0.0.
LiteLLM cannot price a flat per-request charge on the audio paths, so
Tinfoil's per-request models (voxtral-tts, whisper-large-v3-turbo) were
billed at $0. Resolution order: the per-deployment custom pricing the
logging layer builds from our config.yaml model_info, then the shared cost
map, then the call metadata. Inert unless a model declares the field.
"""
def _is_audio(name: Any) -> bool:
text = str(name or "").lower()
return (
"tts" in text or "whisper" in text or "voxtral" in text
)
cost_map = {}
try:
import litellm as _litellm # local import: not every caller binds it globally
cost_map = getattr(_litellm, "model_cost", {}) or {}
except Exception:
cost_map = {}
candidates = []
if isinstance(custom_pricing, dict):
candidates.append(custom_pricing)
map_hits = []
for key in (litellm_model_name, model):
entry = cost_map.get(key) if key else None
if isinstance(entry, dict):
candidates.append(entry)
map_hits.append((key, entry.get("input_cost_per_request")))
try:
details = getattr(litellm_logging_obj, "model_call_details", None) or {}
info = (details.get("metadata") or {}).get("model_info")
if isinstance(info, dict):
candidates.append(info)
except Exception:
pass
# Diagnostic (audio models only): did this run at all, and what could it see?
# If this line never appears for a speech call, the request never reaches the
# model cost calculator and the loss is upstream of this patch.
if _is_audio(model) or _is_audio(litellm_model_name):
try:
print(
"[coop-audio] probe"
f" model={model!r} litellm={litellm_model_name!r}"
f" custom_pricing={sorted(custom_pricing) if isinstance(custom_pricing, dict) else type(custom_pricing).__name__}"
f" map_hits={map_hits!r} candidates={len(candidates)}",
flush=True,
)
except Exception:
pass
for candidate in candidates:
try:
value = candidate.get("input_cost_per_request")
except Exception:
continue
if value:
try:
resolved = float(value)
except (TypeError, ValueError):
continue
if _is_audio(model) or _is_audio(litellm_model_name):
try:
print(f"[coop-audio] resolved per_request={resolved}", flush=True)
except Exception:
pass
return resolved
if _is_audio(model) or _is_audio(litellm_model_name):
try:
print("[coop-audio] resolved per_request=0.0 (nothing declared)", flush=True)
except Exception:
pass
return 0.0
'''
ANCHOR_DEF = "def response_cost_calculator("
ANCHOR_EARLY = """ if provider_response_cost is not None and not custom_pricing:
return provider_response_cost"""
ANCHOR_RETURN = """ region_name=region_name,
)
return response_cost
except Exception as e:
raise e"""
# The unit-price call. On speech/transcription LiteLLM prices by character,
# second or token and raises ValueError when the model declares none of those
# rates — which is exactly a per-request model. The declaration is then the
# whole bill, so price the base at 0 instead of letting the raise swallow the
# spend log write (issue #5: rows recorded spend 0.0 and the tail patch below
# never ran because the exception aborted the function first).
ANCHOR_COST = """ response_cost = completion_cost(
completion_response=response_object,
model=model,
call_type=call_type,
custom_llm_provider=custom_llm_provider,
optional_params=optional_params,
custom_pricing=custom_pricing,
base_model=base_model,
prompt=prompt,
standard_built_in_tools_params=standard_built_in_tools_params,
litellm_model_name=litellm_model_name,
router_model_id=router_model_id,
litellm_logging_obj=litellm_logging_obj,
service_tier=service_tier,
data_residency=data_residency,
vertex_location=vertex_location,
region_name=region_name,
)"""
PATCHED_COST = """ try:
response_cost = completion_cost(
completion_response=response_object,
model=model,
call_type=call_type,
custom_llm_provider=custom_llm_provider,
optional_params=optional_params,
custom_pricing=custom_pricing,
base_model=base_model,
prompt=prompt,
standard_built_in_tools_params=standard_built_in_tools_params,
litellm_model_name=litellm_model_name,
router_model_id=router_model_id,
litellm_logging_obj=litellm_logging_obj,
service_tier=service_tier,
data_residency=data_residency,
vertex_location=vertex_location,
region_name=region_name,
)
except Exception as _coop_unit_exc:
# CO-OP PATCH (audio per-request cost, 2026-11-17): a model billed
# per request declares no per-character/second/token rate, so
# LiteLLM's unit pricing raises. The per-request charge is the
# whole bill; let the tail patch below add it to a zero base.
_coop_declared_pr = (
_coop_per_request_cost(
model=model,
litellm_model_name=litellm_model_name,
custom_pricing=custom_pricing,
litellm_logging_obj=litellm_logging_obj,
)
if call_type in ("transcription", "atranscription", "speech", "aspeech")
else 0.0
)
if _coop_declared_pr > 0:
print(
f"[coop-audio] unit pricing unavailable ({type(_coop_unit_exc).__name__}); "
f"pricing at declared per-request={_coop_declared_pr!r}",
flush=True,
)
response_cost = 0.0
else:
raise"""
PATCHED_EARLY = """ if provider_response_cost is not None and not custom_pricing:
# CO-OP PATCH (2026-11-16): a provider-reported cost is not the
# whole bill on the audio paths. Speech returns here and never
# reaches the tail patch, so add the declared per-request charge
# before returning. Guarded by the DECLARATION, not the call
# type: models without input_cost_per_request (voxtral-small-24b,
# token-priced) stay inert.
return provider_response_cost + _coop_per_request_cost(
model=model,
litellm_model_name=litellm_model_name,
custom_pricing=custom_pricing,
litellm_logging_obj=litellm_logging_obj,
)"""
PATCHED_RETURN = """ region_name=region_name,
)
# CO-OP PATCH (audio per-request cost, 2026-10-26): add the declared
# per-request charge for models LiteLLM can only price per unit
# (Tinfoil's voxtral-tts, whisper-large-v3-turbo). Inert when the model
# declares nothing; the early "provider reported the cost" return above
# is unaffected. Scoped to the AUDIO call types (2026-11-06): a model
# that ALSO carries per-token pricing (voxtral-small-24b) bills its
# chat calls exactly per token and must not take the per-request
# charge on top of that.
try:
_coop_audio_call = call_type in (
"transcription",
"atranscription",
"speech",
"aspeech",
)
_coop_pr = (
_coop_per_request_cost(
model=model,
litellm_model_name=litellm_model_name,
custom_pricing=custom_pricing,
litellm_logging_obj=litellm_logging_obj,
)
if _coop_audio_call
else 0.0
)
response_cost = (response_cost or 0.0) + _coop_pr
_coop_name = str(model or "") + str(litellm_model_name or "")
if _coop_audio_call and (
"tts" in _coop_name.lower()
or "whisper" in _coop_name.lower()
or "voxtral" in _coop_name.lower()
):
print(
f"[coop-audio] add per_request={_coop_pr!r} -> response_cost={response_cost!r}",
flush=True,
)
except Exception as _coop_exc:
print(f"[coop-audio] add failed: {type(_coop_exc).__name__}: {_coop_exc}", flush=True)
return response_cost
except Exception as e:
raise e"""
def target_path() -> str:
"""The installed litellm cost calculator (or an explicit path for testing)."""
if len(sys.argv) > 1:
return sys.argv[1]
env = os.environ.get("LITELLM_COST_CALCULATOR")
if env:
return env
hits = []
try: # most reliable: ask the installed package where it lives
import litellm # noqa: PLC0415
_file = getattr(litellm, "__file__", None)
if _file:
hits.append(os.path.join(os.path.dirname(_file), "cost_calculator.py"))
except Exception:
pass
hits += glob.glob("/usr/lib/python3*/site-packages/litellm/cost_calculator.py")
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/cost_calculator.py")
hits += glob.glob("/usr/local/lib/python3*/dist-packages/litellm/cost_calculator.py")
hits += glob.glob("/opt/**/litellm/cost_calculator.py", recursive=True)
hits = [h for h in hits if os.path.exists(h)]
if not hits:
raise SystemExit("[coop-patch] could not locate litellm/cost_calculator.py")
return sorted(hits)[0]
def main() -> int:
path = target_path()
try:
source = open(path).read()
except OSError as exc:
print(f"[coop-patch] cannot read {path}: {exc}")
return 0
if MARKER in source:
print(f"[coop-patch] {path} already patched, nothing to do")
return 0
if ANCHOR_DEF not in source or ANCHOR_RETURN not in source:
print(f"[coop-patch] WARNING anchors not found in {path}; audio per-request "
f"cost will NOT be billed (check this after a LiteLLM upgrade)")
return 0
patched = source.replace(ANCHOR_DEF, HELPER + ANCHOR_DEF, 1)
patched = patched.replace(ANCHOR_RETURN, PATCHED_RETURN, 1)
if ANCHOR_COST in patched:
patched = patched.replace(ANCHOR_COST, PATCHED_COST, 1)
print("[coop-patch] unit-price call made per-request tolerant (speech/transcription)")
else:
print("[coop-patch] WARNING unit-price anchor not found; per-request models "
"will still raise and bill $0")
if ANCHOR_EARLY in patched:
patched = patched.replace(ANCHOR_EARLY, PATCHED_EARLY, 1)
print("[coop-patch] early provider-cost return also patched (audio speech)")
else:
print("[coop-patch] WARNING early-return anchor not found; speech may stay unbilled")
compile(patched, path, "exec") # refuse to write something that will not import
if not os.access(path, os.W_OK): # practice run against a read-only copy
print(f"[coop-patch] {path} is not writable; verified anchors and syntax only")
return 0
with open(path, "w") as handle:
handle.write(patched)
print(f"[coop-patch] applied to {path}")
return 0
if __name__ == "__main__":
sys.exit(main())