inference-bot
67df968a04
Tinfoil's catalog prices exactly one audio model per token: voxtral-small-24b ($0.20 in / $0.60 out per 1M, $0 per request). Wiring it gives members an STT route without the whisper $0.01/request floor (co-op/support#48). Two billing shapes on one model entry: - chat route (input_audio): returns real token usage -> bills exactly per token. - /v1/audio/transcriptions: Tinfoil returns NO token counts (duration only), so per-token billing cannot fire; Tinfoil also tokenizes audio in ~30s chunks (~375-380 tokens each regardless of clip length), making the chunk-equivalent ~$0.00008/request. Declared input_cost_per_request with that measured value, added by patch_audio_request_cost.py. patch_audio_request_cost.py: scope the per-request charge to the audio call types (transcription/atranscription/speech/aspeech) so this model's chat calls bill per token and are not double-billed; extend the diagnostic name-gate to 'voxtral'. Measured on the live catalog + sidecar 2026-11-06 (see issue #48 follow-up).
249 lines
9.9 KiB
Python
249 lines
9.9 KiB
Python
"""Bill a declared per-request charge on requests LiteLLM prices only per unit.
|
|
|
|
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
|
if the target file or the anchor strings are missing — e.g. after an upstream
|
|
version bump — it prints a warning and exits 0, so the build and the gateway
|
|
still succeed. The build log line ``[coop-patch] applied`` is what confirms it
|
|
took effect; check it on every LiteLLM upgrade (see README, upgrade checklist).
|
|
|
|
Why: Tinfoil charges some models a flat amount PER REQUEST on top of (or instead
|
|
of) per-token pricing — ``voxtral-tts`` $0.002/request and
|
|
``whisper-large-v3-turbo`` $0.01/request, both with $0 per-token. LiteLLM can
|
|
only express per-request pricing on the search / rerank / MCP paths, via
|
|
``input_cost_per_query`` (see ``litellm/search/cost_calculator.py``, and
|
|
``litellm/types/utils.py``: "per-request pricing: rerank, search, and Bedrock
|
|
Marengo embed"). The audio paths price by character, second or token, and
|
|
``input_cost_per_request`` exists only in the proxy's cost-tracking settings —
|
|
the model cost calculator never reads it. So speech and transcription usage is
|
|
billed at $0 however the config is written.
|
|
|
|
Patch: make the model cost calculator honour a declared
|
|
``input_cost_per_request``. ``response_cost_calculator`` is the single funnel for
|
|
every call type (the proxy's audio endpoints dispatch as ``aspeech`` and
|
|
``transcription`` and reach it through ``Logging._response_cost_calculator``), so
|
|
adding the charge at its final return covers both. The declared value is resolved
|
|
from the per-deployment ``custom_pricing`` the logging layer builds out of our
|
|
``config.yaml`` ``model_info``, falling back to the shared cost map and to the
|
|
call metadata.
|
|
|
|
Scope: nothing changes unless a model declares ``input_cost_per_request``, so
|
|
this is inert for every other model on the gateway. Since 2026-11-06 the charge
|
|
is added only on the AUDIO call types (transcription/atranscription/speech/
|
|
aspeech): a model that also carries per-token pricing (``voxtral-small-24b``)
|
|
bills its chat calls exactly per token, and a per-request charge on top of
|
|
that would double-bill them. The audio endpoints are the paths that cannot
|
|
price per token (the provider returns no token counts there), so the flat
|
|
charge belongs to them alone.
|
|
|
|
Upstream: no issue filed; LiteLLM has no per-request audio pricing as of 1.103.2.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import glob
|
|
import os
|
|
import sys
|
|
|
|
MARKER = "CO-OP PATCH (audio per-request cost, 2026-10-26)"
|
|
|
|
HELPER = '''def _coop_per_request_cost(
|
|
*,
|
|
model: str | None = None,
|
|
litellm_model_name: str | None = None,
|
|
custom_pricing: Any | None = None,
|
|
litellm_logging_obj: Any | None = None,
|
|
) -> float:
|
|
"""CO-OP PATCH (audio per-request cost, 2026-10-26)
|
|
|
|
Return the model's declared ``input_cost_per_request``, else 0.0.
|
|
|
|
LiteLLM cannot price a flat per-request charge on the audio paths, so
|
|
Tinfoil's per-request models (voxtral-tts, whisper-large-v3-turbo) were
|
|
billed at $0. Resolution order: the per-deployment custom pricing the
|
|
logging layer builds from our config.yaml model_info, then the shared cost
|
|
map, then the call metadata. Inert unless a model declares the field.
|
|
"""
|
|
def _is_audio(name: Any) -> bool:
|
|
text = str(name or "").lower()
|
|
return (
|
|
"tts" in text or "whisper" in text or "voxtral" in text
|
|
)
|
|
|
|
cost_map = {}
|
|
try:
|
|
import litellm as _litellm # local import: not every caller binds it globally
|
|
cost_map = getattr(_litellm, "model_cost", {}) or {}
|
|
except Exception:
|
|
cost_map = {}
|
|
|
|
candidates = []
|
|
if isinstance(custom_pricing, dict):
|
|
candidates.append(custom_pricing)
|
|
map_hits = []
|
|
for key in (litellm_model_name, model):
|
|
entry = cost_map.get(key) if key else None
|
|
if isinstance(entry, dict):
|
|
candidates.append(entry)
|
|
map_hits.append((key, entry.get("input_cost_per_request")))
|
|
try:
|
|
details = getattr(litellm_logging_obj, "model_call_details", None) or {}
|
|
info = (details.get("metadata") or {}).get("model_info")
|
|
if isinstance(info, dict):
|
|
candidates.append(info)
|
|
except Exception:
|
|
pass
|
|
|
|
# Diagnostic (audio models only): did this run at all, and what could it see?
|
|
# If this line never appears for a speech call, the request never reaches the
|
|
# model cost calculator and the loss is upstream of this patch.
|
|
if _is_audio(model) or _is_audio(litellm_model_name):
|
|
try:
|
|
print(
|
|
"[coop-audio] probe"
|
|
f" model={model!r} litellm={litellm_model_name!r}"
|
|
f" custom_pricing={sorted(custom_pricing) if isinstance(custom_pricing, dict) else type(custom_pricing).__name__}"
|
|
f" map_hits={map_hits!r} candidates={len(candidates)}",
|
|
flush=True,
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
for candidate in candidates:
|
|
try:
|
|
value = candidate.get("input_cost_per_request")
|
|
except Exception:
|
|
continue
|
|
if value:
|
|
try:
|
|
resolved = float(value)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if _is_audio(model) or _is_audio(litellm_model_name):
|
|
try:
|
|
print(f"[coop-audio] resolved per_request={resolved}", flush=True)
|
|
except Exception:
|
|
pass
|
|
return resolved
|
|
if _is_audio(model) or _is_audio(litellm_model_name):
|
|
try:
|
|
print("[coop-audio] resolved per_request=0.0 (nothing declared)", flush=True)
|
|
except Exception:
|
|
pass
|
|
return 0.0
|
|
|
|
|
|
'''
|
|
|
|
ANCHOR_DEF = "def response_cost_calculator("
|
|
|
|
ANCHOR_RETURN = """ region_name=region_name,
|
|
)
|
|
return response_cost
|
|
except Exception as e:
|
|
raise e"""
|
|
|
|
PATCHED_RETURN = """ region_name=region_name,
|
|
)
|
|
|
|
# CO-OP PATCH (audio per-request cost, 2026-10-26): add the declared
|
|
# per-request charge for models LiteLLM can only price per unit
|
|
# (Tinfoil's voxtral-tts, whisper-large-v3-turbo). Inert when the model
|
|
# declares nothing; the early "provider reported the cost" return above
|
|
# is unaffected. Scoped to the AUDIO call types (2026-11-06): a model
|
|
# that ALSO carries per-token pricing (voxtral-small-24b) bills its
|
|
# chat calls exactly per token and must not take the per-request
|
|
# charge on top of that.
|
|
try:
|
|
_coop_audio_call = call_type in (
|
|
"transcription",
|
|
"atranscription",
|
|
"speech",
|
|
"aspeech",
|
|
)
|
|
_coop_pr = (
|
|
_coop_per_request_cost(
|
|
model=model,
|
|
litellm_model_name=litellm_model_name,
|
|
custom_pricing=custom_pricing,
|
|
litellm_logging_obj=litellm_logging_obj,
|
|
)
|
|
if _coop_audio_call
|
|
else 0.0
|
|
)
|
|
response_cost = (response_cost or 0.0) + _coop_pr
|
|
_coop_name = str(model or "") + str(litellm_model_name or "")
|
|
if _coop_audio_call and (
|
|
"tts" in _coop_name.lower()
|
|
or "whisper" in _coop_name.lower()
|
|
or "voxtral" in _coop_name.lower()
|
|
):
|
|
print(
|
|
f"[coop-audio] add per_request={_coop_pr!r} -> response_cost={response_cost!r}",
|
|
flush=True,
|
|
)
|
|
except Exception as _coop_exc:
|
|
print(f"[coop-audio] add failed: {type(_coop_exc).__name__}: {_coop_exc}", flush=True)
|
|
|
|
return response_cost
|
|
except Exception as e:
|
|
raise e"""
|
|
|
|
|
|
def target_path() -> str:
|
|
"""The installed litellm cost calculator (or an explicit path for testing)."""
|
|
if len(sys.argv) > 1:
|
|
return sys.argv[1]
|
|
env = os.environ.get("LITELLM_COST_CALCULATOR")
|
|
if env:
|
|
return env
|
|
hits = []
|
|
try: # most reliable: ask the installed package where it lives
|
|
import litellm # noqa: PLC0415
|
|
_file = getattr(litellm, "__file__", None)
|
|
if _file:
|
|
hits.append(os.path.join(os.path.dirname(_file), "cost_calculator.py"))
|
|
except Exception:
|
|
pass
|
|
hits += glob.glob("/usr/lib/python3*/site-packages/litellm/cost_calculator.py")
|
|
hits += glob.glob("/usr/local/lib/python3*/site-packages/litellm/cost_calculator.py")
|
|
hits += glob.glob("/usr/local/lib/python3*/dist-packages/litellm/cost_calculator.py")
|
|
hits += glob.glob("/opt/**/litellm/cost_calculator.py", recursive=True)
|
|
hits = [h for h in hits if os.path.exists(h)]
|
|
if not hits:
|
|
raise SystemExit("[coop-patch] could not locate litellm/cost_calculator.py")
|
|
return sorted(hits)[0]
|
|
|
|
|
|
def main() -> int:
|
|
path = target_path()
|
|
try:
|
|
source = open(path).read()
|
|
except OSError as exc:
|
|
print(f"[coop-patch] cannot read {path}: {exc}")
|
|
return 0
|
|
|
|
if MARKER in source:
|
|
print(f"[coop-patch] {path} already patched, nothing to do")
|
|
return 0
|
|
|
|
if ANCHOR_DEF not in source or ANCHOR_RETURN not in source:
|
|
print(f"[coop-patch] WARNING anchors not found in {path}; audio per-request "
|
|
f"cost will NOT be billed (check this after a LiteLLM upgrade)")
|
|
return 0
|
|
|
|
patched = source.replace(ANCHOR_DEF, HELPER + ANCHOR_DEF, 1)
|
|
patched = patched.replace(ANCHOR_RETURN, PATCHED_RETURN, 1)
|
|
compile(patched, path, "exec") # refuse to write something that will not import
|
|
|
|
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
|
print(f"[coop-patch] {path} is not writable; verified anchors and syntax only")
|
|
return 0
|
|
|
|
with open(path, "w") as handle:
|
|
handle.write(patched)
|
|
print(f"[coop-patch] applied to {path}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|