From a3acce2107b5dcf0ffdf9070459e1eb0c583d798 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 26 Oct 2026 21:35:02 +0000 Subject: [PATCH] gateway: instrument the audio per-request helper (speech still bills $0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Speech billed $0.000000 across three deploys while transcription billed $0.010000 exactly, so the patch works for one audio call type and not the other. Three hypotheses have now been retired (hidden-params early return; missing character price; lookup miss) — the remaining question is WHERE the speech cost is lost, and guessing is more expensive than measuring. The helper now prints, for TTS/STT models only, whether it ran at all, what it could see (custom_pricing keys, shared cost-map hits) and what it resolved, into the app log. If those lines never appear for a speech call, the request never reaches the model cost calculator and the loss is upstream of this patch. Also fixes a real defect in the previous revision: the addition was wrapped in a bare except, so a TypeError from a None base cost would have been swallowed silently — which is exactly the sort of silence that made this take three deploys. It now coerces a None base to 0.0 and reports any failure. --- patch_audio_request_cost.py | 68 ++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/patch_audio_request_cost.py b/patch_audio_request_cost.py index f07f5e7..2774c4b 100644 --- a/patch_audio_request_cost.py +++ b/patch_audio_request_cost.py @@ -56,18 +56,26 @@ HELPER = '''def _coop_per_request_cost( logging layer builds from our config.yaml model_info, then the shared cost map, then the call metadata. Inert unless a model declares the field. """ + def _is_audio(name: Any) -> bool: + text = str(name or "").lower() + return "tts" in text or "whisper" in text + + cost_map = {} + try: + import litellm as _litellm # local import: not every caller binds it globally + cost_map = getattr(_litellm, "model_cost", {}) or {} + except Exception: + cost_map = {} + candidates = [] if isinstance(custom_pricing, dict): candidates.append(custom_pricing) - try: - import litellm as _litellm # local import: not every caller binds it globally - cost_map = _litellm.model_cost - for key in (litellm_model_name, model): - entry = cost_map.get(key) if key else None - if isinstance(entry, dict): - candidates.append(entry) - except Exception: - pass + map_hits = [] + for key in (litellm_model_name, model): + entry = cost_map.get(key) if key else None + if isinstance(entry, dict): + candidates.append(entry) + map_hits.append((key, entry.get("input_cost_per_request"))) try: details = getattr(litellm_logging_obj, "model_call_details", None) or {} info = (details.get("metadata") or {}).get("model_info") @@ -75,6 +83,22 @@ HELPER = '''def _coop_per_request_cost( candidates.append(info) except Exception: pass + + # Diagnostic (audio models only): did this run at all, and what could it see? + # If this line never appears for a speech call, the request never reaches the + # model cost calculator and the loss is upstream of this patch. + if _is_audio(model) or _is_audio(litellm_model_name): + try: + print( + "[coop-audio] probe" + f" model={model!r} litellm={litellm_model_name!r}" + f" custom_pricing={sorted(custom_pricing) if isinstance(custom_pricing, dict) else type(custom_pricing).__name__}" + f" map_hits={map_hits!r} candidates={len(candidates)}", + flush=True, + ) + except Exception: + pass + for candidate in candidates: try: value = candidate.get("input_cost_per_request") @@ -82,9 +106,20 @@ HELPER = '''def _coop_per_request_cost( continue if value: try: - return float(value) + resolved = float(value) except (TypeError, ValueError): continue + if _is_audio(model) or _is_audio(litellm_model_name): + try: + print(f"[coop-audio] resolved per_request={resolved}", flush=True) + except Exception: + pass + return resolved + if _is_audio(model) or _is_audio(litellm_model_name): + try: + print("[coop-audio] resolved per_request=0.0 (nothing declared)", flush=True) + except Exception: + pass return 0.0 @@ -107,14 +142,21 @@ PATCHED_RETURN = """ region_name=region_name, # declares nothing; the early "provider reported the cost" return above # is unaffected. try: - response_cost = response_cost + _coop_per_request_cost( + _coop_pr = _coop_per_request_cost( model=model, litellm_model_name=litellm_model_name, custom_pricing=custom_pricing, litellm_logging_obj=litellm_logging_obj, ) - except Exception: - pass + response_cost = (response_cost or 0.0) + _coop_pr + _coop_name = str(model or "") + str(litellm_model_name or "") + if "tts" in _coop_name.lower() or "whisper" in _coop_name.lower(): + print( + f"[coop-audio] add per_request={_coop_pr!r} -> response_cost={response_cost!r}", + flush=True, + ) + except Exception as _coop_exc: + print(f"[coop-audio] add failed: {type(_coop_exc).__name__}: {_coop_exc}", flush=True) return response_cost except Exception as e: