"""Price a generic pass-through route from the usage it reports in its OWN body. Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal: if the target file or the anchor is missing — e.g. after an upstream version bump — it prints a warning and exits 0, so the build and the gateway still succeed. The build log line ``[coop-patch] applied`` is what confirms it took effect; check it on every LiteLLM upgrade (see README, upgrade checklist). Why: config-declared ``pass_through_endpoints`` with ``auth: true`` are logged through ``PassThroughEndpointLogging``. A target whose hostname matches none of vertex / anthropic / openai / gemini resolves to ``EndpointType.GENERIC`` and takes the generic branch of ``pass_through_async_success_handler`` — it is logged, but LiteLLM has no model on the route to price, so the only cost it knows is the flat ``cost_per_request`` from the config entry. That is a whole- request estimate used for budget accounting, not an invoice: Metis is billed at $0.12 per 1M *input* tokens and returns the true count in ``usage.input_tokens``, so the flat figure is off by whatever factor the call differed from the estimate. LiteLLM's own supported mechanism for an unpriced pass-through is the ``x-litellm-response-cost`` / ``x-litellm-total-tokens`` response-header contract (``litellm/proxy/pass_through_endpoints/upstream_usage_headers.py``), but that is consumed inside this SAME logging path, so it only helps where the logging path already runs — and GreenPT does not send those headers. Reading the body, which is already in hand here, needs no extra process in front of the provider (a sidecar would be a single point of failure for the route itself, not just for its metering). Patch: make ``PassThroughEndpointLogging._set_cost_per_request`` price a route listed in ``general_settings.coop_passthrough_cost_per_token`` (USD per input token, keyed by the gateway route path, or by the upstream target) from the ``usage.input_tokens`` in the response body: cost = input_tokens * coop_passthrough_cost_per_token[route] The rate lives in ``general_settings`` rather than the endpoint dict because the endpoint model (``PassThroughGenericEndpoint``) drops unknown fields and the exact-route registration copies only ``cost_per_request`` into the logging payload — the ``general_settings`` dict is read straight from the proxy module, the same way ``patch_passthrough_key_access.py`` reads ``coop_passthrough_open_to_keys``. TOKEN COUNTS. Pricing the row from the body is not enough: the row still recorded ``prompt_tokens=0``, ``completion_tokens=0``, ``total_tokens=0``, because the pass-through success handler hands the spend writer a ``StandardPassThroughResponseObject`` whose text carries the body but whose ``usage`` is empty, and the generic pass-through branch builds no ``standard_logging_object`` usage either. Any per-token reporting (panel, dashboard, energy) that sums tokens for this route then under-reports. So the patch also carries the count onto the logging object as ``combined_usage_object`` — the same ``litellm.Usage`` override the streaming handler uses to recover partial usage. The spend writer reads ``combined_usage_object`` (``spend_tracking_utils.get_logging_payload`` and ``StandardLoggingPayloadSetup.get_usage_as_dict``) before it falls back to the response object, so the row records ``prompt_tokens=input_tokens``, ``completion_tokens=0``, ``total_tokens=input_tokens`` while spend stays exactly ``input_tokens × rate``. The exact figure wins; ``cost_per_request`` stays as the fallback for a listed route whose body carries no usable input-token count, and for every route not listed. An upstream that reports its own totals via the x-litellm-* headers still wins outright (the early return is left intact). Inert unless an operator lists a route, so it changes nothing for the chat models or the audio paths. Upstream: no issue filed; LiteLLM has no per-token pricing for pass-through routes whose body is not an OpenAI/Anthropic/Gemini shape (as of 1.103.2). """ from pathlib import Path import os import sys MARKER = "coop_passthrough_cost_per_token" OLD = ( " #########################################################\n" " # Check if cost per request is set\n" " #########################################################\n" " if has_upstream_reported_usage(logging_obj):\n" " return kwargs\n" "\n" " if passthrough_logging_payload.get(\"cost_per_request\") is not None:\n" " kwargs[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n" " logging_obj.model_call_details[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n" "\n" " return kwargs\n" ) NEW = ( " #########################################################\n" " # Check if cost per request is set\n" " #########################################################\n" " if has_upstream_reported_usage(logging_obj):\n" " return kwargs\n" "\n" " # [coop-patch] Exact per-input-token pricing for a generic\n" " # pass-through route. A route the operator lists in\n" " # general_settings.coop_passthrough_cost_per_token (USD per input\n" " # token, keyed by gateway path or upstream target) is priced from the\n" " # usage it reports in its OWN response body, so spend is the real\n" " # token count rather than cost_per_request's flat estimate. GreenPT's\n" " # Metis decision route (/v1/systemone) is the first such route: it\n" " # returns usage.input_tokens and always 0 output tokens, billed at\n" " # $0.12 per 1M input tokens and recorded on the spend row with the\n" " # same count. Inert unless a route is listed; a listed\n" " # route whose body carries no token count falls through to\n" " # cost_per_request below, and every unlisted route is unchanged.\n" " try:\n" " from litellm.proxy.proxy_server import general_settings as _coop_general_settings\n" "\n" " _coop_rates = (_coop_general_settings or {}).get(\"coop_passthrough_cost_per_token\") or {}\n" " if _coop_rates:\n" " _coop_candidates = set()\n" " _coop_psr = ((kwargs.get(\"litellm_params\") or {}).get(\"proxy_server_request\") or {})\n" " _coop_req_url = _coop_psr.get(\"url\")\n" " if _coop_req_url:\n" " _coop_candidates.add(str(_coop_req_url))\n" " _coop_candidates.add(urlparse(str(_coop_req_url)).path)\n" " _coop_upstream_url = passthrough_logging_payload.get(\"url\")\n" " if _coop_upstream_url:\n" " _coop_candidates.add(str(_coop_upstream_url))\n" " _coop_candidates.add(urlparse(str(_coop_upstream_url)).path)\n" "\n" " _coop_rate = None\n" " for _coop_route in _coop_candidates:\n" " if _coop_route in _coop_rates:\n" " _coop_rate = _coop_rates[_coop_route]\n" " break\n" " if _coop_rate is None:\n" " for _coop_key, _coop_val in _coop_rates.items():\n" " _coop_prefix = str(_coop_key).rstrip(\"/\")\n" " if any(\n" " c == _coop_key or (c and c.startswith(_coop_prefix + \"/\"))\n" " for c in _coop_candidates\n" " ):\n" " _coop_rate = _coop_val\n" " break\n" "\n" " _coop_tokens = 0\n" " _coop_body = passthrough_logging_payload.get(\"response_body\")\n" " if isinstance(_coop_body, dict):\n" " _coop_usage = _coop_body.get(\"usage\")\n" " if isinstance(_coop_usage, dict):\n" " _coop_tokens = _coop_usage.get(\"input_tokens\") or 0\n" " if not _coop_tokens:\n" " _coop_tokens = _coop_body.get(\"input_tokens\") or 0\n" "\n" " if _coop_rate is not None and _coop_tokens:\n" " _coop_cost = float(_coop_tokens) * float(_coop_rate)\n" " kwargs[\"response_cost\"] = _coop_cost\n" " logging_obj.model_call_details[\"response_cost\"] = _coop_cost\n" " # [coop-patch] Record the true token counts on the row too.\n" " # The spend writer reads usage from combined_usage_object\n" " # (the override the streaming handler already uses for\n" " # recovered partial usage); without it the generic\n" " # pass-through response object carries no usage and the\n" " # row's prompt/completion/total tokens stay 0.\n" " _coop_usage = None\n" " try:\n" " from litellm.types.utils import Usage as _CoopUsage\n" "\n" " _coop_usage = _CoopUsage(\n" " prompt_tokens=int(_coop_tokens),\n" " completion_tokens=0,\n" " total_tokens=int(_coop_tokens),\n" " )\n" " except Exception as _coop_usage_exc:\n" " print(\n" " \"[coop-passthrough-cost] usage object failed: %s: %s\"\n" " % (type(_coop_usage_exc).__name__, _coop_usage_exc),\n" " flush=True,\n" " )\n" " if _coop_usage is not None:\n" " kwargs[\"combined_usage_object\"] = _coop_usage\n" " logging_obj.model_call_details[\"combined_usage_object\"] = _coop_usage\n" " print(\n" " \"[coop-passthrough-cost] input_tokens=%s rate=%s -> response_cost=%s\"\n" " % (_coop_tokens, _coop_rate, _coop_cost),\n" " flush=True,\n" " )\n" " return kwargs\n" " except Exception as _coop_exc:\n" " print(\n" " \"[coop-passthrough-cost] exact cost failed: %s: %s\"\n" " % (type(_coop_exc).__name__, _coop_exc),\n" " flush=True,\n" " )\n" "\n" " if passthrough_logging_payload.get(\"cost_per_request\") is not None:\n" " kwargs[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n" " logging_obj.model_call_details[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n" "\n" " return kwargs\n" ) def main() -> int: # ROOT override (house style) lets the same script run against a pulled copy # locally: COOP_PATCH_ROOT=. root = Path(os.environ.get("COOP_PATCH_ROOT", "/app")) candidates = sorted( (root / ".venv/lib").glob("python3*/site-packages/litellm/proxy/pass_through_endpoints/success_handler.py") ) if not candidates: print("WARN passthrough-token-cost: success_handler.py not found under /app/.venv/lib — skipping") return 0 target = candidates[-1] src = target.read_text() if MARKER in src: print("passthrough-token-cost: already applied") return 0 if src.count(OLD) != 1: print(f"WARN passthrough-token-cost: expected exactly 1 anchor, found {src.count(OLD)} — skipping") return 0 patched = src.replace(OLD, NEW, 1) try: compile(patched, str(target), "exec") except SyntaxError as e: print(f"WARN passthrough-token-cost: patched source does not compile ({e}) — skipping") return 0 target.write_text(patched) print(f"[coop-patch] applied: passthrough-token-cost -> {target}") return 0 if __name__ == "__main__": sys.exit(main())