From da02f8eb328d090b0017d0b38b5753381d53c3ac Mon Sep 17 00:00:00 2001 From: inference-bot Date: Tue, 3 Nov 2026 18:58:07 +0000 Subject: [PATCH] gateway: price the Metis pass-through route from its reported tokens The /v1/systemone pass-through is an EndpointType.GENERIC route: LiteLLM logs it, but with no model on the route the only cost it can apply is the flat cost_per_request (a budget estimate, not an invoice). So decisions were billed at a fixed 0.0001 instead of the real $0.12 per 1M input tokens GreenPT reports back in usage.input_tokens. Price the route from its own response body via a new build-time patch, patch_passthrough_token_cost.py, in the house style of the existing patch_*.py files: any route listed in general_settings.coop_passthrough_cost_per_token (USD per input token, keyed by path) is billed as usage.input_tokens x rate. cost_per_request stays as the fallback for a listed route whose response carries no token count, and every unlisted route is unchanged. Auth is untouched. config.yaml: list /v1/systemone at 0.00000012 ($0.12 per 1M input tokens); update the metering note. --- Dockerfile | 11 ++ README.md | 1 + config.yaml | 27 ++++- patch_passthrough_token_cost.py | 188 ++++++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+), 6 deletions(-) create mode 100644 patch_passthrough_token_cost.py diff --git a/Dockerfile b/Dockerfile index f73641e..6c264d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,6 +76,17 @@ RUN python3 /app/code/patch_expose_cost_headers.py COPY patch_passthrough_key_access.py /app/code/patch_passthrough_key_access.py RUN python3 /app/code/patch_passthrough_key_access.py +# Exact per-input-token pricing for generic pass-through routes. A route whose +# target matches none of vertex/anthropic/openai/gemini resolves to +# EndpointType.GENERIC: it IS logged, but LiteLLM has no model to price it from, +# so the only cost it knows is the flat cost_per_request. Metis (/v1/systemone) +# is billed at $0.12 per 1M input tokens and returns usage.input_tokens, so this +# patch prices a route listed in general_settings.coop_passthrough_cost_per_token +# from that count and writes the exact figure to member spend. Non-fatal; check +# the "[coop-patch] applied" line in the build log on every LiteLLM upgrade. +COPY patch_passthrough_token_cost.py /app/code/patch_passthrough_token_cost.py +RUN python3 /app/code/patch_passthrough_token_cost.py + # Audio keep-alive: scheduled pings keep the Tinfoil TEE audio enclave # warm so voice mode skips the ~24s cold-start (see audio_keepalive.py header). COPY audio_keepalive.py /app/code/audio_keepalive.py diff --git a/README.md b/README.md index 13fff76..3e7f081 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,7 @@ expect a 15–20 min maintenance window and **never panic-restart mid-migration* - Check the [LiteLLM changelog](https://github.com/BerriAI/litellm/releases) for breaking changes - The `start.sh` and `config.yaml` may need adjustment if LiteLLM changes its env-var or config schema - **Build-time patches ride on LiteLLM internals — verify each one after an upgrade.** `patch_budget_message.py`, `patch_image_cost.py` (keeps image-gen pricing on the shared cost-map key) and `patch_audio_request_cost.py` (adds the declared per-request charge on the audio paths: Tinfoil bills `voxtral-tts` $0.002 and `whisper-large-v3-turbo` $0.01 per request, which the cost map cannot express). Each prints `[coop-patch]` in the build log. They are non-fatal by design: a missing anchor warns and the build still succeeds, so the behaviour can vanish silently on a version bump. Related: code/litellm#5. +- **`patch_passthrough_token_cost.py`** prices a generic pass-through route from the token count it reports in its own response body. A `pass_through_endpoints` target whose host matches none of vertex/anthropic/openai/gemini resolves to `EndpointType.GENERIC`: it is logged, but with no model on the route LiteLLM only knows the flat `cost_per_request`, which is a budget estimate, not an invoice. The patch bills any route listed in `general_settings.coop_passthrough_cost_per_token` (USD per input token, keyed by path) as `usage.input_tokens × rate` — Metis (`/v1/systemone`) at $0.12 per 1M input tokens. Same rules: non-fatal, `[coop-patch]` in the build log, verify after every upgrade. `cost_per_request` stays as the fallback for a listed route whose response carries no token count. The block prints `[coop-passthrough-cost] input_tokens=… rate=… -> response_cost=…` on each priced call, so the log is the field check that it fired. - The CORS env var is `LITELLM_CORS_ORIGINS` (upstream-native since 1.84.0); the old `LITELLM_CORS_ALLOWED_ORIGINS` + sed patch are dead — don't re-add them - Test on a staging instance if the version jump is large diff --git a/config.yaml b/config.yaml index 4ae3f15..a264535 100644 --- a/config.yaml +++ b/config.yaml @@ -245,12 +245,16 @@ general_settings: # GREENPT_API_KEY the green chat models already use. # Price: $0.12 per 1M *input* tokens; output is free. # - # METERING IS APPROXIMATE. LiteLLM cannot price a pass-through route from the - # response body, so cost_per_request (USD per request) is a flat estimate used - # for budget accounting — 0.0001 ≈ a typical ~800-input-token call. Exact - # per-token billing (reading usage.input_tokens from the Metis response) would - # need a patch in the house style of the existing patch_*.py files; until then - # treat the spend figure for this route as an approximation, not an invoice. + # METERING: exact, from the token count GreenPT reports back. Metis is not a + # model_list entry (LiteLLM's chat pipeline has nothing to parse), so it is + # priced through the coop_passthrough_cost_per_token map below, keyed by this + # route's path: a call's cost is usage.input_tokens × the rate. patch_ + # passthrough_token_cost.py (built in via the Dockerfile) reads + # usage.input_tokens from the Metis response body at logging time and writes + # the exact figure to member spend. cost_per_request on the route is now only + # a fallback: it applies if a response ever arrives without a usable token + # count, so a call is never silently free. Keep the two in sync. + # # Routes here (matched by prefix) may be called by ANY authenticated member # key. LiteLLM's own gate for auth=true pass-through routes # (`allowed_passthrough_routes`) is an Enterprise-only field, so on this @@ -261,6 +265,16 @@ general_settings: # endpoint still requires a valid key). coop_passthrough_open_to_keys: - "/v1/systemone" + # Exact per-input-token pricing (USD per token) for pass-through routes that + # report their own usage in the RESPONSE BODY — the shape the x-litellm-* + # response-header contract does not cover. Keyed by the gateway route path + # (an upstream target URL also matches). Any route listed here is billed from + # usage.input_tokens × the rate instead of its flat cost_per_request; a route + # not listed is unchanged. Consumed at build time by + # patch_passthrough_token_cost.py. + coop_passthrough_cost_per_token: + # Metis (GreenPT): $0.12 per 1M input tokens, output always free. + "/v1/systemone": 0.00000012 pass_through_endpoints: - path: "/v1/systemone" target: "https://api.greenpt.ai/v1/systemone" @@ -270,6 +284,7 @@ general_settings: auth: true methods: ["POST"] timeout: 60 + # Fallback only — the exact token cost above is what bills in practice. cost_per_request: 0.0001 litellm_settings: diff --git a/patch_passthrough_token_cost.py b/patch_passthrough_token_cost.py new file mode 100644 index 0000000..87c456b --- /dev/null +++ b/patch_passthrough_token_cost.py @@ -0,0 +1,188 @@ +"""Price a generic pass-through route from the usage it reports in its OWN body. + +Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal: +if the target file or the anchor is missing — e.g. after an upstream version bump +— it prints a warning and exits 0, so the build and the gateway still succeed. +The build log line ``[coop-patch] applied`` is what confirms it took effect; +check it on every LiteLLM upgrade (see README, upgrade checklist). + +Why: config-declared ``pass_through_endpoints`` with ``auth: true`` are logged +through ``PassThroughEndpointLogging``. A target whose hostname matches none of +vertex / anthropic / openai / gemini resolves to ``EndpointType.GENERIC`` and +takes the generic branch of ``pass_through_async_success_handler`` — it is +logged, but LiteLLM has no model on the route to price, so the only cost it +knows is the flat ``cost_per_request`` from the config entry. That is a whole- +request estimate used for budget accounting, not an invoice: Metis is billed at +$0.12 per 1M *input* tokens and returns the true count in ``usage.input_tokens``, +so the flat figure is off by whatever factor the call differed from the estimate. + +LiteLLM's own supported mechanism for an unpriced pass-through is the +``x-litellm-response-cost`` / ``x-litellm-total-tokens`` response-header +contract (``litellm/proxy/pass_through_endpoints/upstream_usage_headers.py``), +but that is consumed inside this SAME logging path, so it only helps where the +logging path already runs — and GreenPT does not send those headers. Reading the +body, which is already in hand here, needs no extra process in front of the +provider (a sidecar would be a single point of failure for the route itself, not +just for its metering). + +Patch: make ``PassThroughEndpointLogging._set_cost_per_request`` price a route +listed in ``general_settings.coop_passthrough_cost_per_token`` (USD per input +token, keyed by the gateway route path, or by the upstream target) from the +``usage.input_tokens`` in the response body: + + cost = input_tokens * coop_passthrough_cost_per_token[route] + +The rate lives in ``general_settings`` rather than the endpoint dict because the +endpoint model (``PassThroughGenericEndpoint``) drops unknown fields and the +exact-route registration copies only ``cost_per_request`` into the logging +payload — the ``general_settings`` dict is read straight from the proxy module, +the same way ``patch_passthrough_key_access.py`` reads +``coop_passthrough_open_to_keys``. + +The exact figure wins; ``cost_per_request`` stays as the fallback for a listed +route whose body carries no usable input-token count, and for every route not +listed. An upstream that reports its own totals via the x-litellm-* headers still +wins outright (the early return is left intact). Inert unless an operator lists +a route, so it changes nothing for the chat models or the audio paths. + +Upstream: no issue filed; LiteLLM has no per-token pricing for pass-through +routes whose body is not an OpenAI/Anthropic/Gemini shape (as of 1.103.2). +""" +from pathlib import Path +import os +import sys + +MARKER = "coop_passthrough_cost_per_token" + +OLD = ( + " #########################################################\n" + " # Check if cost per request is set\n" + " #########################################################\n" + " if has_upstream_reported_usage(logging_obj):\n" + " return kwargs\n" + "\n" + " if passthrough_logging_payload.get(\"cost_per_request\") is not None:\n" + " kwargs[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n" + " logging_obj.model_call_details[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n" + "\n" + " return kwargs\n" +) + +NEW = ( + " #########################################################\n" + " # Check if cost per request is set\n" + " #########################################################\n" + " if has_upstream_reported_usage(logging_obj):\n" + " return kwargs\n" + "\n" + " # [coop-patch] Exact per-input-token pricing for a generic\n" + " # pass-through route. A route the operator lists in\n" + " # general_settings.coop_passthrough_cost_per_token (USD per input\n" + " # token, keyed by gateway path or upstream target) is priced from the\n" + " # usage it reports in its OWN response body, so spend is the real\n" + " # token count rather than cost_per_request's flat estimate. GreenPT's\n" + " # Metis decision route (/v1/systemone) is the first such route: it\n" + " # returns usage.input_tokens and always 0 output tokens, billed at\n" + " # $0.12 per 1M input tokens. Inert unless a route is listed; a listed\n" + " # route whose body carries no token count falls through to\n" + " # cost_per_request below, and every unlisted route is unchanged.\n" + " try:\n" + " from litellm.proxy.proxy_server import general_settings as _coop_general_settings\n" + "\n" + " _coop_rates = (_coop_general_settings or {}).get(\"coop_passthrough_cost_per_token\") or {}\n" + " if _coop_rates:\n" + " _coop_candidates = set()\n" + " _coop_psr = ((kwargs.get(\"litellm_params\") or {}).get(\"proxy_server_request\") or {})\n" + " _coop_req_url = _coop_psr.get(\"url\")\n" + " if _coop_req_url:\n" + " _coop_candidates.add(str(_coop_req_url))\n" + " _coop_candidates.add(urlparse(str(_coop_req_url)).path)\n" + " _coop_upstream_url = passthrough_logging_payload.get(\"url\")\n" + " if _coop_upstream_url:\n" + " _coop_candidates.add(str(_coop_upstream_url))\n" + " _coop_candidates.add(urlparse(str(_coop_upstream_url)).path)\n" + "\n" + " _coop_rate = None\n" + " for _coop_route in _coop_candidates:\n" + " if _coop_route in _coop_rates:\n" + " _coop_rate = _coop_rates[_coop_route]\n" + " break\n" + " if _coop_rate is None:\n" + " for _coop_key, _coop_val in _coop_rates.items():\n" + " _coop_prefix = str(_coop_key).rstrip(\"/\")\n" + " if any(\n" + " c == _coop_key or (c and c.startswith(_coop_prefix + \"/\"))\n" + " for c in _coop_candidates\n" + " ):\n" + " _coop_rate = _coop_val\n" + " break\n" + "\n" + " _coop_tokens = 0\n" + " _coop_body = passthrough_logging_payload.get(\"response_body\")\n" + " if isinstance(_coop_body, dict):\n" + " _coop_usage = _coop_body.get(\"usage\")\n" + " if isinstance(_coop_usage, dict):\n" + " _coop_tokens = _coop_usage.get(\"input_tokens\") or 0\n" + " if not _coop_tokens:\n" + " _coop_tokens = _coop_body.get(\"input_tokens\") or 0\n" + "\n" + " if _coop_rate is not None and _coop_tokens:\n" + " _coop_cost = float(_coop_tokens) * float(_coop_rate)\n" + " kwargs[\"response_cost\"] = _coop_cost\n" + " logging_obj.model_call_details[\"response_cost\"] = _coop_cost\n" + " print(\n" + " \"[coop-passthrough-cost] input_tokens=%s rate=%s -> response_cost=%s\"\n" + " % (_coop_tokens, _coop_rate, _coop_cost),\n" + " flush=True,\n" + " )\n" + " return kwargs\n" + " except Exception as _coop_exc:\n" + " print(\n" + " \"[coop-passthrough-cost] exact cost failed: %s: %s\"\n" + " % (type(_coop_exc).__name__, _coop_exc),\n" + " flush=True,\n" + " )\n" + "\n" + " if passthrough_logging_payload.get(\"cost_per_request\") is not None:\n" + " kwargs[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n" + " logging_obj.model_call_details[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n" + "\n" + " return kwargs\n" +) + + +def main() -> int: + # ROOT override (house style) lets the same script run against a pulled copy + # locally: COOP_PATCH_ROOT=. + root = Path(os.environ.get("COOP_PATCH_ROOT", "/app")) + candidates = sorted( + (root / ".venv/lib").glob("python3*/site-packages/litellm/proxy/pass_through_endpoints/success_handler.py") + ) + if not candidates: + print("WARN passthrough-token-cost: success_handler.py not found under /app/.venv/lib — skipping") + return 0 + + target = candidates[-1] + src = target.read_text() + if MARKER in src: + print("passthrough-token-cost: already applied") + return 0 + + if src.count(OLD) != 1: + print(f"WARN passthrough-token-cost: expected exactly 1 anchor, found {src.count(OLD)} — skipping") + return 0 + + patched = src.replace(OLD, NEW, 1) + try: + compile(patched, str(target), "exec") + except SyntaxError as e: + print(f"WARN passthrough-token-cost: patched source does not compile ({e}) — skipping") + return 0 + + target.write_text(patched) + print(f"[coop-patch] applied: passthrough-token-cost -> {target}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())