inference-bot
da02f8eb32
The /v1/systemone pass-through is an EndpointType.GENERIC route: LiteLLM logs it, but with no model on the route the only cost it can apply is the flat cost_per_request (a budget estimate, not an invoice). So decisions were billed at a fixed 0.0001 instead of the real $0.12 per 1M input tokens GreenPT reports back in usage.input_tokens. Price the route from its own response body via a new build-time patch, patch_passthrough_token_cost.py, in the house style of the existing patch_*.py files: any route listed in general_settings.coop_passthrough_cost_per_token (USD per input token, keyed by path) is billed as usage.input_tokens x rate. cost_per_request stays as the fallback for a listed route whose response carries no token count, and every unlisted route is unchanged. Auth is untouched. config.yaml: list /v1/systemone at 0.00000012 ($0.12 per 1M input tokens); update the metering note.
189 lines
9.6 KiB
Python
189 lines
9.6 KiB
Python
"""Price a generic pass-through route from the usage it reports in its OWN body.
|
|
|
|
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal:
|
|
if the target file or the anchor is missing — e.g. after an upstream version bump
|
|
— it prints a warning and exits 0, so the build and the gateway still succeed.
|
|
The build log line ``[coop-patch] applied`` is what confirms it took effect;
|
|
check it on every LiteLLM upgrade (see README, upgrade checklist).
|
|
|
|
Why: config-declared ``pass_through_endpoints`` with ``auth: true`` are logged
|
|
through ``PassThroughEndpointLogging``. A target whose hostname matches none of
|
|
vertex / anthropic / openai / gemini resolves to ``EndpointType.GENERIC`` and
|
|
takes the generic branch of ``pass_through_async_success_handler`` — it is
|
|
logged, but LiteLLM has no model on the route to price, so the only cost it
|
|
knows is the flat ``cost_per_request`` from the config entry. That is a whole-
|
|
request estimate used for budget accounting, not an invoice: Metis is billed at
|
|
$0.12 per 1M *input* tokens and returns the true count in ``usage.input_tokens``,
|
|
so the flat figure is off by whatever factor the call differed from the estimate.
|
|
|
|
LiteLLM's own supported mechanism for an unpriced pass-through is the
|
|
``x-litellm-response-cost`` / ``x-litellm-total-tokens`` response-header
|
|
contract (``litellm/proxy/pass_through_endpoints/upstream_usage_headers.py``),
|
|
but that is consumed inside this SAME logging path, so it only helps where the
|
|
logging path already runs — and GreenPT does not send those headers. Reading the
|
|
body, which is already in hand here, needs no extra process in front of the
|
|
provider (a sidecar would be a single point of failure for the route itself, not
|
|
just for its metering).
|
|
|
|
Patch: make ``PassThroughEndpointLogging._set_cost_per_request`` price a route
|
|
listed in ``general_settings.coop_passthrough_cost_per_token`` (USD per input
|
|
token, keyed by the gateway route path, or by the upstream target) from the
|
|
``usage.input_tokens`` in the response body:
|
|
|
|
cost = input_tokens * coop_passthrough_cost_per_token[route]
|
|
|
|
The rate lives in ``general_settings`` rather than the endpoint dict because the
|
|
endpoint model (``PassThroughGenericEndpoint``) drops unknown fields and the
|
|
exact-route registration copies only ``cost_per_request`` into the logging
|
|
payload — the ``general_settings`` dict is read straight from the proxy module,
|
|
the same way ``patch_passthrough_key_access.py`` reads
|
|
``coop_passthrough_open_to_keys``.
|
|
|
|
The exact figure wins; ``cost_per_request`` stays as the fallback for a listed
|
|
route whose body carries no usable input-token count, and for every route not
|
|
listed. An upstream that reports its own totals via the x-litellm-* headers still
|
|
wins outright (the early return is left intact). Inert unless an operator lists
|
|
a route, so it changes nothing for the chat models or the audio paths.
|
|
|
|
Upstream: no issue filed; LiteLLM has no per-token pricing for pass-through
|
|
routes whose body is not an OpenAI/Anthropic/Gemini shape (as of 1.103.2).
|
|
"""
|
|
from pathlib import Path
|
|
import os
|
|
import sys
|
|
|
|
MARKER = "coop_passthrough_cost_per_token"
|
|
|
|
OLD = (
|
|
" #########################################################\n"
|
|
" # Check if cost per request is set\n"
|
|
" #########################################################\n"
|
|
" if has_upstream_reported_usage(logging_obj):\n"
|
|
" return kwargs\n"
|
|
"\n"
|
|
" if passthrough_logging_payload.get(\"cost_per_request\") is not None:\n"
|
|
" kwargs[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n"
|
|
" logging_obj.model_call_details[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n"
|
|
"\n"
|
|
" return kwargs\n"
|
|
)
|
|
|
|
NEW = (
|
|
" #########################################################\n"
|
|
" # Check if cost per request is set\n"
|
|
" #########################################################\n"
|
|
" if has_upstream_reported_usage(logging_obj):\n"
|
|
" return kwargs\n"
|
|
"\n"
|
|
" # [coop-patch] Exact per-input-token pricing for a generic\n"
|
|
" # pass-through route. A route the operator lists in\n"
|
|
" # general_settings.coop_passthrough_cost_per_token (USD per input\n"
|
|
" # token, keyed by gateway path or upstream target) is priced from the\n"
|
|
" # usage it reports in its OWN response body, so spend is the real\n"
|
|
" # token count rather than cost_per_request's flat estimate. GreenPT's\n"
|
|
" # Metis decision route (/v1/systemone) is the first such route: it\n"
|
|
" # returns usage.input_tokens and always 0 output tokens, billed at\n"
|
|
" # $0.12 per 1M input tokens. Inert unless a route is listed; a listed\n"
|
|
" # route whose body carries no token count falls through to\n"
|
|
" # cost_per_request below, and every unlisted route is unchanged.\n"
|
|
" try:\n"
|
|
" from litellm.proxy.proxy_server import general_settings as _coop_general_settings\n"
|
|
"\n"
|
|
" _coop_rates = (_coop_general_settings or {}).get(\"coop_passthrough_cost_per_token\") or {}\n"
|
|
" if _coop_rates:\n"
|
|
" _coop_candidates = set()\n"
|
|
" _coop_psr = ((kwargs.get(\"litellm_params\") or {}).get(\"proxy_server_request\") or {})\n"
|
|
" _coop_req_url = _coop_psr.get(\"url\")\n"
|
|
" if _coop_req_url:\n"
|
|
" _coop_candidates.add(str(_coop_req_url))\n"
|
|
" _coop_candidates.add(urlparse(str(_coop_req_url)).path)\n"
|
|
" _coop_upstream_url = passthrough_logging_payload.get(\"url\")\n"
|
|
" if _coop_upstream_url:\n"
|
|
" _coop_candidates.add(str(_coop_upstream_url))\n"
|
|
" _coop_candidates.add(urlparse(str(_coop_upstream_url)).path)\n"
|
|
"\n"
|
|
" _coop_rate = None\n"
|
|
" for _coop_route in _coop_candidates:\n"
|
|
" if _coop_route in _coop_rates:\n"
|
|
" _coop_rate = _coop_rates[_coop_route]\n"
|
|
" break\n"
|
|
" if _coop_rate is None:\n"
|
|
" for _coop_key, _coop_val in _coop_rates.items():\n"
|
|
" _coop_prefix = str(_coop_key).rstrip(\"/\")\n"
|
|
" if any(\n"
|
|
" c == _coop_key or (c and c.startswith(_coop_prefix + \"/\"))\n"
|
|
" for c in _coop_candidates\n"
|
|
" ):\n"
|
|
" _coop_rate = _coop_val\n"
|
|
" break\n"
|
|
"\n"
|
|
" _coop_tokens = 0\n"
|
|
" _coop_body = passthrough_logging_payload.get(\"response_body\")\n"
|
|
" if isinstance(_coop_body, dict):\n"
|
|
" _coop_usage = _coop_body.get(\"usage\")\n"
|
|
" if isinstance(_coop_usage, dict):\n"
|
|
" _coop_tokens = _coop_usage.get(\"input_tokens\") or 0\n"
|
|
" if not _coop_tokens:\n"
|
|
" _coop_tokens = _coop_body.get(\"input_tokens\") or 0\n"
|
|
"\n"
|
|
" if _coop_rate is not None and _coop_tokens:\n"
|
|
" _coop_cost = float(_coop_tokens) * float(_coop_rate)\n"
|
|
" kwargs[\"response_cost\"] = _coop_cost\n"
|
|
" logging_obj.model_call_details[\"response_cost\"] = _coop_cost\n"
|
|
" print(\n"
|
|
" \"[coop-passthrough-cost] input_tokens=%s rate=%s -> response_cost=%s\"\n"
|
|
" % (_coop_tokens, _coop_rate, _coop_cost),\n"
|
|
" flush=True,\n"
|
|
" )\n"
|
|
" return kwargs\n"
|
|
" except Exception as _coop_exc:\n"
|
|
" print(\n"
|
|
" \"[coop-passthrough-cost] exact cost failed: %s: %s\"\n"
|
|
" % (type(_coop_exc).__name__, _coop_exc),\n"
|
|
" flush=True,\n"
|
|
" )\n"
|
|
"\n"
|
|
" if passthrough_logging_payload.get(\"cost_per_request\") is not None:\n"
|
|
" kwargs[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n"
|
|
" logging_obj.model_call_details[\"response_cost\"] = passthrough_logging_payload.get(\"cost_per_request\")\n"
|
|
"\n"
|
|
" return kwargs\n"
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
# ROOT override (house style) lets the same script run against a pulled copy
|
|
# locally: COOP_PATCH_ROOT=<dir containing .venv/lib>.
|
|
root = Path(os.environ.get("COOP_PATCH_ROOT", "/app"))
|
|
candidates = sorted(
|
|
(root / ".venv/lib").glob("python3*/site-packages/litellm/proxy/pass_through_endpoints/success_handler.py")
|
|
)
|
|
if not candidates:
|
|
print("WARN passthrough-token-cost: success_handler.py not found under /app/.venv/lib — skipping")
|
|
return 0
|
|
|
|
target = candidates[-1]
|
|
src = target.read_text()
|
|
if MARKER in src:
|
|
print("passthrough-token-cost: already applied")
|
|
return 0
|
|
|
|
if src.count(OLD) != 1:
|
|
print(f"WARN passthrough-token-cost: expected exactly 1 anchor, found {src.count(OLD)} — skipping")
|
|
return 0
|
|
|
|
patched = src.replace(OLD, NEW, 1)
|
|
try:
|
|
compile(patched, str(target), "exec")
|
|
except SyntaxError as e:
|
|
print(f"WARN passthrough-token-cost: patched source does not compile ({e}) — skipping")
|
|
return 0
|
|
|
|
target.write_text(patched)
|
|
print(f"[coop-patch] applied: passthrough-token-cost -> {target}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|