inference-bot
83c543a0ba
Adds x-litellm-model-name, -call-id, -key-spend, -response-duration-ms, -overhead-duration-ms, -callback-duration-ms, -attempted-retries and -attempted-fallbacks to the CORS expose list, on the GM's go-ahead. Renames COST_HEADERS to EXPOSED_HEADERS since the list is no longer only costs. x-litellm-key-spend discloses the caller's own key spend, which is their own data; noted in the thread so it can be dropped if anyone objects.
91 lines
3.7 KiB
Python
91 lines
3.7 KiB
Python
"""Expose LiteLLM's per-request cost headers to cross-origin browser clients.
|
|
|
|
MEMBER REQUEST (co-op/support#39, 2026-10-31). Browser JS calling the gateway from
|
|
another origin cannot read a response header unless the server names it in
|
|
Access-Control-Expose-Headers. LiteLLM sets
|
|
|
|
expose_headers=LITELLM_UI_ALLOW_HEADERS
|
|
|
|
which covers seven names (semantic-filter, guardrail, cache-key, ...) and none of
|
|
the cost ones — so `x-litellm-response-cost*` reads as null in JS even though the
|
|
proxy sends those headers on the response. Upstream closed the request
|
|
(BerriAI/litellm#33123) as stale, so it has to be handled here at the app layer,
|
|
exactly like the other patches in this directory.
|
|
|
|
Additive only. It lets a caller read per-request pricing for requests they made
|
|
with their own API key — numbers already present in the response body for
|
|
non-streaming calls, and already visible in their dashboard. It exposes no prompt,
|
|
key, or account data, and no other member's usage.
|
|
|
|
Run at BUILD time (the runtime rootfs is read-only under Cloudron). Non-fatal, per
|
|
house rule: if upstream moves or rewrites this call, WARN and exit 0 — a missing
|
|
convenience header must never keep the gateway from booting.
|
|
|
|
Regression check: after a deploy, a browser request from an unrelated origin should
|
|
see the cost headers listed in Access-Control-Expose-Headers on the response.
|
|
"""
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
# The header names the proxy can set, taken from the running image rather than
|
|
# guessed: grep -rhoE 'x-litellm-[a-z0-9-]*cost[a-z0-9-]*' over the litellm package.
|
|
EXPOSED_HEADERS = [
|
|
"x-litellm-response-cost",
|
|
"x-litellm-response-cost-input",
|
|
"x-litellm-response-cost-output",
|
|
"x-litellm-response-cost-original",
|
|
"x-litellm-response-cost-discount-amount",
|
|
"x-litellm-response-cost-margin-amount",
|
|
"x-litellm-response-cost-margin-percent",
|
|
"x-litellm-response-cost-cache-creation",
|
|
"x-litellm-response-cost-cache-read",
|
|
"x-litellm-response-cost-reasoning",
|
|
"x-litellm-response-cost-tool-usage",
|
|
"x-litellm-classifier-cost",
|
|
# Telemetry names, requested in co-op/support#39 (2026-10-31): model identity,
|
|
# call id, the caller's own key spend, timings and retry/fallback counts.
|
|
"x-litellm-model-name",
|
|
"x-litellm-call-id",
|
|
"x-litellm-key-spend",
|
|
"x-litellm-response-duration-ms",
|
|
"x-litellm-overhead-duration-ms",
|
|
"x-litellm-callback-duration-ms",
|
|
"x-litellm-attempted-retries",
|
|
"x-litellm-attempted-fallbacks",
|
|
]
|
|
|
|
OLD = " expose_headers=LITELLM_UI_ALLOW_HEADERS,\n"
|
|
NEW = (
|
|
" expose_headers=LITELLM_UI_ALLOW_HEADERS\n"
|
|
" + [\n"
|
|
+ "".join(f' "{h}",\n' for h in EXPOSED_HEADERS)
|
|
+ " ],\n"
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
# Glob rather than pin a Python version: the venv path moves when the base
|
|
# image bumps its interpreter, and a stale path would silently skip the patch.
|
|
candidates = sorted(Path("/app/.venv/lib").glob("python3*/site-packages/litellm/proxy/proxy_server.py"))
|
|
if not candidates:
|
|
print("WARN expose-cost-headers: proxy_server.py not found under /app/.venv/lib — skipping")
|
|
return 0
|
|
|
|
target = candidates[-1]
|
|
src = target.read_text()
|
|
if "x-litellm-response-cost-input" in src:
|
|
print("expose-cost-headers: already applied")
|
|
return 0
|
|
found = src.count(OLD)
|
|
if found != 1:
|
|
print(f"WARN expose-cost-headers: expected exactly 1 CORS middleware call, found {found} — skipping")
|
|
return 0
|
|
|
|
target.write_text(src.replace(OLD, NEW))
|
|
print(f"expose-cost-headers: appended {len(EXPOSED_HEADERS)} cost header names to expose_headers in {target}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|