Compare commits
26
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ac9b3b17af | ||
|
|
140b443980 | ||
|
|
fe48ca8e38 | ||
|
|
6fcc116199 | ||
|
|
c8816b58c1 | ||
|
|
718644c363 | ||
|
|
870b165321 | ||
|
|
aa9331cff3 | ||
|
|
6f6bb73d16 | ||
|
|
9c81df48b8 | ||
|
|
d1d342d8e6 | ||
|
|
0f7cf17cd8 | ||
|
|
278900684c | ||
|
|
df4e154128 | ||
|
|
4b6583c3c0 | ||
|
|
2483715ef3 | ||
|
|
f66bedfb55 | ||
|
|
0e8517b6be | ||
|
|
d35ee1760f | ||
|
|
faff547ff4 | ||
|
|
b81177c3ad | ||
|
|
d96df6fd5a | ||
|
|
7a167a8354 | ||
|
|
a1613401c3 | ||
|
|
77e38d81e0 | ||
|
|
d877bc8eee |
No files matched your search
@@ -8,7 +8,7 @@
|
||||
"upstreamVersion": "1.108.3",
|
||||
"healthCheckPath": "/health/readiness",
|
||||
"httpPort": 4000,
|
||||
"memoryLimit": 2147483648,
|
||||
"memoryLimit": 3221225472,
|
||||
"manifestVersion": 2,
|
||||
"website": "https://litellm.ai",
|
||||
"contactEmail": "info@inference.coop",
|
||||
@@ -26,5 +26,5 @@
|
||||
],
|
||||
"mediaLinks": [],
|
||||
"changelog": "Initial package for inference.coop",
|
||||
"dockerImage": "registry.inference.coop/code/litellm:20261114-84a1918"
|
||||
"dockerImage": "registry.inference.coop/code/litellm:20261118-140b443"
|
||||
}
|
||||
+14
@@ -22,6 +22,8 @@ COPY config.yaml /app/data/config.yaml
|
||||
# A second copy inside the image: start.sh syncs it over the persistent
|
||||
# /app/data volume at boot, so config changes in the repo actually deploy.
|
||||
COPY config.yaml /app/code/config.yaml
|
||||
COPY fx_sync.py /app/code/fx_sync.py
|
||||
COPY eur_prices.json /app/code/eur_prices.json
|
||||
|
||||
# Fail-closed config validator (canonical copy lives in ops-docs/scripts/;
|
||||
# this byte-identical twin ships with the image and is called from start.sh
|
||||
@@ -59,6 +61,18 @@ RUN python3 /app/code/patch_audio_request_cost.py
|
||||
COPY patch_audio_router_pricing.py /app/code/patch_audio_router_pricing.py
|
||||
RUN python3 /app/code/patch_audio_router_pricing.py
|
||||
|
||||
# Audio per-request prices must live in the SHARED cost map, which is built from
|
||||
# model_prices_and_context_window.json — the per-request helper reads that map, not
|
||||
# model_info, so a router-only patch left every speech call at $0 (issue #5).
|
||||
# This patch fails the build rather than warning.
|
||||
COPY patch_audio_cost_map.py /app/code/patch_audio_cost_map.py
|
||||
RUN python3 /app/code/patch_audio_cost_map.py
|
||||
|
||||
# Runtime registration: the build-time file edit above did not reach the deployed
|
||||
# gateway's model_cost map (issue #5), so register the entries in-process instead.
|
||||
# sitecustomize is imported before any application code.
|
||||
COPY sitecustomize.py /app/.venv/lib/python3.13/site-packages/sitecustomize.py
|
||||
|
||||
# SECURITY (2026-10-27, member-disclosure finding 4): LiteLLM forwards every
|
||||
# provider response header to callers as `llm_provider-*`, which leaked the
|
||||
# co-op's remaining provider balance (x-credits-remaining) to any member API key.
|
||||
|
||||
+44
-31
@@ -6,7 +6,10 @@
|
||||
# via environment variable TINFOIL_API_KEY.
|
||||
#
|
||||
# Pricing (model_info) is per-token, derived from provider per-1M rates:
|
||||
# greenpt/deepseek-v4.1-flash: $0.22 in / $1.10 out / $0.011 cached per 1M
|
||||
# EUR->USD FX = 1.16 (policy, set 2026-11-18). GreenPT and Regolo quote EUR;
|
||||
# LiteLLM's cost fields are USD-shaped, so EUR list prices are converted here.
|
||||
# Never paste EUR numerals into model_info — that under-bills members by the FX gap.
|
||||
# greenpt/deepseek-v4.1-flash: $0.36 in / $1.45 out / $0.04 cached per 1M (1017 snapshot)
|
||||
# tinfoil/deepseek-v4-1-flash: $0.65 in / $1.45 out / $0.13 cached per 1M
|
||||
# tinfoil/gpt-oss-120b: $0.15 in / $0.60 out per 1M
|
||||
# tinfoil/glm-5.3: $1.80 in / $5.75 out / $0.45 cached per 1M
|
||||
@@ -26,20 +29,25 @@
|
||||
# models.md is the member-facing mirror.
|
||||
|
||||
model_list:
|
||||
# DeepSeek V4.1 Flash on GreenPT — DEFAULT model. Same model family as Tinfoil's
|
||||
# DeepSeek V4.1 Flash on GreenPT — DEFAULT model. SERVES THE 1017 SNAPSHOT
|
||||
# (17 Oct 2026) as of 2026-11-16, while KEEPING its version-free model_name: members
|
||||
# pin this id in client configs we cannot see, and renaming would 400 every one of
|
||||
# them (measured: 3,014 calls in 2 days on this name, 84 on the deepseek-flash-green
|
||||
# alias). Repointing is the whole change — no member action.
|
||||
# Same model family as Tinfoil's
|
||||
# deepseek-v4-1-flash, but on renewable energy and NOT marked Experimental by the
|
||||
# provider (Tinfoil's deepseek + glm-flash carry an "Experimental: may be deprecated
|
||||
# with minimal notice" flag). Policy-based privacy (GreenPT) vs TEE (Tinfoil).
|
||||
# €0.22 in / €1.10 out / €0.011 cached per 1M.
|
||||
# $0.36 in / $1.45 out / $0.04 cached per 1M (1017 snapshot; repointed 2026-11-16).
|
||||
- model_name: greenpt/deepseek-v4.1-flash
|
||||
litellm_params:
|
||||
model: openai/deepseek-v4.1-flash
|
||||
model: openai/deepseek-v4.1-flash-1017
|
||||
api_base: https://api.greenpt.ai/v1
|
||||
api_key: os.environ/GREENPT_API_KEY
|
||||
model_info:
|
||||
input_cost_per_token: 0.00000022
|
||||
output_cost_per_token: 0.00000110
|
||||
cache_read_input_token_cost: 0.000000011
|
||||
input_cost_per_token: 4.176e-07
|
||||
output_cost_per_token: 1.682e-06
|
||||
cache_read_input_token_cost: 4.64e-08
|
||||
|
||||
# DeepSeek V4.1 Flash on Tinfoil (TEE) — private counterpart to the default.
|
||||
- model_name: tinfoil/deepseek-v4-1-flash
|
||||
@@ -88,9 +96,9 @@ model_list:
|
||||
api_base: https://api.greenpt.ai/v1
|
||||
api_key: os.environ/GREENPT_API_KEY
|
||||
model_info:
|
||||
input_cost_per_token: 0.00000011
|
||||
output_cost_per_token: 0.00000044
|
||||
cache_read_input_token_cost: 0.000000022
|
||||
input_cost_per_token: 1.276e-07
|
||||
output_cost_per_token: 5.104e-07
|
||||
cache_read_input_token_cost: 2.552e-08
|
||||
|
||||
# GLM-5.3 (full flagship) on GreenPT — strongest agentic model in the green
|
||||
# tier. $1.10 in / $4.40 out per 1M, 1M context.
|
||||
@@ -100,17 +108,20 @@ model_list:
|
||||
api_base: https://api.greenpt.ai/v1
|
||||
api_key: os.environ/GREENPT_API_KEY
|
||||
model_info:
|
||||
input_cost_per_token: 0.0000011
|
||||
output_cost_per_token: 0.0000044
|
||||
cache_read_input_token_cost: 0.000000275
|
||||
input_cost_per_token: 1.276e-06
|
||||
output_cost_per_token: 5.104e-06
|
||||
cache_read_input_token_cost: 3.19e-07
|
||||
|
||||
# --- Audio models (Tinfoil, inside the TEE) ---
|
||||
# NOT chat models: these power /v1/audio/* routes (STT for dictation, TTS for
|
||||
# voice mode). They stay VISIBLE in the API model list so developer members
|
||||
# can call them, but the portal filters them out of the chat-facing list
|
||||
# (Open WebUI) by their `mode` field, so they never appear as pickable chat
|
||||
# options. Pricing per Tinfoil catalog: whisper $0.05/1M in; voxtral-tts
|
||||
# listed $0/$0 (verify on invoice — may be beta-free).
|
||||
# options. Tinfoil bills audio per REQUEST, not per token: whisper
|
||||
# $0.01/request, voxtral-tts $0.002/request (both $0 per token). LiteLLM cannot
|
||||
# price that natively, so each entry declares input_cost_per_request and the
|
||||
# build-time patches make the calculator honour it (issue #5, closed 2026-11-17
|
||||
# with a verified $0.002 spend row).
|
||||
|
||||
# --- Regolo (EU, zero data retention, 100% renewables) ---
|
||||
# Seeweb S.r.l. (Italy). Added 2026-10-16 following member suggestion (#22).
|
||||
@@ -154,8 +165,8 @@ model_list:
|
||||
api_key: os.environ/REGOLO_API_KEY
|
||||
model_info:
|
||||
mode: chat
|
||||
input_cost_per_token: 2.0e-7
|
||||
output_cost_per_token: 7.0e-7
|
||||
input_cost_per_token: 2.32e-07
|
||||
output_cost_per_token: 8.12e-07
|
||||
|
||||
# Gemma 4 (31B) — Google's dense multimodal Gemma 4. $0.40 in / $0.60 out
|
||||
# per 1M, 100K ctx.
|
||||
@@ -166,8 +177,8 @@ model_list:
|
||||
api_key: os.environ/REGOLO_API_KEY
|
||||
model_info:
|
||||
mode: chat
|
||||
input_cost_per_token: 4.0e-7
|
||||
output_cost_per_token: 6.0e-7
|
||||
input_cost_per_token: 4.64e-07
|
||||
output_cost_per_token: 6.96e-07
|
||||
|
||||
- model_name: regolo/qwen-image
|
||||
litellm_params:
|
||||
@@ -249,7 +260,9 @@ model_list:
|
||||
# branch (cost_per_token, cost_metric == "cost_per_character") raises
|
||||
# ValueError("cost for tts call is None") when there is no character price,
|
||||
# and that raise aborts the whole cost calculation — so the spend write is
|
||||
# skipped and speech bills nothing, whatever input_cost_per_request says.
|
||||
# skipped and speech billed nothing, whatever input_cost_per_request said
|
||||
# (that was issue #5; the build patch now prices the base at 0.0 for a
|
||||
# per-request model, so the declared charge is the whole bill).
|
||||
# Declaring the zero lets the path complete; the patch then adds the
|
||||
# per-request charge.
|
||||
input_cost_per_character: 0
|
||||
@@ -295,8 +308,8 @@ model_list:
|
||||
api_key: os.environ/GREENPT_API_KEY
|
||||
model_info:
|
||||
mode: embedding
|
||||
input_cost_per_token: 0.00000025
|
||||
output_cost_per_token: 0.0
|
||||
input_cost_per_token: 2.9e-07
|
||||
output_cost_per_token: 0
|
||||
max_input_tokens: 32000
|
||||
|
||||
# --- Embedding model (GreenPT, member-facing) ---
|
||||
@@ -316,8 +329,8 @@ model_list:
|
||||
api_key: os.environ/GREENPT_API_KEY
|
||||
model_info:
|
||||
mode: embedding
|
||||
input_cost_per_token: 0.0000002
|
||||
output_cost_per_token: 0.0
|
||||
input_cost_per_token: 2.32e-07
|
||||
output_cost_per_token: 0
|
||||
max_input_tokens: 32000
|
||||
|
||||
# Fallback: if DeepSeek is down, use GPT-OSS
|
||||
@@ -342,13 +355,13 @@ model_list:
|
||||
# them at transitions, so pinning one is what makes a config version-free.
|
||||
- model_name: deepseek-flash-green
|
||||
litellm_params:
|
||||
model: openai/deepseek-v4.1-flash
|
||||
model: openai/deepseek-v4.1-flash-1017
|
||||
api_base: https://api.greenpt.ai/v1
|
||||
api_key: os.environ/GREENPT_API_KEY
|
||||
model_info:
|
||||
input_cost_per_token: 0.00000022
|
||||
output_cost_per_token: 0.00000110
|
||||
cache_read_input_token_cost: 0.000000011
|
||||
input_cost_per_token: 4.176e-07
|
||||
output_cost_per_token: 1.682e-06
|
||||
cache_read_input_token_cost: 4.64e-08
|
||||
|
||||
- model_name: deepseek-flash-tee
|
||||
litellm_params:
|
||||
@@ -366,9 +379,9 @@ model_list:
|
||||
api_base: https://api.greenpt.ai/v1
|
||||
api_key: os.environ/GREENPT_API_KEY
|
||||
model_info:
|
||||
input_cost_per_token: 0.00000011
|
||||
output_cost_per_token: 0.00000044
|
||||
cache_read_input_token_cost: 0.000000022
|
||||
input_cost_per_token: 1.276e-07
|
||||
output_cost_per_token: 5.104e-07
|
||||
cache_read_input_token_cost: 2.552e-08
|
||||
|
||||
# Tinfoil's flash GLM was deprecated 2026-10-09; the TEE route for this family
|
||||
# is the flagship, so the alias is named glm-tee rather than glm-flash-tee.
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"_comment": "EUR list prices per 1M tokens (providers quoting EUR). fx_sync.py converts at the ECB daily rate at boot. USD values in config.yaml are fallbacks.",
|
||||
"models": {
|
||||
"greenpt/deepseek-v4.1-flash": {"input": 0.36, "output": 1.45, "cache": 0.04},
|
||||
"greenpt/glm-5.3-flash": {"input": 0.11, "output": 0.44, "cache": 0.022},
|
||||
"greenpt/glm-5.3": {"input": 1.10, "output": 4.40, "cache": 0.275},
|
||||
"greenpt/green-embeddings": {"input": 0.20, "output": 0.0},
|
||||
"greenpt/qwen3-embedding-8b": {"input": 0.25, "output": 0.0},
|
||||
"deepseek-flash-green": {"input": 0.36, "output": 1.45, "cache": 0.04},
|
||||
"glm-flash-green": {"input": 0.11, "output": 0.44, "cache": 0.022},
|
||||
"regolo/apertus-70b": {"input": 1.00, "output": 3.40},
|
||||
"regolo/qwen3.5-122b": {"input": 0.60, "output": 3.40},
|
||||
"regolo/mistral-small-4-119b": {"input": 0.20, "output": 0.70},
|
||||
"regolo/gemma4-31b": {"input": 0.40, "output": 0.60}
|
||||
}
|
||||
}
|
||||
Executable
+85
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Boot-time FX sync: rewrite USD model_info from the EUR table at the ECB rate.
|
||||
|
||||
Fail-open by design: if the feed is unreachable or unparseable, the config is left
|
||||
untouched (the repo's fallback USD values stand) and the boot continues.
|
||||
|
||||
Run by start.sh before config validation, so the validator sees the final prices.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
|
||||
CFG = "/app/data/config.yaml"
|
||||
EUR_FILE = "/app/code/eur_prices.json"
|
||||
STATE = "/app/data/fx_state.json"
|
||||
ECB_URL = os.environ.get(
|
||||
"ECB_FX_URL", "https://www.ecb.europa.eu/stats/eurofxref/eurofxref-daily.xml")
|
||||
|
||||
|
||||
def fetch_rate() -> tuple[float, str]:
|
||||
with urllib.request.urlopen(ECB_URL, timeout=20) as r:
|
||||
body = r.read().decode("utf-8", "ignore")
|
||||
m = re.search(r"currency=['\"]USD['\"]\s+rate=['\"]([0-9.]+)['\"]", body)
|
||||
if not m:
|
||||
raise ValueError("USD rate not found in ECB feed")
|
||||
d = re.search(r"time=['\"](\d{4}-\d{2}-\d{2})['\"]", body)
|
||||
return float(m.group(1)), (d.group(1) if d else "")
|
||||
|
||||
|
||||
def usd(eur: float, rate: float) -> float:
|
||||
return 0.0 if eur == 0 else eur * rate / 1_000_000
|
||||
|
||||
|
||||
def set_cost(text: str, name: str, key: str, value: float) -> tuple[str, bool]:
|
||||
pat = re.compile(r"( - model_name: " + re.escape(name) +
|
||||
r"\n)(.*?)(?=\n - model_name: |\n[a-zA-Z#]|\Z)", re.S)
|
||||
m = pat.search(text)
|
||||
if not m:
|
||||
return text, False
|
||||
block = m.group(2)
|
||||
new = f"{value:.10g}"
|
||||
block2, n = re.subn(key + r": \S+", key + ": " + new, block, count=1)
|
||||
if n == 0:
|
||||
return text, False
|
||||
return text[:m.start(2)] + block2 + text[m.end(2):], True
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
rate, feed_date = fetch_rate()
|
||||
eur = json.load(open(EUR_FILE, encoding="utf-8"))["models"]
|
||||
text = open(CFG, encoding="utf-8").read()
|
||||
changed = []
|
||||
for name, prices in eur.items():
|
||||
for key, eur_val in (("input_cost_per_token", prices.get("input")),
|
||||
("output_cost_per_token", prices.get("output"))):
|
||||
if eur_val is None:
|
||||
continue
|
||||
text, ok = set_cost(text, name, key, usd(eur_val, rate))
|
||||
if ok:
|
||||
changed.append(f"{name}:{key}")
|
||||
if prices.get("cache") is not None:
|
||||
text, ok = set_cost(text, name, "cache_read_input_token_cost",
|
||||
usd(prices["cache"], rate))
|
||||
if ok:
|
||||
changed.append(f"{name}:cache")
|
||||
tmp = CFG + ".fx.tmp"
|
||||
open(tmp, "w", encoding="utf-8").write(text)
|
||||
os.replace(tmp, CFG)
|
||||
json.dump({"rate": rate, "feed_date": feed_date,
|
||||
"checked_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||
"fields": len(changed)},
|
||||
open(STATE, "w", encoding="utf-8"), indent=2)
|
||||
print(f"[fx-sync] ECB USD/EUR={rate} ({feed_date}); updated {len(changed)} fields")
|
||||
return 0
|
||||
except Exception as exc: # fail-open
|
||||
print(f"[fx-sync] FAIL-OPEN: {type(exc).__name__}: {str(exc)[:160]}; keeping existing prices")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Put Tinfoil's per-request audio prices in the SHARED cost map, at BUILD time.
|
||||
|
||||
Why this exists (issue #5): the cost calculator's per-request helper reads
|
||||
``litellm.model_cost`` — the map built from model_prices_and_context_window.json —
|
||||
and our config's ``input_cost_per_request`` never reaches it. Patching the router
|
||||
to keep per-deployment pricing on the shared model_info entry was not enough,
|
||||
because the helper consults the JSON-backed map, not model_info. Result: every
|
||||
successful voxtral-tts speech call logged cost 0.0 (~$0.22/day of unlogged spend).
|
||||
|
||||
This is the same mechanism as patch_image_cost.py: re-state the price where the
|
||||
calculator actually looks. It FAILS THE BUILD rather than warning, because a silent
|
||||
"billed at $0" is exactly the failure mode we are trying to close.
|
||||
|
||||
Prices from the Tinfoil catalogue (2026-11-17): voxtral-tts $0.002/request,
|
||||
whisper-large-v3-turbo $0.01/request, both $0 per token.
|
||||
"""
|
||||
import glob
|
||||
import json
|
||||
import sys
|
||||
|
||||
ENTRIES = {
|
||||
"openai/voxtral-tts": {
|
||||
"mode": "audio_speech",
|
||||
"input_cost_per_request": 0.002,
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
},
|
||||
"openai/whisper-large-v3-turbo": {
|
||||
"mode": "audio_transcription",
|
||||
"input_cost_per_request": 0.01,
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# LiteLLM ships the local map as *_backup.json; cost.json is the download
|
||||
# cache. Patch whichever exists, preferring the loadable backup.
|
||||
files = (glob.glob("/app/.venv/lib/python*/site-packages/litellm/"
|
||||
"model_prices_and_context_window_backup.json")
|
||||
or glob.glob("/app/.venv/lib/python*/site-packages/litellm/"
|
||||
"model_prices_and_context_window.json")
|
||||
or glob.glob("/app/.venv/lib/python*/site-packages/litellm/cost.json"))
|
||||
if not files:
|
||||
print("[coop-patch] FAIL no local cost map found (backup / model_prices / cost.json)")
|
||||
return 1
|
||||
path = files[0]
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
data = json.load(handle)
|
||||
changed = []
|
||||
for key, entry in ENTRIES.items():
|
||||
current = data.get(key) or {}
|
||||
if all(current.get(k) == v for k, v in entry.items()):
|
||||
continue
|
||||
merged = {**current, **entry}
|
||||
data[key] = merged
|
||||
changed.append(key)
|
||||
if not changed:
|
||||
print(f"[coop-patch] audio cost map already correct ({len(ENTRIES)} entries)")
|
||||
return 0
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
json.dump(data, handle, indent=2)
|
||||
print("[coop-patch] applied audio cost map: " + ", ".join(changed))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -135,12 +135,98 @@ HELPER = '''def _coop_per_request_cost(
|
||||
|
||||
ANCHOR_DEF = "def response_cost_calculator("
|
||||
|
||||
ANCHOR_EARLY = """ if provider_response_cost is not None and not custom_pricing:
|
||||
return provider_response_cost"""
|
||||
|
||||
ANCHOR_RETURN = """ region_name=region_name,
|
||||
)
|
||||
return response_cost
|
||||
except Exception as e:
|
||||
raise e"""
|
||||
|
||||
# The unit-price call. On speech/transcription LiteLLM prices by character,
|
||||
# second or token and raises ValueError when the model declares none of those
|
||||
# rates — which is exactly a per-request model. The declaration is then the
|
||||
# whole bill, so price the base at 0 instead of letting the raise swallow the
|
||||
# spend log write (issue #5: rows recorded spend 0.0 and the tail patch below
|
||||
# never ran because the exception aborted the function first).
|
||||
ANCHOR_COST = """ response_cost = completion_cost(
|
||||
completion_response=response_object,
|
||||
model=model,
|
||||
call_type=call_type,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
optional_params=optional_params,
|
||||
custom_pricing=custom_pricing,
|
||||
base_model=base_model,
|
||||
prompt=prompt,
|
||||
standard_built_in_tools_params=standard_built_in_tools_params,
|
||||
litellm_model_name=litellm_model_name,
|
||||
router_model_id=router_model_id,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
region_name=region_name,
|
||||
)"""
|
||||
|
||||
PATCHED_COST = """ try:
|
||||
response_cost = completion_cost(
|
||||
completion_response=response_object,
|
||||
model=model,
|
||||
call_type=call_type,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
optional_params=optional_params,
|
||||
custom_pricing=custom_pricing,
|
||||
base_model=base_model,
|
||||
prompt=prompt,
|
||||
standard_built_in_tools_params=standard_built_in_tools_params,
|
||||
litellm_model_name=litellm_model_name,
|
||||
router_model_id=router_model_id,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
service_tier=service_tier,
|
||||
data_residency=data_residency,
|
||||
vertex_location=vertex_location,
|
||||
region_name=region_name,
|
||||
)
|
||||
except Exception as _coop_unit_exc:
|
||||
# CO-OP PATCH (audio per-request cost, 2026-11-17): a model billed
|
||||
# per request declares no per-character/second/token rate, so
|
||||
# LiteLLM's unit pricing raises. The per-request charge is the
|
||||
# whole bill; let the tail patch below add it to a zero base.
|
||||
_coop_declared_pr = (
|
||||
_coop_per_request_cost(
|
||||
model=model,
|
||||
litellm_model_name=litellm_model_name,
|
||||
custom_pricing=custom_pricing,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
)
|
||||
if call_type in ("transcription", "atranscription", "speech", "aspeech")
|
||||
else 0.0
|
||||
)
|
||||
if _coop_declared_pr > 0:
|
||||
print(
|
||||
f"[coop-audio] unit pricing unavailable ({type(_coop_unit_exc).__name__}); "
|
||||
f"pricing at declared per-request={_coop_declared_pr!r}",
|
||||
flush=True,
|
||||
)
|
||||
response_cost = 0.0
|
||||
else:
|
||||
raise"""
|
||||
|
||||
PATCHED_EARLY = """ if provider_response_cost is not None and not custom_pricing:
|
||||
# CO-OP PATCH (2026-11-16): a provider-reported cost is not the
|
||||
# whole bill on the audio paths. Speech returns here and never
|
||||
# reaches the tail patch, so add the declared per-request charge
|
||||
# before returning. Guarded by the DECLARATION, not the call
|
||||
# type: models without input_cost_per_request (voxtral-small-24b,
|
||||
# token-priced) stay inert.
|
||||
return provider_response_cost + _coop_per_request_cost(
|
||||
model=model,
|
||||
litellm_model_name=litellm_model_name,
|
||||
custom_pricing=custom_pricing,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
)"""
|
||||
|
||||
PATCHED_RETURN = """ region_name=region_name,
|
||||
)
|
||||
|
||||
@@ -232,6 +318,17 @@ def main() -> int:
|
||||
|
||||
patched = source.replace(ANCHOR_DEF, HELPER + ANCHOR_DEF, 1)
|
||||
patched = patched.replace(ANCHOR_RETURN, PATCHED_RETURN, 1)
|
||||
if ANCHOR_COST in patched:
|
||||
patched = patched.replace(ANCHOR_COST, PATCHED_COST, 1)
|
||||
print("[coop-patch] unit-price call made per-request tolerant (speech/transcription)")
|
||||
else:
|
||||
print("[coop-patch] WARNING unit-price anchor not found; per-request models "
|
||||
"will still raise and bill $0")
|
||||
if ANCHOR_EARLY in patched:
|
||||
patched = patched.replace(ANCHOR_EARLY, PATCHED_EARLY, 1)
|
||||
print("[coop-patch] early provider-cost return also patched (audio speech)")
|
||||
else:
|
||||
print("[coop-patch] WARNING early-return anchor not found; speech may stay unbilled")
|
||||
compile(patched, path, "exec") # refuse to write something that will not import
|
||||
|
||||
if not os.access(path, os.W_OK): # practice run against a read-only copy
|
||||
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail if a EUR-quoted provider's model_info is not EUR * FX.
|
||||
|
||||
GreenPT and Regolo quote EUR; LiteLLM costs are USD. Pasting EUR numerals into
|
||||
model_info under-bills members by the FX gap (found 2026-11-18).
|
||||
"""
|
||||
import re
|
||||
import sys
|
||||
|
||||
FX = 1.16
|
||||
EUR = {
|
||||
"greenpt/deepseek-v4.1-flash": (0.36, 1.45, 0.04),
|
||||
"greenpt/glm-5.3-flash": (0.11, 0.44, 0.022),
|
||||
"greenpt/glm-5.3": (1.10, 4.40, 0.275),
|
||||
"greenpt/green-embeddings": (0.20, 0.0, None),
|
||||
"greenpt/qwen3-embedding-8b": (0.25, 0.0, None),
|
||||
"regolo/mistral-small-4-119b": (0.20, 0.70, None),
|
||||
"regolo/gemma4-31b": (0.40, 0.60, None),
|
||||
# Route aliases carry their own model_info; members call these names, so a
|
||||
# deployment-only conversion leaves billing stale (found 2026-11-18).
|
||||
"deepseek-flash-green": (0.36, 1.45, 0.04),
|
||||
"glm-flash-green": (0.11, 0.44, 0.022),
|
||||
}
|
||||
|
||||
text = open("config.yaml", encoding="utf-8").read()
|
||||
bad = []
|
||||
for name, (i, o, c) in EUR.items():
|
||||
m = re.search(r"- model_name: " + re.escape(name) + r"\n(.*?)(?=\n - model_name: |\n[a-zA-Z#])", text, re.S)
|
||||
if not m:
|
||||
bad.append(f"{name}: missing block")
|
||||
continue
|
||||
block = m.group(1)
|
||||
def val(key):
|
||||
mm = re.search(key + r": (\S+)", block)
|
||||
return float(mm.group(1)) if mm else None
|
||||
checks = [("input_cost_per_token", i), ("output_cost_per_token", o)]
|
||||
if c is not None:
|
||||
checks.append(("cache_read_input_token_cost", c))
|
||||
for key, eur in checks:
|
||||
want = eur * FX / 1_000_000
|
||||
got = val(key)
|
||||
if got is None or abs(got - want) > max(1e-12, want * 1e-6):
|
||||
bad.append(f"{name}: {key} got {got}, want {want:.10g}")
|
||||
print("checked", len(EUR), "EUR models at FX", FX)
|
||||
if bad:
|
||||
print("\n".join(bad))
|
||||
sys.exit(1)
|
||||
print("OK: EUR list prices converted to USD")
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Register Tinfoil's per-request audio prices in litellm.model_cost at STARTUP.
|
||||
|
||||
Build-time edits of model_prices_and_context_window_backup.json did not reach the
|
||||
runtime map (the deployed gateway still had no entry — issue #5). sitecustomize is
|
||||
imported by CPython before any application import, so this runs inside the gateway
|
||||
process itself, before LiteLLM prices a request. Idempotent and never fatal.
|
||||
"""
|
||||
import os
|
||||
|
||||
ENTRIES = {
|
||||
"openai/voxtral-tts": {
|
||||
"mode": "audio_speech",
|
||||
"input_cost_per_request": 0.002,
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
},
|
||||
"openai/whisper-large-v3-turbo": {
|
||||
"mode": "audio_transcription",
|
||||
"input_cost_per_request": 0.01,
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
},
|
||||
}
|
||||
|
||||
try:
|
||||
import litellm
|
||||
|
||||
_cost = getattr(litellm, "model_cost", None)
|
||||
if isinstance(_cost, dict):
|
||||
for _k, _v in ENTRIES.items():
|
||||
_cur = _cost.get(_k) or {}
|
||||
if _cur.get("input_cost_per_request") != _v["input_cost_per_request"]:
|
||||
_cost[_k] = {**_cur, **_v}
|
||||
except Exception: # never break the interpreter for a pricing nicety
|
||||
pass
|
||||
@@ -190,6 +190,13 @@ fi
|
||||
# cannot happen through this path, and the shipped /app/code/config.yaml is
|
||||
# the live catalog.
|
||||
|
||||
# --- Daily FX sync (fail-open): convert EUR-priced providers at the ECB rate ---
|
||||
# Prices are refreshed at boot; a nightly timer restarts the gateway at 04:30 UTC.
|
||||
if [ -f /app/code/fx_sync.py ]; then
|
||||
echo "Refreshing EUR->USD prices from the ECB feed (fail-open)..."
|
||||
python3 /app/code/fx_sync.py || echo "fx_sync exited non-zero; continuing with existing prices"
|
||||
fi
|
||||
|
||||
# --- Fail-closed config validation (2026-11-11) ---
|
||||
# Validate the config that will actually be SERVED, before starting anything.
|
||||
# A bad key must fail LOUD at the top of the boot log (the chat outage of
|
||||
|
||||
Reference in new issue
Block a user