# LiteLLM config for inference.coop
# PRODUCTION config: Tinfoil (TEE-protected inference)
# LiteLLM talks plaintext OpenAI to a local Tinfoil proxy (127.0.0.1:3301),
# which verifies the enclave attestation and encrypts request/response bodies
# with EHBP (HPKE) before forwarding to the Tinfoil enclave. Provider key set
# via environment variable TINFOIL_API_KEY.
#
# Pricing (model_info) is per-token, derived from provider per-1M rates:
#   greenpt/deepseek-v4.1-flash: $0.22 in / $1.10 out / $0.011 cached per 1M
#   tinfoil/deepseek-v4-1-flash: $0.65 in / $1.45 out / $0.13 cached per 1M
#   tinfoil/gpt-oss-120b:        $0.15 in / $0.60 out per 1M
#   tinfoil/glm-5.3:             $1.80 in / $5.75 out / $0.45 cached per 1M
#   greenpt/qwen3-embedding-8b:  $0.25 in / $0 out per 1M (embeddings, 32K ctx)
# Without these, LiteLLM can't price these custom models, so spend shows $0.00
# and the team budget cap is not enforced.
#
# SYNC: the per-model pricing here must mirror the user-facing model list in
# co-op/docs/models.md (and vice-versa). When a rate changes, update BOTH this
# file and models.md — this config is the authoritative source of the numbers;
# models.md is the member-facing mirror.

model_list:
  # DeepSeek V4.1 Flash on GreenPT — DEFAULT model. Same model family as Tinfoil's
  # deepseek-v4-1-flash, but on renewable energy and NOT marked Experimental by the
  # provider (Tinfoil's deepseek + glm-flash carry an "Experimental: may be deprecated
  # with minimal notice" flag). Policy-based privacy (GreenPT) vs TEE (Tinfoil).
  # €0.22 in / €1.10 out / €0.011 cached per 1M.
  - model_name: greenpt/deepseek-v4.1-flash
    litellm_params:
      model: openai/deepseek-v4.1-flash
      api_base: https://api.greenpt.ai/v1
      api_key: os.environ/GREENPT_API_KEY
    model_info:
      input_cost_per_token: 0.00000022
      output_cost_per_token: 0.00000110
      cache_read_input_token_cost: 0.000000011

  # DeepSeek V4.1 Flash on Tinfoil (TEE) — private counterpart to the default.
  - model_name: tinfoil/deepseek-v4-1-flash
    litellm_params:
      model: openai/deepseek-v4-1-flash
      api_base: http://127.0.0.1:3301/v1
      api_key: os.environ/TINFOIL_API_KEY
    model_info:
      input_cost_per_token: 0.00000065
      output_cost_per_token: 0.00000145
      cache_read_input_token_cost: 0.00000013

  # GPT-OSS 120B — lightweight fallback
  - model_name: tinfoil/gpt-oss-120b
    litellm_params:
      model: openai/gpt-oss-120b
      api_base: http://127.0.0.1:3301/v1
      api_key: os.environ/TINFOIL_API_KEY
    model_info:
      input_cost_per_token: 0.00000015
      output_cost_per_token: 0.00000060

  # GLM-5.3 (full flagship) on Tinfoil (TEE) — replaced tinfoil/glm-5-3-flash
  # (Tinfoil deprecated the flash variant 2026-10-09). $1.80 in / $5.75 out /
  # $0.45 cached per 1M. Text-only (no vision), reasoning + tool calling.
  - model_name: tinfoil/glm-5.3
    litellm_params:
      model: openai/glm-5-3
      api_base: http://127.0.0.1:3301/v1
      api_key: os.environ/TINFOIL_API_KEY
    model_info:
      input_cost_per_token: 0.0000018
      output_cost_per_token: 0.00000575
      cache_read_input_token_cost: 0.00000045

  # --- GreenPT (renewable energy) ---
  # OpenAI-compatible, EU-hosted on 100% renewable energy. No TEE proxy needed —
  # LiteLLM talks to GreenPT directly.
  # (Removed 2026-10: green-l/green-r — GreenPT now marks them "not recommended".)

  # GLM-5.3 Flash on GreenPT (note the dotted slug — distinct from Tinfoil's
  # glm-5-3-flash). $0.11 in / $0.44 out / $0.022 cached per 1M.
  - model_name: greenpt/glm-5.3-flash
    litellm_params:
      model: openai/glm-5.3-flash
      api_base: https://api.greenpt.ai/v1
      api_key: os.environ/GREENPT_API_KEY
    model_info:
      input_cost_per_token: 0.00000011
      output_cost_per_token: 0.00000044
      cache_read_input_token_cost: 0.000000022

  # GLM-5.3 (full flagship) on GreenPT — strongest agentic model in the green
  # tier. $1.10 in / $4.40 out per 1M, 1M context.
  - model_name: greenpt/glm-5.3
    litellm_params:
      model: openai/glm-5.3
      api_base: https://api.greenpt.ai/v1
      api_key: os.environ/GREENPT_API_KEY
    model_info:
      input_cost_per_token: 0.0000011
      output_cost_per_token: 0.0000044
      cache_read_input_token_cost: 0.000000275

  # --- Audio models (Tinfoil, inside the TEE) ---
  # NOT chat models: these power /v1/audio/* routes (STT for dictation, TTS for
  # voice mode). They stay VISIBLE in the API model list so developer members
  # can call them, but the portal filters them out of the chat-facing list
  # (Open WebUI) by their `mode` field, so they never appear as pickable chat
  # options. Pricing per Tinfoil catalog: whisper $0.05/1M in; voxtral-tts
  # listed $0/$0 (verify on invoice — may be beta-free).

  # --- Regolo (EU, zero data retention, 100% renewables) ---
  # Seeweb S.r.l. (Italy). Added 2026-10-16 following member suggestion (#22).
  # apertus-70b: replaced PublicAI (removed 2026-10-16) as our Apertus provider —
  #   same open-weights model, ~45% cheaper input, EU ZDR + renewables.
  # qwen3.5-122b: large vision+reasoning MoE (262K ctx). REQUIRES
  #   chat_template_kwargs: {enable_thinking: false} — otherwise the reasoning
  #   phase consumes the entire token budget and content comes back empty.
  # Qwen-Image: image generation via /v1/images/generations, billed per pixel.
  - model_name: regolo/apertus-70b
    litellm_params:
      model: openai/apertus-70b
      api_base: https://api.regolo.ai/v1
      api_key: os.environ/REGOLO_API_KEY
    model_info:
      mode: chat
      input_cost_per_token: 1.16e-6
      output_cost_per_token: 3.944e-6

  - model_name: regolo/qwen3.5-122b
    litellm_params:
      model: openai/qwen3.5-122b
      api_base: https://api.regolo.ai/v1
      api_key: os.environ/REGOLO_API_KEY
      chat_template_kwargs: {"enable_thinking": false}
    model_info:
      mode: chat
      input_cost_per_token: 6.96e-7
      output_cost_per_token: 3.944e-6

  - model_name: regolo/qwen-image
    litellm_params:
      model: openai/Qwen-Image
      api_base: https://api.regolo.ai/v1
      api_key: os.environ/REGOLO_API_KEY
    model_info:
      mode: image_generation
      input_cost_per_pixel: 1.74e-8

  # --- Audio models (Tinfoil, in-TEE) ---
  # Used for chat dictation (STT) and voice mode (TTS). These are NOT chat
  # models: the member portal's /v1/models filters them out of the chat model
  # list (see AUDIO_MODEL_PREFIXES there) so members don't see them in the
  # picker, but they remain fully callable via /v1/audio/* for API users.
  # whisper-large-v3-turbo: $0/1M tokens + $0.01 per request.
  # voxtral-tts: $0/1M tokens + $0.002 per request.
  # Tinfoil bills these per REQUEST (its model list carries a `requestPrice`);
  # reading only the two token prices made TTS look free and left STT billed off
  # an older catalog. LiteLLM cannot price per request on the audio paths, so
  # patch_audio_request_cost.py (built in via the Dockerfile) makes the cost
  # calculator honour `input_cost_per_request` below. Re-check that build log
  # line after every LiteLLM upgrade.
  # NOTE: LiteLLM routes audio by the openai/ provider + the endpoint called
  # (/v1/audio/transcriptions or /v1/audio/speech) — there is no
  # "audio_transcription/" provider prefix.
  - model_name: tinfoil/whisper-large-v3-turbo
    litellm_params:
      model: openai/whisper-large-v3-turbo
      api_base: http://127.0.0.1:3301/v1
      api_key: os.environ/TINFOIL_API_KEY
    model_info:
      mode: audio_transcription
      # Tinfoil: $0 per token, $0.01 per request. The per-request charge is the
      # whole cost, and the patch is what makes it billable.
      input_cost_per_token: 0
      input_cost_per_request: 0.01

  - model_name: tinfoil/voxtral-tts
    litellm_params:
      model: openai/voxtral-tts
      api_base: http://127.0.0.1:3301/v1
      api_key: os.environ/TINFOIL_API_KEY
    model_info:
      mode: audio_speech
      # Tinfoil: $0 per token, $0.002 per request — see the audio section note.
      #
      # input_cost_per_character: 0 is load-bearing, not decoration. The TTS cost
      # branch (cost_per_token, cost_metric == "cost_per_character") raises
      # ValueError("cost for tts call is None") when there is no character price,
      # and that raise aborts the whole cost calculation — so the spend write is
      # skipped and speech bills nothing, whatever input_cost_per_request says.
      # Declaring the zero lets the path complete; the patch then adds the
      # per-request charge.
      input_cost_per_character: 0
      input_cost_per_request: 0.002

  # --- Embedding model (Tinfoil, in-TEE) ---
  # nomic-embed-text — 768-dim text embeddings for RAG/semantic search.
  # NOT a chat model: it powers /v1/embeddings only. Scoped to the admin
  # account's team (model_info.team_id) so member keys cannot call it — it is
  # an operator resource for the co-op's own docs-RAG, not a member-facing
  # model. The portal also filters it out of the chat-facing /v1/models list by
  # its `mode: embedding`. Pricing per Tinfoil catalog: $0.05/1M input, $0 out.
  - model_name: tinfoil/nomic-embed-text
    litellm_params:
      model: openai/nomic-embed-text
      api_base: http://127.0.0.1:3301/v1
      api_key: os.environ/TINFOIL_API_KEY
    model_info:
      mode: embedding
      team_id: cdfdf422-633f-4432-8893-cffee82a8325
      input_cost_per_token: 0.00000005
      output_cost_per_token: 0.0

  # --- Embedding model (GreenPT, member-facing) ---
  # qwen3-embedding-8b — text embeddings via /v1/embeddings, callable by any
  # member key (NOT team-scoped, unlike nomic above). Added 2026-11-06 after a
  # member asked for an embedding model: the private (Tinfoil) embedding is an
  # operator-only docs-RAG resource, so this fills the member-facing gap on
  # renewable EU hosting. The portal filters mode:embedding out of the chat
  # picker, so it is API-only — same shape as the audio models above.
  # Pricing per GreenPT /v1/pricing: $0.25/1M input, no output charge
  # (embeddings return vectors; GreenPT lists maxOutputTokens null), 32K
  # context. max_input_tokens declared from the provider's contextWindowTokens
  # so clients don't invent a larger one (the null-caps defect).
  - model_name: greenpt/qwen3-embedding-8b
    litellm_params:
      model: openai/qwen3-embedding-8b
      api_base: https://api.greenpt.ai/v1
      api_key: os.environ/GREENPT_API_KEY
    model_info:
      mode: embedding
      input_cost_per_token: 0.00000025
      output_cost_per_token: 0.0
      max_input_tokens: 32000

# Fallback: if DeepSeek is down, use GPT-OSS
# NOTE: DeepSeek V4.1 Flash is a reasoning model — generations routinely run
# longer than 30s. A 30s timeout caused LiteLLM to abandon in-flight DeepSeek
# requests (falling back to GPT-OSS), while Tinfoil still completed and billed
# the abandoned generation. Result: Tinfoil billed ~2x the tokens LiteLLM logged.
# Fix: a generous timeout (180s) so generations complete inside LiteLLM and get
# logged; and a retry_policy that retries transient connection errors (base
# num_retries) but NEVER retries timeouts (TimeoutErrorRetries: 0) — a timed-out
# generation is already billed by Tinfoil, so re-hitting it re-bills.
router_settings:
  num_retries: 2
  timeout: 180
  retry_policy:
    TimeoutErrorRetries: 0
  fallbacks:
    - greenpt/deepseek-v4.1-flash:
        - tinfoil/deepseek-v4-1-flash
    - tinfoil/deepseek-v4-1-flash:
        - tinfoil/gpt-oss-120b

general_settings:
  master_key: os.environ/LITELLM_MASTER_KEY
  database_url: os.environ/DATABASE_URL
  store_model_in_db: true

  # --- Metis (GreenPT) — decision model, not a chat model ---
  # Metis is a *decision* endpoint, not a chat completion: it takes a `state`
  # plus up to 32 typed questions and returns PROBABILITIES / choices / scores.
  # It generates no text (output_tokens is always 0), so it cannot be a
  # model_list entry — LiteLLM's chat pipeline has nothing to parse. It is
  # exposed as a pass-through route instead, and members call it with their
  # normal co-op API key (auth: true), never the provider key.
  #
  # GreenPT-hosted (policy-based privacy, EU, 100% renewable), same
  # GREENPT_API_KEY the green chat models already use.
  #   Price: $0.12 per 1M *input* tokens; output is free.
  #
  # METERING: exact, from the token count GreenPT reports back. Metis is not a
  # model_list entry (LiteLLM's chat pipeline has nothing to parse), so it is
  # priced through the coop_passthrough_cost_per_token map below, keyed by this
  # route's path: a call's cost is usage.input_tokens × the rate. patch_
  # passthrough_token_cost.py (built in via the Dockerfile) reads
  # usage.input_tokens from the Metis response body at logging time and writes
  # the exact figure to member spend. cost_per_request on the route is now only
  # a fallback: it applies if a response ever arrives without a usable token
  # count, so a call is never silently free. Keep the two in sync.
  #
  # Routes here (matched by prefix) may be called by ANY authenticated member
  # key. LiteLLM's own gate for auth=true pass-through routes
  # (`allowed_passthrough_routes`) is an Enterprise-only field, so on this
  # community-edition proxy it can never be granted to member keys — the only
  # other option, auth:false, would expose the route publicly and forward the
  # co-op's GreenPT key to anonymous callers. This list is consumed at build
  # time by patch_passthrough_key_access.py; authentication is unchanged (the
  # endpoint still requires a valid key).
  coop_passthrough_open_to_keys:
    - "/v1/systemone"
  # Exact per-input-token pricing (USD per token) for pass-through routes that
  # report their own usage in the RESPONSE BODY — the shape the x-litellm-*
  # response-header contract does not cover. Keyed by the gateway route path
  # (an upstream target URL also matches). Any route listed here is billed from
  # usage.input_tokens × the rate instead of its flat cost_per_request; a route
  # not listed is unchanged. Consumed at build time by
  # patch_passthrough_token_cost.py.
  coop_passthrough_cost_per_token:
    # Metis (GreenPT): $0.12 per 1M input tokens, output always free.
    "/v1/systemone": 0.00000012
  pass_through_endpoints:
    - path: "/v1/systemone"
      target: "https://api.greenpt.ai/v1/systemone"
      headers:
        Authorization: "Bearer os.environ/GREENPT_API_KEY"
        Content-Type: "application/json"
      auth: true
      methods: ["POST"]
      timeout: 60
      # Fallback only — the exact token cost above is what bills in practice.
      cost_per_request: 0.0001

litellm_settings:
  salt_key: os.environ/LITELLM_SALT_KEY
  drop_params: true
  num_threads: 4
  request_timeout: 180