litellm/start.sh

#!/bin/bash
set -eu

# ============================================================================
# LiteLLM Cloudron Startup Script
# Configures LiteLLM with Cloudron's PostgreSQL and Redis addons
# ============================================================================

echo "=== LiteLLM Cloudron Startup ==="

# --- Cloudron PostgreSQL addon provides:
#   CLOUDRON_POSTGRESQL_URL - postgresql://user:pass@host:port/dbname
#   CLOUDRON_POSTGRESQL_USERNAME
#   CLOUDRON_POSTGRESQL_PASSWORD
#   CLOUDRON_POSTGRESQL_HOST
#   CLOUDRON_POSTGRESQL_PORT
#   CLOUDRON_POSTGRESQL_DATABASE

# --- Cloudron Redis addon provides:
#   CLOUDRON_REDIS_URL - redis://user:pass@host:port
#   CLOUDRON_REDIS_USERNAME
#   CLOUDRON_REDIS_PASSWORD
#   CLOUDRON_REDIS_HOST
#   CLOUDRON_REDIS_PORT

# --- Cloudron provides:
#   CLOUDRON_APP_DOMAIN - the app's domain (e.g. gateway.inference.coop)
#   CLOUDRON_API_ORIGIN - origin for API calls
#   CLOUDRON_WEB_ORIGIN - origin for web UI

# --- LiteLLM required env vars ---

# Generate a master key if not already in /app/data
MASTER_KEY_FILE="/app/data/.master_key"
if [ ! -f "$MASTER_KEY_FILE" ]; then
    echo "Generating master key..."
    python3 -c "import secrets; print('sk-' + secrets.token_urlsafe(32))" > "$MASTER_KEY_FILE"
fi
export LITELLM_MASTER_KEY=$(cat "$MASTER_KEY_FILE")
# Regolo API key (member-suggested provider, 2026-10-16): stored in /app/data by the operator
# via the Cloudron file manager — kept out of app env config and git.
if [ -f "/app/data/regolo_key.txt" ]; then
  export REGOLO_API_KEY=$(cat /app/data/regolo_key.txt | tr -d '[:space:]')
fi

# Generate a salt key if not already in /app/data
SALT_KEY_FILE="/app/data/.salt_key"
if [ ! -f "$SALT_KEY_FILE" ]; then
    echo "Generating salt key..."
    python3 -c "import secrets; print(secrets.token_urlsafe(32))" > "$SALT_KEY_FILE"
fi
export LITELLM_SALT_KEY=$(cat "$SALT_KEY_FILE")

# Use Cloudron's PostgreSQL
export DATABASE_URL="${CLOUDRON_POSTGRESQL_URL}"

# Use Cloudron's Redis (for rate limiting, caching)
export REDIS_HOST="${CLOUDRON_REDIS_HOST}"
export REDIS_PORT="${CLOUDRON_REDIS_PORT}"
export REDIS_PASSWORD="${CLOUDRON_REDIS_PASSWORD}"

# Store models in DB (manage from Admin UI)
export STORE_MODEL_IN_DB="True"

# Don't run schema migrations on every start — let LiteLLM handle it
# (for single-instance deployment this is fine)
export DISABLE_SCHEMA_UPDATE="false"

# The DB was created by `prisma db push` (no _prisma_migrations ledger), so on
# first boot after a version change LiteLLM hits P3005 ("schema is not empty")
# and must baseline the existing schema. That baseline write needs a WRITABLE
# directory — the package dir is read-only under Cloudron's rootfs, so point it
# at the persistent volume. Without this the boot migration crash-loops.
export LITELLM_MIGRATION_DIR="/app/data/migrations"

# UI configuration — use Cloudron's domain
export UI_USERNAME="admin"
export UI_PASSWORD="${LITELLM_MASTER_KEY}"

# Nothing disables the static /ui admin bundle on the pinned v1.103.2 (there
# is no flag for it): DISABLE_ADMIN_UI="True" only turns off the UI's SSO
# login-redirect routes upstream — GET /ui/ still serves the static login
# page. Logging into it still requires the master key, and the docs surfaces
# below ARE fully disabled (NO_DOCS/NO_REDOC/NO_OPENAPI → 404s). See README
# "Admin access" for the wire-verified state and the patch needed to remove
# the bundle itself. Kept set because the SSO-redirect disable is still real.
export DISABLE_ADMIN_UI="True"
export NO_DOCS="True"
export NO_REDOC="True"
export NO_OPENAPI="True"

# Proxy settings for Cloudron's reverse proxy
export LITELLM_PROXY_BASE_URL="https://${CLOUDRON_APP_DOMAIN}"

# CORS: LiteLLM reads LITELLM_CORS_ORIGINS natively (upstream replaced the
# hardcoded `origins = ["*"]` with this env var in v1.84.0). The value is
# deliberately OVERRIDABLE — a Cloudron app env now wins, where before this
# export silently overrode it and made the app env look effective when it was not.
#
# The co-op decided to allow all origins (co-op/support#38, agreed on Loomio and
# by a member): CORS is not an auth boundary here. Every request carries a
# per-member bearer key, which the browser never attaches implicitly, so an origin
# list buys nothing against a party that already holds a key — and a leaked key is
# curl-able regardless. What actually constrains abuse is the per-key budget,
# model scoping and revocation. The fallback keeps the old chat-only lock if the
# env var is ever unset.
export LITELLM_CORS_ORIGINS="${LITELLM_CORS_ORIGINS:-https://chat.inference.coop}"

echo "PostgreSQL: ${CLOUDRON_POSTGRESQL_HOST}:${CLOUDRON_POSTGRESQL_PORT}"
echo "Redis: ${CLOUDRON_REDIS_HOST}:${CLOUDRON_REDIS_PORT}"
echo "Domain: ${CLOUDRON_APP_DOMAIN}"

# --- Write config.yaml with Cloudron provider keys if set ---

CONFIG_FILE="/app/data/config.yaml"

if [ ! -f "$CONFIG_FILE" ] || [ ! -s "$CONFIG_FILE" ]; then
    echo "Writing default config.yaml..."
    cat > "$CONFIG_FILE" << 'YAML'
# LiteLLM config for inference.coop
# MVP config: Tinfoil only (TEE-protected, privacy-first)
# Two models: DeepSeek V4 Flash (default, best agents) + GPT-OSS 120B (cheapest)
# Provider key set via environment variable or Admin UI

model_list:
  # DeepSeek V4 Flash — default model
  - model_name: deepseek-v4-flash
    litellm_params:
      model: openai/deepseek-v4-flash
      api_base: https://api.tinfoil.sh/v1
      api_key: os.environ/TINFOIL_API_KEY

  # GPT-OSS 120B — lightweight fallback
  - model_name: gpt-oss-120b
    litellm_params:
      model: openai/gpt-oss-120b
      api_base: https://api.tinfoil.sh/v1
      api_key: os.environ/TINFOIL_API_KEY

# Fallback: if DeepSeek is down, use GPT-OSS
router_settings:
  num_retries: 2
  timeout: 30
  fallbacks:
    - deepseek-v4-flash:
        - gpt-oss-120b

general_settings:
  master_key: os.environ/LITELLM_MASTER_KEY
  database_url: os.environ/DATABASE_URL
  store_model_in_db: true

litellm_settings:
  salt_key: os.environ/LITELLM_SALT_KEY
  drop_params: true
  num_threads: 4
  request_timeout: 30
YAML
    echo "Config written to $CONFIG_FILE"
fi

# --- Ship config updates with the image ---
# /app/data is a persistent volume, so the copy the Dockerfile bakes into
# /app/data/config.yaml is shadowed by the volume's file forever — config
# changes in the repo would never reach a running gateway. The repo is the
# source of truth (README: repos mirror production), so sync the shipped copy
# over the volume whenever they differ.
if [ -f /app/code/config.yaml ] && ! cmp -s /app/code/config.yaml "$CONFIG_FILE"; then
    echo "Updating /app/data/config.yaml from the image (repo is the source of truth)"
    cp /app/code/config.yaml "$CONFIG_FILE"
fi
# NOTE on the fallback config written above: it can never be SERVED. The very
# same boot's sync above overwrites it with the image's (repo) config whenever
# they differ — including the boot right after the fallback ran. It is dead
# by construction; it exists only so the container is never left with an
# empty config file if the volume copy vanishes. Historically this block
# carried stale api.tinfoil.sh models that could have resurrected — that
# cannot happen through this path, and the shipped /app/code/config.yaml is
# the live catalog.

# --- Start LiteLLM ---
echo "Starting LiteLLM on port 4000..."

# Start the Tinfoil proxy as a local sidecar. It verifies the enclave
# attestation and encrypts request/response bodies with EHBP (HPKE), so
# LiteLLM can talk plaintext OpenAI to it while the proxy handles the
# end-to-end encryption to the Tinfoil enclave.
#
# The proxy binary is downloaded at runtime into /app/data (persistent,
# writable) because Cloudron's build sandbox has no outbound network access to
# GitHub. The running container does, so we fetch it here on first start.
TINFOIL_PROXY_BIN="/app/data/tinfoil-proxy"
TINFOIL_PROXY_VERSION="v0.2.3"
if [ ! -f "$TINFOIL_PROXY_BIN" ]; then
    echo "Downloading Tinfoil proxy ${TINFOIL_PROXY_VERSION}..."
    python3 -c "import urllib.request; urllib.request.urlretrieve('https://github.com/tinfoilsh/tinfoil-proxy/releases/download/${TINFOIL_PROXY_VERSION}/tinfoil-proxy-linux-amd64', '${TINFOIL_PROXY_BIN}')"
    chmod +x "$TINFOIL_PROXY_BIN"
fi

echo "Starting Tinfoil proxy on 127.0.0.1:3301..."
# Kill any stale proxy from a previous start attempt first — on process-level
# restarts (container NOT recreated) the old proxy still holds 127.0.0.1:3301
# and the new one dies with "bind: address already in use", leaving an
# unmanaged listener running pre-update code/flags.
pkill -f "tinfoil-proxy" 2>/dev/null || true
sleep 1
"$TINFOIL_PROXY_BIN" -b 127.0.0.1 -p 3301 &
TINFOIL_PROXY_PID=$!
echo "Tinfoil proxy PID: $TINFOIL_PROXY_PID"

# Give the proxy a moment to verify the enclave attestation, and FAIL HARD if
# it didn't come up — LiteLLM routing to a dead 3301 breaks every tinfoil
# request, and a silent continue makes that look like a provider outage.
proxy_up=0
for _ in $(seq 1 10); do
    if kill -0 "$TINFOIL_PROXY_PID" 2>/dev/null && \
       (exec 3<>"/dev/tcp/127.0.0.1/3301") 2>/dev/null; then
        exec 3>&- 3<&- || true
        proxy_up=1
        break
    fi
    sleep 1
done
if [ "$proxy_up" != "1" ]; then
    echo "FATAL: tinfoil-proxy did not bind 127.0.0.1:3301" >&2
    exit 1
fi
echo "Tinfoil proxy is up on 127.0.0.1:3301"

exec litellm --config "$CONFIG_FILE" --port 4000 --host 0.0.0.0