213 lines
8.4 KiB
Bash
213 lines
8.4 KiB
Bash
#!/bin/bash
|
|
set -eu
|
|
|
|
# ============================================================================
|
|
# LiteLLM Cloudron Startup Script
|
|
# Configures LiteLLM with Cloudron's PostgreSQL and Redis addons
|
|
# ============================================================================
|
|
|
|
echo "=== LiteLLM Cloudron Startup ==="
|
|
|
|
# --- Cloudron PostgreSQL addon provides:
|
|
# CLOUDRON_POSTGRESQL_URL - postgresql://user:pass@host:port/dbname
|
|
# CLOUDRON_POSTGRESQL_USERNAME
|
|
# CLOUDRON_POSTGRESQL_PASSWORD
|
|
# CLOUDRON_POSTGRESQL_HOST
|
|
# CLOUDRON_POSTGRESQL_PORT
|
|
# CLOUDRON_POSTGRESQL_DATABASE
|
|
|
|
# --- Cloudron Redis addon provides:
|
|
# CLOUDRON_REDIS_URL - redis://user:pass@host:port
|
|
# CLOUDRON_REDIS_USERNAME
|
|
# CLOUDRON_REDIS_PASSWORD
|
|
# CLOUDRON_REDIS_HOST
|
|
# CLOUDRON_REDIS_PORT
|
|
|
|
# --- Cloudron provides:
|
|
# CLOUDRON_APP_DOMAIN - the app's domain (e.g. gateway.inference.coop)
|
|
# CLOUDRON_API_ORIGIN - origin for API calls
|
|
# CLOUDRON_WEB_ORIGIN - origin for web UI
|
|
|
|
# --- LiteLLM required env vars ---
|
|
|
|
# Generate a master key if not already in /app/data
|
|
MASTER_KEY_FILE="/app/data/.master_key"
|
|
if [ ! -f "$MASTER_KEY_FILE" ]; then
|
|
echo "Generating master key..."
|
|
python3 -c "import secrets; print('sk-' + secrets.token_urlsafe(32))" > "$MASTER_KEY_FILE"
|
|
fi
|
|
export LITELLM_MASTER_KEY=$(cat "$MASTER_KEY_FILE")
|
|
|
|
# Generate a salt key if not already in /app/data
|
|
SALT_KEY_FILE="/app/data/.salt_key"
|
|
if [ ! -f "$SALT_KEY_FILE" ]; then
|
|
echo "Generating salt key..."
|
|
python3 -c "import secrets; print(secrets.token_urlsafe(32))" > "$SALT_KEY_FILE"
|
|
fi
|
|
export LITELLM_SALT_KEY=$(cat "$SALT_KEY_FILE")
|
|
|
|
# Use Cloudron's PostgreSQL
|
|
export DATABASE_URL="${CLOUDRON_POSTGRESQL_URL}"
|
|
|
|
# Use Cloudron's Redis (for rate limiting, caching)
|
|
export REDIS_HOST="${CLOUDRON_REDIS_HOST}"
|
|
export REDIS_PORT="${CLOUDRON_REDIS_PORT}"
|
|
export REDIS_PASSWORD="${CLOUDRON_REDIS_PASSWORD}"
|
|
|
|
# Store models in DB (manage from Admin UI)
|
|
export STORE_MODEL_IN_DB="True"
|
|
|
|
# Don't run schema migrations on every start — let LiteLLM handle it
|
|
# (for single-instance deployment this is fine)
|
|
export DISABLE_SCHEMA_UPDATE="false"
|
|
|
|
# The DB was created by `prisma db push` (no _prisma_migrations ledger), so on
|
|
# first boot after a version change LiteLLM hits P3005 ("schema is not empty")
|
|
# and must baseline the existing schema. That baseline write needs a WRITABLE
|
|
# directory — the package dir is read-only under Cloudron's rootfs, so point it
|
|
# at the persistent volume. Without this the boot migration crash-loops.
|
|
export LITELLM_MIGRATION_DIR="/app/data/migrations"
|
|
|
|
# UI configuration — use Cloudron's domain
|
|
export UI_USERNAME="admin"
|
|
export UI_PASSWORD="${LITELLM_MASTER_KEY}"
|
|
|
|
# DISABLE the admin UI and API docs. The gateway must stay publicly reachable
|
|
# for the /v1 API (member keys), but we don't want to advertise a login-gated
|
|
# admin console (/ui) or LiteLLM's Swagger/ReDoc/openapi surfaces. Admin
|
|
# visibility comes from the admin dashboard / the API directly.
|
|
export DISABLE_ADMIN_UI="True"
|
|
export NO_DOCS="True"
|
|
export NO_REDOC="True"
|
|
export NO_OPENAPI="True"
|
|
|
|
# Proxy settings for Cloudron's reverse proxy
|
|
export LITELLM_PROXY_BASE_URL="https://${CLOUDRON_APP_DOMAIN}"
|
|
|
|
# CORS: LiteLLM v1.84.0 reads LITELLM_CORS_ORIGINS natively (upstream added
|
|
# the env var in place of the old hardcoded `origins = ["*"]`). Lock browser
|
|
# access to the co-op's own chat origin. The gateway is called
|
|
# server-to-server (chat → portal → gateway) and by member API keys
|
|
# (curl/SDKs, not browsers), so no wildcard is needed. A `*` here would let
|
|
# any website's JS hit the gateway.
|
|
export LITELLM_CORS_ORIGINS="https://chat.inference.coop"
|
|
|
|
echo "PostgreSQL: ${CLOUDRON_POSTGRESQL_HOST}:${CLOUDRON_POSTGRESQL_PORT}"
|
|
echo "Redis: ${CLOUDRON_REDIS_HOST}:${CLOUDRON_REDIS_PORT}"
|
|
echo "Domain: ${CLOUDRON_APP_DOMAIN}"
|
|
|
|
# --- Write config.yaml with Cloudron provider keys if set ---
|
|
|
|
CONFIG_FILE="/app/data/config.yaml"
|
|
|
|
if [ ! -f "$CONFIG_FILE" ] || [ ! -s "$CONFIG_FILE" ]; then
|
|
echo "Writing default config.yaml..."
|
|
cat > "$CONFIG_FILE" << 'YAML'
|
|
# LiteLLM config for inference.coop
|
|
# MVP config: Tinfoil only (TEE-protected, privacy-first)
|
|
# Two models: DeepSeek V4 Flash (default, best agents) + GPT-OSS 120B (cheapest)
|
|
# Provider key set via environment variable or Admin UI
|
|
|
|
model_list:
|
|
# DeepSeek V4 Flash — default model
|
|
- model_name: deepseek-v4-flash
|
|
litellm_params:
|
|
model: openai/deepseek-v4-flash
|
|
api_base: https://api.tinfoil.sh/v1
|
|
api_key: os.environ/TINFOIL_API_KEY
|
|
|
|
# GPT-OSS 120B — lightweight fallback
|
|
- model_name: gpt-oss-120b
|
|
litellm_params:
|
|
model: openai/gpt-oss-120b
|
|
api_base: https://api.tinfoil.sh/v1
|
|
api_key: os.environ/TINFOIL_API_KEY
|
|
|
|
# Fallback: if DeepSeek is down, use GPT-OSS
|
|
router_settings:
|
|
num_retries: 2
|
|
timeout: 30
|
|
fallbacks:
|
|
- deepseek-v4-flash:
|
|
- gpt-oss-120b
|
|
|
|
general_settings:
|
|
master_key: os.environ/LITELLM_MASTER_KEY
|
|
database_url: os.environ/DATABASE_URL
|
|
store_model_in_db: true
|
|
|
|
litellm_settings:
|
|
salt_key: os.environ/LITELLM_SALT_KEY
|
|
drop_params: true
|
|
num_threads: 4
|
|
request_timeout: 30
|
|
YAML
|
|
echo "Config written to $CONFIG_FILE"
|
|
fi
|
|
|
|
# --- Start LiteLLM ---
|
|
echo "Starting LiteLLM on port 4000..."
|
|
|
|
# Start the Tinfoil proxy as a local sidecar. It verifies the enclave
|
|
# attestation and encrypts request/response bodies with EHBP (HPKE), so
|
|
# LiteLLM can talk plaintext OpenAI to it while the proxy handles the
|
|
# end-to-end encryption to the Tinfoil enclave.
|
|
#
|
|
# The proxy binary is downloaded at runtime into /app/data (persistent,
|
|
# writable) because Cloudron's build sandbox has no outbound network access to
|
|
# GitHub. The running container does, so we fetch it here on first start.
|
|
TINFOIL_PROXY_BIN="/app/data/tinfoil-proxy"
|
|
TINFOIL_PROXY_VERSION="v0.2.3"
|
|
if [ ! -f "$TINFOIL_PROXY_BIN" ]; then
|
|
echo "Downloading Tinfoil proxy ${TINFOIL_PROXY_VERSION}..."
|
|
python3 -c "import urllib.request; urllib.request.urlretrieve('https://github.com/tinfoilsh/tinfoil-proxy/releases/download/${TINFOIL_PROXY_VERSION}/tinfoil-proxy-linux-amd64', '${TINFOIL_PROXY_BIN}')"
|
|
chmod +x "$TINFOIL_PROXY_BIN"
|
|
fi
|
|
|
|
echo "Starting Tinfoil proxy on 127.0.0.1:3301..."
|
|
"$TINFOIL_PROXY_BIN" -b 127.0.0.1 -p 3301 &
|
|
TINFOIL_PROXY_PID=$!
|
|
echo "Tinfoil proxy PID: $TINFOIL_PROXY_PID"
|
|
|
|
# Give the proxy a moment to verify the enclave attestation
|
|
sleep 3
|
|
|
|
# Append a co-op-specific hint to LiteLLM's budget-exceeded messages so a
|
|
# member who hits their limit is told they can top up with a credit pack.
|
|
# LiteLLM hardcodes these strings in
|
|
# litellm/proxy/auth/auth_checks.py (not configurable via env/settings), so we
|
|
# patch the running copy at boot. Idempotent (checks a marker first) and
|
|
# re-applied every start, so it survives upstream file moves on version bumps
|
|
# — unlike the old build-time `RUN sed` CORS patch, which broke silently when
|
|
# LiteLLM relocated the file.
|
|
CREDIT_PACK_HINT='. You can add credit packs at https://opencollective.com/inference-cooperative'
|
|
AUTH_CHECKS_PY="$(python3 -c 'import litellm.proxy.auth.auth_checks as m; print(m.__file__)')"
|
|
if [ -n "$AUTH_CHECKS_PY" ] && ! grep -q "opencollective.com/inference-cooperative" "$AUTH_CHECKS_PY" 2>/dev/null; then
|
|
echo "Patching budget-exceeded messages with credit-pack hint..."
|
|
python3 - "$AUTH_CHECKS_PY" <<'PYEOF'
|
|
import sys
|
|
p = sys.argv[1]
|
|
s = open(p).read()
|
|
hint = ". You can add credit packs at https://opencollective.com/inference-cooperative"
|
|
before = s
|
|
# Team budget message (the one that fires for members, enforced at team level)
|
|
s = s.replace(
|
|
'message=f"Budget has been exceeded! Team={team_object.team_id} Current cost: {spend}, Max budget: {team_object.max_budget}"',
|
|
'message=f"Budget has been exceeded! Team={team_object.team_id} Current cost: {spend}, Max budget: {team_object.max_budget}" + "' + hint.replace('"', '\\"') + '"',
|
|
)
|
|
# Key budget message (a few keys carry individual max_budget)
|
|
s = s.replace(
|
|
'message=f"Budget has been exceeded! Key={key_descriptor} Current cost: {spend}, Max budget: {valid_token.max_budget}"',
|
|
'message=f"Budget has been exceeded! Key={key_descriptor} Current cost: {spend}, Max budget: {valid_token.max_budget}" + "' + hint.replace('"', '\\"') + '"',
|
|
)
|
|
if s != before:
|
|
open(p, "w").write(s)
|
|
print("patched")
|
|
else:
|
|
print("no match (upstream message format changed) — leaving unpatched")
|
|
PYEOF
|
|
else
|
|
echo "Budget-exceeded hint already applied (or auth_checks.py not found); skipping."
|
|
fi
|
|
|
|
exec litellm --config "$CONFIG_FILE" --port 4000 --host 0.0.0.0 |