"""Inference Cooperative page-fetch service. A small internal service that LibreChat's web-search scraper calls instead of a third-party page fetcher. It implements the shape Keenable's fetch API returns (`{content, title, description, url}`) so LibreChat can use it through `KEENABLE_FETCH_URL` with no config fork. Privacy posture: - API-key authenticated; fail closed when FETCHER_KEY is unset. - No access log: full result URLs are not written anywhere. Only errors are logged, and they name the host, never the query. - SSRF guard: http/https only, ports 80/443 only, no credentials in the URL, private/loopback/link-local/reserved/multicast addresses refused, redirects re-validated hop by hop, 2 MB body cap, 10 s timeout. """ import ipaddress import os import re import secrets import socket import urllib.parse from typing import Optional import httpx import trafilatura from fastapi import FastAPI, HTTPException, Request from fastapi.responses import JSONResponse FETCHER_KEY = os.environ.get("FETCHER_KEY", "") TIMEOUT = float(os.environ.get("FETCH_TIMEOUT_SECONDS", "10")) MAX_BYTES = int(os.environ.get("FETCH_MAX_BYTES", "2000000")) MAX_REDIRECTS = 3 ALLOWED_PORTS = {80, 443} ALLOWED_TYPES = ("text/html", "application/xhtml+xml", "text/plain") app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None) _client = httpx.AsyncClient( timeout=httpx.Timeout(TIMEOUT, connect=5.0), follow_redirects=False, headers={"User-Agent": "InferenceCoop-Fetcher/0.1 (+https://inference.coop)"}, ) def _require_key(request: Request) -> None: if not FETCHER_KEY: raise HTTPException(503, "fetcher key not configured") provided = request.headers.get("x-api-key", "") if not secrets.compare_digest(provided, FETCHER_KEY): raise HTTPException(401, "invalid api key") def _check_url(url: str) -> urllib.parse.SplitResult: parsed = urllib.parse.urlsplit(url.strip()) if parsed.scheme not in ("http", "https"): raise HTTPException(400, "only http/https URLs are supported") if parsed.username or parsed.password: raise HTTPException(400, "credentials in URL are refused") if not parsed.hostname: raise HTTPException(400, "URL has no host") port = parsed.port or (443 if parsed.scheme == "https" else 80) if port not in ALLOWED_PORTS: raise HTTPException(400, "only ports 80 and 443 are allowed") try: infos = socket.getaddrinfo(parsed.hostname, port, type=socket.SOCK_STREAM) except socket.gaierror: raise HTTPException(400, "host does not resolve") for info in infos: ip = ipaddress.ip_address(info[4][0]) if ( ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast or ip.is_unspecified ): raise HTTPException(400, "host resolves to a non-public address") return parsed async def _fetch(url: str) -> tuple[str, str]: current = url for _ in range(MAX_REDIRECTS + 1): _check_url(current) async with _client.stream("GET", current) as resp: if resp.status_code in (301, 302, 303, 307, 308): location = resp.headers.get("location") if not location: raise HTTPException(502, "redirect without location") current = urllib.parse.urljoin(current, location) continue if resp.status_code != 200: raise HTTPException(502, f"upstream returned {resp.status_code}") ctype = (resp.headers.get("content-type") or "").split(";")[0].strip().lower() if ctype not in ALLOWED_TYPES: raise HTTPException(415, f"unsupported content type: {ctype or 'unknown'}") chunks = [] size = 0 async for chunk in resp.aiter_bytes(): size += len(chunk) if size > MAX_BYTES: raise HTTPException(413, "page exceeds the size limit") chunks.append(chunk) return current, b"".join(chunks).decode(resp.encoding or "utf-8", "replace") raise HTTPException(508, "too many redirects") def _extract(html: str, url: str) -> dict: content = trafilatura.extract( html, url=url, output_format="markdown", include_links=False, include_comments=False, include_tables=True, favor_recall=True, ) if not content: raise HTTPException(422, "no readable text found") meta = trafilatura.extract_metadata(html) return { "content": content, "title": getattr(meta, "title", None) or "", "description": getattr(meta, "description", None) or "", "url": url, } @app.get("/health") async def health() -> dict: return {"status": "ok"} @app.get("/v1/fetch") @app.get("/v1/fetch/public") async def fetch(request: Request, url: str) -> JSONResponse: _require_key(request) try: final_url, html = await _fetch(url) return JSONResponse(_extract(html, final_url)) except HTTPException: raise except httpx.TimeoutException: raise HTTPException(504, "fetch timed out") except httpx.HTTPError as exc: # Name the host only; never echo the full URL (it carries member search # results) into logs or responses. host = urllib.parse.urlsplit(url).hostname or "unknown" raise HTTPException(502, f"fetch failed for {host}: {exc.__class__.__name__}")