Files

151 lines
5.4 KiB
Python

"""Inference Cooperative page-fetch service.
A small internal service that LibreChat's web-search scraper calls instead of a
third-party page fetcher. It implements the shape Keenable's fetch API returns
(`{content, title, description, url}`) so LibreChat can use it through
`KEENABLE_FETCH_URL` with no config fork.
Privacy posture:
- API-key authenticated; fail closed when FETCHER_KEY is unset.
- No access log: full result URLs are not written anywhere. Only errors are
logged, and they name the host, never the query.
- SSRF guard: http/https only, ports 80/443 only, no credentials in the URL,
private/loopback/link-local/reserved/multicast addresses refused, redirects
re-validated hop by hop, 2 MB body cap, 10 s timeout.
"""
import ipaddress
import os
import re
import secrets
import socket
import urllib.parse
from typing import Optional
import httpx
import trafilatura
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse
FETCHER_KEY = os.environ.get("FETCHER_KEY", "")
TIMEOUT = float(os.environ.get("FETCH_TIMEOUT_SECONDS", "10"))
MAX_BYTES = int(os.environ.get("FETCH_MAX_BYTES", "2000000"))
MAX_REDIRECTS = 3
ALLOWED_PORTS = {80, 443}
ALLOWED_TYPES = ("text/html", "application/xhtml+xml", "text/plain")
app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)
_client = httpx.AsyncClient(
timeout=httpx.Timeout(TIMEOUT, connect=5.0),
follow_redirects=False,
headers={"User-Agent": "InferenceCoop-Fetcher/0.1 (+https://inference.coop)"},
)
def _require_key(request: Request) -> None:
if not FETCHER_KEY:
raise HTTPException(503, "fetcher key not configured")
provided = request.headers.get("x-api-key", "")
if not secrets.compare_digest(provided, FETCHER_KEY):
raise HTTPException(401, "invalid api key")
def _check_url(url: str) -> urllib.parse.SplitResult:
parsed = urllib.parse.urlsplit(url.strip())
if parsed.scheme not in ("http", "https"):
raise HTTPException(400, "only http/https URLs are supported")
if parsed.username or parsed.password:
raise HTTPException(400, "credentials in URL are refused")
if not parsed.hostname:
raise HTTPException(400, "URL has no host")
port = parsed.port or (443 if parsed.scheme == "https" else 80)
if port not in ALLOWED_PORTS:
raise HTTPException(400, "only ports 80 and 443 are allowed")
try:
infos = socket.getaddrinfo(parsed.hostname, port, type=socket.SOCK_STREAM)
except socket.gaierror:
raise HTTPException(400, "host does not resolve")
for info in infos:
ip = ipaddress.ip_address(info[4][0])
if (
ip.is_private
or ip.is_loopback
or ip.is_link_local
or ip.is_reserved
or ip.is_multicast
or ip.is_unspecified
):
raise HTTPException(400, "host resolves to a non-public address")
return parsed
async def _fetch(url: str) -> tuple[str, str]:
current = url
for _ in range(MAX_REDIRECTS + 1):
_check_url(current)
async with _client.stream("GET", current) as resp:
if resp.status_code in (301, 302, 303, 307, 308):
location = resp.headers.get("location")
if not location:
raise HTTPException(502, "redirect without location")
current = urllib.parse.urljoin(current, location)
continue
if resp.status_code != 200:
raise HTTPException(502, f"upstream returned {resp.status_code}")
ctype = (resp.headers.get("content-type") or "").split(";")[0].strip().lower()
if ctype not in ALLOWED_TYPES:
raise HTTPException(415, f"unsupported content type: {ctype or 'unknown'}")
chunks = []
size = 0
async for chunk in resp.aiter_bytes():
size += len(chunk)
if size > MAX_BYTES:
raise HTTPException(413, "page exceeds the size limit")
chunks.append(chunk)
return current, b"".join(chunks).decode(resp.encoding or "utf-8", "replace")
raise HTTPException(508, "too many redirects")
def _extract(html: str, url: str) -> dict:
content = trafilatura.extract(
html,
url=url,
output_format="markdown",
include_links=False,
include_comments=False,
include_tables=True,
favor_recall=True,
)
if not content:
raise HTTPException(422, "no readable text found")
meta = trafilatura.extract_metadata(html)
return {
"content": content,
"title": getattr(meta, "title", None) or "",
"description": getattr(meta, "description", None) or "",
"url": url,
}
@app.get("/health")
async def health() -> dict:
return {"status": "ok"}
@app.get("/v1/fetch")
@app.get("/v1/fetch/public")
async def fetch(request: Request, url: str) -> JSONResponse:
_require_key(request)
try:
final_url, html = await _fetch(url)
return JSONResponse(_extract(html, final_url))
except HTTPException:
raise
except httpx.TimeoutException:
raise HTTPException(504, "fetch timed out")
except httpx.HTTPError as exc:
# Name the host only; never echo the full URL (it carries member search
# results) into logs or responses.
host = urllib.parse.urlsplit(url).hostname or "unknown"
raise HTTPException(502, f"fetch failed for {host}: {exc.__class__.__name__}")