151 lines
5.4 KiB
Python
151 lines
5.4 KiB
Python
"""Inference Cooperative page-fetch service.
|
|
|
|
A small internal service that LibreChat's web-search scraper calls instead of a
|
|
third-party page fetcher. It implements the shape Keenable's fetch API returns
|
|
(`{content, title, description, url}`) so LibreChat can use it through
|
|
`KEENABLE_FETCH_URL` with no config fork.
|
|
|
|
Privacy posture:
|
|
- API-key authenticated; fail closed when FETCHER_KEY is unset.
|
|
- No access log: full result URLs are not written anywhere. Only errors are
|
|
logged, and they name the host, never the query.
|
|
- SSRF guard: http/https only, ports 80/443 only, no credentials in the URL,
|
|
private/loopback/link-local/reserved/multicast addresses refused, redirects
|
|
re-validated hop by hop, 2 MB body cap, 10 s timeout.
|
|
"""
|
|
|
|
import ipaddress
|
|
import os
|
|
import re
|
|
import secrets
|
|
import socket
|
|
import urllib.parse
|
|
from typing import Optional
|
|
|
|
import httpx
|
|
import trafilatura
|
|
from fastapi import FastAPI, HTTPException, Request
|
|
from fastapi.responses import JSONResponse
|
|
|
|
FETCHER_KEY = os.environ.get("FETCHER_KEY", "")
|
|
TIMEOUT = float(os.environ.get("FETCH_TIMEOUT_SECONDS", "10"))
|
|
MAX_BYTES = int(os.environ.get("FETCH_MAX_BYTES", "2000000"))
|
|
MAX_REDIRECTS = 3
|
|
ALLOWED_PORTS = {80, 443}
|
|
ALLOWED_TYPES = ("text/html", "application/xhtml+xml", "text/plain")
|
|
|
|
app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)
|
|
_client = httpx.AsyncClient(
|
|
timeout=httpx.Timeout(TIMEOUT, connect=5.0),
|
|
follow_redirects=False,
|
|
headers={"User-Agent": "InferenceCoop-Fetcher/0.1 (+https://inference.coop)"},
|
|
)
|
|
|
|
|
|
def _require_key(request: Request) -> None:
|
|
if not FETCHER_KEY:
|
|
raise HTTPException(503, "fetcher key not configured")
|
|
provided = request.headers.get("x-api-key", "")
|
|
if not secrets.compare_digest(provided, FETCHER_KEY):
|
|
raise HTTPException(401, "invalid api key")
|
|
|
|
|
|
def _check_url(url: str) -> urllib.parse.SplitResult:
|
|
parsed = urllib.parse.urlsplit(url.strip())
|
|
if parsed.scheme not in ("http", "https"):
|
|
raise HTTPException(400, "only http/https URLs are supported")
|
|
if parsed.username or parsed.password:
|
|
raise HTTPException(400, "credentials in URL are refused")
|
|
if not parsed.hostname:
|
|
raise HTTPException(400, "URL has no host")
|
|
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
|
if port not in ALLOWED_PORTS:
|
|
raise HTTPException(400, "only ports 80 and 443 are allowed")
|
|
try:
|
|
infos = socket.getaddrinfo(parsed.hostname, port, type=socket.SOCK_STREAM)
|
|
except socket.gaierror:
|
|
raise HTTPException(400, "host does not resolve")
|
|
for info in infos:
|
|
ip = ipaddress.ip_address(info[4][0])
|
|
if (
|
|
ip.is_private
|
|
or ip.is_loopback
|
|
or ip.is_link_local
|
|
or ip.is_reserved
|
|
or ip.is_multicast
|
|
or ip.is_unspecified
|
|
):
|
|
raise HTTPException(400, "host resolves to a non-public address")
|
|
return parsed
|
|
|
|
|
|
async def _fetch(url: str) -> tuple[str, str]:
|
|
current = url
|
|
for _ in range(MAX_REDIRECTS + 1):
|
|
_check_url(current)
|
|
async with _client.stream("GET", current) as resp:
|
|
if resp.status_code in (301, 302, 303, 307, 308):
|
|
location = resp.headers.get("location")
|
|
if not location:
|
|
raise HTTPException(502, "redirect without location")
|
|
current = urllib.parse.urljoin(current, location)
|
|
continue
|
|
if resp.status_code != 200:
|
|
raise HTTPException(502, f"upstream returned {resp.status_code}")
|
|
ctype = (resp.headers.get("content-type") or "").split(";")[0].strip().lower()
|
|
if ctype not in ALLOWED_TYPES:
|
|
raise HTTPException(415, f"unsupported content type: {ctype or 'unknown'}")
|
|
chunks = []
|
|
size = 0
|
|
async for chunk in resp.aiter_bytes():
|
|
size += len(chunk)
|
|
if size > MAX_BYTES:
|
|
raise HTTPException(413, "page exceeds the size limit")
|
|
chunks.append(chunk)
|
|
return current, b"".join(chunks).decode(resp.encoding or "utf-8", "replace")
|
|
raise HTTPException(508, "too many redirects")
|
|
|
|
|
|
def _extract(html: str, url: str) -> dict:
|
|
content = trafilatura.extract(
|
|
html,
|
|
url=url,
|
|
output_format="markdown",
|
|
include_links=False,
|
|
include_comments=False,
|
|
include_tables=True,
|
|
favor_recall=True,
|
|
)
|
|
if not content:
|
|
raise HTTPException(422, "no readable text found")
|
|
meta = trafilatura.extract_metadata(html)
|
|
return {
|
|
"content": content,
|
|
"title": getattr(meta, "title", None) or "",
|
|
"description": getattr(meta, "description", None) or "",
|
|
"url": url,
|
|
}
|
|
|
|
|
|
@app.get("/health")
|
|
async def health() -> dict:
|
|
return {"status": "ok"}
|
|
|
|
|
|
@app.get("/v1/fetch")
|
|
@app.get("/v1/fetch/public")
|
|
async def fetch(request: Request, url: str) -> JSONResponse:
|
|
_require_key(request)
|
|
try:
|
|
final_url, html = await _fetch(url)
|
|
return JSONResponse(_extract(html, final_url))
|
|
except HTTPException:
|
|
raise
|
|
except httpx.TimeoutException:
|
|
raise HTTPException(504, "fetch timed out")
|
|
except httpx.HTTPError as exc:
|
|
# Name the host only; never echo the full URL (it carries member search
|
|
# results) into logs or responses.
|
|
host = urllib.parse.urlsplit(url).hostname or "unknown"
|
|
raise HTTPException(502, f"fetch failed for {host}: {exc.__class__.__name__}")
|