Page fetch service: Keenable-shaped API, SSRF guard, no access log
This commit is contained in:
commit
5f88cebede
6 files changed
+255
No files matched your search
+150
@@ -0,0 +1,150 @@
|
||||
"""Inference Cooperative page-fetch service.
|
||||
|
||||
A small internal service that LibreChat's web-search scraper calls instead of a
|
||||
third-party page fetcher. It implements the shape Keenable's fetch API returns
|
||||
(`{content, title, description, url}`) so LibreChat can use it through
|
||||
`KEENABLE_FETCH_URL` with no config fork.
|
||||
|
||||
Privacy posture:
|
||||
- API-key authenticated; fail closed when FETCHER_KEY is unset.
|
||||
- No access log: full result URLs are not written anywhere. Only errors are
|
||||
logged, and they name the host, never the query.
|
||||
- SSRF guard: http/https only, ports 80/443 only, no credentials in the URL,
|
||||
private/loopback/link-local/reserved/multicast addresses refused, redirects
|
||||
re-validated hop by hop, 2 MB body cap, 10 s timeout.
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import socket
|
||||
import urllib.parse
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
import trafilatura
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
FETCHER_KEY = os.environ.get("FETCHER_KEY", "")
|
||||
TIMEOUT = float(os.environ.get("FETCH_TIMEOUT_SECONDS", "10"))
|
||||
MAX_BYTES = int(os.environ.get("FETCH_MAX_BYTES", "2000000"))
|
||||
MAX_REDIRECTS = 3
|
||||
ALLOWED_PORTS = {80, 443}
|
||||
ALLOWED_TYPES = ("text/html", "application/xhtml+xml", "text/plain")
|
||||
|
||||
app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)
|
||||
_client = httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(TIMEOUT, connect=5.0),
|
||||
follow_redirects=False,
|
||||
headers={"User-Agent": "InferenceCoop-Fetcher/0.1 (+https://inference.coop)"},
|
||||
)
|
||||
|
||||
|
||||
def _require_key(request: Request) -> None:
|
||||
if not FETCHER_KEY:
|
||||
raise HTTPException(503, "fetcher key not configured")
|
||||
provided = request.headers.get("x-api-key", "")
|
||||
if not secrets.compare_digest(provided, FETCHER_KEY):
|
||||
raise HTTPException(401, "invalid api key")
|
||||
|
||||
|
||||
def _check_url(url: str) -> urllib.parse.SplitResult:
|
||||
parsed = urllib.parse.urlsplit(url.strip())
|
||||
if parsed.scheme not in ("http", "https"):
|
||||
raise HTTPException(400, "only http/https URLs are supported")
|
||||
if parsed.username or parsed.password:
|
||||
raise HTTPException(400, "credentials in URL are refused")
|
||||
if not parsed.hostname:
|
||||
raise HTTPException(400, "URL has no host")
|
||||
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||||
if port not in ALLOWED_PORTS:
|
||||
raise HTTPException(400, "only ports 80 and 443 are allowed")
|
||||
try:
|
||||
infos = socket.getaddrinfo(parsed.hostname, port, type=socket.SOCK_STREAM)
|
||||
except socket.gaierror:
|
||||
raise HTTPException(400, "host does not resolve")
|
||||
for info in infos:
|
||||
ip = ipaddress.ip_address(info[4][0])
|
||||
if (
|
||||
ip.is_private
|
||||
or ip.is_loopback
|
||||
or ip.is_link_local
|
||||
or ip.is_reserved
|
||||
or ip.is_multicast
|
||||
or ip.is_unspecified
|
||||
):
|
||||
raise HTTPException(400, "host resolves to a non-public address")
|
||||
return parsed
|
||||
|
||||
|
||||
async def _fetch(url: str) -> tuple[str, str]:
|
||||
current = url
|
||||
for _ in range(MAX_REDIRECTS + 1):
|
||||
_check_url(current)
|
||||
async with _client.stream("GET", current) as resp:
|
||||
if resp.status_code in (301, 302, 303, 307, 308):
|
||||
location = resp.headers.get("location")
|
||||
if not location:
|
||||
raise HTTPException(502, "redirect without location")
|
||||
current = urllib.parse.urljoin(current, location)
|
||||
continue
|
||||
if resp.status_code != 200:
|
||||
raise HTTPException(502, f"upstream returned {resp.status_code}")
|
||||
ctype = (resp.headers.get("content-type") or "").split(";")[0].strip().lower()
|
||||
if ctype not in ALLOWED_TYPES:
|
||||
raise HTTPException(415, f"unsupported content type: {ctype or 'unknown'}")
|
||||
chunks = []
|
||||
size = 0
|
||||
async for chunk in resp.aiter_bytes():
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise HTTPException(413, "page exceeds the size limit")
|
||||
chunks.append(chunk)
|
||||
return current, b"".join(chunks).decode(resp.encoding or "utf-8", "replace")
|
||||
raise HTTPException(508, "too many redirects")
|
||||
|
||||
|
||||
def _extract(html: str, url: str) -> dict:
|
||||
content = trafilatura.extract(
|
||||
html,
|
||||
url=url,
|
||||
output_format="markdown",
|
||||
include_links=False,
|
||||
include_comments=False,
|
||||
include_tables=True,
|
||||
favor_recall=True,
|
||||
)
|
||||
if not content:
|
||||
raise HTTPException(422, "no readable text found")
|
||||
meta = trafilatura.extract_metadata(html)
|
||||
return {
|
||||
"content": content,
|
||||
"title": getattr(meta, "title", None) or "",
|
||||
"description": getattr(meta, "description", None) or "",
|
||||
"url": url,
|
||||
}
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health() -> dict:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/v1/fetch")
|
||||
@app.get("/v1/fetch/public")
|
||||
async def fetch(request: Request, url: str) -> JSONResponse:
|
||||
_require_key(request)
|
||||
try:
|
||||
final_url, html = await _fetch(url)
|
||||
return JSONResponse(_extract(html, final_url))
|
||||
except HTTPException:
|
||||
raise
|
||||
except httpx.TimeoutException:
|
||||
raise HTTPException(504, "fetch timed out")
|
||||
except httpx.HTTPError as exc:
|
||||
# Name the host only; never echo the full URL (it carries member search
|
||||
# results) into logs or responses.
|
||||
host = urllib.parse.urlsplit(url).hostname or "unknown"
|
||||
raise HTTPException(502, f"fetch failed for {host}: {exc.__class__.__name__}")
|
||||
Reference in new issue
Block a user