Page fetch service: Keenable-shaped API, SSRF guard, no access log
This commit is contained in:
commit
5f88cebede
6 files changed
+255
No files matched your search
@@ -0,0 +1,3 @@
|
|||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
.venv/
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
{
|
||||||
|
"id": "coop.inference.fetcher",
|
||||||
|
"title": "Page Fetcher",
|
||||||
|
"author": "Inference Cooperative",
|
||||||
|
"description": "Internal page-fetch service for LibreChat web search: fetches result pages and returns readable markdown. Admin-visible only; called server-to-server with an API key.",
|
||||||
|
"tagline": "Readable pages for co-op web search",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"healthCheckPath": "/health",
|
||||||
|
"httpPort": 8000,
|
||||||
|
"manifestVersion": 2,
|
||||||
|
"website": "https://inference.coop",
|
||||||
|
"contactEmail": "info@inference.coop",
|
||||||
|
"tags": [
|
||||||
|
"internal",
|
||||||
|
"search",
|
||||||
|
"fetcher"
|
||||||
|
],
|
||||||
|
"memoryLimit": 536870912,
|
||||||
|
"dockerImage": "registry.inference.coop/coop.inference.fetcher:dev"
|
||||||
|
}
|
||||||
+15
@@ -0,0 +1,15 @@
|
|||||||
|
FROM python:3.12-slim
|
||||||
|
|
||||||
|
WORKDIR /app/code
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
|
||||||
|
|
||||||
|
COPY requirements.txt ./
|
||||||
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
|
||||||
|
COPY app/ ./app/
|
||||||
|
|
||||||
|
EXPOSE 8000
|
||||||
|
|
||||||
|
# No access log: the path carries the fetched URL, which is member search
|
||||||
|
# output. Errors still reach Cloudron's log stream; URLs do not.
|
||||||
|
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"]
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
# Page Fetcher
|
||||||
|
|
||||||
|
Internal page-fetch service for the co-op's LibreChat web search. It replaces
|
||||||
|
the third-party Keenable fetch: LibreChat sends the result URL here, gets
|
||||||
|
readable markdown back, and the model cites the page.
|
||||||
|
|
||||||
|
## Contract
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /v1/fetch?url=<page> (X-API-Key required)
|
||||||
|
GET /v1/fetch/public?url=<page> (same; path LibreChat uses keyless)
|
||||||
|
GET /health (no key — Cloudron health check)
|
||||||
|
```
|
||||||
|
|
||||||
|
Response (Keenable-compatible shape, so LibreChat needs no config fork):
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"content": "markdown…", "title": "…", "description": "…", "url": "https://final.example/page"}
|
||||||
|
```
|
||||||
|
|
||||||
|
Errors are JSON `{"detail": "…"}` with 4xx/5xx. `FETCHER_KEY` unset means the
|
||||||
|
service refuses every fetch (503).
|
||||||
|
|
||||||
|
## Where it sits
|
||||||
|
|
||||||
|
```
|
||||||
|
members → LibreChat → SearXNG (search) → Page Fetcher (this app) → page text
|
||||||
|
↘ gateway (answer), with citations
|
||||||
|
```
|
||||||
|
|
||||||
|
LibreChat reaches it on the internal Cloudron network
|
||||||
|
(`http://fetch:8000`). No member can reach it: the public app is restricted to
|
||||||
|
the `managers` Cloudron group, there is no UI, and every fetch requires
|
||||||
|
`X-API-Key`.
|
||||||
|
|
||||||
|
## Privacy choices
|
||||||
|
|
||||||
|
- **No access log.** The request path carries the fetched URL, which is member
|
||||||
|
search output. Uvicorn runs with `--no-access-log`; errors log the host, never
|
||||||
|
the full URL. Nothing about a fetch is stored.
|
||||||
|
- **SSRF guard.** http/https only, ports 80/443 only, credentials in the URL
|
||||||
|
refused, private/loopback/link-local/reserved/multicast addresses refused,
|
||||||
|
redirects re-validated at every hop, 2 MB body cap, 10 s timeout.
|
||||||
|
- **No browser.** JS-rendered pages yield little or no text; the fetch fails
|
||||||
|
cleanly rather than returning junk.
|
||||||
|
|
||||||
|
## Deploy
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# build (Cloudron builder)
|
||||||
|
~/.hermes/scripts/cloudron_builder_build.sh coop.inference.fetcher <tag> .
|
||||||
|
|
||||||
|
# pin CloudronManifest.json dockerImage to the new tag, commit, push
|
||||||
|
# install once:
|
||||||
|
cloudron install --app fetch.inference.coop --location fetch
|
||||||
|
|
||||||
|
# configure: FETCHER_KEY (32+ random bytes), then restrict access to managers
|
||||||
|
```
|
||||||
|
|
||||||
|
## Not in scope
|
||||||
|
|
||||||
|
- Search itself stays on SearXNG.
|
||||||
|
- No crawling, no index, no cache, no UI.
|
||||||
+150
@@ -0,0 +1,150 @@
|
|||||||
|
"""Inference Cooperative page-fetch service.
|
||||||
|
|
||||||
|
A small internal service that LibreChat's web-search scraper calls instead of a
|
||||||
|
third-party page fetcher. It implements the shape Keenable's fetch API returns
|
||||||
|
(`{content, title, description, url}`) so LibreChat can use it through
|
||||||
|
`KEENABLE_FETCH_URL` with no config fork.
|
||||||
|
|
||||||
|
Privacy posture:
|
||||||
|
- API-key authenticated; fail closed when FETCHER_KEY is unset.
|
||||||
|
- No access log: full result URLs are not written anywhere. Only errors are
|
||||||
|
logged, and they name the host, never the query.
|
||||||
|
- SSRF guard: http/https only, ports 80/443 only, no credentials in the URL,
|
||||||
|
private/loopback/link-local/reserved/multicast addresses refused, redirects
|
||||||
|
re-validated hop by hop, 2 MB body cap, 10 s timeout.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import ipaddress
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import secrets
|
||||||
|
import socket
|
||||||
|
import urllib.parse
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import trafilatura
|
||||||
|
from fastapi import FastAPI, HTTPException, Request
|
||||||
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
|
FETCHER_KEY = os.environ.get("FETCHER_KEY", "")
|
||||||
|
TIMEOUT = float(os.environ.get("FETCH_TIMEOUT_SECONDS", "10"))
|
||||||
|
MAX_BYTES = int(os.environ.get("FETCH_MAX_BYTES", "2000000"))
|
||||||
|
MAX_REDIRECTS = 3
|
||||||
|
ALLOWED_PORTS = {80, 443}
|
||||||
|
ALLOWED_TYPES = ("text/html", "application/xhtml+xml", "text/plain")
|
||||||
|
|
||||||
|
app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None)
|
||||||
|
_client = httpx.AsyncClient(
|
||||||
|
timeout=httpx.Timeout(TIMEOUT, connect=5.0),
|
||||||
|
follow_redirects=False,
|
||||||
|
headers={"User-Agent": "InferenceCoop-Fetcher/0.1 (+https://inference.coop)"},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _require_key(request: Request) -> None:
|
||||||
|
if not FETCHER_KEY:
|
||||||
|
raise HTTPException(503, "fetcher key not configured")
|
||||||
|
provided = request.headers.get("x-api-key", "")
|
||||||
|
if not secrets.compare_digest(provided, FETCHER_KEY):
|
||||||
|
raise HTTPException(401, "invalid api key")
|
||||||
|
|
||||||
|
|
||||||
|
def _check_url(url: str) -> urllib.parse.SplitResult:
|
||||||
|
parsed = urllib.parse.urlsplit(url.strip())
|
||||||
|
if parsed.scheme not in ("http", "https"):
|
||||||
|
raise HTTPException(400, "only http/https URLs are supported")
|
||||||
|
if parsed.username or parsed.password:
|
||||||
|
raise HTTPException(400, "credentials in URL are refused")
|
||||||
|
if not parsed.hostname:
|
||||||
|
raise HTTPException(400, "URL has no host")
|
||||||
|
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||||||
|
if port not in ALLOWED_PORTS:
|
||||||
|
raise HTTPException(400, "only ports 80 and 443 are allowed")
|
||||||
|
try:
|
||||||
|
infos = socket.getaddrinfo(parsed.hostname, port, type=socket.SOCK_STREAM)
|
||||||
|
except socket.gaierror:
|
||||||
|
raise HTTPException(400, "host does not resolve")
|
||||||
|
for info in infos:
|
||||||
|
ip = ipaddress.ip_address(info[4][0])
|
||||||
|
if (
|
||||||
|
ip.is_private
|
||||||
|
or ip.is_loopback
|
||||||
|
or ip.is_link_local
|
||||||
|
or ip.is_reserved
|
||||||
|
or ip.is_multicast
|
||||||
|
or ip.is_unspecified
|
||||||
|
):
|
||||||
|
raise HTTPException(400, "host resolves to a non-public address")
|
||||||
|
return parsed
|
||||||
|
|
||||||
|
|
||||||
|
async def _fetch(url: str) -> tuple[str, str]:
|
||||||
|
current = url
|
||||||
|
for _ in range(MAX_REDIRECTS + 1):
|
||||||
|
_check_url(current)
|
||||||
|
async with _client.stream("GET", current) as resp:
|
||||||
|
if resp.status_code in (301, 302, 303, 307, 308):
|
||||||
|
location = resp.headers.get("location")
|
||||||
|
if not location:
|
||||||
|
raise HTTPException(502, "redirect without location")
|
||||||
|
current = urllib.parse.urljoin(current, location)
|
||||||
|
continue
|
||||||
|
if resp.status_code != 200:
|
||||||
|
raise HTTPException(502, f"upstream returned {resp.status_code}")
|
||||||
|
ctype = (resp.headers.get("content-type") or "").split(";")[0].strip().lower()
|
||||||
|
if ctype not in ALLOWED_TYPES:
|
||||||
|
raise HTTPException(415, f"unsupported content type: {ctype or 'unknown'}")
|
||||||
|
chunks = []
|
||||||
|
size = 0
|
||||||
|
async for chunk in resp.aiter_bytes():
|
||||||
|
size += len(chunk)
|
||||||
|
if size > MAX_BYTES:
|
||||||
|
raise HTTPException(413, "page exceeds the size limit")
|
||||||
|
chunks.append(chunk)
|
||||||
|
return current, b"".join(chunks).decode(resp.encoding or "utf-8", "replace")
|
||||||
|
raise HTTPException(508, "too many redirects")
|
||||||
|
|
||||||
|
|
||||||
|
def _extract(html: str, url: str) -> dict:
|
||||||
|
content = trafilatura.extract(
|
||||||
|
html,
|
||||||
|
url=url,
|
||||||
|
output_format="markdown",
|
||||||
|
include_links=False,
|
||||||
|
include_comments=False,
|
||||||
|
include_tables=True,
|
||||||
|
favor_recall=True,
|
||||||
|
)
|
||||||
|
if not content:
|
||||||
|
raise HTTPException(422, "no readable text found")
|
||||||
|
meta = trafilatura.extract_metadata(html)
|
||||||
|
return {
|
||||||
|
"content": content,
|
||||||
|
"title": getattr(meta, "title", None) or "",
|
||||||
|
"description": getattr(meta, "description", None) or "",
|
||||||
|
"url": url,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/health")
|
||||||
|
async def health() -> dict:
|
||||||
|
return {"status": "ok"}
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/v1/fetch")
|
||||||
|
@app.get("/v1/fetch/public")
|
||||||
|
async def fetch(request: Request, url: str) -> JSONResponse:
|
||||||
|
_require_key(request)
|
||||||
|
try:
|
||||||
|
final_url, html = await _fetch(url)
|
||||||
|
return JSONResponse(_extract(html, final_url))
|
||||||
|
except HTTPException:
|
||||||
|
raise
|
||||||
|
except httpx.TimeoutException:
|
||||||
|
raise HTTPException(504, "fetch timed out")
|
||||||
|
except httpx.HTTPError as exc:
|
||||||
|
# Name the host only; never echo the full URL (it carries member search
|
||||||
|
# results) into logs or responses.
|
||||||
|
host = urllib.parse.urlsplit(url).hostname or "unknown"
|
||||||
|
raise HTTPException(502, f"fetch failed for {host}: {exc.__class__.__name__}")
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
trafilatura==2.3.1
|
||||||
|
fastapi==0.165.0
|
||||||
|
uvicorn[standard]==0.54.0
|
||||||
|
httpx==0.28.1
|
||||||
Reference in new issue
Block a user