fetch: add private-host allowlist + env switch; include resolved IPs in error and recheck redirects; document env vars in README

README.md

@@ -44,6 +44,14 @@ pip install -r requirements.txt
    ```
    If unset, the app automatically prefers `/data` when available, otherwise `./data`.
 
+3. (Optional) Control private/local address policy for `fetch`:
+   - `FETCH_ALLOW_PRIVATE` — set to `1`/`true` to disable the SSRF guard entirely (not recommended except for trusted, local testing).
+   - `FETCH_PRIVATE_ALLOWLIST` — comma/space separated host patterns allowed even if they resolve to private/local IPs, e.g.:
+     ```bash
+     export FETCH_PRIVATE_ALLOWLIST="*.corp.local, my-proxy.internal"
+     ```
+   If neither is set, the fetcher refuses URLs whose host resolves to private, loopback, link‑local, multicast, reserved, or unspecified addresses. It also re-checks the final redirect target.
+
 The request counters live in `<DATA_DIR>/request_counts.json`, guarded by a file lock to support concurrent MCP calls.
 
 ## Running Locally

app.py

@@ -6,6 +6,7 @@ import asyncio
 import ipaddress
 import socket
 from typing import Optional, Dict, Any, List, Tuple
+import fnmatch
 from urllib.parse import urlsplit
 from datetime import datetime, timezone
 
@@ -84,6 +85,23 @@ EXTRACT_CONCURRENCY = max(
 SEARCH_CACHE_TTL = max(0, int(os.getenv("SEARCH_CACHE_TTL", "30")))
 FETCH_CACHE_TTL = max(0, int(os.getenv("FETCH_CACHE_TTL", "300")))
 
+# Controls for private/local address handling in fetch()
+def _env_flag(name: str, default: bool = False) -> bool:
+    """Parse boolean-like env vars such as 1/true/yes/on."""
+    v = os.getenv(name)
+    if v is None:
+        return default
+    return str(v).strip().lower() in {"1", "true", "yes", "on", "y"}
+
+# When True, allow any destination (disables SSRF guard — not recommended)
+FETCH_ALLOW_PRIVATE = _env_flag("FETCH_ALLOW_PRIVATE", False)
+
+# Optional comma/space separated host patterns to allow even if private, e.g.:
+#   FETCH_PRIVATE_ALLOWLIST="*.internal.example.com, my-proxy.local"
+FETCH_PRIVATE_ALLOWLIST = [
+    p for p in re.split(r"[\s,]+", os.getenv("FETCH_PRIVATE_ALLOWLIST", "").strip()) if p
+]
+
 _search_cache: Dict[Tuple[str, str, int], Dict[str, Any]] = {}
 _fetch_cache: Dict[str, Dict[str, Any]] = {}
 _search_cache_lock: Optional[asyncio.Lock] = None
@@ -162,20 +180,39 @@ def _client_ip(request: Optional[gr.Request]) -> str:
     return "unknown"
 
 
-async def _host_is_public(host: str) -> bool:
+def _host_matches_allowlist(host: str) -> bool:
+    """Return True if host matches any pattern in FETCH_PRIVATE_ALLOWLIST."""
     if not host:
         return False
+    for pat in FETCH_PRIVATE_ALLOWLIST:
+        # Support bare host equality and fnmatch-style patterns (*.foo.bar)
+        if host == pat or fnmatch.fnmatch(host, pat):
+            return True
+    return False
+
 
+async def _resolve_addresses(host: str) -> List[str]:
     def _resolve() -> List[str]:
         try:
             return list({ai[4][0] for ai in socket.getaddrinfo(host, None)})
         except Exception:
             return []
 
-    addresses = await asyncio.to_thread(_resolve)
+    return await asyncio.to_thread(_resolve)
+
+
+async def _host_is_public(host: str) -> Tuple[bool, List[str]]:
+    """Return (is_public, resolved_addresses).
+
+    - If resolution fails, treat as public and let HTTP request decide.
+    - Honors allowlist/env flags via the caller.
+    """
+    if not host:
+        return False, []
+
+    addresses = await _resolve_addresses(host)
     if not addresses:
-        # If resolution fails we let the HTTP fetch decide.
-        return True
+        return True, []
 
     for addr in addresses:
         ip_obj = ipaddress.ip_address(addr)
@@ -187,8 +224,8 @@ async def _host_is_public(host: str) -> bool:
             or ip_obj.is_reserved
             or ip_obj.is_unspecified
         ):
-            return False
-    return True
+            return False, addresses
+    return True, addresses
 
 
 async def _check_rate_limits(bucket: str, ip: str) -> Optional[str]:
@@ -374,9 +411,14 @@ async def fetch(
             await record_request("fetch")
             return cached
 
-        if not await _host_is_public(host):
+        is_public, addrs = await _host_is_public(host)
+        if not is_public and not (FETCH_ALLOW_PRIVATE or _host_matches_allowlist(host)):
             await record_request("fetch")
-            return {"error": "Refusing to fetch private or local addresses."}
+            detail = f" (resolved: {', '.join(addrs)})" if addrs else ""
+            return {
+                "error": "Refusing to fetch private or local addresses." + detail,
+                "host": host,
+            }
 
         fetch_sema = _get_semaphore("fetch")
         await fetch_sema.acquire()
@@ -397,6 +439,20 @@ async def fetch(
             fetch_sema.release()
 
         truncated = total > FETCH_MAX_BYTES
+        # Extra guard: if final URL host ended up private due to a redirect and
+        # the user hasn't allowed private hosts, refuse to return body content.
+        try:
+            final_host = urlsplit(final_url_str).hostname or ""
+        except Exception:
+            final_host = ""
+        if final_host and not (FETCH_ALLOW_PRIVATE or _host_matches_allowlist(final_host)):
+            final_public, _ = await _host_is_public(final_host)
+            if not final_public:
+                await record_request("fetch")
+                return {
+                    "error": "Refusing to fetch private or local addresses after redirect.",
+                    "host": final_host,
+                }
         text = body.decode(encoding, errors="ignore")
 
         extract_sema = _get_semaphore("extract")