from __future__ import annotations import hashlib from time import perf_counter from urllib.parse import urlparse import httpx from bs4 import BeautifulSoup from app.core.logging import get_logger from app.services.ai_tools.schemas import FetchedEvidence from app.services.business_logs import emit_business_log, exception_context logger = get_logger(__name__, service="ai_tool") class WebFetchError(RuntimeError): pass def _extract_title_and_text(html: str) -> tuple[str, str]: soup = BeautifulSoup(html, "html.parser") for tag in soup(["script", "style", "noscript", "svg"]): tag.decompose() title = soup.title.get_text(" ", strip=True) if soup.title else "" main = soup.find("main") or soup.find("article") or soup.body or soup text = main.get_text("\n", strip=True) lines = [line.strip() for line in text.splitlines() if line.strip()] return title, "\n".join(lines) async def fetch_url_evidence( url: str, *, timeout_seconds: int = 20, max_bytes: int = 1_500_000, ) -> FetchedEvidence: started_at = perf_counter() if not url: await emit_business_log( logger, event="ai_tool.web_fetch.failed", message="WebFetch failed because URL is empty", category="ai_tool", level="warning", service="ai_tool", module=__name__, context={"reason": "empty_url"}, ) raise WebFetchError("url is required") request_host = urlparse(url).netloc await emit_business_log( logger, event="ai_tool.web_fetch.start", message="WebFetch request started", category="ai_tool", service="ai_tool", module=__name__, context={ "url_host": request_host, "timeout_seconds": timeout_seconds, "max_bytes": max_bytes, }, ) try: async with httpx.AsyncClient( timeout=timeout_seconds, follow_redirects=True, headers={"User-Agent": "PlanetEvidenceFetcher/1.0"}, ) as client: response = await client.get(url) response.raise_for_status() content = response.content[:max_bytes] except httpx.HTTPError as exc: await emit_business_log( logger, event="ai_tool.web_fetch.failed", message="WebFetch request failed", category="ai_tool", level="error", service="ai_tool", module=__name__, context=exception_context( exc, { "url_host": request_host, "status": "failed", "duration_ms": int((perf_counter() - started_at) * 1000), }, ), ) raise WebFetchError(f"failed to fetch page: {exc}") from exc title, text = _extract_title_and_text(content.decode(response.encoding or "utf-8", errors="ignore")) content_hash = hashlib.sha256(text.encode("utf-8")).hexdigest() await emit_business_log( logger, event="ai_tool.web_fetch.success", message="WebFetch request completed", category="ai_tool", service="ai_tool", module=__name__, context={ "url_host": request_host, "final_url_host": urlparse(str(response.url)).netloc, "status": "success", "status_code": response.status_code, "bytes_read": len(content), "content_hash": content_hash, "duration_ms": int((perf_counter() - started_at) * 1000), "extractor": "beautifulsoup_basic", }, ) return FetchedEvidence( url=url, final_url=str(response.url), title=title, text=text, content_hash=content_hash, extractor="beautifulsoup_basic", )