123 lines
3.8 KiB
Python
123 lines
3.8 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from time import perf_counter
|
|
from urllib.parse import urlparse
|
|
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
|
|
from app.core.logging import get_logger
|
|
from app.services.ai_tools.schemas import FetchedEvidence
|
|
from app.services.business_logs import emit_business_log, exception_context
|
|
|
|
|
|
logger = get_logger(__name__, service="ai_tool")
|
|
|
|
|
|
class WebFetchError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def _extract_title_and_text(html: str) -> tuple[str, str]:
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
for tag in soup(["script", "style", "noscript", "svg"]):
|
|
tag.decompose()
|
|
title = soup.title.get_text(" ", strip=True) if soup.title else ""
|
|
main = soup.find("main") or soup.find("article") or soup.body or soup
|
|
text = main.get_text("\n", strip=True)
|
|
lines = [line.strip() for line in text.splitlines() if line.strip()]
|
|
return title, "\n".join(lines)
|
|
|
|
|
|
async def fetch_url_evidence(
|
|
url: str,
|
|
*,
|
|
timeout_seconds: int = 20,
|
|
max_bytes: int = 1_500_000,
|
|
) -> FetchedEvidence:
|
|
started_at = perf_counter()
|
|
if not url:
|
|
await emit_business_log(
|
|
logger,
|
|
event="ai_tool.web_fetch.failed",
|
|
message="WebFetch failed because URL is empty",
|
|
category="ai_tool",
|
|
level="warning",
|
|
service="ai_tool",
|
|
module=__name__,
|
|
context={"reason": "empty_url"},
|
|
)
|
|
raise WebFetchError("url is required")
|
|
request_host = urlparse(url).netloc
|
|
await emit_business_log(
|
|
logger,
|
|
event="ai_tool.web_fetch.start",
|
|
message="WebFetch request started",
|
|
category="ai_tool",
|
|
service="ai_tool",
|
|
module=__name__,
|
|
context={
|
|
"url_host": request_host,
|
|
"timeout_seconds": timeout_seconds,
|
|
"max_bytes": max_bytes,
|
|
},
|
|
)
|
|
try:
|
|
async with httpx.AsyncClient(
|
|
timeout=timeout_seconds,
|
|
follow_redirects=True,
|
|
headers={"User-Agent": "PlanetEvidenceFetcher/1.0"},
|
|
) as client:
|
|
response = await client.get(url)
|
|
response.raise_for_status()
|
|
content = response.content[:max_bytes]
|
|
except httpx.HTTPError as exc:
|
|
await emit_business_log(
|
|
logger,
|
|
event="ai_tool.web_fetch.failed",
|
|
message="WebFetch request failed",
|
|
category="ai_tool",
|
|
level="error",
|
|
service="ai_tool",
|
|
module=__name__,
|
|
context=exception_context(
|
|
exc,
|
|
{
|
|
"url_host": request_host,
|
|
"status": "failed",
|
|
"duration_ms": int((perf_counter() - started_at) * 1000),
|
|
},
|
|
),
|
|
)
|
|
raise WebFetchError(f"failed to fetch page: {exc}") from exc
|
|
|
|
title, text = _extract_title_and_text(content.decode(response.encoding or "utf-8", errors="ignore"))
|
|
content_hash = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
await emit_business_log(
|
|
logger,
|
|
event="ai_tool.web_fetch.success",
|
|
message="WebFetch request completed",
|
|
category="ai_tool",
|
|
service="ai_tool",
|
|
module=__name__,
|
|
context={
|
|
"url_host": request_host,
|
|
"final_url_host": urlparse(str(response.url)).netloc,
|
|
"status": "success",
|
|
"status_code": response.status_code,
|
|
"bytes_read": len(content),
|
|
"content_hash": content_hash,
|
|
"duration_ms": int((perf_counter() - started_at) * 1000),
|
|
"extractor": "beautifulsoup_basic",
|
|
},
|
|
)
|
|
return FetchedEvidence(
|
|
url=url,
|
|
final_url=str(response.url),
|
|
title=title,
|
|
text=text,
|
|
content_hash=content_hash,
|
|
extractor="beautifulsoup_basic",
|
|
)
|