release: bump version to 0.51.0
This commit is contained in:
rayd1o
7
backend/app/services/ai_tools/__init__.py
Normal file
7
backend/app/services/ai_tools/__init__.py
Normal file
@@ -0,0 +1,7 @@
|
||||
"""Backend-owned AI tool services.
|
||||
|
||||
The services in this package are business tools used by Planet's backend
|
||||
orchestrators. They intentionally live outside ``aiprovider`` so model transport
|
||||
stays separate from evidence collection and domain policy.
|
||||
"""
|
||||
|
||||
48
backend/app/services/ai_tools/evidence_store.py
Normal file
48
backend/app/services/ai_tools/evidence_store.py
Normal file
@@ -0,0 +1,48 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import Iterable
|
||||
|
||||
from app.services.ai_tools.schemas import FetchedEvidence, SearchEvidence
|
||||
|
||||
|
||||
def evidence_content_hash(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def normalize_search_evidence(items: Iterable[SearchEvidence], *, limit: int = 5) -> list[dict]:
|
||||
normalized: list[dict] = []
|
||||
seen_urls: set[str] = set()
|
||||
for item in items:
|
||||
if not item.url or item.url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(item.url)
|
||||
normalized.append(
|
||||
{
|
||||
"title": item.title,
|
||||
"url": item.url,
|
||||
"snippet": item.snippet,
|
||||
"content": item.compact_text(),
|
||||
"score": item.score,
|
||||
"source_provider": item.source_provider,
|
||||
"retrieved_at": item.retrieved_at.isoformat(),
|
||||
"metadata": item.metadata,
|
||||
}
|
||||
)
|
||||
if len(normalized) >= limit:
|
||||
break
|
||||
return normalized
|
||||
|
||||
|
||||
def normalize_fetched_evidence(item: FetchedEvidence, *, text_limit: int = 1200) -> dict:
|
||||
text = " ".join(item.text.split())[:text_limit]
|
||||
return {
|
||||
"title": item.title,
|
||||
"url": item.final_url or item.url,
|
||||
"text": text,
|
||||
"content_hash": item.content_hash or evidence_content_hash(item.text),
|
||||
"extractor": item.extractor,
|
||||
"retrieved_at": item.retrieved_at.isoformat(),
|
||||
"metadata": item.metadata,
|
||||
}
|
||||
|
||||
63
backend/app/services/ai_tools/schemas.py
Normal file
63
backend/app/services/ai_tools/schemas.py
Normal file
@@ -0,0 +1,63 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class SearchEvidence(BaseModel):
|
||||
title: str = ""
|
||||
url: str = ""
|
||||
snippet: str = ""
|
||||
content: str = ""
|
||||
score: float | None = None
|
||||
source_provider: str = ""
|
||||
retrieved_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
def compact_text(self, limit: int = 700) -> str:
|
||||
text = " ".join((self.content or self.snippet or "").split())
|
||||
return text[:limit]
|
||||
|
||||
|
||||
class FetchedEvidence(BaseModel):
|
||||
url: str
|
||||
final_url: str = ""
|
||||
title: str = ""
|
||||
text: str = ""
|
||||
content_hash: str = ""
|
||||
extractor: str = "basic_html"
|
||||
retrieved_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class WebSearchProviderConfig(BaseModel):
|
||||
provider: str = "tavily"
|
||||
base_url: str = ""
|
||||
api_key: str = ""
|
||||
max_results: int = Field(default=5, ge=1, le=20)
|
||||
timeout_seconds: int = Field(default=20, ge=3, le=120)
|
||||
endpoint_path: str = ""
|
||||
search_depth: str = "basic"
|
||||
engine: str = "google"
|
||||
include_answer: bool = False
|
||||
include_raw_content: bool = False
|
||||
include_text: bool = False
|
||||
categories: str = "general"
|
||||
engines: list[str] = Field(default_factory=list)
|
||||
search_path: str = ""
|
||||
scrape_path: str = ""
|
||||
scrape_formats: list[str] = Field(default_factory=lambda: ["markdown"])
|
||||
|
||||
|
||||
class WebSearchConfig(BaseModel):
|
||||
enabled: bool = False
|
||||
default_provider: str = "tavily"
|
||||
provider: str = "tavily"
|
||||
providers: dict[str, WebSearchProviderConfig] = Field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def active_provider_config(self) -> WebSearchProviderConfig:
|
||||
return self.providers.get(self.default_provider) or self.providers.get(self.provider) or WebSearchProviderConfig(provider=self.default_provider or self.provider)
|
||||
|
||||
56
backend/app/services/ai_tools/web_fetch.py
Normal file
56
backend/app/services/ai_tools/web_fetch.py
Normal file
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from app.services.ai_tools.schemas import FetchedEvidence
|
||||
|
||||
|
||||
class WebFetchError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _extract_title_and_text(html: str) -> tuple[str, str]:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
for tag in soup(["script", "style", "noscript", "svg"]):
|
||||
tag.decompose()
|
||||
title = soup.title.get_text(" ", strip=True) if soup.title else ""
|
||||
main = soup.find("main") or soup.find("article") or soup.body or soup
|
||||
text = main.get_text("\n", strip=True)
|
||||
lines = [line.strip() for line in text.splitlines() if line.strip()]
|
||||
return title, "\n".join(lines)
|
||||
|
||||
|
||||
async def fetch_url_evidence(
|
||||
url: str,
|
||||
*,
|
||||
timeout_seconds: int = 20,
|
||||
max_bytes: int = 1_500_000,
|
||||
) -> FetchedEvidence:
|
||||
if not url:
|
||||
raise WebFetchError("url is required")
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=timeout_seconds,
|
||||
follow_redirects=True,
|
||||
headers={"User-Agent": "PlanetEvidenceFetcher/1.0"},
|
||||
) as client:
|
||||
response = await client.get(url)
|
||||
response.raise_for_status()
|
||||
content = response.content[:max_bytes]
|
||||
except httpx.HTTPError as exc:
|
||||
raise WebFetchError(f"failed to fetch page: {exc}") from exc
|
||||
|
||||
title, text = _extract_title_and_text(content.decode(response.encoding or "utf-8", errors="ignore"))
|
||||
content_hash = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
return FetchedEvidence(
|
||||
url=url,
|
||||
final_url=str(response.url),
|
||||
title=title,
|
||||
text=text,
|
||||
content_hash=content_hash,
|
||||
extractor="beautifulsoup_basic",
|
||||
)
|
||||
|
||||
391
backend/app/services/ai_tools/web_search.py
Normal file
391
backend/app/services/ai_tools/web_search.py
Normal file
@@ -0,0 +1,391 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.services.ai_tools.schemas import SearchEvidence, WebSearchConfig, WebSearchProviderConfig
|
||||
|
||||
|
||||
WEB_SEARCH_PROVIDER_PRESETS: dict[str, dict[str, Any]] = {
|
||||
"tavily": {
|
||||
"provider": "tavily",
|
||||
"label": "Tavily",
|
||||
"api_key_env": "TAVILY_API_KEY",
|
||||
"base_url": "https://api.tavily.com",
|
||||
"endpoint_path": "/search",
|
||||
"max_results": 5,
|
||||
"timeout_seconds": 20,
|
||||
"search_depth": "basic",
|
||||
"include_answer": False,
|
||||
"include_raw_content": False,
|
||||
},
|
||||
"brave": {
|
||||
"provider": "brave",
|
||||
"label": "Brave Search API",
|
||||
"api_key_env": "BRAVE_SEARCH_API_KEY",
|
||||
"base_url": "https://api.search.brave.com",
|
||||
"endpoint_path": "/res/v1/web/search",
|
||||
"max_results": 5,
|
||||
"timeout_seconds": 20,
|
||||
},
|
||||
"serpapi": {
|
||||
"provider": "serpapi",
|
||||
"label": "SerpAPI",
|
||||
"api_key_env": "SERPAPI_API_KEY",
|
||||
"base_url": "https://serpapi.com",
|
||||
"endpoint_path": "/search.json",
|
||||
"engine": "google",
|
||||
"max_results": 5,
|
||||
"timeout_seconds": 20,
|
||||
},
|
||||
"exa": {
|
||||
"provider": "exa",
|
||||
"label": "Exa",
|
||||
"api_key_env": "EXA_API_KEY",
|
||||
"base_url": "https://api.exa.ai",
|
||||
"endpoint_path": "/search",
|
||||
"max_results": 5,
|
||||
"timeout_seconds": 20,
|
||||
"include_text": False,
|
||||
},
|
||||
"firecrawl": {
|
||||
"provider": "firecrawl",
|
||||
"label": "Firecrawl Search / Scrape",
|
||||
"api_key_env": "FIRECRAWL_API_KEY",
|
||||
"base_url": "https://api.firecrawl.dev",
|
||||
"search_path": "/v2/search",
|
||||
"scrape_path": "/v2/scrape",
|
||||
"max_results": 5,
|
||||
"timeout_seconds": 30,
|
||||
"scrape_formats": ["markdown"],
|
||||
},
|
||||
"searxng": {
|
||||
"provider": "searxng",
|
||||
"label": "SearXNG",
|
||||
"api_key_env": "SEARXNG_API_KEY",
|
||||
"base_url": "http://localhost:8080",
|
||||
"endpoint_path": "/",
|
||||
"max_results": 5,
|
||||
"timeout_seconds": 20,
|
||||
"categories": "general",
|
||||
"engines": [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class WebSearchError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class WebSearchConfigurationError(WebSearchError):
|
||||
pass
|
||||
|
||||
|
||||
def normalize_web_search_provider(provider: str | None) -> str:
|
||||
return (provider or "tavily").strip().lower() or "tavily"
|
||||
|
||||
|
||||
def get_web_search_provider_preset(provider: str) -> dict[str, Any]:
|
||||
provider_id = normalize_web_search_provider(provider)
|
||||
preset = WEB_SEARCH_PROVIDER_PRESETS.get(provider_id)
|
||||
if not preset:
|
||||
raise ValueError(f"Unsupported web search provider: {provider}")
|
||||
return deepcopy(preset)
|
||||
|
||||
|
||||
def list_web_search_provider_presets() -> list[dict[str, Any]]:
|
||||
return [get_web_search_provider_preset(provider) for provider in WEB_SEARCH_PROVIDER_PRESETS]
|
||||
|
||||
|
||||
def provider_defaults(provider: str) -> WebSearchProviderConfig:
|
||||
preset = get_web_search_provider_preset(provider)
|
||||
return WebSearchProviderConfig(**{
|
||||
key: value
|
||||
for key, value in preset.items()
|
||||
if key in WebSearchProviderConfig.model_fields
|
||||
})
|
||||
|
||||
|
||||
class WebSearchClient:
|
||||
def __init__(self, config: WebSearchConfig) -> None:
|
||||
self.config = config
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
max_results: int | None = None,
|
||||
domains: list[str] | None = None,
|
||||
freshness_days: int | None = None,
|
||||
) -> list[SearchEvidence]:
|
||||
if not self.config.enabled:
|
||||
raise WebSearchConfigurationError("WebSearch is disabled.")
|
||||
provider_config = self.config.active_provider_config
|
||||
provider = normalize_web_search_provider(provider_config.provider)
|
||||
if provider != "searxng" and not provider_config.api_key:
|
||||
raise WebSearchConfigurationError(f"{provider} API key is not configured.")
|
||||
query = " ".join(str(query or "").split())
|
||||
if not query:
|
||||
raise WebSearchConfigurationError("search query is required.")
|
||||
limit = max_results or provider_config.max_results
|
||||
if provider == "tavily":
|
||||
return await self._search_tavily(provider_config, query, limit, domains, freshness_days)
|
||||
if provider == "brave":
|
||||
return await self._search_brave(provider_config, query, limit, domains)
|
||||
if provider == "serpapi":
|
||||
return await self._search_serpapi(provider_config, query, limit)
|
||||
if provider == "exa":
|
||||
return await self._search_exa(provider_config, query, limit, domains)
|
||||
if provider == "firecrawl":
|
||||
return await self._search_firecrawl(provider_config, query, limit)
|
||||
if provider == "searxng":
|
||||
return await self._search_searxng(provider_config, query, limit, domains)
|
||||
raise WebSearchConfigurationError(f"Unsupported web search provider: {provider}")
|
||||
|
||||
async def test_connection(self) -> list[SearchEvidence]:
|
||||
return await self.search("Planet WebSearch connectivity test", max_results=1)
|
||||
|
||||
async def _request_json(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
*,
|
||||
provider_config: WebSearchProviderConfig,
|
||||
headers: dict[str, str] | None = None,
|
||||
params: dict[str, Any] | None = None,
|
||||
json: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=provider_config.timeout_seconds) as client:
|
||||
response = await client.request(
|
||||
method,
|
||||
url,
|
||||
headers=headers,
|
||||
params=params,
|
||||
json=json,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except httpx.HTTPStatusError as exc:
|
||||
detail = exc.response.text or exc.response.reason_phrase
|
||||
raise WebSearchError(f"{provider_config.provider} request failed: {detail}") from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise WebSearchError(f"{provider_config.provider} request failed: {exc}") from exc
|
||||
except ValueError as exc:
|
||||
raise WebSearchError(f"{provider_config.provider} returned invalid JSON") from exc
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
async def _search_tavily(
|
||||
self,
|
||||
config: WebSearchProviderConfig,
|
||||
query: str,
|
||||
max_results: int,
|
||||
domains: list[str] | None,
|
||||
freshness_days: int | None,
|
||||
) -> list[SearchEvidence]:
|
||||
body: dict[str, Any] = {
|
||||
"api_key": config.api_key,
|
||||
"query": query,
|
||||
"max_results": max_results,
|
||||
"search_depth": config.search_depth or "basic",
|
||||
"include_answer": config.include_answer,
|
||||
"include_raw_content": config.include_raw_content,
|
||||
}
|
||||
if domains:
|
||||
body["include_domains"] = domains
|
||||
if freshness_days:
|
||||
body["days"] = freshness_days
|
||||
data = await self._request_json(
|
||||
"POST",
|
||||
_join_url(config.base_url, config.endpoint_path or "/search"),
|
||||
provider_config=config,
|
||||
json=body,
|
||||
)
|
||||
return [
|
||||
SearchEvidence(
|
||||
title=str(item.get("title") or ""),
|
||||
url=str(item.get("url") or ""),
|
||||
snippet=str(item.get("content") or ""),
|
||||
content=str(item.get("raw_content") or ""),
|
||||
score=_float_or_none(item.get("score")),
|
||||
source_provider="tavily",
|
||||
metadata={"query": data.get("query") or query},
|
||||
)
|
||||
for item in data.get("results") or []
|
||||
if isinstance(item, dict) and item.get("url")
|
||||
]
|
||||
|
||||
async def _search_brave(
|
||||
self,
|
||||
config: WebSearchProviderConfig,
|
||||
query: str,
|
||||
max_results: int,
|
||||
domains: list[str] | None,
|
||||
) -> list[SearchEvidence]:
|
||||
search_query = query
|
||||
if domains:
|
||||
search_query = f"{query} " + " ".join(f"site:{domain}" for domain in domains)
|
||||
data = await self._request_json(
|
||||
"GET",
|
||||
_join_url(config.base_url, config.endpoint_path or "/res/v1/web/search"),
|
||||
provider_config=config,
|
||||
headers={"X-Subscription-Token": config.api_key},
|
||||
params={"q": search_query, "count": max_results},
|
||||
)
|
||||
results = (data.get("web") or {}).get("results") or []
|
||||
return [
|
||||
SearchEvidence(
|
||||
title=str(item.get("title") or ""),
|
||||
url=str(item.get("url") or ""),
|
||||
snippet=str(item.get("description") or ""),
|
||||
source_provider="brave",
|
||||
metadata={"age": item.get("age")},
|
||||
)
|
||||
for item in results
|
||||
if isinstance(item, dict) and item.get("url")
|
||||
]
|
||||
|
||||
async def _search_serpapi(
|
||||
self,
|
||||
config: WebSearchProviderConfig,
|
||||
query: str,
|
||||
max_results: int,
|
||||
) -> list[SearchEvidence]:
|
||||
data = await self._request_json(
|
||||
"GET",
|
||||
_join_url(config.base_url, config.endpoint_path or "/search.json"),
|
||||
provider_config=config,
|
||||
params={
|
||||
"api_key": config.api_key,
|
||||
"engine": config.engine or "google",
|
||||
"q": query,
|
||||
"num": max_results,
|
||||
},
|
||||
)
|
||||
return [
|
||||
SearchEvidence(
|
||||
title=str(item.get("title") or ""),
|
||||
url=str(item.get("link") or ""),
|
||||
snippet=str(item.get("snippet") or ""),
|
||||
source_provider="serpapi",
|
||||
metadata={"position": item.get("position")},
|
||||
)
|
||||
for item in data.get("organic_results") or []
|
||||
if isinstance(item, dict) and item.get("link")
|
||||
]
|
||||
|
||||
async def _search_exa(
|
||||
self,
|
||||
config: WebSearchProviderConfig,
|
||||
query: str,
|
||||
max_results: int,
|
||||
domains: list[str] | None,
|
||||
) -> list[SearchEvidence]:
|
||||
body: dict[str, Any] = {
|
||||
"query": query,
|
||||
"numResults": max_results,
|
||||
}
|
||||
if domains:
|
||||
body["includeDomains"] = domains
|
||||
if config.include_text:
|
||||
body["contents"] = {"text": True}
|
||||
data = await self._request_json(
|
||||
"POST",
|
||||
_join_url(config.base_url, config.endpoint_path or "/search"),
|
||||
provider_config=config,
|
||||
headers={"Authorization": f"Bearer {config.api_key}"},
|
||||
json=body,
|
||||
)
|
||||
return [
|
||||
SearchEvidence(
|
||||
title=str(item.get("title") or ""),
|
||||
url=str(item.get("url") or ""),
|
||||
snippet=str(item.get("summary") or ""),
|
||||
content=str(item.get("text") or ""),
|
||||
score=_float_or_none(item.get("score")),
|
||||
source_provider="exa",
|
||||
metadata={"id": item.get("id")},
|
||||
)
|
||||
for item in data.get("results") or []
|
||||
if isinstance(item, dict) and item.get("url")
|
||||
]
|
||||
|
||||
async def _search_firecrawl(
|
||||
self,
|
||||
config: WebSearchProviderConfig,
|
||||
query: str,
|
||||
max_results: int,
|
||||
) -> list[SearchEvidence]:
|
||||
data = await self._request_json(
|
||||
"POST",
|
||||
_join_url(config.base_url, config.search_path or "/v2/search"),
|
||||
provider_config=config,
|
||||
headers={"Authorization": f"Bearer {config.api_key}"},
|
||||
json={"query": query, "limit": max_results},
|
||||
)
|
||||
raw_results = data.get("data") or data.get("results") or []
|
||||
return [
|
||||
SearchEvidence(
|
||||
title=str(item.get("title") or ""),
|
||||
url=str(item.get("url") or item.get("sourceURL") or ""),
|
||||
snippet=str(item.get("description") or item.get("markdown") or ""),
|
||||
source_provider="firecrawl",
|
||||
metadata={"status": item.get("status")},
|
||||
)
|
||||
for item in raw_results
|
||||
if isinstance(item, dict) and (item.get("url") or item.get("sourceURL"))
|
||||
]
|
||||
|
||||
async def _search_searxng(
|
||||
self,
|
||||
config: WebSearchProviderConfig,
|
||||
query: str,
|
||||
max_results: int,
|
||||
domains: list[str] | None,
|
||||
) -> list[SearchEvidence]:
|
||||
search_query = query
|
||||
if domains:
|
||||
search_query = f"{query} " + " ".join(f"site:{domain}" for domain in domains)
|
||||
params: dict[str, Any] = {
|
||||
"q": search_query,
|
||||
"format": "json",
|
||||
"categories": config.categories or "general",
|
||||
}
|
||||
if config.engines:
|
||||
params["engines"] = ",".join(config.engines)
|
||||
headers = {"Authorization": f"Bearer {config.api_key}"} if config.api_key else None
|
||||
data = await self._request_json(
|
||||
"GET",
|
||||
_join_url(config.base_url, config.endpoint_path or "/"),
|
||||
provider_config=config,
|
||||
headers=headers,
|
||||
params=params,
|
||||
)
|
||||
results = data.get("results") or []
|
||||
evidence = [
|
||||
SearchEvidence(
|
||||
title=str(item.get("title") or ""),
|
||||
url=str(item.get("url") or ""),
|
||||
snippet=str(item.get("content") or ""),
|
||||
score=_float_or_none(item.get("score")),
|
||||
source_provider="searxng",
|
||||
metadata={"engine": item.get("engine")},
|
||||
)
|
||||
for item in results
|
||||
if isinstance(item, dict) and item.get("url")
|
||||
]
|
||||
return evidence[:max_results]
|
||||
|
||||
|
||||
def _join_url(base_url: str, path: str) -> str:
|
||||
return f"{(base_url or '').rstrip('/')}/{(path or '').lstrip('/')}"
|
||||
|
||||
|
||||
def _float_or_none(value: Any) -> float | None:
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
Reference in New Issue
Block a user