from __future__ import annotations import hashlib from typing import Iterable from app.services.ai_tools.schemas import FetchedEvidence, SearchEvidence def evidence_content_hash(text: str) -> str: return hashlib.sha256(text.encode("utf-8")).hexdigest() def normalize_search_evidence(items: Iterable[SearchEvidence], *, limit: int = 5) -> list[dict]: normalized: list[dict] = [] seen_urls: set[str] = set() for item in items: if not item.url or item.url in seen_urls: continue seen_urls.add(item.url) normalized.append( { "title": item.title, "url": item.url, "snippet": item.snippet, "content": item.compact_text(), "score": item.score, "source_provider": item.source_provider, "retrieved_at": item.retrieved_at.isoformat(), "metadata": item.metadata, } ) if len(normalized) >= limit: break return normalized def normalize_fetched_evidence(item: FetchedEvidence, *, text_limit: int = 1200) -> dict: text = " ".join(item.text.split())[:text_limit] return { "title": item.title, "url": item.final_url or item.url, "text": text, "content_hash": item.content_hash or evidence_content_hash(item.text), "extractor": item.extractor, "retrieved_at": item.retrieved_at.isoformat(), "metadata": item.metadata, }