49 lines
1.5 KiB
Python
49 lines
1.5 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from typing import Iterable
|
|
|
|
from app.services.ai_tools.schemas import FetchedEvidence, SearchEvidence
|
|
|
|
|
|
def evidence_content_hash(text: str) -> str:
|
|
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def normalize_search_evidence(items: Iterable[SearchEvidence], *, limit: int = 5) -> list[dict]:
|
|
normalized: list[dict] = []
|
|
seen_urls: set[str] = set()
|
|
for item in items:
|
|
if not item.url or item.url in seen_urls:
|
|
continue
|
|
seen_urls.add(item.url)
|
|
normalized.append(
|
|
{
|
|
"title": item.title,
|
|
"url": item.url,
|
|
"snippet": item.snippet,
|
|
"content": item.compact_text(),
|
|
"score": item.score,
|
|
"source_provider": item.source_provider,
|
|
"retrieved_at": item.retrieved_at.isoformat(),
|
|
"metadata": item.metadata,
|
|
}
|
|
)
|
|
if len(normalized) >= limit:
|
|
break
|
|
return normalized
|
|
|
|
|
|
def normalize_fetched_evidence(item: FetchedEvidence, *, text_limit: int = 1200) -> dict:
|
|
text = " ".join(item.text.split())[:text_limit]
|
|
return {
|
|
"title": item.title,
|
|
"url": item.final_url or item.url,
|
|
"text": text,
|
|
"content_hash": item.content_hash or evidence_content_hash(item.text),
|
|
"extractor": item.extractor,
|
|
"retrieved_at": item.retrieved_at.isoformat(),
|
|
"metadata": item.metadata,
|
|
}
|
|
|