439 lines
16 KiB
Python
439 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
from sqlalchemy import func, or_, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.models.earth_news import EarthNewsItem
|
|
from app.services.earth_news import (
|
|
ParsedNewsItem,
|
|
apply_enrichment_patch_to_item,
|
|
build_anchor_location_patch,
|
|
_news_meta_patch,
|
|
)
|
|
from app.services.earth_news_classification import (
|
|
breaking_sort_rank,
|
|
normalize_breaking_level,
|
|
normalize_breaking_scope,
|
|
)
|
|
|
|
|
|
def _coerce_datetime(value: datetime | None) -> datetime | None:
|
|
if value is None:
|
|
return None
|
|
if value.tzinfo is None:
|
|
return value.replace(tzinfo=UTC)
|
|
return value.astimezone(UTC)
|
|
|
|
|
|
def _coerce_meta_datetime(value: Any) -> datetime | None:
|
|
if isinstance(value, datetime):
|
|
return _coerce_datetime(value)
|
|
if not isinstance(value, str) or not value.strip():
|
|
return None
|
|
try:
|
|
return _coerce_datetime(datetime.fromisoformat(value.replace("Z", "+00:00")))
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _location_patch_from_record(record: EarthNewsItem) -> dict[str, Any]:
|
|
return {
|
|
"latitude": record.latitude,
|
|
"longitude": record.longitude,
|
|
"location_label": record.location_label,
|
|
"location_source": record.location_source,
|
|
"verified": record.verified,
|
|
"location_meta": dict(record.location_meta or {}),
|
|
}
|
|
|
|
|
|
def record_to_parsed_news_item(record: EarthNewsItem) -> ParsedNewsItem:
|
|
location_meta = dict(record.location_meta or {})
|
|
news_meta = location_meta.get("news_meta") if isinstance(location_meta.get("news_meta"), dict) else {}
|
|
item = ParsedNewsItem(
|
|
id=record.id,
|
|
title=record.title,
|
|
summary=record.summary or "",
|
|
url=record.url,
|
|
source=record.source or "",
|
|
feed_name=record.feed_name or "",
|
|
feed_region=record.region or "global",
|
|
homepage_url=record.homepage_url or "",
|
|
published_at=_coerce_datetime(record.published_at),
|
|
content_language=record.content_language or "en",
|
|
localizations=dict(record.localizations or {}),
|
|
enrichment_status=record.enrichment_status or "pending",
|
|
enrichment_error=record.enrichment_error,
|
|
enriched_at=_coerce_datetime(record.enriched_at),
|
|
source_tags=list(news_meta.get("source_tags") or []),
|
|
feed_id=str(news_meta.get("feed_id") or ""),
|
|
feed_type=str(news_meta.get("feed_type") or "rss"),
|
|
feed_default_category=str(news_meta.get("feed_default_category") or "other"),
|
|
category=str(news_meta.get("category") or "other"),
|
|
item_tags=list(news_meta.get("item_tags") or []),
|
|
tagging_source=str(news_meta.get("tagging_source") or "rules"),
|
|
tagging_confidence=float(news_meta.get("tagging_confidence") or 0),
|
|
importance_score=int(news_meta.get("importance_score") or 0),
|
|
importance_level=str(news_meta.get("importance_level") or "low"),
|
|
importance_reasons=list(news_meta.get("importance_reasons") or []),
|
|
market_impact=str(news_meta.get("market_impact") or "none"),
|
|
breaking_level=normalize_breaking_level(news_meta.get("breaking_level")).value,
|
|
breaking_scope=normalize_breaking_scope(news_meta.get("breaking_scope")).value,
|
|
breaking_reasons=list(news_meta.get("breaking_reasons") or []),
|
|
breaking_source=str(news_meta.get("breaking_source") or "rules"),
|
|
breaking_confidence=float(news_meta.get("breaking_confidence") or 0),
|
|
breaking_expires_at=_coerce_meta_datetime(news_meta.get("breaking_expires_at")),
|
|
)
|
|
return apply_enrichment_patch_to_item(item, _location_patch_from_record(record))
|
|
|
|
|
|
def _sort_parsed_news_items(items: list[ParsedNewsItem], *, active_region: str) -> list[ParsedNewsItem]:
|
|
return sorted(
|
|
items,
|
|
key=lambda item: (
|
|
-breaking_sort_rank(item),
|
|
False
|
|
if active_region == "global"
|
|
or (breaking_sort_rank(item) > 0 and normalize_breaking_scope(item.breaking_scope).value == "global")
|
|
else item.feed_region != active_region,
|
|
item.published_at is None,
|
|
-(item.published_at.timestamp() if item.published_at else 0),
|
|
item.feed_name,
|
|
),
|
|
)
|
|
|
|
|
|
def _query_sort_key(active_region: str):
|
|
if active_region == "global":
|
|
return (
|
|
EarthNewsItem.published_at.is_(None),
|
|
EarthNewsItem.published_at.desc().nullslast(),
|
|
EarthNewsItem.feed_name.asc(),
|
|
)
|
|
return (
|
|
EarthNewsItem.region != active_region,
|
|
EarthNewsItem.published_at.is_(None),
|
|
EarthNewsItem.published_at.desc().nullslast(),
|
|
EarthNewsItem.feed_name.asc(),
|
|
)
|
|
|
|
|
|
def _category_filter_clause(categories: set[str] | None):
|
|
if not categories:
|
|
return None
|
|
return EarthNewsItem.location_meta.op("->")("news_meta").op("->>")("category").in_(sorted(categories))
|
|
|
|
|
|
def _source_filter_clause(source_ids: set[str] | None):
|
|
if not source_ids:
|
|
return None
|
|
return EarthNewsItem.location_meta.op("->")("news_meta").op("->>")("source_id").in_(sorted(source_ids))
|
|
|
|
|
|
def _record_source_id(record: EarthNewsItem) -> str:
|
|
location_meta = dict(record.location_meta or {})
|
|
news_meta = location_meta.get("news_meta") if isinstance(location_meta.get("news_meta"), dict) else {}
|
|
source_id = str(news_meta.get("source_id") or "").strip()
|
|
if source_id:
|
|
return source_id
|
|
if isinstance(record.id, str) and ":" in record.id:
|
|
return record.id.split(":", 1)[0]
|
|
return record.feed_name or record.source or record.id
|
|
|
|
|
|
def _diversify_records_by_source(records: list[EarthNewsItem], *, limit: int) -> list[EarthNewsItem]:
|
|
if limit <= 0 or len(records) <= limit:
|
|
return records[:limit]
|
|
buckets: dict[str, list[EarthNewsItem]] = {}
|
|
order: list[str] = []
|
|
for record in records:
|
|
source_id = _record_source_id(record)
|
|
if source_id not in buckets:
|
|
buckets[source_id] = []
|
|
order.append(source_id)
|
|
buckets[source_id].append(record)
|
|
|
|
diversified: list[EarthNewsItem] = []
|
|
while len(diversified) < limit and order:
|
|
next_order: list[str] = []
|
|
for source_id in order:
|
|
bucket = buckets.get(source_id) or []
|
|
if bucket and len(diversified) < limit:
|
|
diversified.append(bucket.pop(0))
|
|
if bucket:
|
|
next_order.append(source_id)
|
|
order = next_order
|
|
return diversified
|
|
|
|
|
|
async def list_earth_news_items(
|
|
db: AsyncSession,
|
|
*,
|
|
active_region: str,
|
|
limit: int,
|
|
categories: set[str] | None = None,
|
|
source_ids: set[str] | None = None,
|
|
) -> list[ParsedNewsItem]:
|
|
query_limit = limit if source_ids else min(max(limit * 8, limit), 200)
|
|
query = (
|
|
select(EarthNewsItem)
|
|
.order_by(*_query_sort_key(active_region))
|
|
.limit(query_limit)
|
|
)
|
|
if active_region != "global":
|
|
news_meta = EarthNewsItem.location_meta.op("->")("news_meta")
|
|
query = query.where(
|
|
or_(
|
|
EarthNewsItem.region.in_({"global", active_region}),
|
|
news_meta.op("->>")("breaking_scope") == "global",
|
|
)
|
|
)
|
|
category_clause = _category_filter_clause(categories)
|
|
if category_clause is not None:
|
|
query = query.where(category_clause)
|
|
source_clause = _source_filter_clause(source_ids)
|
|
if source_clause is not None:
|
|
query = query.where(source_clause)
|
|
result = await db.execute(query)
|
|
records = list(result.scalars().all())
|
|
items = _sort_parsed_news_items(
|
|
[record_to_parsed_news_item(record) for record in records],
|
|
active_region=active_region,
|
|
)
|
|
return items[:limit]
|
|
|
|
|
|
async def list_earth_news_cruise_items(
|
|
db: AsyncSession,
|
|
*,
|
|
limit: int,
|
|
categories: set[str] | None = None,
|
|
source_ids: set[str] | None = None,
|
|
) -> list[ParsedNewsItem]:
|
|
query = (
|
|
select(EarthNewsItem)
|
|
.order_by(
|
|
EarthNewsItem.region.asc(),
|
|
EarthNewsItem.published_at.desc().nullslast(),
|
|
EarthNewsItem.last_seen_at.desc(),
|
|
EarthNewsItem.feed_name.asc(),
|
|
)
|
|
.limit(min(max(limit * 4, limit), 200))
|
|
)
|
|
category_clause = _category_filter_clause(categories)
|
|
if category_clause is not None:
|
|
query = query.where(category_clause)
|
|
source_clause = _source_filter_clause(source_ids)
|
|
if source_clause is not None:
|
|
query = query.where(source_clause)
|
|
result = await db.execute(query)
|
|
items = _sort_parsed_news_items(
|
|
[record_to_parsed_news_item(record) for record in result.scalars().all()],
|
|
active_region="global",
|
|
)
|
|
return items[:limit]
|
|
|
|
|
|
async def get_earth_news_freshness(
|
|
db: AsyncSession,
|
|
*,
|
|
active_region: str,
|
|
) -> tuple[int, datetime | None]:
|
|
query = select(
|
|
func.count(EarthNewsItem.id),
|
|
func.max(func.coalesce(EarthNewsItem.published_at, EarthNewsItem.last_seen_at)),
|
|
)
|
|
if active_region != "global":
|
|
query = query.where(EarthNewsItem.region.in_({"global", active_region}))
|
|
result = await db.execute(query)
|
|
count, newest = result.one()
|
|
item_count = int(count or 0)
|
|
if item_count == 0:
|
|
return 0, None
|
|
return item_count, _coerce_datetime(newest)
|
|
|
|
|
|
async def get_earth_news_feed_coverage(
|
|
db: AsyncSession,
|
|
*,
|
|
active_region: str,
|
|
recent_after: datetime | None = None,
|
|
) -> set[tuple[str, str]]:
|
|
query = select(
|
|
EarthNewsItem.id,
|
|
EarthNewsItem.location_meta.op("->")("news_meta").op("->>")("source_id"),
|
|
EarthNewsItem.location_meta.op("->")("news_meta").op("->>")("feed_id"),
|
|
)
|
|
if active_region != "global":
|
|
query = query.where(EarthNewsItem.region.in_({"global", active_region}))
|
|
if recent_after is not None:
|
|
query = query.where(func.coalesce(EarthNewsItem.published_at, EarthNewsItem.last_seen_at) >= recent_after)
|
|
result = await db.execute(query)
|
|
coverage: set[tuple[str, str]] = set()
|
|
for item_id, source_id, feed_id in result.all():
|
|
normalized_source_id = str(source_id or "").strip()
|
|
normalized_feed_id = str(feed_id or "").strip()
|
|
if not normalized_source_id and isinstance(item_id, str) and ":" in item_id:
|
|
normalized_source_id = item_id.split(":", 1)[0]
|
|
if normalized_source_id and normalized_feed_id:
|
|
coverage.add((normalized_source_id, normalized_feed_id))
|
|
return coverage
|
|
|
|
|
|
async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem]) -> int:
|
|
if not items:
|
|
return 0
|
|
now = datetime.now(UTC)
|
|
existing_result = await db.execute(
|
|
select(EarthNewsItem).where(EarthNewsItem.id.in_([item.id for item in items]))
|
|
)
|
|
existing = {record.id: record for record in existing_result.scalars().all()}
|
|
changed = 0
|
|
for item in items:
|
|
record = existing.get(item.id)
|
|
if record is None:
|
|
patch = build_anchor_location_patch(item)
|
|
record = EarthNewsItem(
|
|
id=item.id,
|
|
title=item.title,
|
|
summary=item.summary,
|
|
content_language=item.content_language,
|
|
localizations=dict(item.localizations or {}),
|
|
url=item.url,
|
|
source=item.source,
|
|
feed_name=item.feed_name,
|
|
region=item.feed_region,
|
|
homepage_url=item.homepage_url,
|
|
published_at=item.published_at,
|
|
latitude=patch["latitude"],
|
|
longitude=patch["longitude"],
|
|
location_label=patch["location_label"],
|
|
location_source=patch["location_source"],
|
|
verified=patch["verified"],
|
|
location_meta=patch["location_meta"],
|
|
first_seen_at=now,
|
|
last_seen_at=now,
|
|
enrichment_status=item.enrichment_status,
|
|
enrichment_error=item.enrichment_error,
|
|
enriched_at=item.enriched_at,
|
|
)
|
|
db.add(record)
|
|
changed += 1
|
|
continue
|
|
|
|
record.title = item.title
|
|
record.summary = item.summary
|
|
record.url = item.url
|
|
record.source = item.source
|
|
record.feed_name = item.feed_name
|
|
record.region = item.feed_region
|
|
record.homepage_url = item.homepage_url
|
|
record.published_at = item.published_at
|
|
record.last_seen_at = now
|
|
location_meta = dict(record.location_meta or {})
|
|
location_meta["news_meta"] = _news_meta_patch(item)
|
|
record.location_meta = location_meta
|
|
if item.localizations:
|
|
merged_localizations = {
|
|
**dict(record.localizations or {}),
|
|
**dict(item.localizations or {}),
|
|
}
|
|
record.content_language = item.content_language
|
|
record.localizations = merged_localizations
|
|
if item.enrichment_status != "pending" or item.enrichment_error or item.enriched_at:
|
|
record.enrichment_status = item.enrichment_status
|
|
record.enrichment_error = item.enrichment_error
|
|
record.enriched_at = item.enriched_at
|
|
changed += 1
|
|
await db.flush()
|
|
return changed
|
|
|
|
|
|
async def update_earth_news_item_location(
|
|
db: AsyncSession,
|
|
*,
|
|
item_id: str,
|
|
patch: dict[str, Any],
|
|
) -> bool:
|
|
record = await db.get(EarthNewsItem, item_id)
|
|
if record is None:
|
|
return False
|
|
record.latitude = float(patch["latitude"])
|
|
record.longitude = float(patch["longitude"])
|
|
record.location_label = str(patch["location_label"])
|
|
record.location_source = str(patch["location_source"])
|
|
record.verified = bool(patch["verified"])
|
|
record.location_meta = dict(patch.get("location_meta") or {})
|
|
record.resolved_at = datetime.now(UTC) if record.verified else None
|
|
await db.flush()
|
|
return True
|
|
|
|
|
|
async def update_earth_news_item_enrichment(
|
|
db: AsyncSession,
|
|
*,
|
|
item_id: str,
|
|
patch: dict[str, Any],
|
|
) -> bool:
|
|
record = await db.get(EarthNewsItem, item_id)
|
|
if record is None:
|
|
return False
|
|
if "latitude" in patch:
|
|
record.latitude = float(patch["latitude"])
|
|
record.longitude = float(patch["longitude"])
|
|
record.location_label = str(patch["location_label"])
|
|
record.location_source = str(patch["location_source"])
|
|
record.verified = bool(patch["verified"])
|
|
record.location_meta = dict(patch.get("location_meta") or {})
|
|
record.resolved_at = datetime.now(UTC) if record.verified else None
|
|
if "content_language" in patch:
|
|
record.content_language = str(patch.get("content_language") or "en")
|
|
if "localizations" in patch:
|
|
record.localizations = dict(patch.get("localizations") or {})
|
|
if "enrichment_status" in patch:
|
|
record.enrichment_status = str(patch.get("enrichment_status") or "pending")
|
|
if "enrichment_error" in patch:
|
|
record.enrichment_error = patch.get("enrichment_error")
|
|
if patch.get("enriched_at"):
|
|
try:
|
|
parsed_enriched_at = datetime.fromisoformat(
|
|
str(patch["enriched_at"]).replace("Z", "+00:00")
|
|
)
|
|
except ValueError:
|
|
parsed_enriched_at = datetime.now(UTC)
|
|
record.enriched_at = _coerce_datetime(parsed_enriched_at)
|
|
elif patch.get("localizations"):
|
|
record.enriched_at = datetime.now(UTC)
|
|
await db.flush()
|
|
return True
|
|
|
|
|
|
async def list_unverified_earth_news_items(
|
|
db: AsyncSession,
|
|
*,
|
|
active_region: str,
|
|
limit: int,
|
|
) -> list[ParsedNewsItem]:
|
|
regions = {"global", active_region}
|
|
result = await db.execute(
|
|
select(EarthNewsItem)
|
|
.where(EarthNewsItem.region.in_(regions))
|
|
.where(EarthNewsItem.verified.is_(False))
|
|
.order_by(*_query_sort_key(active_region))
|
|
.limit(limit)
|
|
)
|
|
return [record_to_parsed_news_item(record) for record in result.scalars().all()]
|
|
|
|
|
|
async def list_all_earth_news_records(db: AsyncSession) -> list[EarthNewsItem]:
|
|
result = await db.execute(
|
|
select(EarthNewsItem).order_by(
|
|
EarthNewsItem.published_at.desc().nullslast(),
|
|
EarthNewsItem.last_seen_at.desc(),
|
|
)
|
|
)
|
|
return list(result.scalars().all())
|