245 lines
8.3 KiB
Python
245 lines
8.3 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.models.earth_news import EarthNewsItem
|
|
from app.services.earth_news import (
|
|
ParsedNewsItem,
|
|
apply_enrichment_patch_to_item,
|
|
build_anchor_location_patch,
|
|
)
|
|
|
|
|
|
def _coerce_datetime(value: datetime | None) -> datetime | None:
|
|
if value is None:
|
|
return None
|
|
if value.tzinfo is None:
|
|
return value.replace(tzinfo=UTC)
|
|
return value.astimezone(UTC)
|
|
|
|
|
|
def _location_patch_from_record(record: EarthNewsItem) -> dict[str, Any]:
|
|
return {
|
|
"latitude": record.latitude,
|
|
"longitude": record.longitude,
|
|
"location_label": record.location_label,
|
|
"location_source": record.location_source,
|
|
"verified": record.verified,
|
|
"location_meta": dict(record.location_meta or {}),
|
|
}
|
|
|
|
|
|
def record_to_parsed_news_item(record: EarthNewsItem) -> ParsedNewsItem:
|
|
item = ParsedNewsItem(
|
|
id=record.id,
|
|
title=record.title,
|
|
summary=record.summary or "",
|
|
url=record.url,
|
|
source=record.source or "",
|
|
feed_name=record.feed_name or "",
|
|
feed_region=record.region or "global",
|
|
homepage_url=record.homepage_url or "",
|
|
published_at=_coerce_datetime(record.published_at),
|
|
content_language=record.content_language or "en",
|
|
localizations=dict(record.localizations or {}),
|
|
enrichment_status=record.enrichment_status or "pending",
|
|
enrichment_error=record.enrichment_error,
|
|
enriched_at=_coerce_datetime(record.enriched_at),
|
|
)
|
|
return apply_enrichment_patch_to_item(item, _location_patch_from_record(record))
|
|
|
|
|
|
def _query_sort_key(active_region: str):
|
|
return (
|
|
EarthNewsItem.region != active_region,
|
|
EarthNewsItem.published_at.is_(None),
|
|
EarthNewsItem.published_at.desc().nullslast(),
|
|
EarthNewsItem.feed_name.asc(),
|
|
)
|
|
|
|
|
|
async def list_earth_news_items(
|
|
db: AsyncSession,
|
|
*,
|
|
active_region: str,
|
|
limit: int,
|
|
) -> list[ParsedNewsItem]:
|
|
regions = {"global", active_region}
|
|
result = await db.execute(
|
|
select(EarthNewsItem)
|
|
.where(EarthNewsItem.region.in_(regions))
|
|
.order_by(*_query_sort_key(active_region))
|
|
.limit(limit)
|
|
)
|
|
return [record_to_parsed_news_item(record) for record in result.scalars().all()]
|
|
|
|
|
|
async def get_earth_news_freshness(
|
|
db: AsyncSession,
|
|
*,
|
|
active_region: str,
|
|
) -> tuple[int, datetime | None]:
|
|
regions = {"global", active_region}
|
|
result = await db.execute(
|
|
select(
|
|
func.count(EarthNewsItem.id),
|
|
func.max(func.coalesce(EarthNewsItem.published_at, EarthNewsItem.last_seen_at)),
|
|
).where(EarthNewsItem.region.in_(regions))
|
|
)
|
|
count, newest = result.one()
|
|
item_count = int(count or 0)
|
|
if item_count == 0:
|
|
return 0, None
|
|
return item_count, _coerce_datetime(newest)
|
|
|
|
|
|
async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem]) -> int:
|
|
if not items:
|
|
return 0
|
|
now = datetime.now(UTC)
|
|
existing_result = await db.execute(
|
|
select(EarthNewsItem).where(EarthNewsItem.id.in_([item.id for item in items]))
|
|
)
|
|
existing = {record.id: record for record in existing_result.scalars().all()}
|
|
changed = 0
|
|
for item in items:
|
|
record = existing.get(item.id)
|
|
if record is None:
|
|
patch = build_anchor_location_patch(item)
|
|
record = EarthNewsItem(
|
|
id=item.id,
|
|
title=item.title,
|
|
summary=item.summary,
|
|
content_language=item.content_language,
|
|
localizations=dict(item.localizations or {}),
|
|
url=item.url,
|
|
source=item.source,
|
|
feed_name=item.feed_name,
|
|
region=item.feed_region,
|
|
homepage_url=item.homepage_url,
|
|
published_at=item.published_at,
|
|
latitude=patch["latitude"],
|
|
longitude=patch["longitude"],
|
|
location_label=patch["location_label"],
|
|
location_source=patch["location_source"],
|
|
verified=patch["verified"],
|
|
location_meta=patch["location_meta"],
|
|
first_seen_at=now,
|
|
last_seen_at=now,
|
|
enrichment_status=item.enrichment_status,
|
|
enrichment_error=item.enrichment_error,
|
|
enriched_at=item.enriched_at,
|
|
)
|
|
db.add(record)
|
|
changed += 1
|
|
continue
|
|
|
|
record.title = item.title
|
|
record.summary = item.summary
|
|
record.url = item.url
|
|
record.source = item.source
|
|
record.feed_name = item.feed_name
|
|
record.region = item.feed_region
|
|
record.homepage_url = item.homepage_url
|
|
record.published_at = item.published_at
|
|
record.last_seen_at = now
|
|
if item.localizations:
|
|
record.content_language = item.content_language
|
|
record.localizations = dict(item.localizations or {})
|
|
record.enrichment_status = item.enrichment_status
|
|
record.enrichment_error = item.enrichment_error
|
|
record.enriched_at = item.enriched_at
|
|
changed += 1
|
|
await db.flush()
|
|
return changed
|
|
|
|
|
|
async def update_earth_news_item_location(
|
|
db: AsyncSession,
|
|
*,
|
|
item_id: str,
|
|
patch: dict[str, Any],
|
|
) -> bool:
|
|
record = await db.get(EarthNewsItem, item_id)
|
|
if record is None:
|
|
return False
|
|
record.latitude = float(patch["latitude"])
|
|
record.longitude = float(patch["longitude"])
|
|
record.location_label = str(patch["location_label"])
|
|
record.location_source = str(patch["location_source"])
|
|
record.verified = bool(patch["verified"])
|
|
record.location_meta = dict(patch.get("location_meta") or {})
|
|
record.resolved_at = datetime.now(UTC) if record.verified else None
|
|
await db.flush()
|
|
return True
|
|
|
|
|
|
async def update_earth_news_item_enrichment(
|
|
db: AsyncSession,
|
|
*,
|
|
item_id: str,
|
|
patch: dict[str, Any],
|
|
) -> bool:
|
|
record = await db.get(EarthNewsItem, item_id)
|
|
if record is None:
|
|
return False
|
|
if "latitude" in patch:
|
|
record.latitude = float(patch["latitude"])
|
|
record.longitude = float(patch["longitude"])
|
|
record.location_label = str(patch["location_label"])
|
|
record.location_source = str(patch["location_source"])
|
|
record.verified = bool(patch["verified"])
|
|
record.location_meta = dict(patch.get("location_meta") or {})
|
|
record.resolved_at = datetime.now(UTC) if record.verified else None
|
|
if "content_language" in patch:
|
|
record.content_language = str(patch.get("content_language") or "en")
|
|
if "localizations" in patch:
|
|
record.localizations = dict(patch.get("localizations") or {})
|
|
if "enrichment_status" in patch:
|
|
record.enrichment_status = str(patch.get("enrichment_status") or "pending")
|
|
if "enrichment_error" in patch:
|
|
record.enrichment_error = patch.get("enrichment_error")
|
|
if patch.get("enriched_at"):
|
|
try:
|
|
parsed_enriched_at = datetime.fromisoformat(
|
|
str(patch["enriched_at"]).replace("Z", "+00:00")
|
|
)
|
|
except ValueError:
|
|
parsed_enriched_at = datetime.now(UTC)
|
|
record.enriched_at = _coerce_datetime(parsed_enriched_at)
|
|
elif patch.get("localizations"):
|
|
record.enriched_at = datetime.now(UTC)
|
|
await db.flush()
|
|
return True
|
|
|
|
|
|
async def list_unverified_earth_news_items(
|
|
db: AsyncSession,
|
|
*,
|
|
active_region: str,
|
|
limit: int,
|
|
) -> list[ParsedNewsItem]:
|
|
regions = {"global", active_region}
|
|
result = await db.execute(
|
|
select(EarthNewsItem)
|
|
.where(EarthNewsItem.region.in_(regions))
|
|
.where(EarthNewsItem.verified.is_(False))
|
|
.order_by(*_query_sort_key(active_region))
|
|
.limit(limit)
|
|
)
|
|
return [record_to_parsed_news_item(record) for record in result.scalars().all()]
|
|
|
|
|
|
async def list_all_earth_news_records(db: AsyncSession) -> list[EarthNewsItem]:
|
|
result = await db.execute(
|
|
select(EarthNewsItem).order_by(
|
|
EarthNewsItem.published_at.desc().nullslast(),
|
|
EarthNewsItem.last_seen_at.desc(),
|
|
)
|
|
)
|
|
return list(result.scalars().all())
|