planet/backend/app/services/earth_news.py

from __future__ import annotations

import asyncio
from dataclasses import dataclass, field
from datetime import UTC, datetime
from email.utils import parsedate_to_datetime
import hashlib
import html
import json
import math
import re
from typing import Any
from urllib.parse import quote
import xml.etree.ElementTree as ET

import httpx
from bs4 import BeautifulSoup
from sqlalchemy.ext.asyncio import AsyncSession

from app.core.countries import COUNTRY_VARIANTS_MAP, get_country_centroid, normalize_country
from app.ai_tasks.prompts import EffectiveAIPrompt, get_effective_prompt
from app.schemas.ai import SituationalAnalysisRequest
from app.services.ai_client import AIProviderClient
from app.services.location.resolvers.nominatim import build_default_nominatim_geocoder


USER_AGENT = "PlanetEarthNewsBoard/1.0 (+https://planet.local)"
REQUEST_TIMEOUT = 12.0
MAX_ITEMS_PER_SOURCE = 6
MAX_ITEMS_TOTAL = 12
STALE_CACHE_MAX_AGE_SECONDS = 60 * 45
RSS_SUPPLEMENT_MAX_AGE_SECONDS = STALE_CACHE_MAX_AGE_SECONDS
MAX_TARGET_INFERENCE_CONCURRENCY = 3
TARGET_INFERENCE_TIMEOUT_SECONDS = 6.0
DEFAULT_NEWS_LOCALE = "zh-CN"
NEWS_ENRICH_PROMPT_KEY = "earth.news.enrich"


@dataclass(frozen=True)
class RegionProfile:
    key: str
    label: str
    query: str
    accent: str


@dataclass(frozen=True)
class RegionAnchor:
    region: str
    label: str
    latitude: float
    longitude: float


@dataclass(frozen=True)
class NewsFeedSource:
    id: str
    name: str
    region: str
    feed_url: str
    homepage_url: str
    source_type: str = "rss"
    priority: int = 100


@dataclass(frozen=True)
class NewsTargetLocation:
    latitude: float
    longitude: float
    label: str
    source: str
    confidence: float | None = None
    country: str | None = None
    city: str | None = None


@dataclass
class ParsedNewsItem:
    id: str
    title: str
    summary: str
    url: str
    source: str
    feed_name: str
    feed_region: str
    homepage_url: str
    published_at: datetime | None
    content_language: str = "en"
    localizations: dict[str, dict[str, str]] = field(default_factory=dict)
    enrichment_status: str = "pending"
    enrichment_error: str | None = None
    enriched_at: datetime | None = None
    target_location: NewsTargetLocation | None = None
    target_resolution_stage: str = "unresolved"
    target_ai_attempted: bool = False
    target_ai_status: str = "not_attempted"
    target_ai_error: str | None = None
    target_debug_note: str | None = None
    location_patch: dict[str, Any] | None = None


@dataclass
class CachedRegionFeed:
    region: str
    fetched_at: datetime
    items: list[ParsedNewsItem]
    sources: list[NewsFeedSource]


REGION_PROFILES: dict[str, RegionProfile] = {
    "americas": RegionProfile(
        key="americas",
        label="美洲焦点",
        query='Americas geopolitics OR Latin America OR "United States" OR Canada',
        accent="#79d3ff",
    ),
    "europe": RegionProfile(
        key="europe",
        label="欧洲焦点",
        query='Europe geopolitics OR EU OR NATO OR "Eastern Europe"',
        accent="#8fd4ff",
    ),
    "middle-east-africa": RegionProfile(
        key="middle-east-africa",
        label="中东与非洲焦点",
        query='"Middle East" OR Africa geopolitics OR Red Sea OR Gulf',
        accent="#ffb56a",
    ),
    "asia-pacific": RegionProfile(
        key="asia-pacific",
        label="亚太焦点",
        query='"Asia Pacific" OR Indo-Pacific OR China OR Japan OR Korea OR ASEAN',
        accent="#78f2cf",
    ),
    "global": RegionProfile(
        key="global",
        label="全球焦点",
        query='"world news" OR geopolitics OR "global affairs"',
        accent="#d6e6ff",
    ),
}

REGION_ANCHORS: dict[str, RegionAnchor] = {
    "americas": RegionAnchor(
        region="americas",
        label="美洲",
        latitude=37.0902,
        longitude=-95.7129,
    ),
    "europe": RegionAnchor(
        region="europe",
        label="欧洲",
        latitude=50.1109,
        longitude=8.6821,
    ),
    "middle-east-africa": RegionAnchor(
        region="middle-east-africa",
        label="中东与非洲",
        latitude=25.2048,
        longitude=55.2708,
    ),
    "asia-pacific": RegionAnchor(
        region="asia-pacific",
        label="亚太",
        latitude=1.3521,
        longitude=103.8198,
    ),
    "global": RegionAnchor(
        region="global",
        label="全球",
        latitude=20.0,
        longitude=0.0,
    ),
}


def _google_news_feed(query: str, *, hl: str, gl: str, ceid: str) -> str:
    return (
        "https://news.google.com/rss/search?q="
        + quote(query, safe="")
        + f"&hl={hl}&gl={gl}&ceid={ceid}"
    )


NEWS_FEED_SOURCES: tuple[NewsFeedSource, ...] = (
    NewsFeedSource(
        id="bbc-world",
        name="BBC World",
        region="global",
        feed_url="https://feeds.bbci.co.uk/news/world/rss.xml",
        homepage_url="https://www.bbc.com/news/world",
        priority=10,
    ),
    NewsFeedSource(
        id="dw-top",
        name="DW Top Stories",
        region="europe",
        feed_url="https://rss.dw.com/rdf/rss-en-top",
        homepage_url="https://www.dw.com/en/top-stories/s-9097",
        priority=20,
    ),
    NewsFeedSource(
        id="global-scan",
        name="Global Monitor / World",
        region="global",
        feed_url=_google_news_feed(
            REGION_PROFILES["global"].query,
            hl="en-US",
            gl="US",
            ceid="US:en",
        ),
        homepage_url="https://news.google.com/",
        source_type="aggregated",
        priority=30,
    ),
    NewsFeedSource(
        id="google-americas",
        name="Global Monitor / Americas",
        region="americas",
        feed_url=_google_news_feed(
            REGION_PROFILES["americas"].query,
            hl="en-US",
            gl="US",
            ceid="US:en",
        ),
        homepage_url="https://news.google.com/",
        source_type="aggregated",
        priority=40,
    ),
    NewsFeedSource(
        id="google-europe",
        name="Global Monitor / Europe",
        region="europe",
        feed_url=_google_news_feed(
            REGION_PROFILES["europe"].query,
            hl="en-GB",
            gl="GB",
            ceid="GB:en",
        ),
        homepage_url="https://news.google.com/",
        source_type="aggregated",
        priority=40,
    ),
    NewsFeedSource(
        id="google-mea",
        name="Global Monitor / MEA",
        region="middle-east-africa",
        feed_url=_google_news_feed(
            REGION_PROFILES["middle-east-africa"].query,
            hl="en-US",
            gl="US",
            ceid="US:en",
        ),
        homepage_url="https://news.google.com/",
        source_type="aggregated",
        priority=40,
    ),
    NewsFeedSource(
        id="google-apac",
        name="Global Monitor / APAC",
        region="asia-pacific",
        feed_url=_google_news_feed(
            REGION_PROFILES["asia-pacific"].query,
            hl="en-SG",
            gl="SG",
            ceid="SG:en",
        ),
        homepage_url="https://news.google.com/",
        source_type="aggregated",
        priority=40,
    ),
)


_REGION_CACHE: dict[str, CachedRegionFeed] = {}
_news_target_geocode = build_default_nominatim_geocoder(user_agent=USER_AGENT)
_CITY_HINTS: tuple[dict[str, str | None], ...] = (
    {"name": "Beijing", "country": "中国"},
    {"name": "Havana", "country": "古巴"},
    {"name": "Kyiv", "country": "乌克兰"},
    {"name": "Bangkok", "country": "泰国"},
    {"name": "Tehran", "country": "伊朗"},
    {"name": "Moscow", "country": "俄罗斯"},
    {"name": "Taipei", "country": "中国（台湾）"},
    {"name": "Hong Kong", "country": "中国（香港）"},
)


def determine_focus_region(lat: float | None, lon: float | None) -> str:
    if lat is None or lon is None:
        return "global"
    if -170 <= lon <= -30:
        return "americas"
    if -30 < lon <= 45:
        return "europe" if lat >= 30 else "middle-east-africa"
    if 45 < lon <= 150:
        return "middle-east-africa" if lat < 10 else "asia-pacific"
    return "asia-pacific"


def get_region_profile(region: str) -> RegionProfile:
    return REGION_PROFILES.get(region, REGION_PROFILES["global"])


def get_region_anchor(region: str) -> RegionAnchor:
    return REGION_ANCHORS.get(region, REGION_ANCHORS["global"])


def _coerce_str(value: Any) -> str | None:
    if value is None:
        return None
    if not isinstance(value, str):
        value = str(value)
    cleaned = re.sub(r"\s+", " ", value).strip()
    return cleaned or None


def _contains_location_alias(text: str, alias: str) -> bool:
    normalized_alias = _coerce_str(alias)
    if not normalized_alias:
        return False
    if re.search(r"[A-Za-z]", normalized_alias):
        pattern = r"(?<![A-Za-z])" + re.escape(normalized_alias) + r"(?![A-Za-z])"
        return re.search(pattern, text, flags=re.IGNORECASE) is not None
    return normalized_alias in text


def _iter_searchable_country_variants(
    canonical: str,
    variants: list[str],
) -> tuple[str, ...]:
    searchable: list[str] = []
    seen: set[str] = set()
    for variant in (canonical, *variants):
        normalized = _coerce_str(variant)
        if not normalized:
            continue
        if re.fullmatch(r"[A-Z]{2,3}", normalized):
            continue
        if len(normalized) <= 2:
            continue
        key = normalized.casefold()
        if key in seen:
            continue
        seen.add(key)
        searchable.append(normalized)
    return tuple(searchable)


def _coerce_float(value: Any) -> float | None:
    try:
        parsed = float(value)
    except (TypeError, ValueError):
        return None
    if not math.isfinite(parsed):
        return None
    return parsed


def _first_json_object(text: str) -> dict[str, Any] | None:
    if not text:
        return None
    decoder = json.JSONDecoder()
    for index, char in enumerate(text):
        if char != "{":
            continue
        try:
            payload, _ = decoder.raw_decode(text[index:])
        except ValueError:
            continue
        if isinstance(payload, dict):
            return payload
    return None


def _normalize_localizations(value: Any) -> dict[str, dict[str, str]]:
    if not isinstance(value, dict):
        return {}
    normalized: dict[str, dict[str, str]] = {}
    for locale, payload in value.items():
        locale_key = _coerce_str(locale)
        if not locale_key or not isinstance(payload, dict):
            continue
        title = _coerce_str(payload.get("title"))
        summary = _coerce_str(payload.get("summary"))
        entry: dict[str, str] = {}
        if title:
            entry["title"] = title
        if summary:
            entry["summary"] = summary
        if entry:
            normalized[locale_key] = entry
    return normalized


def _get_locale_text(
    item: ParsedNewsItem,
    key: str,
    *,
    locale: str = DEFAULT_NEWS_LOCALE,
) -> str:
    localized = item.localizations.get(locale)
    if isinstance(localized, dict):
        value = _coerce_str(localized.get(key))
        if value:
            return value
    return ""


def _has_default_localization(item: ParsedNewsItem) -> bool:
    localized = item.localizations.get(DEFAULT_NEWS_LOCALE)
    if not isinstance(localized, dict):
        return False
    return bool(_coerce_str(localized.get("title")) and _coerce_str(localized.get("summary")))


def apply_enrichment_patch_to_item(
    item: ParsedNewsItem,
    patch: dict[str, Any],
) -> ParsedNewsItem:
    item.location_patch = patch
    if "content_language" in patch:
        item.content_language = _coerce_str(patch.get("content_language")) or item.content_language
    if "localizations" in patch:
        item.localizations = _normalize_localizations(patch.get("localizations"))
    if "enrichment_status" in patch:
        item.enrichment_status = _coerce_str(patch.get("enrichment_status")) or item.enrichment_status
    if "enrichment_error" in patch:
        item.enrichment_error = _coerce_str(patch.get("enrichment_error"))
    if "enriched_at" in patch:
        item.enriched_at = _parse_datetime(_coerce_str(patch.get("enriched_at")))
    return item


async def _geocode_target_location(query: str) -> dict[str, Any] | None:
    return await asyncio.to_thread(_news_target_geocode, query)


async def _build_target_location_from_payload(
    payload: dict[str, Any],
) -> NewsTargetLocation | None:
    country = normalize_country(payload.get("country"))
    city = _coerce_str(payload.get("city"))
    matched_location_name = _coerce_str(payload.get("matched_location_name"))
    confidence = _coerce_float(payload.get("confidence"))
    if confidence is not None:
        confidence = max(0.0, min(confidence, 1.0))

    latitude = _coerce_float(payload.get("latitude"))
    longitude = _coerce_float(payload.get("longitude"))
    if latitude not in (None, 0.0) and longitude not in (None, 0.0):
        label = matched_location_name or ", ".join(part for part in (city, country) if part) or "关联位置"
        return NewsTargetLocation(
            latitude=latitude,
            longitude=longitude,
            label=label,
            source="ai_inferred_target",
            confidence=confidence,
            country=country,
            city=city,
        )

    geocode_queries: list[str] = []
    for value in (
        ", ".join(part for part in (city, country) if part),
        matched_location_name,
        city,
        country,
    ):
        normalized = _coerce_str(value)
        if normalized and normalized not in geocode_queries:
            geocode_queries.append(normalized)

    for query in geocode_queries:
        try:
            result = await _geocode_target_location(query)
        except Exception:
            continue
        if not isinstance(result, dict):
            continue
        latitude = _coerce_float(result.get("lat"))
        longitude = _coerce_float(result.get("lon"))
        if latitude in (None, 0.0) or longitude in (None, 0.0):
            continue
        label = (
            _coerce_str(result.get("display_name"))
            or matched_location_name
            or ", ".join(part for part in (city, country) if part)
            or query
        )
        return NewsTargetLocation(
            latitude=latitude,
            longitude=longitude,
            label=label,
            source="ai_inferred_target",
            confidence=confidence,
            country=country,
            city=city,
        )

    centroid = get_country_centroid(country)
    if centroid:
        label = matched_location_name or city or country or "关联位置"
        return NewsTargetLocation(
            latitude=centroid["latitude"],
            longitude=centroid["longitude"],
            label=label,
            source="ai_inferred_target",
            confidence=confidence,
            country=country,
            city=city,
        )

    return None


async def _extract_target_location_from_text(item: ParsedNewsItem) -> NewsTargetLocation | None:
    combined_text = " ".join(part for part in (item.title, item.summary) if part).strip()
    if not combined_text:
        return None

    for hint in _CITY_HINTS:
        city_name = _coerce_str(hint.get("name"))
        if not city_name or not _contains_location_alias(combined_text, city_name):
            continue
        country = normalize_country(hint.get("country"))
        geocode_query = ", ".join(part for part in (city_name, country) if part)
        try:
            result = await _geocode_target_location(geocode_query)
        except Exception:
            result = None
        if isinstance(result, dict):
            latitude = _coerce_float(result.get("lat"))
            longitude = _coerce_float(result.get("lon"))
            if latitude not in (None, 0.0) and longitude not in (None, 0.0):
                return NewsTargetLocation(
                    latitude=latitude,
                    longitude=longitude,
                    label=_coerce_str(result.get("display_name")) or geocode_query,
                    source="headline_location_hint",
                    confidence=0.78,
                    country=country,
                    city=city_name,
                )
        centroid = get_country_centroid(country)
        if centroid:
            return NewsTargetLocation(
                latitude=centroid["latitude"],
                longitude=centroid["longitude"],
                label=geocode_query,
                source="headline_location_hint",
                confidence=0.68,
                country=country,
                city=city_name,
            )

    for canonical, variants in COUNTRY_VARIANTS_MAP.items():
        if not get_country_centroid(canonical):
            continue
        searchable_variants = _iter_searchable_country_variants(canonical, variants)
        if not any(_contains_location_alias(combined_text, variant) for variant in searchable_variants):
            continue
        centroid = get_country_centroid(canonical)
        if not centroid:
            continue
        return NewsTargetLocation(
            latitude=centroid["latitude"],
            longitude=centroid["longitude"],
            label=canonical,
            source="headline_country_hint",
            confidence=0.62,
            country=canonical,
            city=None,
        )

    return None


async def _infer_news_target_location(
    item: ParsedNewsItem,
    *,
    provider_client: AIProviderClient | None,
    prompt: EffectiveAIPrompt | None = None,
) -> NewsTargetLocation | None:
    target, _localizations = await _infer_news_enrichment(
        item,
        provider_client=provider_client,
        prompt=prompt,
    )
    return target


async def _infer_news_enrichment(
    item: ParsedNewsItem,
    *,
    provider_client: AIProviderClient | None,
    prompt: EffectiveAIPrompt | None = None,
) -> tuple[NewsTargetLocation | None, dict[str, dict[str, str]]]:
    text_hint = await _extract_target_location_from_text(item)
    content_error: str | None = None
    if text_hint is not None and text_hint.city:
        item.target_resolution_stage = text_hint.source
        item.target_ai_attempted = False
        item.target_ai_status = "skipped_text_hint"
        item.target_ai_error = None
        item.target_debug_note = f"text hint matched {text_hint.label}"
    localizations: dict[str, dict[str, str]] = {}

    if provider_client is None:
        if text_hint is None or not text_hint.city:
            item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
            item.target_ai_attempted = False
            item.target_ai_status = "unavailable"
            item.target_ai_error = "AI provider is not configured or unavailable for earth-feed."
            item.target_debug_note = (
                f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
            )
        item.enrichment_status = "unavailable"
        item.enrichment_error = "AI provider is not configured or unavailable for earth-feed."
        return text_hint, localizations

    if text_hint is None or not text_hint.city:
        item.target_ai_attempted = True
        item.target_ai_status = "attempted"
        item.target_ai_error = None
        item.target_debug_note = (
            f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
        )
    item.enrichment_status = "attempted"
    item.enrichment_error = None

    prompt = prompt or await get_effective_prompt(None, NEWS_ENRICH_PROMPT_KEY)
    request = SituationalAnalysisRequest(
        title="Enrich Earth news item with event location and zh-CN content",
        objective=prompt.prompt,
        system_prompt=prompt.system_prompt or None,
        context={
            "news_item": {
                "title": item.title,
                "summary": item.summary,
                "source": item.source,
                "feed_name": item.feed_name,
                "feed_region": item.feed_region,
                "url": item.url,
                "published_at": (
                    item.published_at.isoformat().replace("+00:00", "Z")
                    if item.published_at
                    else None
                ),
            },
            "required_json_schema": {
                "location": {
                    "country": "string|null",
                    "city": "string|null",
                    "matched_location_name": "string|null",
                    "latitude": "number|null",
                    "longitude": "number|null",
                    "confidence": "number from 0 to 1",
                    "reasoning_summary": "short string",
                },
                "localizations": {
                    "zh-CN": {
                        "title": "faithful Simplified Chinese title",
                        "summary": "one-sentence newswire-style Simplified Chinese lead summary",
                    }
                },
            },
        },
        constraints=[
            "Return only strict JSON. Do not wrap it in markdown.",
            "For localizations, do not add facts that are absent from the RSS headline, description, source, or date.",
            "Write zh-CN summary as one concise newswire-style sentence, like a breaking-news lead.",
            "If the RSS description is thin, write a conservative one-sentence summary that says only what is supported.",
            "Keep zh-CN summary factual, non-promotional, and avoid colon-heavy keyword labels.",
            "Prefer the event location, not the newsroom or publisher headquarters.",
            "When a country visit or summit is the clear topic but the city is omitted, use the most likely host city only if it is broadly public knowledge.",
            "Use null for unknown fields instead of inventing details.",
            "Calibrate confidence conservatively: 0.75+ only when the city is strongly supported, 0.55-0.74 for country-level or likely city inference, below 0.55 when weak.",
        ],
    )
    try:
        response = await provider_client.analyze(request)
    except Exception as exc:
        if text_hint is None or not text_hint.city:
            item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
            item.target_ai_status = "provider_error"
            item.target_ai_error = str(exc)
        item.enrichment_status = "provider_error"
        item.enrichment_error = str(exc)
        return text_hint, localizations

    payload = _first_json_object(response.content)
    if not isinstance(payload, dict):
        if text_hint is None or not text_hint.city:
            item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
            item.target_ai_status = "parse_error"
            item.target_ai_error = "AI response did not contain a parseable JSON object."
        item.enrichment_status = "parse_error"
        item.enrichment_error = "AI response did not contain a parseable JSON object."
        return text_hint, localizations

    localizations = _normalize_localizations(payload.get("localizations"))
    if not localizations:
        content_error = "AI returned no usable localizations."

    location_payload = payload.get("location") if isinstance(payload.get("location"), dict) else payload
    if text_hint is not None and text_hint.city:
        target = text_hint
    else:
        target = await _build_target_location_from_payload(location_payload)
    if target is None:
        item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
        item.target_ai_status = "no_result"
        item.target_ai_error = "AI returned no usable target coordinates or geocodeable location."
        target = text_hint
    elif target.confidence is not None and target.confidence < 0.45:
        item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
        item.target_ai_status = "low_confidence"
        item.target_ai_error = f"AI target confidence too low: {target.confidence:.2f}"
        target = text_hint
    else:
        item.target_resolution_stage = target.source
        item.target_ai_status = "success"
        item.target_ai_error = None
        item.target_debug_note = f"ai inferred {target.label}"

    item.localizations = localizations
    if localizations and item.target_ai_status in {"success", "skipped_text_hint"}:
        item.enrichment_status = "success"
        item.enrichment_error = None
    elif localizations:
        item.enrichment_status = "content_only"
        item.enrichment_error = item.target_ai_error
    else:
        item.enrichment_status = "location_only" if target is not None else "no_result"
        item.enrichment_error = content_error or item.target_ai_error
    item.enriched_at = datetime.now(UTC) if localizations else None
    return target, localizations


async def _enrich_items_with_target_locations(
    items: list[ParsedNewsItem],
    *,
    provider_client: AIProviderClient | None,
    prompt: EffectiveAIPrompt | None = None,
) -> list[ParsedNewsItem]:
    if not items:
        return items

    semaphore = asyncio.Semaphore(MAX_TARGET_INFERENCE_CONCURRENCY)

    async def enrich(item: ParsedNewsItem) -> ParsedNewsItem:
        async with semaphore:
            target = await _infer_news_target_location(
                item,
                provider_client=provider_client,
                prompt=prompt,
            )
        item.target_location = target
        return item

    return list(await asyncio.gather(*(enrich(item) for item in items)))


def get_sources_for_region(region: str) -> list[NewsFeedSource]:
    return sorted(
        [source for source in NEWS_FEED_SOURCES if source.region in {"global", region}],
        key=lambda source: (source.priority, source.name),
    )


def _strip_html(value: str) -> str:
    if not value:
        return ""
    soup = BeautifulSoup(value, "html.parser")
    return re.sub(r"\s+", " ", soup.get_text(" ", strip=True)).strip()


def _truncate(value: str, limit: int = 180) -> str:
    text = value.strip()
    if len(text) <= limit:
        return text
    return text[: limit - 1].rstrip() + "…"


def _normalize_source_name(raw: str, fallback: str) -> str:
    text = html.unescape((raw or "").strip())
    if " - " in text:
        return text.split(" - ")[-1].strip() or fallback
    return text or fallback


def _parse_datetime(raw: str | None) -> datetime | None:
    if not raw:
        return None
    text = raw.strip()
    if not text:
        return None

    for parser in (
        lambda value: parsedate_to_datetime(value),
        lambda value: datetime.fromisoformat(value.replace("Z", "+00:00")),
    ):
        try:
            parsed = parser(text)
            if parsed.tzinfo is None:
                parsed = parsed.replace(tzinfo=UTC)
            return parsed.astimezone(UTC)
        except Exception:
            continue
    return None


def _extract_item_text(element: ET.Element, *names: str) -> str:
    for name in names:
        node = element.find(name)
        if node is not None and node.text:
            return node.text.strip()
    return ""


def _parse_feed_entries(xml_text: str, source: NewsFeedSource) -> list[ParsedNewsItem]:
    root = ET.fromstring(xml_text)
    items: list[ParsedNewsItem] = []

    rss_items = root.findall("./channel/item")
    atom_entries = root.findall("{http://www.w3.org/2005/Atom}entry")
    nodes = rss_items or atom_entries

    for node in nodes[:MAX_ITEMS_PER_SOURCE]:
        if node.tag.endswith("entry"):
            title = _extract_item_text(node, "{http://www.w3.org/2005/Atom}title")
            summary = _extract_item_text(
                node,
                "{http://www.w3.org/2005/Atom}summary",
                "{http://www.w3.org/2005/Atom}content",
            )
            link_node = node.find("{http://www.w3.org/2005/Atom}link")
            link = link_node.get("href", "").strip() if link_node is not None else ""
            published = _extract_item_text(
                node,
                "{http://www.w3.org/2005/Atom}updated",
                "{http://www.w3.org/2005/Atom}published",
            )
        else:
            title = _extract_item_text(node, "title")
            summary = _extract_item_text(node, "description", "content")
            link = _extract_item_text(node, "link")
            published = _extract_item_text(node, "pubDate", "published", "updated")

        clean_title = html.unescape(title).strip()
        clean_summary = _truncate(_strip_html(summary), 180)
        if not clean_title or not link:
            continue

        item_source = source.name
        display_title = clean_title
        if source.source_type == "aggregated" and " - " in clean_title:
            parts = clean_title.rsplit(" - ", 1)
            display_title = parts[0].strip()
            item_source = _normalize_source_name(parts[1], source.name)

        items.append(
            ParsedNewsItem(
                id=f"{source.id}:{hashlib.sha1(link.encode('utf-8')).hexdigest()[:12]}",
                title=display_title,
                summary=clean_summary,
                url=link,
                source=item_source,
                feed_name=source.name,
                feed_region=source.region,
                homepage_url=source.homepage_url,
                published_at=_parse_datetime(published),
            )
        )

    return items


def _serialize_sources(sources: list[NewsFeedSource]) -> list[dict[str, Any]]:
    return [
        {
            "id": source.id,
            "name": source.name,
            "region": source.region,
            "homepage_url": source.homepage_url,
        }
        for source in sources
    ]


def _serialize_anchor(anchor: RegionAnchor) -> dict[str, Any]:
    return {
        "region": anchor.region,
        "label": anchor.label,
        "latitude": anchor.latitude,
        "longitude": anchor.longitude,
    }


def _serialize_target(target: NewsTargetLocation | None) -> dict[str, Any] | None:
    if target is None:
        return None
    return {
        "latitude": target.latitude,
        "longitude": target.longitude,
        "label": target.label,
        "source": target.source,
        "confidence": target.confidence,
        "country": target.country,
        "city": target.city,
    }


def _serialize_enriched_at(value: datetime | None) -> str | None:
    return value.isoformat().replace("+00:00", "Z") if value else None


def _content_patch(item: ParsedNewsItem) -> dict[str, Any]:
    return {
        "content_language": item.content_language,
        "localizations": item.localizations,
        "enrichment_status": item.enrichment_status,
        "enrichment_error": item.enrichment_error,
        "enriched_at": _serialize_enriched_at(item.enriched_at),
    }


def build_anchor_location_patch(
    item: ParsedNewsItem,
    *,
    queued: bool = False,
    queue_available: bool | None = None,
) -> dict[str, Any]:
    anchor = get_region_anchor(item.feed_region)
    if queued:
        resolution_stage = "queued"
        ai_status = "queued"
        debug_note = "queued for async target location inference"
    else:
        resolution_stage = item.target_resolution_stage
        ai_status = item.target_ai_status
        debug_note = item.target_debug_note
    content_patch = _content_patch(item)
    if queued and content_patch["enrichment_status"] == "pending":
        content_patch["enrichment_status"] = "queued"
    return {
        "latitude": anchor.latitude,
        "longitude": anchor.longitude,
        "location_label": anchor.label,
        "location_source": "region_anchor",
        "verified": False,
        "location_meta": {
            "resolution_stage": resolution_stage,
            "ai_attempted": item.target_ai_attempted,
            "ai_status": ai_status,
            "ai_error": item.target_ai_error,
            "debug_note": debug_note,
            "queue_available": queue_available,
            "target": None,
            "anchor": _serialize_anchor(anchor),
        },
        **content_patch,
    }


def build_target_location_patch(item: ParsedNewsItem, target: NewsTargetLocation | None) -> dict[str, Any]:
    if target is None:
        return build_anchor_location_patch(item)
    anchor = get_region_anchor(item.feed_region)
    return {
        "latitude": target.latitude,
        "longitude": target.longitude,
        "location_label": target.label,
        "location_source": target.source,
        "verified": True,
        "location_meta": {
            "resolution_stage": item.target_resolution_stage,
            "ai_attempted": item.target_ai_attempted,
            "ai_status": item.target_ai_status,
            "ai_error": item.target_ai_error,
            "debug_note": item.target_debug_note,
            "target": _serialize_target(target),
            "anchor": _serialize_anchor(anchor),
        },
        **_content_patch(item),
    }


def build_target_location_job_payload(item: ParsedNewsItem) -> dict[str, Any]:
    published_at = item.published_at
    return {
        "id": item.id,
        "title": item.title,
        "summary": item.summary,
        "content_language": item.content_language,
        "localizations": item.localizations,
        "enrichment_status": item.enrichment_status,
        "enrichment_error": item.enrichment_error,
        "enriched_at": _serialize_enriched_at(item.enriched_at),
        "url": item.url,
        "source": item.source,
        "feed_name": item.feed_name,
        "feed_region": item.feed_region,
        "homepage_url": item.homepage_url,
        "published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None,
    }


def parsed_news_item_from_job_payload(payload: dict[str, Any]) -> ParsedNewsItem:
    return ParsedNewsItem(
        id=str(payload.get("id") or ""),
        title=str(payload.get("title") or ""),
        summary=str(payload.get("summary") or ""),
        content_language=str(payload.get("content_language") or "en"),
        localizations=_normalize_localizations(payload.get("localizations")),
        enrichment_status=str(payload.get("enrichment_status") or "pending"),
        enrichment_error=_coerce_str(payload.get("enrichment_error")),
        enriched_at=_parse_datetime(_coerce_str(payload.get("enriched_at"))),
        url=str(payload.get("url") or ""),
        source=str(payload.get("source") or ""),
        feed_name=str(payload.get("feed_name") or ""),
        feed_region=str(payload.get("feed_region") or "global"),
        homepage_url=str(payload.get("homepage_url") or ""),
        published_at=_parse_datetime(_coerce_str(payload.get("published_at"))),
    )


def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, Any]:
    published_at = item.published_at
    location_patch = item.location_patch or build_target_location_patch(item, item.target_location)
    return {
        "id": item.id,
        "title": item.title,
        "summary": item.summary,
        "content_language": item.content_language,
        "localizations": item.localizations,
        "display_title": _get_locale_text(item, "title"),
        "display_summary": _get_locale_text(item, "summary"),
        "url": item.url,
        "source": item.source,
        "feed_name": item.feed_name,
        "region": item.feed_region,
        "display_region": get_region_anchor(item.feed_region).label,
        "homepage_url": item.homepage_url,
        "published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None,
        "latitude": location_patch["latitude"],
        "longitude": location_patch["longitude"],
        "location_label": location_patch["location_label"],
        "location_source": location_patch["location_source"],
        "verified": location_patch["verified"],
        "location_meta": location_patch["location_meta"],
        "enrichment_status": item.enrichment_status,
        "enrichment_error": item.enrichment_error,
        "enriched_at": _serialize_enriched_at(item.enriched_at),
        "is_focus_match": item.feed_region == active_region,
    }


def _build_payload(
    *,
    lat: float | None,
    lon: float | None,
    active_region: str,
    items: list[ParsedNewsItem],
    sources: list[NewsFeedSource],
    errors: list[str],
    stale: bool,
    generated_at: datetime | None = None,
) -> dict[str, Any]:
    profile = get_region_profile(active_region)
    timestamp = generated_at or datetime.now(UTC)
    return {
        "generated_at": timestamp.isoformat().replace("+00:00", "Z"),
        "focus": {
            "lat": lat,
            "lon": lon,
            "region": active_region,
            "label": profile.label,
            "display_region": get_region_anchor(active_region).label,
            "accent": profile.accent,
        },
        "sources": _serialize_sources(sources),
        "items": [_serialize_item(item, active_region=active_region) for item in items],
        "errors": errors,
        "stale": stale,
    }


def _rank_and_trim_items(items: list[ParsedNewsItem], *, active_region: str) -> list[ParsedNewsItem]:
    deduped: dict[str, ParsedNewsItem] = {}
    for item in items:
        key = item.url.strip() or item.title.strip().lower()
        if key not in deduped:
            deduped[key] = item

    return sorted(
        deduped.values(),
        key=lambda item: (
            item.feed_region != active_region,
            item.published_at is None,
            -(item.published_at.timestamp() if item.published_at else 0),
            item.feed_name,
        ),
    )[:MAX_ITEMS_TOTAL]


def _get_cached_region_feed(region: str) -> CachedRegionFeed | None:
    cached = _REGION_CACHE.get(region)
    if not cached:
        return None
    age_seconds = (datetime.now(UTC) - cached.fetched_at).total_seconds()
    if age_seconds > STALE_CACHE_MAX_AGE_SECONDS:
        return None
    return cached


def _store_region_cache(region: str, *, items: list[ParsedNewsItem], sources: list[NewsFeedSource]) -> None:
    _REGION_CACHE[region] = CachedRegionFeed(
        region=region,
        fetched_at=datetime.now(UTC),
        items=list(items),
        sources=list(sources),
    )


async def _apply_cached_locations_and_enqueue(items: list[ParsedNewsItem]) -> list[ParsedNewsItem]:
    if not items:
        return items

    from app.services.earth_news_queue import (
        enqueue_target_location_job,
        get_cached_target_location_patch,
    )

    async def enqueue_item(item: ParsedNewsItem, *, force: bool = False) -> bool:
        return await enqueue_target_location_job(build_target_location_job_payload(item), force=force)

    async def apply_location(item: ParsedNewsItem) -> ParsedNewsItem:
        cached_patch = await get_cached_target_location_patch(item.id)
        if cached_patch:
            apply_enrichment_patch_to_item(item, cached_patch)
            if not _has_default_localization(item):
                queued = await enqueue_item(item, force=True)
                if queued and item.enrichment_status in {
                    "pending",
                    "unavailable",
                    "provider_error",
                    "parse_error",
                    "no_result",
                    "location_only",
                }:
                    item.enrichment_status = "queued"
            return item

        queued = await enqueue_item(item)
        item.location_patch = build_anchor_location_patch(
            item,
            queued=queued,
            queue_available=queued,
        )
        return item

    return list(await asyncio.gather(*(apply_location(item) for item in items)))


async def _enqueue_unverified_locations(items: list[ParsedNewsItem]) -> None:
    if not items:
        return

    from app.services.earth_news_queue import enqueue_target_location_job

    await asyncio.gather(
        *(
            enqueue_target_location_job(
                build_target_location_job_payload(item),
                force=not _has_default_localization(item),
            )
            for item in items
            if (
                item.location_patch is None
                or item.location_patch.get("verified") is False
                or not _has_default_localization(item)
            )
        )
    )


async def _fetch_source(
    client: httpx.AsyncClient,
    source: NewsFeedSource,
) -> tuple[NewsFeedSource, list[ParsedNewsItem], str | None]:
    try:
        response = await client.get(source.feed_url)
        response.raise_for_status()
        return source, _parse_feed_entries(response.text, source), None
    except Exception as exc:
        return source, [], str(exc)


async def _fetch_rss_items_for_sources(
    sources: list[NewsFeedSource],
) -> tuple[list[ParsedNewsItem], list[str]]:
    errors: list[str] = []
    async with httpx.AsyncClient(
        timeout=REQUEST_TIMEOUT,
        follow_redirects=True,
        headers={"User-Agent": USER_AGENT},
    ) as client:
        results = await asyncio.gather(*(_fetch_source(client, source) for source in sources))

    fetched_items: list[ParsedNewsItem] = []
    for source, items, error in results:
        if error:
            errors.append(f"{source.name}: {error}")
            continue
        fetched_items.extend(items)
    return fetched_items, errors


def _needs_rss_supplement(*, item_count: int, newest_at: datetime | None) -> bool:
    if item_count < MAX_ITEMS_TOTAL:
        return True
    if newest_at is None:
        return True
    age_seconds = (datetime.now(UTC) - newest_at).total_seconds()
    return age_seconds > RSS_SUPPLEMENT_MAX_AGE_SECONDS


async def _get_earth_news_payload_from_rss_only(
    *,
    lat: float | None,
    lon: float | None,
    active_region: str,
    sources: list[NewsFeedSource],
) -> dict[str, Any]:
    fetched_items, errors = await _fetch_rss_items_for_sources(sources)
    ranked_items = _rank_and_trim_items(fetched_items, active_region=active_region)
    if ranked_items:
        ranked_items = await _apply_cached_locations_and_enqueue(ranked_items)
        _store_region_cache(active_region, items=ranked_items, sources=sources)
        return _build_payload(
            lat=lat,
            lon=lon,
            active_region=active_region,
            items=ranked_items,
            sources=sources,
            errors=errors,
            stale=False,
        )

    cached = _get_cached_region_feed(active_region)
    if cached:
        cached.items = await _apply_cached_locations_and_enqueue(cached.items)
        return _build_payload(
            lat=lat,
            lon=lon,
            active_region=active_region,
            items=cached.items,
            sources=cached.sources,
            errors=errors,
            stale=True,
            generated_at=cached.fetched_at,
        )

    return _build_payload(
        lat=lat,
        lon=lon,
        active_region=active_region,
        items=[],
        sources=sources,
        errors=errors,
        stale=False,
    )


async def get_earth_news_payload(
    lat: float | None = None,
    lon: float | None = None,
    *,
    provider_client: AIProviderClient | None = None,
    db: AsyncSession | None = None,
) -> dict[str, Any]:
    del provider_client
    active_region = determine_focus_region(lat, lon)
    sources = get_sources_for_region(active_region)

    if db is None:
        return await _get_earth_news_payload_from_rss_only(
            lat=lat,
            lon=lon,
            active_region=active_region,
            sources=sources,
        )

    from app.services.earth_news_store import (
        get_earth_news_freshness,
        list_earth_news_items,
        upsert_earth_news_items,
    )

    errors: list[str] = []
    item_count, newest_at = await get_earth_news_freshness(db, active_region=active_region)
    should_supplement = _needs_rss_supplement(item_count=item_count, newest_at=newest_at)
    if should_supplement:
        fetched_items, errors = await _fetch_rss_items_for_sources(sources)
        ranked_fetched_items = _rank_and_trim_items(fetched_items, active_region=active_region)
        await upsert_earth_news_items(db, ranked_fetched_items)

    items = await list_earth_news_items(
        db,
        active_region=active_region,
        limit=MAX_ITEMS_TOTAL,
    )
    await _enqueue_unverified_locations(items)
    stale = bool(errors and items)

    return _build_payload(
        lat=lat,
        lon=lon,
        active_region=active_region,
        items=items,
        sources=sources,
        errors=errors,
        stale=stale,
    )