93eb41a9f7
Release 0.58.0 includes the Earth high-precision boundary PMTiles/MVT pipeline, standardized Earth boundary source collectors, China POV boundary configuration templates, and removal of the legacy low-precision GeoJSON fallback. It also adds Earth news target-location queueing/archive support, fixes datasource task status visibility, documents the Earth surface depth-spacing rules that prevent far-zoom z-fighting snow/black blocks, and updates bilingual operations/developer docs.
1139 lines
36 KiB
Python
1139 lines
36 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime
|
|
from email.utils import parsedate_to_datetime
|
|
import hashlib
|
|
import html
|
|
import json
|
|
import math
|
|
import re
|
|
from typing import Any
|
|
from urllib.parse import quote
|
|
import xml.etree.ElementTree as ET
|
|
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.core.countries import COUNTRY_VARIANTS_MAP, get_country_centroid, normalize_country
|
|
from app.schemas.ai import SituationalAnalysisRequest
|
|
from app.services.ai_client import AIProviderClient
|
|
from app.services.location.resolvers.nominatim import build_default_nominatim_geocoder
|
|
|
|
|
|
USER_AGENT = "PlanetEarthNewsBoard/1.0 (+https://planet.local)"
|
|
REQUEST_TIMEOUT = 12.0
|
|
MAX_ITEMS_PER_SOURCE = 6
|
|
MAX_ITEMS_TOTAL = 12
|
|
STALE_CACHE_MAX_AGE_SECONDS = 60 * 45
|
|
RSS_SUPPLEMENT_MAX_AGE_SECONDS = STALE_CACHE_MAX_AGE_SECONDS
|
|
MAX_TARGET_INFERENCE_CONCURRENCY = 3
|
|
TARGET_INFERENCE_TIMEOUT_SECONDS = 6.0
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RegionProfile:
|
|
key: str
|
|
label: str
|
|
query: str
|
|
accent: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RegionAnchor:
|
|
region: str
|
|
label: str
|
|
latitude: float
|
|
longitude: float
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class NewsFeedSource:
|
|
id: str
|
|
name: str
|
|
region: str
|
|
feed_url: str
|
|
homepage_url: str
|
|
source_type: str = "rss"
|
|
priority: int = 100
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class NewsTargetLocation:
|
|
latitude: float
|
|
longitude: float
|
|
label: str
|
|
source: str
|
|
confidence: float | None = None
|
|
country: str | None = None
|
|
city: str | None = None
|
|
|
|
|
|
@dataclass
|
|
class ParsedNewsItem:
|
|
id: str
|
|
title: str
|
|
summary: str
|
|
url: str
|
|
source: str
|
|
feed_name: str
|
|
feed_region: str
|
|
homepage_url: str
|
|
published_at: datetime | None
|
|
target_location: NewsTargetLocation | None = None
|
|
target_resolution_stage: str = "unresolved"
|
|
target_ai_attempted: bool = False
|
|
target_ai_status: str = "not_attempted"
|
|
target_ai_error: str | None = None
|
|
target_debug_note: str | None = None
|
|
location_patch: dict[str, Any] | None = None
|
|
|
|
|
|
@dataclass
|
|
class CachedRegionFeed:
|
|
region: str
|
|
fetched_at: datetime
|
|
items: list[ParsedNewsItem]
|
|
sources: list[NewsFeedSource]
|
|
|
|
|
|
REGION_PROFILES: dict[str, RegionProfile] = {
|
|
"americas": RegionProfile(
|
|
key="americas",
|
|
label="美洲焦点",
|
|
query='Americas geopolitics OR Latin America OR "United States" OR Canada',
|
|
accent="#79d3ff",
|
|
),
|
|
"europe": RegionProfile(
|
|
key="europe",
|
|
label="欧洲焦点",
|
|
query='Europe geopolitics OR EU OR NATO OR "Eastern Europe"',
|
|
accent="#8fd4ff",
|
|
),
|
|
"middle-east-africa": RegionProfile(
|
|
key="middle-east-africa",
|
|
label="中东与非洲焦点",
|
|
query='"Middle East" OR Africa geopolitics OR Red Sea OR Gulf',
|
|
accent="#ffb56a",
|
|
),
|
|
"asia-pacific": RegionProfile(
|
|
key="asia-pacific",
|
|
label="亚太焦点",
|
|
query='"Asia Pacific" OR Indo-Pacific OR China OR Japan OR Korea OR ASEAN',
|
|
accent="#78f2cf",
|
|
),
|
|
"global": RegionProfile(
|
|
key="global",
|
|
label="全球焦点",
|
|
query='"world news" OR geopolitics OR "global affairs"',
|
|
accent="#d6e6ff",
|
|
),
|
|
}
|
|
|
|
REGION_ANCHORS: dict[str, RegionAnchor] = {
|
|
"americas": RegionAnchor(
|
|
region="americas",
|
|
label="美洲",
|
|
latitude=37.0902,
|
|
longitude=-95.7129,
|
|
),
|
|
"europe": RegionAnchor(
|
|
region="europe",
|
|
label="欧洲",
|
|
latitude=50.1109,
|
|
longitude=8.6821,
|
|
),
|
|
"middle-east-africa": RegionAnchor(
|
|
region="middle-east-africa",
|
|
label="中东与非洲",
|
|
latitude=25.2048,
|
|
longitude=55.2708,
|
|
),
|
|
"asia-pacific": RegionAnchor(
|
|
region="asia-pacific",
|
|
label="亚太",
|
|
latitude=1.3521,
|
|
longitude=103.8198,
|
|
),
|
|
"global": RegionAnchor(
|
|
region="global",
|
|
label="全球",
|
|
latitude=20.0,
|
|
longitude=0.0,
|
|
),
|
|
}
|
|
|
|
|
|
def _google_news_feed(query: str, *, hl: str, gl: str, ceid: str) -> str:
|
|
return (
|
|
"https://news.google.com/rss/search?q="
|
|
+ quote(query, safe="")
|
|
+ f"&hl={hl}&gl={gl}&ceid={ceid}"
|
|
)
|
|
|
|
|
|
NEWS_FEED_SOURCES: tuple[NewsFeedSource, ...] = (
|
|
NewsFeedSource(
|
|
id="bbc-world",
|
|
name="BBC World",
|
|
region="global",
|
|
feed_url="https://feeds.bbci.co.uk/news/world/rss.xml",
|
|
homepage_url="https://www.bbc.com/news/world",
|
|
priority=10,
|
|
),
|
|
NewsFeedSource(
|
|
id="dw-top",
|
|
name="DW Top Stories",
|
|
region="europe",
|
|
feed_url="https://rss.dw.com/rdf/rss-en-top",
|
|
homepage_url="https://www.dw.com/en/top-stories/s-9097",
|
|
priority=20,
|
|
),
|
|
NewsFeedSource(
|
|
id="global-scan",
|
|
name="Global Monitor / World",
|
|
region="global",
|
|
feed_url=_google_news_feed(
|
|
REGION_PROFILES["global"].query,
|
|
hl="en-US",
|
|
gl="US",
|
|
ceid="US:en",
|
|
),
|
|
homepage_url="https://news.google.com/",
|
|
source_type="aggregated",
|
|
priority=30,
|
|
),
|
|
NewsFeedSource(
|
|
id="google-americas",
|
|
name="Global Monitor / Americas",
|
|
region="americas",
|
|
feed_url=_google_news_feed(
|
|
REGION_PROFILES["americas"].query,
|
|
hl="en-US",
|
|
gl="US",
|
|
ceid="US:en",
|
|
),
|
|
homepage_url="https://news.google.com/",
|
|
source_type="aggregated",
|
|
priority=40,
|
|
),
|
|
NewsFeedSource(
|
|
id="google-europe",
|
|
name="Global Monitor / Europe",
|
|
region="europe",
|
|
feed_url=_google_news_feed(
|
|
REGION_PROFILES["europe"].query,
|
|
hl="en-GB",
|
|
gl="GB",
|
|
ceid="GB:en",
|
|
),
|
|
homepage_url="https://news.google.com/",
|
|
source_type="aggregated",
|
|
priority=40,
|
|
),
|
|
NewsFeedSource(
|
|
id="google-mea",
|
|
name="Global Monitor / MEA",
|
|
region="middle-east-africa",
|
|
feed_url=_google_news_feed(
|
|
REGION_PROFILES["middle-east-africa"].query,
|
|
hl="en-US",
|
|
gl="US",
|
|
ceid="US:en",
|
|
),
|
|
homepage_url="https://news.google.com/",
|
|
source_type="aggregated",
|
|
priority=40,
|
|
),
|
|
NewsFeedSource(
|
|
id="google-apac",
|
|
name="Global Monitor / APAC",
|
|
region="asia-pacific",
|
|
feed_url=_google_news_feed(
|
|
REGION_PROFILES["asia-pacific"].query,
|
|
hl="en-SG",
|
|
gl="SG",
|
|
ceid="SG:en",
|
|
),
|
|
homepage_url="https://news.google.com/",
|
|
source_type="aggregated",
|
|
priority=40,
|
|
),
|
|
)
|
|
|
|
|
|
_REGION_CACHE: dict[str, CachedRegionFeed] = {}
|
|
_news_target_geocode = build_default_nominatim_geocoder(user_agent=USER_AGENT)
|
|
_CITY_HINTS: tuple[dict[str, str | None], ...] = (
|
|
{"name": "Beijing", "country": "中国"},
|
|
{"name": "Havana", "country": "古巴"},
|
|
{"name": "Kyiv", "country": "乌克兰"},
|
|
{"name": "Bangkok", "country": "泰国"},
|
|
{"name": "Tehran", "country": "伊朗"},
|
|
{"name": "Moscow", "country": "俄罗斯"},
|
|
{"name": "Taipei", "country": "中国（台湾）"},
|
|
{"name": "Hong Kong", "country": "中国（香港）"},
|
|
)
|
|
|
|
|
|
def determine_focus_region(lat: float | None, lon: float | None) -> str:
|
|
if lat is None or lon is None:
|
|
return "global"
|
|
if -170 <= lon <= -30:
|
|
return "americas"
|
|
if -30 < lon <= 45:
|
|
return "europe" if lat >= 30 else "middle-east-africa"
|
|
if 45 < lon <= 150:
|
|
return "middle-east-africa" if lat < 10 else "asia-pacific"
|
|
return "asia-pacific"
|
|
|
|
|
|
def get_region_profile(region: str) -> RegionProfile:
|
|
return REGION_PROFILES.get(region, REGION_PROFILES["global"])
|
|
|
|
|
|
def get_region_anchor(region: str) -> RegionAnchor:
|
|
return REGION_ANCHORS.get(region, REGION_ANCHORS["global"])
|
|
|
|
|
|
def _coerce_str(value: Any) -> str | None:
|
|
if value is None:
|
|
return None
|
|
if not isinstance(value, str):
|
|
value = str(value)
|
|
cleaned = re.sub(r"\s+", " ", value).strip()
|
|
return cleaned or None
|
|
|
|
|
|
def _contains_location_alias(text: str, alias: str) -> bool:
|
|
normalized_alias = _coerce_str(alias)
|
|
if not normalized_alias:
|
|
return False
|
|
if re.search(r"[A-Za-z]", normalized_alias):
|
|
pattern = r"(?<![A-Za-z])" + re.escape(normalized_alias) + r"(?![A-Za-z])"
|
|
return re.search(pattern, text, flags=re.IGNORECASE) is not None
|
|
return normalized_alias in text
|
|
|
|
|
|
def _iter_searchable_country_variants(
|
|
canonical: str,
|
|
variants: list[str],
|
|
) -> tuple[str, ...]:
|
|
searchable: list[str] = []
|
|
seen: set[str] = set()
|
|
for variant in (canonical, *variants):
|
|
normalized = _coerce_str(variant)
|
|
if not normalized:
|
|
continue
|
|
if re.fullmatch(r"[A-Z]{2,3}", normalized):
|
|
continue
|
|
if len(normalized) <= 2:
|
|
continue
|
|
key = normalized.casefold()
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
searchable.append(normalized)
|
|
return tuple(searchable)
|
|
|
|
|
|
def _coerce_float(value: Any) -> float | None:
|
|
try:
|
|
parsed = float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
if not math.isfinite(parsed):
|
|
return None
|
|
return parsed
|
|
|
|
|
|
def _first_json_object(text: str) -> dict[str, Any] | None:
|
|
if not text:
|
|
return None
|
|
decoder = json.JSONDecoder()
|
|
for index, char in enumerate(text):
|
|
if char != "{":
|
|
continue
|
|
try:
|
|
payload, _ = decoder.raw_decode(text[index:])
|
|
except ValueError:
|
|
continue
|
|
if isinstance(payload, dict):
|
|
return payload
|
|
return None
|
|
|
|
|
|
async def _geocode_target_location(query: str) -> dict[str, Any] | None:
|
|
return await asyncio.to_thread(_news_target_geocode, query)
|
|
|
|
|
|
async def _build_target_location_from_payload(
|
|
payload: dict[str, Any],
|
|
) -> NewsTargetLocation | None:
|
|
country = normalize_country(payload.get("country"))
|
|
city = _coerce_str(payload.get("city"))
|
|
matched_location_name = _coerce_str(payload.get("matched_location_name"))
|
|
confidence = _coerce_float(payload.get("confidence"))
|
|
if confidence is not None:
|
|
confidence = max(0.0, min(confidence, 1.0))
|
|
|
|
latitude = _coerce_float(payload.get("latitude"))
|
|
longitude = _coerce_float(payload.get("longitude"))
|
|
if latitude not in (None, 0.0) and longitude not in (None, 0.0):
|
|
label = matched_location_name or ", ".join(part for part in (city, country) if part) or "关联位置"
|
|
return NewsTargetLocation(
|
|
latitude=latitude,
|
|
longitude=longitude,
|
|
label=label,
|
|
source="ai_inferred_target",
|
|
confidence=confidence,
|
|
country=country,
|
|
city=city,
|
|
)
|
|
|
|
geocode_queries: list[str] = []
|
|
for value in (
|
|
", ".join(part for part in (city, country) if part),
|
|
matched_location_name,
|
|
city,
|
|
country,
|
|
):
|
|
normalized = _coerce_str(value)
|
|
if normalized and normalized not in geocode_queries:
|
|
geocode_queries.append(normalized)
|
|
|
|
for query in geocode_queries:
|
|
try:
|
|
result = await _geocode_target_location(query)
|
|
except Exception:
|
|
continue
|
|
if not isinstance(result, dict):
|
|
continue
|
|
latitude = _coerce_float(result.get("lat"))
|
|
longitude = _coerce_float(result.get("lon"))
|
|
if latitude in (None, 0.0) or longitude in (None, 0.0):
|
|
continue
|
|
label = (
|
|
_coerce_str(result.get("display_name"))
|
|
or matched_location_name
|
|
or ", ".join(part for part in (city, country) if part)
|
|
or query
|
|
)
|
|
return NewsTargetLocation(
|
|
latitude=latitude,
|
|
longitude=longitude,
|
|
label=label,
|
|
source="ai_inferred_target",
|
|
confidence=confidence,
|
|
country=country,
|
|
city=city,
|
|
)
|
|
|
|
centroid = get_country_centroid(country)
|
|
if centroid:
|
|
label = matched_location_name or city or country or "关联位置"
|
|
return NewsTargetLocation(
|
|
latitude=centroid["latitude"],
|
|
longitude=centroid["longitude"],
|
|
label=label,
|
|
source="ai_inferred_target",
|
|
confidence=confidence,
|
|
country=country,
|
|
city=city,
|
|
)
|
|
|
|
return None
|
|
|
|
|
|
async def _extract_target_location_from_text(item: ParsedNewsItem) -> NewsTargetLocation | None:
|
|
combined_text = " ".join(part for part in (item.title, item.summary) if part).strip()
|
|
if not combined_text:
|
|
return None
|
|
|
|
for hint in _CITY_HINTS:
|
|
city_name = _coerce_str(hint.get("name"))
|
|
if not city_name or not _contains_location_alias(combined_text, city_name):
|
|
continue
|
|
country = normalize_country(hint.get("country"))
|
|
geocode_query = ", ".join(part for part in (city_name, country) if part)
|
|
try:
|
|
result = await _geocode_target_location(geocode_query)
|
|
except Exception:
|
|
result = None
|
|
if isinstance(result, dict):
|
|
latitude = _coerce_float(result.get("lat"))
|
|
longitude = _coerce_float(result.get("lon"))
|
|
if latitude not in (None, 0.0) and longitude not in (None, 0.0):
|
|
return NewsTargetLocation(
|
|
latitude=latitude,
|
|
longitude=longitude,
|
|
label=_coerce_str(result.get("display_name")) or geocode_query,
|
|
source="headline_location_hint",
|
|
confidence=0.78,
|
|
country=country,
|
|
city=city_name,
|
|
)
|
|
centroid = get_country_centroid(country)
|
|
if centroid:
|
|
return NewsTargetLocation(
|
|
latitude=centroid["latitude"],
|
|
longitude=centroid["longitude"],
|
|
label=geocode_query,
|
|
source="headline_location_hint",
|
|
confidence=0.68,
|
|
country=country,
|
|
city=city_name,
|
|
)
|
|
|
|
for canonical, variants in COUNTRY_VARIANTS_MAP.items():
|
|
if not get_country_centroid(canonical):
|
|
continue
|
|
searchable_variants = _iter_searchable_country_variants(canonical, variants)
|
|
if not any(_contains_location_alias(combined_text, variant) for variant in searchable_variants):
|
|
continue
|
|
centroid = get_country_centroid(canonical)
|
|
if not centroid:
|
|
continue
|
|
return NewsTargetLocation(
|
|
latitude=centroid["latitude"],
|
|
longitude=centroid["longitude"],
|
|
label=canonical,
|
|
source="headline_country_hint",
|
|
confidence=0.62,
|
|
country=canonical,
|
|
city=None,
|
|
)
|
|
|
|
return None
|
|
|
|
|
|
async def _infer_news_target_location(
|
|
item: ParsedNewsItem,
|
|
*,
|
|
provider_client: AIProviderClient | None,
|
|
) -> NewsTargetLocation | None:
|
|
text_hint = await _extract_target_location_from_text(item)
|
|
if text_hint is not None and text_hint.city:
|
|
item.target_resolution_stage = text_hint.source
|
|
item.target_ai_attempted = False
|
|
item.target_ai_status = "skipped_text_hint"
|
|
item.target_ai_error = None
|
|
item.target_debug_note = f"text hint matched {text_hint.label}"
|
|
return text_hint
|
|
|
|
if provider_client is None:
|
|
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
|
item.target_ai_attempted = False
|
|
item.target_ai_status = "unavailable"
|
|
item.target_ai_error = "AI provider is not configured or unavailable for earth-feed."
|
|
item.target_debug_note = (
|
|
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
|
|
)
|
|
return text_hint
|
|
|
|
item.target_ai_attempted = True
|
|
item.target_ai_status = "attempted"
|
|
item.target_ai_error = None
|
|
item.target_debug_note = (
|
|
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
|
|
)
|
|
|
|
request = SituationalAnalysisRequest(
|
|
title="Infer likely event location for Earth news cruise",
|
|
objective=(
|
|
"Return exactly one strict JSON object for the most likely physical "
|
|
"location the news event is about. Prefer the host city when a state "
|
|
"visit, summit, meeting, attack, or disaster is clearly centered in a "
|
|
"known city. Fall back to the best-supported country only when a city "
|
|
"cannot be inferred."
|
|
),
|
|
context={
|
|
"news_item": {
|
|
"title": item.title,
|
|
"summary": item.summary,
|
|
"source": item.source,
|
|
"feed_name": item.feed_name,
|
|
"feed_region": item.feed_region,
|
|
"url": item.url,
|
|
"published_at": (
|
|
item.published_at.isoformat().replace("+00:00", "Z")
|
|
if item.published_at
|
|
else None
|
|
),
|
|
},
|
|
"required_json_schema": {
|
|
"country": "string|null",
|
|
"city": "string|null",
|
|
"matched_location_name": "string|null",
|
|
"latitude": "number|null",
|
|
"longitude": "number|null",
|
|
"confidence": "number from 0 to 1",
|
|
"reasoning_summary": "short string",
|
|
},
|
|
},
|
|
constraints=[
|
|
"Return only strict JSON. Do not wrap it in markdown.",
|
|
"Prefer the event location, not the newsroom or publisher headquarters.",
|
|
"When a country visit or summit is the clear topic but the city is omitted, use the most likely host city only if it is broadly public knowledge.",
|
|
"Use null for unknown fields instead of inventing details.",
|
|
"Calibrate confidence conservatively: 0.75+ only when the city is strongly supported, 0.55-0.74 for country-level or likely city inference, below 0.55 when weak.",
|
|
],
|
|
)
|
|
try:
|
|
response = await provider_client.analyze(request)
|
|
except Exception as exc:
|
|
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
|
item.target_ai_status = "provider_error"
|
|
item.target_ai_error = str(exc)
|
|
return text_hint
|
|
|
|
payload = _first_json_object(response.content)
|
|
if not isinstance(payload, dict):
|
|
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
|
item.target_ai_status = "parse_error"
|
|
item.target_ai_error = "AI response did not contain a parseable JSON object."
|
|
return text_hint
|
|
|
|
target = await _build_target_location_from_payload(payload)
|
|
if target is None:
|
|
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
|
item.target_ai_status = "no_result"
|
|
item.target_ai_error = "AI returned no usable target coordinates or geocodeable location."
|
|
return text_hint
|
|
if target.confidence is not None and target.confidence < 0.45:
|
|
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
|
item.target_ai_status = "low_confidence"
|
|
item.target_ai_error = f"AI target confidence too low: {target.confidence:.2f}"
|
|
return text_hint
|
|
item.target_resolution_stage = target.source
|
|
item.target_ai_status = "success"
|
|
item.target_ai_error = None
|
|
item.target_debug_note = f"ai inferred {target.label}"
|
|
return target
|
|
|
|
|
|
async def _enrich_items_with_target_locations(
|
|
items: list[ParsedNewsItem],
|
|
*,
|
|
provider_client: AIProviderClient | None,
|
|
) -> list[ParsedNewsItem]:
|
|
if not items:
|
|
return items
|
|
|
|
semaphore = asyncio.Semaphore(MAX_TARGET_INFERENCE_CONCURRENCY)
|
|
|
|
async def enrich(item: ParsedNewsItem) -> ParsedNewsItem:
|
|
async with semaphore:
|
|
target = await _infer_news_target_location(item, provider_client=provider_client)
|
|
item.target_location = target
|
|
return item
|
|
|
|
return list(await asyncio.gather(*(enrich(item) for item in items)))
|
|
|
|
|
|
def get_sources_for_region(region: str) -> list[NewsFeedSource]:
|
|
return sorted(
|
|
[source for source in NEWS_FEED_SOURCES if source.region in {"global", region}],
|
|
key=lambda source: (source.priority, source.name),
|
|
)
|
|
|
|
|
|
def _strip_html(value: str) -> str:
|
|
if not value:
|
|
return ""
|
|
soup = BeautifulSoup(value, "html.parser")
|
|
return re.sub(r"\s+", " ", soup.get_text(" ", strip=True)).strip()
|
|
|
|
|
|
def _truncate(value: str, limit: int = 180) -> str:
|
|
text = value.strip()
|
|
if len(text) <= limit:
|
|
return text
|
|
return text[: limit - 1].rstrip() + "…"
|
|
|
|
|
|
def _normalize_source_name(raw: str, fallback: str) -> str:
|
|
text = html.unescape((raw or "").strip())
|
|
if " - " in text:
|
|
return text.split(" - ")[-1].strip() or fallback
|
|
return text or fallback
|
|
|
|
|
|
def _parse_datetime(raw: str | None) -> datetime | None:
|
|
if not raw:
|
|
return None
|
|
text = raw.strip()
|
|
if not text:
|
|
return None
|
|
|
|
for parser in (
|
|
lambda value: parsedate_to_datetime(value),
|
|
lambda value: datetime.fromisoformat(value.replace("Z", "+00:00")),
|
|
):
|
|
try:
|
|
parsed = parser(text)
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=UTC)
|
|
return parsed.astimezone(UTC)
|
|
except Exception:
|
|
continue
|
|
return None
|
|
|
|
|
|
def _extract_item_text(element: ET.Element, *names: str) -> str:
|
|
for name in names:
|
|
node = element.find(name)
|
|
if node is not None and node.text:
|
|
return node.text.strip()
|
|
return ""
|
|
|
|
|
|
def _parse_feed_entries(xml_text: str, source: NewsFeedSource) -> list[ParsedNewsItem]:
|
|
root = ET.fromstring(xml_text)
|
|
items: list[ParsedNewsItem] = []
|
|
|
|
rss_items = root.findall("./channel/item")
|
|
atom_entries = root.findall("{http://www.w3.org/2005/Atom}entry")
|
|
nodes = rss_items or atom_entries
|
|
|
|
for node in nodes[:MAX_ITEMS_PER_SOURCE]:
|
|
if node.tag.endswith("entry"):
|
|
title = _extract_item_text(node, "{http://www.w3.org/2005/Atom}title")
|
|
summary = _extract_item_text(
|
|
node,
|
|
"{http://www.w3.org/2005/Atom}summary",
|
|
"{http://www.w3.org/2005/Atom}content",
|
|
)
|
|
link_node = node.find("{http://www.w3.org/2005/Atom}link")
|
|
link = link_node.get("href", "").strip() if link_node is not None else ""
|
|
published = _extract_item_text(
|
|
node,
|
|
"{http://www.w3.org/2005/Atom}updated",
|
|
"{http://www.w3.org/2005/Atom}published",
|
|
)
|
|
else:
|
|
title = _extract_item_text(node, "title")
|
|
summary = _extract_item_text(node, "description", "content")
|
|
link = _extract_item_text(node, "link")
|
|
published = _extract_item_text(node, "pubDate", "published", "updated")
|
|
|
|
clean_title = html.unescape(title).strip()
|
|
clean_summary = _truncate(_strip_html(summary), 180)
|
|
if not clean_title or not link:
|
|
continue
|
|
|
|
item_source = _normalize_source_name(clean_title, source.name)
|
|
display_title = clean_title
|
|
if source.source_type == "aggregated" and " - " in clean_title:
|
|
parts = clean_title.rsplit(" - ", 1)
|
|
display_title = parts[0].strip()
|
|
item_source = _normalize_source_name(parts[1], source.name)
|
|
|
|
items.append(
|
|
ParsedNewsItem(
|
|
id=f"{source.id}:{hashlib.sha1(link.encode('utf-8')).hexdigest()[:12]}",
|
|
title=display_title,
|
|
summary=clean_summary,
|
|
url=link,
|
|
source=item_source,
|
|
feed_name=source.name,
|
|
feed_region=source.region,
|
|
homepage_url=source.homepage_url,
|
|
published_at=_parse_datetime(published),
|
|
)
|
|
)
|
|
|
|
return items
|
|
|
|
|
|
def _serialize_sources(sources: list[NewsFeedSource]) -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
"id": source.id,
|
|
"name": source.name,
|
|
"region": source.region,
|
|
"homepage_url": source.homepage_url,
|
|
}
|
|
for source in sources
|
|
]
|
|
|
|
|
|
def _serialize_anchor(anchor: RegionAnchor) -> dict[str, Any]:
|
|
return {
|
|
"region": anchor.region,
|
|
"label": anchor.label,
|
|
"latitude": anchor.latitude,
|
|
"longitude": anchor.longitude,
|
|
}
|
|
|
|
|
|
def _serialize_target(target: NewsTargetLocation | None) -> dict[str, Any] | None:
|
|
if target is None:
|
|
return None
|
|
return {
|
|
"latitude": target.latitude,
|
|
"longitude": target.longitude,
|
|
"label": target.label,
|
|
"source": target.source,
|
|
"confidence": target.confidence,
|
|
"country": target.country,
|
|
"city": target.city,
|
|
}
|
|
|
|
|
|
def build_anchor_location_patch(
|
|
item: ParsedNewsItem,
|
|
*,
|
|
queued: bool = False,
|
|
queue_available: bool | None = None,
|
|
) -> dict[str, Any]:
|
|
anchor = get_region_anchor(item.feed_region)
|
|
if queued:
|
|
resolution_stage = "queued"
|
|
ai_status = "queued"
|
|
debug_note = "queued for async target location inference"
|
|
else:
|
|
resolution_stage = item.target_resolution_stage
|
|
ai_status = item.target_ai_status
|
|
debug_note = item.target_debug_note
|
|
return {
|
|
"latitude": anchor.latitude,
|
|
"longitude": anchor.longitude,
|
|
"location_label": anchor.label,
|
|
"location_source": "region_anchor",
|
|
"verified": False,
|
|
"location_meta": {
|
|
"resolution_stage": resolution_stage,
|
|
"ai_attempted": item.target_ai_attempted,
|
|
"ai_status": ai_status,
|
|
"ai_error": item.target_ai_error,
|
|
"debug_note": debug_note,
|
|
"queue_available": queue_available,
|
|
"target": None,
|
|
"anchor": _serialize_anchor(anchor),
|
|
},
|
|
}
|
|
|
|
|
|
def build_target_location_patch(item: ParsedNewsItem, target: NewsTargetLocation | None) -> dict[str, Any]:
|
|
if target is None:
|
|
return build_anchor_location_patch(item)
|
|
anchor = get_region_anchor(item.feed_region)
|
|
return {
|
|
"latitude": target.latitude,
|
|
"longitude": target.longitude,
|
|
"location_label": target.label,
|
|
"location_source": target.source,
|
|
"verified": True,
|
|
"location_meta": {
|
|
"resolution_stage": item.target_resolution_stage,
|
|
"ai_attempted": item.target_ai_attempted,
|
|
"ai_status": item.target_ai_status,
|
|
"ai_error": item.target_ai_error,
|
|
"debug_note": item.target_debug_note,
|
|
"target": _serialize_target(target),
|
|
"anchor": _serialize_anchor(anchor),
|
|
},
|
|
}
|
|
|
|
|
|
def build_target_location_job_payload(item: ParsedNewsItem) -> dict[str, Any]:
|
|
published_at = item.published_at
|
|
return {
|
|
"id": item.id,
|
|
"title": item.title,
|
|
"summary": item.summary,
|
|
"url": item.url,
|
|
"source": item.source,
|
|
"feed_name": item.feed_name,
|
|
"feed_region": item.feed_region,
|
|
"homepage_url": item.homepage_url,
|
|
"published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None,
|
|
}
|
|
|
|
|
|
def parsed_news_item_from_job_payload(payload: dict[str, Any]) -> ParsedNewsItem:
|
|
return ParsedNewsItem(
|
|
id=str(payload.get("id") or ""),
|
|
title=str(payload.get("title") or ""),
|
|
summary=str(payload.get("summary") or ""),
|
|
url=str(payload.get("url") or ""),
|
|
source=str(payload.get("source") or ""),
|
|
feed_name=str(payload.get("feed_name") or ""),
|
|
feed_region=str(payload.get("feed_region") or "global"),
|
|
homepage_url=str(payload.get("homepage_url") or ""),
|
|
published_at=_parse_datetime(_coerce_str(payload.get("published_at"))),
|
|
)
|
|
|
|
|
|
def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, Any]:
|
|
published_at = item.published_at
|
|
location_patch = item.location_patch or build_target_location_patch(item, item.target_location)
|
|
return {
|
|
"id": item.id,
|
|
"title": item.title,
|
|
"summary": item.summary,
|
|
"url": item.url,
|
|
"source": item.source,
|
|
"feed_name": item.feed_name,
|
|
"region": item.feed_region,
|
|
"homepage_url": item.homepage_url,
|
|
"published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None,
|
|
"latitude": location_patch["latitude"],
|
|
"longitude": location_patch["longitude"],
|
|
"location_label": location_patch["location_label"],
|
|
"location_source": location_patch["location_source"],
|
|
"verified": location_patch["verified"],
|
|
"location_meta": location_patch["location_meta"],
|
|
"is_focus_match": item.feed_region == active_region,
|
|
}
|
|
|
|
|
|
def _build_payload(
|
|
*,
|
|
lat: float | None,
|
|
lon: float | None,
|
|
active_region: str,
|
|
items: list[ParsedNewsItem],
|
|
sources: list[NewsFeedSource],
|
|
errors: list[str],
|
|
stale: bool,
|
|
generated_at: datetime | None = None,
|
|
) -> dict[str, Any]:
|
|
profile = get_region_profile(active_region)
|
|
timestamp = generated_at or datetime.now(UTC)
|
|
return {
|
|
"generated_at": timestamp.isoformat().replace("+00:00", "Z"),
|
|
"focus": {
|
|
"lat": lat,
|
|
"lon": lon,
|
|
"region": active_region,
|
|
"label": profile.label,
|
|
"accent": profile.accent,
|
|
},
|
|
"sources": _serialize_sources(sources),
|
|
"items": [_serialize_item(item, active_region=active_region) for item in items],
|
|
"errors": errors,
|
|
"stale": stale,
|
|
}
|
|
|
|
|
|
def _rank_and_trim_items(items: list[ParsedNewsItem], *, active_region: str) -> list[ParsedNewsItem]:
|
|
deduped: dict[str, ParsedNewsItem] = {}
|
|
for item in items:
|
|
key = item.url.strip() or item.title.strip().lower()
|
|
if key not in deduped:
|
|
deduped[key] = item
|
|
|
|
return sorted(
|
|
deduped.values(),
|
|
key=lambda item: (
|
|
item.feed_region != active_region,
|
|
item.published_at is None,
|
|
-(item.published_at.timestamp() if item.published_at else 0),
|
|
item.feed_name,
|
|
),
|
|
)[:MAX_ITEMS_TOTAL]
|
|
|
|
|
|
def _get_cached_region_feed(region: str) -> CachedRegionFeed | None:
|
|
cached = _REGION_CACHE.get(region)
|
|
if not cached:
|
|
return None
|
|
age_seconds = (datetime.now(UTC) - cached.fetched_at).total_seconds()
|
|
if age_seconds > STALE_CACHE_MAX_AGE_SECONDS:
|
|
return None
|
|
return cached
|
|
|
|
|
|
def _store_region_cache(region: str, *, items: list[ParsedNewsItem], sources: list[NewsFeedSource]) -> None:
|
|
_REGION_CACHE[region] = CachedRegionFeed(
|
|
region=region,
|
|
fetched_at=datetime.now(UTC),
|
|
items=list(items),
|
|
sources=list(sources),
|
|
)
|
|
|
|
|
|
async def _apply_cached_locations_and_enqueue(items: list[ParsedNewsItem]) -> list[ParsedNewsItem]:
|
|
if not items:
|
|
return items
|
|
|
|
from app.services.earth_news_queue import (
|
|
enqueue_target_location_job,
|
|
get_cached_target_location_patch,
|
|
)
|
|
|
|
async def apply_location(item: ParsedNewsItem) -> ParsedNewsItem:
|
|
cached_patch = await get_cached_target_location_patch(item.id)
|
|
if cached_patch:
|
|
item.location_patch = cached_patch
|
|
return item
|
|
|
|
queued = await enqueue_target_location_job(build_target_location_job_payload(item))
|
|
item.location_patch = build_anchor_location_patch(
|
|
item,
|
|
queued=queued,
|
|
queue_available=queued,
|
|
)
|
|
return item
|
|
|
|
return list(await asyncio.gather(*(apply_location(item) for item in items)))
|
|
|
|
|
|
async def _enqueue_unverified_locations(items: list[ParsedNewsItem]) -> None:
|
|
if not items:
|
|
return
|
|
|
|
from app.services.earth_news_queue import enqueue_target_location_job
|
|
|
|
await asyncio.gather(
|
|
*(
|
|
enqueue_target_location_job(build_target_location_job_payload(item))
|
|
for item in items
|
|
if item.location_patch is None or item.location_patch.get("verified") is False
|
|
)
|
|
)
|
|
|
|
|
|
async def _fetch_source(
|
|
client: httpx.AsyncClient,
|
|
source: NewsFeedSource,
|
|
) -> tuple[NewsFeedSource, list[ParsedNewsItem], str | None]:
|
|
try:
|
|
response = await client.get(source.feed_url)
|
|
response.raise_for_status()
|
|
return source, _parse_feed_entries(response.text, source), None
|
|
except Exception as exc:
|
|
return source, [], str(exc)
|
|
|
|
|
|
async def _fetch_rss_items_for_sources(
|
|
sources: list[NewsFeedSource],
|
|
) -> tuple[list[ParsedNewsItem], list[str]]:
|
|
errors: list[str] = []
|
|
async with httpx.AsyncClient(
|
|
timeout=REQUEST_TIMEOUT,
|
|
follow_redirects=True,
|
|
headers={"User-Agent": USER_AGENT},
|
|
) as client:
|
|
results = await asyncio.gather(*(_fetch_source(client, source) for source in sources))
|
|
|
|
fetched_items: list[ParsedNewsItem] = []
|
|
for source, items, error in results:
|
|
if error:
|
|
errors.append(f"{source.name}: {error}")
|
|
continue
|
|
fetched_items.extend(items)
|
|
return fetched_items, errors
|
|
|
|
|
|
def _needs_rss_supplement(*, item_count: int, newest_at: datetime | None) -> bool:
|
|
if item_count < MAX_ITEMS_TOTAL:
|
|
return True
|
|
if newest_at is None:
|
|
return True
|
|
age_seconds = (datetime.now(UTC) - newest_at).total_seconds()
|
|
return age_seconds > RSS_SUPPLEMENT_MAX_AGE_SECONDS
|
|
|
|
|
|
async def _get_earth_news_payload_from_rss_only(
|
|
*,
|
|
lat: float | None,
|
|
lon: float | None,
|
|
active_region: str,
|
|
sources: list[NewsFeedSource],
|
|
) -> dict[str, Any]:
|
|
fetched_items, errors = await _fetch_rss_items_for_sources(sources)
|
|
ranked_items = _rank_and_trim_items(fetched_items, active_region=active_region)
|
|
if ranked_items:
|
|
ranked_items = await _apply_cached_locations_and_enqueue(ranked_items)
|
|
_store_region_cache(active_region, items=ranked_items, sources=sources)
|
|
return _build_payload(
|
|
lat=lat,
|
|
lon=lon,
|
|
active_region=active_region,
|
|
items=ranked_items,
|
|
sources=sources,
|
|
errors=errors,
|
|
stale=False,
|
|
)
|
|
|
|
cached = _get_cached_region_feed(active_region)
|
|
if cached:
|
|
cached.items = await _apply_cached_locations_and_enqueue(cached.items)
|
|
return _build_payload(
|
|
lat=lat,
|
|
lon=lon,
|
|
active_region=active_region,
|
|
items=cached.items,
|
|
sources=cached.sources,
|
|
errors=errors,
|
|
stale=True,
|
|
generated_at=cached.fetched_at,
|
|
)
|
|
|
|
return _build_payload(
|
|
lat=lat,
|
|
lon=lon,
|
|
active_region=active_region,
|
|
items=[],
|
|
sources=sources,
|
|
errors=errors,
|
|
stale=False,
|
|
)
|
|
|
|
|
|
async def get_earth_news_payload(
|
|
lat: float | None = None,
|
|
lon: float | None = None,
|
|
*,
|
|
provider_client: AIProviderClient | None = None,
|
|
db: AsyncSession | None = None,
|
|
) -> dict[str, Any]:
|
|
del provider_client
|
|
active_region = determine_focus_region(lat, lon)
|
|
sources = get_sources_for_region(active_region)
|
|
|
|
if db is None:
|
|
return await _get_earth_news_payload_from_rss_only(
|
|
lat=lat,
|
|
lon=lon,
|
|
active_region=active_region,
|
|
sources=sources,
|
|
)
|
|
|
|
from app.services.earth_news_store import (
|
|
get_earth_news_freshness,
|
|
list_earth_news_items,
|
|
upsert_earth_news_items,
|
|
)
|
|
|
|
errors: list[str] = []
|
|
item_count, newest_at = await get_earth_news_freshness(db, active_region=active_region)
|
|
should_supplement = _needs_rss_supplement(item_count=item_count, newest_at=newest_at)
|
|
if should_supplement:
|
|
fetched_items, errors = await _fetch_rss_items_for_sources(sources)
|
|
ranked_fetched_items = _rank_and_trim_items(fetched_items, active_region=active_region)
|
|
await upsert_earth_news_items(db, ranked_fetched_items)
|
|
|
|
items = await list_earth_news_items(
|
|
db,
|
|
active_region=active_region,
|
|
limit=MAX_ITEMS_TOTAL,
|
|
)
|
|
await _enqueue_unverified_locations(items)
|
|
stale = bool(errors and items)
|
|
|
|
return _build_payload(
|
|
lat=lat,
|
|
lon=lon,
|
|
active_region=active_region,
|
|
items=items,
|
|
sources=sources,
|
|
errors=errors,
|
|
stale=stale,
|
|
)
|