feat: refine earth hud panel behaviors and news board
This commit is contained in:
linkong
490
backend/app/services/earth_news.py
Normal file
490
backend/app/services/earth_news.py
Normal file
@@ -0,0 +1,490 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
import hashlib
|
||||
import html
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
USER_AGENT = "PlanetEarthNewsBoard/1.0 (+https://planet.local)"
|
||||
REQUEST_TIMEOUT = 12.0
|
||||
MAX_ITEMS_PER_SOURCE = 6
|
||||
MAX_ITEMS_TOTAL = 12
|
||||
STALE_CACHE_MAX_AGE_SECONDS = 60 * 45
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RegionProfile:
|
||||
key: str
|
||||
label: str
|
||||
query: str
|
||||
accent: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NewsFeedSource:
|
||||
id: str
|
||||
name: str
|
||||
region: str
|
||||
feed_url: str
|
||||
homepage_url: str
|
||||
source_type: str = "rss"
|
||||
priority: int = 100
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsedNewsItem:
|
||||
id: str
|
||||
title: str
|
||||
summary: str
|
||||
url: str
|
||||
source: str
|
||||
feed_name: str
|
||||
feed_region: str
|
||||
homepage_url: str
|
||||
published_at: datetime | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class CachedRegionFeed:
|
||||
region: str
|
||||
fetched_at: datetime
|
||||
items: list[ParsedNewsItem]
|
||||
sources: list[NewsFeedSource]
|
||||
|
||||
|
||||
REGION_PROFILES: dict[str, RegionProfile] = {
|
||||
"americas": RegionProfile(
|
||||
key="americas",
|
||||
label="美洲焦点",
|
||||
query='Americas geopolitics OR Latin America OR "United States" OR Canada',
|
||||
accent="#79d3ff",
|
||||
),
|
||||
"europe": RegionProfile(
|
||||
key="europe",
|
||||
label="欧洲焦点",
|
||||
query='Europe geopolitics OR EU OR NATO OR "Eastern Europe"',
|
||||
accent="#8fd4ff",
|
||||
),
|
||||
"middle-east-africa": RegionProfile(
|
||||
key="middle-east-africa",
|
||||
label="中东与非洲焦点",
|
||||
query='"Middle East" OR Africa geopolitics OR Red Sea OR Gulf',
|
||||
accent="#ffb56a",
|
||||
),
|
||||
"asia-pacific": RegionProfile(
|
||||
key="asia-pacific",
|
||||
label="亚太焦点",
|
||||
query='"Asia Pacific" OR Indo-Pacific OR China OR Japan OR Korea OR ASEAN',
|
||||
accent="#78f2cf",
|
||||
),
|
||||
"global": RegionProfile(
|
||||
key="global",
|
||||
label="全球焦点",
|
||||
query='"world news" OR geopolitics OR "global affairs"',
|
||||
accent="#d6e6ff",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _google_news_feed(query: str, *, hl: str, gl: str, ceid: str) -> str:
|
||||
return (
|
||||
"https://news.google.com/rss/search?q="
|
||||
+ quote(query, safe="")
|
||||
+ f"&hl={hl}&gl={gl}&ceid={ceid}"
|
||||
)
|
||||
|
||||
|
||||
NEWS_FEED_SOURCES: tuple[NewsFeedSource, ...] = (
|
||||
NewsFeedSource(
|
||||
id="bbc-world",
|
||||
name="BBC World",
|
||||
region="global",
|
||||
feed_url="https://feeds.bbci.co.uk/news/world/rss.xml",
|
||||
homepage_url="https://www.bbc.com/news/world",
|
||||
priority=10,
|
||||
),
|
||||
NewsFeedSource(
|
||||
id="dw-top",
|
||||
name="DW Top Stories",
|
||||
region="europe",
|
||||
feed_url="https://rss.dw.com/rdf/rss-en-top",
|
||||
homepage_url="https://www.dw.com/en/top-stories/s-9097",
|
||||
priority=20,
|
||||
),
|
||||
NewsFeedSource(
|
||||
id="global-scan",
|
||||
name="Global Monitor / World",
|
||||
region="global",
|
||||
feed_url=_google_news_feed(
|
||||
REGION_PROFILES["global"].query,
|
||||
hl="en-US",
|
||||
gl="US",
|
||||
ceid="US:en",
|
||||
),
|
||||
homepage_url="https://news.google.com/",
|
||||
source_type="aggregated",
|
||||
priority=30,
|
||||
),
|
||||
NewsFeedSource(
|
||||
id="google-americas",
|
||||
name="Global Monitor / Americas",
|
||||
region="americas",
|
||||
feed_url=_google_news_feed(
|
||||
REGION_PROFILES["americas"].query,
|
||||
hl="en-US",
|
||||
gl="US",
|
||||
ceid="US:en",
|
||||
),
|
||||
homepage_url="https://news.google.com/",
|
||||
source_type="aggregated",
|
||||
priority=40,
|
||||
),
|
||||
NewsFeedSource(
|
||||
id="google-europe",
|
||||
name="Global Monitor / Europe",
|
||||
region="europe",
|
||||
feed_url=_google_news_feed(
|
||||
REGION_PROFILES["europe"].query,
|
||||
hl="en-GB",
|
||||
gl="GB",
|
||||
ceid="GB:en",
|
||||
),
|
||||
homepage_url="https://news.google.com/",
|
||||
source_type="aggregated",
|
||||
priority=40,
|
||||
),
|
||||
NewsFeedSource(
|
||||
id="google-mea",
|
||||
name="Global Monitor / MEA",
|
||||
region="middle-east-africa",
|
||||
feed_url=_google_news_feed(
|
||||
REGION_PROFILES["middle-east-africa"].query,
|
||||
hl="en-US",
|
||||
gl="US",
|
||||
ceid="US:en",
|
||||
),
|
||||
homepage_url="https://news.google.com/",
|
||||
source_type="aggregated",
|
||||
priority=40,
|
||||
),
|
||||
NewsFeedSource(
|
||||
id="google-apac",
|
||||
name="Global Monitor / APAC",
|
||||
region="asia-pacific",
|
||||
feed_url=_google_news_feed(
|
||||
REGION_PROFILES["asia-pacific"].query,
|
||||
hl="en-SG",
|
||||
gl="SG",
|
||||
ceid="SG:en",
|
||||
),
|
||||
homepage_url="https://news.google.com/",
|
||||
source_type="aggregated",
|
||||
priority=40,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
_REGION_CACHE: dict[str, CachedRegionFeed] = {}
|
||||
|
||||
|
||||
def determine_focus_region(lat: float | None, lon: float | None) -> str:
|
||||
if lat is None or lon is None:
|
||||
return "global"
|
||||
if -170 <= lon <= -30:
|
||||
return "americas"
|
||||
if -30 < lon <= 45:
|
||||
return "europe" if lat >= 30 else "middle-east-africa"
|
||||
if 45 < lon <= 150:
|
||||
return "middle-east-africa" if lat < 10 else "asia-pacific"
|
||||
return "asia-pacific"
|
||||
|
||||
|
||||
def get_region_profile(region: str) -> RegionProfile:
|
||||
return REGION_PROFILES.get(region, REGION_PROFILES["global"])
|
||||
|
||||
|
||||
def get_sources_for_region(region: str) -> list[NewsFeedSource]:
|
||||
return sorted(
|
||||
[source for source in NEWS_FEED_SOURCES if source.region in {"global", region}],
|
||||
key=lambda source: (source.priority, source.name),
|
||||
)
|
||||
|
||||
|
||||
def _strip_html(value: str) -> str:
|
||||
if not value:
|
||||
return ""
|
||||
soup = BeautifulSoup(value, "html.parser")
|
||||
return re.sub(r"\s+", " ", soup.get_text(" ", strip=True)).strip()
|
||||
|
||||
|
||||
def _truncate(value: str, limit: int = 180) -> str:
|
||||
text = value.strip()
|
||||
if len(text) <= limit:
|
||||
return text
|
||||
return text[: limit - 1].rstrip() + "…"
|
||||
|
||||
|
||||
def _normalize_source_name(raw: str, fallback: str) -> str:
|
||||
text = html.unescape((raw or "").strip())
|
||||
if " - " in text:
|
||||
return text.split(" - ")[-1].strip() or fallback
|
||||
return text or fallback
|
||||
|
||||
|
||||
def _parse_datetime(raw: str | None) -> datetime | None:
|
||||
if not raw:
|
||||
return None
|
||||
text = raw.strip()
|
||||
if not text:
|
||||
return None
|
||||
|
||||
for parser in (
|
||||
lambda value: parsedate_to_datetime(value),
|
||||
lambda value: datetime.fromisoformat(value.replace("Z", "+00:00")),
|
||||
):
|
||||
try:
|
||||
parsed = parser(text)
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=UTC)
|
||||
return parsed.astimezone(UTC)
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _extract_item_text(element: ET.Element, *names: str) -> str:
|
||||
for name in names:
|
||||
node = element.find(name)
|
||||
if node is not None and node.text:
|
||||
return node.text.strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _parse_feed_entries(xml_text: str, source: NewsFeedSource) -> list[ParsedNewsItem]:
|
||||
root = ET.fromstring(xml_text)
|
||||
items: list[ParsedNewsItem] = []
|
||||
|
||||
rss_items = root.findall("./channel/item")
|
||||
atom_entries = root.findall("{http://www.w3.org/2005/Atom}entry")
|
||||
nodes = rss_items or atom_entries
|
||||
|
||||
for node in nodes[:MAX_ITEMS_PER_SOURCE]:
|
||||
if node.tag.endswith("entry"):
|
||||
title = _extract_item_text(node, "{http://www.w3.org/2005/Atom}title")
|
||||
summary = _extract_item_text(
|
||||
node,
|
||||
"{http://www.w3.org/2005/Atom}summary",
|
||||
"{http://www.w3.org/2005/Atom}content",
|
||||
)
|
||||
link_node = node.find("{http://www.w3.org/2005/Atom}link")
|
||||
link = link_node.get("href", "").strip() if link_node is not None else ""
|
||||
published = _extract_item_text(
|
||||
node,
|
||||
"{http://www.w3.org/2005/Atom}updated",
|
||||
"{http://www.w3.org/2005/Atom}published",
|
||||
)
|
||||
else:
|
||||
title = _extract_item_text(node, "title")
|
||||
summary = _extract_item_text(node, "description", "content")
|
||||
link = _extract_item_text(node, "link")
|
||||
published = _extract_item_text(node, "pubDate", "published", "updated")
|
||||
|
||||
clean_title = html.unescape(title).strip()
|
||||
clean_summary = _truncate(_strip_html(summary), 180)
|
||||
if not clean_title or not link:
|
||||
continue
|
||||
|
||||
item_source = _normalize_source_name(clean_title, source.name)
|
||||
display_title = clean_title
|
||||
if source.source_type == "aggregated" and " - " in clean_title:
|
||||
parts = clean_title.rsplit(" - ", 1)
|
||||
display_title = parts[0].strip()
|
||||
item_source = _normalize_source_name(parts[1], source.name)
|
||||
|
||||
items.append(
|
||||
ParsedNewsItem(
|
||||
id=f"{source.id}:{hashlib.sha1(link.encode('utf-8')).hexdigest()[:12]}",
|
||||
title=display_title,
|
||||
summary=clean_summary,
|
||||
url=link,
|
||||
source=item_source,
|
||||
feed_name=source.name,
|
||||
feed_region=source.region,
|
||||
homepage_url=source.homepage_url,
|
||||
published_at=_parse_datetime(published),
|
||||
)
|
||||
)
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def _serialize_sources(sources: list[NewsFeedSource]) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": source.id,
|
||||
"name": source.name,
|
||||
"region": source.region,
|
||||
"homepage_url": source.homepage_url,
|
||||
}
|
||||
for source in sources
|
||||
]
|
||||
|
||||
|
||||
def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, Any]:
|
||||
published_at = item.published_at
|
||||
return {
|
||||
"id": item.id,
|
||||
"title": item.title,
|
||||
"summary": item.summary,
|
||||
"url": item.url,
|
||||
"source": item.source,
|
||||
"feed_name": item.feed_name,
|
||||
"region": item.feed_region,
|
||||
"homepage_url": item.homepage_url,
|
||||
"published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None,
|
||||
"is_focus_match": item.feed_region == active_region,
|
||||
}
|
||||
|
||||
|
||||
def _build_payload(
|
||||
*,
|
||||
lat: float | None,
|
||||
lon: float | None,
|
||||
active_region: str,
|
||||
items: list[ParsedNewsItem],
|
||||
sources: list[NewsFeedSource],
|
||||
errors: list[str],
|
||||
stale: bool,
|
||||
generated_at: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
profile = get_region_profile(active_region)
|
||||
timestamp = generated_at or datetime.now(UTC)
|
||||
return {
|
||||
"generated_at": timestamp.isoformat().replace("+00:00", "Z"),
|
||||
"focus": {
|
||||
"lat": lat,
|
||||
"lon": lon,
|
||||
"region": active_region,
|
||||
"label": profile.label,
|
||||
"accent": profile.accent,
|
||||
},
|
||||
"sources": _serialize_sources(sources),
|
||||
"items": [_serialize_item(item, active_region=active_region) for item in items],
|
||||
"errors": errors,
|
||||
"stale": stale,
|
||||
}
|
||||
|
||||
|
||||
def _rank_and_trim_items(items: list[ParsedNewsItem], *, active_region: str) -> list[ParsedNewsItem]:
|
||||
deduped: dict[str, ParsedNewsItem] = {}
|
||||
for item in items:
|
||||
key = item.url.strip() or item.title.strip().lower()
|
||||
if key not in deduped:
|
||||
deduped[key] = item
|
||||
|
||||
return sorted(
|
||||
deduped.values(),
|
||||
key=lambda item: (
|
||||
item.feed_region != active_region,
|
||||
item.published_at is None,
|
||||
-(item.published_at.timestamp() if item.published_at else 0),
|
||||
item.feed_name,
|
||||
),
|
||||
)[:MAX_ITEMS_TOTAL]
|
||||
|
||||
|
||||
def _get_cached_region_feed(region: str) -> CachedRegionFeed | None:
|
||||
cached = _REGION_CACHE.get(region)
|
||||
if not cached:
|
||||
return None
|
||||
age_seconds = (datetime.now(UTC) - cached.fetched_at).total_seconds()
|
||||
if age_seconds > STALE_CACHE_MAX_AGE_SECONDS:
|
||||
return None
|
||||
return cached
|
||||
|
||||
|
||||
def _store_region_cache(region: str, *, items: list[ParsedNewsItem], sources: list[NewsFeedSource]) -> None:
|
||||
_REGION_CACHE[region] = CachedRegionFeed(
|
||||
region=region,
|
||||
fetched_at=datetime.now(UTC),
|
||||
items=list(items),
|
||||
sources=list(sources),
|
||||
)
|
||||
|
||||
|
||||
async def _fetch_source(
|
||||
client: httpx.AsyncClient,
|
||||
source: NewsFeedSource,
|
||||
) -> tuple[NewsFeedSource, list[ParsedNewsItem], str | None]:
|
||||
try:
|
||||
response = await client.get(source.feed_url)
|
||||
response.raise_for_status()
|
||||
return source, _parse_feed_entries(response.text, source), None
|
||||
except Exception as exc:
|
||||
return source, [], str(exc)
|
||||
|
||||
|
||||
async def get_earth_news_payload(lat: float | None = None, lon: float | None = None) -> dict[str, Any]:
|
||||
active_region = determine_focus_region(lat, lon)
|
||||
sources = get_sources_for_region(active_region)
|
||||
errors: list[str] = []
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
timeout=REQUEST_TIMEOUT,
|
||||
follow_redirects=True,
|
||||
headers={"User-Agent": USER_AGENT},
|
||||
) as client:
|
||||
results = await asyncio.gather(*(_fetch_source(client, source) for source in sources))
|
||||
|
||||
fetched_items: list[ParsedNewsItem] = []
|
||||
for source, items, error in results:
|
||||
if error:
|
||||
errors.append(f"{source.name}: {error}")
|
||||
continue
|
||||
fetched_items.extend(items)
|
||||
|
||||
ranked_items = _rank_and_trim_items(fetched_items, active_region=active_region)
|
||||
if ranked_items:
|
||||
_store_region_cache(active_region, items=ranked_items, sources=sources)
|
||||
return _build_payload(
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
active_region=active_region,
|
||||
items=ranked_items,
|
||||
sources=sources,
|
||||
errors=errors,
|
||||
stale=False,
|
||||
)
|
||||
|
||||
cached = _get_cached_region_feed(active_region)
|
||||
if cached:
|
||||
return _build_payload(
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
active_region=active_region,
|
||||
items=cached.items,
|
||||
sources=cached.sources,
|
||||
errors=errors,
|
||||
stale=True,
|
||||
generated_at=cached.fetched_at,
|
||||
)
|
||||
|
||||
return _build_payload(
|
||||
lat=lat,
|
||||
lon=lon,
|
||||
active_region=active_region,
|
||||
items=[],
|
||||
sources=sources,
|
||||
errors=errors,
|
||||
stale=False,
|
||||
)
|
||||
Reference in New Issue
Block a user