491 lines
14 KiB
Python
491 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime
|
|
from email.utils import parsedate_to_datetime
|
|
import hashlib
|
|
import html
|
|
import re
|
|
from typing import Any
|
|
from urllib.parse import quote
|
|
import xml.etree.ElementTree as ET
|
|
|
|
import httpx
|
|
from bs4 import BeautifulSoup
|
|
|
|
|
|
USER_AGENT = "PlanetEarthNewsBoard/1.0 (+https://planet.local)"
|
|
REQUEST_TIMEOUT = 12.0
|
|
MAX_ITEMS_PER_SOURCE = 6
|
|
MAX_ITEMS_TOTAL = 12
|
|
STALE_CACHE_MAX_AGE_SECONDS = 60 * 45
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RegionProfile:
|
|
key: str
|
|
label: str
|
|
query: str
|
|
accent: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class NewsFeedSource:
|
|
id: str
|
|
name: str
|
|
region: str
|
|
feed_url: str
|
|
homepage_url: str
|
|
source_type: str = "rss"
|
|
priority: int = 100
|
|
|
|
|
|
@dataclass
|
|
class ParsedNewsItem:
|
|
id: str
|
|
title: str
|
|
summary: str
|
|
url: str
|
|
source: str
|
|
feed_name: str
|
|
feed_region: str
|
|
homepage_url: str
|
|
published_at: datetime | None
|
|
|
|
|
|
@dataclass
|
|
class CachedRegionFeed:
|
|
region: str
|
|
fetched_at: datetime
|
|
items: list[ParsedNewsItem]
|
|
sources: list[NewsFeedSource]
|
|
|
|
|
|
REGION_PROFILES: dict[str, RegionProfile] = {
|
|
"americas": RegionProfile(
|
|
key="americas",
|
|
label="美洲焦点",
|
|
query='Americas geopolitics OR Latin America OR "United States" OR Canada',
|
|
accent="#79d3ff",
|
|
),
|
|
"europe": RegionProfile(
|
|
key="europe",
|
|
label="欧洲焦点",
|
|
query='Europe geopolitics OR EU OR NATO OR "Eastern Europe"',
|
|
accent="#8fd4ff",
|
|
),
|
|
"middle-east-africa": RegionProfile(
|
|
key="middle-east-africa",
|
|
label="中东与非洲焦点",
|
|
query='"Middle East" OR Africa geopolitics OR Red Sea OR Gulf',
|
|
accent="#ffb56a",
|
|
),
|
|
"asia-pacific": RegionProfile(
|
|
key="asia-pacific",
|
|
label="亚太焦点",
|
|
query='"Asia Pacific" OR Indo-Pacific OR China OR Japan OR Korea OR ASEAN',
|
|
accent="#78f2cf",
|
|
),
|
|
"global": RegionProfile(
|
|
key="global",
|
|
label="全球焦点",
|
|
query='"world news" OR geopolitics OR "global affairs"',
|
|
accent="#d6e6ff",
|
|
),
|
|
}
|
|
|
|
|
|
def _google_news_feed(query: str, *, hl: str, gl: str, ceid: str) -> str:
|
|
return (
|
|
"https://news.google.com/rss/search?q="
|
|
+ quote(query, safe="")
|
|
+ f"&hl={hl}&gl={gl}&ceid={ceid}"
|
|
)
|
|
|
|
|
|
NEWS_FEED_SOURCES: tuple[NewsFeedSource, ...] = (
|
|
NewsFeedSource(
|
|
id="bbc-world",
|
|
name="BBC World",
|
|
region="global",
|
|
feed_url="https://feeds.bbci.co.uk/news/world/rss.xml",
|
|
homepage_url="https://www.bbc.com/news/world",
|
|
priority=10,
|
|
),
|
|
NewsFeedSource(
|
|
id="dw-top",
|
|
name="DW Top Stories",
|
|
region="europe",
|
|
feed_url="https://rss.dw.com/rdf/rss-en-top",
|
|
homepage_url="https://www.dw.com/en/top-stories/s-9097",
|
|
priority=20,
|
|
),
|
|
NewsFeedSource(
|
|
id="global-scan",
|
|
name="Global Monitor / World",
|
|
region="global",
|
|
feed_url=_google_news_feed(
|
|
REGION_PROFILES["global"].query,
|
|
hl="en-US",
|
|
gl="US",
|
|
ceid="US:en",
|
|
),
|
|
homepage_url="https://news.google.com/",
|
|
source_type="aggregated",
|
|
priority=30,
|
|
),
|
|
NewsFeedSource(
|
|
id="google-americas",
|
|
name="Global Monitor / Americas",
|
|
region="americas",
|
|
feed_url=_google_news_feed(
|
|
REGION_PROFILES["americas"].query,
|
|
hl="en-US",
|
|
gl="US",
|
|
ceid="US:en",
|
|
),
|
|
homepage_url="https://news.google.com/",
|
|
source_type="aggregated",
|
|
priority=40,
|
|
),
|
|
NewsFeedSource(
|
|
id="google-europe",
|
|
name="Global Monitor / Europe",
|
|
region="europe",
|
|
feed_url=_google_news_feed(
|
|
REGION_PROFILES["europe"].query,
|
|
hl="en-GB",
|
|
gl="GB",
|
|
ceid="GB:en",
|
|
),
|
|
homepage_url="https://news.google.com/",
|
|
source_type="aggregated",
|
|
priority=40,
|
|
),
|
|
NewsFeedSource(
|
|
id="google-mea",
|
|
name="Global Monitor / MEA",
|
|
region="middle-east-africa",
|
|
feed_url=_google_news_feed(
|
|
REGION_PROFILES["middle-east-africa"].query,
|
|
hl="en-US",
|
|
gl="US",
|
|
ceid="US:en",
|
|
),
|
|
homepage_url="https://news.google.com/",
|
|
source_type="aggregated",
|
|
priority=40,
|
|
),
|
|
NewsFeedSource(
|
|
id="google-apac",
|
|
name="Global Monitor / APAC",
|
|
region="asia-pacific",
|
|
feed_url=_google_news_feed(
|
|
REGION_PROFILES["asia-pacific"].query,
|
|
hl="en-SG",
|
|
gl="SG",
|
|
ceid="SG:en",
|
|
),
|
|
homepage_url="https://news.google.com/",
|
|
source_type="aggregated",
|
|
priority=40,
|
|
),
|
|
)
|
|
|
|
|
|
_REGION_CACHE: dict[str, CachedRegionFeed] = {}
|
|
|
|
|
|
def determine_focus_region(lat: float | None, lon: float | None) -> str:
|
|
if lat is None or lon is None:
|
|
return "global"
|
|
if -170 <= lon <= -30:
|
|
return "americas"
|
|
if -30 < lon <= 45:
|
|
return "europe" if lat >= 30 else "middle-east-africa"
|
|
if 45 < lon <= 150:
|
|
return "middle-east-africa" if lat < 10 else "asia-pacific"
|
|
return "asia-pacific"
|
|
|
|
|
|
def get_region_profile(region: str) -> RegionProfile:
|
|
return REGION_PROFILES.get(region, REGION_PROFILES["global"])
|
|
|
|
|
|
def get_sources_for_region(region: str) -> list[NewsFeedSource]:
|
|
return sorted(
|
|
[source for source in NEWS_FEED_SOURCES if source.region in {"global", region}],
|
|
key=lambda source: (source.priority, source.name),
|
|
)
|
|
|
|
|
|
def _strip_html(value: str) -> str:
|
|
if not value:
|
|
return ""
|
|
soup = BeautifulSoup(value, "html.parser")
|
|
return re.sub(r"\s+", " ", soup.get_text(" ", strip=True)).strip()
|
|
|
|
|
|
def _truncate(value: str, limit: int = 180) -> str:
|
|
text = value.strip()
|
|
if len(text) <= limit:
|
|
return text
|
|
return text[: limit - 1].rstrip() + "…"
|
|
|
|
|
|
def _normalize_source_name(raw: str, fallback: str) -> str:
|
|
text = html.unescape((raw or "").strip())
|
|
if " - " in text:
|
|
return text.split(" - ")[-1].strip() or fallback
|
|
return text or fallback
|
|
|
|
|
|
def _parse_datetime(raw: str | None) -> datetime | None:
|
|
if not raw:
|
|
return None
|
|
text = raw.strip()
|
|
if not text:
|
|
return None
|
|
|
|
for parser in (
|
|
lambda value: parsedate_to_datetime(value),
|
|
lambda value: datetime.fromisoformat(value.replace("Z", "+00:00")),
|
|
):
|
|
try:
|
|
parsed = parser(text)
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=UTC)
|
|
return parsed.astimezone(UTC)
|
|
except Exception:
|
|
continue
|
|
return None
|
|
|
|
|
|
def _extract_item_text(element: ET.Element, *names: str) -> str:
|
|
for name in names:
|
|
node = element.find(name)
|
|
if node is not None and node.text:
|
|
return node.text.strip()
|
|
return ""
|
|
|
|
|
|
def _parse_feed_entries(xml_text: str, source: NewsFeedSource) -> list[ParsedNewsItem]:
|
|
root = ET.fromstring(xml_text)
|
|
items: list[ParsedNewsItem] = []
|
|
|
|
rss_items = root.findall("./channel/item")
|
|
atom_entries = root.findall("{http://www.w3.org/2005/Atom}entry")
|
|
nodes = rss_items or atom_entries
|
|
|
|
for node in nodes[:MAX_ITEMS_PER_SOURCE]:
|
|
if node.tag.endswith("entry"):
|
|
title = _extract_item_text(node, "{http://www.w3.org/2005/Atom}title")
|
|
summary = _extract_item_text(
|
|
node,
|
|
"{http://www.w3.org/2005/Atom}summary",
|
|
"{http://www.w3.org/2005/Atom}content",
|
|
)
|
|
link_node = node.find("{http://www.w3.org/2005/Atom}link")
|
|
link = link_node.get("href", "").strip() if link_node is not None else ""
|
|
published = _extract_item_text(
|
|
node,
|
|
"{http://www.w3.org/2005/Atom}updated",
|
|
"{http://www.w3.org/2005/Atom}published",
|
|
)
|
|
else:
|
|
title = _extract_item_text(node, "title")
|
|
summary = _extract_item_text(node, "description", "content")
|
|
link = _extract_item_text(node, "link")
|
|
published = _extract_item_text(node, "pubDate", "published", "updated")
|
|
|
|
clean_title = html.unescape(title).strip()
|
|
clean_summary = _truncate(_strip_html(summary), 180)
|
|
if not clean_title or not link:
|
|
continue
|
|
|
|
item_source = _normalize_source_name(clean_title, source.name)
|
|
display_title = clean_title
|
|
if source.source_type == "aggregated" and " - " in clean_title:
|
|
parts = clean_title.rsplit(" - ", 1)
|
|
display_title = parts[0].strip()
|
|
item_source = _normalize_source_name(parts[1], source.name)
|
|
|
|
items.append(
|
|
ParsedNewsItem(
|
|
id=f"{source.id}:{hashlib.sha1(link.encode('utf-8')).hexdigest()[:12]}",
|
|
title=display_title,
|
|
summary=clean_summary,
|
|
url=link,
|
|
source=item_source,
|
|
feed_name=source.name,
|
|
feed_region=source.region,
|
|
homepage_url=source.homepage_url,
|
|
published_at=_parse_datetime(published),
|
|
)
|
|
)
|
|
|
|
return items
|
|
|
|
|
|
def _serialize_sources(sources: list[NewsFeedSource]) -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
"id": source.id,
|
|
"name": source.name,
|
|
"region": source.region,
|
|
"homepage_url": source.homepage_url,
|
|
}
|
|
for source in sources
|
|
]
|
|
|
|
|
|
def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, Any]:
|
|
published_at = item.published_at
|
|
return {
|
|
"id": item.id,
|
|
"title": item.title,
|
|
"summary": item.summary,
|
|
"url": item.url,
|
|
"source": item.source,
|
|
"feed_name": item.feed_name,
|
|
"region": item.feed_region,
|
|
"homepage_url": item.homepage_url,
|
|
"published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None,
|
|
"is_focus_match": item.feed_region == active_region,
|
|
}
|
|
|
|
|
|
def _build_payload(
|
|
*,
|
|
lat: float | None,
|
|
lon: float | None,
|
|
active_region: str,
|
|
items: list[ParsedNewsItem],
|
|
sources: list[NewsFeedSource],
|
|
errors: list[str],
|
|
stale: bool,
|
|
generated_at: datetime | None = None,
|
|
) -> dict[str, Any]:
|
|
profile = get_region_profile(active_region)
|
|
timestamp = generated_at or datetime.now(UTC)
|
|
return {
|
|
"generated_at": timestamp.isoformat().replace("+00:00", "Z"),
|
|
"focus": {
|
|
"lat": lat,
|
|
"lon": lon,
|
|
"region": active_region,
|
|
"label": profile.label,
|
|
"accent": profile.accent,
|
|
},
|
|
"sources": _serialize_sources(sources),
|
|
"items": [_serialize_item(item, active_region=active_region) for item in items],
|
|
"errors": errors,
|
|
"stale": stale,
|
|
}
|
|
|
|
|
|
def _rank_and_trim_items(items: list[ParsedNewsItem], *, active_region: str) -> list[ParsedNewsItem]:
|
|
deduped: dict[str, ParsedNewsItem] = {}
|
|
for item in items:
|
|
key = item.url.strip() or item.title.strip().lower()
|
|
if key not in deduped:
|
|
deduped[key] = item
|
|
|
|
return sorted(
|
|
deduped.values(),
|
|
key=lambda item: (
|
|
item.feed_region != active_region,
|
|
item.published_at is None,
|
|
-(item.published_at.timestamp() if item.published_at else 0),
|
|
item.feed_name,
|
|
),
|
|
)[:MAX_ITEMS_TOTAL]
|
|
|
|
|
|
def _get_cached_region_feed(region: str) -> CachedRegionFeed | None:
|
|
cached = _REGION_CACHE.get(region)
|
|
if not cached:
|
|
return None
|
|
age_seconds = (datetime.now(UTC) - cached.fetched_at).total_seconds()
|
|
if age_seconds > STALE_CACHE_MAX_AGE_SECONDS:
|
|
return None
|
|
return cached
|
|
|
|
|
|
def _store_region_cache(region: str, *, items: list[ParsedNewsItem], sources: list[NewsFeedSource]) -> None:
|
|
_REGION_CACHE[region] = CachedRegionFeed(
|
|
region=region,
|
|
fetched_at=datetime.now(UTC),
|
|
items=list(items),
|
|
sources=list(sources),
|
|
)
|
|
|
|
|
|
async def _fetch_source(
|
|
client: httpx.AsyncClient,
|
|
source: NewsFeedSource,
|
|
) -> tuple[NewsFeedSource, list[ParsedNewsItem], str | None]:
|
|
try:
|
|
response = await client.get(source.feed_url)
|
|
response.raise_for_status()
|
|
return source, _parse_feed_entries(response.text, source), None
|
|
except Exception as exc:
|
|
return source, [], str(exc)
|
|
|
|
|
|
async def get_earth_news_payload(lat: float | None = None, lon: float | None = None) -> dict[str, Any]:
|
|
active_region = determine_focus_region(lat, lon)
|
|
sources = get_sources_for_region(active_region)
|
|
errors: list[str] = []
|
|
|
|
async with httpx.AsyncClient(
|
|
timeout=REQUEST_TIMEOUT,
|
|
follow_redirects=True,
|
|
headers={"User-Agent": USER_AGENT},
|
|
) as client:
|
|
results = await asyncio.gather(*(_fetch_source(client, source) for source in sources))
|
|
|
|
fetched_items: list[ParsedNewsItem] = []
|
|
for source, items, error in results:
|
|
if error:
|
|
errors.append(f"{source.name}: {error}")
|
|
continue
|
|
fetched_items.extend(items)
|
|
|
|
ranked_items = _rank_and_trim_items(fetched_items, active_region=active_region)
|
|
if ranked_items:
|
|
_store_region_cache(active_region, items=ranked_items, sources=sources)
|
|
return _build_payload(
|
|
lat=lat,
|
|
lon=lon,
|
|
active_region=active_region,
|
|
items=ranked_items,
|
|
sources=sources,
|
|
errors=errors,
|
|
stale=False,
|
|
)
|
|
|
|
cached = _get_cached_region_feed(active_region)
|
|
if cached:
|
|
return _build_payload(
|
|
lat=lat,
|
|
lon=lon,
|
|
active_region=active_region,
|
|
items=cached.items,
|
|
sources=cached.sources,
|
|
errors=errors,
|
|
stale=True,
|
|
generated_at=cached.fetched_at,
|
|
)
|
|
|
|
return _build_payload(
|
|
lat=lat,
|
|
lon=lon,
|
|
active_region=active_region,
|
|
items=[],
|
|
sources=sources,
|
|
errors=errors,
|
|
stale=False,
|
|
)
|