from __future__ import annotations import asyncio from dataclasses import dataclass, field from datetime import UTC, datetime, timedelta from email.utils import parsedate_to_datetime import hashlib import html import json import math import re from time import perf_counter from typing import Any from urllib.parse import quote import xml.etree.ElementTree as ET import httpx from bs4 import BeautifulSoup from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.core.countries import COUNTRY_VARIANTS_MAP, get_country_centroid, normalize_country from app.core.enums import ( BreakingLevel, BreakingScope, BreakingSource, NewsEnrichmentStatus, NewsImportanceLevel, NewsMarketImpact, NewsSourceType, NewsTaggingSource, ) from app.models.system_setting import SystemSetting from app.ai_tasks.prompts import EffectiveAIPrompt, get_effective_prompt from app.schemas.ai import SituationalAnalysisRequest from app.services.ai_client import AIProviderClient from app.services.earth_news_classification import ( apply_news_classification as _apply_news_classification, breaking_sort_rank as _breaking_sort_rank, highest_breaking_level as _highest_breaking_level, normalize_breaking_level as _normalize_breaking_level_enum, normalize_breaking_scope as _normalize_breaking_scope_enum, ) from app.services.location.resolvers.nominatim import build_default_nominatim_geocoder USER_AGENT = "PlanetEarthNewsBoard/1.0 (+https://planet.local)" REQUEST_TIMEOUT = 12.0 MAX_ITEMS_PER_SOURCE = 6 MAX_ITEMS_TOTAL = 12 STALE_CACHE_MAX_AGE_SECONDS = 60 * 45 RSS_SUPPLEMENT_MAX_AGE_SECONDS = STALE_CACHE_MAX_AGE_SECONDS MAX_TARGET_INFERENCE_CONCURRENCY = 3 TARGET_INFERENCE_TIMEOUT_SECONDS = 6.0 DEFAULT_NEWS_LOCALE = "zh-CN" SUPPORTED_NEWS_LOCALES = frozenset({"zh-CN", "en-US"}) NEWS_ENRICH_PROMPT_KEY = "earth.news.enrich" EARTH_NEWS_SOURCES_CATEGORY = "earth_news_sources" DEFAULT_NEWS_HEALTH_POLICY = { "timeout_seconds": REQUEST_TIMEOUT, "failure_threshold": 3, "cooldown_minutes": 30, "fetch_interval_minutes": 45, "circuit_breaker": True, } @dataclass(frozen=True) class RegionProfile: key: str label: str query: str accent: str @dataclass(frozen=True) class RegionAnchor: region: str label: str latitude: float longitude: float @dataclass(frozen=True) class NewsFeedEndpoint: id: str name: str url: str type: str = NewsSourceType.RSS.value region: str = "" enabled: bool = True default_category: str = "other" tags: tuple[str, ...] = () priority: int = 100 @dataclass(frozen=True) class NewsFeedSource: id: str name: str region: str feed_url: str homepage_url: str feed_directory_url: str = "" source_type: str = NewsSourceType.RSS.value feed_urls: tuple[str, ...] = () feeds: tuple[NewsFeedEndpoint, ...] = () priority: int = 100 enabled: bool = True source_tags: tuple[str, ...] = () default_category: str = "other" importance_weight: int = 0 health_policy: dict[str, Any] = field(default_factory=dict) @dataclass(frozen=True) class NewsTargetLocation: latitude: float longitude: float label: str source: str confidence: float | None = None country: str | None = None city: str | None = None @dataclass class ParsedNewsItem: id: str title: str summary: str url: str source: str feed_name: str feed_region: str homepage_url: str published_at: datetime | None content_language: str = "en" localizations: dict[str, dict[str, str]] = field(default_factory=dict) enrichment_status: str = NewsEnrichmentStatus.PENDING.value enrichment_error: str | None = None enriched_at: datetime | None = None target_location: NewsTargetLocation | None = None target_resolution_stage: str = "unresolved" target_ai_attempted: bool = False target_ai_status: str = "not_attempted" target_ai_error: str | None = None target_debug_note: str | None = None location_patch: dict[str, Any] | None = None source_tags: list[str] = field(default_factory=list) feed_id: str = "" feed_type: str = NewsSourceType.RSS.value feed_default_category: str = "other" category: str = "other" item_tags: list[str] = field(default_factory=list) tagging_source: str = NewsTaggingSource.RULES.value tagging_confidence: float = 0.0 importance_score: int = 0 importance_level: str = NewsImportanceLevel.LOW.value importance_reasons: list[str] = field(default_factory=list) market_impact: str = NewsMarketImpact.NONE.value breaking_level: str = BreakingLevel.NONE.value breaking_scope: str = BreakingScope.REGIONAL.value breaking_reasons: list[str] = field(default_factory=list) breaking_source: str = BreakingSource.RULES.value breaking_confidence: float = 0.0 breaking_expires_at: datetime | None = None @dataclass class CachedRegionFeed: region: str fetched_at: datetime items: list[ParsedNewsItem] sources: list[NewsFeedSource] REGION_PROFILES: dict[str, RegionProfile] = { "americas": RegionProfile( key="americas", label="美洲焦点", query='Americas geopolitics OR Latin America OR "United States" OR Canada', accent="#79d3ff", ), "europe": RegionProfile( key="europe", label="欧洲焦点", query='Europe geopolitics OR EU OR NATO OR "Eastern Europe"', accent="#8fd4ff", ), "middle-east-africa": RegionProfile( key="middle-east-africa", label="中东与非洲焦点", query='"Middle East" OR Africa geopolitics OR Red Sea OR Gulf', accent="#ffb56a", ), "asia-pacific": RegionProfile( key="asia-pacific", label="亚太焦点", query='"Asia Pacific" OR Indo-Pacific OR China OR Japan OR Korea OR ASEAN', accent="#78f2cf", ), "global": RegionProfile( key="global", label="全球焦点", query='"world news" OR geopolitics OR "global affairs"', accent="#d6e6ff", ), } REGION_ANCHORS: dict[str, RegionAnchor] = { "americas": RegionAnchor( region="americas", label="美洲", latitude=37.0902, longitude=-95.7129, ), "europe": RegionAnchor( region="europe", label="欧洲", latitude=50.1109, longitude=8.6821, ), "middle-east-africa": RegionAnchor( region="middle-east-africa", label="中东与非洲", latitude=25.2048, longitude=55.2708, ), "asia-pacific": RegionAnchor( region="asia-pacific", label="亚太", latitude=1.3521, longitude=103.8198, ), "global": RegionAnchor( region="global", label="全球", latitude=20.0, longitude=0.0, ), } def _google_news_feed(query: str, *, hl: str, gl: str, ceid: str) -> str: return ( "https://news.google.com/rss/search?q=" + quote(query, safe="") + f"&hl={hl}&gl={gl}&ceid={ceid}" ) NEWS_FEED_SOURCES: tuple[NewsFeedSource, ...] = ( NewsFeedSource( id="bbc-world", name="BBC World", region="global", feed_url="https://feeds.bbci.co.uk/news/world/rss.xml", homepage_url="https://www.bbc.com/news/world", feeds=( NewsFeedEndpoint(id="world", name="World", url="https://feeds.bbci.co.uk/news/world/rss.xml", default_category="politics", priority=1), ), source_type="rss", priority=10, source_tags=("official_media", "business_news", "global"), default_category="politics", importance_weight=12, health_policy=DEFAULT_NEWS_HEALTH_POLICY, ), NewsFeedSource( id="dw-top", name="DW Top Stories", region="europe", feed_url="https://rss.dw.com/rdf/rss-en-top", homepage_url="https://www.dw.com/en/top-stories/s-9097", feeds=( NewsFeedEndpoint(id="top", name="Top Stories", url="https://rss.dw.com/rdf/rss-en-top", default_category="politics", priority=1), ), source_type="rss", priority=20, source_tags=("official_media", "business_news", "global", "europe"), default_category="politics", importance_weight=12, health_policy=DEFAULT_NEWS_HEALTH_POLICY, ), NewsFeedSource( id="cnbc-business", name="CNBC Business", region="global", feed_url="https://www.cnbc.com/id/10001147/device/rss/rss.html", homepage_url="https://www.cnbc.com/business/", feeds=( NewsFeedEndpoint(id="business", name="Business", url="https://www.cnbc.com/id/10001147/device/rss/rss.html", default_category="business", priority=1), ), source_type="rss", priority=21, source_tags=("business_news", "global", "us"), default_category="business", importance_weight=16, health_policy=DEFAULT_NEWS_HEALTH_POLICY, ), NewsFeedSource( id="bbc-business", name="BBC Business", region="global", feed_url="https://feeds.bbci.co.uk/news/business/rss.xml", homepage_url="https://www.bbc.com/news/business", feeds=( NewsFeedEndpoint(id="business", name="Business", url="https://feeds.bbci.co.uk/news/business/rss.xml", default_category="business", priority=1), ), source_type="rss", priority=22, source_tags=("official_media", "business_news", "global"), default_category="business", importance_weight=14, health_policy=DEFAULT_NEWS_HEALTH_POLICY, ), NewsFeedSource( id="guardian-business", name="The Guardian Business", region="europe", feed_url="https://www.theguardian.com/uk/business/rss", homepage_url="https://www.theguardian.com/uk/business", feeds=( NewsFeedEndpoint(id="business", name="Business", url="https://www.theguardian.com/uk/business/rss", default_category="business", priority=1), ), source_type="rss", priority=24, source_tags=("business_news", "global", "europe"), default_category="business", importance_weight=12, health_policy=DEFAULT_NEWS_HEALTH_POLICY, ), NewsFeedSource( id="npr-business", name="NPR Business", region="americas", feed_url="https://feeds.npr.org/1006/rss.xml", homepage_url="https://www.npr.org/sections/business/", feeds=( NewsFeedEndpoint(id="business", name="Business", url="https://feeds.npr.org/1006/rss.xml", default_category="business", priority=1), ), source_type="rss", priority=25, source_tags=("business_news", "global", "us"), default_category="business", importance_weight=12, health_policy=DEFAULT_NEWS_HEALTH_POLICY, ), NewsFeedSource( id="marketwatch-top", name="MarketWatch Top Stories", region="americas", feed_url="https://feeds.content.dowjones.io/public/rss/mw_topstories", homepage_url="https://www.marketwatch.com/", feeds=( NewsFeedEndpoint(id="top", name="Top Stories", url="https://feeds.content.dowjones.io/public/rss/mw_topstories", default_category="finance", priority=1), ), source_type="rss", priority=26, source_tags=("business_news", "finance", "global", "us"), default_category="finance", importance_weight=12, health_policy=DEFAULT_NEWS_HEALTH_POLICY, ), NewsFeedSource( id="techcrunch", name="TechCrunch", region="global", feed_url="https://techcrunch.com/feed/", homepage_url="https://techcrunch.com/", feeds=( NewsFeedEndpoint(id="main", name="Main Feed", url="https://techcrunch.com/feed/", default_category="technology", priority=1), ), source_type="rss", priority=32, source_tags=("business_news", "ecommerce", "global"), default_category="technology", importance_weight=8, health_policy=DEFAULT_NEWS_HEALTH_POLICY, ), NewsFeedSource( id="retaildive", name="Retail Dive", region="global", feed_url="https://www.retaildive.com/feeds/news/", homepage_url="https://www.retaildive.com/", feeds=( NewsFeedEndpoint(id="news", name="News", url="https://www.retaildive.com/feeds/news/", default_category="business", priority=1), ), source_type="rss", priority=34, source_tags=("business_news", "retail", "ecommerce", "global"), default_category="business", importance_weight=10, health_policy=DEFAULT_NEWS_HEALTH_POLICY, ), NewsFeedSource( id="prnewswire-retail", name="PR Newswire Consumer Products & Retail", region="global", feed_url="https://www.prnewswire.com/rss/consumer-products-retail-latest-news/consumer-products-retail-latest-news-list.rss", homepage_url="https://www.prnewswire.com/news-releases/consumer-products-retail-latest-news/", feeds=( NewsFeedEndpoint(id="consumer-retail", name="Consumer Products & Retail", url="https://www.prnewswire.com/rss/consumer-products-retail-latest-news/consumer-products-retail-latest-news-list.rss", default_category="business", priority=1), ), source_type="rss", priority=48, source_tags=("press_release", "retail", "ecommerce", "global"), default_category="business", importance_weight=-4, health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "failure_threshold": 2}, ), NewsFeedSource( id="36kr", name="36氪", region="asia-pacific", feed_url="https://36kr.com/feed-article", homepage_url="https://www.36kr.com/", feed_directory_url="https://www.36kr.com/rss-center", source_type="rss", feeds=( NewsFeedEndpoint(id="feed", name="综合资讯", url="https://36kr.com/feed", default_category="business", priority=1), NewsFeedEndpoint(id="article", name="文章资讯", url="https://36kr.com/feed-article", default_category="business", priority=2), NewsFeedEndpoint(id="newsflash", name="最新快讯", url="https://36kr.com/feed-newsflash", default_category="business", priority=3), NewsFeedEndpoint(id="moment", name="动态内容", url="https://36kr.com/feed-moment", default_category="business", priority=4), ), priority=35, source_tags=("business_news", "ecommerce", "china"), default_category="business", importance_weight=12, health_policy=DEFAULT_NEWS_HEALTH_POLICY, ), NewsFeedSource( id="businesswire-ecommerce", name="BusinessWire Electronic Commerce", region="global", feed_url="https://www.businesswire.com/newsroom/industry/technology/ecommerce", homepage_url="https://www.businesswire.com/newsroom/industry/technology/ecommerce", source_type="reference", priority=62, enabled=False, source_tags=("press_release", "ecommerce", "global", "low_stability"), default_category="ecommerce", importance_weight=-6, health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "failure_threshold": 2}, ), NewsFeedSource( id="mckinsey-retail", name="McKinsey Retail Insights", region="global", feed_url="https://www.mckinsey.com/industries/retail/our-insights", homepage_url="https://www.mckinsey.com/industries/retail/our-insights", source_type="reference", priority=64, enabled=False, source_tags=("industry_insight", "retail", "global"), default_category="business", importance_weight=18, health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 720}, ), NewsFeedSource( id="deloitte-retail", name="Deloitte Retail", region="global", feed_url="https://www.deloitte.com/us/en/Industries/retail/about.html", homepage_url="https://www.deloitte.com/us/en/Industries/retail/about.html", source_type="reference", priority=66, enabled=False, source_tags=("industry_insight", "retail", "global", "us"), default_category="business", importance_weight=16, health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 720}, ), NewsFeedSource( id="us-census-ecommerce", name="US Census Retail / Quarterly E-Commerce", region="americas", feed_url="https://www.census.gov/retail/index.html#ecommerce", homepage_url="https://www.census.gov/retail/index.html#ecommerce", source_type="reference", priority=18, enabled=False, source_tags=("official_data", "ecommerce", "retail", "us"), default_category="ecommerce", importance_weight=34, health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 1440}, ), NewsFeedSource( id="mofcom-data", name="商务数据中心", region="asia-pacific", feed_url="https://data.mofcom.gov.cn/index.shtml", homepage_url="https://data.mofcom.gov.cn/index.shtml", source_type="reference", priority=18, enabled=False, source_tags=("official_data", "business_news", "china"), default_category="business", importance_weight=34, health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 1440}, ), NewsFeedSource( id="mofcom-ecommerce", name="商务部电商动态", region="asia-pacific", feed_url="https://www.mofcom.gov.cn/", homepage_url="https://www.mofcom.gov.cn/", source_type="reference", priority=19, enabled=False, source_tags=("official_data", "ecommerce", "china"), default_category="ecommerce", importance_weight=34, health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 1440}, ), NewsFeedSource( id="stats-china-online-retail", name="国家统计局数据发布", region="asia-pacific", feed_url="https://www.stats.gov.cn/sj/zxfb/rss.xml", homepage_url="https://www.stats.gov.cn/sj/zxfb/", feeds=( NewsFeedEndpoint(id="release", name="数据发布", url="https://www.stats.gov.cn/sj/zxfb/rss.xml", default_category="ecommerce", priority=1), ), source_type="rss", priority=19, source_tags=("official_data", "ecommerce", "retail", "china"), default_category="ecommerce", importance_weight=36, health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 1440}, ), NewsFeedSource( id="china-ecommerce-logistics-index", name="电商物流指数", region="asia-pacific", feed_url="https://www.gov.cn/", homepage_url="https://www.gov.cn/", source_type="reference", priority=20, enabled=False, source_tags=("official_data", "ecommerce", "retail", "china"), default_category="ecommerce", importance_weight=36, health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 1440}, ), NewsFeedSource( id="ebrun", name="亿邦动力", region="asia-pacific", feed_url="https://www.ebrun.com/rss/news_b2c.xml", homepage_url="https://www.ebrun.com/", feed_directory_url="https://www.ebrun.com/rss/", source_type="rss", feeds=( NewsFeedEndpoint(id="b2c", name="B2C", url="https://www.ebrun.com/rss/news_b2c.xml", default_category="ecommerce", priority=1), NewsFeedEndpoint(id="b2b", name="B2B", url="https://www.ebrun.com/rss/news_b2b.xml", default_category="ecommerce", priority=2), NewsFeedEndpoint(id="retail", name="零售", url="https://www.ebrun.com/rss/news_retail.xml", default_category="ecommerce", priority=3), NewsFeedEndpoint(id="o2o", name="O2O", url="https://www.ebrun.com/rss/news_o2o.xml", default_category="ecommerce", priority=4), NewsFeedEndpoint(id="service", name="服务", url="https://www.ebrun.com/rss/news_service.xml", default_category="ecommerce", priority=5), NewsFeedEndpoint(id="data", name="数据", url="https://www.ebrun.com/rss/news_data.xml", default_category="ecommerce", priority=6), NewsFeedEndpoint(id="policy", name="政策", url="https://www.ebrun.com/rss/news_policy.xml", default_category="ecommerce", priority=7), ), priority=38, source_tags=("business_news", "ecommerce", "retail", "china"), default_category="ecommerce", importance_weight=14, health_policy=DEFAULT_NEWS_HEALTH_POLICY, ), NewsFeedSource( id="google-news", name="Google News", region="global", feed_url=_google_news_feed( REGION_PROFILES["global"].query, hl="en-US", gl="US", ceid="US:en", ), homepage_url="https://news.google.com/", feed_directory_url="https://news.google.com/rss", feeds=( NewsFeedEndpoint(id="world", name="全球", url=_google_news_feed(REGION_PROFILES["global"].query, hl="en-US", gl="US", ceid="US:en"), type="aggregated", region="global", default_category="politics", priority=1), NewsFeedEndpoint(id="americas", name="美洲", url=_google_news_feed(REGION_PROFILES["americas"].query, hl="en-US", gl="US", ceid="US:en"), type="aggregated", region="americas", default_category="politics", priority=2), NewsFeedEndpoint(id="europe", name="欧洲", url=_google_news_feed(REGION_PROFILES["europe"].query, hl="en-GB", gl="GB", ceid="GB:en"), type="aggregated", region="europe", default_category="politics", priority=3), NewsFeedEndpoint(id="middle-east-africa", name="中东与非洲", url=_google_news_feed(REGION_PROFILES["middle-east-africa"].query, hl="en-US", gl="US", ceid="US:en"), type="aggregated", region="middle-east-africa", default_category="politics", priority=4), NewsFeedEndpoint(id="asia-pacific", name="亚太", url=_google_news_feed(REGION_PROFILES["asia-pacific"].query, hl="en-SG", gl="SG", ceid="SG:en"), type="aggregated", region="asia-pacific", default_category="politics", priority=5), ), source_type="aggregated", priority=90, source_tags=("aggregated", "business_news", "global", "europe", "low_stability"), default_category="politics", importance_weight=-2, health_policy=DEFAULT_NEWS_HEALTH_POLICY, ), ) DEFAULT_NEWS_SOURCE_BY_ID: dict[str, NewsFeedSource] = {source.id: source for source in NEWS_FEED_SOURCES} LEGACY_GOOGLE_NEWS_SOURCE_IDS = { "global-scan", "google-americas", "google-europe", "google-mea", "google-apac", } DEFAULT_SOURCE_TAGS: tuple[dict[str, Any], ...] = ( {"key": "official_media", "label": "官方媒体", "color": "#2563eb", "enabled": True, "sort_order": 10}, {"key": "official_data", "label": "官方数据", "color": "#059669", "enabled": True, "sort_order": 20}, {"key": "business_news", "label": "商业新闻", "color": "#0f766e", "enabled": True, "sort_order": 30}, {"key": "ecommerce", "label": "电商", "color": "#7c3aed", "enabled": True, "sort_order": 40}, {"key": "finance", "label": "金融", "color": "#1d4ed8", "enabled": True, "sort_order": 45}, {"key": "retail", "label": "零售", "color": "#ea580c", "enabled": True, "sort_order": 50}, {"key": "logistics", "label": "物流", "color": "#16a34a", "enabled": True, "sort_order": 55}, {"key": "industry_insight", "label": "行业洞察", "color": "#0891b2", "enabled": True, "sort_order": 60}, {"key": "press_release", "label": "企业公告", "color": "#64748b", "enabled": True, "sort_order": 70}, {"key": "china", "label": "中国", "color": "#dc2626", "enabled": True, "sort_order": 80}, {"key": "global", "label": "全球", "color": "#475569", "enabled": True, "sort_order": 90}, {"key": "us", "label": "美国", "color": "#1d4ed8", "enabled": True, "sort_order": 100}, {"key": "europe", "label": "欧洲", "color": "#0284c7", "enabled": True, "sort_order": 110}, {"key": "aggregated", "label": "聚合源", "color": "#9333ea", "enabled": True, "sort_order": 120}, {"key": "low_stability", "label": "低稳定性", "color": "#f59e0b", "enabled": True, "sort_order": 130}, ) DEFAULT_CATEGORIES: tuple[dict[str, Any], ...] = ( {"key": "politics", "label": "政治", "color": "#2563eb", "enabled": True, "sort_order": 10, "keywords": ["election", "government", "minister", "parliament", "sanction", "diplomatic", "policy", "summit", "选举", "政府", "制裁", "外交", "政策"]}, {"key": "business", "label": "商业", "color": "#0f766e", "enabled": True, "sort_order": 20, "keywords": ["market", "company", "earnings", "trade", "supply chain", "merger", "retail", "consumer", "商业", "公司", "贸易", "供应链", "消费", "零售"]}, {"key": "ecommerce", "label": "电商", "color": "#7c3aed", "enabled": True, "sort_order": 30, "keywords": ["e-commerce", "ecommerce", "online retail", "gmv", "marketplace", "shopify", "amazon", "tiktok shop", "网上零售", "电商", "直播电商", "跨境电商", "订单量", "物流指数", "履约"]}, {"key": "finance", "label": "金融", "color": "#0369a1", "enabled": True, "sort_order": 40, "keywords": ["stock", "bond", "inflation", "central bank", "rate cut", "rate hike", "bank", "金融", "股市", "通胀", "央行", "利率"]}, {"key": "sports", "label": "体育", "color": "#16a34a", "enabled": True, "sort_order": 50, "keywords": ["football", "basketball", "tennis", "match", "league", "world cup", "olympics", "体育", "足球", "篮球", "世界杯", "奥运"]}, {"key": "technology", "label": "科技", "color": "#0891b2", "enabled": True, "sort_order": 60, "keywords": ["ai", "semiconductor", "chip", "cyber", "satellite", "software", "科技", "人工智能", "半导体", "芯片", "网络安全"]}, {"key": "military", "label": "军事", "color": "#4b5563", "enabled": True, "sort_order": 70, "keywords": ["military", "missile", "airstrike", "defense", "warship", "军事", "导弹", "空袭", "防务", "军舰"]}, {"key": "disaster", "label": "灾害", "color": "#dc2626", "enabled": True, "sort_order": 80, "keywords": ["earthquake", "flood", "wildfire", "typhoon", "hurricane", "灾害", "地震", "洪水", "山火", "台风"]}, {"key": "energy", "label": "能源", "color": "#ca8a04", "enabled": True, "sort_order": 90, "keywords": ["oil", "gas", "opec", "energy", "power grid", "能源", "石油", "天然气", "电网"]}, {"key": "society", "label": "社会", "color": "#64748b", "enabled": True, "sort_order": 100, "keywords": ["health", "education", "crime", "migration", "社会", "医疗", "教育", "犯罪", "移民"]}, {"key": "culture", "label": "文化", "color": "#db2777", "enabled": True, "sort_order": 110, "keywords": ["film", "music", "art", "culture", "文化", "电影", "音乐", "艺术"]}, {"key": "other", "label": "其他", "color": "#64748b", "enabled": True, "sort_order": 999, "keywords": []}, ) ALLOWED_NEWS_CATEGORY_KEYS = tuple(item["key"] for item in DEFAULT_CATEGORIES) DEFAULT_ITEM_TAG_RULES: tuple[dict[str, Any], ...] = ( {"key": "cross_border_ecommerce", "label": "跨境电商", "category": "ecommerce", "keywords": ["cross-border e-commerce", "cross border ecommerce", "跨境电商"]}, {"key": "live_commerce", "label": "直播电商", "category": "ecommerce", "keywords": ["live commerce", "livestream shopping", "直播电商", "直播带货"]}, {"key": "retail_data", "label": "零售数据", "category": "ecommerce", "keywords": ["online retail sales", "retail sales", "网上零售额", "社零", "社会消费品零售总额"]}, {"key": "logistics_fulfillment", "label": "物流履约", "category": "ecommerce", "keywords": ["logistics", "fulfillment", "delivery", "物流指数", "履约", "配送"]}, {"key": "platform_governance", "label": "平台治理", "category": "ecommerce", "keywords": ["platform regulation", "marketplace rules", "平台治理", "平台监管"]}, {"key": "ai", "label": "AI", "category": "technology", "keywords": ["ai", "artificial intelligence", "人工智能", "大模型"]}, {"key": "semiconductor", "label": "半导体", "category": "technology", "keywords": ["semiconductor", "chip", "半导体", "芯片"]}, {"key": "election", "label": "选举", "category": "politics", "keywords": ["election", "vote", "campaign", "选举", "投票"]}, {"key": "oil_price", "label": "油价", "category": "energy", "keywords": ["oil price", "crude", "油价", "原油"]}, {"key": "football", "label": "足球", "category": "sports", "keywords": ["football", "soccer", "premier league", "足球"]}, {"key": "supply_chain", "label": "供应链", "category": "business", "keywords": ["supply chain", "供应链"]}, ) def default_earth_news_sources_payload() -> dict[str, Any]: return { "cache_version": 1, "source_tags": [dict(item) for item in DEFAULT_SOURCE_TAGS], "categories": [dict(item) for item in DEFAULT_CATEGORIES], "item_tag_rules": [dict(item) for item in DEFAULT_ITEM_TAG_RULES], "sources": [serialize_news_source_config(source) for source in NEWS_FEED_SOURCES], "health": {}, } _REGION_CACHE: dict[str, CachedRegionFeed] = {} _news_target_geocode = build_default_nominatim_geocoder(user_agent=USER_AGENT) _CITY_HINTS: tuple[dict[str, str | None], ...] = ( {"name": "Beijing", "country": "中国"}, {"name": "Havana", "country": "古巴"}, {"name": "Kyiv", "country": "乌克兰"}, {"name": "Bangkok", "country": "泰国"}, {"name": "Tehran", "country": "伊朗"}, {"name": "Moscow", "country": "俄罗斯"}, {"name": "Taipei", "country": "中国（台湾）"}, {"name": "Hong Kong", "country": "中国（香港）"}, ) def determine_focus_region(lat: float | None, lon: float | None) -> str: if lat is None or lon is None: return "global" if -170 <= lon <= -30: return "americas" if -30 < lon <= 45: return "europe" if lat >= 30 else "middle-east-africa" if 45 < lon <= 150: return "middle-east-africa" if lat < 10 else "asia-pacific" return "asia-pacific" def get_region_profile(region: str) -> RegionProfile: return REGION_PROFILES.get(region, REGION_PROFILES["global"]) def get_region_anchor(region: str) -> RegionAnchor: return REGION_ANCHORS.get(region, REGION_ANCHORS["global"]) def _coerce_str(value: Any) -> str | None: if value is None: return None if not isinstance(value, str): value = str(value) cleaned = re.sub(r"\s+", " ", value).strip() return cleaned or None def _contains_location_alias(text: str, alias: str) -> bool: normalized_alias = _coerce_str(alias) if not normalized_alias: return False if re.search(r"[A-Za-z]", normalized_alias): pattern = r"(? tuple[str, ...]: searchable: list[str] = [] seen: set[str] = set() for variant in (canonical, *variants): normalized = _coerce_str(variant) if not normalized: continue if re.fullmatch(r"[A-Z]{2,3}", normalized): continue if len(normalized) <= 2: continue key = normalized.casefold() if key in seen: continue seen.add(key) searchable.append(normalized) return tuple(searchable) def _coerce_float(value: Any) -> float | None: try: parsed = float(value) except (TypeError, ValueError): return None if not math.isfinite(parsed): return None return parsed def _first_json_object(text: str) -> dict[str, Any] | None: if not text: return None decoder = json.JSONDecoder() for index, char in enumerate(text): if char != "{": continue try: payload, _ = decoder.raw_decode(text[index:]) except ValueError: continue if isinstance(payload, dict): return payload return None def _normalize_localizations(value: Any) -> dict[str, dict[str, str]]: if not isinstance(value, dict): return {} normalized: dict[str, dict[str, str]] = {} for locale, payload in value.items(): locale_key = _coerce_str(locale) if not locale_key or not isinstance(payload, dict): continue title = _coerce_str(payload.get("title")) summary = _coerce_str(payload.get("summary")) entry: dict[str, str] = {} if title: entry["title"] = title if summary: entry["summary"] = summary if entry: normalized[locale_key] = entry return normalized def _is_chinese_language(language: str | None) -> bool: return str(language or "").lower() in {"zh", "zh-cn", "zh-hans", "chinese"} def _is_english_language(language: str | None) -> bool: return str(language or "").lower() in {"en", "en-us", "english"} def _contains_cjk_text(value: str) -> bool: return bool(re.search(r"[\u3400-\u9fff]", value or "")) def _detect_content_language( *, title: str, summary: str, source: NewsFeedSource, feed: NewsFeedEndpoint | None = None, ) -> str: text = f"{title}\n{summary}" if _contains_cjk_text(text): return "zh-CN" identity = {source.id.lower(), *(tag.lower() for tag in source.source_tags)} if feed is not None: identity.add(feed.id.lower()) identity.update(tag.lower() for tag in feed.tags) if identity & {"china", "chinese", "36kr", "ebrun", "stats-china-online-retail"}: return "zh-CN" return "en" def _source_language_localizations( *, title: str, summary: str, content_language: str, ) -> dict[str, dict[str, str]]: if _is_chinese_language(content_language): return {"zh-CN": {"title": title, "summary": summary or title}} return {} def _target_localization_locale(item: ParsedNewsItem) -> str: return "en-US" if _is_chinese_language(item.content_language) else DEFAULT_NEWS_LOCALE def _target_locale_schema(locale: str) -> dict[str, dict[str, str]]: if locale == "en-US": return { "en-US": { "title": "faithful English title", "summary": "one-sentence newswire-style English lead summary", } } return { "zh-CN": { "title": "faithful Simplified Chinese title", "summary": "one-sentence newswire-style Simplified Chinese lead summary", } } def _get_locale_text( item: ParsedNewsItem, key: str, *, locale: str = DEFAULT_NEWS_LOCALE, ) -> str: localized = item.localizations.get(locale) if isinstance(localized, dict): value = _coerce_str(localized.get(key)) if value: return value if locale == "zh-CN" and _is_chinese_language(item.content_language): return item.title if key == "title" else item.summary if locale == "en-US" and _is_english_language(item.content_language): return item.title if key == "title" else item.summary fallback = item.localizations.get(DEFAULT_NEWS_LOCALE) if isinstance(fallback, dict): value = _coerce_str(fallback.get(key)) if value: return value if locale == "en-US" and _is_chinese_language(item.content_language): return item.title if key == "title" else item.summary return "" def _has_localization(item: ParsedNewsItem, locale: str) -> bool: localized = item.localizations.get(locale) if not isinstance(localized, dict): return False return bool(_coerce_str(localized.get("title")) and _coerce_str(localized.get("summary"))) def _has_default_localization(item: ParsedNewsItem) -> bool: return _has_localization(item, DEFAULT_NEWS_LOCALE) def _has_required_localization(item: ParsedNewsItem) -> bool: return _has_localization(item, _target_localization_locale(item)) def apply_enrichment_patch_to_item( item: ParsedNewsItem, patch: dict[str, Any], ) -> ParsedNewsItem: item.location_patch = patch if "content_language" in patch: item.content_language = _coerce_str(patch.get("content_language")) or item.content_language if "localizations" in patch: item.localizations = _normalize_localizations(patch.get("localizations")) if "enrichment_status" in patch: item.enrichment_status = _coerce_str(patch.get("enrichment_status")) or item.enrichment_status if "enrichment_error" in patch: item.enrichment_error = _coerce_str(patch.get("enrichment_error")) if "enriched_at" in patch: item.enriched_at = _parse_datetime(_coerce_str(patch.get("enriched_at"))) return item async def _geocode_target_location(query: str) -> dict[str, Any] | None: return await asyncio.to_thread(_news_target_geocode, query) async def _build_target_location_from_payload( payload: dict[str, Any], ) -> NewsTargetLocation | None: country = normalize_country(payload.get("country")) city = _coerce_str(payload.get("city")) matched_location_name = _coerce_str(payload.get("matched_location_name")) confidence = _coerce_float(payload.get("confidence")) if confidence is not None: confidence = max(0.0, min(confidence, 1.0)) latitude = _coerce_float(payload.get("latitude")) longitude = _coerce_float(payload.get("longitude")) if latitude not in (None, 0.0) and longitude not in (None, 0.0): label = matched_location_name or ", ".join(part for part in (city, country) if part) or "关联位置" return NewsTargetLocation( latitude=latitude, longitude=longitude, label=label, source="ai_inferred_target", confidence=confidence, country=country, city=city, ) geocode_queries: list[str] = [] for value in ( ", ".join(part for part in (city, country) if part), matched_location_name, city, country, ): normalized = _coerce_str(value) if normalized and normalized not in geocode_queries: geocode_queries.append(normalized) for query in geocode_queries: try: result = await _geocode_target_location(query) except Exception: continue if not isinstance(result, dict): continue latitude = _coerce_float(result.get("lat")) longitude = _coerce_float(result.get("lon")) if latitude in (None, 0.0) or longitude in (None, 0.0): continue label = ( _coerce_str(result.get("display_name")) or matched_location_name or ", ".join(part for part in (city, country) if part) or query ) return NewsTargetLocation( latitude=latitude, longitude=longitude, label=label, source="ai_inferred_target", confidence=confidence, country=country, city=city, ) centroid = get_country_centroid(country) if centroid: label = matched_location_name or city or country or "关联位置" return NewsTargetLocation( latitude=centroid["latitude"], longitude=centroid["longitude"], label=label, source="ai_inferred_target", confidence=confidence, country=country, city=city, ) return None async def _extract_target_location_from_text(item: ParsedNewsItem) -> NewsTargetLocation | None: combined_text = " ".join(part for part in (item.title, item.summary) if part).strip() if not combined_text: return None for hint in _CITY_HINTS: city_name = _coerce_str(hint.get("name")) if not city_name or not _contains_location_alias(combined_text, city_name): continue country = normalize_country(hint.get("country")) geocode_query = ", ".join(part for part in (city_name, country) if part) try: result = await _geocode_target_location(geocode_query) except Exception: result = None if isinstance(result, dict): latitude = _coerce_float(result.get("lat")) longitude = _coerce_float(result.get("lon")) if latitude not in (None, 0.0) and longitude not in (None, 0.0): return NewsTargetLocation( latitude=latitude, longitude=longitude, label=_coerce_str(result.get("display_name")) or geocode_query, source="headline_location_hint", confidence=0.78, country=country, city=city_name, ) centroid = get_country_centroid(country) if centroid: return NewsTargetLocation( latitude=centroid["latitude"], longitude=centroid["longitude"], label=geocode_query, source="headline_location_hint", confidence=0.68, country=country, city=city_name, ) for canonical, variants in COUNTRY_VARIANTS_MAP.items(): if not get_country_centroid(canonical): continue searchable_variants = _iter_searchable_country_variants(canonical, variants) if not any(_contains_location_alias(combined_text, variant) for variant in searchable_variants): continue centroid = get_country_centroid(canonical) if not centroid: continue return NewsTargetLocation( latitude=centroid["latitude"], longitude=centroid["longitude"], label=canonical, source="headline_country_hint", confidence=0.62, country=canonical, city=None, ) return None async def _infer_news_target_location( item: ParsedNewsItem, *, provider_client: AIProviderClient | None, prompt: EffectiveAIPrompt | None = None, ) -> NewsTargetLocation | None: target, _localizations = await _infer_news_enrichment( item, provider_client=provider_client, prompt=prompt, ) return target async def _infer_news_enrichment( item: ParsedNewsItem, *, provider_client: AIProviderClient | None, prompt: EffectiveAIPrompt | None = None, ) -> tuple[NewsTargetLocation | None, dict[str, dict[str, str]]]: text_hint = await _extract_target_location_from_text(item) content_error: str | None = None if text_hint is not None and text_hint.city: item.target_resolution_stage = text_hint.source item.target_ai_attempted = False item.target_ai_status = "skipped_text_hint" item.target_ai_error = None item.target_debug_note = f"text hint matched {text_hint.label}" localizations: dict[str, dict[str, str]] = {} if provider_client is None: if text_hint is None or not text_hint.city: item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved" item.target_ai_attempted = False item.target_ai_status = "unavailable" item.target_ai_error = "AI provider is not configured or unavailable for earth-feed." item.target_debug_note = ( f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched" ) item.enrichment_status = "unavailable" item.enrichment_error = "AI provider is not configured or unavailable for earth-feed." return text_hint, localizations if text_hint is None or not text_hint.city: item.target_ai_attempted = True item.target_ai_status = "attempted" item.target_ai_error = None item.target_debug_note = ( f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched" ) item.enrichment_status = "attempted" item.enrichment_error = None prompt = prompt or await get_effective_prompt(None, NEWS_ENRICH_PROMPT_KEY) target_locale = _target_localization_locale(item) target_locale_name = "English" if target_locale == "en-US" else "Simplified Chinese" request = SituationalAnalysisRequest( title=f"Enrich Earth news item with event location and {target_locale} content", objective=prompt.prompt, system_prompt=prompt.system_prompt or None, context={ "news_item": { "title": item.title, "summary": item.summary, "content_language": item.content_language, "source": item.source, "feed_name": item.feed_name, "feed_id": item.feed_id, "feed_type": item.feed_type, "feed_region": item.feed_region, "url": item.url, "published_at": ( item.published_at.isoformat().replace("+00:00", "Z") if item.published_at else None ), }, "required_json_schema": { "location": { "country": "string|null", "city": "string|null", "matched_location_name": "string|null", "latitude": "number|null", "longitude": "number|null", "confidence": "number from 0 to 1", "reasoning_summary": "short string", }, "localizations": _target_locale_schema(target_locale), }, }, constraints=[ "Return only strict JSON. Do not wrap it in markdown.", "For localizations, do not add facts that are absent from the RSS headline, description, source, or date.", f"Write {target_locale} summary as one concise {target_locale_name} newswire-style sentence, like a breaking-news lead.", "If the RSS description is thin, write a conservative one-sentence summary that says only what is supported.", f"Keep {target_locale} summary factual, non-promotional, and avoid colon-heavy keyword labels.", "If the source content is Chinese and target locale is en-US, translate faithfully into English instead of rewriting the story.", "Prefer the event location, not the newsroom or publisher headquarters.", "When a country visit or summit is the clear topic but the city is omitted, use the most likely host city only if it is broadly public knowledge.", "Use null for unknown fields instead of inventing details.", "Calibrate confidence conservatively: 0.75+ only when the city is strongly supported, 0.55-0.74 for country-level or likely city inference, below 0.55 when weak.", ], ) try: response = await provider_client.analyze(request) except Exception as exc: if text_hint is None or not text_hint.city: item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved" item.target_ai_status = "provider_error" item.target_ai_error = str(exc) item.enrichment_status = "provider_error" item.enrichment_error = str(exc) return text_hint, localizations payload = _first_json_object(response.content) if not isinstance(payload, dict): if text_hint is None or not text_hint.city: item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved" item.target_ai_status = "parse_error" item.target_ai_error = "AI response did not contain a parseable JSON object." item.enrichment_status = "parse_error" item.enrichment_error = "AI response did not contain a parseable JSON object." return text_hint, localizations localizations = _normalize_localizations(payload.get("localizations")) if not localizations: content_error = "AI returned no usable localizations." location_payload = payload.get("location") if isinstance(payload.get("location"), dict) else payload if text_hint is not None and text_hint.city: target = text_hint else: target = await _build_target_location_from_payload(location_payload) if target is None: item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved" item.target_ai_status = "no_result" item.target_ai_error = "AI returned no usable target coordinates or geocodeable location." target = text_hint elif target.confidence is not None and target.confidence < 0.45: item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved" item.target_ai_status = "low_confidence" item.target_ai_error = f"AI target confidence too low: {target.confidence:.2f}" target = text_hint else: item.target_resolution_stage = target.source item.target_ai_status = "success" item.target_ai_error = None item.target_debug_note = f"ai inferred {target.label}" item.localizations = {**dict(item.localizations or {}), **localizations} if localizations and item.target_ai_status in {"success", "skipped_text_hint"}: item.enrichment_status = "success" item.enrichment_error = None elif localizations: item.enrichment_status = "content_only" item.enrichment_error = item.target_ai_error else: item.enrichment_status = "location_only" if target is not None else "no_result" item.enrichment_error = content_error or item.target_ai_error item.enriched_at = datetime.now(UTC) if localizations else None return target, localizations async def _enrich_items_with_target_locations( items: list[ParsedNewsItem], *, provider_client: AIProviderClient | None, prompt: EffectiveAIPrompt | None = None, ) -> list[ParsedNewsItem]: if not items: return items semaphore = asyncio.Semaphore(MAX_TARGET_INFERENCE_CONCURRENCY) async def enrich(item: ParsedNewsItem) -> ParsedNewsItem: async with semaphore: target = await _infer_news_target_location( item, provider_client=provider_client, prompt=prompt, ) item.target_location = target return item return list(await asyncio.gather(*(enrich(item) for item in items))) def get_sources_for_region(region: str) -> list[NewsFeedSource]: return sorted( [ source for source in NEWS_FEED_SOURCES if source.enabled and source.source_type in {"rss", "atom", "aggregated"} and _source_matches_active_region(source, region) ], key=lambda source: (source.priority, source.name), ) def _clean_feed_urls(*values: Any) -> tuple[str, ...]: urls: list[str] = [] for value in values: if isinstance(value, str): parts = re.split(r"[\n,]+", value) elif isinstance(value, (list, tuple)): parts = [str(item) for item in value] else: parts = [] for part in parts: url = str(part or "").strip() if url and url not in urls: urls.append(url) return tuple(urls) def _clean_feed_tags(value: Any) -> tuple[str, ...]: if isinstance(value, str): parts = re.split(r"[\n,，]+", value) elif isinstance(value, (list, tuple)): parts = [str(item) for item in value] else: parts = [] tags: list[str] = [] for part in parts: tag = str(part or "").strip() if tag and tag not in tags: tags.append(tag) return tuple(tags) def _slug_feed_id(value: str, fallback: str) -> str: text_value = str(value or "").strip().lower() slug = re.sub(r"[^a-z0-9_-]+", "-", text_value).strip("-") return slug or fallback def _feed_from_config(raw: Any, *, source: NewsFeedSource | None = None, index: int = 0) -> NewsFeedEndpoint | None: if not isinstance(raw, dict): return None url = str(raw.get("url") or raw.get("feed_url") or "").strip() if not url: return None feed_id = _slug_feed_id(str(raw.get("id") or raw.get("key") or raw.get("name") or ""), f"feed-{index + 1}") fallback_type = source.source_type if source else "rss" feed_type = str(raw.get("type") or raw.get("source_type") or fallback_type).strip().lower() or "rss" default_category = str(raw.get("default_category") or (source.default_category if source else "other") or "other").strip() or "other" try: priority = int(raw.get("priority", index + 1)) except (TypeError, ValueError): priority = index + 1 return NewsFeedEndpoint( id=feed_id, name=str(raw.get("name") or raw.get("label") or feed_id).strip() or feed_id, url=url, type=feed_type, region=str(raw.get("region") or (source.region if source else "") or "").strip(), enabled=raw.get("enabled") is not False and feed_type in {"rss", "atom", "aggregated"}, default_category=default_category, tags=_clean_feed_tags(raw.get("tags")), priority=priority, ) def _source_feed_urls(source: NewsFeedSource) -> tuple[str, ...]: return tuple(feed.url for feed in _source_feeds(source)) def _source_feeds(source: NewsFeedSource) -> tuple[NewsFeedEndpoint, ...]: if source.source_type == "reference": return tuple() feeds = tuple(feed for feed in source.feeds if feed.url) if feeds: return feeds urls = _clean_feed_urls(source.feed_urls, source.feed_url) return tuple( NewsFeedEndpoint( id=f"feed-{index + 1}", name=source.name if len(urls) == 1 else f"{source.name} {index + 1}", url=url, type=source.source_type, region=source.region, enabled=source.enabled and source.source_type in {"rss", "atom", "aggregated"}, default_category=source.default_category, priority=index + 1, ) for index, url in enumerate(urls) ) def _feed_matches_active_region(feed: NewsFeedEndpoint, active_region: str | None) -> bool: return ( active_region is None or active_region == "global" or not feed.region or feed.region in {"global", active_region} ) def _source_matches_active_region(source: NewsFeedSource, active_region: str) -> bool: return active_region == "global" or source.region in {"global", active_region} def _expected_feed_keys(sources: list[NewsFeedSource], *, active_region: str) -> set[tuple[str, str]]: return { (source.id, feed.id) for source in sources for feed in _source_feeds(source) if source.enabled and source.source_type in {"rss", "atom", "aggregated"} and feed.enabled and feed.type in {"rss", "atom", "aggregated"} and _feed_matches_active_region(feed, active_region) } def _serialize_feed_endpoint(feed: NewsFeedEndpoint) -> dict[str, Any]: return { "id": feed.id, "name": feed.name, "url": feed.url, "type": feed.type, "region": feed.region, "enabled": feed.enabled and feed.type in {"rss", "atom", "aggregated"}, "default_category": feed.default_category, "tags": list(feed.tags), "priority": feed.priority, } def _should_repair_builtin_source_feeds( source_id: str, *, raw_feeds: list[Any], feed_urls: tuple[str, ...], feed_url: str, homepage_url: str, feed_directory_url: str, ) -> bool: default_source = DEFAULT_NEWS_SOURCE_BY_ID.get(source_id) if not default_source or not default_source.feeds: return False if not raw_feeds: return True current_urls: set[str] = set(feed_urls) if feed_url: current_urls.add(feed_url) for raw_feed in raw_feeds: if isinstance(raw_feed, dict): current_url = str(raw_feed.get("url") or raw_feed.get("feed_url") or "").strip() if current_url: current_urls.add(current_url) non_feed_urls = { url for url in ( homepage_url, feed_directory_url, default_source.homepage_url, default_source.feed_directory_url, ) if url } return bool(current_urls & non_feed_urls) def serialize_news_source_config(source: NewsFeedSource) -> dict[str, Any]: feeds = _source_feeds(source) feed_urls = tuple(feed.url for feed in feeds) return { "id": source.id, "name": source.name, "region": source.region, "feed_url": source.feed_url or (feed_urls[0] if feed_urls else ""), "feed_urls": list(feed_urls), "feeds": [_serialize_feed_endpoint(feed) for feed in feeds], "homepage_url": source.homepage_url, "feed_directory_url": source.feed_directory_url, "source_type": source.source_type, "priority": source.priority, "enabled": source.enabled and source.source_type in {"rss", "atom", "aggregated"}, "source_tags": list(source.source_tags), "default_category": source.default_category, "importance_weight": source.importance_weight, "health_policy": dict(source.health_policy or DEFAULT_NEWS_HEALTH_POLICY), } def _source_from_config(payload: dict[str, Any]) -> NewsFeedSource | None: source_id = str(payload.get("id") or "").strip() name = str(payload.get("name") or "").strip() source_type = str(payload.get("source_type") or payload.get("type") or "rss").strip().lower() or "rss" default_source = DEFAULT_NEWS_SOURCE_BY_ID.get(source_id) homepage_url = str(payload.get("homepage_url") or "").strip() feed_directory_url = str(payload.get("feed_directory_url") or "").strip() raw_feeds = payload.get("feeds") if isinstance(payload.get("feeds"), list) else [] feed_urls = _clean_feed_urls(payload.get("feed_urls"), payload.get("feed_url")) feed_url = str(payload.get("feed_url") or "").strip() or (feed_urls[0] if feed_urls else "") enabled_override: bool | None = None if default_source: homepage_url = homepage_url or default_source.homepage_url feed_directory_url = feed_directory_url or default_source.feed_directory_url if source_type == "reference" and default_source.source_type in {"rss", "atom", "aggregated"}: source_type = default_source.source_type if not raw_feeds: enabled_override = default_source.enabled if _should_repair_builtin_source_feeds( source_id, raw_feeds=raw_feeds, feed_urls=feed_urls, feed_url=feed_url, homepage_url=homepage_url, feed_directory_url=feed_directory_url, ): raw_feeds = [_serialize_feed_endpoint(feed) for feed in default_source.feeds] feed_urls = tuple(feed.url for feed in default_source.feeds) feed_url = default_source.feed_url or (feed_urls[0] if feed_urls else "") if not feed_url and source_type == "reference": feed_url = homepage_url feed_urls = _clean_feed_urls(feed_url) if not source_id or not name or (not feed_url and not raw_feeds): return None try: priority = int(payload.get("priority", default_source.priority if default_source else 100)) except (TypeError, ValueError): priority = default_source.priority if default_source else 100 try: importance_weight = int(payload.get("importance_weight", default_source.importance_weight if default_source else 0)) except (TypeError, ValueError): importance_weight = default_source.importance_weight if default_source else 0 raw_tags = payload.get("source_tags") source_tags = ( tuple(str(tag).strip() for tag in raw_tags if str(tag).strip()) if isinstance(raw_tags, list) else (default_source.source_tags if default_source else ()) ) health_policy = payload.get("health_policy") if isinstance(payload.get("health_policy"), dict) else {} base_source = NewsFeedSource( id=source_id, name=name, region=str(payload.get("region") or "global").strip() or "global", feed_url=feed_url, homepage_url=homepage_url, feed_directory_url=feed_directory_url, source_type=source_type, priority=priority, enabled=(enabled_override if enabled_override is not None else payload.get("enabled") is not False) and source_type in {"rss", "atom", "aggregated"}, source_tags=source_tags, default_category=str(payload.get("default_category") or (default_source.default_category if default_source else "other") or "other").strip() or "other", importance_weight=importance_weight, health_policy={**DEFAULT_NEWS_HEALTH_POLICY, **health_policy}, ) feeds = tuple( feed for index, raw_feed in enumerate(raw_feeds) for feed in [_feed_from_config(raw_feed, source=base_source, index=index)] if feed is not None ) return NewsFeedSource( **{ **base_source.__dict__, "feed_urls": feed_urls or ((feed_url,) if feed_url else tuple()), "feeds": feeds, } ) def _merge_legacy_google_news_sources(sources: list[dict[str, Any]]) -> list[dict[str, Any]]: has_legacy_google = any(str(source.get("id") or "") in LEGACY_GOOGLE_NEWS_SOURCE_IDS for source in sources) if not has_legacy_google: return sources google_default = DEFAULT_NEWS_SOURCE_BY_ID.get("google-news") if google_default is None: return sources merged = [source for source in sources if str(source.get("id") or "") not in LEGACY_GOOGLE_NEWS_SOURCE_IDS] if not any(str(source.get("id") or "") == "google-news" for source in merged): merged.append(serialize_news_source_config(google_default)) return sorted(merged, key=lambda source: (int(source.get("priority") or 100), str(source.get("name") or ""))) def normalize_earth_news_sources_payload(payload: dict[str, Any] | None) -> dict[str, Any]: defaults = default_earth_news_sources_payload() if not isinstance(payload, dict): return defaults normalized = { "cache_version": int(payload.get("cache_version") or defaults["cache_version"]), "source_tags": payload.get("source_tags") if isinstance(payload.get("source_tags"), list) else defaults["source_tags"], "categories": payload.get("categories") if isinstance(payload.get("categories"), list) else defaults["categories"], "item_tag_rules": payload.get("item_tag_rules") if isinstance(payload.get("item_tag_rules"), list) else defaults["item_tag_rules"], "health": payload.get("health") if isinstance(payload.get("health"), dict) else {}, "sources": [], } for source in payload.get("sources") if isinstance(payload.get("sources"), list) else defaults["sources"]: if isinstance(source, dict): coerced = _source_from_config(source) if coerced: normalized["sources"].append(serialize_news_source_config(coerced)) if not normalized["sources"]: normalized["sources"] = defaults["sources"] else: normalized["sources"] = _merge_legacy_google_news_sources(normalized["sources"]) return normalized async def get_earth_news_sources_payload(db: AsyncSession | None) -> dict[str, Any]: if db is None: payload = default_earth_news_sources_payload() payload["is_default"] = True return payload result = await db.execute(select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_SOURCES_CATEGORY)) record = result.scalar_one_or_none() payload = normalize_earth_news_sources_payload(record.payload if record else None) payload["is_default"] = record is None return payload async def save_earth_news_sources_payload(db: AsyncSession, payload: dict[str, Any]) -> dict[str, Any]: normalized = normalize_earth_news_sources_payload(payload) normalized["cache_version"] = int(normalized.get("cache_version") or 1) + 1 result = await db.execute(select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_SOURCES_CATEGORY)) record = result.scalar_one_or_none() if record is None: record = SystemSetting(category=EARTH_NEWS_SOURCES_CATEGORY, payload=normalized) db.add(record) else: record.payload = normalized await db.commit() await db.refresh(record) clear_earth_news_region_cache() return {**normalize_earth_news_sources_payload(record.payload), "is_default": False} async def reset_earth_news_sources_payload(db: AsyncSession) -> dict[str, Any]: result = await db.execute(select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_SOURCES_CATEGORY)) record = result.scalar_one_or_none() if record is not None: await db.delete(record) await db.commit() clear_earth_news_region_cache() payload = default_earth_news_sources_payload() payload["is_default"] = True return payload async def record_earth_news_source_health( db: AsyncSession | None, source_id: str, health: dict[str, Any], ) -> None: if db is None or not source_id or not callable(getattr(db, "execute", None)): return result = await db.execute(select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_SOURCES_CATEGORY)) record = result.scalar_one_or_none() payload = normalize_earth_news_sources_payload(record.payload if record else None) payload_health = payload.get("health") if isinstance(payload.get("health"), dict) else {} payload_health[source_id] = health payload["health"] = payload_health if record is None: record = SystemSetting(category=EARTH_NEWS_SOURCES_CATEGORY, payload=payload) db.add(record) else: record.payload = payload await db.commit() async def record_earth_news_sources_health( db: AsyncSession | None, health_by_source: dict[str, dict[str, Any]], ) -> None: if db is None or not health_by_source or not callable(getattr(db, "execute", None)): return result = await db.execute(select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_SOURCES_CATEGORY)) record = result.scalar_one_or_none() payload = normalize_earth_news_sources_payload(record.payload if record else None) payload_health = payload.get("health") if isinstance(payload.get("health"), dict) else {} payload_health.update(health_by_source) payload["health"] = payload_health if record is None: record = SystemSetting(category=EARTH_NEWS_SOURCES_CATEGORY, payload=payload) db.add(record) else: record.payload = payload await db.commit() async def get_configured_sources_for_region(db: AsyncSession | None, region: str) -> list[NewsFeedSource]: if db is None: return get_sources_for_region(region) payload = await get_earth_news_sources_payload(db) sources = [ source for raw in payload.get("sources", []) if isinstance(raw, dict) for source in [_source_from_config(raw)] if source ] enabled = [ source for source in sources if source.enabled and source.source_type in {"rss", "atom", "aggregated"} and _source_matches_active_region(source, region) ] return sorted(enabled, key=lambda source: (source.priority, source.name)) def _source_health_result( source: NewsFeedSource, *, ok: bool, count: int, error: str | None = None, status: str | None = None, status_code: int | None = None, content_type: str | None = None, latency_ms: int | None = None, final_url: str | None = None, feed: NewsFeedEndpoint | None = None, ) -> dict[str, Any]: return { "source_id": source.id, "feed_id": feed.id if feed else "", "feed_name": feed.name if feed else "", "feed_type": feed.type if feed else source.source_type, "feed_url": feed.url if feed else source.feed_url, "ok": ok, "status": status or ("ok" if ok else "failed"), "count": int(count or 0), "item_count": int(count or 0), "error": error, "status_code": status_code, "content_type": content_type or "", "latency_ms": latency_ms, "final_url": final_url or (feed.url if feed else source.feed_url), "fetched_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), } def _format_source_fetch_error( *, status_code: int | None, content_type: str, body: str, parsed_count: int, ) -> tuple[str | None, str]: if status_code is not None and status_code >= 400: return f"HTTP {status_code}", "http_error" lower_content_type = content_type.lower() body_prefix = body[:500].lower() looks_like_html = "text/html" in lower_content_type or " dict[str, Any]: source = _source_from_config(raw_source) if source is None: return {"ok": False, "count": 0, "error": "新闻源缺少 id、名称或 URL"} if source.source_type not in {"rss", "atom", "aggregated"}: health = _source_health_result( source, ok=False, count=0, error="参考链接仅用于记录官网、报告页或未来采集器线索，不参与 RSS/Atom 抓取。", status="reference", ) await record_earth_news_source_health(db, source.id, health) return { "ok": False, "count": 0, "error": health["error"], "health": health, "source": serialize_news_source_config(source), } async with httpx.AsyncClient( timeout=float(source.health_policy.get("timeout_seconds", REQUEST_TIMEOUT)), follow_redirects=True, headers={"User-Agent": USER_AGENT}, ) as client: fetched_source, items, error, health = await _fetch_source(client, source) del fetched_source await record_earth_news_source_health(db, source.id, health) return { "ok": error is None, "count": len(items), "item_count": len(items), "error": error, "health": health, "source": serialize_news_source_config(source), } def _strip_html(value: str) -> str: if not value: return "" soup = BeautifulSoup(value, "html.parser") return re.sub(r"\s+", " ", soup.get_text(" ", strip=True)).strip() def _truncate(value: str, limit: int = 180) -> str: text = value.strip() if len(text) <= limit: return text return text[: limit - 1].rstrip() + "…" def _normalize_source_name(raw: str, fallback: str) -> str: text = html.unescape((raw or "").strip()) if " - " in text: return text.split(" - ")[-1].strip() or fallback return text or fallback def _parse_datetime(raw: str | None) -> datetime | None: if not raw: return None text = raw.strip() if not text: return None for parser in ( lambda value: parsedate_to_datetime(value), lambda value: datetime.fromisoformat(value.replace("Z", "+00:00")), ): try: parsed = parser(text) if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=UTC) return parsed.astimezone(UTC) except Exception: continue return None def _extract_item_text(element: ET.Element, *names: str) -> str: for name in names: node = element.find(name) if node is None and not name.startswith("{"): node = element.find(f"{{*}}{name}") if node is not None and node.text: return node.text.strip() return "" def _parse_feed_entries( xml_text: str, source: NewsFeedSource, *, feed: NewsFeedEndpoint | None = None, config_payload: dict[str, Any] | None = None, ) -> list[ParsedNewsItem]: root = ET.fromstring(xml_text) items: list[ParsedNewsItem] = [] rss_items = root.findall("./channel/item") or root.findall(".//item") or root.findall(".//{*}item") atom_entries = root.findall("{http://www.w3.org/2005/Atom}entry") or root.findall(".//{*}entry") nodes = rss_items or atom_entries for node in nodes[:MAX_ITEMS_PER_SOURCE]: if node.tag.endswith("entry"): title = _extract_item_text(node, "{http://www.w3.org/2005/Atom}title", "title") summary = _extract_item_text( node, "{http://www.w3.org/2005/Atom}summary", "{http://www.w3.org/2005/Atom}content", "summary", "content", ) link_node = node.find("{http://www.w3.org/2005/Atom}link") or node.find("{*}link") link = link_node.get("href", "").strip() if link_node is not None else "" published = _extract_item_text( node, "{http://www.w3.org/2005/Atom}updated", "{http://www.w3.org/2005/Atom}published", "updated", "published", ) else: title = _extract_item_text(node, "title") summary = _extract_item_text(node, "description", "content", "encoded") link = _extract_item_text(node, "link") published = _extract_item_text(node, "pubDate", "published", "updated") clean_title = html.unescape(title).strip() clean_summary = _truncate(_strip_html(summary), 180) if not clean_title or not link: continue item_source = source.name display_title = clean_title feed_type = feed.type if feed else source.source_type if feed_type == "aggregated" and " - " in clean_title: parts = clean_title.rsplit(" - ", 1) display_title = parts[0].strip() item_source = _normalize_source_name(parts[1], source.name) content_language = _detect_content_language( title=display_title, summary=clean_summary, source=source, feed=feed, ) feed_id_segment = f"{feed.id}:" if feed is not None and len(source.feeds or ()) > 1 else "" item = ParsedNewsItem( id=f"{source.id}:{feed_id_segment}{hashlib.sha1(link.encode('utf-8')).hexdigest()[:12]}", title=display_title, summary=clean_summary, url=link, source=item_source, feed_name=feed.name if feed else source.name, feed_region=(feed.region if feed and feed.region else source.region), homepage_url=source.homepage_url, published_at=_parse_datetime(published), content_language=content_language, localizations=_source_language_localizations( title=display_title, summary=clean_summary, content_language=content_language, ), feed_id=feed.id if feed else "", feed_type=feed_type, feed_default_category=(feed.default_category if feed else source.default_category) or "other", source_tags=list(source.source_tags), ) apply_news_classification(item, source, feed=feed, config_payload=config_payload) items.append(item) return items def _normalize_breaking_level(value: str | None) -> str: return _normalize_breaking_level_enum(value).value def _normalize_breaking_scope(value: str | None) -> str: return _normalize_breaking_scope_enum(value).value def apply_news_classification( item: ParsedNewsItem, source: NewsFeedSource, *, feed: NewsFeedEndpoint | None = None, config_payload: dict[str, Any] | None = None, ) -> ParsedNewsItem: config = normalize_earth_news_sources_payload(config_payload) return _apply_news_classification(item, source, feed=feed, config=config) def _serialize_sources(sources: list[NewsFeedSource]) -> list[dict[str, Any]]: return [ { "id": source.id, "name": source.name, "region": source.region, "feed_url": source.feed_url, "feed_urls": list(_source_feed_urls(source)), "feeds": [_serialize_feed_endpoint(feed) for feed in _source_feeds(source)], "homepage_url": source.homepage_url, "feed_directory_url": source.feed_directory_url, "source_type": source.source_type, "priority": source.priority, "enabled": source.enabled, "source_tags": list(source.source_tags), "default_category": source.default_category, "importance_weight": source.importance_weight, } for source in sources ] def _serialize_anchor(anchor: RegionAnchor) -> dict[str, Any]: return { "region": anchor.region, "label": anchor.label, "latitude": anchor.latitude, "longitude": anchor.longitude, } def _serialize_target(target: NewsTargetLocation | None) -> dict[str, Any] | None: if target is None: return None return { "latitude": target.latitude, "longitude": target.longitude, "label": target.label, "source": target.source, "confidence": target.confidence, "country": target.country, "city": target.city, } def _serialize_enriched_at(value: datetime | None) -> str | None: return value.isoformat().replace("+00:00", "Z") if value else None def _serialize_breaking_expires_at(value: datetime | None) -> str | None: return value.isoformat().replace("+00:00", "Z") if value else None def _content_patch(item: ParsedNewsItem) -> dict[str, Any]: return { "content_language": item.content_language, "localizations": item.localizations, "enrichment_status": item.enrichment_status, "enrichment_error": item.enrichment_error, "enriched_at": _serialize_enriched_at(item.enriched_at), } def _news_meta_patch(item: ParsedNewsItem) -> dict[str, Any]: source_id = item.id.split(":", 1)[0] if ":" in item.id else "" return { "source_id": source_id, "source_tags": list(item.source_tags or []), "feed_id": item.feed_id, "feed_name": item.feed_name, "feed_type": item.feed_type, "feed_default_category": item.feed_default_category, "category": item.category, "item_tags": list(item.item_tags or []), "tagging_source": item.tagging_source, "tagging_confidence": item.tagging_confidence, "importance_score": item.importance_score, "importance_level": item.importance_level, "importance_reasons": list(item.importance_reasons or []), "market_impact": item.market_impact, "breaking_level": _normalize_breaking_level(item.breaking_level), "breaking_scope": _normalize_breaking_scope(item.breaking_scope), "breaking_reasons": list(item.breaking_reasons or []), "breaking_source": item.breaking_source, "breaking_confidence": item.breaking_confidence, "breaking_expires_at": _serialize_breaking_expires_at(item.breaking_expires_at), } def build_anchor_location_patch( item: ParsedNewsItem, *, queued: bool = False, queue_available: bool | None = None, ) -> dict[str, Any]: anchor = get_region_anchor(item.feed_region) if queued: resolution_stage = "queued" ai_status = "queued" debug_note = "queued for async target location inference" else: resolution_stage = item.target_resolution_stage ai_status = item.target_ai_status debug_note = item.target_debug_note content_patch = _content_patch(item) if queued and content_patch["enrichment_status"] == "pending": content_patch["enrichment_status"] = "queued" return { "latitude": anchor.latitude, "longitude": anchor.longitude, "location_label": anchor.label, "location_source": "region_anchor", "verified": False, "location_meta": { "resolution_stage": resolution_stage, "ai_attempted": item.target_ai_attempted, "ai_status": ai_status, "ai_error": item.target_ai_error, "debug_note": debug_note, "queue_available": queue_available, "target": None, "anchor": _serialize_anchor(anchor), "news_meta": _news_meta_patch(item), }, **content_patch, } def build_target_location_patch(item: ParsedNewsItem, target: NewsTargetLocation | None) -> dict[str, Any]: if target is None: return build_anchor_location_patch(item) anchor = get_region_anchor(item.feed_region) return { "latitude": target.latitude, "longitude": target.longitude, "location_label": target.label, "location_source": target.source, "verified": True, "location_meta": { "resolution_stage": item.target_resolution_stage, "ai_attempted": item.target_ai_attempted, "ai_status": item.target_ai_status, "ai_error": item.target_ai_error, "debug_note": item.target_debug_note, "target": _serialize_target(target), "anchor": _serialize_anchor(anchor), "news_meta": _news_meta_patch(item), }, **_content_patch(item), } def build_target_location_job_payload(item: ParsedNewsItem) -> dict[str, Any]: published_at = item.published_at return { "id": item.id, "title": item.title, "summary": item.summary, "content_language": item.content_language, "localizations": item.localizations, "enrichment_status": item.enrichment_status, "enrichment_error": item.enrichment_error, "enriched_at": _serialize_enriched_at(item.enriched_at), "url": item.url, "source": item.source, "feed_name": item.feed_name, "feed_id": item.feed_id, "feed_type": item.feed_type, "feed_default_category": item.feed_default_category, "feed_region": item.feed_region, "homepage_url": item.homepage_url, "published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None, "source_tags": list(item.source_tags or []), "category": item.category, "item_tags": list(item.item_tags or []), "tagging_source": item.tagging_source, "tagging_confidence": item.tagging_confidence, "importance_score": item.importance_score, "importance_level": item.importance_level, "importance_reasons": list(item.importance_reasons or []), "market_impact": item.market_impact, "breaking_level": _normalize_breaking_level(item.breaking_level), "breaking_scope": _normalize_breaking_scope(item.breaking_scope), "breaking_reasons": list(item.breaking_reasons or []), "breaking_source": item.breaking_source, "breaking_confidence": item.breaking_confidence, "breaking_expires_at": _serialize_breaking_expires_at(item.breaking_expires_at), } def parsed_news_item_from_job_payload(payload: dict[str, Any]) -> ParsedNewsItem: return ParsedNewsItem( id=str(payload.get("id") or ""), title=str(payload.get("title") or ""), summary=str(payload.get("summary") or ""), content_language=str(payload.get("content_language") or "en"), localizations=_normalize_localizations(payload.get("localizations")), enrichment_status=str(payload.get("enrichment_status") or "pending"), enrichment_error=_coerce_str(payload.get("enrichment_error")), enriched_at=_parse_datetime(_coerce_str(payload.get("enriched_at"))), url=str(payload.get("url") or ""), source=str(payload.get("source") or ""), feed_name=str(payload.get("feed_name") or ""), feed_id=str(payload.get("feed_id") or ""), feed_type=str(payload.get("feed_type") or "rss"), feed_default_category=str(payload.get("feed_default_category") or payload.get("category") or "other"), feed_region=str(payload.get("feed_region") or "global"), homepage_url=str(payload.get("homepage_url") or ""), published_at=_parse_datetime(_coerce_str(payload.get("published_at"))), source_tags=list(payload.get("source_tags") or []), category=str(payload.get("category") or "other"), item_tags=list(payload.get("item_tags") or []), tagging_source=str(payload.get("tagging_source") or "rules"), tagging_confidence=float(payload.get("tagging_confidence") or 0), importance_score=int(payload.get("importance_score") or 0), importance_level=str(payload.get("importance_level") or "low"), importance_reasons=list(payload.get("importance_reasons") or []), market_impact=str(payload.get("market_impact") or "none"), breaking_level=_normalize_breaking_level(str(payload.get("breaking_level") or "none")), breaking_scope=_normalize_breaking_scope(str(payload.get("breaking_scope") or "regional")), breaking_reasons=list(payload.get("breaking_reasons") or []), breaking_source=str(payload.get("breaking_source") or "rules"), breaking_confidence=float(payload.get("breaking_confidence") or 0), breaking_expires_at=_parse_datetime(_coerce_str(payload.get("breaking_expires_at"))), ) def _serialize_item(item: ParsedNewsItem, *, active_region: str, locale: str = DEFAULT_NEWS_LOCALE) -> dict[str, Any]: published_at = item.published_at location_patch = item.location_patch or build_target_location_patch(item, item.target_location) source_id = item.id.split(":", 1)[0] if ":" in item.id else "" return { "id": item.id, "source_id": source_id, "title": item.title, "summary": item.summary, "content_language": item.content_language, "localizations": item.localizations, "display_title": _get_locale_text(item, "title", locale=locale), "display_summary": _get_locale_text(item, "summary", locale=locale), "url": item.url, "source": item.source, "feed_name": item.feed_name, "feed_id": item.feed_id, "feed_type": item.feed_type, "feed_default_category": item.feed_default_category, "region": item.feed_region, "display_region": get_region_anchor(item.feed_region).label, "homepage_url": item.homepage_url, "published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None, "latitude": location_patch["latitude"], "longitude": location_patch["longitude"], "location_label": location_patch["location_label"], "location_source": location_patch["location_source"], "verified": location_patch["verified"], "location_meta": location_patch["location_meta"], "enrichment_status": item.enrichment_status, "enrichment_error": item.enrichment_error, "enriched_at": _serialize_enriched_at(item.enriched_at), "is_focus_match": item.feed_region == active_region, "source_tags": list(item.source_tags or []), "category": item.category, "item_tags": list(item.item_tags or []), "tagging_source": item.tagging_source, "tagging_confidence": item.tagging_confidence, "importance_score": item.importance_score, "importance_level": item.importance_level, "importance_reasons": list(item.importance_reasons or []), "market_impact": item.market_impact, "breaking_level": _normalize_breaking_level(item.breaking_level), "breaking_scope": _normalize_breaking_scope(item.breaking_scope), "breaking_reasons": list(item.breaking_reasons or []), "breaking_source": item.breaking_source, "breaking_confidence": item.breaking_confidence, "breaking_expires_at": _serialize_breaking_expires_at(item.breaking_expires_at), } def _build_payload( *, lat: float | None, lon: float | None, active_region: str, items: list[ParsedNewsItem], cruise_items: list[ParsedNewsItem] | None = None, sources: list[NewsFeedSource], errors: list[str], stale: bool, categories: set[str] | None = None, source_ids: set[str] | None = None, limit: int = MAX_ITEMS_TOTAL, locale: str = DEFAULT_NEWS_LOCALE, generated_at: datetime | None = None, ) -> dict[str, Any]: profile = get_region_profile(active_region) timestamp = generated_at or datetime.now(UTC) visible_items = list(items) cruise_visible_items = list(cruise_items if cruise_items is not None else items) highest_breaking_level = _highest_breaking_level(visible_items + cruise_visible_items) return { "generated_at": timestamp.isoformat().replace("+00:00", "Z"), "focus": { "lat": lat, "lon": lon, "region": active_region, "label": profile.label, "display_region": get_region_anchor(active_region).label, "accent": profile.accent, }, "sources": _serialize_sources(sources), "filters": { "region": active_region, "categories": sorted(categories or []), "sources": sorted(source_ids or []), "limit": limit, "locale": locale, "has_breaking": highest_breaking_level != "none", "highest_breaking_level": highest_breaking_level, }, "items": [_serialize_item(item, active_region=active_region, locale=locale) for item in visible_items], "cruise_items": [ _serialize_item(item, active_region=active_region, locale=locale) for item in cruise_visible_items ], "errors": errors, "stale": stale, } def _rank_and_trim_items( items: list[ParsedNewsItem], *, active_region: str, limit: int = MAX_ITEMS_TOTAL, ) -> list[ParsedNewsItem]: deduped: dict[str, ParsedNewsItem] = {} for item in items: key = item.url.strip() or item.title.strip().lower() if key not in deduped: deduped[key] = item return sorted( deduped.values(), key=lambda item: ( -_breaking_sort_rank(item), False if active_region == "global" or (_breaking_sort_rank(item) > 0 and _normalize_breaking_scope(item.breaking_scope) == "global") else item.feed_region != active_region, item.published_at is None, -(item.published_at.timestamp() if item.published_at else 0), item.feed_name, ), )[:limit] def _filter_news_items_by_categories( items: list[ParsedNewsItem], categories: set[str] | None, ) -> list[ParsedNewsItem]: if not categories: return items return [item for item in items if (item.category or "other") in categories] def _filter_news_items_by_source_ids( items: list[ParsedNewsItem], source_ids: set[str] | None, ) -> list[ParsedNewsItem]: if not source_ids: return items return [ item for item in items if (item.id.split(":", 1)[0] if ":" in item.id else "") in source_ids ] def _news_item_source_id(item: ParsedNewsItem) -> str: return item.id.split(":", 1)[0] if ":" in item.id else "" def _is_news_item_display_ready(item: ParsedNewsItem, *, locale: str) -> bool: return bool( _get_locale_text(item, "title", locale=locale) and _get_locale_text(item, "summary", locale=locale) ) def _diversify_news_items_for_locale( items: list[ParsedNewsItem], *, active_region: str, limit: int, locale: str, ) -> list[ParsedNewsItem]: ranked = sorted( _rank_and_trim_items(items, active_region=active_region, limit=max(len(items), limit)), key=lambda item: (not _is_news_item_display_ready(item, locale=locale),), ) buckets: dict[str, list[ParsedNewsItem]] = {} order: list[str] = [] for item in ranked: if active_region == "global": key = item.feed_region or "global" else: key = _news_item_source_id(item) or item.source or item.feed_name or item.id if key not in buckets: buckets[key] = [] order.append(key) buckets[key].append(item) diversified: list[ParsedNewsItem] = [] while len(diversified) < limit and order: next_order: list[str] = [] for source_id in order: bucket = buckets.get(source_id) or [] if bucket and len(diversified) < limit: diversified.append(bucket.pop(0)) if bucket: next_order.append(source_id) order = next_order return diversified async def _call_store_list_items(list_fn, db: AsyncSession, **kwargs): try: return await list_fn(db, **kwargs) except TypeError as error: if "source_ids" not in str(error): raise legacy_kwargs = dict(kwargs) legacy_kwargs.pop("source_ids", None) return await list_fn(db, **legacy_kwargs) def _get_cached_region_feed(region: str) -> CachedRegionFeed | None: cached = _REGION_CACHE.get(region) if not cached: return None age_seconds = (datetime.now(UTC) - cached.fetched_at).total_seconds() if age_seconds > STALE_CACHE_MAX_AGE_SECONDS: return None return cached def clear_earth_news_region_cache() -> None: _REGION_CACHE.clear() def _store_region_cache(region: str, *, items: list[ParsedNewsItem], sources: list[NewsFeedSource]) -> None: _REGION_CACHE[region] = CachedRegionFeed( region=region, fetched_at=datetime.now(UTC), items=list(items), sources=list(sources), ) async def _apply_cached_locations_and_enqueue(items: list[ParsedNewsItem]) -> list[ParsedNewsItem]: if not items: return items from app.services.earth_news_queue import ( enqueue_target_location_job, get_cached_target_location_patch, ) async def enqueue_item(item: ParsedNewsItem, *, force: bool = False) -> bool: return await enqueue_target_location_job(build_target_location_job_payload(item), force=force) async def apply_location(item: ParsedNewsItem) -> ParsedNewsItem: cached_patch = await get_cached_target_location_patch(item.id) if cached_patch: apply_enrichment_patch_to_item(item, cached_patch) if not _has_required_localization(item): queued = await enqueue_item(item, force=True) if queued and item.enrichment_status in { "pending", "unavailable", "provider_error", "parse_error", "no_result", "location_only", }: item.enrichment_status = "queued" return item queued = await enqueue_item(item) item.location_patch = build_anchor_location_patch( item, queued=queued, queue_available=queued, ) return item return list(await asyncio.gather(*(apply_location(item) for item in items))) async def _enqueue_unverified_locations(items: list[ParsedNewsItem]) -> None: if not items: return from app.services.earth_news_queue import enqueue_target_location_job await asyncio.gather( *( enqueue_target_location_job( build_target_location_job_payload(item), force=not _has_default_localization(item), ) for item in items if ( item.location_patch is None or item.location_patch.get("verified") is False or not _has_required_localization(item) ) ) ) async def _fetch_single_feed_url( client: httpx.AsyncClient, source: NewsFeedSource, feed: NewsFeedEndpoint, *, config_payload: dict[str, Any] | None = None, ) -> tuple[NewsFeedSource, list[ParsedNewsItem], str | None, dict[str, Any]]: started_at = perf_counter() try: response = await client.get(feed.url) latency_ms = int((perf_counter() - started_at) * 1000) status_code = response.status_code content_type = response.headers.get("content-type", "") if status_code >= 400: error = f"HTTP {status_code}" return source, [], error, _source_health_result( source, ok=False, count=0, error=error, status="http_error", status_code=status_code, content_type=content_type, latency_ms=latency_ms, final_url=str(response.url), feed=feed, ) try: items = _parse_feed_entries(response.text, source, feed=feed, config_payload=config_payload) except Exception as exc: error, status = _format_source_fetch_error( status_code=status_code, content_type=content_type, body=response.text, parsed_count=0, ) if error is None: error = str(exc) status = "format_error" return source, [], error, _source_health_result( source, ok=False, count=0, error=error, status=status, status_code=status_code, content_type=content_type, latency_ms=latency_ms, final_url=str(response.url), feed=feed, ) error, status = _format_source_fetch_error( status_code=status_code, content_type=content_type, body=response.text, parsed_count=len(items), ) return source, items, error, _source_health_result( source, ok=error is None, count=len(items), error=error, status=status, status_code=status_code, content_type=content_type, latency_ms=latency_ms, final_url=str(response.url), feed=feed, ) except httpx.TimeoutException as exc: error = f"请求超时: {exc}" return source, [], error, _source_health_result( source, ok=False, count=0, error=error, status="timeout", latency_ms=int((perf_counter() - started_at) * 1000), feed=feed, ) except Exception as exc: error = str(exc) return source, [], error, _source_health_result( source, ok=False, count=0, error=error, status="network_error", latency_ms=int((perf_counter() - started_at) * 1000), feed=feed, ) async def _fetch_source( client: httpx.AsyncClient, source: NewsFeedSource, *, active_region: str | None = None, config_payload: dict[str, Any] | None = None, ) -> tuple[NewsFeedSource, list[ParsedNewsItem], str | None, dict[str, Any]]: feeds = tuple( feed for feed in _source_feeds(source) if feed.enabled and feed.type in {"rss", "atom", "aggregated"} and _feed_matches_active_region(feed, active_region) ) if not feeds: health = _source_health_result(source, ok=False, count=0, error="新闻源缺少已启用的 Feed 子项。", status="format_error") return source, [], health["error"], health results = await asyncio.gather(*( _fetch_single_feed_url(client, source, feed, config_payload=config_payload) for feed in sorted(feeds, key=lambda item: (item.priority, item.name)) )) merged_items: list[ParsedNewsItem] = [] feed_results: list[dict[str, Any]] = [] errors: list[str] = [] for _source, items, error, health in results: feed_results.append(health) if error: errors.append(f"{health.get('final_url') or source.feed_url}: {error}") continue merged_items.extend(items) deduped: dict[str, ParsedNewsItem] = {} for item in merged_items: key = item.url.strip() or item.title.strip().lower() if key and key not in deduped: deduped[key] = item items = list(deduped.values()) ok = bool(items) error = None if ok else "; ".join(errors) or "未解析到 RSS/Atom 条目，请检查 Feed 地址或源格式。" latency_ms = sum(int(result.get("latency_ms") or 0) for result in feed_results) status = "ok" if ok else (feed_results[0].get("status") if len(feed_results) == 1 else "failed") status_code = next((result.get("status_code") for result in feed_results if result.get("status_code")), None) content_type = ", ".join(sorted({str(result.get("content_type") or "") for result in feed_results if result.get("content_type")})) health = _source_health_result( source, ok=ok, count=len(items), error=error, status=status, status_code=int(status_code) if isinstance(status_code, int) else None, content_type=content_type, latency_ms=latency_ms, final_url=source.feed_url, ) health["feed_results"] = feed_results return source, items, error, health async def _fetch_rss_items_for_sources( sources: list[NewsFeedSource], *, active_region: str | None = None, config_payload: dict[str, Any] | None = None, ) -> tuple[list[ParsedNewsItem], list[str], dict[str, dict[str, Any]]]: errors: list[str] = [] async with httpx.AsyncClient( timeout=REQUEST_TIMEOUT, follow_redirects=True, headers={"User-Agent": USER_AGENT}, ) as client: if config_payload is None: results = await asyncio.gather(*(_fetch_source(client, source, active_region=active_region) for source in sources)) else: results = await asyncio.gather(*( _fetch_source(client, source, active_region=active_region, config_payload=config_payload) for source in sources )) fetched_items: list[ParsedNewsItem] = [] health_by_source: dict[str, dict[str, Any]] = {} for source, items, error, health in results: health_by_source[source.id] = health if error: errors.append(f"{source.name}: {error}") continue fetched_items.extend(items) return fetched_items, errors, health_by_source def _needs_rss_supplement(*, item_count: int, newest_at: datetime | None) -> bool: if item_count < MAX_ITEMS_TOTAL: return True if newest_at is None: return True age_seconds = (datetime.now(UTC) - newest_at).total_seconds() return age_seconds > RSS_SUPPLEMENT_MAX_AGE_SECONDS async def _get_earth_news_payload_from_rss_only( *, lat: float | None, lon: float | None, active_region: str, sources: list[NewsFeedSource], categories: set[str] | None = None, source_ids: set[str] | None = None, limit: int = MAX_ITEMS_TOTAL, locale: str = DEFAULT_NEWS_LOCALE, ) -> dict[str, Any]: fetched_items, errors, _health_by_source = await _fetch_rss_items_for_sources(sources, active_region=active_region) ranked_items = _filter_news_items_by_source_ids( _filter_news_items_by_categories( _rank_and_trim_items(fetched_items, active_region=active_region, limit=limit), categories, ), source_ids, )[:limit] if ranked_items: ranked_items = await _apply_cached_locations_and_enqueue(ranked_items) _store_region_cache(active_region, items=ranked_items, sources=sources) return _build_payload( lat=lat, lon=lon, active_region=active_region, items=ranked_items, sources=sources, errors=errors, stale=False, categories=categories, source_ids=source_ids, limit=limit, locale=locale, ) cached = _get_cached_region_feed(active_region) if cached: cached_items = _filter_news_items_by_source_ids( _filter_news_items_by_categories(cached.items, categories), source_ids, )[:limit] cached.items = await _apply_cached_locations_and_enqueue(cached_items) return _build_payload( lat=lat, lon=lon, active_region=active_region, items=cached.items, sources=cached.sources, errors=errors, stale=True, categories=categories, source_ids=source_ids, limit=limit, locale=locale, generated_at=cached.fetched_at, ) return _build_payload( lat=lat, lon=lon, active_region=active_region, items=[], sources=sources, errors=errors, stale=False, categories=categories, source_ids=source_ids, limit=limit, locale=locale, ) async def get_earth_news_payload( lat: float | None = None, lon: float | None = None, *, region: str | None = None, categories: set[str] | None = None, source_ids: set[str] | None = None, limit: int = MAX_ITEMS_TOTAL, locale: str = DEFAULT_NEWS_LOCALE, provider_client: AIProviderClient | None = None, db: AsyncSession | None = None, ) -> dict[str, Any]: del provider_client active_region = region if region in REGION_ANCHORS else determine_focus_region(lat, lon) has_settings_db = db is not None and callable(getattr(db, "execute", None)) source_config_payload = await get_earth_news_sources_payload(db) if has_settings_db else None sources = ( await get_configured_sources_for_region(db, active_region) if has_settings_db else get_sources_for_region(active_region) ) if db is None: return await _get_earth_news_payload_from_rss_only( lat=lat, lon=lon, active_region=active_region, sources=sources, categories=categories, source_ids=source_ids, limit=limit, locale=locale, ) from app.services.earth_news_store import ( get_earth_news_feed_coverage, get_earth_news_freshness, list_earth_news_cruise_items, list_earth_news_items, upsert_earth_news_items, ) errors: list[str] = [] item_count, newest_at = await get_earth_news_freshness(db, active_region=active_region) should_supplement = _needs_rss_supplement(item_count=item_count, newest_at=newest_at) if not should_supplement and callable(getattr(db, "execute", None)): expected_feed_keys = _expected_feed_keys(sources, active_region=active_region) if expected_feed_keys: covered_feed_keys = await get_earth_news_feed_coverage( db, active_region=active_region, recent_after=datetime.now(UTC) - timedelta(seconds=RSS_SUPPLEMENT_MAX_AGE_SECONDS), ) should_supplement = bool(expected_feed_keys - covered_feed_keys) if should_supplement: if source_config_payload is None: fetched_items, errors, health_by_source = await _fetch_rss_items_for_sources(sources, active_region=active_region) else: fetched_items, errors, health_by_source = await _fetch_rss_items_for_sources( sources, active_region=active_region, config_payload=source_config_payload, ) await record_earth_news_sources_health(db, health_by_source) ranked_fetched_items = _rank_and_trim_items(fetched_items, active_region=active_region) await _enqueue_unverified_locations(ranked_fetched_items) await upsert_earth_news_items(db, ranked_fetched_items) items = await _call_store_list_items( list_earth_news_items, db, active_region=active_region, limit=limit, categories=categories, source_ids=source_ids, ) if not source_ids: ready_sources = { _news_item_source_id(item) for item in items if _is_news_item_display_ready(item, locale=locale) } missing_ready_sources = [ source.id for source in sources if source.id and source.id not in ready_sources ] if missing_ready_sources: extra_items: list[ParsedNewsItem] = [] for missing_source_id in missing_ready_sources: extra_items.extend( await _call_store_list_items( list_earth_news_items, db, active_region=active_region, limit=3, categories=categories, source_ids={missing_source_id}, ) ) if extra_items: items = _diversify_news_items_for_locale( [*items, *extra_items], active_region=active_region, limit=limit, locale=locale, ) if hasattr(db, "execute"): cruise_items = await _call_store_list_items( list_earth_news_cruise_items, db, limit=min(max(limit, MAX_ITEMS_TOTAL) * len(REGION_ANCHORS), 500), categories=categories, source_ids=source_ids, ) else: cruise_items = items enqueue_candidates = {item.id: item for item in [*items, *cruise_items] if item.id} await _enqueue_unverified_locations(list(enqueue_candidates.values())) stale = bool(errors and items) return _build_payload( lat=lat, lon=lon, active_region=active_region, items=items, cruise_items=cruise_items, sources=sources, errors=errors, stale=stale, categories=categories, source_ids=source_ids, limit=limit, locale=locale, )