2747 lines
107 KiB
Python
2747 lines
107 KiB
Python
from __future__ import annotations
|
||
|
||
import asyncio
|
||
from dataclasses import dataclass, field
|
||
from datetime import UTC, datetime, timedelta
|
||
from email.utils import parsedate_to_datetime
|
||
import hashlib
|
||
import html
|
||
import json
|
||
import math
|
||
import re
|
||
from time import perf_counter
|
||
from typing import Any
|
||
from urllib.parse import quote
|
||
import xml.etree.ElementTree as ET
|
||
|
||
import httpx
|
||
from bs4 import BeautifulSoup
|
||
from sqlalchemy import select
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.core.countries import COUNTRY_VARIANTS_MAP, get_country_centroid, normalize_country
|
||
from app.core.enums import (
|
||
BreakingLevel,
|
||
BreakingScope,
|
||
BreakingSource,
|
||
NewsEnrichmentStatus,
|
||
NewsImportanceLevel,
|
||
NewsMarketImpact,
|
||
NewsSourceType,
|
||
NewsTaggingSource,
|
||
)
|
||
from app.models.system_setting import SystemSetting
|
||
from app.ai_tasks.prompts import EffectiveAIPrompt, get_effective_prompt
|
||
from app.schemas.ai import SituationalAnalysisRequest
|
||
from app.services.ai_client import AIProviderClient
|
||
from app.services.earth_news_classification import (
|
||
apply_news_classification as _apply_news_classification,
|
||
breaking_sort_rank as _breaking_sort_rank,
|
||
highest_breaking_level as _highest_breaking_level,
|
||
normalize_breaking_level as _normalize_breaking_level_enum,
|
||
normalize_breaking_scope as _normalize_breaking_scope_enum,
|
||
)
|
||
from app.services.location.resolvers.nominatim import build_default_nominatim_geocoder
|
||
|
||
|
||
USER_AGENT = "PlanetEarthNewsBoard/1.0 (+https://planet.local)"
|
||
REQUEST_TIMEOUT = 12.0
|
||
MAX_ITEMS_PER_SOURCE = 6
|
||
MAX_ITEMS_TOTAL = 12
|
||
STALE_CACHE_MAX_AGE_SECONDS = 60 * 45
|
||
RSS_SUPPLEMENT_MAX_AGE_SECONDS = STALE_CACHE_MAX_AGE_SECONDS
|
||
MAX_TARGET_INFERENCE_CONCURRENCY = 3
|
||
TARGET_INFERENCE_TIMEOUT_SECONDS = 6.0
|
||
DEFAULT_NEWS_LOCALE = "zh-CN"
|
||
SUPPORTED_NEWS_LOCALES = frozenset({"zh-CN", "en-US"})
|
||
NEWS_ENRICH_PROMPT_KEY = "earth.news.enrich"
|
||
EARTH_NEWS_SOURCES_CATEGORY = "earth_news_sources"
|
||
DEFAULT_NEWS_HEALTH_POLICY = {
|
||
"timeout_seconds": REQUEST_TIMEOUT,
|
||
"failure_threshold": 3,
|
||
"cooldown_minutes": 30,
|
||
"fetch_interval_minutes": 45,
|
||
"circuit_breaker": True,
|
||
}
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class RegionProfile:
|
||
key: str
|
||
label: str
|
||
query: str
|
||
accent: str
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class RegionAnchor:
|
||
region: str
|
||
label: str
|
||
latitude: float
|
||
longitude: float
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class NewsFeedEndpoint:
|
||
id: str
|
||
name: str
|
||
url: str
|
||
type: str = NewsSourceType.RSS.value
|
||
region: str = ""
|
||
enabled: bool = True
|
||
default_category: str = "other"
|
||
tags: tuple[str, ...] = ()
|
||
priority: int = 100
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class NewsFeedSource:
|
||
id: str
|
||
name: str
|
||
region: str
|
||
feed_url: str
|
||
homepage_url: str
|
||
feed_directory_url: str = ""
|
||
source_type: str = NewsSourceType.RSS.value
|
||
feed_urls: tuple[str, ...] = ()
|
||
feeds: tuple[NewsFeedEndpoint, ...] = ()
|
||
priority: int = 100
|
||
enabled: bool = True
|
||
source_tags: tuple[str, ...] = ()
|
||
default_category: str = "other"
|
||
importance_weight: int = 0
|
||
health_policy: dict[str, Any] = field(default_factory=dict)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class NewsTargetLocation:
|
||
latitude: float
|
||
longitude: float
|
||
label: str
|
||
source: str
|
||
confidence: float | None = None
|
||
country: str | None = None
|
||
city: str | None = None
|
||
|
||
|
||
@dataclass
|
||
class ParsedNewsItem:
|
||
id: str
|
||
title: str
|
||
summary: str
|
||
url: str
|
||
source: str
|
||
feed_name: str
|
||
feed_region: str
|
||
homepage_url: str
|
||
published_at: datetime | None
|
||
content_language: str = "en"
|
||
localizations: dict[str, dict[str, str]] = field(default_factory=dict)
|
||
enrichment_status: str = NewsEnrichmentStatus.PENDING.value
|
||
enrichment_error: str | None = None
|
||
enriched_at: datetime | None = None
|
||
target_location: NewsTargetLocation | None = None
|
||
target_resolution_stage: str = "unresolved"
|
||
target_ai_attempted: bool = False
|
||
target_ai_status: str = "not_attempted"
|
||
target_ai_error: str | None = None
|
||
target_debug_note: str | None = None
|
||
location_patch: dict[str, Any] | None = None
|
||
source_tags: list[str] = field(default_factory=list)
|
||
feed_id: str = ""
|
||
feed_type: str = NewsSourceType.RSS.value
|
||
feed_default_category: str = "other"
|
||
category: str = "other"
|
||
item_tags: list[str] = field(default_factory=list)
|
||
tagging_source: str = NewsTaggingSource.RULES.value
|
||
tagging_confidence: float = 0.0
|
||
importance_score: int = 0
|
||
importance_level: str = NewsImportanceLevel.LOW.value
|
||
importance_reasons: list[str] = field(default_factory=list)
|
||
market_impact: str = NewsMarketImpact.NONE.value
|
||
breaking_level: str = BreakingLevel.NONE.value
|
||
breaking_scope: str = BreakingScope.REGIONAL.value
|
||
breaking_reasons: list[str] = field(default_factory=list)
|
||
breaking_source: str = BreakingSource.RULES.value
|
||
breaking_confidence: float = 0.0
|
||
breaking_expires_at: datetime | None = None
|
||
|
||
|
||
@dataclass
|
||
class CachedRegionFeed:
|
||
region: str
|
||
fetched_at: datetime
|
||
items: list[ParsedNewsItem]
|
||
sources: list[NewsFeedSource]
|
||
|
||
|
||
REGION_PROFILES: dict[str, RegionProfile] = {
|
||
"americas": RegionProfile(
|
||
key="americas",
|
||
label="美洲焦点",
|
||
query='Americas geopolitics OR Latin America OR "United States" OR Canada',
|
||
accent="#79d3ff",
|
||
),
|
||
"europe": RegionProfile(
|
||
key="europe",
|
||
label="欧洲焦点",
|
||
query='Europe geopolitics OR EU OR NATO OR "Eastern Europe"',
|
||
accent="#8fd4ff",
|
||
),
|
||
"middle-east-africa": RegionProfile(
|
||
key="middle-east-africa",
|
||
label="中东与非洲焦点",
|
||
query='"Middle East" OR Africa geopolitics OR Red Sea OR Gulf',
|
||
accent="#ffb56a",
|
||
),
|
||
"asia-pacific": RegionProfile(
|
||
key="asia-pacific",
|
||
label="亚太焦点",
|
||
query='"Asia Pacific" OR Indo-Pacific OR China OR Japan OR Korea OR ASEAN',
|
||
accent="#78f2cf",
|
||
),
|
||
"global": RegionProfile(
|
||
key="global",
|
||
label="全球焦点",
|
||
query='"world news" OR geopolitics OR "global affairs"',
|
||
accent="#d6e6ff",
|
||
),
|
||
}
|
||
|
||
REGION_ANCHORS: dict[str, RegionAnchor] = {
|
||
"americas": RegionAnchor(
|
||
region="americas",
|
||
label="美洲",
|
||
latitude=37.0902,
|
||
longitude=-95.7129,
|
||
),
|
||
"europe": RegionAnchor(
|
||
region="europe",
|
||
label="欧洲",
|
||
latitude=50.1109,
|
||
longitude=8.6821,
|
||
),
|
||
"middle-east-africa": RegionAnchor(
|
||
region="middle-east-africa",
|
||
label="中东与非洲",
|
||
latitude=25.2048,
|
||
longitude=55.2708,
|
||
),
|
||
"asia-pacific": RegionAnchor(
|
||
region="asia-pacific",
|
||
label="亚太",
|
||
latitude=1.3521,
|
||
longitude=103.8198,
|
||
),
|
||
"global": RegionAnchor(
|
||
region="global",
|
||
label="全球",
|
||
latitude=20.0,
|
||
longitude=0.0,
|
||
),
|
||
}
|
||
|
||
|
||
def _google_news_feed(query: str, *, hl: str, gl: str, ceid: str) -> str:
|
||
return (
|
||
"https://news.google.com/rss/search?q="
|
||
+ quote(query, safe="")
|
||
+ f"&hl={hl}&gl={gl}&ceid={ceid}"
|
||
)
|
||
|
||
|
||
NEWS_FEED_SOURCES: tuple[NewsFeedSource, ...] = (
|
||
NewsFeedSource(
|
||
id="bbc-world",
|
||
name="BBC World",
|
||
region="global",
|
||
feed_url="https://feeds.bbci.co.uk/news/world/rss.xml",
|
||
homepage_url="https://www.bbc.com/news/world",
|
||
feeds=(
|
||
NewsFeedEndpoint(id="world", name="World", url="https://feeds.bbci.co.uk/news/world/rss.xml", default_category="politics", priority=1),
|
||
),
|
||
source_type="rss",
|
||
priority=10,
|
||
source_tags=("official_media", "business_news", "global"),
|
||
default_category="politics",
|
||
importance_weight=12,
|
||
health_policy=DEFAULT_NEWS_HEALTH_POLICY,
|
||
),
|
||
NewsFeedSource(
|
||
id="dw-top",
|
||
name="DW Top Stories",
|
||
region="europe",
|
||
feed_url="https://rss.dw.com/rdf/rss-en-top",
|
||
homepage_url="https://www.dw.com/en/top-stories/s-9097",
|
||
feeds=(
|
||
NewsFeedEndpoint(id="top", name="Top Stories", url="https://rss.dw.com/rdf/rss-en-top", default_category="politics", priority=1),
|
||
),
|
||
source_type="rss",
|
||
priority=20,
|
||
source_tags=("official_media", "business_news", "global", "europe"),
|
||
default_category="politics",
|
||
importance_weight=12,
|
||
health_policy=DEFAULT_NEWS_HEALTH_POLICY,
|
||
),
|
||
NewsFeedSource(
|
||
id="cnbc-business",
|
||
name="CNBC Business",
|
||
region="global",
|
||
feed_url="https://www.cnbc.com/id/10001147/device/rss/rss.html",
|
||
homepage_url="https://www.cnbc.com/business/",
|
||
feeds=(
|
||
NewsFeedEndpoint(id="business", name="Business", url="https://www.cnbc.com/id/10001147/device/rss/rss.html", default_category="business", priority=1),
|
||
),
|
||
source_type="rss",
|
||
priority=21,
|
||
source_tags=("business_news", "global", "us"),
|
||
default_category="business",
|
||
importance_weight=16,
|
||
health_policy=DEFAULT_NEWS_HEALTH_POLICY,
|
||
),
|
||
NewsFeedSource(
|
||
id="bbc-business",
|
||
name="BBC Business",
|
||
region="global",
|
||
feed_url="https://feeds.bbci.co.uk/news/business/rss.xml",
|
||
homepage_url="https://www.bbc.com/news/business",
|
||
feeds=(
|
||
NewsFeedEndpoint(id="business", name="Business", url="https://feeds.bbci.co.uk/news/business/rss.xml", default_category="business", priority=1),
|
||
),
|
||
source_type="rss",
|
||
priority=22,
|
||
source_tags=("official_media", "business_news", "global"),
|
||
default_category="business",
|
||
importance_weight=14,
|
||
health_policy=DEFAULT_NEWS_HEALTH_POLICY,
|
||
),
|
||
NewsFeedSource(
|
||
id="guardian-business",
|
||
name="The Guardian Business",
|
||
region="europe",
|
||
feed_url="https://www.theguardian.com/uk/business/rss",
|
||
homepage_url="https://www.theguardian.com/uk/business",
|
||
feeds=(
|
||
NewsFeedEndpoint(id="business", name="Business", url="https://www.theguardian.com/uk/business/rss", default_category="business", priority=1),
|
||
),
|
||
source_type="rss",
|
||
priority=24,
|
||
source_tags=("business_news", "global", "europe"),
|
||
default_category="business",
|
||
importance_weight=12,
|
||
health_policy=DEFAULT_NEWS_HEALTH_POLICY,
|
||
),
|
||
NewsFeedSource(
|
||
id="npr-business",
|
||
name="NPR Business",
|
||
region="americas",
|
||
feed_url="https://feeds.npr.org/1006/rss.xml",
|
||
homepage_url="https://www.npr.org/sections/business/",
|
||
feeds=(
|
||
NewsFeedEndpoint(id="business", name="Business", url="https://feeds.npr.org/1006/rss.xml", default_category="business", priority=1),
|
||
),
|
||
source_type="rss",
|
||
priority=25,
|
||
source_tags=("business_news", "global", "us"),
|
||
default_category="business",
|
||
importance_weight=12,
|
||
health_policy=DEFAULT_NEWS_HEALTH_POLICY,
|
||
),
|
||
NewsFeedSource(
|
||
id="marketwatch-top",
|
||
name="MarketWatch Top Stories",
|
||
region="americas",
|
||
feed_url="https://feeds.content.dowjones.io/public/rss/mw_topstories",
|
||
homepage_url="https://www.marketwatch.com/",
|
||
feeds=(
|
||
NewsFeedEndpoint(id="top", name="Top Stories", url="https://feeds.content.dowjones.io/public/rss/mw_topstories", default_category="finance", priority=1),
|
||
),
|
||
source_type="rss",
|
||
priority=26,
|
||
source_tags=("business_news", "finance", "global", "us"),
|
||
default_category="finance",
|
||
importance_weight=12,
|
||
health_policy=DEFAULT_NEWS_HEALTH_POLICY,
|
||
),
|
||
NewsFeedSource(
|
||
id="techcrunch",
|
||
name="TechCrunch",
|
||
region="global",
|
||
feed_url="https://techcrunch.com/feed/",
|
||
homepage_url="https://techcrunch.com/",
|
||
feeds=(
|
||
NewsFeedEndpoint(id="main", name="Main Feed", url="https://techcrunch.com/feed/", default_category="technology", priority=1),
|
||
),
|
||
source_type="rss",
|
||
priority=32,
|
||
source_tags=("business_news", "ecommerce", "global"),
|
||
default_category="technology",
|
||
importance_weight=8,
|
||
health_policy=DEFAULT_NEWS_HEALTH_POLICY,
|
||
),
|
||
NewsFeedSource(
|
||
id="retaildive",
|
||
name="Retail Dive",
|
||
region="global",
|
||
feed_url="https://www.retaildive.com/feeds/news/",
|
||
homepage_url="https://www.retaildive.com/",
|
||
feeds=(
|
||
NewsFeedEndpoint(id="news", name="News", url="https://www.retaildive.com/feeds/news/", default_category="business", priority=1),
|
||
),
|
||
source_type="rss",
|
||
priority=34,
|
||
source_tags=("business_news", "retail", "ecommerce", "global"),
|
||
default_category="business",
|
||
importance_weight=10,
|
||
health_policy=DEFAULT_NEWS_HEALTH_POLICY,
|
||
),
|
||
NewsFeedSource(
|
||
id="prnewswire-retail",
|
||
name="PR Newswire Consumer Products & Retail",
|
||
region="global",
|
||
feed_url="https://www.prnewswire.com/rss/consumer-products-retail-latest-news/consumer-products-retail-latest-news-list.rss",
|
||
homepage_url="https://www.prnewswire.com/news-releases/consumer-products-retail-latest-news/",
|
||
feeds=(
|
||
NewsFeedEndpoint(id="consumer-retail", name="Consumer Products & Retail", url="https://www.prnewswire.com/rss/consumer-products-retail-latest-news/consumer-products-retail-latest-news-list.rss", default_category="business", priority=1),
|
||
),
|
||
source_type="rss",
|
||
priority=48,
|
||
source_tags=("press_release", "retail", "ecommerce", "global"),
|
||
default_category="business",
|
||
importance_weight=-4,
|
||
health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "failure_threshold": 2},
|
||
),
|
||
NewsFeedSource(
|
||
id="36kr",
|
||
name="36氪",
|
||
region="asia-pacific",
|
||
feed_url="https://36kr.com/feed-article",
|
||
homepage_url="https://www.36kr.com/",
|
||
feed_directory_url="https://www.36kr.com/rss-center",
|
||
source_type="rss",
|
||
feeds=(
|
||
NewsFeedEndpoint(id="feed", name="综合资讯", url="https://36kr.com/feed", default_category="business", priority=1),
|
||
NewsFeedEndpoint(id="article", name="文章资讯", url="https://36kr.com/feed-article", default_category="business", priority=2),
|
||
NewsFeedEndpoint(id="newsflash", name="最新快讯", url="https://36kr.com/feed-newsflash", default_category="business", priority=3),
|
||
NewsFeedEndpoint(id="moment", name="动态内容", url="https://36kr.com/feed-moment", default_category="business", priority=4),
|
||
),
|
||
priority=35,
|
||
source_tags=("business_news", "ecommerce", "china"),
|
||
default_category="business",
|
||
importance_weight=12,
|
||
health_policy=DEFAULT_NEWS_HEALTH_POLICY,
|
||
),
|
||
NewsFeedSource(
|
||
id="businesswire-ecommerce",
|
||
name="BusinessWire Electronic Commerce",
|
||
region="global",
|
||
feed_url="https://www.businesswire.com/newsroom/industry/technology/ecommerce",
|
||
homepage_url="https://www.businesswire.com/newsroom/industry/technology/ecommerce",
|
||
source_type="reference",
|
||
priority=62,
|
||
enabled=False,
|
||
source_tags=("press_release", "ecommerce", "global", "low_stability"),
|
||
default_category="ecommerce",
|
||
importance_weight=-6,
|
||
health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "failure_threshold": 2},
|
||
),
|
||
NewsFeedSource(
|
||
id="mckinsey-retail",
|
||
name="McKinsey Retail Insights",
|
||
region="global",
|
||
feed_url="https://www.mckinsey.com/industries/retail/our-insights",
|
||
homepage_url="https://www.mckinsey.com/industries/retail/our-insights",
|
||
source_type="reference",
|
||
priority=64,
|
||
enabled=False,
|
||
source_tags=("industry_insight", "retail", "global"),
|
||
default_category="business",
|
||
importance_weight=18,
|
||
health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 720},
|
||
),
|
||
NewsFeedSource(
|
||
id="deloitte-retail",
|
||
name="Deloitte Retail",
|
||
region="global",
|
||
feed_url="https://www.deloitte.com/us/en/Industries/retail/about.html",
|
||
homepage_url="https://www.deloitte.com/us/en/Industries/retail/about.html",
|
||
source_type="reference",
|
||
priority=66,
|
||
enabled=False,
|
||
source_tags=("industry_insight", "retail", "global", "us"),
|
||
default_category="business",
|
||
importance_weight=16,
|
||
health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 720},
|
||
),
|
||
NewsFeedSource(
|
||
id="us-census-ecommerce",
|
||
name="US Census Retail / Quarterly E-Commerce",
|
||
region="americas",
|
||
feed_url="https://www.census.gov/retail/index.html#ecommerce",
|
||
homepage_url="https://www.census.gov/retail/index.html#ecommerce",
|
||
source_type="reference",
|
||
priority=18,
|
||
enabled=False,
|
||
source_tags=("official_data", "ecommerce", "retail", "us"),
|
||
default_category="ecommerce",
|
||
importance_weight=34,
|
||
health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 1440},
|
||
),
|
||
NewsFeedSource(
|
||
id="mofcom-data",
|
||
name="商务数据中心",
|
||
region="asia-pacific",
|
||
feed_url="https://data.mofcom.gov.cn/index.shtml",
|
||
homepage_url="https://data.mofcom.gov.cn/index.shtml",
|
||
source_type="reference",
|
||
priority=18,
|
||
enabled=False,
|
||
source_tags=("official_data", "business_news", "china"),
|
||
default_category="business",
|
||
importance_weight=34,
|
||
health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 1440},
|
||
),
|
||
NewsFeedSource(
|
||
id="mofcom-ecommerce",
|
||
name="商务部电商动态",
|
||
region="asia-pacific",
|
||
feed_url="https://www.mofcom.gov.cn/",
|
||
homepage_url="https://www.mofcom.gov.cn/",
|
||
source_type="reference",
|
||
priority=19,
|
||
enabled=False,
|
||
source_tags=("official_data", "ecommerce", "china"),
|
||
default_category="ecommerce",
|
||
importance_weight=34,
|
||
health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 1440},
|
||
),
|
||
NewsFeedSource(
|
||
id="stats-china-online-retail",
|
||
name="国家统计局数据发布",
|
||
region="asia-pacific",
|
||
feed_url="https://www.stats.gov.cn/sj/zxfb/rss.xml",
|
||
homepage_url="https://www.stats.gov.cn/sj/zxfb/",
|
||
feeds=(
|
||
NewsFeedEndpoint(id="release", name="数据发布", url="https://www.stats.gov.cn/sj/zxfb/rss.xml", default_category="ecommerce", priority=1),
|
||
),
|
||
source_type="rss",
|
||
priority=19,
|
||
source_tags=("official_data", "ecommerce", "retail", "china"),
|
||
default_category="ecommerce",
|
||
importance_weight=36,
|
||
health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 1440},
|
||
),
|
||
NewsFeedSource(
|
||
id="china-ecommerce-logistics-index",
|
||
name="电商物流指数",
|
||
region="asia-pacific",
|
||
feed_url="https://www.gov.cn/",
|
||
homepage_url="https://www.gov.cn/",
|
||
source_type="reference",
|
||
priority=20,
|
||
enabled=False,
|
||
source_tags=("official_data", "ecommerce", "retail", "china"),
|
||
default_category="ecommerce",
|
||
importance_weight=36,
|
||
health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 1440},
|
||
),
|
||
NewsFeedSource(
|
||
id="ebrun",
|
||
name="亿邦动力",
|
||
region="asia-pacific",
|
||
feed_url="https://www.ebrun.com/rss/news_b2c.xml",
|
||
homepage_url="https://www.ebrun.com/",
|
||
feed_directory_url="https://www.ebrun.com/rss/",
|
||
source_type="rss",
|
||
feeds=(
|
||
NewsFeedEndpoint(id="b2c", name="B2C", url="https://www.ebrun.com/rss/news_b2c.xml", default_category="ecommerce", priority=1),
|
||
NewsFeedEndpoint(id="b2b", name="B2B", url="https://www.ebrun.com/rss/news_b2b.xml", default_category="ecommerce", priority=2),
|
||
NewsFeedEndpoint(id="retail", name="零售", url="https://www.ebrun.com/rss/news_retail.xml", default_category="ecommerce", priority=3),
|
||
NewsFeedEndpoint(id="o2o", name="O2O", url="https://www.ebrun.com/rss/news_o2o.xml", default_category="ecommerce", priority=4),
|
||
NewsFeedEndpoint(id="service", name="服务", url="https://www.ebrun.com/rss/news_service.xml", default_category="ecommerce", priority=5),
|
||
NewsFeedEndpoint(id="data", name="数据", url="https://www.ebrun.com/rss/news_data.xml", default_category="ecommerce", priority=6),
|
||
NewsFeedEndpoint(id="policy", name="政策", url="https://www.ebrun.com/rss/news_policy.xml", default_category="ecommerce", priority=7),
|
||
),
|
||
priority=38,
|
||
source_tags=("business_news", "ecommerce", "retail", "china"),
|
||
default_category="ecommerce",
|
||
importance_weight=14,
|
||
health_policy=DEFAULT_NEWS_HEALTH_POLICY,
|
||
),
|
||
NewsFeedSource(
|
||
id="google-news",
|
||
name="Google News",
|
||
region="global",
|
||
feed_url=_google_news_feed(
|
||
REGION_PROFILES["global"].query,
|
||
hl="en-US",
|
||
gl="US",
|
||
ceid="US:en",
|
||
),
|
||
homepage_url="https://news.google.com/",
|
||
feed_directory_url="https://news.google.com/rss",
|
||
feeds=(
|
||
NewsFeedEndpoint(id="world", name="全球", url=_google_news_feed(REGION_PROFILES["global"].query, hl="en-US", gl="US", ceid="US:en"), type="aggregated", region="global", default_category="politics", priority=1),
|
||
NewsFeedEndpoint(id="americas", name="美洲", url=_google_news_feed(REGION_PROFILES["americas"].query, hl="en-US", gl="US", ceid="US:en"), type="aggregated", region="americas", default_category="politics", priority=2),
|
||
NewsFeedEndpoint(id="europe", name="欧洲", url=_google_news_feed(REGION_PROFILES["europe"].query, hl="en-GB", gl="GB", ceid="GB:en"), type="aggregated", region="europe", default_category="politics", priority=3),
|
||
NewsFeedEndpoint(id="middle-east-africa", name="中东与非洲", url=_google_news_feed(REGION_PROFILES["middle-east-africa"].query, hl="en-US", gl="US", ceid="US:en"), type="aggregated", region="middle-east-africa", default_category="politics", priority=4),
|
||
NewsFeedEndpoint(id="asia-pacific", name="亚太", url=_google_news_feed(REGION_PROFILES["asia-pacific"].query, hl="en-SG", gl="SG", ceid="SG:en"), type="aggregated", region="asia-pacific", default_category="politics", priority=5),
|
||
),
|
||
source_type="aggregated",
|
||
priority=90,
|
||
source_tags=("aggregated", "business_news", "global", "europe", "low_stability"),
|
||
default_category="politics",
|
||
importance_weight=-2,
|
||
health_policy=DEFAULT_NEWS_HEALTH_POLICY,
|
||
),
|
||
)
|
||
DEFAULT_NEWS_SOURCE_BY_ID: dict[str, NewsFeedSource] = {source.id: source for source in NEWS_FEED_SOURCES}
|
||
LEGACY_GOOGLE_NEWS_SOURCE_IDS = {
|
||
"global-scan",
|
||
"google-americas",
|
||
"google-europe",
|
||
"google-mea",
|
||
"google-apac",
|
||
}
|
||
|
||
DEFAULT_SOURCE_TAGS: tuple[dict[str, Any], ...] = (
|
||
{"key": "official_media", "label": "官方媒体", "color": "#2563eb", "enabled": True, "sort_order": 10},
|
||
{"key": "official_data", "label": "官方数据", "color": "#059669", "enabled": True, "sort_order": 20},
|
||
{"key": "business_news", "label": "商业新闻", "color": "#0f766e", "enabled": True, "sort_order": 30},
|
||
{"key": "ecommerce", "label": "电商", "color": "#7c3aed", "enabled": True, "sort_order": 40},
|
||
{"key": "finance", "label": "金融", "color": "#1d4ed8", "enabled": True, "sort_order": 45},
|
||
{"key": "retail", "label": "零售", "color": "#ea580c", "enabled": True, "sort_order": 50},
|
||
{"key": "logistics", "label": "物流", "color": "#16a34a", "enabled": True, "sort_order": 55},
|
||
{"key": "industry_insight", "label": "行业洞察", "color": "#0891b2", "enabled": True, "sort_order": 60},
|
||
{"key": "press_release", "label": "企业公告", "color": "#64748b", "enabled": True, "sort_order": 70},
|
||
{"key": "china", "label": "中国", "color": "#dc2626", "enabled": True, "sort_order": 80},
|
||
{"key": "global", "label": "全球", "color": "#475569", "enabled": True, "sort_order": 90},
|
||
{"key": "us", "label": "美国", "color": "#1d4ed8", "enabled": True, "sort_order": 100},
|
||
{"key": "europe", "label": "欧洲", "color": "#0284c7", "enabled": True, "sort_order": 110},
|
||
{"key": "aggregated", "label": "聚合源", "color": "#9333ea", "enabled": True, "sort_order": 120},
|
||
{"key": "low_stability", "label": "低稳定性", "color": "#f59e0b", "enabled": True, "sort_order": 130},
|
||
)
|
||
|
||
DEFAULT_CATEGORIES: tuple[dict[str, Any], ...] = (
|
||
{"key": "politics", "label": "政治", "color": "#2563eb", "enabled": True, "sort_order": 10, "keywords": ["election", "government", "minister", "parliament", "sanction", "diplomatic", "policy", "summit", "选举", "政府", "制裁", "外交", "政策"]},
|
||
{"key": "business", "label": "商业", "color": "#0f766e", "enabled": True, "sort_order": 20, "keywords": ["market", "company", "earnings", "trade", "supply chain", "merger", "retail", "consumer", "商业", "公司", "贸易", "供应链", "消费", "零售"]},
|
||
{"key": "ecommerce", "label": "电商", "color": "#7c3aed", "enabled": True, "sort_order": 30, "keywords": ["e-commerce", "ecommerce", "online retail", "gmv", "marketplace", "shopify", "amazon", "tiktok shop", "网上零售", "电商", "直播电商", "跨境电商", "订单量", "物流指数", "履约"]},
|
||
{"key": "finance", "label": "金融", "color": "#0369a1", "enabled": True, "sort_order": 40, "keywords": ["stock", "bond", "inflation", "central bank", "rate cut", "rate hike", "bank", "金融", "股市", "通胀", "央行", "利率"]},
|
||
{"key": "sports", "label": "体育", "color": "#16a34a", "enabled": True, "sort_order": 50, "keywords": ["football", "basketball", "tennis", "match", "league", "world cup", "olympics", "体育", "足球", "篮球", "世界杯", "奥运"]},
|
||
{"key": "technology", "label": "科技", "color": "#0891b2", "enabled": True, "sort_order": 60, "keywords": ["ai", "semiconductor", "chip", "cyber", "satellite", "software", "科技", "人工智能", "半导体", "芯片", "网络安全"]},
|
||
{"key": "military", "label": "军事", "color": "#4b5563", "enabled": True, "sort_order": 70, "keywords": ["military", "missile", "airstrike", "defense", "warship", "军事", "导弹", "空袭", "防务", "军舰"]},
|
||
{"key": "disaster", "label": "灾害", "color": "#dc2626", "enabled": True, "sort_order": 80, "keywords": ["earthquake", "flood", "wildfire", "typhoon", "hurricane", "灾害", "地震", "洪水", "山火", "台风"]},
|
||
{"key": "energy", "label": "能源", "color": "#ca8a04", "enabled": True, "sort_order": 90, "keywords": ["oil", "gas", "opec", "energy", "power grid", "能源", "石油", "天然气", "电网"]},
|
||
{"key": "society", "label": "社会", "color": "#64748b", "enabled": True, "sort_order": 100, "keywords": ["health", "education", "crime", "migration", "社会", "医疗", "教育", "犯罪", "移民"]},
|
||
{"key": "culture", "label": "文化", "color": "#db2777", "enabled": True, "sort_order": 110, "keywords": ["film", "music", "art", "culture", "文化", "电影", "音乐", "艺术"]},
|
||
{"key": "other", "label": "其他", "color": "#64748b", "enabled": True, "sort_order": 999, "keywords": []},
|
||
)
|
||
ALLOWED_NEWS_CATEGORY_KEYS = tuple(item["key"] for item in DEFAULT_CATEGORIES)
|
||
|
||
DEFAULT_ITEM_TAG_RULES: tuple[dict[str, Any], ...] = (
|
||
{"key": "cross_border_ecommerce", "label": "跨境电商", "category": "ecommerce", "keywords": ["cross-border e-commerce", "cross border ecommerce", "跨境电商"]},
|
||
{"key": "live_commerce", "label": "直播电商", "category": "ecommerce", "keywords": ["live commerce", "livestream shopping", "直播电商", "直播带货"]},
|
||
{"key": "retail_data", "label": "零售数据", "category": "ecommerce", "keywords": ["online retail sales", "retail sales", "网上零售额", "社零", "社会消费品零售总额"]},
|
||
{"key": "logistics_fulfillment", "label": "物流履约", "category": "ecommerce", "keywords": ["logistics", "fulfillment", "delivery", "物流指数", "履约", "配送"]},
|
||
{"key": "platform_governance", "label": "平台治理", "category": "ecommerce", "keywords": ["platform regulation", "marketplace rules", "平台治理", "平台监管"]},
|
||
{"key": "ai", "label": "AI", "category": "technology", "keywords": ["ai", "artificial intelligence", "人工智能", "大模型"]},
|
||
{"key": "semiconductor", "label": "半导体", "category": "technology", "keywords": ["semiconductor", "chip", "半导体", "芯片"]},
|
||
{"key": "election", "label": "选举", "category": "politics", "keywords": ["election", "vote", "campaign", "选举", "投票"]},
|
||
{"key": "oil_price", "label": "油价", "category": "energy", "keywords": ["oil price", "crude", "油价", "原油"]},
|
||
{"key": "football", "label": "足球", "category": "sports", "keywords": ["football", "soccer", "premier league", "足球"]},
|
||
{"key": "supply_chain", "label": "供应链", "category": "business", "keywords": ["supply chain", "供应链"]},
|
||
)
|
||
|
||
def default_earth_news_sources_payload() -> dict[str, Any]:
|
||
return {
|
||
"cache_version": 1,
|
||
"source_tags": [dict(item) for item in DEFAULT_SOURCE_TAGS],
|
||
"categories": [dict(item) for item in DEFAULT_CATEGORIES],
|
||
"item_tag_rules": [dict(item) for item in DEFAULT_ITEM_TAG_RULES],
|
||
"sources": [serialize_news_source_config(source) for source in NEWS_FEED_SOURCES],
|
||
"health": {},
|
||
}
|
||
|
||
|
||
_REGION_CACHE: dict[str, CachedRegionFeed] = {}
|
||
_news_target_geocode = build_default_nominatim_geocoder(user_agent=USER_AGENT)
|
||
_CITY_HINTS: tuple[dict[str, str | None], ...] = (
|
||
{"name": "Beijing", "country": "中国"},
|
||
{"name": "Havana", "country": "古巴"},
|
||
{"name": "Kyiv", "country": "乌克兰"},
|
||
{"name": "Bangkok", "country": "泰国"},
|
||
{"name": "Tehran", "country": "伊朗"},
|
||
{"name": "Moscow", "country": "俄罗斯"},
|
||
{"name": "Taipei", "country": "中国（台湾）"},
|
||
{"name": "Hong Kong", "country": "中国（香港）"},
|
||
)
|
||
|
||
|
||
def determine_focus_region(lat: float | None, lon: float | None) -> str:
|
||
if lat is None or lon is None:
|
||
return "global"
|
||
if -170 <= lon <= -30:
|
||
return "americas"
|
||
if -30 < lon <= 45:
|
||
return "europe" if lat >= 30 else "middle-east-africa"
|
||
if 45 < lon <= 150:
|
||
return "middle-east-africa" if lat < 10 else "asia-pacific"
|
||
return "asia-pacific"
|
||
|
||
|
||
def get_region_profile(region: str) -> RegionProfile:
|
||
return REGION_PROFILES.get(region, REGION_PROFILES["global"])
|
||
|
||
|
||
def get_region_anchor(region: str) -> RegionAnchor:
|
||
return REGION_ANCHORS.get(region, REGION_ANCHORS["global"])
|
||
|
||
|
||
def _coerce_str(value: Any) -> str | None:
|
||
if value is None:
|
||
return None
|
||
if not isinstance(value, str):
|
||
value = str(value)
|
||
cleaned = re.sub(r"\s+", " ", value).strip()
|
||
return cleaned or None
|
||
|
||
|
||
def _contains_location_alias(text: str, alias: str) -> bool:
|
||
normalized_alias = _coerce_str(alias)
|
||
if not normalized_alias:
|
||
return False
|
||
if re.search(r"[A-Za-z]", normalized_alias):
|
||
pattern = r"(?<![A-Za-z])" + re.escape(normalized_alias) + r"(?![A-Za-z])"
|
||
return re.search(pattern, text, flags=re.IGNORECASE) is not None
|
||
return normalized_alias in text
|
||
|
||
|
||
def _iter_searchable_country_variants(
|
||
canonical: str,
|
||
variants: list[str],
|
||
) -> tuple[str, ...]:
|
||
searchable: list[str] = []
|
||
seen: set[str] = set()
|
||
for variant in (canonical, *variants):
|
||
normalized = _coerce_str(variant)
|
||
if not normalized:
|
||
continue
|
||
if re.fullmatch(r"[A-Z]{2,3}", normalized):
|
||
continue
|
||
if len(normalized) <= 2:
|
||
continue
|
||
key = normalized.casefold()
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
searchable.append(normalized)
|
||
return tuple(searchable)
|
||
|
||
|
||
def _coerce_float(value: Any) -> float | None:
|
||
try:
|
||
parsed = float(value)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
if not math.isfinite(parsed):
|
||
return None
|
||
return parsed
|
||
|
||
|
||
def _first_json_object(text: str) -> dict[str, Any] | None:
|
||
if not text:
|
||
return None
|
||
decoder = json.JSONDecoder()
|
||
for index, char in enumerate(text):
|
||
if char != "{":
|
||
continue
|
||
try:
|
||
payload, _ = decoder.raw_decode(text[index:])
|
||
except ValueError:
|
||
continue
|
||
if isinstance(payload, dict):
|
||
return payload
|
||
return None
|
||
|
||
|
||
def _normalize_localizations(value: Any) -> dict[str, dict[str, str]]:
|
||
if not isinstance(value, dict):
|
||
return {}
|
||
normalized: dict[str, dict[str, str]] = {}
|
||
for locale, payload in value.items():
|
||
locale_key = _coerce_str(locale)
|
||
if not locale_key or not isinstance(payload, dict):
|
||
continue
|
||
title = _coerce_str(payload.get("title"))
|
||
summary = _coerce_str(payload.get("summary"))
|
||
entry: dict[str, str] = {}
|
||
if title:
|
||
entry["title"] = title
|
||
if summary:
|
||
entry["summary"] = summary
|
||
if entry:
|
||
normalized[locale_key] = entry
|
||
return normalized
|
||
|
||
|
||
def _is_chinese_language(language: str | None) -> bool:
|
||
return str(language or "").lower() in {"zh", "zh-cn", "zh-hans", "chinese"}
|
||
|
||
|
||
def _is_english_language(language: str | None) -> bool:
|
||
return str(language or "").lower() in {"en", "en-us", "english"}
|
||
|
||
|
||
def _contains_cjk_text(value: str) -> bool:
|
||
return bool(re.search(r"[\u3400-\u9fff]", value or ""))
|
||
|
||
|
||
def _detect_content_language(
|
||
*,
|
||
title: str,
|
||
summary: str,
|
||
source: NewsFeedSource,
|
||
feed: NewsFeedEndpoint | None = None,
|
||
) -> str:
|
||
text = f"{title}\n{summary}"
|
||
if _contains_cjk_text(text):
|
||
return "zh-CN"
|
||
identity = {source.id.lower(), *(tag.lower() for tag in source.source_tags)}
|
||
if feed is not None:
|
||
identity.add(feed.id.lower())
|
||
identity.update(tag.lower() for tag in feed.tags)
|
||
if identity & {"china", "chinese", "36kr", "ebrun", "stats-china-online-retail"}:
|
||
return "zh-CN"
|
||
return "en"
|
||
|
||
|
||
def _source_language_localizations(
|
||
*,
|
||
title: str,
|
||
summary: str,
|
||
content_language: str,
|
||
) -> dict[str, dict[str, str]]:
|
||
if _is_chinese_language(content_language):
|
||
return {"zh-CN": {"title": title, "summary": summary or title}}
|
||
return {}
|
||
|
||
|
||
def _target_localization_locale(item: ParsedNewsItem) -> str:
|
||
return "en-US" if _is_chinese_language(item.content_language) else DEFAULT_NEWS_LOCALE
|
||
|
||
|
||
def _target_locale_schema(locale: str) -> dict[str, dict[str, str]]:
|
||
if locale == "en-US":
|
||
return {
|
||
"en-US": {
|
||
"title": "faithful English title",
|
||
"summary": "one-sentence newswire-style English lead summary",
|
||
}
|
||
}
|
||
return {
|
||
"zh-CN": {
|
||
"title": "faithful Simplified Chinese title",
|
||
"summary": "one-sentence newswire-style Simplified Chinese lead summary",
|
||
}
|
||
}
|
||
|
||
|
||
def _get_locale_text(
|
||
item: ParsedNewsItem,
|
||
key: str,
|
||
*,
|
||
locale: str = DEFAULT_NEWS_LOCALE,
|
||
) -> str:
|
||
localized = item.localizations.get(locale)
|
||
if isinstance(localized, dict):
|
||
value = _coerce_str(localized.get(key))
|
||
if value:
|
||
return value
|
||
if locale == "zh-CN" and _is_chinese_language(item.content_language):
|
||
return item.title if key == "title" else item.summary
|
||
if locale == "en-US" and _is_english_language(item.content_language):
|
||
return item.title if key == "title" else item.summary
|
||
fallback = item.localizations.get(DEFAULT_NEWS_LOCALE)
|
||
if isinstance(fallback, dict):
|
||
value = _coerce_str(fallback.get(key))
|
||
if value:
|
||
return value
|
||
if locale == "en-US" and _is_chinese_language(item.content_language):
|
||
return item.title if key == "title" else item.summary
|
||
return ""
|
||
|
||
|
||
def _has_localization(item: ParsedNewsItem, locale: str) -> bool:
|
||
localized = item.localizations.get(locale)
|
||
if not isinstance(localized, dict):
|
||
return False
|
||
return bool(_coerce_str(localized.get("title")) and _coerce_str(localized.get("summary")))
|
||
|
||
|
||
def _has_default_localization(item: ParsedNewsItem) -> bool:
|
||
return _has_localization(item, DEFAULT_NEWS_LOCALE)
|
||
|
||
|
||
def _has_required_localization(item: ParsedNewsItem) -> bool:
|
||
return _has_localization(item, _target_localization_locale(item))
|
||
|
||
|
||
def apply_enrichment_patch_to_item(
|
||
item: ParsedNewsItem,
|
||
patch: dict[str, Any],
|
||
) -> ParsedNewsItem:
|
||
item.location_patch = patch
|
||
if "content_language" in patch:
|
||
item.content_language = _coerce_str(patch.get("content_language")) or item.content_language
|
||
if "localizations" in patch:
|
||
item.localizations = _normalize_localizations(patch.get("localizations"))
|
||
if "enrichment_status" in patch:
|
||
item.enrichment_status = _coerce_str(patch.get("enrichment_status")) or item.enrichment_status
|
||
if "enrichment_error" in patch:
|
||
item.enrichment_error = _coerce_str(patch.get("enrichment_error"))
|
||
if "enriched_at" in patch:
|
||
item.enriched_at = _parse_datetime(_coerce_str(patch.get("enriched_at")))
|
||
return item
|
||
|
||
|
||
async def _geocode_target_location(query: str) -> dict[str, Any] | None:
|
||
return await asyncio.to_thread(_news_target_geocode, query)
|
||
|
||
|
||
async def _build_target_location_from_payload(
|
||
payload: dict[str, Any],
|
||
) -> NewsTargetLocation | None:
|
||
country = normalize_country(payload.get("country"))
|
||
city = _coerce_str(payload.get("city"))
|
||
matched_location_name = _coerce_str(payload.get("matched_location_name"))
|
||
confidence = _coerce_float(payload.get("confidence"))
|
||
if confidence is not None:
|
||
confidence = max(0.0, min(confidence, 1.0))
|
||
|
||
latitude = _coerce_float(payload.get("latitude"))
|
||
longitude = _coerce_float(payload.get("longitude"))
|
||
if latitude not in (None, 0.0) and longitude not in (None, 0.0):
|
||
label = matched_location_name or ", ".join(part for part in (city, country) if part) or "关联位置"
|
||
return NewsTargetLocation(
|
||
latitude=latitude,
|
||
longitude=longitude,
|
||
label=label,
|
||
source="ai_inferred_target",
|
||
confidence=confidence,
|
||
country=country,
|
||
city=city,
|
||
)
|
||
|
||
geocode_queries: list[str] = []
|
||
for value in (
|
||
", ".join(part for part in (city, country) if part),
|
||
matched_location_name,
|
||
city,
|
||
country,
|
||
):
|
||
normalized = _coerce_str(value)
|
||
if normalized and normalized not in geocode_queries:
|
||
geocode_queries.append(normalized)
|
||
|
||
for query in geocode_queries:
|
||
try:
|
||
result = await _geocode_target_location(query)
|
||
except Exception:
|
||
continue
|
||
if not isinstance(result, dict):
|
||
continue
|
||
latitude = _coerce_float(result.get("lat"))
|
||
longitude = _coerce_float(result.get("lon"))
|
||
if latitude in (None, 0.0) or longitude in (None, 0.0):
|
||
continue
|
||
label = (
|
||
_coerce_str(result.get("display_name"))
|
||
or matched_location_name
|
||
or ", ".join(part for part in (city, country) if part)
|
||
or query
|
||
)
|
||
return NewsTargetLocation(
|
||
latitude=latitude,
|
||
longitude=longitude,
|
||
label=label,
|
||
source="ai_inferred_target",
|
||
confidence=confidence,
|
||
country=country,
|
||
city=city,
|
||
)
|
||
|
||
centroid = get_country_centroid(country)
|
||
if centroid:
|
||
label = matched_location_name or city or country or "关联位置"
|
||
return NewsTargetLocation(
|
||
latitude=centroid["latitude"],
|
||
longitude=centroid["longitude"],
|
||
label=label,
|
||
source="ai_inferred_target",
|
||
confidence=confidence,
|
||
country=country,
|
||
city=city,
|
||
)
|
||
|
||
return None
|
||
|
||
|
||
async def _extract_target_location_from_text(item: ParsedNewsItem) -> NewsTargetLocation | None:
|
||
combined_text = " ".join(part for part in (item.title, item.summary) if part).strip()
|
||
if not combined_text:
|
||
return None
|
||
|
||
for hint in _CITY_HINTS:
|
||
city_name = _coerce_str(hint.get("name"))
|
||
if not city_name or not _contains_location_alias(combined_text, city_name):
|
||
continue
|
||
country = normalize_country(hint.get("country"))
|
||
geocode_query = ", ".join(part for part in (city_name, country) if part)
|
||
try:
|
||
result = await _geocode_target_location(geocode_query)
|
||
except Exception:
|
||
result = None
|
||
if isinstance(result, dict):
|
||
latitude = _coerce_float(result.get("lat"))
|
||
longitude = _coerce_float(result.get("lon"))
|
||
if latitude not in (None, 0.0) and longitude not in (None, 0.0):
|
||
return NewsTargetLocation(
|
||
latitude=latitude,
|
||
longitude=longitude,
|
||
label=_coerce_str(result.get("display_name")) or geocode_query,
|
||
source="headline_location_hint",
|
||
confidence=0.78,
|
||
country=country,
|
||
city=city_name,
|
||
)
|
||
centroid = get_country_centroid(country)
|
||
if centroid:
|
||
return NewsTargetLocation(
|
||
latitude=centroid["latitude"],
|
||
longitude=centroid["longitude"],
|
||
label=geocode_query,
|
||
source="headline_location_hint",
|
||
confidence=0.68,
|
||
country=country,
|
||
city=city_name,
|
||
)
|
||
|
||
for canonical, variants in COUNTRY_VARIANTS_MAP.items():
|
||
if not get_country_centroid(canonical):
|
||
continue
|
||
searchable_variants = _iter_searchable_country_variants(canonical, variants)
|
||
if not any(_contains_location_alias(combined_text, variant) for variant in searchable_variants):
|
||
continue
|
||
centroid = get_country_centroid(canonical)
|
||
if not centroid:
|
||
continue
|
||
return NewsTargetLocation(
|
||
latitude=centroid["latitude"],
|
||
longitude=centroid["longitude"],
|
||
label=canonical,
|
||
source="headline_country_hint",
|
||
confidence=0.62,
|
||
country=canonical,
|
||
city=None,
|
||
)
|
||
|
||
return None
|
||
|
||
|
||
async def _infer_news_target_location(
|
||
item: ParsedNewsItem,
|
||
*,
|
||
provider_client: AIProviderClient | None,
|
||
prompt: EffectiveAIPrompt | None = None,
|
||
) -> NewsTargetLocation | None:
|
||
target, _localizations = await _infer_news_enrichment(
|
||
item,
|
||
provider_client=provider_client,
|
||
prompt=prompt,
|
||
)
|
||
return target
|
||
|
||
|
||
async def _infer_news_enrichment(
|
||
item: ParsedNewsItem,
|
||
*,
|
||
provider_client: AIProviderClient | None,
|
||
prompt: EffectiveAIPrompt | None = None,
|
||
) -> tuple[NewsTargetLocation | None, dict[str, dict[str, str]]]:
|
||
text_hint = await _extract_target_location_from_text(item)
|
||
content_error: str | None = None
|
||
if text_hint is not None and text_hint.city:
|
||
item.target_resolution_stage = text_hint.source
|
||
item.target_ai_attempted = False
|
||
item.target_ai_status = "skipped_text_hint"
|
||
item.target_ai_error = None
|
||
item.target_debug_note = f"text hint matched {text_hint.label}"
|
||
localizations: dict[str, dict[str, str]] = {}
|
||
|
||
if provider_client is None:
|
||
if text_hint is None or not text_hint.city:
|
||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||
item.target_ai_attempted = False
|
||
item.target_ai_status = "unavailable"
|
||
item.target_ai_error = "AI provider is not configured or unavailable for earth-feed."
|
||
item.target_debug_note = (
|
||
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
|
||
)
|
||
item.enrichment_status = "unavailable"
|
||
item.enrichment_error = "AI provider is not configured or unavailable for earth-feed."
|
||
return text_hint, localizations
|
||
|
||
if text_hint is None or not text_hint.city:
|
||
item.target_ai_attempted = True
|
||
item.target_ai_status = "attempted"
|
||
item.target_ai_error = None
|
||
item.target_debug_note = (
|
||
f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
|
||
)
|
||
item.enrichment_status = "attempted"
|
||
item.enrichment_error = None
|
||
|
||
prompt = prompt or await get_effective_prompt(None, NEWS_ENRICH_PROMPT_KEY)
|
||
target_locale = _target_localization_locale(item)
|
||
target_locale_name = "English" if target_locale == "en-US" else "Simplified Chinese"
|
||
request = SituationalAnalysisRequest(
|
||
title=f"Enrich Earth news item with event location and {target_locale} content",
|
||
objective=prompt.prompt,
|
||
system_prompt=prompt.system_prompt or None,
|
||
context={
|
||
"news_item": {
|
||
"title": item.title,
|
||
"summary": item.summary,
|
||
"content_language": item.content_language,
|
||
"source": item.source,
|
||
"feed_name": item.feed_name,
|
||
"feed_id": item.feed_id,
|
||
"feed_type": item.feed_type,
|
||
"feed_region": item.feed_region,
|
||
"url": item.url,
|
||
"published_at": (
|
||
item.published_at.isoformat().replace("+00:00", "Z")
|
||
if item.published_at
|
||
else None
|
||
),
|
||
},
|
||
"required_json_schema": {
|
||
"location": {
|
||
"country": "string|null",
|
||
"city": "string|null",
|
||
"matched_location_name": "string|null",
|
||
"latitude": "number|null",
|
||
"longitude": "number|null",
|
||
"confidence": "number from 0 to 1",
|
||
"reasoning_summary": "short string",
|
||
},
|
||
"localizations": _target_locale_schema(target_locale),
|
||
},
|
||
},
|
||
constraints=[
|
||
"Return only strict JSON. Do not wrap it in markdown.",
|
||
"For localizations, do not add facts that are absent from the RSS headline, description, source, or date.",
|
||
f"Write {target_locale} summary as one concise {target_locale_name} newswire-style sentence, like a breaking-news lead.",
|
||
"If the RSS description is thin, write a conservative one-sentence summary that says only what is supported.",
|
||
f"Keep {target_locale} summary factual, non-promotional, and avoid colon-heavy keyword labels.",
|
||
"If the source content is Chinese and target locale is en-US, translate faithfully into English instead of rewriting the story.",
|
||
"Prefer the event location, not the newsroom or publisher headquarters.",
|
||
"When a country visit or summit is the clear topic but the city is omitted, use the most likely host city only if it is broadly public knowledge.",
|
||
"Use null for unknown fields instead of inventing details.",
|
||
"Calibrate confidence conservatively: 0.75+ only when the city is strongly supported, 0.55-0.74 for country-level or likely city inference, below 0.55 when weak.",
|
||
],
|
||
)
|
||
try:
|
||
response = await provider_client.analyze(request)
|
||
except Exception as exc:
|
||
if text_hint is None or not text_hint.city:
|
||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||
item.target_ai_status = "provider_error"
|
||
item.target_ai_error = str(exc)
|
||
item.enrichment_status = "provider_error"
|
||
item.enrichment_error = str(exc)
|
||
return text_hint, localizations
|
||
|
||
payload = _first_json_object(response.content)
|
||
if not isinstance(payload, dict):
|
||
if text_hint is None or not text_hint.city:
|
||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||
item.target_ai_status = "parse_error"
|
||
item.target_ai_error = "AI response did not contain a parseable JSON object."
|
||
item.enrichment_status = "parse_error"
|
||
item.enrichment_error = "AI response did not contain a parseable JSON object."
|
||
return text_hint, localizations
|
||
|
||
localizations = _normalize_localizations(payload.get("localizations"))
|
||
if not localizations:
|
||
content_error = "AI returned no usable localizations."
|
||
|
||
location_payload = payload.get("location") if isinstance(payload.get("location"), dict) else payload
|
||
if text_hint is not None and text_hint.city:
|
||
target = text_hint
|
||
else:
|
||
target = await _build_target_location_from_payload(location_payload)
|
||
if target is None:
|
||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||
item.target_ai_status = "no_result"
|
||
item.target_ai_error = "AI returned no usable target coordinates or geocodeable location."
|
||
target = text_hint
|
||
elif target.confidence is not None and target.confidence < 0.45:
|
||
item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
|
||
item.target_ai_status = "low_confidence"
|
||
item.target_ai_error = f"AI target confidence too low: {target.confidence:.2f}"
|
||
target = text_hint
|
||
else:
|
||
item.target_resolution_stage = target.source
|
||
item.target_ai_status = "success"
|
||
item.target_ai_error = None
|
||
item.target_debug_note = f"ai inferred {target.label}"
|
||
|
||
item.localizations = {**dict(item.localizations or {}), **localizations}
|
||
if localizations and item.target_ai_status in {"success", "skipped_text_hint"}:
|
||
item.enrichment_status = "success"
|
||
item.enrichment_error = None
|
||
elif localizations:
|
||
item.enrichment_status = "content_only"
|
||
item.enrichment_error = item.target_ai_error
|
||
else:
|
||
item.enrichment_status = "location_only" if target is not None else "no_result"
|
||
item.enrichment_error = content_error or item.target_ai_error
|
||
item.enriched_at = datetime.now(UTC) if localizations else None
|
||
return target, localizations
|
||
|
||
|
||
async def _enrich_items_with_target_locations(
|
||
items: list[ParsedNewsItem],
|
||
*,
|
||
provider_client: AIProviderClient | None,
|
||
prompt: EffectiveAIPrompt | None = None,
|
||
) -> list[ParsedNewsItem]:
|
||
if not items:
|
||
return items
|
||
|
||
semaphore = asyncio.Semaphore(MAX_TARGET_INFERENCE_CONCURRENCY)
|
||
|
||
async def enrich(item: ParsedNewsItem) -> ParsedNewsItem:
|
||
async with semaphore:
|
||
target = await _infer_news_target_location(
|
||
item,
|
||
provider_client=provider_client,
|
||
prompt=prompt,
|
||
)
|
||
item.target_location = target
|
||
return item
|
||
|
||
return list(await asyncio.gather(*(enrich(item) for item in items)))
|
||
|
||
|
||
def get_sources_for_region(region: str) -> list[NewsFeedSource]:
|
||
return sorted(
|
||
[
|
||
source
|
||
for source in NEWS_FEED_SOURCES
|
||
if source.enabled
|
||
and source.source_type in {"rss", "atom", "aggregated"}
|
||
and _source_matches_active_region(source, region)
|
||
],
|
||
key=lambda source: (source.priority, source.name),
|
||
)
|
||
|
||
|
||
def _clean_feed_urls(*values: Any) -> tuple[str, ...]:
|
||
urls: list[str] = []
|
||
for value in values:
|
||
if isinstance(value, str):
|
||
parts = re.split(r"[\n,]+", value)
|
||
elif isinstance(value, (list, tuple)):
|
||
parts = [str(item) for item in value]
|
||
else:
|
||
parts = []
|
||
for part in parts:
|
||
url = str(part or "").strip()
|
||
if url and url not in urls:
|
||
urls.append(url)
|
||
return tuple(urls)
|
||
|
||
|
||
def _clean_feed_tags(value: Any) -> tuple[str, ...]:
|
||
if isinstance(value, str):
|
||
parts = re.split(r"[\n,，]+", value)
|
||
elif isinstance(value, (list, tuple)):
|
||
parts = [str(item) for item in value]
|
||
else:
|
||
parts = []
|
||
tags: list[str] = []
|
||
for part in parts:
|
||
tag = str(part or "").strip()
|
||
if tag and tag not in tags:
|
||
tags.append(tag)
|
||
return tuple(tags)
|
||
|
||
|
||
def _slug_feed_id(value: str, fallback: str) -> str:
|
||
text_value = str(value or "").strip().lower()
|
||
slug = re.sub(r"[^a-z0-9_-]+", "-", text_value).strip("-")
|
||
return slug or fallback
|
||
|
||
|
||
def _feed_from_config(raw: Any, *, source: NewsFeedSource | None = None, index: int = 0) -> NewsFeedEndpoint | None:
|
||
if not isinstance(raw, dict):
|
||
return None
|
||
url = str(raw.get("url") or raw.get("feed_url") or "").strip()
|
||
if not url:
|
||
return None
|
||
feed_id = _slug_feed_id(str(raw.get("id") or raw.get("key") or raw.get("name") or ""), f"feed-{index + 1}")
|
||
fallback_type = source.source_type if source else "rss"
|
||
feed_type = str(raw.get("type") or raw.get("source_type") or fallback_type).strip().lower() or "rss"
|
||
default_category = str(raw.get("default_category") or (source.default_category if source else "other") or "other").strip() or "other"
|
||
try:
|
||
priority = int(raw.get("priority", index + 1))
|
||
except (TypeError, ValueError):
|
||
priority = index + 1
|
||
return NewsFeedEndpoint(
|
||
id=feed_id,
|
||
name=str(raw.get("name") or raw.get("label") or feed_id).strip() or feed_id,
|
||
url=url,
|
||
type=feed_type,
|
||
region=str(raw.get("region") or (source.region if source else "") or "").strip(),
|
||
enabled=raw.get("enabled") is not False and feed_type in {"rss", "atom", "aggregated"},
|
||
default_category=default_category,
|
||
tags=_clean_feed_tags(raw.get("tags")),
|
||
priority=priority,
|
||
)
|
||
|
||
|
||
def _source_feed_urls(source: NewsFeedSource) -> tuple[str, ...]:
|
||
return tuple(feed.url for feed in _source_feeds(source))
|
||
|
||
|
||
def _source_feeds(source: NewsFeedSource) -> tuple[NewsFeedEndpoint, ...]:
|
||
if source.source_type == "reference":
|
||
return tuple()
|
||
feeds = tuple(feed for feed in source.feeds if feed.url)
|
||
if feeds:
|
||
return feeds
|
||
urls = _clean_feed_urls(source.feed_urls, source.feed_url)
|
||
return tuple(
|
||
NewsFeedEndpoint(
|
||
id=f"feed-{index + 1}",
|
||
name=source.name if len(urls) == 1 else f"{source.name} {index + 1}",
|
||
url=url,
|
||
type=source.source_type,
|
||
region=source.region,
|
||
enabled=source.enabled and source.source_type in {"rss", "atom", "aggregated"},
|
||
default_category=source.default_category,
|
||
priority=index + 1,
|
||
)
|
||
for index, url in enumerate(urls)
|
||
)
|
||
|
||
|
||
def _feed_matches_active_region(feed: NewsFeedEndpoint, active_region: str | None) -> bool:
|
||
return (
|
||
active_region is None
|
||
or active_region == "global"
|
||
or not feed.region
|
||
or feed.region in {"global", active_region}
|
||
)
|
||
|
||
|
||
def _source_matches_active_region(source: NewsFeedSource, active_region: str) -> bool:
|
||
return active_region == "global" or source.region in {"global", active_region}
|
||
|
||
|
||
def _expected_feed_keys(sources: list[NewsFeedSource], *, active_region: str) -> set[tuple[str, str]]:
|
||
return {
|
||
(source.id, feed.id)
|
||
for source in sources
|
||
for feed in _source_feeds(source)
|
||
if source.enabled
|
||
and source.source_type in {"rss", "atom", "aggregated"}
|
||
and feed.enabled
|
||
and feed.type in {"rss", "atom", "aggregated"}
|
||
and _feed_matches_active_region(feed, active_region)
|
||
}
|
||
|
||
|
||
def _serialize_feed_endpoint(feed: NewsFeedEndpoint) -> dict[str, Any]:
|
||
return {
|
||
"id": feed.id,
|
||
"name": feed.name,
|
||
"url": feed.url,
|
||
"type": feed.type,
|
||
"region": feed.region,
|
||
"enabled": feed.enabled and feed.type in {"rss", "atom", "aggregated"},
|
||
"default_category": feed.default_category,
|
||
"tags": list(feed.tags),
|
||
"priority": feed.priority,
|
||
}
|
||
|
||
|
||
def _should_repair_builtin_source_feeds(
|
||
source_id: str,
|
||
*,
|
||
raw_feeds: list[Any],
|
||
feed_urls: tuple[str, ...],
|
||
feed_url: str,
|
||
homepage_url: str,
|
||
feed_directory_url: str,
|
||
) -> bool:
|
||
default_source = DEFAULT_NEWS_SOURCE_BY_ID.get(source_id)
|
||
if not default_source or not default_source.feeds:
|
||
return False
|
||
if not raw_feeds:
|
||
return True
|
||
|
||
current_urls: set[str] = set(feed_urls)
|
||
if feed_url:
|
||
current_urls.add(feed_url)
|
||
for raw_feed in raw_feeds:
|
||
if isinstance(raw_feed, dict):
|
||
current_url = str(raw_feed.get("url") or raw_feed.get("feed_url") or "").strip()
|
||
if current_url:
|
||
current_urls.add(current_url)
|
||
|
||
non_feed_urls = {
|
||
url
|
||
for url in (
|
||
homepage_url,
|
||
feed_directory_url,
|
||
default_source.homepage_url,
|
||
default_source.feed_directory_url,
|
||
)
|
||
if url
|
||
}
|
||
return bool(current_urls & non_feed_urls)
|
||
|
||
|
||
def serialize_news_source_config(source: NewsFeedSource) -> dict[str, Any]:
|
||
feeds = _source_feeds(source)
|
||
feed_urls = tuple(feed.url for feed in feeds)
|
||
return {
|
||
"id": source.id,
|
||
"name": source.name,
|
||
"region": source.region,
|
||
"feed_url": source.feed_url or (feed_urls[0] if feed_urls else ""),
|
||
"feed_urls": list(feed_urls),
|
||
"feeds": [_serialize_feed_endpoint(feed) for feed in feeds],
|
||
"homepage_url": source.homepage_url,
|
||
"feed_directory_url": source.feed_directory_url,
|
||
"source_type": source.source_type,
|
||
"priority": source.priority,
|
||
"enabled": source.enabled and source.source_type in {"rss", "atom", "aggregated"},
|
||
"source_tags": list(source.source_tags),
|
||
"default_category": source.default_category,
|
||
"importance_weight": source.importance_weight,
|
||
"health_policy": dict(source.health_policy or DEFAULT_NEWS_HEALTH_POLICY),
|
||
}
|
||
|
||
|
||
def _source_from_config(payload: dict[str, Any]) -> NewsFeedSource | None:
|
||
source_id = str(payload.get("id") or "").strip()
|
||
name = str(payload.get("name") or "").strip()
|
||
source_type = str(payload.get("source_type") or payload.get("type") or "rss").strip().lower() or "rss"
|
||
default_source = DEFAULT_NEWS_SOURCE_BY_ID.get(source_id)
|
||
homepage_url = str(payload.get("homepage_url") or "").strip()
|
||
feed_directory_url = str(payload.get("feed_directory_url") or "").strip()
|
||
raw_feeds = payload.get("feeds") if isinstance(payload.get("feeds"), list) else []
|
||
feed_urls = _clean_feed_urls(payload.get("feed_urls"), payload.get("feed_url"))
|
||
feed_url = str(payload.get("feed_url") or "").strip() or (feed_urls[0] if feed_urls else "")
|
||
enabled_override: bool | None = None
|
||
if default_source:
|
||
homepage_url = homepage_url or default_source.homepage_url
|
||
feed_directory_url = feed_directory_url or default_source.feed_directory_url
|
||
if source_type == "reference" and default_source.source_type in {"rss", "atom", "aggregated"}:
|
||
source_type = default_source.source_type
|
||
if not raw_feeds:
|
||
enabled_override = default_source.enabled
|
||
if _should_repair_builtin_source_feeds(
|
||
source_id,
|
||
raw_feeds=raw_feeds,
|
||
feed_urls=feed_urls,
|
||
feed_url=feed_url,
|
||
homepage_url=homepage_url,
|
||
feed_directory_url=feed_directory_url,
|
||
):
|
||
raw_feeds = [_serialize_feed_endpoint(feed) for feed in default_source.feeds]
|
||
feed_urls = tuple(feed.url for feed in default_source.feeds)
|
||
feed_url = default_source.feed_url or (feed_urls[0] if feed_urls else "")
|
||
if not feed_url and source_type == "reference":
|
||
feed_url = homepage_url
|
||
feed_urls = _clean_feed_urls(feed_url)
|
||
if not source_id or not name or (not feed_url and not raw_feeds):
|
||
return None
|
||
try:
|
||
priority = int(payload.get("priority", default_source.priority if default_source else 100))
|
||
except (TypeError, ValueError):
|
||
priority = default_source.priority if default_source else 100
|
||
try:
|
||
importance_weight = int(payload.get("importance_weight", default_source.importance_weight if default_source else 0))
|
||
except (TypeError, ValueError):
|
||
importance_weight = default_source.importance_weight if default_source else 0
|
||
raw_tags = payload.get("source_tags")
|
||
source_tags = (
|
||
tuple(str(tag).strip() for tag in raw_tags if str(tag).strip())
|
||
if isinstance(raw_tags, list)
|
||
else (default_source.source_tags if default_source else ())
|
||
)
|
||
health_policy = payload.get("health_policy") if isinstance(payload.get("health_policy"), dict) else {}
|
||
base_source = NewsFeedSource(
|
||
id=source_id,
|
||
name=name,
|
||
region=str(payload.get("region") or "global").strip() or "global",
|
||
feed_url=feed_url,
|
||
homepage_url=homepage_url,
|
||
feed_directory_url=feed_directory_url,
|
||
source_type=source_type,
|
||
priority=priority,
|
||
enabled=(enabled_override if enabled_override is not None else payload.get("enabled") is not False)
|
||
and source_type in {"rss", "atom", "aggregated"},
|
||
source_tags=source_tags,
|
||
default_category=str(payload.get("default_category") or (default_source.default_category if default_source else "other") or "other").strip() or "other",
|
||
importance_weight=importance_weight,
|
||
health_policy={**DEFAULT_NEWS_HEALTH_POLICY, **health_policy},
|
||
)
|
||
feeds = tuple(
|
||
feed
|
||
for index, raw_feed in enumerate(raw_feeds)
|
||
for feed in [_feed_from_config(raw_feed, source=base_source, index=index)]
|
||
if feed is not None
|
||
)
|
||
return NewsFeedSource(
|
||
**{
|
||
**base_source.__dict__,
|
||
"feed_urls": feed_urls or ((feed_url,) if feed_url else tuple()),
|
||
"feeds": feeds,
|
||
}
|
||
)
|
||
|
||
|
||
def _merge_legacy_google_news_sources(sources: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
has_legacy_google = any(str(source.get("id") or "") in LEGACY_GOOGLE_NEWS_SOURCE_IDS for source in sources)
|
||
if not has_legacy_google:
|
||
return sources
|
||
|
||
google_default = DEFAULT_NEWS_SOURCE_BY_ID.get("google-news")
|
||
if google_default is None:
|
||
return sources
|
||
|
||
merged = [source for source in sources if str(source.get("id") or "") not in LEGACY_GOOGLE_NEWS_SOURCE_IDS]
|
||
if not any(str(source.get("id") or "") == "google-news" for source in merged):
|
||
merged.append(serialize_news_source_config(google_default))
|
||
return sorted(merged, key=lambda source: (int(source.get("priority") or 100), str(source.get("name") or "")))
|
||
|
||
|
||
def normalize_earth_news_sources_payload(payload: dict[str, Any] | None) -> dict[str, Any]:
|
||
defaults = default_earth_news_sources_payload()
|
||
if not isinstance(payload, dict):
|
||
return defaults
|
||
|
||
normalized = {
|
||
"cache_version": int(payload.get("cache_version") or defaults["cache_version"]),
|
||
"source_tags": payload.get("source_tags") if isinstance(payload.get("source_tags"), list) else defaults["source_tags"],
|
||
"categories": payload.get("categories") if isinstance(payload.get("categories"), list) else defaults["categories"],
|
||
"item_tag_rules": payload.get("item_tag_rules") if isinstance(payload.get("item_tag_rules"), list) else defaults["item_tag_rules"],
|
||
"health": payload.get("health") if isinstance(payload.get("health"), dict) else {},
|
||
"sources": [],
|
||
}
|
||
for source in payload.get("sources") if isinstance(payload.get("sources"), list) else defaults["sources"]:
|
||
if isinstance(source, dict):
|
||
coerced = _source_from_config(source)
|
||
if coerced:
|
||
normalized["sources"].append(serialize_news_source_config(coerced))
|
||
if not normalized["sources"]:
|
||
normalized["sources"] = defaults["sources"]
|
||
else:
|
||
normalized["sources"] = _merge_legacy_google_news_sources(normalized["sources"])
|
||
return normalized
|
||
|
||
|
||
async def get_earth_news_sources_payload(db: AsyncSession | None) -> dict[str, Any]:
|
||
if db is None:
|
||
payload = default_earth_news_sources_payload()
|
||
payload["is_default"] = True
|
||
return payload
|
||
result = await db.execute(select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_SOURCES_CATEGORY))
|
||
record = result.scalar_one_or_none()
|
||
payload = normalize_earth_news_sources_payload(record.payload if record else None)
|
||
payload["is_default"] = record is None
|
||
return payload
|
||
|
||
|
||
async def save_earth_news_sources_payload(db: AsyncSession, payload: dict[str, Any]) -> dict[str, Any]:
|
||
normalized = normalize_earth_news_sources_payload(payload)
|
||
normalized["cache_version"] = int(normalized.get("cache_version") or 1) + 1
|
||
result = await db.execute(select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_SOURCES_CATEGORY))
|
||
record = result.scalar_one_or_none()
|
||
if record is None:
|
||
record = SystemSetting(category=EARTH_NEWS_SOURCES_CATEGORY, payload=normalized)
|
||
db.add(record)
|
||
else:
|
||
record.payload = normalized
|
||
await db.commit()
|
||
await db.refresh(record)
|
||
clear_earth_news_region_cache()
|
||
return {**normalize_earth_news_sources_payload(record.payload), "is_default": False}
|
||
|
||
|
||
async def reset_earth_news_sources_payload(db: AsyncSession) -> dict[str, Any]:
|
||
result = await db.execute(select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_SOURCES_CATEGORY))
|
||
record = result.scalar_one_or_none()
|
||
if record is not None:
|
||
await db.delete(record)
|
||
await db.commit()
|
||
clear_earth_news_region_cache()
|
||
payload = default_earth_news_sources_payload()
|
||
payload["is_default"] = True
|
||
return payload
|
||
|
||
|
||
async def record_earth_news_source_health(
|
||
db: AsyncSession | None,
|
||
source_id: str,
|
||
health: dict[str, Any],
|
||
) -> None:
|
||
if db is None or not source_id or not callable(getattr(db, "execute", None)):
|
||
return
|
||
result = await db.execute(select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_SOURCES_CATEGORY))
|
||
record = result.scalar_one_or_none()
|
||
payload = normalize_earth_news_sources_payload(record.payload if record else None)
|
||
payload_health = payload.get("health") if isinstance(payload.get("health"), dict) else {}
|
||
payload_health[source_id] = health
|
||
payload["health"] = payload_health
|
||
if record is None:
|
||
record = SystemSetting(category=EARTH_NEWS_SOURCES_CATEGORY, payload=payload)
|
||
db.add(record)
|
||
else:
|
||
record.payload = payload
|
||
await db.commit()
|
||
|
||
|
||
async def record_earth_news_sources_health(
|
||
db: AsyncSession | None,
|
||
health_by_source: dict[str, dict[str, Any]],
|
||
) -> None:
|
||
if db is None or not health_by_source or not callable(getattr(db, "execute", None)):
|
||
return
|
||
result = await db.execute(select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_SOURCES_CATEGORY))
|
||
record = result.scalar_one_or_none()
|
||
payload = normalize_earth_news_sources_payload(record.payload if record else None)
|
||
payload_health = payload.get("health") if isinstance(payload.get("health"), dict) else {}
|
||
payload_health.update(health_by_source)
|
||
payload["health"] = payload_health
|
||
if record is None:
|
||
record = SystemSetting(category=EARTH_NEWS_SOURCES_CATEGORY, payload=payload)
|
||
db.add(record)
|
||
else:
|
||
record.payload = payload
|
||
await db.commit()
|
||
|
||
|
||
async def get_configured_sources_for_region(db: AsyncSession | None, region: str) -> list[NewsFeedSource]:
|
||
if db is None:
|
||
return get_sources_for_region(region)
|
||
payload = await get_earth_news_sources_payload(db)
|
||
sources = [
|
||
source
|
||
for raw in payload.get("sources", [])
|
||
if isinstance(raw, dict)
|
||
for source in [_source_from_config(raw)]
|
||
if source
|
||
]
|
||
enabled = [
|
||
source
|
||
for source in sources
|
||
if source.enabled
|
||
and source.source_type in {"rss", "atom", "aggregated"}
|
||
and _source_matches_active_region(source, region)
|
||
]
|
||
return sorted(enabled, key=lambda source: (source.priority, source.name))
|
||
|
||
|
||
def _source_health_result(
|
||
source: NewsFeedSource,
|
||
*,
|
||
ok: bool,
|
||
count: int,
|
||
error: str | None = None,
|
||
status: str | None = None,
|
||
status_code: int | None = None,
|
||
content_type: str | None = None,
|
||
latency_ms: int | None = None,
|
||
final_url: str | None = None,
|
||
feed: NewsFeedEndpoint | None = None,
|
||
) -> dict[str, Any]:
|
||
return {
|
||
"source_id": source.id,
|
||
"feed_id": feed.id if feed else "",
|
||
"feed_name": feed.name if feed else "",
|
||
"feed_type": feed.type if feed else source.source_type,
|
||
"feed_url": feed.url if feed else source.feed_url,
|
||
"ok": ok,
|
||
"status": status or ("ok" if ok else "failed"),
|
||
"count": int(count or 0),
|
||
"item_count": int(count or 0),
|
||
"error": error,
|
||
"status_code": status_code,
|
||
"content_type": content_type or "",
|
||
"latency_ms": latency_ms,
|
||
"final_url": final_url or (feed.url if feed else source.feed_url),
|
||
"fetched_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
|
||
}
|
||
|
||
|
||
def _format_source_fetch_error(
|
||
*,
|
||
status_code: int | None,
|
||
content_type: str,
|
||
body: str,
|
||
parsed_count: int,
|
||
) -> tuple[str | None, str]:
|
||
if status_code is not None and status_code >= 400:
|
||
return f"HTTP {status_code}", "http_error"
|
||
lower_content_type = content_type.lower()
|
||
body_prefix = body[:500].lower()
|
||
looks_like_html = "text/html" in lower_content_type or "<html" in body_prefix or "<!doctype html" in body_prefix
|
||
if looks_like_html:
|
||
return "返回 HTML 页面，不是可解析的 RSS/Atom XML。", "format_error"
|
||
if parsed_count <= 0:
|
||
return "未解析到 RSS/Atom 条目，请检查 Feed 地址或源格式。", "empty"
|
||
return None, "ok"
|
||
|
||
|
||
async def test_news_source_config(raw_source: dict[str, Any], db: AsyncSession | None = None) -> dict[str, Any]:
|
||
source = _source_from_config(raw_source)
|
||
if source is None:
|
||
return {"ok": False, "count": 0, "error": "新闻源缺少 id、名称或 URL"}
|
||
if source.source_type not in {"rss", "atom", "aggregated"}:
|
||
health = _source_health_result(
|
||
source,
|
||
ok=False,
|
||
count=0,
|
||
error="参考链接仅用于记录官网、报告页或未来采集器线索，不参与 RSS/Atom 抓取。",
|
||
status="reference",
|
||
)
|
||
await record_earth_news_source_health(db, source.id, health)
|
||
return {
|
||
"ok": False,
|
||
"count": 0,
|
||
"error": health["error"],
|
||
"health": health,
|
||
"source": serialize_news_source_config(source),
|
||
}
|
||
async with httpx.AsyncClient(
|
||
timeout=float(source.health_policy.get("timeout_seconds", REQUEST_TIMEOUT)),
|
||
follow_redirects=True,
|
||
headers={"User-Agent": USER_AGENT},
|
||
) as client:
|
||
fetched_source, items, error, health = await _fetch_source(client, source)
|
||
del fetched_source
|
||
await record_earth_news_source_health(db, source.id, health)
|
||
return {
|
||
"ok": error is None,
|
||
"count": len(items),
|
||
"item_count": len(items),
|
||
"error": error,
|
||
"health": health,
|
||
"source": serialize_news_source_config(source),
|
||
}
|
||
|
||
|
||
def _strip_html(value: str) -> str:
|
||
if not value:
|
||
return ""
|
||
soup = BeautifulSoup(value, "html.parser")
|
||
return re.sub(r"\s+", " ", soup.get_text(" ", strip=True)).strip()
|
||
|
||
|
||
def _truncate(value: str, limit: int = 180) -> str:
|
||
text = value.strip()
|
||
if len(text) <= limit:
|
||
return text
|
||
return text[: limit - 1].rstrip() + "…"
|
||
|
||
|
||
def _normalize_source_name(raw: str, fallback: str) -> str:
|
||
text = html.unescape((raw or "").strip())
|
||
if " - " in text:
|
||
return text.split(" - ")[-1].strip() or fallback
|
||
return text or fallback
|
||
|
||
|
||
def _parse_datetime(raw: str | None) -> datetime | None:
|
||
if not raw:
|
||
return None
|
||
text = raw.strip()
|
||
if not text:
|
||
return None
|
||
|
||
for parser in (
|
||
lambda value: parsedate_to_datetime(value),
|
||
lambda value: datetime.fromisoformat(value.replace("Z", "+00:00")),
|
||
):
|
||
try:
|
||
parsed = parser(text)
|
||
if parsed.tzinfo is None:
|
||
parsed = parsed.replace(tzinfo=UTC)
|
||
return parsed.astimezone(UTC)
|
||
except Exception:
|
||
continue
|
||
return None
|
||
|
||
|
||
def _extract_item_text(element: ET.Element, *names: str) -> str:
|
||
for name in names:
|
||
node = element.find(name)
|
||
if node is None and not name.startswith("{"):
|
||
node = element.find(f"{{*}}{name}")
|
||
if node is not None and node.text:
|
||
return node.text.strip()
|
||
return ""
|
||
|
||
|
||
def _parse_feed_entries(
|
||
xml_text: str,
|
||
source: NewsFeedSource,
|
||
*,
|
||
feed: NewsFeedEndpoint | None = None,
|
||
config_payload: dict[str, Any] | None = None,
|
||
) -> list[ParsedNewsItem]:
|
||
root = ET.fromstring(xml_text)
|
||
items: list[ParsedNewsItem] = []
|
||
|
||
rss_items = root.findall("./channel/item") or root.findall(".//item") or root.findall(".//{*}item")
|
||
atom_entries = root.findall("{http://www.w3.org/2005/Atom}entry") or root.findall(".//{*}entry")
|
||
nodes = rss_items or atom_entries
|
||
|
||
for node in nodes[:MAX_ITEMS_PER_SOURCE]:
|
||
if node.tag.endswith("entry"):
|
||
title = _extract_item_text(node, "{http://www.w3.org/2005/Atom}title", "title")
|
||
summary = _extract_item_text(
|
||
node,
|
||
"{http://www.w3.org/2005/Atom}summary",
|
||
"{http://www.w3.org/2005/Atom}content",
|
||
"summary",
|
||
"content",
|
||
)
|
||
link_node = node.find("{http://www.w3.org/2005/Atom}link") or node.find("{*}link")
|
||
link = link_node.get("href", "").strip() if link_node is not None else ""
|
||
published = _extract_item_text(
|
||
node,
|
||
"{http://www.w3.org/2005/Atom}updated",
|
||
"{http://www.w3.org/2005/Atom}published",
|
||
"updated",
|
||
"published",
|
||
)
|
||
else:
|
||
title = _extract_item_text(node, "title")
|
||
summary = _extract_item_text(node, "description", "content", "encoded")
|
||
link = _extract_item_text(node, "link")
|
||
published = _extract_item_text(node, "pubDate", "published", "updated")
|
||
|
||
clean_title = html.unescape(title).strip()
|
||
clean_summary = _truncate(_strip_html(summary), 180)
|
||
if not clean_title or not link:
|
||
continue
|
||
|
||
item_source = source.name
|
||
display_title = clean_title
|
||
feed_type = feed.type if feed else source.source_type
|
||
if feed_type == "aggregated" and " - " in clean_title:
|
||
parts = clean_title.rsplit(" - ", 1)
|
||
display_title = parts[0].strip()
|
||
item_source = _normalize_source_name(parts[1], source.name)
|
||
|
||
content_language = _detect_content_language(
|
||
title=display_title,
|
||
summary=clean_summary,
|
||
source=source,
|
||
feed=feed,
|
||
)
|
||
feed_id_segment = f"{feed.id}:" if feed is not None and len(source.feeds or ()) > 1 else ""
|
||
item = ParsedNewsItem(
|
||
id=f"{source.id}:{feed_id_segment}{hashlib.sha1(link.encode('utf-8')).hexdigest()[:12]}",
|
||
title=display_title,
|
||
summary=clean_summary,
|
||
url=link,
|
||
source=item_source,
|
||
feed_name=feed.name if feed else source.name,
|
||
feed_region=(feed.region if feed and feed.region else source.region),
|
||
homepage_url=source.homepage_url,
|
||
published_at=_parse_datetime(published),
|
||
content_language=content_language,
|
||
localizations=_source_language_localizations(
|
||
title=display_title,
|
||
summary=clean_summary,
|
||
content_language=content_language,
|
||
),
|
||
feed_id=feed.id if feed else "",
|
||
feed_type=feed_type,
|
||
feed_default_category=(feed.default_category if feed else source.default_category) or "other",
|
||
source_tags=list(source.source_tags),
|
||
)
|
||
apply_news_classification(item, source, feed=feed, config_payload=config_payload)
|
||
items.append(item)
|
||
|
||
return items
|
||
|
||
|
||
def _normalize_breaking_level(value: str | None) -> str:
|
||
return _normalize_breaking_level_enum(value).value
|
||
|
||
|
||
def _normalize_breaking_scope(value: str | None) -> str:
|
||
return _normalize_breaking_scope_enum(value).value
|
||
|
||
|
||
def apply_news_classification(
|
||
item: ParsedNewsItem,
|
||
source: NewsFeedSource,
|
||
*,
|
||
feed: NewsFeedEndpoint | None = None,
|
||
config_payload: dict[str, Any] | None = None,
|
||
) -> ParsedNewsItem:
|
||
config = normalize_earth_news_sources_payload(config_payload)
|
||
return _apply_news_classification(item, source, feed=feed, config=config)
|
||
|
||
|
||
def _serialize_sources(sources: list[NewsFeedSource]) -> list[dict[str, Any]]:
|
||
return [
|
||
{
|
||
"id": source.id,
|
||
"name": source.name,
|
||
"region": source.region,
|
||
"feed_url": source.feed_url,
|
||
"feed_urls": list(_source_feed_urls(source)),
|
||
"feeds": [_serialize_feed_endpoint(feed) for feed in _source_feeds(source)],
|
||
"homepage_url": source.homepage_url,
|
||
"feed_directory_url": source.feed_directory_url,
|
||
"source_type": source.source_type,
|
||
"priority": source.priority,
|
||
"enabled": source.enabled,
|
||
"source_tags": list(source.source_tags),
|
||
"default_category": source.default_category,
|
||
"importance_weight": source.importance_weight,
|
||
}
|
||
for source in sources
|
||
]
|
||
|
||
|
||
def _serialize_anchor(anchor: RegionAnchor) -> dict[str, Any]:
|
||
return {
|
||
"region": anchor.region,
|
||
"label": anchor.label,
|
||
"latitude": anchor.latitude,
|
||
"longitude": anchor.longitude,
|
||
}
|
||
|
||
|
||
def _serialize_target(target: NewsTargetLocation | None) -> dict[str, Any] | None:
|
||
if target is None:
|
||
return None
|
||
return {
|
||
"latitude": target.latitude,
|
||
"longitude": target.longitude,
|
||
"label": target.label,
|
||
"source": target.source,
|
||
"confidence": target.confidence,
|
||
"country": target.country,
|
||
"city": target.city,
|
||
}
|
||
|
||
|
||
def _serialize_enriched_at(value: datetime | None) -> str | None:
|
||
return value.isoformat().replace("+00:00", "Z") if value else None
|
||
|
||
|
||
def _serialize_breaking_expires_at(value: datetime | None) -> str | None:
|
||
return value.isoformat().replace("+00:00", "Z") if value else None
|
||
|
||
|
||
def _content_patch(item: ParsedNewsItem) -> dict[str, Any]:
|
||
return {
|
||
"content_language": item.content_language,
|
||
"localizations": item.localizations,
|
||
"enrichment_status": item.enrichment_status,
|
||
"enrichment_error": item.enrichment_error,
|
||
"enriched_at": _serialize_enriched_at(item.enriched_at),
|
||
}
|
||
|
||
|
||
def _news_meta_patch(item: ParsedNewsItem) -> dict[str, Any]:
|
||
source_id = item.id.split(":", 1)[0] if ":" in item.id else ""
|
||
return {
|
||
"source_id": source_id,
|
||
"source_tags": list(item.source_tags or []),
|
||
"feed_id": item.feed_id,
|
||
"feed_name": item.feed_name,
|
||
"feed_type": item.feed_type,
|
||
"feed_default_category": item.feed_default_category,
|
||
"category": item.category,
|
||
"item_tags": list(item.item_tags or []),
|
||
"tagging_source": item.tagging_source,
|
||
"tagging_confidence": item.tagging_confidence,
|
||
"importance_score": item.importance_score,
|
||
"importance_level": item.importance_level,
|
||
"importance_reasons": list(item.importance_reasons or []),
|
||
"market_impact": item.market_impact,
|
||
"breaking_level": _normalize_breaking_level(item.breaking_level),
|
||
"breaking_scope": _normalize_breaking_scope(item.breaking_scope),
|
||
"breaking_reasons": list(item.breaking_reasons or []),
|
||
"breaking_source": item.breaking_source,
|
||
"breaking_confidence": item.breaking_confidence,
|
||
"breaking_expires_at": _serialize_breaking_expires_at(item.breaking_expires_at),
|
||
}
|
||
|
||
|
||
def build_anchor_location_patch(
|
||
item: ParsedNewsItem,
|
||
*,
|
||
queued: bool = False,
|
||
queue_available: bool | None = None,
|
||
) -> dict[str, Any]:
|
||
anchor = get_region_anchor(item.feed_region)
|
||
if queued:
|
||
resolution_stage = "queued"
|
||
ai_status = "queued"
|
||
debug_note = "queued for async target location inference"
|
||
else:
|
||
resolution_stage = item.target_resolution_stage
|
||
ai_status = item.target_ai_status
|
||
debug_note = item.target_debug_note
|
||
content_patch = _content_patch(item)
|
||
if queued and content_patch["enrichment_status"] == "pending":
|
||
content_patch["enrichment_status"] = "queued"
|
||
return {
|
||
"latitude": anchor.latitude,
|
||
"longitude": anchor.longitude,
|
||
"location_label": anchor.label,
|
||
"location_source": "region_anchor",
|
||
"verified": False,
|
||
"location_meta": {
|
||
"resolution_stage": resolution_stage,
|
||
"ai_attempted": item.target_ai_attempted,
|
||
"ai_status": ai_status,
|
||
"ai_error": item.target_ai_error,
|
||
"debug_note": debug_note,
|
||
"queue_available": queue_available,
|
||
"target": None,
|
||
"anchor": _serialize_anchor(anchor),
|
||
"news_meta": _news_meta_patch(item),
|
||
},
|
||
**content_patch,
|
||
}
|
||
|
||
|
||
def build_target_location_patch(item: ParsedNewsItem, target: NewsTargetLocation | None) -> dict[str, Any]:
|
||
if target is None:
|
||
return build_anchor_location_patch(item)
|
||
anchor = get_region_anchor(item.feed_region)
|
||
return {
|
||
"latitude": target.latitude,
|
||
"longitude": target.longitude,
|
||
"location_label": target.label,
|
||
"location_source": target.source,
|
||
"verified": True,
|
||
"location_meta": {
|
||
"resolution_stage": item.target_resolution_stage,
|
||
"ai_attempted": item.target_ai_attempted,
|
||
"ai_status": item.target_ai_status,
|
||
"ai_error": item.target_ai_error,
|
||
"debug_note": item.target_debug_note,
|
||
"target": _serialize_target(target),
|
||
"anchor": _serialize_anchor(anchor),
|
||
"news_meta": _news_meta_patch(item),
|
||
},
|
||
**_content_patch(item),
|
||
}
|
||
|
||
|
||
def build_target_location_job_payload(item: ParsedNewsItem) -> dict[str, Any]:
|
||
published_at = item.published_at
|
||
return {
|
||
"id": item.id,
|
||
"title": item.title,
|
||
"summary": item.summary,
|
||
"content_language": item.content_language,
|
||
"localizations": item.localizations,
|
||
"enrichment_status": item.enrichment_status,
|
||
"enrichment_error": item.enrichment_error,
|
||
"enriched_at": _serialize_enriched_at(item.enriched_at),
|
||
"url": item.url,
|
||
"source": item.source,
|
||
"feed_name": item.feed_name,
|
||
"feed_id": item.feed_id,
|
||
"feed_type": item.feed_type,
|
||
"feed_default_category": item.feed_default_category,
|
||
"feed_region": item.feed_region,
|
||
"homepage_url": item.homepage_url,
|
||
"published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None,
|
||
"source_tags": list(item.source_tags or []),
|
||
"category": item.category,
|
||
"item_tags": list(item.item_tags or []),
|
||
"tagging_source": item.tagging_source,
|
||
"tagging_confidence": item.tagging_confidence,
|
||
"importance_score": item.importance_score,
|
||
"importance_level": item.importance_level,
|
||
"importance_reasons": list(item.importance_reasons or []),
|
||
"market_impact": item.market_impact,
|
||
"breaking_level": _normalize_breaking_level(item.breaking_level),
|
||
"breaking_scope": _normalize_breaking_scope(item.breaking_scope),
|
||
"breaking_reasons": list(item.breaking_reasons or []),
|
||
"breaking_source": item.breaking_source,
|
||
"breaking_confidence": item.breaking_confidence,
|
||
"breaking_expires_at": _serialize_breaking_expires_at(item.breaking_expires_at),
|
||
}
|
||
|
||
|
||
def parsed_news_item_from_job_payload(payload: dict[str, Any]) -> ParsedNewsItem:
|
||
return ParsedNewsItem(
|
||
id=str(payload.get("id") or ""),
|
||
title=str(payload.get("title") or ""),
|
||
summary=str(payload.get("summary") or ""),
|
||
content_language=str(payload.get("content_language") or "en"),
|
||
localizations=_normalize_localizations(payload.get("localizations")),
|
||
enrichment_status=str(payload.get("enrichment_status") or "pending"),
|
||
enrichment_error=_coerce_str(payload.get("enrichment_error")),
|
||
enriched_at=_parse_datetime(_coerce_str(payload.get("enriched_at"))),
|
||
url=str(payload.get("url") or ""),
|
||
source=str(payload.get("source") or ""),
|
||
feed_name=str(payload.get("feed_name") or ""),
|
||
feed_id=str(payload.get("feed_id") or ""),
|
||
feed_type=str(payload.get("feed_type") or "rss"),
|
||
feed_default_category=str(payload.get("feed_default_category") or payload.get("category") or "other"),
|
||
feed_region=str(payload.get("feed_region") or "global"),
|
||
homepage_url=str(payload.get("homepage_url") or ""),
|
||
published_at=_parse_datetime(_coerce_str(payload.get("published_at"))),
|
||
source_tags=list(payload.get("source_tags") or []),
|
||
category=str(payload.get("category") or "other"),
|
||
item_tags=list(payload.get("item_tags") or []),
|
||
tagging_source=str(payload.get("tagging_source") or "rules"),
|
||
tagging_confidence=float(payload.get("tagging_confidence") or 0),
|
||
importance_score=int(payload.get("importance_score") or 0),
|
||
importance_level=str(payload.get("importance_level") or "low"),
|
||
importance_reasons=list(payload.get("importance_reasons") or []),
|
||
market_impact=str(payload.get("market_impact") or "none"),
|
||
breaking_level=_normalize_breaking_level(str(payload.get("breaking_level") or "none")),
|
||
breaking_scope=_normalize_breaking_scope(str(payload.get("breaking_scope") or "regional")),
|
||
breaking_reasons=list(payload.get("breaking_reasons") or []),
|
||
breaking_source=str(payload.get("breaking_source") or "rules"),
|
||
breaking_confidence=float(payload.get("breaking_confidence") or 0),
|
||
breaking_expires_at=_parse_datetime(_coerce_str(payload.get("breaking_expires_at"))),
|
||
)
|
||
|
||
|
||
def _serialize_item(item: ParsedNewsItem, *, active_region: str, locale: str = DEFAULT_NEWS_LOCALE) -> dict[str, Any]:
|
||
published_at = item.published_at
|
||
location_patch = item.location_patch or build_target_location_patch(item, item.target_location)
|
||
source_id = item.id.split(":", 1)[0] if ":" in item.id else ""
|
||
return {
|
||
"id": item.id,
|
||
"source_id": source_id,
|
||
"title": item.title,
|
||
"summary": item.summary,
|
||
"content_language": item.content_language,
|
||
"localizations": item.localizations,
|
||
"display_title": _get_locale_text(item, "title", locale=locale),
|
||
"display_summary": _get_locale_text(item, "summary", locale=locale),
|
||
"url": item.url,
|
||
"source": item.source,
|
||
"feed_name": item.feed_name,
|
||
"feed_id": item.feed_id,
|
||
"feed_type": item.feed_type,
|
||
"feed_default_category": item.feed_default_category,
|
||
"region": item.feed_region,
|
||
"display_region": get_region_anchor(item.feed_region).label,
|
||
"homepage_url": item.homepage_url,
|
||
"published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None,
|
||
"latitude": location_patch["latitude"],
|
||
"longitude": location_patch["longitude"],
|
||
"location_label": location_patch["location_label"],
|
||
"location_source": location_patch["location_source"],
|
||
"verified": location_patch["verified"],
|
||
"location_meta": location_patch["location_meta"],
|
||
"enrichment_status": item.enrichment_status,
|
||
"enrichment_error": item.enrichment_error,
|
||
"enriched_at": _serialize_enriched_at(item.enriched_at),
|
||
"is_focus_match": item.feed_region == active_region,
|
||
"source_tags": list(item.source_tags or []),
|
||
"category": item.category,
|
||
"item_tags": list(item.item_tags or []),
|
||
"tagging_source": item.tagging_source,
|
||
"tagging_confidence": item.tagging_confidence,
|
||
"importance_score": item.importance_score,
|
||
"importance_level": item.importance_level,
|
||
"importance_reasons": list(item.importance_reasons or []),
|
||
"market_impact": item.market_impact,
|
||
"breaking_level": _normalize_breaking_level(item.breaking_level),
|
||
"breaking_scope": _normalize_breaking_scope(item.breaking_scope),
|
||
"breaking_reasons": list(item.breaking_reasons or []),
|
||
"breaking_source": item.breaking_source,
|
||
"breaking_confidence": item.breaking_confidence,
|
||
"breaking_expires_at": _serialize_breaking_expires_at(item.breaking_expires_at),
|
||
}
|
||
|
||
|
||
def _build_payload(
|
||
*,
|
||
lat: float | None,
|
||
lon: float | None,
|
||
active_region: str,
|
||
items: list[ParsedNewsItem],
|
||
cruise_items: list[ParsedNewsItem] | None = None,
|
||
sources: list[NewsFeedSource],
|
||
errors: list[str],
|
||
stale: bool,
|
||
categories: set[str] | None = None,
|
||
source_ids: set[str] | None = None,
|
||
limit: int = MAX_ITEMS_TOTAL,
|
||
locale: str = DEFAULT_NEWS_LOCALE,
|
||
generated_at: datetime | None = None,
|
||
) -> dict[str, Any]:
|
||
profile = get_region_profile(active_region)
|
||
timestamp = generated_at or datetime.now(UTC)
|
||
visible_items = list(items)
|
||
cruise_visible_items = list(cruise_items if cruise_items is not None else items)
|
||
highest_breaking_level = _highest_breaking_level(visible_items + cruise_visible_items)
|
||
return {
|
||
"generated_at": timestamp.isoformat().replace("+00:00", "Z"),
|
||
"focus": {
|
||
"lat": lat,
|
||
"lon": lon,
|
||
"region": active_region,
|
||
"label": profile.label,
|
||
"display_region": get_region_anchor(active_region).label,
|
||
"accent": profile.accent,
|
||
},
|
||
"sources": _serialize_sources(sources),
|
||
"filters": {
|
||
"region": active_region,
|
||
"categories": sorted(categories or []),
|
||
"sources": sorted(source_ids or []),
|
||
"limit": limit,
|
||
"locale": locale,
|
||
"has_breaking": highest_breaking_level != "none",
|
||
"highest_breaking_level": highest_breaking_level,
|
||
},
|
||
"items": [_serialize_item(item, active_region=active_region, locale=locale) for item in visible_items],
|
||
"cruise_items": [
|
||
_serialize_item(item, active_region=active_region, locale=locale)
|
||
for item in cruise_visible_items
|
||
],
|
||
"errors": errors,
|
||
"stale": stale,
|
||
}
|
||
|
||
|
||
def _rank_and_trim_items(
|
||
items: list[ParsedNewsItem],
|
||
*,
|
||
active_region: str,
|
||
limit: int = MAX_ITEMS_TOTAL,
|
||
) -> list[ParsedNewsItem]:
|
||
deduped: dict[str, ParsedNewsItem] = {}
|
||
for item in items:
|
||
key = item.url.strip() or item.title.strip().lower()
|
||
if key not in deduped:
|
||
deduped[key] = item
|
||
|
||
return sorted(
|
||
deduped.values(),
|
||
key=lambda item: (
|
||
-_breaking_sort_rank(item),
|
||
False
|
||
if active_region == "global"
|
||
or (_breaking_sort_rank(item) > 0 and _normalize_breaking_scope(item.breaking_scope) == "global")
|
||
else item.feed_region != active_region,
|
||
item.published_at is None,
|
||
-(item.published_at.timestamp() if item.published_at else 0),
|
||
item.feed_name,
|
||
),
|
||
)[:limit]
|
||
|
||
|
||
def _filter_news_items_by_categories(
|
||
items: list[ParsedNewsItem],
|
||
categories: set[str] | None,
|
||
) -> list[ParsedNewsItem]:
|
||
if not categories:
|
||
return items
|
||
return [item for item in items if (item.category or "other") in categories]
|
||
|
||
|
||
def _filter_news_items_by_source_ids(
|
||
items: list[ParsedNewsItem],
|
||
source_ids: set[str] | None,
|
||
) -> list[ParsedNewsItem]:
|
||
if not source_ids:
|
||
return items
|
||
return [
|
||
item for item in items
|
||
if (item.id.split(":", 1)[0] if ":" in item.id else "") in source_ids
|
||
]
|
||
|
||
|
||
async def _call_store_list_items(list_fn, db: AsyncSession, **kwargs):
|
||
try:
|
||
return await list_fn(db, **kwargs)
|
||
except TypeError as error:
|
||
if "source_ids" not in str(error):
|
||
raise
|
||
legacy_kwargs = dict(kwargs)
|
||
legacy_kwargs.pop("source_ids", None)
|
||
return await list_fn(db, **legacy_kwargs)
|
||
|
||
|
||
def _get_cached_region_feed(region: str) -> CachedRegionFeed | None:
|
||
cached = _REGION_CACHE.get(region)
|
||
if not cached:
|
||
return None
|
||
age_seconds = (datetime.now(UTC) - cached.fetched_at).total_seconds()
|
||
if age_seconds > STALE_CACHE_MAX_AGE_SECONDS:
|
||
return None
|
||
return cached
|
||
|
||
|
||
def clear_earth_news_region_cache() -> None:
|
||
_REGION_CACHE.clear()
|
||
|
||
|
||
def _store_region_cache(region: str, *, items: list[ParsedNewsItem], sources: list[NewsFeedSource]) -> None:
|
||
_REGION_CACHE[region] = CachedRegionFeed(
|
||
region=region,
|
||
fetched_at=datetime.now(UTC),
|
||
items=list(items),
|
||
sources=list(sources),
|
||
)
|
||
|
||
|
||
async def _apply_cached_locations_and_enqueue(items: list[ParsedNewsItem]) -> list[ParsedNewsItem]:
|
||
if not items:
|
||
return items
|
||
|
||
from app.services.earth_news_queue import (
|
||
enqueue_target_location_job,
|
||
get_cached_target_location_patch,
|
||
)
|
||
|
||
async def enqueue_item(item: ParsedNewsItem, *, force: bool = False) -> bool:
|
||
return await enqueue_target_location_job(build_target_location_job_payload(item), force=force)
|
||
|
||
async def apply_location(item: ParsedNewsItem) -> ParsedNewsItem:
|
||
cached_patch = await get_cached_target_location_patch(item.id)
|
||
if cached_patch:
|
||
apply_enrichment_patch_to_item(item, cached_patch)
|
||
if not _has_required_localization(item):
|
||
queued = await enqueue_item(item, force=True)
|
||
if queued and item.enrichment_status in {
|
||
"pending",
|
||
"unavailable",
|
||
"provider_error",
|
||
"parse_error",
|
||
"no_result",
|
||
"location_only",
|
||
}:
|
||
item.enrichment_status = "queued"
|
||
return item
|
||
|
||
queued = await enqueue_item(item)
|
||
item.location_patch = build_anchor_location_patch(
|
||
item,
|
||
queued=queued,
|
||
queue_available=queued,
|
||
)
|
||
return item
|
||
|
||
return list(await asyncio.gather(*(apply_location(item) for item in items)))
|
||
|
||
|
||
async def _enqueue_unverified_locations(items: list[ParsedNewsItem]) -> None:
|
||
if not items:
|
||
return
|
||
|
||
from app.services.earth_news_queue import enqueue_target_location_job
|
||
|
||
await asyncio.gather(
|
||
*(
|
||
enqueue_target_location_job(
|
||
build_target_location_job_payload(item),
|
||
force=not _has_default_localization(item),
|
||
)
|
||
for item in items
|
||
if (
|
||
item.location_patch is None
|
||
or item.location_patch.get("verified") is False
|
||
or not _has_required_localization(item)
|
||
)
|
||
)
|
||
)
|
||
|
||
|
||
async def _fetch_single_feed_url(
|
||
client: httpx.AsyncClient,
|
||
source: NewsFeedSource,
|
||
feed: NewsFeedEndpoint,
|
||
*,
|
||
config_payload: dict[str, Any] | None = None,
|
||
) -> tuple[NewsFeedSource, list[ParsedNewsItem], str | None, dict[str, Any]]:
|
||
started_at = perf_counter()
|
||
try:
|
||
response = await client.get(feed.url)
|
||
latency_ms = int((perf_counter() - started_at) * 1000)
|
||
status_code = response.status_code
|
||
content_type = response.headers.get("content-type", "")
|
||
if status_code >= 400:
|
||
error = f"HTTP {status_code}"
|
||
return source, [], error, _source_health_result(
|
||
source,
|
||
ok=False,
|
||
count=0,
|
||
error=error,
|
||
status="http_error",
|
||
status_code=status_code,
|
||
content_type=content_type,
|
||
latency_ms=latency_ms,
|
||
final_url=str(response.url),
|
||
feed=feed,
|
||
)
|
||
try:
|
||
items = _parse_feed_entries(response.text, source, feed=feed, config_payload=config_payload)
|
||
except Exception as exc:
|
||
error, status = _format_source_fetch_error(
|
||
status_code=status_code,
|
||
content_type=content_type,
|
||
body=response.text,
|
||
parsed_count=0,
|
||
)
|
||
if error is None:
|
||
error = str(exc)
|
||
status = "format_error"
|
||
return source, [], error, _source_health_result(
|
||
source,
|
||
ok=False,
|
||
count=0,
|
||
error=error,
|
||
status=status,
|
||
status_code=status_code,
|
||
content_type=content_type,
|
||
latency_ms=latency_ms,
|
||
final_url=str(response.url),
|
||
feed=feed,
|
||
)
|
||
error, status = _format_source_fetch_error(
|
||
status_code=status_code,
|
||
content_type=content_type,
|
||
body=response.text,
|
||
parsed_count=len(items),
|
||
)
|
||
return source, items, error, _source_health_result(
|
||
source,
|
||
ok=error is None,
|
||
count=len(items),
|
||
error=error,
|
||
status=status,
|
||
status_code=status_code,
|
||
content_type=content_type,
|
||
latency_ms=latency_ms,
|
||
final_url=str(response.url),
|
||
feed=feed,
|
||
)
|
||
except httpx.TimeoutException as exc:
|
||
error = f"请求超时: {exc}"
|
||
return source, [], error, _source_health_result(
|
||
source,
|
||
ok=False,
|
||
count=0,
|
||
error=error,
|
||
status="timeout",
|
||
latency_ms=int((perf_counter() - started_at) * 1000),
|
||
feed=feed,
|
||
)
|
||
except Exception as exc:
|
||
error = str(exc)
|
||
return source, [], error, _source_health_result(
|
||
source,
|
||
ok=False,
|
||
count=0,
|
||
error=error,
|
||
status="network_error",
|
||
latency_ms=int((perf_counter() - started_at) * 1000),
|
||
feed=feed,
|
||
)
|
||
|
||
|
||
async def _fetch_source(
|
||
client: httpx.AsyncClient,
|
||
source: NewsFeedSource,
|
||
*,
|
||
active_region: str | None = None,
|
||
config_payload: dict[str, Any] | None = None,
|
||
) -> tuple[NewsFeedSource, list[ParsedNewsItem], str | None, dict[str, Any]]:
|
||
feeds = tuple(
|
||
feed
|
||
for feed in _source_feeds(source)
|
||
if feed.enabled and feed.type in {"rss", "atom", "aggregated"}
|
||
and _feed_matches_active_region(feed, active_region)
|
||
)
|
||
if not feeds:
|
||
health = _source_health_result(source, ok=False, count=0, error="新闻源缺少已启用的 Feed 子项。", status="format_error")
|
||
return source, [], health["error"], health
|
||
|
||
results = await asyncio.gather(*(
|
||
_fetch_single_feed_url(client, source, feed, config_payload=config_payload)
|
||
for feed in sorted(feeds, key=lambda item: (item.priority, item.name))
|
||
))
|
||
merged_items: list[ParsedNewsItem] = []
|
||
feed_results: list[dict[str, Any]] = []
|
||
errors: list[str] = []
|
||
for _source, items, error, health in results:
|
||
feed_results.append(health)
|
||
if error:
|
||
errors.append(f"{health.get('final_url') or source.feed_url}: {error}")
|
||
continue
|
||
merged_items.extend(items)
|
||
|
||
deduped: dict[str, ParsedNewsItem] = {}
|
||
for item in merged_items:
|
||
key = item.url.strip() or item.title.strip().lower()
|
||
if key and key not in deduped:
|
||
deduped[key] = item
|
||
items = list(deduped.values())
|
||
ok = bool(items)
|
||
error = None if ok else "; ".join(errors) or "未解析到 RSS/Atom 条目，请检查 Feed 地址或源格式。"
|
||
latency_ms = sum(int(result.get("latency_ms") or 0) for result in feed_results)
|
||
status = "ok" if ok else (feed_results[0].get("status") if len(feed_results) == 1 else "failed")
|
||
status_code = next((result.get("status_code") for result in feed_results if result.get("status_code")), None)
|
||
content_type = ", ".join(sorted({str(result.get("content_type") or "") for result in feed_results if result.get("content_type")}))
|
||
health = _source_health_result(
|
||
source,
|
||
ok=ok,
|
||
count=len(items),
|
||
error=error,
|
||
status=status,
|
||
status_code=int(status_code) if isinstance(status_code, int) else None,
|
||
content_type=content_type,
|
||
latency_ms=latency_ms,
|
||
final_url=source.feed_url,
|
||
)
|
||
health["feed_results"] = feed_results
|
||
return source, items, error, health
|
||
|
||
|
||
async def _fetch_rss_items_for_sources(
|
||
sources: list[NewsFeedSource],
|
||
*,
|
||
active_region: str | None = None,
|
||
config_payload: dict[str, Any] | None = None,
|
||
) -> tuple[list[ParsedNewsItem], list[str], dict[str, dict[str, Any]]]:
|
||
errors: list[str] = []
|
||
async with httpx.AsyncClient(
|
||
timeout=REQUEST_TIMEOUT,
|
||
follow_redirects=True,
|
||
headers={"User-Agent": USER_AGENT},
|
||
) as client:
|
||
if config_payload is None:
|
||
results = await asyncio.gather(*(_fetch_source(client, source, active_region=active_region) for source in sources))
|
||
else:
|
||
results = await asyncio.gather(*(
|
||
_fetch_source(client, source, active_region=active_region, config_payload=config_payload)
|
||
for source in sources
|
||
))
|
||
|
||
fetched_items: list[ParsedNewsItem] = []
|
||
health_by_source: dict[str, dict[str, Any]] = {}
|
||
for source, items, error, health in results:
|
||
health_by_source[source.id] = health
|
||
if error:
|
||
errors.append(f"{source.name}: {error}")
|
||
continue
|
||
fetched_items.extend(items)
|
||
return fetched_items, errors, health_by_source
|
||
|
||
|
||
def _needs_rss_supplement(*, item_count: int, newest_at: datetime | None) -> bool:
|
||
if item_count < MAX_ITEMS_TOTAL:
|
||
return True
|
||
if newest_at is None:
|
||
return True
|
||
age_seconds = (datetime.now(UTC) - newest_at).total_seconds()
|
||
return age_seconds > RSS_SUPPLEMENT_MAX_AGE_SECONDS
|
||
|
||
|
||
async def _get_earth_news_payload_from_rss_only(
|
||
*,
|
||
lat: float | None,
|
||
lon: float | None,
|
||
active_region: str,
|
||
sources: list[NewsFeedSource],
|
||
categories: set[str] | None = None,
|
||
source_ids: set[str] | None = None,
|
||
limit: int = MAX_ITEMS_TOTAL,
|
||
locale: str = DEFAULT_NEWS_LOCALE,
|
||
) -> dict[str, Any]:
|
||
fetched_items, errors, _health_by_source = await _fetch_rss_items_for_sources(sources, active_region=active_region)
|
||
ranked_items = _filter_news_items_by_source_ids(
|
||
_filter_news_items_by_categories(
|
||
_rank_and_trim_items(fetched_items, active_region=active_region, limit=limit),
|
||
categories,
|
||
),
|
||
source_ids,
|
||
)[:limit]
|
||
if ranked_items:
|
||
ranked_items = await _apply_cached_locations_and_enqueue(ranked_items)
|
||
_store_region_cache(active_region, items=ranked_items, sources=sources)
|
||
return _build_payload(
|
||
lat=lat,
|
||
lon=lon,
|
||
active_region=active_region,
|
||
items=ranked_items,
|
||
sources=sources,
|
||
errors=errors,
|
||
stale=False,
|
||
categories=categories,
|
||
source_ids=source_ids,
|
||
limit=limit,
|
||
locale=locale,
|
||
)
|
||
|
||
cached = _get_cached_region_feed(active_region)
|
||
if cached:
|
||
cached_items = _filter_news_items_by_source_ids(
|
||
_filter_news_items_by_categories(cached.items, categories),
|
||
source_ids,
|
||
)[:limit]
|
||
cached.items = await _apply_cached_locations_and_enqueue(cached_items)
|
||
return _build_payload(
|
||
lat=lat,
|
||
lon=lon,
|
||
active_region=active_region,
|
||
items=cached.items,
|
||
sources=cached.sources,
|
||
errors=errors,
|
||
stale=True,
|
||
categories=categories,
|
||
source_ids=source_ids,
|
||
limit=limit,
|
||
locale=locale,
|
||
generated_at=cached.fetched_at,
|
||
)
|
||
|
||
return _build_payload(
|
||
lat=lat,
|
||
lon=lon,
|
||
active_region=active_region,
|
||
items=[],
|
||
sources=sources,
|
||
errors=errors,
|
||
stale=False,
|
||
categories=categories,
|
||
source_ids=source_ids,
|
||
limit=limit,
|
||
locale=locale,
|
||
)
|
||
|
||
|
||
async def get_earth_news_payload(
|
||
lat: float | None = None,
|
||
lon: float | None = None,
|
||
*,
|
||
region: str | None = None,
|
||
categories: set[str] | None = None,
|
||
source_ids: set[str] | None = None,
|
||
limit: int = MAX_ITEMS_TOTAL,
|
||
locale: str = DEFAULT_NEWS_LOCALE,
|
||
provider_client: AIProviderClient | None = None,
|
||
db: AsyncSession | None = None,
|
||
) -> dict[str, Any]:
|
||
del provider_client
|
||
active_region = region if region in REGION_ANCHORS else determine_focus_region(lat, lon)
|
||
has_settings_db = db is not None and callable(getattr(db, "execute", None))
|
||
source_config_payload = await get_earth_news_sources_payload(db) if has_settings_db else None
|
||
sources = (
|
||
await get_configured_sources_for_region(db, active_region)
|
||
if has_settings_db
|
||
else get_sources_for_region(active_region)
|
||
)
|
||
|
||
if db is None:
|
||
return await _get_earth_news_payload_from_rss_only(
|
||
lat=lat,
|
||
lon=lon,
|
||
active_region=active_region,
|
||
sources=sources,
|
||
categories=categories,
|
||
source_ids=source_ids,
|
||
limit=limit,
|
||
locale=locale,
|
||
)
|
||
|
||
from app.services.earth_news_store import (
|
||
get_earth_news_feed_coverage,
|
||
get_earth_news_freshness,
|
||
list_earth_news_cruise_items,
|
||
list_earth_news_items,
|
||
upsert_earth_news_items,
|
||
)
|
||
|
||
errors: list[str] = []
|
||
item_count, newest_at = await get_earth_news_freshness(db, active_region=active_region)
|
||
should_supplement = _needs_rss_supplement(item_count=item_count, newest_at=newest_at)
|
||
if not should_supplement and callable(getattr(db, "execute", None)):
|
||
expected_feed_keys = _expected_feed_keys(sources, active_region=active_region)
|
||
if expected_feed_keys:
|
||
covered_feed_keys = await get_earth_news_feed_coverage(
|
||
db,
|
||
active_region=active_region,
|
||
recent_after=datetime.now(UTC) - timedelta(seconds=RSS_SUPPLEMENT_MAX_AGE_SECONDS),
|
||
)
|
||
should_supplement = bool(expected_feed_keys - covered_feed_keys)
|
||
if should_supplement:
|
||
if source_config_payload is None:
|
||
fetched_items, errors, health_by_source = await _fetch_rss_items_for_sources(sources, active_region=active_region)
|
||
else:
|
||
fetched_items, errors, health_by_source = await _fetch_rss_items_for_sources(
|
||
sources,
|
||
active_region=active_region,
|
||
config_payload=source_config_payload,
|
||
)
|
||
await record_earth_news_sources_health(db, health_by_source)
|
||
ranked_fetched_items = _rank_and_trim_items(fetched_items, active_region=active_region)
|
||
await upsert_earth_news_items(db, ranked_fetched_items)
|
||
|
||
items = await _call_store_list_items(
|
||
list_earth_news_items,
|
||
db,
|
||
active_region=active_region,
|
||
limit=limit,
|
||
categories=categories,
|
||
source_ids=source_ids,
|
||
)
|
||
if hasattr(db, "execute"):
|
||
cruise_items = await _call_store_list_items(
|
||
list_earth_news_cruise_items,
|
||
db,
|
||
limit=min(max(limit, MAX_ITEMS_TOTAL) * len(REGION_ANCHORS), 500),
|
||
categories=categories,
|
||
source_ids=source_ids,
|
||
)
|
||
else:
|
||
cruise_items = items
|
||
await _enqueue_unverified_locations(items)
|
||
stale = bool(errors and items)
|
||
|
||
return _build_payload(
|
||
lat=lat,
|
||
lon=lon,
|
||
active_region=active_region,
|
||
items=items,
|
||
cruise_items=cruise_items,
|
||
sources=sources,
|
||
errors=errors,
|
||
stale=stale,
|
||
categories=categories,
|
||
source_ids=source_ids,
|
||
limit=limit,
|
||
locale=locale,
|
||
)
|