diff --git a/VERSION b/VERSION index 7d530da8..106d4ac0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.68.1 +0.69.0 diff --git a/backend/app/api/v1/earth.py b/backend/app/api/v1/earth.py index b01a884a..89e7d7e2 100644 --- a/backend/app/api/v1/earth.py +++ b/backend/app/api/v1/earth.py @@ -21,6 +21,12 @@ from app.models.datasource_config import DataSourceConfig from app.models.system_setting import SystemSetting from app.models.user import User from app.services.tv_streams import get_tv_settings_payload +from app.services.earth_news import ( + get_earth_news_sources_payload, + reset_earth_news_sources_payload, + save_earth_news_sources_payload, + test_news_source_config, +) from app.services.earth_boundaries import ( EarthBoundaryBuildError, get_boundary_build_status, @@ -100,6 +106,19 @@ class EarthAboutPayload(BaseModel): meta: list[EarthAboutMetaItem] = Field(default_factory=list) +class EarthNewsSourcesPayload(BaseModel): + cache_version: int | None = None + source_tags: list[dict[str, Any]] = Field(default_factory=list) + categories: list[dict[str, Any]] = Field(default_factory=list) + item_tag_rules: list[dict[str, Any]] = Field(default_factory=list) + sources: list[dict[str, Any]] = Field(default_factory=list) + health: dict[str, Any] = Field(default_factory=dict) + + +class EarthNewsSourceTestPayload(BaseModel): + source: dict[str, Any] = Field(default_factory=dict) + + def _normalize_earth_brand_payload(payload: dict[str, Any] | None) -> dict[str, str]: merged = DEFAULT_EARTH_BRAND.copy() if payload: @@ -324,6 +343,38 @@ async def reset_earth_about( return {"status": "reset", "about": _normalize_earth_about_payload(None), "is_default": True} +@router.get("/news-sources") +async def get_earth_news_sources(db: AsyncSession = Depends(get_db)): + return await get_earth_news_sources_payload(db) + + +@router.put("/news-sources") +async def update_earth_news_sources( + payload: EarthNewsSourcesPayload, + _current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + return await save_earth_news_sources_payload(db, payload.model_dump()) + + +@router.delete("/news-sources") +@router.post("/news-sources/reset") +async def reset_earth_news_sources( + _current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + return await reset_earth_news_sources_payload(db) + + +@router.post("/news-sources/test") +async def test_earth_news_source( + payload: EarthNewsSourceTestPayload, + _current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + return await test_news_source_config(payload.source, db=db) + + @router.get("/oobe-status") async def get_earth_oobe_status( current_user: User | None = Depends(_get_optional_current_user), diff --git a/backend/app/api/v1/interactables.py b/backend/app/api/v1/interactables.py index f541d56a..055ff9a7 100644 --- a/backend/app/api/v1/interactables.py +++ b/backend/app/api/v1/interactables.py @@ -90,7 +90,7 @@ async def get_interactables_geojson( return interactables_to_geojson(items) payload = await get_or_build_layer_payload( - key=earth_layer_cache.key("interactables", layer=layer or "all"), + key=earth_layer_cache.key("interactables", interactable_layer=layer or "all"), policy=INTERACTABLE_CACHE_POLICY, builder=build_payload, response=response, diff --git a/backend/app/api/v1/news.py b/backend/app/api/v1/news.py index 12cc7d0b..47389e63 100644 --- a/backend/app/api/v1/news.py +++ b/backend/app/api/v1/news.py @@ -1,16 +1,92 @@ -from fastapi import APIRouter, Depends, Query +from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy.ext.asyncio import AsyncSession from app.db.session import get_db -from app.services.earth_news import get_earth_news_payload +from app.services.earth_news import ( + ALLOWED_NEWS_CATEGORY_KEYS, + SUPPORTED_NEWS_LOCALES, + REGION_ANCHORS, + get_earth_news_payload, +) router = APIRouter() +def _parse_categories(raw: str | None) -> set[str] | None: + if raw is None or not raw.strip(): + return None + requested = {item.strip().lower() for item in raw.split(",") if item.strip()} + invalid = sorted(requested - set(ALLOWED_NEWS_CATEGORY_KEYS)) + if invalid: + raise HTTPException( + status_code=422, + detail={ + "message": "Unsupported news categories.", + "invalid_categories": invalid, + "allowed_categories": list(ALLOWED_NEWS_CATEGORY_KEYS), + }, + ) + return requested or None + + +def _parse_source_ids(raw: str | None) -> set[str] | None: + if raw is None or not raw.strip(): + return None + return {item.strip() for item in raw.split(",") if item.strip()} or None + + +def _parse_limit(raw: int | None) -> int: + if raw is None: + return 12 + if raw < 1: + raise HTTPException(status_code=422, detail={"message": "News limit must be greater than 0."}) + return min(raw, 100) + + +def _parse_locale(raw: str | None) -> str: + if raw is None or not raw.strip(): + return "zh-CN" + requested = raw.strip() + if requested not in SUPPORTED_NEWS_LOCALES: + raise HTTPException( + status_code=422, + detail={ + "message": "Unsupported news locale.", + "invalid_locale": requested, + "allowed_locales": sorted(SUPPORTED_NEWS_LOCALES), + }, + ) + return requested + + @router.get("/earth-feed") async def get_earth_feed( lat: float | None = Query(None, description="Current Earth view center latitude"), lon: float | None = Query(None, description="Current Earth view center longitude"), + region: str | None = Query(None, description="Explicit Earth news region for UE/client integrations"), + categories: str | None = Query(None, description="Comma-separated news category keys"), + sources: str | None = Query(None, description="Comma-separated news source ids"), + limit: int | None = Query(None, description="Maximum news items to return, capped at 100"), + locale: str | None = Query(None, description="Display locale, zh-CN or en-US"), db: AsyncSession = Depends(get_db), ): - return await get_earth_news_payload(lat=lat, lon=lon, db=db) + normalized_region = region.strip().lower() if isinstance(region, str) and region.strip() else None + if normalized_region is not None and normalized_region not in REGION_ANCHORS: + raise HTTPException( + status_code=422, + detail={ + "message": "Unsupported news region.", + "invalid_region": normalized_region, + "allowed_regions": list(REGION_ANCHORS.keys()), + }, + ) + return await get_earth_news_payload( + lat=lat, + lon=lon, + region=normalized_region, + categories=_parse_categories(categories), + source_ids=_parse_source_ids(sources), + limit=_parse_limit(limit), + locale=_parse_locale(locale), + db=db, + ) diff --git a/backend/app/api/v1/system_control.py b/backend/app/api/v1/system_control.py index f481d633..f614c553 100644 --- a/backend/app/api/v1/system_control.py +++ b/backend/app/api/v1/system_control.py @@ -1,16 +1,17 @@ from __future__ import annotations import os +import secrets import subprocess import sys from datetime import datetime -from fastapi import APIRouter, Depends, HTTPException, Query, Request, status +from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, status from pydantic import BaseModel from sqlalchemy.ext.asyncio import AsyncSession -from app.core.config import ROOT_DIR +from app.core.config import ROOT_DIR, settings from app.core.security import get_current_user from app.db.session import get_db from app.models.user import User @@ -37,6 +38,9 @@ from app.services.system_logs import ( normalize_log_level, read_database_log_snapshot, read_log_snapshot, + read_observability_group_events, + read_observability_groups, + read_observability_raw_events, ) from app.services.earth_layer_cache import earth_layer_cache @@ -108,12 +112,34 @@ class EarthClientLogEventCreate(BaseModel): url: str | None = None module: str | None = None detail: str | None = None + fingerprint: str | None = None + occurrence_count: int = 1 + metadata: dict[str, object] | None = None class EarthClientLogEventResponse(BaseModel): accepted: bool source_id: str level: str + fingerprint: str | None = None + + +class ServiceLogEventCreate(BaseModel): + source: str = "ai-provider" + service: str = "ai-provider" + module: str | None = None + category: str | None = None + event: str = "service.runtime_log" + level: str = "error" + message: str + fingerprint: str | None = None + occurrence_count: int = 1 + request_id: str | None = None + trace_id: str | None = None + task_id: str | None = None + source_id: int | str | None = None + provider: str | None = None + context: dict[str, object] | None = None async def ingest_client_log_event( @@ -136,6 +162,9 @@ async def ingest_client_log_event( "url": payload.url or "", "module": payload.module or "", "detail": payload.detail or "", + "fingerprint": payload.fingerprint or "", + "occurrence_count": max(1, int(payload.occurrence_count or 1)), + "metadata": payload.metadata or {}, }, ) await record_system_log( @@ -151,9 +180,36 @@ async def ingest_client_log_event( "detail": payload.detail or "", "module": payload.module or "", "client_ip": request.client.host if request.client else "", + "metadata": payload.metadata or {}, }, + fingerprint=payload.fingerprint, + occurrence_count=max(1, int(payload.occurrence_count or 1)), ) - return EarthClientLogEventResponse(accepted=True, source_id=source_id, level=normalized_level) + return EarthClientLogEventResponse(accepted=True, source_id=source_id, level=normalized_level, fingerprint=payload.fingerprint) + + +def require_observability_ingest_token( + authorization: str | None, + ingest_token: str | None, +) -> None: + expected_token = settings.OBSERVABILITY_INGEST_TOKEN.strip() + if not expected_token: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Observability service ingestion is not configured", + ) + provided = "" + if ingest_token: + provided = ingest_token.strip() + elif authorization: + scheme, _, token = authorization.partition(" ") + if scheme.lower() == "bearer": + provided = token.strip() + if not provided or not secrets.compare_digest(provided, expected_token): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Invalid observability ingestion token", + ) class EarthLayerCacheStatusResponse(BaseModel): @@ -378,6 +434,118 @@ async def get_system_log_sources( } +@router.get("/logs/observability/groups") +async def get_observability_log_groups( + limit: int = DEFAULT_LOG_LINE_LIMIT, + level: str = "all", + levels: str | None = Query(None, description="Comma-separated log levels"), + start_date: str | None = Query(None, description="Filter logs from this date (YYYY-MM-DD)"), + end_date: str | None = Query(None, description="Filter logs until this date (YYYY-MM-DD)"), + search: str | None = Query(None, description="Case-insensitive substring search"), + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + ensure_super_admin(current_user) + if limit < 1 or limit > MAX_LOG_LINE_LIMIT: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"limit must be between 1 and {MAX_LOG_LINE_LIMIT}") + normalized_start_date = validate_log_date(start_date, "start_date") + normalized_end_date = validate_log_date(end_date, "end_date") + if normalized_start_date and normalized_end_date and normalized_start_date > normalized_end_date: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="start_date must be earlier than or equal to end_date") + return await read_observability_groups( + limit=limit, + level=level, + levels=levels, + start_date=normalized_start_date, + end_date=normalized_end_date, + search=search, + db=db, + ) + + +@router.get("/logs/observability/groups/{fingerprint}/events") +async def get_observability_group_events( + fingerprint: str, + limit: int = DEFAULT_LOG_LINE_LIMIT, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + ensure_super_admin(current_user) + if limit < 1 or limit > MAX_LOG_LINE_LIMIT: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"limit must be between 1 and {MAX_LOG_LINE_LIMIT}") + payload = await read_observability_group_events(fingerprint, limit=limit, db=db) + if payload is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Observability group not found") + return payload + + +@router.get("/logs/observability/raw") +async def get_observability_raw_events( + limit: int = DEFAULT_LOG_LINE_LIMIT, + level: str = "all", + levels: str | None = Query(None, description="Comma-separated log levels"), + start_date: str | None = Query(None, description="Filter logs from this date (YYYY-MM-DD)"), + end_date: str | None = Query(None, description="Filter logs until this date (YYYY-MM-DD)"), + search: str | None = Query(None, description="Case-insensitive substring search"), + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + ensure_super_admin(current_user) + if limit < 1 or limit > MAX_LOG_LINE_LIMIT: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"limit must be between 1 and {MAX_LOG_LINE_LIMIT}") + normalized_start_date = validate_log_date(start_date, "start_date") + normalized_end_date = validate_log_date(end_date, "end_date") + return await read_observability_raw_events( + limit=limit, + level=level, + levels=levels, + start_date=normalized_start_date, + end_date=normalized_end_date, + search=search, + db=db, + ) + + +@router.post("/logs/service", response_model=EarthClientLogEventResponse) +async def ingest_service_log( + payload: ServiceLogEventCreate, + authorization: str | None = Header(default=None), + ingest_token: str | None = Header(default=None, alias="X-Planet-Observability-Token"), +): + require_observability_ingest_token(authorization, ingest_token) + normalized_level = normalize_log_level(payload.level) + source = (payload.source or "ai-provider").strip() or "ai-provider" + context = dict(payload.context or {}) + if payload.request_id: + context["request_id"] = payload.request_id + if payload.trace_id: + context["trace_id"] = payload.trace_id + if payload.task_id: + context["task_id"] = payload.task_id + if payload.source_id is not None: + context["source_id"] = payload.source_id + if payload.provider: + context["provider"] = payload.provider + await record_system_log( + source=source, + service=(payload.service or source).strip() or source, + module=payload.module or source, + event=(payload.event or "service.runtime_log").strip() or "service.runtime_log", + level=normalized_level, + message=payload.message, + category=payload.category or "service-runtime", + context=context, + fingerprint=payload.fingerprint, + occurrence_count=max(1, int(payload.occurrence_count or 1)), + ) + return EarthClientLogEventResponse( + accepted=True, + source_id=source, + level=normalized_level, + fingerprint=payload.fingerprint, + ) + + @router.get("/logs/{source_id}", response_model=SystemLogSnapshotResponse) async def get_system_log_snapshot( source_id: str, diff --git a/backend/app/api/v1/tv.py b/backend/app/api/v1/tv.py index 75f01ea5..78687a5c 100644 --- a/backend/app/api/v1/tv.py +++ b/backend/app/api/v1/tv.py @@ -1,3 +1,4 @@ +import re from urllib.parse import quote, urljoin import httpx @@ -10,6 +11,26 @@ from app.services.tv_streams import get_public_tv_payload, is_allowed_tv_proxy_u router = APIRouter() +_HLS_URI_ATTRIBUTE_RE = re.compile(r'URI="([^"]+)"') + + +def _proxied_tv_url(url: str) -> str: + return f"/api/v1/tv/proxy?url={quote(url, safe='')}" + + +def _rewrite_hls_uri_attributes(line: str, *, base_url: str) -> str: + def replace(match: re.Match[str]) -> str: + uri = match.group(1) + absolute_url = urljoin(base_url, uri) + return f'URI="{_proxied_tv_url(absolute_url)}"' + + return _HLS_URI_ATTRIBUTE_RE.sub(replace, line) + + +def _should_strip_hls_metadata_line(line: str) -> bool: + normalized = line.strip().upper() + return normalized.startswith("#EXT-X-MEDIA:") and "TYPE=SUBTITLES" in normalized + @router.get("/streams") async def list_public_tv_streams( @@ -56,11 +77,16 @@ async def proxy_tv_stream( rewritten_lines: list[str] = [] for line in manifest_text.splitlines(): stripped = line.strip() - if not stripped or stripped.startswith("#"): + if not stripped: rewritten_lines.append(line) continue + if stripped.startswith("#"): + if _should_strip_hls_metadata_line(line): + continue + rewritten_lines.append(_rewrite_hls_uri_attributes(line, base_url=response_url)) + continue absolute_url = urljoin(response_url, stripped) - rewritten_lines.append(f"/api/v1/tv/proxy?url={quote(absolute_url, safe='')}") + rewritten_lines.append(_proxied_tv_url(absolute_url)) return Response( content="\n".join(rewritten_lines), media_type="application/vnd.apple.mpegurl", diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 24e3559b..1210d881 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -41,6 +41,7 @@ class Settings(BaseSettings): AI_PROVIDER_SERVICE_TOKEN: str = "" AI_PROVIDER_TIMEOUT_SECONDS: int = 60 AI_PROVIDER_RETRY_ATTEMPTS: int = 2 + OBSERVABILITY_INGEST_TOKEN: str = "" @property def REDIS_URL(self) -> str: diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 460ec258..563b6223 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -13,7 +13,7 @@ from app.models.compute_center_location import ComputeCenterLocationRecord from app.models.system_setting import SystemSetting from app.models.playground_session import PlaygroundSession from app.models.playground_message import PlaygroundMessage -from app.models.system_log import SystemLog, AuditLog +from app.models.system_log import AuditLog, ObservabilityEvent, ObservabilityEventGroup, SystemLog from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth, VesselPosition, VesselStatic from app.models.datasource_mapping import DataSourceMappingTemplate from app.models.earth_news import EarthNewsItem @@ -37,6 +37,8 @@ __all__ = [ "ComputeCenterLocationRecord", "SystemLog", "AuditLog", + "ObservabilityEvent", + "ObservabilityEventGroup", "PlaygroundSession", "PlaygroundMessage", "VesselPosition", diff --git a/backend/app/models/system_log.py b/backend/app/models/system_log.py index 5024c9b4..9b98f82a 100644 --- a/backend/app/models/system_log.py +++ b/backend/app/models/system_log.py @@ -38,3 +38,46 @@ class AuditLog(Base): ip = Column(String(64), nullable=True) details = Column(JSON, nullable=False, default=dict) created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class ObservabilityEvent(Base): + __tablename__ = "observability_events" + + id = Column(Integer, primary_key=True, autoincrement=True) + occurred_at = Column(DateTime(timezone=True), server_default=func.now(), index=True) + source = Column(String(50), nullable=False, index=True) + service = Column(String(50), nullable=True, index=True) + module = Column(String(120), nullable=True, index=True) + category = Column(String(80), nullable=True, index=True) + event = Column(String(160), nullable=True, index=True) + level = Column(String(20), nullable=False, index=True) + message = Column(Text, nullable=False) + fingerprint = Column(String(80), nullable=False, index=True) + request_id = Column(String(64), nullable=True, index=True) + trace_id = Column(String(64), nullable=True, index=True) + task_id = Column(String(120), nullable=True, index=True) + source_ref_id = Column(String(120), nullable=True, index=True) + provider = Column(String(120), nullable=True, index=True) + user_id = Column(Integer, nullable=True, index=True) + context = Column(JSON, nullable=False, default=dict) + occurrence_count = Column(Integer, nullable=False, default=1) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class ObservabilityEventGroup(Base): + __tablename__ = "observability_event_groups" + + fingerprint = Column(String(80), primary_key=True) + source = Column(String(50), nullable=False, index=True) + service = Column(String(50), nullable=True, index=True) + module = Column(String(120), nullable=True, index=True) + category = Column(String(80), nullable=True, index=True) + event = Column(String(160), nullable=True, index=True) + last_level = Column(String(20), nullable=False, index=True) + sample_message = Column(Text, nullable=False) + sample_detail = Column(Text, nullable=True) + affected_sources = Column(JSON, nullable=False, default=list) + count = Column(Integer, nullable=False, default=0) + first_seen_at = Column(DateTime(timezone=True), nullable=False, index=True) + last_seen_at = Column(DateTime(timezone=True), nullable=False, index=True) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) diff --git a/backend/app/services/earth_interactables.py b/backend/app/services/earth_interactables.py index e65596c6..f1b5f365 100644 --- a/backend/app/services/earth_interactables.py +++ b/backend/app/services/earth_interactables.py @@ -62,11 +62,11 @@ def interactables_to_geojson(items: list[EarthInteractable]) -> dict[str, Any]: def invalidate_interactable_cache(layer: str | None = None) -> int: layer_key = str(layer or "*").strip() or "*" deleted = earth_layer_cache.delete_pattern( - f"{EARTH_LAYER_CACHE_PREFIX}:interactables:layer:{layer_key}*" + f"{EARTH_LAYER_CACHE_PREFIX}:interactables:interactable_layer:{layer_key}*" ) if layer_key != "all": deleted += earth_layer_cache.delete_pattern( - f"{EARTH_LAYER_CACHE_PREFIX}:interactables:layer:all*" + f"{EARTH_LAYER_CACHE_PREFIX}:interactables:interactable_layer:all*" ) return deleted diff --git a/backend/app/services/earth_news.py b/backend/app/services/earth_news.py index 77b7293a..6495a8b7 100644 --- a/backend/app/services/earth_news.py +++ b/backend/app/services/earth_news.py @@ -2,22 +2,25 @@ from __future__ import annotations import asyncio from dataclasses import dataclass, field -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from email.utils import parsedate_to_datetime import hashlib import html import json import math import re +from time import perf_counter from typing import Any from urllib.parse import quote import xml.etree.ElementTree as ET import httpx from bs4 import BeautifulSoup +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.core.countries import COUNTRY_VARIANTS_MAP, get_country_centroid, normalize_country +from app.models.system_setting import SystemSetting from app.ai_tasks.prompts import EffectiveAIPrompt, get_effective_prompt from app.schemas.ai import SituationalAnalysisRequest from app.services.ai_client import AIProviderClient @@ -33,7 +36,16 @@ RSS_SUPPLEMENT_MAX_AGE_SECONDS = STALE_CACHE_MAX_AGE_SECONDS MAX_TARGET_INFERENCE_CONCURRENCY = 3 TARGET_INFERENCE_TIMEOUT_SECONDS = 6.0 DEFAULT_NEWS_LOCALE = "zh-CN" +SUPPORTED_NEWS_LOCALES = frozenset({"zh-CN", "en-US"}) NEWS_ENRICH_PROMPT_KEY = "earth.news.enrich" +EARTH_NEWS_SOURCES_CATEGORY = "earth_news_sources" +DEFAULT_NEWS_HEALTH_POLICY = { + "timeout_seconds": REQUEST_TIMEOUT, + "failure_threshold": 3, + "cooldown_minutes": 30, + "fetch_interval_minutes": 45, + "circuit_breaker": True, +} @dataclass(frozen=True) @@ -52,6 +64,19 @@ class RegionAnchor: longitude: float +@dataclass(frozen=True) +class NewsFeedEndpoint: + id: str + name: str + url: str + type: str = "rss" + region: str = "" + enabled: bool = True + default_category: str = "other" + tags: tuple[str, ...] = () + priority: int = 100 + + @dataclass(frozen=True) class NewsFeedSource: id: str @@ -59,8 +84,16 @@ class NewsFeedSource: region: str feed_url: str homepage_url: str + feed_directory_url: str = "" source_type: str = "rss" + feed_urls: tuple[str, ...] = () + feeds: tuple[NewsFeedEndpoint, ...] = () priority: int = 100 + enabled: bool = True + source_tags: tuple[str, ...] = () + default_category: str = "other" + importance_weight: int = 0 + health_policy: dict[str, Any] = field(default_factory=dict) @dataclass(frozen=True) @@ -97,6 +130,18 @@ class ParsedNewsItem: target_ai_error: str | None = None target_debug_note: str | None = None location_patch: dict[str, Any] | None = None + source_tags: list[str] = field(default_factory=list) + feed_id: str = "" + feed_type: str = "rss" + feed_default_category: str = "other" + category: str = "other" + item_tags: list[str] = field(default_factory=list) + tagging_source: str = "rules" + tagging_confidence: float = 0.0 + importance_score: int = 0 + importance_level: str = "low" + importance_reasons: list[str] = field(default_factory=list) + market_impact: str = "none" @dataclass @@ -189,7 +234,15 @@ NEWS_FEED_SOURCES: tuple[NewsFeedSource, ...] = ( region="global", feed_url="https://feeds.bbci.co.uk/news/world/rss.xml", homepage_url="https://www.bbc.com/news/world", + feeds=( + NewsFeedEndpoint(id="world", name="World", url="https://feeds.bbci.co.uk/news/world/rss.xml", default_category="politics", priority=1), + ), + source_type="rss", priority=10, + source_tags=("official_media", "business_news", "global"), + default_category="politics", + importance_weight=12, + health_policy=DEFAULT_NEWS_HEALTH_POLICY, ), NewsFeedSource( id="dw-top", @@ -197,11 +250,304 @@ NEWS_FEED_SOURCES: tuple[NewsFeedSource, ...] = ( region="europe", feed_url="https://rss.dw.com/rdf/rss-en-top", homepage_url="https://www.dw.com/en/top-stories/s-9097", + feeds=( + NewsFeedEndpoint(id="top", name="Top Stories", url="https://rss.dw.com/rdf/rss-en-top", default_category="politics", priority=1), + ), + source_type="rss", priority=20, + source_tags=("official_media", "business_news", "global", "europe"), + default_category="politics", + importance_weight=12, + health_policy=DEFAULT_NEWS_HEALTH_POLICY, ), NewsFeedSource( - id="global-scan", - name="Global Monitor / World", + id="cnbc-business", + name="CNBC Business", + region="global", + feed_url="https://www.cnbc.com/id/10001147/device/rss/rss.html", + homepage_url="https://www.cnbc.com/business/", + feeds=( + NewsFeedEndpoint(id="business", name="Business", url="https://www.cnbc.com/id/10001147/device/rss/rss.html", default_category="business", priority=1), + ), + source_type="rss", + priority=21, + source_tags=("business_news", "global", "us"), + default_category="business", + importance_weight=16, + health_policy=DEFAULT_NEWS_HEALTH_POLICY, + ), + NewsFeedSource( + id="bbc-business", + name="BBC Business", + region="global", + feed_url="https://feeds.bbci.co.uk/news/business/rss.xml", + homepage_url="https://www.bbc.com/news/business", + feeds=( + NewsFeedEndpoint(id="business", name="Business", url="https://feeds.bbci.co.uk/news/business/rss.xml", default_category="business", priority=1), + ), + source_type="rss", + priority=22, + source_tags=("official_media", "business_news", "global"), + default_category="business", + importance_weight=14, + health_policy=DEFAULT_NEWS_HEALTH_POLICY, + ), + NewsFeedSource( + id="guardian-business", + name="The Guardian Business", + region="europe", + feed_url="https://www.theguardian.com/uk/business/rss", + homepage_url="https://www.theguardian.com/uk/business", + feeds=( + NewsFeedEndpoint(id="business", name="Business", url="https://www.theguardian.com/uk/business/rss", default_category="business", priority=1), + ), + source_type="rss", + priority=24, + source_tags=("business_news", "global", "europe"), + default_category="business", + importance_weight=12, + health_policy=DEFAULT_NEWS_HEALTH_POLICY, + ), + NewsFeedSource( + id="npr-business", + name="NPR Business", + region="americas", + feed_url="https://feeds.npr.org/1006/rss.xml", + homepage_url="https://www.npr.org/sections/business/", + feeds=( + NewsFeedEndpoint(id="business", name="Business", url="https://feeds.npr.org/1006/rss.xml", default_category="business", priority=1), + ), + source_type="rss", + priority=25, + source_tags=("business_news", "global", "us"), + default_category="business", + importance_weight=12, + health_policy=DEFAULT_NEWS_HEALTH_POLICY, + ), + NewsFeedSource( + id="marketwatch-top", + name="MarketWatch Top Stories", + region="americas", + feed_url="https://feeds.content.dowjones.io/public/rss/mw_topstories", + homepage_url="https://www.marketwatch.com/", + feeds=( + NewsFeedEndpoint(id="top", name="Top Stories", url="https://feeds.content.dowjones.io/public/rss/mw_topstories", default_category="finance", priority=1), + ), + source_type="rss", + priority=26, + source_tags=("business_news", "finance", "global", "us"), + default_category="finance", + importance_weight=12, + health_policy=DEFAULT_NEWS_HEALTH_POLICY, + ), + NewsFeedSource( + id="techcrunch", + name="TechCrunch", + region="global", + feed_url="https://techcrunch.com/feed/", + homepage_url="https://techcrunch.com/", + feeds=( + NewsFeedEndpoint(id="main", name="Main Feed", url="https://techcrunch.com/feed/", default_category="technology", priority=1), + ), + source_type="rss", + priority=32, + source_tags=("business_news", "ecommerce", "global"), + default_category="technology", + importance_weight=8, + health_policy=DEFAULT_NEWS_HEALTH_POLICY, + ), + NewsFeedSource( + id="retaildive", + name="Retail Dive", + region="global", + feed_url="https://www.retaildive.com/feeds/news/", + homepage_url="https://www.retaildive.com/", + feeds=( + NewsFeedEndpoint(id="news", name="News", url="https://www.retaildive.com/feeds/news/", default_category="business", priority=1), + ), + source_type="rss", + priority=34, + source_tags=("business_news", "retail", "ecommerce", "global"), + default_category="business", + importance_weight=10, + health_policy=DEFAULT_NEWS_HEALTH_POLICY, + ), + NewsFeedSource( + id="prnewswire-retail", + name="PR Newswire Consumer Products & Retail", + region="global", + feed_url="https://www.prnewswire.com/rss/consumer-products-retail-latest-news/consumer-products-retail-latest-news-list.rss", + homepage_url="https://www.prnewswire.com/news-releases/consumer-products-retail-latest-news/", + feeds=( + NewsFeedEndpoint(id="consumer-retail", name="Consumer Products & Retail", url="https://www.prnewswire.com/rss/consumer-products-retail-latest-news/consumer-products-retail-latest-news-list.rss", default_category="business", priority=1), + ), + source_type="rss", + priority=48, + source_tags=("press_release", "retail", "ecommerce", "global"), + default_category="business", + importance_weight=-4, + health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "failure_threshold": 2}, + ), + NewsFeedSource( + id="36kr", + name="36氪", + region="asia-pacific", + feed_url="https://36kr.com/feed-article", + homepage_url="https://www.36kr.com/", + feed_directory_url="https://www.36kr.com/rss-center", + source_type="rss", + feeds=( + NewsFeedEndpoint(id="feed", name="综合资讯", url="https://36kr.com/feed", default_category="business", priority=1), + NewsFeedEndpoint(id="article", name="文章资讯", url="https://36kr.com/feed-article", default_category="business", priority=2), + NewsFeedEndpoint(id="newsflash", name="最新快讯", url="https://36kr.com/feed-newsflash", default_category="business", priority=3), + NewsFeedEndpoint(id="moment", name="动态内容", url="https://36kr.com/feed-moment", default_category="business", priority=4), + ), + priority=35, + source_tags=("business_news", "ecommerce", "china"), + default_category="business", + importance_weight=12, + health_policy=DEFAULT_NEWS_HEALTH_POLICY, + ), + NewsFeedSource( + id="businesswire-ecommerce", + name="BusinessWire Electronic Commerce", + region="global", + feed_url="https://www.businesswire.com/newsroom/industry/technology/ecommerce", + homepage_url="https://www.businesswire.com/newsroom/industry/technology/ecommerce", + source_type="reference", + priority=62, + enabled=False, + source_tags=("press_release", "ecommerce", "global", "low_stability"), + default_category="ecommerce", + importance_weight=-6, + health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "failure_threshold": 2}, + ), + NewsFeedSource( + id="mckinsey-retail", + name="McKinsey Retail Insights", + region="global", + feed_url="https://www.mckinsey.com/industries/retail/our-insights", + homepage_url="https://www.mckinsey.com/industries/retail/our-insights", + source_type="reference", + priority=64, + enabled=False, + source_tags=("industry_insight", "retail", "global"), + default_category="business", + importance_weight=18, + health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 720}, + ), + NewsFeedSource( + id="deloitte-retail", + name="Deloitte Retail", + region="global", + feed_url="https://www.deloitte.com/us/en/Industries/retail/about.html", + homepage_url="https://www.deloitte.com/us/en/Industries/retail/about.html", + source_type="reference", + priority=66, + enabled=False, + source_tags=("industry_insight", "retail", "global", "us"), + default_category="business", + importance_weight=16, + health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 720}, + ), + NewsFeedSource( + id="us-census-ecommerce", + name="US Census Retail / Quarterly E-Commerce", + region="americas", + feed_url="https://www.census.gov/retail/index.html#ecommerce", + homepage_url="https://www.census.gov/retail/index.html#ecommerce", + source_type="reference", + priority=18, + enabled=False, + source_tags=("official_data", "ecommerce", "retail", "us"), + default_category="ecommerce", + importance_weight=34, + health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 1440}, + ), + NewsFeedSource( + id="mofcom-data", + name="商务数据中心", + region="asia-pacific", + feed_url="https://data.mofcom.gov.cn/index.shtml", + homepage_url="https://data.mofcom.gov.cn/index.shtml", + source_type="reference", + priority=18, + enabled=False, + source_tags=("official_data", "business_news", "china"), + default_category="business", + importance_weight=34, + health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 1440}, + ), + NewsFeedSource( + id="mofcom-ecommerce", + name="商务部电商动态", + region="asia-pacific", + feed_url="https://www.mofcom.gov.cn/", + homepage_url="https://www.mofcom.gov.cn/", + source_type="reference", + priority=19, + enabled=False, + source_tags=("official_data", "ecommerce", "china"), + default_category="ecommerce", + importance_weight=34, + health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 1440}, + ), + NewsFeedSource( + id="stats-china-online-retail", + name="国家统计局数据发布", + region="asia-pacific", + feed_url="https://www.stats.gov.cn/sj/zxfb/rss.xml", + homepage_url="https://www.stats.gov.cn/sj/zxfb/", + feeds=( + NewsFeedEndpoint(id="release", name="数据发布", url="https://www.stats.gov.cn/sj/zxfb/rss.xml", default_category="ecommerce", priority=1), + ), + source_type="rss", + priority=19, + source_tags=("official_data", "ecommerce", "retail", "china"), + default_category="ecommerce", + importance_weight=36, + health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 1440}, + ), + NewsFeedSource( + id="china-ecommerce-logistics-index", + name="电商物流指数", + region="asia-pacific", + feed_url="https://www.gov.cn/", + homepage_url="https://www.gov.cn/", + source_type="reference", + priority=20, + enabled=False, + source_tags=("official_data", "ecommerce", "retail", "china"), + default_category="ecommerce", + importance_weight=36, + health_policy={**DEFAULT_NEWS_HEALTH_POLICY, "fetch_interval_minutes": 1440}, + ), + NewsFeedSource( + id="ebrun", + name="亿邦动力", + region="asia-pacific", + feed_url="https://www.ebrun.com/rss/news_b2c.xml", + homepage_url="https://www.ebrun.com/", + feed_directory_url="https://www.ebrun.com/rss/", + source_type="rss", + feeds=( + NewsFeedEndpoint(id="b2c", name="B2C", url="https://www.ebrun.com/rss/news_b2c.xml", default_category="ecommerce", priority=1), + NewsFeedEndpoint(id="b2b", name="B2B", url="https://www.ebrun.com/rss/news_b2b.xml", default_category="ecommerce", priority=2), + NewsFeedEndpoint(id="retail", name="零售", url="https://www.ebrun.com/rss/news_retail.xml", default_category="ecommerce", priority=3), + NewsFeedEndpoint(id="o2o", name="O2O", url="https://www.ebrun.com/rss/news_o2o.xml", default_category="ecommerce", priority=4), + NewsFeedEndpoint(id="service", name="服务", url="https://www.ebrun.com/rss/news_service.xml", default_category="ecommerce", priority=5), + NewsFeedEndpoint(id="data", name="数据", url="https://www.ebrun.com/rss/news_data.xml", default_category="ecommerce", priority=6), + NewsFeedEndpoint(id="policy", name="政策", url="https://www.ebrun.com/rss/news_policy.xml", default_category="ecommerce", priority=7), + ), + priority=38, + source_tags=("business_news", "ecommerce", "retail", "china"), + default_category="ecommerce", + importance_weight=14, + health_policy=DEFAULT_NEWS_HEALTH_POLICY, + ), + NewsFeedSource( + id="google-news", + name="Google News", region="global", feed_url=_google_news_feed( REGION_PROFILES["global"].query, @@ -210,66 +556,88 @@ NEWS_FEED_SOURCES: tuple[NewsFeedSource, ...] = ( ceid="US:en", ), homepage_url="https://news.google.com/", - source_type="aggregated", - priority=30, - ), - NewsFeedSource( - id="google-americas", - name="Global Monitor / Americas", - region="americas", - feed_url=_google_news_feed( - REGION_PROFILES["americas"].query, - hl="en-US", - gl="US", - ceid="US:en", + feed_directory_url="https://news.google.com/rss", + feeds=( + NewsFeedEndpoint(id="world", name="全球", url=_google_news_feed(REGION_PROFILES["global"].query, hl="en-US", gl="US", ceid="US:en"), type="aggregated", region="global", default_category="politics", priority=1), + NewsFeedEndpoint(id="americas", name="美洲", url=_google_news_feed(REGION_PROFILES["americas"].query, hl="en-US", gl="US", ceid="US:en"), type="aggregated", region="americas", default_category="politics", priority=2), + NewsFeedEndpoint(id="europe", name="欧洲", url=_google_news_feed(REGION_PROFILES["europe"].query, hl="en-GB", gl="GB", ceid="GB:en"), type="aggregated", region="europe", default_category="politics", priority=3), + NewsFeedEndpoint(id="middle-east-africa", name="中东与非洲", url=_google_news_feed(REGION_PROFILES["middle-east-africa"].query, hl="en-US", gl="US", ceid="US:en"), type="aggregated", region="middle-east-africa", default_category="politics", priority=4), + NewsFeedEndpoint(id="asia-pacific", name="亚太", url=_google_news_feed(REGION_PROFILES["asia-pacific"].query, hl="en-SG", gl="SG", ceid="SG:en"), type="aggregated", region="asia-pacific", default_category="politics", priority=5), ), - homepage_url="https://news.google.com/", source_type="aggregated", - priority=40, - ), - NewsFeedSource( - id="google-europe", - name="Global Monitor / Europe", - region="europe", - feed_url=_google_news_feed( - REGION_PROFILES["europe"].query, - hl="en-GB", - gl="GB", - ceid="GB:en", - ), - homepage_url="https://news.google.com/", - source_type="aggregated", - priority=40, - ), - NewsFeedSource( - id="google-mea", - name="Global Monitor / MEA", - region="middle-east-africa", - feed_url=_google_news_feed( - REGION_PROFILES["middle-east-africa"].query, - hl="en-US", - gl="US", - ceid="US:en", - ), - homepage_url="https://news.google.com/", - source_type="aggregated", - priority=40, - ), - NewsFeedSource( - id="google-apac", - name="Global Monitor / APAC", - region="asia-pacific", - feed_url=_google_news_feed( - REGION_PROFILES["asia-pacific"].query, - hl="en-SG", - gl="SG", - ceid="SG:en", - ), - homepage_url="https://news.google.com/", - source_type="aggregated", - priority=40, + priority=90, + source_tags=("aggregated", "business_news", "global", "europe", "low_stability"), + default_category="politics", + importance_weight=-2, + health_policy=DEFAULT_NEWS_HEALTH_POLICY, ), ) +DEFAULT_NEWS_SOURCE_BY_ID: dict[str, NewsFeedSource] = {source.id: source for source in NEWS_FEED_SOURCES} +LEGACY_GOOGLE_NEWS_SOURCE_IDS = { + "global-scan", + "google-americas", + "google-europe", + "google-mea", + "google-apac", +} + +DEFAULT_SOURCE_TAGS: tuple[dict[str, Any], ...] = ( + {"key": "official_media", "label": "官方媒体", "color": "#2563eb", "enabled": True, "sort_order": 10}, + {"key": "official_data", "label": "官方数据", "color": "#059669", "enabled": True, "sort_order": 20}, + {"key": "business_news", "label": "商业新闻", "color": "#0f766e", "enabled": True, "sort_order": 30}, + {"key": "ecommerce", "label": "电商", "color": "#7c3aed", "enabled": True, "sort_order": 40}, + {"key": "finance", "label": "金融", "color": "#1d4ed8", "enabled": True, "sort_order": 45}, + {"key": "retail", "label": "零售", "color": "#ea580c", "enabled": True, "sort_order": 50}, + {"key": "logistics", "label": "物流", "color": "#16a34a", "enabled": True, "sort_order": 55}, + {"key": "industry_insight", "label": "行业洞察", "color": "#0891b2", "enabled": True, "sort_order": 60}, + {"key": "press_release", "label": "企业公告", "color": "#64748b", "enabled": True, "sort_order": 70}, + {"key": "china", "label": "中国", "color": "#dc2626", "enabled": True, "sort_order": 80}, + {"key": "global", "label": "全球", "color": "#475569", "enabled": True, "sort_order": 90}, + {"key": "us", "label": "美国", "color": "#1d4ed8", "enabled": True, "sort_order": 100}, + {"key": "europe", "label": "欧洲", "color": "#0284c7", "enabled": True, "sort_order": 110}, + {"key": "aggregated", "label": "聚合源", "color": "#9333ea", "enabled": True, "sort_order": 120}, + {"key": "low_stability", "label": "低稳定性", "color": "#f59e0b", "enabled": True, "sort_order": 130}, +) + +DEFAULT_CATEGORIES: tuple[dict[str, Any], ...] = ( + {"key": "politics", "label": "政治", "color": "#2563eb", "enabled": True, "sort_order": 10, "keywords": ["election", "government", "minister", "parliament", "sanction", "diplomatic", "policy", "summit", "选举", "政府", "制裁", "外交", "政策"]}, + {"key": "business", "label": "商业", "color": "#0f766e", "enabled": True, "sort_order": 20, "keywords": ["market", "company", "earnings", "trade", "supply chain", "merger", "retail", "consumer", "商业", "公司", "贸易", "供应链", "消费", "零售"]}, + {"key": "ecommerce", "label": "电商", "color": "#7c3aed", "enabled": True, "sort_order": 30, "keywords": ["e-commerce", "ecommerce", "online retail", "gmv", "marketplace", "shopify", "amazon", "tiktok shop", "网上零售", "电商", "直播电商", "跨境电商", "订单量", "物流指数", "履约"]}, + {"key": "finance", "label": "金融", "color": "#0369a1", "enabled": True, "sort_order": 40, "keywords": ["stock", "bond", "inflation", "central bank", "rate cut", "rate hike", "bank", "金融", "股市", "通胀", "央行", "利率"]}, + {"key": "sports", "label": "体育", "color": "#16a34a", "enabled": True, "sort_order": 50, "keywords": ["football", "basketball", "tennis", "match", "league", "world cup", "olympics", "体育", "足球", "篮球", "世界杯", "奥运"]}, + {"key": "technology", "label": "科技", "color": "#0891b2", "enabled": True, "sort_order": 60, "keywords": ["ai", "semiconductor", "chip", "cyber", "satellite", "software", "科技", "人工智能", "半导体", "芯片", "网络安全"]}, + {"key": "military", "label": "军事", "color": "#4b5563", "enabled": True, "sort_order": 70, "keywords": ["military", "missile", "airstrike", "defense", "warship", "军事", "导弹", "空袭", "防务", "军舰"]}, + {"key": "disaster", "label": "灾害", "color": "#dc2626", "enabled": True, "sort_order": 80, "keywords": ["earthquake", "flood", "wildfire", "typhoon", "hurricane", "灾害", "地震", "洪水", "山火", "台风"]}, + {"key": "energy", "label": "能源", "color": "#ca8a04", "enabled": True, "sort_order": 90, "keywords": ["oil", "gas", "opec", "energy", "power grid", "能源", "石油", "天然气", "电网"]}, + {"key": "society", "label": "社会", "color": "#64748b", "enabled": True, "sort_order": 100, "keywords": ["health", "education", "crime", "migration", "社会", "医疗", "教育", "犯罪", "移民"]}, + {"key": "culture", "label": "文化", "color": "#db2777", "enabled": True, "sort_order": 110, "keywords": ["film", "music", "art", "culture", "文化", "电影", "音乐", "艺术"]}, + {"key": "other", "label": "其他", "color": "#64748b", "enabled": True, "sort_order": 999, "keywords": []}, +) +ALLOWED_NEWS_CATEGORY_KEYS = tuple(item["key"] for item in DEFAULT_CATEGORIES) + +DEFAULT_ITEM_TAG_RULES: tuple[dict[str, Any], ...] = ( + {"key": "cross_border_ecommerce", "label": "跨境电商", "category": "ecommerce", "keywords": ["cross-border e-commerce", "cross border ecommerce", "跨境电商"]}, + {"key": "live_commerce", "label": "直播电商", "category": "ecommerce", "keywords": ["live commerce", "livestream shopping", "直播电商", "直播带货"]}, + {"key": "retail_data", "label": "零售数据", "category": "ecommerce", "keywords": ["online retail sales", "retail sales", "网上零售额", "社零", "社会消费品零售总额"]}, + {"key": "logistics_fulfillment", "label": "物流履约", "category": "ecommerce", "keywords": ["logistics", "fulfillment", "delivery", "物流指数", "履约", "配送"]}, + {"key": "platform_governance", "label": "平台治理", "category": "ecommerce", "keywords": ["platform regulation", "marketplace rules", "平台治理", "平台监管"]}, + {"key": "ai", "label": "AI", "category": "technology", "keywords": ["ai", "artificial intelligence", "人工智能", "大模型"]}, + {"key": "semiconductor", "label": "半导体", "category": "technology", "keywords": ["semiconductor", "chip", "半导体", "芯片"]}, + {"key": "election", "label": "选举", "category": "politics", "keywords": ["election", "vote", "campaign", "选举", "投票"]}, + {"key": "oil_price", "label": "油价", "category": "energy", "keywords": ["oil price", "crude", "油价", "原油"]}, + {"key": "football", "label": "足球", "category": "sports", "keywords": ["football", "soccer", "premier league", "足球"]}, + {"key": "supply_chain", "label": "供应链", "category": "business", "keywords": ["supply chain", "供应链"]}, +) + +def default_earth_news_sources_payload() -> dict[str, Any]: + return { + "cache_version": 1, + "source_tags": [dict(item) for item in DEFAULT_SOURCE_TAGS], + "categories": [dict(item) for item in DEFAULT_CATEGORIES], + "item_tag_rules": [dict(item) for item in DEFAULT_ITEM_TAG_RULES], + "sources": [serialize_news_source_config(source) for source in NEWS_FEED_SOURCES], + "health": {}, + } _REGION_CACHE: dict[str, CachedRegionFeed] = {} @@ -393,6 +761,68 @@ def _normalize_localizations(value: Any) -> dict[str, dict[str, str]]: return normalized +def _is_chinese_language(language: str | None) -> bool: + return str(language or "").lower() in {"zh", "zh-cn", "zh-hans", "chinese"} + + +def _is_english_language(language: str | None) -> bool: + return str(language or "").lower() in {"en", "en-us", "english"} + + +def _contains_cjk_text(value: str) -> bool: + return bool(re.search(r"[\u3400-\u9fff]", value or "")) + + +def _detect_content_language( + *, + title: str, + summary: str, + source: NewsFeedSource, + feed: NewsFeedEndpoint | None = None, +) -> str: + text = f"{title}\n{summary}" + if _contains_cjk_text(text): + return "zh-CN" + identity = {source.id.lower(), *(tag.lower() for tag in source.source_tags)} + if feed is not None: + identity.add(feed.id.lower()) + identity.update(tag.lower() for tag in feed.tags) + if identity & {"china", "chinese", "36kr", "ebrun", "stats-china-online-retail"}: + return "zh-CN" + return "en" + + +def _source_language_localizations( + *, + title: str, + summary: str, + content_language: str, +) -> dict[str, dict[str, str]]: + if _is_chinese_language(content_language): + return {"zh-CN": {"title": title, "summary": summary or title}} + return {} + + +def _target_localization_locale(item: ParsedNewsItem) -> str: + return "en-US" if _is_chinese_language(item.content_language) else DEFAULT_NEWS_LOCALE + + +def _target_locale_schema(locale: str) -> dict[str, dict[str, str]]: + if locale == "en-US": + return { + "en-US": { + "title": "faithful English title", + "summary": "one-sentence newswire-style English lead summary", + } + } + return { + "zh-CN": { + "title": "faithful Simplified Chinese title", + "summary": "one-sentence newswire-style Simplified Chinese lead summary", + } + } + + def _get_locale_text( item: ParsedNewsItem, key: str, @@ -404,16 +834,35 @@ def _get_locale_text( value = _coerce_str(localized.get(key)) if value: return value + if locale == "zh-CN" and _is_chinese_language(item.content_language): + return item.title if key == "title" else item.summary + if locale == "en-US" and _is_english_language(item.content_language): + return item.title if key == "title" else item.summary + fallback = item.localizations.get(DEFAULT_NEWS_LOCALE) + if isinstance(fallback, dict): + value = _coerce_str(fallback.get(key)) + if value: + return value + if locale == "en-US" and _is_chinese_language(item.content_language): + return item.title if key == "title" else item.summary return "" -def _has_default_localization(item: ParsedNewsItem) -> bool: - localized = item.localizations.get(DEFAULT_NEWS_LOCALE) +def _has_localization(item: ParsedNewsItem, locale: str) -> bool: + localized = item.localizations.get(locale) if not isinstance(localized, dict): return False return bool(_coerce_str(localized.get("title")) and _coerce_str(localized.get("summary"))) +def _has_default_localization(item: ParsedNewsItem) -> bool: + return _has_localization(item, DEFAULT_NEWS_LOCALE) + + +def _has_required_localization(item: ParsedNewsItem) -> bool: + return _has_localization(item, _target_localization_locale(item)) + + def apply_enrichment_patch_to_item( item: ParsedNewsItem, patch: dict[str, Any], @@ -630,16 +1079,21 @@ async def _infer_news_enrichment( item.enrichment_error = None prompt = prompt or await get_effective_prompt(None, NEWS_ENRICH_PROMPT_KEY) + target_locale = _target_localization_locale(item) + target_locale_name = "English" if target_locale == "en-US" else "Simplified Chinese" request = SituationalAnalysisRequest( - title="Enrich Earth news item with event location and zh-CN content", + title=f"Enrich Earth news item with event location and {target_locale} content", objective=prompt.prompt, system_prompt=prompt.system_prompt or None, context={ "news_item": { "title": item.title, "summary": item.summary, + "content_language": item.content_language, "source": item.source, "feed_name": item.feed_name, + "feed_id": item.feed_id, + "feed_type": item.feed_type, "feed_region": item.feed_region, "url": item.url, "published_at": ( @@ -658,20 +1112,16 @@ async def _infer_news_enrichment( "confidence": "number from 0 to 1", "reasoning_summary": "short string", }, - "localizations": { - "zh-CN": { - "title": "faithful Simplified Chinese title", - "summary": "one-sentence newswire-style Simplified Chinese lead summary", - } - }, + "localizations": _target_locale_schema(target_locale), }, }, constraints=[ "Return only strict JSON. Do not wrap it in markdown.", "For localizations, do not add facts that are absent from the RSS headline, description, source, or date.", - "Write zh-CN summary as one concise newswire-style sentence, like a breaking-news lead.", + f"Write {target_locale} summary as one concise {target_locale_name} newswire-style sentence, like a breaking-news lead.", "If the RSS description is thin, write a conservative one-sentence summary that says only what is supported.", - "Keep zh-CN summary factual, non-promotional, and avoid colon-heavy keyword labels.", + f"Keep {target_locale} summary factual, non-promotional, and avoid colon-heavy keyword labels.", + "If the source content is Chinese and target locale is en-US, translate faithfully into English instead of rewriting the story.", "Prefer the event location, not the newsroom or publisher headquarters.", "When a country visit or summit is the clear topic but the city is omitted, use the most likely host city only if it is broadly public knowledge.", "Use null for unknown fields instead of inventing details.", @@ -724,7 +1174,7 @@ async def _infer_news_enrichment( item.target_ai_error = None item.target_debug_note = f"ai inferred {target.label}" - item.localizations = localizations + item.localizations = {**dict(item.localizations or {}), **localizations} if localizations and item.target_ai_status in {"success", "skipped_text_hint"}: item.enrichment_status = "success" item.enrichment_error = None @@ -764,11 +1214,517 @@ async def _enrich_items_with_target_locations( def get_sources_for_region(region: str) -> list[NewsFeedSource]: return sorted( - [source for source in NEWS_FEED_SOURCES if source.region in {"global", region}], + [ + source + for source in NEWS_FEED_SOURCES + if source.enabled + and source.source_type in {"rss", "atom", "aggregated"} + and _source_matches_active_region(source, region) + ], key=lambda source: (source.priority, source.name), ) +def _clean_feed_urls(*values: Any) -> tuple[str, ...]: + urls: list[str] = [] + for value in values: + if isinstance(value, str): + parts = re.split(r"[\n,]+", value) + elif isinstance(value, (list, tuple)): + parts = [str(item) for item in value] + else: + parts = [] + for part in parts: + url = str(part or "").strip() + if url and url not in urls: + urls.append(url) + return tuple(urls) + + +def _clean_feed_tags(value: Any) -> tuple[str, ...]: + if isinstance(value, str): + parts = re.split(r"[\n,,]+", value) + elif isinstance(value, (list, tuple)): + parts = [str(item) for item in value] + else: + parts = [] + tags: list[str] = [] + for part in parts: + tag = str(part or "").strip() + if tag and tag not in tags: + tags.append(tag) + return tuple(tags) + + +def _slug_feed_id(value: str, fallback: str) -> str: + text_value = str(value or "").strip().lower() + slug = re.sub(r"[^a-z0-9_-]+", "-", text_value).strip("-") + return slug or fallback + + +def _feed_from_config(raw: Any, *, source: NewsFeedSource | None = None, index: int = 0) -> NewsFeedEndpoint | None: + if not isinstance(raw, dict): + return None + url = str(raw.get("url") or raw.get("feed_url") or "").strip() + if not url: + return None + feed_id = _slug_feed_id(str(raw.get("id") or raw.get("key") or raw.get("name") or ""), f"feed-{index + 1}") + fallback_type = source.source_type if source else "rss" + feed_type = str(raw.get("type") or raw.get("source_type") or fallback_type).strip().lower() or "rss" + default_category = str(raw.get("default_category") or (source.default_category if source else "other") or "other").strip() or "other" + try: + priority = int(raw.get("priority", index + 1)) + except (TypeError, ValueError): + priority = index + 1 + return NewsFeedEndpoint( + id=feed_id, + name=str(raw.get("name") or raw.get("label") or feed_id).strip() or feed_id, + url=url, + type=feed_type, + region=str(raw.get("region") or (source.region if source else "") or "").strip(), + enabled=raw.get("enabled") is not False and feed_type in {"rss", "atom", "aggregated"}, + default_category=default_category, + tags=_clean_feed_tags(raw.get("tags")), + priority=priority, + ) + + +def _source_feed_urls(source: NewsFeedSource) -> tuple[str, ...]: + return tuple(feed.url for feed in _source_feeds(source)) + + +def _source_feeds(source: NewsFeedSource) -> tuple[NewsFeedEndpoint, ...]: + if source.source_type == "reference": + return tuple() + feeds = tuple(feed for feed in source.feeds if feed.url) + if feeds: + return feeds + urls = _clean_feed_urls(source.feed_urls, source.feed_url) + return tuple( + NewsFeedEndpoint( + id=f"feed-{index + 1}", + name=source.name if len(urls) == 1 else f"{source.name} {index + 1}", + url=url, + type=source.source_type, + region=source.region, + enabled=source.enabled and source.source_type in {"rss", "atom", "aggregated"}, + default_category=source.default_category, + priority=index + 1, + ) + for index, url in enumerate(urls) + ) + + +def _feed_matches_active_region(feed: NewsFeedEndpoint, active_region: str | None) -> bool: + return ( + active_region is None + or active_region == "global" + or not feed.region + or feed.region in {"global", active_region} + ) + + +def _source_matches_active_region(source: NewsFeedSource, active_region: str) -> bool: + return active_region == "global" or source.region in {"global", active_region} + + +def _expected_feed_keys(sources: list[NewsFeedSource], *, active_region: str) -> set[tuple[str, str]]: + return { + (source.id, feed.id) + for source in sources + for feed in _source_feeds(source) + if source.enabled + and source.source_type in {"rss", "atom", "aggregated"} + and feed.enabled + and feed.type in {"rss", "atom", "aggregated"} + and _feed_matches_active_region(feed, active_region) + } + + +def _serialize_feed_endpoint(feed: NewsFeedEndpoint) -> dict[str, Any]: + return { + "id": feed.id, + "name": feed.name, + "url": feed.url, + "type": feed.type, + "region": feed.region, + "enabled": feed.enabled and feed.type in {"rss", "atom", "aggregated"}, + "default_category": feed.default_category, + "tags": list(feed.tags), + "priority": feed.priority, + } + + +def _should_repair_builtin_source_feeds( + source_id: str, + *, + raw_feeds: list[Any], + feed_urls: tuple[str, ...], + feed_url: str, + homepage_url: str, + feed_directory_url: str, +) -> bool: + default_source = DEFAULT_NEWS_SOURCE_BY_ID.get(source_id) + if not default_source or not default_source.feeds: + return False + if not raw_feeds: + return True + + current_urls: set[str] = set(feed_urls) + if feed_url: + current_urls.add(feed_url) + for raw_feed in raw_feeds: + if isinstance(raw_feed, dict): + current_url = str(raw_feed.get("url") or raw_feed.get("feed_url") or "").strip() + if current_url: + current_urls.add(current_url) + + non_feed_urls = { + url + for url in ( + homepage_url, + feed_directory_url, + default_source.homepage_url, + default_source.feed_directory_url, + ) + if url + } + return bool(current_urls & non_feed_urls) + + +def serialize_news_source_config(source: NewsFeedSource) -> dict[str, Any]: + feeds = _source_feeds(source) + feed_urls = tuple(feed.url for feed in feeds) + return { + "id": source.id, + "name": source.name, + "region": source.region, + "feed_url": source.feed_url or (feed_urls[0] if feed_urls else ""), + "feed_urls": list(feed_urls), + "feeds": [_serialize_feed_endpoint(feed) for feed in feeds], + "homepage_url": source.homepage_url, + "feed_directory_url": source.feed_directory_url, + "source_type": source.source_type, + "priority": source.priority, + "enabled": source.enabled and source.source_type in {"rss", "atom", "aggregated"}, + "source_tags": list(source.source_tags), + "default_category": source.default_category, + "importance_weight": source.importance_weight, + "health_policy": dict(source.health_policy or DEFAULT_NEWS_HEALTH_POLICY), + } + + +def _source_from_config(payload: dict[str, Any]) -> NewsFeedSource | None: + source_id = str(payload.get("id") or "").strip() + name = str(payload.get("name") or "").strip() + source_type = str(payload.get("source_type") or payload.get("type") or "rss").strip().lower() or "rss" + default_source = DEFAULT_NEWS_SOURCE_BY_ID.get(source_id) + homepage_url = str(payload.get("homepage_url") or "").strip() + feed_directory_url = str(payload.get("feed_directory_url") or "").strip() + raw_feeds = payload.get("feeds") if isinstance(payload.get("feeds"), list) else [] + feed_urls = _clean_feed_urls(payload.get("feed_urls"), payload.get("feed_url")) + feed_url = str(payload.get("feed_url") or "").strip() or (feed_urls[0] if feed_urls else "") + enabled_override: bool | None = None + if default_source: + homepage_url = homepage_url or default_source.homepage_url + feed_directory_url = feed_directory_url or default_source.feed_directory_url + if source_type == "reference" and default_source.source_type in {"rss", "atom", "aggregated"}: + source_type = default_source.source_type + if not raw_feeds: + enabled_override = default_source.enabled + if _should_repair_builtin_source_feeds( + source_id, + raw_feeds=raw_feeds, + feed_urls=feed_urls, + feed_url=feed_url, + homepage_url=homepage_url, + feed_directory_url=feed_directory_url, + ): + raw_feeds = [_serialize_feed_endpoint(feed) for feed in default_source.feeds] + feed_urls = tuple(feed.url for feed in default_source.feeds) + feed_url = default_source.feed_url or (feed_urls[0] if feed_urls else "") + if not feed_url and source_type == "reference": + feed_url = homepage_url + feed_urls = _clean_feed_urls(feed_url) + if not source_id or not name or (not feed_url and not raw_feeds): + return None + try: + priority = int(payload.get("priority", default_source.priority if default_source else 100)) + except (TypeError, ValueError): + priority = default_source.priority if default_source else 100 + try: + importance_weight = int(payload.get("importance_weight", default_source.importance_weight if default_source else 0)) + except (TypeError, ValueError): + importance_weight = default_source.importance_weight if default_source else 0 + raw_tags = payload.get("source_tags") + source_tags = ( + tuple(str(tag).strip() for tag in raw_tags if str(tag).strip()) + if isinstance(raw_tags, list) + else (default_source.source_tags if default_source else ()) + ) + health_policy = payload.get("health_policy") if isinstance(payload.get("health_policy"), dict) else {} + base_source = NewsFeedSource( + id=source_id, + name=name, + region=str(payload.get("region") or "global").strip() or "global", + feed_url=feed_url, + homepage_url=homepage_url, + feed_directory_url=feed_directory_url, + source_type=source_type, + priority=priority, + enabled=(enabled_override if enabled_override is not None else payload.get("enabled") is not False) + and source_type in {"rss", "atom", "aggregated"}, + source_tags=source_tags, + default_category=str(payload.get("default_category") or (default_source.default_category if default_source else "other") or "other").strip() or "other", + importance_weight=importance_weight, + health_policy={**DEFAULT_NEWS_HEALTH_POLICY, **health_policy}, + ) + feeds = tuple( + feed + for index, raw_feed in enumerate(raw_feeds) + for feed in [_feed_from_config(raw_feed, source=base_source, index=index)] + if feed is not None + ) + return NewsFeedSource( + **{ + **base_source.__dict__, + "feed_urls": feed_urls or ((feed_url,) if feed_url else tuple()), + "feeds": feeds, + } + ) + + +def _merge_legacy_google_news_sources(sources: list[dict[str, Any]]) -> list[dict[str, Any]]: + has_legacy_google = any(str(source.get("id") or "") in LEGACY_GOOGLE_NEWS_SOURCE_IDS for source in sources) + if not has_legacy_google: + return sources + + google_default = DEFAULT_NEWS_SOURCE_BY_ID.get("google-news") + if google_default is None: + return sources + + merged = [source for source in sources if str(source.get("id") or "") not in LEGACY_GOOGLE_NEWS_SOURCE_IDS] + if not any(str(source.get("id") or "") == "google-news" for source in merged): + merged.append(serialize_news_source_config(google_default)) + return sorted(merged, key=lambda source: (int(source.get("priority") or 100), str(source.get("name") or ""))) + + +def normalize_earth_news_sources_payload(payload: dict[str, Any] | None) -> dict[str, Any]: + defaults = default_earth_news_sources_payload() + if not isinstance(payload, dict): + return defaults + + normalized = { + "cache_version": int(payload.get("cache_version") or defaults["cache_version"]), + "source_tags": payload.get("source_tags") if isinstance(payload.get("source_tags"), list) else defaults["source_tags"], + "categories": payload.get("categories") if isinstance(payload.get("categories"), list) else defaults["categories"], + "item_tag_rules": payload.get("item_tag_rules") if isinstance(payload.get("item_tag_rules"), list) else defaults["item_tag_rules"], + "health": payload.get("health") if isinstance(payload.get("health"), dict) else {}, + "sources": [], + } + for source in payload.get("sources") if isinstance(payload.get("sources"), list) else defaults["sources"]: + if isinstance(source, dict): + coerced = _source_from_config(source) + if coerced: + normalized["sources"].append(serialize_news_source_config(coerced)) + if not normalized["sources"]: + normalized["sources"] = defaults["sources"] + else: + normalized["sources"] = _merge_legacy_google_news_sources(normalized["sources"]) + return normalized + + +async def get_earth_news_sources_payload(db: AsyncSession | None) -> dict[str, Any]: + if db is None: + payload = default_earth_news_sources_payload() + payload["is_default"] = True + return payload + result = await db.execute(select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_SOURCES_CATEGORY)) + record = result.scalar_one_or_none() + payload = normalize_earth_news_sources_payload(record.payload if record else None) + payload["is_default"] = record is None + return payload + + +async def save_earth_news_sources_payload(db: AsyncSession, payload: dict[str, Any]) -> dict[str, Any]: + normalized = normalize_earth_news_sources_payload(payload) + normalized["cache_version"] = int(normalized.get("cache_version") or 1) + 1 + result = await db.execute(select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_SOURCES_CATEGORY)) + record = result.scalar_one_or_none() + if record is None: + record = SystemSetting(category=EARTH_NEWS_SOURCES_CATEGORY, payload=normalized) + db.add(record) + else: + record.payload = normalized + await db.commit() + await db.refresh(record) + clear_earth_news_region_cache() + return {**normalize_earth_news_sources_payload(record.payload), "is_default": False} + + +async def reset_earth_news_sources_payload(db: AsyncSession) -> dict[str, Any]: + result = await db.execute(select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_SOURCES_CATEGORY)) + record = result.scalar_one_or_none() + if record is not None: + await db.delete(record) + await db.commit() + clear_earth_news_region_cache() + payload = default_earth_news_sources_payload() + payload["is_default"] = True + return payload + + +async def record_earth_news_source_health( + db: AsyncSession | None, + source_id: str, + health: dict[str, Any], +) -> None: + if db is None or not source_id or not callable(getattr(db, "execute", None)): + return + result = await db.execute(select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_SOURCES_CATEGORY)) + record = result.scalar_one_or_none() + payload = normalize_earth_news_sources_payload(record.payload if record else None) + payload_health = payload.get("health") if isinstance(payload.get("health"), dict) else {} + payload_health[source_id] = health + payload["health"] = payload_health + if record is None: + record = SystemSetting(category=EARTH_NEWS_SOURCES_CATEGORY, payload=payload) + db.add(record) + else: + record.payload = payload + await db.commit() + + +async def record_earth_news_sources_health( + db: AsyncSession | None, + health_by_source: dict[str, dict[str, Any]], +) -> None: + if db is None or not health_by_source or not callable(getattr(db, "execute", None)): + return + result = await db.execute(select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_SOURCES_CATEGORY)) + record = result.scalar_one_or_none() + payload = normalize_earth_news_sources_payload(record.payload if record else None) + payload_health = payload.get("health") if isinstance(payload.get("health"), dict) else {} + payload_health.update(health_by_source) + payload["health"] = payload_health + if record is None: + record = SystemSetting(category=EARTH_NEWS_SOURCES_CATEGORY, payload=payload) + db.add(record) + else: + record.payload = payload + await db.commit() + + +async def get_configured_sources_for_region(db: AsyncSession | None, region: str) -> list[NewsFeedSource]: + if db is None: + return get_sources_for_region(region) + payload = await get_earth_news_sources_payload(db) + sources = [ + source + for raw in payload.get("sources", []) + if isinstance(raw, dict) + for source in [_source_from_config(raw)] + if source + ] + enabled = [ + source + for source in sources + if source.enabled + and source.source_type in {"rss", "atom", "aggregated"} + and _source_matches_active_region(source, region) + ] + return sorted(enabled, key=lambda source: (source.priority, source.name)) + + +def _source_health_result( + source: NewsFeedSource, + *, + ok: bool, + count: int, + error: str | None = None, + status: str | None = None, + status_code: int | None = None, + content_type: str | None = None, + latency_ms: int | None = None, + final_url: str | None = None, + feed: NewsFeedEndpoint | None = None, +) -> dict[str, Any]: + return { + "source_id": source.id, + "feed_id": feed.id if feed else "", + "feed_name": feed.name if feed else "", + "feed_type": feed.type if feed else source.source_type, + "feed_url": feed.url if feed else source.feed_url, + "ok": ok, + "status": status or ("ok" if ok else "failed"), + "count": int(count or 0), + "item_count": int(count or 0), + "error": error, + "status_code": status_code, + "content_type": content_type or "", + "latency_ms": latency_ms, + "final_url": final_url or (feed.url if feed else source.feed_url), + "fetched_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), + } + + +def _format_source_fetch_error( + *, + status_code: int | None, + content_type: str, + body: str, + parsed_count: int, +) -> tuple[str | None, str]: + if status_code is not None and status_code >= 400: + return f"HTTP {status_code}", "http_error" + lower_content_type = content_type.lower() + body_prefix = body[:500].lower() + looks_like_html = "text/html" in lower_content_type or " dict[str, Any]: + source = _source_from_config(raw_source) + if source is None: + return {"ok": False, "count": 0, "error": "新闻源缺少 id、名称或 URL"} + if source.source_type not in {"rss", "atom", "aggregated"}: + health = _source_health_result( + source, + ok=False, + count=0, + error="参考链接仅用于记录官网、报告页或未来采集器线索,不参与 RSS/Atom 抓取。", + status="reference", + ) + await record_earth_news_source_health(db, source.id, health) + return { + "ok": False, + "count": 0, + "error": health["error"], + "health": health, + "source": serialize_news_source_config(source), + } + async with httpx.AsyncClient( + timeout=float(source.health_policy.get("timeout_seconds", REQUEST_TIMEOUT)), + follow_redirects=True, + headers={"User-Agent": USER_AGENT}, + ) as client: + fetched_source, items, error, health = await _fetch_source(client, source) + del fetched_source + await record_earth_news_source_health(db, source.id, health) + return { + "ok": error is None, + "count": len(items), + "item_count": len(items), + "error": error, + "health": health, + "source": serialize_news_source_config(source), + } + + def _strip_html(value: str) -> str: if not value: return "" @@ -814,37 +1770,49 @@ def _parse_datetime(raw: str | None) -> datetime | None: def _extract_item_text(element: ET.Element, *names: str) -> str: for name in names: node = element.find(name) + if node is None and not name.startswith("{"): + node = element.find(f"{{*}}{name}") if node is not None and node.text: return node.text.strip() return "" -def _parse_feed_entries(xml_text: str, source: NewsFeedSource) -> list[ParsedNewsItem]: +def _parse_feed_entries( + xml_text: str, + source: NewsFeedSource, + *, + feed: NewsFeedEndpoint | None = None, + config_payload: dict[str, Any] | None = None, +) -> list[ParsedNewsItem]: root = ET.fromstring(xml_text) items: list[ParsedNewsItem] = [] - rss_items = root.findall("./channel/item") - atom_entries = root.findall("{http://www.w3.org/2005/Atom}entry") + rss_items = root.findall("./channel/item") or root.findall(".//item") or root.findall(".//{*}item") + atom_entries = root.findall("{http://www.w3.org/2005/Atom}entry") or root.findall(".//{*}entry") nodes = rss_items or atom_entries for node in nodes[:MAX_ITEMS_PER_SOURCE]: if node.tag.endswith("entry"): - title = _extract_item_text(node, "{http://www.w3.org/2005/Atom}title") + title = _extract_item_text(node, "{http://www.w3.org/2005/Atom}title", "title") summary = _extract_item_text( node, "{http://www.w3.org/2005/Atom}summary", "{http://www.w3.org/2005/Atom}content", + "summary", + "content", ) - link_node = node.find("{http://www.w3.org/2005/Atom}link") + link_node = node.find("{http://www.w3.org/2005/Atom}link") or node.find("{*}link") link = link_node.get("href", "").strip() if link_node is not None else "" published = _extract_item_text( node, "{http://www.w3.org/2005/Atom}updated", "{http://www.w3.org/2005/Atom}published", + "updated", + "published", ) else: title = _extract_item_text(node, "title") - summary = _extract_item_text(node, "description", "content") + summary = _extract_item_text(node, "description", "content", "encoded") link = _extract_item_text(node, "link") published = _extract_item_text(node, "pubDate", "published", "updated") @@ -855,35 +1823,171 @@ def _parse_feed_entries(xml_text: str, source: NewsFeedSource) -> list[ParsedNew item_source = source.name display_title = clean_title - if source.source_type == "aggregated" and " - " in clean_title: + feed_type = feed.type if feed else source.source_type + if feed_type == "aggregated" and " - " in clean_title: parts = clean_title.rsplit(" - ", 1) display_title = parts[0].strip() item_source = _normalize_source_name(parts[1], source.name) - items.append( - ParsedNewsItem( - id=f"{source.id}:{hashlib.sha1(link.encode('utf-8')).hexdigest()[:12]}", + content_language = _detect_content_language( + title=display_title, + summary=clean_summary, + source=source, + feed=feed, + ) + feed_id_segment = f"{feed.id}:" if feed is not None and len(source.feeds or ()) > 1 else "" + item = ParsedNewsItem( + id=f"{source.id}:{feed_id_segment}{hashlib.sha1(link.encode('utf-8')).hexdigest()[:12]}", + title=display_title, + summary=clean_summary, + url=link, + source=item_source, + feed_name=feed.name if feed else source.name, + feed_region=(feed.region if feed and feed.region else source.region), + homepage_url=source.homepage_url, + published_at=_parse_datetime(published), + content_language=content_language, + localizations=_source_language_localizations( title=display_title, summary=clean_summary, - url=link, - source=item_source, - feed_name=source.name, - feed_region=source.region, - homepage_url=source.homepage_url, - published_at=_parse_datetime(published), - ) + content_language=content_language, + ), + feed_id=feed.id if feed else "", + feed_type=feed_type, + feed_default_category=(feed.default_category if feed else source.default_category) or "other", + source_tags=list(source.source_tags), ) + apply_news_classification(item, source, feed=feed, config_payload=config_payload) + items.append(item) return items +def _contains_keyword(text: str, keyword: str) -> bool: + keyword_text = str(keyword or "").strip().lower() + if not keyword_text: + return False + if re.search(r"[\u4e00-\u9fff]", keyword_text): + return keyword_text in text + return re.search(rf"(? int: + score = 0 + keywords = category.get("keywords") if isinstance(category.get("keywords"), list) else [] + for keyword in keywords: + if _contains_keyword(title_text, keyword): + score += 3 + elif _contains_keyword(text, keyword): + score += 1 + return score + + +def _importance_level(score: int) -> str: + if score >= 80: + return "critical" + if score >= 60: + return "high" + if score >= 35: + return "medium" + return "low" + + +def apply_news_classification( + item: ParsedNewsItem, + source: NewsFeedSource, + *, + feed: NewsFeedEndpoint | None = None, + config_payload: dict[str, Any] | None = None, +) -> ParsedNewsItem: + config = normalize_earth_news_sources_payload(config_payload) + title_text = item.title.lower() + combined_text = f"{item.title} {item.summary} {item.source} {item.feed_name}".lower() + + feed_default_category = (feed.default_category if feed else item.feed_default_category) or source.default_category or "other" + best_key = feed_default_category + best_score = 0 + second_score = 0 + for category in config["categories"]: + if not isinstance(category, dict) or category.get("enabled") is False: + continue + score = _score_category(combined_text, title_text, category) + if score > best_score: + second_score = best_score + best_score = score + best_key = str(category.get("key") or "other") + elif score > second_score: + second_score = score + + item_tags: list[str] = [] + for rule in config["item_tag_rules"]: + if not isinstance(rule, dict): + continue + keywords = rule.get("keywords") if isinstance(rule.get("keywords"), list) else [] + if any(_contains_keyword(combined_text, keyword) for keyword in keywords): + tag_key = str(rule.get("key") or "").strip() + if tag_key and tag_key not in item_tags: + item_tags.append(tag_key) + if best_score < 3 and rule.get("category"): + best_key = str(rule["category"]) + best_score = 3 + + confidence = round(best_score / (best_score + second_score + 1), 2) if best_score else 0.35 + if best_score < 3 and feed_default_category: + best_key = feed_default_category + confidence = 0.45 + + importance_score = max(0, min(100, 18 + source.importance_weight + best_score * 6)) + reasons: list[str] = [] + source_tag_set = set(source.source_tags) + if "official_data" in source_tag_set: + importance_score += 20 + reasons.append("官方数据源") + if "press_release" in source_tag_set: + importance_score = max(0, importance_score - 12) + reasons.append("企业公告基础权重较低") + ecommerce_terms = ("网上零售额", "电商物流指数", "gmv", "订单量", "物流指数", "履约", "直播电商", "跨境电商") + if any(_contains_keyword(combined_text, term) for term in ecommerce_terms): + importance_score += 25 + reasons.append("命中电商数据指标") + major_platforms = ("amazon", "shopify", "walmart", "alibaba", "jd.com", "pinduoduo", "tiktok shop", "shein", "阿里", "京东", "拼多多", "抖音") + if any(_contains_keyword(combined_text, term) for term in major_platforms): + importance_score += 15 + reasons.append("涉及大型平台") + if any(term in combined_text for term in ("同比", "环比", "%", "billion", "million", "增长", "下降")): + importance_score += 10 + reasons.append("包含量化指标") + + importance_score = max(0, min(100, importance_score)) + item.category = best_key or "other" + item.item_tags = item_tags + item.tagging_source = "rules" + item.tagging_confidence = confidence + item.importance_score = importance_score + item.importance_level = _importance_level(importance_score) + item.importance_reasons = reasons or ["按来源权重和分类规则计算"] + item.market_impact = "global" if "global" in source_tag_set else "national" if {"china", "us"} & source_tag_set else "sector" + item.source_tags = list(source.source_tags) + return item + + def _serialize_sources(sources: list[NewsFeedSource]) -> list[dict[str, Any]]: return [ { "id": source.id, "name": source.name, "region": source.region, + "feed_url": source.feed_url, + "feed_urls": list(_source_feed_urls(source)), + "feeds": [_serialize_feed_endpoint(feed) for feed in _source_feeds(source)], "homepage_url": source.homepage_url, + "feed_directory_url": source.feed_directory_url, + "source_type": source.source_type, + "priority": source.priority, + "enabled": source.enabled, + "source_tags": list(source.source_tags), + "default_category": source.default_category, + "importance_weight": source.importance_weight, } for source in sources ] @@ -926,6 +2030,26 @@ def _content_patch(item: ParsedNewsItem) -> dict[str, Any]: } +def _news_meta_patch(item: ParsedNewsItem) -> dict[str, Any]: + source_id = item.id.split(":", 1)[0] if ":" in item.id else "" + return { + "source_id": source_id, + "source_tags": list(item.source_tags or []), + "feed_id": item.feed_id, + "feed_name": item.feed_name, + "feed_type": item.feed_type, + "feed_default_category": item.feed_default_category, + "category": item.category, + "item_tags": list(item.item_tags or []), + "tagging_source": item.tagging_source, + "tagging_confidence": item.tagging_confidence, + "importance_score": item.importance_score, + "importance_level": item.importance_level, + "importance_reasons": list(item.importance_reasons or []), + "market_impact": item.market_impact, + } + + def build_anchor_location_patch( item: ParsedNewsItem, *, @@ -959,6 +2083,7 @@ def build_anchor_location_patch( "queue_available": queue_available, "target": None, "anchor": _serialize_anchor(anchor), + "news_meta": _news_meta_patch(item), }, **content_patch, } @@ -982,6 +2107,7 @@ def build_target_location_patch(item: ParsedNewsItem, target: NewsTargetLocation "debug_note": item.target_debug_note, "target": _serialize_target(target), "anchor": _serialize_anchor(anchor), + "news_meta": _news_meta_patch(item), }, **_content_patch(item), } @@ -1001,9 +2127,21 @@ def build_target_location_job_payload(item: ParsedNewsItem) -> dict[str, Any]: "url": item.url, "source": item.source, "feed_name": item.feed_name, + "feed_id": item.feed_id, + "feed_type": item.feed_type, + "feed_default_category": item.feed_default_category, "feed_region": item.feed_region, "homepage_url": item.homepage_url, "published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None, + "source_tags": list(item.source_tags or []), + "category": item.category, + "item_tags": list(item.item_tags or []), + "tagging_source": item.tagging_source, + "tagging_confidence": item.tagging_confidence, + "importance_score": item.importance_score, + "importance_level": item.importance_level, + "importance_reasons": list(item.importance_reasons or []), + "market_impact": item.market_impact, } @@ -1020,26 +2158,43 @@ def parsed_news_item_from_job_payload(payload: dict[str, Any]) -> ParsedNewsItem url=str(payload.get("url") or ""), source=str(payload.get("source") or ""), feed_name=str(payload.get("feed_name") or ""), + feed_id=str(payload.get("feed_id") or ""), + feed_type=str(payload.get("feed_type") or "rss"), + feed_default_category=str(payload.get("feed_default_category") or payload.get("category") or "other"), feed_region=str(payload.get("feed_region") or "global"), homepage_url=str(payload.get("homepage_url") or ""), published_at=_parse_datetime(_coerce_str(payload.get("published_at"))), + source_tags=list(payload.get("source_tags") or []), + category=str(payload.get("category") or "other"), + item_tags=list(payload.get("item_tags") or []), + tagging_source=str(payload.get("tagging_source") or "rules"), + tagging_confidence=float(payload.get("tagging_confidence") or 0), + importance_score=int(payload.get("importance_score") or 0), + importance_level=str(payload.get("importance_level") or "low"), + importance_reasons=list(payload.get("importance_reasons") or []), + market_impact=str(payload.get("market_impact") or "none"), ) -def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, Any]: +def _serialize_item(item: ParsedNewsItem, *, active_region: str, locale: str = DEFAULT_NEWS_LOCALE) -> dict[str, Any]: published_at = item.published_at location_patch = item.location_patch or build_target_location_patch(item, item.target_location) + source_id = item.id.split(":", 1)[0] if ":" in item.id else "" return { "id": item.id, + "source_id": source_id, "title": item.title, "summary": item.summary, "content_language": item.content_language, "localizations": item.localizations, - "display_title": _get_locale_text(item, "title"), - "display_summary": _get_locale_text(item, "summary"), + "display_title": _get_locale_text(item, "title", locale=locale), + "display_summary": _get_locale_text(item, "summary", locale=locale), "url": item.url, "source": item.source, "feed_name": item.feed_name, + "feed_id": item.feed_id, + "feed_type": item.feed_type, + "feed_default_category": item.feed_default_category, "region": item.feed_region, "display_region": get_region_anchor(item.feed_region).label, "homepage_url": item.homepage_url, @@ -1054,6 +2209,15 @@ def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, An "enrichment_error": item.enrichment_error, "enriched_at": _serialize_enriched_at(item.enriched_at), "is_focus_match": item.feed_region == active_region, + "source_tags": list(item.source_tags or []), + "category": item.category, + "item_tags": list(item.item_tags or []), + "tagging_source": item.tagging_source, + "tagging_confidence": item.tagging_confidence, + "importance_score": item.importance_score, + "importance_level": item.importance_level, + "importance_reasons": list(item.importance_reasons or []), + "market_impact": item.market_impact, } @@ -1067,6 +2231,10 @@ def _build_payload( sources: list[NewsFeedSource], errors: list[str], stale: bool, + categories: set[str] | None = None, + source_ids: set[str] | None = None, + limit: int = MAX_ITEMS_TOTAL, + locale: str = DEFAULT_NEWS_LOCALE, generated_at: datetime | None = None, ) -> dict[str, Any]: profile = get_region_profile(active_region) @@ -1082,9 +2250,16 @@ def _build_payload( "accent": profile.accent, }, "sources": _serialize_sources(sources), - "items": [_serialize_item(item, active_region=active_region) for item in items], + "filters": { + "region": active_region, + "categories": sorted(categories or []), + "sources": sorted(source_ids or []), + "limit": limit, + "locale": locale, + }, + "items": [_serialize_item(item, active_region=active_region, locale=locale) for item in items], "cruise_items": [ - _serialize_item(item, active_region=active_region) + _serialize_item(item, active_region=active_region, locale=locale) for item in (cruise_items if cruise_items is not None else items) ], "errors": errors, @@ -1092,7 +2267,12 @@ def _build_payload( } -def _rank_and_trim_items(items: list[ParsedNewsItem], *, active_region: str) -> list[ParsedNewsItem]: +def _rank_and_trim_items( + items: list[ParsedNewsItem], + *, + active_region: str, + limit: int = MAX_ITEMS_TOTAL, +) -> list[ParsedNewsItem]: deduped: dict[str, ParsedNewsItem] = {} for item in items: key = item.url.strip() or item.title.strip().lower() @@ -1102,12 +2282,44 @@ def _rank_and_trim_items(items: list[ParsedNewsItem], *, active_region: str) -> return sorted( deduped.values(), key=lambda item: ( - item.feed_region != active_region, + False if active_region == "global" else item.feed_region != active_region, item.published_at is None, -(item.published_at.timestamp() if item.published_at else 0), item.feed_name, ), - )[:MAX_ITEMS_TOTAL] + )[:limit] + + +def _filter_news_items_by_categories( + items: list[ParsedNewsItem], + categories: set[str] | None, +) -> list[ParsedNewsItem]: + if not categories: + return items + return [item for item in items if (item.category or "other") in categories] + + +def _filter_news_items_by_source_ids( + items: list[ParsedNewsItem], + source_ids: set[str] | None, +) -> list[ParsedNewsItem]: + if not source_ids: + return items + return [ + item for item in items + if (item.id.split(":", 1)[0] if ":" in item.id else "") in source_ids + ] + + +async def _call_store_list_items(list_fn, db: AsyncSession, **kwargs): + try: + return await list_fn(db, **kwargs) + except TypeError as error: + if "source_ids" not in str(error): + raise + legacy_kwargs = dict(kwargs) + legacy_kwargs.pop("source_ids", None) + return await list_fn(db, **legacy_kwargs) def _get_cached_region_feed(region: str) -> CachedRegionFeed | None: @@ -1120,6 +2332,10 @@ def _get_cached_region_feed(region: str) -> CachedRegionFeed | None: return cached +def clear_earth_news_region_cache() -> None: + _REGION_CACHE.clear() + + def _store_region_cache(region: str, *, items: list[ParsedNewsItem], sources: list[NewsFeedSource]) -> None: _REGION_CACHE[region] = CachedRegionFeed( region=region, @@ -1145,7 +2361,7 @@ async def _apply_cached_locations_and_enqueue(items: list[ParsedNewsItem]) -> li cached_patch = await get_cached_target_location_patch(item.id) if cached_patch: apply_enrichment_patch_to_item(item, cached_patch) - if not _has_default_localization(item): + if not _has_required_localization(item): queued = await enqueue_item(item, force=True) if queued and item.enrichment_status in { "pending", @@ -1185,42 +2401,192 @@ async def _enqueue_unverified_locations(items: list[ParsedNewsItem]) -> None: if ( item.location_patch is None or item.location_patch.get("verified") is False - or not _has_default_localization(item) + or not _has_required_localization(item) ) ) ) +async def _fetch_single_feed_url( + client: httpx.AsyncClient, + source: NewsFeedSource, + feed: NewsFeedEndpoint, + *, + config_payload: dict[str, Any] | None = None, +) -> tuple[NewsFeedSource, list[ParsedNewsItem], str | None, dict[str, Any]]: + started_at = perf_counter() + try: + response = await client.get(feed.url) + latency_ms = int((perf_counter() - started_at) * 1000) + status_code = response.status_code + content_type = response.headers.get("content-type", "") + if status_code >= 400: + error = f"HTTP {status_code}" + return source, [], error, _source_health_result( + source, + ok=False, + count=0, + error=error, + status="http_error", + status_code=status_code, + content_type=content_type, + latency_ms=latency_ms, + final_url=str(response.url), + feed=feed, + ) + try: + items = _parse_feed_entries(response.text, source, feed=feed, config_payload=config_payload) + except Exception as exc: + error, status = _format_source_fetch_error( + status_code=status_code, + content_type=content_type, + body=response.text, + parsed_count=0, + ) + if error is None: + error = str(exc) + status = "format_error" + return source, [], error, _source_health_result( + source, + ok=False, + count=0, + error=error, + status=status, + status_code=status_code, + content_type=content_type, + latency_ms=latency_ms, + final_url=str(response.url), + feed=feed, + ) + error, status = _format_source_fetch_error( + status_code=status_code, + content_type=content_type, + body=response.text, + parsed_count=len(items), + ) + return source, items, error, _source_health_result( + source, + ok=error is None, + count=len(items), + error=error, + status=status, + status_code=status_code, + content_type=content_type, + latency_ms=latency_ms, + final_url=str(response.url), + feed=feed, + ) + except httpx.TimeoutException as exc: + error = f"请求超时: {exc}" + return source, [], error, _source_health_result( + source, + ok=False, + count=0, + error=error, + status="timeout", + latency_ms=int((perf_counter() - started_at) * 1000), + feed=feed, + ) + except Exception as exc: + error = str(exc) + return source, [], error, _source_health_result( + source, + ok=False, + count=0, + error=error, + status="network_error", + latency_ms=int((perf_counter() - started_at) * 1000), + feed=feed, + ) + + async def _fetch_source( client: httpx.AsyncClient, source: NewsFeedSource, -) -> tuple[NewsFeedSource, list[ParsedNewsItem], str | None]: - try: - response = await client.get(source.feed_url) - response.raise_for_status() - return source, _parse_feed_entries(response.text, source), None - except Exception as exc: - return source, [], str(exc) + *, + active_region: str | None = None, + config_payload: dict[str, Any] | None = None, +) -> tuple[NewsFeedSource, list[ParsedNewsItem], str | None, dict[str, Any]]: + feeds = tuple( + feed + for feed in _source_feeds(source) + if feed.enabled and feed.type in {"rss", "atom", "aggregated"} + and _feed_matches_active_region(feed, active_region) + ) + if not feeds: + health = _source_health_result(source, ok=False, count=0, error="新闻源缺少已启用的 Feed 子项。", status="format_error") + return source, [], health["error"], health + + results = await asyncio.gather(*( + _fetch_single_feed_url(client, source, feed, config_payload=config_payload) + for feed in sorted(feeds, key=lambda item: (item.priority, item.name)) + )) + merged_items: list[ParsedNewsItem] = [] + feed_results: list[dict[str, Any]] = [] + errors: list[str] = [] + for _source, items, error, health in results: + feed_results.append(health) + if error: + errors.append(f"{health.get('final_url') or source.feed_url}: {error}") + continue + merged_items.extend(items) + + deduped: dict[str, ParsedNewsItem] = {} + for item in merged_items: + key = item.url.strip() or item.title.strip().lower() + if key and key not in deduped: + deduped[key] = item + items = list(deduped.values()) + ok = bool(items) + error = None if ok else "; ".join(errors) or "未解析到 RSS/Atom 条目,请检查 Feed 地址或源格式。" + latency_ms = sum(int(result.get("latency_ms") or 0) for result in feed_results) + status = "ok" if ok else (feed_results[0].get("status") if len(feed_results) == 1 else "failed") + status_code = next((result.get("status_code") for result in feed_results if result.get("status_code")), None) + content_type = ", ".join(sorted({str(result.get("content_type") or "") for result in feed_results if result.get("content_type")})) + health = _source_health_result( + source, + ok=ok, + count=len(items), + error=error, + status=status, + status_code=int(status_code) if isinstance(status_code, int) else None, + content_type=content_type, + latency_ms=latency_ms, + final_url=source.feed_url, + ) + health["feed_results"] = feed_results + return source, items, error, health async def _fetch_rss_items_for_sources( sources: list[NewsFeedSource], -) -> tuple[list[ParsedNewsItem], list[str]]: + *, + active_region: str | None = None, + config_payload: dict[str, Any] | None = None, +) -> tuple[list[ParsedNewsItem], list[str], dict[str, dict[str, Any]]]: errors: list[str] = [] async with httpx.AsyncClient( timeout=REQUEST_TIMEOUT, follow_redirects=True, headers={"User-Agent": USER_AGENT}, ) as client: - results = await asyncio.gather(*(_fetch_source(client, source) for source in sources)) + if config_payload is None: + results = await asyncio.gather(*(_fetch_source(client, source, active_region=active_region) for source in sources)) + else: + results = await asyncio.gather(*( + _fetch_source(client, source, active_region=active_region, config_payload=config_payload) + for source in sources + )) fetched_items: list[ParsedNewsItem] = [] - for source, items, error in results: + health_by_source: dict[str, dict[str, Any]] = {} + for source, items, error, health in results: + health_by_source[source.id] = health if error: errors.append(f"{source.name}: {error}") continue fetched_items.extend(items) - return fetched_items, errors + return fetched_items, errors, health_by_source def _needs_rss_supplement(*, item_count: int, newest_at: datetime | None) -> bool: @@ -1238,9 +2604,19 @@ async def _get_earth_news_payload_from_rss_only( lon: float | None, active_region: str, sources: list[NewsFeedSource], + categories: set[str] | None = None, + source_ids: set[str] | None = None, + limit: int = MAX_ITEMS_TOTAL, + locale: str = DEFAULT_NEWS_LOCALE, ) -> dict[str, Any]: - fetched_items, errors = await _fetch_rss_items_for_sources(sources) - ranked_items = _rank_and_trim_items(fetched_items, active_region=active_region) + fetched_items, errors, _health_by_source = await _fetch_rss_items_for_sources(sources, active_region=active_region) + ranked_items = _filter_news_items_by_source_ids( + _filter_news_items_by_categories( + _rank_and_trim_items(fetched_items, active_region=active_region, limit=limit), + categories, + ), + source_ids, + )[:limit] if ranked_items: ranked_items = await _apply_cached_locations_and_enqueue(ranked_items) _store_region_cache(active_region, items=ranked_items, sources=sources) @@ -1252,11 +2628,19 @@ async def _get_earth_news_payload_from_rss_only( sources=sources, errors=errors, stale=False, + categories=categories, + source_ids=source_ids, + limit=limit, + locale=locale, ) cached = _get_cached_region_feed(active_region) if cached: - cached.items = await _apply_cached_locations_and_enqueue(cached.items) + cached_items = _filter_news_items_by_source_ids( + _filter_news_items_by_categories(cached.items, categories), + source_ids, + )[:limit] + cached.items = await _apply_cached_locations_and_enqueue(cached_items) return _build_payload( lat=lat, lon=lon, @@ -1265,6 +2649,10 @@ async def _get_earth_news_payload_from_rss_only( sources=cached.sources, errors=errors, stale=True, + categories=categories, + source_ids=source_ids, + limit=limit, + locale=locale, generated_at=cached.fetched_at, ) @@ -1276,6 +2664,10 @@ async def _get_earth_news_payload_from_rss_only( sources=sources, errors=errors, stale=False, + categories=categories, + source_ids=source_ids, + limit=limit, + locale=locale, ) @@ -1283,12 +2675,23 @@ async def get_earth_news_payload( lat: float | None = None, lon: float | None = None, *, + region: str | None = None, + categories: set[str] | None = None, + source_ids: set[str] | None = None, + limit: int = MAX_ITEMS_TOTAL, + locale: str = DEFAULT_NEWS_LOCALE, provider_client: AIProviderClient | None = None, db: AsyncSession | None = None, ) -> dict[str, Any]: del provider_client - active_region = determine_focus_region(lat, lon) - sources = get_sources_for_region(active_region) + active_region = region if region in REGION_ANCHORS else determine_focus_region(lat, lon) + has_settings_db = db is not None and callable(getattr(db, "execute", None)) + source_config_payload = await get_earth_news_sources_payload(db) if has_settings_db else None + sources = ( + await get_configured_sources_for_region(db, active_region) + if has_settings_db + else get_sources_for_region(active_region) + ) if db is None: return await _get_earth_news_payload_from_rss_only( @@ -1296,9 +2699,14 @@ async def get_earth_news_payload( lon=lon, active_region=active_region, sources=sources, + categories=categories, + source_ids=source_ids, + limit=limit, + locale=locale, ) from app.services.earth_news_store import ( + get_earth_news_feed_coverage, get_earth_news_freshness, list_earth_news_cruise_items, list_earth_news_items, @@ -1308,20 +2716,43 @@ async def get_earth_news_payload( errors: list[str] = [] item_count, newest_at = await get_earth_news_freshness(db, active_region=active_region) should_supplement = _needs_rss_supplement(item_count=item_count, newest_at=newest_at) + if not should_supplement and callable(getattr(db, "execute", None)): + expected_feed_keys = _expected_feed_keys(sources, active_region=active_region) + if expected_feed_keys: + covered_feed_keys = await get_earth_news_feed_coverage( + db, + active_region=active_region, + recent_after=datetime.now(UTC) - timedelta(seconds=RSS_SUPPLEMENT_MAX_AGE_SECONDS), + ) + should_supplement = bool(expected_feed_keys - covered_feed_keys) if should_supplement: - fetched_items, errors = await _fetch_rss_items_for_sources(sources) + if source_config_payload is None: + fetched_items, errors, health_by_source = await _fetch_rss_items_for_sources(sources, active_region=active_region) + else: + fetched_items, errors, health_by_source = await _fetch_rss_items_for_sources( + sources, + active_region=active_region, + config_payload=source_config_payload, + ) + await record_earth_news_sources_health(db, health_by_source) ranked_fetched_items = _rank_and_trim_items(fetched_items, active_region=active_region) await upsert_earth_news_items(db, ranked_fetched_items) - items = await list_earth_news_items( + items = await _call_store_list_items( + list_earth_news_items, db, active_region=active_region, - limit=MAX_ITEMS_TOTAL, + limit=limit, + categories=categories, + source_ids=source_ids, ) if hasattr(db, "execute"): - cruise_items = await list_earth_news_cruise_items( + cruise_items = await _call_store_list_items( + list_earth_news_cruise_items, db, - limit=MAX_ITEMS_TOTAL * len(REGION_ANCHORS), + limit=min(max(limit, MAX_ITEMS_TOTAL) * len(REGION_ANCHORS), 500), + categories=categories, + source_ids=source_ids, ) else: cruise_items = items @@ -1337,4 +2768,8 @@ async def get_earth_news_payload( sources=sources, errors=errors, stale=stale, + categories=categories, + source_ids=source_ids, + limit=limit, + locale=locale, ) diff --git a/backend/app/services/earth_news_store.py b/backend/app/services/earth_news_store.py index 4cebef81..5264cce4 100644 --- a/backend/app/services/earth_news_store.py +++ b/backend/app/services/earth_news_store.py @@ -11,6 +11,7 @@ from app.services.earth_news import ( ParsedNewsItem, apply_enrichment_patch_to_item, build_anchor_location_patch, + _news_meta_patch, ) @@ -34,6 +35,8 @@ def _location_patch_from_record(record: EarthNewsItem) -> dict[str, Any]: def record_to_parsed_news_item(record: EarthNewsItem) -> ParsedNewsItem: + location_meta = dict(record.location_meta or {}) + news_meta = location_meta.get("news_meta") if isinstance(location_meta.get("news_meta"), dict) else {} item = ParsedNewsItem( id=record.id, title=record.title, @@ -49,11 +52,29 @@ def record_to_parsed_news_item(record: EarthNewsItem) -> ParsedNewsItem: enrichment_status=record.enrichment_status or "pending", enrichment_error=record.enrichment_error, enriched_at=_coerce_datetime(record.enriched_at), + source_tags=list(news_meta.get("source_tags") or []), + feed_id=str(news_meta.get("feed_id") or ""), + feed_type=str(news_meta.get("feed_type") or "rss"), + feed_default_category=str(news_meta.get("feed_default_category") or "other"), + category=str(news_meta.get("category") or "other"), + item_tags=list(news_meta.get("item_tags") or []), + tagging_source=str(news_meta.get("tagging_source") or "rules"), + tagging_confidence=float(news_meta.get("tagging_confidence") or 0), + importance_score=int(news_meta.get("importance_score") or 0), + importance_level=str(news_meta.get("importance_level") or "low"), + importance_reasons=list(news_meta.get("importance_reasons") or []), + market_impact=str(news_meta.get("market_impact") or "none"), ) return apply_enrichment_patch_to_item(item, _location_patch_from_record(record)) def _query_sort_key(active_region: str): + if active_region == "global": + return ( + EarthNewsItem.published_at.is_(None), + EarthNewsItem.published_at.desc().nullslast(), + EarthNewsItem.feed_name.asc(), + ) return ( EarthNewsItem.region != active_region, EarthNewsItem.published_at.is_(None), @@ -62,28 +83,93 @@ def _query_sort_key(active_region: str): ) +def _category_filter_clause(categories: set[str] | None): + if not categories: + return None + return EarthNewsItem.location_meta.op("->")("news_meta").op("->>")("category").in_(sorted(categories)) + + +def _source_filter_clause(source_ids: set[str] | None): + if not source_ids: + return None + return EarthNewsItem.location_meta.op("->")("news_meta").op("->>")("source_id").in_(sorted(source_ids)) + + +def _record_source_id(record: EarthNewsItem) -> str: + location_meta = dict(record.location_meta or {}) + news_meta = location_meta.get("news_meta") if isinstance(location_meta.get("news_meta"), dict) else {} + source_id = str(news_meta.get("source_id") or "").strip() + if source_id: + return source_id + if isinstance(record.id, str) and ":" in record.id: + return record.id.split(":", 1)[0] + return record.feed_name or record.source or record.id + + +def _diversify_records_by_source(records: list[EarthNewsItem], *, limit: int) -> list[EarthNewsItem]: + if limit <= 0 or len(records) <= limit: + return records[:limit] + buckets: dict[str, list[EarthNewsItem]] = {} + order: list[str] = [] + for record in records: + source_id = _record_source_id(record) + if source_id not in buckets: + buckets[source_id] = [] + order.append(source_id) + buckets[source_id].append(record) + + diversified: list[EarthNewsItem] = [] + while len(diversified) < limit and order: + next_order: list[str] = [] + for source_id in order: + bucket = buckets.get(source_id) or [] + if bucket and len(diversified) < limit: + diversified.append(bucket.pop(0)) + if bucket: + next_order.append(source_id) + order = next_order + return diversified + + async def list_earth_news_items( db: AsyncSession, *, active_region: str, limit: int, + categories: set[str] | None = None, + source_ids: set[str] | None = None, ) -> list[ParsedNewsItem]: - regions = {"global", active_region} - result = await db.execute( + query_limit = limit if source_ids else min(max(limit * 4, limit), 100) + query = ( select(EarthNewsItem) - .where(EarthNewsItem.region.in_(regions)) .order_by(*_query_sort_key(active_region)) - .limit(limit) + .limit(query_limit) ) - return [record_to_parsed_news_item(record) for record in result.scalars().all()] + if active_region != "global": + query = query.where(EarthNewsItem.region.in_({"global", active_region})) + category_clause = _category_filter_clause(categories) + if category_clause is not None: + query = query.where(category_clause) + source_clause = _source_filter_clause(source_ids) + if source_clause is not None: + query = query.where(source_clause) + result = await db.execute(query) + records = list(result.scalars().all()) + if not source_ids: + records = _diversify_records_by_source(records, limit=limit) + else: + records = records[:limit] + return [record_to_parsed_news_item(record) for record in records] async def list_earth_news_cruise_items( db: AsyncSession, *, limit: int, + categories: set[str] | None = None, + source_ids: set[str] | None = None, ) -> list[ParsedNewsItem]: - result = await db.execute( + query = ( select(EarthNewsItem) .order_by( EarthNewsItem.region.asc(), @@ -93,6 +179,13 @@ async def list_earth_news_cruise_items( ) .limit(limit) ) + category_clause = _category_filter_clause(categories) + if category_clause is not None: + query = query.where(category_clause) + source_clause = _source_filter_clause(source_ids) + if source_clause is not None: + query = query.where(source_clause) + result = await db.execute(query) return [record_to_parsed_news_item(record) for record in result.scalars().all()] @@ -101,13 +194,13 @@ async def get_earth_news_freshness( *, active_region: str, ) -> tuple[int, datetime | None]: - regions = {"global", active_region} - result = await db.execute( - select( - func.count(EarthNewsItem.id), - func.max(func.coalesce(EarthNewsItem.published_at, EarthNewsItem.last_seen_at)), - ).where(EarthNewsItem.region.in_(regions)) + query = select( + func.count(EarthNewsItem.id), + func.max(func.coalesce(EarthNewsItem.published_at, EarthNewsItem.last_seen_at)), ) + if active_region != "global": + query = query.where(EarthNewsItem.region.in_({"global", active_region})) + result = await db.execute(query) count, newest = result.one() item_count = int(count or 0) if item_count == 0: @@ -115,6 +208,33 @@ async def get_earth_news_freshness( return item_count, _coerce_datetime(newest) +async def get_earth_news_feed_coverage( + db: AsyncSession, + *, + active_region: str, + recent_after: datetime | None = None, +) -> set[tuple[str, str]]: + query = select( + EarthNewsItem.id, + EarthNewsItem.location_meta.op("->")("news_meta").op("->>")("source_id"), + EarthNewsItem.location_meta.op("->")("news_meta").op("->>")("feed_id"), + ) + if active_region != "global": + query = query.where(EarthNewsItem.region.in_({"global", active_region})) + if recent_after is not None: + query = query.where(func.coalesce(EarthNewsItem.published_at, EarthNewsItem.last_seen_at) >= recent_after) + result = await db.execute(query) + coverage: set[tuple[str, str]] = set() + for item_id, source_id, feed_id in result.all(): + normalized_source_id = str(source_id or "").strip() + normalized_feed_id = str(feed_id or "").strip() + if not normalized_source_id and isinstance(item_id, str) and ":" in item_id: + normalized_source_id = item_id.split(":", 1)[0] + if normalized_source_id and normalized_feed_id: + coverage.add((normalized_source_id, normalized_feed_id)) + return coverage + + async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem]) -> int: if not items: return 0 @@ -165,12 +285,20 @@ async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem]) record.homepage_url = item.homepage_url record.published_at = item.published_at record.last_seen_at = now + location_meta = dict(record.location_meta or {}) + location_meta["news_meta"] = _news_meta_patch(item) + record.location_meta = location_meta if item.localizations: + merged_localizations = { + **dict(record.localizations or {}), + **dict(item.localizations or {}), + } record.content_language = item.content_language - record.localizations = dict(item.localizations or {}) - record.enrichment_status = item.enrichment_status - record.enrichment_error = item.enrichment_error - record.enriched_at = item.enriched_at + record.localizations = merged_localizations + if item.enrichment_status != "pending" or item.enrichment_error or item.enriched_at: + record.enrichment_status = item.enrichment_status + record.enrichment_error = item.enrichment_error + record.enriched_at = item.enriched_at changed += 1 await db.flush() return changed diff --git a/backend/app/services/persistent_logs.py b/backend/app/services/persistent_logs.py index 838c9f7c..8f326ecc 100644 --- a/backend/app/services/persistent_logs.py +++ b/backend/app/services/persistent_logs.py @@ -1,14 +1,185 @@ from __future__ import annotations +import hashlib +import re + +from datetime import UTC, datetime from typing import Any from app.core.logging import get_logger, sanitize_log_value from app.core.request_context import get_request_id from app.db.session import async_session_factory -from app.models.system_log import AuditLog, SystemLog +from app.models.system_log import AuditLog, ObservabilityEvent, ObservabilityEventGroup, SystemLog logger = get_logger(__name__) +HLS_TRANSIENT_RE = re.compile(r"(index|chunk|segment)[_-]?\d+(?:_\d+)?\.(?:ts|m4s|vtt)", re.IGNORECASE) +QUERY_RE = re.compile(r"([?&](?:m|t|token|expires|signature|X-Amz-[^=]+)=[^&\\s]+)", re.IGNORECASE) +UUID_RE = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", re.IGNORECASE) +CONNECTION_RE = re.compile(r"\bconn_[A-Za-z0-9:._-]+\b") +NUMBER_RE = re.compile(r"\b\d{5,}\b") + + +def normalize_observability_text(value: Any) -> str: + text = str(sanitize_log_value(value or "")).strip() + text = QUERY_RE.sub("", text) + text = HLS_TRANSIENT_RE.sub("", text) + text = UUID_RE.sub("", text) + text = CONNECTION_RE.sub("", text) + text = NUMBER_RE.sub("", text) + return re.sub(r"\s+", " ", text).strip() + + +def build_observability_fingerprint( + *, + source: str, + service: str | None = None, + module: str | None = None, + category: str | None = None, + event: str | None = None, + message: str, + context: dict[str, Any] | None = None, +) -> str: + context = context or {} + stable_context = { + key: context.get(key) + for key in ( + "task_type", + "source_id", + "source", + "provider", + "status_code", + "error_type", + "details", + ) + if context.get(key) not in (None, "") + } + raw = "|".join( + [ + normalize_observability_text(source), + normalize_observability_text(service), + normalize_observability_text(module), + normalize_observability_text(category), + normalize_observability_text(event), + normalize_observability_text(message), + normalize_observability_text(stable_context), + ] + ) + return hashlib.sha1(raw.encode("utf-8", errors="replace")).hexdigest() + + +def _context_text(context: dict[str, Any] | None, key: str) -> str | None: + value = (context or {}).get(key) + if value in (None, ""): + return None + return str(value) + + +async def record_observability_event( + *, + source: str, + level: str, + message: str, + service: str | None = None, + module: str | None = None, + event: str | None = None, + request_id: str | None = None, + trace_id: str | None = None, + user_id: int | None = None, + category: str | None = None, + context: dict[str, Any] | None = None, + fingerprint: str | None = None, + occurred_at: datetime | None = None, + occurrence_count: int = 1, +) -> None: + normalized_context = sanitize_log_value(context or {}) + if not isinstance(normalized_context, dict): + normalized_context = {"value": normalized_context} + safe_message = str(sanitize_log_value(message)) + normalized_level = str(level or "info").lower() + count = max(1, int(occurrence_count or 1)) + event_time = occurred_at or datetime.now(UTC) + event_fingerprint = fingerprint or build_observability_fingerprint( + source=source, + service=service, + module=module, + category=category, + event=event, + message=safe_message, + context=normalized_context, + ) + detail = _context_text(normalized_context, "detail") or _context_text(normalized_context, "error") + affected_sources = sorted( + { + item + for item in ( + source, + service, + module, + _context_text(normalized_context, "source_id"), + _context_text(normalized_context, "source"), + ) + if item + } + ) + try: + async with async_session_factory() as session: + session.add( + ObservabilityEvent( + source=source, + service=service, + module=module, + category=category, + event=event, + level=normalized_level, + message=safe_message, + fingerprint=event_fingerprint, + occurred_at=event_time, + request_id=request_id or get_request_id(), + trace_id=trace_id, + user_id=user_id, + task_id=_context_text(normalized_context, "task_id"), + source_ref_id=_context_text(normalized_context, "source_id") or _context_text(normalized_context, "source"), + provider=_context_text(normalized_context, "provider"), + context=normalized_context, + occurrence_count=count, + ) + ) + group = await session.get(ObservabilityEventGroup, event_fingerprint) + if group is None: + session.add( + ObservabilityEventGroup( + fingerprint=event_fingerprint, + source=source, + service=service, + module=module, + category=category, + event=event, + last_level=normalized_level, + sample_message=safe_message, + sample_detail=detail, + affected_sources=affected_sources, + count=count, + first_seen_at=event_time, + last_seen_at=event_time, + ) + ) + else: + group.count = int(group.count or 0) + count + group.last_seen_at = event_time + group.last_level = normalized_level + group.sample_message = safe_message + group.sample_detail = detail + merged_sources = sorted(set(group.affected_sources or []) | set(affected_sources)) + group.affected_sources = merged_sources + await session.commit() + except Exception: + logger.exception_event( + "Failed to persist observability event", + event="observability_event.persist.failed", + context={"event_name": event, "source": source}, + ) + async def record_system_log( *, @@ -23,6 +194,8 @@ async def record_system_log( user_id: int | None = None, category: str | None = None, context: dict[str, Any] | None = None, + fingerprint: str | None = None, + occurrence_count: int = 1, ) -> None: try: async with async_session_factory() as session: @@ -48,6 +221,21 @@ async def record_system_log( event="system_log.persist.failed", context={"event_name": event, "source": source}, ) + await record_observability_event( + source=source, + service=service, + module=module, + event=event, + level=level, + message=message, + request_id=request_id, + trace_id=trace_id, + user_id=user_id, + category=category, + context=context, + fingerprint=fingerprint, + occurrence_count=occurrence_count, + ) async def record_audit_log( diff --git a/backend/app/services/system_logs.py b/backend/app/services/system_logs.py index 7850c364..97655340 100644 --- a/backend/app/services/system_logs.py +++ b/backend/app/services/system_logs.py @@ -14,7 +14,7 @@ from pathlib import Path from typing import Any from app.core.security import redis_client -from app.models.system_log import AuditLog, SystemLog +from app.models.system_log import AuditLog, ObservabilityEvent, ObservabilityEventGroup, SystemLog from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -120,6 +120,10 @@ class DailyLogMarker: dominant_level: str +def _normalize_search_query(search: str | None) -> str: + return (search or "").strip().lower() + + def _planet_state_dir() -> Path: configured = os.getenv("PLANET_STATE_DIR") if configured: @@ -660,6 +664,228 @@ async def read_database_log_snapshot( } +def _observability_group_matches( + group: ObservabilityEventGroup, + *, + selected_levels: tuple[str, ...], + start_date: str | None, + end_date: str | None, + search: str | None, +) -> bool: + if selected_levels and group.last_level not in selected_levels: + return False + if start_date or end_date: + if group.last_seen_at is None: + return False + date_token = group.last_seen_at.astimezone(UTC).date().isoformat() + if start_date and date_token < start_date: + return False + if end_date and date_token > end_date: + return False + query = _normalize_search_query(search) + if not query: + return True + haystack = " ".join( + [ + group.fingerprint or "", + group.source or "", + group.service or "", + group.module or "", + group.category or "", + group.event or "", + group.last_level or "", + group.sample_message or "", + group.sample_detail or "", + json.dumps(group.affected_sources or [], ensure_ascii=False, sort_keys=True), + ] + ).lower() + return query in haystack + + +def _serialize_observability_group(group: ObservabilityEventGroup) -> dict[str, Any]: + return { + "fingerprint": group.fingerprint, + "source": group.source, + "service": group.service, + "module": group.module, + "category": group.category, + "event": group.event, + "level": group.last_level, + "message": group.sample_message, + "detail": group.sample_detail, + "affected_sources": group.affected_sources or [], + "count": group.count or 0, + "first_seen_at": group.first_seen_at.isoformat() if group.first_seen_at else None, + "last_seen_at": group.last_seen_at.isoformat() if group.last_seen_at else None, + } + + +def _serialize_observability_event(record: ObservabilityEvent) -> dict[str, Any]: + return { + "id": record.id, + "source": record.source, + "service": record.service, + "module": record.module, + "category": record.category, + "event": record.event, + "level": record.level, + "message": record.message, + "fingerprint": record.fingerprint, + "occurred_at": record.occurred_at.isoformat() if record.occurred_at else None, + "request_id": record.request_id, + "trace_id": record.trace_id, + "task_id": record.task_id, + "source_id": record.source_ref_id, + "provider": record.provider, + "user_id": record.user_id, + "context": record.context or {}, + "occurrence_count": record.occurrence_count or 1, + } + + +async def read_observability_groups( + *, + limit: int, + level: str = LOG_LEVEL_ALL, + levels: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + search: str | None = None, + db: AsyncSession, +) -> dict[str, Any]: + selected_levels = normalize_log_levels(level, levels) + scan_limit = max(limit * 5, limit, DEFAULT_LOG_LINE_LIMIT) + result = await db.execute( + select(ObservabilityEventGroup) + .order_by(ObservabilityEventGroup.last_seen_at.desc().nullslast()) + .limit(scan_limit) + ) + groups = [ + group + for group in result.scalars().all() + if _observability_group_matches( + group, + selected_levels=selected_levels, + start_date=start_date, + end_date=end_date, + search=search, + ) + ][:limit] + return { + "mode": "grouped", + "line_limit": limit, + "line_count": len(groups), + "groups": [_serialize_observability_group(group) for group in groups], + "filters": { + "level": level, + "levels": list(selected_levels), + "start_date": start_date, + "end_date": end_date, + "search": search or "", + }, + } + + +async def read_observability_group_events( + fingerprint: str, + *, + limit: int, + db: AsyncSession, +) -> dict[str, Any] | None: + group = await db.get(ObservabilityEventGroup, fingerprint) + if group is None: + return None + result = await db.execute( + select(ObservabilityEvent) + .where(ObservabilityEvent.fingerprint == fingerprint) + .order_by(ObservabilityEvent.occurred_at.desc().nullslast(), ObservabilityEvent.id.desc()) + .limit(limit) + ) + events = list(reversed(result.scalars().all())) + return { + "fingerprint": fingerprint, + "group": _serialize_observability_group(group), + "line_limit": limit, + "line_count": len(events), + "events": [_serialize_observability_event(record) for record in events], + } + + +async def read_observability_raw_events( + *, + limit: int, + level: str = LOG_LEVEL_ALL, + levels: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + search: str | None = None, + db: AsyncSession, +) -> dict[str, Any]: + selected_levels = normalize_log_levels(level, levels) + query = select(ObservabilityEvent).order_by(ObservabilityEvent.occurred_at.desc().nullslast(), ObservabilityEvent.id.desc()) + if selected_levels: + query = query.where(ObservabilityEvent.level.in_(selected_levels)) + result = await db.execute(query.limit(max(limit * 5, limit))) + records = result.scalars().all() + search_query = _normalize_search_query(search) + visible: list[ObservabilityEvent] = [] + for record in records: + if start_date or end_date: + if record.occurred_at is None: + continue + date_token = record.occurred_at.astimezone(UTC).date().isoformat() + if start_date and date_token < start_date: + continue + if end_date and date_token > end_date: + continue + if search_query: + haystack = " ".join( + [ + record.source or "", + record.service or "", + record.module or "", + record.category or "", + record.event or "", + record.message or "", + record.fingerprint or "", + record.request_id or "", + record.trace_id or "", + record.task_id or "", + record.source_ref_id or "", + record.provider or "", + json.dumps(record.context or {}, ensure_ascii=False, sort_keys=True), + ] + ).lower() + if search_query not in haystack: + continue + visible.append(record) + if len(visible) >= limit: + break + visible = list(reversed(visible)) + return { + "mode": "raw", + "line_limit": limit, + "line_count": len(visible), + "events": [_serialize_observability_event(record) for record in visible], + "lines": [ + " ".join( + part + for part in [ + record.occurred_at.isoformat() if record.occurred_at else "", + record.level.upper(), + record.source, + record.category or "", + record.event or "", + f"fingerprint={record.fingerprint}", + record.message, + ] + if part + ) + for record in visible + ], + } + + def _stable_hash(value: str) -> str: return hashlib.sha1(value.encode("utf-8", errors="replace")).hexdigest()[:16] diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 975ea877..8b35e54f 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -655,6 +655,8 @@ async def test_ingest_earth_client_log_accepts_public_events(): "message": "登陆点加载失败: 登陆点接口返回 HTTP 500", "category": "startup-load", "module": "layer-startup", + "fingerprint": "client-test", + "occurrence_count": 3, }, ) assert response.status_code == 200 @@ -668,6 +670,8 @@ async def test_ingest_earth_client_log_accepts_public_events(): assert persisted_kwargs["event"] == "earth.client.runtime_log" assert persisted_kwargs["category"] == "startup-load" assert persisted_kwargs["level"] == "error" + assert persisted_kwargs["fingerprint"] == "client-test" + assert persisted_kwargs["occurrence_count"] == 3 finally: app.dependency_overrides.clear() @@ -705,6 +709,59 @@ async def test_ingest_admin_client_log_accepts_public_events(): app.dependency_overrides.clear() +@pytest.mark.asyncio +async def test_ingest_service_log_requires_configured_token(monkeypatch): + monkeypatch.setattr(settings, "OBSERVABILITY_INGEST_TOKEN", "") + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.post( + "/api/v1/system/logs/service", + json={"message": "AI provider failed"}, + headers={"X-Planet-Observability-Token": "secret"}, + ) + + assert response.status_code == 503 + + +@pytest.mark.asyncio +async def test_ingest_service_log_accepts_internal_token(monkeypatch): + monkeypatch.setattr(settings, "OBSERVABILITY_INGEST_TOKEN", "service-secret") + transport = ASGITransport(app=app) + with patch("app.api.v1.system_control.record_system_log", new_callable=AsyncMock) as mock_record_system_log: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.post( + "/api/v1/system/logs/service", + json={ + "source": "ai-provider", + "service": "ai-provider", + "module": "provider", + "category": "connectivity", + "event": "ai.provider.test.failed", + "level": "error", + "message": "Provider connectivity failed", + "fingerprint": "ai-provider-test", + "occurrence_count": 4, + "provider": "minimax", + "trace_id": "trace-123", + "context": {"status_code": 502}, + }, + headers={"Authorization": "Bearer service-secret"}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["accepted"] is True + assert data["source_id"] == "ai-provider" + mock_record_system_log.assert_awaited_once() + persisted_kwargs = mock_record_system_log.await_args.kwargs + assert persisted_kwargs["event"] == "ai.provider.test.failed" + assert persisted_kwargs["fingerprint"] == "ai-provider-test" + assert persisted_kwargs["occurrence_count"] == 4 + assert persisted_kwargs["context"]["provider"] == "minimax" + assert persisted_kwargs["context"]["trace_id"] == "trace-123" + assert persisted_kwargs["context"]["status_code"] == 502 + + @pytest.mark.asyncio async def test_earth_layer_cache_status_requires_super_admin(auth_headers, monkeypatch): def override_get_current_user(): diff --git a/backend/tests/test_earth_interactables.py b/backend/tests/test_earth_interactables.py index d5d18d23..bf464cec 100644 --- a/backend/tests/test_earth_interactables.py +++ b/backend/tests/test_earth_interactables.py @@ -74,6 +74,6 @@ def test_interactable_cache_invalidation_clears_layer_and_all(monkeypatch): assert deleted == 2 assert patterns == [ - "earth:layer:v1:interactables:layer:places*", - "earth:layer:v1:interactables:layer:all*", + "earth:layer:v1:interactables:interactable_layer:places*", + "earth:layer:v1:interactables:interactable_layer:all*", ] diff --git a/backend/tests/test_earth_news.py b/backend/tests/test_earth_news.py index 93313797..7edffb17 100644 --- a/backend/tests/test_earth_news.py +++ b/backend/tests/test_earth_news.py @@ -4,14 +4,20 @@ from types import SimpleNamespace import pytest from app.services.earth_news import ( + NewsFeedEndpoint, NewsFeedSource, NewsTargetLocation, ParsedNewsItem, + apply_news_classification, + default_earth_news_sources_payload, + normalize_earth_news_sources_payload, + _fetch_source, _enrich_items_with_target_locations, _extract_target_location_from_text, _parse_feed_entries, _serialize_item, get_earth_news_payload, + test_news_source_config as run_news_source_config_test, ) from app.services.earth_news_queue import NewsTargetLocationMessage from app.services.earth_news_worker import process_target_location_message @@ -164,6 +170,479 @@ def test_parse_aggregated_rss_splits_publisher_from_title(): assert items[0].source == "Reuters" +def test_parse_chinese_rss_marks_source_language_and_keeps_zh_localization(): + source = NewsFeedSource( + id="36kr", + name="36氪", + region="asia-pacific", + feed_url="https://36kr.com/feed", + homepage_url="https://www.36kr.com/", + source_tags=("china", "business_news"), + default_category="business", + ) + xml = """ + + + + 中国电商平台发布季度增长数据 + 平台表示,跨境电商订单量同比增长。 + https://36kr.com/p/example + + + + """ + + items = _parse_feed_entries(xml, source) + payload_zh = _serialize_item(items[0], active_region="global", locale="zh-CN") + payload_en = _serialize_item(items[0], active_region="global", locale="en-US") + + assert items[0].content_language == "zh-CN" + assert items[0].localizations["zh-CN"]["title"] == "中国电商平台发布季度增长数据" + assert payload_zh["display_title"] == "中国电商平台发布季度增长数据" + assert payload_en["display_title"] == "中国电商平台发布季度增长数据" + + +def test_default_news_sources_include_business_and_ecommerce_sources(): + payload = default_earth_news_sources_payload() + sources_by_id = {source["id"]: source for source in payload["sources"]} + source_ids = {source["id"] for source in payload["sources"]} + category_keys = {category["key"] for category in payload["categories"]} + tag_keys = {tag["key"] for tag in payload["source_tags"]} + + assert "cnbc-business" in source_ids + assert "36kr" in source_ids + assert "techcrunch" in source_ids + assert "retaildive" in source_ids + assert "prnewswire-retail" in source_ids + assert "google-news" in source_ids + assert "global-scan" not in source_ids + assert "google-americas" not in source_ids + assert "google-europe" not in source_ids + assert "google-mea" not in source_ids + assert "google-apac" not in source_ids + assert "businesswire-ecommerce" in source_ids + assert "us-census-ecommerce" in source_ids + assert "mofcom-data" in source_ids + assert "stats-china-online-retail" in source_ids + assert "ebrun" in source_ids + assert sources_by_id["36kr"]["source_type"] == "rss" + assert sources_by_id["36kr"]["homepage_url"] == "https://www.36kr.com/" + assert sources_by_id["36kr"]["feed_directory_url"] == "https://www.36kr.com/rss-center" + kr_feeds = {feed["id"]: feed for feed in sources_by_id["36kr"]["feeds"]} + assert set(kr_feeds) == {"feed", "article", "newsflash", "moment"} + assert kr_feeds["feed"]["url"] == "https://36kr.com/feed" + assert kr_feeds["article"]["url"] == "https://36kr.com/feed-article" + assert kr_feeds["newsflash"]["url"] == "https://36kr.com/feed-newsflash" + assert kr_feeds["moment"]["url"] == "https://36kr.com/feed-moment" + assert all(feed["enabled"] is True for feed in kr_feeds.values()) + assert all(feed["default_category"] == "business" for feed in kr_feeds.values()) + assert "https://36kr.com/feed-article" in sources_by_id["36kr"]["feed_urls"] + assert "https://36kr.com/feed-newsflash" in sources_by_id["36kr"]["feed_urls"] + assert "https://36kr.com/feed-moment" in sources_by_id["36kr"]["feed_urls"] + assert sources_by_id["ebrun"]["source_type"] == "rss" + assert sources_by_id["ebrun"]["homepage_url"] == "https://www.ebrun.com/" + assert sources_by_id["ebrun"]["feed_directory_url"] == "https://www.ebrun.com/rss/" + ebrun_feeds = {feed["id"]: feed for feed in sources_by_id["ebrun"]["feeds"]} + assert {"b2c", "b2b", "retail", "o2o", "service", "data", "policy"}.issubset(ebrun_feeds) + assert all(feed["enabled"] is True for feed in ebrun_feeds.values()) + assert all(feed["default_category"] == "ecommerce" for feed in ebrun_feeds.values()) + assert "https://www.ebrun.com/rss/news_b2c.xml" in sources_by_id["ebrun"]["feed_urls"] + assert "https://www.ebrun.com/rss/news_retail.xml" in sources_by_id["ebrun"]["feed_urls"] + assert sources_by_id["businesswire-ecommerce"]["source_type"] == "reference" + assert sources_by_id["businesswire-ecommerce"]["enabled"] is False + assert sources_by_id["google-news"]["source_type"] == "aggregated" + assert sources_by_id["google-news"]["homepage_url"] == "https://news.google.com/" + assert sources_by_id["google-news"]["feed_directory_url"] == "https://news.google.com/rss" + google_feeds = {feed["id"]: feed for feed in sources_by_id["google-news"]["feeds"]} + assert set(google_feeds) == {"world", "americas", "europe", "middle-east-africa", "asia-pacific"} + assert all(feed["type"] == "aggregated" for feed in google_feeds.values()) + assert all(feed["enabled"] is True for feed in google_feeds.values()) + assert google_feeds["world"]["region"] == "global" + assert google_feeds["europe"]["region"] == "europe" + assert sources_by_id["stats-china-online-retail"]["source_type"] == "rss" + assert sources_by_id["stats-china-online-retail"]["enabled"] is True + assert "https://www.stats.gov.cn/sj/zxfb/rss.xml" in sources_by_id["stats-china-online-retail"]["feed_urls"] + assert {"business", "ecommerce", "finance"}.issubset(category_keys) + assert {"official_data", "business_news", "ecommerce", "press_release", "finance", "logistics"}.issubset(tag_keys) + + +def test_default_enabled_fetchable_sources_have_explicit_types_and_urls(): + payload = default_earth_news_sources_payload() + for source in payload["sources"]: + source_type = source["source_type"] + assert source_type in {"rss", "atom", "aggregated", "reference"} + if source_type == "reference": + assert source["enabled"] is False + assert source["feeds"] == [] + continue + if source["enabled"]: + assert source["feed_url"] + assert source["feed_urls"] + assert source["feeds"] + assert any(feed["enabled"] for feed in source["feeds"]) + for feed in source["feeds"]: + assert feed["url"] != source["homepage_url"] + assert feed["url"] != source.get("feed_directory_url", "") + + +def test_legacy_news_source_urls_migrate_to_feed_children(): + payload = normalize_earth_news_sources_payload( + { + "sources": [ + { + "id": "legacy-source", + "name": "Legacy Source", + "region": "global", + "source_type": "rss", + "feed_urls": ["https://example.com/a.xml", "https://example.com/b.xml"], + "default_category": "business", + } + ] + } + ) + + source = payload["sources"][0] + + assert source["feed_urls"] == ["https://example.com/a.xml", "https://example.com/b.xml"] + assert [feed["url"] for feed in source["feeds"]] == ["https://example.com/a.xml", "https://example.com/b.xml"] + assert [feed["id"] for feed in source["feeds"]] == ["feed-1", "feed-2"] + assert all(feed["default_category"] == "business" for feed in source["feeds"]) + + +def test_builtin_news_source_legacy_directory_url_is_repaired(): + payload = normalize_earth_news_sources_payload( + { + "sources": [ + { + "id": "36kr", + "name": "36氪", + "region": "asia-pacific", + "source_type": "rss", + "homepage_url": "https://www.36kr.com/", + "feed_url": "https://www.36kr.com/rss-center", + "feed_urls": ["https://www.36kr.com/rss-center"], + "feeds": [ + { + "id": "feed-1", + "name": "36氪", + "url": "https://www.36kr.com/rss-center", + "type": "rss", + "enabled": True, + "default_category": "business", + } + ], + "default_category": "business", + } + ] + } + ) + + source = payload["sources"][0] + feed_urls = {feed["url"] for feed in source["feeds"]} + + assert source["homepage_url"] == "https://www.36kr.com/" + assert source["feed_directory_url"] == "https://www.36kr.com/rss-center" + assert "https://www.36kr.com/rss-center" not in feed_urls + assert { + "https://36kr.com/feed", + "https://36kr.com/feed-article", + "https://36kr.com/feed-newsflash", + "https://36kr.com/feed-moment", + }.issubset(feed_urls) + + +def test_builtin_news_source_without_feed_children_gets_explicit_defaults(): + payload = normalize_earth_news_sources_payload( + { + "sources": [ + { + "id": "ebrun", + "name": "亿邦动力", + "region": "asia-pacific", + "source_type": "rss", + "homepage_url": "https://www.ebrun.com/", + "feed_url": "https://www.ebrun.com/rss/news_b2c.xml", + "feed_urls": ["https://www.ebrun.com/rss/news_b2c.xml"], + "default_category": "ecommerce", + } + ] + } + ) + + source = payload["sources"][0] + feed_urls = {feed["url"] for feed in source["feeds"]} + + assert source["feed_directory_url"] == "https://www.ebrun.com/rss/" + assert "https://www.ebrun.com/rss/" not in feed_urls + assert { + "https://www.ebrun.com/rss/news_b2c.xml", + "https://www.ebrun.com/rss/news_b2b.xml", + "https://www.ebrun.com/rss/news_retail.xml", + "https://www.ebrun.com/rss/news_o2o.xml", + "https://www.ebrun.com/rss/news_service.xml", + "https://www.ebrun.com/rss/news_data.xml", + "https://www.ebrun.com/rss/news_policy.xml", + }.issubset(feed_urls) + + +def test_builtin_fetchable_source_saved_as_reference_is_repaired(): + payload = normalize_earth_news_sources_payload( + { + "sources": [ + { + "id": "stats-china-online-retail", + "name": "国家统计局数据发布", + "region": "asia-pacific", + "source_type": "reference", + "enabled": False, + "homepage_url": "https://www.stats.gov.cn/sj/zxfb/", + "feed_url": "https://www.stats.gov.cn/sj/zxfb/", + "default_category": "ecommerce", + } + ] + } + ) + + source = payload["sources"][0] + + assert source["source_type"] == "rss" + assert source["enabled"] is True + assert source["priority"] == 19 + assert source["source_tags"] == ["official_data", "ecommerce", "retail", "china"] + assert source["default_category"] == "ecommerce" + assert source["importance_weight"] == 36 + assert source["feed_directory_url"] == "" + assert source["feeds"] == [ + { + "id": "release", + "name": "数据发布", + "url": "https://www.stats.gov.cn/sj/zxfb/rss.xml", + "type": "rss", + "region": "asia-pacific", + "enabled": True, + "default_category": "ecommerce", + "tags": [], + "priority": 1, + } + ] + + +def test_legacy_google_sources_merge_into_google_news_source(): + payload = normalize_earth_news_sources_payload( + { + "sources": [ + { + "id": "global-scan", + "name": "Global Monitor / World", + "region": "global", + "source_type": "aggregated", + "feed_url": "https://news.google.com/rss/search?q=world", + "homepage_url": "https://news.google.com/", + }, + { + "id": "google-europe", + "name": "Global Monitor / Europe", + "region": "europe", + "source_type": "aggregated", + "feed_url": "https://news.google.com/rss/search?q=europe", + "homepage_url": "https://news.google.com/", + }, + ] + } + ) + + sources_by_id = {source["id"]: source for source in payload["sources"]} + + assert "global-scan" not in sources_by_id + assert "google-europe" not in sources_by_id + assert "google-news" in sources_by_id + assert {feed["id"] for feed in sources_by_id["google-news"]["feeds"]} == { + "world", + "americas", + "europe", + "middle-east-africa", + "asia-pacific", + } + + +def test_feed_child_default_category_overrides_source_default(): + source = NewsFeedSource( + id="multi-feed", + name="Multi Feed", + region="global", + feed_url="https://example.com/source.xml", + homepage_url="https://example.com", + default_category="business", + ) + feed = NewsFeedEndpoint( + id="ecommerce-feed", + name="Ecommerce Feed", + url="https://example.com/ecommerce.xml", + default_category="ecommerce", + ) + xml = """ + + + + Quarterly results released + Company update. + https://example.com/results + + + + """ + + items = _parse_feed_entries(xml, source, feed=feed) + + assert items[0].feed_id == "ecommerce-feed" + assert items[0].feed_name == "Ecommerce Feed" + assert items[0].feed_default_category == "ecommerce" + assert items[0].category == "ecommerce" + + +@pytest.mark.asyncio +async def test_fetch_source_only_requests_enabled_feed_children(monkeypatch): + source = NewsFeedSource( + id="multi-feed", + name="Multi Feed", + region="global", + feed_url="https://example.com/source.xml", + homepage_url="https://example.com", + feeds=( + NewsFeedEndpoint(id="enabled", name="Enabled", url="https://example.com/enabled.xml", enabled=True), + NewsFeedEndpoint(id="disabled", name="Disabled", url="https://example.com/disabled.xml", enabled=False), + ), + ) + calls = [] + + async def fake_fetch_single(_client, feed_source, feed, *, config_payload=None): + calls.append(feed.id) + item = ParsedNewsItem( + id=f"{feed_source.id}:{feed.id}:1", + title="Fetched story", + summary="Fetched summary", + url=f"https://example.com/{feed.id}", + source="Example", + feed_name=feed.name, + feed_region="global", + homepage_url="https://example.com", + published_at=None, + feed_id=feed.id, + ) + return feed_source, [item], None, {"source_id": feed_source.id, "feed_id": feed.id, "ok": True, "status": "ok", "item_count": 1, "count": 1} + + monkeypatch.setattr("app.services.earth_news._fetch_single_feed_url", fake_fetch_single) + + source_result, items, error, health = await _fetch_source(object(), source) + + assert source_result.id == "multi-feed" + assert calls == ["enabled"] + assert error is None + assert [item.feed_id for item in items] == ["enabled"] + assert health["ok"] is True + assert [result["feed_id"] for result in health["feed_results"]] == ["enabled"] + + +@pytest.mark.asyncio +async def test_fetch_source_filters_google_feed_children_by_active_region(monkeypatch): + source = NewsFeedSource( + id="google-news", + name="Google News", + region="global", + feed_url="https://news.google.com/rss", + homepage_url="https://news.google.com/", + source_type="aggregated", + feeds=( + NewsFeedEndpoint(id="world", name="全球", url="https://example.com/world.xml", type="aggregated", region="global"), + NewsFeedEndpoint(id="europe", name="欧洲", url="https://example.com/europe.xml", type="aggregated", region="europe"), + NewsFeedEndpoint(id="americas", name="美洲", url="https://example.com/americas.xml", type="aggregated", region="americas"), + ), + ) + calls = [] + + async def fake_fetch_single(_client, feed_source, feed, *, config_payload=None): + calls.append(feed.id) + item = ParsedNewsItem( + id=f"{feed_source.id}:{feed.id}:1", + title=f"{feed.name} headline", + summary="Fetched summary", + url=f"https://example.com/{feed.id}", + source="Example", + feed_name=feed.name, + feed_region=feed.region, + homepage_url="https://example.com", + published_at=None, + feed_id=feed.id, + ) + return feed_source, [item], None, {"source_id": feed_source.id, "feed_id": feed.id, "ok": True, "status": "ok", "item_count": 1, "count": 1} + + monkeypatch.setattr("app.services.earth_news._fetch_single_feed_url", fake_fetch_single) + + _source_result, items, error, health = await _fetch_source(object(), source, active_region="europe") + + assert error is None + assert calls == ["world", "europe"] + assert [item.feed_region for item in items] == ["global", "europe"] + assert [result["feed_id"] for result in health["feed_results"]] == ["world", "europe"] + + +def test_parse_rdf_rss_items_with_namespaces(): + source = NewsFeedSource( + id="dw-top", + name="DW Top Stories", + region="europe", + feed_url="https://rss.dw.com/rdf/rss-en-top", + homepage_url="https://www.dw.com/en/top-stories/s-9097", + ) + xml = """ + + + German retail sales rise + https://example.com/dw + Retail summary + + + """ + + items = _parse_feed_entries(xml, source) + + assert len(items) == 1 + assert items[0].title == "German retail sales rise" + + +def test_news_classification_marks_ecommerce_and_importance(): + source = NewsFeedSource( + id="ebrun", + name="亿邦动力", + region="asia-pacific", + feed_url="https://www.ebrun.com/rss/", + homepage_url="https://www.ebrun.com/", + source_tags=("business_news", "ecommerce", "china"), + default_category="ecommerce", + importance_weight=14, + ) + item = ParsedNewsItem( + id="ebrun:test", + title="跨境电商平台 GMV 同比增长,物流履约效率提升", + summary="订单量和网上零售额继续增长。", + url="https://example.com/ecommerce", + source="亿邦动力", + feed_name="亿邦动力", + feed_region="asia-pacific", + homepage_url="https://www.ebrun.com/", + published_at=None, + ) + + apply_news_classification(item, source) + + assert item.category == "ecommerce" + assert "cross_border_ecommerce" in item.item_tags + assert "logistics_fulfillment" in item.item_tags + assert item.importance_level in {"high", "critical"} + assert "命中电商数据指标" in item.importance_reasons + + @pytest.mark.asyncio async def test_enrich_items_with_target_locations_uses_ai_and_geocode(monkeypatch): item = ParsedNewsItem( @@ -338,8 +817,8 @@ async def test_earth_news_payload_returns_anchor_items_and_enqueues_location_job published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC), ) - async def fake_fetch_source(_client, feed_source): - return feed_source, [item], None + async def fake_fetch_source(_client, feed_source, **_kwargs): + return feed_source, [item], None, {"source_id": feed_source.id, "ok": True, "status": "ok", "count": 1} async def fake_get_cached_target_location_patch(_item_id): return None @@ -402,7 +881,7 @@ async def test_earth_news_payload_uses_fresh_database_items_without_rss(monkeypa async def fake_get_earth_news_freshness(_db, *, active_region): return 12, datetime.now(UTC) - async def fake_list_earth_news_items(_db, *, active_region, limit): + async def fake_list_earth_news_items(_db, *, active_region, limit, categories=None): assert limit == 12 return [item] @@ -452,10 +931,10 @@ async def test_earth_news_payload_keeps_current_items_and_all_cruise_items(monke async def fake_get_earth_news_freshness(_db, *, active_region): return 12, datetime.now(UTC) - async def fake_list_earth_news_items(_db, *, active_region, limit): + async def fake_list_earth_news_items(_db, *, active_region, limit, categories=None): return [current_item] - async def fake_list_earth_news_cruise_items(_db, *, limit): + async def fake_list_earth_news_cruise_items(_db, *, limit, categories=None): return [current_item, cruise_item] async def fake_enqueue_target_location_job(_payload, **_kwargs): @@ -477,6 +956,89 @@ async def test_earth_news_payload_keeps_current_items_and_all_cruise_items(monke assert payload["cruise_items"][1]["region"] == "asia-pacific" +@pytest.mark.asyncio +async def test_earth_news_payload_passes_region_and_category_filters_to_store(monkeypatch): + class FakeDb: + execute = object() + + captured = {} + item = ParsedNewsItem( + id="db:business", + title="Business story", + summary="Business summary", + url="https://example.com/business", + source="Stored Source", + feed_name="Stored Feed", + feed_region="europe", + homepage_url="https://example.com", + published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC), + category="business", + ) + + async def fake_get_earth_news_freshness(_db, *, active_region): + captured["freshness_region"] = active_region + return 12, datetime.now(UTC) + + async def fake_list_earth_news_items(_db, *, active_region, limit, categories=None, source_ids=None): + captured["items_region"] = active_region + captured["items_categories"] = categories + captured["items_source_ids"] = source_ids + return [item] + + async def fake_list_earth_news_cruise_items(_db, *, limit, categories=None, source_ids=None): + captured["cruise_categories"] = categories + captured["cruise_source_ids"] = source_ids + return [item] + + async def fake_enqueue_target_location_job(_payload, **_kwargs): + return True + + monkeypatch.setattr("app.services.earth_news_store.get_earth_news_freshness", fake_get_earth_news_freshness) + monkeypatch.setattr("app.services.earth_news_store.list_earth_news_items", fake_list_earth_news_items) + monkeypatch.setattr("app.services.earth_news_store.list_earth_news_cruise_items", fake_list_earth_news_cruise_items) + monkeypatch.setattr("app.services.earth_news_queue.enqueue_target_location_job", fake_enqueue_target_location_job) + monkeypatch.setattr("app.services.earth_news._fetch_rss_items_for_sources", lambda _sources: (_ for _ in ()).throw(AssertionError("fresh database items should not fetch RSS"))) + + payload = await get_earth_news_payload( + lat=35.0, + lon=-100.0, + region="europe", + categories={"business", "ecommerce"}, + db=FakeDb(), + ) + + assert captured["freshness_region"] == "europe" + assert captured["items_region"] == "europe" + assert captured["items_categories"] == {"business", "ecommerce"} + assert captured["items_source_ids"] is None + assert captured["cruise_categories"] == {"business", "ecommerce"} + assert captured["cruise_source_ids"] is None + assert payload["filters"] == { + "region": "europe", + "categories": ["business", "ecommerce"], + "sources": [], + "limit": 12, + "locale": "zh-CN", + } + assert payload["items"][0]["category"] == "business" + + +@pytest.mark.asyncio +async def test_news_source_test_treats_type_reference_as_non_fetching(): + result = await run_news_source_config_test( + { + "id": "reference-only", + "name": "Reference Only", + "type": "reference", + "feed_url": "https://example.com", + } + ) + + assert result["ok"] is False + assert result["health"]["status"] == "reference" + assert "不参与 RSS/Atom 抓取" in result["error"] + + @pytest.mark.asyncio async def test_earth_news_payload_initializes_empty_database_from_rss(monkeypatch): db = object() @@ -512,14 +1074,14 @@ async def test_earth_news_payload_initializes_empty_database_from_rss(monkeypatc async def fake_get_earth_news_freshness(_db, *, active_region): return 0, None - async def fake_fetch_rss_items_for_sources(_sources): - return [item], [] + async def fake_fetch_rss_items_for_sources(_sources, **_kwargs): + return [item], [], {"test-feed": {"source_id": "test-feed", "ok": True, "status": "ok", "count": 1}} async def fake_upsert_earth_news_items(_db, items): upserted.extend(items) return len(items) - async def fake_list_earth_news_items(_db, *, active_region, limit): + async def fake_list_earth_news_items(_db, *, active_region, limit, categories=None): return [item] async def fake_enqueue_target_location_job(payload, **_kwargs): @@ -568,14 +1130,14 @@ async def test_earth_news_payload_supplements_stale_database_items(monkeypatch): async def fake_get_earth_news_freshness(_db, *, active_region): return 12, datetime(2026, 5, 14, 3, 0, tzinfo=UTC) - async def fake_fetch_rss_items_for_sources(_sources): + async def fake_fetch_rss_items_for_sources(_sources, **_kwargs): fetched.append(True) - return [old_item], [] + return [old_item], [], {"stored": {"source_id": "stored", "ok": True, "status": "ok", "count": 1}} async def fake_upsert_earth_news_items(_db, items): return len(items) - async def fake_list_earth_news_items(_db, *, active_region, limit): + async def fake_list_earth_news_items(_db, *, active_region, limit, categories=None): return [old_item] async def fake_enqueue_target_location_job(_payload, **_kwargs): @@ -630,8 +1192,8 @@ async def test_earth_news_payload_merges_cached_location_patch(monkeypatch): }, } - async def fake_fetch_source(_client, feed_source): - return feed_source, [item], None + async def fake_fetch_source(_client, feed_source, **_kwargs): + return feed_source, [item], None, {"source_id": feed_source.id, "ok": True, "status": "ok", "count": 1} async def fake_get_cached_target_location_patch(_item_id): return cached_patch @@ -697,8 +1259,8 @@ async def test_earth_news_payload_requeues_cached_failed_localization(monkeypatc } enqueued = [] - async def fake_fetch_source(_client, feed_source): - return feed_source, [item], None + async def fake_fetch_source(_client, feed_source, **_kwargs): + return feed_source, [item], None, {"source_id": feed_source.id, "ok": True, "status": "ok", "count": 1} async def fake_get_cached_target_location_patch(_item_id): return cached_patch diff --git a/backend/tests/test_logging.py b/backend/tests/test_logging.py index 8eb6389b..9573f4e8 100644 --- a/backend/tests/test_logging.py +++ b/backend/tests/test_logging.py @@ -9,6 +9,8 @@ import pytest from app.core.logging import PlanetContextFilter, PlanetFormatter, get_logger from app.core.request_context import set_request_id from app.services import business_logs +from app.services import persistent_logs +from app.models.system_log import ObservabilityEvent, ObservabilityEventGroup def _capture_output(callback): @@ -98,6 +100,86 @@ def test_business_context_redacts_nested_sensitive_values(): assert context["nested"]["safe"] == "visible" +def test_observability_fingerprint_normalizes_hls_fragments(): + first = persistent_logs.build_observability_fingerprint( + source="earth-client", + service="earth", + module="tv", + category="hls-proxy", + event="hls.fragment.failed", + message="HLS 分片加载失败: index_5_9086220.ts?m=1725933270", + context={"status_code": 502}, + ) + second = persistent_logs.build_observability_fingerprint( + source="earth-client", + service="earth", + module="tv", + category="hls-proxy", + event="hls.fragment.failed", + message="HLS 分片加载失败: index_5_9086361.ts?m=1725934270", + context={"status_code": 502}, + ) + + assert first == second + + +@pytest.mark.asyncio +async def test_record_observability_event_updates_group_count(monkeypatch): + events: list[ObservabilityEvent] = [] + groups: dict[str, ObservabilityEventGroup] = {} + + class FakeSession: + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + def add(self, item): + if isinstance(item, ObservabilityEvent): + events.append(item) + elif isinstance(item, ObservabilityEventGroup): + groups[item.fingerprint] = item + + async def get(self, model, key): + if model is ObservabilityEventGroup: + return groups.get(key) + return None + + async def commit(self): + return None + + monkeypatch.setattr(persistent_logs, "async_session_factory", lambda: FakeSession()) + + await persistent_logs.record_observability_event( + source="earth-client", + level="error", + service="earth", + module="tv", + category="hls-proxy", + event="hls.fragment.failed", + message="HLS 分片加载失败: index_5_9086220.ts?m=1725933270", + context={"status_code": 502}, + occurrence_count=2, + ) + await persistent_logs.record_observability_event( + source="earth-client", + level="error", + service="earth", + module="tv", + category="hls-proxy", + event="hls.fragment.failed", + message="HLS 分片加载失败: index_5_9086361.ts?m=1725934270", + context={"status_code": 502}, + occurrence_count=1, + ) + + assert len(events) == 2 + assert len(groups) == 1 + group = next(iter(groups.values())) + assert group.count == 3 + + @pytest.mark.asyncio async def test_emit_business_log_persists_sanitized_system_event(monkeypatch): events = [] diff --git a/backend/tests/test_tv_proxy.py b/backend/tests/test_tv_proxy.py new file mode 100644 index 00000000..0c06de5a --- /dev/null +++ b/backend/tests/test_tv_proxy.py @@ -0,0 +1,35 @@ +from app.api.v1.tv import _rewrite_hls_uri_attributes, _should_strip_hls_metadata_line + + +def test_rewrite_hls_uri_attributes_rewrites_subtitle_manifest_url(): + line = '#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="subs",NAME="English",URI="index_3_0.m3u8"' + + rewritten = _rewrite_hls_uri_attributes( + line, + base_url="https://example.com/live/master.m3u8", + ) + + assert 'URI="/api/v1/tv/proxy?url=https%3A%2F%2Fexample.com%2Flive%2Findex_3_0.m3u8"' in rewritten + + +def test_rewrite_hls_uri_attributes_rewrites_absolute_uri(): + line = '#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=1234,URI="https://cdn.example.com/live/iframe.m3u8"' + + rewritten = _rewrite_hls_uri_attributes( + line, + base_url="https://example.com/live/master.m3u8", + ) + + assert 'URI="/api/v1/tv/proxy?url=https%3A%2F%2Fcdn.example.com%2Flive%2Fiframe.m3u8"' in rewritten + + +def test_strip_hls_subtitle_media_metadata(): + line = '#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="subs",NAME="English",URI="index_3_0.m3u8"' + + assert _should_strip_hls_metadata_line(line) is True + + +def test_keep_hls_audio_media_metadata(): + line = '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="English",URI="audio.m3u8"' + + assert _should_strip_hls_metadata_line(line) is False diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e59974fa..ebf88407 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -8,6 +8,23 @@ This project follows the repository versioning rule: - `improvement` -> `+0.0.1`(bugfix + 小功能混合) - `bugfix` -> `+0.0.1` +## [0.69.0] — 2026-06-03 + +Released: 2026-06-03 + +### Highlights +- 新增 Earth 新闻源治理能力,支持多 Feed 子项、源属性标签、新闻类型过滤、重要度规则和健康测试。 +- 新增观测日志聚合视图,按 fingerprint 汇总 Earth、Admin 和服务端重复运行时事件,并保留原始发生明细。 +- 改进 TV/HLS 播放恢复和代理重写,降低字幕、分片和源站波动导致的直播不可用噪声。 + +### Added / Fixed / Improved +- Earth 新闻面板和 UE 端统一通过 `/api/v1/news/earth-feed` 使用 `categories` 与 `locale` 服务端过滤,Web 端新闻类型偏好仅保存在当前浏览器。 +- 控制台日志页新增重复统计、原始日志和审计日志模式,前端上报器会合并短窗口内的重复错误并提交 `occurrence_count`。 +- AI Provider / 服务端运行时可通过受保护的 observability ingest 入口写入结构化事件。 +- 新闻源文档新增中英文配置说明,并补齐 Earth 前端、控制台日志和公开 Docs 索引。 + +--- + ## [0.68.1] — 2026-05-28 Released: 2026-05-28 diff --git a/docs/technical/en/README.md b/docs/technical/en/README.md index 5381756d..a578e64c 100644 --- a/docs/technical/en/README.md +++ b/docs/technical/en/README.md @@ -23,6 +23,7 @@ This is the current Intelligent Planet documentation entry point. Docs are organ - [Earth Interactable Usage](/home/ray/dev/linkong/planet/docs/technical/en/earth-interactable-usage.md): `Interactable` API, lifecycle, and integration examples - [Earth Interactable Clustering](/home/ray/dev/linkong/planet/docs/technical/en/earth-interactable-clustering.md): pluggable cluster strategies, stable spherical clustering, and dynamic screen clustering boundaries - [Earth Toolbar and Overlay Coordination](/home/ray/dev/linkong/planet/docs/technical/en/earth-toolbar-overlay-coordination.md): close matrix for toolbar buttons, search, settings, news, and layer overlays +- [Intelligent Planet News Source Configuration](/home/ray/dev/linkong/planet/docs/technical/en/earth-news-sources.md): default sources, feed children, source property tags, content categories, importance rules, and configuration APIs ## Frontend Implementation diff --git a/docs/technical/en/earth-frontend-context.md b/docs/technical/en/earth-frontend-context.md index f7bacc29..8f1ecbf4 100644 --- a/docs/technical/en/earth-frontend-context.md +++ b/docs/technical/en/earth-frontend-context.md @@ -75,6 +75,8 @@ This is currently the most critical UI control entry point for the Earth fronten Earth settings are now grouped by `data-settings-tab` and `data-settings-tab-panel`. Desktop and mobile share the same category semantics: Runtime, Display, Panels, Motion, Shortcuts, and System. When adding a setting, first choose its category, then add the DOM, persistence field, and restore logic; do not keep growing one long undifferentiated panel. +The news category selector in Display reuses the same chip-selector pattern as Cruise Modules. It only filters news categories for the current browser on the Earth frontend. It does not toggle layers, basemap, boundaries, TV, data points, BGP, vessels, satellites, or compute centers; those remain owned by the layer panel, media panel, and admin configuration. `controls.js` persists only `shared.newsCategoryFilters` and broadcasts `earth:news-category-filters-change`; `news.js` sends the selected categories to `/api/v1/news/earth-feed?categories=...&locale=zh-CN`, so Web and UE clients share the same backend category filtering path. + Shortcut configuration is a device-local preference owned by `controls.js`: read, capture, enable/disable, and reset all stay in the Earth frontend. It should not be written to backend user settings and should not affect other browsers. New shortcuts must provide a default key, display label, disabled/enabled state, and reset path instead of being hard-coded only in a keydown handler. ### 4. UI and Status Messages diff --git a/docs/technical/en/earth-news-sources.md b/docs/technical/en/earth-news-sources.md new file mode 100644 index 00000000..657fa4d6 --- /dev/null +++ b/docs/technical/en/earth-news-sources.md @@ -0,0 +1,161 @@ +# Earth News Source Configuration + +Earth situational news is served through `/api/v1/news/earth-feed`. News source configuration lives in `SystemSetting.category = "earth_news_sources"`; when no database configuration exists, the backend uses the built-in default sources as the fallback seed. + +## Default Sources + +The default set contains four groups: + +- **News feeds**: BBC World, DW Top Stories, CNBC Business, BBC Business, Guardian Business, NPR Business, MarketWatch, TechCrunch, Retail Dive, PR Newswire Retail, 36Kr, Ebrun, and China NBS data releases. +- **Industry insight sources**: McKinsey Retail and Deloitte Retail. +- **Official data sources**: China NBS data releases, US Census Retail / E-Commerce, MOFCOM Data, MOFCOM e-commerce updates, and China e-commerce logistics index. +- **Lead sources**: BusinessWire Electronic Commerce; Google News is one aggregated source with feed children for global, Americas, Europe, Middle East / Africa, and Asia Pacific. + +Config data sources are visible in Admin by default. If a source is not a stable RSS/Atom feed, it is kept disabled for automatic fetching until an administrator replaces it with a fetchable URL and enables it. + +| Source | Type | Default state | Default category | Main tags | Purpose | +| --- | --- | --- | --- | --- | --- | +| BBC World | RSS | Enabled | Politics | `official_media`, `global` | Global public-news baseline | +| DW Top Stories | RSS | Enabled | Politics | `official_media`, `europe` | Europe and international baseline | +| CNBC Business | RSS | Enabled | Business | `business_news`, `us`, `global` | International business news | +| BBC Business / Guardian Business / NPR Business / MarketWatch | RSS | Enabled | Business / Finance | `business_news`, `finance` | UK / US business and finance baseline | +| TechCrunch / Retail Dive / PR Newswire Retail | RSS | Enabled | Technology / Business | `business_news`, `ecommerce`, `retail`, `press_release` | Technology, e-commerce, retail, and company announcements | +| 36Kr | RSS | Enabled | Business | `business_news`, `ecommerce`, `china` | China business, venture, and newsflash feeds; homepage is `https://www.36kr.com/`, the Feed Directory is `https://www.36kr.com/rss-center`, and feed children are the general, article, newsflash, and moment feeds | +| Ebrun | RSS | Enabled | E-commerce | `ecommerce`, `business_news`, `china`, `retail` | China e-commerce industry news; homepage is `https://www.ebrun.com/`, the Feed Directory is `https://www.ebrun.com/rss/`, and feed children are B2C, B2B, retail, O2O, service, data, and policy XML feeds | +| China NBS data releases | RSS | Enabled | E-commerce | `official_data`, `ecommerce`, `retail`, `china` | Official data release RSS; retail and online retail items are identified by category and importance rules | +| Google News | Aggregated | Enabled | Politics | `aggregated`, `low_stability` | One aggregated source with global, Americas, Europe, Middle East / Africa, and Asia Pacific feed children; lower priority than real RSS | +| BusinessWire Electronic Commerce | Reference | Disabled | E-commerce | `press_release`, `ecommerce`, `low_stability` | Corporate announcement leads | +| McKinsey Retail Insights | Reference | Disabled | Business | `industry_insight`, `retail` | Retail industry insight | +| Deloitte Retail | Reference | Disabled | Business | `industry_insight`, `retail` | Retail industry insight | +| US Census Retail / E-Commerce | Reference | Disabled | E-commerce | `official_data`, `ecommerce`, `retail`, `us` | US retail and e-commerce official data | +| MOFCOM Data | Reference | Disabled | Business | `official_data`, `china` | China commerce data | +| MOFCOM e-commerce updates | Reference | Disabled | E-commerce | `official_data`, `ecommerce`, `china` | China e-commerce policy and updates | +| China e-commerce logistics index | Reference | Disabled | E-commerce | `official_data`, `ecommerce`, `logistics`, `china` | Logistics fulfillment and e-commerce activity | + +`Reference` means a reference link or future collector lead. It records a homepage, report page, or data page and does not participate in RSS/Atom fetching. This lets commercial and official sources enter Admin governance without letting non-feed pages break the live news feed. + +The news source model has two levels: + +- `source` is the brand or aggregator, such as 36Kr, Ebrun, Google News, or BBC. +- `homepage_url` is the source homepage, section page, or report page. +- `feed_directory_url` is the Feed Directory page, such as an RSS subscription center or feed index. It is for human inspection and is not fetched. +- `feeds` are the actual RSS, Atom, or Aggregated child entries under that source. Each feed child has `id / name / url / type / enabled / default_category / tags / priority`. + +The backend iterates over every enabled feed child under the same source, fetches them independently, merges and deduplicates items, and writes per-feed diagnostics into `health.feed_results`. This is not a backup URL model: all four 36Kr subscription feeds, multiple Ebrun category XML feeds, and the five Google News regional RSS feeds can be enabled at the same time, and each feed can have its own default category and enabled state. HTML subscription-center pages belong in `feed_directory_url`, not in feed URLs. Every default enabled fetchable feed is tested item by item: RSS/Atom/Aggregated feeds must parse at least one item, while Reference sources only retain a reference URL and future collector lead. + +Items that still remain Reference are not treated as broken feeds; no stable directly consumable RSS/Atom feed was verified: + +- BusinessWire documents customizable RSS/Atom support, but the public pages do not expose a stable industry feed URL; the e-commerce industry page is kept as an announcement lead. +- McKinsey and Deloitte retail insight pages are report/article collections, not public RSS feeds. +- The US Census press-release RSS is reachable, but its items currently have empty links; the Quarterly E-Commerce page remains an official data reference. +- MOFCOM data and China e-commerce logistics index pages do not expose stable RSS feeds yet; they should become dedicated collectors or be replaced with administrator-provided fetchable feeds. + +## Source Property Tags and News Categories + +News sources have `source_tags`, shown in Admin as source property tags. They describe the source, not the media name and not the content category of an individual story: + +- `official_data` +- `business_news` +- `ecommerce` +- `finance` +- `retail` +- `logistics` +- `industry_insight` +- `press_release` +- `china`, `global`, `us` +- `aggregated`, `low_stability` + +Each news item has exactly one primary `category`. Defaults are politics, business, e-commerce, finance, sports, technology, military, disaster, energy, society, culture, and other. `item_tags` are item-level secondary tags, such as cross-border e-commerce, live commerce, retail data, logistics fulfillment, platform governance, AI, semiconductor, election, oil price, football, and supply chain. + +The primary category is generated by a rule-based scorer over title, summary, and source text. If the rules do not match, the feed child default category is used first, then the source default category. AI enrichment does not block news display. + +## Importance + +Each item includes: + +- `importance_score` +- `importance_level` +- `importance_reasons` +- `market_impact` + +Official data, e-commerce metrics, major platforms, and numeric business signals increase importance. Press releases start with a lower baseline and rise only when they match stronger platform, amount, M&A, or regulatory signals. + +## Configuration and Cache + +`GET /api/v1/earth/news-sources` returns the default or saved configuration. `PUT /api/v1/earth/news-sources` saves it, increments `cache_version`, and clears the process region cache. `POST /api/v1/earth/news-sources/reset` restores defaults. `POST /api/v1/earth/news-sources/test` tests one RSS/Atom/Aggregated source without writing news items. + +## Feed Query and Category Filtering + +The Web Earth client and UE client both consume `GET /api/v1/news/earth-feed`. The endpoint supports server-side filtering, so clients do not need to fetch the full list and apply the primary category filter locally. + +- `lat` / `lon`: infer the active region from the current view, used by the Web Earth client. +- `region`: explicitly select a region for UE or service integrations. Supported values include `global`, `americas`, `europe`, `asia-pacific`, and `middle-east-africa`. `global` is an aggregate view and can include every region; non-global regions include only their own region plus `global` sources. +- `categories`: comma-separated news category keys, for example `business,ecommerce`. Omit it when all categories are selected. +- `locale`: display locale, currently `zh-CN` or `en-US`, defaulting to `zh-CN`. Chinese RSS items are stored as Chinese source content and enriched with `en-US`; English RSS items are enriched with `zh-CN`. + +Examples: + +```http +GET /api/v1/news/earth-feed?region=europe&categories=business,ecommerce +GET /api/v1/news/earth-feed?lat=48&lon=10&categories=technology +GET /api/v1/news/earth-feed?region=global&categories=business,ecommerce&locale=zh-CN +``` + +Unknown category or locale values return `422` with the allowed values. The response includes `filters`, which confirms the region, category, and locale filters applied by the backend. `items` and `cruise_items` use the same category filter set. + +The Web Earth category chips only store the current browser preference; changing them triggers a new API request. UE should pass its selected categories through the `categories` query parameter and does not need to perform the primary filtering itself. + +Source testing only proves that a specific RSS/Atom/XML feed can be parsed. It does not mean those items have already been written to the news table or are visible in the current region/category view. Saving or resetting news sources increments the configuration version and clears cache; if an enabled feed has no recent stored items, the next `earth-feed` request supplements from RSS so newly enabled sources such as 36Kr and Ebrun are not masked by fresh Google News rows. + +## Connectivity Monitoring + +`POST /api/v1/earth/news-sources/test` tests one source and writes the result to `earth_news_sources.health[source_id]`. Normal RSS/Atom fetches update the same health map. + +Health results include: + +- `status`: `ok`, `empty`, `format_error`, `http_error`, `timeout`, `network_error`, or `reference`. +- `status_code`, `content_type`, `item_count`, `latency_ms`, `error`, and `fetched_at`. +- `feed_results`: per-feed diagnostics for multi-feed sources, including `feed_id`, `feed_name`, `feed_type`, `feed_url`, status, item count, and error. + +Common diagnostics: + +- HTML response: the configured URL is not an RSS/Atom feed, for example a web page listing RSS options. +- HTTP 403: the source or CDN rejected the crawler request. +- Reference: the source is a reference link only and must be converted to RSS, Atom, or Aggregated before fetch testing. + +The Admin entry is `Earth Content -> News Sources`. It is not a raw whole-payload JSON editor. The UI has two layers: + +- **News sources**: a left-side source list with filters for enabled, disabled, reference links, RSS/Atom/Aggregated, region, and source property tags; the right side edits one selected source and its feed child list. +- **Policy rules**: global source property tags, news categories, item tag rules, and default health policy. Advanced JSON is reserved for diagnostics, not the default edit path. + +The single-source form is split into source information and feed children: + +- Source information covers name, ID, region, homepage URL, Feed Directory URL, source type, enabled state, source property tags, importance weight, fetch interval, timeout, failure threshold, and circuit breaker. +- Feed children cover feed ID, name, real feed URL, type, enabled switch, default news category, priority, and feed tags. The `+` button under the feed child list creates a frontend-only draft; saving the source persists it, while canceling destroys the draft. + +The per-source “test source” action tests all enabled feed children under the current source. The feed-row test action tests only that feed child. Both send to `/api/v1/earth/news-sources/test`, but the feed-row action submits the current source with only the selected feed child. + +Reference links show that they only record a homepage, report page, or future collector lead and do not participate in RSS/Atom fetching. They can remain as commercial or official-data leads, but must be converted to RSS, Atom, or Aggregated with fetchable feed URLs before they can be enabled for fetching. + +```mermaid +flowchart LR + Admin["Admin: Earth Content / News Sources"] --> Source["Source config"] + Source --> Feed["Feed children"] + Feed --> ConfigAPI["/api/v1/earth/news-sources"] + ConfigAPI --> Config["SystemSetting: earth_news_sources"] + + Earth["Earth News Panel"] --> NewsAPI["/api/v1/news/earth-feed"] + NewsAPI --> Resolver["Source Resolver"] + Resolver --> Config + Resolver --> Cache["Region Feed Cache"] + Resolver --> Fetcher["RSS / Atom Fetcher"] + Fetcher --> Parser["Feed Parser"] + Parser --> Classifier["Classifier: category + item_tags + importance"] + Classifier --> Store["earth_news_items"] + Fetcher --> Health["source health"] + Health --> Config + Store --> EnrichQueue["Location / Localization Queue"] + EnrichQueue --> AI["AI Provider"] + Store --> NewsAPI + UE["UE Client"] --> NewsAPI +``` diff --git a/docs/technical/en/frontend-admin-frontend-context.md b/docs/technical/en/frontend-admin-frontend-context.md index 15451218..8e87c63d 100644 --- a/docs/technical/en/frontend-admin-frontend-context.md +++ b/docs/technical/en/frontend-admin-frontend-context.md @@ -117,6 +117,8 @@ Admin runtime errors are reported through [runtimeLogs.ts](/home/ray/dev/linkong The Logs page follows log increments through the `/ws` `logs_tail` channel. File logs and database logs are both normalized into line events by the backend. When adding a new log source, wire it through the backend source registry and tail manager instead of adding a page-local poller. +The Logs page now opens in the grouped view by default. It reads `/api/v1/system/logs/observability/groups`, groups Earth, Admin, and service runtime reports by `fingerprint`, and then reads `/api/v1/system/logs/observability/groups/{fingerprint}/events` when an operator opens one group. Raw logs and audit logs remain separate views; only the raw-log view can follow WebSocket updates. Frontend reporters coalesce repeated errors in a short window and submit `occurrence_count`, while the backend writes both `system_logs` and `observability_events` / `observability_event_groups`, so the page should not add another browser-side aggregation pass over identical messages. + The datasource task queue `View Logs` action opens `/logs?source=system-db&search=task_id=`. Backend database-log search indexes must expand simple JSON context fields into `key=value` aliases such as `task_id=26906` and `datasource_id=20`, so historical task logs remain discoverable without rerunning the task. ## Current Shared Components diff --git a/docs/technical/en/manual.md b/docs/technical/en/manual.md index 711fdf88..71808bcf 100644 --- a/docs/technical/en/manual.md +++ b/docs/technical/en/manual.md @@ -300,6 +300,8 @@ Adopt All is for batch processing the compute-center unresolved queue. It starts The settings panel is grouped into Runtime, Display, Panels, Motion, Shortcuts, and System. It covers rotate / cruise / motion mode, cruise modules (BGP/news/compute centers/vessels/cables/satellites), view (satellite display style, hover tooltip, satellite idle breathing, real satellite altitude, track display, compact dots, day-night mode, panel toggles), motion debug mode / input source / skeleton-only, shortcut enablement and remapping, default globe size, terrain opacity, reset. +News categories use the same chip selector as Cruise Modules. They only filter the news panel and news cruise items in the current browser; they do not affect layers, TV, data points, basemap, boundaries, collector jobs, or admin news-source configuration. + "Real Satellite Altitude" is enabled by default: satellite positions use a compressed display height based on TLE/SGP4 orbital altitude. LEO satellites remain close to the globe, while high-orbit satellites render farther out without leaving the normal view. The high-orbit display height is capped at about one quarter of the globe radius, so GEO / MEO objects remain visually separated from LEO without spreading trails and selection targets too far apart. Turning it off restores the legacy same-sphere satellite display. "Track Display" controls satellite trail visibility; trails are unavailable while the satellite layer is hidden. "Hover Tooltip" controls the tooltip shown when the pointer hovers over the globe surface: `Country` shows country details only when land matches a country, and stays silent over oceans such as the Pacific; `Position` shows latitude, longitude, and elevation over land and ocean; `Full` is the default and shows country + position over land and position over ocean. diff --git a/docs/technical/zh/README.md b/docs/technical/zh/README.md index 776649f8..ca2d2fb4 100644 --- a/docs/technical/zh/README.md +++ b/docs/technical/zh/README.md @@ -23,6 +23,7 @@ - [智能星球可交互图标接入](/home/ray/dev/linkong/planet/docs/technical/zh/earth-interactable-usage.md):`Interactable` 的接口、生命周期和接入示例 - [智能星球可交互图标聚类策略](/home/ray/dev/linkong/planet/docs/technical/zh/earth-interactable-clustering.md):可插拔 cluster strategy、稳定球面聚类和动态屏幕聚类的适用边界 - [智能星球工具栏与浮层协同](/home/ray/dev/linkong/planet/docs/technical/zh/earth-toolbar-overlay-coordination.md):工具栏按钮与搜索、设置、新闻、图层浮层的关闭矩阵 +- [智能星球新闻源配置](/home/ray/dev/linkong/planet/docs/technical/zh/earth-news-sources.md):默认新闻源、Feed 子项、源属性标签、内容类型、重要度规则和配置接口 ## 前端技术实现 diff --git a/docs/technical/zh/earth-frontend-context.md b/docs/technical/zh/earth-frontend-context.md index 804d5972..6d33a38a 100644 --- a/docs/technical/zh/earth-frontend-context.md +++ b/docs/technical/zh/earth-frontend-context.md @@ -75,6 +75,8 @@ Earth 收到 `/ws` 的 `earth_updates` 时只把它当作刷新提示,真实 Earth 设置面板现在按 `data-settings-tab` 和 `data-settings-tab-panel` 分类组织。桌面端和移动端使用同一组分类语义:运行、显示、面板、动捕、快捷键、系统。新增设置项时应先判断它属于哪个分类,再补 DOM、持久化字段和恢复逻辑;不要把所有控件继续堆到一个长面板里。 +`显示` 分类里的新闻类型选择复用巡航模块的 chip 选择器形态,只控制星球端当前浏览器的新闻分类显示。它不会打开或关闭图层、底图、边界、TV、数据点、BGP、船舶、卫星或算力中心;这些仍由图层面板、媒体面板和控制台配置各自负责。`controls.js` 只持久化 `shared.newsCategoryFilters` 并广播 `earth:news-category-filters-change`,`news.js` 会把选中的类型拼到 `/api/v1/news/earth-feed?categories=...&locale=zh-CN`,让 Web 和 UE 走同一套后端类型过滤。 + 快捷键配置属于设备本地偏好,由 `controls.js` 负责读取、捕获、启用/禁用和重置。它不应写入后端用户设置,也不应影响其它浏览器。后续新增快捷键时,必须同时提供默认键、显示标签、可禁用状态和重置路径,避免只在 keydown handler 中硬编码。 ### 4. UI 与状态消息 diff --git a/docs/technical/zh/earth-news-sources.md b/docs/technical/zh/earth-news-sources.md new file mode 100644 index 00000000..8918d976 --- /dev/null +++ b/docs/technical/zh/earth-news-sources.md @@ -0,0 +1,161 @@ +# Earth 新闻源配置 + +Earth 态势新闻使用 `/api/v1/news/earth-feed` 输出给前端。新闻源配置存放在 `SystemSetting.category = "earth_news_sources"`;没有数据库配置时,后端使用内置默认源作为 fallback seed。 + +## 默认源 + +默认源包含四类: + +- **新闻源**:BBC World、DW Top Stories、CNBC Business、BBC Business、Guardian Business、NPR Business、MarketWatch、TechCrunch、Retail Dive、PR Newswire Retail、36氪、亿邦动力、国家统计局数据发布。 +- **行业洞察源**:McKinsey Retail、Deloitte Retail。 +- **官方数据源**:国家统计局数据发布、US Census Retail / E-Commerce、商务数据中心、商务部电商动态、电商物流指数。 +- **线索源**:BusinessWire Electronic Commerce;Google News 作为一个聚合 source,下面挂 global / americas / europe / middle-east-africa / asia-pacific 五个区域 Feed 子项。 + +配置型数据源默认保留在 Admin 配置中,但若不是稳定 RSS/Atom,则默认不参与自动抓取。管理员可以在 `Earth 内容 -> 新闻源` 中改成可抓取 RSS、启用或禁用。 + +| 来源 | 类型 | 默认状态 | 默认主类型 | 主要标签 | 用途 | +| --- | --- | --- | --- | --- | --- | +| BBC World | RSS | 启用 | 政治 | `official_media`, `global` | 全球公共新闻基线 | +| DW Top Stories | RSS | 启用 | 政治 | `official_media`, `europe` | 欧洲与国际新闻基线 | +| CNBC Business | RSS | 启用 | 商业 | `business_news`, `us`, `global` | 国际商业新闻 | +| BBC Business / Guardian Business / NPR Business / MarketWatch | RSS | 启用 | 商业 / 金融 | `business_news`, `finance` | 英美商业与金融基线 | +| TechCrunch / Retail Dive / PR Newswire Retail | RSS | 启用 | 科技 / 商业 | `business_news`, `ecommerce`, `retail`, `press_release` | 科技、电商、零售和企业公告 | +| 36氪 | RSS | 启用 | 商业 | `business_news`, `ecommerce`, `china` | 国内商业、创投和快讯;主页是 `https://www.36kr.com/`,Feed 信息页是 `https://www.36kr.com/rss-center`,Feed 子项是综合资讯、文章资讯、最新快讯、动态内容 | +| 亿邦动力 | RSS | 启用 | 电商 | `ecommerce`, `business_news`, `china`, `retail` | 国内电商行业新闻;主页是 `https://www.ebrun.com/`,Feed 信息页是 `https://www.ebrun.com/rss/`,Feed 子项是 B2C、B2B、零售、O2O、服务、数据、政策 XML | +| 国家统计局数据发布 | RSS | 启用 | 电商 | `official_data`, `ecommerce`, `retail`, `china` | 官方数据发布 RSS;社零和网上零售条目由分类/重要度规则识别 | +| Google News | Aggregated | 启用 | 政治 | `aggregated`, `low_stability` | 一个聚合 source,Feed 子项为全球、美洲、欧洲、中东与非洲、亚太区域兜底 RSS;优先级低于真实 RSS | +| BusinessWire Electronic Commerce | Reference | 禁用 | 电商 | `press_release`, `ecommerce`, `low_stability` | 企业公告线索 | +| McKinsey Retail Insights | Reference | 禁用 | 商业 | `industry_insight`, `retail` | 零售行业洞察 | +| Deloitte Retail | Reference | 禁用 | 商业 | `industry_insight`, `retail` | 零售行业洞察 | +| US Census Retail / E-Commerce | Reference | 禁用 | 电商 | `official_data`, `ecommerce`, `retail`, `us` | 美国零售和电商官方数据 | +| 商务数据中心 | Reference | 禁用 | 商业 | `official_data`, `china` | 国内商务数据 | +| 商务部电商动态 | Reference | 禁用 | 电商 | `official_data`, `ecommerce`, `china` | 国内电商政策与动态 | +| 电商物流指数 | Reference | 禁用 | 电商 | `official_data`, `ecommerce`, `logistics`, `china` | 物流履约与电商景气度 | + +`Reference` 源表示参考链接/未来采集器线索,只记录官网、报告页或数据页,不参与 RSS/Atom 抓取。这样可以把商业与官方数据源先纳入后台治理,同时避免不可抓取页面拖垮新闻 feed。 + +新闻源模型是两层结构: + +- `source` 表示来源品牌或聚合器,例如 36氪、亿邦动力、Google News、BBC。 +- `homepage_url` 表示来源官网、栏目页或报告页。 +- `feed_directory_url` 表示 Feed 信息页,也就是 RSS 订阅中心或 Feed 聚合页,只用于人工查看,不参与抓取。 +- `feeds` 表示该来源下真正抓取的 RSS、Atom 或 Aggregated 子项。每个 Feed 子项都有 `id / name / url / type / enabled / default_category / tags / priority`。 + +后端会遍历同一 source 下所有启用的 Feed 子项,逐个抓取、合并去重,并把单个子项的检测结果写入 `health.feed_results`。这不是“备用地址”逻辑;36氪的四个订阅地址、亿邦的多个分类 XML、Google News 的五个区域 RSS 都可以同时启用,并且每个 Feed 可以单独配置默认新闻类型和启用状态。HTML 订阅中心或聚合页只能放在 `feed_directory_url`,不能放进 Feed 地址。默认启用的可抓 Feed 已逐项连通性检测:RSS/Atom/Aggregated Feed 必须解析到条目,Reference 源只保留参考地址和后续采集器线索。 + +当前仍保留为 Reference 的项不是“坏源”,而是没有找到稳定、可直接消费的 RSS/Atom: + +- BusinessWire 官方说明支持可定制 RSS/Atom,但公开页面未暴露稳定行业 feed URL;当前保留电子商务行业页作为公告线索。 +- McKinsey / Deloitte 的零售洞察页是报告和文章集合,不是公开 RSS。 +- US Census 的 press release RSS 可访问,但条目链接为空;Quarterly E-Commerce 页面保留为官方数据参考链接。 +- 商务部数据、电商物流指数目前未找到稳定 RSS,后续应做专用 collector 或人工配置可抓 feed。 + +## 源属性标签与新闻类型 + +新闻源有 `source_tags`,在 Admin 中显示为“源属性标签”。它用于描述 source 的属性,不是媒体来源名,也不是新闻条目的内容类型。例如: + +- `official_data`:官方数据 +- `business_news`:商业新闻 +- `ecommerce`:电商 +- `finance`:金融 +- `retail`:零售 +- `logistics`:物流 +- `industry_insight`:行业洞察 +- `press_release`:企业公告 +- `china`、`global`、`us` +- `aggregated`、`low_stability` + +单条新闻有一个主类型 `category`,默认类型包括:政治、商业、电商、金融、体育、科技、军事、灾害、能源、社会、文化、其他。`item_tags` 是条目级补充标签,例如跨境电商、直播电商、零售数据、物流履约、平台治理、AI、半导体、选举、油价、足球、供应链。 + +主类型优先由规则引擎根据标题、摘要、来源名打分生成;规则未命中时优先使用 Feed 子项的默认类型,再回退 source 默认类型。AI enrichment 不阻塞新闻展示。 + +## 重要度 + +每条新闻输出: + +- `importance_score` +- `importance_level` +- `importance_reasons` +- `market_impact` + +官方数据源、电商指标、平台型公司、量化指标会提高重要度;企业公告基础权重较低,只有命中大平台、金额、并购、监管等信号时提升。 + +## 配置与缓存 + +`GET /api/v1/earth/news-sources` 返回默认或已保存配置。`PUT /api/v1/earth/news-sources` 保存配置并递增 `cache_version`,同时清理进程内 region cache。`POST /api/v1/earth/news-sources/reset` 恢复默认源。`POST /api/v1/earth/news-sources/test` 只测试单个 RSS/Atom/Aggregated 源,不写入新闻表。 + +## Feed 查询与类型过滤 + +星球端和 UE 端统一使用 `GET /api/v1/news/earth-feed` 获取新闻。接口支持服务端过滤,不要求客户端拿全量列表后自行筛选。 + +- `lat` / `lon`:按当前视角推断区域,适合 Web 星球端。 +- `region`:显式指定区域,适合 UE 端或服务端集成;可选值包括 `global`、`americas`、`europe`、`asia-pacific`、`middle-east-africa`。`global` 是全局聚合视图,会展示所有区域来源;其它区域只展示该区域和 `global` 来源。 +- `categories`:逗号分隔的新闻类型 key,例如 `business,ecommerce`。全选时可以不传。 +- `locale`:展示语言,支持 `zh-CN` 和 `en-US`,默认 `zh-CN`。中文 RSS 会以中文原文入库,并由后台补 `en-US`;英文 RSS 则由后台补 `zh-CN`。 + +示例: + +```http +GET /api/v1/news/earth-feed?region=europe&categories=business,ecommerce +GET /api/v1/news/earth-feed?lat=48&lon=10&categories=technology +GET /api/v1/news/earth-feed?region=global&categories=business,ecommerce&locale=zh-CN +``` + +非法新闻类型或语言会返回 `422`,响应中包含允许值。响应体会带 `filters`,用于确认后端实际应用的区域、类型和语言过滤。`items` 和 `cruise_items` 使用同一套类型过滤规则。 + +Web 星球端的新闻类型按钮只保存当前浏览器的显示偏好;偏好变化后会重新请求接口。UE 端应直接把类型选择拼到 `categories` 参数里,不需要再做主过滤。 + +源测试只证明当前 RSS/Atom/XML 能解析到条目,不等于这些条目已经入库展示。展示链路还会检查区域、类型过滤和数据库新鲜度。保存或重置新闻源会递增配置版本并清理缓存;如果当前启用的 Feed 子项在库里没有近期条目,下一次 `earth-feed` 请求会补抓,避免新启用的 36氪、亿邦被旧 Google News 缓存挡住。 + +## 连通性监测 + +`POST /api/v1/earth/news-sources/test` 会测试单个源并把结果写入 `earth_news_sources.health[source_id]`。实际 RSS/Atom 抓取也会更新同一份健康状态。 + +健康结果包含: + +- `status`:`ok`、`empty`、`format_error`、`http_error`、`timeout`、`network_error`、`reference`。 +- `status_code`、`content_type`、`item_count`、`latency_ms`、`error`、`fetched_at`。 +- `feed_results`:多 Feed source 的逐 Feed 子项检测结果,包含 `feed_id`、`feed_name`、`feed_type`、`feed_url`、状态、条数和错误。 + +常见诊断: + +- 返回 HTML 页面:说明配置 URL 不是 RSS/Atom feed,例如把网页中心页当成 feed。 +- HTTP 403:通常是 CDN、反爬或源站拒绝抓取。 +- Reference:参考链接,不参与抓取;需要改为 RSS、Atom 或 Aggregated 后才可测试抓取。 + +Admin 入口是 `Earth 内容 -> 新闻源`。界面不是整包 JSON 编辑,而是两层: + +- **新闻源**:左侧逐个 source 列表,支持按启用、停用、参考链接、RSS/Atom/Aggregated、区域和源属性标签筛选;右侧编辑当前 source 字段和 Feed 子项列表。 +- **策略规则**:保留源属性标签、新闻类型、条目标签规则、默认健康策略等全局规则。高级 JSON 只用于排障,不作为默认编辑路径。 + +单源表单分为“来源信息”和“Feed 子项”: + +- 来源信息包括名称、ID、区域、主页 URL、Feed 信息页、源类型、启用开关、源属性标签、重要度权重、抓取间隔、超时、失败阈值和熔断开关。 +- Feed 子项包括 Feed ID、名称、真实 Feed URL、类型、启用开关、默认新闻类型、优先级和 Feed 标签。Feed 子项底部的 `+` 只新增一个前端草稿;保存 source 后才写入配置,取消会销毁草稿。 + +单源“测试源”会测试当前 source 下全部启用 Feed;Feed 子项上的测试按钮只测试当前 Feed。测试请求仍发送到 `/api/v1/earth/news-sources/test`,但 payload 里只带当前 source 和选中的 Feed 子项。 + +参考链接会显示“只记录官网、报告页或未来采集器线索,不参与 RSS/Atom 抓取”。它可作为商业或官方数据线索保留在配置中,但启用抓取前必须改成 RSS、Atom 或 Aggregated,并提供可抓取的 Feed 地址。 + +```mermaid +flowchart LR + Admin["Admin: Earth 内容 / 新闻源"] --> Source["Source 配置"] + Source --> Feed["Feed 子项"] + Feed --> ConfigAPI["/api/v1/earth/news-sources"] + ConfigAPI --> Config["SystemSetting: earth_news_sources"] + + Earth["Earth 新闻面板"] --> NewsAPI["/api/v1/news/earth-feed"] + NewsAPI --> Resolver["Source Resolver"] + Resolver --> Config + Resolver --> Cache["Region Feed Cache"] + Resolver --> Fetcher["RSS / Atom Fetcher"] + Fetcher --> Parser["Feed Parser"] + Parser --> Classifier["Classifier: category + item_tags + importance"] + Classifier --> Store["earth_news_items"] + Fetcher --> Health["source health"] + Health --> Config + Store --> EnrichQueue["Location / Localization Queue"] + EnrichQueue --> AI["AI Provider"] + Store --> NewsAPI + UE["UE Client"] --> NewsAPI +``` diff --git a/docs/technical/zh/frontend-admin-frontend-context.md b/docs/technical/zh/frontend-admin-frontend-context.md index 2a525caa..d57e21bf 100644 --- a/docs/technical/zh/frontend-admin-frontend-context.md +++ b/docs/technical/zh/frontend-admin-frontend-context.md @@ -117,6 +117,8 @@ Admin 的状态标签统一走 [StatusText](/home/ray/dev/linkong/planet/fronten 日志页通过 `/ws` 的 `logs_tail` channel 跟随日志增量;文件日志和数据库日志都由后端统一转换成行事件。新增日志源时优先接入后端 source registry 和 tail manager,不要在日志页写独立轮询器。 +日志页默认进入“重复统计”视图,读取 `/api/v1/system/logs/observability/groups`,按 `fingerprint` 聚合 Earth、Admin 和服务端运行时上报;点击聚合项再读取 `/api/v1/system/logs/observability/groups/{fingerprint}/events` 展示发生明细。原始日志和审计日志仍保留为独立视图;只有原始日志视图允许通过 WebSocket 跟随。前端上报器会在短时间窗口内合并同一错误并提交 `occurrence_count`,后端同时写 `system_logs` 和 `observability_events` / `observability_event_groups`,所以日志页不要再按相同消息在浏览器端二次聚合。 + 数据源任务队列的“查看日志”入口跳转到 `/logs?source=system-db&search=task_id=`。后端数据库日志搜索索引必须把 JSON context 中的简单字段同时展开为 `key=value` 别名,例如 `task_id=26906`、`datasource_id=20`,这样历史任务日志不依赖重新执行任务也能被精确查到。 ## 当前共享组件 diff --git a/docs/technical/zh/manual.md b/docs/technical/zh/manual.md index 586cf289..68a4588b 100644 --- a/docs/technical/zh/manual.md +++ b/docs/technical/zh/manual.md @@ -299,6 +299,8 @@ AIS 船只图例按船型显示颜色:货轮、油轮、客船、渔船、军 设置面板按分类组织:运行、显示、面板、动捕、快捷键、系统。里面包含旋转模式 / 巡航模式 / 动捕模式、巡航模块(BGP/新闻/算力中心/船只/海缆/卫星)、视图设置(卫星显示风格、悬停提示、卫星呼吸闪烁、真实卫星高度、轨迹显示、低缩放圆点、日夜模式、面板显示开关)、动捕调试模式 / 输入源 / 只显示骨骼、快捷键启用与改键、地球默认大小、地形透明度、重置设置。 +新闻类型使用与巡航模块一致的标签选择器,只筛选当前浏览器里的新闻面板和新闻巡航条目,不影响图层、TV、数据点、底图、边界、采集任务或后台新闻源配置。 + “真实卫星高度”默认开启:卫星会按 TLE/SGP4 算出的真实轨道高度做压缩分层显示,低轨仍靠近地球,高轨会更远但不会脱离当前视图。高轨显示高度会被压到地球半径外约四分之一以内,这样 GEO / MEO 仍能和 LEO 分层,但不会把视线、轨迹和选择操作拉得过散;关闭后恢复旧版所有卫星位于同一显示球面的效果。“轨迹显示”控制卫星轨迹线显隐,卫星图层关闭时轨迹也不可见。 “悬停提示”控制鼠标悬停地表时的 tooltip 内容:`国家` 只在陆地命中国家时显示国家信息,太平洋等海洋区域不弹出地表提示;`位置` 在陆地和海洋都显示纬度、经度和海拔;`完整` 是默认模式,陆地显示国家 + 位置,海洋显示位置。 diff --git a/docs/version-history.md b/docs/version-history.md index 4c1add01..0d0656e3 100644 --- a/docs/version-history.md +++ b/docs/version-history.md @@ -16,12 +16,13 @@ ## Current Version - `main` 当前主线历史推导到:`0.16.5` -- `dev` 当前开发分支历史推导到:`0.68.1` +- `dev` 当前开发分支历史推导到:`0.69.0` ## Timeline | Version | Type | Branch | Commit | Summary | | --- | --- | --- | --- | --- | +| `0.69.0` | feature | `dev` | `pending` | 新增 Earth 新闻源治理、新闻类型服务端过滤、观测日志 fingerprint 聚合和 TV/HLS 播放恢复改进 | | `0.68.1` | bugfix | `dev` | `pending` | 修复 CelesTrak fallback group/cache 恢复链路,并让数据源任务日志可按 task_id / datasource_id 搜索 | | `0.68.0` | feature | `dev` | `pending` | 新增数据源任务队列实时指标、AIS 大表分批删除和智能星球可插拔聚类策略,并让新设备启动前同步前端依赖 | | `0.67.0` | feature | `dev` | `pending` | 新增控制台日志实时跟随和运行时错误上报,重构智能星球 Interactable 聚合、wheel 缩放输入、国界壳半径和开发脚本锁文件保护 | diff --git a/frontend/package.json b/frontend/package.json index aec4d646..ffda52c5 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "planet-frontend", - "version": "0.68.1", + "version": "0.69.0", "private": true, "packageManager": "bun@1", "dependencies": { diff --git a/frontend/public/earth/css/hud.css b/frontend/public/earth/css/hud.css index ba82c006..00c45af7 100644 --- a/frontend/public/earth/css/hud.css +++ b/frontend/public/earth/css/hud.css @@ -1086,6 +1086,12 @@ font-weight: 600; } +.earth-mobile-news-filter-popover { + position: relative; + inset: auto; + max-height: 34vh; +} + .earth-mobile-news-board-list .news-story-card { margin: 0; } diff --git a/frontend/public/earth/css/info-panel.css b/frontend/public/earth/css/info-panel.css index 0dd024c1..b6f1fb59 100644 --- a/frontend/public/earth/css/info-panel.css +++ b/frontend/public/earth/css/info-panel.css @@ -427,6 +427,43 @@ text-wrap: balance; } +.info-card-news-meta-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: calc(6px * var(--hud-scale)); +} + +.info-card-news-meta-grid--mobile { + grid-template-columns: 1fr; +} + +.info-card-news-meta-item { + min-width: 0; + padding: calc(6px * var(--hud-scale)) calc(8px * var(--hud-scale)); + border: 1px solid rgba(201, 225, 247, 0.08); + background: rgba(255, 255, 255, 0.035); +} + +.info-card-news-meta-item span, +.info-card-news-meta-item strong { + display: block; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.info-card-news-meta-item span { + color: rgba(188, 212, 238, 0.62); + font-size: calc(0.58rem * var(--hud-scale)); +} + +.info-card-news-meta-item strong { + color: rgba(236, 246, 255, 0.9); + font-size: calc(0.68rem * var(--hud-scale)); + font-weight: 650; + margin-top: calc(2px * var(--hud-scale)); +} + .info-card-news-summary-shell { position: relative; padding: calc(10px * var(--hud-scale)) calc(12px * var(--hud-scale)); diff --git a/frontend/public/earth/css/news-panel.css b/frontend/public/earth/css/news-panel.css index 14714c1a..883a548e 100644 --- a/frontend/public/earth/css/news-panel.css +++ b/frontend/public/earth/css/news-panel.css @@ -303,6 +303,121 @@ font-size: calc(0.72rem * var(--hud-scale)); } +.news-filter-bar { + position: relative; + display: flex; + align-items: center; + gap: calc(8px * var(--hud-scale)); + flex-wrap: wrap; +} + +.news-filter-pill { + display: inline-flex; + align-items: center; + gap: calc(6px * var(--hud-scale)); + min-height: calc(30px * var(--hud-scale)); + border: 1px solid rgba(201, 225, 247, 0.1); + border-radius: calc(12px * var(--hud-scale)); + padding: calc(5px * var(--hud-scale)) calc(9px * var(--hud-scale)); + color: var(--hud-text-soft); + background: + linear-gradient(180deg, rgba(255, 255, 255, 0.07), rgba(116, 166, 224, 0.04)); + box-shadow: + inset 0 1px 0 rgba(255, 255, 255, 0.06), + 0 8px 18px rgba(2, 10, 22, 0.12); + font: inherit; + font-size: calc(0.72rem * var(--hud-scale)); + cursor: pointer; +} + +.news-filter-pill:hover, +.news-filter-pill[aria-expanded="true"] { + color: var(--hud-text); + border-color: rgba(147, 202, 255, 0.22); + background: + linear-gradient(180deg, rgba(255, 255, 255, 0.1), rgba(116, 166, 224, 0.07)); +} + +.news-filter-pill strong { + color: var(--hud-accent-strong); + font-size: calc(0.7rem * var(--hud-scale)); + font-weight: 700; +} + +.news-filter-pill--view { + margin-left: auto; +} + +.news-filter-popover { + position: absolute; + top: calc(154px * var(--hud-scale)); + left: calc(14px * var(--hud-scale)); + right: calc(14px * var(--hud-scale)); + z-index: 30; + display: grid; + gap: calc(10px * var(--hud-scale)); + max-height: min(calc(280px * var(--hud-scale)), 44vh); + overflow-y: auto; + border: 1px solid rgba(205, 231, 255, 0.12); + border-radius: calc(16px * var(--hud-scale)); + padding: calc(12px * var(--hud-scale)); + color: var(--hud-text); + background: + radial-gradient(circle at 18% 10%, rgba(122, 187, 255, 0.16), transparent 42%), + linear-gradient(180deg, rgba(20, 35, 58, 0.96), rgba(10, 20, 35, 0.96)); + box-shadow: + 0 20px 50px rgba(2, 8, 20, 0.38), + inset 0 1px 0 rgba(255, 255, 255, 0.07); + backdrop-filter: blur(18px) saturate(125%); + -webkit-backdrop-filter: blur(18px) saturate(125%); +} + +.news-filter-popover[hidden] { + display: none; +} + +.news-filter-popover__header { + display: flex; + align-items: baseline; + justify-content: space-between; + gap: calc(10px * var(--hud-scale)); +} + +.news-filter-popover__title { + font-size: calc(0.8rem * var(--hud-scale)); + font-weight: 700; +} + +.news-filter-popover__hint { + color: var(--hud-text-muted); + font-size: calc(0.66rem * var(--hud-scale)); +} + +.news-filter-chip-group { + display: flex; + flex-wrap: wrap; + gap: calc(8px * var(--hud-scale)); +} + +.news-filter-chip { + border: 1px solid rgba(201, 225, 247, 0.12); + border-radius: calc(14px * var(--hud-scale)); + padding: calc(7px * var(--hud-scale)) calc(10px * var(--hud-scale)); + color: var(--hud-text-soft); + background: rgba(255, 255, 255, 0.04); + font: inherit; + font-size: calc(0.74rem * var(--hud-scale)); + cursor: pointer; +} + +.news-filter-chip.is-active { + color: var(--hud-text); + border-color: rgba(120, 190, 255, 0.36); + background: + linear-gradient(180deg, rgba(79, 143, 232, 0.22), rgba(64, 111, 191, 0.12)); + box-shadow: inset 0 0 0 1px rgba(206, 232, 255, 0.08); +} + .news-board { display: flex; flex: 1 1 auto; @@ -379,13 +494,21 @@ .news-story-tags { display: flex; align-items: center; - justify-content: space-between; gap: calc(8px * var(--hud-scale)); flex-wrap: wrap; } +.news-story-meta { + justify-content: space-between; +} + +.news-story-tags { + justify-content: flex-start; +} + .news-story-source, .news-story-time, +.news-story-origin, .news-story-tag { color: var(--hud-text-soft); font-size: calc(0.66rem * var(--hud-scale)); @@ -393,6 +516,17 @@ .news-story-source { color: var(--hud-accent-strong); + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.news-story-origin { + color: rgba(188, 212, 238, 0.56); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; } .news-story-title { diff --git a/frontend/public/earth/index.html b/frontend/public/earth/index.html index a547fbf9..05254a50 100644 --- a/frontend/public/earth/index.html +++ b/frontend/public/earth/index.html @@ -606,6 +606,22 @@
0 路聚合源
+
+ + + +
+ +
正在准备全球态势新闻...
@@ -768,6 +784,21 @@
0 路聚合源
+
+ + + +
+
正在准备全球态势新闻...
@@ -1049,6 +1080,27 @@ +
新闻类型
+
+
+ 新闻类型 + 只筛选当前浏览器的新闻面板与新闻巡航,不改变后台新闻源。 +
+
+ + + + + + + + + + + + +
+