release: bump version to 0.58.0

Release 0.58.0 includes the Earth high-precision boundary PMTiles/MVT pipeline, standardized Earth boundary source collectors, China POV boundary configuration templates, and removal of the legacy low-precision GeoJSON fallback. It also adds Earth news target-location queueing/archive support, fixes datasource task status visibility, documents the Earth surface depth-spacing rules that prevent far-zoom z-fighting snow/black blocks, and updates bilingual operations/developer docs.

backend/app/services/collectors/__init__.py

@@ -36,8 +36,15 @@ from app.services.collectors.iptoasn import IPtoASNPrefixGeoCollector
 from app.services.collectors.opengeofeed import OpenGeoFeedPrefixGeoCollector
 from app.services.collectors.nro_delegated import NRODelegatedPrefixGeoCollector
 from app.services.collectors.news_live_streams import NewsLiveStreamsCollector
+from app.services.collectors.media_news_archive import MediaNewsArchiveCollector
 from app.services.collectors.aisstream import AISStreamCollector
 from app.services.collectors.vessel_ais import VesselAISCollector
+from app.services.collectors.earth_boundaries import (
+    EarthAdmin0BoundaryCollector,
+    EarthBoundaryTileCollector,
+    EarthClaimLinesCollector,
+    EarthCoastlineCollector,
+)
 
 collector_registry.register(TOP500Collector())
 collector_registry.register(EpochAIGPUCollector())
@@ -65,8 +72,13 @@ collector_registry.register(IPtoASNPrefixGeoCollector())
 collector_registry.register(OpenGeoFeedPrefixGeoCollector())
 collector_registry.register(NRODelegatedPrefixGeoCollector())
 collector_registry.register(NewsLiveStreamsCollector())
+collector_registry.register(MediaNewsArchiveCollector())
 collector_registry.register(VesselAISCollector())
 collector_registry.register(AISStreamCollector())
+collector_registry.register(EarthAdmin0BoundaryCollector())
+collector_registry.register(EarthCoastlineCollector())
+collector_registry.register(EarthClaimLinesCollector())
+collector_registry.register(EarthBoundaryTileCollector())
 
 __all__ = [
     "BaseCollector",
@@ -100,6 +112,11 @@ __all__ = [
     "OpenGeoFeedPrefixGeoCollector",
     "NRODelegatedPrefixGeoCollector",
     "NewsLiveStreamsCollector",
+    "MediaNewsArchiveCollector",
     "VesselAISCollector",
     "AISStreamCollector",
+    "EarthAdmin0BoundaryCollector",
+    "EarthCoastlineCollector",
+    "EarthClaimLinesCollector",
+    "EarthBoundaryTileCollector",
 ]

backend/app/services/collectors/earth_boundaries.py

@@ -0,0 +1,578 @@
+"""Earth boundary source and static tile collector."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import shutil
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
+
+import httpx
+from sqlalchemy import select
+
+from app.core.earth_boundary_defaults import EARTH_BOUNDARY_DEFAULT_SOURCES, default_earth_boundary_config
+from app.models.collected_data import CollectedData
+from app.models.datasource_config import DataSourceConfig
+from app.services.collectors.base import BaseCollector
+from app.services.custom_datasource_runtime import build_query_params, build_request_headers
+
+
+REPO_ROOT = Path(__file__).resolve().parents[4]
+SOURCE_OUTPUT_DIR = REPO_ROOT / "data/earth-boundary-sources"
+SOURCE_MANIFEST_PATH = SOURCE_OUTPUT_DIR / "manifest.json"
+BOUNDARY_OUTPUT_DIR = REPO_ROOT / "frontend/public/earth/data/boundaries/v1"
+BOUNDARY_MANIFEST_PATH = BOUNDARY_OUTPUT_DIR / "manifest.json"
+PMTILES_ARTIFACT_PATH = REPO_ROOT / "frontend/public/earth/data/boundaries/earth-boundaries-china-pov-v1.pmtiles"
+POV_POLICY_PATH = REPO_ROOT / "config/earth-boundary-pov-policy.china-v1.json"
+BUILD_CONFIG = {
+    "builder": "scripts/build_earth_boundary_china_pov_geojson.py",
+    "format": "geojson-high-precision",
+    "production_target": "geojson-high-precision",
+}
+
+EARTH_BOUNDARY_SOURCE_COLLECTORS = {
+    "earth_admin0_boundaries": "admin0-boundaries",
+    "earth_coastline": "coastline",
+    "earth_claim_lines": "claim-lines",
+}
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    if not path.exists():
+        return {}
+    with path.open("r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def _directory_stats(path: Path) -> dict[str, int]:
+    if not path.exists():
+        return {"file_count": 0, "size_bytes": 0}
+    files = [item for item in path.rglob("*") if item.is_file()]
+    return {
+        "file_count": len(files),
+        "size_bytes": sum(item.stat().st_size for item in files),
+    }
+
+
+def _stable_json_hash(payload: Any) -> str:
+    encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+
+
+def _build_input_hash(source_manifest: dict[str, Any]) -> str:
+    return _stable_json_hash(
+        {
+            "source_manifest_schema": source_manifest.get("schema"),
+            "sources": [
+                {
+                    "id": source.get("id"),
+                    "sha256": source.get("sha256"),
+                    "kind": source.get("kind"),
+                    "pov": source.get("pov"),
+                }
+                for source in source_manifest.get("sources", [])
+            ],
+            "pov_policy": source_manifest.get("povPolicy"),
+            "build_config": BUILD_CONFIG,
+        }
+    )
+
+
+def _has_current_artifacts(boundary_manifest: dict[str, Any], build_input_hash: str) -> bool:
+    if not boundary_manifest:
+        return False
+    if boundary_manifest.get("buildInputHash") != build_input_hash:
+        return False
+    if not BOUNDARY_OUTPUT_DIR.exists():
+        return False
+    if boundary_manifest.get("tileProvider") == "pmtiles-mvt":
+        return PMTILES_ARTIFACT_PATH.exists()
+    return (BOUNDARY_OUTPUT_DIR / "base.geojson").exists() and (
+        BOUNDARY_OUTPUT_DIR / "hover-index.geojson"
+    ).exists()
+
+
+def _sha256_bytes(payload: bytes) -> str:
+    return hashlib.sha256(payload).hexdigest()
+
+
+def _json_feature_count(payload: Any) -> int:
+    if isinstance(payload, dict) and isinstance(payload.get("features"), list):
+        return len(payload["features"])
+    if isinstance(payload, list):
+        return len(payload)
+    return 1 if payload else 0
+
+
+def _json_sample_properties(payload: Any) -> dict[str, Any]:
+    feature = None
+    if isinstance(payload, dict) and isinstance(payload.get("features"), list) and payload["features"]:
+        feature = payload["features"][0]
+    elif isinstance(payload, list) and payload:
+        feature = payload[0]
+    elif isinstance(payload, dict):
+        feature = payload
+    if not isinstance(feature, dict):
+        return {}
+    props = feature.get("properties") if isinstance(feature.get("properties"), dict) else feature
+    return {str(key): value for key, value in list(props.items())[:20]}
+
+
+def _artifact_extension(endpoint: str, content_type: str, payload: bytes) -> str:
+    suffix = Path(endpoint.split("?", 1)[0]).suffix.lower()
+    if suffix in {".json", ".geojson", ".zip", ".pbf"}:
+        return suffix
+    if "geo+json" in content_type or b'"FeatureCollection"' in payload[:4096]:
+        return ".geojson"
+    if "json" in content_type:
+        return ".json"
+    return ".dat"
+
+
+async def _load_datasource_config(db, name: str) -> DataSourceConfig | None:
+    if db is None:
+        return None
+    result = await db.execute(
+        select(DataSourceConfig)
+        .where(DataSourceConfig.name == name)
+        .where(DataSourceConfig.is_active.is_(True))
+        .order_by(DataSourceConfig.id.desc())
+        .limit(1)
+    )
+    return result.scalar_one_or_none()
+
+
+async def _latest_boundary_source_record(db, source_name: str) -> CollectedData | None:
+    result = await db.execute(
+        select(CollectedData)
+        .where(CollectedData.source == source_name)
+        .where(CollectedData.data_type == "earth_boundary_source")
+        .where(CollectedData.is_current.is_(True))
+        .order_by(CollectedData.collected_at.desc().nullslast(), CollectedData.id.desc())
+        .limit(1)
+    )
+    return result.scalar_one_or_none()
+
+
+class EarthBoundarySourceCollector(BaseCollector):
+    priority = "P1"
+    module = "L3"
+    frequency_hours = 168
+    data_type = "earth_boundary_source"
+    fail_on_empty = True
+    source_kind = "unknown"
+
+    def _default_config(self):
+        source = EARTH_BOUNDARY_DEFAULT_SOURCES.get(self.name)
+        if not source:
+            return None
+        return SimpleNamespace(
+            name=self.name,
+            description=f"内置默认源：{self.name}",
+            endpoint=source["endpoint"],
+            source_type="http",
+            auth_type="none",
+            auth_config={},
+            headers={},
+            config=default_earth_boundary_config(self.name),
+        )
+
+    async def _download_payload(self, config: DataSourceConfig) -> tuple[bytes, str]:
+        request_config = config.config or {}
+        method = str(request_config.get("method") or request_config.get("request_method") or "GET").upper()
+        if method not in {"GET", "POST"}:
+            raise RuntimeError("Earth boundary source collectors support GET and POST only")
+
+        endpoint = str(config.endpoint or "").strip()
+        if not endpoint:
+            raise RuntimeError(
+                f"{self.name} requires an endpoint in Collector Settings before it can collect data"
+            )
+
+        if endpoint.startswith("file://") or Path(endpoint).expanduser().exists():
+            path = Path(endpoint.removeprefix("file://")).expanduser()
+            return path.read_bytes(), "application/octet-stream"
+
+        headers = build_request_headers(config.auth_type, config.auth_config or {}, config.headers or {})
+        params = build_query_params(config.auth_type, config.auth_config or {}, request_config)
+        timeout = float(request_config.get("timeout", 120))
+        json_body = request_config.get("json_body") or request_config.get("body")
+        async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
+            response = await client.request(
+                method,
+                endpoint,
+                headers=headers,
+                params=params or None,
+                json=json_body if isinstance(json_body, (dict, list)) else None,
+            )
+            response.raise_for_status()
+            return response.content, response.headers.get("content-type", "")
+
+    async def fetch(self) -> list[dict[str, Any]]:
+        db = getattr(self, "_db_session", None)
+        config = await _load_datasource_config(db, self.name) or self._default_config()
+        if config is None:
+            raise RuntimeError(
+                f"{self.name} has no active Collector Settings config and no built-in default source."
+            )
+
+        await self.set_phase("fetching_source", message=f"正在下载 {self.source_kind} 源数据")
+        payload, content_type = await self._download_payload(config)
+        sha256 = _sha256_bytes(payload)
+        endpoint = str(config.endpoint or "")
+        extension = _artifact_extension(endpoint, content_type, payload)
+        source_dir = SOURCE_OUTPUT_DIR / self.name
+        source_dir.mkdir(parents=True, exist_ok=True)
+        artifact_path = source_dir / f"{sha256}{extension}"
+        artifact_path.write_bytes(payload)
+
+        parsed: Any = None
+        if extension in {".json", ".geojson"}:
+            parsed = json.loads(payload.decode("utf-8"))
+
+        feature_count = _json_feature_count(parsed)
+        if feature_count <= 0:
+            raise RuntimeError(f"{self.name} downloaded data but found no JSON/GeoJSON features")
+
+        config_body = config.config or {}
+        target_schema = str(config_body.get("target_schema") or "earth_boundary_source")
+        if target_schema != "earth_boundary_source":
+            raise RuntimeError(f"{self.name} target_schema must be earth_boundary_source")
+
+        await self.update_phase_progress(
+            current=1,
+            total=1,
+            unit="artifact",
+            message=f"已保存 {feature_count} 个 {self.source_kind} feature",
+            progress=100,
+            commit=True,
+            force=True,
+        )
+
+        relative_artifact_path = str(artifact_path.relative_to(REPO_ROOT))
+        return [
+            {
+                "id": f"{self.source_kind}:{sha256}",
+                "name": config.description or self.name,
+                "description": f"Earth boundary source artifact collected from configured endpoint",
+                "source_kind": self.source_kind,
+                "source_id": sha256,
+                "value": feature_count,
+                "unit": "features",
+                "metadata": {
+                    "target_schema": target_schema,
+                    "source_kind": self.source_kind,
+                    "endpoint": endpoint,
+                    "method": str(config_body.get("method") or config_body.get("request_method") or "GET").upper(),
+                    "artifact_path": relative_artifact_path,
+                    "sha256": sha256,
+                    "feature_count": feature_count,
+                    "size_bytes": len(payload),
+                    "content_type": content_type,
+                    "license": config_body.get("license"),
+                    "mapping_json": config_body.get("mapping_json"),
+                    "sample_properties": _json_sample_properties(parsed),
+                },
+            }
+        ]
+
+
+class EarthAdmin0BoundaryCollector(EarthBoundarySourceCollector):
+    name = "earth_admin0_boundaries"
+    source_kind = "admin0-boundaries"
+
+
+class EarthCoastlineCollector(EarthBoundarySourceCollector):
+    name = "earth_coastline"
+    source_kind = "coastline"
+
+
+class EarthClaimLinesCollector(EarthBoundarySourceCollector):
+    name = "earth_claim_lines"
+    source_kind = "claim-lines"
+
+
+class EarthBoundaryTileCollector(BaseCollector):
+    name = "earth_boundary_tiles"
+    priority = "P1"
+    module = "L3"
+    frequency_hours = 168
+    data_type = "earth_boundary_tiles"
+    fail_on_empty = False
+
+    async def _run_step(self, args: list[str], *, allow_failure: bool = False) -> dict[str, Any]:
+        process = await asyncio.create_subprocess_exec(
+            sys.executable,
+            *args,
+            cwd=REPO_ROOT,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        stdout_bytes, stderr_bytes = await process.communicate()
+        stdout = stdout_bytes.decode("utf-8", errors="replace").strip()
+        stderr = stderr_bytes.decode("utf-8", errors="replace").strip()
+        if process.returncode != 0 and not allow_failure:
+            raise RuntimeError(stderr or stdout or f"command failed: {' '.join(args)}")
+
+        payload: dict[str, Any] = {
+            "stdout": stdout,
+            "stderr": stderr,
+            "returncode": process.returncode,
+        }
+        last_line = stdout.splitlines()[-1:] or []
+        if last_line:
+            try:
+                payload["result"] = json.loads(last_line[0])
+            except json.JSONDecodeError:
+                payload["result"] = last_line[0]
+        return payload
+
+    async def fetch(self) -> list[dict[str, Any]]:
+        db = getattr(self, "_db_session", None)
+        if db is None:
+            raise RuntimeError("Earth PMTiles builder requires an active database session")
+
+        await self.set_phase("checking_sources", message="正在检查三类 Earth 边界源")
+        await self.update_phase_progress(
+            current=0,
+            total=3,
+            unit="steps",
+            message="读取 admin0 / coastline / claim-lines 最新采集结果",
+            progress=5,
+            commit=True,
+            force=True,
+        )
+
+        source_records: dict[str, CollectedData] = {}
+        missing_sources: list[str] = []
+        for source_name in EARTH_BOUNDARY_SOURCE_COLLECTORS:
+            record = await _latest_boundary_source_record(db, source_name)
+            if record is None:
+                missing_sources.append(source_name)
+            else:
+                source_records[source_name] = record
+
+        if missing_sources:
+            missing_text = ", ".join(missing_sources)
+            raise RuntimeError(f"未就绪：缺少 {missing_text}。不会更新 Earth 国界。")
+
+        sources = []
+        for source_name, record in source_records.items():
+            metadata = record.extra_data or {}
+            artifact_path = metadata.get("artifact_path")
+            if not artifact_path or not (REPO_ROOT / str(artifact_path)).exists():
+                missing_sources.append(f"{source_name}: artifact missing")
+                continue
+            sources.append(
+                {
+                    "id": source_name,
+                    "kind": metadata.get("source_kind") or EARTH_BOUNDARY_SOURCE_COLLECTORS[source_name],
+                    "path": str(artifact_path),
+                    "sha256": metadata.get("sha256"),
+                    "featureCount": metadata.get("feature_count"),
+                    "license": metadata.get("license"),
+                }
+            )
+
+        if missing_sources:
+            missing_text = ", ".join(missing_sources)
+            raise RuntimeError(f"未就绪：{missing_text}。不会更新 Earth 国界。")
+
+        source_manifest = {
+            "schema": "planet-earth-boundary-sources/v2",
+            "sources": sources,
+            "povPolicy": _read_json(POV_POLICY_PATH),
+        }
+        SOURCE_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+        with SOURCE_MANIFEST_PATH.open("w", encoding="utf-8") as f:
+            json.dump(source_manifest, f, ensure_ascii=False, indent=2)
+            f.write("\n")
+        build_input_hash = _build_input_hash(source_manifest)
+        await self.update_phase_progress(
+            current=1,
+            total=3,
+            unit="steps",
+            message="三类边界源已就绪",
+            progress=35,
+            commit=True,
+            force=True,
+        )
+
+        await self.set_phase("checking_readiness", message="正在检查 Earth 国界构建就绪状态")
+        missing_tools = [tool for tool in ("tippecanoe", "pmtiles") if shutil.which(tool) is None]
+        readiness_result = {
+            "returncode": 0,
+            "result": {
+                "ready": True,
+                "failures": [f"external tool not found in PATH: {tool}" for tool in missing_tools],
+                "sources": sources,
+                "artifact": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
+                "fallback_builder": "geojson-high-precision" if missing_tools else None,
+            },
+        }
+
+        await self.set_phase("building_tiles", message="正在检查 Earth 国界瓦片产物")
+        boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
+        build_skipped = _has_current_artifacts(boundary_manifest, build_input_hash)
+        if build_skipped:
+            build_result = {
+                "result": {
+                    "status": "unchanged",
+                    "reason": "source manifest and build config hash unchanged",
+                    "buildInputHash": build_input_hash,
+                }
+            }
+        elif shutil.which("tippecanoe") and shutil.which("pmtiles"):
+            admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
+            coastline = next(source for source in sources if source["kind"] == "coastline")
+            claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
+            build_result = await self._run_step(
+                [
+                    "scripts/build_earth_boundary_pmtiles.py",
+                    "--admin0-source",
+                    admin0["path"],
+                    "--coastline-source",
+                    coastline["path"],
+                    "--claims-source",
+                    claim_lines["path"],
+                    "--output",
+                    str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
+                    "--manifest",
+                    str(BOUNDARY_MANIFEST_PATH.relative_to(REPO_ROOT)),
+                    "--build-input-hash",
+                    build_input_hash,
+                    "--pov-policy",
+                    str(POV_POLICY_PATH.relative_to(REPO_ROOT)),
+                ]
+            )
+            boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
+        else:
+            admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
+            coastline = next(source for source in sources if source["kind"] == "coastline")
+            claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
+            build_result = await self._run_step(
+                [
+                    "scripts/build_earth_boundary_china_pov_geojson.py",
+                    "--admin0-source",
+                    admin0["path"],
+                    "--coastline-source",
+                    coastline["path"],
+                    "--claims-source",
+                    claim_lines["path"],
+                    "--output-dir",
+                    str(BOUNDARY_OUTPUT_DIR.relative_to(REPO_ROOT)),
+                    "--build-input-hash",
+                    build_input_hash,
+                ]
+            )
+            boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
+        boundary_stats = _directory_stats(BOUNDARY_OUTPUT_DIR)
+        pmtiles_exists = PMTILES_ARTIFACT_PATH.exists()
+        pmtiles_size = PMTILES_ARTIFACT_PATH.stat().st_size if pmtiles_exists else 0
+        await self.update_phase_progress(
+            current=2,
+            total=3,
+            unit="steps",
+            message=(
+                "边界源未变化，已跳过瓦片重建"
+                if build_skipped
+                else f"已生成 {boundary_manifest.get('tileProvider') or 'boundary'} 国界产物"
+            ),
+            progress=80,
+            commit=True,
+            force=True,
+        )
+
+        await self.set_phase("indexing_artifacts", message="正在登记边界瓦片产物")
+        tile_counts = boundary_manifest.get("tiles", {}).get("countsByZoom", {})
+        records = [
+            {
+                "id": "source-manifest",
+                "name": "Earth boundary source manifest",
+                "description": "Offline source collection manifest for Earth boundary tiles",
+                "value": len(source_manifest.get("sources", [])),
+                "unit": "sources",
+                "metadata": {
+                    "manifest_path": str(SOURCE_MANIFEST_PATH.relative_to(REPO_ROOT)),
+                    "manifest": source_manifest,
+                    "collector_result": {
+                        "status": "loaded_from_collected_data",
+                        "sources": [source["id"] for source in sources],
+                    },
+                },
+            },
+            {
+                "id": "production-readiness",
+                "name": "Earth boundary production readiness",
+                "description": "Checks whether all source artifacts and boundary build tooling are available",
+                "value": 1 if readiness_result.get("returncode") == 0 else 0,
+                "unit": "ready",
+                "metadata": {
+                    "result": readiness_result.get("result"),
+                    "returncode": readiness_result.get("returncode"),
+                },
+            },
+            {
+                "id": "boundary-manifest",
+                "name": "Earth boundary tile manifest",
+                "description": "Versioned static vector tile manifest for Earth country boundaries",
+                "value": boundary_stats["file_count"],
+                "unit": "files",
+                "metadata": {
+                    "manifest_path": str(BOUNDARY_MANIFEST_PATH.relative_to(REPO_ROOT)),
+                    "output_dir": str(BOUNDARY_OUTPUT_DIR.relative_to(REPO_ROOT)),
+                    "size_bytes": boundary_stats["size_bytes"],
+                    "manifest": boundary_manifest,
+                    "collector_result": build_result.get("result"),
+                    "build_skipped": build_skipped,
+                    "production_target": BUILD_CONFIG["production_target"],
+                    "pmtiles_artifact": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
+                    "pmtiles_exists": pmtiles_exists,
+                },
+            },
+        ]
+        if pmtiles_exists:
+            records.append(
+                {
+                    "id": "pmtiles-artifact",
+                    "name": "Earth boundary PMTiles artifact",
+                    "description": "Single-file PMTiles/MVT artifact for Earth boundaries",
+                    "value": pmtiles_size,
+                    "unit": "bytes",
+                    "metadata": {
+                        "path": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
+                        "exists": pmtiles_exists,
+                        "provider": "pmtiles-mvt",
+                    },
+                }
+            )
+        for zoom, count in sorted(tile_counts.items(), key=lambda item: int(item[0])):
+            records.append(
+                {
+                    "id": f"tile-z{zoom}",
+                    "name": f"Earth boundary tiles z{zoom}",
+                    "description": f"Generated Earth boundary tile count for zoom {zoom}",
+                    "value": int(count),
+                    "unit": "tiles",
+                    "metadata": {
+                        "zoom": int(zoom),
+                        "tile_count": int(count),
+                        "output_dir": str((BOUNDARY_OUTPUT_DIR / str(zoom)).relative_to(REPO_ROOT)),
+                    },
+                }
+            )
+
+        await self.update_phase_progress(
+            current=3,
+            total=3,
+            unit="steps",
+            message="边界瓦片产物已登记",
+            progress=100,
+            commit=True,
+            force=True,
+        )
+        return records

backend/app/services/collectors/media_news_archive.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+from typing import Any
+
+from app.services.collectors.base import BaseCollector
+from app.services.earth_news_store import list_all_earth_news_records
+
+
+class MediaNewsArchiveCollector(BaseCollector):
+    name = "media_news_archive"
+    priority = "P2"
+    module = "L4"
+    frequency_hours = 12
+    data_type = "news_item"
+    fail_on_empty = False
+
+    async def fetch(self) -> list[dict[str, Any]]:
+        if not self._db_session:
+            return []
+
+        records = await list_all_earth_news_records(self._db_session)
+        items: list[dict[str, Any]] = []
+        for record in records:
+            location_meta = dict(record.location_meta or {})
+            target = location_meta.get("target") if isinstance(location_meta.get("target"), dict) else {}
+            country = target.get("country")
+            city = target.get("city")
+            items.append(
+                {
+                    "id": record.id,
+                    "source_id": record.id,
+                    "name": record.title,
+                    "title": record.title,
+                    "description": record.summary,
+                    "country": country,
+                    "city": city,
+                    "latitude": record.latitude,
+                    "longitude": record.longitude,
+                    "reference_date": record.published_at,
+                    "metadata": {
+                        "url": record.url,
+                        "source": record.source,
+                        "feed_name": record.feed_name,
+                        "region": record.region,
+                        "homepage_url": record.homepage_url,
+                        "published_at": record.published_at.isoformat() if record.published_at else None,
+                        "location_label": record.location_label,
+                        "location_source": record.location_source,
+                        "verified": record.verified,
+                        "location_meta": location_meta,
+                        "first_seen_at": record.first_seen_at.isoformat() if record.first_seen_at else None,
+                        "last_seen_at": record.last_seen_at.isoformat() if record.last_seen_at else None,
+                        "resolved_at": record.resolved_at.isoformat() if record.resolved_at else None,
+                    },
+                }
+            )
+        return items

backend/app/services/datasource_mapping.py

@@ -8,6 +8,7 @@ import re
 from datetime import UTC, datetime
 from typing import Any
 
+from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.core.target_schema_registry import TargetSchema, get_target_schema
@@ -254,6 +255,12 @@ def _best_field_match(field_name: str, candidates: list[str]) -> str | None:
         "lat": ("lat", "latitude", "y"),
         "lon": ("lon", "lng", "longitude", "x"),
         "mmsi": ("mmsi",),
+        "geometry": ("geometry", "geom"),
+        "properties": ("properties", "props"),
+        "source_kind": ("source_kind", "kind", "type"),
+        "feature_count": ("feature_count", "features_count", "count"),
+        "artifact_path": ("artifact_path", "path", "file"),
+        "sha256": ("sha256", "hash", "checksum"),
         "sog": ("sog", "speed", "speedOverGround"),
         "cog": ("cog", "course", "courseOverGround"),
         "received_at": ("received_at", "timestamp", "time", "updated_at"),
@@ -294,6 +301,52 @@ async def persist_mapped_records(
     transport: str | None = None,
 ) -> int:
     """Persist validated mapped records to the destination for a target schema."""
+    if target_schema == "earth_boundary_source":
+        from app.models.collected_data import CollectedData
+
+        now = datetime.now(UTC)
+        written_count = 0
+        for index, record in enumerate(records):
+            source_id = record.get("source_id") or record.get("sha256") or str(index)
+            entity_key = f"{datasource_name}:{source_id}"
+            previous_result = await db.execute(
+                select(CollectedData)
+                .where(CollectedData.source == datasource_name)
+                .where(CollectedData.entity_key == entity_key)
+                .where(CollectedData.is_current.is_(True))
+                .order_by(CollectedData.id.desc())
+                .limit(1)
+            )
+            previous = previous_result.scalar_one_or_none()
+            if previous is not None:
+                previous.is_current = False
+            db.add(
+                CollectedData(
+                    source=datasource_name,
+                    source_id=str(source_id),
+                    entity_key=entity_key,
+                    data_type=target_schema,
+                    name=record.get("name") or str(source_id),
+                    description=f"Earth boundary source artifact: {record.get('source_kind')}",
+                    extra_data={
+                        **record,
+                        "datasource_config_id": datasource_config_id,
+                        "mapping_version": mapping_version,
+                        "delivery_mode": delivery_mode or "polling",
+                        "transport": transport or "http",
+                    },
+                    collected_at=now,
+                    is_valid=1,
+                    is_current=True,
+                    previous_record_id=previous.id if previous else None,
+                    change_type="updated" if previous else "created",
+                    change_summary={},
+                )
+            )
+            written_count += 1
+        await db.commit()
+        return written_count
+
     if target_schema == "vessel_ais":
         from app.core.time import to_iso8601_utc
         from app.core.websocket.broadcaster import broadcaster

backend/app/services/earth_news.py

@@ -6,6 +6,8 @@ from datetime import UTC, datetime
 from email.utils import parsedate_to_datetime
 import hashlib
 import html
+import json
+import math
 import re
 from typing import Any
 from urllib.parse import quote
@@ -13,6 +15,12 @@ import xml.etree.ElementTree as ET
 
 import httpx
 from bs4 import BeautifulSoup
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.countries import COUNTRY_VARIANTS_MAP, get_country_centroid, normalize_country
+from app.schemas.ai import SituationalAnalysisRequest
+from app.services.ai_client import AIProviderClient
+from app.services.location.resolvers.nominatim import build_default_nominatim_geocoder
 
 
 USER_AGENT = "PlanetEarthNewsBoard/1.0 (+https://planet.local)"
@@ -20,6 +28,9 @@ REQUEST_TIMEOUT = 12.0
 MAX_ITEMS_PER_SOURCE = 6
 MAX_ITEMS_TOTAL = 12
 STALE_CACHE_MAX_AGE_SECONDS = 60 * 45
+RSS_SUPPLEMENT_MAX_AGE_SECONDS = STALE_CACHE_MAX_AGE_SECONDS
+MAX_TARGET_INFERENCE_CONCURRENCY = 3
+TARGET_INFERENCE_TIMEOUT_SECONDS = 6.0
 
 
 @dataclass(frozen=True)
@@ -49,6 +60,17 @@ class NewsFeedSource:
     priority: int = 100
 
 
+@dataclass(frozen=True)
+class NewsTargetLocation:
+    latitude: float
+    longitude: float
+    label: str
+    source: str
+    confidence: float | None = None
+    country: str | None = None
+    city: str | None = None
+
+
 @dataclass
 class ParsedNewsItem:
     id: str
@@ -60,6 +82,13 @@ class ParsedNewsItem:
     feed_region: str
     homepage_url: str
     published_at: datetime | None
+    target_location: NewsTargetLocation | None = None
+    target_resolution_stage: str = "unresolved"
+    target_ai_attempted: bool = False
+    target_ai_status: str = "not_attempted"
+    target_ai_error: str | None = None
+    target_debug_note: str | None = None
+    location_patch: dict[str, Any] | None = None
 
 
 @dataclass
@@ -236,6 +265,17 @@ NEWS_FEED_SOURCES: tuple[NewsFeedSource, ...] = (
 
 
 _REGION_CACHE: dict[str, CachedRegionFeed] = {}
+_news_target_geocode = build_default_nominatim_geocoder(user_agent=USER_AGENT)
+_CITY_HINTS: tuple[dict[str, str | None], ...] = (
+    {"name": "Beijing", "country": "中国"},
+    {"name": "Havana", "country": "古巴"},
+    {"name": "Kyiv", "country": "乌克兰"},
+    {"name": "Bangkok", "country": "泰国"},
+    {"name": "Tehran", "country": "伊朗"},
+    {"name": "Moscow", "country": "俄罗斯"},
+    {"name": "Taipei", "country": "中国（台湾）"},
+    {"name": "Hong Kong", "country": "中国（香港）"},
+)
 
 
 def determine_focus_region(lat: float | None, lon: float | None) -> str:
@@ -258,6 +298,341 @@ def get_region_anchor(region: str) -> RegionAnchor:
     return REGION_ANCHORS.get(region, REGION_ANCHORS["global"])
 
 
+def _coerce_str(value: Any) -> str | None:
+    if value is None:
+        return None
+    if not isinstance(value, str):
+        value = str(value)
+    cleaned = re.sub(r"\s+", " ", value).strip()
+    return cleaned or None
+
+
+def _contains_location_alias(text: str, alias: str) -> bool:
+    normalized_alias = _coerce_str(alias)
+    if not normalized_alias:
+        return False
+    if re.search(r"[A-Za-z]", normalized_alias):
+        pattern = r"(?<![A-Za-z])" + re.escape(normalized_alias) + r"(?![A-Za-z])"
+        return re.search(pattern, text, flags=re.IGNORECASE) is not None
+    return normalized_alias in text
+
+
+def _iter_searchable_country_variants(
+    canonical: str,
+    variants: list[str],
+) -> tuple[str, ...]:
+    searchable: list[str] = []
+    seen: set[str] = set()
+    for variant in (canonical, *variants):
+        normalized = _coerce_str(variant)
+        if not normalized:
+            continue
+        if re.fullmatch(r"[A-Z]{2,3}", normalized):
+            continue
+        if len(normalized) <= 2:
+            continue
+        key = normalized.casefold()
+        if key in seen:
+            continue
+        seen.add(key)
+        searchable.append(normalized)
+    return tuple(searchable)
+
+
+def _coerce_float(value: Any) -> float | None:
+    try:
+        parsed = float(value)
+    except (TypeError, ValueError):
+        return None
+    if not math.isfinite(parsed):
+        return None
+    return parsed
+
+
+def _first_json_object(text: str) -> dict[str, Any] | None:
+    if not text:
+        return None
+    decoder = json.JSONDecoder()
+    for index, char in enumerate(text):
+        if char != "{":
+            continue
+        try:
+            payload, _ = decoder.raw_decode(text[index:])
+        except ValueError:
+            continue
+        if isinstance(payload, dict):
+            return payload
+    return None
+
+
+async def _geocode_target_location(query: str) -> dict[str, Any] | None:
+    return await asyncio.to_thread(_news_target_geocode, query)
+
+
+async def _build_target_location_from_payload(
+    payload: dict[str, Any],
+) -> NewsTargetLocation | None:
+    country = normalize_country(payload.get("country"))
+    city = _coerce_str(payload.get("city"))
+    matched_location_name = _coerce_str(payload.get("matched_location_name"))
+    confidence = _coerce_float(payload.get("confidence"))
+    if confidence is not None:
+        confidence = max(0.0, min(confidence, 1.0))
+
+    latitude = _coerce_float(payload.get("latitude"))
+    longitude = _coerce_float(payload.get("longitude"))
+    if latitude not in (None, 0.0) and longitude not in (None, 0.0):
+        label = matched_location_name or ", ".join(part for part in (city, country) if part) or "关联位置"
+        return NewsTargetLocation(
+            latitude=latitude,
+            longitude=longitude,
+            label=label,
+            source="ai_inferred_target",
+            confidence=confidence,
+            country=country,
+            city=city,
+        )
+
+    geocode_queries: list[str] = []
+    for value in (
+        ", ".join(part for part in (city, country) if part),
+        matched_location_name,
+        city,
+        country,
+    ):
+        normalized = _coerce_str(value)
+        if normalized and normalized not in geocode_queries:
+            geocode_queries.append(normalized)
+
+    for query in geocode_queries:
+        try:
+            result = await _geocode_target_location(query)
+        except Exception:
+            continue
+        if not isinstance(result, dict):
+            continue
+        latitude = _coerce_float(result.get("lat"))
+        longitude = _coerce_float(result.get("lon"))
+        if latitude in (None, 0.0) or longitude in (None, 0.0):
+            continue
+        label = (
+            _coerce_str(result.get("display_name"))
+            or matched_location_name
+            or ", ".join(part for part in (city, country) if part)
+            or query
+        )
+        return NewsTargetLocation(
+            latitude=latitude,
+            longitude=longitude,
+            label=label,
+            source="ai_inferred_target",
+            confidence=confidence,
+            country=country,
+            city=city,
+        )
+
+    centroid = get_country_centroid(country)
+    if centroid:
+        label = matched_location_name or city or country or "关联位置"
+        return NewsTargetLocation(
+            latitude=centroid["latitude"],
+            longitude=centroid["longitude"],
+            label=label,
+            source="ai_inferred_target",
+            confidence=confidence,
+            country=country,
+            city=city,
+        )
+
+    return None
+
+
+async def _extract_target_location_from_text(item: ParsedNewsItem) -> NewsTargetLocation | None:
+    combined_text = " ".join(part for part in (item.title, item.summary) if part).strip()
+    if not combined_text:
+        return None
+
+    for hint in _CITY_HINTS:
+        city_name = _coerce_str(hint.get("name"))
+        if not city_name or not _contains_location_alias(combined_text, city_name):
+            continue
+        country = normalize_country(hint.get("country"))
+        geocode_query = ", ".join(part for part in (city_name, country) if part)
+        try:
+            result = await _geocode_target_location(geocode_query)
+        except Exception:
+            result = None
+        if isinstance(result, dict):
+            latitude = _coerce_float(result.get("lat"))
+            longitude = _coerce_float(result.get("lon"))
+            if latitude not in (None, 0.0) and longitude not in (None, 0.0):
+                return NewsTargetLocation(
+                    latitude=latitude,
+                    longitude=longitude,
+                    label=_coerce_str(result.get("display_name")) or geocode_query,
+                    source="headline_location_hint",
+                    confidence=0.78,
+                    country=country,
+                    city=city_name,
+                )
+        centroid = get_country_centroid(country)
+        if centroid:
+            return NewsTargetLocation(
+                latitude=centroid["latitude"],
+                longitude=centroid["longitude"],
+                label=geocode_query,
+                source="headline_location_hint",
+                confidence=0.68,
+                country=country,
+                city=city_name,
+            )
+
+    for canonical, variants in COUNTRY_VARIANTS_MAP.items():
+        if not get_country_centroid(canonical):
+            continue
+        searchable_variants = _iter_searchable_country_variants(canonical, variants)
+        if not any(_contains_location_alias(combined_text, variant) for variant in searchable_variants):
+            continue
+        centroid = get_country_centroid(canonical)
+        if not centroid:
+            continue
+        return NewsTargetLocation(
+            latitude=centroid["latitude"],
+            longitude=centroid["longitude"],
+            label=canonical,
+            source="headline_country_hint",
+            confidence=0.62,
+            country=canonical,
+            city=None,
+        )
+
+    return None
+
+
+async def _infer_news_target_location(
+    item: ParsedNewsItem,
+    *,
+    provider_client: AIProviderClient | None,
+) -> NewsTargetLocation | None:
+    text_hint = await _extract_target_location_from_text(item)
+    if text_hint is not None and text_hint.city:
+        item.target_resolution_stage = text_hint.source
+        item.target_ai_attempted = False
+        item.target_ai_status = "skipped_text_hint"
+        item.target_ai_error = None
+        item.target_debug_note = f"text hint matched {text_hint.label}"
+        return text_hint
+
+    if provider_client is None:
+        item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
+        item.target_ai_attempted = False
+        item.target_ai_status = "unavailable"
+        item.target_ai_error = "AI provider is not configured or unavailable for earth-feed."
+        item.target_debug_note = (
+            f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
+        )
+        return text_hint
+
+    item.target_ai_attempted = True
+    item.target_ai_status = "attempted"
+    item.target_ai_error = None
+    item.target_debug_note = (
+        f"text hint matched {text_hint.label}" if text_hint is not None else "no text location hint matched"
+    )
+
+    request = SituationalAnalysisRequest(
+        title="Infer likely event location for Earth news cruise",
+        objective=(
+            "Return exactly one strict JSON object for the most likely physical "
+            "location the news event is about. Prefer the host city when a state "
+            "visit, summit, meeting, attack, or disaster is clearly centered in a "
+            "known city. Fall back to the best-supported country only when a city "
+            "cannot be inferred."
+        ),
+        context={
+            "news_item": {
+                "title": item.title,
+                "summary": item.summary,
+                "source": item.source,
+                "feed_name": item.feed_name,
+                "feed_region": item.feed_region,
+                "url": item.url,
+                "published_at": (
+                    item.published_at.isoformat().replace("+00:00", "Z")
+                    if item.published_at
+                    else None
+                ),
+            },
+            "required_json_schema": {
+                "country": "string|null",
+                "city": "string|null",
+                "matched_location_name": "string|null",
+                "latitude": "number|null",
+                "longitude": "number|null",
+                "confidence": "number from 0 to 1",
+                "reasoning_summary": "short string",
+            },
+        },
+        constraints=[
+            "Return only strict JSON. Do not wrap it in markdown.",
+            "Prefer the event location, not the newsroom or publisher headquarters.",
+            "When a country visit or summit is the clear topic but the city is omitted, use the most likely host city only if it is broadly public knowledge.",
+            "Use null for unknown fields instead of inventing details.",
+            "Calibrate confidence conservatively: 0.75+ only when the city is strongly supported, 0.55-0.74 for country-level or likely city inference, below 0.55 when weak.",
+        ],
+    )
+    try:
+        response = await provider_client.analyze(request)
+    except Exception as exc:
+        item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
+        item.target_ai_status = "provider_error"
+        item.target_ai_error = str(exc)
+        return text_hint
+
+    payload = _first_json_object(response.content)
+    if not isinstance(payload, dict):
+        item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
+        item.target_ai_status = "parse_error"
+        item.target_ai_error = "AI response did not contain a parseable JSON object."
+        return text_hint
+
+    target = await _build_target_location_from_payload(payload)
+    if target is None:
+        item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
+        item.target_ai_status = "no_result"
+        item.target_ai_error = "AI returned no usable target coordinates or geocodeable location."
+        return text_hint
+    if target.confidence is not None and target.confidence < 0.45:
+        item.target_resolution_stage = text_hint.source if text_hint is not None else "unresolved"
+        item.target_ai_status = "low_confidence"
+        item.target_ai_error = f"AI target confidence too low: {target.confidence:.2f}"
+        return text_hint
+    item.target_resolution_stage = target.source
+    item.target_ai_status = "success"
+    item.target_ai_error = None
+    item.target_debug_note = f"ai inferred {target.label}"
+    return target
+
+
+async def _enrich_items_with_target_locations(
+    items: list[ParsedNewsItem],
+    *,
+    provider_client: AIProviderClient | None,
+) -> list[ParsedNewsItem]:
+    if not items:
+        return items
+
+    semaphore = asyncio.Semaphore(MAX_TARGET_INFERENCE_CONCURRENCY)
+
+    async def enrich(item: ParsedNewsItem) -> ParsedNewsItem:
+        async with semaphore:
+            target = await _infer_news_target_location(item, provider_client=provider_client)
+        item.target_location = target
+        return item
+
+    return list(await asyncio.gather(*(enrich(item) for item in items)))
+
+
 def get_sources_for_region(region: str) -> list[NewsFeedSource]:
     return sorted(
         [source for source in NEWS_FEED_SOURCES if source.region in {"global", region}],
@@ -385,9 +760,117 @@ def _serialize_sources(sources: list[NewsFeedSource]) -> list[dict[str, Any]]:
     ]
 
 
+def _serialize_anchor(anchor: RegionAnchor) -> dict[str, Any]:
+    return {
+        "region": anchor.region,
+        "label": anchor.label,
+        "latitude": anchor.latitude,
+        "longitude": anchor.longitude,
+    }
+
+
+def _serialize_target(target: NewsTargetLocation | None) -> dict[str, Any] | None:
+    if target is None:
+        return None
+    return {
+        "latitude": target.latitude,
+        "longitude": target.longitude,
+        "label": target.label,
+        "source": target.source,
+        "confidence": target.confidence,
+        "country": target.country,
+        "city": target.city,
+    }
+
+
+def build_anchor_location_patch(
+    item: ParsedNewsItem,
+    *,
+    queued: bool = False,
+    queue_available: bool | None = None,
+) -> dict[str, Any]:
+    anchor = get_region_anchor(item.feed_region)
+    if queued:
+        resolution_stage = "queued"
+        ai_status = "queued"
+        debug_note = "queued for async target location inference"
+    else:
+        resolution_stage = item.target_resolution_stage
+        ai_status = item.target_ai_status
+        debug_note = item.target_debug_note
+    return {
+        "latitude": anchor.latitude,
+        "longitude": anchor.longitude,
+        "location_label": anchor.label,
+        "location_source": "region_anchor",
+        "verified": False,
+        "location_meta": {
+            "resolution_stage": resolution_stage,
+            "ai_attempted": item.target_ai_attempted,
+            "ai_status": ai_status,
+            "ai_error": item.target_ai_error,
+            "debug_note": debug_note,
+            "queue_available": queue_available,
+            "target": None,
+            "anchor": _serialize_anchor(anchor),
+        },
+    }
+
+
+def build_target_location_patch(item: ParsedNewsItem, target: NewsTargetLocation | None) -> dict[str, Any]:
+    if target is None:
+        return build_anchor_location_patch(item)
+    anchor = get_region_anchor(item.feed_region)
+    return {
+        "latitude": target.latitude,
+        "longitude": target.longitude,
+        "location_label": target.label,
+        "location_source": target.source,
+        "verified": True,
+        "location_meta": {
+            "resolution_stage": item.target_resolution_stage,
+            "ai_attempted": item.target_ai_attempted,
+            "ai_status": item.target_ai_status,
+            "ai_error": item.target_ai_error,
+            "debug_note": item.target_debug_note,
+            "target": _serialize_target(target),
+            "anchor": _serialize_anchor(anchor),
+        },
+    }
+
+
+def build_target_location_job_payload(item: ParsedNewsItem) -> dict[str, Any]:
+    published_at = item.published_at
+    return {
+        "id": item.id,
+        "title": item.title,
+        "summary": item.summary,
+        "url": item.url,
+        "source": item.source,
+        "feed_name": item.feed_name,
+        "feed_region": item.feed_region,
+        "homepage_url": item.homepage_url,
+        "published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None,
+    }
+
+
+def parsed_news_item_from_job_payload(payload: dict[str, Any]) -> ParsedNewsItem:
+    return ParsedNewsItem(
+        id=str(payload.get("id") or ""),
+        title=str(payload.get("title") or ""),
+        summary=str(payload.get("summary") or ""),
+        url=str(payload.get("url") or ""),
+        source=str(payload.get("source") or ""),
+        feed_name=str(payload.get("feed_name") or ""),
+        feed_region=str(payload.get("feed_region") or "global"),
+        homepage_url=str(payload.get("homepage_url") or ""),
+        published_at=_parse_datetime(_coerce_str(payload.get("published_at"))),
+    )
+
+
 def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, Any]:
     published_at = item.published_at
-    anchor = get_region_anchor(item.feed_region)
+    location_patch = item.location_patch or build_target_location_patch(item, item.target_location)
     return {
         "id": item.id,
         "title": item.title,
@@ -398,10 +881,12 @@ def _serialize_item(item: ParsedNewsItem, *, active_region: str) -> dict[str, An
         "region": item.feed_region,
         "homepage_url": item.homepage_url,
         "published_at": published_at.isoformat().replace("+00:00", "Z") if published_at else None,
-        "latitude": anchor.latitude,
-        "longitude": anchor.longitude,
-        "location_label": anchor.label,
-        "location_inferred": True,
+        "latitude": location_patch["latitude"],
+        "longitude": location_patch["longitude"],
+        "location_label": location_patch["location_label"],
+        "location_source": location_patch["location_source"],
+        "verified": location_patch["verified"],
+        "location_meta": location_patch["location_meta"],
         "is_focus_match": item.feed_region == active_region,
     }
 
@@ -472,6 +957,47 @@ def _store_region_cache(region: str, *, items: list[ParsedNewsItem], sources: li
     )
 
 
+async def _apply_cached_locations_and_enqueue(items: list[ParsedNewsItem]) -> list[ParsedNewsItem]:
+    if not items:
+        return items
+
+    from app.services.earth_news_queue import (
+        enqueue_target_location_job,
+        get_cached_target_location_patch,
+    )
+
+    async def apply_location(item: ParsedNewsItem) -> ParsedNewsItem:
+        cached_patch = await get_cached_target_location_patch(item.id)
+        if cached_patch:
+            item.location_patch = cached_patch
+            return item
+
+        queued = await enqueue_target_location_job(build_target_location_job_payload(item))
+        item.location_patch = build_anchor_location_patch(
+            item,
+            queued=queued,
+            queue_available=queued,
+        )
+        return item
+
+    return list(await asyncio.gather(*(apply_location(item) for item in items)))
+
+
+async def _enqueue_unverified_locations(items: list[ParsedNewsItem]) -> None:
+    if not items:
+        return
+
+    from app.services.earth_news_queue import enqueue_target_location_job
+
+    await asyncio.gather(
+        *(
+            enqueue_target_location_job(build_target_location_job_payload(item))
+            for item in items
+            if item.location_patch is None or item.location_patch.get("verified") is False
+        )
+    )
+
+
 async def _fetch_source(
     client: httpx.AsyncClient,
     source: NewsFeedSource,
@@ -484,11 +1010,10 @@ async def _fetch_source(
         return source, [], str(exc)
 
 
-async def get_earth_news_payload(lat: float | None = None, lon: float | None = None) -> dict[str, Any]:
-    active_region = determine_focus_region(lat, lon)
-    sources = get_sources_for_region(active_region)
+async def _fetch_rss_items_for_sources(
+    sources: list[NewsFeedSource],
+) -> tuple[list[ParsedNewsItem], list[str]]:
     errors: list[str] = []
-
     async with httpx.AsyncClient(
         timeout=REQUEST_TIMEOUT,
         follow_redirects=True,
@@ -502,9 +1027,29 @@ async def get_earth_news_payload(lat: float | None = None, lon: float | None = N
             errors.append(f"{source.name}: {error}")
             continue
         fetched_items.extend(items)
+    return fetched_items, errors
 
+
+def _needs_rss_supplement(*, item_count: int, newest_at: datetime | None) -> bool:
+    if item_count < MAX_ITEMS_TOTAL:
+        return True
+    if newest_at is None:
+        return True
+    age_seconds = (datetime.now(UTC) - newest_at).total_seconds()
+    return age_seconds > RSS_SUPPLEMENT_MAX_AGE_SECONDS
+
+
+async def _get_earth_news_payload_from_rss_only(
+    *,
+    lat: float | None,
+    lon: float | None,
+    active_region: str,
+    sources: list[NewsFeedSource],
+) -> dict[str, Any]:
+    fetched_items, errors = await _fetch_rss_items_for_sources(sources)
     ranked_items = _rank_and_trim_items(fetched_items, active_region=active_region)
     if ranked_items:
+        ranked_items = await _apply_cached_locations_and_enqueue(ranked_items)
         _store_region_cache(active_region, items=ranked_items, sources=sources)
         return _build_payload(
             lat=lat,
@@ -518,6 +1063,7 @@ async def get_earth_news_payload(lat: float | None = None, lon: float | None = N
 
     cached = _get_cached_region_feed(active_region)
     if cached:
+        cached.items = await _apply_cached_locations_and_enqueue(cached.items)
         return _build_payload(
             lat=lat,
             lon=lon,
@@ -538,3 +1084,55 @@ async def get_earth_news_payload(lat: float | None = None, lon: float | None = N
         errors=errors,
         stale=False,
     )
+
+
+async def get_earth_news_payload(
+    lat: float | None = None,
+    lon: float | None = None,
+    *,
+    provider_client: AIProviderClient | None = None,
+    db: AsyncSession | None = None,
+) -> dict[str, Any]:
+    del provider_client
+    active_region = determine_focus_region(lat, lon)
+    sources = get_sources_for_region(active_region)
+
+    if db is None:
+        return await _get_earth_news_payload_from_rss_only(
+            lat=lat,
+            lon=lon,
+            active_region=active_region,
+            sources=sources,
+        )
+
+    from app.services.earth_news_store import (
+        get_earth_news_freshness,
+        list_earth_news_items,
+        upsert_earth_news_items,
+    )
+
+    errors: list[str] = []
+    item_count, newest_at = await get_earth_news_freshness(db, active_region=active_region)
+    should_supplement = _needs_rss_supplement(item_count=item_count, newest_at=newest_at)
+    if should_supplement:
+        fetched_items, errors = await _fetch_rss_items_for_sources(sources)
+        ranked_fetched_items = _rank_and_trim_items(fetched_items, active_region=active_region)
+        await upsert_earth_news_items(db, ranked_fetched_items)
+
+    items = await list_earth_news_items(
+        db,
+        active_region=active_region,
+        limit=MAX_ITEMS_TOTAL,
+    )
+    await _enqueue_unverified_locations(items)
+    stale = bool(errors and items)
+
+    return _build_payload(
+        lat=lat,
+        lon=lon,
+        active_region=active_region,
+        items=items,
+        sources=sources,
+        errors=errors,
+        stale=stale,
+    )

backend/app/services/earth_news_queue.py

@@ -0,0 +1,232 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import json
+from typing import Any, Protocol
+
+import redis.asyncio as redis
+from redis.exceptions import ResponseError
+
+from app.core.config import settings
+from app.core.logging import get_logger
+
+
+logger = get_logger(__name__, service="earth_news")
+
+TARGET_LOCATION_STREAM = "earth_news:target_location:jobs"
+TARGET_LOCATION_GROUP = "earth_news_target_location"
+TARGET_LOCATION_DEAD_LETTER_STREAM = "earth_news:target_location:dead"
+TARGET_LOCATION_RESULT_TTL_SECONDS = 60 * 60 * 12
+TARGET_LOCATION_JOB_DEDUP_TTL_SECONDS = 60 * 60 * 6
+TARGET_LOCATION_MAX_ATTEMPTS = 3
+
+_redis_client: redis.Redis | None = None
+
+
+@dataclass(frozen=True)
+class NewsTargetLocationMessage:
+    message_id: str
+    item_id: str
+    payload: dict[str, Any]
+    attempts: int = 0
+
+
+class NewsTargetLocationQueue(Protocol):
+    async def enqueue(self, *, item_id: str, payload: dict[str, Any]) -> bool:
+        ...
+
+    async def consume_batch(
+        self,
+        *,
+        consumer_name: str,
+        count: int,
+        block_ms: int,
+    ) -> list[NewsTargetLocationMessage]:
+        ...
+
+    async def ack(self, message_id: str) -> None:
+        ...
+
+    async def retry_or_dead_letter(
+        self,
+        message: NewsTargetLocationMessage,
+        *,
+        error: str,
+    ) -> None:
+        ...
+
+
+def _get_redis_client() -> redis.Redis:
+    global _redis_client
+    if _redis_client is None:
+        _redis_client = redis.from_url(settings.REDIS_URL, decode_responses=True)
+    return _redis_client
+
+
+def _result_key(item_id: str) -> str:
+    return f"earth_news:target_location:result:{item_id}"
+
+
+def _queued_key(item_id: str) -> str:
+    return f"earth_news:target_location:queued:{item_id}"
+
+
+class RedisStreamsNewsTargetLocationQueue:
+    def __init__(self, client: redis.Redis | None = None) -> None:
+        self.client = client or _get_redis_client()
+        self._group_ready = False
+
+    async def _ensure_group(self) -> None:
+        if self._group_ready:
+            return
+        try:
+            await self.client.xgroup_create(
+                TARGET_LOCATION_STREAM,
+                TARGET_LOCATION_GROUP,
+                id="0",
+                mkstream=True,
+            )
+        except ResponseError as exc:
+            if "BUSYGROUP" not in str(exc):
+                raise
+        self._group_ready = True
+
+    async def enqueue(self, *, item_id: str, payload: dict[str, Any]) -> bool:
+        await self._ensure_group()
+        if await self.client.exists(_result_key(item_id)):
+            return False
+        queued = await self.client.set(
+            _queued_key(item_id),
+            "1",
+            nx=True,
+            ex=TARGET_LOCATION_JOB_DEDUP_TTL_SECONDS,
+        )
+        if not queued:
+            return bool(await self.client.exists(_queued_key(item_id)))
+        await self.client.xadd(
+            TARGET_LOCATION_STREAM,
+            {
+                "item_id": item_id,
+                "attempts": "0",
+                "payload": json.dumps(payload, ensure_ascii=False),
+            },
+        )
+        return True
+
+    async def consume_batch(
+        self,
+        *,
+        consumer_name: str,
+        count: int,
+        block_ms: int,
+    ) -> list[NewsTargetLocationMessage]:
+        await self._ensure_group()
+        streams = await self.client.xreadgroup(
+            TARGET_LOCATION_GROUP,
+            consumer_name,
+            {TARGET_LOCATION_STREAM: ">"},
+            count=count,
+            block=block_ms,
+        )
+        messages: list[NewsTargetLocationMessage] = []
+        for _stream_name, stream_messages in streams:
+            for message_id, fields in stream_messages:
+                raw_payload = fields.get("payload")
+                item_id = fields.get("item_id")
+                if not raw_payload or not item_id:
+                    await self.ack(message_id)
+                    continue
+                try:
+                    payload = json.loads(raw_payload)
+                except json.JSONDecodeError:
+                    await self.ack(message_id)
+                    continue
+                attempts = int(fields.get("attempts") or 0)
+                messages.append(
+                    NewsTargetLocationMessage(
+                        message_id=message_id,
+                        item_id=item_id,
+                        payload=payload,
+                        attempts=attempts,
+                    )
+                )
+        return messages
+
+    async def ack(self, message_id: str) -> None:
+        await self.client.xack(TARGET_LOCATION_STREAM, TARGET_LOCATION_GROUP, message_id)
+
+    async def retry_or_dead_letter(
+        self,
+        message: NewsTargetLocationMessage,
+        *,
+        error: str,
+    ) -> None:
+        await self.ack(message.message_id)
+        if message.attempts + 1 >= TARGET_LOCATION_MAX_ATTEMPTS:
+            await self.client.xadd(
+                TARGET_LOCATION_DEAD_LETTER_STREAM,
+                {
+                    "item_id": message.item_id,
+                    "attempts": str(message.attempts + 1),
+                    "error": error,
+                    "payload": json.dumps(message.payload, ensure_ascii=False),
+                },
+            )
+            return
+        await self.client.xadd(
+            TARGET_LOCATION_STREAM,
+            {
+                "item_id": message.item_id,
+                "attempts": str(message.attempts + 1),
+                "payload": json.dumps(message.payload, ensure_ascii=False),
+            },
+        )
+
+
+def get_news_target_location_queue() -> NewsTargetLocationQueue:
+    return RedisStreamsNewsTargetLocationQueue()
+
+
+async def enqueue_target_location_job(payload: dict[str, Any]) -> bool:
+    item_id = str(payload.get("id") or "")
+    if not item_id:
+        return False
+    try:
+        queue = get_news_target_location_queue()
+        return await queue.enqueue(item_id=item_id, payload=payload)
+    except Exception as exc:
+        logger.warning_event(
+            "Failed to enqueue Earth news target location job",
+            event="earth_news.target_location.enqueue_failed",
+            context={"item_id": item_id, "error": str(exc)},
+        )
+        return False
+
+
+async def get_cached_target_location_patch(item_id: str) -> dict[str, Any] | None:
+    try:
+        raw_value = await _get_redis_client().get(_result_key(item_id))
+    except Exception as exc:
+        logger.warning_event(
+            "Failed to read Earth news target location cache",
+            event="earth_news.target_location.cache_read_failed",
+            context={"item_id": item_id, "error": str(exc)},
+        )
+        return None
+    if not raw_value:
+        return None
+    try:
+        value = json.loads(raw_value)
+    except json.JSONDecodeError:
+        return None
+    return value if isinstance(value, dict) else None
+
+
+async def save_target_location_patch(item_id: str, patch: dict[str, Any]) -> None:
+    client = _get_redis_client()
+    await client.setex(
+        _result_key(item_id),
+        TARGET_LOCATION_RESULT_TTL_SECONDS,
+        json.dumps(patch, ensure_ascii=False),
+    )
+    await client.delete(_queued_key(item_id))

backend/app/services/earth_news_store.py

@@ -0,0 +1,188 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.earth_news import EarthNewsItem
+from app.services.earth_news import (
+    ParsedNewsItem,
+    build_anchor_location_patch,
+)
+
+
+def _coerce_datetime(value: datetime | None) -> datetime | None:
+    if value is None:
+        return None
+    if value.tzinfo is None:
+        return value.replace(tzinfo=UTC)
+    return value.astimezone(UTC)
+
+
+def _location_patch_from_record(record: EarthNewsItem) -> dict[str, Any]:
+    return {
+        "latitude": record.latitude,
+        "longitude": record.longitude,
+        "location_label": record.location_label,
+        "location_source": record.location_source,
+        "verified": record.verified,
+        "location_meta": dict(record.location_meta or {}),
+    }
+
+
+def record_to_parsed_news_item(record: EarthNewsItem) -> ParsedNewsItem:
+    return ParsedNewsItem(
+        id=record.id,
+        title=record.title,
+        summary=record.summary or "",
+        url=record.url,
+        source=record.source or "",
+        feed_name=record.feed_name or "",
+        feed_region=record.region or "global",
+        homepage_url=record.homepage_url or "",
+        published_at=_coerce_datetime(record.published_at),
+        location_patch=_location_patch_from_record(record),
+    )
+
+
+def _query_sort_key(active_region: str):
+    return (
+        EarthNewsItem.region != active_region,
+        EarthNewsItem.published_at.is_(None),
+        EarthNewsItem.published_at.desc().nullslast(),
+        EarthNewsItem.feed_name.asc(),
+    )
+
+
+async def list_earth_news_items(
+    db: AsyncSession,
+    *,
+    active_region: str,
+    limit: int,
+) -> list[ParsedNewsItem]:
+    regions = {"global", active_region}
+    result = await db.execute(
+        select(EarthNewsItem)
+        .where(EarthNewsItem.region.in_(regions))
+        .order_by(*_query_sort_key(active_region))
+        .limit(limit)
+    )
+    return [record_to_parsed_news_item(record) for record in result.scalars().all()]
+
+
+async def get_earth_news_freshness(
+    db: AsyncSession,
+    *,
+    active_region: str,
+) -> tuple[int, datetime | None]:
+    regions = {"global", active_region}
+    result = await db.execute(
+        select(
+            func.count(EarthNewsItem.id),
+            func.max(func.coalesce(EarthNewsItem.published_at, EarthNewsItem.last_seen_at)),
+        ).where(EarthNewsItem.region.in_(regions))
+    )
+    count, newest = result.one()
+    item_count = int(count or 0)
+    if item_count == 0:
+        return 0, None
+    return item_count, _coerce_datetime(newest)
+
+
+async def upsert_earth_news_items(db: AsyncSession, items: list[ParsedNewsItem]) -> int:
+    if not items:
+        return 0
+    now = datetime.now(UTC)
+    existing_result = await db.execute(
+        select(EarthNewsItem).where(EarthNewsItem.id.in_([item.id for item in items]))
+    )
+    existing = {record.id: record for record in existing_result.scalars().all()}
+    changed = 0
+    for item in items:
+        record = existing.get(item.id)
+        if record is None:
+            patch = build_anchor_location_patch(item)
+            record = EarthNewsItem(
+                id=item.id,
+                title=item.title,
+                summary=item.summary,
+                url=item.url,
+                source=item.source,
+                feed_name=item.feed_name,
+                region=item.feed_region,
+                homepage_url=item.homepage_url,
+                published_at=item.published_at,
+                latitude=patch["latitude"],
+                longitude=patch["longitude"],
+                location_label=patch["location_label"],
+                location_source=patch["location_source"],
+                verified=patch["verified"],
+                location_meta=patch["location_meta"],
+                first_seen_at=now,
+                last_seen_at=now,
+            )
+            db.add(record)
+            changed += 1
+            continue
+
+        record.title = item.title
+        record.summary = item.summary
+        record.url = item.url
+        record.source = item.source
+        record.feed_name = item.feed_name
+        record.region = item.feed_region
+        record.homepage_url = item.homepage_url
+        record.published_at = item.published_at
+        record.last_seen_at = now
+        changed += 1
+    await db.flush()
+    return changed
+
+
+async def update_earth_news_item_location(
+    db: AsyncSession,
+    *,
+    item_id: str,
+    patch: dict[str, Any],
+) -> bool:
+    record = await db.get(EarthNewsItem, item_id)
+    if record is None:
+        return False
+    record.latitude = float(patch["latitude"])
+    record.longitude = float(patch["longitude"])
+    record.location_label = str(patch["location_label"])
+    record.location_source = str(patch["location_source"])
+    record.verified = bool(patch["verified"])
+    record.location_meta = dict(patch.get("location_meta") or {})
+    record.resolved_at = datetime.now(UTC) if record.verified else None
+    await db.flush()
+    return True
+
+
+async def list_unverified_earth_news_items(
+    db: AsyncSession,
+    *,
+    active_region: str,
+    limit: int,
+) -> list[ParsedNewsItem]:
+    regions = {"global", active_region}
+    result = await db.execute(
+        select(EarthNewsItem)
+        .where(EarthNewsItem.region.in_(regions))
+        .where(EarthNewsItem.verified.is_(False))
+        .order_by(*_query_sort_key(active_region))
+        .limit(limit)
+    )
+    return [record_to_parsed_news_item(record) for record in result.scalars().all()]
+
+
+async def list_all_earth_news_records(db: AsyncSession) -> list[EarthNewsItem]:
+    result = await db.execute(
+        select(EarthNewsItem).order_by(
+            EarthNewsItem.published_at.desc().nullslast(),
+            EarthNewsItem.last_seen_at.desc(),
+        )
+    )
+    return list(result.scalars().all())

backend/app/services/earth_news_worker.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+import asyncio
+from contextlib import suppress
+from socket import gethostname
+from typing import Any
+
+from app.core.logging import get_logger
+from app.core.websocket.broadcaster import broadcaster
+from app.db.session import async_session_factory
+from app.services.ai_client import AIProviderClient
+from app.services.earth_news import (
+    _infer_news_target_location,
+    build_target_location_patch,
+    parsed_news_item_from_job_payload,
+)
+from app.services.earth_news_queue import (
+    NewsTargetLocationMessage,
+    get_news_target_location_queue,
+    save_target_location_patch,
+)
+from app.services.earth_news_store import update_earth_news_item_location
+
+
+logger = get_logger(__name__, service="earth_news")
+
+WORKER_BATCH_SIZE = 4
+WORKER_BLOCK_MS = 5000
+WORKER_BACKOFF_SECONDS = 5.0
+
+_worker_task: asyncio.Task | None = None
+
+
+async def _build_provider_client() -> AIProviderClient | None:
+    try:
+        from app.api.v1.settings import get_runtime_ai_provider_config
+
+        async with async_session_factory() as session:
+            runtime_config = await get_runtime_ai_provider_config(session)
+        return AIProviderClient(
+            service_url=runtime_config["service_url"],
+            service_token=runtime_config["service_token"],
+            timeout=runtime_config["timeout_seconds"],
+            retry_attempts=runtime_config["retry_attempts"],
+            llm_config=runtime_config.get("llm_config") or {},
+        )
+    except Exception as exc:
+        logger.warning_event(
+            "Failed to build Earth news AI provider client",
+            event="earth_news.target_location.provider_unavailable",
+            context={"error": str(exc)},
+        )
+        return None
+
+
+async def process_target_location_message(
+    message: NewsTargetLocationMessage,
+    *,
+    provider_client: AIProviderClient | None,
+) -> dict[str, Any]:
+    item = parsed_news_item_from_job_payload(message.payload)
+    target = await _infer_news_target_location(item, provider_client=provider_client)
+    item.target_location = target
+    patch = build_target_location_patch(item, target)
+    await save_target_location_patch(item.id, patch)
+    async with async_session_factory() as session:
+        await update_earth_news_item_location(session, item_id=item.id, patch=patch)
+        await session.commit()
+    await broadcaster.broadcast_custom(
+        "earth_news",
+        {
+            "item_id": item.id,
+            "patch": patch,
+        },
+    )
+    return patch
+
+
+async def _run_target_location_worker() -> None:
+    consumer_name = f"{gethostname()}:{id(asyncio.current_task())}"
+    queue = get_news_target_location_queue()
+    while True:
+        try:
+            messages = await queue.consume_batch(
+                consumer_name=consumer_name,
+                count=WORKER_BATCH_SIZE,
+                block_ms=WORKER_BLOCK_MS,
+            )
+        except asyncio.CancelledError:
+            raise
+        except Exception as exc:
+            logger.warning_event(
+                "Earth news target location worker queue read failed",
+                event="earth_news.target_location.worker_read_failed",
+                context={"error": str(exc)},
+            )
+            await asyncio.sleep(WORKER_BACKOFF_SECONDS)
+            continue
+
+        if not messages:
+            continue
+        provider_client = await _build_provider_client()
+        for message in messages:
+            try:
+                await process_target_location_message(message, provider_client=provider_client)
+                await queue.ack(message.message_id)
+            except asyncio.CancelledError:
+                raise
+            except Exception as exc:
+                logger.warning_event(
+                    "Earth news target location worker job failed",
+                    event="earth_news.target_location.worker_job_failed",
+                    context={"item_id": message.item_id, "error": str(exc)},
+                )
+                with suppress(Exception):
+                    await queue.retry_or_dead_letter(message, error=str(exc))
+
+
+def start_earth_news_target_worker() -> None:
+    global _worker_task
+    if _worker_task is None or _worker_task.done():
+        _worker_task = asyncio.create_task(_run_target_location_worker())
+
+
+async def stop_earth_news_target_worker() -> None:
+    global _worker_task
+    task = _worker_task
+    if task is None:
+        return
+    task.cancel()
+    with suppress(asyncio.CancelledError):
+        await task
+    _worker_task = None

backend/app/services/scheduler.py

@@ -175,19 +175,29 @@ async def run_collector_task(collector_name: str):
             )
 
         try:
-            collector._datasource_id = datasource.id
+            datasource_id = datasource.id
+            datasource_source = datasource.source
+            collector._datasource_id = datasource_id
             logger.info_event(
                 "Running collector",
                 event="collector.run.started",
-                context={"collector_name": collector_name, "datasource_id": datasource.id},
+                context={"collector_name": collector_name, "datasource_id": datasource_id},
             )
             task_result = await collector.run(db)
+            datasource = await db.get(DataSource, datasource_id)
+            if datasource is None:
+                logger.error_event(
+                    "Datasource disappeared after collector run",
+                    event="collector.run.datasource_missing_after_run",
+                    context={"collector_name": collector_name, "datasource_id": datasource_id},
+                )
+                return
             datasource.last_run_at = datetime.now(UTC)
             datasource.last_status = task_result.get("status")
             if datasource.last_status == "success":
-                effective_candidate = await get_builtin_effective_candidate(db, datasource.source)
+                effective_candidate = await get_builtin_effective_candidate(db, datasource_source)
                 checksum, _credential_context = await build_builtin_connectivity_checksum(
-                    datasource.source,
+                    datasource_source,
                     effective_candidate["endpoint"],
                     effective_candidate["auth_type"],
                     effective_candidate["headers"],
@@ -196,7 +206,7 @@ async def run_collector_task(collector_name: str):
                 )
                 await save_connectivity_success(
                     db,
-                    datasource.source,
+                    datasource_source,
                     checksum,
                     {"status_code": None},
                     connected_by="collection",
@@ -205,9 +215,11 @@ async def run_collector_task(collector_name: str):
             logger.info_event(
                 "Collector completed",
                 event="collector.run.completed",
-                context={"collector_name": collector_name, "datasource_id": datasource.id, "result": task_result},
+                context={"collector_name": collector_name, "datasource_id": datasource_id, "result": task_result},
             )
         except asyncio.CancelledError:
+            await db.rollback()
+            datasource = await db.get(DataSource, datasource_id)
             datasource.last_run_at = datetime.now(UTC)
             datasource.last_status = "cancelled"
             await db.commit()
@@ -218,6 +230,8 @@ async def run_collector_task(collector_name: str):
             )
             raise
         except Exception as exc:
+            await db.rollback()
+            datasource = await db.get(DataSource, datasource_id)
             datasource.last_run_at = datetime.now(UTC)
             datasource.last_status = "failed"
             await db.commit()