"""Earth boundary source and static tile collector.""" from __future__ import annotations import asyncio import hashlib import json import shutil import sys from pathlib import Path from types import SimpleNamespace from typing import Any import httpx from sqlalchemy import select from app.core.earth_boundary_defaults import EARTH_BOUNDARY_DEFAULT_SOURCES, default_earth_boundary_config from app.models.collected_data import CollectedData from app.models.datasource_config import DataSourceConfig from app.services.collectors.base import BaseCollector from app.services.custom_datasource_runtime import build_query_params, build_request_headers REPO_ROOT = Path(__file__).resolve().parents[4] SOURCE_OUTPUT_DIR = REPO_ROOT / "data/earth-boundary-sources" SOURCE_MANIFEST_PATH = SOURCE_OUTPUT_DIR / "manifest.json" BOUNDARY_OUTPUT_DIR = REPO_ROOT / "frontend/public/earth/data/boundaries/v1" BOUNDARY_MANIFEST_PATH = BOUNDARY_OUTPUT_DIR / "manifest.json" PMTILES_ARTIFACT_PATH = REPO_ROOT / "frontend/public/earth/data/boundaries/earth-boundaries-china-pov-v1.pmtiles" POV_POLICY_PATH = REPO_ROOT / "config/earth-boundary-pov-policy.china-v1.json" BUILD_CONFIG = { "builder": "scripts/build_earth_boundary_china_pov_geojson.py", "format": "geojson-high-precision", "production_target": "geojson-high-precision", } EARTH_BOUNDARY_SOURCE_COLLECTORS = { "earth_admin0_boundaries": "admin0-boundaries", "earth_coastline": "coastline", "earth_claim_lines": "claim-lines", } def _read_json(path: Path) -> dict[str, Any]: if not path.exists(): return {} with path.open("r", encoding="utf-8") as f: return json.load(f) def _directory_stats(path: Path) -> dict[str, int]: if not path.exists(): return {"file_count": 0, "size_bytes": 0} files = [item for item in path.rglob("*") if item.is_file()] return { "file_count": len(files), "size_bytes": sum(item.stat().st_size for item in files), } def _stable_json_hash(payload: Any) -> str: encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) return hashlib.sha256(encoded.encode("utf-8")).hexdigest() def _build_input_hash(source_manifest: dict[str, Any]) -> str: return _stable_json_hash( { "source_manifest_schema": source_manifest.get("schema"), "sources": [ { "id": source.get("id"), "sha256": source.get("sha256"), "kind": source.get("kind"), "pov": source.get("pov"), } for source in source_manifest.get("sources", []) ], "pov_policy": source_manifest.get("povPolicy"), "build_config": BUILD_CONFIG, } ) def _has_current_artifacts(boundary_manifest: dict[str, Any], build_input_hash: str) -> bool: if not boundary_manifest: return False if boundary_manifest.get("buildInputHash") != build_input_hash: return False if not BOUNDARY_OUTPUT_DIR.exists(): return False if boundary_manifest.get("tileProvider") == "pmtiles-mvt": return PMTILES_ARTIFACT_PATH.exists() return (BOUNDARY_OUTPUT_DIR / "base.geojson").exists() and ( BOUNDARY_OUTPUT_DIR / "hover-index.geojson" ).exists() def _sha256_bytes(payload: bytes) -> str: return hashlib.sha256(payload).hexdigest() def _json_feature_count(payload: Any) -> int: if isinstance(payload, dict) and isinstance(payload.get("features"), list): return len(payload["features"]) if isinstance(payload, list): return len(payload) return 1 if payload else 0 def _json_sample_properties(payload: Any) -> dict[str, Any]: feature = None if isinstance(payload, dict) and isinstance(payload.get("features"), list) and payload["features"]: feature = payload["features"][0] elif isinstance(payload, list) and payload: feature = payload[0] elif isinstance(payload, dict): feature = payload if not isinstance(feature, dict): return {} props = feature.get("properties") if isinstance(feature.get("properties"), dict) else feature return {str(key): value for key, value in list(props.items())[:20]} def _artifact_extension(endpoint: str, content_type: str, payload: bytes) -> str: suffix = Path(endpoint.split("?", 1)[0]).suffix.lower() if suffix in {".json", ".geojson", ".zip", ".pbf"}: return suffix if "geo+json" in content_type or b'"FeatureCollection"' in payload[:4096]: return ".geojson" if "json" in content_type: return ".json" return ".dat" async def _load_datasource_config(db, name: str) -> DataSourceConfig | None: if db is None: return None result = await db.execute( select(DataSourceConfig) .where(DataSourceConfig.name == name) .where(DataSourceConfig.is_active.is_(True)) .order_by(DataSourceConfig.id.desc()) .limit(1) ) return result.scalar_one_or_none() async def _latest_boundary_source_record(db, source_name: str) -> CollectedData | None: result = await db.execute( select(CollectedData) .where(CollectedData.source == source_name) .where(CollectedData.data_type == "earth_boundary_source") .where(CollectedData.is_current.is_(True)) .order_by(CollectedData.collected_at.desc().nullslast(), CollectedData.id.desc()) .limit(1) ) return result.scalar_one_or_none() class EarthBoundarySourceCollector(BaseCollector): priority = "P1" module = "L3" frequency_hours = 168 data_type = "earth_boundary_source" fail_on_empty = True source_kind = "unknown" def _default_config(self): source = EARTH_BOUNDARY_DEFAULT_SOURCES.get(self.name) if not source: return None return SimpleNamespace( name=self.name, description=f"内置默认源：{self.name}", endpoint=source["endpoint"], source_type="http", auth_type="none", auth_config={}, headers={}, config=default_earth_boundary_config(self.name), ) async def _download_payload(self, config: DataSourceConfig) -> tuple[bytes, str]: request_config = config.config or {} method = str(request_config.get("method") or request_config.get("request_method") or "GET").upper() if method not in {"GET", "POST"}: raise RuntimeError("Earth boundary source collectors support GET and POST only") endpoint = str(config.endpoint or "").strip() if not endpoint: raise RuntimeError( f"{self.name} requires an endpoint in Collector Settings before it can collect data" ) if endpoint.startswith("file://") or Path(endpoint).expanduser().exists(): path = Path(endpoint.removeprefix("file://")).expanduser() return path.read_bytes(), "application/octet-stream" headers = build_request_headers(config.auth_type, config.auth_config or {}, config.headers or {}) params = build_query_params(config.auth_type, config.auth_config or {}, request_config) timeout = float(request_config.get("timeout", 120)) json_body = request_config.get("json_body") or request_config.get("body") async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client: response = await client.request( method, endpoint, headers=headers, params=params or None, json=json_body if isinstance(json_body, (dict, list)) else None, ) response.raise_for_status() return response.content, response.headers.get("content-type", "") async def fetch(self) -> list[dict[str, Any]]: db = getattr(self, "_db_session", None) config = await _load_datasource_config(db, self.name) or self._default_config() if config is None: raise RuntimeError( f"{self.name} has no active Collector Settings config and no built-in default source." ) await self.set_phase("fetching_source", message=f"正在下载 {self.source_kind} 源数据") payload, content_type = await self._download_payload(config) sha256 = _sha256_bytes(payload) endpoint = str(config.endpoint or "") extension = _artifact_extension(endpoint, content_type, payload) source_dir = SOURCE_OUTPUT_DIR / self.name source_dir.mkdir(parents=True, exist_ok=True) artifact_path = source_dir / f"{sha256}{extension}" artifact_path.write_bytes(payload) parsed: Any = None if extension in {".json", ".geojson"}: parsed = json.loads(payload.decode("utf-8")) feature_count = _json_feature_count(parsed) if feature_count <= 0: raise RuntimeError(f"{self.name} downloaded data but found no JSON/GeoJSON features") config_body = config.config or {} target_schema = str(config_body.get("target_schema") or "earth_boundary_source") if target_schema != "earth_boundary_source": raise RuntimeError(f"{self.name} target_schema must be earth_boundary_source") await self.update_phase_progress( current=1, total=1, unit="artifact", message=f"已保存 {feature_count} 个 {self.source_kind} feature", progress=100, commit=True, force=True, ) relative_artifact_path = str(artifact_path.relative_to(REPO_ROOT)) return [ { "id": f"{self.source_kind}:{sha256}", "name": config.description or self.name, "description": f"Earth boundary source artifact collected from configured endpoint", "source_kind": self.source_kind, "source_id": sha256, "value": feature_count, "unit": "features", "metadata": { "target_schema": target_schema, "source_kind": self.source_kind, "endpoint": endpoint, "method": str(config_body.get("method") or config_body.get("request_method") or "GET").upper(), "artifact_path": relative_artifact_path, "sha256": sha256, "feature_count": feature_count, "size_bytes": len(payload), "content_type": content_type, "license": config_body.get("license"), "mapping_json": config_body.get("mapping_json"), "sample_properties": _json_sample_properties(parsed), }, } ] class EarthAdmin0BoundaryCollector(EarthBoundarySourceCollector): name = "earth_admin0_boundaries" source_kind = "admin0-boundaries" class EarthCoastlineCollector(EarthBoundarySourceCollector): name = "earth_coastline" source_kind = "coastline" class EarthClaimLinesCollector(EarthBoundarySourceCollector): name = "earth_claim_lines" source_kind = "claim-lines" class EarthBoundaryTileCollector(BaseCollector): name = "earth_boundary_tiles" priority = "P1" module = "L3" frequency_hours = 168 data_type = "earth_boundary_tiles" fail_on_empty = False async def _run_step(self, args: list[str], *, allow_failure: bool = False) -> dict[str, Any]: process = await asyncio.create_subprocess_exec( sys.executable, *args, cwd=REPO_ROOT, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) stdout_bytes, stderr_bytes = await process.communicate() stdout = stdout_bytes.decode("utf-8", errors="replace").strip() stderr = stderr_bytes.decode("utf-8", errors="replace").strip() if process.returncode != 0 and not allow_failure: raise RuntimeError(stderr or stdout or f"command failed: {' '.join(args)}") payload: dict[str, Any] = { "stdout": stdout, "stderr": stderr, "returncode": process.returncode, } last_line = stdout.splitlines()[-1:] or [] if last_line: try: payload["result"] = json.loads(last_line[0]) except json.JSONDecodeError: payload["result"] = last_line[0] return payload async def fetch(self) -> list[dict[str, Any]]: db = getattr(self, "_db_session", None) if db is None: raise RuntimeError("Earth PMTiles builder requires an active database session") await self.set_phase("checking_sources", message="正在检查三类 Earth 边界源") await self.update_phase_progress( current=0, total=3, unit="steps", message="读取 admin0 / coastline / claim-lines 最新采集结果", progress=5, commit=True, force=True, ) source_records: dict[str, CollectedData] = {} missing_sources: list[str] = [] for source_name in EARTH_BOUNDARY_SOURCE_COLLECTORS: record = await _latest_boundary_source_record(db, source_name) if record is None: missing_sources.append(source_name) else: source_records[source_name] = record if missing_sources: missing_text = ", ".join(missing_sources) raise RuntimeError(f"未就绪：缺少 {missing_text}。不会更新 Earth 国界。") sources = [] for source_name, record in source_records.items(): metadata = record.extra_data or {} artifact_path = metadata.get("artifact_path") if not artifact_path or not (REPO_ROOT / str(artifact_path)).exists(): missing_sources.append(f"{source_name}: artifact missing") continue sources.append( { "id": source_name, "kind": metadata.get("source_kind") or EARTH_BOUNDARY_SOURCE_COLLECTORS[source_name], "path": str(artifact_path), "sha256": metadata.get("sha256"), "featureCount": metadata.get("feature_count"), "license": metadata.get("license"), } ) if missing_sources: missing_text = ", ".join(missing_sources) raise RuntimeError(f"未就绪：{missing_text}。不会更新 Earth 国界。") source_manifest = { "schema": "planet-earth-boundary-sources/v2", "sources": sources, "povPolicy": _read_json(POV_POLICY_PATH), } SOURCE_OUTPUT_DIR.mkdir(parents=True, exist_ok=True) with SOURCE_MANIFEST_PATH.open("w", encoding="utf-8") as f: json.dump(source_manifest, f, ensure_ascii=False, indent=2) f.write("\n") build_input_hash = _build_input_hash(source_manifest) await self.update_phase_progress( current=1, total=3, unit="steps", message="三类边界源已就绪", progress=35, commit=True, force=True, ) await self.set_phase("checking_readiness", message="正在检查 Earth 国界构建就绪状态") missing_tools = [tool for tool in ("tippecanoe", "pmtiles") if shutil.which(tool) is None] readiness_result = { "returncode": 0, "result": { "ready": True, "failures": [f"external tool not found in PATH: {tool}" for tool in missing_tools], "sources": sources, "artifact": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)), "fallback_builder": "geojson-high-precision" if missing_tools else None, }, } await self.set_phase("building_tiles", message="正在检查 Earth 国界瓦片产物") boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH) build_skipped = _has_current_artifacts(boundary_manifest, build_input_hash) if build_skipped: build_result = { "result": { "status": "unchanged", "reason": "source manifest and build config hash unchanged", "buildInputHash": build_input_hash, } } elif shutil.which("tippecanoe") and shutil.which("pmtiles"): admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries") coastline = next(source for source in sources if source["kind"] == "coastline") claim_lines = next(source for source in sources if source["kind"] == "claim-lines") build_result = await self._run_step( [ "scripts/build_earth_boundary_pmtiles.py", "--admin0-source", admin0["path"], "--coastline-source", coastline["path"], "--claims-source", claim_lines["path"], "--output", str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)), "--manifest", str(BOUNDARY_MANIFEST_PATH.relative_to(REPO_ROOT)), "--build-input-hash", build_input_hash, "--pov-policy", str(POV_POLICY_PATH.relative_to(REPO_ROOT)), ] ) boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH) else: admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries") coastline = next(source for source in sources if source["kind"] == "coastline") claim_lines = next(source for source in sources if source["kind"] == "claim-lines") build_result = await self._run_step( [ "scripts/build_earth_boundary_china_pov_geojson.py", "--admin0-source", admin0["path"], "--coastline-source", coastline["path"], "--claims-source", claim_lines["path"], "--output-dir", str(BOUNDARY_OUTPUT_DIR.relative_to(REPO_ROOT)), "--build-input-hash", build_input_hash, ] ) boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH) boundary_stats = _directory_stats(BOUNDARY_OUTPUT_DIR) pmtiles_exists = PMTILES_ARTIFACT_PATH.exists() pmtiles_size = PMTILES_ARTIFACT_PATH.stat().st_size if pmtiles_exists else 0 await self.update_phase_progress( current=2, total=3, unit="steps", message=( "边界源未变化，已跳过瓦片重建" if build_skipped else f"已生成 {boundary_manifest.get('tileProvider') or 'boundary'} 国界产物" ), progress=80, commit=True, force=True, ) await self.set_phase("indexing_artifacts", message="正在登记边界瓦片产物") tile_counts = boundary_manifest.get("tiles", {}).get("countsByZoom", {}) records = [ { "id": "source-manifest", "name": "Earth boundary source manifest", "description": "Offline source collection manifest for Earth boundary tiles", "value": len(source_manifest.get("sources", [])), "unit": "sources", "metadata": { "manifest_path": str(SOURCE_MANIFEST_PATH.relative_to(REPO_ROOT)), "manifest": source_manifest, "collector_result": { "status": "loaded_from_collected_data", "sources": [source["id"] for source in sources], }, }, }, { "id": "production-readiness", "name": "Earth boundary production readiness", "description": "Checks whether all source artifacts and boundary build tooling are available", "value": 1 if readiness_result.get("returncode") == 0 else 0, "unit": "ready", "metadata": { "result": readiness_result.get("result"), "returncode": readiness_result.get("returncode"), }, }, { "id": "boundary-manifest", "name": "Earth boundary tile manifest", "description": "Versioned static vector tile manifest for Earth country boundaries", "value": boundary_stats["file_count"], "unit": "files", "metadata": { "manifest_path": str(BOUNDARY_MANIFEST_PATH.relative_to(REPO_ROOT)), "output_dir": str(BOUNDARY_OUTPUT_DIR.relative_to(REPO_ROOT)), "size_bytes": boundary_stats["size_bytes"], "manifest": boundary_manifest, "collector_result": build_result.get("result"), "build_skipped": build_skipped, "production_target": BUILD_CONFIG["production_target"], "pmtiles_artifact": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)), "pmtiles_exists": pmtiles_exists, }, }, ] if pmtiles_exists: records.append( { "id": "pmtiles-artifact", "name": "Earth boundary PMTiles artifact", "description": "Single-file PMTiles/MVT artifact for Earth boundaries", "value": pmtiles_size, "unit": "bytes", "metadata": { "path": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)), "exists": pmtiles_exists, "provider": "pmtiles-mvt", }, } ) for zoom, count in sorted(tile_counts.items(), key=lambda item: int(item[0])): records.append( { "id": f"tile-z{zoom}", "name": f"Earth boundary tiles z{zoom}", "description": f"Generated Earth boundary tile count for zoom {zoom}", "value": int(count), "unit": "tiles", "metadata": { "zoom": int(zoom), "tile_count": int(count), "output_dir": str((BOUNDARY_OUTPUT_DIR / str(zoom)).relative_to(REPO_ROOT)), }, } ) await self.update_phase_progress( current=3, total=3, unit="steps", message="边界瓦片产物已登记", progress=100, commit=True, force=True, ) return records