"""Earth boundary static asset service.""" from __future__ import annotations import asyncio import hashlib import json import shutil import sys from datetime import UTC, datetime from pathlib import Path from typing import Any from uuid import uuid4 import httpx REPO_ROOT = Path(__file__).resolve().parents[3] SOURCE_OUTPUT_DIR = REPO_ROOT / "data/earth-boundary-sources" SOURCE_MANIFEST_PATH = SOURCE_OUTPUT_DIR / "manifest.json" BUILD_RESULT_PATH = SOURCE_OUTPUT_DIR / "build-result.json" BUILD_JOB_PATH = SOURCE_OUTPUT_DIR / "build-job.json" BOUNDARY_OUTPUT_DIR = REPO_ROOT / "frontend/public/earth/data/boundaries/v1" BOUNDARY_MANIFEST_PATH = BOUNDARY_OUTPUT_DIR / "manifest.json" PMTILES_ARTIFACT_PATH = ( REPO_ROOT / "frontend/public/earth/data/boundaries/earth-boundaries-china-pov-v1.pmtiles" ) LEGACY_GEOJSON_PATH = REPO_ROOT / "frontend/public/earth/data/countries-admin0.min.geojson" POV_POLICY_PATH = REPO_ROOT / "config/earth-boundary-pov-policy.china-v1.json" LOCAL_CONFIG_PATH = REPO_ROOT / "config/earth-boundary-sources.local.json" EXAMPLE_CONFIG_PATH = REPO_ROOT / "config/earth-boundary-sources.example.json" BOUNDARY_SOURCE_KINDS = { "earth_admin0_boundaries": "admin0-boundaries", "earth_coastline": "coastline", "earth_claim_lines": "claim-lines", } DEFAULT_PUBLIC_BOUNDARY_SOURCES = { "earth_admin0_boundaries": { "displayName": "Natural Earth Admin-0 Countries", "sourceKind": "admin0-boundaries", "endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_0_countries.geojson", "method": "GET", "headers": {}, "auth_type": "none", "license": "Natural Earth public domain", }, "earth_coastline": { "displayName": "Natural Earth Coastline", "sourceKind": "coastline", "endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_coastline.geojson", "method": "GET", "headers": {}, "auth_type": "none", "license": "Natural Earth public domain", }, "earth_claim_lines": { "displayName": "Natural Earth Disputed Boundaries", "sourceKind": "claim-lines", "endpoint": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_10m_admin_0_boundary_lines_disputed_areas.geojson", "method": "GET", "headers": {}, "auth_type": "none", "license": "Natural Earth public domain", }, } BUILD_CONFIG = { "builder": "scripts/build_earth_boundary_pmtiles.py", "format": "pmtiles+mvt", "production_target": "pmtiles-mvt", } class EarthBoundaryBuildError(RuntimeError): def __init__(self, message: str, *, code: str = "build_failed", details: Any = None) -> None: super().__init__(message) self.code = code self.details = details _build_job_lock = asyncio.Lock() _build_task: asyncio.Task | None = None _build_job_state: dict[str, Any] = {} def _utc_now_iso() -> str: return datetime.now(UTC).isoformat() def _public_job_state() -> dict[str, Any]: if _build_job_state: return dict(_build_job_state) return _read_json(BUILD_JOB_PATH) def get_boundary_build_status() -> dict[str, Any]: return {"job": _public_job_state()} def _set_job_state(**updates: Any) -> dict[str, Any]: global _build_job_state current = dict(_build_job_state) current.update(updates) current["updated_at"] = _utc_now_iso() _build_job_state = current _write_json(BUILD_JOB_PATH, current) return current def _append_job_log(message: str) -> None: logs = list(_build_job_state.get("logs") or []) logs.append({"time": _utc_now_iso(), "message": message}) _set_job_state(logs=logs[-40:]) def _update_job_progress(progress: float, phase: str, message: str, **extra: Any) -> None: bounded_progress = max(0, min(100, int(round(progress)))) _set_job_state( status="running", progress=bounded_progress, phase=phase, message=message, **extra, ) def _read_json(path: Path) -> dict[str, Any]: if not path.exists(): return {} with path.open("r", encoding="utf-8") as f: payload = json.load(f) return payload if isinstance(payload, dict) else {} def _write_json(path: Path, payload: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8") as f: json.dump(payload, f, ensure_ascii=False, indent=2) f.write("\n") def _sha256_bytes(payload: bytes) -> str: return hashlib.sha256(payload).hexdigest() def _stable_json_hash(payload: Any) -> str: encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) return hashlib.sha256(encoded.encode("utf-8")).hexdigest() def _artifact_extension(endpoint: str, content_type: str, payload: bytes) -> str: suffix = Path(endpoint.split("?", 1)[0]).suffix.lower() if suffix in {".json", ".geojson", ".zip", ".pbf"}: return suffix if "geo+json" in content_type or b'"FeatureCollection"' in payload[:4096]: return ".geojson" if "json" in content_type: return ".json" return ".dat" def _json_feature_count(payload: Any) -> int: if isinstance(payload, dict) and isinstance(payload.get("features"), list): return len(payload["features"]) if isinstance(payload, list): return len(payload) return 1 if payload else 0 def _directory_stats(path: Path) -> dict[str, int]: if not path.exists(): return {"file_count": 0, "size_bytes": 0} files = [item for item in path.rglob("*") if item.is_file()] return {"file_count": len(files), "size_bytes": sum(item.stat().st_size for item in files)} def _load_source_feature_collection(source: dict[str, Any]) -> dict[str, Any]: path = REPO_ROOT / source["path"] payload = _read_json(path) features = payload.get("features") if isinstance(payload, dict) else None return { "type": "FeatureCollection", "features": features if isinstance(features, list) else [], } def _write_high_precision_geojson_manifest( sources: list[dict[str, Any]], build_input_hash: str, missing_tools: list[str], ) -> dict[str, Any]: BOUNDARY_OUTPUT_DIR.mkdir(parents=True, exist_ok=True) admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries") coastline = next(source for source in sources if source["kind"] == "coastline") claim_lines = next(source for source in sources if source["kind"] == "claim-lines") admin0_payload = _load_source_feature_collection(admin0) coastline_payload = _load_source_feature_collection(coastline) claim_payload = _load_source_feature_collection(claim_lines) for feature in coastline_payload["features"]: props = feature.setdefault("properties", {}) if isinstance(props, dict): props["PLANET_LAYER"] = "coastline" base_payload = { "type": "FeatureCollection", "features": [*admin0_payload["features"], *coastline_payload["features"]], } base_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-base.geojson" hover_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-hover.geojson" claim_path = BOUNDARY_OUTPUT_DIR / "earth-boundaries-high-precision-claims.geojson" _write_json(base_path, base_payload) _write_json(hover_path, admin0_payload) _write_json(claim_path, claim_payload) manifest = { "version": "natural-earth-v1", "builtAt": _utc_now_iso(), "tileProvider": "geojson-high-precision", "format": "geojson-directory", "buildInputHash": build_input_hash, "base": base_path.name, "hoverIndex": hover_path.name, "claimLine": claim_path.name, "sourceFeatureCount": { "admin0": len(admin0_payload["features"]), "coastline": len(coastline_payload["features"]), "claimLines": len(claim_payload["features"]), }, "pmtiles": None, "missingTools": missing_tools, } _write_json(BOUNDARY_MANIFEST_PATH, manifest) return manifest def _relative(path: Path) -> str: return str(path.relative_to(REPO_ROOT)) def load_boundary_config() -> tuple[dict[str, Any], str]: if LOCAL_CONFIG_PATH.exists(): return _read_json(LOCAL_CONFIG_PATH), "local" return _read_json(EXAMPLE_CONFIG_PATH), "example" def save_boundary_config(payload: dict[str, Any]) -> dict[str, Any]: if not isinstance(payload, dict): raise EarthBoundaryBuildError("Earth boundary config must be a JSON object", code="invalid_config") _write_json(LOCAL_CONFIG_PATH, payload) return get_boundary_status() def _source_configs(payload: dict[str, Any]) -> dict[str, Any]: raw_sources = payload.get("collectorConfigs") or payload.get("sources") or {} return raw_sources if isinstance(raw_sources, dict) else {} def _is_placeholder_endpoint(endpoint: Any) -> bool: value = str(endpoint or "").strip() return not value or "example.com" in value def _source_configs_with_defaults(payload: dict[str, Any]) -> dict[str, Any]: raw_sources = _source_configs(payload) merged: dict[str, Any] = {} for source_key, default_config in DEFAULT_PUBLIC_BOUNDARY_SOURCES.items(): configured = raw_sources.get(source_key) if not isinstance(configured, dict) or _is_placeholder_endpoint(configured.get("endpoint")): merged[source_key] = dict(default_config) else: merged[source_key] = {**default_config, **configured} for source_key, source_config in raw_sources.items(): if source_key not in merged: merged[source_key] = source_config return merged def _build_input_hash(source_manifest: dict[str, Any]) -> str: return _stable_json_hash( { "source_manifest_schema": source_manifest.get("schema"), "sources": [ { "id": source.get("id"), "sha256": source.get("sha256"), "kind": source.get("kind"), } for source in source_manifest.get("sources", []) ], "pov_policy": source_manifest.get("povPolicy"), "build_config": BUILD_CONFIG, } ) def _has_current_artifacts(boundary_manifest: dict[str, Any], build_input_hash: str) -> bool: return ( bool(boundary_manifest) and boundary_manifest.get("buildInputHash") == build_input_hash and boundary_manifest.get("tileProvider") == "pmtiles-mvt" and PMTILES_ARTIFACT_PATH.exists() ) def get_boundary_status() -> dict[str, Any]: config_payload, config_source = load_boundary_config() effective_source_configs = _source_configs_with_defaults(config_payload) boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH) pmtiles_exists = PMTILES_ARTIFACT_PATH.exists() manifest_exists = BOUNDARY_MANIFEST_PATH.exists() high_precision_ready = ( manifest_exists and ( ( boundary_manifest.get("tileProvider") == "pmtiles-mvt" and pmtiles_exists ) or boundary_manifest.get("tileProvider") == "geojson-high-precision" ) ) legacy_exists = LEGACY_GEOJSON_PATH.exists() provider = ( boundary_manifest.get("tileProvider") if high_precision_ready else "legacy-geojson" if legacy_exists else "missing" ) return { "provider": provider, "high_precision_ready": high_precision_ready, "fallback_available": legacy_exists, "config_source": config_source, "config_path": _relative(LOCAL_CONFIG_PATH), "config_exists": LOCAL_CONFIG_PATH.exists(), "config": config_payload, "effective_default_sources": [ source_key for source_key, source_config in effective_source_configs.items() if source_key in DEFAULT_PUBLIC_BOUNDARY_SOURCES and source_config.get("endpoint") == DEFAULT_PUBLIC_BOUNDARY_SOURCES[source_key]["endpoint"] ], "manifest": { "path": _relative(BOUNDARY_MANIFEST_PATH), "exists": manifest_exists, "tileProvider": boundary_manifest.get("tileProvider"), "buildInputHash": boundary_manifest.get("buildInputHash"), "builtAt": boundary_manifest.get("builtAt"), }, "pmtiles": { "path": _relative(PMTILES_ARTIFACT_PATH), "exists": pmtiles_exists, "size_bytes": PMTILES_ARTIFACT_PATH.stat().st_size if pmtiles_exists else 0, }, "legacy": { "path": _relative(LEGACY_GEOJSON_PATH), "exists": legacy_exists, "size_bytes": LEGACY_GEOJSON_PATH.stat().st_size if legacy_exists else 0, }, "source_manifest": { "path": _relative(SOURCE_MANIFEST_PATH), "exists": SOURCE_MANIFEST_PATH.exists(), }, "last_build": _read_json(BUILD_RESULT_PATH), "current_job": _public_job_state(), } async def _download_source( source_key: str, source_config: dict[str, Any], progress_callback: Any = None, ) -> dict[str, Any]: endpoint = str(source_config.get("endpoint") or "").strip() if _is_placeholder_endpoint(endpoint): raise EarthBoundaryBuildError( f"{source_key} endpoint is not configured", code="source_not_configured", details={"source": source_key}, ) method = str(source_config.get("method") or "GET").upper() if method not in {"GET", "POST"}: raise EarthBoundaryBuildError( f"{source_key} method must be GET or POST", code="invalid_config", details={"source": source_key, "method": method}, ) if endpoint.startswith("file://") or Path(endpoint).expanduser().exists(): payload = Path(endpoint.removeprefix("file://")).expanduser().read_bytes() content_type = "application/octet-stream" if progress_callback: progress_callback(1, len(payload), len(payload)) else: timeout = float(source_config.get("timeout") or 120) headers = source_config.get("headers") if isinstance(source_config.get("headers"), dict) else {} async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client: async with client.stream(method, endpoint, headers=headers) as response: response.raise_for_status() content_type = response.headers.get("content-type", "") total = int(response.headers.get("content-length") or 0) chunks = [] downloaded = 0 async for chunk in response.aiter_bytes(): if not chunk: continue chunks.append(chunk) downloaded += len(chunk) if progress_callback: progress_callback( (downloaded / total) if total else None, downloaded, total, ) payload = b"".join(chunks) extension = _artifact_extension(endpoint, content_type, payload) parsed: Any = None if extension in {".json", ".geojson"}: parsed = json.loads(payload.decode("utf-8")) feature_count = _json_feature_count(parsed) if feature_count <= 0: raise EarthBoundaryBuildError( f"{source_key} downloaded payload contains no features", code="empty_source", details={"source": source_key}, ) sha256 = _sha256_bytes(payload) source_dir = SOURCE_OUTPUT_DIR / source_key source_dir.mkdir(parents=True, exist_ok=True) artifact_path = source_dir / f"{sha256}{extension}" artifact_path.write_bytes(payload) return { "id": source_key, "kind": source_config.get("sourceKind") or BOUNDARY_SOURCE_KINDS[source_key], "path": _relative(artifact_path), "sha256": sha256, "featureCount": feature_count, "license": source_config.get("license"), } async def _run_step(args: list[str]) -> dict[str, Any]: process = await asyncio.create_subprocess_exec( sys.executable, *args, cwd=REPO_ROOT, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) stdout_bytes, stderr_bytes = await process.communicate() stdout = stdout_bytes.decode("utf-8", errors="replace").strip() stderr = stderr_bytes.decode("utf-8", errors="replace").strip() payload: dict[str, Any] = {"stdout": stdout, "stderr": stderr, "returncode": process.returncode} last_line = stdout.splitlines()[-1:] or [] if last_line: try: payload["result"] = json.loads(last_line[0]) except json.JSONDecodeError: payload["result"] = last_line[0] if process.returncode != 0: raise EarthBoundaryBuildError( stderr or stdout or f"command failed: {' '.join(args)}", code="build_command_failed", details=payload, ) return payload async def build_boundary_assets(progress_callback: Any = None) -> dict[str, Any]: config_payload, config_source = load_boundary_config() source_configs = _source_configs_with_defaults(config_payload) missing = [source for source in BOUNDARY_SOURCE_KINDS if source not in source_configs] if missing: raise EarthBoundaryBuildError( f"Missing Earth boundary source configs: {', '.join(missing)}", code="missing_sources", details={"missing": missing}, ) sources = [] source_keys = list(BOUNDARY_SOURCE_KINDS) for index, source_key in enumerate(source_keys): source_config = source_configs[source_key] if not isinstance(source_config, dict): raise EarthBoundaryBuildError( f"{source_key} config must be an object", code="invalid_config", details={"source": source_key}, ) source_start = 8 + index * 18 source_end = source_start + 18 if progress_callback: progress_callback(source_start, "download", f"正在下载 {source_key}") def report_download_progress(ratio: float | None, downloaded: int, total: int) -> None: if not progress_callback: return if ratio is None: progress_callback(source_start + 8, "download", f"{source_key} 已下载 {downloaded} bytes") return progress_callback( source_start + (source_end - source_start) * ratio, "download", f"{source_key} 下载 {int(ratio * 100)}%", downloaded_bytes=downloaded, total_bytes=total, ) sources.append(await _download_source(source_key, source_config, report_download_progress)) source_manifest = { "schema": "planet-earth-boundary-sources/v2", "sources": sources, "povPolicy": _read_json(POV_POLICY_PATH), } if progress_callback: progress_callback(65, "manifest", "正在写入边界源 manifest") _write_json(SOURCE_MANIFEST_PATH, source_manifest) build_input_hash = _build_input_hash(source_manifest) boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH) build_skipped = _has_current_artifacts(boundary_manifest, build_input_hash) missing_tools = [tool for tool in ("tippecanoe", "pmtiles") if shutil.which(tool) is None] if missing_tools and not build_skipped: if progress_callback: progress_callback(82, "build", "缺少 PMTiles 工具，正在生成 GeoJSON 高清包") boundary_manifest = _write_high_precision_geojson_manifest( sources, build_input_hash, missing_tools, ) result = { "status": "built_geojson_fallback", "code": "missing_tools", "missing_tools": missing_tools, "sources": sources, "boundary_manifest": _relative(BOUNDARY_MANIFEST_PATH), "manifest": boundary_manifest, } _write_json(BUILD_RESULT_PATH, result) if progress_callback: progress_callback(96, "finalize", "GeoJSON 高清国界包已生成") return {**get_boundary_status(), "build": result} if build_skipped: if progress_callback: progress_callback(96, "unchanged", "高精国界已是最新") build_result = { "status": "unchanged", "reason": "source manifest and build config hash unchanged", "buildInputHash": build_input_hash, } else: if progress_callback: progress_callback(72, "build", "正在构建 PMTiles/MVT") admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries") coastline = next(source for source in sources if source["kind"] == "coastline") claim_lines = next(source for source in sources if source["kind"] == "claim-lines") build_result = await _run_step( [ "scripts/build_earth_boundary_pmtiles.py", "--admin0-source", admin0["path"], "--coastline-source", coastline["path"], "--claims-source", claim_lines["path"], "--output", _relative(PMTILES_ARTIFACT_PATH), "--manifest", _relative(BOUNDARY_MANIFEST_PATH), "--build-input-hash", build_input_hash, "--pov-policy", _relative(POV_POLICY_PATH), ] ) if progress_callback: progress_callback(95, "finalize", "正在校验构建产物") boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH) boundary_stats = _directory_stats(BOUNDARY_OUTPUT_DIR) result = { "status": "unchanged" if build_skipped else "built", "sources": sources, "source_manifest": _relative(SOURCE_MANIFEST_PATH), "boundary_manifest": _relative(BOUNDARY_MANIFEST_PATH), "pmtiles_artifact": _relative(PMTILES_ARTIFACT_PATH), "pmtiles_exists": PMTILES_ARTIFACT_PATH.exists(), "boundary_stats": boundary_stats, "manifest": boundary_manifest, "build_result": build_result, } _write_json(BUILD_RESULT_PATH, result) return {**get_boundary_status(), "build": result} async def _run_boundary_build_job(job_id: str) -> None: def report(progress: float, phase: str, message: str, **extra: Any) -> None: if _build_job_state.get("id") != job_id: return _update_job_progress(progress, phase, message, **extra) try: report(3, "prepare", "正在准备高精国界构建") result = await build_boundary_assets(report) _set_job_state( id=job_id, status="succeeded", progress=100, phase="complete", message="高精国界构建完成", finished_at=_utc_now_iso(), result={ "provider": result.get("provider"), "high_precision_ready": result.get("high_precision_ready"), "pmtiles": result.get("pmtiles"), "manifest": result.get("manifest"), }, ) _append_job_log("高精国界构建完成") except EarthBoundaryBuildError as exc: _set_job_state( id=job_id, status="failed", progress=_build_job_state.get("progress", 0), phase="failed", message=str(exc), code=exc.code, details=exc.details, finished_at=_utc_now_iso(), ) _append_job_log(str(exc)) except Exception as exc: # pragma: no cover - defensive guard for background task _set_job_state( id=job_id, status="failed", progress=_build_job_state.get("progress", 0), phase="failed", message=str(exc), code="build_failed", finished_at=_utc_now_iso(), ) _append_job_log(str(exc)) async def start_boundary_build_job() -> dict[str, Any]: global _build_task async with _build_job_lock: if _build_task and not _build_task.done(): return {"accepted": False, "job": _public_job_state()} job_id = uuid4().hex _set_job_state( id=job_id, status="queued", progress=0, phase="queued", message="高精国界构建已加入队列", logs=[], started_at=_utc_now_iso(), finished_at=None, code=None, details=None, ) _append_job_log("高精国界构建已启动") _build_task = asyncio.create_task(_run_boundary_build_job(job_id)) return {"accepted": True, "job": _public_job_state()}