Files
planet/backend/app/services/collectors/earth_boundaries.py
linkong 93eb41a9f7
Some checks failed
ci / backend (push) Has been cancelled
ci / frontend (push) Has been cancelled
ci / delivery (push) Has been cancelled
release / images (push) Has been cancelled
release: bump version to 0.58.0
Release 0.58.0 includes the Earth high-precision boundary PMTiles/MVT pipeline, standardized Earth boundary source collectors, China POV boundary configuration templates, and removal of the legacy low-precision GeoJSON fallback. It also adds Earth news target-location queueing/archive support, fixes datasource task status visibility, documents the Earth surface depth-spacing rules that prevent far-zoom z-fighting snow/black blocks, and updates bilingual operations/developer docs.
2026-05-15 17:40:07 +08:00

579 lines
23 KiB
Python

"""Earth boundary source and static tile collector."""
from __future__ import annotations
import asyncio
import hashlib
import json
import shutil
import sys
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import httpx
from sqlalchemy import select
from app.core.earth_boundary_defaults import EARTH_BOUNDARY_DEFAULT_SOURCES, default_earth_boundary_config
from app.models.collected_data import CollectedData
from app.models.datasource_config import DataSourceConfig
from app.services.collectors.base import BaseCollector
from app.services.custom_datasource_runtime import build_query_params, build_request_headers
REPO_ROOT = Path(__file__).resolve().parents[4]
SOURCE_OUTPUT_DIR = REPO_ROOT / "data/earth-boundary-sources"
SOURCE_MANIFEST_PATH = SOURCE_OUTPUT_DIR / "manifest.json"
BOUNDARY_OUTPUT_DIR = REPO_ROOT / "frontend/public/earth/data/boundaries/v1"
BOUNDARY_MANIFEST_PATH = BOUNDARY_OUTPUT_DIR / "manifest.json"
PMTILES_ARTIFACT_PATH = REPO_ROOT / "frontend/public/earth/data/boundaries/earth-boundaries-china-pov-v1.pmtiles"
POV_POLICY_PATH = REPO_ROOT / "config/earth-boundary-pov-policy.china-v1.json"
BUILD_CONFIG = {
"builder": "scripts/build_earth_boundary_china_pov_geojson.py",
"format": "geojson-high-precision",
"production_target": "geojson-high-precision",
}
EARTH_BOUNDARY_SOURCE_COLLECTORS = {
"earth_admin0_boundaries": "admin0-boundaries",
"earth_coastline": "coastline",
"earth_claim_lines": "claim-lines",
}
def _read_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
with path.open("r", encoding="utf-8") as f:
return json.load(f)
def _directory_stats(path: Path) -> dict[str, int]:
if not path.exists():
return {"file_count": 0, "size_bytes": 0}
files = [item for item in path.rglob("*") if item.is_file()]
return {
"file_count": len(files),
"size_bytes": sum(item.stat().st_size for item in files),
}
def _stable_json_hash(payload: Any) -> str:
encoded = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
def _build_input_hash(source_manifest: dict[str, Any]) -> str:
return _stable_json_hash(
{
"source_manifest_schema": source_manifest.get("schema"),
"sources": [
{
"id": source.get("id"),
"sha256": source.get("sha256"),
"kind": source.get("kind"),
"pov": source.get("pov"),
}
for source in source_manifest.get("sources", [])
],
"pov_policy": source_manifest.get("povPolicy"),
"build_config": BUILD_CONFIG,
}
)
def _has_current_artifacts(boundary_manifest: dict[str, Any], build_input_hash: str) -> bool:
if not boundary_manifest:
return False
if boundary_manifest.get("buildInputHash") != build_input_hash:
return False
if not BOUNDARY_OUTPUT_DIR.exists():
return False
if boundary_manifest.get("tileProvider") == "pmtiles-mvt":
return PMTILES_ARTIFACT_PATH.exists()
return (BOUNDARY_OUTPUT_DIR / "base.geojson").exists() and (
BOUNDARY_OUTPUT_DIR / "hover-index.geojson"
).exists()
def _sha256_bytes(payload: bytes) -> str:
return hashlib.sha256(payload).hexdigest()
def _json_feature_count(payload: Any) -> int:
if isinstance(payload, dict) and isinstance(payload.get("features"), list):
return len(payload["features"])
if isinstance(payload, list):
return len(payload)
return 1 if payload else 0
def _json_sample_properties(payload: Any) -> dict[str, Any]:
feature = None
if isinstance(payload, dict) and isinstance(payload.get("features"), list) and payload["features"]:
feature = payload["features"][0]
elif isinstance(payload, list) and payload:
feature = payload[0]
elif isinstance(payload, dict):
feature = payload
if not isinstance(feature, dict):
return {}
props = feature.get("properties") if isinstance(feature.get("properties"), dict) else feature
return {str(key): value for key, value in list(props.items())[:20]}
def _artifact_extension(endpoint: str, content_type: str, payload: bytes) -> str:
suffix = Path(endpoint.split("?", 1)[0]).suffix.lower()
if suffix in {".json", ".geojson", ".zip", ".pbf"}:
return suffix
if "geo+json" in content_type or b'"FeatureCollection"' in payload[:4096]:
return ".geojson"
if "json" in content_type:
return ".json"
return ".dat"
async def _load_datasource_config(db, name: str) -> DataSourceConfig | None:
if db is None:
return None
result = await db.execute(
select(DataSourceConfig)
.where(DataSourceConfig.name == name)
.where(DataSourceConfig.is_active.is_(True))
.order_by(DataSourceConfig.id.desc())
.limit(1)
)
return result.scalar_one_or_none()
async def _latest_boundary_source_record(db, source_name: str) -> CollectedData | None:
result = await db.execute(
select(CollectedData)
.where(CollectedData.source == source_name)
.where(CollectedData.data_type == "earth_boundary_source")
.where(CollectedData.is_current.is_(True))
.order_by(CollectedData.collected_at.desc().nullslast(), CollectedData.id.desc())
.limit(1)
)
return result.scalar_one_or_none()
class EarthBoundarySourceCollector(BaseCollector):
priority = "P1"
module = "L3"
frequency_hours = 168
data_type = "earth_boundary_source"
fail_on_empty = True
source_kind = "unknown"
def _default_config(self):
source = EARTH_BOUNDARY_DEFAULT_SOURCES.get(self.name)
if not source:
return None
return SimpleNamespace(
name=self.name,
description=f"内置默认源:{self.name}",
endpoint=source["endpoint"],
source_type="http",
auth_type="none",
auth_config={},
headers={},
config=default_earth_boundary_config(self.name),
)
async def _download_payload(self, config: DataSourceConfig) -> tuple[bytes, str]:
request_config = config.config or {}
method = str(request_config.get("method") or request_config.get("request_method") or "GET").upper()
if method not in {"GET", "POST"}:
raise RuntimeError("Earth boundary source collectors support GET and POST only")
endpoint = str(config.endpoint or "").strip()
if not endpoint:
raise RuntimeError(
f"{self.name} requires an endpoint in Collector Settings before it can collect data"
)
if endpoint.startswith("file://") or Path(endpoint).expanduser().exists():
path = Path(endpoint.removeprefix("file://")).expanduser()
return path.read_bytes(), "application/octet-stream"
headers = build_request_headers(config.auth_type, config.auth_config or {}, config.headers or {})
params = build_query_params(config.auth_type, config.auth_config or {}, request_config)
timeout = float(request_config.get("timeout", 120))
json_body = request_config.get("json_body") or request_config.get("body")
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
response = await client.request(
method,
endpoint,
headers=headers,
params=params or None,
json=json_body if isinstance(json_body, (dict, list)) else None,
)
response.raise_for_status()
return response.content, response.headers.get("content-type", "")
async def fetch(self) -> list[dict[str, Any]]:
db = getattr(self, "_db_session", None)
config = await _load_datasource_config(db, self.name) or self._default_config()
if config is None:
raise RuntimeError(
f"{self.name} has no active Collector Settings config and no built-in default source."
)
await self.set_phase("fetching_source", message=f"正在下载 {self.source_kind} 源数据")
payload, content_type = await self._download_payload(config)
sha256 = _sha256_bytes(payload)
endpoint = str(config.endpoint or "")
extension = _artifact_extension(endpoint, content_type, payload)
source_dir = SOURCE_OUTPUT_DIR / self.name
source_dir.mkdir(parents=True, exist_ok=True)
artifact_path = source_dir / f"{sha256}{extension}"
artifact_path.write_bytes(payload)
parsed: Any = None
if extension in {".json", ".geojson"}:
parsed = json.loads(payload.decode("utf-8"))
feature_count = _json_feature_count(parsed)
if feature_count <= 0:
raise RuntimeError(f"{self.name} downloaded data but found no JSON/GeoJSON features")
config_body = config.config or {}
target_schema = str(config_body.get("target_schema") or "earth_boundary_source")
if target_schema != "earth_boundary_source":
raise RuntimeError(f"{self.name} target_schema must be earth_boundary_source")
await self.update_phase_progress(
current=1,
total=1,
unit="artifact",
message=f"已保存 {feature_count}{self.source_kind} feature",
progress=100,
commit=True,
force=True,
)
relative_artifact_path = str(artifact_path.relative_to(REPO_ROOT))
return [
{
"id": f"{self.source_kind}:{sha256}",
"name": config.description or self.name,
"description": f"Earth boundary source artifact collected from configured endpoint",
"source_kind": self.source_kind,
"source_id": sha256,
"value": feature_count,
"unit": "features",
"metadata": {
"target_schema": target_schema,
"source_kind": self.source_kind,
"endpoint": endpoint,
"method": str(config_body.get("method") or config_body.get("request_method") or "GET").upper(),
"artifact_path": relative_artifact_path,
"sha256": sha256,
"feature_count": feature_count,
"size_bytes": len(payload),
"content_type": content_type,
"license": config_body.get("license"),
"mapping_json": config_body.get("mapping_json"),
"sample_properties": _json_sample_properties(parsed),
},
}
]
class EarthAdmin0BoundaryCollector(EarthBoundarySourceCollector):
name = "earth_admin0_boundaries"
source_kind = "admin0-boundaries"
class EarthCoastlineCollector(EarthBoundarySourceCollector):
name = "earth_coastline"
source_kind = "coastline"
class EarthClaimLinesCollector(EarthBoundarySourceCollector):
name = "earth_claim_lines"
source_kind = "claim-lines"
class EarthBoundaryTileCollector(BaseCollector):
name = "earth_boundary_tiles"
priority = "P1"
module = "L3"
frequency_hours = 168
data_type = "earth_boundary_tiles"
fail_on_empty = False
async def _run_step(self, args: list[str], *, allow_failure: bool = False) -> dict[str, Any]:
process = await asyncio.create_subprocess_exec(
sys.executable,
*args,
cwd=REPO_ROOT,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout_bytes, stderr_bytes = await process.communicate()
stdout = stdout_bytes.decode("utf-8", errors="replace").strip()
stderr = stderr_bytes.decode("utf-8", errors="replace").strip()
if process.returncode != 0 and not allow_failure:
raise RuntimeError(stderr or stdout or f"command failed: {' '.join(args)}")
payload: dict[str, Any] = {
"stdout": stdout,
"stderr": stderr,
"returncode": process.returncode,
}
last_line = stdout.splitlines()[-1:] or []
if last_line:
try:
payload["result"] = json.loads(last_line[0])
except json.JSONDecodeError:
payload["result"] = last_line[0]
return payload
async def fetch(self) -> list[dict[str, Any]]:
db = getattr(self, "_db_session", None)
if db is None:
raise RuntimeError("Earth PMTiles builder requires an active database session")
await self.set_phase("checking_sources", message="正在检查三类 Earth 边界源")
await self.update_phase_progress(
current=0,
total=3,
unit="steps",
message="读取 admin0 / coastline / claim-lines 最新采集结果",
progress=5,
commit=True,
force=True,
)
source_records: dict[str, CollectedData] = {}
missing_sources: list[str] = []
for source_name in EARTH_BOUNDARY_SOURCE_COLLECTORS:
record = await _latest_boundary_source_record(db, source_name)
if record is None:
missing_sources.append(source_name)
else:
source_records[source_name] = record
if missing_sources:
missing_text = ", ".join(missing_sources)
raise RuntimeError(f"未就绪:缺少 {missing_text}。不会更新 Earth 国界。")
sources = []
for source_name, record in source_records.items():
metadata = record.extra_data or {}
artifact_path = metadata.get("artifact_path")
if not artifact_path or not (REPO_ROOT / str(artifact_path)).exists():
missing_sources.append(f"{source_name}: artifact missing")
continue
sources.append(
{
"id": source_name,
"kind": metadata.get("source_kind") or EARTH_BOUNDARY_SOURCE_COLLECTORS[source_name],
"path": str(artifact_path),
"sha256": metadata.get("sha256"),
"featureCount": metadata.get("feature_count"),
"license": metadata.get("license"),
}
)
if missing_sources:
missing_text = ", ".join(missing_sources)
raise RuntimeError(f"未就绪:{missing_text}。不会更新 Earth 国界。")
source_manifest = {
"schema": "planet-earth-boundary-sources/v2",
"sources": sources,
"povPolicy": _read_json(POV_POLICY_PATH),
}
SOURCE_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
with SOURCE_MANIFEST_PATH.open("w", encoding="utf-8") as f:
json.dump(source_manifest, f, ensure_ascii=False, indent=2)
f.write("\n")
build_input_hash = _build_input_hash(source_manifest)
await self.update_phase_progress(
current=1,
total=3,
unit="steps",
message="三类边界源已就绪",
progress=35,
commit=True,
force=True,
)
await self.set_phase("checking_readiness", message="正在检查 Earth 国界构建就绪状态")
missing_tools = [tool for tool in ("tippecanoe", "pmtiles") if shutil.which(tool) is None]
readiness_result = {
"returncode": 0,
"result": {
"ready": True,
"failures": [f"external tool not found in PATH: {tool}" for tool in missing_tools],
"sources": sources,
"artifact": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
"fallback_builder": "geojson-high-precision" if missing_tools else None,
},
}
await self.set_phase("building_tiles", message="正在检查 Earth 国界瓦片产物")
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
build_skipped = _has_current_artifacts(boundary_manifest, build_input_hash)
if build_skipped:
build_result = {
"result": {
"status": "unchanged",
"reason": "source manifest and build config hash unchanged",
"buildInputHash": build_input_hash,
}
}
elif shutil.which("tippecanoe") and shutil.which("pmtiles"):
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
coastline = next(source for source in sources if source["kind"] == "coastline")
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
build_result = await self._run_step(
[
"scripts/build_earth_boundary_pmtiles.py",
"--admin0-source",
admin0["path"],
"--coastline-source",
coastline["path"],
"--claims-source",
claim_lines["path"],
"--output",
str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
"--manifest",
str(BOUNDARY_MANIFEST_PATH.relative_to(REPO_ROOT)),
"--build-input-hash",
build_input_hash,
"--pov-policy",
str(POV_POLICY_PATH.relative_to(REPO_ROOT)),
]
)
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
else:
admin0 = next(source for source in sources if source["kind"] == "admin0-boundaries")
coastline = next(source for source in sources if source["kind"] == "coastline")
claim_lines = next(source for source in sources if source["kind"] == "claim-lines")
build_result = await self._run_step(
[
"scripts/build_earth_boundary_china_pov_geojson.py",
"--admin0-source",
admin0["path"],
"--coastline-source",
coastline["path"],
"--claims-source",
claim_lines["path"],
"--output-dir",
str(BOUNDARY_OUTPUT_DIR.relative_to(REPO_ROOT)),
"--build-input-hash",
build_input_hash,
]
)
boundary_manifest = _read_json(BOUNDARY_MANIFEST_PATH)
boundary_stats = _directory_stats(BOUNDARY_OUTPUT_DIR)
pmtiles_exists = PMTILES_ARTIFACT_PATH.exists()
pmtiles_size = PMTILES_ARTIFACT_PATH.stat().st_size if pmtiles_exists else 0
await self.update_phase_progress(
current=2,
total=3,
unit="steps",
message=(
"边界源未变化,已跳过瓦片重建"
if build_skipped
else f"已生成 {boundary_manifest.get('tileProvider') or 'boundary'} 国界产物"
),
progress=80,
commit=True,
force=True,
)
await self.set_phase("indexing_artifacts", message="正在登记边界瓦片产物")
tile_counts = boundary_manifest.get("tiles", {}).get("countsByZoom", {})
records = [
{
"id": "source-manifest",
"name": "Earth boundary source manifest",
"description": "Offline source collection manifest for Earth boundary tiles",
"value": len(source_manifest.get("sources", [])),
"unit": "sources",
"metadata": {
"manifest_path": str(SOURCE_MANIFEST_PATH.relative_to(REPO_ROOT)),
"manifest": source_manifest,
"collector_result": {
"status": "loaded_from_collected_data",
"sources": [source["id"] for source in sources],
},
},
},
{
"id": "production-readiness",
"name": "Earth boundary production readiness",
"description": "Checks whether all source artifacts and boundary build tooling are available",
"value": 1 if readiness_result.get("returncode") == 0 else 0,
"unit": "ready",
"metadata": {
"result": readiness_result.get("result"),
"returncode": readiness_result.get("returncode"),
},
},
{
"id": "boundary-manifest",
"name": "Earth boundary tile manifest",
"description": "Versioned static vector tile manifest for Earth country boundaries",
"value": boundary_stats["file_count"],
"unit": "files",
"metadata": {
"manifest_path": str(BOUNDARY_MANIFEST_PATH.relative_to(REPO_ROOT)),
"output_dir": str(BOUNDARY_OUTPUT_DIR.relative_to(REPO_ROOT)),
"size_bytes": boundary_stats["size_bytes"],
"manifest": boundary_manifest,
"collector_result": build_result.get("result"),
"build_skipped": build_skipped,
"production_target": BUILD_CONFIG["production_target"],
"pmtiles_artifact": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
"pmtiles_exists": pmtiles_exists,
},
},
]
if pmtiles_exists:
records.append(
{
"id": "pmtiles-artifact",
"name": "Earth boundary PMTiles artifact",
"description": "Single-file PMTiles/MVT artifact for Earth boundaries",
"value": pmtiles_size,
"unit": "bytes",
"metadata": {
"path": str(PMTILES_ARTIFACT_PATH.relative_to(REPO_ROOT)),
"exists": pmtiles_exists,
"provider": "pmtiles-mvt",
},
}
)
for zoom, count in sorted(tile_counts.items(), key=lambda item: int(item[0])):
records.append(
{
"id": f"tile-z{zoom}",
"name": f"Earth boundary tiles z{zoom}",
"description": f"Generated Earth boundary tile count for zoom {zoom}",
"value": int(count),
"unit": "tiles",
"metadata": {
"zoom": int(zoom),
"tile_count": int(count),
"output_dir": str((BOUNDARY_OUTPUT_DIR / str(zoom)).relative_to(REPO_ROOT)),
},
}
)
await self.update_phase_progress(
current=3,
total=3,
unit="steps",
message="边界瓦片产物已登记",
progress=100,
commit=True,
force=True,
)
return records