release: bump version to 0.58.0

Release 0.58.0 includes the Earth high-precision boundary PMTiles/MVT pipeline, standardized Earth boundary source collectors, China POV boundary configuration templates, and removal of the legacy low-precision GeoJSON fallback. It also adds Earth news target-location queueing/archive support, fixes datasource task status visibility, documents the Earth surface depth-spacing rules that prevent far-zoom z-fighting snow/black blocks, and updates bilingual operations/developer docs.

backend/app/api/v1/datasources.py

@@ -35,6 +35,7 @@ PRODUCT_SOURCE_KEYWORDS: tuple[tuple[str, tuple[str, ...]], ...] = (
     ("cables", ("cable", "landing", "telegeography", "arcgis", "fao")),
     ("satellites", ("tle", "satellite", "spacetrack", "celestrak")),
     ("bgp", ("bgp", "asn", "prefix_geo", "opengeofeed", "nro")),
+    ("earth", ("earth_", "earth_boundary", "boundary", "country boundary", "coastline", "claim")),
     ("compute", ("top500", "gpu", "supercomputer", "compute")),
     ("ai", ("huggingface", "epoch_ai")),
     ("media", ("news", "tv", "live_stream")),
@@ -150,6 +151,33 @@ async def _load_latest_task_ids(
     return {datasource_id: task_id for datasource_id, task_id in result.all()}
 
 
+async def _load_latest_tasks(
+    db: AsyncSession,
+    datasource_ids: list[int],
+) -> dict[int, CollectionTask]:
+    if not datasource_ids:
+        return {}
+
+    ranked_tasks = (
+        select(
+            CollectionTask.id.label("task_id"),
+            CollectionTask.datasource_id.label("datasource_id"),
+            func.row_number().over(
+                partition_by=CollectionTask.datasource_id,
+                order_by=(CollectionTask.created_at.desc().nullslast(), CollectionTask.id.desc()),
+            ).label("row_num"),
+        )
+        .where(CollectionTask.datasource_id.in_(datasource_ids))
+        .subquery()
+    )
+    result = await db.execute(
+        select(CollectionTask)
+        .join(ranked_tasks, CollectionTask.id == ranked_tasks.c.task_id)
+        .where(ranked_tasks.c.row_num == 1)
+    )
+    return {task.datasource_id: task for task in result.scalars().all()}
+
+
 async def _load_collected_record_counts(
     db: AsyncSession,
     sources: list[str],
@@ -208,7 +236,7 @@ async def _load_datasource_endpoint_overrides(
 async def _load_datasource_list_context(
     db: AsyncSession,
     datasources: list[DataSource],
-) -> tuple[dict[int, CollectionTask], dict[str, str]]:
+) -> tuple[dict[int, CollectionTask], dict[int, CollectionTask], dict[str, str]]:
     datasource_ids = [datasource.id for datasource in datasources]
     sources = [datasource.source for datasource in datasources]
 
@@ -232,8 +260,9 @@ async def _load_datasource_list_context(
     if stale_datasource_ids:
         running_tasks = await _load_latest_running_tasks(db, datasource_ids)
 
+    latest_tasks = await _load_latest_tasks(db, datasource_ids)
     endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources)
-    return running_tasks, endpoint_overrides
+    return running_tasks, latest_tasks, endpoint_overrides
 
 
 def _apply_datasource_query_filters(
@@ -251,11 +280,6 @@ def _apply_datasource_query_filters(
         query = query.where(DataSource.is_active == is_active)
     if priority:
         query = query.where(DataSource.priority == priority)
-    if run_status and run_status not in {"running", "collected", "uncollected"}:
-        if run_status == "not_run":
-            query = query.where(DataSource.last_status.is_(None))
-        else:
-            query = query.where(DataSource.last_status == run_status)
     if q:
         like_value = f"%{q.strip()}%"
         query = query.where(
@@ -272,6 +296,7 @@ def _filter_datasources_in_memory(
     datasources: list[DataSource],
     *,
     running_tasks: dict[int, CollectionTask],
+    latest_tasks: dict[int, CollectionTask],
     record_counts: dict[str, int],
     product: Optional[str] = None,
     run_status: Optional[str] = None,
@@ -281,6 +306,14 @@ def _filter_datasources_in_memory(
     filtered: list[DataSource] = []
     for datasource in datasources:
         record_count = record_counts.get(datasource.source, 0)
+        latest_task = latest_tasks.get(datasource.id)
+        effective_status = (
+            "running"
+            if datasource.id in running_tasks
+            else latest_task.status
+            if latest_task is not None
+            else datasource.last_status
+        )
         if product and datasource_product_key(datasource) != product:
             continue
         if collected is not None and (record_count > 0) != collected:
@@ -291,6 +324,10 @@ def _filter_datasources_in_memory(
                 continue
         if run_status == "running" and datasource.id not in running_tasks:
             continue
+        if run_status == "not_run" and effective_status is not None:
+            continue
+        if run_status not in {None, "running", "not_run", "collected", "uncollected"} and effective_status != run_status:
+            continue
         if run_status == "collected" and record_count <= 0:
             continue
         if run_status == "uncollected" and record_count > 0:
@@ -639,11 +676,12 @@ async def list_datasources(
 
     collector_list = []
     config = get_data_sources_config()
-    running_tasks, endpoint_overrides = await _load_datasource_list_context(db, datasources)
+    running_tasks, latest_tasks, endpoint_overrides = await _load_datasource_list_context(db, datasources)
     record_counts = await _load_collected_record_counts(db, [datasource.source for datasource in datasources])
     datasources = _filter_datasources_in_memory(
         datasources,
         running_tasks=running_tasks,
+        latest_tasks=latest_tasks,
         record_counts=record_counts,
         product=product,
         run_status=run_status,
@@ -652,9 +690,11 @@ async def list_datasources(
     )
     for datasource in datasources:
         running_task = running_tasks.get(datasource.id)
+        latest_task = latest_tasks.get(datasource.id)
+        display_task = running_task or latest_task
         endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
-        last_run_at = datasource.last_run_at
-        last_status = datasource.last_status
+        last_run_at = datasource.last_run_at or (latest_task.completed_at if latest_task else None)
+        last_status = datasource.last_status or (latest_task.status if latest_task else None)
         collected_records = record_counts.get(datasource.source, 0)
 
         collector_list.append(
@@ -675,16 +715,17 @@ async def list_datasources(
                 "last_run_at": to_iso8601_utc(last_run_at),
                 "last_status": last_status,
                 "is_running": running_task is not None,
-                "task_id": running_task.id if running_task else None,
-                "progress": running_task.progress if running_task else None,
-                "phase": running_task.phase if running_task else None,
-                "phase_progress": running_task.phase_progress if running_task else None,
-                "phase_message": running_task.phase_message if running_task else None,
-                "phase_current": running_task.phase_current if running_task else None,
-                "phase_total": running_task.phase_total if running_task else None,
-                "phase_unit": running_task.phase_unit if running_task else None,
-                "records_processed": running_task.records_processed if running_task else None,
-                "total_records": running_task.total_records if running_task else None,
+                "task_id": display_task.id if display_task else None,
+                "progress": display_task.progress if display_task else None,
+                "phase": display_task.phase if display_task else None,
+                "phase_progress": display_task.phase_progress if display_task else None,
+                "phase_message": display_task.phase_message if display_task else None,
+                "phase_current": display_task.phase_current if display_task else None,
+                "phase_total": display_task.phase_total if display_task else None,
+                "phase_unit": display_task.phase_unit if display_task else None,
+                "records_processed": display_task.records_processed if display_task else None,
+                "total_records": display_task.total_records if display_task else None,
+                "error_message": display_task.error_message if display_task else None,
                 "collected_records": collected_records,
                 "has_collected_data": collected_records > 0,
             }
@@ -729,11 +770,12 @@ async def trigger_datasource_batch(
 
     result = await db.execute(query)
     datasources = result.scalars().all()
-    running_tasks, _ = await _load_datasource_list_context(db, datasources)
+    running_tasks, latest_tasks, _ = await _load_datasource_list_context(db, datasources)
     record_counts = await _load_collected_record_counts(db, [datasource.source for datasource in datasources])
     datasources = _filter_datasources_in_memory(
         datasources,
         running_tasks=running_tasks,
+        latest_tasks=latest_tasks,
         record_counts=record_counts,
         product=None if payload.source_ids else payload.product,
         run_status=None if payload.source_ids else payload.run_status,
@@ -935,6 +977,14 @@ async def get_task_status(
             raise HTTPException(status_code=404, detail="Task not found")
     else:
         task = await get_running_task(db, datasource.id)
+        if task is None:
+            result = await db.execute(
+                select(CollectionTask)
+                .where(CollectionTask.datasource_id == datasource.id)
+                .order_by(CollectionTask.created_at.desc().nullslast(), CollectionTask.id.desc())
+                .limit(1)
+            )
+            task = result.scalar_one_or_none()
 
     if not task:
         return {
@@ -963,4 +1013,5 @@ async def get_task_status(
         "records_processed": task.records_processed,
         "total_records": task.total_records,
         "status": task.status,
+        "error_message": task.error_message,
     }