release: bump version to 0.62.0

backend/app/services/collectors/base.py

@@ -9,12 +9,38 @@ from sqlalchemy import select, text
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.core.collected_data_fields import build_dynamic_metadata, get_record_field
-from app.core.config import settings
 from app.core.countries import normalize_country
 from app.core.time import to_iso8601_utc
 from app.core.websocket.broadcaster import broadcaster
 from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
 
+EARTH_UPDATE_LAYER_HINTS: dict[str, list[str]] = {
+    "ris_live_bgp": ["bgp"],
+    "bgpstream_bgp": ["bgp"],
+    "top500_supercomputers": ["computeCenters"],
+    "epoch_ai_gpu": ["computeCenters"],
+    "huggingface_models": ["computeCenters"],
+    "huggingface_datasets": ["computeCenters"],
+    "huggingface_spaces": ["computeCenters"],
+    "telegeography_cables": ["cables"],
+    "telegeography_landing_points": ["cables"],
+    "telegeography_cable_systems": ["cables"],
+    "arcgis_cables": ["cables"],
+    "fao_landing_points": ["cables"],
+    "arcgis_landing_points": ["cables"],
+    "arcgis_cable_landing_relations": ["cables"],
+    "spacetrack_tle": ["satellites"],
+    "celestrak_tle": ["satellites"],
+    "barentswatch_vessels": ["vessels"],
+    "aisstream_vessels": ["vessels"],
+    "news_live_streams": ["media"],
+    "media_news_archive": ["news"],
+}
+
+
+def get_earth_update_layers_for_source(source: str) -> list[str]:
+    return EARTH_UPDATE_LAYER_HINTS.get(source, [])
+
 
 class BaseCollector(ABC):
     """Abstract base class for data collectors"""
@@ -70,6 +96,29 @@ class BaseCollector(ABC):
         )
         self._last_broadcast_progress = rounded_progress
 
+    async def _publish_earth_update(
+        self,
+        *,
+        action: str,
+        records_processed: int,
+        task_id: int | None = None,
+    ) -> None:
+        layers = get_earth_update_layers_for_source(self.name)
+        if not layers:
+            return
+        await broadcaster.broadcast_earth_update(
+            {
+                "action": action,
+                "source": self.name,
+                "data_type": self.data_type,
+                "layers": layers,
+                "datasource_id": getattr(self, "_datasource_id", None),
+                "task_id": task_id,
+                "records_processed": records_processed,
+                "timestamp": to_iso8601_utc(datetime.now(UTC)),
+            }
+        )
+
     async def update_progress(self, records_processed: int, *, commit: bool = False, force: bool = False):
         """Update task progress - call this during data processing"""
         if self._current_task and self._db_session:
@@ -187,7 +236,7 @@ class BaseCollector(ABC):
 
         result = await db.execute(
             select(DataSnapshot)
-            .where(DataSnapshot.source == self.name, DataSnapshot.is_current == True)
+            .where(DataSnapshot.source == self.name, DataSnapshot.is_current.is_(True))
             .order_by(DataSnapshot.completed_at.desc().nullslast(), DataSnapshot.id.desc())
             .limit(1)
         )
@@ -325,6 +374,11 @@ class BaseCollector(ABC):
             task.completed_at = datetime.now(UTC)
             await db.commit()
             await self._publish_task_update(force=True)
+            await self._publish_earth_update(
+                action="collector_completed",
+                records_processed=records_count,
+                task_id=task_id,
+            )
 
             return {
                 "status": "success",
@@ -406,7 +460,7 @@ class BaseCollector(ABC):
             select(CollectedData)
             .where(
                 CollectedData.source == self.name,
-                CollectedData.is_current == True,
+                CollectedData.is_current.is_(True),
             )
             .order_by(CollectedData.entity_key.asc(), CollectedData.collected_at.desc().nullslast(), CollectedData.id.desc())
         )