fix: expand bgp pipeline and stabilize backend tests

backend/app/api/v1/visualization.py

@@ -202,6 +202,44 @@ def convert_satellite_to_geojson(records: List[CollectedData]) -> Dict[str, Any]
     return {"type": "FeatureCollection", "features": features}
 
 
+def dedupe_satellite_records(records: List[CollectedData]) -> List[CollectedData]:
+    """Keep only the newest record for each satellite identity."""
+    latest_by_key: Dict[str, CollectedData] = {}
+
+    for record in records:
+        metadata = record.extra_data or {}
+        norad_id = metadata.get("norad_cat_id")
+        dedupe_key = (
+            str(norad_id)
+            if norad_id not in (None, "")
+            else str(record.source_id or record.entity_key or record.name or record.id)
+        )
+
+        existing = latest_by_key.get(dedupe_key)
+        if existing is None or (record.id or 0) > (existing.id or 0):
+            latest_by_key[dedupe_key] = record
+
+    return sorted(latest_by_key.values(), key=lambda item: item.id or 0, reverse=True)
+
+
+def dedupe_collected_records(records: List[CollectedData]) -> List[CollectedData]:
+    """Keep only the newest record for each collected entity."""
+    latest_by_key: Dict[str, CollectedData] = {}
+
+    for record in records:
+        dedupe_key = str(
+            record.source_id
+            or record.entity_key
+            or record.name
+            or record.id
+        )
+        existing = latest_by_key.get(dedupe_key)
+        if existing is None or (record.id or 0) > (existing.id or 0):
+            latest_by_key[dedupe_key] = record
+
+    return sorted(latest_by_key.values(), key=lambda item: item.id or 0, reverse=True)
+
+
 def convert_supercomputer_to_geojson(records: List[CollectedData]) -> Dict[str, Any]:
     """Convert TOP500 supercomputer records to GeoJSON"""
     features = []
@@ -410,7 +448,7 @@ async def get_cables_geojson(db: AsyncSession = Depends(get_db)):
     try:
         stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
         result = await db.execute(stmt)
-        records = result.scalars().all()
+        records = dedupe_collected_records(list(result.scalars().all()))
 
         if not records:
             raise HTTPException(
@@ -430,15 +468,15 @@ async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
     try:
         landing_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
         landing_result = await db.execute(landing_stmt)
-        records = landing_result.scalars().all()
+        records = dedupe_collected_records(list(landing_result.scalars().all()))
         
         relation_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cable_landing_relation")
         relation_result = await db.execute(relation_stmt)
-        relation_records = relation_result.scalars().all()
+        relation_records = dedupe_collected_records(list(relation_result.scalars().all()))
         
         cable_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
         cable_result = await db.execute(cable_stmt)
-        cable_records = cable_result.scalars().all()
+        cable_records = dedupe_collected_records(list(cable_result.scalars().all()))
         
         city_to_cable_ids_map = {}
         for rel in relation_records:
@@ -476,15 +514,15 @@ async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
 async def get_all_geojson(db: AsyncSession = Depends(get_db)):
     cables_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
     cables_result = await db.execute(cables_stmt)
-    cables_records = cables_result.scalars().all()
+    cables_records = dedupe_collected_records(list(cables_result.scalars().all()))
 
     points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
     points_result = await db.execute(points_stmt)
-    points_records = points_result.scalars().all()
+    points_records = dedupe_collected_records(list(points_result.scalars().all()))
     
     relation_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cable_landing_relation")
     relation_result = await db.execute(relation_stmt)
-    relation_records = relation_result.scalars().all()
+    relation_records = dedupe_collected_records(list(relation_result.scalars().all()))
     
     city_to_cable_ids_map = {}
     for rel in relation_records:
@@ -542,10 +580,11 @@ async def get_satellites_geojson(
         .where(CollectedData.name != "Unknown")
         .order_by(CollectedData.id.desc())
     )
-    if limit is not None:
-        stmt = stmt.limit(limit)
     result = await db.execute(stmt)
-    records = result.scalars().all()
+    records = dedupe_satellite_records(list(result.scalars().all()))
+
+    if limit is not None:
+        records = records[:limit]
 
     if not records:
         return {"type": "FeatureCollection", "features": [], "count": 0}
@@ -567,10 +606,11 @@ async def get_supercomputers_geojson(
         select(CollectedData)
         .where(CollectedData.source == "top500")
         .where(CollectedData.name != "Unknown")
-        .limit(limit)
+        .order_by(CollectedData.id.desc())
     )
     result = await db.execute(stmt)
-    records = result.scalars().all()
+    records = dedupe_collected_records(list(result.scalars().all()))
+    records = records[:limit]
 
     if not records:
         return {"type": "FeatureCollection", "features": [], "count": 0}
@@ -592,10 +632,11 @@ async def get_gpu_clusters_geojson(
         select(CollectedData)
         .where(CollectedData.source == "epoch_ai_gpu")
         .where(CollectedData.name != "Unknown")
-        .limit(limit)
+        .order_by(CollectedData.id.desc())
     )
     result = await db.execute(stmt)
-    records = result.scalars().all()
+    records = dedupe_collected_records(list(result.scalars().all()))
+    records = records[:limit]
 
     if not records:
         return {"type": "FeatureCollection", "features": [], "count": 0}
@@ -645,11 +686,11 @@ async def get_all_visualization_data(db: AsyncSession = Depends(get_db)):
     """
     cables_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
     cables_result = await db.execute(cables_stmt)
-    cables_records = list(cables_result.scalars().all())
+    cables_records = dedupe_collected_records(list(cables_result.scalars().all()))
 
     points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
     points_result = await db.execute(points_stmt)
-    points_records = list(points_result.scalars().all())
+    points_records = dedupe_collected_records(list(points_result.scalars().all()))
 
     satellites_stmt = (
         select(CollectedData)
@@ -657,7 +698,7 @@ async def get_all_visualization_data(db: AsyncSession = Depends(get_db)):
         .where(CollectedData.name != "Unknown")
     )
     satellites_result = await db.execute(satellites_stmt)
-    satellites_records = list(satellites_result.scalars().all())
+    satellites_records = dedupe_satellite_records(list(satellites_result.scalars().all()))
 
     supercomputers_stmt = (
         select(CollectedData)
@@ -665,7 +706,7 @@ async def get_all_visualization_data(db: AsyncSession = Depends(get_db)):
         .where(CollectedData.name != "Unknown")
     )
     supercomputers_result = await db.execute(supercomputers_stmt)
-    supercomputers_records = list(supercomputers_result.scalars().all())
+    supercomputers_records = dedupe_collected_records(list(supercomputers_result.scalars().all()))
 
     gpu_stmt = (
         select(CollectedData)
@@ -673,7 +714,7 @@ async def get_all_visualization_data(db: AsyncSession = Depends(get_db)):
         .where(CollectedData.name != "Unknown")
     )
     gpu_result = await db.execute(gpu_stmt)
-    gpu_records = list(gpu_result.scalars().all())
+    gpu_records = dedupe_collected_records(list(gpu_result.scalars().all()))
 
     cables = (
         convert_cable_to_geojson(cables_records)