fix: optimize visualization hot paths and refine bgp brief workspace

backend/app/api/v1/alerts.py

@@ -101,25 +101,44 @@ async def get_alert_stats(
     current_user: User = Depends(get_current_user),
     db: AsyncSession = Depends(get_db),
 ):
-    critical_query = select(func.count(Alert.id)).where(
-        Alert.severity == AlertSeverity.CRITICAL,
-        Alert.status == AlertStatus.ACTIVE,
+    result = await db.execute(
+        select(
+            func.sum(
+                case(
+                    (
+                        (Alert.severity == AlertSeverity.CRITICAL)
+                        & (Alert.status == AlertStatus.ACTIVE),
+                        1,
+                    ),
+                    else_=0,
+                )
+            ).label("critical"),
+            func.sum(
+                case(
+                    (
+                        (Alert.severity == AlertSeverity.WARNING)
+                        & (Alert.status == AlertStatus.ACTIVE),
+                        1,
+                    ),
+                    else_=0,
+                )
+            ).label("warning"),
+            func.sum(
+                case(
+                    (
+                        (Alert.severity == AlertSeverity.INFO)
+                        & (Alert.status == AlertStatus.ACTIVE),
+                        1,
+                    ),
+                    else_=0,
+                )
+            ).label("info"),
+        )
     )
-    warning_query = select(func.count(Alert.id)).where(
-        Alert.severity == AlertSeverity.WARNING,
-        Alert.status == AlertStatus.ACTIVE,
-    )
-    info_query = select(func.count(Alert.id)).where(
-        Alert.severity == AlertSeverity.INFO,
-        Alert.status == AlertStatus.ACTIVE,
-    )
-
-    critical_result = await db.execute(critical_query)
-    warning_result = await db.execute(warning_query)
-    info_result = await db.execute(info_query)
+    row = result.one()
 
     return {
-        "critical": critical_result.scalar() or 0,
-        "warning": warning_result.scalar() or 0,
-        "info": info_result.scalar() or 0,
+        "critical": row.critical or 0,
+        "warning": row.warning or 0,
+        "info": row.info or 0,
     }

backend/app/api/v1/dashboard.py

@@ -118,58 +118,77 @@ async def get_stats(
     built_in_count = len(COLLECTOR_INFO)
     built_in_active = built_in_count  # Built-in are always "active" for counting purposes
 
-    # Count custom configs from database
-    result = await db.execute(select(func.count(DataSourceConfig.id)))
-    custom_count = result.scalar() or 0
-
     result = await db.execute(
-        select(func.count(DataSourceConfig.id)).where(DataSourceConfig.is_active == True)
+        select(
+            func.count(DataSourceConfig.id).label("custom_count"),
+            func.sum(
+                case((DataSourceConfig.is_active == True, 1), else_=0)
+            ).label("custom_active"),
+        )
     )
-    custom_active = result.scalar() or 0
+    datasource_stats = result.one()
+    custom_count = datasource_stats.custom_count or 0
+    custom_active = datasource_stats.custom_active or 0
 
     # Total datasources
     total_datasources = built_in_count + custom_count
     active_datasources = built_in_active + custom_active
 
-    # Tasks today (from database)
     result = await db.execute(
-        select(func.count(CollectionTask.id)).where(CollectionTask.started_at >= today_start)
-    )
-    tasks_today = result.scalar() or 0
-
-    result = await db.execute(
-        select(func.count(CollectionTask.id)).where(
-            CollectionTask.status == "success",
-            CollectionTask.started_at >= today_start,
+        select(
+            func.count(CollectionTask.id).label("tasks_today"),
+            func.sum(
+                case(
+                    (CollectionTask.status == "success", 1),
+                    else_=0,
+                )
+            ).label("success_tasks"),
         )
+        .where(CollectionTask.started_at >= today_start)
     )
-    success_tasks = result.scalar() or 0
+    task_stats = result.one()
+    tasks_today = task_stats.tasks_today or 0
+    success_tasks = task_stats.success_tasks or 0
     success_rate = (success_tasks / tasks_today * 100) if tasks_today > 0 else 0
 
-    # Alerts
     result = await db.execute(
-        select(func.count(Alert.id)).where(
-            Alert.severity == AlertSeverity.CRITICAL,
-            Alert.status == "active",
+        select(
+            func.sum(
+                case(
+                    (
+                        (Alert.severity == AlertSeverity.CRITICAL)
+                        & (Alert.status == "active"),
+                        1,
+                    ),
+                    else_=0,
+                )
+            ).label("critical_alerts"),
+            func.sum(
+                case(
+                    (
+                        (Alert.severity == AlertSeverity.WARNING)
+                        & (Alert.status == "active"),
+                        1,
+                    ),
+                    else_=0,
+                )
+            ).label("warning_alerts"),
+            func.sum(
+                case(
+                    (
+                        (Alert.severity == AlertSeverity.INFO)
+                        & (Alert.status == "active"),
+                        1,
+                    ),
+                    else_=0,
+                )
+            ).label("info_alerts"),
         )
     )
-    critical_alerts = result.scalar() or 0
-
-    result = await db.execute(
-        select(func.count(Alert.id)).where(
-            Alert.severity == AlertSeverity.WARNING,
-            Alert.status == "active",
-        )
-    )
-    warning_alerts = result.scalar() or 0
-
-    result = await db.execute(
-        select(func.count(Alert.id)).where(
-            Alert.severity == AlertSeverity.INFO,
-            Alert.status == "active",
-        )
-    )
-    info_alerts = result.scalar() or 0
+    alert_stats = result.one()
+    critical_alerts = alert_stats.critical_alerts or 0
+    warning_alerts = alert_stats.warning_alerts or 0
+    info_alerts = alert_stats.info_alerts or 0
 
     response = {
         "total_datasources": total_datasources,

backend/app/api/v1/datasources.py

@@ -13,6 +13,7 @@ from app.db.session import get_db
 from app.models.collected_data import CollectedData
 from app.models.data_snapshot import DataSnapshot
 from app.models.datasource import DataSource
+from app.models.datasource_config import DataSourceConfig
 from app.models.task import CollectionTask
 from app.models.user import User
 from app.services.scheduler import (
@@ -40,6 +41,156 @@ def is_due_for_collection(datasource: DataSource, now: datetime) -> bool:
     return datasource.last_run_at + timedelta(minutes=datasource.frequency_minutes) <= now
 
 
+def _task_rank_column(order_column):
+    return func.row_number().over(
+        partition_by=CollectionTask.datasource_id,
+        order_by=(order_column.desc().nullslast(), CollectionTask.id.desc()),
+    ).label("row_num")
+
+
+async def _load_latest_running_tasks(
+    db: AsyncSession,
+    datasource_ids: list[int],
+) -> dict[int, CollectionTask]:
+    if not datasource_ids:
+        return {}
+
+    ranked_tasks = (
+        select(
+            CollectionTask.id.label("task_id"),
+            _task_rank_column(CollectionTask.started_at),
+        )
+        .where(CollectionTask.datasource_id.in_(datasource_ids))
+        .where(CollectionTask.status == "running")
+        .subquery()
+    )
+    result = await db.execute(
+        select(CollectionTask)
+        .join(ranked_tasks, CollectionTask.id == ranked_tasks.c.task_id)
+        .where(ranked_tasks.c.row_num == 1)
+    )
+    return {task.datasource_id: task for task in result.scalars().all()}
+
+
+async def _load_latest_completed_tasks(
+    db: AsyncSession,
+    datasource_ids: list[int],
+) -> dict[int, CollectionTask]:
+    if not datasource_ids:
+        return {}
+
+    ranked_tasks = (
+        select(
+            CollectionTask.id.label("task_id"),
+            _task_rank_column(CollectionTask.completed_at),
+        )
+        .where(CollectionTask.datasource_id.in_(datasource_ids))
+        .where(CollectionTask.completed_at.isnot(None))
+        .where(CollectionTask.status.in_(("success", "failed", "cancelled")))
+        .subquery()
+    )
+    result = await db.execute(
+        select(CollectionTask)
+        .join(ranked_tasks, CollectionTask.id == ranked_tasks.c.task_id)
+        .where(ranked_tasks.c.row_num == 1)
+    )
+    return {task.datasource_id: task for task in result.scalars().all()}
+
+
+async def _load_latest_task_ids(
+    db: AsyncSession,
+    datasource_ids: list[int],
+) -> dict[int, int]:
+    if not datasource_ids:
+        return {}
+
+    ranked_tasks = (
+        select(
+            CollectionTask.id.label("task_id"),
+            CollectionTask.datasource_id.label("datasource_id"),
+            func.row_number().over(
+                partition_by=CollectionTask.datasource_id,
+                order_by=CollectionTask.id.desc(),
+            ).label("row_num"),
+        )
+        .where(CollectionTask.datasource_id.in_(datasource_ids))
+        .subquery()
+    )
+    result = await db.execute(
+        select(ranked_tasks.c.datasource_id, ranked_tasks.c.task_id)
+        .where(ranked_tasks.c.row_num == 1)
+    )
+    return {datasource_id: task_id for datasource_id, task_id in result.all()}
+
+
+async def _load_datasource_data_counts(
+    db: AsyncSession,
+    sources: list[str],
+) -> dict[str, int]:
+    if not sources:
+        return {}
+
+    result = await db.execute(
+        select(CollectedData.source, func.count(CollectedData.id))
+        .where(CollectedData.source.in_(sources))
+        .group_by(CollectedData.source)
+    )
+    return {source: count for source, count in result.all()}
+
+
+async def _load_datasource_endpoint_overrides(
+    db: AsyncSession,
+    sources: list[str],
+) -> dict[str, str]:
+    if not sources:
+        return {}
+
+    result = await db.execute(
+        select(DataSourceConfig.name, DataSourceConfig.endpoint)
+        .where(DataSourceConfig.name.in_(sources))
+        .where(DataSourceConfig.is_active.is_(True))
+        .where(DataSourceConfig.endpoint.isnot(None))
+    )
+    return {
+        name: endpoint
+        for name, endpoint in result.all()
+        if endpoint
+    }
+
+
+async def _load_datasource_list_context(
+    db: AsyncSession,
+    datasources: list[DataSource],
+) -> tuple[dict[int, CollectionTask], dict[int, CollectionTask], dict[str, int], dict[str, str]]:
+    datasource_ids = [datasource.id for datasource in datasources]
+    sources = [datasource.source for datasource in datasources]
+
+    running_tasks = await _load_latest_running_tasks(db, datasource_ids)
+    datasource_by_id = {datasource.id: datasource for datasource in datasources}
+    now = datetime.now(timezone.utc)
+
+    stale_datasource_ids: list[int] = []
+    for datasource_id, task in running_tasks.items():
+        started_at = task.started_at
+        if started_at is None:
+            continue
+        if started_at.tzinfo is None:
+            started_at = started_at.replace(tzinfo=timezone.utc)
+        if now - started_at > timedelta(minutes=STALE_RUNNING_TASK_TIMEOUT_MINUTES):
+            datasource = datasource_by_id.get(datasource_id)
+            if datasource is not None:
+                await fail_and_rollback_stale_running_task(db, datasource, task)
+                stale_datasource_ids.append(datasource_id)
+
+    if stale_datasource_ids:
+        running_tasks = await _load_latest_running_tasks(db, datasource_ids)
+
+    completed_tasks = await _load_latest_completed_tasks(db, datasource_ids)
+    data_counts = await _load_datasource_data_counts(db, sources)
+    endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources)
+    return running_tasks, completed_tasks, data_counts, endpoint_overrides
+
+
 async def get_datasource_record(db: AsyncSession, source_id: str) -> Optional[DataSource]:
     datasource = None
     try:
@@ -58,18 +209,6 @@ async def get_datasource_record(db: AsyncSession, source_id: str) -> Optional[Da
     return result.scalar_one_or_none()
 
 
-async def get_last_completed_task(db: AsyncSession, datasource_id: int) -> Optional[CollectionTask]:
-    result = await db.execute(
-        select(CollectionTask)
-        .where(CollectionTask.datasource_id == datasource_id)
-        .where(CollectionTask.completed_at.isnot(None))
-        .where(CollectionTask.status.in_(("success", "failed", "cancelled")))
-        .order_by(CollectionTask.completed_at.desc())
-        .limit(1)
-    )
-    return result.scalar_one_or_none()
-
-
 async def get_running_task(db: AsyncSession, datasource_id: int) -> Optional[CollectionTask]:
     result = await db.execute(
         select(CollectionTask)
@@ -262,14 +401,17 @@ async def list_datasources(
 
     collector_list = []
     config = get_data_sources_config()
+    running_tasks, completed_tasks, data_counts, endpoint_overrides = await _load_datasource_list_context(
+        db,
+        datasources,
+    )
     for datasource in datasources:
-        running_task = await get_running_task(db, datasource.id)
-        last_task = await get_last_completed_task(db, datasource.id)
-        endpoint = await config.get_url(datasource.source, db)
-        data_count_result = await db.execute(
-            select(func.count(CollectedData.id)).where(CollectedData.source == datasource.source)
+        running_task = running_tasks.get(datasource.id)
+        last_task = completed_tasks.get(datasource.id)
+        endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(
+            datasource.source,
         )
-        data_count = data_count_result.scalar() or 0
+        data_count = data_counts.get(datasource.source, 0)
 
         last_run_at = datasource.last_run_at or (last_task.completed_at if last_task else None)
         last_run = to_iso8601_utc(last_run_at)
@@ -330,9 +472,13 @@ async def trigger_all_datasources(
     skipped_sources: list[dict] = []
     failed_sources: list[dict] = []
     now = datetime.now(timezone.utc)
+    running_tasks = await _load_latest_running_tasks(
+        db,
+        [datasource.id for datasource in datasources],
+    )
 
     for datasource in datasources:
-        running_task = await get_running_task(db, datasource.id)
+        running_task = running_tasks.get(datasource.id)
         if running_task is not None:
             skipped_sources.append(
                 {
@@ -360,7 +506,7 @@ async def trigger_all_datasources(
             )
             continue
 
-        previous_task_ids[datasource.id] = await get_latest_task_id_for_datasource(datasource.id)
+        previous_task_ids[datasource.id] = None
         success = run_collector_now(datasource.source)
         if not success:
             failed_sources.append(
@@ -382,13 +528,24 @@ async def trigger_all_datasources(
             }
         )
 
+    latest_task_ids = await _load_latest_task_ids(
+        db,
+        [datasource.id for datasource in datasources],
+    )
+    for datasource_id in previous_task_ids:
+        previous_task_ids[datasource_id] = latest_task_ids.get(datasource_id)
+
     for _ in range(20):
         await asyncio.sleep(0.1)
         pending = [item for item in triggered_sources if item["task_id"] is None]
         if not pending:
             break
+        latest_task_ids = await _load_latest_task_ids(
+            db,
+            [item["id"] for item in pending],
+        )
         for item in pending:
-            task_id = await get_latest_task_id_for_datasource(item["id"])
+            task_id = latest_task_ids.get(item["id"])
             if task_id is not None and task_id != previous_task_ids.get(item["id"]):
                 item["task_id"] = task_id
 

backend/app/api/v1/settings.py

@@ -79,6 +79,26 @@ async def get_setting_record(db: AsyncSession, category: str) -> Optional[System
     return result.scalar_one_or_none()
 
 
+async def get_setting_payloads(db: AsyncSession, categories: list[str]) -> dict[str, dict]:
+    if not categories:
+        return {}
+
+    result = await db.execute(
+        select(SystemSetting).where(SystemSetting.category.in_(categories))
+    )
+    records_by_category = {
+        record.category: record
+        for record in result.scalars().all()
+    }
+    return {
+        category: merge_with_defaults(
+            category,
+            records_by_category.get(category).payload if records_by_category.get(category) else None,
+        )
+        for category in categories
+    }
+
+
 async def get_setting_payload(db: AsyncSession, category: str) -> dict:
     record = await get_setting_record(db, category)
     return merge_with_defaults(category, record.payload if record else None)
@@ -212,10 +232,14 @@ async def get_all_settings(
 ):
     result = await db.execute(select(DataSource).order_by(DataSource.module, DataSource.id))
     datasources = result.scalars().all()
+    setting_payloads = await get_setting_payloads(
+        db,
+        ["system", "notifications", "security"],
+    )
     return {
-        "system": await get_setting_payload(db, "system"),
-        "notifications": await get_setting_payload(db, "notifications"),
-        "security": await get_setting_payload(db, "security"),
+        "system": setting_payloads["system"],
+        "notifications": setting_payloads["notifications"],
+        "security": setting_payloads["security"],
         "collectors": [serialize_collector(datasource) for datasource in datasources],
         "generated_at": to_iso8601_utc(datetime.now(UTC)),
     }

backend/app/api/v1/visualization.py

@@ -205,42 +205,84 @@ def convert_satellite_to_geojson(records: List[CollectedData]) -> Dict[str, Any]
     return {"type": "FeatureCollection", "features": features}
 
 
-def dedupe_satellite_records(records: List[CollectedData]) -> List[CollectedData]:
-    """Keep only the newest record for each satellite identity."""
-    latest_by_key: Dict[str, CollectedData] = {}
-
-    for record in records:
-        metadata = record.extra_data or {}
-        norad_id = metadata.get("norad_cat_id")
-        dedupe_key = (
-            str(norad_id)
-            if norad_id not in (None, "")
-            else str(record.source_id or record.entity_key or record.name or record.id)
-        )
-
-        existing = latest_by_key.get(dedupe_key)
-        if existing is None or (record.id or 0) > (existing.id or 0):
-            latest_by_key[dedupe_key] = record
-
-    return sorted(latest_by_key.values(), key=lambda item: item.id or 0, reverse=True)
+def _current_collected_data_stmt(source: str):
+    return (
+        select(CollectedData)
+        .where(CollectedData.source == source)
+        .where(CollectedData.is_current.is_(True))
+        .order_by(CollectedData.id.desc())
+    )
 
 
-def dedupe_collected_records(records: List[CollectedData]) -> List[CollectedData]:
-    """Keep only the newest record for each collected entity."""
-    latest_by_key: Dict[str, CollectedData] = {}
+async def _load_current_collected_data(
+    db: AsyncSession,
+    source: str,
+    *,
+    exclude_unknown_name: bool = False,
+    limit: Optional[int] = None,
+) -> List[CollectedData]:
+    stmt = _current_collected_data_stmt(source)
+    if exclude_unknown_name:
+        stmt = stmt.where(CollectedData.name != "Unknown")
+    if limit is not None:
+        stmt = stmt.limit(limit)
 
-    for record in records:
-        dedupe_key = str(
-            record.source_id
-            or record.entity_key
-            or record.name
-            or record.id
-        )
-        existing = latest_by_key.get(dedupe_key)
-        if existing is None or (record.id or 0) > (existing.id or 0):
-            latest_by_key[dedupe_key] = record
+    result = await db.execute(stmt)
+    return list(result.scalars().all())
 
-    return sorted(latest_by_key.values(), key=lambda item: item.id or 0, reverse=True)
+
+async def _load_current_collected_data_by_sources(
+    db: AsyncSession,
+    sources: List[str],
+) -> Dict[str, List[CollectedData]]:
+    if not sources:
+        return {}
+
+    stmt = (
+        select(CollectedData)
+        .where(CollectedData.source.in_(sources))
+        .where(CollectedData.is_current.is_(True))
+        .order_by(CollectedData.source.asc(), CollectedData.id.desc())
+    )
+    result = await db.execute(stmt)
+
+    grouped_records: Dict[str, List[CollectedData]] = {source: [] for source in sources}
+    for record in result.scalars().all():
+        grouped_records.setdefault(record.source, []).append(record)
+
+    return grouped_records
+
+
+def _build_landing_point_cable_maps(
+    relation_records: List[CollectedData],
+    cable_records: List[CollectedData],
+) -> tuple[Dict[int, List[int]], Dict[int, str]]:
+    city_to_cable_ids_map: Dict[int, List[int]] = {}
+    for relation_record in relation_records:
+        if not relation_record.extra_data:
+            continue
+        city_id = relation_record.extra_data.get("city_id")
+        cable_id = relation_record.extra_data.get("cable_id")
+        if city_id is None or cable_id is None:
+            continue
+        city_to_cable_ids_map.setdefault(city_id, [])
+        if cable_id not in city_to_cable_ids_map[city_id]:
+            city_to_cable_ids_map[city_id].append(cable_id)
+
+    cable_id_to_name_map: Dict[int, str] = {}
+    for cable_record in cable_records:
+        if not cable_record.extra_data:
+            continue
+        cable_id = cable_record.extra_data.get("cable_id")
+        cable_name = cable_record.name
+        if cable_id and cable_name:
+            cable_id_to_name_map[cable_id] = cable_name
+
+    return city_to_cable_ids_map, cable_id_to_name_map
+
+
+def _filter_known_records(records: List[CollectedData]) -> List[CollectedData]:
+    return [record for record in records if record.name != "Unknown"]
 
 
 def convert_supercomputer_to_geojson(records: List[CollectedData]) -> Dict[str, Any]:
@@ -722,9 +764,7 @@ def convert_bgp_incidents_to_geojson(
 async def get_cables_geojson(db: AsyncSession = Depends(get_db)):
     """获取海底电缆 GeoJSON 数据 (LineString)"""
     try:
-        stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
-        result = await db.execute(stmt)
-        records = dedupe_collected_records(list(result.scalars().all()))
+        records = await _load_current_collected_data(db, "arcgis_cables")
 
         if not records:
             raise HTTPException(
@@ -742,36 +782,14 @@ async def get_cables_geojson(db: AsyncSession = Depends(get_db)):
 @router.get("/geo/landing-points")
 async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
     try:
-        landing_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
-        landing_result = await db.execute(landing_stmt)
-        records = dedupe_collected_records(list(landing_result.scalars().all()))
-        
-        relation_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cable_landing_relation")
-        relation_result = await db.execute(relation_stmt)
-        relation_records = dedupe_collected_records(list(relation_result.scalars().all()))
-        
-        cable_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
-        cable_result = await db.execute(cable_stmt)
-        cable_records = dedupe_collected_records(list(cable_result.scalars().all()))
-        
-        city_to_cable_ids_map = {}
-        for rel in relation_records:
-            if rel.extra_data:
-                city_id = rel.extra_data.get("city_id")
-                cable_id = rel.extra_data.get("cable_id")
-                if city_id is not None and cable_id is not None:
-                    if city_id not in city_to_cable_ids_map:
-                        city_to_cable_ids_map[city_id] = []
-                    if cable_id not in city_to_cable_ids_map[city_id]:
-                        city_to_cable_ids_map[city_id].append(cable_id)
-        
-        cable_id_to_name_map = {}
-        for cable in cable_records:
-            if cable.extra_data:
-                cable_id = cable.extra_data.get("cable_id")
-                cable_name = cable.name
-                if cable_id and cable_name:
-                    cable_id_to_name_map[cable_id] = cable_name
+        records = await _load_current_collected_data(db, "arcgis_landing_points")
+        relation_records = await _load_current_collected_data(db, "arcgis_cable_landing_relation")
+        cable_records = await _load_current_collected_data(db, "arcgis_cables")
+
+        city_to_cable_ids_map, cable_id_to_name_map = _build_landing_point_cable_maps(
+            relation_records,
+            cable_records,
+        )
         
         if not records:
             raise HTTPException(
@@ -788,36 +806,21 @@ async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
 
 @router.get("/geo/all")
 async def get_all_geojson(db: AsyncSession = Depends(get_db)):
-    cables_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
-    cables_result = await db.execute(cables_stmt)
-    cables_records = dedupe_collected_records(list(cables_result.scalars().all()))
-
-    points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
-    points_result = await db.execute(points_stmt)
-    points_records = dedupe_collected_records(list(points_result.scalars().all()))
-    
-    relation_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cable_landing_relation")
-    relation_result = await db.execute(relation_stmt)
-    relation_records = dedupe_collected_records(list(relation_result.scalars().all()))
-    
-    city_to_cable_ids_map = {}
-    for rel in relation_records:
-        if rel.extra_data:
-            city_id = rel.extra_data.get("city_id")
-            cable_id = rel.extra_data.get("cable_id")
-            if city_id is not None and cable_id is not None:
-                if city_id not in city_to_cable_ids_map:
-                    city_to_cable_ids_map[city_id] = []
-                if cable_id not in city_to_cable_ids_map[city_id]:
-                    city_to_cable_ids_map[city_id].append(cable_id)
-    
-    cable_id_to_name_map = {}
-    for cable in cables_records:
-        if cable.extra_data:
-            cable_id = cable.extra_data.get("cable_id")
-            cable_name = cable.name
-            if cable_id and cable_name:
-                cable_id_to_name_map[cable_id] = cable_name
+    records_by_source = await _load_current_collected_data_by_sources(
+        db,
+        [
+            "arcgis_cables",
+            "arcgis_landing_points",
+            "arcgis_cable_landing_relation",
+        ],
+    )
+    cables_records = records_by_source.get("arcgis_cables", [])
+    points_records = records_by_source.get("arcgis_landing_points", [])
+    relation_records = records_by_source.get("arcgis_cable_landing_relation", [])
+    city_to_cable_ids_map, cable_id_to_name_map = _build_landing_point_cable_maps(
+        relation_records,
+        cables_records,
+    )
 
     cables = (
         convert_cable_to_geojson(cables_records)
@@ -850,17 +853,12 @@ async def get_satellites_geojson(
     db: AsyncSession = Depends(get_db),
 ):
     """获取卫星 TLE GeoJSON 数据"""
-    stmt = (
-        select(CollectedData)
-        .where(CollectedData.source == "celestrak_tle")
-        .where(CollectedData.name != "Unknown")
-        .order_by(CollectedData.id.desc())
+    records = await _load_current_collected_data(
+        db,
+        "celestrak_tle",
+        exclude_unknown_name=True,
+        limit=limit,
     )
-    result = await db.execute(stmt)
-    records = dedupe_satellite_records(list(result.scalars().all()))
-
-    if limit is not None:
-        records = records[:limit]
 
     if not records:
         return {"type": "FeatureCollection", "features": [], "count": 0}
@@ -878,15 +876,12 @@ async def get_supercomputers_geojson(
     db: AsyncSession = Depends(get_db),
 ):
     """获取 TOP500 超算中心 GeoJSON 数据"""
-    stmt = (
-        select(CollectedData)
-        .where(CollectedData.source == "top500")
-        .where(CollectedData.name != "Unknown")
-        .order_by(CollectedData.id.desc())
+    records = await _load_current_collected_data(
+        db,
+        "top500",
+        exclude_unknown_name=True,
+        limit=limit,
     )
-    result = await db.execute(stmt)
-    records = dedupe_collected_records(list(result.scalars().all()))
-    records = records[:limit]
 
     if not records:
         return {"type": "FeatureCollection", "features": [], "count": 0}
@@ -904,15 +899,12 @@ async def get_gpu_clusters_geojson(
     db: AsyncSession = Depends(get_db),
 ):
     """获取 GPU 集群 GeoJSON 数据"""
-    stmt = (
-        select(CollectedData)
-        .where(CollectedData.source == "epoch_ai_gpu")
-        .where(CollectedData.name != "Unknown")
-        .order_by(CollectedData.id.desc())
+    records = await _load_current_collected_data(
+        db,
+        "epoch_ai_gpu",
+        exclude_unknown_name=True,
+        limit=limit,
     )
-    result = await db.execute(stmt)
-    records = dedupe_collected_records(list(result.scalars().all()))
-    records = records[:limit]
 
     if not records:
         return {"type": "FeatureCollection", "features": [], "count": 0}
@@ -990,37 +982,27 @@ async def get_all_visualization_data(db: AsyncSession = Depends(get_db)):
     - supercomputers: TOP500 超算
     - gpu_clusters: GPU 集群
     """
-    cables_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
-    cables_result = await db.execute(cables_stmt)
-    cables_records = dedupe_collected_records(list(cables_result.scalars().all()))
-
-    points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
-    points_result = await db.execute(points_stmt)
-    points_records = dedupe_collected_records(list(points_result.scalars().all()))
-
-    satellites_stmt = (
-        select(CollectedData)
-        .where(CollectedData.source == "celestrak_tle")
-        .where(CollectedData.name != "Unknown")
+    records_by_source = await _load_current_collected_data_by_sources(
+        db,
+        [
+            "arcgis_cables",
+            "arcgis_landing_points",
+            "celestrak_tle",
+            "top500",
+            "epoch_ai_gpu",
+        ],
     )
-    satellites_result = await db.execute(satellites_stmt)
-    satellites_records = dedupe_satellite_records(list(satellites_result.scalars().all()))
-
-    supercomputers_stmt = (
-        select(CollectedData)
-        .where(CollectedData.source == "top500")
-        .where(CollectedData.name != "Unknown")
+    cables_records = records_by_source.get("arcgis_cables", [])
+    points_records = records_by_source.get("arcgis_landing_points", [])
+    satellites_records = _filter_known_records(
+        records_by_source.get("celestrak_tle", []),
     )
-    supercomputers_result = await db.execute(supercomputers_stmt)
-    supercomputers_records = dedupe_collected_records(list(supercomputers_result.scalars().all()))
-
-    gpu_stmt = (
-        select(CollectedData)
-        .where(CollectedData.source == "epoch_ai_gpu")
-        .where(CollectedData.name != "Unknown")
+    supercomputers_records = _filter_known_records(
+        records_by_source.get("top500", []),
+    )
+    gpu_records = _filter_known_records(
+        records_by_source.get("epoch_ai_gpu", []),
     )
-    gpu_result = await db.execute(gpu_stmt)
-    gpu_records = dedupe_collected_records(list(gpu_result.scalars().all()))
 
     cables = (
         convert_cable_to_geojson(cables_records)
@@ -1084,13 +1066,8 @@ async def get_cable_graph(db: AsyncSession) -> CableGraph:
     global _cable_graph
 
     if _cable_graph is None:
-        cables_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
-        cables_result = await db.execute(cables_stmt)
-        cables_records = list(cables_result.scalars().all())
-
-        points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
-        points_result = await db.execute(points_stmt)
-        points_records = list(points_result.scalars().all())
+        cables_records = await _load_current_collected_data(db, "arcgis_cables")
+        points_records = await _load_current_collected_data(db, "arcgis_landing_points")
 
         cables_data = convert_cable_to_geojson(cables_records)
         points_data = convert_landing_point_to_geojson(points_records)

backend/app/services/bgp_collectors.py

@@ -35,8 +35,8 @@ async def build_bgp_collector_coverage(
     recent_7d_threshold = now - timedelta(days=7)
 
     filters = _collector_base_filters(source_filter)
-    country_expr = func.nullif(BGPObservation.collector_geo["country"].astext, "")
-    city_expr = func.nullif(BGPObservation.collector_geo["city"].astext, "")
+    country_expr = func.nullif(BGPObservation.collector_geo["country"].as_string(), "")
+    city_expr = func.nullif(BGPObservation.collector_geo["city"].as_string(), "")
 
     aggregate_stmt = (
         select(

backend/app/services/bgp_enrichment.py

@@ -7,7 +7,7 @@ from collections import defaultdict
 from datetime import UTC, datetime
 from typing import Any
 
-from sqlalchemy import select, text
+from sqlalchemy import Integer, cast, select, text
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.core.countries import get_country_centroid, normalize_country
@@ -261,29 +261,40 @@ async def enrich_bgp_events_for_batch(
     historical_prefix_baseline: dict[str, dict[str, Any]] = {}
     if prefix_values:
         previous_result = await db.execute(
-            select(BGPObservation).where(
+            select(
+                BGPObservation.prefix,
+                BGPObservation.origin_asn,
+                BGPObservation.collector,
+                BGPObservation.collector_geo,
+            ).where(
                 BGPObservation.source == source,
                 BGPObservation.prefix.in_(prefix_values),
             )
         )
-        by_prefix: defaultdict[str, list[BGPObservation]] = defaultdict(list)
-        for observation in previous_result.scalars().all():
-            if observation.prefix:
-                by_prefix[observation.prefix].append(observation)
+        by_prefix: defaultdict[str, list[dict[str, Any]]] = defaultdict(list)
+        for prefix, origin_asn, collector, collector_geo in previous_result.all():
+            if prefix:
+                by_prefix[str(prefix)].append(
+                    {
+                        "origin_asn": origin_asn,
+                        "collector": collector,
+                        "collector_geo": collector_geo or {},
+                    }
+                )
 
         for prefix, observations in by_prefix.items():
             unique_origins = sorted(
                 {
-                    observation.origin_asn
+                    observation["origin_asn"]
                     for observation in observations
-                    if observation.origin_asn is not None
+                    if observation["origin_asn"] is not None
                 }
             )
             unique_collectors = sorted(
                 {
-                    observation.collector
+                    observation["collector"]
                     for observation in observations
-                    if observation.collector
+                    if observation["collector"]
                 }
             )
             historical_prefix_baseline[prefix] = {
@@ -292,9 +303,9 @@ async def enrich_bgp_events_for_batch(
                 "historical_observation_count": len(observations),
                 "historical_regions": _compact_locations(
                     [
-                        observation.collector_geo or {}
+                        observation["collector_geo"] or {}
                         for observation in observations
-                        if observation.collector_geo
+                        if observation["collector_geo"]
                     ]
                 ),
             }
@@ -303,7 +314,13 @@ async def enrich_bgp_events_for_batch(
     prefix_geographies = await _lookup_prefix_geography(db, prefix_values) if prefix_values else {}
     if origin_asns:
         peeringdb_result = await db.execute(
-            select(CollectedData).where(CollectedData.source == "peeringdb_network")
+            select(CollectedData)
+            .where(CollectedData.source == "peeringdb_network")
+            .where(CollectedData.is_current.is_(True))
+            .where(
+                cast(CollectedData.extra_data["asn"].as_string(), Integer).in_(origin_asns),
+            )
+            .order_by(CollectedData.id.desc())
         )
         for record in peeringdb_result.scalars().all():
             metadata = record.extra_data or {}

backend/app/services/bgp_incidents.py

@@ -48,14 +48,36 @@ def _collector_regions_from_anomaly(anomaly: BGPAnomaly) -> list[dict]:
     return collected
 
 
-def _dedupe_collected_records(records: list[CollectedData]) -> list[CollectedData]:
-    latest_by_key: dict[str, CollectedData] = {}
-    for record in records:
-        dedupe_key = str(record.source_id or record.entity_key or record.name or record.id)
-        existing = latest_by_key.get(dedupe_key)
-        if existing is None or (record.id or 0) > (existing.id or 0):
-            latest_by_key[dedupe_key] = record
-    return list(latest_by_key.values())
+async def _load_current_infrastructure_records(
+    db: AsyncSession,
+) -> tuple[list[CollectedData], list[CollectedData], list[CollectedData]]:
+    result = await db.execute(
+        select(CollectedData)
+        .where(
+            CollectedData.source.in_(
+                (
+                    "arcgis_landing_points",
+                    "arcgis_cable_landing_relation",
+                    "arcgis_cables",
+                )
+            )
+        )
+        .where(CollectedData.is_current.is_(True))
+        .order_by(CollectedData.source.asc(), CollectedData.id.desc())
+    )
+    grouped_records = {
+        "arcgis_landing_points": [],
+        "arcgis_cable_landing_relation": [],
+        "arcgis_cables": [],
+    }
+    for record in result.scalars().all():
+        grouped_records.setdefault(record.source, []).append(record)
+
+    return (
+        grouped_records["arcgis_landing_points"],
+        grouped_records["arcgis_cable_landing_relation"],
+        grouped_records["arcgis_cables"],
+    )
 
 
 async def infer_related_infrastructure(
@@ -75,19 +97,9 @@ async def infer_related_infrastructure(
     if not valid_regions:
         return {"related_cables": [], "related_ixps": []}
 
-    landing_result = await db.execute(
-        select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
+    landing_records, relation_records, cable_records = await _load_current_infrastructure_records(
+        db,
     )
-    relation_result = await db.execute(
-        select(CollectedData).where(CollectedData.source == "arcgis_cable_landing_relation")
-    )
-    cable_result = await db.execute(
-        select(CollectedData).where(CollectedData.source == "arcgis_cables")
-    )
-
-    landing_records = _dedupe_collected_records(list(landing_result.scalars().all()))
-    relation_records = _dedupe_collected_records(list(relation_result.scalars().all()))
-    cable_records = _dedupe_collected_records(list(cable_result.scalars().all()))
 
     city_to_cable_ids: dict[int, list[int]] = {}
     for relation in relation_records: