fix: ship persistent bgp ai briefs and optimize bgp queries

backend/app/api/v1/bgp.py

@@ -22,16 +22,161 @@ def _parse_dt(value: Optional[str]) -> Optional[datetime]:
     if not value:
         return None
     return datetime.fromisoformat(value.replace("Z", "+00:00"))
+def _event_filters(
+    *,
+    prefix: Optional[str],
+    origin_asn: Optional[int],
+    peer_asn: Optional[int],
+    collector: Optional[str],
+    event_type: Optional[str],
+    source: Optional[str],
+    time_from: Optional[datetime],
+    time_to: Optional[datetime],
+):
+    filters = [BGPObservation.source.in_(BGP_SOURCES)]
+    if source:
+        filters.append(BGPObservation.source == source)
+    if prefix:
+        filters.append(BGPObservation.prefix == prefix)
+    if origin_asn is not None:
+        filters.append(BGPObservation.origin_asn == origin_asn)
+    if peer_asn is not None:
+        filters.append(BGPObservation.peer_asn == peer_asn)
+    if collector:
+        filters.append(BGPObservation.collector == collector)
+    if event_type:
+        filters.append(BGPObservation.event_type == event_type)
+    if time_from:
+        filters.append(BGPObservation.observed_at >= time_from)
+    if time_to:
+        filters.append(BGPObservation.observed_at <= time_to)
+    return filters
 
 
-def _matches_time(value: Optional[datetime], time_from: Optional[datetime], time_to: Optional[datetime]) -> bool:
-    if value is None:
-        return False
-    if time_from and value < time_from:
-        return False
-    if time_to and value > time_to:
-        return False
-    return True
+def _anomaly_filters(
+    *,
+    severity: Optional[str],
+    anomaly_type: Optional[str],
+    status: Optional[str],
+    prefix: Optional[str],
+    origin_asn: Optional[int],
+    time_from: Optional[datetime],
+    time_to: Optional[datetime],
+):
+    filters = []
+    if severity:
+        filters.append(BGPAnomaly.severity == severity)
+    if anomaly_type:
+        filters.append(BGPAnomaly.anomaly_type == anomaly_type)
+    if status:
+        filters.append(BGPAnomaly.status == status)
+    if prefix:
+        filters.append(BGPAnomaly.prefix == prefix)
+    if origin_asn is not None:
+        filters.append(BGPAnomaly.origin_asn == origin_asn)
+    if time_from:
+        filters.append(BGPAnomaly.created_at >= time_from)
+    if time_to:
+        filters.append(BGPAnomaly.created_at <= time_to)
+    return filters
+
+
+def _incident_filters(
+    *,
+    severity: Optional[str],
+    incident_type: Optional[str],
+    status: Optional[str],
+):
+    filters = []
+    if severity:
+        filters.append(BGPIncident.severity == severity)
+    if incident_type:
+        filters.append(BGPIncident.incident_type == incident_type)
+    if status:
+        filters.append(BGPIncident.status == status)
+    return filters
+
+
+async def _build_event_summary_payload(db: AsyncSession) -> dict:
+    base_filters = [BGPObservation.source.in_(BGP_SOURCES)]
+
+    total_result = await db.execute(
+        select(func.count(BGPObservation.id)).where(*base_filters)
+    )
+    collectors_result = await db.execute(
+        select(func.count(func.distinct(BGPObservation.collector))).where(
+            *base_filters, BGPObservation.collector.isnot(None)
+        )
+    )
+    prefixes_result = await db.execute(
+        select(func.count(func.distinct(BGPObservation.prefix))).where(
+            *base_filters, BGPObservation.prefix.isnot(None)
+        )
+    )
+    type_result = await db.execute(
+        select(BGPObservation.event_type, func.count(BGPObservation.id))
+        .where(*base_filters)
+        .group_by(BGPObservation.event_type)
+    )
+
+    return {
+        "total": total_result.scalar() or 0,
+        "collector_count": collectors_result.scalar() or 0,
+        "prefix_count": prefixes_result.scalar() or 0,
+        "by_type": {row[0]: row[1] for row in type_result.fetchall()},
+    }
+
+
+async def _build_anomaly_summary_payload(db: AsyncSession) -> dict:
+    total_result = await db.execute(select(func.count(BGPAnomaly.id)))
+    type_result = await db.execute(
+        select(BGPAnomaly.anomaly_type, func.count(BGPAnomaly.id))
+        .group_by(BGPAnomaly.anomaly_type)
+        .order_by(func.count(BGPAnomaly.id).desc())
+    )
+    severity_result = await db.execute(
+        select(BGPAnomaly.severity, func.count(BGPAnomaly.id))
+        .group_by(BGPAnomaly.severity)
+        .order_by(func.count(BGPAnomaly.id).desc())
+    )
+    status_result = await db.execute(
+        select(BGPAnomaly.status, func.count(BGPAnomaly.id))
+        .group_by(BGPAnomaly.status)
+        .order_by(func.count(BGPAnomaly.id).desc())
+    )
+
+    return {
+        "total": total_result.scalar() or 0,
+        "by_type": {row[0]: row[1] for row in type_result.fetchall()},
+        "by_severity": {row[0]: row[1] for row in severity_result.fetchall()},
+        "by_status": {row[0]: row[1] for row in status_result.fetchall()},
+    }
+
+
+async def _build_incident_summary_payload(db: AsyncSession) -> dict:
+    total_result = await db.execute(select(func.count(BGPIncident.id)))
+    type_result = await db.execute(
+        select(BGPIncident.incident_type, func.count(BGPIncident.id))
+        .group_by(BGPIncident.incident_type)
+        .order_by(func.count(BGPIncident.id).desc())
+    )
+    severity_result = await db.execute(
+        select(BGPIncident.severity, func.count(BGPIncident.id))
+        .group_by(BGPIncident.severity)
+        .order_by(func.count(BGPIncident.id).desc())
+    )
+    status_result = await db.execute(
+        select(BGPIncident.status, func.count(BGPIncident.id))
+        .group_by(BGPIncident.status)
+        .order_by(func.count(BGPIncident.id).desc())
+    )
+
+    return {
+        "total": total_result.scalar() or 0,
+        "by_type": {row[0]: row[1] for row in type_result.fetchall()},
+        "by_severity": {row[0]: row[1] for row in severity_result.fetchall()},
+        "by_status": {row[0]: row[1] for row in status_result.fetchall()},
+    }
 
 
 @router.get("/events")
@@ -49,41 +194,36 @@ async def list_bgp_events(
     current_user: User = Depends(get_current_user),
     db: AsyncSession = Depends(get_db),
 ):
-    stmt = (
-        select(BGPObservation)
-        .where(BGPObservation.source.in_(BGP_SOURCES))
-        .order_by(BGPObservation.observed_at.desc(), BGPObservation.id.desc())
-    )
-    if source:
-        stmt = stmt.where(BGPObservation.source == source)
-
-    result = await db.execute(stmt)
-    records = result.scalars().all()
     dt_from = _parse_dt(time_from)
     dt_to = _parse_dt(time_to)
-
-    filtered = []
-    for record in records:
-        if prefix and record.prefix != prefix:
-            continue
-        if origin_asn is not None and record.origin_asn != origin_asn:
-            continue
-        if peer_asn is not None and record.peer_asn != peer_asn:
-            continue
-        if collector and record.collector != collector:
-            continue
-        if event_type and record.event_type != event_type:
-            continue
-        if (dt_from or dt_to) and not _matches_time(record.observed_at, dt_from, dt_to):
-            continue
-        filtered.append(record)
-
+    filters = _event_filters(
+        prefix=prefix,
+        origin_asn=origin_asn,
+        peer_asn=peer_asn,
+        collector=collector,
+        event_type=event_type,
+        source=source,
+        time_from=dt_from,
+        time_to=dt_to,
+    )
     offset = (page - 1) * page_size
+    count_result = await db.execute(
+        select(func.count(BGPObservation.id)).where(*filters)
+    )
+    data_result = await db.execute(
+        select(BGPObservation)
+        .where(*filters)
+        .order_by(BGPObservation.observed_at.desc(), BGPObservation.id.desc())
+        .offset(offset)
+        .limit(page_size)
+    )
+    records = data_result.scalars().all()
+
     return {
-        "total": len(filtered),
+        "total": count_result.scalar() or 0,
         "page": page,
         "page_size": page_size,
-        "data": [record.to_dict() for record in filtered[offset : offset + page_size]],
+        "data": [record.to_dict() for record in records],
     }
 
 
@@ -92,21 +232,7 @@ async def get_bgp_event_summary(
     current_user: User = Depends(get_current_user),
     db: AsyncSession = Depends(get_db),
 ):
-    result = await db.execute(select(BGPObservation).where(BGPObservation.source.in_(BGP_SOURCES)))
-    records = result.scalars().all()
-
-    collectors = sorted({record.collector for record in records if record.collector})
-    prefixes = sorted({record.prefix for record in records if record.prefix})
-    by_type: dict[str, int] = {}
-    for record in records:
-        by_type[record.event_type] = by_type.get(record.event_type, 0) + 1
-
-    return {
-        "total": len(records),
-        "collector_count": len(collectors),
-        "prefix_count": len(prefixes),
-        "by_type": by_type,
-    }
+    return await _build_event_summary_payload(db)
 
 
 @router.get("/collectors")
@@ -138,6 +264,32 @@ async def get_bgp_collector_summary(
     }
 
 
+@router.get("/overview/summary")
+async def get_bgp_overview_summary(
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    event_summary = await _build_event_summary_payload(db)
+    anomaly_summary = await _build_anomaly_summary_payload(db)
+    incident_summary = await _build_incident_summary_payload(db)
+    collectors = await build_bgp_collector_coverage(db, source_filter=BGP_SOURCES)
+    active_collectors = [item for item in collectors if item["observation_count"] > 0]
+
+    return {
+        "incidentSummary": incident_summary,
+        "anomalySummary": anomaly_summary,
+        "eventSummary": event_summary,
+        "collectorSummary": {
+            "total": len(collectors),
+            "active_collectors": len(active_collectors),
+            "observed_prefixes": sum(item["prefix_count"] for item in active_collectors),
+            "observed_origins": sum(item["origin_asn_count"] for item in active_collectors),
+            "recent_24h_events": sum(item["recent_24h_observation_count"] for item in active_collectors),
+            "recent_7d_events": sum(item["recent_7d_observation_count"] for item in active_collectors),
+        },
+    }
+
+
 @router.get("/events/{event_id}")
 async def get_bgp_event(
     event_id: int,
@@ -164,31 +316,35 @@ async def list_bgp_anomalies(
     current_user: User = Depends(get_current_user),
     db: AsyncSession = Depends(get_db),
 ):
-    stmt = select(BGPAnomaly).order_by(BGPAnomaly.created_at.desc(), BGPAnomaly.id.desc())
-    if severity:
-        stmt = stmt.where(BGPAnomaly.severity == severity)
-    if anomaly_type:
-        stmt = stmt.where(BGPAnomaly.anomaly_type == anomaly_type)
-    if status:
-        stmt = stmt.where(BGPAnomaly.status == status)
-    if prefix:
-        stmt = stmt.where(BGPAnomaly.prefix == prefix)
-    if origin_asn is not None:
-        stmt = stmt.where(BGPAnomaly.origin_asn == origin_asn)
-
-    result = await db.execute(stmt)
-    records = result.scalars().all()
     dt_from = _parse_dt(time_from)
     dt_to = _parse_dt(time_to)
-    if dt_from or dt_to:
-        records = [record for record in records if _matches_time(record.created_at, dt_from, dt_to)]
-
+    filters = _anomaly_filters(
+        severity=severity,
+        anomaly_type=anomaly_type,
+        status=status,
+        prefix=prefix,
+        origin_asn=origin_asn,
+        time_from=dt_from,
+        time_to=dt_to,
+    )
     offset = (page - 1) * page_size
+    total_result = await db.execute(
+        select(func.count(BGPAnomaly.id)).where(*filters)
+    )
+    data_result = await db.execute(
+        select(BGPAnomaly)
+        .where(*filters)
+        .order_by(BGPAnomaly.created_at.desc(), BGPAnomaly.id.desc())
+        .offset(offset)
+        .limit(page_size)
+    )
+    records = data_result.scalars().all()
+
     return {
-        "total": len(records),
+        "total": total_result.scalar() or 0,
         "page": page,
         "page_size": page_size,
-        "data": [record.to_dict() for record in records[offset : offset + page_size]],
+        "data": [record.to_dict() for record in records],
     }
 
 
@@ -197,29 +353,7 @@ async def get_bgp_anomaly_summary(
     current_user: User = Depends(get_current_user),
     db: AsyncSession = Depends(get_db),
 ):
-    total_result = await db.execute(select(func.count(BGPAnomaly.id)))
-    type_result = await db.execute(
-        select(BGPAnomaly.anomaly_type, func.count(BGPAnomaly.id))
-        .group_by(BGPAnomaly.anomaly_type)
-        .order_by(func.count(BGPAnomaly.id).desc())
-    )
-    severity_result = await db.execute(
-        select(BGPAnomaly.severity, func.count(BGPAnomaly.id))
-        .group_by(BGPAnomaly.severity)
-        .order_by(func.count(BGPAnomaly.id).desc())
-    )
-    status_result = await db.execute(
-        select(BGPAnomaly.status, func.count(BGPAnomaly.id))
-        .group_by(BGPAnomaly.status)
-        .order_by(func.count(BGPAnomaly.id).desc())
-    )
-
-    return {
-        "total": total_result.scalar() or 0,
-        "by_type": {row[0]: row[1] for row in type_result.fetchall()},
-        "by_severity": {row[0]: row[1] for row in severity_result.fetchall()},
-        "by_status": {row[0]: row[1] for row in status_result.fetchall()},
-    }
+    return await _build_anomaly_summary_payload(db)
 
 
 @router.get("/anomalies/{anomaly_id}")
@@ -244,22 +378,29 @@ async def list_bgp_incidents(
     current_user: User = Depends(get_current_user),
     db: AsyncSession = Depends(get_db),
 ):
-    stmt = select(BGPIncident).order_by(BGPIncident.created_at.desc(), BGPIncident.id.desc())
-    if severity:
-        stmt = stmt.where(BGPIncident.severity == severity)
-    if incident_type:
-        stmt = stmt.where(BGPIncident.incident_type == incident_type)
-    if status:
-        stmt = stmt.where(BGPIncident.status == status)
-
-    result = await db.execute(stmt)
-    records = result.scalars().all()
+    filters = _incident_filters(
+        severity=severity,
+        incident_type=incident_type,
+        status=status,
+    )
     offset = (page - 1) * page_size
+    total_result = await db.execute(
+        select(func.count(BGPIncident.id)).where(*filters)
+    )
+    data_result = await db.execute(
+        select(BGPIncident)
+        .where(*filters)
+        .order_by(BGPIncident.created_at.desc(), BGPIncident.id.desc())
+        .offset(offset)
+        .limit(page_size)
+    )
+    records = data_result.scalars().all()
+
     return {
-        "total": len(records),
+        "total": total_result.scalar() or 0,
         "page": page,
         "page_size": page_size,
-        "data": [record.to_dict() for record in records[offset : offset + page_size]],
+        "data": [record.to_dict() for record in records],
     }
 
 
@@ -268,29 +409,7 @@ async def get_bgp_incident_summary(
     current_user: User = Depends(get_current_user),
     db: AsyncSession = Depends(get_db),
 ):
-    total_result = await db.execute(select(func.count(BGPIncident.id)))
-    type_result = await db.execute(
-        select(BGPIncident.incident_type, func.count(BGPIncident.id))
-        .group_by(BGPIncident.incident_type)
-        .order_by(func.count(BGPIncident.id).desc())
-    )
-    severity_result = await db.execute(
-        select(BGPIncident.severity, func.count(BGPIncident.id))
-        .group_by(BGPIncident.severity)
-        .order_by(func.count(BGPIncident.id).desc())
-    )
-    status_result = await db.execute(
-        select(BGPIncident.status, func.count(BGPIncident.id))
-        .group_by(BGPIncident.status)
-        .order_by(func.count(BGPIncident.id).desc())
-    )
-
-    return {
-        "total": total_result.scalar() or 0,
-        "by_type": {row[0]: row[1] for row in type_result.fetchall()},
-        "by_severity": {row[0]: row[1] for row in severity_result.fetchall()},
-        "by_status": {row[0]: row[1] for row in status_result.fetchall()},
-    }
+    return await _build_incident_summary_payload(db)
 
 
 @router.get("/incidents/{incident_id}")