linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix: expand bgp pipeline and stabilize backend tests

Browse Source
This commit is contained in:
linkong
2026-03-30 16:13:36 +08:00
parent 2015ab79bd
commit 945786cee5
22 changed files with 1787 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
backend/app/api/v1/bgp.py
View File

@@ -8,7 +8,8 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.core.security import get_current_user
from app.db.session import get_db
from app.models.bgp_anomaly import BGPAnomaly
from app.models.collected_data import CollectedData
from app.models.bgp_incident import BGPIncident
from app.models.bgp_observation import BGPObservation
from app.models.user import User
router = APIRouter()
@@ -48,12 +49,12 @@ async def list_bgp_events(
db: AsyncSession = Depends(get_db),
):
stmt = (
select(CollectedData)
.where(CollectedData.source.in_(BGP_SOURCES))
.order_by(CollectedData.reference_date.desc().nullslast(), CollectedData.id.desc())
select(BGPObservation)
.where(BGPObservation.source.in_(BGP_SOURCES))
.order_by(BGPObservation.observed_at.desc(), BGPObservation.id.desc())
)
if source:
stmt = stmt.where(CollectedData.source == source)
stmt = stmt.where(BGPObservation.source == source)
result = await db.execute(stmt)
records = result.scalars().all()
@@ -62,18 +63,17 @@ async def list_bgp_events(
filtered = []
for record in records:
metadata = record.extra_data or {}
if prefix and metadata.get("prefix") != prefix:
if prefix and record.prefix != prefix:
continue
if origin_asn is not None and metadata.get("origin_asn") != origin_asn:
if origin_asn is not None and record.origin_asn != origin_asn:
continue
if peer_asn is not None and metadata.get("peer_asn") != peer_asn:
if peer_asn is not None and record.peer_asn != peer_asn:
continue
if collector and metadata.get("collector") != collector:
if collector and record.collector != collector:
continue
if event_type and metadata.get("event_type") != event_type:
if event_type and record.event_type != event_type:
continue
if (dt_from or dt_to) and not _matches_time(record.reference_date, dt_from, dt_to):
if (dt_from or dt_to) and not _matches_time(record.observed_at, dt_from, dt_to):
continue
filtered.append(record)
@@ -92,7 +92,7 @@ async def get_bgp_event(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
record = await db.get(CollectedData, event_id)
record = await db.get(BGPObservation, event_id)
if not record or record.source not in BGP_SOURCES:
raise HTTPException(status_code=404, detail="BGP event not found")
return record.to_dict()
@@ -180,3 +180,44 @@ async def get_bgp_anomaly(
if not record:
raise HTTPException(status_code=404, detail="BGP anomaly not found")
return record.to_dict()
@router.get("/incidents")
async def list_bgp_incidents(
severity: Optional[str] = Query(None),
incident_type: Optional[str] = Query(None),
status: Optional[str] = Query(None),
page: int = Query(1, ge=1),
page_size: int = Query(50, ge=1, le=200),
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
stmt = select(BGPIncident).order_by(BGPIncident.created_at.desc(), BGPIncident.id.desc())
if severity:
stmt = stmt.where(BGPIncident.severity == severity)
if incident_type:
stmt = stmt.where(BGPIncident.incident_type == incident_type)
if status:
stmt = stmt.where(BGPIncident.status == status)
result = await db.execute(stmt)
records = result.scalars().all()
offset = (page - 1) * page_size
return {
"total": len(records),
"page": page,
"page_size": page_size,
"data": [record.to_dict() for record in records[offset : offset + page_size]],
}
@router.get("/incidents/{incident_id}")
async def get_bgp_incident(
incident_id: int,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
record = await db.get(BGPIncident, incident_id)
if not record:
raise HTTPException(status_code=404, detail="BGP incident not found")
return record.to_dict()

79
backend/app/api/v1/visualization.py
View File

@@ -202,6 +202,44 @@ def convert_satellite_to_geojson(records: List[CollectedData]) -> Dict[str, Any]
return {"type": "FeatureCollection", "features": features}
def dedupe_satellite_records(records: List[CollectedData]) -> List[CollectedData]:
"""Keep only the newest record for each satellite identity."""
latest_by_key: Dict[str, CollectedData] = {}
for record in records:
metadata = record.extra_data or {}
norad_id = metadata.get("norad_cat_id")
dedupe_key = (
str(norad_id)
if norad_id not in (None, "")
else str(record.source_id or record.entity_key or record.name or record.id)
)
existing = latest_by_key.get(dedupe_key)
if existing is None or (record.id or 0) > (existing.id or 0):
latest_by_key[dedupe_key] = record
return sorted(latest_by_key.values(), key=lambda item: item.id or 0, reverse=True)
def dedupe_collected_records(records: List[CollectedData]) -> List[CollectedData]:
"""Keep only the newest record for each collected entity."""
latest_by_key: Dict[str, CollectedData] = {}
for record in records:
dedupe_key = str(
record.source_id
or record.entity_key
or record.name
or record.id
)
existing = latest_by_key.get(dedupe_key)
if existing is None or (record.id or 0) > (existing.id or 0):
latest_by_key[dedupe_key] = record
return sorted(latest_by_key.values(), key=lambda item: item.id or 0, reverse=True)
def convert_supercomputer_to_geojson(records: List[CollectedData]) -> Dict[str, Any]:
"""Convert TOP500 supercomputer records to GeoJSON"""
features = []
@@ -410,7 +448,7 @@ async def get_cables_geojson(db: AsyncSession = Depends(get_db)):
try:
stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
result = await db.execute(stmt)
records = result.scalars().all()
records = dedupe_collected_records(list(result.scalars().all()))
if not records:
raise HTTPException(
@@ -430,15 +468,15 @@ async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
try:
landing_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
landing_result = await db.execute(landing_stmt)
records = landing_result.scalars().all()
records = dedupe_collected_records(list(landing_result.scalars().all()))
relation_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cable_landing_relation")
relation_result = await db.execute(relation_stmt)
relation_records = relation_result.scalars().all()
relation_records = dedupe_collected_records(list(relation_result.scalars().all()))
cable_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
cable_result = await db.execute(cable_stmt)
cable_records = cable_result.scalars().all()
cable_records = dedupe_collected_records(list(cable_result.scalars().all()))
city_to_cable_ids_map = {}
for rel in relation_records:
@@ -476,15 +514,15 @@ async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
async def get_all_geojson(db: AsyncSession = Depends(get_db)):
cables_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
cables_result = await db.execute(cables_stmt)
cables_records = cables_result.scalars().all()
cables_records = dedupe_collected_records(list(cables_result.scalars().all()))
points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
points_result = await db.execute(points_stmt)
points_records = points_result.scalars().all()
points_records = dedupe_collected_records(list(points_result.scalars().all()))
relation_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cable_landing_relation")
relation_result = await db.execute(relation_stmt)
relation_records = relation_result.scalars().all()
relation_records = dedupe_collected_records(list(relation_result.scalars().all()))
city_to_cable_ids_map = {}
for rel in relation_records:
@@ -542,10 +580,11 @@ async def get_satellites_geojson(
.where(CollectedData.name != "Unknown")
.order_by(CollectedData.id.desc())
)
if limit is not None:
stmt = stmt.limit(limit)
result = await db.execute(stmt)
records = result.scalars().all()
records = dedupe_satellite_records(list(result.scalars().all()))
if limit is not None:
records = records[:limit]
if not records:
return {"type": "FeatureCollection", "features": [], "count": 0}
@@ -567,10 +606,11 @@ async def get_supercomputers_geojson(
select(CollectedData)
.where(CollectedData.source == "top500")
.where(CollectedData.name != "Unknown")
.limit(limit)
.order_by(CollectedData.id.desc())
)
result = await db.execute(stmt)
records = result.scalars().all()
records = dedupe_collected_records(list(result.scalars().all()))
records = records[:limit]
if not records:
return {"type": "FeatureCollection", "features": [], "count": 0}
@@ -592,10 +632,11 @@ async def get_gpu_clusters_geojson(
select(CollectedData)
.where(CollectedData.source == "epoch_ai_gpu")
.where(CollectedData.name != "Unknown")
.limit(limit)
.order_by(CollectedData.id.desc())
)
result = await db.execute(stmt)
records = result.scalars().all()
records = dedupe_collected_records(list(result.scalars().all()))
records = records[:limit]
if not records:
return {"type": "FeatureCollection", "features": [], "count": 0}
@@ -645,11 +686,11 @@ async def get_all_visualization_data(db: AsyncSession = Depends(get_db)):
"""
cables_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
cables_result = await db.execute(cables_stmt)
cables_records = list(cables_result.scalars().all())
cables_records = dedupe_collected_records(list(cables_result.scalars().all()))
points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
points_result = await db.execute(points_stmt)
points_records = list(points_result.scalars().all())
points_records = dedupe_collected_records(list(points_result.scalars().all()))
satellites_stmt = (
select(CollectedData)
@@ -657,7 +698,7 @@ async def get_all_visualization_data(db: AsyncSession = Depends(get_db)):
.where(CollectedData.name != "Unknown")
)
satellites_result = await db.execute(satellites_stmt)
satellites_records = list(satellites_result.scalars().all())
satellites_records = dedupe_satellite_records(list(satellites_result.scalars().all()))
supercomputers_stmt = (
select(CollectedData)
@@ -665,7 +706,7 @@ async def get_all_visualization_data(db: AsyncSession = Depends(get_db)):
.where(CollectedData.name != "Unknown")
)
supercomputers_result = await db.execute(supercomputers_stmt)
supercomputers_records = list(supercomputers_result.scalars().all())
supercomputers_records = dedupe_collected_records(list(supercomputers_result.scalars().all()))
gpu_stmt = (
select(CollectedData)
@@ -673,7 +714,7 @@ async def get_all_visualization_data(db: AsyncSession = Depends(get_db)):
.where(CollectedData.name != "Unknown")
)
gpu_result = await db.execute(gpu_stmt)
gpu_records = list(gpu_result.scalars().all())
gpu_records = dedupe_collected_records(list(gpu_result.scalars().all()))
cables = (
convert_cable_to_geojson(cables_records)