linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

fix: stabilize earth bgp geography and rendering

Browse Source
This commit is contained in:
linkong
2026-04-02 15:36:20 +08:00
parent 07e4f519a1
commit e5fec8ba3d
26 changed files with 1788 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

284
backend/app/api/v1/visualization.py
View File

@@ -5,6 +5,7 @@ Returns GeoJSON format compatible with Three.js, CesiumJS, and Unreal Cesium.
"""
from datetime import UTC, datetime
import math
from fastapi import APIRouter, HTTPException, Depends, Query
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, func
@@ -18,7 +19,8 @@ from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_incident import BGPIncident
from app.models.collected_data import CollectedData
from app.services.bgp_collectors import build_bgp_collector_coverage
from app.services.cable_graph import build_graph_from_data, CableGraph
from app.services.bgp_enrichment import _lookup_prefix_geography
from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance
from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
router = APIRouter()
@@ -316,11 +318,16 @@ def convert_gpu_cluster_to_geojson(records: List[CollectedData]) -> Dict[str, An
return {"type": "FeatureCollection", "features": features}
def convert_bgp_anomalies_to_geojson(records: List[BGPAnomaly]) -> Dict[str, Any]:
def convert_bgp_anomalies_to_geojson(
records: List[BGPAnomaly],
geography_hints: Optional[Dict[str, Dict[str, Any]]] = None,
) -> Dict[str, Any]:
features = []
geography_hints = geography_hints or {}
for record in records:
evidence = record.evidence or {}
hint = geography_hints.get(str(record.entity_key or record.id), {})
collectors = evidence.get("collectors") or record.peer_scope or []
if not collectors:
nested = evidence.get("events") or []
@@ -334,23 +341,6 @@ def convert_bgp_anomalies_to_geojson(records: List[BGPAnomaly]) -> Dict[str, Any
if not collectors:
collectors = []
collector = collectors[0] if collectors else None
location = None
if collector:
location = RIPE_RIS_COLLECTOR_COORDS.get(str(collector))
if location is None:
nested = evidence.get("events") or []
for item in nested:
collector_name = (item or {}).get("collector")
if collector_name and collector_name in RIPE_RIS_COLLECTOR_COORDS:
location = RIPE_RIS_COLLECTOR_COORDS[collector_name]
collector = collector_name
break
if location is None:
continue
as_path = []
if isinstance(evidence.get("as_path"), list):
as_path = evidence.get("as_path") or []
@@ -385,6 +375,28 @@ def convert_bgp_anomalies_to_geojson(records: List[BGPAnomaly]) -> Dict[str, Any
}
)
geography_regions = _normalize_geo_regions(hint.get("regions") or [])
geography_mode = hint.get("geography_mode") or "collector_centroid"
collector = collectors[0] if collectors else None
location = geography_regions[0] if geography_regions else None
if location is None and collector:
location = RIPE_RIS_COLLECTOR_COORDS.get(str(collector))
if location is None:
nested = evidence.get("events") or []
for item in nested:
collector_name = (item or {}).get("collector")
if collector_name and collector_name in RIPE_RIS_COLLECTOR_COORDS:
location = RIPE_RIS_COLLECTOR_COORDS[collector_name]
collector = collector_name
geography_mode = "collector_centroid"
break
if location is None:
continue
features.append(
{
"type": "Feature",
@@ -408,6 +420,7 @@ def convert_bgp_anomalies_to_geojson(records: List[BGPAnomaly]) -> Dict[str, Any
"collector_count": len(collectors) or 1,
"as_path": as_path,
"impacted_regions": impacted_regions,
"geography_mode": geography_mode,
"confidence": record.confidence,
"summary": record.summary,
"created_at": to_iso8601_utc(record.created_at),
@@ -418,6 +431,54 @@ def convert_bgp_anomalies_to_geojson(records: List[BGPAnomaly]) -> Dict[str, Any
return {"type": "FeatureCollection", "features": features}
async def build_anomaly_geography_hints(
db: AsyncSession,
records: List[BGPAnomaly],
) -> Dict[str, Dict[str, Any]]:
hints: Dict[str, Dict[str, Any]] = {}
for record in records:
evidence = record.evidence or {}
key = str(record.entity_key or record.id)
prefix_geo_regions = []
prefix_regions = []
asn_regions = []
evidence_prefix_geography = evidence.get("prefix_geography") or {}
prefix_geo_regions.extend(
_normalize_geo_regions(evidence_prefix_geography.get("regions") or [])
)
prefix_scope = evidence.get("prefix_scope") or {}
prefix_regions.extend(_normalize_geo_regions(prefix_scope.get("regions") or []))
for profile_key in ("origin_asn_profile", "new_origin_asn_profile"):
profile = evidence.get(profile_key) or {}
latitude = profile.get("latitude")
longitude = profile.get("longitude")
if isinstance(latitude, (int, float)) and isinstance(longitude, (int, float)):
asn_regions.append(
{
"country": profile.get("country"),
"city": profile.get("city"),
"latitude": float(latitude),
"longitude": float(longitude),
}
)
prefix_geo_regions = _normalize_geo_regions(prefix_geo_regions)
prefix_regions = _normalize_geo_regions(prefix_regions)
asn_regions = _normalize_geo_regions(asn_regions)
if prefix_geo_regions:
hints[key] = {"regions": prefix_geo_regions, "geography_mode": "prefix_geography"}
elif prefix_regions:
hints[key] = {"regions": prefix_regions, "geography_mode": "prefix_scope"}
elif asn_regions:
hints[key] = {"regions": asn_regions, "geography_mode": "asn_region"}
return hints
def convert_bgp_collectors_to_geojson(
coverage_by_collector: Dict[str, Dict[str, Any]] | None = None,
) -> Dict[str, Any]:
@@ -442,8 +503,10 @@ def convert_bgp_collectors_to_geojson(
"prefix_count": coverage.get("prefix_count", 0),
"origin_asn_count": coverage.get("origin_asn_count", 0),
"peer_asn_count": coverage.get("peer_asn_count", 0),
"recent_15m_observation_count": coverage.get("recent_15m_observation_count", 0),
"recent_24h_observation_count": coverage.get("recent_24h_observation_count", 0),
"recent_7d_observation_count": coverage.get("recent_7d_observation_count", 0),
"recent_15m_prefix_count": coverage.get("recent_15m_prefix_count", 0),
"recent_24h_prefix_count": coverage.get("recent_24h_prefix_count", 0),
"recent_7d_prefix_count": coverage.get("recent_7d_prefix_count", 0),
"top_event_types": coverage.get("top_event_types", []),
@@ -463,33 +526,183 @@ def convert_bgp_collectors_to_geojson(
return {"type": "FeatureCollection", "features": features}
def convert_bgp_incidents_to_geojson(records: List[BGPIncident]) -> Dict[str, Any]:
def _incident_estimated_center(valid_regions: List[Dict[str, Any]]) -> Dict[str, float]:
x = 0.0
y = 0.0
z = 0.0
for region in valid_regions:
lat_rad = math.radians(float(region["latitude"]))
lon_rad = math.radians(float(region["longitude"]))
x += math.cos(lat_rad) * math.cos(lon_rad)
y += math.cos(lat_rad) * math.sin(lon_rad)
z += math.sin(lat_rad)
total = float(len(valid_regions))
if total <= 0:
return {"latitude": 0.0, "longitude": 0.0}
x /= total
y /= total
z /= total
hyp = math.sqrt((x * x) + (y * y))
if hyp == 0:
return {"latitude": 0.0, "longitude": 0.0}
return {
"latitude": math.degrees(math.atan2(z, hyp)),
"longitude": math.degrees(math.atan2(y, x)),
}
def _incident_estimated_radius_km(center: Dict[str, float], valid_regions: List[Dict[str, Any]]) -> float:
center_coords = (float(center["longitude"]), float(center["latitude"]))
distances = [
haversine_distance(
center_coords,
(float(region["longitude"]), float(region["latitude"])),
)
for region in valid_regions
]
return round(max(distances) if distances else 0.0, 1)
def _normalize_geo_regions(regions: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
normalized: list[dict[str, Any]] = []
seen: set[tuple[Any, ...]] = set()
for region in regions:
if not isinstance(region, dict):
continue
latitude = region.get("latitude")
longitude = region.get("longitude")
if not isinstance(latitude, (int, float)) or not isinstance(longitude, (int, float)):
continue
item = {
"collector": region.get("collector"),
"country": region.get("country"),
"city": region.get("city"),
"latitude": float(latitude),
"longitude": float(longitude),
}
key = (
item["collector"],
item["country"],
item["city"],
item["latitude"],
item["longitude"],
)
if key in seen:
continue
seen.add(key)
normalized.append(item)
return normalized
async def build_incident_geography_hints(
db: AsyncSession,
records: List[BGPIncident],
) -> Dict[str, Dict[str, Any]]:
evidence_refs = sorted(
{
str(ref)
for record in records
for ref in (record.evidence_refs or [])
if ref
}
)
anomalies = []
if evidence_refs:
result = await db.execute(
select(BGPAnomaly).where(BGPAnomaly.entity_key.in_(evidence_refs))
)
anomalies = result.scalars().all()
anomaly_by_key = {
str(anomaly.entity_key): anomaly
for anomaly in anomalies
if anomaly.entity_key
}
hints: Dict[str, Dict[str, Any]] = {}
for record in records:
prefix_geo_regions: list[dict[str, Any]] = []
prefix_regions: list[dict[str, Any]] = []
asn_regions: list[dict[str, Any]] = []
for ref in record.evidence_refs or []:
anomaly = anomaly_by_key.get(str(ref))
if anomaly is None:
continue
evidence = anomaly.evidence or {}
if not prefix_geo_regions:
prefix_geography = evidence.get("prefix_geography") or {}
prefix_geo_regions.extend(_normalize_geo_regions(prefix_geography.get("regions") or []))
prefix_scope = evidence.get("prefix_scope") or {}
prefix_regions.extend(_normalize_geo_regions(prefix_scope.get("regions") or []))
for key in ("origin_asn_profile", "new_origin_asn_profile"):
profile = evidence.get(key) or {}
latitude = profile.get("latitude")
longitude = profile.get("longitude")
if isinstance(latitude, (int, float)) and isinstance(longitude, (int, float)):
asn_regions.append(
{
"country": profile.get("country"),
"city": profile.get("city"),
"latitude": float(latitude),
"longitude": float(longitude),
}
)
prefix_geo_regions = _normalize_geo_regions(prefix_geo_regions)
prefix_regions = _normalize_geo_regions(prefix_regions)
asn_regions = _normalize_geo_regions(asn_regions)
if prefix_geo_regions:
hints[record.incident_key] = {
"regions": prefix_geo_regions,
"geography_mode": "prefix_geography",
}
elif prefix_regions:
hints[record.incident_key] = {
"regions": prefix_regions,
"geography_mode": "prefix_scope",
}
elif asn_regions:
hints[record.incident_key] = {
"regions": asn_regions,
"geography_mode": "asn_region",
}
return hints
def convert_bgp_incidents_to_geojson(
records: List[BGPIncident],
geography_hints: Optional[Dict[str, Dict[str, Any]]] = None,
) -> Dict[str, Any]:
features = []
for record in records:
regions = record.affected_regions or []
hint = (geography_hints or {}).get(record.incident_key, {})
regions = hint.get("regions") or (record.affected_regions or [])
if not regions:
continue
valid_regions = [
region
for region in regions
if isinstance(region, dict)
and isinstance(region.get("latitude"), (int, float))
and isinstance(region.get("longitude"), (int, float))
]
valid_regions = _normalize_geo_regions(regions)
if not valid_regions:
continue
avg_lat = sum(float(region["latitude"]) for region in valid_regions) / len(valid_regions)
avg_lon = sum(float(region["longitude"]) for region in valid_regions) / len(valid_regions)
estimated_center = _incident_estimated_center(valid_regions)
estimated_radius_km = _incident_estimated_radius_km(estimated_center, valid_regions)
features.append(
{
"type": "Feature",
"geometry": {
"type": "Point",
"coordinates": [avg_lon, avg_lat],
"coordinates": [
estimated_center["longitude"],
estimated_center["latitude"],
],
},
"properties": {
"id": record.id,
@@ -504,6 +717,9 @@ def convert_bgp_incidents_to_geojson(records: List[BGPIncident]) -> Dict[str, An
"affected_asns": record.affected_asns or [],
"affected_collectors": record.affected_collectors or [],
"affected_regions": valid_regions,
"estimated_center": estimated_center,
"estimated_radius_km": estimated_radius_km,
"geography_mode": hint.get("geography_mode") or "collector_centroid",
"related_cables": record.related_cables or [],
"related_ixps": record.related_ixps or [],
"created_at": to_iso8601_utc(record.created_at),
@@ -739,7 +955,8 @@ async def get_bgp_anomalies_geojson(
result = await db.execute(stmt)
records = list(result.scalars().all())
geojson = convert_bgp_anomalies_to_geojson(records)
geography_hints = await build_anomaly_geography_hints(db, records)
geojson = convert_bgp_anomalies_to_geojson(records, geography_hints)
return {**geojson, "count": len(geojson.get("features", []))}
@@ -758,7 +975,8 @@ async def get_bgp_incidents_geojson(
result = await db.execute(stmt)
records = list(result.scalars().all())
geojson = convert_bgp_incidents_to_geojson(records)
geography_hints = await build_incident_geography_hints(db, records)
geojson = convert_bgp_incidents_to_geojson(records, geography_hints)
return {**geojson, "count": len(geojson.get("features", []))}