feat: expand bgp observability surfaces
This commit is contained in:
linkong
162
backend/app/services/bgp_collectors.py
Normal file
162
backend/app/services/bgp_collectors.py
Normal file
@@ -0,0 +1,162 @@
|
||||
"""Collector baseline and coverage helpers for BGP observations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.models.bgp_observation import BGPObservation
|
||||
from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
|
||||
|
||||
|
||||
async def build_bgp_collector_coverage(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
source_filter: tuple[str, ...] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
now = datetime.now(UTC)
|
||||
recent_24h_threshold = now - timedelta(hours=24)
|
||||
recent_7d_threshold = now - timedelta(days=7)
|
||||
|
||||
stmt = select(BGPObservation).order_by(BGPObservation.observed_at.desc(), BGPObservation.id.desc())
|
||||
if source_filter:
|
||||
stmt = stmt.where(BGPObservation.source.in_(source_filter))
|
||||
|
||||
result = await db.execute(stmt)
|
||||
records = list(result.scalars().all())
|
||||
|
||||
by_collector: dict[str, dict[str, Any]] = {}
|
||||
for record in records:
|
||||
collector = str(record.collector or "").strip()
|
||||
if not collector:
|
||||
continue
|
||||
|
||||
coverage = by_collector.get(collector)
|
||||
if coverage is None:
|
||||
location = record.collector_geo or RIPE_RIS_COLLECTOR_COORDS.get(collector, {})
|
||||
coverage = {
|
||||
"collector": collector,
|
||||
"city": location.get("city"),
|
||||
"country": location.get("country"),
|
||||
"latitude": location.get("latitude"),
|
||||
"longitude": location.get("longitude"),
|
||||
"observation_count": 0,
|
||||
"prefixes": set(),
|
||||
"origin_asns": set(),
|
||||
"peer_asns": set(),
|
||||
"event_types": defaultdict(int),
|
||||
"countries": set(),
|
||||
"cities": set(),
|
||||
"recent_24h_observation_count": 0,
|
||||
"recent_7d_observation_count": 0,
|
||||
"recent_24h_prefixes": set(),
|
||||
"recent_7d_prefixes": set(),
|
||||
"latest_observed_at": None,
|
||||
"latest_event_type": None,
|
||||
}
|
||||
by_collector[collector] = coverage
|
||||
|
||||
coverage["observation_count"] += 1
|
||||
if record.prefix:
|
||||
coverage["prefixes"].add(record.prefix)
|
||||
if record.origin_asn is not None:
|
||||
coverage["origin_asns"].add(record.origin_asn)
|
||||
if record.peer_asn is not None:
|
||||
coverage["peer_asns"].add(record.peer_asn)
|
||||
if record.event_type:
|
||||
coverage["event_types"][record.event_type] += 1
|
||||
|
||||
observed_at = record.observed_at
|
||||
if observed_at is not None:
|
||||
aware_observed_at = (
|
||||
observed_at.astimezone(UTC)
|
||||
if observed_at.tzinfo
|
||||
else observed_at.replace(tzinfo=UTC)
|
||||
)
|
||||
if aware_observed_at >= recent_24h_threshold:
|
||||
coverage["recent_24h_observation_count"] += 1
|
||||
if record.prefix:
|
||||
coverage["recent_24h_prefixes"].add(record.prefix)
|
||||
if aware_observed_at >= recent_7d_threshold:
|
||||
coverage["recent_7d_observation_count"] += 1
|
||||
if record.prefix:
|
||||
coverage["recent_7d_prefixes"].add(record.prefix)
|
||||
|
||||
geo = record.collector_geo or {}
|
||||
if geo.get("country"):
|
||||
coverage["countries"].add(geo["country"])
|
||||
if geo.get("city"):
|
||||
coverage["cities"].add(geo["city"])
|
||||
|
||||
current_latest = coverage["latest_observed_at"]
|
||||
if current_latest is None or (
|
||||
record.observed_at is not None and record.observed_at > current_latest
|
||||
):
|
||||
coverage["latest_observed_at"] = record.observed_at
|
||||
coverage["latest_event_type"] = record.event_type
|
||||
|
||||
for collector, location in RIPE_RIS_COLLECTOR_COORDS.items():
|
||||
if collector in by_collector:
|
||||
continue
|
||||
by_collector[collector] = {
|
||||
"collector": collector,
|
||||
"city": location.get("city"),
|
||||
"country": location.get("country"),
|
||||
"latitude": location.get("latitude"),
|
||||
"longitude": location.get("longitude"),
|
||||
"observation_count": 0,
|
||||
"prefixes": set(),
|
||||
"origin_asns": set(),
|
||||
"peer_asns": set(),
|
||||
"event_types": defaultdict(int),
|
||||
"countries": {location.get("country")} if location.get("country") else set(),
|
||||
"cities": {location.get("city")} if location.get("city") else set(),
|
||||
"recent_24h_observation_count": 0,
|
||||
"recent_7d_observation_count": 0,
|
||||
"recent_24h_prefixes": set(),
|
||||
"recent_7d_prefixes": set(),
|
||||
"latest_observed_at": None,
|
||||
"latest_event_type": None,
|
||||
}
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
for collector in sorted(by_collector.keys()):
|
||||
item = by_collector[collector]
|
||||
top_event_types = sorted(
|
||||
item["event_types"].items(),
|
||||
key=lambda pair: (-pair[1], pair[0]),
|
||||
)
|
||||
results.append(
|
||||
{
|
||||
"collector": item["collector"],
|
||||
"city": item["city"],
|
||||
"country": item["country"],
|
||||
"latitude": item["latitude"],
|
||||
"longitude": item["longitude"],
|
||||
"observation_count": item["observation_count"],
|
||||
"prefix_count": len(item["prefixes"]),
|
||||
"origin_asn_count": len(item["origin_asns"]),
|
||||
"peer_asn_count": len(item["peer_asns"]),
|
||||
"recent_24h_observation_count": item["recent_24h_observation_count"],
|
||||
"recent_7d_observation_count": item["recent_7d_observation_count"],
|
||||
"recent_24h_prefix_count": len(item["recent_24h_prefixes"]),
|
||||
"recent_7d_prefix_count": len(item["recent_7d_prefixes"]),
|
||||
"top_event_types": [
|
||||
{"event_type": event_type, "count": count}
|
||||
for event_type, count in top_event_types[:3]
|
||||
],
|
||||
"latest_observed_at": to_iso8601_utc(item["latest_observed_at"]),
|
||||
"latest_event_type": item["latest_event_type"],
|
||||
"baseline_scope": {
|
||||
"countries": sorted(country for country in item["countries"] if country),
|
||||
"cities": sorted(city for city in item["cities"] if city),
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
return results
|
||||
@@ -3,12 +3,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.collected_data_fields import get_record_field
|
||||
from app.models.bgp_anomaly import BGPAnomaly
|
||||
from app.models.bgp_incident import BGPIncident
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.services.cable_graph import haversine_distance
|
||||
|
||||
|
||||
def _severity_rank(value: str | None) -> int:
|
||||
@@ -44,6 +48,150 @@ def _collector_regions_from_anomaly(anomaly: BGPAnomaly) -> list[dict]:
|
||||
return collected
|
||||
|
||||
|
||||
def _dedupe_collected_records(records: list[CollectedData]) -> list[CollectedData]:
|
||||
latest_by_key: dict[str, CollectedData] = {}
|
||||
for record in records:
|
||||
dedupe_key = str(record.source_id or record.entity_key or record.name or record.id)
|
||||
existing = latest_by_key.get(dedupe_key)
|
||||
if existing is None or (record.id or 0) > (existing.id or 0):
|
||||
latest_by_key[dedupe_key] = record
|
||||
return list(latest_by_key.values())
|
||||
|
||||
|
||||
async def infer_related_infrastructure(
|
||||
db: AsyncSession,
|
||||
affected_regions: list[dict],
|
||||
*,
|
||||
max_matches: int = 6,
|
||||
max_distance_km: float = 450.0,
|
||||
) -> dict[str, list[dict[str, Any]]]:
|
||||
valid_regions = [
|
||||
region
|
||||
for region in affected_regions
|
||||
if isinstance(region, dict)
|
||||
and isinstance(region.get("latitude"), (int, float))
|
||||
and isinstance(region.get("longitude"), (int, float))
|
||||
]
|
||||
if not valid_regions:
|
||||
return {"related_cables": [], "related_ixps": []}
|
||||
|
||||
landing_result = await db.execute(
|
||||
select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
|
||||
)
|
||||
relation_result = await db.execute(
|
||||
select(CollectedData).where(CollectedData.source == "arcgis_cable_landing_relation")
|
||||
)
|
||||
cable_result = await db.execute(
|
||||
select(CollectedData).where(CollectedData.source == "arcgis_cables")
|
||||
)
|
||||
|
||||
landing_records = _dedupe_collected_records(list(landing_result.scalars().all()))
|
||||
relation_records = _dedupe_collected_records(list(relation_result.scalars().all()))
|
||||
cable_records = _dedupe_collected_records(list(cable_result.scalars().all()))
|
||||
|
||||
city_to_cable_ids: dict[int, list[int]] = {}
|
||||
for relation in relation_records:
|
||||
metadata = relation.extra_data or {}
|
||||
city_id = metadata.get("city_id")
|
||||
cable_id = metadata.get("cable_id")
|
||||
if city_id is None or cable_id is None:
|
||||
continue
|
||||
city_key = int(city_id)
|
||||
cable_key = int(cable_id)
|
||||
city_to_cable_ids.setdefault(city_key, [])
|
||||
if cable_key not in city_to_cable_ids[city_key]:
|
||||
city_to_cable_ids[city_key].append(cable_key)
|
||||
|
||||
cable_id_to_name: dict[int, str] = {}
|
||||
for cable in cable_records:
|
||||
metadata = cable.extra_data or {}
|
||||
cable_id = metadata.get("cable_id")
|
||||
if cable_id is None or not cable.name:
|
||||
continue
|
||||
cable_id_to_name[int(cable_id)] = cable.name
|
||||
|
||||
matches: list[dict[str, Any]] = []
|
||||
seen_match_keys: set[tuple[Any, ...]] = set()
|
||||
|
||||
for region in valid_regions:
|
||||
region_coords = (float(region["longitude"]), float(region["latitude"]))
|
||||
|
||||
for landing in landing_records:
|
||||
try:
|
||||
latitude = get_record_field(landing, "latitude")
|
||||
longitude = get_record_field(landing, "longitude")
|
||||
landing_lat = float(latitude) if latitude is not None else None
|
||||
landing_lon = float(longitude) if longitude is not None else None
|
||||
except (TypeError, ValueError):
|
||||
landing_lat = None
|
||||
landing_lon = None
|
||||
|
||||
if landing_lat is None or landing_lon is None:
|
||||
continue
|
||||
|
||||
distance_km = haversine_distance(region_coords, (landing_lon, landing_lat))
|
||||
if distance_km > max_distance_km:
|
||||
continue
|
||||
|
||||
landing_meta = landing.extra_data or {}
|
||||
city_id = landing_meta.get("city_id")
|
||||
cable_names = []
|
||||
if city_id is not None:
|
||||
for cable_id in city_to_cable_ids.get(int(city_id), []):
|
||||
cable_name = cable_id_to_name.get(int(cable_id))
|
||||
if cable_name and cable_name not in cable_names:
|
||||
cable_names.append(cable_name)
|
||||
|
||||
match = {
|
||||
"landing_point": landing.name or "Unknown",
|
||||
"city": get_record_field(landing, "city"),
|
||||
"country": get_record_field(landing, "country"),
|
||||
"distance_km": round(distance_km, 1),
|
||||
"collector": region.get("collector"),
|
||||
"cable_names": cable_names,
|
||||
}
|
||||
match_key = (
|
||||
match["landing_point"],
|
||||
match["city"],
|
||||
match["country"],
|
||||
)
|
||||
if match_key in seen_match_keys:
|
||||
continue
|
||||
seen_match_keys.add(match_key)
|
||||
matches.append(match)
|
||||
|
||||
matches.sort(
|
||||
key=lambda item: (
|
||||
item.get("distance_km", 999999),
|
||||
str(item.get("landing_point") or ""),
|
||||
)
|
||||
)
|
||||
matches = matches[:max_matches]
|
||||
|
||||
related_ixps = []
|
||||
seen_ixp_keys: set[tuple[str, str]] = set()
|
||||
for item in matches:
|
||||
city = str(item.get("city") or "").strip()
|
||||
country = str(item.get("country") or "").strip()
|
||||
if not city and not country:
|
||||
continue
|
||||
key = (city, country)
|
||||
if key in seen_ixp_keys:
|
||||
continue
|
||||
seen_ixp_keys.add(key)
|
||||
related_ixps.append(
|
||||
{
|
||||
"name": ", ".join(part for part in [city, country] if part),
|
||||
"type": "regional_exchange_hint",
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"related_cables": matches,
|
||||
"related_ixps": related_ixps,
|
||||
}
|
||||
|
||||
|
||||
async def create_bgp_incidents_for_anomalies(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
@@ -120,6 +268,7 @@ async def create_bgp_incidents_for_anomalies(
|
||||
f"{len(items)} anomaly signal(s) grouped into one {primary.anomaly_type} incident, "
|
||||
f"affecting {len(prefixes) or 1} prefix scope(s) across {len(collectors)} collector(s)."
|
||||
)
|
||||
related_infrastructure = await infer_related_infrastructure(db, regions)
|
||||
|
||||
db.add(
|
||||
BGPIncident(
|
||||
@@ -138,8 +287,8 @@ async def create_bgp_incidents_for_anomalies(
|
||||
affected_asns=asns,
|
||||
affected_collectors=collectors,
|
||||
affected_regions=regions,
|
||||
related_cables=[],
|
||||
related_ixps=[],
|
||||
related_cables=related_infrastructure["related_cables"],
|
||||
related_ixps=related_infrastructure["related_ixps"],
|
||||
evidence_refs=evidence_refs,
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user