linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix: stabilize earth bgp geography and rendering

Browse Source
This commit is contained in:
linkong
2026-04-02 15:36:20 +08:00
parent 07e4f519a1
commit e5fec8ba3d
26 changed files with 1788 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

371
backend/tests/test_bgp.py
View File

@@ -13,6 +13,8 @@ from app.main import app
from app.services.bgp_detectors import (
detect_mass_withdrawal_anomalies,
detect_origin_change_anomalies,
detect_path_flap_anomalies,
detect_route_leak_anomalies,
)
from app.services.collectors.bgp_common import (
create_bgp_anomalies_for_batch,
@@ -24,6 +26,7 @@ from app.services.bgp_incidents import (
infer_related_infrastructure,
)
from app.services.bgp_collectors import build_bgp_collector_coverage
from app.api.v1.visualization import convert_bgp_incidents_to_geojson, build_incident_geography_hints
from app.models.bgp_anomaly import BGPAnomaly
from app.models.collected_data import CollectedData
from app.models.bgp_incident import BGPIncident
@@ -31,6 +34,7 @@ from app.models.bgp_observation import BGPObservation
from app.models.user import User
from app.services.collectors.bgp_common import normalize_bgp_event
from app.services.collectors.bgpstream import BGPStreamBackfillCollector
from app.services.collectors.iptoasn import IPtoASNPrefixGeoCollector
class _FakeScalarResult:
@@ -51,6 +55,9 @@ class _FakeResult:
def fetchall(self):
return self._rows
def fetchone(self):
return self._rows[0] if self._rows else None
class _FakeAsyncSession:
def __init__(self, results, gets=None):
@@ -59,7 +66,7 @@ class _FakeAsyncSession:
self.added = []
self.commits = 0
async def execute(self, _stmt):
async def execute(self, _stmt, _params=None):
if not self._results:
return _FakeResult([])
return _FakeResult(self._results.pop(0))
@@ -140,6 +147,29 @@ def test_bgpstream_transform_preserves_broker_record():
assert record["metadata"]["broker_record"]["filename"] == "rib.20260326.0800.gz"
def test_iptoasn_transform_creates_prefix_geography_records():
collector = IPtoASNPrefixGeoCollector()
transformed = collector.transform(
[
{
"range_start": "1.0.0.0",
"range_end": "1.0.0.255",
"asn": "13335",
"country_code": "AU",
"as_name": "CLOUDFLARENET",
}
]
)
assert len(transformed) == 1
record = transformed[0]
assert record["name"] == "1.0.0.0/24"
assert record["metadata"]["family"] == "ipv4"
assert record["metadata"]["country_code"] == "AU"
assert record["metadata"]["asn"] == 13335
assert record["metadata"]["source_dataset"] == "iptoasn_combined"
def test_bgp_anomaly_to_dict():
anomaly = BGPAnomaly(
source="ris_live_bgp",
@@ -315,6 +345,75 @@ def test_detect_mass_withdrawal_anomalies_accepts_cross_collector_pair():
assert anomalies[0].evidence["collector_count"] == 2
def test_detect_route_leak_anomalies_creates_candidate_for_divergent_long_paths():
events = [
{
"metadata": {
"collector": "rrc00",
"event_type": "announcement",
"prefix": "203.0.113.0/24",
"origin_asn": 64496,
"as_path": [64500, 64496],
"collector_location": {"country": "NL", "city": "Amsterdam", "latitude": 52.3, "longitude": 4.9},
"enrichment": {"prefix_scope": {"regions": [{"country": "NL"}]}},
}
},
{
"metadata": {
"collector": "rrc01",
"event_type": "announcement",
"prefix": "203.0.113.0/24",
"origin_asn": 64496,
"as_path": [64510, 64520, 64530, 64540, 64496],
"collector_location": {"country": "GB", "city": "London", "latitude": 51.5, "longitude": -0.1},
"enrichment": {"prefix_scope": {"regions": [{"country": "GB"}]}},
}
},
]
anomalies = detect_route_leak_anomalies(
source="ris_live_bgp",
snapshot_id=1,
task_id=2,
events=events,
)
assert len(anomalies) == 1
assert anomalies[0].anomaly_type == "route_leak_candidate"
assert anomalies[0].evidence["max_path_length"] == 5
def test_detect_path_flap_anomalies_creates_signal_for_repeated_state_changes():
base_timestamp = datetime(2026, 3, 27, 0, 0, tzinfo=UTC)
events = []
for index, event_type in enumerate(["announcement", "withdrawal", "announcement", "withdrawal"]):
events.append(
{
"metadata": {
"collector": "rrc00",
"event_type": event_type,
"timestamp": (base_timestamp + timedelta(minutes=index)).isoformat(),
"prefix": "198.51.100.0/24",
"origin_asn": 64512,
"as_path": [64500 + index, 64512] if event_type == "announcement" else [],
"collector_location": {"country": "NL", "city": "Amsterdam", "latitude": 52.3, "longitude": 4.9},
"enrichment": {"prefix_scope": {"regions": [{"country": "NL"}]}},
}
}
)
anomalies = detect_path_flap_anomalies(
source="ris_live_bgp",
snapshot_id=1,
task_id=2,
events=events,
)
assert len(anomalies) == 1
assert anomalies[0].anomaly_type == "path_flap"
assert anomalies[0].evidence["transitions"] == 3
def test_bgp_incident_to_dict():
incident = BGPIncident(
source="ris_live_bgp",
@@ -338,6 +437,131 @@ def test_bgp_incident_to_dict():
assert data["affected_collectors"] == ["rrc00", "rrc01"]
def test_convert_bgp_incidents_to_geojson_adds_estimated_geography():
incident = BGPIncident(
source="ris_live_bgp",
incident_key="origin_change:203.0.113.0/24:64497",
incident_type="origin_change",
title="Origin Change incident on 203.0.113.0/24",
summary="Grouped incident summary",
severity="critical",
status="active",
confidence=0.91,
affected_prefixes=["203.0.113.0/24"],
affected_collectors=["rrc00", "rrc01"],
affected_regions=[
{
"collector": "rrc00",
"country": "Netherlands",
"city": "Amsterdam",
"latitude": 52.3676,
"longitude": 4.9041,
},
{
"collector": "rrc01",
"country": "United Kingdom",
"city": "London",
"latitude": 51.5072,
"longitude": -0.1276,
},
],
)
payload = convert_bgp_incidents_to_geojson([incident])
feature = payload["features"][0]
assert feature["properties"]["geography_mode"] == "collector_centroid"
assert feature["properties"]["estimated_radius_km"] > 0
assert feature["properties"]["estimated_center"]["latitude"] != 0
def test_convert_bgp_incidents_to_geojson_prefers_prefix_scope_hint():
incident = BGPIncident(
source="ris_live_bgp",
incident_key="origin_change:203.0.113.0/24:64497",
incident_type="origin_change",
title="Origin Change incident on 203.0.113.0/24",
summary="Grouped incident summary",
severity="critical",
status="active",
confidence=0.91,
affected_prefixes=["203.0.113.0/24"],
affected_collectors=["rrc00"],
affected_regions=[
{
"collector": "rrc00",
"country": "Netherlands",
"city": "Amsterdam",
"latitude": 52.3676,
"longitude": 4.9041,
}
],
)
payload = convert_bgp_incidents_to_geojson(
[incident],
{
incident.incident_key: {
"geography_mode": "prefix_scope",
"regions": [
{
"country": "Japan",
"city": "Tokyo",
"latitude": 35.6764,
"longitude": 139.65,
}
],
}
},
)
feature = payload["features"][0]
assert feature["properties"]["geography_mode"] == "prefix_scope"
assert feature["geometry"]["coordinates"] == [139.65, 35.6764]
def test_convert_bgp_incidents_to_geojson_prefers_prefix_geography_hint():
incident = BGPIncident(
source="ris_live_bgp",
incident_key="origin_change:198.51.100.0/24:64512",
incident_type="origin_change",
title="Origin Change incident on 198.51.100.0/24",
summary="Grouped incident summary",
severity="critical",
status="active",
confidence=0.91,
affected_prefixes=["198.51.100.0/24"],
affected_collectors=["rrc00"],
affected_regions=[
{
"collector": "rrc00",
"country": "Netherlands",
"city": "Amsterdam",
"latitude": 52.3676,
"longitude": 4.9041,
}
],
)
payload = convert_bgp_incidents_to_geojson(
[incident],
{
incident.incident_key: {
"geography_mode": "prefix_geography",
"regions": [
{
"country": "日本",
"city": None,
"latitude": 35.6764,
"longitude": 139.65,
}
],
}
},
)
feature = payload["features"][0]
assert feature["properties"]["geography_mode"] == "prefix_geography"
assert feature["geometry"]["coordinates"] == [139.65, 35.6764]
@pytest.mark.asyncio
async def test_enrich_bgp_events_for_batch_adds_profiles_and_prefix_scope():
historical_observation = BGPObservation(
@@ -367,8 +591,21 @@ async def test_enrich_bgp_events_for_batch_adds_profiles_and_prefix_scope():
},
)
peeringdb_record.id = 99
iptoasn_row = {
"extra_data": {
"family": "ipv4",
"range_start": "203.0.113.0",
"range_end": "203.0.113.255",
"prefix": "203.0.113.0/24",
"country": "英国",
"country_code": "GB",
"asn": 64497,
"as_name": "Example ASN",
"source_dataset": "iptoasn_combined",
}
}
db = _FakeAsyncSession([[historical_observation], [peeringdb_record]])
db = _FakeAsyncSession([[historical_observation], [iptoasn_row], [peeringdb_record]])
events = [
{
"metadata": {
@@ -396,8 +633,10 @@ async def test_enrich_bgp_events_for_batch_adds_profiles_and_prefix_scope():
assert enrichment["is_new_origin_for_prefix"] is True
assert enrichment["rpki_validation"]["status"] == "unknown"
assert enrichment["origin_asn_profile"]["name"] == "ExampleNet"
assert enrichment["prefix_scope"]["countries"] == ["Netherlands", "United Kingdom"]
assert enrichment["prefix_scope"]["cities"] == ["Amsterdam", "London"]
assert enrichment["prefix_geography"]["country"] == "英国"
assert enrichment["prefix_geography"]["source"] == "iptoasn_combined"
assert enrichment["prefix_scope"]["countries"] == ["United Kingdom"]
assert enrichment["prefix_scope"]["cities"] == ["London"]
@pytest.mark.asyncio
@@ -448,6 +687,126 @@ async def test_create_bgp_incidents_for_anomalies_aggregates_regions_and_collect
assert incident.affected_regions[0]["city"] == "Amsterdam"
@pytest.mark.asyncio
async def test_create_bgp_incidents_for_anomalies_refreshes_existing_incident():
existing = BGPIncident(
source="ris_live_bgp",
incident_key="origin_change:203.0.113.0/24:64497",
incident_type="origin_change",
title="Old title",
summary="Old summary",
severity="medium",
status="active",
confidence=0.4,
affected_prefixes=["203.0.113.0/24"],
affected_asns=[64496, 64497],
affected_collectors=["rrc00"],
affected_regions=[
{
"collector": "rrc00",
"country": "Netherlands",
"city": "Amsterdam",
"latitude": 52.3676,
"longitude": 4.9041,
}
],
related_cables=[],
related_ixps=[],
evidence_refs=["old-key"],
)
db = _FakeAsyncSession([[existing]])
anomaly = BGPAnomaly(
source="ris_live_bgp",
anomaly_type="origin_change",
severity="critical",
status="active",
entity_key="origin_change:203.0.113.0/24:64497",
prefix="203.0.113.0/24",
origin_asn=64496,
new_origin_asn=64497,
summary="Origin ASN changed",
confidence=0.9,
evidence={
"impacted_regions": [
{
"collector": None,
"country": "United States",
"city": None,
"latitude": 39.8283,
"longitude": -98.5795,
}
]
},
)
with patch(
"app.services.bgp_incidents.infer_related_infrastructure",
new=AsyncMock(return_value={"related_cables": [{"landing_point": "NYC"}], "related_ixps": []}),
):
created = await create_bgp_incidents_for_anomalies(
db,
source="ris_live_bgp",
snapshot_id=1,
task_id=2,
anomalies=[anomaly],
)
assert created == 0
assert db.commits == 1
assert len(db.added) == 0
assert existing.summary != "Old summary"
assert existing.severity == "critical"
assert existing.confidence == 0.9
assert existing.affected_regions[0]["country"] == "United States"
assert existing.related_cables == [{"landing_point": "NYC"}]
assert existing.evidence_refs == ["origin_change:203.0.113.0/24:64497"]
@pytest.mark.asyncio
async def test_build_incident_geography_hints_prefers_evidence_prefix_scope_when_no_cached_prefix_geo():
incident = BGPIncident(
source="ris_live_bgp",
incident_key="origin_change:93.175.153.0/24:16509",
incident_type="origin_change",
title="Origin Change incident on 93.175.153.0/24",
summary="summary",
severity="critical",
status="active",
affected_prefixes=["93.175.153.0/24"],
evidence_refs=["origin_change:93.175.153.0/24:16509"],
)
anomaly = BGPAnomaly(
source="ris_live_bgp",
anomaly_type="origin_change",
severity="critical",
status="active",
entity_key="origin_change:93.175.153.0/24:16509",
prefix="93.175.153.0/24",
origin_asn=12654,
new_origin_asn=16509,
summary="summary",
confidence=0.8,
evidence={
"prefix_scope": {
"regions": [
{
"country": "Netherlands",
"city": "Amsterdam",
"latitude": 52.3676,
"longitude": 4.9041,
}
]
}
},
)
db = _FakeAsyncSession([[anomaly]])
hints = await build_incident_geography_hints(db, [incident])
assert hints["origin_change:93.175.153.0/24:16509"]["geography_mode"] == "prefix_scope"
assert hints["origin_change:93.175.153.0/24:16509"]["regions"][0]["country"] == "Netherlands"
@pytest.mark.asyncio
async def test_infer_related_infrastructure_links_nearby_cables():
landing = CollectedData(
@@ -524,9 +883,11 @@ async def test_build_bgp_collector_coverage_summarizes_observations():
first = next(item for item in coverage if item["collector"] == "rrc00")
assert first["observation_count"] == 2
assert first["recent_15m_observation_count"] == 2
assert first["recent_24h_observation_count"] == 2
assert first["recent_7d_observation_count"] == 2
assert first["prefix_count"] == 2
assert first["recent_15m_prefix_count"] == 2
assert first["recent_24h_prefix_count"] == 2
assert first["origin_asn_count"] == 2
assert first["latest_event_type"] == "withdrawal"
@@ -583,6 +944,7 @@ async def test_create_bgp_anomalies_for_batch_calls_incident_aggregation():
extra_data={"prefix": "203.0.113.0/24", "origin_asn": 64496},
)
db = _FakeAsyncSession([
[],
[],
[],
[previous_record],
@@ -647,6 +1009,7 @@ async def test_create_bgp_anomalies_for_batch_skips_existing_entity_keys():
new_origin_asn=64497,
)
db = _FakeAsyncSession([
[],
[],
[],
[previous_record],