linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat(bgp): improve prefix-geo pipeline and collector reliability

Browse Source
This commit is contained in:
linkong
2026-04-07 15:23:26 +08:00
parent 5b9ef0223d
commit c439e91d12
11 changed files with 621 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
backend/tests/test_bgp.py
View File

@@ -35,6 +35,8 @@ from app.models.user import User
from app.services.collectors.bgp_common import normalize_bgp_event
from app.services.collectors.bgpstream import BGPStreamBackfillCollector
from app.services.collectors.iptoasn import IPtoASNPrefixGeoCollector
from app.services.collectors.opengeofeed import OpenGeoFeedPrefixGeoCollector
from app.services.collectors.nro_delegated import NRODelegatedPrefixGeoCollector
class _FakeScalarResult:
@@ -170,6 +172,75 @@ def test_iptoasn_transform_creates_prefix_geography_records():
assert record["metadata"]["source_dataset"] == "iptoasn_combined"
def test_iptoasn_build_dataset_urls_from_combined():
urls = IPtoASNPrefixGeoCollector._build_dataset_urls(
"https://iptoasn.com/data/ip2asn-combined.tsv.gz"
)
assert urls == [
"https://iptoasn.com/data/ip2asn-v4.tsv.gz",
"https://iptoasn.com/data/ip2asn-v6.tsv.gz",
]
def test_iptoasn_build_dataset_urls_passthrough_non_combined():
url = "https://example.com/custom.tsv.gz"
urls = IPtoASNPrefixGeoCollector._build_dataset_urls(url)
assert urls == [url]
def test_opengeofeed_transform_creates_prefix_geography_records():
collector = OpenGeoFeedPrefixGeoCollector()
transformed = collector.transform(
[
{
"prefix": "203.0.113.0/24",
"country_code": "GB",
"region": "GB-LND",
"city": "London",
"postal_code": "EC1A",
"extra_columns": ["source:example"],
}
]
)
assert len(transformed) == 1
record = transformed[0]
assert record["name"] == "203.0.113.0/24"
assert record["metadata"]["family"] == "ipv4"
assert record["metadata"]["range_start"] == "203.0.113.0"
assert record["metadata"]["range_end"] == "203.0.113.255"
assert record["metadata"]["country_code"] == "GB"
assert record["metadata"]["source_dataset"] == "opengeofeed_public"
assert record["metadata"]["confidence"] == "geofeed"
def test_nro_delegated_transform_creates_prefix_geography_records():
collector = NRODelegatedPrefixGeoCollector()
transformed = collector.transform(
[
{
"rir": "ripencc",
"country_code": "DE",
"type": "ipv4",
"start": "198.51.100.0",
"value": "256",
"allocated_date": "20250401",
"status": "allocated",
}
]
)
assert len(transformed) == 1
record = transformed[0]
assert record["name"] == "198.51.100.0/24"
assert record["metadata"]["family"] == "ipv4"
assert record["metadata"]["range_start"] == "198.51.100.0"
assert record["metadata"]["range_end"] == "198.51.100.255"
assert record["metadata"]["country_code"] == "DE"
assert record["metadata"]["source_dataset"] == "nro_delegated_stats"
assert record["metadata"]["confidence"] == "registry_allocated"
def test_bgp_anomaly_to_dict():
anomaly = BGPAnomaly(
source="ris_live_bgp",
@@ -605,7 +676,7 @@ async def test_enrich_bgp_events_for_batch_adds_profiles_and_prefix_scope():
}
}
db = _FakeAsyncSession([[historical_observation], [iptoasn_row], [peeringdb_record]])
db = _FakeAsyncSession([[historical_observation], [], [iptoasn_row], [peeringdb_record]])
events = [
{
"metadata": {
@@ -639,6 +710,90 @@ async def test_enrich_bgp_events_for_batch_adds_profiles_and_prefix_scope():
assert enrichment["prefix_scope"]["cities"] == ["London"]
@pytest.mark.asyncio
async def test_enrich_bgp_events_for_batch_prefers_opengeofeed_over_iptoasn():
opengeofeed_row = {
"extra_data": {
"family": "ipv4",
"range_start": "203.0.113.0",
"range_end": "203.0.113.255",
"prefix": "203.0.113.0/24",
"country_code": "GB",
"region": "GB-LND",
"city": "London",
"source_dataset": "opengeofeed_public",
"confidence": "geofeed",
}
}
db = _FakeAsyncSession([[], [opengeofeed_row], []])
events = [
{
"metadata": {
"prefix": "203.0.113.0/24",
"origin_asn": 64497,
"collector": "rrc00",
"collector_location": {
"country": "Netherlands",
"city": "Amsterdam",
"latitude": 52.3676,
"longitude": 4.9041,
},
"as_path": [3333, 64497],
"timestamp": "2026-03-30T10:00:00Z",
},
"reference_date": "2026-03-30T10:00:00Z",
}
]
enriched = await enrich_bgp_events_for_batch(db, source="ris_live_bgp", events=events)
geography = enriched[0]["metadata"]["enrichment"]["prefix_geography"]
assert geography["source"] == "opengeofeed_public"
assert geography["confidence"] == "geofeed"
assert geography["city"] == "London"
@pytest.mark.asyncio
async def test_enrich_bgp_events_for_batch_falls_back_to_nro_delegated():
nro_row = {
"extra_data": {
"family": "ipv4",
"range_start": "198.51.100.0",
"range_end": "198.51.100.255",
"prefix": "198.51.100.0/24",
"country_code": "DE",
"rir": "ripencc",
"source_dataset": "nro_delegated_stats",
"confidence": "registry_allocated",
}
}
db = _FakeAsyncSession([[], [], [nro_row], []])
events = [
{
"metadata": {
"prefix": "198.51.100.0/24",
"origin_asn": 64512,
"collector": "rrc00",
"collector_location": {
"country": "Netherlands",
"city": "Amsterdam",
"latitude": 52.3676,
"longitude": 4.9041,
},
"as_path": [3333, 64512],
"timestamp": "2026-03-30T10:00:00Z",
},
"reference_date": "2026-03-30T10:00:00Z",
}
]
enriched = await enrich_bgp_events_for_batch(db, source="ris_live_bgp", events=events)
geography = enriched[0]["metadata"]["enrichment"]["prefix_geography"]
assert geography["source"] == "nro_delegated_stats"
assert geography["confidence"] == "registry_allocated"
assert geography["country"] == "德国"
@pytest.mark.asyncio
async def test_create_bgp_incidents_for_anomalies_aggregates_regions_and_collectors():
db = _FakeAsyncSession([[]])