linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix: expand bgp pipeline and stabilize backend tests

Browse Source
This commit is contained in:
linkong
2026-03-30 16:13:36 +08:00
parent 2015ab79bd
commit 945786cee5
22 changed files with 1787 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

491
backend/tests/test_bgp.py
View File

@@ -1,10 +1,72 @@
"""Tests for BGP observability helpers."""
from datetime import UTC, datetime
import pytest
from httpx import ASGITransport, AsyncClient
from unittest.mock import AsyncMock, patch
from app.api.v1.bgp import BGP_SOURCES
from app.core.security import get_current_user
from app.db.session import get_db
from app.main import app
from app.services.bgp_detectors import detect_mass_withdrawal_anomalies
from app.services.collectors.bgp_common import (
create_bgp_anomalies_for_batch,
save_bgp_observations_for_batch,
)
from app.services.bgp_enrichment import enrich_bgp_events_for_batch, extract_bgp_network_fields
from app.services.bgp_incidents import create_bgp_incidents_for_anomalies
from app.models.bgp_anomaly import BGPAnomaly
from app.models.collected_data import CollectedData
from app.models.bgp_incident import BGPIncident
from app.models.bgp_observation import BGPObservation
from app.models.user import User
from app.services.collectors.bgp_common import normalize_bgp_event
from app.services.collectors.bgpstream import BGPStreamBackfillCollector
class _FakeScalarResult:
def __init__(self, rows):
self._rows = rows
def all(self):
return self._rows
class _FakeResult:
def __init__(self, rows):
self._rows = rows
def scalars(self):
return _FakeScalarResult(self._rows)
def fetchall(self):
return self._rows
class _FakeAsyncSession:
def __init__(self, results, gets=None):
self._results = list(results)
self._gets = gets or {}
self.added = []
self.commits = 0
async def execute(self, _stmt):
if not self._results:
return _FakeResult([])
return _FakeResult(self._results.pop(0))
async def get(self, model, item_id):
return self._gets.get((model, item_id))
def add(self, item):
self.added.append(item)
async def commit(self):
self.commits += 1
def test_normalize_bgp_event_from_live_payload():
event = normalize_bgp_event(
{
@@ -30,6 +92,25 @@ def test_normalize_bgp_event_from_live_payload():
assert event["metadata"]["is_more_specific"] is False
def test_normalize_bgp_event_uses_peer_and_community_fallbacks():
event = normalize_bgp_event(
{
"collector": "rrc00",
"peer_asn": "3333",
"peer": "2405:a640::50",
"type": "UPDATE",
"prefix": "2401:2260::/32",
"path": [3333, 15412, 9304, 151650],
"community": [[15412, 603], [3333, 100]],
"timestamp": "2026-03-27T06:07:18.470000+00:00",
},
project="ris-live",
)
assert event["metadata"]["peer_ip"] == "2405:a640::50"
assert event["metadata"]["communities"] == [[15412, 603], [3333, 100]]
def test_bgpstream_transform_preserves_broker_record():
collector = BGPStreamBackfillCollector()
transformed = collector.transform(
@@ -72,3 +153,413 @@ def test_bgp_anomaly_to_dict():
assert data["anomaly_type"] == "origin_change"
assert data["new_origin_asn"] == 64497
assert data["evidence"]["previous_origins"] == [64496]
def test_bgp_observation_to_dict():
observation = BGPObservation(
source="ris_live_bgp",
ingest_batch_id="ris_live_bgp:1:1",
source_event_id="evt-1",
collector="rrc00",
peer_asn=3333,
peer_ip="2001:db8::1",
prefix="203.0.113.0/24",
event_type="announcement",
as_path=[3333, 64500, 64496],
origin_asn=64496,
next_hop="2001:db8::2",
communities=["3333:100"],
collector_geo={"city": "Amsterdam", "country": "Netherlands"},
raw_payload={"raw": "deadbeef"},
)
data = observation.to_dict()
assert data["source"] == "ris_live_bgp"
assert data["collector"] == "rrc00"
assert data["event_type"] == "announcement"
assert data["as_path"] == [3333, 64500, 64496]
assert data["collector_geo"]["city"] == "Amsterdam"
def test_extract_bgp_network_fields():
ipv4 = extract_bgp_network_fields("203.0.113.0/24")
assert ipv4["prefix_family"] == "ipv4"
assert ipv4["prefix_length"] == 24
assert ipv4["prefix_supernet"] == "203.0.0.0/16"
assert ipv4["is_more_specific"] is False
ipv6 = extract_bgp_network_fields("2001:db8:1::/48")
assert ipv6["prefix_family"] == "ipv6"
assert ipv6["prefix_length"] == 48
assert ipv6["prefix_supernet"] == "2001:db8::/32"
assert ipv6["is_more_specific"] is False
def test_detect_mass_withdrawal_anomalies():
events = [
{
"metadata": {
"prefix": "203.0.113.0/24",
"origin_asn": 64496,
"event_type": "withdrawal",
}
}
for _ in range(3)
]
anomalies = detect_mass_withdrawal_anomalies(
source="ris_live_bgp",
snapshot_id=1,
task_id=2,
events=events,
)
assert len(anomalies) == 1
assert anomalies[0].anomaly_type == "mass_withdrawal"
assert anomalies[0].prefix == "203.0.113.0/24"
def test_bgp_incident_to_dict():
incident = BGPIncident(
source="ris_live_bgp",
incident_key="origin_change:203.0.113.0/24:64497",
incident_type="origin_change",
title="Origin Change incident on 203.0.113.0/24",
summary="Grouped incident summary",
severity="critical",
status="active",
confidence=0.91,
affected_prefixes=["203.0.113.0/24"],
affected_asns=[64496, 64497],
affected_collectors=["rrc00", "rrc01"],
affected_regions=[{"country": "Netherlands", "city": "Amsterdam"}],
evidence_refs=["origin_change:203.0.113.0/24:64497"],
)
data = incident.to_dict()
assert data["incident_type"] == "origin_change"
assert data["affected_prefixes"] == ["203.0.113.0/24"]
assert data["affected_collectors"] == ["rrc00", "rrc01"]
@pytest.mark.asyncio
async def test_enrich_bgp_events_for_batch_adds_profiles_and_prefix_scope():
historical_observation = BGPObservation(
source="ris_live_bgp",
collector="rrc01",
prefix="203.0.113.0/24",
origin_asn=64496,
observed_at=datetime(2026, 3, 28, 0, 0, tzinfo=UTC),
collector_geo={
"country": "United Kingdom",
"city": "London",
"latitude": 51.5072,
"longitude": -0.1276,
},
event_type="announcement",
)
peeringdb_record = CollectedData(
source="peeringdb_network",
name="ExampleNet",
extra_data={
"asn": 64497,
"country": "NL",
"city": "Amsterdam",
"info_type": "Content",
"ix_count": 3,
"url": "https://example.net",
},
)
peeringdb_record.id = 99
db = _FakeAsyncSession([[historical_observation], [peeringdb_record]])
events = [
{
"metadata": {
"prefix": "203.0.113.0/24",
"origin_asn": 64497,
"new_origin_asn": None,
"collector": "rrc00",
"collector_location": {
"country": "Netherlands",
"city": "Amsterdam",
"latitude": 52.3676,
"longitude": 4.9041,
},
"as_path": [3333, 64497, 64497],
"timestamp": "2026-03-30T10:00:00Z",
},
"reference_date": "2026-03-30T10:00:00Z",
}
]
enriched = await enrich_bgp_events_for_batch(db, source="ris_live_bgp", events=events)
enrichment = enriched[0]["metadata"]["enrichment"]
assert enrichment["path_prepending"] is True
assert enrichment["is_new_origin_for_prefix"] is True
assert enrichment["rpki_validation"]["status"] == "unknown"
assert enrichment["origin_asn_profile"]["name"] == "ExampleNet"
assert enrichment["prefix_scope"]["countries"] == ["Netherlands", "United Kingdom"]
assert enrichment["prefix_scope"]["cities"] == ["Amsterdam", "London"]
@pytest.mark.asyncio
async def test_create_bgp_incidents_for_anomalies_aggregates_regions_and_collectors():
db = _FakeAsyncSession([[]])
anomaly = BGPAnomaly(
source="ris_live_bgp",
anomaly_type="origin_change",
severity="critical",
status="active",
entity_key="origin_change:203.0.113.0/24:64497",
prefix="203.0.113.0/24",
origin_asn=64496,
new_origin_asn=64497,
summary="Origin ASN changed",
confidence=0.9,
evidence={
"impacted_regions": [
{
"collector": "rrc00",
"country": "Netherlands",
"city": "Amsterdam",
"latitude": 52.3676,
"longitude": 4.9041,
}
]
},
)
created = await create_bgp_incidents_for_anomalies(
db,
source="ris_live_bgp",
snapshot_id=1,
task_id=2,
anomalies=[anomaly],
)
assert created == 1
assert db.commits == 1
assert len(db.added) == 1
incident = db.added[0]
assert incident.incident_type == "origin_change"
assert incident.affected_collectors == ["rrc00"]
assert incident.affected_regions[0]["city"] == "Amsterdam"
@pytest.mark.asyncio
async def test_save_bgp_observations_for_batch_adds_rows():
db = _FakeAsyncSession([])
events = [
{
"source_id": "evt-1",
"description": "rrc00 observed announcement for 203.0.113.0/24",
"reference_date": "2026-03-30T10:00:00Z",
"metadata": {
"collector": "rrc00",
"peer_asn": 3333,
"peer_ip": "2001:db8::1",
"prefix": "203.0.113.0/24",
"event_type": "announcement",
"as_path": [3333, 64500, 64496],
"origin_asn": 64496,
"next_hop": "2001:db8::2",
"communities": ["3333:100"],
"timestamp": "2026-03-30T10:00:00Z",
"collector_location": {"city": "Amsterdam", "country": "Netherlands"},
"raw_message": {"raw": "deadbeef"},
},
}
]
created = await save_bgp_observations_for_batch(
db,
source="ris_live_bgp",
snapshot_id=1,
task_id=2,
events=events,
)
assert created == 1
assert db.commits == 1
assert len(db.added) == 1
observation = db.added[0]
assert observation.ingest_batch_id == "ris_live_bgp:2:1"
assert observation.collector == "rrc00"
assert observation.prefix == "203.0.113.0/24"
@pytest.mark.asyncio
async def test_create_bgp_anomalies_for_batch_calls_incident_aggregation():
previous_record = CollectedData(
source="ris_live_bgp",
extra_data={"prefix": "203.0.113.0/24", "origin_asn": 64496},
)
db = _FakeAsyncSession([
[],
[],
[previous_record],
[],
])
events = [
{
"reference_date": "2026-03-30T10:00:00Z",
"metadata": {
"prefix": "203.0.113.0/24",
"origin_asn": 64497,
"collector": "rrc00",
"collector_location": {
"country": "Netherlands",
"city": "Amsterdam",
"latitude": 52.3676,
"longitude": 4.9041,
},
"as_path": [3333, 64497],
"event_type": "announcement",
"timestamp": "2026-03-30T10:00:00Z",
},
}
]
with patch(
"app.services.collectors.bgp_common.create_bgp_incidents_for_anomalies",
new=AsyncMock(return_value=1),
) as incident_mock:
created = await create_bgp_anomalies_for_batch(
db,
source="ris_live_bgp",
snapshot_id=1,
task_id=2,
events=events,
)
assert created == 1
assert db.commits == 1
assert len(db.added) == 1
anomaly = db.added[0]
assert anomaly.anomaly_type == "origin_change"
assert anomaly.prefix == "203.0.113.0/24"
incident_mock.assert_awaited_once()
@pytest.mark.asyncio
async def test_create_bgp_anomalies_for_batch_skips_existing_entity_keys():
previous_record = CollectedData(
source="ris_live_bgp",
extra_data={"prefix": "203.0.113.0/24", "origin_asn": 64496},
)
existing_key = ("origin_change:203.0.113.0/24:64497",)
db = _FakeAsyncSession([
[],
[],
[previous_record],
[existing_key],
])
events = [
{
"reference_date": "2026-03-30T10:00:00Z",
"metadata": {
"prefix": "203.0.113.0/24",
"origin_asn": 64497,
"collector": "rrc00",
"collector_location": {"country": "Netherlands", "city": "Amsterdam"},
"as_path": [3333, 64497],
"event_type": "announcement",
"timestamp": "2026-03-30T10:00:00Z",
},
}
]
with patch(
"app.services.collectors.bgp_common.create_bgp_incidents_for_anomalies",
new=AsyncMock(return_value=0),
) as incident_mock:
created = await create_bgp_anomalies_for_batch(
db,
source="ris_live_bgp",
snapshot_id=1,
task_id=2,
events=events,
)
assert created == 0
assert len(db.added) == 0
incident_mock.assert_not_awaited()
async def _bgp_test_client(db_session):
async def override_get_db():
yield db_session
def override_get_current_user():
return User(id=1, username="testuser", email="test@example.com", password_hash="x", role="admin")
app.dependency_overrides[get_db] = override_get_db
app.dependency_overrides[get_current_user] = override_get_current_user
transport = ASGITransport(app=app)
client = AsyncClient(transport=transport, base_url="http://test")
return client
@pytest.mark.asyncio
async def test_bgp_events_api_lists_observations():
observation = BGPObservation(
id=1,
source=BGP_SOURCES[0],
collector="rrc00",
peer_asn=3333,
prefix="203.0.113.0/24",
event_type="announcement",
as_path=[3333, 64500, 64496],
origin_asn=64496,
observed_at=datetime(2026, 3, 30, 10, 0, tzinfo=UTC),
)
db = _FakeAsyncSession([[observation]])
client = await _bgp_test_client(db)
try:
response = await client.get("/api/v1/bgp/events")
finally:
await client.aclose()
app.dependency_overrides.clear()
assert response.status_code == 200
payload = response.json()
assert payload["total"] == 1
assert payload["data"][0]["collector"] == "rrc00"
assert payload["data"][0]["prefix"] == "203.0.113.0/24"
@pytest.mark.asyncio
async def test_bgp_incidents_api_returns_incident():
incident = BGPIncident(
id=7,
source="ris_live_bgp",
incident_key="origin_change:203.0.113.0/24:64497",
incident_type="origin_change",
title="Origin Change incident on 203.0.113.0/24",
summary="Grouped incident summary",
severity="critical",
status="active",
confidence=0.91,
affected_prefixes=["203.0.113.0/24"],
affected_collectors=["rrc00"],
)
db = _FakeAsyncSession(
[[incident]],
gets={(BGPIncident, 7): incident},
)
client = await _bgp_test_client(db)
try:
list_response = await client.get("/api/v1/bgp/incidents")
detail_response = await client.get("/api/v1/bgp/incidents/7")
finally:
await client.aclose()
app.dependency_overrides.clear()
assert list_response.status_code == 200
assert list_response.json()["total"] == 1
assert detail_response.status_code == 200
assert detail_response.json()["incident_type"] == "origin_change"