fix: expand bgp pipeline and stabilize backend tests
This commit is contained in:
linkong
@@ -3,8 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import ipaddress
|
||||
from collections import Counter, defaultdict
|
||||
from collections import defaultdict
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
@@ -12,7 +11,15 @@ from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.bgp_anomaly import BGPAnomaly
|
||||
from app.models.bgp_observation import BGPObservation
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.services.bgp_incidents import create_bgp_incidents_for_anomalies
|
||||
from app.services.bgp_detectors import (
|
||||
detect_mass_withdrawal_anomalies,
|
||||
detect_more_specific_burst_anomalies,
|
||||
detect_origin_change_anomalies,
|
||||
)
|
||||
from app.services.bgp_enrichment import enrich_bgp_events_for_batch, extract_bgp_network_fields
|
||||
|
||||
|
||||
RIPE_RIS_COLLECTOR_COORDS: dict[str, dict[str, Any]] = {
|
||||
@@ -105,6 +112,10 @@ def normalize_bgp_event(payload: dict[str, Any], *, project: str) -> dict[str, A
|
||||
timestamp = _parse_timestamp(payload.get("timestamp") or payload.get("time") or payload.get("ts"))
|
||||
collector = str(payload.get("collector") or payload.get("host") or payload.get("router") or "unknown")
|
||||
peer_asn = _safe_int(payload.get("peer_asn") or payload.get("peer"))
|
||||
peer_ip = payload.get("peer_ip") or payload.get("peer_address")
|
||||
if peer_ip in (None, ""):
|
||||
peer_candidate = payload.get("peer")
|
||||
peer_ip = str(peer_candidate) if isinstance(peer_candidate, str) and ":" in peer_candidate else peer_candidate
|
||||
origin_asn = _safe_int(payload.get("origin_asn")) or (as_path[-1] if as_path else None)
|
||||
source_material = "|".join(
|
||||
[
|
||||
@@ -118,37 +129,33 @@ def normalize_bgp_event(payload: dict[str, Any], *, project: str) -> dict[str, A
|
||||
)
|
||||
source_id = hashlib.sha1(source_material.encode("utf-8")).hexdigest()[:24]
|
||||
|
||||
prefix_length = None
|
||||
is_more_specific = False
|
||||
if prefix:
|
||||
try:
|
||||
network = ipaddress.ip_network(prefix, strict=False)
|
||||
prefix_length = int(network.prefixlen)
|
||||
is_more_specific = prefix_length > (24 if network.version == 4 else 48)
|
||||
except ValueError:
|
||||
prefix_length = None
|
||||
|
||||
collector_location = RIPE_RIS_COLLECTOR_COORDS.get(collector, {})
|
||||
network_fields = extract_bgp_network_fields(prefix)
|
||||
metadata = {
|
||||
"project": project,
|
||||
"collector": collector,
|
||||
"peer_asn": peer_asn,
|
||||
"peer_ip": payload.get("peer_ip") or payload.get("peer_address"),
|
||||
"peer_ip": peer_ip,
|
||||
"event_type": event_type,
|
||||
"prefix": prefix,
|
||||
"origin_asn": origin_asn,
|
||||
"as_path": as_path,
|
||||
"communities": payload.get("communities") or payload.get("attrs", {}).get("communities") or [],
|
||||
"communities": payload.get("communities")
|
||||
or payload.get("community")
|
||||
or payload.get("attrs", {}).get("communities")
|
||||
or [],
|
||||
"next_hop": payload.get("next_hop") or payload.get("attrs", {}).get("next_hop"),
|
||||
"med": payload.get("med") or payload.get("attrs", {}).get("med"),
|
||||
"local_pref": payload.get("local_pref") or payload.get("attrs", {}).get("local_pref"),
|
||||
"timestamp": timestamp.isoformat(),
|
||||
"as_path_length": len(as_path),
|
||||
"prefix_length": prefix_length,
|
||||
"is_more_specific": is_more_specific,
|
||||
"visibility_weight": 1,
|
||||
"collector_location": collector_location,
|
||||
"raw_message": raw_message,
|
||||
"prefix_family": network_fields.get("prefix_family"),
|
||||
"prefix_length": network_fields.get("prefix_length"),
|
||||
"prefix_supernet": network_fields.get("prefix_supernet"),
|
||||
"is_more_specific": network_fields.get("is_more_specific", False),
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -165,6 +172,57 @@ def normalize_bgp_event(payload: dict[str, Any], *, project: str) -> dict[str, A
|
||||
}
|
||||
|
||||
|
||||
async def save_bgp_observations_for_batch(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
source: str,
|
||||
snapshot_id: int | None,
|
||||
task_id: int | None,
|
||||
events: list[dict[str, Any]],
|
||||
) -> int:
|
||||
if not events:
|
||||
return 0
|
||||
|
||||
ingest_batch_id = f"{source}:{task_id or 'adhoc'}:{snapshot_id or 'nosnapshot'}"
|
||||
created = 0
|
||||
|
||||
for event in events:
|
||||
metadata = event.get("metadata", {}) or {}
|
||||
collector_location = metadata.get("collector_location") or {}
|
||||
observed_at = _parse_timestamp(
|
||||
metadata.get("timestamp") or event.get("reference_date")
|
||||
)
|
||||
|
||||
db.add(
|
||||
BGPObservation(
|
||||
snapshot_id=snapshot_id,
|
||||
task_id=task_id,
|
||||
source=source,
|
||||
ingest_batch_id=ingest_batch_id,
|
||||
source_event_id=event.get("source_id"),
|
||||
collector=metadata.get("collector"),
|
||||
peer_asn=_safe_int(metadata.get("peer_asn")),
|
||||
peer_ip=metadata.get("peer_ip"),
|
||||
prefix=metadata.get("prefix"),
|
||||
event_type=str(metadata.get("event_type") or "announcement"),
|
||||
as_path=metadata.get("as_path") or [],
|
||||
origin_asn=_safe_int(metadata.get("origin_asn")),
|
||||
next_hop=metadata.get("next_hop"),
|
||||
communities=metadata.get("communities") or [],
|
||||
observed_at=observed_at,
|
||||
collector_geo=collector_location,
|
||||
raw_payload=metadata.get("raw_message") or {},
|
||||
note=event.get("description"),
|
||||
)
|
||||
)
|
||||
created += 1
|
||||
|
||||
if created:
|
||||
await db.commit()
|
||||
|
||||
return created
|
||||
|
||||
|
||||
async def create_bgp_anomalies_for_batch(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
@@ -176,12 +234,17 @@ async def create_bgp_anomalies_for_batch(
|
||||
if not events:
|
||||
return 0
|
||||
|
||||
pending_anomalies: list[BGPAnomaly] = []
|
||||
prefix_to_origins: defaultdict[str, set[int]] = defaultdict(set)
|
||||
prefix_to_more_specifics: defaultdict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
withdrawal_counter: Counter[tuple[str, int | None]] = Counter()
|
||||
enriched_events = await enrich_bgp_events_for_batch(
|
||||
db,
|
||||
source=source,
|
||||
events=events,
|
||||
)
|
||||
|
||||
prefixes = {event["metadata"].get("prefix") for event in events if event.get("metadata", {}).get("prefix")}
|
||||
prefixes = {
|
||||
event["metadata"].get("prefix")
|
||||
for event in enriched_events
|
||||
if event.get("metadata", {}).get("prefix")
|
||||
}
|
||||
previous_origin_map: dict[str, set[int]] = defaultdict(set)
|
||||
|
||||
if prefixes:
|
||||
@@ -199,97 +262,27 @@ async def create_bgp_anomalies_for_batch(
|
||||
if prefix and origin is not None:
|
||||
previous_origin_map[prefix].add(origin)
|
||||
|
||||
for event in events:
|
||||
metadata = event.get("metadata", {})
|
||||
prefix = metadata.get("prefix")
|
||||
origin_asn = _safe_int(metadata.get("origin_asn"))
|
||||
if not prefix:
|
||||
continue
|
||||
|
||||
if origin_asn is not None:
|
||||
prefix_to_origins[prefix].add(origin_asn)
|
||||
|
||||
if metadata.get("is_more_specific"):
|
||||
prefix_to_more_specifics[prefix.split("/")[0]].append(event)
|
||||
|
||||
if metadata.get("event_type") == "withdrawal":
|
||||
withdrawal_counter[(prefix, origin_asn)] += 1
|
||||
|
||||
for prefix, origins in prefix_to_origins.items():
|
||||
historic = previous_origin_map.get(prefix, set())
|
||||
new_origins = sorted(origin for origin in origins if origin not in historic)
|
||||
if historic and new_origins:
|
||||
for new_origin in new_origins:
|
||||
pending_anomalies.append(
|
||||
BGPAnomaly(
|
||||
snapshot_id=snapshot_id,
|
||||
task_id=task_id,
|
||||
source=source,
|
||||
anomaly_type="origin_change",
|
||||
severity="critical",
|
||||
status="active",
|
||||
entity_key=f"origin_change:{prefix}:{new_origin}",
|
||||
prefix=prefix,
|
||||
origin_asn=sorted(historic)[0],
|
||||
new_origin_asn=new_origin,
|
||||
peer_scope=[],
|
||||
started_at=datetime.now(UTC),
|
||||
confidence=0.86,
|
||||
summary=f"Prefix {prefix} is now originated by AS{new_origin}, outside the current baseline.",
|
||||
evidence={"previous_origins": sorted(historic), "current_origins": sorted(origins)},
|
||||
)
|
||||
)
|
||||
|
||||
for root_prefix, more_specifics in prefix_to_more_specifics.items():
|
||||
if len(more_specifics) >= 2:
|
||||
sample = more_specifics[0]["metadata"]
|
||||
pending_anomalies.append(
|
||||
BGPAnomaly(
|
||||
snapshot_id=snapshot_id,
|
||||
task_id=task_id,
|
||||
source=source,
|
||||
anomaly_type="more_specific_burst",
|
||||
severity="high",
|
||||
status="active",
|
||||
entity_key=f"more_specific_burst:{root_prefix}:{len(more_specifics)}",
|
||||
prefix=sample.get("prefix"),
|
||||
origin_asn=_safe_int(sample.get("origin_asn")),
|
||||
new_origin_asn=None,
|
||||
peer_scope=sorted(
|
||||
{
|
||||
str(item.get("metadata", {}).get("collector") or "")
|
||||
for item in more_specifics
|
||||
if item.get("metadata", {}).get("collector")
|
||||
}
|
||||
),
|
||||
started_at=datetime.now(UTC),
|
||||
confidence=0.72,
|
||||
summary=f"{len(more_specifics)} more-specific announcements clustered around {root_prefix}.",
|
||||
evidence={"events": [item.get("metadata") for item in more_specifics[:10]]},
|
||||
)
|
||||
)
|
||||
|
||||
for (prefix, origin_asn), count in withdrawal_counter.items():
|
||||
if count >= 3:
|
||||
pending_anomalies.append(
|
||||
BGPAnomaly(
|
||||
snapshot_id=snapshot_id,
|
||||
task_id=task_id,
|
||||
source=source,
|
||||
anomaly_type="mass_withdrawal",
|
||||
severity="high" if count < 8 else "critical",
|
||||
status="active",
|
||||
entity_key=f"mass_withdrawal:{prefix}:{origin_asn}:{count}",
|
||||
prefix=prefix,
|
||||
origin_asn=origin_asn,
|
||||
new_origin_asn=None,
|
||||
peer_scope=[],
|
||||
started_at=datetime.now(UTC),
|
||||
confidence=min(0.55 + (count * 0.05), 0.95),
|
||||
summary=f"{count} withdrawal events observed for {prefix} in the current ingest window.",
|
||||
evidence={"withdrawal_count": count},
|
||||
)
|
||||
)
|
||||
pending_anomalies = [
|
||||
*detect_origin_change_anomalies(
|
||||
source=source,
|
||||
snapshot_id=snapshot_id,
|
||||
task_id=task_id,
|
||||
events=enriched_events,
|
||||
previous_origin_map=previous_origin_map,
|
||||
),
|
||||
*detect_more_specific_burst_anomalies(
|
||||
source=source,
|
||||
snapshot_id=snapshot_id,
|
||||
task_id=task_id,
|
||||
events=enriched_events,
|
||||
),
|
||||
*detect_mass_withdrawal_anomalies(
|
||||
source=source,
|
||||
snapshot_id=snapshot_id,
|
||||
task_id=task_id,
|
||||
events=enriched_events,
|
||||
),
|
||||
]
|
||||
|
||||
if not pending_anomalies:
|
||||
return 0
|
||||
@@ -302,12 +295,21 @@ async def create_bgp_anomalies_for_batch(
|
||||
existing_keys = {row[0] for row in existing_result.fetchall()}
|
||||
|
||||
created = 0
|
||||
created_anomalies: list[BGPAnomaly] = []
|
||||
for anomaly in pending_anomalies:
|
||||
if anomaly.entity_key in existing_keys:
|
||||
continue
|
||||
db.add(anomaly)
|
||||
created_anomalies.append(anomaly)
|
||||
created += 1
|
||||
|
||||
if created:
|
||||
await db.commit()
|
||||
await create_bgp_incidents_for_anomalies(
|
||||
db,
|
||||
source=source,
|
||||
snapshot_id=snapshot_id,
|
||||
task_id=task_id,
|
||||
anomalies=created_anomalies,
|
||||
)
|
||||
return created
|
||||
|
||||
Reference in New Issue
Block a user