468 lines
20 KiB
Python
468 lines
20 KiB
Python
"""Detector helpers for BGP anomaly generation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections import Counter, defaultdict
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
from app.core.enums import BGPStatus
|
|
from app.models.bgp_anomaly import BGPAnomaly
|
|
|
|
|
|
def _iter_event_regions(events: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
regions: list[dict[str, Any]] = []
|
|
seen: set[tuple[Any, ...]] = set()
|
|
for event in events:
|
|
metadata = event.get("metadata") or {}
|
|
location = metadata.get("collector_location") or {}
|
|
region = {
|
|
"collector": metadata.get("collector"),
|
|
"country": location.get("country"),
|
|
"city": location.get("city"),
|
|
"latitude": location.get("latitude"),
|
|
"longitude": location.get("longitude"),
|
|
}
|
|
region_key = (
|
|
region.get("collector"),
|
|
region.get("country"),
|
|
region.get("city"),
|
|
region.get("latitude"),
|
|
region.get("longitude"),
|
|
)
|
|
if region_key in seen:
|
|
continue
|
|
seen.add(region_key)
|
|
regions.append(region)
|
|
return regions
|
|
|
|
|
|
def _unique_collectors(events: list[dict[str, Any]]) -> list[str]:
|
|
return sorted(
|
|
{
|
|
str((event.get("metadata") or {}).get("collector"))
|
|
for event in events
|
|
if (event.get("metadata") or {}).get("collector")
|
|
}
|
|
)
|
|
|
|
|
|
def _unique_peers(events: list[dict[str, Any]]) -> list[int]:
|
|
peers: set[int] = set()
|
|
for event in events:
|
|
peer_asn = (event.get("metadata") or {}).get("peer_asn")
|
|
if peer_asn is not None:
|
|
peers.add(int(peer_asn))
|
|
return sorted(peers)
|
|
|
|
|
|
def _path_signature(metadata: dict[str, Any]) -> tuple[int, ...]:
|
|
path = metadata.get("as_path") or []
|
|
return tuple(int(asn) for asn in path if asn is not None)
|
|
|
|
|
|
def detect_origin_change_anomalies(
|
|
*,
|
|
source: str,
|
|
snapshot_id: int | None,
|
|
task_id: int | None,
|
|
events: list[dict[str, Any]],
|
|
previous_origin_map: dict[str, set[int]],
|
|
) -> list[BGPAnomaly]:
|
|
prefix_to_origins: defaultdict[str, set[int]] = defaultdict(set)
|
|
for event in events:
|
|
metadata = event.get("metadata") or {}
|
|
prefix = metadata.get("prefix")
|
|
origin_asn = metadata.get("origin_asn")
|
|
if prefix and origin_asn is not None:
|
|
prefix_to_origins[str(prefix)].add(int(origin_asn))
|
|
|
|
anomalies: list[BGPAnomaly] = []
|
|
for prefix, origins in prefix_to_origins.items():
|
|
historic = previous_origin_map.get(prefix, set())
|
|
new_origins = sorted(origin for origin in origins if origin not in historic)
|
|
related_events = [
|
|
event
|
|
for event in events
|
|
if (event.get("metadata") or {}).get("prefix") == prefix
|
|
]
|
|
related_collectors = _unique_collectors(related_events)
|
|
related_regions = _iter_event_regions(related_events)
|
|
|
|
moas_candidate = not historic and len(origins) >= 2 and len(related_collectors) >= 2
|
|
if (not historic or not new_origins) and not moas_candidate:
|
|
continue
|
|
|
|
target_origins = new_origins or sorted(origins)
|
|
for new_origin in target_origins:
|
|
sample_event = next(
|
|
(
|
|
event
|
|
for event in related_events
|
|
if (event.get("metadata") or {}).get("prefix") == prefix
|
|
and int((event.get("metadata") or {}).get("origin_asn") or -1) == new_origin
|
|
),
|
|
{},
|
|
)
|
|
sample_metadata = sample_event.get("metadata") or {}
|
|
sample_enrichment = sample_metadata.get("enrichment") or {}
|
|
sample_prefix_geography = sample_enrichment.get("prefix_geography") or {}
|
|
anomaly_type = "origin_change"
|
|
severity = "critical"
|
|
confidence = 0.86
|
|
summary = f"Prefix {prefix} is now originated by AS{new_origin}, outside the current baseline."
|
|
evidence_previous_origins = sorted(historic)
|
|
if moas_candidate and not historic:
|
|
anomaly_type = "origin_conflict"
|
|
severity = "high"
|
|
confidence = 0.74
|
|
summary = (
|
|
f"Prefix {prefix} is being originated by multiple ASNs "
|
|
f"{sorted(origins)} across {len(related_collectors)} collectors."
|
|
)
|
|
evidence_previous_origins = []
|
|
anomalies.append(
|
|
BGPAnomaly(
|
|
snapshot_id=snapshot_id,
|
|
task_id=task_id,
|
|
source=source,
|
|
anomaly_type=anomaly_type,
|
|
severity=severity,
|
|
status=BGPStatus.ACTIVE.value,
|
|
entity_key=f"{anomaly_type}:{prefix}:{new_origin}",
|
|
prefix=prefix,
|
|
origin_asn=sorted(historic)[0] if historic else None,
|
|
new_origin_asn=new_origin,
|
|
peer_scope=related_collectors,
|
|
started_at=datetime.now(UTC),
|
|
confidence=confidence,
|
|
summary=summary,
|
|
evidence={
|
|
"previous_origins": evidence_previous_origins,
|
|
"current_origins": sorted(origins),
|
|
"events": [
|
|
(item.get("metadata") or {})
|
|
for item in related_events[:10]
|
|
],
|
|
"origin_asn_profile": sample_enrichment.get("origin_asn_profile"),
|
|
"new_origin_asn_profile": sample_enrichment.get("new_origin_asn_profile"),
|
|
"rpki_validation": sample_enrichment.get("rpki_validation"),
|
|
"prefix_geography": sample_prefix_geography,
|
|
"prefix_scope": sample_enrichment.get("prefix_scope"),
|
|
"impacted_regions": sample_prefix_geography.get("regions")
|
|
or related_regions
|
|
or sample_enrichment.get("prefix_scope", {}).get("regions", []),
|
|
},
|
|
)
|
|
)
|
|
|
|
return anomalies
|
|
|
|
|
|
def detect_more_specific_burst_anomalies(
|
|
*,
|
|
source: str,
|
|
snapshot_id: int | None,
|
|
task_id: int | None,
|
|
events: list[dict[str, Any]],
|
|
) -> list[BGPAnomaly]:
|
|
prefix_to_more_specifics: defaultdict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for event in events:
|
|
metadata = event.get("metadata") or {}
|
|
enrichment = metadata.get("enrichment") or {}
|
|
root_prefix = enrichment.get("prefix_supernet")
|
|
if root_prefix and enrichment.get("is_more_specific"):
|
|
prefix_to_more_specifics[str(root_prefix)].append(event)
|
|
|
|
anomalies: list[BGPAnomaly] = []
|
|
for root_prefix, more_specifics in prefix_to_more_specifics.items():
|
|
unique_prefixes = sorted(
|
|
{
|
|
str((item.get("metadata") or {}).get("prefix"))
|
|
for item in more_specifics
|
|
if (item.get("metadata") or {}).get("prefix")
|
|
}
|
|
)
|
|
related_collectors = _unique_collectors(more_specifics)
|
|
if len(unique_prefixes) < 2 and len(related_collectors) < 2:
|
|
continue
|
|
|
|
sample = more_specifics[0].get("metadata") or {}
|
|
sample_enrichment = sample.get("enrichment") or {}
|
|
sample_prefix_geography = sample_enrichment.get("prefix_geography") or {}
|
|
event_count = len(more_specifics)
|
|
anomalies.append(
|
|
BGPAnomaly(
|
|
snapshot_id=snapshot_id,
|
|
task_id=task_id,
|
|
source=source,
|
|
anomaly_type="more_specific_burst",
|
|
severity="high",
|
|
status=BGPStatus.ACTIVE.value,
|
|
entity_key=f"more_specific_burst:{root_prefix}:{len(unique_prefixes)}:{len(related_collectors)}",
|
|
prefix=sample.get("prefix"),
|
|
origin_asn=sample.get("origin_asn"),
|
|
new_origin_asn=None,
|
|
peer_scope=related_collectors,
|
|
started_at=datetime.now(UTC),
|
|
confidence=min(0.64 + (0.04 * min(event_count, 5)), 0.88),
|
|
summary=(
|
|
f"{len(unique_prefixes)} more-specific prefixes clustered under {root_prefix} "
|
|
f"across {len(related_collectors) or 1} collectors."
|
|
),
|
|
evidence={
|
|
"events": [item.get("metadata") for item in more_specifics[:10]],
|
|
"unique_prefixes": unique_prefixes,
|
|
"rpki_validation": sample_enrichment.get("rpki_validation"),
|
|
"origin_asn_profile": sample_enrichment.get("origin_asn_profile"),
|
|
"prefix_geography": sample_prefix_geography,
|
|
"prefix_scope": sample_enrichment.get("prefix_scope"),
|
|
"impacted_regions": sample_prefix_geography.get("regions")
|
|
or _iter_event_regions(more_specifics)
|
|
or sample_enrichment.get("prefix_scope", {}).get("regions", []),
|
|
},
|
|
)
|
|
)
|
|
|
|
return anomalies
|
|
|
|
|
|
def detect_mass_withdrawal_anomalies(
|
|
*,
|
|
source: str,
|
|
snapshot_id: int | None,
|
|
task_id: int | None,
|
|
events: list[dict[str, Any]],
|
|
) -> list[BGPAnomaly]:
|
|
withdrawal_counter: Counter[tuple[str, int | None]] = Counter()
|
|
withdrawal_events_by_key: defaultdict[tuple[str, int | None], list[dict[str, Any]]] = defaultdict(list)
|
|
for event in events:
|
|
metadata = event.get("metadata") or {}
|
|
prefix = metadata.get("prefix")
|
|
if prefix and metadata.get("event_type") == "withdrawal":
|
|
key = (str(prefix), metadata.get("origin_asn"))
|
|
withdrawal_counter[key] += 1
|
|
withdrawal_events_by_key[key].append(event)
|
|
|
|
anomalies: list[BGPAnomaly] = []
|
|
for (prefix, origin_asn), count in withdrawal_counter.items():
|
|
related_events = withdrawal_events_by_key[(prefix, origin_asn)]
|
|
related_collectors = _unique_collectors(related_events)
|
|
related_peers = _unique_peers(related_events)
|
|
if count < 3 and not (count >= 2 and len(related_collectors) >= 2):
|
|
continue
|
|
sample_event = related_events[0] if related_events else {}
|
|
sample_metadata = sample_event.get("metadata") or {}
|
|
sample_enrichment = sample_metadata.get("enrichment") or {}
|
|
sample_prefix_geography = sample_enrichment.get("prefix_geography") or {}
|
|
severity = "medium"
|
|
if count >= 4 or len(related_collectors) >= 3:
|
|
severity = "high"
|
|
if count >= 8:
|
|
severity = "critical"
|
|
|
|
anomalies.append(
|
|
BGPAnomaly(
|
|
snapshot_id=snapshot_id,
|
|
task_id=task_id,
|
|
source=source,
|
|
anomaly_type="mass_withdrawal",
|
|
severity=severity,
|
|
status=BGPStatus.ACTIVE.value,
|
|
entity_key=f"mass_withdrawal:{prefix}:{origin_asn}:{len(related_collectors)}:{count}",
|
|
prefix=prefix,
|
|
origin_asn=origin_asn,
|
|
new_origin_asn=None,
|
|
peer_scope=related_collectors,
|
|
started_at=datetime.now(UTC),
|
|
confidence=min(0.5 + (count * 0.06) + (0.04 * max(len(related_collectors) - 1, 0)), 0.95),
|
|
summary=(
|
|
f"{count} withdrawal events observed for {prefix} "
|
|
f"across {len(related_collectors) or 1} collectors in the current ingest window."
|
|
),
|
|
evidence={
|
|
"withdrawal_count": count,
|
|
"collector_count": len(related_collectors),
|
|
"peer_count": len(related_peers),
|
|
"events": [
|
|
(item.get("metadata") or {})
|
|
for item in related_events[:10]
|
|
],
|
|
"origin_asn_profile": sample_enrichment.get("origin_asn_profile"),
|
|
"rpki_validation": sample_enrichment.get("rpki_validation"),
|
|
"prefix_geography": sample_prefix_geography,
|
|
"prefix_scope": sample_enrichment.get("prefix_scope"),
|
|
"impacted_regions": sample_prefix_geography.get("regions")
|
|
or _iter_event_regions(related_events)
|
|
or sample_enrichment.get("prefix_scope", {}).get("regions", []),
|
|
},
|
|
)
|
|
)
|
|
|
|
return anomalies
|
|
|
|
|
|
def detect_route_leak_anomalies(
|
|
*,
|
|
source: str,
|
|
snapshot_id: int | None,
|
|
task_id: int | None,
|
|
events: list[dict[str, Any]],
|
|
) -> list[BGPAnomaly]:
|
|
events_by_prefix: defaultdict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for event in events:
|
|
metadata = event.get("metadata") or {}
|
|
prefix = metadata.get("prefix")
|
|
if prefix and metadata.get("event_type") == "announcement":
|
|
events_by_prefix[str(prefix)].append(event)
|
|
|
|
anomalies: list[BGPAnomaly] = []
|
|
for prefix, related_events in events_by_prefix.items():
|
|
related_collectors = _unique_collectors(related_events)
|
|
if len(related_collectors) < 2:
|
|
continue
|
|
|
|
path_signatures = Counter()
|
|
max_path_length = 0
|
|
for event in related_events:
|
|
metadata = event.get("metadata") or {}
|
|
signature = _path_signature(metadata)
|
|
if signature:
|
|
path_signatures[signature] += 1
|
|
max_path_length = max(max_path_length, len(signature))
|
|
|
|
if len(path_signatures) < 2:
|
|
continue
|
|
|
|
dominant_length = len(path_signatures.most_common(1)[0][0])
|
|
if max_path_length < max(dominant_length + 2, 5):
|
|
continue
|
|
|
|
sample_event = max(
|
|
related_events,
|
|
key=lambda event: len(_path_signature((event.get("metadata") or {}))),
|
|
)
|
|
sample_metadata = sample_event.get("metadata") or {}
|
|
sample_enrichment = sample_metadata.get("enrichment") or {}
|
|
sample_prefix_geography = sample_enrichment.get("prefix_geography") or {}
|
|
peer_scope = related_collectors
|
|
path_lengths = sorted({len(signature) for signature in path_signatures if signature})
|
|
|
|
anomalies.append(
|
|
BGPAnomaly(
|
|
snapshot_id=snapshot_id,
|
|
task_id=task_id,
|
|
source=source,
|
|
anomaly_type="route_leak_candidate",
|
|
severity="high" if max_path_length >= dominant_length + 3 else "medium",
|
|
status=BGPStatus.ACTIVE.value,
|
|
entity_key=f"route_leak_candidate:{prefix}:{max_path_length}:{len(related_collectors)}",
|
|
prefix=prefix,
|
|
origin_asn=sample_metadata.get("origin_asn"),
|
|
new_origin_asn=None,
|
|
peer_scope=peer_scope,
|
|
started_at=datetime.now(UTC),
|
|
confidence=min(0.58 + (0.05 * min(len(related_collectors), 4)) + (0.03 * min(max_path_length - dominant_length, 4)), 0.88),
|
|
summary=(
|
|
f"Prefix {prefix} shows divergent long AS paths across "
|
|
f"{len(related_collectors)} collectors, suggesting a possible route leak."
|
|
),
|
|
evidence={
|
|
"path_lengths": path_lengths,
|
|
"dominant_path_length": dominant_length,
|
|
"max_path_length": max_path_length,
|
|
"path_signatures": [
|
|
{"path": list(signature), "count": count}
|
|
for signature, count in path_signatures.most_common(5)
|
|
],
|
|
"events": [(item.get("metadata") or {}) for item in related_events[:10]],
|
|
"origin_asn_profile": sample_enrichment.get("origin_asn_profile"),
|
|
"rpki_validation": sample_enrichment.get("rpki_validation"),
|
|
"prefix_geography": sample_prefix_geography,
|
|
"prefix_scope": sample_enrichment.get("prefix_scope"),
|
|
"impacted_regions": sample_prefix_geography.get("regions")
|
|
or _iter_event_regions(related_events)
|
|
or sample_enrichment.get("prefix_scope", {}).get("regions", []),
|
|
},
|
|
)
|
|
)
|
|
|
|
return anomalies
|
|
|
|
|
|
def detect_path_flap_anomalies(
|
|
*,
|
|
source: str,
|
|
snapshot_id: int | None,
|
|
task_id: int | None,
|
|
events: list[dict[str, Any]],
|
|
) -> list[BGPAnomaly]:
|
|
events_by_prefix: defaultdict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for event in events:
|
|
metadata = event.get("metadata") or {}
|
|
prefix = metadata.get("prefix")
|
|
if prefix:
|
|
events_by_prefix[str(prefix)].append(event)
|
|
|
|
anomalies: list[BGPAnomaly] = []
|
|
for prefix, related_events in events_by_prefix.items():
|
|
ordered = sorted(
|
|
related_events,
|
|
key=lambda event: str((event.get("metadata") or {}).get("timestamp") or ""),
|
|
)
|
|
event_types = [str((item.get("metadata") or {}).get("event_type") or "") for item in ordered]
|
|
transitions = sum(1 for index in range(1, len(event_types)) if event_types[index] != event_types[index - 1])
|
|
distinct_paths = {
|
|
_path_signature(item.get("metadata") or {})
|
|
for item in ordered
|
|
if _path_signature(item.get("metadata") or {})
|
|
}
|
|
related_collectors = _unique_collectors(ordered)
|
|
|
|
if transitions < 3 and len(distinct_paths) < 3:
|
|
continue
|
|
|
|
sample_metadata = (ordered[0].get("metadata") or {}) if ordered else {}
|
|
sample_enrichment = sample_metadata.get("enrichment") or {}
|
|
sample_prefix_geography = sample_enrichment.get("prefix_geography") or {}
|
|
severity = "medium"
|
|
if transitions >= 5 or len(distinct_paths) >= 4:
|
|
severity = "high"
|
|
|
|
anomalies.append(
|
|
BGPAnomaly(
|
|
snapshot_id=snapshot_id,
|
|
task_id=task_id,
|
|
source=source,
|
|
anomaly_type="path_flap",
|
|
severity=severity,
|
|
status=BGPStatus.ACTIVE.value,
|
|
entity_key=f"path_flap:{prefix}:{transitions}:{len(distinct_paths)}",
|
|
prefix=prefix,
|
|
origin_asn=sample_metadata.get("origin_asn"),
|
|
new_origin_asn=None,
|
|
peer_scope=related_collectors,
|
|
started_at=datetime.now(UTC),
|
|
confidence=min(0.54 + (0.05 * min(transitions, 5)) + (0.03 * min(len(distinct_paths), 4)), 0.9),
|
|
summary=(
|
|
f"Prefix {prefix} shows repeated state/path changes "
|
|
f"({transitions} transitions, {len(distinct_paths)} distinct paths) in the current window."
|
|
),
|
|
evidence={
|
|
"transitions": transitions,
|
|
"event_types": event_types[:12],
|
|
"distinct_paths": [list(path) for path in list(distinct_paths)[:6]],
|
|
"events": [(item.get("metadata") or {}) for item in ordered[:10]],
|
|
"origin_asn_profile": sample_enrichment.get("origin_asn_profile"),
|
|
"rpki_validation": sample_enrichment.get("rpki_validation"),
|
|
"prefix_geography": sample_prefix_geography,
|
|
"prefix_scope": sample_enrichment.get("prefix_scope"),
|
|
"impacted_regions": sample_prefix_geography.get("regions")
|
|
or _iter_event_regions(ordered)
|
|
or sample_enrichment.get("prefix_scope", {}).get("regions", []),
|
|
},
|
|
)
|
|
)
|
|
|
|
return anomalies
|