353 lines
13 KiB
Python
353 lines
13 KiB
Python
"""Shared helpers for BGP collectors."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from collections import defaultdict
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.models.bgp_anomaly import BGPAnomaly
|
|
from app.models.bgp_observation import BGPObservation
|
|
from app.models.collected_data import CollectedData
|
|
from app.services.bgp_collector_locations import (
|
|
RIPE_RIS_COLLECTOR_COORDS,
|
|
get_bgp_collector_location_dict,
|
|
)
|
|
from app.services.bgp_event_locations import resolve_bgp_event_geo_dict
|
|
from app.services.bgp_incidents import create_bgp_incidents_for_anomalies
|
|
from app.services.bgp_detectors import (
|
|
detect_mass_withdrawal_anomalies,
|
|
detect_more_specific_burst_anomalies,
|
|
detect_origin_change_anomalies,
|
|
detect_path_flap_anomalies,
|
|
detect_route_leak_anomalies,
|
|
)
|
|
from app.services.bgp_enrichment import enrich_bgp_events_for_batch, extract_bgp_network_fields
|
|
|
|
# Re-exported for backward compatibility with anything that imports
|
|
# ``RIPE_RIS_COLLECTOR_COORDS`` from this module. New code should call
|
|
# ``app.services.bgp_collector_locations.get_bgp_collector_location_dict()``
|
|
# or ``resolve_bgp_collector_location()`` instead — those use the DB-backed
|
|
# collector-location cache.
|
|
__all__ = [
|
|
"RIPE_RIS_COLLECTOR_COORDS",
|
|
"normalize_bgp_event",
|
|
"save_bgp_observations_for_batch",
|
|
"create_bgp_anomalies_for_batch",
|
|
]
|
|
|
|
|
|
def _safe_int(value: Any) -> int | None:
|
|
try:
|
|
if value in (None, ""):
|
|
return None
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _parse_timestamp(value: Any) -> datetime:
|
|
if isinstance(value, datetime):
|
|
return value.astimezone(UTC) if value.tzinfo else value.replace(tzinfo=UTC)
|
|
|
|
if isinstance(value, (int, float)):
|
|
return datetime.fromtimestamp(value, tz=UTC)
|
|
|
|
if isinstance(value, str) and value:
|
|
normalized = value.replace("Z", "+00:00")
|
|
parsed = datetime.fromisoformat(normalized)
|
|
return parsed.astimezone(UTC) if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
|
|
|
return datetime.now(UTC)
|
|
|
|
|
|
def _normalize_as_path(raw_path: Any) -> list[int]:
|
|
if raw_path in (None, ""):
|
|
return []
|
|
if isinstance(raw_path, list):
|
|
return [asn for asn in (_safe_int(item) for item in raw_path) if asn is not None]
|
|
if isinstance(raw_path, str):
|
|
parts = raw_path.replace("{", "").replace("}", "").split()
|
|
return [asn for asn in (_safe_int(item) for item in parts) if asn is not None]
|
|
return []
|
|
|
|
|
|
def normalize_bgp_event(payload: dict[str, Any], *, project: str) -> dict[str, Any]:
|
|
raw_message = payload.get("raw_message", payload)
|
|
raw_path = (
|
|
payload.get("path")
|
|
or payload.get("as_path")
|
|
or payload.get("attrs", {}).get("path")
|
|
or payload.get("attrs", {}).get("as_path")
|
|
or []
|
|
)
|
|
as_path = _normalize_as_path(raw_path)
|
|
|
|
raw_type = str(payload.get("event_type") or payload.get("type") or payload.get("msg_type") or "").lower()
|
|
if raw_type in {"a", "announce", "announcement"}:
|
|
event_type = "announcement"
|
|
elif raw_type in {"w", "withdraw", "withdrawal"}:
|
|
event_type = "withdrawal"
|
|
elif raw_type in {"r", "rib"}:
|
|
event_type = "rib"
|
|
else:
|
|
event_type = raw_type or "announcement"
|
|
|
|
prefix = str(payload.get("prefix") or payload.get("prefixes") or payload.get("target_prefix") or "").strip()
|
|
if prefix.startswith("[") and prefix.endswith("]"):
|
|
prefix = prefix[1:-1]
|
|
|
|
timestamp = _parse_timestamp(payload.get("timestamp") or payload.get("time") or payload.get("ts"))
|
|
collector = str(payload.get("collector") or payload.get("host") or payload.get("router") or "unknown")
|
|
peer_asn = _safe_int(payload.get("peer_asn") or payload.get("peer"))
|
|
peer_ip = payload.get("peer_ip") or payload.get("peer_address")
|
|
if peer_ip in (None, ""):
|
|
peer_candidate = payload.get("peer")
|
|
peer_ip = str(peer_candidate) if isinstance(peer_candidate, str) and ":" in peer_candidate else peer_candidate
|
|
origin_asn = _safe_int(payload.get("origin_asn")) or (as_path[-1] if as_path else None)
|
|
source_material = "|".join(
|
|
[
|
|
collector,
|
|
str(peer_asn or ""),
|
|
prefix,
|
|
event_type,
|
|
timestamp.isoformat(),
|
|
",".join(str(asn) for asn in as_path),
|
|
]
|
|
)
|
|
source_id = hashlib.sha1(source_material.encode("utf-8")).hexdigest()[:24]
|
|
|
|
# Routes through the BGP event pipeline: source coords (if any) →
|
|
# collector inheritance. Returned dict keeps the legacy
|
|
# {city, country, latitude, longitude} keys plus richer
|
|
# {precision, source, needs_confirmation, matched_location_name, confidence}.
|
|
collector_location = resolve_bgp_event_geo_dict(
|
|
collector,
|
|
source_latitude=payload.get("latitude"),
|
|
source_longitude=payload.get("longitude"),
|
|
)
|
|
# Empty result (unknown collector & no source coords) — keep the
|
|
# downstream-expected dict shape so detectors / serializers don't crash.
|
|
if not collector_location:
|
|
collector_location = get_bgp_collector_location_dict(collector)
|
|
network_fields = extract_bgp_network_fields(prefix)
|
|
metadata = {
|
|
"project": project,
|
|
"collector": collector,
|
|
"peer_asn": peer_asn,
|
|
"peer_ip": peer_ip,
|
|
"event_type": event_type,
|
|
"prefix": prefix,
|
|
"origin_asn": origin_asn,
|
|
"as_path": as_path,
|
|
"communities": payload.get("communities")
|
|
or payload.get("community")
|
|
or payload.get("attrs", {}).get("communities")
|
|
or [],
|
|
"next_hop": payload.get("next_hop") or payload.get("attrs", {}).get("next_hop"),
|
|
"med": payload.get("med") or payload.get("attrs", {}).get("med"),
|
|
"local_pref": payload.get("local_pref") or payload.get("attrs", {}).get("local_pref"),
|
|
"timestamp": timestamp.isoformat(),
|
|
"as_path_length": len(as_path),
|
|
"visibility_weight": 1,
|
|
"collector_location": collector_location,
|
|
"raw_message": raw_message,
|
|
"prefix_family": network_fields.get("prefix_family"),
|
|
"prefix_length": network_fields.get("prefix_length"),
|
|
"prefix_supernet": network_fields.get("prefix_supernet"),
|
|
"is_more_specific": network_fields.get("is_more_specific", False),
|
|
}
|
|
|
|
return {
|
|
"source_id": source_id,
|
|
"name": prefix or f"{collector}:{event_type}",
|
|
"title": f"{event_type} {prefix}".strip(),
|
|
"description": f"{collector} observed {event_type} for {prefix}".strip(),
|
|
"reference_date": timestamp.isoformat(),
|
|
"country": collector_location.get("country"),
|
|
"city": collector_location.get("city"),
|
|
"latitude": collector_location.get("latitude"),
|
|
"longitude": collector_location.get("longitude"),
|
|
"metadata": metadata,
|
|
}
|
|
|
|
|
|
async def save_bgp_observations_for_batch(
|
|
db: AsyncSession,
|
|
*,
|
|
source: str,
|
|
snapshot_id: int | None,
|
|
task_id: int | None,
|
|
events: list[dict[str, Any]],
|
|
) -> int:
|
|
if not events:
|
|
return 0
|
|
|
|
ingest_batch_id = f"{source}:{task_id or 'adhoc'}:{snapshot_id or 'nosnapshot'}"
|
|
created = 0
|
|
|
|
for event in events:
|
|
metadata = event.get("metadata", {}) or {}
|
|
collector_location = metadata.get("collector_location") or {}
|
|
observed_at = _parse_timestamp(
|
|
metadata.get("timestamp") or event.get("reference_date")
|
|
)
|
|
|
|
db.add(
|
|
BGPObservation(
|
|
snapshot_id=snapshot_id,
|
|
task_id=task_id,
|
|
source=source,
|
|
ingest_batch_id=ingest_batch_id,
|
|
source_event_id=event.get("source_id"),
|
|
collector=metadata.get("collector"),
|
|
peer_asn=_safe_int(metadata.get("peer_asn")),
|
|
peer_ip=metadata.get("peer_ip"),
|
|
prefix=metadata.get("prefix"),
|
|
event_type=str(metadata.get("event_type") or "announcement"),
|
|
as_path=metadata.get("as_path") or [],
|
|
origin_asn=_safe_int(metadata.get("origin_asn")),
|
|
next_hop=metadata.get("next_hop"),
|
|
communities=metadata.get("communities") or [],
|
|
observed_at=observed_at,
|
|
collector_geo=collector_location,
|
|
raw_payload=metadata.get("raw_message") or {},
|
|
note=event.get("description"),
|
|
)
|
|
)
|
|
created += 1
|
|
|
|
if created:
|
|
await db.commit()
|
|
|
|
return created
|
|
|
|
|
|
async def create_bgp_anomalies_for_batch(
|
|
db: AsyncSession,
|
|
*,
|
|
source: str,
|
|
snapshot_id: int | None,
|
|
task_id: int | None,
|
|
events: list[dict[str, Any]],
|
|
) -> int:
|
|
if not events:
|
|
return 0
|
|
|
|
enriched_events = await enrich_bgp_events_for_batch(
|
|
db,
|
|
source=source,
|
|
events=events,
|
|
)
|
|
|
|
prefixes = {
|
|
event["metadata"].get("prefix")
|
|
for event in enriched_events
|
|
if event.get("metadata", {}).get("prefix")
|
|
}
|
|
previous_origin_map: dict[str, set[int]] = defaultdict(set)
|
|
|
|
if prefixes:
|
|
previous_query = await db.execute(
|
|
select(CollectedData).where(
|
|
CollectedData.source == source,
|
|
CollectedData.snapshot_id != snapshot_id,
|
|
CollectedData.extra_data["prefix"].as_string().in_(sorted(prefixes)),
|
|
)
|
|
)
|
|
for record in previous_query.scalars().all():
|
|
metadata = record.extra_data or {}
|
|
prefix = metadata.get("prefix")
|
|
origin = _safe_int(metadata.get("origin_asn"))
|
|
if prefix and origin is not None:
|
|
previous_origin_map[prefix].add(origin)
|
|
|
|
pending_anomalies = [
|
|
*detect_origin_change_anomalies(
|
|
source=source,
|
|
snapshot_id=snapshot_id,
|
|
task_id=task_id,
|
|
events=enriched_events,
|
|
previous_origin_map=previous_origin_map,
|
|
),
|
|
*detect_more_specific_burst_anomalies(
|
|
source=source,
|
|
snapshot_id=snapshot_id,
|
|
task_id=task_id,
|
|
events=enriched_events,
|
|
),
|
|
*detect_mass_withdrawal_anomalies(
|
|
source=source,
|
|
snapshot_id=snapshot_id,
|
|
task_id=task_id,
|
|
events=enriched_events,
|
|
),
|
|
*detect_route_leak_anomalies(
|
|
source=source,
|
|
snapshot_id=snapshot_id,
|
|
task_id=task_id,
|
|
events=enriched_events,
|
|
),
|
|
*detect_path_flap_anomalies(
|
|
source=source,
|
|
snapshot_id=snapshot_id,
|
|
task_id=task_id,
|
|
events=enriched_events,
|
|
),
|
|
]
|
|
|
|
if not pending_anomalies:
|
|
return 0
|
|
|
|
existing_result = await db.execute(
|
|
select(BGPAnomaly.entity_key).where(
|
|
BGPAnomaly.entity_key.in_([item.entity_key for item in pending_anomalies])
|
|
)
|
|
)
|
|
existing_keys = {row[0] for row in existing_result.fetchall()}
|
|
existing_anomalies: list[BGPAnomaly] = []
|
|
if existing_keys:
|
|
existing_anomaly_result = await db.execute(
|
|
select(BGPAnomaly).where(BGPAnomaly.entity_key.in_(sorted(existing_keys)))
|
|
)
|
|
existing_anomalies = existing_anomaly_result.scalars().all()
|
|
|
|
created = 0
|
|
created_anomalies: list[BGPAnomaly] = []
|
|
refreshed_anomalies: list[BGPAnomaly] = []
|
|
existing_map = {item.entity_key: item for item in existing_anomalies if item.entity_key}
|
|
for anomaly in pending_anomalies:
|
|
if anomaly.entity_key in existing_keys:
|
|
existing = existing_map.get(anomaly.entity_key)
|
|
if existing is not None:
|
|
existing.severity = anomaly.severity
|
|
existing.status = anomaly.status
|
|
existing.summary = anomaly.summary
|
|
existing.confidence = anomaly.confidence
|
|
existing.peer_scope = anomaly.peer_scope
|
|
existing.evidence = anomaly.evidence
|
|
existing.new_origin_asn = anomaly.new_origin_asn
|
|
existing.origin_asn = anomaly.origin_asn
|
|
refreshed_anomalies.append(existing)
|
|
continue
|
|
db.add(anomaly)
|
|
created_anomalies.append(anomaly)
|
|
created += 1
|
|
|
|
if created or refreshed_anomalies:
|
|
await db.commit()
|
|
incident_seed_anomalies = [*created_anomalies, *refreshed_anomalies]
|
|
if incident_seed_anomalies:
|
|
await create_bgp_incidents_for_anomalies(
|
|
db,
|
|
source=source,
|
|
snapshot_id=snapshot_id,
|
|
task_id=task_id,
|
|
anomalies=incident_seed_anomalies,
|
|
)
|
|
return created
|