352 lines
12 KiB
Python
352 lines
12 KiB
Python
"""Enrichment helpers for BGP observation and anomaly pipelines."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ipaddress
|
|
from collections import defaultdict
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
from sqlalchemy import select, text
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.core.countries import get_country_centroid, normalize_country
|
|
from app.models.bgp_observation import BGPObservation
|
|
from app.models.collected_data import CollectedData
|
|
|
|
|
|
def _safe_int(value: Any) -> int | None:
|
|
try:
|
|
if value in (None, ""):
|
|
return None
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _parse_timestamp(value: Any) -> datetime:
|
|
if isinstance(value, datetime):
|
|
return value.astimezone(UTC) if value.tzinfo else value.replace(tzinfo=UTC)
|
|
|
|
if isinstance(value, (int, float)):
|
|
return datetime.fromtimestamp(value, tz=UTC)
|
|
|
|
if isinstance(value, str) and value:
|
|
normalized = value.replace("Z", "+00:00")
|
|
parsed = datetime.fromisoformat(normalized)
|
|
return parsed.astimezone(UTC) if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
|
|
|
return datetime.now(UTC)
|
|
|
|
|
|
def _dedupe_as_path(as_path: list[int]) -> list[int]:
|
|
deduped: list[int] = []
|
|
for asn in as_path:
|
|
if not deduped or deduped[-1] != asn:
|
|
deduped.append(asn)
|
|
return deduped
|
|
|
|
|
|
def _compact_locations(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
results: list[dict[str, Any]] = []
|
|
seen: set[tuple[Any, ...]] = set()
|
|
for item in items:
|
|
key = (
|
|
item.get("country"),
|
|
item.get("city"),
|
|
item.get("latitude"),
|
|
item.get("longitude"),
|
|
)
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
results.append(item)
|
|
return results
|
|
|
|
|
|
def extract_bgp_network_fields(prefix: str) -> dict[str, Any]:
|
|
if not prefix:
|
|
return {
|
|
"prefix_family": None,
|
|
"prefix_length": None,
|
|
"prefix_supernet": None,
|
|
"is_more_specific": False,
|
|
}
|
|
|
|
try:
|
|
network = ipaddress.ip_network(prefix, strict=False)
|
|
except ValueError:
|
|
return {
|
|
"prefix_family": None,
|
|
"prefix_length": None,
|
|
"prefix_supernet": None,
|
|
"is_more_specific": False,
|
|
}
|
|
|
|
supernet_prefix = 16 if network.version == 4 else 32
|
|
if network.prefixlen > supernet_prefix:
|
|
prefix_supernet = str(network.supernet(new_prefix=supernet_prefix))
|
|
else:
|
|
prefix_supernet = str(network)
|
|
|
|
return {
|
|
"prefix_family": f"ipv{network.version}",
|
|
"prefix_length": int(network.prefixlen),
|
|
"prefix_supernet": prefix_supernet,
|
|
"is_more_specific": network.prefixlen > (24 if network.version == 4 else 48),
|
|
}
|
|
|
|
|
|
async def _lookup_prefix_geography(
|
|
db: AsyncSession,
|
|
prefix_values: list[str],
|
|
) -> dict[str, dict[str, Any]]:
|
|
results: dict[str, dict[str, Any]] = {}
|
|
|
|
for prefix in prefix_values:
|
|
try:
|
|
network = ipaddress.ip_network(prefix, strict=False)
|
|
except ValueError:
|
|
continue
|
|
|
|
family = f"ipv{network.version}"
|
|
range_start = str(network.network_address)
|
|
range_end = str(network.broadcast_address)
|
|
result = await db.execute(
|
|
text(
|
|
"""
|
|
SELECT metadata
|
|
FROM collected_data
|
|
WHERE source = 'iptoasn_prefix_geo'
|
|
AND COALESCE(is_current, TRUE) = TRUE
|
|
AND metadata->>'family' = :family
|
|
AND CAST(metadata->>'range_start' AS inet) <= CAST(:range_start AS inet)
|
|
AND CAST(metadata->>'range_end' AS inet) >= CAST(:range_end AS inet)
|
|
ORDER BY id DESC
|
|
LIMIT 1
|
|
"""
|
|
),
|
|
{
|
|
"family": family,
|
|
"range_start": range_start,
|
|
"range_end": range_end,
|
|
},
|
|
)
|
|
row = result.fetchone()
|
|
if not row:
|
|
continue
|
|
|
|
if isinstance(row, dict):
|
|
payload = row.get("metadata") or row.get("extra_data")
|
|
elif hasattr(row, "_mapping"):
|
|
payload = row._mapping.get("metadata") or row._mapping.get("extra_data")
|
|
else:
|
|
payload = row[0]
|
|
if not isinstance(payload, dict):
|
|
continue
|
|
|
|
country = normalize_country(payload.get("country") or payload.get("country_code"))
|
|
prefix_hint = payload.get("prefix") or prefix
|
|
asn = _safe_int(payload.get("asn"))
|
|
as_name = payload.get("as_name")
|
|
centroid = get_country_centroid(country)
|
|
regions = []
|
|
if country:
|
|
regions.append(
|
|
{
|
|
"country": country,
|
|
"city": None,
|
|
"latitude": centroid.get("latitude") if centroid else None,
|
|
"longitude": centroid.get("longitude") if centroid else None,
|
|
}
|
|
)
|
|
|
|
results[prefix] = {
|
|
"prefix": prefix_hint,
|
|
"country": country,
|
|
"asn": asn,
|
|
"as_name": as_name,
|
|
"source": payload.get("source_dataset") or "iptoasn_combined",
|
|
"confidence": "country_range",
|
|
"regions": regions,
|
|
}
|
|
|
|
return results
|
|
|
|
|
|
async def enrich_bgp_events_for_batch(
|
|
db: AsyncSession,
|
|
*,
|
|
source: str,
|
|
events: list[dict[str, Any]],
|
|
) -> list[dict[str, Any]]:
|
|
if not events:
|
|
return []
|
|
|
|
prefixes = {
|
|
str((event.get("metadata") or {}).get("prefix") or "").strip()
|
|
for event in events
|
|
if (event.get("metadata") or {}).get("prefix")
|
|
}
|
|
prefix_values = sorted(prefix for prefix in prefixes if prefix)
|
|
origin_asns = sorted(
|
|
{
|
|
asn
|
|
for event in events
|
|
for asn in [
|
|
_safe_int((event.get("metadata") or {}).get("origin_asn")),
|
|
_safe_int((event.get("metadata") or {}).get("new_origin_asn")),
|
|
]
|
|
if asn is not None
|
|
}
|
|
)
|
|
|
|
historical_prefix_baseline: dict[str, dict[str, Any]] = {}
|
|
if prefix_values:
|
|
previous_result = await db.execute(
|
|
select(BGPObservation).where(
|
|
BGPObservation.source == source,
|
|
BGPObservation.prefix.in_(prefix_values),
|
|
)
|
|
)
|
|
by_prefix: defaultdict[str, list[BGPObservation]] = defaultdict(list)
|
|
for observation in previous_result.scalars().all():
|
|
if observation.prefix:
|
|
by_prefix[observation.prefix].append(observation)
|
|
|
|
for prefix, observations in by_prefix.items():
|
|
unique_origins = sorted(
|
|
{
|
|
observation.origin_asn
|
|
for observation in observations
|
|
if observation.origin_asn is not None
|
|
}
|
|
)
|
|
unique_collectors = sorted(
|
|
{
|
|
observation.collector
|
|
for observation in observations
|
|
if observation.collector
|
|
}
|
|
)
|
|
historical_prefix_baseline[prefix] = {
|
|
"historical_origin_asns": unique_origins,
|
|
"historical_collectors": unique_collectors,
|
|
"historical_observation_count": len(observations),
|
|
"historical_regions": _compact_locations(
|
|
[
|
|
observation.collector_geo or {}
|
|
for observation in observations
|
|
if observation.collector_geo
|
|
]
|
|
),
|
|
}
|
|
|
|
asn_profiles: dict[int, dict[str, Any]] = {}
|
|
prefix_geographies = await _lookup_prefix_geography(db, prefix_values) if prefix_values else {}
|
|
if origin_asns:
|
|
peeringdb_result = await db.execute(
|
|
select(CollectedData).where(CollectedData.source == "peeringdb_network")
|
|
)
|
|
for record in peeringdb_result.scalars().all():
|
|
metadata = record.extra_data or {}
|
|
asn = _safe_int(metadata.get("asn"))
|
|
if asn is None or asn not in origin_asns:
|
|
continue
|
|
current = asn_profiles.get(asn)
|
|
if current and (current.get("id") or 0) > (record.id or 0):
|
|
continue
|
|
asn_profiles[asn] = {
|
|
"id": record.id,
|
|
"asn": asn,
|
|
"name": record.name,
|
|
"country": metadata.get("country"),
|
|
"city": metadata.get("city"),
|
|
"source": "peeringdb_network",
|
|
"info_type": metadata.get("info_type"),
|
|
"info_traffic": metadata.get("info_traffic"),
|
|
"info_ratio": metadata.get("info_ratio"),
|
|
"ix_count": metadata.get("ix_count"),
|
|
"url": metadata.get("url"),
|
|
}
|
|
|
|
collector_counts: defaultdict[str, int] = defaultdict(int)
|
|
for event in events:
|
|
collector = (event.get("metadata") or {}).get("collector")
|
|
if collector:
|
|
collector_counts[str(collector)] += 1
|
|
|
|
enriched: list[dict[str, Any]] = []
|
|
for event in events:
|
|
metadata = dict(event.get("metadata") or {})
|
|
prefix = str(metadata.get("prefix") or "").strip()
|
|
as_path = metadata.get("as_path") or []
|
|
normalized_as_path = [asn for asn in (_safe_int(item) for item in as_path) if asn is not None]
|
|
deduped_as_path = _dedupe_as_path(normalized_as_path)
|
|
collector = str(metadata.get("collector") or "").strip()
|
|
collector_location = metadata.get("collector_location") or {}
|
|
baseline = historical_prefix_baseline.get(prefix, {})
|
|
prefix_geography = prefix_geographies.get(prefix)
|
|
observed_at = _parse_timestamp(metadata.get("timestamp") or event.get("reference_date"))
|
|
origin_asn = _safe_int(metadata.get("origin_asn"))
|
|
new_origin_asn = _safe_int(metadata.get("new_origin_asn"))
|
|
baseline_regions = baseline.get("historical_regions", [])
|
|
prefix_scope_regions = _compact_locations([*baseline_regions])
|
|
|
|
enrichment = {
|
|
**extract_bgp_network_fields(prefix),
|
|
"observed_at": observed_at.isoformat(),
|
|
"normalized_as_path": normalized_as_path,
|
|
"deduped_as_path": deduped_as_path,
|
|
"deduped_as_path_length": len(deduped_as_path),
|
|
"path_prepending": len(normalized_as_path) > len(deduped_as_path),
|
|
"collector_region": {
|
|
"city": collector_location.get("city"),
|
|
"country": collector_location.get("country"),
|
|
},
|
|
"collector_observation_count_in_batch": collector_counts.get(collector, 0),
|
|
"batch_visibility_collectors": sorted(collector_counts.keys()),
|
|
"prefix_baseline": baseline,
|
|
"is_new_origin_for_prefix": (
|
|
origin_asn is not None
|
|
and origin_asn
|
|
not in set(baseline.get("historical_origin_asns", []))
|
|
),
|
|
"rpki_validation": {
|
|
"status": "unknown",
|
|
"reason": "no_rpki_roa_dataset_configured",
|
|
},
|
|
"origin_asn_profile": asn_profiles.get(origin_asn),
|
|
"new_origin_asn_profile": asn_profiles.get(new_origin_asn),
|
|
"prefix_geography": prefix_geography,
|
|
"prefix_scope": {
|
|
"countries": sorted(
|
|
{
|
|
item.get("country")
|
|
for item in prefix_scope_regions
|
|
if item.get("country")
|
|
}
|
|
),
|
|
"cities": sorted(
|
|
{
|
|
item.get("city")
|
|
for item in prefix_scope_regions
|
|
if item.get("city")
|
|
}
|
|
),
|
|
"regions": prefix_scope_regions,
|
|
},
|
|
}
|
|
|
|
enriched.append(
|
|
{
|
|
**event,
|
|
"metadata": {
|
|
**metadata,
|
|
"enrichment": enrichment,
|
|
},
|
|
}
|
|
)
|
|
|
|
return enriched
|