"""Enrichment helpers for BGP observation and anomaly pipelines.""" from __future__ import annotations import ipaddress from collections import defaultdict from datetime import UTC, datetime from typing import Any from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.models.bgp_observation import BGPObservation from app.models.collected_data import CollectedData def _safe_int(value: Any) -> int | None: try: if value in (None, ""): return None return int(value) except (TypeError, ValueError): return None def _parse_timestamp(value: Any) -> datetime: if isinstance(value, datetime): return value.astimezone(UTC) if value.tzinfo else value.replace(tzinfo=UTC) if isinstance(value, (int, float)): return datetime.fromtimestamp(value, tz=UTC) if isinstance(value, str) and value: normalized = value.replace("Z", "+00:00") parsed = datetime.fromisoformat(normalized) return parsed.astimezone(UTC) if parsed.tzinfo else parsed.replace(tzinfo=UTC) return datetime.now(UTC) def _dedupe_as_path(as_path: list[int]) -> list[int]: deduped: list[int] = [] for asn in as_path: if not deduped or deduped[-1] != asn: deduped.append(asn) return deduped def _compact_locations(items: list[dict[str, Any]]) -> list[dict[str, Any]]: results: list[dict[str, Any]] = [] seen: set[tuple[Any, ...]] = set() for item in items: key = ( item.get("country"), item.get("city"), item.get("latitude"), item.get("longitude"), ) if key in seen: continue seen.add(key) results.append(item) return results def extract_bgp_network_fields(prefix: str) -> dict[str, Any]: if not prefix: return { "prefix_family": None, "prefix_length": None, "prefix_supernet": None, "is_more_specific": False, } try: network = ipaddress.ip_network(prefix, strict=False) except ValueError: return { "prefix_family": None, "prefix_length": None, "prefix_supernet": None, "is_more_specific": False, } supernet_prefix = 16 if network.version == 4 else 32 if network.prefixlen > supernet_prefix: prefix_supernet = str(network.supernet(new_prefix=supernet_prefix)) else: prefix_supernet = str(network) return { "prefix_family": f"ipv{network.version}", "prefix_length": int(network.prefixlen), "prefix_supernet": prefix_supernet, "is_more_specific": network.prefixlen > (24 if network.version == 4 else 48), } async def enrich_bgp_events_for_batch( db: AsyncSession, *, source: str, events: list[dict[str, Any]], ) -> list[dict[str, Any]]: if not events: return [] prefixes = { str((event.get("metadata") or {}).get("prefix") or "").strip() for event in events if (event.get("metadata") or {}).get("prefix") } prefix_values = sorted(prefix for prefix in prefixes if prefix) origin_asns = sorted( { asn for event in events for asn in [ _safe_int((event.get("metadata") or {}).get("origin_asn")), _safe_int((event.get("metadata") or {}).get("new_origin_asn")), ] if asn is not None } ) historical_prefix_baseline: dict[str, dict[str, Any]] = {} if prefix_values: previous_result = await db.execute( select(BGPObservation).where( BGPObservation.source == source, BGPObservation.prefix.in_(prefix_values), ) ) by_prefix: defaultdict[str, list[BGPObservation]] = defaultdict(list) for observation in previous_result.scalars().all(): if observation.prefix: by_prefix[observation.prefix].append(observation) for prefix, observations in by_prefix.items(): unique_origins = sorted( { observation.origin_asn for observation in observations if observation.origin_asn is not None } ) unique_collectors = sorted( { observation.collector for observation in observations if observation.collector } ) historical_prefix_baseline[prefix] = { "historical_origin_asns": unique_origins, "historical_collectors": unique_collectors, "historical_observation_count": len(observations), "historical_regions": _compact_locations( [ observation.collector_geo or {} for observation in observations if observation.collector_geo ] ), } asn_profiles: dict[int, dict[str, Any]] = {} if origin_asns: peeringdb_result = await db.execute( select(CollectedData).where(CollectedData.source == "peeringdb_network") ) for record in peeringdb_result.scalars().all(): metadata = record.extra_data or {} asn = _safe_int(metadata.get("asn")) if asn is None or asn not in origin_asns: continue current = asn_profiles.get(asn) if current and (current.get("id") or 0) > (record.id or 0): continue asn_profiles[asn] = { "id": record.id, "asn": asn, "name": record.name, "country": metadata.get("country"), "city": metadata.get("city"), "source": "peeringdb_network", "info_type": metadata.get("info_type"), "info_traffic": metadata.get("info_traffic"), "info_ratio": metadata.get("info_ratio"), "ix_count": metadata.get("ix_count"), "url": metadata.get("url"), } collector_counts: defaultdict[str, int] = defaultdict(int) for event in events: collector = (event.get("metadata") or {}).get("collector") if collector: collector_counts[str(collector)] += 1 enriched: list[dict[str, Any]] = [] for event in events: metadata = dict(event.get("metadata") or {}) prefix = str(metadata.get("prefix") or "").strip() as_path = metadata.get("as_path") or [] normalized_as_path = [asn for asn in (_safe_int(item) for item in as_path) if asn is not None] deduped_as_path = _dedupe_as_path(normalized_as_path) collector = str(metadata.get("collector") or "").strip() collector_location = metadata.get("collector_location") or {} baseline = historical_prefix_baseline.get(prefix, {}) observed_at = _parse_timestamp(metadata.get("timestamp") or event.get("reference_date")) origin_asn = _safe_int(metadata.get("origin_asn")) new_origin_asn = _safe_int(metadata.get("new_origin_asn")) observed_regions = _compact_locations( [ { "country": collector_location.get("country"), "city": collector_location.get("city"), "latitude": collector_location.get("latitude"), "longitude": collector_location.get("longitude"), } ] ) baseline_regions = baseline.get("historical_regions", []) prefix_scope_regions = _compact_locations([*observed_regions, *baseline_regions]) enrichment = { **extract_bgp_network_fields(prefix), "observed_at": observed_at.isoformat(), "normalized_as_path": normalized_as_path, "deduped_as_path": deduped_as_path, "deduped_as_path_length": len(deduped_as_path), "path_prepending": len(normalized_as_path) > len(deduped_as_path), "collector_region": { "city": collector_location.get("city"), "country": collector_location.get("country"), }, "collector_observation_count_in_batch": collector_counts.get(collector, 0), "batch_visibility_collectors": sorted(collector_counts.keys()), "prefix_baseline": baseline, "is_new_origin_for_prefix": ( origin_asn is not None and origin_asn not in set(baseline.get("historical_origin_asns", [])) ), "rpki_validation": { "status": "unknown", "reason": "no_rpki_roa_dataset_configured", }, "origin_asn_profile": asn_profiles.get(origin_asn), "new_origin_asn_profile": asn_profiles.get(new_origin_asn), "prefix_scope": { "countries": sorted( { item.get("country") for item in prefix_scope_regions if item.get("country") } ), "cities": sorted( { item.get("city") for item in prefix_scope_regions if item.get("city") } ), "regions": prefix_scope_regions, }, } enriched.append( { **event, "metadata": { **metadata, "enrichment": enrichment, }, } ) return enriched