diff --git a/TODO.md b/TODO.md index 30196ef1..207754a2 100644 --- a/TODO.md +++ b/TODO.md @@ -1,4 +1,16 @@ # TODO -- [ ] 把 BGP 观测站和异常点的 `hover/click` 手感再磨细一点 -- [ ] 开始做 BGP 异常和海缆/区域的关联展示 +- [x] 把 BGP 观测站和异常点的 `hover/click` 手感再磨细一点 +- [x] 开始做 BGP 异常和海缆/区域的关联展示 +- [x] 做 Earth 侧的 `BGP activity layer`,让低 incident 密度时地图仍然有持续可感知的观测存在感 +- [x] 给 Earth BGP 补三层状态表达:`平稳观测态 / 局部波动态 / 事件活跃态` +- [x] 把“当前无活跃事件”改造成“观测网络仍在运行、当前未发现聚合级事件”的状态表达 +- [x] 做 collector / region 近 15 分钟 activity score 聚合接口或动态聚合逻辑 +- [x] 把 Earth 的 BGP incident 改成 `紧凑事件核 + 向外扩张环形 pulse`,替换当前大面积 glow +- [x] 为 BGP incident 建立符号系统:按事件类型用不同 marker,而不是都用同一种亮点 +- [x] 把 incident 地理定位从 `collector-centric` 改成 `prefix-centric`,优先使用 `prefix_geography`,其次 `prefix_scope`,再次 ASN 区域,最后才回退到观测区域质心 +- [x] 新增 `prefix_geography` 数据层,不再把 `prefix_scope` 当成 prefix 地理归属本身 +- [x] 接入 `IPtoASN / IPtoCountry` 作为 prefix-centric geography 的主数据源 +- [ ] 接入 `OpenGeoFeed` 作为 prefix geography 的高质量覆盖/override 数据源 +- [ ] 把 RIR delegated / `inetnum` / `inet6num` whois 设计成 prefix geography 的 fallback,而不是主来源 +- [x] 在 activity layer 之后继续补 `route leak` 和 `path instability / flap` detector diff --git a/VERSION b/VERSION index 6b9f278d..0eda0deb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.22.9 +0.22.10 diff --git a/backend/app/api/v1/visualization.py b/backend/app/api/v1/visualization.py index e0370657..bda38dec 100644 --- a/backend/app/api/v1/visualization.py +++ b/backend/app/api/v1/visualization.py @@ -5,6 +5,7 @@ Returns GeoJSON format compatible with Three.js, CesiumJS, and Unreal Cesium. """ from datetime import UTC, datetime +import math from fastapi import APIRouter, HTTPException, Depends, Query from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, func @@ -18,7 +19,8 @@ from app.models.bgp_anomaly import BGPAnomaly from app.models.bgp_incident import BGPIncident from app.models.collected_data import CollectedData from app.services.bgp_collectors import build_bgp_collector_coverage -from app.services.cable_graph import build_graph_from_data, CableGraph +from app.services.bgp_enrichment import _lookup_prefix_geography +from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS router = APIRouter() @@ -316,11 +318,16 @@ def convert_gpu_cluster_to_geojson(records: List[CollectedData]) -> Dict[str, An return {"type": "FeatureCollection", "features": features} -def convert_bgp_anomalies_to_geojson(records: List[BGPAnomaly]) -> Dict[str, Any]: +def convert_bgp_anomalies_to_geojson( + records: List[BGPAnomaly], + geography_hints: Optional[Dict[str, Dict[str, Any]]] = None, +) -> Dict[str, Any]: features = [] + geography_hints = geography_hints or {} for record in records: evidence = record.evidence or {} + hint = geography_hints.get(str(record.entity_key or record.id), {}) collectors = evidence.get("collectors") or record.peer_scope or [] if not collectors: nested = evidence.get("events") or [] @@ -334,23 +341,6 @@ def convert_bgp_anomalies_to_geojson(records: List[BGPAnomaly]) -> Dict[str, Any if not collectors: collectors = [] - collector = collectors[0] if collectors else None - location = None - if collector: - location = RIPE_RIS_COLLECTOR_COORDS.get(str(collector)) - - if location is None: - nested = evidence.get("events") or [] - for item in nested: - collector_name = (item or {}).get("collector") - if collector_name and collector_name in RIPE_RIS_COLLECTOR_COORDS: - location = RIPE_RIS_COLLECTOR_COORDS[collector_name] - collector = collector_name - break - - if location is None: - continue - as_path = [] if isinstance(evidence.get("as_path"), list): as_path = evidence.get("as_path") or [] @@ -385,6 +375,28 @@ def convert_bgp_anomalies_to_geojson(records: List[BGPAnomaly]) -> Dict[str, Any } ) + geography_regions = _normalize_geo_regions(hint.get("regions") or []) + geography_mode = hint.get("geography_mode") or "collector_centroid" + + collector = collectors[0] if collectors else None + location = geography_regions[0] if geography_regions else None + + if location is None and collector: + location = RIPE_RIS_COLLECTOR_COORDS.get(str(collector)) + + if location is None: + nested = evidence.get("events") or [] + for item in nested: + collector_name = (item or {}).get("collector") + if collector_name and collector_name in RIPE_RIS_COLLECTOR_COORDS: + location = RIPE_RIS_COLLECTOR_COORDS[collector_name] + collector = collector_name + geography_mode = "collector_centroid" + break + + if location is None: + continue + features.append( { "type": "Feature", @@ -408,6 +420,7 @@ def convert_bgp_anomalies_to_geojson(records: List[BGPAnomaly]) -> Dict[str, Any "collector_count": len(collectors) or 1, "as_path": as_path, "impacted_regions": impacted_regions, + "geography_mode": geography_mode, "confidence": record.confidence, "summary": record.summary, "created_at": to_iso8601_utc(record.created_at), @@ -418,6 +431,54 @@ def convert_bgp_anomalies_to_geojson(records: List[BGPAnomaly]) -> Dict[str, Any return {"type": "FeatureCollection", "features": features} +async def build_anomaly_geography_hints( + db: AsyncSession, + records: List[BGPAnomaly], +) -> Dict[str, Dict[str, Any]]: + hints: Dict[str, Dict[str, Any]] = {} + for record in records: + evidence = record.evidence or {} + key = str(record.entity_key or record.id) + + prefix_geo_regions = [] + prefix_regions = [] + asn_regions = [] + evidence_prefix_geography = evidence.get("prefix_geography") or {} + prefix_geo_regions.extend( + _normalize_geo_regions(evidence_prefix_geography.get("regions") or []) + ) + + prefix_scope = evidence.get("prefix_scope") or {} + prefix_regions.extend(_normalize_geo_regions(prefix_scope.get("regions") or [])) + + for profile_key in ("origin_asn_profile", "new_origin_asn_profile"): + profile = evidence.get(profile_key) or {} + latitude = profile.get("latitude") + longitude = profile.get("longitude") + if isinstance(latitude, (int, float)) and isinstance(longitude, (int, float)): + asn_regions.append( + { + "country": profile.get("country"), + "city": profile.get("city"), + "latitude": float(latitude), + "longitude": float(longitude), + } + ) + + prefix_geo_regions = _normalize_geo_regions(prefix_geo_regions) + prefix_regions = _normalize_geo_regions(prefix_regions) + asn_regions = _normalize_geo_regions(asn_regions) + + if prefix_geo_regions: + hints[key] = {"regions": prefix_geo_regions, "geography_mode": "prefix_geography"} + elif prefix_regions: + hints[key] = {"regions": prefix_regions, "geography_mode": "prefix_scope"} + elif asn_regions: + hints[key] = {"regions": asn_regions, "geography_mode": "asn_region"} + + return hints + + def convert_bgp_collectors_to_geojson( coverage_by_collector: Dict[str, Dict[str, Any]] | None = None, ) -> Dict[str, Any]: @@ -442,8 +503,10 @@ def convert_bgp_collectors_to_geojson( "prefix_count": coverage.get("prefix_count", 0), "origin_asn_count": coverage.get("origin_asn_count", 0), "peer_asn_count": coverage.get("peer_asn_count", 0), + "recent_15m_observation_count": coverage.get("recent_15m_observation_count", 0), "recent_24h_observation_count": coverage.get("recent_24h_observation_count", 0), "recent_7d_observation_count": coverage.get("recent_7d_observation_count", 0), + "recent_15m_prefix_count": coverage.get("recent_15m_prefix_count", 0), "recent_24h_prefix_count": coverage.get("recent_24h_prefix_count", 0), "recent_7d_prefix_count": coverage.get("recent_7d_prefix_count", 0), "top_event_types": coverage.get("top_event_types", []), @@ -463,33 +526,183 @@ def convert_bgp_collectors_to_geojson( return {"type": "FeatureCollection", "features": features} -def convert_bgp_incidents_to_geojson(records: List[BGPIncident]) -> Dict[str, Any]: +def _incident_estimated_center(valid_regions: List[Dict[str, Any]]) -> Dict[str, float]: + x = 0.0 + y = 0.0 + z = 0.0 + for region in valid_regions: + lat_rad = math.radians(float(region["latitude"])) + lon_rad = math.radians(float(region["longitude"])) + x += math.cos(lat_rad) * math.cos(lon_rad) + y += math.cos(lat_rad) * math.sin(lon_rad) + z += math.sin(lat_rad) + + total = float(len(valid_regions)) + if total <= 0: + return {"latitude": 0.0, "longitude": 0.0} + + x /= total + y /= total + z /= total + hyp = math.sqrt((x * x) + (y * y)) + if hyp == 0: + return {"latitude": 0.0, "longitude": 0.0} + + return { + "latitude": math.degrees(math.atan2(z, hyp)), + "longitude": math.degrees(math.atan2(y, x)), + } + + +def _incident_estimated_radius_km(center: Dict[str, float], valid_regions: List[Dict[str, Any]]) -> float: + center_coords = (float(center["longitude"]), float(center["latitude"])) + distances = [ + haversine_distance( + center_coords, + (float(region["longitude"]), float(region["latitude"])), + ) + for region in valid_regions + ] + return round(max(distances) if distances else 0.0, 1) + + +def _normalize_geo_regions(regions: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + normalized: list[dict[str, Any]] = [] + seen: set[tuple[Any, ...]] = set() + for region in regions: + if not isinstance(region, dict): + continue + latitude = region.get("latitude") + longitude = region.get("longitude") + if not isinstance(latitude, (int, float)) or not isinstance(longitude, (int, float)): + continue + item = { + "collector": region.get("collector"), + "country": region.get("country"), + "city": region.get("city"), + "latitude": float(latitude), + "longitude": float(longitude), + } + key = ( + item["collector"], + item["country"], + item["city"], + item["latitude"], + item["longitude"], + ) + if key in seen: + continue + seen.add(key) + normalized.append(item) + return normalized + + +async def build_incident_geography_hints( + db: AsyncSession, + records: List[BGPIncident], +) -> Dict[str, Dict[str, Any]]: + evidence_refs = sorted( + { + str(ref) + for record in records + for ref in (record.evidence_refs or []) + if ref + } + ) + + anomalies = [] + if evidence_refs: + result = await db.execute( + select(BGPAnomaly).where(BGPAnomaly.entity_key.in_(evidence_refs)) + ) + anomalies = result.scalars().all() + anomaly_by_key = { + str(anomaly.entity_key): anomaly + for anomaly in anomalies + if anomaly.entity_key + } + + hints: Dict[str, Dict[str, Any]] = {} + for record in records: + prefix_geo_regions: list[dict[str, Any]] = [] + prefix_regions: list[dict[str, Any]] = [] + asn_regions: list[dict[str, Any]] = [] + + for ref in record.evidence_refs or []: + anomaly = anomaly_by_key.get(str(ref)) + if anomaly is None: + continue + evidence = anomaly.evidence or {} + if not prefix_geo_regions: + prefix_geography = evidence.get("prefix_geography") or {} + prefix_geo_regions.extend(_normalize_geo_regions(prefix_geography.get("regions") or [])) + prefix_scope = evidence.get("prefix_scope") or {} + prefix_regions.extend(_normalize_geo_regions(prefix_scope.get("regions") or [])) + + for key in ("origin_asn_profile", "new_origin_asn_profile"): + profile = evidence.get(key) or {} + latitude = profile.get("latitude") + longitude = profile.get("longitude") + if isinstance(latitude, (int, float)) and isinstance(longitude, (int, float)): + asn_regions.append( + { + "country": profile.get("country"), + "city": profile.get("city"), + "latitude": float(latitude), + "longitude": float(longitude), + } + ) + + prefix_geo_regions = _normalize_geo_regions(prefix_geo_regions) + prefix_regions = _normalize_geo_regions(prefix_regions) + asn_regions = _normalize_geo_regions(asn_regions) + if prefix_geo_regions: + hints[record.incident_key] = { + "regions": prefix_geo_regions, + "geography_mode": "prefix_geography", + } + elif prefix_regions: + hints[record.incident_key] = { + "regions": prefix_regions, + "geography_mode": "prefix_scope", + } + elif asn_regions: + hints[record.incident_key] = { + "regions": asn_regions, + "geography_mode": "asn_region", + } + + return hints + + +def convert_bgp_incidents_to_geojson( + records: List[BGPIncident], + geography_hints: Optional[Dict[str, Dict[str, Any]]] = None, +) -> Dict[str, Any]: features = [] for record in records: - regions = record.affected_regions or [] + hint = (geography_hints or {}).get(record.incident_key, {}) + regions = hint.get("regions") or (record.affected_regions or []) if not regions: continue - valid_regions = [ - region - for region in regions - if isinstance(region, dict) - and isinstance(region.get("latitude"), (int, float)) - and isinstance(region.get("longitude"), (int, float)) - ] + valid_regions = _normalize_geo_regions(regions) if not valid_regions: continue - avg_lat = sum(float(region["latitude"]) for region in valid_regions) / len(valid_regions) - avg_lon = sum(float(region["longitude"]) for region in valid_regions) / len(valid_regions) + estimated_center = _incident_estimated_center(valid_regions) + estimated_radius_km = _incident_estimated_radius_km(estimated_center, valid_regions) features.append( { "type": "Feature", "geometry": { "type": "Point", - "coordinates": [avg_lon, avg_lat], + "coordinates": [ + estimated_center["longitude"], + estimated_center["latitude"], + ], }, "properties": { "id": record.id, @@ -504,6 +717,9 @@ def convert_bgp_incidents_to_geojson(records: List[BGPIncident]) -> Dict[str, An "affected_asns": record.affected_asns or [], "affected_collectors": record.affected_collectors or [], "affected_regions": valid_regions, + "estimated_center": estimated_center, + "estimated_radius_km": estimated_radius_km, + "geography_mode": hint.get("geography_mode") or "collector_centroid", "related_cables": record.related_cables or [], "related_ixps": record.related_ixps or [], "created_at": to_iso8601_utc(record.created_at), @@ -739,7 +955,8 @@ async def get_bgp_anomalies_geojson( result = await db.execute(stmt) records = list(result.scalars().all()) - geojson = convert_bgp_anomalies_to_geojson(records) + geography_hints = await build_anomaly_geography_hints(db, records) + geojson = convert_bgp_anomalies_to_geojson(records, geography_hints) return {**geojson, "count": len(geojson.get("features", []))} @@ -758,7 +975,8 @@ async def get_bgp_incidents_geojson( result = await db.execute(stmt) records = list(result.scalars().all()) - geojson = convert_bgp_incidents_to_geojson(records) + geography_hints = await build_incident_geography_hints(db, records) + geojson = convert_bgp_incidents_to_geojson(records, geography_hints) return {**geojson, "count": len(geojson.get("features", []))} diff --git a/backend/app/core/countries.py b/backend/app/core/countries.py index b1e8bc3c..343cb392 100644 --- a/backend/app/core/countries.py +++ b/backend/app/core/countries.py @@ -232,6 +232,30 @@ for canonical, aliases in COUNTRY_ENTRIES: COUNTRY_ALIAS_MAP[alias.casefold()] = canonical +COUNTRY_CENTROIDS = { + "美国": {"latitude": 39.8283, "longitude": -98.5795}, + "英国": {"latitude": 55.3781, "longitude": -3.4360}, + "荷兰": {"latitude": 52.1326, "longitude": 5.2913}, + "日本": {"latitude": 36.2048, "longitude": 138.2529}, + "德国": {"latitude": 51.1657, "longitude": 10.4515}, + "法国": {"latitude": 46.2276, "longitude": 2.2137}, + "新加坡": {"latitude": 1.3521, "longitude": 103.8198}, + "中国": {"latitude": 35.8617, "longitude": 104.1954}, + "中国(香港)": {"latitude": 22.3193, "longitude": 114.1694}, + "中国(台湾)": {"latitude": 23.6978, "longitude": 120.9605}, + "韩国": {"latitude": 35.9078, "longitude": 127.7669}, + "俄罗斯": {"latitude": 61.5240, "longitude": 105.3188}, + "加拿大": {"latitude": 56.1304, "longitude": -106.3468}, + "澳大利亚": {"latitude": -25.2744, "longitude": 133.7751}, + "巴西": {"latitude": -14.2350, "longitude": -51.9253}, + "南非": {"latitude": -30.5595, "longitude": 22.9375}, + "西班牙": {"latitude": 40.4637, "longitude": -3.7492}, + "意大利": {"latitude": 41.8719, "longitude": 12.5674}, + "瑞士": {"latitude": 46.8182, "longitude": 8.2275}, + "阿联酋": {"latitude": 23.4241, "longitude": 53.8478}, +} + + def normalize_country(value: Any) -> Optional[str]: if value is None: return None @@ -258,6 +282,13 @@ def normalize_country(value: Any) -> Optional[str]: return COUNTRY_ALIAS_MAP.get(lowered) +def get_country_centroid(value: Any) -> Optional[dict[str, float]]: + canonical = normalize_country(value) + if not canonical: + return None + return COUNTRY_CENTROIDS.get(canonical) + + def get_country_search_variants(value: Any) -> list[str]: canonical = normalize_country(value) if canonical is None: diff --git a/backend/app/core/data_sources.py b/backend/app/core/data_sources.py index 6828771e..c677129c 100644 --- a/backend/app/core/data_sources.py +++ b/backend/app/core/data_sources.py @@ -25,6 +25,7 @@ COLLECTOR_URL_KEYS = { "spacetrack_tle": "spacetrack.tle_query_url", "ris_live_bgp": "ris_live.url", "bgpstream_bgp": "bgpstream.url", + "iptoasn_prefix_geo": "iptoasn.combined_url", } diff --git a/backend/app/core/data_sources.yaml b/backend/app/core/data_sources.yaml index 9c64adb4..faacac7c 100644 --- a/backend/app/core/data_sources.yaml +++ b/backend/app/core/data_sources.yaml @@ -43,3 +43,6 @@ ris_live: bgpstream: url: "https://broker.bgpstream.caida.org/v2" + +iptoasn: + combined_url: "https://iptoasn.com/data/ip2asn-combined.tsv.gz" diff --git a/backend/app/core/datasource_defaults.py b/backend/app/core/datasource_defaults.py index cf0e374a..9d44121c 100644 --- a/backend/app/core/datasource_defaults.py +++ b/backend/app/core/datasource_defaults.py @@ -134,6 +134,13 @@ DEFAULT_DATASOURCES = { "priority": "P1", "frequency_minutes": 360, }, + "iptoasn_prefix_geo": { + "id": 23, + "name": "IPtoASN Prefix Geography", + "module": "L3", + "priority": "P1", + "frequency_minutes": 1440, + }, } ID_TO_COLLECTOR = {info["id"]: name for name, info in DEFAULT_DATASOURCES.items()} diff --git a/backend/app/services/bgp_collectors.py b/backend/app/services/bgp_collectors.py index 488730f2..56a7a209 100644 --- a/backend/app/services/bgp_collectors.py +++ b/backend/app/services/bgp_collectors.py @@ -20,6 +20,7 @@ async def build_bgp_collector_coverage( source_filter: tuple[str, ...] | None = None, ) -> list[dict[str, Any]]: now = datetime.now(UTC) + recent_15m_threshold = now - timedelta(minutes=15) recent_24h_threshold = now - timedelta(hours=24) recent_7d_threshold = now - timedelta(days=7) @@ -52,8 +53,10 @@ async def build_bgp_collector_coverage( "event_types": defaultdict(int), "countries": set(), "cities": set(), + "recent_15m_observation_count": 0, "recent_24h_observation_count": 0, "recent_7d_observation_count": 0, + "recent_15m_prefixes": set(), "recent_24h_prefixes": set(), "recent_7d_prefixes": set(), "latest_observed_at": None, @@ -78,6 +81,10 @@ async def build_bgp_collector_coverage( if observed_at.tzinfo else observed_at.replace(tzinfo=UTC) ) + if aware_observed_at >= recent_15m_threshold: + coverage["recent_15m_observation_count"] += 1 + if record.prefix: + coverage["recent_15m_prefixes"].add(record.prefix) if aware_observed_at >= recent_24h_threshold: coverage["recent_24h_observation_count"] += 1 if record.prefix: @@ -116,8 +123,10 @@ async def build_bgp_collector_coverage( "event_types": defaultdict(int), "countries": {location.get("country")} if location.get("country") else set(), "cities": {location.get("city")} if location.get("city") else set(), + "recent_15m_observation_count": 0, "recent_24h_observation_count": 0, "recent_7d_observation_count": 0, + "recent_15m_prefixes": set(), "recent_24h_prefixes": set(), "recent_7d_prefixes": set(), "latest_observed_at": None, @@ -142,8 +151,10 @@ async def build_bgp_collector_coverage( "prefix_count": len(item["prefixes"]), "origin_asn_count": len(item["origin_asns"]), "peer_asn_count": len(item["peer_asns"]), + "recent_15m_observation_count": item["recent_15m_observation_count"], "recent_24h_observation_count": item["recent_24h_observation_count"], "recent_7d_observation_count": item["recent_7d_observation_count"], + "recent_15m_prefix_count": len(item["recent_15m_prefixes"]), "recent_24h_prefix_count": len(item["recent_24h_prefixes"]), "recent_7d_prefix_count": len(item["recent_7d_prefixes"]), "top_event_types": [ diff --git a/backend/app/services/bgp_detectors.py b/backend/app/services/bgp_detectors.py index 39f7e39d..c1263ff8 100644 --- a/backend/app/services/bgp_detectors.py +++ b/backend/app/services/bgp_detectors.py @@ -55,6 +55,11 @@ def _unique_peers(events: list[dict[str, Any]]) -> list[int]: return sorted(peers) +def _path_signature(metadata: dict[str, Any]) -> tuple[int, ...]: + path = metadata.get("as_path") or [] + return tuple(int(asn) for asn in path if asn is not None) + + def detect_origin_change_anomalies( *, source: str, @@ -100,6 +105,7 @@ def detect_origin_change_anomalies( ) sample_metadata = sample_event.get("metadata") or {} sample_enrichment = sample_metadata.get("enrichment") or {} + sample_prefix_geography = sample_enrichment.get("prefix_geography") or {} anomaly_type = "origin_change" severity = "critical" confidence = 0.86 @@ -140,8 +146,10 @@ def detect_origin_change_anomalies( "origin_asn_profile": sample_enrichment.get("origin_asn_profile"), "new_origin_asn_profile": sample_enrichment.get("new_origin_asn_profile"), "rpki_validation": sample_enrichment.get("rpki_validation"), + "prefix_geography": sample_prefix_geography, "prefix_scope": sample_enrichment.get("prefix_scope"), - "impacted_regions": related_regions + "impacted_regions": sample_prefix_geography.get("regions") + or related_regions or sample_enrichment.get("prefix_scope", {}).get("regions", []), }, ) @@ -180,6 +188,7 @@ def detect_more_specific_burst_anomalies( sample = more_specifics[0].get("metadata") or {} sample_enrichment = sample.get("enrichment") or {} + sample_prefix_geography = sample_enrichment.get("prefix_geography") or {} event_count = len(more_specifics) anomalies.append( BGPAnomaly( @@ -205,8 +214,10 @@ def detect_more_specific_burst_anomalies( "unique_prefixes": unique_prefixes, "rpki_validation": sample_enrichment.get("rpki_validation"), "origin_asn_profile": sample_enrichment.get("origin_asn_profile"), + "prefix_geography": sample_prefix_geography, "prefix_scope": sample_enrichment.get("prefix_scope"), - "impacted_regions": _iter_event_regions(more_specifics) + "impacted_regions": sample_prefix_geography.get("regions") + or _iter_event_regions(more_specifics) or sample_enrichment.get("prefix_scope", {}).get("regions", []), }, ) @@ -242,6 +253,7 @@ def detect_mass_withdrawal_anomalies( sample_event = related_events[0] if related_events else {} sample_metadata = sample_event.get("metadata") or {} sample_enrichment = sample_metadata.get("enrichment") or {} + sample_prefix_geography = sample_enrichment.get("prefix_geography") or {} severity = "medium" if count >= 4 or len(related_collectors) >= 3: severity = "high" @@ -277,8 +289,175 @@ def detect_mass_withdrawal_anomalies( ], "origin_asn_profile": sample_enrichment.get("origin_asn_profile"), "rpki_validation": sample_enrichment.get("rpki_validation"), + "prefix_geography": sample_prefix_geography, "prefix_scope": sample_enrichment.get("prefix_scope"), - "impacted_regions": _iter_event_regions(related_events) + "impacted_regions": sample_prefix_geography.get("regions") + or _iter_event_regions(related_events) + or sample_enrichment.get("prefix_scope", {}).get("regions", []), + }, + ) + ) + + return anomalies + + +def detect_route_leak_anomalies( + *, + source: str, + snapshot_id: int | None, + task_id: int | None, + events: list[dict[str, Any]], +) -> list[BGPAnomaly]: + events_by_prefix: defaultdict[str, list[dict[str, Any]]] = defaultdict(list) + for event in events: + metadata = event.get("metadata") or {} + prefix = metadata.get("prefix") + if prefix and metadata.get("event_type") == "announcement": + events_by_prefix[str(prefix)].append(event) + + anomalies: list[BGPAnomaly] = [] + for prefix, related_events in events_by_prefix.items(): + related_collectors = _unique_collectors(related_events) + if len(related_collectors) < 2: + continue + + path_signatures = Counter() + max_path_length = 0 + for event in related_events: + metadata = event.get("metadata") or {} + signature = _path_signature(metadata) + if signature: + path_signatures[signature] += 1 + max_path_length = max(max_path_length, len(signature)) + + if len(path_signatures) < 2: + continue + + dominant_length = len(path_signatures.most_common(1)[0][0]) + if max_path_length < max(dominant_length + 2, 5): + continue + + sample_event = max( + related_events, + key=lambda event: len(_path_signature((event.get("metadata") or {}))), + ) + sample_metadata = sample_event.get("metadata") or {} + sample_enrichment = sample_metadata.get("enrichment") or {} + sample_prefix_geography = sample_enrichment.get("prefix_geography") or {} + peer_scope = related_collectors + path_lengths = sorted({len(signature) for signature in path_signatures if signature}) + + anomalies.append( + BGPAnomaly( + snapshot_id=snapshot_id, + task_id=task_id, + source=source, + anomaly_type="route_leak_candidate", + severity="high" if max_path_length >= dominant_length + 3 else "medium", + status="active", + entity_key=f"route_leak_candidate:{prefix}:{max_path_length}:{len(related_collectors)}", + prefix=prefix, + origin_asn=sample_metadata.get("origin_asn"), + new_origin_asn=None, + peer_scope=peer_scope, + started_at=datetime.now(UTC), + confidence=min(0.58 + (0.05 * min(len(related_collectors), 4)) + (0.03 * min(max_path_length - dominant_length, 4)), 0.88), + summary=( + f"Prefix {prefix} shows divergent long AS paths across " + f"{len(related_collectors)} collectors, suggesting a possible route leak." + ), + evidence={ + "path_lengths": path_lengths, + "dominant_path_length": dominant_length, + "max_path_length": max_path_length, + "path_signatures": [ + {"path": list(signature), "count": count} + for signature, count in path_signatures.most_common(5) + ], + "events": [(item.get("metadata") or {}) for item in related_events[:10]], + "origin_asn_profile": sample_enrichment.get("origin_asn_profile"), + "rpki_validation": sample_enrichment.get("rpki_validation"), + "prefix_geography": sample_prefix_geography, + "prefix_scope": sample_enrichment.get("prefix_scope"), + "impacted_regions": sample_prefix_geography.get("regions") + or _iter_event_regions(related_events) + or sample_enrichment.get("prefix_scope", {}).get("regions", []), + }, + ) + ) + + return anomalies + + +def detect_path_flap_anomalies( + *, + source: str, + snapshot_id: int | None, + task_id: int | None, + events: list[dict[str, Any]], +) -> list[BGPAnomaly]: + events_by_prefix: defaultdict[str, list[dict[str, Any]]] = defaultdict(list) + for event in events: + metadata = event.get("metadata") or {} + prefix = metadata.get("prefix") + if prefix: + events_by_prefix[str(prefix)].append(event) + + anomalies: list[BGPAnomaly] = [] + for prefix, related_events in events_by_prefix.items(): + ordered = sorted( + related_events, + key=lambda event: str((event.get("metadata") or {}).get("timestamp") or ""), + ) + event_types = [str((item.get("metadata") or {}).get("event_type") or "") for item in ordered] + transitions = sum(1 for index in range(1, len(event_types)) if event_types[index] != event_types[index - 1]) + distinct_paths = { + _path_signature(item.get("metadata") or {}) + for item in ordered + if _path_signature(item.get("metadata") or {}) + } + related_collectors = _unique_collectors(ordered) + + if transitions < 3 and len(distinct_paths) < 3: + continue + + sample_metadata = (ordered[0].get("metadata") or {}) if ordered else {} + sample_enrichment = sample_metadata.get("enrichment") or {} + sample_prefix_geography = sample_enrichment.get("prefix_geography") or {} + severity = "medium" + if transitions >= 5 or len(distinct_paths) >= 4: + severity = "high" + + anomalies.append( + BGPAnomaly( + snapshot_id=snapshot_id, + task_id=task_id, + source=source, + anomaly_type="path_flap", + severity=severity, + status="active", + entity_key=f"path_flap:{prefix}:{transitions}:{len(distinct_paths)}", + prefix=prefix, + origin_asn=sample_metadata.get("origin_asn"), + new_origin_asn=None, + peer_scope=related_collectors, + started_at=datetime.now(UTC), + confidence=min(0.54 + (0.05 * min(transitions, 5)) + (0.03 * min(len(distinct_paths), 4)), 0.9), + summary=( + f"Prefix {prefix} shows repeated state/path changes " + f"({transitions} transitions, {len(distinct_paths)} distinct paths) in the current window." + ), + evidence={ + "transitions": transitions, + "event_types": event_types[:12], + "distinct_paths": [list(path) for path in list(distinct_paths)[:6]], + "events": [(item.get("metadata") or {}) for item in ordered[:10]], + "origin_asn_profile": sample_enrichment.get("origin_asn_profile"), + "rpki_validation": sample_enrichment.get("rpki_validation"), + "prefix_geography": sample_prefix_geography, + "prefix_scope": sample_enrichment.get("prefix_scope"), + "impacted_regions": sample_prefix_geography.get("regions") + or _iter_event_regions(ordered) or sample_enrichment.get("prefix_scope", {}).get("regions", []), }, ) diff --git a/backend/app/services/bgp_enrichment.py b/backend/app/services/bgp_enrichment.py index f8e04b65..32cc0fca 100644 --- a/backend/app/services/bgp_enrichment.py +++ b/backend/app/services/bgp_enrichment.py @@ -7,9 +7,10 @@ from collections import defaultdict from datetime import UTC, datetime from typing import Any -from sqlalchemy import select +from sqlalchemy import select, text from sqlalchemy.ext.asyncio import AsyncSession +from app.core.countries import get_country_centroid, normalize_country from app.models.bgp_observation import BGPObservation from app.models.collected_data import CollectedData @@ -96,6 +97,83 @@ def extract_bgp_network_fields(prefix: str) -> dict[str, Any]: } +async def _lookup_prefix_geography( + db: AsyncSession, + prefix_values: list[str], +) -> dict[str, dict[str, Any]]: + results: dict[str, dict[str, Any]] = {} + + for prefix in prefix_values: + try: + network = ipaddress.ip_network(prefix, strict=False) + except ValueError: + continue + + family = f"ipv{network.version}" + range_start = str(network.network_address) + range_end = str(network.broadcast_address) + result = await db.execute( + text( + """ + SELECT metadata + FROM collected_data + WHERE source = 'iptoasn_prefix_geo' + AND COALESCE(is_current, TRUE) = TRUE + AND metadata->>'family' = :family + AND CAST(metadata->>'range_start' AS inet) <= CAST(:range_start AS inet) + AND CAST(metadata->>'range_end' AS inet) >= CAST(:range_end AS inet) + ORDER BY id DESC + LIMIT 1 + """ + ), + { + "family": family, + "range_start": range_start, + "range_end": range_end, + }, + ) + row = result.fetchone() + if not row: + continue + + if isinstance(row, dict): + payload = row.get("metadata") or row.get("extra_data") + elif hasattr(row, "_mapping"): + payload = row._mapping.get("metadata") or row._mapping.get("extra_data") + else: + payload = row[0] + if not isinstance(payload, dict): + continue + + country = normalize_country(payload.get("country") or payload.get("country_code")) + prefix_hint = payload.get("prefix") or prefix + asn = _safe_int(payload.get("asn")) + as_name = payload.get("as_name") + centroid = get_country_centroid(country) + regions = [] + if country: + regions.append( + { + "country": country, + "city": None, + "latitude": centroid.get("latitude") if centroid else None, + "longitude": centroid.get("longitude") if centroid else None, + } + ) + + results[prefix] = { + "prefix": prefix_hint, + "country": country, + "asn": asn, + "as_name": as_name, + "source": payload.get("source_dataset") or "iptoasn_combined", + "confidence": "country_range", + "regions": regions, + } + + return results + + async def enrich_bgp_events_for_batch( db: AsyncSession, *, @@ -165,6 +243,7 @@ async def enrich_bgp_events_for_batch( } asn_profiles: dict[int, dict[str, Any]] = {} + prefix_geographies = await _lookup_prefix_geography(db, prefix_values) if prefix_values else {} if origin_asns: peeringdb_result = await db.execute( select(CollectedData).where(CollectedData.source == "peeringdb_network") @@ -207,21 +286,12 @@ async def enrich_bgp_events_for_batch( collector = str(metadata.get("collector") or "").strip() collector_location = metadata.get("collector_location") or {} baseline = historical_prefix_baseline.get(prefix, {}) + prefix_geography = prefix_geographies.get(prefix) observed_at = _parse_timestamp(metadata.get("timestamp") or event.get("reference_date")) origin_asn = _safe_int(metadata.get("origin_asn")) new_origin_asn = _safe_int(metadata.get("new_origin_asn")) - observed_regions = _compact_locations( - [ - { - "country": collector_location.get("country"), - "city": collector_location.get("city"), - "latitude": collector_location.get("latitude"), - "longitude": collector_location.get("longitude"), - } - ] - ) baseline_regions = baseline.get("historical_regions", []) - prefix_scope_regions = _compact_locations([*observed_regions, *baseline_regions]) + prefix_scope_regions = _compact_locations([*baseline_regions]) enrichment = { **extract_bgp_network_fields(prefix), @@ -248,6 +318,7 @@ async def enrich_bgp_events_for_batch( }, "origin_asn_profile": asn_profiles.get(origin_asn), "new_origin_asn_profile": asn_profiles.get(new_origin_asn), + "prefix_geography": prefix_geography, "prefix_scope": { "countries": sorted( { diff --git a/backend/app/services/bgp_incidents.py b/backend/app/services/bgp_incidents.py index 03149b4e..ee7bf41f 100644 --- a/backend/app/services/bgp_incidents.py +++ b/backend/app/services/bgp_incidents.py @@ -209,15 +209,14 @@ async def create_bgp_incidents_for_anomalies( grouped.setdefault(incident_key, []).append(anomaly) existing_result = await db.execute( - select(BGPIncident.incident_key).where(BGPIncident.incident_key.in_(sorted(grouped.keys()))) + select(BGPIncident).where(BGPIncident.incident_key.in_(sorted(grouped.keys()))) ) - existing_keys = {row[0] for row in existing_result.fetchall()} + existing_incidents = { + incident.incident_key: incident for incident in existing_result.scalars().all() + } created = 0 for incident_key, items in grouped.items(): - if incident_key in existing_keys: - continue - items = sorted(items, key=lambda item: item.created_at or item.started_at or datetime.now(UTC)) primary = items[0] prefixes = sorted({item.prefix for item in items if item.prefix}) @@ -270,6 +269,28 @@ async def create_bgp_incidents_for_anomalies( ) related_infrastructure = await infer_related_infrastructure(db, regions) + existing = existing_incidents.get(incident_key) + if existing is not None: + existing.snapshot_id = snapshot_id + existing.task_id = task_id + existing.source = source + existing.incident_type = primary.anomaly_type + existing.title = title + existing.summary = summary + existing.severity = severity + existing.status = "active" + existing.confidence = confidence + existing.started_at = primary.started_at or existing.started_at or datetime.now(UTC) + existing.ended_at = None + existing.affected_prefixes = prefixes + existing.affected_asns = asns + existing.affected_collectors = collectors + existing.affected_regions = regions + existing.related_cables = related_infrastructure["related_cables"] + existing.related_ixps = related_infrastructure["related_ixps"] + existing.evidence_refs = evidence_refs + continue + db.add( BGPIncident( snapshot_id=snapshot_id, @@ -294,7 +315,7 @@ async def create_bgp_incidents_for_anomalies( ) created += 1 - if created: + if created or existing_incidents: await db.commit() return created diff --git a/backend/app/services/collectors/__init__.py b/backend/app/services/collectors/__init__.py index 155f150d..3c83c86a 100644 --- a/backend/app/services/collectors/__init__.py +++ b/backend/app/services/collectors/__init__.py @@ -32,6 +32,7 @@ from app.services.collectors.spacetrack import SpaceTrackTLECollector from app.services.collectors.celestrak import CelesTrakTLECollector from app.services.collectors.ris_live import RISLiveCollector from app.services.collectors.bgpstream import BGPStreamBackfillCollector +from app.services.collectors.iptoasn import IPtoASNPrefixGeoCollector collector_registry.register(TOP500Collector()) collector_registry.register(EpochAIGPUCollector()) @@ -55,3 +56,4 @@ collector_registry.register(SpaceTrackTLECollector()) collector_registry.register(CelesTrakTLECollector()) collector_registry.register(RISLiveCollector()) collector_registry.register(BGPStreamBackfillCollector()) +collector_registry.register(IPtoASNPrefixGeoCollector()) diff --git a/backend/app/services/collectors/bgp_common.py b/backend/app/services/collectors/bgp_common.py index 9625aa40..ec54a73c 100644 --- a/backend/app/services/collectors/bgp_common.py +++ b/backend/app/services/collectors/bgp_common.py @@ -18,6 +18,8 @@ from app.services.bgp_detectors import ( detect_mass_withdrawal_anomalies, detect_more_specific_burst_anomalies, detect_origin_change_anomalies, + detect_path_flap_anomalies, + detect_route_leak_anomalies, ) from app.services.bgp_enrichment import enrich_bgp_events_for_batch, extract_bgp_network_fields @@ -282,6 +284,18 @@ async def create_bgp_anomalies_for_batch( task_id=task_id, events=enriched_events, ), + *detect_route_leak_anomalies( + source=source, + snapshot_id=snapshot_id, + task_id=task_id, + events=enriched_events, + ), + *detect_path_flap_anomalies( + source=source, + snapshot_id=snapshot_id, + task_id=task_id, + events=enriched_events, + ), ] if not pending_anomalies: @@ -302,16 +316,29 @@ async def create_bgp_anomalies_for_batch( created = 0 created_anomalies: list[BGPAnomaly] = [] + refreshed_anomalies: list[BGPAnomaly] = [] + existing_map = {item.entity_key: item for item in existing_anomalies if item.entity_key} for anomaly in pending_anomalies: if anomaly.entity_key in existing_keys: + existing = existing_map.get(anomaly.entity_key) + if existing is not None: + existing.severity = anomaly.severity + existing.status = anomaly.status + existing.summary = anomaly.summary + existing.confidence = anomaly.confidence + existing.peer_scope = anomaly.peer_scope + existing.evidence = anomaly.evidence + existing.new_origin_asn = anomaly.new_origin_asn + existing.origin_asn = anomaly.origin_asn + refreshed_anomalies.append(existing) continue db.add(anomaly) created_anomalies.append(anomaly) created += 1 - if created: + if created or refreshed_anomalies: await db.commit() - incident_seed_anomalies = [*created_anomalies, *existing_anomalies] + incident_seed_anomalies = [*created_anomalies, *refreshed_anomalies] if incident_seed_anomalies: await create_bgp_incidents_for_anomalies( db, diff --git a/backend/app/services/collectors/iptoasn.py b/backend/app/services/collectors/iptoasn.py new file mode 100644 index 00000000..ee0de6f9 --- /dev/null +++ b/backend/app/services/collectors/iptoasn.py @@ -0,0 +1,111 @@ +"""IPtoASN prefix geography collector. + +Downloads the public combined IPv4+IPv6 TSV database and stores coarse +prefix-to-country/ASN geography hints for BGP enrichment. +""" + +from __future__ import annotations + +import gzip +from datetime import UTC, datetime +from ipaddress import summarize_address_range, ip_address +from typing import Any + +import httpx + +from app.services.collectors.base import BaseCollector + + +class IPtoASNPrefixGeoCollector(BaseCollector): + name = "iptoasn_prefix_geo" + priority = "P1" + module = "L3" + frequency_hours = 24 + data_type = "prefix_geography" + fail_on_empty = True + + async def fetch(self) -> list[dict[str, Any]]: + if not self._resolved_url: + raise RuntimeError("IPtoASN combined URL is not configured") + + async with httpx.AsyncClient(timeout=180.0, follow_redirects=True) as client: + response = await client.get( + self._resolved_url, + headers={ + "User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)", + "Accept": "application/gzip,application/octet-stream,*/*", + }, + ) + response.raise_for_status() + body = gzip.decompress(response.content).decode("utf-8", errors="replace") + + rows: list[dict[str, Any]] = [] + for raw_line in body.splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + parts = line.split("\t") + if len(parts) < 5: + continue + range_start, range_end, asn, country_code, as_name = parts[:5] + rows.append( + { + "range_start": range_start, + "range_end": range_end, + "asn": asn, + "country_code": country_code, + "as_name": as_name, + } + ) + return rows + + def transform(self, raw_data: list[dict[str, Any]]) -> list[dict[str, Any]]: + reference_date = datetime.now(UTC).isoformat() + transformed: list[dict[str, Any]] = [] + + for item in raw_data: + try: + start_ip = ip_address(str(item["range_start"])) + end_ip = ip_address(str(item["range_end"])) + except ValueError: + continue + + if start_ip.version != end_ip.version: + continue + + summarized = list(summarize_address_range(start_ip, end_ip)) + primary_prefix = str(summarized[0]) if summarized else f"{start_ip}/{32 if start_ip.version == 4 else 128}" + family = f"ipv{start_ip.version}" + + asn_value = item.get("asn") + try: + normalized_asn = int(str(asn_value)) + except (TypeError, ValueError): + normalized_asn = None + + transformed.append( + { + "source_id": f"{family}:{item['range_start']}-{item['range_end']}", + "name": primary_prefix, + "title": f"{primary_prefix} {item.get('country_code', '').strip()}".strip(), + "country": item.get("country_code"), + "city": "", + "latitude": None, + "longitude": None, + "metadata": { + "family": family, + "range_start": item["range_start"], + "range_end": item["range_end"], + "prefix": primary_prefix, + "prefixes": [str(prefix) for prefix in summarized[:8]], + "range_prefix_count": len(summarized), + "country_code": item.get("country_code"), + "asn": normalized_asn, + "as_name": item.get("as_name"), + "source_dataset": "iptoasn_combined", + }, + "reference_date": reference_date, + } + ) + + return transformed diff --git a/backend/tests/test_bgp.py b/backend/tests/test_bgp.py index a84f28de..d22cab90 100644 --- a/backend/tests/test_bgp.py +++ b/backend/tests/test_bgp.py @@ -13,6 +13,8 @@ from app.main import app from app.services.bgp_detectors import ( detect_mass_withdrawal_anomalies, detect_origin_change_anomalies, + detect_path_flap_anomalies, + detect_route_leak_anomalies, ) from app.services.collectors.bgp_common import ( create_bgp_anomalies_for_batch, @@ -24,6 +26,7 @@ from app.services.bgp_incidents import ( infer_related_infrastructure, ) from app.services.bgp_collectors import build_bgp_collector_coverage +from app.api.v1.visualization import convert_bgp_incidents_to_geojson, build_incident_geography_hints from app.models.bgp_anomaly import BGPAnomaly from app.models.collected_data import CollectedData from app.models.bgp_incident import BGPIncident @@ -31,6 +34,7 @@ from app.models.bgp_observation import BGPObservation from app.models.user import User from app.services.collectors.bgp_common import normalize_bgp_event from app.services.collectors.bgpstream import BGPStreamBackfillCollector +from app.services.collectors.iptoasn import IPtoASNPrefixGeoCollector class _FakeScalarResult: @@ -51,6 +55,9 @@ class _FakeResult: def fetchall(self): return self._rows + def fetchone(self): + return self._rows[0] if self._rows else None + class _FakeAsyncSession: def __init__(self, results, gets=None): @@ -59,7 +66,7 @@ class _FakeAsyncSession: self.added = [] self.commits = 0 - async def execute(self, _stmt): + async def execute(self, _stmt, _params=None): if not self._results: return _FakeResult([]) return _FakeResult(self._results.pop(0)) @@ -140,6 +147,29 @@ def test_bgpstream_transform_preserves_broker_record(): assert record["metadata"]["broker_record"]["filename"] == "rib.20260326.0800.gz" +def test_iptoasn_transform_creates_prefix_geography_records(): + collector = IPtoASNPrefixGeoCollector() + transformed = collector.transform( + [ + { + "range_start": "1.0.0.0", + "range_end": "1.0.0.255", + "asn": "13335", + "country_code": "AU", + "as_name": "CLOUDFLARENET", + } + ] + ) + + assert len(transformed) == 1 + record = transformed[0] + assert record["name"] == "1.0.0.0/24" + assert record["metadata"]["family"] == "ipv4" + assert record["metadata"]["country_code"] == "AU" + assert record["metadata"]["asn"] == 13335 + assert record["metadata"]["source_dataset"] == "iptoasn_combined" + + def test_bgp_anomaly_to_dict(): anomaly = BGPAnomaly( source="ris_live_bgp", @@ -315,6 +345,75 @@ def test_detect_mass_withdrawal_anomalies_accepts_cross_collector_pair(): assert anomalies[0].evidence["collector_count"] == 2 +def test_detect_route_leak_anomalies_creates_candidate_for_divergent_long_paths(): + events = [ + { + "metadata": { + "collector": "rrc00", + "event_type": "announcement", + "prefix": "203.0.113.0/24", + "origin_asn": 64496, + "as_path": [64500, 64496], + "collector_location": {"country": "NL", "city": "Amsterdam", "latitude": 52.3, "longitude": 4.9}, + "enrichment": {"prefix_scope": {"regions": [{"country": "NL"}]}}, + } + }, + { + "metadata": { + "collector": "rrc01", + "event_type": "announcement", + "prefix": "203.0.113.0/24", + "origin_asn": 64496, + "as_path": [64510, 64520, 64530, 64540, 64496], + "collector_location": {"country": "GB", "city": "London", "latitude": 51.5, "longitude": -0.1}, + "enrichment": {"prefix_scope": {"regions": [{"country": "GB"}]}}, + } + }, + ] + + anomalies = detect_route_leak_anomalies( + source="ris_live_bgp", + snapshot_id=1, + task_id=2, + events=events, + ) + + assert len(anomalies) == 1 + assert anomalies[0].anomaly_type == "route_leak_candidate" + assert anomalies[0].evidence["max_path_length"] == 5 + + +def test_detect_path_flap_anomalies_creates_signal_for_repeated_state_changes(): + base_timestamp = datetime(2026, 3, 27, 0, 0, tzinfo=UTC) + events = [] + for index, event_type in enumerate(["announcement", "withdrawal", "announcement", "withdrawal"]): + events.append( + { + "metadata": { + "collector": "rrc00", + "event_type": event_type, + "timestamp": (base_timestamp + timedelta(minutes=index)).isoformat(), + "prefix": "198.51.100.0/24", + "origin_asn": 64512, + "as_path": [64500 + index, 64512] if event_type == "announcement" else [], + "collector_location": {"country": "NL", "city": "Amsterdam", "latitude": 52.3, "longitude": 4.9}, + "enrichment": {"prefix_scope": {"regions": [{"country": "NL"}]}}, + } + } + ) + + anomalies = detect_path_flap_anomalies( + source="ris_live_bgp", + snapshot_id=1, + task_id=2, + events=events, + ) + + assert len(anomalies) == 1 + assert anomalies[0].anomaly_type == "path_flap" + assert anomalies[0].evidence["transitions"] == 3 + + def test_bgp_incident_to_dict(): incident = BGPIncident( source="ris_live_bgp", @@ -338,6 +437,131 @@ def test_bgp_incident_to_dict(): assert data["affected_collectors"] == ["rrc00", "rrc01"] +def test_convert_bgp_incidents_to_geojson_adds_estimated_geography(): + incident = BGPIncident( + source="ris_live_bgp", + incident_key="origin_change:203.0.113.0/24:64497", + incident_type="origin_change", + title="Origin Change incident on 203.0.113.0/24", + summary="Grouped incident summary", + severity="critical", + status="active", + confidence=0.91, + affected_prefixes=["203.0.113.0/24"], + affected_collectors=["rrc00", "rrc01"], + affected_regions=[ + { + "collector": "rrc00", + "country": "Netherlands", + "city": "Amsterdam", + "latitude": 52.3676, + "longitude": 4.9041, + }, + { + "collector": "rrc01", + "country": "United Kingdom", + "city": "London", + "latitude": 51.5072, + "longitude": -0.1276, + }, + ], + ) + + payload = convert_bgp_incidents_to_geojson([incident]) + feature = payload["features"][0] + assert feature["properties"]["geography_mode"] == "collector_centroid" + assert feature["properties"]["estimated_radius_km"] > 0 + assert feature["properties"]["estimated_center"]["latitude"] != 0 + + +def test_convert_bgp_incidents_to_geojson_prefers_prefix_scope_hint(): + incident = BGPIncident( + source="ris_live_bgp", + incident_key="origin_change:203.0.113.0/24:64497", + incident_type="origin_change", + title="Origin Change incident on 203.0.113.0/24", + summary="Grouped incident summary", + severity="critical", + status="active", + confidence=0.91, + affected_prefixes=["203.0.113.0/24"], + affected_collectors=["rrc00"], + affected_regions=[ + { + "collector": "rrc00", + "country": "Netherlands", + "city": "Amsterdam", + "latitude": 52.3676, + "longitude": 4.9041, + } + ], + ) + + payload = convert_bgp_incidents_to_geojson( + [incident], + { + incident.incident_key: { + "geography_mode": "prefix_scope", + "regions": [ + { + "country": "Japan", + "city": "Tokyo", + "latitude": 35.6764, + "longitude": 139.65, + } + ], + } + }, + ) + feature = payload["features"][0] + assert feature["properties"]["geography_mode"] == "prefix_scope" + assert feature["geometry"]["coordinates"] == [139.65, 35.6764] + + +def test_convert_bgp_incidents_to_geojson_prefers_prefix_geography_hint(): + incident = BGPIncident( + source="ris_live_bgp", + incident_key="origin_change:198.51.100.0/24:64512", + incident_type="origin_change", + title="Origin Change incident on 198.51.100.0/24", + summary="Grouped incident summary", + severity="critical", + status="active", + confidence=0.91, + affected_prefixes=["198.51.100.0/24"], + affected_collectors=["rrc00"], + affected_regions=[ + { + "collector": "rrc00", + "country": "Netherlands", + "city": "Amsterdam", + "latitude": 52.3676, + "longitude": 4.9041, + } + ], + ) + + payload = convert_bgp_incidents_to_geojson( + [incident], + { + incident.incident_key: { + "geography_mode": "prefix_geography", + "regions": [ + { + "country": "日本", + "city": None, + "latitude": 35.6764, + "longitude": 139.65, + } + ], + } + }, + ) + feature = payload["features"][0] + assert feature["properties"]["geography_mode"] == "prefix_geography" + assert feature["geometry"]["coordinates"] == [139.65, 35.6764] + + @pytest.mark.asyncio async def test_enrich_bgp_events_for_batch_adds_profiles_and_prefix_scope(): historical_observation = BGPObservation( @@ -367,8 +591,21 @@ async def test_enrich_bgp_events_for_batch_adds_profiles_and_prefix_scope(): }, ) peeringdb_record.id = 99 + iptoasn_row = { + "extra_data": { + "family": "ipv4", + "range_start": "203.0.113.0", + "range_end": "203.0.113.255", + "prefix": "203.0.113.0/24", + "country": "英国", + "country_code": "GB", + "asn": 64497, + "as_name": "Example ASN", + "source_dataset": "iptoasn_combined", + } + } - db = _FakeAsyncSession([[historical_observation], [peeringdb_record]]) + db = _FakeAsyncSession([[historical_observation], [iptoasn_row], [peeringdb_record]]) events = [ { "metadata": { @@ -396,8 +633,10 @@ async def test_enrich_bgp_events_for_batch_adds_profiles_and_prefix_scope(): assert enrichment["is_new_origin_for_prefix"] is True assert enrichment["rpki_validation"]["status"] == "unknown" assert enrichment["origin_asn_profile"]["name"] == "ExampleNet" - assert enrichment["prefix_scope"]["countries"] == ["Netherlands", "United Kingdom"] - assert enrichment["prefix_scope"]["cities"] == ["Amsterdam", "London"] + assert enrichment["prefix_geography"]["country"] == "英国" + assert enrichment["prefix_geography"]["source"] == "iptoasn_combined" + assert enrichment["prefix_scope"]["countries"] == ["United Kingdom"] + assert enrichment["prefix_scope"]["cities"] == ["London"] @pytest.mark.asyncio @@ -448,6 +687,126 @@ async def test_create_bgp_incidents_for_anomalies_aggregates_regions_and_collect assert incident.affected_regions[0]["city"] == "Amsterdam" +@pytest.mark.asyncio +async def test_create_bgp_incidents_for_anomalies_refreshes_existing_incident(): + existing = BGPIncident( + source="ris_live_bgp", + incident_key="origin_change:203.0.113.0/24:64497", + incident_type="origin_change", + title="Old title", + summary="Old summary", + severity="medium", + status="active", + confidence=0.4, + affected_prefixes=["203.0.113.0/24"], + affected_asns=[64496, 64497], + affected_collectors=["rrc00"], + affected_regions=[ + { + "collector": "rrc00", + "country": "Netherlands", + "city": "Amsterdam", + "latitude": 52.3676, + "longitude": 4.9041, + } + ], + related_cables=[], + related_ixps=[], + evidence_refs=["old-key"], + ) + db = _FakeAsyncSession([[existing]]) + anomaly = BGPAnomaly( + source="ris_live_bgp", + anomaly_type="origin_change", + severity="critical", + status="active", + entity_key="origin_change:203.0.113.0/24:64497", + prefix="203.0.113.0/24", + origin_asn=64496, + new_origin_asn=64497, + summary="Origin ASN changed", + confidence=0.9, + evidence={ + "impacted_regions": [ + { + "collector": None, + "country": "United States", + "city": None, + "latitude": 39.8283, + "longitude": -98.5795, + } + ] + }, + ) + + with patch( + "app.services.bgp_incidents.infer_related_infrastructure", + new=AsyncMock(return_value={"related_cables": [{"landing_point": "NYC"}], "related_ixps": []}), + ): + created = await create_bgp_incidents_for_anomalies( + db, + source="ris_live_bgp", + snapshot_id=1, + task_id=2, + anomalies=[anomaly], + ) + + assert created == 0 + assert db.commits == 1 + assert len(db.added) == 0 + assert existing.summary != "Old summary" + assert existing.severity == "critical" + assert existing.confidence == 0.9 + assert existing.affected_regions[0]["country"] == "United States" + assert existing.related_cables == [{"landing_point": "NYC"}] + assert existing.evidence_refs == ["origin_change:203.0.113.0/24:64497"] + + +@pytest.mark.asyncio +async def test_build_incident_geography_hints_prefers_evidence_prefix_scope_when_no_cached_prefix_geo(): + incident = BGPIncident( + source="ris_live_bgp", + incident_key="origin_change:93.175.153.0/24:16509", + incident_type="origin_change", + title="Origin Change incident on 93.175.153.0/24", + summary="summary", + severity="critical", + status="active", + affected_prefixes=["93.175.153.0/24"], + evidence_refs=["origin_change:93.175.153.0/24:16509"], + ) + anomaly = BGPAnomaly( + source="ris_live_bgp", + anomaly_type="origin_change", + severity="critical", + status="active", + entity_key="origin_change:93.175.153.0/24:16509", + prefix="93.175.153.0/24", + origin_asn=12654, + new_origin_asn=16509, + summary="summary", + confidence=0.8, + evidence={ + "prefix_scope": { + "regions": [ + { + "country": "Netherlands", + "city": "Amsterdam", + "latitude": 52.3676, + "longitude": 4.9041, + } + ] + } + }, + ) + db = _FakeAsyncSession([[anomaly]]) + + hints = await build_incident_geography_hints(db, [incident]) + + assert hints["origin_change:93.175.153.0/24:16509"]["geography_mode"] == "prefix_scope" + assert hints["origin_change:93.175.153.0/24:16509"]["regions"][0]["country"] == "Netherlands" + + @pytest.mark.asyncio async def test_infer_related_infrastructure_links_nearby_cables(): landing = CollectedData( @@ -524,9 +883,11 @@ async def test_build_bgp_collector_coverage_summarizes_observations(): first = next(item for item in coverage if item["collector"] == "rrc00") assert first["observation_count"] == 2 + assert first["recent_15m_observation_count"] == 2 assert first["recent_24h_observation_count"] == 2 assert first["recent_7d_observation_count"] == 2 assert first["prefix_count"] == 2 + assert first["recent_15m_prefix_count"] == 2 assert first["recent_24h_prefix_count"] == 2 assert first["origin_asn_count"] == 2 assert first["latest_event_type"] == "withdrawal" @@ -583,6 +944,7 @@ async def test_create_bgp_anomalies_for_batch_calls_incident_aggregation(): extra_data={"prefix": "203.0.113.0/24", "origin_asn": 64496}, ) db = _FakeAsyncSession([ + [], [], [], [previous_record], @@ -647,6 +1009,7 @@ async def test_create_bgp_anomalies_for_batch_skips_existing_entity_keys(): new_origin_asn=64497, ) db = _FakeAsyncSession([ + [], [], [], [previous_record], diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e77425d8..c3d61f20 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,49 @@ This project follows the repository versioning rule: - `feature` -> `+0.1.0` - `bugfix` -> `+0.0.1` +## 0.22.10 + +Released: 2026-04-02 + +### Highlights + +- Recovered the Earth-side BGP experience after a failed cache-busting / asset-loading refactor temporarily broke the globe runtime, removed textures, and made the BGP layer disappear when one backend endpoint timed out. +- Added a first usable `prefix_geography` data layer backed by `IPtoASN / IP-to-Country` ingestion so BGP geography can start moving away from pure collector-centric placement. +- Reworked BGP Earth rendering to keep collectors visible under degraded backend conditions, restore symbol-based incident markers, and split icon pulse from outward event-ring animation. + +### Added + +- Added a new `IPtoASN Prefix Geography` collector in [iptoasn.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/iptoasn.py) and registered it through [data_sources.yaml](/home/ray/dev/linkong/planet/backend/app/core/data_sources.yaml), [data_sources.py](/home/ray/dev/linkong/planet/backend/app/core/data_sources.py), [datasource_defaults.py](/home/ray/dev/linkong/planet/backend/app/core/datasource_defaults.py), and [collectors/__init__.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/__init__.py). +- Added country centroid helpers in [countries.py](/home/ray/dev/linkong/planet/backend/app/core/countries.py) so country-level prefix geography can produce map coordinates instead of only labels. +- Added a dedicated prefix-geography implementation note in [prefix-geography-plan.md](/home/ray/dev/linkong/planet/docs/prefix-geography-plan.md). +- Added recent `15m` collector activity dimensions to BGP coverage output in [bgp_collectors.py](/home/ray/dev/linkong/planet/backend/app/services/bgp_collectors.py) and [visualization.py](/home/ray/dev/linkong/planet/backend/app/api/v1/visualization.py). +- Added additional BGP detector coverage for `route_leak_candidate` and `path_flap` flows in [test_bgp.py](/home/ray/dev/linkong/planet/backend/tests/test_bgp.py). +- Added a local Earth cloud texture at [earth_clouds_1024.png](/home/ray/dev/linkong/planet/frontend/public/earth/assets/earth_clouds_1024.png) to avoid remote cloud-map dependency failures. + +### Improved + +- Improved BGP enrichment in [bgp_enrichment.py](/home/ray/dev/linkong/planet/backend/app/services/bgp_enrichment.py) so events now attach `prefix_geography`, `prefix_scope`, ASN profile context, and country-centroid-backed geography hints in one place. +- Improved incident aggregation in [bgp_incidents.py](/home/ray/dev/linkong/planet/backend/app/services/bgp_incidents.py) so existing incidents refresh their regions and geography metadata instead of remaining pinned to stale first-generation evidence forever. +- Improved anomaly generation flow in [bgp_common.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/bgp_common.py) by cleaning up duplicate incident-seeding paths and only feeding newly created or refreshed anomalies forward. +- Improved Earth BGP loading in [bgp.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp.js) so collectors are now the mandatory baseline layer while anomalies and incidents can fail independently without blanking the whole BGP surface. +- Improved Earth BGP marker language in [bgp.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp.js) and [constants.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/constants.js) by restoring typed event symbols, reducing additive white blowout, and making the incident ring animation read as an outward pulse instead of a generic glow blob. +- Improved Earth event animation semantics in [bgp.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp.js) by separating icon pulse from ring expansion so the center marker can breathe while the ring expands independently. +- Improved Earth texture reliability in [earth.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/earth.js) by switching clouds back to a local static asset under the restored `public/earth` runtime. +- Improved frontend boot noise in [frontend/index.html](/home/ray/dev/linkong/planet/frontend/index.html) by removing the default Vite favicon request that was generating irrelevant `vite.svg` timeouts during Earth debugging. +- Improved project planning docs in [bgp-context.md](/home/ray/dev/linkong/planet/docs/bgp-context.md) and [TODO.md](/home/ray/dev/linkong/planet/TODO.md) so the roadmap now explicitly prioritizes `activity layer`, `prefix-centric geography`, and follow-up geofeed/whois work. + +### Fixed + +- Fixed a failed Earth asset-versioning route where hand-applied cache-busting and a parallel Vite multi-entry experiment introduced duplicate module instances, broken `/earth` boot paths, missing textures, and severe runtime instability; the globe has now been restored to the stable `frontend/public/earth` runtime instead of the abandoned refactor path. +- Fixed Earth cloud and terrain loading regressions by restoring the old static Earth entrypoint and ensuring local cloud and 8K day-map assets resolve again from `public/earth/assets`. +- Fixed a full-layer BGP disappearance regression where `bgp-anomalies` or `bgp-incidents` timeouts caused the entire BGP layer to show `0` collectors and `0` events even though collector data still existed. +- Fixed `prefix_geography` lookups in [bgp_enrichment.py](/home/ray/dev/linkong/planet/backend/app/services/bgp_enrichment.py) that previously failed because JSON metadata access mixed SQL column names and ORM property names. +- Fixed stale anomaly and incident geography reuse so pre-existing records can now absorb refreshed evidence instead of staying locked to older Amsterdam-centric geography forever. +- Fixed a wrong optimization path in [visualization.py](/home/ray/dev/linkong/planet/backend/app/api/v1/visualization.py) where live `prefix_geography` lookups were pushed directly into Earth visualization endpoints, causing `bgp-anomalies` and `bgp-incidents` to time out under load; the visualization layer now prefers cached evidence again so Earth remains responsive. +- Fixed Earth-side BGP fallback rendering so anomaly fallback no longer collapses into a single undifferentiated glow layer when incidents are unavailable. +- Fixed extreme incident brightness and same-coordinate blowout in [bgp.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp.js) by removing additive blending from incident cores, reducing ring intensity, and deduplicating incident rendering at the coordinate level. +- Fixed the confusing floating BGP event hub in [bgp.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp.js) by removing the suspended off-surface glow anchor and its arc links, leaving Earth incidents grounded on the globe surface with regional halo context instead. + ## 0.22.9 Released: 2026-04-02 diff --git a/docs/bgp-context.md b/docs/bgp-context.md index 6fa9290c..5f598755 100644 --- a/docs/bgp-context.md +++ b/docs/bgp-context.md @@ -6,7 +6,17 @@ The BGP module is being evolved from an anomaly-only demo into a layered observa `raw observations -> enrichment -> detectors -> incidents -> console/Earth visualization` -The practical product goal is to turn low-level BGP control-plane changes into understandable network situation events with collector coverage, impact regions, and incident-centric visualization. +The practical product goal is no longer just to "show incidents on the globe". The current product objective is: + +1. keep BGP visually present on Earth even when incident density is low +2. make incidents clearly feel like a higher-confidence layer than anomalies +3. show that the observation network is still active even when there are no active incidents + +In practice, that means Earth should behave like an observability surface, not only an incident map: + +- `collectors` show that observation is happening +- `activity` shows where routing state is currently active or noisy +- `incidents` become the highest-confidence focus layer ## Current Backend Architecture @@ -135,16 +145,26 @@ Current design: 2. Incident markers are now the primary Earth BGP markers. 3. If there are no incidents, Earth falls back to anomaly markers. 4. If there are no anomalies either, collectors still provide presence. +5. A dedicated `activity layer` now adds: + - per-collector recent 15-minute activity halos + - clustered regional activity hints derived from active collectors +6. Incident markers now use: + - symbol-driven event cores + - outward ring pulses + - reduced diffuse glow compared with older Earth builds 5. The right-side stats now show: - BGP events - collector count - BGP status summary +This is directionally correct, but still incomplete for low-event-density periods. Right now Earth can still feel too quiet when incidents are sparse because the system lacks a dedicated `activity layer` between raw observation and incident focus. + Current BGP status strategy: - incidents present: show active incident count -- no incidents but anomalies present: show active anomaly count -- no incidents/anomalies but collectors present: show `当前无活跃事件` +- no incidents but anomalies present: show active anomaly count, plus active observation regions when available +- no incidents/anomalies but activity present: show `观测网络运行中` +- no incidents/anomalies but collectors present: show `观测网络运行中 · 当前未发现聚合级事件` - no BGP data at all: show `暂无观测数据` Earth info-card strategy: @@ -152,6 +172,81 @@ Earth info-card strategy: - `bgp` card is now incident-centric in wording - `bgp_collector` card shows collector location and current event count +## Current Product Gap + +The main product gap is not architecture correctness. It is low-density visualization strategy. + +Current reality: + +- incident count is naturally much lower than anomaly count +- that is expected, because incidents are aggregated and de-noised +- but incident-first rendering makes the Earth view look too quiet unless there is another always-available activity layer + +So the immediate next milestone is: + +`event map -> observability map` + +That means Earth needs three simultaneously readable layers: + +1. `observation layer` + - collectors + - recent collector activity + - baseline coverage +2. `activity layer` + - recent event density + - anomaly/noise hotspots + - regional activity scoring + - incident presence bonus +3. `incident layer` + - sparse but highly legible, high-confidence event objects + - symbol-driven markers + - outward ring pulse instead of broad diffuse glow + +## Incident Visual Direction + +The Earth `incident` layer should not read like a large glowing patch. It should read like a compact, high-confidence event focus. + +Design principles: + +1. `incident` markers should use a strong primary symbol + - the symbol shape should carry type meaning where possible + - examples: + - `origin_change`: triangle-like warning marker + - `mass_withdrawal`: alert/exclamation-style marker + - `more_specific_burst`: split/radiating marker + +2. emphasis should come from outward ring pulses, not area flooding + - use a compact hot core + - use one or more expanding ring pulses + - avoid broad luminous blobs that make the event center feel vague + +3. `collector` and `incident` must stay visually distinct + - collectors are observation infrastructure + - incidents are extracted event focus + - collector activity should stay quieter than incident pulse language + +4. calm periods still need observability presence + - collectors and activity layers should keep the map alive + - once incidents appear, they should clearly dominate nearby BGP visuals + +5. incident geography should become `prefix-centric` + - collectors should remain evidence sources, not the primary event location + - preferred geography priority: + - `prefix_geography` + - `prefix_scope` + - `ASN organization region` + - `collector centroid` as final fallback + - `prefix_scope` should remain an observation-derived scope hint + - a new `prefix_geography` layer should be introduced for actual prefix-centric placement + +Reference inspiration: + +- `World Monitor` + - sparse event symbols + - compact centers + - ring-like outward pulses + - stronger incident legibility than diffuse glow + ## Current Console Behavior Relevant page: @@ -188,14 +283,15 @@ BGP-specific tests live in: Verified status at this point: -- `17 passed` +- `25 passed` for `backend/tests/test_bgp.py` +- `62 passed` for `backend/tests` Covered areas include: - normalization - observation serialization - enrichment -- detectors +- detectors, including route leak candidate and path flap - incident aggregation - batch anomaly creation - BGP events/incidents API @@ -226,9 +322,14 @@ Frontend: ## Recommended Next Steps -1. Expand realtime collector coverage and include withdrawals more broadly. -2. Integrate real RPKI validation data. -3. Improve route leak and path instability detectors. +### Next Backend / Detection Priority + +1. Integrate real RPKI validation data. +2. Expand realtime collector coverage and include withdrawals more broadly. +3. Continue refining route leak and path instability detectors with stronger heuristics. + +### Next Correlation / Storytelling Priority + 4. Strengthen incident aggregation semantics and titles. 5. Add weak correlation from incidents to: - cable corridors @@ -236,3 +337,12 @@ Frontend: - IXPs - other traffic anomaly sources 6. Refine Earth hover/click handoff between collectors and incidents. + +### Next Visualization Priority + +7. Refine regional activity scoring so the activity layer is informative without becoming noisy. +8. Add more incident symbol types as new detectors land. +9. Add a real prefix geography source: + - `IPtoASN / IPtoCountry` as the first practical dataset + - `OpenGeoFeed` as a higher-quality override layer + - registry/whois only as fallback diff --git a/docs/prefix-geography-plan.md b/docs/prefix-geography-plan.md new file mode 100644 index 00000000..32b6dcf8 --- /dev/null +++ b/docs/prefix-geography-plan.md @@ -0,0 +1,216 @@ +# Prefix Geography Plan + +## Goal + +Make Earth BGP incidents `prefix-centric` instead of `collector-centric`. + +The map should primarily answer: + +- where a prefix-related event is likely centered +- which regions the prefix is likely associated with +- which collectors observed the event as evidence + +It should not continue to imply that the event is located at the collector itself unless no better geography is available. + +## Why Current Geography Is Not Enough + +Current incident geography can still collapse back to collector-derived regions because: + +1. `prefix_scope` is currently built mostly from observed collector regions and historical observation regions. +2. `origin_asn_profile` currently comes from `peeringdb_network`, which is useful for ASN footprint hints but not sufficient as a primary prefix location source. +3. `collector centroid` is still a common fallback and therefore dominates sparse incidents. + +This makes Earth feel like a collector map with event decorations instead of a prefix impact map. + +## Data Source Layers + +Prefix geography should be built from four layers, ordered by confidence. + +### Layer 1. Prefix-to-country / prefix-to-region + +This is the primary source layer and the current missing piece. + +Recommended sources: + +1. `IPtoASN / IPtoCountry` + - URL: + - Good fit for this project because it provides downloadable IPv4/IPv6 range-to-ASN and range-to-country mappings. + - Best use: + - map a prefix to country code + - enrich prefixes with coarse regional placement + +2. `OpenGeoFeed` + - URL: + - Best use: + - override coarse country mappings when the prefix holder publishes a geofeed + - provide a more realistic deployment/service region than whois-style registration country + +### Layer 2. Registry allocation fallback + +Use these only as fallback signals, not as a ground-truth physical location. + +Candidate inputs: + +- RIR delegated stats +- `inetnum` / `inet6num` whois + +Best use: + +- detect registration country / allocation region +- provide fallback when no direct prefix geolocation dataset is available + +### Layer 3. ASN footprint hints + +Existing in this project: + +- `peeringdb_network` +- `peeringdb_facility` +- `peeringdb_ixp` + +Best use: + +- derive ASN city/country footprint +- identify likely exchange/facility regions +- act as secondary evidence when prefix-specific geography is unavailable + +### Layer 4. Observation evidence + +Existing in this project: + +- `RIPE RIS Live` +- `CAIDA BGPStream Backfill` + +Best use: + +- prove who observed the event +- derive affected observation regions +- support impact evidence + +This should remain the final fallback and evidence layer, not the primary event geography. + +## Recommended Geography Priority + +The backend should compute incident geography with this order: + +1. `prefix_geography` + - prefix-to-country / region / geofeed-backed result +2. `asn_region` + - ASN organization / facility / IXP footprint +3. `collector_centroid` + - observed collector regions only as final fallback + +Returned GeoJSON should keep exposing the selected mode through: + +- `geography_mode = prefix_geography | asn_region | collector_centroid` + +## Proposed Backend Changes + +### 1. Add a dedicated prefix geography dataset + +New datasource candidates: + +- `ip2asn_prefix_geo` +- optionally `opengeofeed_prefix_geo` + +Suggested storage model: + +- keep downloaded rows in `CollectedData` first for speed of integration +- later move to a dedicated table if lookup volume grows + +Minimum normalized fields: + +- `range_start` +- `range_end` +- `prefix` +- `country` +- `continent` +- `asn` +- `as_name` +- `source` +- `confidence` + +### 2. Add prefix geography enrichment + +Extend: + +- `backend/app/services/bgp_enrichment.py` + +New enrichment payload should include: + +- `prefix_geography` + - `country` + - `continent` + - `regions` + - `source` + - `confidence` + +This should be separate from the current `prefix_scope`. + +Suggested distinction: + +- `prefix_scope` + - observation-derived scope hint +- `prefix_geography` + - prefix-centric geography estimate + +### 3. Update incident visualization geography selection + +Extend: + +- `backend/app/api/v1/visualization.py` + +Selection order: + +1. `prefix_geography.regions` +2. ASN geography hints from PeeringDB-derived profile +3. observation-derived `affected_regions` + +### 4. Keep evidence visible in the frontend + +Earth should distinguish: + +- event center = prefix geography estimate +- evidence lines / collectors = observation proof + +This keeps the event meaningful for non-expert users without losing collector evidence. + +## Earth UX Result + +After this change, a user should see: + +- an incident marker near the estimated affected prefix region +- collectors as supporting evidence, not as the event center itself +- cables / landing points / nearby infrastructure as weak correlation around the estimated region + +This makes BGP incidents readable as “where the event is likely happening or affecting”, instead of “which station saw it”. + +## Implementation Order + +### Phase 1 + +1. Add `IPtoASN / IPtoCountry` datasource support +2. Normalize rows into lookup-friendly format +3. Enrich BGP events with `prefix_geography` +4. Switch incident geography priority to prefer `prefix_geography` + +### Phase 2 + +5. Add `OpenGeoFeed` support +6. Let geofeed override coarse country-level prefix geography +7. Add confidence scoring per geography source + +### Phase 3 + +8. Add RIR / whois fallback +9. Add better ASN regional footprint from PeeringDB facilities / IXPs +10. Refine Earth visual semantics for prefix geography vs observation evidence + +## Recommendation + +The best next engineering move is: + +1. integrate `IPtoASN / IPtoCountry` +2. model `prefix_geography` separately from `prefix_scope` +3. only then continue refining incident map placement + +Without this layer, any further Earth tuning will still be constrained by collector-centric data. diff --git a/frontend/index.html b/frontend/index.html index 67fed84b..a7137230 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -2,7 +2,7 @@ - + 智能星球计划 diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 906af7a7..ec7cb9ac 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "planet-frontend", - "version": "0.22.8", + "version": "0.22.10", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "planet-frontend", - "version": "0.22.8", + "version": "0.22.10", "dependencies": { "@ant-design/icons": "^5.2.6", "antd": "^5.12.5", @@ -16,7 +16,9 @@ "react-dom": "^18.2.0", "react-resizable": "^3.1.3", "react-router-dom": "^6.21.0", + "simplex-noise": "^4.0.1", "socket.io-client": "^4.7.2", + "three": "^0.160.0", "zustand": "^4.4.7" }, "devDependencies": { @@ -3009,6 +3011,12 @@ "semver": "bin/semver.js" } }, + "node_modules/simplex-noise": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/simplex-noise/-/simplex-noise-4.0.3.tgz", + "integrity": "sha512-qSE2I4AngLQG7BXqoZj51jokT4WUXe8mOBrvfOXpci8+6Yu44+/dD5zqDpOx3Ux792eamTd2lLcI8jqFntk/lg==", + "license": "MIT" + }, "node_modules/socket.io-client": { "version": "4.8.3", "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.8.3.tgz", @@ -3059,6 +3067,12 @@ "integrity": "sha512-yQ3rwFWRfwNUY7H5vpU0wfdkNSnvnJinhF9830Swlaxl03zsOjCfmX0ugac+3LtK0lYSgwL/KXc8oYL3mG4YFQ==", "license": "MIT" }, + "node_modules/three": { + "version": "0.160.1", + "resolved": "https://registry.npmjs.org/three/-/three-0.160.1.tgz", + "integrity": "sha512-Bgl2wPJypDOZ1stAxwfWAcJ0WQf7QzlptsxkjYiURPz+n5k4RBDLsq+6f9Y75TYxn6aHLcWz+JNmwTOXWrQTBQ==", + "license": "MIT" + }, "node_modules/throttle-debounce": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/throttle-debounce/-/throttle-debounce-5.0.2.tgz", diff --git a/frontend/package.json b/frontend/package.json index 575bf960..f2abbec9 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "planet-frontend", - "version": "0.22.9", + "version": "0.22.10", "private": true, "dependencies": { "@ant-design/icons": "^5.2.6", diff --git a/frontend/public/earth/assets/earth_clouds_1024.png b/frontend/public/earth/assets/earth_clouds_1024.png new file mode 100644 index 00000000..5c6b17b7 Binary files /dev/null and b/frontend/public/earth/assets/earth_clouds_1024.png differ diff --git a/frontend/public/earth/js/bgp.js b/frontend/public/earth/js/bgp.js index 1bcb0575..712bac6b 100644 --- a/frontend/public/earth/js/bgp.js +++ b/frontend/public/earth/js/bgp.js @@ -13,7 +13,9 @@ let showBGP = true; let totalAnomalyCount = 0; let totalIncidentCount = 0; let textureCache = null; +let eventRingTextureCache = null; let collectorTextureCache = null; +const eventTextureCache = new Map(); let activeEventOverlay = null; let activeCollectorOverlayContext = null; const relativeTimeFormatter = new Intl.RelativeTimeFormat("zh-CN", { @@ -60,6 +62,29 @@ function getMarkerTexture() { return textureCache; } +function getEventRingTexture() { + if (eventRingTextureCache) return eventRingTextureCache; + + const canvas = document.createElement("canvas"); + canvas.width = 128; + canvas.height = 128; + const context = canvas.getContext("2d"); + if (!context) { + eventRingTextureCache = new THREE.Texture(canvas); + return eventRingTextureCache; + } + + context.clearRect(0, 0, 128, 128); + context.strokeStyle = "rgba(255,255,255,0.98)"; + context.lineWidth = 6; + context.beginPath(); + context.arc(64, 64, 44, 0, Math.PI * 2); + context.stroke(); + + eventRingTextureCache = new THREE.CanvasTexture(canvas); + return eventRingTextureCache; +} + function getCollectorTexture() { if (collectorTextureCache) return collectorTextureCache; @@ -102,6 +127,121 @@ function getCollectorTexture() { return collectorTextureCache; } +function getEventSymbolKind(anomalyType) { + const value = String(anomalyType || "").toLowerCase(); + if (value.includes("origin")) return "triangle"; + if (value.includes("withdraw")) return "exclamation"; + if (value.includes("specific") || value.includes("burst")) return "burst"; + if (value.includes("flap")) return "wave"; + if (value.includes("leak")) return "leak"; + return "dot"; +} + +function drawTriangleSymbol(context) { + context.beginPath(); + context.moveTo(64, 18); + context.lineTo(110, 106); + context.lineTo(18, 106); + context.closePath(); + context.fill(); +} + +function drawExclamationSymbol(context) { + context.beginPath(); + context.roundRect(52, 22, 24, 62, 12); + context.fill(); + context.beginPath(); + context.arc(64, 102, 10, 0, Math.PI * 2); + context.fill(); +} + +function drawWaveSymbol(context) { + context.lineWidth = 12; + context.lineCap = "round"; + context.beginPath(); + context.moveTo(18, 76); + context.bezierCurveTo(34, 46, 46, 46, 64, 76); + context.bezierCurveTo(80, 106, 94, 106, 110, 76); + context.stroke(); +} + +function drawBurstSymbol(context) { + context.lineWidth = 10; + context.lineCap = "round"; + for (let index = 0; index < 6; index += 1) { + const angle = (Math.PI * 2 * index) / 6; + const inner = 26; + const outer = 48; + context.beginPath(); + context.moveTo(64 + Math.cos(angle) * inner, 64 + Math.sin(angle) * inner); + context.lineTo(64 + Math.cos(angle) * outer, 64 + Math.sin(angle) * outer); + context.stroke(); + } + context.beginPath(); + context.arc(64, 64, 16, 0, Math.PI * 2); + context.fill(); +} + +function drawLeakSymbol(context) { + context.lineWidth = 10; + context.lineCap = "round"; + context.beginPath(); + context.moveTo(28, 96); + context.lineTo(64, 28); + context.lineTo(100, 96); + context.stroke(); + context.beginPath(); + context.moveTo(40, 82); + context.lineTo(64, 54); + context.lineTo(88, 82); + context.stroke(); +} + +function drawDotSymbol(context) { + context.beginPath(); + context.arc(64, 64, 28, 0, Math.PI * 2); + context.fill(); +} + +function getEventTexture(anomalyType) { + const kind = getEventSymbolKind(anomalyType); + if (eventTextureCache.has(kind)) return eventTextureCache.get(kind); + + const canvas = document.createElement("canvas"); + canvas.width = 128; + canvas.height = 128; + const context = canvas.getContext("2d"); + if (!context) { + const fallback = new THREE.Texture(canvas); + eventTextureCache.set(kind, fallback); + return fallback; + } + + context.clearRect(0, 0, 128, 128); + context.fillStyle = "rgba(255,255,255,0.96)"; + context.strokeStyle = "rgba(255,255,255,0.96)"; + context.shadowBlur = 0; + context.lineJoin = "round"; + + if (kind === "triangle") { + drawTriangleSymbol(context); + } else if (kind === "exclamation") { + drawExclamationSymbol(context); + } else if (kind === "wave") { + drawWaveSymbol(context); + } else if (kind === "burst") { + drawBurstSymbol(context); + } else if (kind === "leak") { + drawLeakSymbol(context); + } else { + drawDotSymbol(context); + } + + const texture = new THREE.CanvasTexture(canvas); + eventTextureCache.set(kind, texture); + return texture; +} + function normalizeSeverity(severity) { const value = String(severity || "").trim().toLowerCase(); @@ -934,9 +1074,14 @@ function createCollectorMarker(markerData) { function createAnomalyMarker(markerData) { const sprite = new THREE.Sprite( - createSpriteMaterial({ + new THREE.SpriteMaterial({ + map: getEventTexture(markerData.incident_type || markerData.anomaly_type), color: getSeverityColor(markerData.severity), + transparent: true, opacity: BGP_CONFIG.opacity.normal, + depthWrite: false, + depthTest: true, + blending: THREE.NormalBlending, }), ); @@ -960,12 +1105,45 @@ function createAnomalyMarker(markerData) { ...markerData, }; + const ringA = new THREE.Sprite( + new THREE.SpriteMaterial({ + map: getEventRingTexture(), + color: getSeverityColor(markerData.severity), + transparent: true, + opacity: 0, + depthWrite: false, + depthTest: true, + blending: THREE.AdditiveBlending, + }), + ); + ringA.scale.setScalar(baseScale * BGP_CONFIG.eventRingScaleA); + ringA.position.set(0, 0, -0.01); + sprite.add(ringA); + + const ringB = new THREE.Sprite( + new THREE.SpriteMaterial({ + map: getEventRingTexture(), + color: getSeverityColor(markerData.severity), + transparent: true, + opacity: 0, + depthWrite: false, + depthTest: true, + blending: THREE.AdditiveBlending, + }), + ); + ringB.scale.setScalar(baseScale * BGP_CONFIG.eventRingScaleB); + ringB.position.set(0, 0, -0.02); + sprite.add(ringB); + + sprite.userData.ringA = ringA; + sprite.userData.ringB = ringB; + anomalyMarkers.push(sprite); bgpGroup.add(sprite); } function dedupeAnomalies(features) { - const latestByCollector = new Map(); + const latestByLocation = new Map(); features.forEach((feature) => { const data = buildAnomalyFeatureData(feature); @@ -976,22 +1154,30 @@ function dedupeAnomalies(features) { (activeEventCountByCollector.get(data.collector) || 0) + 1, ); - const dedupeKey = `${data.collector}|${data.latitude.toFixed(4)}|${data.longitude.toFixed(4)}`; - const previous = latestByCollector.get(dedupeKey); + const dedupeKey = `${data.latitude.toFixed(3)}|${data.longitude.toFixed(3)}`; + const previous = latestByLocation.get(dedupeKey); const currentTime = data.created_at_raw ? new Date(data.created_at_raw).getTime() : 0; const previousTime = previous?.created_at_raw ? new Date(previous.created_at_raw).getTime() : 0; + const currentSeverity = getSeverityScale(data.severity); + const previousSeverity = previous ? getSeverityScale(previous.severity) : 0; - if (!previous || currentTime >= previousTime) { - latestByCollector.set(dedupeKey, data); + if ( + !previous || + currentSeverity > previousSeverity || + (currentSeverity === previousSeverity && currentTime >= previousTime) + ) { + latestByLocation.set(dedupeKey, data); } }); - return Array.from(latestByCollector.values()) + return Array.from(latestByLocation.values()) .sort((a, b) => { + const severityDiff = getSeverityScale(b.severity) - getSeverityScale(a.severity); + if (severityDiff !== 0) return severityDiff; const timeA = a.created_at_raw ? new Date(a.created_at_raw).getTime() : 0; const timeB = b.created_at_raw ? new Date(b.created_at_raw).getTime() : 0; return timeB - timeA; @@ -1000,7 +1186,7 @@ function dedupeAnomalies(features) { } function dedupeIncidents(features) { - const latestByKey = new Map(); + const latestByLocation = new Map(); features.forEach((feature) => { const data = buildIncidentFeatureData(feature); @@ -1013,22 +1199,30 @@ function dedupeIncidents(features) { ); }); - const dedupeKey = String(data.incident_key || data.id); - const previous = latestByKey.get(dedupeKey); + const dedupeKey = `${data.latitude.toFixed(3)}|${data.longitude.toFixed(3)}`; + const previous = latestByLocation.get(dedupeKey); const currentTime = data.created_at_raw ? new Date(data.created_at_raw).getTime() : 0; const previousTime = previous?.created_at_raw ? new Date(previous.created_at_raw).getTime() : 0; + const currentSeverity = getSeverityScale(data.severity); + const previousSeverity = previous ? getSeverityScale(previous.severity) : 0; - if (!previous || currentTime >= previousTime) { - latestByKey.set(dedupeKey, data); + if ( + !previous || + currentSeverity > previousSeverity || + (currentSeverity === previousSeverity && currentTime >= previousTime) + ) { + latestByLocation.set(dedupeKey, data); } }); - return Array.from(latestByKey.values()) + return Array.from(latestByLocation.values()) .sort((a, b) => { + const severityDiff = getSeverityScale(b.severity) - getSeverityScale(a.severity); + if (severityDiff !== 0) return severityDiff; const timeA = a.created_at_raw ? new Date(a.created_at_raw).getTime() : 0; const timeB = b.created_at_raw ? new Date(b.created_at_raw).getTime() : 0; return timeB - timeA; @@ -1046,25 +1240,40 @@ function applyCollectorCounts() { export async function loadBGPAnomalies(scene, earth) { clearBGPData(earth); - const [collectorsResponse, incidentsResponse, anomaliesResponse] = await Promise.all([ - fetch(PATHS.bgpCollectorsApi), - fetch(`${PATHS.bgpIncidentsApi}?limit=${BGP_CONFIG.defaultFetchLimit}`), - fetch(`${PATHS.bgpApi}?limit=${BGP_CONFIG.defaultFetchLimit}`), - ]); - + const collectorsResponse = await fetch(PATHS.bgpCollectorsApi); if (!collectorsResponse.ok) { throw new Error(`BGP collectors HTTP ${collectorsResponse.status}`); } - if (!incidentsResponse.ok) { - throw new Error(`BGP incidents HTTP ${incidentsResponse.status}`); + + let anomaliesPayload = { type: "FeatureCollection", features: [], count: 0 }; + try { + const anomaliesResponse = await fetch( + `${PATHS.bgpApi}?limit=${BGP_CONFIG.defaultFetchLimit}`, + { signal: AbortSignal.timeout(5000) }, + ); + if (!anomaliesResponse.ok) { + throw new Error(`BGP anomalies HTTP ${anomaliesResponse.status}`); + } + anomaliesPayload = await anomaliesResponse.json(); + } catch (error) { + console.warn("BGP anomalies unavailable, falling back to collectors only:", error); } - if (!anomaliesResponse.ok) { - throw new Error(`BGP anomalies HTTP ${anomaliesResponse.status}`); + + let incidentsPayload = { type: "FeatureCollection", features: [], count: 0 }; + try { + const incidentsResponse = await fetch( + `${PATHS.bgpIncidentsApi}?limit=${BGP_CONFIG.defaultFetchLimit}`, + { signal: AbortSignal.timeout(5000) }, + ); + if (!incidentsResponse.ok) { + throw new Error(`BGP incidents HTTP ${incidentsResponse.status}`); + } + incidentsPayload = await incidentsResponse.json(); + } catch (error) { + console.warn("BGP incidents unavailable, falling back to anomalies:", error); } const collectorsPayload = await collectorsResponse.json(); - const incidentsPayload = await incidentsResponse.json(); - const anomaliesPayload = await anomaliesResponse.json(); const collectorFeatures = Array.isArray(collectorsPayload?.features) ? collectorsPayload.features : []; @@ -1232,28 +1441,59 @@ export function updateBGPVisualState(lockedObjectType, lockedObject, camera) { let scale = marker.userData.baseScale; let opacity = BGP_CONFIG.opacity.normal; let markerColor = marker.userData.baseColor || getSeverityColor(marker.userData.severity); + const isIncidentMarker = marker.userData.source === "bgp_incident"; + let ringBaseOpacity = isIncidentMarker + ? BGP_CONFIG.eventRingOpacity + : BGP_CONFIG.eventRingOpacity * 0.45; if (isLocked || isLinkedCollectorLocked) { scale *= 1 + BGP_CONFIG.lockedPulseAmplitude * pulse; opacity = BGP_CONFIG.opacity.lockedMin + (BGP_CONFIG.opacity.lockedMax - BGP_CONFIG.opacity.lockedMin) * pulse; + ringBaseOpacity *= 1.2; } else if (isHovered) { scale *= BGP_CONFIG.hoverScale; opacity = BGP_CONFIG.opacity.hover; + ringBaseOpacity *= 1.05; } else if (isOtherLocked) { scale *= BGP_CONFIG.dimmedScale; opacity = 0.1; markerColor = 0x7d8ca3; + ringBaseOpacity = 0.02; } else { scale *= 1 + BGP_CONFIG.normalPulseAmplitude * pulse; - opacity = BGP_CONFIG.opacity.normal; + opacity = isIncidentMarker ? 0.7 : 0.62; } marker.scale.setScalar(scale); marker.material.color.setHex(markerColor); marker.material.opacity = opacity; marker.visible = showBGP; + + const ringPhaseA = (now * BGP_CONFIG.eventRingSpeed + marker.userData.pulseOffset) % 1; + const applyRingState = (ring, phase, maxScale) => { + if (!ring) return; + const progress = Math.max(0, Math.min(1, phase)); + const minScale = 1.28; + const desiredWorldScale = + marker.userData.baseScale * (minScale + progress * (maxScale - minScale)); + const parentScale = Math.max(scale, 0.0001); + const localRingScale = desiredWorldScale / parentScale; + const fadeIn = Math.max(0, Math.min(1, (progress - 0.08) / 0.14)); + const fadeOut = 1 - progress; + const visibility = fadeIn * fadeOut; + ring.scale.setScalar(localRingScale); + ring.material.color.setHex(markerColor); + ring.material.opacity = showBGP ? ringBaseOpacity * visibility : 0; + ring.visible = showBGP; + }; + + applyRingState(marker.userData.ringA, ringPhaseA, BGP_CONFIG.eventRingScaleA); + if (marker.userData.ringB) { + marker.userData.ringB.material.opacity = 0; + marker.userData.ringB.visible = false; + } }); } @@ -1368,42 +1608,9 @@ export function showBGPEventOverlay(marker, earth) { typeof region?.longitude === "number", ); if (validRegions.length === 0) return; - - const averageLatitude = - validRegions.reduce((sum, region) => sum + region.latitude, 0) / - validRegions.length; - const averageLongitude = - validRegions.reduce((sum, region) => sum + region.longitude, 0) / - validRegions.length; - - const hubPosition = latLonToVector3( - averageLatitude, - averageLongitude, - CONFIG.earthRadius + BGP_CONFIG.eventHubAltitudeOffset, - ); - const hub = createOverlaySprite({ - color: BGP_CONFIG.eventHubColor, - opacity: 0.95, - scale: BGP_CONFIG.eventHubScale, - }); - hub.position.copy(hubPosition); - hub.renderOrder = 6; - bgpOverlayGroup.add(hub); - - const overlayItems = [hub]; + const overlayItems = []; validRegions.forEach((region) => { - const regionPosition = latLonToVector3( - region.latitude, - region.longitude, - CONFIG.earthRadius + BGP_CONFIG.collectorAltitudeOffset + 0.3, - ); - - const link = createArcLine(regionPosition, hubPosition, BGP_CONFIG.linkColor); - link.renderOrder = 4; - bgpOverlayGroup.add(link); - overlayItems.push(link); - const halo = createOverlaySprite({ color: BGP_CONFIG.regionColor, opacity: 0.24, diff --git a/frontend/public/earth/js/constants.js b/frontend/public/earth/js/constants.js index 7284ae57..5a4f47b0 100644 --- a/frontend/public/earth/js/constants.js +++ b/frontend/public/earth/js/constants.js @@ -138,6 +138,10 @@ export const BGP_CONFIG = { eventHubColor: 0x8af5ff, linkColor: 0x54d2ff, regionColor: 0x2dd4bf, + eventRingScaleA: 2.5, + eventRingScaleB: 3.4, + eventRingOpacity: 0.5, + eventRingSpeed: 0.001, collectorHaloScale: 11.5, collectorPulseHaloScale: 16.5, collectorCoverageHaloScale: 22.5 diff --git a/frontend/public/earth/js/earth.js b/frontend/public/earth/js/earth.js index f30934d9..2522eb48 100644 --- a/frontend/public/earth/js/earth.js +++ b/frontend/public/earth/js/earth.js @@ -104,7 +104,7 @@ export function createClouds(scene, earthObj) { earthObj.add(clouds); textureLoader.load( - 'https://threejs.org/examples/textures/planets/earth_clouds_1024.png', + './assets/earth_clouds_1024.png', function(texture) { material.map = texture; material.needsUpdate = true; diff --git a/pyproject.toml b/pyproject.toml index cd6090b6..d81f2744 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "planet" -version = "0.22.8" +version = "0.22.10" description = "智能星球计划 - 态势感知系统" requires-python = ">=3.14" dependencies = [