fix: stabilize earth bgp geography and rendering

This commit is contained in:
linkong
2026-04-02 15:36:20 +08:00
parent 07e4f519a1
commit e5fec8ba3d
26 changed files with 1788 additions and 137 deletions

16
TODO.md
View File

@@ -1,4 +1,16 @@
# TODO # TODO
- [ ] 把 BGP 观测站和异常点的 `hover/click` 手感再磨细一点 - [x] 把 BGP 观测站和异常点的 `hover/click` 手感再磨细一点
- [ ] 开始做 BGP 异常和海缆/区域的关联展示 - [x] 开始做 BGP 异常和海缆/区域的关联展示
- [x] 做 Earth 侧的 `BGP activity layer`,让低 incident 密度时地图仍然有持续可感知的观测存在感
- [x] 给 Earth BGP 补三层状态表达:`平稳观测态 / 局部波动态 / 事件活跃态`
- [x] 把“当前无活跃事件”改造成“观测网络仍在运行、当前未发现聚合级事件”的状态表达
- [x] 做 collector / region 近 15 分钟 activity score 聚合接口或动态聚合逻辑
- [x] 把 Earth 的 BGP incident 改成 `紧凑事件核 + 向外扩张环形 pulse`,替换当前大面积 glow
- [x] 为 BGP incident 建立符号系统:按事件类型用不同 marker而不是都用同一种亮点
- [x] 把 incident 地理定位从 `collector-centric` 改成 `prefix-centric`,优先使用 `prefix_geography`,其次 `prefix_scope`,再次 ASN 区域,最后才回退到观测区域质心
- [x] 新增 `prefix_geography` 数据层,不再把 `prefix_scope` 当成 prefix 地理归属本身
- [x] 接入 `IPtoASN / IPtoCountry` 作为 prefix-centric geography 的主数据源
- [ ] 接入 `OpenGeoFeed` 作为 prefix geography 的高质量覆盖/override 数据源
- [ ] 把 RIR delegated / `inetnum` / `inet6num` whois 设计成 prefix geography 的 fallback而不是主来源
- [x] 在 activity layer 之后继续补 `route leak``path instability / flap` detector

View File

@@ -1 +1 @@
0.22.9 0.22.10

View File

@@ -5,6 +5,7 @@ Returns GeoJSON format compatible with Three.js, CesiumJS, and Unreal Cesium.
""" """
from datetime import UTC, datetime from datetime import UTC, datetime
import math
from fastapi import APIRouter, HTTPException, Depends, Query from fastapi import APIRouter, HTTPException, Depends, Query
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import select, func from sqlalchemy import select, func
@@ -18,7 +19,8 @@ from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_incident import BGPIncident from app.models.bgp_incident import BGPIncident
from app.models.collected_data import CollectedData from app.models.collected_data import CollectedData
from app.services.bgp_collectors import build_bgp_collector_coverage from app.services.bgp_collectors import build_bgp_collector_coverage
from app.services.cable_graph import build_graph_from_data, CableGraph from app.services.bgp_enrichment import _lookup_prefix_geography
from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance
from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
router = APIRouter() router = APIRouter()
@@ -316,11 +318,16 @@ def convert_gpu_cluster_to_geojson(records: List[CollectedData]) -> Dict[str, An
return {"type": "FeatureCollection", "features": features} return {"type": "FeatureCollection", "features": features}
def convert_bgp_anomalies_to_geojson(records: List[BGPAnomaly]) -> Dict[str, Any]: def convert_bgp_anomalies_to_geojson(
records: List[BGPAnomaly],
geography_hints: Optional[Dict[str, Dict[str, Any]]] = None,
) -> Dict[str, Any]:
features = [] features = []
geography_hints = geography_hints or {}
for record in records: for record in records:
evidence = record.evidence or {} evidence = record.evidence or {}
hint = geography_hints.get(str(record.entity_key or record.id), {})
collectors = evidence.get("collectors") or record.peer_scope or [] collectors = evidence.get("collectors") or record.peer_scope or []
if not collectors: if not collectors:
nested = evidence.get("events") or [] nested = evidence.get("events") or []
@@ -334,23 +341,6 @@ def convert_bgp_anomalies_to_geojson(records: List[BGPAnomaly]) -> Dict[str, Any
if not collectors: if not collectors:
collectors = [] collectors = []
collector = collectors[0] if collectors else None
location = None
if collector:
location = RIPE_RIS_COLLECTOR_COORDS.get(str(collector))
if location is None:
nested = evidence.get("events") or []
for item in nested:
collector_name = (item or {}).get("collector")
if collector_name and collector_name in RIPE_RIS_COLLECTOR_COORDS:
location = RIPE_RIS_COLLECTOR_COORDS[collector_name]
collector = collector_name
break
if location is None:
continue
as_path = [] as_path = []
if isinstance(evidence.get("as_path"), list): if isinstance(evidence.get("as_path"), list):
as_path = evidence.get("as_path") or [] as_path = evidence.get("as_path") or []
@@ -385,6 +375,28 @@ def convert_bgp_anomalies_to_geojson(records: List[BGPAnomaly]) -> Dict[str, Any
} }
) )
geography_regions = _normalize_geo_regions(hint.get("regions") or [])
geography_mode = hint.get("geography_mode") or "collector_centroid"
collector = collectors[0] if collectors else None
location = geography_regions[0] if geography_regions else None
if location is None and collector:
location = RIPE_RIS_COLLECTOR_COORDS.get(str(collector))
if location is None:
nested = evidence.get("events") or []
for item in nested:
collector_name = (item or {}).get("collector")
if collector_name and collector_name in RIPE_RIS_COLLECTOR_COORDS:
location = RIPE_RIS_COLLECTOR_COORDS[collector_name]
collector = collector_name
geography_mode = "collector_centroid"
break
if location is None:
continue
features.append( features.append(
{ {
"type": "Feature", "type": "Feature",
@@ -408,6 +420,7 @@ def convert_bgp_anomalies_to_geojson(records: List[BGPAnomaly]) -> Dict[str, Any
"collector_count": len(collectors) or 1, "collector_count": len(collectors) or 1,
"as_path": as_path, "as_path": as_path,
"impacted_regions": impacted_regions, "impacted_regions": impacted_regions,
"geography_mode": geography_mode,
"confidence": record.confidence, "confidence": record.confidence,
"summary": record.summary, "summary": record.summary,
"created_at": to_iso8601_utc(record.created_at), "created_at": to_iso8601_utc(record.created_at),
@@ -418,6 +431,54 @@ def convert_bgp_anomalies_to_geojson(records: List[BGPAnomaly]) -> Dict[str, Any
return {"type": "FeatureCollection", "features": features} return {"type": "FeatureCollection", "features": features}
async def build_anomaly_geography_hints(
db: AsyncSession,
records: List[BGPAnomaly],
) -> Dict[str, Dict[str, Any]]:
hints: Dict[str, Dict[str, Any]] = {}
for record in records:
evidence = record.evidence or {}
key = str(record.entity_key or record.id)
prefix_geo_regions = []
prefix_regions = []
asn_regions = []
evidence_prefix_geography = evidence.get("prefix_geography") or {}
prefix_geo_regions.extend(
_normalize_geo_regions(evidence_prefix_geography.get("regions") or [])
)
prefix_scope = evidence.get("prefix_scope") or {}
prefix_regions.extend(_normalize_geo_regions(prefix_scope.get("regions") or []))
for profile_key in ("origin_asn_profile", "new_origin_asn_profile"):
profile = evidence.get(profile_key) or {}
latitude = profile.get("latitude")
longitude = profile.get("longitude")
if isinstance(latitude, (int, float)) and isinstance(longitude, (int, float)):
asn_regions.append(
{
"country": profile.get("country"),
"city": profile.get("city"),
"latitude": float(latitude),
"longitude": float(longitude),
}
)
prefix_geo_regions = _normalize_geo_regions(prefix_geo_regions)
prefix_regions = _normalize_geo_regions(prefix_regions)
asn_regions = _normalize_geo_regions(asn_regions)
if prefix_geo_regions:
hints[key] = {"regions": prefix_geo_regions, "geography_mode": "prefix_geography"}
elif prefix_regions:
hints[key] = {"regions": prefix_regions, "geography_mode": "prefix_scope"}
elif asn_regions:
hints[key] = {"regions": asn_regions, "geography_mode": "asn_region"}
return hints
def convert_bgp_collectors_to_geojson( def convert_bgp_collectors_to_geojson(
coverage_by_collector: Dict[str, Dict[str, Any]] | None = None, coverage_by_collector: Dict[str, Dict[str, Any]] | None = None,
) -> Dict[str, Any]: ) -> Dict[str, Any]:
@@ -442,8 +503,10 @@ def convert_bgp_collectors_to_geojson(
"prefix_count": coverage.get("prefix_count", 0), "prefix_count": coverage.get("prefix_count", 0),
"origin_asn_count": coverage.get("origin_asn_count", 0), "origin_asn_count": coverage.get("origin_asn_count", 0),
"peer_asn_count": coverage.get("peer_asn_count", 0), "peer_asn_count": coverage.get("peer_asn_count", 0),
"recent_15m_observation_count": coverage.get("recent_15m_observation_count", 0),
"recent_24h_observation_count": coverage.get("recent_24h_observation_count", 0), "recent_24h_observation_count": coverage.get("recent_24h_observation_count", 0),
"recent_7d_observation_count": coverage.get("recent_7d_observation_count", 0), "recent_7d_observation_count": coverage.get("recent_7d_observation_count", 0),
"recent_15m_prefix_count": coverage.get("recent_15m_prefix_count", 0),
"recent_24h_prefix_count": coverage.get("recent_24h_prefix_count", 0), "recent_24h_prefix_count": coverage.get("recent_24h_prefix_count", 0),
"recent_7d_prefix_count": coverage.get("recent_7d_prefix_count", 0), "recent_7d_prefix_count": coverage.get("recent_7d_prefix_count", 0),
"top_event_types": coverage.get("top_event_types", []), "top_event_types": coverage.get("top_event_types", []),
@@ -463,33 +526,183 @@ def convert_bgp_collectors_to_geojson(
return {"type": "FeatureCollection", "features": features} return {"type": "FeatureCollection", "features": features}
def convert_bgp_incidents_to_geojson(records: List[BGPIncident]) -> Dict[str, Any]: def _incident_estimated_center(valid_regions: List[Dict[str, Any]]) -> Dict[str, float]:
x = 0.0
y = 0.0
z = 0.0
for region in valid_regions:
lat_rad = math.radians(float(region["latitude"]))
lon_rad = math.radians(float(region["longitude"]))
x += math.cos(lat_rad) * math.cos(lon_rad)
y += math.cos(lat_rad) * math.sin(lon_rad)
z += math.sin(lat_rad)
total = float(len(valid_regions))
if total <= 0:
return {"latitude": 0.0, "longitude": 0.0}
x /= total
y /= total
z /= total
hyp = math.sqrt((x * x) + (y * y))
if hyp == 0:
return {"latitude": 0.0, "longitude": 0.0}
return {
"latitude": math.degrees(math.atan2(z, hyp)),
"longitude": math.degrees(math.atan2(y, x)),
}
def _incident_estimated_radius_km(center: Dict[str, float], valid_regions: List[Dict[str, Any]]) -> float:
center_coords = (float(center["longitude"]), float(center["latitude"]))
distances = [
haversine_distance(
center_coords,
(float(region["longitude"]), float(region["latitude"])),
)
for region in valid_regions
]
return round(max(distances) if distances else 0.0, 1)
def _normalize_geo_regions(regions: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
normalized: list[dict[str, Any]] = []
seen: set[tuple[Any, ...]] = set()
for region in regions:
if not isinstance(region, dict):
continue
latitude = region.get("latitude")
longitude = region.get("longitude")
if not isinstance(latitude, (int, float)) or not isinstance(longitude, (int, float)):
continue
item = {
"collector": region.get("collector"),
"country": region.get("country"),
"city": region.get("city"),
"latitude": float(latitude),
"longitude": float(longitude),
}
key = (
item["collector"],
item["country"],
item["city"],
item["latitude"],
item["longitude"],
)
if key in seen:
continue
seen.add(key)
normalized.append(item)
return normalized
async def build_incident_geography_hints(
db: AsyncSession,
records: List[BGPIncident],
) -> Dict[str, Dict[str, Any]]:
evidence_refs = sorted(
{
str(ref)
for record in records
for ref in (record.evidence_refs or [])
if ref
}
)
anomalies = []
if evidence_refs:
result = await db.execute(
select(BGPAnomaly).where(BGPAnomaly.entity_key.in_(evidence_refs))
)
anomalies = result.scalars().all()
anomaly_by_key = {
str(anomaly.entity_key): anomaly
for anomaly in anomalies
if anomaly.entity_key
}
hints: Dict[str, Dict[str, Any]] = {}
for record in records:
prefix_geo_regions: list[dict[str, Any]] = []
prefix_regions: list[dict[str, Any]] = []
asn_regions: list[dict[str, Any]] = []
for ref in record.evidence_refs or []:
anomaly = anomaly_by_key.get(str(ref))
if anomaly is None:
continue
evidence = anomaly.evidence or {}
if not prefix_geo_regions:
prefix_geography = evidence.get("prefix_geography") or {}
prefix_geo_regions.extend(_normalize_geo_regions(prefix_geography.get("regions") or []))
prefix_scope = evidence.get("prefix_scope") or {}
prefix_regions.extend(_normalize_geo_regions(prefix_scope.get("regions") or []))
for key in ("origin_asn_profile", "new_origin_asn_profile"):
profile = evidence.get(key) or {}
latitude = profile.get("latitude")
longitude = profile.get("longitude")
if isinstance(latitude, (int, float)) and isinstance(longitude, (int, float)):
asn_regions.append(
{
"country": profile.get("country"),
"city": profile.get("city"),
"latitude": float(latitude),
"longitude": float(longitude),
}
)
prefix_geo_regions = _normalize_geo_regions(prefix_geo_regions)
prefix_regions = _normalize_geo_regions(prefix_regions)
asn_regions = _normalize_geo_regions(asn_regions)
if prefix_geo_regions:
hints[record.incident_key] = {
"regions": prefix_geo_regions,
"geography_mode": "prefix_geography",
}
elif prefix_regions:
hints[record.incident_key] = {
"regions": prefix_regions,
"geography_mode": "prefix_scope",
}
elif asn_regions:
hints[record.incident_key] = {
"regions": asn_regions,
"geography_mode": "asn_region",
}
return hints
def convert_bgp_incidents_to_geojson(
records: List[BGPIncident],
geography_hints: Optional[Dict[str, Dict[str, Any]]] = None,
) -> Dict[str, Any]:
features = [] features = []
for record in records: for record in records:
regions = record.affected_regions or [] hint = (geography_hints or {}).get(record.incident_key, {})
regions = hint.get("regions") or (record.affected_regions or [])
if not regions: if not regions:
continue continue
valid_regions = [ valid_regions = _normalize_geo_regions(regions)
region
for region in regions
if isinstance(region, dict)
and isinstance(region.get("latitude"), (int, float))
and isinstance(region.get("longitude"), (int, float))
]
if not valid_regions: if not valid_regions:
continue continue
avg_lat = sum(float(region["latitude"]) for region in valid_regions) / len(valid_regions) estimated_center = _incident_estimated_center(valid_regions)
avg_lon = sum(float(region["longitude"]) for region in valid_regions) / len(valid_regions) estimated_radius_km = _incident_estimated_radius_km(estimated_center, valid_regions)
features.append( features.append(
{ {
"type": "Feature", "type": "Feature",
"geometry": { "geometry": {
"type": "Point", "type": "Point",
"coordinates": [avg_lon, avg_lat], "coordinates": [
estimated_center["longitude"],
estimated_center["latitude"],
],
}, },
"properties": { "properties": {
"id": record.id, "id": record.id,
@@ -504,6 +717,9 @@ def convert_bgp_incidents_to_geojson(records: List[BGPIncident]) -> Dict[str, An
"affected_asns": record.affected_asns or [], "affected_asns": record.affected_asns or [],
"affected_collectors": record.affected_collectors or [], "affected_collectors": record.affected_collectors or [],
"affected_regions": valid_regions, "affected_regions": valid_regions,
"estimated_center": estimated_center,
"estimated_radius_km": estimated_radius_km,
"geography_mode": hint.get("geography_mode") or "collector_centroid",
"related_cables": record.related_cables or [], "related_cables": record.related_cables or [],
"related_ixps": record.related_ixps or [], "related_ixps": record.related_ixps or [],
"created_at": to_iso8601_utc(record.created_at), "created_at": to_iso8601_utc(record.created_at),
@@ -739,7 +955,8 @@ async def get_bgp_anomalies_geojson(
result = await db.execute(stmt) result = await db.execute(stmt)
records = list(result.scalars().all()) records = list(result.scalars().all())
geojson = convert_bgp_anomalies_to_geojson(records) geography_hints = await build_anomaly_geography_hints(db, records)
geojson = convert_bgp_anomalies_to_geojson(records, geography_hints)
return {**geojson, "count": len(geojson.get("features", []))} return {**geojson, "count": len(geojson.get("features", []))}
@@ -758,7 +975,8 @@ async def get_bgp_incidents_geojson(
result = await db.execute(stmt) result = await db.execute(stmt)
records = list(result.scalars().all()) records = list(result.scalars().all())
geojson = convert_bgp_incidents_to_geojson(records) geography_hints = await build_incident_geography_hints(db, records)
geojson = convert_bgp_incidents_to_geojson(records, geography_hints)
return {**geojson, "count": len(geojson.get("features", []))} return {**geojson, "count": len(geojson.get("features", []))}

View File

@@ -232,6 +232,30 @@ for canonical, aliases in COUNTRY_ENTRIES:
COUNTRY_ALIAS_MAP[alias.casefold()] = canonical COUNTRY_ALIAS_MAP[alias.casefold()] = canonical
COUNTRY_CENTROIDS = {
"美国": {"latitude": 39.8283, "longitude": -98.5795},
"英国": {"latitude": 55.3781, "longitude": -3.4360},
"荷兰": {"latitude": 52.1326, "longitude": 5.2913},
"日本": {"latitude": 36.2048, "longitude": 138.2529},
"德国": {"latitude": 51.1657, "longitude": 10.4515},
"法国": {"latitude": 46.2276, "longitude": 2.2137},
"新加坡": {"latitude": 1.3521, "longitude": 103.8198},
"中国": {"latitude": 35.8617, "longitude": 104.1954},
"中国(香港)": {"latitude": 22.3193, "longitude": 114.1694},
"中国(台湾)": {"latitude": 23.6978, "longitude": 120.9605},
"韩国": {"latitude": 35.9078, "longitude": 127.7669},
"俄罗斯": {"latitude": 61.5240, "longitude": 105.3188},
"加拿大": {"latitude": 56.1304, "longitude": -106.3468},
"澳大利亚": {"latitude": -25.2744, "longitude": 133.7751},
"巴西": {"latitude": -14.2350, "longitude": -51.9253},
"南非": {"latitude": -30.5595, "longitude": 22.9375},
"西班牙": {"latitude": 40.4637, "longitude": -3.7492},
"意大利": {"latitude": 41.8719, "longitude": 12.5674},
"瑞士": {"latitude": 46.8182, "longitude": 8.2275},
"阿联酋": {"latitude": 23.4241, "longitude": 53.8478},
}
def normalize_country(value: Any) -> Optional[str]: def normalize_country(value: Any) -> Optional[str]:
if value is None: if value is None:
return None return None
@@ -258,6 +282,13 @@ def normalize_country(value: Any) -> Optional[str]:
return COUNTRY_ALIAS_MAP.get(lowered) return COUNTRY_ALIAS_MAP.get(lowered)
def get_country_centroid(value: Any) -> Optional[dict[str, float]]:
canonical = normalize_country(value)
if not canonical:
return None
return COUNTRY_CENTROIDS.get(canonical)
def get_country_search_variants(value: Any) -> list[str]: def get_country_search_variants(value: Any) -> list[str]:
canonical = normalize_country(value) canonical = normalize_country(value)
if canonical is None: if canonical is None:

View File

@@ -25,6 +25,7 @@ COLLECTOR_URL_KEYS = {
"spacetrack_tle": "spacetrack.tle_query_url", "spacetrack_tle": "spacetrack.tle_query_url",
"ris_live_bgp": "ris_live.url", "ris_live_bgp": "ris_live.url",
"bgpstream_bgp": "bgpstream.url", "bgpstream_bgp": "bgpstream.url",
"iptoasn_prefix_geo": "iptoasn.combined_url",
} }

View File

@@ -43,3 +43,6 @@ ris_live:
bgpstream: bgpstream:
url: "https://broker.bgpstream.caida.org/v2" url: "https://broker.bgpstream.caida.org/v2"
iptoasn:
combined_url: "https://iptoasn.com/data/ip2asn-combined.tsv.gz"

View File

@@ -134,6 +134,13 @@ DEFAULT_DATASOURCES = {
"priority": "P1", "priority": "P1",
"frequency_minutes": 360, "frequency_minutes": 360,
}, },
"iptoasn_prefix_geo": {
"id": 23,
"name": "IPtoASN Prefix Geography",
"module": "L3",
"priority": "P1",
"frequency_minutes": 1440,
},
} }
ID_TO_COLLECTOR = {info["id"]: name for name, info in DEFAULT_DATASOURCES.items()} ID_TO_COLLECTOR = {info["id"]: name for name, info in DEFAULT_DATASOURCES.items()}

View File

@@ -20,6 +20,7 @@ async def build_bgp_collector_coverage(
source_filter: tuple[str, ...] | None = None, source_filter: tuple[str, ...] | None = None,
) -> list[dict[str, Any]]: ) -> list[dict[str, Any]]:
now = datetime.now(UTC) now = datetime.now(UTC)
recent_15m_threshold = now - timedelta(minutes=15)
recent_24h_threshold = now - timedelta(hours=24) recent_24h_threshold = now - timedelta(hours=24)
recent_7d_threshold = now - timedelta(days=7) recent_7d_threshold = now - timedelta(days=7)
@@ -52,8 +53,10 @@ async def build_bgp_collector_coverage(
"event_types": defaultdict(int), "event_types": defaultdict(int),
"countries": set(), "countries": set(),
"cities": set(), "cities": set(),
"recent_15m_observation_count": 0,
"recent_24h_observation_count": 0, "recent_24h_observation_count": 0,
"recent_7d_observation_count": 0, "recent_7d_observation_count": 0,
"recent_15m_prefixes": set(),
"recent_24h_prefixes": set(), "recent_24h_prefixes": set(),
"recent_7d_prefixes": set(), "recent_7d_prefixes": set(),
"latest_observed_at": None, "latest_observed_at": None,
@@ -78,6 +81,10 @@ async def build_bgp_collector_coverage(
if observed_at.tzinfo if observed_at.tzinfo
else observed_at.replace(tzinfo=UTC) else observed_at.replace(tzinfo=UTC)
) )
if aware_observed_at >= recent_15m_threshold:
coverage["recent_15m_observation_count"] += 1
if record.prefix:
coverage["recent_15m_prefixes"].add(record.prefix)
if aware_observed_at >= recent_24h_threshold: if aware_observed_at >= recent_24h_threshold:
coverage["recent_24h_observation_count"] += 1 coverage["recent_24h_observation_count"] += 1
if record.prefix: if record.prefix:
@@ -116,8 +123,10 @@ async def build_bgp_collector_coverage(
"event_types": defaultdict(int), "event_types": defaultdict(int),
"countries": {location.get("country")} if location.get("country") else set(), "countries": {location.get("country")} if location.get("country") else set(),
"cities": {location.get("city")} if location.get("city") else set(), "cities": {location.get("city")} if location.get("city") else set(),
"recent_15m_observation_count": 0,
"recent_24h_observation_count": 0, "recent_24h_observation_count": 0,
"recent_7d_observation_count": 0, "recent_7d_observation_count": 0,
"recent_15m_prefixes": set(),
"recent_24h_prefixes": set(), "recent_24h_prefixes": set(),
"recent_7d_prefixes": set(), "recent_7d_prefixes": set(),
"latest_observed_at": None, "latest_observed_at": None,
@@ -142,8 +151,10 @@ async def build_bgp_collector_coverage(
"prefix_count": len(item["prefixes"]), "prefix_count": len(item["prefixes"]),
"origin_asn_count": len(item["origin_asns"]), "origin_asn_count": len(item["origin_asns"]),
"peer_asn_count": len(item["peer_asns"]), "peer_asn_count": len(item["peer_asns"]),
"recent_15m_observation_count": item["recent_15m_observation_count"],
"recent_24h_observation_count": item["recent_24h_observation_count"], "recent_24h_observation_count": item["recent_24h_observation_count"],
"recent_7d_observation_count": item["recent_7d_observation_count"], "recent_7d_observation_count": item["recent_7d_observation_count"],
"recent_15m_prefix_count": len(item["recent_15m_prefixes"]),
"recent_24h_prefix_count": len(item["recent_24h_prefixes"]), "recent_24h_prefix_count": len(item["recent_24h_prefixes"]),
"recent_7d_prefix_count": len(item["recent_7d_prefixes"]), "recent_7d_prefix_count": len(item["recent_7d_prefixes"]),
"top_event_types": [ "top_event_types": [

View File

@@ -55,6 +55,11 @@ def _unique_peers(events: list[dict[str, Any]]) -> list[int]:
return sorted(peers) return sorted(peers)
def _path_signature(metadata: dict[str, Any]) -> tuple[int, ...]:
path = metadata.get("as_path") or []
return tuple(int(asn) for asn in path if asn is not None)
def detect_origin_change_anomalies( def detect_origin_change_anomalies(
*, *,
source: str, source: str,
@@ -100,6 +105,7 @@ def detect_origin_change_anomalies(
) )
sample_metadata = sample_event.get("metadata") or {} sample_metadata = sample_event.get("metadata") or {}
sample_enrichment = sample_metadata.get("enrichment") or {} sample_enrichment = sample_metadata.get("enrichment") or {}
sample_prefix_geography = sample_enrichment.get("prefix_geography") or {}
anomaly_type = "origin_change" anomaly_type = "origin_change"
severity = "critical" severity = "critical"
confidence = 0.86 confidence = 0.86
@@ -140,8 +146,10 @@ def detect_origin_change_anomalies(
"origin_asn_profile": sample_enrichment.get("origin_asn_profile"), "origin_asn_profile": sample_enrichment.get("origin_asn_profile"),
"new_origin_asn_profile": sample_enrichment.get("new_origin_asn_profile"), "new_origin_asn_profile": sample_enrichment.get("new_origin_asn_profile"),
"rpki_validation": sample_enrichment.get("rpki_validation"), "rpki_validation": sample_enrichment.get("rpki_validation"),
"prefix_geography": sample_prefix_geography,
"prefix_scope": sample_enrichment.get("prefix_scope"), "prefix_scope": sample_enrichment.get("prefix_scope"),
"impacted_regions": related_regions "impacted_regions": sample_prefix_geography.get("regions")
or related_regions
or sample_enrichment.get("prefix_scope", {}).get("regions", []), or sample_enrichment.get("prefix_scope", {}).get("regions", []),
}, },
) )
@@ -180,6 +188,7 @@ def detect_more_specific_burst_anomalies(
sample = more_specifics[0].get("metadata") or {} sample = more_specifics[0].get("metadata") or {}
sample_enrichment = sample.get("enrichment") or {} sample_enrichment = sample.get("enrichment") or {}
sample_prefix_geography = sample_enrichment.get("prefix_geography") or {}
event_count = len(more_specifics) event_count = len(more_specifics)
anomalies.append( anomalies.append(
BGPAnomaly( BGPAnomaly(
@@ -205,8 +214,10 @@ def detect_more_specific_burst_anomalies(
"unique_prefixes": unique_prefixes, "unique_prefixes": unique_prefixes,
"rpki_validation": sample_enrichment.get("rpki_validation"), "rpki_validation": sample_enrichment.get("rpki_validation"),
"origin_asn_profile": sample_enrichment.get("origin_asn_profile"), "origin_asn_profile": sample_enrichment.get("origin_asn_profile"),
"prefix_geography": sample_prefix_geography,
"prefix_scope": sample_enrichment.get("prefix_scope"), "prefix_scope": sample_enrichment.get("prefix_scope"),
"impacted_regions": _iter_event_regions(more_specifics) "impacted_regions": sample_prefix_geography.get("regions")
or _iter_event_regions(more_specifics)
or sample_enrichment.get("prefix_scope", {}).get("regions", []), or sample_enrichment.get("prefix_scope", {}).get("regions", []),
}, },
) )
@@ -242,6 +253,7 @@ def detect_mass_withdrawal_anomalies(
sample_event = related_events[0] if related_events else {} sample_event = related_events[0] if related_events else {}
sample_metadata = sample_event.get("metadata") or {} sample_metadata = sample_event.get("metadata") or {}
sample_enrichment = sample_metadata.get("enrichment") or {} sample_enrichment = sample_metadata.get("enrichment") or {}
sample_prefix_geography = sample_enrichment.get("prefix_geography") or {}
severity = "medium" severity = "medium"
if count >= 4 or len(related_collectors) >= 3: if count >= 4 or len(related_collectors) >= 3:
severity = "high" severity = "high"
@@ -277,8 +289,175 @@ def detect_mass_withdrawal_anomalies(
], ],
"origin_asn_profile": sample_enrichment.get("origin_asn_profile"), "origin_asn_profile": sample_enrichment.get("origin_asn_profile"),
"rpki_validation": sample_enrichment.get("rpki_validation"), "rpki_validation": sample_enrichment.get("rpki_validation"),
"prefix_geography": sample_prefix_geography,
"prefix_scope": sample_enrichment.get("prefix_scope"), "prefix_scope": sample_enrichment.get("prefix_scope"),
"impacted_regions": _iter_event_regions(related_events) "impacted_regions": sample_prefix_geography.get("regions")
or _iter_event_regions(related_events)
or sample_enrichment.get("prefix_scope", {}).get("regions", []),
},
)
)
return anomalies
def detect_route_leak_anomalies(
*,
source: str,
snapshot_id: int | None,
task_id: int | None,
events: list[dict[str, Any]],
) -> list[BGPAnomaly]:
events_by_prefix: defaultdict[str, list[dict[str, Any]]] = defaultdict(list)
for event in events:
metadata = event.get("metadata") or {}
prefix = metadata.get("prefix")
if prefix and metadata.get("event_type") == "announcement":
events_by_prefix[str(prefix)].append(event)
anomalies: list[BGPAnomaly] = []
for prefix, related_events in events_by_prefix.items():
related_collectors = _unique_collectors(related_events)
if len(related_collectors) < 2:
continue
path_signatures = Counter()
max_path_length = 0
for event in related_events:
metadata = event.get("metadata") or {}
signature = _path_signature(metadata)
if signature:
path_signatures[signature] += 1
max_path_length = max(max_path_length, len(signature))
if len(path_signatures) < 2:
continue
dominant_length = len(path_signatures.most_common(1)[0][0])
if max_path_length < max(dominant_length + 2, 5):
continue
sample_event = max(
related_events,
key=lambda event: len(_path_signature((event.get("metadata") or {}))),
)
sample_metadata = sample_event.get("metadata") or {}
sample_enrichment = sample_metadata.get("enrichment") or {}
sample_prefix_geography = sample_enrichment.get("prefix_geography") or {}
peer_scope = related_collectors
path_lengths = sorted({len(signature) for signature in path_signatures if signature})
anomalies.append(
BGPAnomaly(
snapshot_id=snapshot_id,
task_id=task_id,
source=source,
anomaly_type="route_leak_candidate",
severity="high" if max_path_length >= dominant_length + 3 else "medium",
status="active",
entity_key=f"route_leak_candidate:{prefix}:{max_path_length}:{len(related_collectors)}",
prefix=prefix,
origin_asn=sample_metadata.get("origin_asn"),
new_origin_asn=None,
peer_scope=peer_scope,
started_at=datetime.now(UTC),
confidence=min(0.58 + (0.05 * min(len(related_collectors), 4)) + (0.03 * min(max_path_length - dominant_length, 4)), 0.88),
summary=(
f"Prefix {prefix} shows divergent long AS paths across "
f"{len(related_collectors)} collectors, suggesting a possible route leak."
),
evidence={
"path_lengths": path_lengths,
"dominant_path_length": dominant_length,
"max_path_length": max_path_length,
"path_signatures": [
{"path": list(signature), "count": count}
for signature, count in path_signatures.most_common(5)
],
"events": [(item.get("metadata") or {}) for item in related_events[:10]],
"origin_asn_profile": sample_enrichment.get("origin_asn_profile"),
"rpki_validation": sample_enrichment.get("rpki_validation"),
"prefix_geography": sample_prefix_geography,
"prefix_scope": sample_enrichment.get("prefix_scope"),
"impacted_regions": sample_prefix_geography.get("regions")
or _iter_event_regions(related_events)
or sample_enrichment.get("prefix_scope", {}).get("regions", []),
},
)
)
return anomalies
def detect_path_flap_anomalies(
*,
source: str,
snapshot_id: int | None,
task_id: int | None,
events: list[dict[str, Any]],
) -> list[BGPAnomaly]:
events_by_prefix: defaultdict[str, list[dict[str, Any]]] = defaultdict(list)
for event in events:
metadata = event.get("metadata") or {}
prefix = metadata.get("prefix")
if prefix:
events_by_prefix[str(prefix)].append(event)
anomalies: list[BGPAnomaly] = []
for prefix, related_events in events_by_prefix.items():
ordered = sorted(
related_events,
key=lambda event: str((event.get("metadata") or {}).get("timestamp") or ""),
)
event_types = [str((item.get("metadata") or {}).get("event_type") or "") for item in ordered]
transitions = sum(1 for index in range(1, len(event_types)) if event_types[index] != event_types[index - 1])
distinct_paths = {
_path_signature(item.get("metadata") or {})
for item in ordered
if _path_signature(item.get("metadata") or {})
}
related_collectors = _unique_collectors(ordered)
if transitions < 3 and len(distinct_paths) < 3:
continue
sample_metadata = (ordered[0].get("metadata") or {}) if ordered else {}
sample_enrichment = sample_metadata.get("enrichment") or {}
sample_prefix_geography = sample_enrichment.get("prefix_geography") or {}
severity = "medium"
if transitions >= 5 or len(distinct_paths) >= 4:
severity = "high"
anomalies.append(
BGPAnomaly(
snapshot_id=snapshot_id,
task_id=task_id,
source=source,
anomaly_type="path_flap",
severity=severity,
status="active",
entity_key=f"path_flap:{prefix}:{transitions}:{len(distinct_paths)}",
prefix=prefix,
origin_asn=sample_metadata.get("origin_asn"),
new_origin_asn=None,
peer_scope=related_collectors,
started_at=datetime.now(UTC),
confidence=min(0.54 + (0.05 * min(transitions, 5)) + (0.03 * min(len(distinct_paths), 4)), 0.9),
summary=(
f"Prefix {prefix} shows repeated state/path changes "
f"({transitions} transitions, {len(distinct_paths)} distinct paths) in the current window."
),
evidence={
"transitions": transitions,
"event_types": event_types[:12],
"distinct_paths": [list(path) for path in list(distinct_paths)[:6]],
"events": [(item.get("metadata") or {}) for item in ordered[:10]],
"origin_asn_profile": sample_enrichment.get("origin_asn_profile"),
"rpki_validation": sample_enrichment.get("rpki_validation"),
"prefix_geography": sample_prefix_geography,
"prefix_scope": sample_enrichment.get("prefix_scope"),
"impacted_regions": sample_prefix_geography.get("regions")
or _iter_event_regions(ordered)
or sample_enrichment.get("prefix_scope", {}).get("regions", []), or sample_enrichment.get("prefix_scope", {}).get("regions", []),
}, },
) )

View File

@@ -7,9 +7,10 @@ from collections import defaultdict
from datetime import UTC, datetime from datetime import UTC, datetime
from typing import Any from typing import Any
from sqlalchemy import select from sqlalchemy import select, text
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.core.countries import get_country_centroid, normalize_country
from app.models.bgp_observation import BGPObservation from app.models.bgp_observation import BGPObservation
from app.models.collected_data import CollectedData from app.models.collected_data import CollectedData
@@ -96,6 +97,83 @@ def extract_bgp_network_fields(prefix: str) -> dict[str, Any]:
} }
async def _lookup_prefix_geography(
db: AsyncSession,
prefix_values: list[str],
) -> dict[str, dict[str, Any]]:
results: dict[str, dict[str, Any]] = {}
for prefix in prefix_values:
try:
network = ipaddress.ip_network(prefix, strict=False)
except ValueError:
continue
family = f"ipv{network.version}"
range_start = str(network.network_address)
range_end = str(network.broadcast_address)
result = await db.execute(
text(
"""
SELECT metadata
FROM collected_data
WHERE source = 'iptoasn_prefix_geo'
AND COALESCE(is_current, TRUE) = TRUE
AND metadata->>'family' = :family
AND CAST(metadata->>'range_start' AS inet) <= CAST(:range_start AS inet)
AND CAST(metadata->>'range_end' AS inet) >= CAST(:range_end AS inet)
ORDER BY id DESC
LIMIT 1
"""
),
{
"family": family,
"range_start": range_start,
"range_end": range_end,
},
)
row = result.fetchone()
if not row:
continue
if isinstance(row, dict):
payload = row.get("metadata") or row.get("extra_data")
elif hasattr(row, "_mapping"):
payload = row._mapping.get("metadata") or row._mapping.get("extra_data")
else:
payload = row[0]
if not isinstance(payload, dict):
continue
country = normalize_country(payload.get("country") or payload.get("country_code"))
prefix_hint = payload.get("prefix") or prefix
asn = _safe_int(payload.get("asn"))
as_name = payload.get("as_name")
centroid = get_country_centroid(country)
regions = []
if country:
regions.append(
{
"country": country,
"city": None,
"latitude": centroid.get("latitude") if centroid else None,
"longitude": centroid.get("longitude") if centroid else None,
}
)
results[prefix] = {
"prefix": prefix_hint,
"country": country,
"asn": asn,
"as_name": as_name,
"source": payload.get("source_dataset") or "iptoasn_combined",
"confidence": "country_range",
"regions": regions,
}
return results
async def enrich_bgp_events_for_batch( async def enrich_bgp_events_for_batch(
db: AsyncSession, db: AsyncSession,
*, *,
@@ -165,6 +243,7 @@ async def enrich_bgp_events_for_batch(
} }
asn_profiles: dict[int, dict[str, Any]] = {} asn_profiles: dict[int, dict[str, Any]] = {}
prefix_geographies = await _lookup_prefix_geography(db, prefix_values) if prefix_values else {}
if origin_asns: if origin_asns:
peeringdb_result = await db.execute( peeringdb_result = await db.execute(
select(CollectedData).where(CollectedData.source == "peeringdb_network") select(CollectedData).where(CollectedData.source == "peeringdb_network")
@@ -207,21 +286,12 @@ async def enrich_bgp_events_for_batch(
collector = str(metadata.get("collector") or "").strip() collector = str(metadata.get("collector") or "").strip()
collector_location = metadata.get("collector_location") or {} collector_location = metadata.get("collector_location") or {}
baseline = historical_prefix_baseline.get(prefix, {}) baseline = historical_prefix_baseline.get(prefix, {})
prefix_geography = prefix_geographies.get(prefix)
observed_at = _parse_timestamp(metadata.get("timestamp") or event.get("reference_date")) observed_at = _parse_timestamp(metadata.get("timestamp") or event.get("reference_date"))
origin_asn = _safe_int(metadata.get("origin_asn")) origin_asn = _safe_int(metadata.get("origin_asn"))
new_origin_asn = _safe_int(metadata.get("new_origin_asn")) new_origin_asn = _safe_int(metadata.get("new_origin_asn"))
observed_regions = _compact_locations(
[
{
"country": collector_location.get("country"),
"city": collector_location.get("city"),
"latitude": collector_location.get("latitude"),
"longitude": collector_location.get("longitude"),
}
]
)
baseline_regions = baseline.get("historical_regions", []) baseline_regions = baseline.get("historical_regions", [])
prefix_scope_regions = _compact_locations([*observed_regions, *baseline_regions]) prefix_scope_regions = _compact_locations([*baseline_regions])
enrichment = { enrichment = {
**extract_bgp_network_fields(prefix), **extract_bgp_network_fields(prefix),
@@ -248,6 +318,7 @@ async def enrich_bgp_events_for_batch(
}, },
"origin_asn_profile": asn_profiles.get(origin_asn), "origin_asn_profile": asn_profiles.get(origin_asn),
"new_origin_asn_profile": asn_profiles.get(new_origin_asn), "new_origin_asn_profile": asn_profiles.get(new_origin_asn),
"prefix_geography": prefix_geography,
"prefix_scope": { "prefix_scope": {
"countries": sorted( "countries": sorted(
{ {

View File

@@ -209,15 +209,14 @@ async def create_bgp_incidents_for_anomalies(
grouped.setdefault(incident_key, []).append(anomaly) grouped.setdefault(incident_key, []).append(anomaly)
existing_result = await db.execute( existing_result = await db.execute(
select(BGPIncident.incident_key).where(BGPIncident.incident_key.in_(sorted(grouped.keys()))) select(BGPIncident).where(BGPIncident.incident_key.in_(sorted(grouped.keys())))
) )
existing_keys = {row[0] for row in existing_result.fetchall()} existing_incidents = {
incident.incident_key: incident for incident in existing_result.scalars().all()
}
created = 0 created = 0
for incident_key, items in grouped.items(): for incident_key, items in grouped.items():
if incident_key in existing_keys:
continue
items = sorted(items, key=lambda item: item.created_at or item.started_at or datetime.now(UTC)) items = sorted(items, key=lambda item: item.created_at or item.started_at or datetime.now(UTC))
primary = items[0] primary = items[0]
prefixes = sorted({item.prefix for item in items if item.prefix}) prefixes = sorted({item.prefix for item in items if item.prefix})
@@ -270,6 +269,28 @@ async def create_bgp_incidents_for_anomalies(
) )
related_infrastructure = await infer_related_infrastructure(db, regions) related_infrastructure = await infer_related_infrastructure(db, regions)
existing = existing_incidents.get(incident_key)
if existing is not None:
existing.snapshot_id = snapshot_id
existing.task_id = task_id
existing.source = source
existing.incident_type = primary.anomaly_type
existing.title = title
existing.summary = summary
existing.severity = severity
existing.status = "active"
existing.confidence = confidence
existing.started_at = primary.started_at or existing.started_at or datetime.now(UTC)
existing.ended_at = None
existing.affected_prefixes = prefixes
existing.affected_asns = asns
existing.affected_collectors = collectors
existing.affected_regions = regions
existing.related_cables = related_infrastructure["related_cables"]
existing.related_ixps = related_infrastructure["related_ixps"]
existing.evidence_refs = evidence_refs
continue
db.add( db.add(
BGPIncident( BGPIncident(
snapshot_id=snapshot_id, snapshot_id=snapshot_id,
@@ -294,7 +315,7 @@ async def create_bgp_incidents_for_anomalies(
) )
created += 1 created += 1
if created: if created or existing_incidents:
await db.commit() await db.commit()
return created return created

View File

@@ -32,6 +32,7 @@ from app.services.collectors.spacetrack import SpaceTrackTLECollector
from app.services.collectors.celestrak import CelesTrakTLECollector from app.services.collectors.celestrak import CelesTrakTLECollector
from app.services.collectors.ris_live import RISLiveCollector from app.services.collectors.ris_live import RISLiveCollector
from app.services.collectors.bgpstream import BGPStreamBackfillCollector from app.services.collectors.bgpstream import BGPStreamBackfillCollector
from app.services.collectors.iptoasn import IPtoASNPrefixGeoCollector
collector_registry.register(TOP500Collector()) collector_registry.register(TOP500Collector())
collector_registry.register(EpochAIGPUCollector()) collector_registry.register(EpochAIGPUCollector())
@@ -55,3 +56,4 @@ collector_registry.register(SpaceTrackTLECollector())
collector_registry.register(CelesTrakTLECollector()) collector_registry.register(CelesTrakTLECollector())
collector_registry.register(RISLiveCollector()) collector_registry.register(RISLiveCollector())
collector_registry.register(BGPStreamBackfillCollector()) collector_registry.register(BGPStreamBackfillCollector())
collector_registry.register(IPtoASNPrefixGeoCollector())

View File

@@ -18,6 +18,8 @@ from app.services.bgp_detectors import (
detect_mass_withdrawal_anomalies, detect_mass_withdrawal_anomalies,
detect_more_specific_burst_anomalies, detect_more_specific_burst_anomalies,
detect_origin_change_anomalies, detect_origin_change_anomalies,
detect_path_flap_anomalies,
detect_route_leak_anomalies,
) )
from app.services.bgp_enrichment import enrich_bgp_events_for_batch, extract_bgp_network_fields from app.services.bgp_enrichment import enrich_bgp_events_for_batch, extract_bgp_network_fields
@@ -282,6 +284,18 @@ async def create_bgp_anomalies_for_batch(
task_id=task_id, task_id=task_id,
events=enriched_events, events=enriched_events,
), ),
*detect_route_leak_anomalies(
source=source,
snapshot_id=snapshot_id,
task_id=task_id,
events=enriched_events,
),
*detect_path_flap_anomalies(
source=source,
snapshot_id=snapshot_id,
task_id=task_id,
events=enriched_events,
),
] ]
if not pending_anomalies: if not pending_anomalies:
@@ -302,16 +316,29 @@ async def create_bgp_anomalies_for_batch(
created = 0 created = 0
created_anomalies: list[BGPAnomaly] = [] created_anomalies: list[BGPAnomaly] = []
refreshed_anomalies: list[BGPAnomaly] = []
existing_map = {item.entity_key: item for item in existing_anomalies if item.entity_key}
for anomaly in pending_anomalies: for anomaly in pending_anomalies:
if anomaly.entity_key in existing_keys: if anomaly.entity_key in existing_keys:
existing = existing_map.get(anomaly.entity_key)
if existing is not None:
existing.severity = anomaly.severity
existing.status = anomaly.status
existing.summary = anomaly.summary
existing.confidence = anomaly.confidence
existing.peer_scope = anomaly.peer_scope
existing.evidence = anomaly.evidence
existing.new_origin_asn = anomaly.new_origin_asn
existing.origin_asn = anomaly.origin_asn
refreshed_anomalies.append(existing)
continue continue
db.add(anomaly) db.add(anomaly)
created_anomalies.append(anomaly) created_anomalies.append(anomaly)
created += 1 created += 1
if created: if created or refreshed_anomalies:
await db.commit() await db.commit()
incident_seed_anomalies = [*created_anomalies, *existing_anomalies] incident_seed_anomalies = [*created_anomalies, *refreshed_anomalies]
if incident_seed_anomalies: if incident_seed_anomalies:
await create_bgp_incidents_for_anomalies( await create_bgp_incidents_for_anomalies(
db, db,

View File

@@ -0,0 +1,111 @@
"""IPtoASN prefix geography collector.
Downloads the public combined IPv4+IPv6 TSV database and stores coarse
prefix-to-country/ASN geography hints for BGP enrichment.
"""
from __future__ import annotations
import gzip
from datetime import UTC, datetime
from ipaddress import summarize_address_range, ip_address
from typing import Any
import httpx
from app.services.collectors.base import BaseCollector
class IPtoASNPrefixGeoCollector(BaseCollector):
name = "iptoasn_prefix_geo"
priority = "P1"
module = "L3"
frequency_hours = 24
data_type = "prefix_geography"
fail_on_empty = True
async def fetch(self) -> list[dict[str, Any]]:
if not self._resolved_url:
raise RuntimeError("IPtoASN combined URL is not configured")
async with httpx.AsyncClient(timeout=180.0, follow_redirects=True) as client:
response = await client.get(
self._resolved_url,
headers={
"User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
"Accept": "application/gzip,application/octet-stream,*/*",
},
)
response.raise_for_status()
body = gzip.decompress(response.content).decode("utf-8", errors="replace")
rows: list[dict[str, Any]] = []
for raw_line in body.splitlines():
line = raw_line.strip()
if not line or line.startswith("#"):
continue
parts = line.split("\t")
if len(parts) < 5:
continue
range_start, range_end, asn, country_code, as_name = parts[:5]
rows.append(
{
"range_start": range_start,
"range_end": range_end,
"asn": asn,
"country_code": country_code,
"as_name": as_name,
}
)
return rows
def transform(self, raw_data: list[dict[str, Any]]) -> list[dict[str, Any]]:
reference_date = datetime.now(UTC).isoformat()
transformed: list[dict[str, Any]] = []
for item in raw_data:
try:
start_ip = ip_address(str(item["range_start"]))
end_ip = ip_address(str(item["range_end"]))
except ValueError:
continue
if start_ip.version != end_ip.version:
continue
summarized = list(summarize_address_range(start_ip, end_ip))
primary_prefix = str(summarized[0]) if summarized else f"{start_ip}/{32 if start_ip.version == 4 else 128}"
family = f"ipv{start_ip.version}"
asn_value = item.get("asn")
try:
normalized_asn = int(str(asn_value))
except (TypeError, ValueError):
normalized_asn = None
transformed.append(
{
"source_id": f"{family}:{item['range_start']}-{item['range_end']}",
"name": primary_prefix,
"title": f"{primary_prefix} {item.get('country_code', '').strip()}".strip(),
"country": item.get("country_code"),
"city": "",
"latitude": None,
"longitude": None,
"metadata": {
"family": family,
"range_start": item["range_start"],
"range_end": item["range_end"],
"prefix": primary_prefix,
"prefixes": [str(prefix) for prefix in summarized[:8]],
"range_prefix_count": len(summarized),
"country_code": item.get("country_code"),
"asn": normalized_asn,
"as_name": item.get("as_name"),
"source_dataset": "iptoasn_combined",
},
"reference_date": reference_date,
}
)
return transformed

View File

@@ -13,6 +13,8 @@ from app.main import app
from app.services.bgp_detectors import ( from app.services.bgp_detectors import (
detect_mass_withdrawal_anomalies, detect_mass_withdrawal_anomalies,
detect_origin_change_anomalies, detect_origin_change_anomalies,
detect_path_flap_anomalies,
detect_route_leak_anomalies,
) )
from app.services.collectors.bgp_common import ( from app.services.collectors.bgp_common import (
create_bgp_anomalies_for_batch, create_bgp_anomalies_for_batch,
@@ -24,6 +26,7 @@ from app.services.bgp_incidents import (
infer_related_infrastructure, infer_related_infrastructure,
) )
from app.services.bgp_collectors import build_bgp_collector_coverage from app.services.bgp_collectors import build_bgp_collector_coverage
from app.api.v1.visualization import convert_bgp_incidents_to_geojson, build_incident_geography_hints
from app.models.bgp_anomaly import BGPAnomaly from app.models.bgp_anomaly import BGPAnomaly
from app.models.collected_data import CollectedData from app.models.collected_data import CollectedData
from app.models.bgp_incident import BGPIncident from app.models.bgp_incident import BGPIncident
@@ -31,6 +34,7 @@ from app.models.bgp_observation import BGPObservation
from app.models.user import User from app.models.user import User
from app.services.collectors.bgp_common import normalize_bgp_event from app.services.collectors.bgp_common import normalize_bgp_event
from app.services.collectors.bgpstream import BGPStreamBackfillCollector from app.services.collectors.bgpstream import BGPStreamBackfillCollector
from app.services.collectors.iptoasn import IPtoASNPrefixGeoCollector
class _FakeScalarResult: class _FakeScalarResult:
@@ -51,6 +55,9 @@ class _FakeResult:
def fetchall(self): def fetchall(self):
return self._rows return self._rows
def fetchone(self):
return self._rows[0] if self._rows else None
class _FakeAsyncSession: class _FakeAsyncSession:
def __init__(self, results, gets=None): def __init__(self, results, gets=None):
@@ -59,7 +66,7 @@ class _FakeAsyncSession:
self.added = [] self.added = []
self.commits = 0 self.commits = 0
async def execute(self, _stmt): async def execute(self, _stmt, _params=None):
if not self._results: if not self._results:
return _FakeResult([]) return _FakeResult([])
return _FakeResult(self._results.pop(0)) return _FakeResult(self._results.pop(0))
@@ -140,6 +147,29 @@ def test_bgpstream_transform_preserves_broker_record():
assert record["metadata"]["broker_record"]["filename"] == "rib.20260326.0800.gz" assert record["metadata"]["broker_record"]["filename"] == "rib.20260326.0800.gz"
def test_iptoasn_transform_creates_prefix_geography_records():
collector = IPtoASNPrefixGeoCollector()
transformed = collector.transform(
[
{
"range_start": "1.0.0.0",
"range_end": "1.0.0.255",
"asn": "13335",
"country_code": "AU",
"as_name": "CLOUDFLARENET",
}
]
)
assert len(transformed) == 1
record = transformed[0]
assert record["name"] == "1.0.0.0/24"
assert record["metadata"]["family"] == "ipv4"
assert record["metadata"]["country_code"] == "AU"
assert record["metadata"]["asn"] == 13335
assert record["metadata"]["source_dataset"] == "iptoasn_combined"
def test_bgp_anomaly_to_dict(): def test_bgp_anomaly_to_dict():
anomaly = BGPAnomaly( anomaly = BGPAnomaly(
source="ris_live_bgp", source="ris_live_bgp",
@@ -315,6 +345,75 @@ def test_detect_mass_withdrawal_anomalies_accepts_cross_collector_pair():
assert anomalies[0].evidence["collector_count"] == 2 assert anomalies[0].evidence["collector_count"] == 2
def test_detect_route_leak_anomalies_creates_candidate_for_divergent_long_paths():
events = [
{
"metadata": {
"collector": "rrc00",
"event_type": "announcement",
"prefix": "203.0.113.0/24",
"origin_asn": 64496,
"as_path": [64500, 64496],
"collector_location": {"country": "NL", "city": "Amsterdam", "latitude": 52.3, "longitude": 4.9},
"enrichment": {"prefix_scope": {"regions": [{"country": "NL"}]}},
}
},
{
"metadata": {
"collector": "rrc01",
"event_type": "announcement",
"prefix": "203.0.113.0/24",
"origin_asn": 64496,
"as_path": [64510, 64520, 64530, 64540, 64496],
"collector_location": {"country": "GB", "city": "London", "latitude": 51.5, "longitude": -0.1},
"enrichment": {"prefix_scope": {"regions": [{"country": "GB"}]}},
}
},
]
anomalies = detect_route_leak_anomalies(
source="ris_live_bgp",
snapshot_id=1,
task_id=2,
events=events,
)
assert len(anomalies) == 1
assert anomalies[0].anomaly_type == "route_leak_candidate"
assert anomalies[0].evidence["max_path_length"] == 5
def test_detect_path_flap_anomalies_creates_signal_for_repeated_state_changes():
base_timestamp = datetime(2026, 3, 27, 0, 0, tzinfo=UTC)
events = []
for index, event_type in enumerate(["announcement", "withdrawal", "announcement", "withdrawal"]):
events.append(
{
"metadata": {
"collector": "rrc00",
"event_type": event_type,
"timestamp": (base_timestamp + timedelta(minutes=index)).isoformat(),
"prefix": "198.51.100.0/24",
"origin_asn": 64512,
"as_path": [64500 + index, 64512] if event_type == "announcement" else [],
"collector_location": {"country": "NL", "city": "Amsterdam", "latitude": 52.3, "longitude": 4.9},
"enrichment": {"prefix_scope": {"regions": [{"country": "NL"}]}},
}
}
)
anomalies = detect_path_flap_anomalies(
source="ris_live_bgp",
snapshot_id=1,
task_id=2,
events=events,
)
assert len(anomalies) == 1
assert anomalies[0].anomaly_type == "path_flap"
assert anomalies[0].evidence["transitions"] == 3
def test_bgp_incident_to_dict(): def test_bgp_incident_to_dict():
incident = BGPIncident( incident = BGPIncident(
source="ris_live_bgp", source="ris_live_bgp",
@@ -338,6 +437,131 @@ def test_bgp_incident_to_dict():
assert data["affected_collectors"] == ["rrc00", "rrc01"] assert data["affected_collectors"] == ["rrc00", "rrc01"]
def test_convert_bgp_incidents_to_geojson_adds_estimated_geography():
incident = BGPIncident(
source="ris_live_bgp",
incident_key="origin_change:203.0.113.0/24:64497",
incident_type="origin_change",
title="Origin Change incident on 203.0.113.0/24",
summary="Grouped incident summary",
severity="critical",
status="active",
confidence=0.91,
affected_prefixes=["203.0.113.0/24"],
affected_collectors=["rrc00", "rrc01"],
affected_regions=[
{
"collector": "rrc00",
"country": "Netherlands",
"city": "Amsterdam",
"latitude": 52.3676,
"longitude": 4.9041,
},
{
"collector": "rrc01",
"country": "United Kingdom",
"city": "London",
"latitude": 51.5072,
"longitude": -0.1276,
},
],
)
payload = convert_bgp_incidents_to_geojson([incident])
feature = payload["features"][0]
assert feature["properties"]["geography_mode"] == "collector_centroid"
assert feature["properties"]["estimated_radius_km"] > 0
assert feature["properties"]["estimated_center"]["latitude"] != 0
def test_convert_bgp_incidents_to_geojson_prefers_prefix_scope_hint():
incident = BGPIncident(
source="ris_live_bgp",
incident_key="origin_change:203.0.113.0/24:64497",
incident_type="origin_change",
title="Origin Change incident on 203.0.113.0/24",
summary="Grouped incident summary",
severity="critical",
status="active",
confidence=0.91,
affected_prefixes=["203.0.113.0/24"],
affected_collectors=["rrc00"],
affected_regions=[
{
"collector": "rrc00",
"country": "Netherlands",
"city": "Amsterdam",
"latitude": 52.3676,
"longitude": 4.9041,
}
],
)
payload = convert_bgp_incidents_to_geojson(
[incident],
{
incident.incident_key: {
"geography_mode": "prefix_scope",
"regions": [
{
"country": "Japan",
"city": "Tokyo",
"latitude": 35.6764,
"longitude": 139.65,
}
],
}
},
)
feature = payload["features"][0]
assert feature["properties"]["geography_mode"] == "prefix_scope"
assert feature["geometry"]["coordinates"] == [139.65, 35.6764]
def test_convert_bgp_incidents_to_geojson_prefers_prefix_geography_hint():
incident = BGPIncident(
source="ris_live_bgp",
incident_key="origin_change:198.51.100.0/24:64512",
incident_type="origin_change",
title="Origin Change incident on 198.51.100.0/24",
summary="Grouped incident summary",
severity="critical",
status="active",
confidence=0.91,
affected_prefixes=["198.51.100.0/24"],
affected_collectors=["rrc00"],
affected_regions=[
{
"collector": "rrc00",
"country": "Netherlands",
"city": "Amsterdam",
"latitude": 52.3676,
"longitude": 4.9041,
}
],
)
payload = convert_bgp_incidents_to_geojson(
[incident],
{
incident.incident_key: {
"geography_mode": "prefix_geography",
"regions": [
{
"country": "日本",
"city": None,
"latitude": 35.6764,
"longitude": 139.65,
}
],
}
},
)
feature = payload["features"][0]
assert feature["properties"]["geography_mode"] == "prefix_geography"
assert feature["geometry"]["coordinates"] == [139.65, 35.6764]
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_enrich_bgp_events_for_batch_adds_profiles_and_prefix_scope(): async def test_enrich_bgp_events_for_batch_adds_profiles_and_prefix_scope():
historical_observation = BGPObservation( historical_observation = BGPObservation(
@@ -367,8 +591,21 @@ async def test_enrich_bgp_events_for_batch_adds_profiles_and_prefix_scope():
}, },
) )
peeringdb_record.id = 99 peeringdb_record.id = 99
iptoasn_row = {
"extra_data": {
"family": "ipv4",
"range_start": "203.0.113.0",
"range_end": "203.0.113.255",
"prefix": "203.0.113.0/24",
"country": "英国",
"country_code": "GB",
"asn": 64497,
"as_name": "Example ASN",
"source_dataset": "iptoasn_combined",
}
}
db = _FakeAsyncSession([[historical_observation], [peeringdb_record]]) db = _FakeAsyncSession([[historical_observation], [iptoasn_row], [peeringdb_record]])
events = [ events = [
{ {
"metadata": { "metadata": {
@@ -396,8 +633,10 @@ async def test_enrich_bgp_events_for_batch_adds_profiles_and_prefix_scope():
assert enrichment["is_new_origin_for_prefix"] is True assert enrichment["is_new_origin_for_prefix"] is True
assert enrichment["rpki_validation"]["status"] == "unknown" assert enrichment["rpki_validation"]["status"] == "unknown"
assert enrichment["origin_asn_profile"]["name"] == "ExampleNet" assert enrichment["origin_asn_profile"]["name"] == "ExampleNet"
assert enrichment["prefix_scope"]["countries"] == ["Netherlands", "United Kingdom"] assert enrichment["prefix_geography"]["country"] == "英国"
assert enrichment["prefix_scope"]["cities"] == ["Amsterdam", "London"] assert enrichment["prefix_geography"]["source"] == "iptoasn_combined"
assert enrichment["prefix_scope"]["countries"] == ["United Kingdom"]
assert enrichment["prefix_scope"]["cities"] == ["London"]
@pytest.mark.asyncio @pytest.mark.asyncio
@@ -448,6 +687,126 @@ async def test_create_bgp_incidents_for_anomalies_aggregates_regions_and_collect
assert incident.affected_regions[0]["city"] == "Amsterdam" assert incident.affected_regions[0]["city"] == "Amsterdam"
@pytest.mark.asyncio
async def test_create_bgp_incidents_for_anomalies_refreshes_existing_incident():
existing = BGPIncident(
source="ris_live_bgp",
incident_key="origin_change:203.0.113.0/24:64497",
incident_type="origin_change",
title="Old title",
summary="Old summary",
severity="medium",
status="active",
confidence=0.4,
affected_prefixes=["203.0.113.0/24"],
affected_asns=[64496, 64497],
affected_collectors=["rrc00"],
affected_regions=[
{
"collector": "rrc00",
"country": "Netherlands",
"city": "Amsterdam",
"latitude": 52.3676,
"longitude": 4.9041,
}
],
related_cables=[],
related_ixps=[],
evidence_refs=["old-key"],
)
db = _FakeAsyncSession([[existing]])
anomaly = BGPAnomaly(
source="ris_live_bgp",
anomaly_type="origin_change",
severity="critical",
status="active",
entity_key="origin_change:203.0.113.0/24:64497",
prefix="203.0.113.0/24",
origin_asn=64496,
new_origin_asn=64497,
summary="Origin ASN changed",
confidence=0.9,
evidence={
"impacted_regions": [
{
"collector": None,
"country": "United States",
"city": None,
"latitude": 39.8283,
"longitude": -98.5795,
}
]
},
)
with patch(
"app.services.bgp_incidents.infer_related_infrastructure",
new=AsyncMock(return_value={"related_cables": [{"landing_point": "NYC"}], "related_ixps": []}),
):
created = await create_bgp_incidents_for_anomalies(
db,
source="ris_live_bgp",
snapshot_id=1,
task_id=2,
anomalies=[anomaly],
)
assert created == 0
assert db.commits == 1
assert len(db.added) == 0
assert existing.summary != "Old summary"
assert existing.severity == "critical"
assert existing.confidence == 0.9
assert existing.affected_regions[0]["country"] == "United States"
assert existing.related_cables == [{"landing_point": "NYC"}]
assert existing.evidence_refs == ["origin_change:203.0.113.0/24:64497"]
@pytest.mark.asyncio
async def test_build_incident_geography_hints_prefers_evidence_prefix_scope_when_no_cached_prefix_geo():
incident = BGPIncident(
source="ris_live_bgp",
incident_key="origin_change:93.175.153.0/24:16509",
incident_type="origin_change",
title="Origin Change incident on 93.175.153.0/24",
summary="summary",
severity="critical",
status="active",
affected_prefixes=["93.175.153.0/24"],
evidence_refs=["origin_change:93.175.153.0/24:16509"],
)
anomaly = BGPAnomaly(
source="ris_live_bgp",
anomaly_type="origin_change",
severity="critical",
status="active",
entity_key="origin_change:93.175.153.0/24:16509",
prefix="93.175.153.0/24",
origin_asn=12654,
new_origin_asn=16509,
summary="summary",
confidence=0.8,
evidence={
"prefix_scope": {
"regions": [
{
"country": "Netherlands",
"city": "Amsterdam",
"latitude": 52.3676,
"longitude": 4.9041,
}
]
}
},
)
db = _FakeAsyncSession([[anomaly]])
hints = await build_incident_geography_hints(db, [incident])
assert hints["origin_change:93.175.153.0/24:16509"]["geography_mode"] == "prefix_scope"
assert hints["origin_change:93.175.153.0/24:16509"]["regions"][0]["country"] == "Netherlands"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_infer_related_infrastructure_links_nearby_cables(): async def test_infer_related_infrastructure_links_nearby_cables():
landing = CollectedData( landing = CollectedData(
@@ -524,9 +883,11 @@ async def test_build_bgp_collector_coverage_summarizes_observations():
first = next(item for item in coverage if item["collector"] == "rrc00") first = next(item for item in coverage if item["collector"] == "rrc00")
assert first["observation_count"] == 2 assert first["observation_count"] == 2
assert first["recent_15m_observation_count"] == 2
assert first["recent_24h_observation_count"] == 2 assert first["recent_24h_observation_count"] == 2
assert first["recent_7d_observation_count"] == 2 assert first["recent_7d_observation_count"] == 2
assert first["prefix_count"] == 2 assert first["prefix_count"] == 2
assert first["recent_15m_prefix_count"] == 2
assert first["recent_24h_prefix_count"] == 2 assert first["recent_24h_prefix_count"] == 2
assert first["origin_asn_count"] == 2 assert first["origin_asn_count"] == 2
assert first["latest_event_type"] == "withdrawal" assert first["latest_event_type"] == "withdrawal"
@@ -583,6 +944,7 @@ async def test_create_bgp_anomalies_for_batch_calls_incident_aggregation():
extra_data={"prefix": "203.0.113.0/24", "origin_asn": 64496}, extra_data={"prefix": "203.0.113.0/24", "origin_asn": 64496},
) )
db = _FakeAsyncSession([ db = _FakeAsyncSession([
[],
[], [],
[], [],
[previous_record], [previous_record],
@@ -647,6 +1009,7 @@ async def test_create_bgp_anomalies_for_batch_skips_existing_entity_keys():
new_origin_asn=64497, new_origin_asn=64497,
) )
db = _FakeAsyncSession([ db = _FakeAsyncSession([
[],
[], [],
[], [],
[previous_record], [previous_record],

View File

@@ -7,6 +7,49 @@ This project follows the repository versioning rule:
- `feature` -> `+0.1.0` - `feature` -> `+0.1.0`
- `bugfix` -> `+0.0.1` - `bugfix` -> `+0.0.1`
## 0.22.10
Released: 2026-04-02
### Highlights
- Recovered the Earth-side BGP experience after a failed cache-busting / asset-loading refactor temporarily broke the globe runtime, removed textures, and made the BGP layer disappear when one backend endpoint timed out.
- Added a first usable `prefix_geography` data layer backed by `IPtoASN / IP-to-Country` ingestion so BGP geography can start moving away from pure collector-centric placement.
- Reworked BGP Earth rendering to keep collectors visible under degraded backend conditions, restore symbol-based incident markers, and split icon pulse from outward event-ring animation.
### Added
- Added a new `IPtoASN Prefix Geography` collector in [iptoasn.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/iptoasn.py) and registered it through [data_sources.yaml](/home/ray/dev/linkong/planet/backend/app/core/data_sources.yaml), [data_sources.py](/home/ray/dev/linkong/planet/backend/app/core/data_sources.py), [datasource_defaults.py](/home/ray/dev/linkong/planet/backend/app/core/datasource_defaults.py), and [collectors/__init__.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/__init__.py).
- Added country centroid helpers in [countries.py](/home/ray/dev/linkong/planet/backend/app/core/countries.py) so country-level prefix geography can produce map coordinates instead of only labels.
- Added a dedicated prefix-geography implementation note in [prefix-geography-plan.md](/home/ray/dev/linkong/planet/docs/prefix-geography-plan.md).
- Added recent `15m` collector activity dimensions to BGP coverage output in [bgp_collectors.py](/home/ray/dev/linkong/planet/backend/app/services/bgp_collectors.py) and [visualization.py](/home/ray/dev/linkong/planet/backend/app/api/v1/visualization.py).
- Added additional BGP detector coverage for `route_leak_candidate` and `path_flap` flows in [test_bgp.py](/home/ray/dev/linkong/planet/backend/tests/test_bgp.py).
- Added a local Earth cloud texture at [earth_clouds_1024.png](/home/ray/dev/linkong/planet/frontend/public/earth/assets/earth_clouds_1024.png) to avoid remote cloud-map dependency failures.
### Improved
- Improved BGP enrichment in [bgp_enrichment.py](/home/ray/dev/linkong/planet/backend/app/services/bgp_enrichment.py) so events now attach `prefix_geography`, `prefix_scope`, ASN profile context, and country-centroid-backed geography hints in one place.
- Improved incident aggregation in [bgp_incidents.py](/home/ray/dev/linkong/planet/backend/app/services/bgp_incidents.py) so existing incidents refresh their regions and geography metadata instead of remaining pinned to stale first-generation evidence forever.
- Improved anomaly generation flow in [bgp_common.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/bgp_common.py) by cleaning up duplicate incident-seeding paths and only feeding newly created or refreshed anomalies forward.
- Improved Earth BGP loading in [bgp.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp.js) so collectors are now the mandatory baseline layer while anomalies and incidents can fail independently without blanking the whole BGP surface.
- Improved Earth BGP marker language in [bgp.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp.js) and [constants.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/constants.js) by restoring typed event symbols, reducing additive white blowout, and making the incident ring animation read as an outward pulse instead of a generic glow blob.
- Improved Earth event animation semantics in [bgp.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp.js) by separating icon pulse from ring expansion so the center marker can breathe while the ring expands independently.
- Improved Earth texture reliability in [earth.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/earth.js) by switching clouds back to a local static asset under the restored `public/earth` runtime.
- Improved frontend boot noise in [frontend/index.html](/home/ray/dev/linkong/planet/frontend/index.html) by removing the default Vite favicon request that was generating irrelevant `vite.svg` timeouts during Earth debugging.
- Improved project planning docs in [bgp-context.md](/home/ray/dev/linkong/planet/docs/bgp-context.md) and [TODO.md](/home/ray/dev/linkong/planet/TODO.md) so the roadmap now explicitly prioritizes `activity layer`, `prefix-centric geography`, and follow-up geofeed/whois work.
### Fixed
- Fixed a failed Earth asset-versioning route where hand-applied cache-busting and a parallel Vite multi-entry experiment introduced duplicate module instances, broken `/earth` boot paths, missing textures, and severe runtime instability; the globe has now been restored to the stable `frontend/public/earth` runtime instead of the abandoned refactor path.
- Fixed Earth cloud and terrain loading regressions by restoring the old static Earth entrypoint and ensuring local cloud and 8K day-map assets resolve again from `public/earth/assets`.
- Fixed a full-layer BGP disappearance regression where `bgp-anomalies` or `bgp-incidents` timeouts caused the entire BGP layer to show `0` collectors and `0` events even though collector data still existed.
- Fixed `prefix_geography` lookups in [bgp_enrichment.py](/home/ray/dev/linkong/planet/backend/app/services/bgp_enrichment.py) that previously failed because JSON metadata access mixed SQL column names and ORM property names.
- Fixed stale anomaly and incident geography reuse so pre-existing records can now absorb refreshed evidence instead of staying locked to older Amsterdam-centric geography forever.
- Fixed a wrong optimization path in [visualization.py](/home/ray/dev/linkong/planet/backend/app/api/v1/visualization.py) where live `prefix_geography` lookups were pushed directly into Earth visualization endpoints, causing `bgp-anomalies` and `bgp-incidents` to time out under load; the visualization layer now prefers cached evidence again so Earth remains responsive.
- Fixed Earth-side BGP fallback rendering so anomaly fallback no longer collapses into a single undifferentiated glow layer when incidents are unavailable.
- Fixed extreme incident brightness and same-coordinate blowout in [bgp.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp.js) by removing additive blending from incident cores, reducing ring intensity, and deduplicating incident rendering at the coordinate level.
- Fixed the confusing floating BGP event hub in [bgp.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/bgp.js) by removing the suspended off-surface glow anchor and its arc links, leaving Earth incidents grounded on the globe surface with regional halo context instead.
## 0.22.9 ## 0.22.9
Released: 2026-04-02 Released: 2026-04-02

View File

@@ -6,7 +6,17 @@ The BGP module is being evolved from an anomaly-only demo into a layered observa
`raw observations -> enrichment -> detectors -> incidents -> console/Earth visualization` `raw observations -> enrichment -> detectors -> incidents -> console/Earth visualization`
The practical product goal is to turn low-level BGP control-plane changes into understandable network situation events with collector coverage, impact regions, and incident-centric visualization. The practical product goal is no longer just to "show incidents on the globe". The current product objective is:
1. keep BGP visually present on Earth even when incident density is low
2. make incidents clearly feel like a higher-confidence layer than anomalies
3. show that the observation network is still active even when there are no active incidents
In practice, that means Earth should behave like an observability surface, not only an incident map:
- `collectors` show that observation is happening
- `activity` shows where routing state is currently active or noisy
- `incidents` become the highest-confidence focus layer
## Current Backend Architecture ## Current Backend Architecture
@@ -135,16 +145,26 @@ Current design:
2. Incident markers are now the primary Earth BGP markers. 2. Incident markers are now the primary Earth BGP markers.
3. If there are no incidents, Earth falls back to anomaly markers. 3. If there are no incidents, Earth falls back to anomaly markers.
4. If there are no anomalies either, collectors still provide presence. 4. If there are no anomalies either, collectors still provide presence.
5. A dedicated `activity layer` now adds:
- per-collector recent 15-minute activity halos
- clustered regional activity hints derived from active collectors
6. Incident markers now use:
- symbol-driven event cores
- outward ring pulses
- reduced diffuse glow compared with older Earth builds
5. The right-side stats now show: 5. The right-side stats now show:
- BGP events - BGP events
- collector count - collector count
- BGP status summary - BGP status summary
This is directionally correct, but still incomplete for low-event-density periods. Right now Earth can still feel too quiet when incidents are sparse because the system lacks a dedicated `activity layer` between raw observation and incident focus.
Current BGP status strategy: Current BGP status strategy:
- incidents present: show active incident count - incidents present: show active incident count
- no incidents but anomalies present: show active anomaly count - no incidents but anomalies present: show active anomaly count, plus active observation regions when available
- no incidents/anomalies but collectors present: show `当前无活跃事件` - no incidents/anomalies but activity present: show `观测网络运行中`
- no incidents/anomalies but collectors present: show `观测网络运行中 · 当前未发现聚合级事件`
- no BGP data at all: show `暂无观测数据` - no BGP data at all: show `暂无观测数据`
Earth info-card strategy: Earth info-card strategy:
@@ -152,6 +172,81 @@ Earth info-card strategy:
- `bgp` card is now incident-centric in wording - `bgp` card is now incident-centric in wording
- `bgp_collector` card shows collector location and current event count - `bgp_collector` card shows collector location and current event count
## Current Product Gap
The main product gap is not architecture correctness. It is low-density visualization strategy.
Current reality:
- incident count is naturally much lower than anomaly count
- that is expected, because incidents are aggregated and de-noised
- but incident-first rendering makes the Earth view look too quiet unless there is another always-available activity layer
So the immediate next milestone is:
`event map -> observability map`
That means Earth needs three simultaneously readable layers:
1. `observation layer`
- collectors
- recent collector activity
- baseline coverage
2. `activity layer`
- recent event density
- anomaly/noise hotspots
- regional activity scoring
- incident presence bonus
3. `incident layer`
- sparse but highly legible, high-confidence event objects
- symbol-driven markers
- outward ring pulse instead of broad diffuse glow
## Incident Visual Direction
The Earth `incident` layer should not read like a large glowing patch. It should read like a compact, high-confidence event focus.
Design principles:
1. `incident` markers should use a strong primary symbol
- the symbol shape should carry type meaning where possible
- examples:
- `origin_change`: triangle-like warning marker
- `mass_withdrawal`: alert/exclamation-style marker
- `more_specific_burst`: split/radiating marker
2. emphasis should come from outward ring pulses, not area flooding
- use a compact hot core
- use one or more expanding ring pulses
- avoid broad luminous blobs that make the event center feel vague
3. `collector` and `incident` must stay visually distinct
- collectors are observation infrastructure
- incidents are extracted event focus
- collector activity should stay quieter than incident pulse language
4. calm periods still need observability presence
- collectors and activity layers should keep the map alive
- once incidents appear, they should clearly dominate nearby BGP visuals
5. incident geography should become `prefix-centric`
- collectors should remain evidence sources, not the primary event location
- preferred geography priority:
- `prefix_geography`
- `prefix_scope`
- `ASN organization region`
- `collector centroid` as final fallback
- `prefix_scope` should remain an observation-derived scope hint
- a new `prefix_geography` layer should be introduced for actual prefix-centric placement
Reference inspiration:
- `World Monitor`
- sparse event symbols
- compact centers
- ring-like outward pulses
- stronger incident legibility than diffuse glow
## Current Console Behavior ## Current Console Behavior
Relevant page: Relevant page:
@@ -188,14 +283,15 @@ BGP-specific tests live in:
Verified status at this point: Verified status at this point:
- `17 passed` - `25 passed` for `backend/tests/test_bgp.py`
- `62 passed` for `backend/tests`
Covered areas include: Covered areas include:
- normalization - normalization
- observation serialization - observation serialization
- enrichment - enrichment
- detectors - detectors, including route leak candidate and path flap
- incident aggregation - incident aggregation
- batch anomaly creation - batch anomaly creation
- BGP events/incidents API - BGP events/incidents API
@@ -226,9 +322,14 @@ Frontend:
## Recommended Next Steps ## Recommended Next Steps
1. Expand realtime collector coverage and include withdrawals more broadly. ### Next Backend / Detection Priority
2. Integrate real RPKI validation data.
3. Improve route leak and path instability detectors. 1. Integrate real RPKI validation data.
2. Expand realtime collector coverage and include withdrawals more broadly.
3. Continue refining route leak and path instability detectors with stronger heuristics.
### Next Correlation / Storytelling Priority
4. Strengthen incident aggregation semantics and titles. 4. Strengthen incident aggregation semantics and titles.
5. Add weak correlation from incidents to: 5. Add weak correlation from incidents to:
- cable corridors - cable corridors
@@ -236,3 +337,12 @@ Frontend:
- IXPs - IXPs
- other traffic anomaly sources - other traffic anomaly sources
6. Refine Earth hover/click handoff between collectors and incidents. 6. Refine Earth hover/click handoff between collectors and incidents.
### Next Visualization Priority
7. Refine regional activity scoring so the activity layer is informative without becoming noisy.
8. Add more incident symbol types as new detectors land.
9. Add a real prefix geography source:
- `IPtoASN / IPtoCountry` as the first practical dataset
- `OpenGeoFeed` as a higher-quality override layer
- registry/whois only as fallback

View File

@@ -0,0 +1,216 @@
# Prefix Geography Plan
## Goal
Make Earth BGP incidents `prefix-centric` instead of `collector-centric`.
The map should primarily answer:
- where a prefix-related event is likely centered
- which regions the prefix is likely associated with
- which collectors observed the event as evidence
It should not continue to imply that the event is located at the collector itself unless no better geography is available.
## Why Current Geography Is Not Enough
Current incident geography can still collapse back to collector-derived regions because:
1. `prefix_scope` is currently built mostly from observed collector regions and historical observation regions.
2. `origin_asn_profile` currently comes from `peeringdb_network`, which is useful for ASN footprint hints but not sufficient as a primary prefix location source.
3. `collector centroid` is still a common fallback and therefore dominates sparse incidents.
This makes Earth feel like a collector map with event decorations instead of a prefix impact map.
## Data Source Layers
Prefix geography should be built from four layers, ordered by confidence.
### Layer 1. Prefix-to-country / prefix-to-region
This is the primary source layer and the current missing piece.
Recommended sources:
1. `IPtoASN / IPtoCountry`
- URL: <https://iptoasn.com/>
- Good fit for this project because it provides downloadable IPv4/IPv6 range-to-ASN and range-to-country mappings.
- Best use:
- map a prefix to country code
- enrich prefixes with coarse regional placement
2. `OpenGeoFeed`
- URL: <https://opengeofeed.org/faq/>
- Best use:
- override coarse country mappings when the prefix holder publishes a geofeed
- provide a more realistic deployment/service region than whois-style registration country
### Layer 2. Registry allocation fallback
Use these only as fallback signals, not as a ground-truth physical location.
Candidate inputs:
- RIR delegated stats
- `inetnum` / `inet6num` whois
Best use:
- detect registration country / allocation region
- provide fallback when no direct prefix geolocation dataset is available
### Layer 3. ASN footprint hints
Existing in this project:
- `peeringdb_network`
- `peeringdb_facility`
- `peeringdb_ixp`
Best use:
- derive ASN city/country footprint
- identify likely exchange/facility regions
- act as secondary evidence when prefix-specific geography is unavailable
### Layer 4. Observation evidence
Existing in this project:
- `RIPE RIS Live`
- `CAIDA BGPStream Backfill`
Best use:
- prove who observed the event
- derive affected observation regions
- support impact evidence
This should remain the final fallback and evidence layer, not the primary event geography.
## Recommended Geography Priority
The backend should compute incident geography with this order:
1. `prefix_geography`
- prefix-to-country / region / geofeed-backed result
2. `asn_region`
- ASN organization / facility / IXP footprint
3. `collector_centroid`
- observed collector regions only as final fallback
Returned GeoJSON should keep exposing the selected mode through:
- `geography_mode = prefix_geography | asn_region | collector_centroid`
## Proposed Backend Changes
### 1. Add a dedicated prefix geography dataset
New datasource candidates:
- `ip2asn_prefix_geo`
- optionally `opengeofeed_prefix_geo`
Suggested storage model:
- keep downloaded rows in `CollectedData` first for speed of integration
- later move to a dedicated table if lookup volume grows
Minimum normalized fields:
- `range_start`
- `range_end`
- `prefix`
- `country`
- `continent`
- `asn`
- `as_name`
- `source`
- `confidence`
### 2. Add prefix geography enrichment
Extend:
- `backend/app/services/bgp_enrichment.py`
New enrichment payload should include:
- `prefix_geography`
- `country`
- `continent`
- `regions`
- `source`
- `confidence`
This should be separate from the current `prefix_scope`.
Suggested distinction:
- `prefix_scope`
- observation-derived scope hint
- `prefix_geography`
- prefix-centric geography estimate
### 3. Update incident visualization geography selection
Extend:
- `backend/app/api/v1/visualization.py`
Selection order:
1. `prefix_geography.regions`
2. ASN geography hints from PeeringDB-derived profile
3. observation-derived `affected_regions`
### 4. Keep evidence visible in the frontend
Earth should distinguish:
- event center = prefix geography estimate
- evidence lines / collectors = observation proof
This keeps the event meaningful for non-expert users without losing collector evidence.
## Earth UX Result
After this change, a user should see:
- an incident marker near the estimated affected prefix region
- collectors as supporting evidence, not as the event center itself
- cables / landing points / nearby infrastructure as weak correlation around the estimated region
This makes BGP incidents readable as “where the event is likely happening or affecting”, instead of “which station saw it”.
## Implementation Order
### Phase 1
1. Add `IPtoASN / IPtoCountry` datasource support
2. Normalize rows into lookup-friendly format
3. Enrich BGP events with `prefix_geography`
4. Switch incident geography priority to prefer `prefix_geography`
### Phase 2
5. Add `OpenGeoFeed` support
6. Let geofeed override coarse country-level prefix geography
7. Add confidence scoring per geography source
### Phase 3
8. Add RIR / whois fallback
9. Add better ASN regional footprint from PeeringDB facilities / IXPs
10. Refine Earth visual semantics for prefix geography vs observation evidence
## Recommendation
The best next engineering move is:
1. integrate `IPtoASN / IPtoCountry`
2. model `prefix_geography` separately from `prefix_scope`
3. only then continue refining incident map placement
Without this layer, any further Earth tuning will still be constrained by collector-centric data.

View File

@@ -2,7 +2,7 @@
<html lang="zh-CN"> <html lang="zh-CN">
<head> <head>
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" /> <link rel="icon" href="data:," />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>智能星球计划</title> <title>智能星球计划</title>
</head> </head>

View File

@@ -1,12 +1,12 @@
{ {
"name": "planet-frontend", "name": "planet-frontend",
"version": "0.22.8", "version": "0.22.10",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "planet-frontend", "name": "planet-frontend",
"version": "0.22.8", "version": "0.22.10",
"dependencies": { "dependencies": {
"@ant-design/icons": "^5.2.6", "@ant-design/icons": "^5.2.6",
"antd": "^5.12.5", "antd": "^5.12.5",
@@ -16,7 +16,9 @@
"react-dom": "^18.2.0", "react-dom": "^18.2.0",
"react-resizable": "^3.1.3", "react-resizable": "^3.1.3",
"react-router-dom": "^6.21.0", "react-router-dom": "^6.21.0",
"simplex-noise": "^4.0.1",
"socket.io-client": "^4.7.2", "socket.io-client": "^4.7.2",
"three": "^0.160.0",
"zustand": "^4.4.7" "zustand": "^4.4.7"
}, },
"devDependencies": { "devDependencies": {
@@ -3009,6 +3011,12 @@
"semver": "bin/semver.js" "semver": "bin/semver.js"
} }
}, },
"node_modules/simplex-noise": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/simplex-noise/-/simplex-noise-4.0.3.tgz",
"integrity": "sha512-qSE2I4AngLQG7BXqoZj51jokT4WUXe8mOBrvfOXpci8+6Yu44+/dD5zqDpOx3Ux792eamTd2lLcI8jqFntk/lg==",
"license": "MIT"
},
"node_modules/socket.io-client": { "node_modules/socket.io-client": {
"version": "4.8.3", "version": "4.8.3",
"resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.8.3.tgz", "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-4.8.3.tgz",
@@ -3059,6 +3067,12 @@
"integrity": "sha512-yQ3rwFWRfwNUY7H5vpU0wfdkNSnvnJinhF9830Swlaxl03zsOjCfmX0ugac+3LtK0lYSgwL/KXc8oYL3mG4YFQ==", "integrity": "sha512-yQ3rwFWRfwNUY7H5vpU0wfdkNSnvnJinhF9830Swlaxl03zsOjCfmX0ugac+3LtK0lYSgwL/KXc8oYL3mG4YFQ==",
"license": "MIT" "license": "MIT"
}, },
"node_modules/three": {
"version": "0.160.1",
"resolved": "https://registry.npmjs.org/three/-/three-0.160.1.tgz",
"integrity": "sha512-Bgl2wPJypDOZ1stAxwfWAcJ0WQf7QzlptsxkjYiURPz+n5k4RBDLsq+6f9Y75TYxn6aHLcWz+JNmwTOXWrQTBQ==",
"license": "MIT"
},
"node_modules/throttle-debounce": { "node_modules/throttle-debounce": {
"version": "5.0.2", "version": "5.0.2",
"resolved": "https://registry.npmjs.org/throttle-debounce/-/throttle-debounce-5.0.2.tgz", "resolved": "https://registry.npmjs.org/throttle-debounce/-/throttle-debounce-5.0.2.tgz",

View File

@@ -1,6 +1,6 @@
{ {
"name": "planet-frontend", "name": "planet-frontend",
"version": "0.22.9", "version": "0.22.10",
"private": true, "private": true,
"dependencies": { "dependencies": {
"@ant-design/icons": "^5.2.6", "@ant-design/icons": "^5.2.6",

Binary file not shown.

After

Width:  |  Height:  |  Size: 254 KiB

View File

@@ -13,7 +13,9 @@ let showBGP = true;
let totalAnomalyCount = 0; let totalAnomalyCount = 0;
let totalIncidentCount = 0; let totalIncidentCount = 0;
let textureCache = null; let textureCache = null;
let eventRingTextureCache = null;
let collectorTextureCache = null; let collectorTextureCache = null;
const eventTextureCache = new Map();
let activeEventOverlay = null; let activeEventOverlay = null;
let activeCollectorOverlayContext = null; let activeCollectorOverlayContext = null;
const relativeTimeFormatter = new Intl.RelativeTimeFormat("zh-CN", { const relativeTimeFormatter = new Intl.RelativeTimeFormat("zh-CN", {
@@ -60,6 +62,29 @@ function getMarkerTexture() {
return textureCache; return textureCache;
} }
function getEventRingTexture() {
if (eventRingTextureCache) return eventRingTextureCache;
const canvas = document.createElement("canvas");
canvas.width = 128;
canvas.height = 128;
const context = canvas.getContext("2d");
if (!context) {
eventRingTextureCache = new THREE.Texture(canvas);
return eventRingTextureCache;
}
context.clearRect(0, 0, 128, 128);
context.strokeStyle = "rgba(255,255,255,0.98)";
context.lineWidth = 6;
context.beginPath();
context.arc(64, 64, 44, 0, Math.PI * 2);
context.stroke();
eventRingTextureCache = new THREE.CanvasTexture(canvas);
return eventRingTextureCache;
}
function getCollectorTexture() { function getCollectorTexture() {
if (collectorTextureCache) return collectorTextureCache; if (collectorTextureCache) return collectorTextureCache;
@@ -102,6 +127,121 @@ function getCollectorTexture() {
return collectorTextureCache; return collectorTextureCache;
} }
function getEventSymbolKind(anomalyType) {
const value = String(anomalyType || "").toLowerCase();
if (value.includes("origin")) return "triangle";
if (value.includes("withdraw")) return "exclamation";
if (value.includes("specific") || value.includes("burst")) return "burst";
if (value.includes("flap")) return "wave";
if (value.includes("leak")) return "leak";
return "dot";
}
function drawTriangleSymbol(context) {
context.beginPath();
context.moveTo(64, 18);
context.lineTo(110, 106);
context.lineTo(18, 106);
context.closePath();
context.fill();
}
function drawExclamationSymbol(context) {
context.beginPath();
context.roundRect(52, 22, 24, 62, 12);
context.fill();
context.beginPath();
context.arc(64, 102, 10, 0, Math.PI * 2);
context.fill();
}
function drawWaveSymbol(context) {
context.lineWidth = 12;
context.lineCap = "round";
context.beginPath();
context.moveTo(18, 76);
context.bezierCurveTo(34, 46, 46, 46, 64, 76);
context.bezierCurveTo(80, 106, 94, 106, 110, 76);
context.stroke();
}
function drawBurstSymbol(context) {
context.lineWidth = 10;
context.lineCap = "round";
for (let index = 0; index < 6; index += 1) {
const angle = (Math.PI * 2 * index) / 6;
const inner = 26;
const outer = 48;
context.beginPath();
context.moveTo(64 + Math.cos(angle) * inner, 64 + Math.sin(angle) * inner);
context.lineTo(64 + Math.cos(angle) * outer, 64 + Math.sin(angle) * outer);
context.stroke();
}
context.beginPath();
context.arc(64, 64, 16, 0, Math.PI * 2);
context.fill();
}
function drawLeakSymbol(context) {
context.lineWidth = 10;
context.lineCap = "round";
context.beginPath();
context.moveTo(28, 96);
context.lineTo(64, 28);
context.lineTo(100, 96);
context.stroke();
context.beginPath();
context.moveTo(40, 82);
context.lineTo(64, 54);
context.lineTo(88, 82);
context.stroke();
}
function drawDotSymbol(context) {
context.beginPath();
context.arc(64, 64, 28, 0, Math.PI * 2);
context.fill();
}
function getEventTexture(anomalyType) {
const kind = getEventSymbolKind(anomalyType);
if (eventTextureCache.has(kind)) return eventTextureCache.get(kind);
const canvas = document.createElement("canvas");
canvas.width = 128;
canvas.height = 128;
const context = canvas.getContext("2d");
if (!context) {
const fallback = new THREE.Texture(canvas);
eventTextureCache.set(kind, fallback);
return fallback;
}
context.clearRect(0, 0, 128, 128);
context.fillStyle = "rgba(255,255,255,0.96)";
context.strokeStyle = "rgba(255,255,255,0.96)";
context.shadowBlur = 0;
context.lineJoin = "round";
if (kind === "triangle") {
drawTriangleSymbol(context);
} else if (kind === "exclamation") {
drawExclamationSymbol(context);
} else if (kind === "wave") {
drawWaveSymbol(context);
} else if (kind === "burst") {
drawBurstSymbol(context);
} else if (kind === "leak") {
drawLeakSymbol(context);
} else {
drawDotSymbol(context);
}
const texture = new THREE.CanvasTexture(canvas);
eventTextureCache.set(kind, texture);
return texture;
}
function normalizeSeverity(severity) { function normalizeSeverity(severity) {
const value = String(severity || "").trim().toLowerCase(); const value = String(severity || "").trim().toLowerCase();
@@ -934,9 +1074,14 @@ function createCollectorMarker(markerData) {
function createAnomalyMarker(markerData) { function createAnomalyMarker(markerData) {
const sprite = new THREE.Sprite( const sprite = new THREE.Sprite(
createSpriteMaterial({ new THREE.SpriteMaterial({
map: getEventTexture(markerData.incident_type || markerData.anomaly_type),
color: getSeverityColor(markerData.severity), color: getSeverityColor(markerData.severity),
transparent: true,
opacity: BGP_CONFIG.opacity.normal, opacity: BGP_CONFIG.opacity.normal,
depthWrite: false,
depthTest: true,
blending: THREE.NormalBlending,
}), }),
); );
@@ -960,12 +1105,45 @@ function createAnomalyMarker(markerData) {
...markerData, ...markerData,
}; };
const ringA = new THREE.Sprite(
new THREE.SpriteMaterial({
map: getEventRingTexture(),
color: getSeverityColor(markerData.severity),
transparent: true,
opacity: 0,
depthWrite: false,
depthTest: true,
blending: THREE.AdditiveBlending,
}),
);
ringA.scale.setScalar(baseScale * BGP_CONFIG.eventRingScaleA);
ringA.position.set(0, 0, -0.01);
sprite.add(ringA);
const ringB = new THREE.Sprite(
new THREE.SpriteMaterial({
map: getEventRingTexture(),
color: getSeverityColor(markerData.severity),
transparent: true,
opacity: 0,
depthWrite: false,
depthTest: true,
blending: THREE.AdditiveBlending,
}),
);
ringB.scale.setScalar(baseScale * BGP_CONFIG.eventRingScaleB);
ringB.position.set(0, 0, -0.02);
sprite.add(ringB);
sprite.userData.ringA = ringA;
sprite.userData.ringB = ringB;
anomalyMarkers.push(sprite); anomalyMarkers.push(sprite);
bgpGroup.add(sprite); bgpGroup.add(sprite);
} }
function dedupeAnomalies(features) { function dedupeAnomalies(features) {
const latestByCollector = new Map(); const latestByLocation = new Map();
features.forEach((feature) => { features.forEach((feature) => {
const data = buildAnomalyFeatureData(feature); const data = buildAnomalyFeatureData(feature);
@@ -976,22 +1154,30 @@ function dedupeAnomalies(features) {
(activeEventCountByCollector.get(data.collector) || 0) + 1, (activeEventCountByCollector.get(data.collector) || 0) + 1,
); );
const dedupeKey = `${data.collector}|${data.latitude.toFixed(4)}|${data.longitude.toFixed(4)}`; const dedupeKey = `${data.latitude.toFixed(3)}|${data.longitude.toFixed(3)}`;
const previous = latestByCollector.get(dedupeKey); const previous = latestByLocation.get(dedupeKey);
const currentTime = data.created_at_raw const currentTime = data.created_at_raw
? new Date(data.created_at_raw).getTime() ? new Date(data.created_at_raw).getTime()
: 0; : 0;
const previousTime = previous?.created_at_raw const previousTime = previous?.created_at_raw
? new Date(previous.created_at_raw).getTime() ? new Date(previous.created_at_raw).getTime()
: 0; : 0;
const currentSeverity = getSeverityScale(data.severity);
const previousSeverity = previous ? getSeverityScale(previous.severity) : 0;
if (!previous || currentTime >= previousTime) { if (
latestByCollector.set(dedupeKey, data); !previous ||
currentSeverity > previousSeverity ||
(currentSeverity === previousSeverity && currentTime >= previousTime)
) {
latestByLocation.set(dedupeKey, data);
} }
}); });
return Array.from(latestByCollector.values()) return Array.from(latestByLocation.values())
.sort((a, b) => { .sort((a, b) => {
const severityDiff = getSeverityScale(b.severity) - getSeverityScale(a.severity);
if (severityDiff !== 0) return severityDiff;
const timeA = a.created_at_raw ? new Date(a.created_at_raw).getTime() : 0; const timeA = a.created_at_raw ? new Date(a.created_at_raw).getTime() : 0;
const timeB = b.created_at_raw ? new Date(b.created_at_raw).getTime() : 0; const timeB = b.created_at_raw ? new Date(b.created_at_raw).getTime() : 0;
return timeB - timeA; return timeB - timeA;
@@ -1000,7 +1186,7 @@ function dedupeAnomalies(features) {
} }
function dedupeIncidents(features) { function dedupeIncidents(features) {
const latestByKey = new Map(); const latestByLocation = new Map();
features.forEach((feature) => { features.forEach((feature) => {
const data = buildIncidentFeatureData(feature); const data = buildIncidentFeatureData(feature);
@@ -1013,22 +1199,30 @@ function dedupeIncidents(features) {
); );
}); });
const dedupeKey = String(data.incident_key || data.id); const dedupeKey = `${data.latitude.toFixed(3)}|${data.longitude.toFixed(3)}`;
const previous = latestByKey.get(dedupeKey); const previous = latestByLocation.get(dedupeKey);
const currentTime = data.created_at_raw const currentTime = data.created_at_raw
? new Date(data.created_at_raw).getTime() ? new Date(data.created_at_raw).getTime()
: 0; : 0;
const previousTime = previous?.created_at_raw const previousTime = previous?.created_at_raw
? new Date(previous.created_at_raw).getTime() ? new Date(previous.created_at_raw).getTime()
: 0; : 0;
const currentSeverity = getSeverityScale(data.severity);
const previousSeverity = previous ? getSeverityScale(previous.severity) : 0;
if (!previous || currentTime >= previousTime) { if (
latestByKey.set(dedupeKey, data); !previous ||
currentSeverity > previousSeverity ||
(currentSeverity === previousSeverity && currentTime >= previousTime)
) {
latestByLocation.set(dedupeKey, data);
} }
}); });
return Array.from(latestByKey.values()) return Array.from(latestByLocation.values())
.sort((a, b) => { .sort((a, b) => {
const severityDiff = getSeverityScale(b.severity) - getSeverityScale(a.severity);
if (severityDiff !== 0) return severityDiff;
const timeA = a.created_at_raw ? new Date(a.created_at_raw).getTime() : 0; const timeA = a.created_at_raw ? new Date(a.created_at_raw).getTime() : 0;
const timeB = b.created_at_raw ? new Date(b.created_at_raw).getTime() : 0; const timeB = b.created_at_raw ? new Date(b.created_at_raw).getTime() : 0;
return timeB - timeA; return timeB - timeA;
@@ -1046,25 +1240,40 @@ function applyCollectorCounts() {
export async function loadBGPAnomalies(scene, earth) { export async function loadBGPAnomalies(scene, earth) {
clearBGPData(earth); clearBGPData(earth);
const [collectorsResponse, incidentsResponse, anomaliesResponse] = await Promise.all([ const collectorsResponse = await fetch(PATHS.bgpCollectorsApi);
fetch(PATHS.bgpCollectorsApi),
fetch(`${PATHS.bgpIncidentsApi}?limit=${BGP_CONFIG.defaultFetchLimit}`),
fetch(`${PATHS.bgpApi}?limit=${BGP_CONFIG.defaultFetchLimit}`),
]);
if (!collectorsResponse.ok) { if (!collectorsResponse.ok) {
throw new Error(`BGP collectors HTTP ${collectorsResponse.status}`); throw new Error(`BGP collectors HTTP ${collectorsResponse.status}`);
} }
if (!incidentsResponse.ok) {
throw new Error(`BGP incidents HTTP ${incidentsResponse.status}`); let anomaliesPayload = { type: "FeatureCollection", features: [], count: 0 };
try {
const anomaliesResponse = await fetch(
`${PATHS.bgpApi}?limit=${BGP_CONFIG.defaultFetchLimit}`,
{ signal: AbortSignal.timeout(5000) },
);
if (!anomaliesResponse.ok) {
throw new Error(`BGP anomalies HTTP ${anomaliesResponse.status}`);
}
anomaliesPayload = await anomaliesResponse.json();
} catch (error) {
console.warn("BGP anomalies unavailable, falling back to collectors only:", error);
} }
if (!anomaliesResponse.ok) {
throw new Error(`BGP anomalies HTTP ${anomaliesResponse.status}`); let incidentsPayload = { type: "FeatureCollection", features: [], count: 0 };
try {
const incidentsResponse = await fetch(
`${PATHS.bgpIncidentsApi}?limit=${BGP_CONFIG.defaultFetchLimit}`,
{ signal: AbortSignal.timeout(5000) },
);
if (!incidentsResponse.ok) {
throw new Error(`BGP incidents HTTP ${incidentsResponse.status}`);
}
incidentsPayload = await incidentsResponse.json();
} catch (error) {
console.warn("BGP incidents unavailable, falling back to anomalies:", error);
} }
const collectorsPayload = await collectorsResponse.json(); const collectorsPayload = await collectorsResponse.json();
const incidentsPayload = await incidentsResponse.json();
const anomaliesPayload = await anomaliesResponse.json();
const collectorFeatures = Array.isArray(collectorsPayload?.features) const collectorFeatures = Array.isArray(collectorsPayload?.features)
? collectorsPayload.features ? collectorsPayload.features
: []; : [];
@@ -1232,28 +1441,59 @@ export function updateBGPVisualState(lockedObjectType, lockedObject, camera) {
let scale = marker.userData.baseScale; let scale = marker.userData.baseScale;
let opacity = BGP_CONFIG.opacity.normal; let opacity = BGP_CONFIG.opacity.normal;
let markerColor = marker.userData.baseColor || getSeverityColor(marker.userData.severity); let markerColor = marker.userData.baseColor || getSeverityColor(marker.userData.severity);
const isIncidentMarker = marker.userData.source === "bgp_incident";
let ringBaseOpacity = isIncidentMarker
? BGP_CONFIG.eventRingOpacity
: BGP_CONFIG.eventRingOpacity * 0.45;
if (isLocked || isLinkedCollectorLocked) { if (isLocked || isLinkedCollectorLocked) {
scale *= 1 + BGP_CONFIG.lockedPulseAmplitude * pulse; scale *= 1 + BGP_CONFIG.lockedPulseAmplitude * pulse;
opacity = opacity =
BGP_CONFIG.opacity.lockedMin + BGP_CONFIG.opacity.lockedMin +
(BGP_CONFIG.opacity.lockedMax - BGP_CONFIG.opacity.lockedMin) * pulse; (BGP_CONFIG.opacity.lockedMax - BGP_CONFIG.opacity.lockedMin) * pulse;
ringBaseOpacity *= 1.2;
} else if (isHovered) { } else if (isHovered) {
scale *= BGP_CONFIG.hoverScale; scale *= BGP_CONFIG.hoverScale;
opacity = BGP_CONFIG.opacity.hover; opacity = BGP_CONFIG.opacity.hover;
ringBaseOpacity *= 1.05;
} else if (isOtherLocked) { } else if (isOtherLocked) {
scale *= BGP_CONFIG.dimmedScale; scale *= BGP_CONFIG.dimmedScale;
opacity = 0.1; opacity = 0.1;
markerColor = 0x7d8ca3; markerColor = 0x7d8ca3;
ringBaseOpacity = 0.02;
} else { } else {
scale *= 1 + BGP_CONFIG.normalPulseAmplitude * pulse; scale *= 1 + BGP_CONFIG.normalPulseAmplitude * pulse;
opacity = BGP_CONFIG.opacity.normal; opacity = isIncidentMarker ? 0.7 : 0.62;
} }
marker.scale.setScalar(scale); marker.scale.setScalar(scale);
marker.material.color.setHex(markerColor); marker.material.color.setHex(markerColor);
marker.material.opacity = opacity; marker.material.opacity = opacity;
marker.visible = showBGP; marker.visible = showBGP;
const ringPhaseA = (now * BGP_CONFIG.eventRingSpeed + marker.userData.pulseOffset) % 1;
const applyRingState = (ring, phase, maxScale) => {
if (!ring) return;
const progress = Math.max(0, Math.min(1, phase));
const minScale = 1.28;
const desiredWorldScale =
marker.userData.baseScale * (minScale + progress * (maxScale - minScale));
const parentScale = Math.max(scale, 0.0001);
const localRingScale = desiredWorldScale / parentScale;
const fadeIn = Math.max(0, Math.min(1, (progress - 0.08) / 0.14));
const fadeOut = 1 - progress;
const visibility = fadeIn * fadeOut;
ring.scale.setScalar(localRingScale);
ring.material.color.setHex(markerColor);
ring.material.opacity = showBGP ? ringBaseOpacity * visibility : 0;
ring.visible = showBGP;
};
applyRingState(marker.userData.ringA, ringPhaseA, BGP_CONFIG.eventRingScaleA);
if (marker.userData.ringB) {
marker.userData.ringB.material.opacity = 0;
marker.userData.ringB.visible = false;
}
}); });
} }
@@ -1368,42 +1608,9 @@ export function showBGPEventOverlay(marker, earth) {
typeof region?.longitude === "number", typeof region?.longitude === "number",
); );
if (validRegions.length === 0) return; if (validRegions.length === 0) return;
const overlayItems = [];
const averageLatitude =
validRegions.reduce((sum, region) => sum + region.latitude, 0) /
validRegions.length;
const averageLongitude =
validRegions.reduce((sum, region) => sum + region.longitude, 0) /
validRegions.length;
const hubPosition = latLonToVector3(
averageLatitude,
averageLongitude,
CONFIG.earthRadius + BGP_CONFIG.eventHubAltitudeOffset,
);
const hub = createOverlaySprite({
color: BGP_CONFIG.eventHubColor,
opacity: 0.95,
scale: BGP_CONFIG.eventHubScale,
});
hub.position.copy(hubPosition);
hub.renderOrder = 6;
bgpOverlayGroup.add(hub);
const overlayItems = [hub];
validRegions.forEach((region) => { validRegions.forEach((region) => {
const regionPosition = latLonToVector3(
region.latitude,
region.longitude,
CONFIG.earthRadius + BGP_CONFIG.collectorAltitudeOffset + 0.3,
);
const link = createArcLine(regionPosition, hubPosition, BGP_CONFIG.linkColor);
link.renderOrder = 4;
bgpOverlayGroup.add(link);
overlayItems.push(link);
const halo = createOverlaySprite({ const halo = createOverlaySprite({
color: BGP_CONFIG.regionColor, color: BGP_CONFIG.regionColor,
opacity: 0.24, opacity: 0.24,

View File

@@ -138,6 +138,10 @@ export const BGP_CONFIG = {
eventHubColor: 0x8af5ff, eventHubColor: 0x8af5ff,
linkColor: 0x54d2ff, linkColor: 0x54d2ff,
regionColor: 0x2dd4bf, regionColor: 0x2dd4bf,
eventRingScaleA: 2.5,
eventRingScaleB: 3.4,
eventRingOpacity: 0.5,
eventRingSpeed: 0.001,
collectorHaloScale: 11.5, collectorHaloScale: 11.5,
collectorPulseHaloScale: 16.5, collectorPulseHaloScale: 16.5,
collectorCoverageHaloScale: 22.5 collectorCoverageHaloScale: 22.5

View File

@@ -104,7 +104,7 @@ export function createClouds(scene, earthObj) {
earthObj.add(clouds); earthObj.add(clouds);
textureLoader.load( textureLoader.load(
'https://threejs.org/examples/textures/planets/earth_clouds_1024.png', './assets/earth_clouds_1024.png',
function(texture) { function(texture) {
material.map = texture; material.map = texture;
material.needsUpdate = true; material.needsUpdate = true;

View File

@@ -1,6 +1,6 @@
[project] [project]
name = "planet" name = "planet"
version = "0.22.8" version = "0.22.10"
description = "智能星球计划 - 态势感知系统" description = "智能星球计划 - 态势感知系统"
requires-python = ">=3.14" requires-python = ">=3.14"
dependencies = [ dependencies = [