fix: stabilize earth bgp geography and rendering

This commit is contained in:
linkong
2026-04-02 15:36:20 +08:00
parent 07e4f519a1
commit e5fec8ba3d
26 changed files with 1788 additions and 137 deletions

View File

@@ -7,9 +7,10 @@ from collections import defaultdict
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy import select, text
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.countries import get_country_centroid, normalize_country
from app.models.bgp_observation import BGPObservation
from app.models.collected_data import CollectedData
@@ -96,6 +97,83 @@ def extract_bgp_network_fields(prefix: str) -> dict[str, Any]:
}
async def _lookup_prefix_geography(
db: AsyncSession,
prefix_values: list[str],
) -> dict[str, dict[str, Any]]:
results: dict[str, dict[str, Any]] = {}
for prefix in prefix_values:
try:
network = ipaddress.ip_network(prefix, strict=False)
except ValueError:
continue
family = f"ipv{network.version}"
range_start = str(network.network_address)
range_end = str(network.broadcast_address)
result = await db.execute(
text(
"""
SELECT metadata
FROM collected_data
WHERE source = 'iptoasn_prefix_geo'
AND COALESCE(is_current, TRUE) = TRUE
AND metadata->>'family' = :family
AND CAST(metadata->>'range_start' AS inet) <= CAST(:range_start AS inet)
AND CAST(metadata->>'range_end' AS inet) >= CAST(:range_end AS inet)
ORDER BY id DESC
LIMIT 1
"""
),
{
"family": family,
"range_start": range_start,
"range_end": range_end,
},
)
row = result.fetchone()
if not row:
continue
if isinstance(row, dict):
payload = row.get("metadata") or row.get("extra_data")
elif hasattr(row, "_mapping"):
payload = row._mapping.get("metadata") or row._mapping.get("extra_data")
else:
payload = row[0]
if not isinstance(payload, dict):
continue
country = normalize_country(payload.get("country") or payload.get("country_code"))
prefix_hint = payload.get("prefix") or prefix
asn = _safe_int(payload.get("asn"))
as_name = payload.get("as_name")
centroid = get_country_centroid(country)
regions = []
if country:
regions.append(
{
"country": country,
"city": None,
"latitude": centroid.get("latitude") if centroid else None,
"longitude": centroid.get("longitude") if centroid else None,
}
)
results[prefix] = {
"prefix": prefix_hint,
"country": country,
"asn": asn,
"as_name": as_name,
"source": payload.get("source_dataset") or "iptoasn_combined",
"confidence": "country_range",
"regions": regions,
}
return results
async def enrich_bgp_events_for_batch(
db: AsyncSession,
*,
@@ -165,6 +243,7 @@ async def enrich_bgp_events_for_batch(
}
asn_profiles: dict[int, dict[str, Any]] = {}
prefix_geographies = await _lookup_prefix_geography(db, prefix_values) if prefix_values else {}
if origin_asns:
peeringdb_result = await db.execute(
select(CollectedData).where(CollectedData.source == "peeringdb_network")
@@ -207,21 +286,12 @@ async def enrich_bgp_events_for_batch(
collector = str(metadata.get("collector") or "").strip()
collector_location = metadata.get("collector_location") or {}
baseline = historical_prefix_baseline.get(prefix, {})
prefix_geography = prefix_geographies.get(prefix)
observed_at = _parse_timestamp(metadata.get("timestamp") or event.get("reference_date"))
origin_asn = _safe_int(metadata.get("origin_asn"))
new_origin_asn = _safe_int(metadata.get("new_origin_asn"))
observed_regions = _compact_locations(
[
{
"country": collector_location.get("country"),
"city": collector_location.get("city"),
"latitude": collector_location.get("latitude"),
"longitude": collector_location.get("longitude"),
}
]
)
baseline_regions = baseline.get("historical_regions", [])
prefix_scope_regions = _compact_locations([*observed_regions, *baseline_regions])
prefix_scope_regions = _compact_locations([*baseline_regions])
enrichment = {
**extract_bgp_network_fields(prefix),
@@ -248,6 +318,7 @@ async def enrich_bgp_events_for_batch(
},
"origin_asn_profile": asn_profiles.get(origin_asn),
"new_origin_asn_profile": asn_profiles.get(new_origin_asn),
"prefix_geography": prefix_geography,
"prefix_scope": {
"countries": sorted(
{