fix: stabilize earth bgp geography and rendering
This commit is contained in:
@@ -7,9 +7,10 @@ from collections import defaultdict
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.countries import get_country_centroid, normalize_country
|
||||
from app.models.bgp_observation import BGPObservation
|
||||
from app.models.collected_data import CollectedData
|
||||
|
||||
@@ -96,6 +97,83 @@ def extract_bgp_network_fields(prefix: str) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
async def _lookup_prefix_geography(
|
||||
db: AsyncSession,
|
||||
prefix_values: list[str],
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
results: dict[str, dict[str, Any]] = {}
|
||||
|
||||
for prefix in prefix_values:
|
||||
try:
|
||||
network = ipaddress.ip_network(prefix, strict=False)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
family = f"ipv{network.version}"
|
||||
range_start = str(network.network_address)
|
||||
range_end = str(network.broadcast_address)
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT metadata
|
||||
FROM collected_data
|
||||
WHERE source = 'iptoasn_prefix_geo'
|
||||
AND COALESCE(is_current, TRUE) = TRUE
|
||||
AND metadata->>'family' = :family
|
||||
AND CAST(metadata->>'range_start' AS inet) <= CAST(:range_start AS inet)
|
||||
AND CAST(metadata->>'range_end' AS inet) >= CAST(:range_end AS inet)
|
||||
ORDER BY id DESC
|
||||
LIMIT 1
|
||||
"""
|
||||
),
|
||||
{
|
||||
"family": family,
|
||||
"range_start": range_start,
|
||||
"range_end": range_end,
|
||||
},
|
||||
)
|
||||
row = result.fetchone()
|
||||
if not row:
|
||||
continue
|
||||
|
||||
if isinstance(row, dict):
|
||||
payload = row.get("metadata") or row.get("extra_data")
|
||||
elif hasattr(row, "_mapping"):
|
||||
payload = row._mapping.get("metadata") or row._mapping.get("extra_data")
|
||||
else:
|
||||
payload = row[0]
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
|
||||
country = normalize_country(payload.get("country") or payload.get("country_code"))
|
||||
prefix_hint = payload.get("prefix") or prefix
|
||||
asn = _safe_int(payload.get("asn"))
|
||||
as_name = payload.get("as_name")
|
||||
centroid = get_country_centroid(country)
|
||||
regions = []
|
||||
if country:
|
||||
regions.append(
|
||||
{
|
||||
"country": country,
|
||||
"city": None,
|
||||
"latitude": centroid.get("latitude") if centroid else None,
|
||||
"longitude": centroid.get("longitude") if centroid else None,
|
||||
}
|
||||
)
|
||||
|
||||
results[prefix] = {
|
||||
"prefix": prefix_hint,
|
||||
"country": country,
|
||||
"asn": asn,
|
||||
"as_name": as_name,
|
||||
"source": payload.get("source_dataset") or "iptoasn_combined",
|
||||
"confidence": "country_range",
|
||||
"regions": regions,
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
|
||||
async def enrich_bgp_events_for_batch(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
@@ -165,6 +243,7 @@ async def enrich_bgp_events_for_batch(
|
||||
}
|
||||
|
||||
asn_profiles: dict[int, dict[str, Any]] = {}
|
||||
prefix_geographies = await _lookup_prefix_geography(db, prefix_values) if prefix_values else {}
|
||||
if origin_asns:
|
||||
peeringdb_result = await db.execute(
|
||||
select(CollectedData).where(CollectedData.source == "peeringdb_network")
|
||||
@@ -207,21 +286,12 @@ async def enrich_bgp_events_for_batch(
|
||||
collector = str(metadata.get("collector") or "").strip()
|
||||
collector_location = metadata.get("collector_location") or {}
|
||||
baseline = historical_prefix_baseline.get(prefix, {})
|
||||
prefix_geography = prefix_geographies.get(prefix)
|
||||
observed_at = _parse_timestamp(metadata.get("timestamp") or event.get("reference_date"))
|
||||
origin_asn = _safe_int(metadata.get("origin_asn"))
|
||||
new_origin_asn = _safe_int(metadata.get("new_origin_asn"))
|
||||
observed_regions = _compact_locations(
|
||||
[
|
||||
{
|
||||
"country": collector_location.get("country"),
|
||||
"city": collector_location.get("city"),
|
||||
"latitude": collector_location.get("latitude"),
|
||||
"longitude": collector_location.get("longitude"),
|
||||
}
|
||||
]
|
||||
)
|
||||
baseline_regions = baseline.get("historical_regions", [])
|
||||
prefix_scope_regions = _compact_locations([*observed_regions, *baseline_regions])
|
||||
prefix_scope_regions = _compact_locations([*baseline_regions])
|
||||
|
||||
enrichment = {
|
||||
**extract_bgp_network_fields(prefix),
|
||||
@@ -248,6 +318,7 @@ async def enrich_bgp_events_for_batch(
|
||||
},
|
||||
"origin_asn_profile": asn_profiles.get(origin_asn),
|
||||
"new_origin_asn_profile": asn_profiles.get(new_origin_asn),
|
||||
"prefix_geography": prefix_geography,
|
||||
"prefix_scope": {
|
||||
"countries": sorted(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user