112 lines
4.0 KiB
Python
112 lines
4.0 KiB
Python
"""IPtoASN prefix geography collector.
|
|
|
|
Downloads the public combined IPv4+IPv6 TSV database and stores coarse
|
|
prefix-to-country/ASN geography hints for BGP enrichment.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import gzip
|
|
from datetime import UTC, datetime
|
|
from ipaddress import summarize_address_range, ip_address
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from app.services.collectors.base import BaseCollector
|
|
|
|
|
|
class IPtoASNPrefixGeoCollector(BaseCollector):
|
|
name = "iptoasn_prefix_geo"
|
|
priority = "P1"
|
|
module = "L3"
|
|
frequency_hours = 24
|
|
data_type = "prefix_geography"
|
|
fail_on_empty = True
|
|
|
|
async def fetch(self) -> list[dict[str, Any]]:
|
|
if not self._resolved_url:
|
|
raise RuntimeError("IPtoASN combined URL is not configured")
|
|
|
|
async with httpx.AsyncClient(timeout=180.0, follow_redirects=True) as client:
|
|
response = await client.get(
|
|
self._resolved_url,
|
|
headers={
|
|
"User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
|
|
"Accept": "application/gzip,application/octet-stream,*/*",
|
|
},
|
|
)
|
|
response.raise_for_status()
|
|
body = gzip.decompress(response.content).decode("utf-8", errors="replace")
|
|
|
|
rows: list[dict[str, Any]] = []
|
|
for raw_line in body.splitlines():
|
|
line = raw_line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
parts = line.split("\t")
|
|
if len(parts) < 5:
|
|
continue
|
|
range_start, range_end, asn, country_code, as_name = parts[:5]
|
|
rows.append(
|
|
{
|
|
"range_start": range_start,
|
|
"range_end": range_end,
|
|
"asn": asn,
|
|
"country_code": country_code,
|
|
"as_name": as_name,
|
|
}
|
|
)
|
|
return rows
|
|
|
|
def transform(self, raw_data: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
reference_date = datetime.now(UTC).isoformat()
|
|
transformed: list[dict[str, Any]] = []
|
|
|
|
for item in raw_data:
|
|
try:
|
|
start_ip = ip_address(str(item["range_start"]))
|
|
end_ip = ip_address(str(item["range_end"]))
|
|
except ValueError:
|
|
continue
|
|
|
|
if start_ip.version != end_ip.version:
|
|
continue
|
|
|
|
summarized = list(summarize_address_range(start_ip, end_ip))
|
|
primary_prefix = str(summarized[0]) if summarized else f"{start_ip}/{32 if start_ip.version == 4 else 128}"
|
|
family = f"ipv{start_ip.version}"
|
|
|
|
asn_value = item.get("asn")
|
|
try:
|
|
normalized_asn = int(str(asn_value))
|
|
except (TypeError, ValueError):
|
|
normalized_asn = None
|
|
|
|
transformed.append(
|
|
{
|
|
"source_id": f"{family}:{item['range_start']}-{item['range_end']}",
|
|
"name": primary_prefix,
|
|
"title": f"{primary_prefix} {item.get('country_code', '').strip()}".strip(),
|
|
"country": item.get("country_code"),
|
|
"city": "",
|
|
"latitude": None,
|
|
"longitude": None,
|
|
"metadata": {
|
|
"family": family,
|
|
"range_start": item["range_start"],
|
|
"range_end": item["range_end"],
|
|
"prefix": primary_prefix,
|
|
"prefixes": [str(prefix) for prefix in summarized[:8]],
|
|
"range_prefix_count": len(summarized),
|
|
"country_code": item.get("country_code"),
|
|
"asn": normalized_asn,
|
|
"as_name": item.get("as_name"),
|
|
"source_dataset": "iptoasn_combined",
|
|
},
|
|
"reference_date": reference_date,
|
|
}
|
|
)
|
|
|
|
return transformed
|