119 lines
4.2 KiB
Python
119 lines
4.2 KiB
Python
"""OpenGeoFeed prefix geography collector.
|
|
|
|
Fetches public OpenGeoFeed CSV data and stores higher-confidence
|
|
prefix-to-location hints for BGP prefix-centric enrichment.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import ipaddress
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from app.services.collectors.base import BaseCollector
|
|
|
|
|
|
class OpenGeoFeedPrefixGeoCollector(BaseCollector):
|
|
name = "opengeofeed_prefix_geo"
|
|
priority = "P1"
|
|
module = "L3"
|
|
frequency_hours = 24
|
|
data_type = "prefix_geography"
|
|
fail_on_empty = True
|
|
|
|
async def fetch(self) -> list[dict[str, Any]]:
|
|
if not self._resolved_url:
|
|
raise RuntimeError("OpenGeoFeed URL is not configured")
|
|
|
|
async with httpx.AsyncClient(timeout=180.0, follow_redirects=True) as client:
|
|
response = await client.get(
|
|
self._resolved_url,
|
|
headers={
|
|
"User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
|
|
"Accept": "text/csv,*/*",
|
|
},
|
|
)
|
|
response.raise_for_status()
|
|
body = response.text
|
|
|
|
rows: list[dict[str, Any]] = []
|
|
reader = csv.reader(body.splitlines())
|
|
for fields in reader:
|
|
if not fields:
|
|
continue
|
|
first = (fields[0] or "").strip().lower()
|
|
if not first or first.startswith("#") or first == "prefix":
|
|
continue
|
|
|
|
prefix = (fields[0] or "").strip()
|
|
country_code = (fields[1] if len(fields) > 1 else "").strip()
|
|
region = (fields[2] if len(fields) > 2 else "").strip()
|
|
city = (fields[3] if len(fields) > 3 else "").strip()
|
|
postal_code = (fields[4] if len(fields) > 4 else "").strip()
|
|
|
|
# Keep additional columns for future enrichment without breaking
|
|
# current normalized schema.
|
|
extras = [value.strip() for value in fields[5:]] if len(fields) > 5 else []
|
|
|
|
rows.append(
|
|
{
|
|
"prefix": prefix,
|
|
"country_code": country_code,
|
|
"region": region,
|
|
"city": city,
|
|
"postal_code": postal_code,
|
|
"extra_columns": extras,
|
|
}
|
|
)
|
|
return rows
|
|
|
|
def transform(self, raw_data: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
reference_date = datetime.now(UTC).isoformat()
|
|
transformed: list[dict[str, Any]] = []
|
|
|
|
for item in raw_data:
|
|
prefix = str(item.get("prefix") or "").strip()
|
|
if not prefix:
|
|
continue
|
|
try:
|
|
network = ipaddress.ip_network(prefix, strict=False)
|
|
except ValueError:
|
|
continue
|
|
|
|
family = f"ipv{network.version}"
|
|
country_code = str(item.get("country_code") or "").strip().upper()
|
|
region = str(item.get("region") or "").strip()
|
|
city = str(item.get("city") or "").strip()
|
|
postal_code = str(item.get("postal_code") or "").strip()
|
|
|
|
transformed.append(
|
|
{
|
|
"source_id": f"{family}:{prefix}:{country_code}:{region}:{city}",
|
|
"name": prefix,
|
|
"title": f"{prefix} {country_code}".strip(),
|
|
"country": country_code,
|
|
"city": city,
|
|
"latitude": None,
|
|
"longitude": None,
|
|
"metadata": {
|
|
"family": family,
|
|
"prefix": prefix,
|
|
"range_start": str(network.network_address),
|
|
"range_end": str(network.broadcast_address),
|
|
"country_code": country_code,
|
|
"region": region,
|
|
"city": city,
|
|
"postal_code": postal_code,
|
|
"extra_columns": item.get("extra_columns") or [],
|
|
"source_dataset": "opengeofeed_public",
|
|
"confidence": "geofeed",
|
|
},
|
|
"reference_date": reference_date,
|
|
}
|
|
)
|
|
|
|
return transformed
|