"""OpenGeoFeed prefix geography collector. Fetches public OpenGeoFeed CSV data and stores higher-confidence prefix-to-location hints for BGP prefix-centric enrichment. """ from __future__ import annotations import csv import ipaddress from datetime import UTC, datetime from typing import Any import httpx from app.services.collectors.base import BaseCollector from app.services.collectors.downloads import ResumableFileDownloader class OpenGeoFeedPrefixGeoCollector(BaseCollector): name = "opengeofeed_prefix_geo" priority = "P1" module = "L3" frequency_hours = 24 data_type = "prefix_geography" fail_on_empty = True _downloader = ResumableFileDownloader( cache_namespace="opengeofeed", default_accept="text/csv,*/*", ) async def fetch(self) -> list[dict[str, Any]]: if not self._resolved_url: raise RuntimeError("OpenGeoFeed URL is not configured") async with httpx.AsyncClient(timeout=180.0, follow_redirects=True) as client: remote = await self._downloader.fetch_remote_info(client, self._resolved_url) total_expected = remote.get("content_length") or 0 if total_expected > 0 and self._current_task and self._db_session: self._current_task.total_records = total_expected self._current_task.records_processed = 0 self._current_task.progress = 0.0 self._current_task.phase_progress = 0.0 self._current_task.phase_message = "正在下载 OpenGeoFeed 数据" self._current_task.phase_current = 0 self._current_task.phase_total = total_expected self._current_task.phase_unit = "bytes" await self._db_session.commit() await self._publish_task_update(force=True) async def on_progress(downloaded: int, total: int | None) -> None: if not total or total <= 0: return await self.update_phase_progress( current=min(downloaded, total), total=total, unit="bytes", message="正在下载 OpenGeoFeed 数据", ) await self.update_progress(min(downloaded, total), commit=True) body_path = await self._downloader.download_file( client, self._resolved_url, extension=".csv", progress_callback=on_progress, ) body = body_path.read_text(encoding="utf-8", errors="replace") rows: list[dict[str, Any]] = [] reader = csv.reader(body.splitlines()) for fields in reader: if not fields: continue first = (fields[0] or "").strip().lower() if not first or first.startswith("#") or first == "prefix": continue prefix = (fields[0] or "").strip() country_code = (fields[1] if len(fields) > 1 else "").strip() region = (fields[2] if len(fields) > 2 else "").strip() city = (fields[3] if len(fields) > 3 else "").strip() postal_code = (fields[4] if len(fields) > 4 else "").strip() # Keep additional columns for future enrichment without breaking # current normalized schema. extras = [value.strip() for value in fields[5:]] if len(fields) > 5 else [] rows.append( { "prefix": prefix, "country_code": country_code, "region": region, "city": city, "postal_code": postal_code, "extra_columns": extras, } ) return rows def transform(self, raw_data: list[dict[str, Any]]) -> list[dict[str, Any]]: reference_date = datetime.now(UTC).isoformat() transformed: list[dict[str, Any]] = [] for item in raw_data: prefix = str(item.get("prefix") or "").strip() if not prefix: continue try: network = ipaddress.ip_network(prefix, strict=False) except ValueError: continue family = f"ipv{network.version}" country_code = str(item.get("country_code") or "").strip().upper() region = str(item.get("region") or "").strip() city = str(item.get("city") or "").strip() postal_code = str(item.get("postal_code") or "").strip() transformed.append( { "source_id": f"{family}:{prefix}:{country_code}:{region}:{city}", "name": prefix, "title": f"{prefix} {country_code}".strip(), "country": country_code, "city": city, "latitude": None, "longitude": None, "metadata": { "family": family, "prefix": prefix, "range_start": str(network.network_address), "range_end": str(network.broadcast_address), "country_code": country_code, "region": region, "city": city, "postal_code": postal_code, "extra_columns": item.get("extra_columns") or [], "source_dataset": "opengeofeed_public", "confidence": "geofeed", }, "reference_date": reference_date, } ) return transformed