from typing import Dict, Any, List from datetime import datetime import httpx from app.services.collectors.base import BaseCollector from app.core.config import settings class ArcGISLandingPointCollector(BaseCollector): name = "arcgis_landing_points" priority = "P1" module = "L2" frequency_hours = 168 data_type = "landing_point" @property def base_url(self) -> str: return settings.ARCGIS_LANDING_POINT_URL async def fetch(self) -> List[Dict[str, Any]]: params = {"where": "1=1", "outFields": "*", "returnGeometry": "true", "f": "geojson"} async with httpx.AsyncClient(timeout=60.0) as client: response = await client.get(self.base_url, params=params) response.raise_for_status() return self.parse_response(response.json()) def parse_response(self, data: Dict[str, Any]) -> List[Dict[str, Any]]: result = [] features = data.get("features", []) for feature in features: props = feature.get("properties", {}) geometry = feature.get("geometry", {}) lat = geometry.get("y") if geometry else None lon = geometry.get("x") if geometry else None try: entry = { "source_id": f"arcgis_lp_{props.get('OBJECTID', props.get('id', ''))}", "name": props.get("Name", props.get("name", "Unknown")), "country": props.get("country", ""), "city": props.get("city", ""), "latitude": str(lat) if lat else "", "longitude": str(lon) if lon else "", "value": "", "unit": "", "metadata": { "objectid": props.get("OBJECTID"), "cable_id": props.get("cable_id"), "cable_name": props.get("cable_name"), "facility": props.get("facility"), "facility_type": props.get("facility_type"), "status": props.get("status"), "landing_point_id": props.get("landing_point_id"), }, "reference_date": datetime.utcnow().strftime("%Y-%m-%d"), } result.append(entry) except (ValueError, TypeError, KeyError): continue return result