from typing import Dict, Any, List from datetime import datetime import httpx from app.services.collectors.base import BaseCollector from app.core.data_sources import get_data_sources_config class ArcGISCableLandingRelationCollector(BaseCollector): name = "arcgis_cable_landing_relation" priority = "P1" module = "L2" frequency_hours = 168 data_type = "cable_landing_relation" @property def base_url(self) -> str: if self._resolved_url: return self._resolved_url from app.core.data_sources import get_data_sources_config config = get_data_sources_config() return config.get_yaml_url("arcgis_cable_landing_relation") async def fetch(self) -> List[Dict[str, Any]]: params = {"where": "1=1", "outFields": "*", "returnGeometry": "true", "f": "geojson"} async with httpx.AsyncClient(timeout=60.0) as client: response = await client.get(self.base_url, params=params) response.raise_for_status() return self.parse_response(response.json()) def parse_response(self, data: Dict[str, Any]) -> List[Dict[str, Any]]: result = [] features = data.get("features", []) for feature in features: props = feature.get("properties", {}) try: entry = { "source_id": f"arcgis_relation_{props.get('OBJECTID', props.get('id', ''))}", "name": f"{props.get('cable_name', 'Unknown')} - {props.get('landing_point_name', 'Unknown')}", "country": props.get("country", ""), "city": props.get("landing_point_name", ""), "latitude": str(props.get("latitude", "")) if props.get("latitude") else "", "longitude": str(props.get("longitude", "")) if props.get("longitude") else "", "value": "", "unit": "", "metadata": { "objectid": props.get("OBJECTID"), "city_id": props.get("city_id"), "cable_id": props.get("cable_id"), "cable_name": props.get("cable_name"), "landing_point_id": props.get("landing_point_id"), "landing_point_name": props.get("landing_point_name"), "facility": props.get("facility"), "status": props.get("status"), }, "reference_date": datetime.utcnow().strftime("%Y-%m-%d"), } result.append(entry) except (ValueError, TypeError, KeyError): continue return result