99771a88c5
- Add ARCGIS_CABLE_URL, ARCGIS_LANDING_POINT_URL, ARCGIS_CABLE_LANDING_RELATION_URL to config - Use @property to read URL from settings in collectors - URLs can now be configured via environment variables
65 lines
2.4 KiB
Python
65 lines
2.4 KiB
Python
from typing import Dict, Any, List
|
|
from datetime import datetime
|
|
import httpx
|
|
|
|
from app.services.collectors.base import BaseCollector
|
|
from app.core.config import settings
|
|
|
|
|
|
class ArcGISLandingPointCollector(BaseCollector):
|
|
name = "arcgis_landing_points"
|
|
priority = "P1"
|
|
module = "L2"
|
|
frequency_hours = 168
|
|
data_type = "landing_point"
|
|
|
|
@property
|
|
def base_url(self) -> str:
|
|
return settings.ARCGIS_LANDING_POINT_URL
|
|
|
|
async def fetch(self) -> List[Dict[str, Any]]:
|
|
params = {"where": "1=1", "outFields": "*", "returnGeometry": "true", "f": "geojson"}
|
|
|
|
async with httpx.AsyncClient(timeout=60.0) as client:
|
|
response = await client.get(self.base_url, params=params)
|
|
response.raise_for_status()
|
|
return self.parse_response(response.json())
|
|
|
|
def parse_response(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
result = []
|
|
|
|
features = data.get("features", [])
|
|
for feature in features:
|
|
props = feature.get("properties", {})
|
|
geometry = feature.get("geometry", {})
|
|
|
|
lat = geometry.get("y") if geometry else None
|
|
lon = geometry.get("x") if geometry else None
|
|
|
|
try:
|
|
entry = {
|
|
"source_id": f"arcgis_lp_{props.get('OBJECTID', props.get('id', ''))}",
|
|
"name": props.get("Name", props.get("name", "Unknown")),
|
|
"country": props.get("country", ""),
|
|
"city": props.get("city", ""),
|
|
"latitude": str(lat) if lat else "",
|
|
"longitude": str(lon) if lon else "",
|
|
"value": "",
|
|
"unit": "",
|
|
"metadata": {
|
|
"objectid": props.get("OBJECTID"),
|
|
"cable_id": props.get("cable_id"),
|
|
"cable_name": props.get("cable_name"),
|
|
"facility": props.get("facility"),
|
|
"facility_type": props.get("facility_type"),
|
|
"status": props.get("status"),
|
|
"landing_point_id": props.get("landing_point_id"),
|
|
},
|
|
"reference_date": datetime.utcnow().strftime("%Y-%m-%d"),
|
|
}
|
|
result.append(entry)
|
|
except (ValueError, TypeError, KeyError):
|
|
continue
|
|
|
|
return result
|