de32552159
- Add data_sources.yaml for configurable data source URLs - Add data_sources.py to load config with database override support - Add arcgis_landing_points and arcgis_cable_landing_relation collectors - Change visualization API to query arcgis_landing_points - Add /api/v1/datasources/configs/all endpoint - Update Earth to fetch from API instead of static files - Fix scheduler collector ID mappings
94 lines
3.3 KiB
Python
94 lines
3.3 KiB
Python
"""ArcGIS Submarine Cables Collector
|
|
|
|
Collects submarine cable data from ArcGIS GeoJSON API.
|
|
"""
|
|
|
|
import json
|
|
from typing import Dict, Any, List
|
|
from datetime import datetime
|
|
import httpx
|
|
|
|
from app.services.collectors.base import BaseCollector
|
|
from app.core.data_sources import get_data_sources_config
|
|
|
|
|
|
|
|
class ArcGISCableCollector(BaseCollector):
|
|
name = "arcgis_cables"
|
|
priority = "P1"
|
|
module = "L2"
|
|
frequency_hours = 168
|
|
data_type = "submarine_cable"
|
|
|
|
@property
|
|
def base_url(self) -> str:
|
|
if self._resolved_url:
|
|
return self._resolved_url
|
|
from app.core.data_sources import get_data_sources_config
|
|
|
|
config = get_data_sources_config()
|
|
return config.get_yaml_url("arcgis_cables")
|
|
|
|
async def fetch(self) -> List[Dict[str, Any]]:
|
|
params = {"where": "1=1", "outFields": "*", "returnGeometry": "true", "f": "geojson"}
|
|
|
|
async with httpx.AsyncClient(timeout=60.0) as client:
|
|
response = await client.get(self.base_url, params=params)
|
|
response.raise_for_status()
|
|
return self.parse_response(response.json())
|
|
|
|
def parse_response(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
result = []
|
|
|
|
features = data.get("features", [])
|
|
for feature in features:
|
|
props = feature.get("properties", {})
|
|
geometry = feature.get("geometry", {})
|
|
|
|
route_coordinates = []
|
|
if geometry.get("type") == "MultiLineString":
|
|
coords = geometry.get("coordinates", [])
|
|
for line in coords:
|
|
line_coords = []
|
|
for point in line:
|
|
if len(point) >= 2:
|
|
line_coords.append(point)
|
|
if line_coords:
|
|
route_coordinates.append(line_coords)
|
|
elif geometry.get("type") == "LineString":
|
|
coords = geometry.get("coordinates", [])
|
|
line_coords = []
|
|
for point in coords:
|
|
if len(point) >= 2:
|
|
line_coords.append(point)
|
|
if line_coords:
|
|
route_coordinates.append(line_coords)
|
|
|
|
try:
|
|
entry = {
|
|
"source_id": f"arcgis_cable_{props.get('cable_id', props.get('OBJECTID', ''))}",
|
|
"name": props.get("Name", "Unknown"),
|
|
"country": "",
|
|
"city": "",
|
|
"latitude": "",
|
|
"longitude": "",
|
|
"value": str(props.get("length", "")).replace(",", ""),
|
|
"unit": "km",
|
|
"metadata": {
|
|
"cable_id": props.get("cable_id"),
|
|
"owners": props.get("owners"),
|
|
"rfs": props.get("rfs"),
|
|
"status": "active",
|
|
"year": props.get("year"),
|
|
"url": props.get("url"),
|
|
"color": props.get("color"),
|
|
"route_coordinates": route_coordinates,
|
|
},
|
|
"reference_date": datetime.utcnow().strftime("%Y-%m-%d"),
|
|
}
|
|
result.append(entry)
|
|
except (ValueError, TypeError, KeyError):
|
|
continue
|
|
|
|
return result
|