153 lines
6.1 KiB
Python
153 lines
6.1 KiB
Python
import asyncio
|
|
from datetime import datetime
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
import httpx
|
|
|
|
from app.core.data_sources import get_data_sources_config
|
|
from app.services.collectors.base import BaseCollector
|
|
|
|
|
|
class ArcGISCableLandingRelationCollector(BaseCollector):
|
|
name = "arcgis_cable_landing_relation"
|
|
priority = "P1"
|
|
module = "L2"
|
|
frequency_hours = 168
|
|
data_type = "cable_landing_relation"
|
|
|
|
@property
|
|
def base_url(self) -> str:
|
|
if self._resolved_url:
|
|
return self._resolved_url
|
|
config = get_data_sources_config()
|
|
return config.get_yaml_url("arcgis_cable_landing_relation")
|
|
|
|
def _layer_url(self, layer_id: int) -> str:
|
|
if "/FeatureServer/" not in self.base_url:
|
|
return self.base_url
|
|
prefix = self.base_url.split("/FeatureServer/")[0]
|
|
return f"{prefix}/FeatureServer/{layer_id}/query"
|
|
|
|
async def _fetch_layer_attributes(
|
|
self, client: httpx.AsyncClient, layer_id: int
|
|
) -> List[Dict[str, Any]]:
|
|
response = await client.get(
|
|
self._layer_url(layer_id),
|
|
params={
|
|
"where": "1=1",
|
|
"outFields": "*",
|
|
"returnGeometry": "false",
|
|
"f": "json",
|
|
},
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
return [feature.get("attributes", {}) for feature in data.get("features", [])]
|
|
|
|
async def _fetch_relation_features(self, client: httpx.AsyncClient) -> List[Dict[str, Any]]:
|
|
response = await client.get(
|
|
self.base_url,
|
|
params={
|
|
"where": "1=1",
|
|
"outFields": "*",
|
|
"returnGeometry": "true",
|
|
"f": "geojson",
|
|
},
|
|
)
|
|
response.raise_for_status()
|
|
data = response.json()
|
|
return data.get("features", [])
|
|
|
|
async def fetch(self) -> List[Dict[str, Any]]:
|
|
async with httpx.AsyncClient(timeout=60.0) as client:
|
|
relation_features, landing_rows, cable_rows = await asyncio.gather(
|
|
self._fetch_relation_features(client),
|
|
self._fetch_layer_attributes(client, 1),
|
|
self._fetch_layer_attributes(client, 2),
|
|
)
|
|
return self.parse_response(relation_features, landing_rows, cable_rows)
|
|
|
|
def _build_landing_lookup(self, landing_rows: List[Dict[str, Any]]) -> Dict[int, Dict[str, Any]]:
|
|
lookup: Dict[int, Dict[str, Any]] = {}
|
|
for row in landing_rows:
|
|
city_id = row.get("city_id")
|
|
if city_id is None:
|
|
continue
|
|
lookup[int(city_id)] = {
|
|
"landing_point_id": row.get("landing_point_id") or city_id,
|
|
"landing_point_name": row.get("Name") or row.get("name") or "",
|
|
"facility": row.get("facility") or "",
|
|
"status": row.get("status") or "",
|
|
"country": row.get("country") or "",
|
|
}
|
|
return lookup
|
|
|
|
def _build_cable_lookup(self, cable_rows: List[Dict[str, Any]]) -> Dict[int, Dict[str, Any]]:
|
|
lookup: Dict[int, Dict[str, Any]] = {}
|
|
for row in cable_rows:
|
|
cable_id = row.get("cable_id")
|
|
if cable_id is None:
|
|
continue
|
|
lookup[int(cable_id)] = {
|
|
"cable_name": row.get("Name") or "",
|
|
"status": row.get("status") or "active",
|
|
}
|
|
return lookup
|
|
|
|
def parse_response(
|
|
self,
|
|
relation_features: List[Dict[str, Any]],
|
|
landing_rows: List[Dict[str, Any]],
|
|
cable_rows: List[Dict[str, Any]],
|
|
) -> List[Dict[str, Any]]:
|
|
result: List[Dict[str, Any]] = []
|
|
landing_lookup = self._build_landing_lookup(landing_rows)
|
|
cable_lookup = self._build_cable_lookup(cable_rows)
|
|
|
|
for feature in relation_features:
|
|
props = feature.get("properties", {})
|
|
|
|
try:
|
|
city_id = props.get("city_id")
|
|
cable_id = props.get("cable_id")
|
|
landing_info = landing_lookup.get(int(city_id), {}) if city_id is not None else {}
|
|
cable_info = cable_lookup.get(int(cable_id), {}) if cable_id is not None else {}
|
|
|
|
cable_name = cable_info.get("cable_name") or props.get("cable_name") or "Unknown"
|
|
landing_point_name = (
|
|
landing_info.get("landing_point_name")
|
|
or props.get("landing_point_name")
|
|
or "Unknown"
|
|
)
|
|
facility = landing_info.get("facility") or props.get("facility") or "-"
|
|
status = cable_info.get("status") or landing_info.get("status") or props.get("status") or "-"
|
|
country = landing_info.get("country") or props.get("country") or ""
|
|
landing_point_id = landing_info.get("landing_point_id") or props.get("landing_point_id") or city_id
|
|
|
|
entry = {
|
|
"source_id": f"arcgis_relation_{props.get('OBJECTID', props.get('id', ''))}",
|
|
"name": f"{cable_name} - {landing_point_name}",
|
|
"country": country,
|
|
"city": landing_point_name,
|
|
"latitude": str(props.get("latitude", "")) if props.get("latitude") else "",
|
|
"longitude": str(props.get("longitude", "")) if props.get("longitude") else "",
|
|
"value": "",
|
|
"unit": "",
|
|
"metadata": {
|
|
"objectid": props.get("OBJECTID"),
|
|
"city_id": city_id,
|
|
"cable_id": cable_id,
|
|
"cable_name": cable_name,
|
|
"landing_point_id": landing_point_id,
|
|
"landing_point_name": landing_point_name,
|
|
"facility": facility,
|
|
"status": status,
|
|
},
|
|
"reference_date": datetime.utcnow().strftime("%Y-%m-%d"),
|
|
}
|
|
result.append(entry)
|
|
except (ValueError, TypeError, KeyError):
|
|
continue
|
|
|
|
return result
|