feat(collectors): add ArcGIS landing points and cable-landing relation collectors
- Add ArcGISLandingPointCollector for FeatureServer/1 (landing points) - Add ArcGISCableLandingRelationCollector for FeatureServer/3 (cable-landing relations) - Register new collectors in __init__.py - Fix earth cables.js with cable_id grouping for highlight
This commit is contained in:
@@ -26,6 +26,8 @@ from app.services.collectors.cloudflare import (
|
||||
)
|
||||
from app.services.collectors.arcgis_cables import ArcGISCableCollector
|
||||
from app.services.collectors.fao_landing import FAOLandingPointCollector
|
||||
from app.services.collectors.arcgis_landing import ArcGISLandingPointCollector
|
||||
from app.services.collectors.arcgis_relation import ArcGISCableLandingRelationCollector
|
||||
|
||||
collector_registry.register(TOP500Collector())
|
||||
collector_registry.register(EpochAIGPUCollector())
|
||||
@@ -43,3 +45,5 @@ collector_registry.register(CloudflareRadarTrafficCollector())
|
||||
collector_registry.register(CloudflareRadarTopASCollector())
|
||||
collector_registry.register(ArcGISCableCollector())
|
||||
collector_registry.register(FAOLandingPointCollector())
|
||||
collector_registry.register(ArcGISLandingPointCollector())
|
||||
collector_registry.register(ArcGISCableLandingRelationCollector())
|
||||
|
||||
70
backend/app/services/collectors/arcgis_landing.py
Normal file
70
backend/app/services/collectors/arcgis_landing.py
Normal file
@@ -0,0 +1,70 @@
|
||||
"""ArcGIS Landing Points Collector
|
||||
|
||||
Collects landing point data from ArcGIS GeoJSON API.
|
||||
"""
|
||||
|
||||
from typing import Dict, Any, List
|
||||
from datetime import datetime
|
||||
|
||||
from app.services.collectors.base import BaseCollector
|
||||
|
||||
|
||||
class ArcGISLandingPointCollector(BaseCollector):
|
||||
name = "arcgis_landing_points"
|
||||
priority = "P1"
|
||||
module = "L2"
|
||||
frequency_hours = 168
|
||||
data_type = "landing_point"
|
||||
|
||||
base_url = "https://services.arcgis.com/6DIQcwlPy8knb6sg/arcgis/rest/services/SubmarineCables/FeatureServer/1/query"
|
||||
|
||||
async def fetch(self) -> List[Dict[str, Any]]:
|
||||
params = {"where": "1=1", "outFields": "*", "returnGeometry": "true", "f": "geojson"}
|
||||
|
||||
async with self._get_client() as client:
|
||||
response = await client.get(self.base_url, params=params)
|
||||
response.raise_for_status()
|
||||
return self.parse_response(response.json())
|
||||
|
||||
def _get_client(self):
|
||||
import httpx
|
||||
|
||||
return httpx.AsyncClient(timeout=60.0)
|
||||
|
||||
def parse_response(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
result = []
|
||||
|
||||
features = data.get("features", [])
|
||||
for feature in features:
|
||||
props = feature.get("properties", {})
|
||||
geometry = feature.get("geometry", {})
|
||||
|
||||
lat = geometry.get("y") if geometry else None
|
||||
lon = geometry.get("x") if geometry else None
|
||||
|
||||
try:
|
||||
entry = {
|
||||
"source_id": f"arcgis_lp_{props.get('OBJECTID', props.get('id', ''))}",
|
||||
"name": props.get("Name", props.get("name", "Unknown")),
|
||||
"country": props.get("country", ""),
|
||||
"city": props.get("city", ""),
|
||||
"latitude": str(lat) if lat else "",
|
||||
"longitude": str(lon) if lon else "",
|
||||
"value": "",
|
||||
"unit": "",
|
||||
"metadata": {
|
||||
"objectid": props.get("OBJECTID"),
|
||||
"cable_id": props.get("cable_id"),
|
||||
"cable_name": props.get("cable_name"),
|
||||
"facility": props.get("facility"),
|
||||
"facility_type": props.get("facility_type"),
|
||||
"status": props.get("status"),
|
||||
"landing_point_id": props.get("landing_point_id"),
|
||||
},
|
||||
"reference_date": datetime.utcnow().strftime("%Y-%m-%d"),
|
||||
}
|
||||
result.append(entry)
|
||||
except (ValueError, TypeError, KeyError):
|
||||
continue
|
||||
|
||||
return result
|
||||
58
backend/app/services/collectors/arcgis_relation.py
Normal file
58
backend/app/services/collectors/arcgis_relation.py
Normal file
@@ -0,0 +1,58 @@
|
||||
from typing import Dict, Any, List
|
||||
from datetime import datetime
|
||||
|
||||
from app.services.collectors.base import BaseCollector
|
||||
|
||||
|
||||
class ArcGISCableLandingRelationCollector(BaseCollector):
|
||||
name = "arcgis_cable_landing_relation"
|
||||
priority = "P1"
|
||||
module = "L2"
|
||||
frequency_hours = 168
|
||||
data_type = "cable_landing_relation"
|
||||
|
||||
base_url = "https://services.arcgis.com/6DIQcwlPy8knb6sg/arcgis/rest/services/SubmarineCables/FeatureServer/3/query"
|
||||
|
||||
async def fetch(self) -> List[Dict[str, Any]]:
|
||||
import httpx
|
||||
|
||||
params = {"where": "1=1", "outFields": "*", "returnGeometry": "true", "f": "geojson"}
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
response = await client.get(self.base_url, params=params)
|
||||
response.raise_for_status()
|
||||
return self.parse_response(response.json())
|
||||
|
||||
def parse_response(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
result = []
|
||||
|
||||
features = data.get("features", [])
|
||||
for feature in features:
|
||||
props = feature.get("properties", {})
|
||||
|
||||
try:
|
||||
entry = {
|
||||
"source_id": f"arcgis_relation_{props.get('OBJECTID', props.get('id', ''))}",
|
||||
"name": f"{props.get('cable_name', 'Unknown')} - {props.get('landing_point_name', 'Unknown')}",
|
||||
"country": props.get("country", ""),
|
||||
"city": props.get("landing_point_name", ""),
|
||||
"latitude": str(props.get("latitude", "")) if props.get("latitude") else "",
|
||||
"longitude": str(props.get("longitude", "")) if props.get("longitude") else "",
|
||||
"value": "",
|
||||
"unit": "",
|
||||
"metadata": {
|
||||
"objectid": props.get("OBJECTID"),
|
||||
"cable_id": props.get("cable_id"),
|
||||
"cable_name": props.get("cable_name"),
|
||||
"landing_point_id": props.get("landing_point_id"),
|
||||
"landing_point_name": props.get("landing_point_name"),
|
||||
"facility": props.get("facility"),
|
||||
"status": props.get("status"),
|
||||
},
|
||||
"reference_date": datetime.utcnow().strftime("%Y-%m-%d"),
|
||||
}
|
||||
result.append(entry)
|
||||
except (ValueError, TypeError, KeyError):
|
||||
continue
|
||||
|
||||
return result
|
||||
@@ -312,6 +312,7 @@ export function handleCableClick(cable) {
|
||||
lockedCable = cable;
|
||||
|
||||
const data = cable.userData;
|
||||
// console.log(data)
|
||||
updateCableDetails({
|
||||
name: data.name,
|
||||
owner: data.owner,
|
||||
|
||||
Reference in New Issue
Block a user