feat(backend): Add cable graph service and data collectors
## Changelog ### New Features #### Cable Graph Service - Add cable_graph.py for finding shortest path between landing points - Implement haversine distance calculation for great circle distances - Support for dateline crossing (longitude normalization) - NetworkX-based graph for optimal path finding #### Data Collectors - Add ArcGISCableCollector for fetching submarine cable data from ArcGIS GeoJSON API - Add FAOLandingPointCollector for fetching landing point data from FAO CSV API ### Backend Changes #### API Updates - auth.py: Update authentication logic - datasources.py: Add datasource endpoints and management - visualization.py: Add visualization API endpoints - config.py: Update configuration settings - security.py: Improve security settings #### Models & Schemas - task.py: Update task model with new fields - token.py: Update token schema #### Services - collectors/base.py: Improve base collector with better error handling - collectors/__init__.py: Register new collectors - scheduler.py: Update scheduler logic - tasks/scheduler.py: Add task scheduling ### Frontend Changes - AppLayout.tsx: Improve layout component - index.css: Add global styles - DataSources.tsx: Enhance data sources management page - vite.config.ts: Add Vite configuration for earth module
This commit is contained in:
84
backend/app/services/collectors/arcgis_cables.py
Normal file
84
backend/app/services/collectors/arcgis_cables.py
Normal file
@@ -0,0 +1,84 @@
|
||||
"""ArcGIS Submarine Cables Collector
|
||||
|
||||
Collects submarine cable data from ArcGIS GeoJSON API.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Dict, Any, List
|
||||
from datetime import datetime
|
||||
import httpx
|
||||
|
||||
from app.services.collectors.base import BaseCollector
|
||||
|
||||
|
||||
class ArcGISCableCollector(BaseCollector):
|
||||
name = "arcgis_cables"
|
||||
priority = "P1"
|
||||
module = "L2"
|
||||
frequency_hours = 168
|
||||
data_type = "submarine_cable"
|
||||
|
||||
base_url = "https://services.arcgis.com/6DIQcwlPy8knb6sg/arcgis/rest/services/SubmarineCables/FeatureServer/2/query"
|
||||
|
||||
async def fetch(self) -> List[Dict[str, Any]]:
|
||||
params = {"where": "1=1", "outFields": "*", "returnGeometry": "true", "f": "geojson"}
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
response = await client.get(self.base_url, params=params)
|
||||
response.raise_for_status()
|
||||
return self.parse_response(response.json())
|
||||
|
||||
def parse_response(self, data: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
result = []
|
||||
|
||||
features = data.get("features", [])
|
||||
for feature in features:
|
||||
props = feature.get("properties", {})
|
||||
geometry = feature.get("geometry", {})
|
||||
|
||||
route_coordinates = []
|
||||
if geometry.get("type") == "MultiLineString":
|
||||
coords = geometry.get("coordinates", [])
|
||||
for line in coords:
|
||||
line_coords = []
|
||||
for point in line:
|
||||
if len(point) >= 2:
|
||||
line_coords.append(point)
|
||||
if line_coords:
|
||||
route_coordinates.append(line_coords)
|
||||
elif geometry.get("type") == "LineString":
|
||||
coords = geometry.get("coordinates", [])
|
||||
line_coords = []
|
||||
for point in coords:
|
||||
if len(point) >= 2:
|
||||
line_coords.append(point)
|
||||
if line_coords:
|
||||
route_coordinates.append(line_coords)
|
||||
|
||||
try:
|
||||
entry = {
|
||||
"source_id": f"arcgis_cable_{props.get('cable_id', props.get('OBJECTID', ''))}",
|
||||
"name": props.get("Name", "Unknown"),
|
||||
"country": "",
|
||||
"city": "",
|
||||
"latitude": "",
|
||||
"longitude": "",
|
||||
"value": str(props.get("length", "")).replace(",", ""),
|
||||
"unit": "km",
|
||||
"metadata": {
|
||||
"cable_id": props.get("cable_id"),
|
||||
"owners": props.get("owners"),
|
||||
"rfs": props.get("rfs"),
|
||||
"status": "active",
|
||||
"year": props.get("year"),
|
||||
"url": props.get("url"),
|
||||
"color": props.get("color"),
|
||||
"route_coordinates": route_coordinates,
|
||||
},
|
||||
"reference_date": datetime.utcnow().strftime("%Y-%m-%d"),
|
||||
}
|
||||
result.append(entry)
|
||||
except (ValueError, TypeError, KeyError):
|
||||
continue
|
||||
|
||||
return result
|
||||
Reference in New Issue
Block a user