feat(backend): Add cable graph service and data collectors
## Changelog ### New Features #### Cable Graph Service - Add cable_graph.py for finding shortest path between landing points - Implement haversine distance calculation for great circle distances - Support for dateline crossing (longitude normalization) - NetworkX-based graph for optimal path finding #### Data Collectors - Add ArcGISCableCollector for fetching submarine cable data from ArcGIS GeoJSON API - Add FAOLandingPointCollector for fetching landing point data from FAO CSV API ### Backend Changes #### API Updates - auth.py: Update authentication logic - datasources.py: Add datasource endpoints and management - visualization.py: Add visualization API endpoints - config.py: Update configuration settings - security.py: Improve security settings #### Models & Schemas - task.py: Update task model with new fields - token.py: Update token schema #### Services - collectors/base.py: Improve base collector with better error handling - collectors/__init__.py: Register new collectors - scheduler.py: Update scheduler logic - tasks/scheduler.py: Add task scheduling ### Frontend Changes - AppLayout.tsx: Improve layout component - index.css: Add global styles - DataSources.tsx: Enhance data sources management page - vite.config.ts: Add Vite configuration for earth module
This commit is contained in:
rayd1o
@@ -61,10 +61,14 @@ async def login(
|
||||
access_token = create_access_token(data={"sub": user.id})
|
||||
refresh_token = create_refresh_token(data={"sub": user.id})
|
||||
|
||||
expires_in = None
|
||||
if settings.ACCESS_TOKEN_EXPIRE_MINUTES > 0:
|
||||
expires_in = settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60
|
||||
|
||||
return {
|
||||
"access_token": access_token,
|
||||
"token_type": "bearer",
|
||||
"expires_in": settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60,
|
||||
"expires_in": expires_in,
|
||||
"user": {
|
||||
"id": user.id,
|
||||
"username": user.username,
|
||||
@@ -79,10 +83,14 @@ async def refresh_token(
|
||||
):
|
||||
access_token = create_access_token(data={"sub": current_user.id})
|
||||
|
||||
expires_in = None
|
||||
if settings.ACCESS_TOKEN_EXPIRE_MINUTES > 0:
|
||||
expires_in = settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60
|
||||
|
||||
return {
|
||||
"access_token": access_token,
|
||||
"token_type": "bearer",
|
||||
"expires_in": settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60,
|
||||
"expires_in": expires_in,
|
||||
"user": {
|
||||
"id": current_user.id,
|
||||
"username": current_user.username,
|
||||
|
||||
@@ -7,6 +7,8 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.db.session import get_db
|
||||
from app.models.user import User
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.task import CollectionTask
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.core.security import get_current_user
|
||||
from app.services.collectors.registry import collector_registry
|
||||
|
||||
@@ -90,6 +92,20 @@ COLLECTOR_INFO = {
|
||||
"priority": "P2",
|
||||
"frequency_hours": 168,
|
||||
},
|
||||
"arcgis_cables": {
|
||||
"id": 15,
|
||||
"name": "ArcGIS Submarine Cables",
|
||||
"module": "L2",
|
||||
"priority": "P1",
|
||||
"frequency_hours": 168,
|
||||
},
|
||||
"fao_landing_points": {
|
||||
"id": 16,
|
||||
"name": "FAO Landing Points",
|
||||
"module": "L2",
|
||||
"priority": "P1",
|
||||
"frequency_hours": 168,
|
||||
},
|
||||
}
|
||||
|
||||
ID_TO_COLLECTOR = {info["id"]: name for name, info in COLLECTOR_INFO.items()}
|
||||
@@ -135,6 +151,35 @@ async def list_datasources(
|
||||
collector_list = []
|
||||
for name, info in COLLECTOR_INFO.items():
|
||||
is_active_status = collector_registry.is_active(name)
|
||||
|
||||
running_task_query = (
|
||||
select(CollectionTask)
|
||||
.where(CollectionTask.datasource_id == info["id"])
|
||||
.where(CollectionTask.status == "running")
|
||||
.order_by(CollectionTask.started_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
running_result = await db.execute(running_task_query)
|
||||
running_task = running_result.scalar_one_or_none()
|
||||
|
||||
last_run_query = (
|
||||
select(CollectionTask)
|
||||
.where(CollectionTask.datasource_id == info["id"])
|
||||
.where(CollectionTask.completed_at.isnot(None))
|
||||
.order_by(CollectionTask.completed_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
last_run_result = await db.execute(last_run_query)
|
||||
last_task = last_run_result.scalar_one_or_none()
|
||||
|
||||
data_count_query = select(func.count(CollectedData.id)).where(CollectedData.source == name)
|
||||
data_count_result = await db.execute(data_count_query)
|
||||
data_count = data_count_result.scalar() or 0
|
||||
|
||||
last_run = None
|
||||
if last_task and last_task.completed_at and data_count > 0:
|
||||
last_run = last_task.completed_at.strftime("%Y-%m-%d %H:%M")
|
||||
|
||||
collector_list.append(
|
||||
{
|
||||
"id": info["id"],
|
||||
@@ -144,6 +189,12 @@ async def list_datasources(
|
||||
"frequency": f"{info['frequency_hours']}h",
|
||||
"is_active": is_active_status,
|
||||
"collector_class": name,
|
||||
"last_run": last_run,
|
||||
"is_running": running_task is not None,
|
||||
"task_id": running_task.id if running_task else None,
|
||||
"progress": running_task.progress if running_task else None,
|
||||
"records_processed": running_task.records_processed if running_task else None,
|
||||
"total_records": running_task.total_records if running_task else None,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -215,16 +266,22 @@ async def get_datasource_stats(
|
||||
raise HTTPException(status_code=404, detail="Data source not found")
|
||||
|
||||
info = COLLECTOR_INFO[collector_name]
|
||||
total_query = select(func.count(DataSource.id)).where(DataSource.source == info["name"])
|
||||
result = await db.execute(total_query)
|
||||
source_name = info["name"]
|
||||
|
||||
query = select(func.count(CollectedData.id)).where(CollectedData.source == collector_name)
|
||||
result = await db.execute(query)
|
||||
total = result.scalar() or 0
|
||||
|
||||
if total == 0:
|
||||
query = select(func.count(CollectedData.id)).where(CollectedData.source == source_name)
|
||||
result = await db.execute(query)
|
||||
total = result.scalar() or 0
|
||||
|
||||
return {
|
||||
"source_id": source_id,
|
||||
"collector_name": collector_name,
|
||||
"name": info["name"],
|
||||
"total_records": total,
|
||||
"last_updated": datetime.utcnow().isoformat(),
|
||||
}
|
||||
|
||||
|
||||
@@ -256,3 +313,80 @@ async def trigger_datasource(
|
||||
status_code=500,
|
||||
detail=f"Failed to trigger collector '{collector_name}'",
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{source_id}/data")
|
||||
async def clear_datasource_data(
|
||||
source_id: str,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
collector_name = get_collector_name(source_id)
|
||||
if not collector_name:
|
||||
raise HTTPException(status_code=404, detail="Data source not found")
|
||||
|
||||
info = COLLECTOR_INFO[collector_name]
|
||||
source_name = info["name"]
|
||||
|
||||
query = select(func.count(CollectedData.id)).where(CollectedData.source == collector_name)
|
||||
result = await db.execute(query)
|
||||
count = result.scalar() or 0
|
||||
|
||||
if count == 0:
|
||||
query = select(func.count(CollectedData.id)).where(CollectedData.source == source_name)
|
||||
result = await db.execute(query)
|
||||
count = result.scalar() or 0
|
||||
delete_source = source_name
|
||||
else:
|
||||
delete_source = collector_name
|
||||
|
||||
if count == 0:
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "No data to clear",
|
||||
"deleted_count": 0,
|
||||
}
|
||||
|
||||
delete_query = CollectedData.__table__.delete().where(CollectedData.source == delete_source)
|
||||
await db.execute(delete_query)
|
||||
await db.commit()
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Cleared {count} records for data source '{info['name']}'",
|
||||
"deleted_count": count,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{source_id}/task-status")
|
||||
async def get_task_status(
|
||||
source_id: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
collector_name = get_collector_name(source_id)
|
||||
if not collector_name:
|
||||
raise HTTPException(status_code=404, detail="Data source not found")
|
||||
|
||||
info = COLLECTOR_INFO[collector_name]
|
||||
|
||||
running_task_query = (
|
||||
select(CollectionTask)
|
||||
.where(CollectionTask.datasource_id == info["id"])
|
||||
.where(CollectionTask.status == "running")
|
||||
.order_by(CollectionTask.started_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
running_result = await db.execute(running_task_query)
|
||||
running_task = running_result.scalar_one_or_none()
|
||||
|
||||
if not running_task:
|
||||
return {"is_running": False, "task_id": None, "progress": None}
|
||||
|
||||
return {
|
||||
"is_running": True,
|
||||
"task_id": running_task.id,
|
||||
"progress": running_task.progress,
|
||||
"records_processed": running_task.records_processed,
|
||||
"total_records": running_task.total_records,
|
||||
"status": running_task.status,
|
||||
}
|
||||
|
||||
@@ -1,75 +1,189 @@
|
||||
"""Visualization API - GeoJSON endpoints for 3D Earth display"""
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
import httpx
|
||||
from fastapi import APIRouter, HTTPException, Depends
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
from typing import List, Dict, Any, Optional
|
||||
|
||||
from app.db.session import get_db
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.services.cable_graph import build_graph_from_data, CableGraph
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
CABLE_DATA_URL = "https://services.arcgis.com/6DIQcwlPy8knb6sg/arcgis/rest/services/SubmarineCables/FeatureServer/2/query?where=1%3D1&outFields=*&returnGeometry=true&f=geojson"
|
||||
LANDING_POINT_CSV_URL = "https://data.apps.fao.org/catalog/dataset/1b75ff21-92f2-4b96-9b7b-98e8aa65ad5d/resource/b6071077-d1d4-4e97-aa00-42e902847c87/download/landing-point-geo.csv"
|
||||
|
||||
def convert_cable_to_geojson(records: List[CollectedData]) -> Dict[str, Any]:
|
||||
"""Convert cable records to GeoJSON FeatureCollection"""
|
||||
features = []
|
||||
|
||||
for record in records:
|
||||
metadata = record.extra_data or {}
|
||||
route_coords = metadata.get("route_coordinates", [])
|
||||
|
||||
if not route_coords:
|
||||
continue
|
||||
|
||||
all_lines = []
|
||||
|
||||
# Handle both old format (flat array) and new format (array of arrays)
|
||||
if route_coords and isinstance(route_coords[0], list):
|
||||
# New format: array of arrays (MultiLineString structure)
|
||||
if route_coords and isinstance(route_coords[0][0], list):
|
||||
# Array of arrays of arrays - multiple lines
|
||||
for line in route_coords:
|
||||
line_coords = []
|
||||
for point in line:
|
||||
if len(point) >= 2:
|
||||
try:
|
||||
lon = float(point[0])
|
||||
lat = float(point[1])
|
||||
line_coords.append([lon, lat])
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if len(line_coords) >= 2:
|
||||
all_lines.append(line_coords)
|
||||
else:
|
||||
# Old format: flat array of points - treat as single line
|
||||
line_coords = []
|
||||
for point in route_coords:
|
||||
if len(point) >= 2:
|
||||
try:
|
||||
lon = float(point[0])
|
||||
lat = float(point[1])
|
||||
line_coords.append([lon, lat])
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
if len(line_coords) >= 2:
|
||||
all_lines.append(line_coords)
|
||||
|
||||
if not all_lines:
|
||||
continue
|
||||
|
||||
# Use MultiLineString format to preserve cable segments
|
||||
features.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "MultiLineString", "coordinates": all_lines},
|
||||
"properties": {
|
||||
"id": record.id,
|
||||
"source_id": record.source_id,
|
||||
"Name": record.name,
|
||||
"name": record.name,
|
||||
"owner": metadata.get("owners"),
|
||||
"owners": metadata.get("owners"),
|
||||
"rfs": metadata.get("rfs"),
|
||||
"RFS": metadata.get("rfs"),
|
||||
"status": metadata.get("status", "active"),
|
||||
"length": record.value,
|
||||
"length_km": record.value,
|
||||
"SHAPE__Length": record.value,
|
||||
"url": metadata.get("url"),
|
||||
"color": metadata.get("color"),
|
||||
"year": metadata.get("year"),
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
return {"type": "FeatureCollection", "features": features}
|
||||
|
||||
|
||||
def convert_landing_point_to_geojson(records: List[CollectedData]) -> Dict[str, Any]:
|
||||
"""Convert landing point records to GeoJSON FeatureCollection"""
|
||||
features = []
|
||||
|
||||
for record in records:
|
||||
try:
|
||||
lat = float(record.latitude) if record.latitude else None
|
||||
lon = float(record.longitude) if record.longitude else None
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
if lat is None or lon is None:
|
||||
continue
|
||||
|
||||
metadata = record.extra_data or {}
|
||||
|
||||
features.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [lon, lat]},
|
||||
"properties": {
|
||||
"id": record.id,
|
||||
"source_id": record.source_id,
|
||||
"name": record.name,
|
||||
"country": record.country,
|
||||
"city": record.city,
|
||||
"is_tbd": metadata.get("is_tbd", False),
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
return {"type": "FeatureCollection", "features": features}
|
||||
|
||||
|
||||
@router.get("/geo/cables")
|
||||
async def get_cables_geojson():
|
||||
async def get_cables_geojson(db: AsyncSession = Depends(get_db)):
|
||||
"""获取海底电缆 GeoJSON 数据 (LineString)"""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
response = await client.get(CABLE_DATA_URL)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except httpx.HTTPError as e:
|
||||
raise HTTPException(status_code=502, detail=f"Failed to fetch cable data: {str(e)}")
|
||||
stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
|
||||
result = await db.execute(stmt)
|
||||
records = result.scalars().all()
|
||||
|
||||
if not records:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="No cable data found. Please run the arcgis_cables collector first.",
|
||||
)
|
||||
|
||||
return convert_cable_to_geojson(records)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Internal error: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/geo/landing-points")
|
||||
async def get_landing_points_geojson():
|
||||
async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
|
||||
"""获取登陆点 GeoJSON 数据 (Point)"""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
response = await client.get(LANDING_POINT_CSV_URL)
|
||||
response.raise_for_status()
|
||||
stmt = select(CollectedData).where(CollectedData.source == "fao_landing_points")
|
||||
result = await db.execute(stmt)
|
||||
records = result.scalars().all()
|
||||
|
||||
lines = response.text.strip().split("\n")
|
||||
if not lines:
|
||||
raise HTTPException(status_code=500, detail="Empty CSV data")
|
||||
if not records:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="No landing point data found. Please run the fao_landing_points collector first.",
|
||||
)
|
||||
|
||||
features = []
|
||||
for line in lines[1:]:
|
||||
if not line.strip():
|
||||
continue
|
||||
parts = line.split(",")
|
||||
if len(parts) >= 4:
|
||||
try:
|
||||
lon = float(parts[0])
|
||||
lat = float(parts[1])
|
||||
feature_id = parts[2]
|
||||
name = parts[3].strip('"')
|
||||
is_tbd = parts[4].strip() == "true" if len(parts) > 4 else False
|
||||
|
||||
features.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [lon, lat]},
|
||||
"properties": {"id": feature_id, "name": name, "is_tbd": is_tbd},
|
||||
}
|
||||
)
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
|
||||
return {"type": "FeatureCollection", "features": features}
|
||||
except httpx.HTTPError as e:
|
||||
raise HTTPException(status_code=502, detail=f"Failed to fetch landing point data: {str(e)}")
|
||||
return convert_landing_point_to_geojson(records)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"Internal error: {str(e)}")
|
||||
|
||||
|
||||
@router.get("/geo/all")
|
||||
async def get_all_geojson():
|
||||
async def get_all_geojson(db: AsyncSession = Depends(get_db)):
|
||||
"""获取所有可视化数据 (电缆 + 登陆点)"""
|
||||
cables = await get_cables_geojson()
|
||||
points = await get_landing_points_geojson()
|
||||
cables_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
|
||||
cables_result = await db.execute(cables_stmt)
|
||||
cables_records = cables_result.scalars().all()
|
||||
|
||||
points_stmt = select(CollectedData).where(CollectedData.source == "fao_landing_points")
|
||||
points_result = await db.execute(points_stmt)
|
||||
points_records = points_result.scalars().all()
|
||||
|
||||
cables = (
|
||||
convert_cable_to_geojson(cables_records)
|
||||
if cables_records
|
||||
else {"type": "FeatureCollection", "features": []}
|
||||
)
|
||||
points = (
|
||||
convert_landing_point_to_geojson(points_records)
|
||||
if points_records
|
||||
else {"type": "FeatureCollection", "features": []}
|
||||
)
|
||||
|
||||
return {
|
||||
"cables": cables,
|
||||
@@ -79,3 +193,52 @@ async def get_all_geojson():
|
||||
"landing_point_count": len(points.get("features", [])) if points else 0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# Cache for cable graph
|
||||
_cable_graph: Optional[CableGraph] = None
|
||||
|
||||
|
||||
async def get_cable_graph(db: AsyncSession) -> CableGraph:
|
||||
"""Get or build cable graph (cached)"""
|
||||
global _cable_graph
|
||||
|
||||
if _cable_graph is None:
|
||||
cables_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
|
||||
cables_result = await db.execute(cables_stmt)
|
||||
cables_records = list(cables_result.scalars().all())
|
||||
|
||||
points_stmt = select(CollectedData).where(CollectedData.source == "fao_landing_points")
|
||||
points_result = await db.execute(points_stmt)
|
||||
points_records = list(points_result.scalars().all())
|
||||
|
||||
cables_data = convert_cable_to_geojson(cables_records)
|
||||
points_data = convert_landing_point_to_geojson(points_records)
|
||||
|
||||
_cable_graph = build_graph_from_data(cables_data, points_data)
|
||||
|
||||
return _cable_graph
|
||||
|
||||
|
||||
@router.post("/geo/path")
|
||||
async def find_path(
|
||||
start: List[float],
|
||||
end: List[float],
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Find shortest path between two coordinates via cable network"""
|
||||
if not start or len(start) != 2:
|
||||
raise HTTPException(status_code=400, detail="Start must be [lon, lat]")
|
||||
if not end or len(end) != 2:
|
||||
raise HTTPException(status_code=400, detail="End must be [lon, lat]")
|
||||
|
||||
graph = await get_cable_graph(db)
|
||||
result = graph.find_shortest_path(start, end)
|
||||
|
||||
if not result:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="No path found between these points. They may be too far from any landing point.",
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
Reference in New Issue
Block a user