aaae6a53c3
## Changelog ### New Features #### Cable Graph Service - Add cable_graph.py for finding shortest path between landing points - Implement haversine distance calculation for great circle distances - Support for dateline crossing (longitude normalization) - NetworkX-based graph for optimal path finding #### Data Collectors - Add ArcGISCableCollector for fetching submarine cable data from ArcGIS GeoJSON API - Add FAOLandingPointCollector for fetching landing point data from FAO CSV API ### Backend Changes #### API Updates - auth.py: Update authentication logic - datasources.py: Add datasource endpoints and management - visualization.py: Add visualization API endpoints - config.py: Update configuration settings - security.py: Improve security settings #### Models & Schemas - task.py: Update task model with new fields - token.py: Update token schema #### Services - collectors/base.py: Improve base collector with better error handling - collectors/__init__.py: Register new collectors - scheduler.py: Update scheduler logic - tasks/scheduler.py: Add task scheduling ### Frontend Changes - AppLayout.tsx: Improve layout component - index.css: Add global styles - DataSources.tsx: Enhance data sources management page - vite.config.ts: Add Vite configuration for earth module
39 lines
1.2 KiB
Python
39 lines
1.2 KiB
Python
"""Celery tasks for data collection"""
|
|
|
|
import asyncio
|
|
from datetime import datetime
|
|
from typing import Dict, Any
|
|
|
|
from app.db.session import async_session_factory
|
|
from app.services.collectors.registry import collector_registry
|
|
|
|
|
|
async def run_collector_task(collector_name: str) -> Dict[str, Any]:
|
|
"""Run a single collector task"""
|
|
from sqlalchemy import select
|
|
from app.models.datasource import DataSource
|
|
|
|
collector = collector_registry.get(collector_name)
|
|
if not collector:
|
|
return {"status": "failed", "error": f"Collector {collector_name} not found"}
|
|
|
|
if not collector_registry.is_active(collector_name):
|
|
return {"status": "skipped", "reason": "Collector is disabled"}
|
|
|
|
async with async_session_factory() as db:
|
|
result = await db.execute(
|
|
select(DataSource.id).where(DataSource.collector_class == collector_name)
|
|
)
|
|
datasource = result.scalar_one_or_none()
|
|
|
|
if datasource:
|
|
collector._datasource_id = datasource
|
|
|
|
result = await collector.run(db)
|
|
return result
|
|
|
|
|
|
def run_collector_sync(collector_name: str) -> Dict[str, Any]:
|
|
"""Synchronous wrapper for running collectors"""
|
|
return asyncio.run(run_collector_task(collector_name))
|