feat: add data sources config system and Earth API integration
- Add data_sources.yaml for configurable data source URLs - Add data_sources.py to load config with database override support - Add arcgis_landing_points and arcgis_cable_landing_relation collectors - Change visualization API to query arcgis_landing_points - Add /api/v1/datasources/configs/all endpoint - Update Earth to fetch from API instead of static files - Fix scheduler collector ID mappings
This commit is contained in:
@@ -10,6 +10,7 @@ from app.models.user import User
|
||||
from app.core.security import get_current_user
|
||||
from app.models.alert import Alert, AlertSeverity, AlertStatus
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ from app.models.task import CollectionTask
|
||||
from app.core.security import get_current_user
|
||||
from app.core.cache import cache
|
||||
|
||||
|
||||
# Built-in collectors info (mirrored from datasources.py)
|
||||
COLLECTOR_INFO = {
|
||||
"top500": {
|
||||
|
||||
@@ -307,3 +307,40 @@ async def test_new_config(
|
||||
"error": "Connection failed",
|
||||
"message": str(e),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/configs/all")
|
||||
async def list_all_datasources(
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""List all data sources: YAML defaults + DB overrides"""
|
||||
from app.core.data_sources import COLLECTOR_URL_KEYS, get_data_sources_config
|
||||
|
||||
config = get_data_sources_config()
|
||||
|
||||
db_query = await db.execute(select(DataSourceConfig))
|
||||
db_configs = {c.name: c for c in db_query.scalars().all()}
|
||||
|
||||
result = []
|
||||
for name, yaml_key in COLLECTOR_URL_KEYS.items():
|
||||
yaml_url = config.get_yaml_url(name)
|
||||
db_config = db_configs.get(name)
|
||||
|
||||
result.append(
|
||||
{
|
||||
"name": name,
|
||||
"default_url": yaml_url,
|
||||
"endpoint": db_config.endpoint if db_config else yaml_url,
|
||||
"is_overridden": db_config is not None and db_config.endpoint != yaml_url
|
||||
if yaml_url
|
||||
else db_config is not None,
|
||||
"is_active": db_config.is_active if db_config else True,
|
||||
"source_type": db_config.source_type if db_config else "http",
|
||||
"description": db_config.description
|
||||
if db_config
|
||||
else f"Data source from YAML: {yaml_key}",
|
||||
}
|
||||
)
|
||||
|
||||
return {"total": len(result), "data": result}
|
||||
|
||||
@@ -99,8 +99,22 @@ COLLECTOR_INFO = {
|
||||
"priority": "P1",
|
||||
"frequency_hours": 168,
|
||||
},
|
||||
"fao_landing_points": {
|
||||
"arcgis_landing_points": {
|
||||
"id": 16,
|
||||
"name": "ArcGIS Landing Points",
|
||||
"module": "L2",
|
||||
"priority": "P1",
|
||||
"frequency_hours": 168,
|
||||
},
|
||||
"arcgis_cable_landing_relation": {
|
||||
"id": 17,
|
||||
"name": "ArcGIS Cable-Landing Relations",
|
||||
"module": "L2",
|
||||
"priority": "P1",
|
||||
"frequency_hours": 168,
|
||||
},
|
||||
"fao_landing_points": {
|
||||
"id": 18,
|
||||
"name": "FAO Landing Points",
|
||||
"module": "L2",
|
||||
"priority": "P1",
|
||||
|
||||
@@ -10,6 +10,7 @@ from app.models.user import User
|
||||
from app.core.security import get_current_user
|
||||
from app.services.collectors.registry import collector_registry
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
|
||||
@@ -146,14 +146,14 @@ async def get_cables_geojson(db: AsyncSession = Depends(get_db)):
|
||||
async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
|
||||
"""获取登陆点 GeoJSON 数据 (Point)"""
|
||||
try:
|
||||
stmt = select(CollectedData).where(CollectedData.source == "fao_landing_points")
|
||||
stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
|
||||
result = await db.execute(stmt)
|
||||
records = result.scalars().all()
|
||||
|
||||
if not records:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="No landing point data found. Please run the fao_landing_points collector first.",
|
||||
detail="No landing point data found. Please run the arcgis_landing_points collector first.",
|
||||
)
|
||||
|
||||
return convert_landing_point_to_geojson(records)
|
||||
@@ -170,7 +170,7 @@ async def get_all_geojson(db: AsyncSession = Depends(get_db)):
|
||||
cables_result = await db.execute(cables_stmt)
|
||||
cables_records = cables_result.scalars().all()
|
||||
|
||||
points_stmt = select(CollectedData).where(CollectedData.source == "fao_landing_points")
|
||||
points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
|
||||
points_result = await db.execute(points_stmt)
|
||||
points_records = points_result.scalars().all()
|
||||
|
||||
@@ -208,7 +208,7 @@ async def get_cable_graph(db: AsyncSession) -> CableGraph:
|
||||
cables_result = await db.execute(cables_stmt)
|
||||
cables_records = list(cables_result.scalars().all())
|
||||
|
||||
points_stmt = select(CollectedData).where(CollectedData.source == "fao_landing_points")
|
||||
points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
|
||||
points_result = await db.execute(points_stmt)
|
||||
points_records = list(points_result.scalars().all())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user