feat: add data sources config system and Earth API integration

- Add data_sources.yaml for configurable data source URLs
- Add data_sources.py to load config with database override support
- Add arcgis_landing_points and arcgis_cable_landing_relation collectors
- Change visualization API to query arcgis_landing_points
- Add /api/v1/datasources/configs/all endpoint
- Update Earth to fetch from API instead of static files
- Fix scheduler collector ID mappings
This commit is contained in:
rayd1o
2026-03-13 10:54:02 +08:00
parent 99771a88c5
commit de32552159
25 changed files with 222 additions and 23 deletions

View File

@@ -10,6 +10,7 @@ from app.models.user import User
from app.core.security import get_current_user
from app.models.alert import Alert, AlertSeverity, AlertStatus
router = APIRouter()

View File

@@ -14,6 +14,7 @@ from app.models.task import CollectionTask
from app.core.security import get_current_user
from app.core.cache import cache
# Built-in collectors info (mirrored from datasources.py)
COLLECTOR_INFO = {
"top500": {

View File

@@ -307,3 +307,40 @@ async def test_new_config(
"error": "Connection failed",
"message": str(e),
}
@router.get("/configs/all")
async def list_all_datasources(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""List all data sources: YAML defaults + DB overrides"""
from app.core.data_sources import COLLECTOR_URL_KEYS, get_data_sources_config
config = get_data_sources_config()
db_query = await db.execute(select(DataSourceConfig))
db_configs = {c.name: c for c in db_query.scalars().all()}
result = []
for name, yaml_key in COLLECTOR_URL_KEYS.items():
yaml_url = config.get_yaml_url(name)
db_config = db_configs.get(name)
result.append(
{
"name": name,
"default_url": yaml_url,
"endpoint": db_config.endpoint if db_config else yaml_url,
"is_overridden": db_config is not None and db_config.endpoint != yaml_url
if yaml_url
else db_config is not None,
"is_active": db_config.is_active if db_config else True,
"source_type": db_config.source_type if db_config else "http",
"description": db_config.description
if db_config
else f"Data source from YAML: {yaml_key}",
}
)
return {"total": len(result), "data": result}

View File

@@ -99,8 +99,22 @@ COLLECTOR_INFO = {
"priority": "P1",
"frequency_hours": 168,
},
"fao_landing_points": {
"arcgis_landing_points": {
"id": 16,
"name": "ArcGIS Landing Points",
"module": "L2",
"priority": "P1",
"frequency_hours": 168,
},
"arcgis_cable_landing_relation": {
"id": 17,
"name": "ArcGIS Cable-Landing Relations",
"module": "L2",
"priority": "P1",
"frequency_hours": 168,
},
"fao_landing_points": {
"id": 18,
"name": "FAO Landing Points",
"module": "L2",
"priority": "P1",

View File

@@ -10,6 +10,7 @@ from app.models.user import User
from app.core.security import get_current_user
from app.services.collectors.registry import collector_registry
router = APIRouter()

View File

@@ -146,14 +146,14 @@ async def get_cables_geojson(db: AsyncSession = Depends(get_db)):
async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
"""获取登陆点 GeoJSON 数据 (Point)"""
try:
stmt = select(CollectedData).where(CollectedData.source == "fao_landing_points")
stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
result = await db.execute(stmt)
records = result.scalars().all()
if not records:
raise HTTPException(
status_code=404,
detail="No landing point data found. Please run the fao_landing_points collector first.",
detail="No landing point data found. Please run the arcgis_landing_points collector first.",
)
return convert_landing_point_to_geojson(records)
@@ -170,7 +170,7 @@ async def get_all_geojson(db: AsyncSession = Depends(get_db)):
cables_result = await db.execute(cables_stmt)
cables_records = cables_result.scalars().all()
points_stmt = select(CollectedData).where(CollectedData.source == "fao_landing_points")
points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
points_result = await db.execute(points_stmt)
points_records = points_result.scalars().all()
@@ -208,7 +208,7 @@ async def get_cable_graph(db: AsyncSession) -> CableGraph:
cables_result = await db.execute(cables_stmt)
cables_records = list(cables_result.scalars().all())
points_stmt = select(CollectedData).where(CollectedData.source == "fao_landing_points")
points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
points_result = await db.execute(points_stmt)
points_records = list(points_result.scalars().all())