feat: add data sources config system and Earth API integration

- Add data_sources.yaml for configurable data source URLs
- Add data_sources.py to load config with database override support
- Add arcgis_landing_points and arcgis_cable_landing_relation collectors
- Change visualization API to query arcgis_landing_points
- Add /api/v1/datasources/configs/all endpoint
- Update Earth to fetch from API instead of static files
- Fix scheduler collector ID mappings
This commit is contained in:
rayd1o
2026-03-13 10:54:02 +08:00
parent 99771a88c5
commit de32552159
25 changed files with 222 additions and 23 deletions

View File

@@ -23,6 +23,13 @@ class BaseCollector(ABC):
self._current_task = None
self._db_session = None
self._datasource_id = 1
self._resolved_url: Optional[str] = None
async def resolve_url(self, db: AsyncSession) -> None:
from app.core.data_sources import get_data_sources_config
config = get_data_sources_config()
self._resolved_url = await config.get_url(self.name, db)
def update_progress(self, records_processed: int):
"""Update task progress - call this during data processing"""
@@ -65,6 +72,8 @@ class BaseCollector(ABC):
self._current_task = task
self._db_session = db
await self.resolve_url(db)
try:
raw_data = await self.fetch()
task.total_records = len(raw_data)
@@ -87,7 +96,6 @@ class BaseCollector(ABC):
"execution_time_seconds": (datetime.utcnow() - start_time).total_seconds(),
}
except Exception as e:
# Log task failure
task.status = "failed"
task.error_message = str(e)
task.completed_at = datetime.utcnow()