feat: add data sources config system and Earth API integration
- Add data_sources.yaml for configurable data source URLs - Add data_sources.py to load config with database override support - Add arcgis_landing_points and arcgis_cable_landing_relation collectors - Change visualization API to query arcgis_landing_points - Add /api/v1/datasources/configs/all endpoint - Update Earth to fetch from API instead of static files - Fix scheduler collector ID mappings
This commit is contained in:
35
backend/app/core/data_sources.yaml
Normal file
35
backend/app/core/data_sources.yaml
Normal file
@@ -0,0 +1,35 @@
|
||||
# Data Sources Configuration
|
||||
# All external data source URLs should be configured here
|
||||
|
||||
arcgis:
|
||||
cable_url: "https://services.arcgis.com/6DIQcwlPy8knb6sg/ArcGIS/rest/services/SubmarineCables/FeatureServer/2/query"
|
||||
landing_point_url: "https://services.arcgis.com/6DIQcwlPy8knb6sg/ArcGIS/rest/services/SubmarineCables/FeatureServer/1/query"
|
||||
cable_landing_relation_url: "https://services.arcgis.com/6DIQcwlPy8knb6sg/ArcGIS/rest/services/SubmarineCables/FeatureServer/3/query"
|
||||
|
||||
fao:
|
||||
landing_point_url: "https://data.apps.fao.org/catalog/dataset/1b75ff21-92f2-4b96-9b7b-98e8aa65ad5d/resource/b6071077-d1d4-4e97-aa00-42e902847c87/download/landing-point-geo.csv"
|
||||
|
||||
telegeography:
|
||||
cable_url: "https://raw.githubusercontent.com/lintaojlu/submarine_cable_information/main/cable.json"
|
||||
landing_point_url: "https://raw.githubusercontent.com/lintaojlu/submarine_cable_information/main/landing_point.json"
|
||||
|
||||
huggingface:
|
||||
models_url: "https://huggingface.co/api/models"
|
||||
datasets_url: "https://huggingface.co/api/datasets"
|
||||
spaces_url: "https://huggingface.co/api/spaces"
|
||||
|
||||
cloudflare:
|
||||
radar_device_url: "https://api.cloudflare.com/client/v4/radar/http/summary/device_type"
|
||||
radar_traffic_url: "https://api.cloudflare.com/client/v4/radar/http/timeseries/requests"
|
||||
radar_top_locations_url: "https://api.cloudflare.com/client/v4/radar/http/top/locations"
|
||||
|
||||
peeringdb:
|
||||
ixp_url: "https://www.peeringdb.com/api/ix"
|
||||
network_url: "https://www.peeringdb.com/api/net"
|
||||
facility_url: "https://www.peeringdb.com/api/fac"
|
||||
|
||||
top500:
|
||||
url: "https://top500.org/lists/top500/list/2025/11/"
|
||||
|
||||
epoch_ai:
|
||||
gpu_clusters_url: "https://epoch.ai/data/gpu-clusters"
|
||||
Reference in New Issue
Block a user