linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: configure collector endpoints and health plan

Browse Source
This commit is contained in:
linkong
2026-04-08 10:15:33 +08:00
parent 2d43263b9e
commit 8bd9d34376
14 changed files with 673 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/app/services/collectors/celestrak.py
View File

@@ -21,7 +21,7 @@ class CelesTrakTLECollector(BaseCollector):
@property
def base_url(self) -> str:
return "https://celestrak.org/NORAD/elements/gp.php"
return self._resolved_url or ""
async def fetch(self) -> List[Dict[str, Any]]:
satellite_groups = [
@@ -40,7 +40,7 @@ class CelesTrakTLECollector(BaseCollector):
async with httpx.AsyncClient(timeout=120.0) as client:
for group in satellite_groups:
try:
url = f"https://celestrak.org/NORAD/elements/gp.php?GROUP={group}&FORMAT=json"
url = f"{self.base_url}?GROUP={group}&FORMAT=json"
response = await client.get(url)
if response.status_code == 200:

30
backend/app/services/collectors/cloudflare.py
View File

@@ -39,6 +39,16 @@ class CloudflareRadarDeviceCollector(HTTPCollector):
if CLOUDFLARE_API_TOKEN:
self.headers["Authorization"] = f"Bearer {CLOUDFLARE_API_TOKEN}"
@property
def request_url(self) -> str:
return self._resolved_url or self.base_url
async def fetch(self) -> List[Dict[str, Any]]:
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(self.request_url, headers=self.headers)
response.raise_for_status()
return self.parse_response(response.json())
def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse Cloudflare Radar device type response"""
data = []
@@ -87,6 +97,16 @@ class CloudflareRadarTrafficCollector(HTTPCollector):
if CLOUDFLARE_API_TOKEN:
self.headers["Authorization"] = f"Bearer {CLOUDFLARE_API_TOKEN}"
@property
def request_url(self) -> str:
return self._resolved_url or self.base_url
async def fetch(self) -> List[Dict[str, Any]]:
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(self.request_url, headers=self.headers)
response.raise_for_status()
return self.parse_response(response.json())
def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse Cloudflare Radar traffic timeseries response"""
data = []
@@ -135,6 +155,16 @@ class CloudflareRadarTopASCollector(HTTPCollector):
if CLOUDFLARE_API_TOKEN:
self.headers["Authorization"] = f"Bearer {CLOUDFLARE_API_TOKEN}"
@property
def request_url(self) -> str:
return self._resolved_url or self.base_url
async def fetch(self) -> List[Dict[str, Any]]:
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(self.request_url, headers=self.headers)
response.raise_for_status()
return self.parse_response(response.json())
def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse Cloudflare Radar top locations response"""
data = []

2
backend/app/services/collectors/epoch_ai.py
View File

@@ -23,7 +23,7 @@ class EpochAIGPUCollector(BaseCollector):
async def fetch(self) -> List[Dict[str, Any]]:
"""Fetch Epoch AI GPU clusters data from webpage"""
url = "https://epoch.ai/data/gpu-clusters"
url = self._resolved_url or ""
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(url)

4
backend/app/services/collectors/fao_landing.py
View File

@@ -18,11 +18,9 @@ class FAOLandingPointCollector(BaseCollector):
frequency_hours = 168
data_type = "landing_point"
csv_url = "https://data.apps.fao.org/catalog/dataset/1b75ff21-92f2-4b96-9b7b-98e8aa65ad5d/resource/b6071077-d1d4-4e97-aa00-42e902847c87/download/landing-point-geo.csv"
async def fetch(self) -> List[Dict[str, Any]]:
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(self.csv_url)
response = await client.get(self._resolved_url or "")
response.raise_for_status()
return self.parse_csv(response.text)

36
backend/app/services/collectors/huggingface.py
View File

@@ -21,6 +21,18 @@ class HuggingFaceModelCollector(HTTPCollector):
data_type = "model"
base_url = "https://huggingface.co/api/models"
@property
def request_url(self) -> str:
return self._resolved_url or self.base_url
async def fetch(self) -> List[Dict[str, Any]]:
from httpx import AsyncClient
async with AsyncClient(timeout=60.0) as client:
response = await client.get(self.request_url, headers=self.headers)
response.raise_for_status()
return self.parse_response(response.json())
def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse Hugging Face models API response"""
data = []
@@ -63,6 +75,18 @@ class HuggingFaceDatasetCollector(HTTPCollector):
data_type = "dataset"
base_url = "https://huggingface.co/api/datasets"
@property
def request_url(self) -> str:
return self._resolved_url or self.base_url
async def fetch(self) -> List[Dict[str, Any]]:
from httpx import AsyncClient
async with AsyncClient(timeout=60.0) as client:
response = await client.get(self.request_url, headers=self.headers)
response.raise_for_status()
return self.parse_response(response.json())
def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse Hugging Face datasets API response"""
data = []
@@ -104,6 +128,18 @@ class HuggingFaceSpacesCollector(HTTPCollector):
data_type = "space"
base_url = "https://huggingface.co/api/spaces"
@property
def request_url(self) -> str:
return self._resolved_url or self.base_url
async def fetch(self) -> List[Dict[str, Any]]:
from httpx import AsyncClient
async with AsyncClient(timeout=60.0) as client:
response = await client.get(self.request_url, headers=self.headers)
response.raise_for_status()
return self.parse_response(response.json())
def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Parse Hugging Face Spaces API response"""
data = []

35
backend/app/services/collectors/peeringdb.py
View File

@@ -16,6 +16,7 @@ from typing import Dict, Any, List
from datetime import UTC, datetime
import httpx
from urllib.parse import urlencode
from app.services.collectors.base import HTTPCollector
@@ -38,9 +39,13 @@ class PeeringDBIXPCollector(HTTPCollector):
"User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
"Accept": "application/json",
}
# API key is added to URL as query parameter
if PEERINGDB_API_KEY:
self.base_url = f"{self.base_url}?key={PEERINGDB_API_KEY}"
@property
def request_url(self) -> str:
base = self._resolved_url or self.base_url
if not PEERINGDB_API_KEY:
return base
separator = "&" if "?" in base else "?"
return f"{base}{separator}{urlencode({'key': PEERINGDB_API_KEY})}"
async def fetch_with_retry(
self, max_retries: int = 3, base_delay: float = 2.0
@@ -51,7 +56,7 @@ class PeeringDBIXPCollector(HTTPCollector):
for attempt in range(max_retries):
try:
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(self.base_url, headers=self.headers)
response = await client.get(self.request_url, headers=self.headers)
if response.status_code == 429:
# Rate limited - wait and retry with exponential backoff
@@ -141,8 +146,13 @@ class PeeringDBNetworkCollector(HTTPCollector):
"User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
"Accept": "application/json",
}
if PEERINGDB_API_KEY:
self.base_url = f"{self.base_url}?key={PEERINGDB_API_KEY}"
@property
def request_url(self) -> str:
base = self._resolved_url or self.base_url
if not PEERINGDB_API_KEY:
return base
separator = "&" if "?" in base else "?"
return f"{base}{separator}{urlencode({'key': PEERINGDB_API_KEY})}"
async def fetch_with_retry(
self, max_retries: int = 3, base_delay: float = 2.0
@@ -153,7 +163,7 @@ class PeeringDBNetworkCollector(HTTPCollector):
for attempt in range(max_retries):
try:
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(self.base_url, headers=self.headers)
response = await client.get(self.request_url, headers=self.headers)
if response.status_code == 429:
delay = base_delay * (2**attempt)
@@ -244,8 +254,13 @@ class PeeringDBFacilityCollector(HTTPCollector):
"User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
"Accept": "application/json",
}
if PEERINGDB_API_KEY:
self.base_url = f"{self.base_url}?key={PEERINGDB_API_KEY}"
@property
def request_url(self) -> str:
base = self._resolved_url or self.base_url
if not PEERINGDB_API_KEY:
return base
separator = "&" if "?" in base else "?"
return f"{base}{separator}{urlencode({'key': PEERINGDB_API_KEY})}"
async def fetch_with_retry(
self, max_retries: int = 3, base_delay: float = 2.0
@@ -256,7 +271,7 @@ class PeeringDBFacilityCollector(HTTPCollector):
for attempt in range(max_retries):
try:
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(self.base_url, headers=self.headers)
response = await client.get(self.request_url, headers=self.headers)
if response.status_code == 429:
delay = base_delay * (2**attempt)

2
backend/app/services/collectors/ris_live.py
View File

@@ -33,7 +33,7 @@ class RISLiveCollector(BaseCollector):
def _fetch_via_stream(self) -> list[dict[str, Any]]:
events: list[dict[str, Any]] = []
stream_url = "https://ris-live.ripe.net/v1/stream/?format=json&client=planet-ris-live"
stream_url = self._resolved_url or ""
subscribe = json.dumps(
{
"host": "rrc00",

51
backend/app/services/collectors/spacetrack.py
View File

@@ -7,6 +7,7 @@ API documentation: https://www.space-track.org/documentation
import json
from typing import Dict, Any, List
import httpx
from urllib.parse import urlparse
from app.services.collectors.base import BaseCollector
from app.core.data_sources import get_data_sources_config
@@ -21,12 +22,30 @@ class SpaceTrackTLECollector(BaseCollector):
data_type = "satellite_tle"
@property
def base_url(self) -> str:
def query_url(self) -> str:
config = get_data_sources_config()
if self._resolved_url:
return self._resolved_url
return config.get_yaml_url("spacetrack_tle")
@property
def site_root(self) -> str:
config = get_data_sources_config()
configured_root = config.get_yaml_value("spacetrack.base_url")
if isinstance(configured_root, str) and configured_root:
return configured_root.rstrip("/")
parsed = urlparse(self.query_url)
return f"{parsed.scheme}://{parsed.netloc}".rstrip("/")
@property
def login_url(self) -> str:
return f"{self.site_root}/ajaxauth/login"
@property
def probe_url(self) -> str:
return f"{self.site_root}/basicspacedata/query/class/gp/NORAD_CAT_ID/25544/format/json"
async def fetch(self) -> List[Dict[str, Any]]:
from app.core.config import settings
@@ -47,13 +66,13 @@ class SpaceTrackTLECollector(BaseCollector):
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Accept": "application/json, text/html, */*",
"Accept-Language": "en-US,en;q=0.9",
"Referer": "https://www.space-track.org/",
"Referer": f"{self.site_root}/",
},
) as client:
await client.get("https://www.space-track.org/")
await client.get(f"{self.site_root}/")
login_response = await client.post(
"https://www.space-track.org/ajaxauth/login",
self.login_url,
data={
"identity": username,
"password": password,
@@ -69,7 +88,7 @@ class SpaceTrackTLECollector(BaseCollector):
timeout=120.0,
follow_redirects=True,
) as alt_client:
await alt_client.get("https://www.space-track.org/")
await alt_client.get(f"{self.site_root}/")
form_data = {
"username": username,
@@ -77,7 +96,7 @@ class SpaceTrackTLECollector(BaseCollector):
"query": "class/gp/NORAD_CAT_ID/25544/format/json",
}
alt_login = await alt_client.post(
"https://www.space-track.org/ajaxauth/login",
self.login_url,
data={
"identity": username,
"password": password,
@@ -86,9 +105,7 @@ class SpaceTrackTLECollector(BaseCollector):
print(f"SPACETRACK: Alt login status: {alt_login.status_code}")
if alt_login.status_code == 200:
tle_response = await alt_client.get(
"https://www.space-track.org/basicspacedata/query/class/gp/NORAD_CAT_ID/25544/format/json"
)
tle_response = await alt_client.get(self.probe_url)
if tle_response.status_code == 200:
data = tle_response.json()
print(f"SPACETRACK: Received {len(data)} records via alt method")
@@ -98,9 +115,7 @@ class SpaceTrackTLECollector(BaseCollector):
print(f"SPACETRACK: Login failed, using sample data")
return self._get_sample_data()
tle_response = await client.get(
"https://www.space-track.org/basicspacedata/query/class/gp/NORAD_CAT_ID/25544/format/json"
)
tle_response = await client.get(self.probe_url)
print(f"SPACETRACK: TLE query status: {tle_response.status_code}")
if tle_response.status_code != 200:
@@ -127,11 +142,11 @@ class SpaceTrackTLECollector(BaseCollector):
},
) as client:
# First, visit the main page to get any cookies
await client.get("https://www.space-track.org/")
await client.get(f"{self.site_root}/")
# Login to get session cookie
login_response = await client.post(
"https://www.space-track.org/ajaxauth/login",
self.login_url,
data={
"identity": username,
"password": password,
@@ -146,13 +161,7 @@ class SpaceTrackTLECollector(BaseCollector):
return self._get_sample_data()
# Query for TLE data (get first 1000 satellites)
tle_response = await client.get(
"https://www.space-track.org/basicspacedata/query"
"/class/gp"
"/orderby/EPOCH%20desc"
"/limit/1000"
"/format/json"
)
tle_response = await client.get(self.query_url)
print(f"SPACETRACK: TLE query status: {tle_response.status_code}")
if tle_response.status_code != 200:

15
backend/app/services/collectors/telegeography.py
View File

@@ -11,6 +11,7 @@ from datetime import UTC, datetime
from bs4 import BeautifulSoup
import httpx
from app.core.data_sources import get_data_sources_config
from app.services.collectors.base import BaseCollector
@@ -24,15 +25,17 @@ class TeleGeographyCableCollector(BaseCollector):
async def fetch(self) -> List[Dict[str, Any]]:
"""Fetch submarine cable data from Wayback Machine"""
config = get_data_sources_config()
# Try multiple data sources
sources = [
# Wayback Machine archive of TeleGeography
"https://web.archive.org/web/2024/https://www.submarinecablemap.com/api/v3/cable",
# Alternative: Try scraping the page
"https://www.submarinecablemap.com",
self._resolved_url or "",
str(config.get_yaml_value("telegeography.archived_cable_url") or ""),
str(config.get_yaml_value("telegeography.live_map_url") or ""),
]
for url in sources:
if not url:
continue
try:
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
response = await client.get(url)
@@ -161,7 +164,7 @@ class TeleGeographyLandingPointCollector(BaseCollector):
async def fetch(self) -> List[Dict[str, Any]]:
"""Fetch landing point data from GitHub mirror"""
url = "https://raw.githubusercontent.com/lintaojlu/submarine_cable_information/main/landing_point.json"
url = self._resolved_url or ""
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(url)
@@ -225,7 +228,7 @@ class TeleGeographyCableSystemCollector(BaseCollector):
async def fetch(self) -> List[Dict[str, Any]]:
"""Fetch cable system data"""
url = "https://raw.githubusercontent.com/lintaojlu/submarine_cable_information/main/cable.json"
url = self._resolved_url or ""
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.get(url)

7
backend/app/services/collectors/top500.py
View File

@@ -10,6 +10,7 @@ from typing import Dict, Any, List
from bs4 import BeautifulSoup
import httpx
from app.core.data_sources import get_data_sources_config
from app.services.collectors.base import BaseCollector
@@ -22,7 +23,7 @@ class TOP500Collector(BaseCollector):
async def fetch(self) -> List[Dict[str, Any]]:
"""Fetch TOP500 list data and enrich each row with detail-page metadata."""
url = "https://top500.org/lists/top500/list/2025/11/"
url = self._resolved_url or ""
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
response = await client.get(url)
@@ -48,11 +49,13 @@ class TOP500Collector(BaseCollector):
return await asyncio.gather(*(enrich(entry) for entry in entries))
def _extract_system_fields(self, system_cell) -> Dict[str, str]:
config = get_data_sources_config()
top500_base_url = config.get_yaml_value("top500.base_url") or "https://top500.org"
link = system_cell.find("a")
system_name = link.get_text(" ", strip=True) if link else system_cell.get_text(" ", strip=True)
detail_url = ""
if link and link.get("href"):
detail_url = f"https://top500.org{link.get('href')}"
detail_url = f"{str(top500_base_url).rstrip('/')}{link.get('href')}"
manufacturer = ""
if link and link.next_sibling: