linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

release: bump version to 0.72.0
Some checks failed
ci / backend (push) Has been cancelled
Details
ci / frontend (push) Has been cancelled
Details
ci / delivery (push) Has been cancelled
Details
release / images (push) Has been cancelled
Details

Browse Source
This commit is contained in:
linkong
2026-06-29 14:05:06 +08:00
parent 3265d22af5
commit 19d5ac0fee
60 changed files with 3702 additions and 678 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
backend/app/services/collectors/peeringdb.py
View File

@@ -11,17 +11,20 @@ To get higher limits, set PEERINGDB_API_KEY environment variable.
"""
import asyncio
import os
from typing import Dict, Any, List
from datetime import UTC, datetime
import os
from typing import Any, Dict, List
from urllib.parse import urlencode
import httpx
from urllib.parse import urlencode
from app.core.logging import get_logger
from app.services.collectors.base import HTTPCollector
# PeeringDB API key - read from environment variable
PEERINGDB_API_KEY = os.environ.get("PEERINGDB_API_KEY", "")
logger = get_logger(__name__, service="collector")
class PeeringDBIXPCollector(HTTPCollector):
@@ -39,6 +42,7 @@ class PeeringDBIXPCollector(HTTPCollector):
"User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
"Accept": "application/json",
}
@property
def request_url(self) -> str:
base = self._resolved_url or self.base_url
@@ -61,7 +65,11 @@ class PeeringDBIXPCollector(HTTPCollector):
if response.status_code == 429:
# Rate limited - wait and retry with exponential backoff
delay = base_delay * (2**attempt)
print(f"PeeringDB rate limited, waiting {delay}s before retry...")
logger.warning_event(
"PeeringDB rate limited; retrying after delay",
event="collector.peeringdb.rate_limited",
context={"delay_seconds": delay, "attempt": attempt + 1},
)
await asyncio.sleep(delay)
last_error = "Rate limited"
continue
@@ -72,13 +80,21 @@ class PeeringDBIXPCollector(HTTPCollector):
except httpx.HTTPStatusError as e:
if e.response.status_code == 429:
delay = base_delay * (2**attempt)
print(f"PeeringDB rate limited, waiting {delay}s before retry...")
logger.warning_event(
"PeeringDB rate limited; retrying after delay",
event="collector.peeringdb.rate_limited",
context={"delay_seconds": delay, "attempt": attempt + 1},
)
await asyncio.sleep(delay)
last_error = "Rate limited"
continue
raise
print(f"Warning: PeeringDB collection failed after {max_retries} retries: {last_error}")
logger.warning_event(
"PeeringDB collection failed after retries",
event="collector.peeringdb.retries_exhausted",
context={"max_retries": max_retries, "last_error": last_error},
)
return {}
async def fetch(self) -> List[Dict[str, Any]]:
@@ -146,6 +162,7 @@ class PeeringDBNetworkCollector(HTTPCollector):
"User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
"Accept": "application/json",
}
@property
def request_url(self) -> str:
base = self._resolved_url or self.base_url
@@ -167,7 +184,11 @@ class PeeringDBNetworkCollector(HTTPCollector):
if response.status_code == 429:
delay = base_delay * (2**attempt)
print(f"PeeringDB rate limited, waiting {delay}s before retry...")
logger.warning_event(
"PeeringDB rate limited; retrying after delay",
event="collector.peeringdb.rate_limited",
context={"delay_seconds": delay, "attempt": attempt + 1},
)
await asyncio.sleep(delay)
last_error = "Rate limited"
continue
@@ -178,13 +199,21 @@ class PeeringDBNetworkCollector(HTTPCollector):
except httpx.HTTPStatusError as e:
if e.response.status_code == 429:
delay = base_delay * (2**attempt)
print(f"PeeringDB rate limited, waiting {delay}s before retry...")
logger.warning_event(
"PeeringDB rate limited; retrying after delay",
event="collector.peeringdb.rate_limited",
context={"delay_seconds": delay, "attempt": attempt + 1},
)
await asyncio.sleep(delay)
last_error = "Rate limited"
continue
raise
print(f"Warning: PeeringDB collection failed after {max_retries} retries: {last_error}")
logger.warning_event(
"PeeringDB collection failed after retries",
event="collector.peeringdb.retries_exhausted",
context={"max_retries": max_retries, "last_error": last_error},
)
return {}
async def fetch(self) -> List[Dict[str, Any]]:
@@ -254,6 +283,7 @@ class PeeringDBFacilityCollector(HTTPCollector):
"User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
"Accept": "application/json",
}
@property
def request_url(self) -> str:
base = self._resolved_url or self.base_url
@@ -275,7 +305,11 @@ class PeeringDBFacilityCollector(HTTPCollector):
if response.status_code == 429:
delay = base_delay * (2**attempt)
print(f"PeeringDB rate limited, waiting {delay}s before retry...")
logger.warning_event(
"PeeringDB rate limited; retrying after delay",
event="collector.peeringdb.rate_limited",
context={"delay_seconds": delay, "attempt": attempt + 1},
)
await asyncio.sleep(delay)
last_error = "Rate limited"
continue
@@ -286,13 +320,21 @@ class PeeringDBFacilityCollector(HTTPCollector):
except httpx.HTTPStatusError as e:
if e.response.status_code == 429:
delay = base_delay * (2**attempt)
print(f"PeeringDB rate limited, waiting {delay}s before retry...")
logger.warning_event(
"PeeringDB rate limited; retrying after delay",
event="collector.peeringdb.rate_limited",
context={"delay_seconds": delay, "attempt": attempt + 1},
)
await asyncio.sleep(delay)
last_error = "Rate limited"
continue
raise
print(f"Warning: PeeringDB collection failed after {max_retries} retries: {last_error}")
logger.warning_event(
"PeeringDB collection failed after retries",
event="collector.peeringdb.retries_exhausted",
context={"max_retries": max_retries, "last_error": last_error},
)
return {}
async def fetch(self) -> List[Dict[str, Any]]:

127
backend/app/services/collectors/spacetrack.py
View File

@@ -1,17 +1,21 @@
"""Space-Track TLE Collector
"""Space-Track TLE Collector.
Collects satellite TLE (Two-Line Element) data from Space-Track.org.
API documentation: https://www.space-track.org/documentation
"""
import json
from typing import Dict, Any, List
import httpx
from typing import Any, Dict, List
from urllib.parse import urlparse
from app.services.collectors.base import BaseCollector
import httpx
from app.core.data_sources import get_data_sources_config
from app.core.logging import get_logger
from app.core.satellite_tle import build_tle_lines_from_elements
from app.services.collectors.base import BaseCollector
logger = get_logger(__name__, service="collector")
class SpaceTrackTLECollector(BaseCollector):
@@ -53,10 +57,16 @@ class SpaceTrackTLECollector(BaseCollector):
password = settings.SPACETRACK_PASSWORD
if not username or not password:
print("SPACETRACK: No credentials configured, using sample data")
logger.warning_event(
"Space-Track credentials are not configured; using sample data",
event="collector.spacetrack.credentials_missing",
)
return self._get_sample_data()
print(f"SPACETRACK: Attempting to fetch TLE data with username: {username}")
logger.info_event(
"Space-Track TLE fetch started",
event="collector.spacetrack.fetch.start",
)
try:
async with httpx.AsyncClient(
@@ -78,11 +88,17 @@ class SpaceTrackTLECollector(BaseCollector):
"password": password,
},
)
print(f"SPACETRACK: Login response status: {login_response.status_code}")
print(f"SPACETRACK: Login response URL: {login_response.url}")
logger.info_event(
"Space-Track login response received",
event="collector.spacetrack.login.response",
context={"status_code": login_response.status_code},
)
if login_response.status_code == 403:
print("SPACETRACK: Trying alternate login method...")
logger.warning_event(
"Space-Track login returned forbidden; trying alternate method",
event="collector.spacetrack.login.forbidden",
)
async with httpx.AsyncClient(
timeout=120.0,
@@ -90,11 +106,6 @@ class SpaceTrackTLECollector(BaseCollector):
) as alt_client:
await alt_client.get(f"{self.site_root}/")
form_data = {
"username": username,
"password": password,
"query": "class/gp/NORAD_CAT_ID/25544/format/json",
}
alt_login = await alt_client.post(
self.login_url,
data={
@@ -102,77 +113,59 @@ class SpaceTrackTLECollector(BaseCollector):
"password": password,
},
)
print(f"SPACETRACK: Alt login status: {alt_login.status_code}")
logger.info_event(
"Space-Track alternate login response received",
event="collector.spacetrack.alt_login.response",
context={"status_code": alt_login.status_code},
)
if alt_login.status_code == 200:
tle_response = await alt_client.get(self.probe_url)
if tle_response.status_code == 200:
data = tle_response.json()
print(f"SPACETRACK: Received {len(data)} records via alt method")
logger.info_event(
"Space-Track alternate query completed",
event="collector.spacetrack.alt_query.completed",
context={"record_count": len(data)},
)
return data
if login_response.status_code != 200:
print(f"SPACETRACK: Login failed, using sample data")
logger.warning_event(
"Space-Track login failed; using sample data",
event="collector.spacetrack.login.failed",
context={"status_code": login_response.status_code},
)
return self._get_sample_data()
tle_response = await client.get(self.probe_url)
print(f"SPACETRACK: TLE query status: {tle_response.status_code}")
if tle_response.status_code != 200:
print(f"SPACETRACK: Query failed, using sample data")
return self._get_sample_data()
data = tle_response.json()
print(f"SPACETRACK: Received {len(data)} records")
return data
except Exception as e:
print(f"SPACETRACK: Error - {e}, using sample data")
return self._get_sample_data()
print(f"SPACETRACK: Attempting to fetch TLE data with username: {username}")
try:
async with httpx.AsyncClient(
timeout=120.0,
follow_redirects=True,
headers={
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"Accept": "application/json, text/html, */*",
"Accept-Language": "en-US,en;q=0.9",
},
) as client:
# First, visit the main page to get any cookies
await client.get(f"{self.site_root}/")
# Login to get session cookie
login_response = await client.post(
self.login_url,
data={
"identity": username,
"password": password,
},
logger.info_event(
"Space-Track TLE query response received",
event="collector.spacetrack.query.response",
context={"status_code": tle_response.status_code},
)
print(f"SPACETRACK: Login response status: {login_response.status_code}")
print(f"SPACETRACK: Login response URL: {login_response.url}")
print(f"SPACETRACK: Login response body: {login_response.text[:500]}")
if login_response.status_code != 200:
print(f"SPACETRACK: Login failed, using sample data")
return self._get_sample_data()
# Query for TLE data (get first 1000 satellites)
tle_response = await client.get(self.query_url)
print(f"SPACETRACK: TLE query status: {tle_response.status_code}")
if tle_response.status_code != 200:
print(f"SPACETRACK: Query failed, using sample data")
logger.warning_event(
"Space-Track TLE query failed; using sample data",
event="collector.spacetrack.query.failed",
context={"status_code": tle_response.status_code},
)
return self._get_sample_data()
data = tle_response.json()
print(f"SPACETRACK: Received {len(data)} records")
logger.info_event(
"Space-Track TLE fetch completed",
event="collector.spacetrack.fetch.completed",
context={"record_count": len(data)},
)
return data
except Exception as e:
print(f"SPACETRACK: Error - {e}, using sample data")
logger.warning_event(
"Space-Track TLE fetch failed; using sample data",
event="collector.spacetrack.fetch.failed",
context={"error": str(e)},
)
return self._get_sample_data()
def transform(self, raw_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:

4
backend/app/services/docs_gatekeeper.py
View File

@@ -45,7 +45,7 @@ DOCS_METADATA: tuple[DocsMetadata, ...] = (
DocsMetadata("earth-bgp-context.md", "earth-bgp-context", "docs_developer", "Earth", 14, "BGP 态势上下文", "BGP Context"),
DocsMetadata("earth-interactable-usage.md", "earth-interactable-usage", "docs_developer", "Earth", 16, "智能星球可交互图标接入", "Intelligent Planet Interactable Usage"),
DocsMetadata("earth-interactable-clustering.md", "earth-interactable-clustering", "docs_developer", "Earth", 17, "智能星球可交互图标聚类策略", "Intelligent Planet Interactable Clustering"),
DocsMetadata("earth-toolbar-overlay-coordination.md", "earth-toolbar-overlay-coordination", "docs_developer", "Earth", 17, "智能星球工具栏与浮层协同", "Intelligent Planet Toolbar and Overlay Coordination"),
DocsMetadata("earth-toolbar-overlay-coordination.md", "earth-toolbar-overlay-coordination", "docs_developer", "Earth", 18, "智能星球工具栏与浮层协同", "Intelligent Planet Toolbar and Overlay Coordination"),
DocsMetadata("earth-news-sources.md", "earth-news-sources", "docs_developer", "Earth", 19, "智能星球新闻源配置", "Intelligent Planet News Source Configuration"),
DocsMetadata("frontend-admin-frontend-context.md", "frontend-admin-frontend-context", "docs_developer", "Frontend", 20, "控制台前端结构", "Admin Frontend Context"),
DocsMetadata("frontend-layout-guidelines.md", "frontend-layout-guidelines", "docs_developer", "Frontend", 21, "前端布局指南", "Frontend Layout Guidelines"),
@@ -56,7 +56,7 @@ DOCS_METADATA: tuple[DocsMetadata, ...] = (
DocsMetadata("backend-datasources-api-performance.md", "backend-datasources-api-performance", "docs_developer", "Backend", 33, "数据源 API 性能", "Datasource API Performance"),
DocsMetadata("data-job-earth-sync-architecture.md", "data-job-earth-sync-architecture", "docs_developer", "Backend", 34, "数据作业与 Outbox 技术架构", "Data Jobs and Outbox Architecture"),
DocsMetadata("backend-enum-contracts.md", "backend-enum-contracts", "docs_developer", "Backend", 35, "后端枚举与字符串兼容契约", "Backend Enum and String Compatibility Contract"),
DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 36, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"),
DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 35, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"),
DocsMetadata("earth-news-live-streams-collector-format.md", "earth-news-live-streams-collector-format", "docs_developer", "Backend", 36, "新闻直播采集格式", "News Live Streams Collector Format"),
DocsMetadata("docs-gatekeeper-development.md", "docs-gatekeeper-development", "docs_developer", "Backend", 37, "Docs Gatekeeper 开发说明", "Docs Gatekeeper Development Guide"),
DocsMetadata("agents-aiprovider.md", "agents-aiprovider", "docs_developer", "Agents", 40, "AI Provider 指南", "AI Provider Guide"),