465 lines
20 KiB
Python
465 lines
20 KiB
Python
"""CelesTrak TLE Collector.
|
|
|
|
Collects the full active satellite GP element set from CelesTrak.
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
from pathlib import Path
|
|
from time import perf_counter
|
|
from typing import Any, Dict, List
|
|
from urllib.parse import urlencode, urlparse
|
|
|
|
import httpx
|
|
|
|
from app.core.logging import get_logger
|
|
from app.core.satellite_tle import build_tle_lines_from_elements
|
|
from app.services.business_logs import emit_business_log, exception_context
|
|
from app.services.collectors.base import BaseCollector
|
|
from app.services.collectors.downloads import DownloadHTTPStatusError, ResumableFileDownloader
|
|
|
|
|
|
logger = get_logger(__name__, service="collector")
|
|
ACTIVE_GROUP = "active"
|
|
FALLBACK_GROUPS = (
|
|
"starlink",
|
|
"gps-ops",
|
|
"galileo",
|
|
"glonass",
|
|
"beidou",
|
|
"leo",
|
|
"geo",
|
|
"iridium-next",
|
|
)
|
|
FETCH_RETRY_ATTEMPTS = 3
|
|
FETCH_RETRY_BASE_DELAY_SECONDS = 0.8
|
|
CELESTRAK_NOT_UPDATED_MARKER = "GP data has not updated since your last successful"
|
|
|
|
|
|
class CelesTrakTLECollector(BaseCollector):
|
|
name = "celestrak_tle"
|
|
priority = "P2"
|
|
module = "L3"
|
|
frequency_hours = 24
|
|
data_type = "satellite_tle"
|
|
_downloader = ResumableFileDownloader(
|
|
cache_namespace="celestrak",
|
|
default_accept="application/json",
|
|
)
|
|
|
|
@property
|
|
def base_url(self) -> str:
|
|
return self._resolved_url or ""
|
|
|
|
def _active_url(self) -> str:
|
|
return self._group_url(ACTIVE_GROUP)
|
|
|
|
def _group_url(self, group: str) -> str:
|
|
if not self.base_url:
|
|
raise RuntimeError("CelesTrak base URL is not configured")
|
|
return f"{self.base_url}?{urlencode({'GROUP': group, 'FORMAT': 'json'})}"
|
|
|
|
async def fetch(self) -> List[Dict[str, Any]]:
|
|
url = self._active_url()
|
|
last_error: Exception | None = None
|
|
|
|
async with httpx.AsyncClient(timeout=180.0, follow_redirects=True) as client:
|
|
for attempt in range(1, FETCH_RETRY_ATTEMPTS + 1):
|
|
started_at = perf_counter()
|
|
try:
|
|
await emit_business_log(
|
|
logger,
|
|
event="collector.celestrak.download.start",
|
|
message="CelesTrak active satellite download started",
|
|
category="collector",
|
|
service="collector",
|
|
module=__name__,
|
|
context={
|
|
"collector_name": self.name,
|
|
"datasource_id": getattr(self, "_datasource_id", None),
|
|
"group": ACTIVE_GROUP,
|
|
"attempt": attempt,
|
|
"url_host": urlparse(url).netloc,
|
|
},
|
|
)
|
|
body_path = await self._downloader.download_file(
|
|
client,
|
|
url,
|
|
extension=".json",
|
|
accept="application/json",
|
|
progress_callback=self._report_download_progress,
|
|
validate_existing=self._validate_json_file,
|
|
)
|
|
data = await self._load_downloaded_payload(body_path, url)
|
|
await emit_business_log(
|
|
logger,
|
|
event="collector.celestrak.download.success",
|
|
message="CelesTrak active satellite download completed",
|
|
category="collector",
|
|
service="collector",
|
|
module=__name__,
|
|
context={
|
|
"collector_name": self.name,
|
|
"datasource_id": getattr(self, "_datasource_id", None),
|
|
"group": ACTIVE_GROUP,
|
|
"attempt": attempt,
|
|
"record_count": len(data),
|
|
"duration_ms": self._duration_ms(started_at),
|
|
},
|
|
)
|
|
return data
|
|
except DownloadHTTPStatusError as exc:
|
|
if self._is_not_updated_response(exc):
|
|
cached_path = self._downloader.get_cached_file(
|
|
url,
|
|
".json",
|
|
validate_existing=self._validate_json_file,
|
|
)
|
|
if cached_path is not None:
|
|
data = await self._load_downloaded_payload(cached_path, url)
|
|
await emit_business_log(
|
|
logger,
|
|
event="collector.celestrak.download.cached_not_updated",
|
|
message="CelesTrak active satellite data has not changed; using cached download",
|
|
category="collector",
|
|
level="warning",
|
|
service="collector",
|
|
module=__name__,
|
|
context={
|
|
"collector_name": self.name,
|
|
"datasource_id": getattr(self, "_datasource_id", None),
|
|
"group": ACTIVE_GROUP,
|
|
"attempt": attempt,
|
|
"record_count": len(data),
|
|
"duration_ms": self._duration_ms(started_at),
|
|
},
|
|
)
|
|
return data
|
|
await emit_business_log(
|
|
logger,
|
|
event="collector.celestrak.download.not_updated_no_cache",
|
|
message="CelesTrak active satellite data has not changed; trying fallback groups",
|
|
category="collector",
|
|
level="warning",
|
|
service="collector",
|
|
module=__name__,
|
|
context=exception_context(
|
|
exc,
|
|
{
|
|
"collector_name": self.name,
|
|
"datasource_id": getattr(self, "_datasource_id", None),
|
|
"group": ACTIVE_GROUP,
|
|
"attempt": attempt,
|
|
"duration_ms": self._duration_ms(started_at),
|
|
},
|
|
),
|
|
)
|
|
return await self._fetch_fallback_groups(client, active_error=exc)
|
|
raise
|
|
except Exception as exc:
|
|
last_error = exc
|
|
is_final_attempt = attempt >= FETCH_RETRY_ATTEMPTS
|
|
await emit_business_log(
|
|
logger,
|
|
event=(
|
|
"collector.celestrak.download.failed"
|
|
if is_final_attempt
|
|
else "collector.celestrak.download.retry"
|
|
),
|
|
message=(
|
|
"CelesTrak active satellite download failed"
|
|
if is_final_attempt
|
|
else "CelesTrak active satellite download will retry"
|
|
),
|
|
category="collector",
|
|
level="error" if is_final_attempt else "warning",
|
|
service="collector",
|
|
module=__name__,
|
|
context=exception_context(
|
|
exc,
|
|
{
|
|
"collector_name": self.name,
|
|
"datasource_id": getattr(self, "_datasource_id", None),
|
|
"group": ACTIVE_GROUP,
|
|
"attempt": attempt,
|
|
"duration_ms": self._duration_ms(started_at),
|
|
},
|
|
),
|
|
)
|
|
if not is_final_attempt:
|
|
await asyncio.sleep(FETCH_RETRY_BASE_DELAY_SECONDS * attempt)
|
|
|
|
raise RuntimeError(f"CelesTrak active satellite download failed after retries: {last_error}")
|
|
|
|
async def _fetch_fallback_groups(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
*,
|
|
active_error: DownloadHTTPStatusError,
|
|
) -> List[Dict[str, Any]]:
|
|
started_at = perf_counter()
|
|
records_by_norad: dict[str, Dict[str, Any]] = {}
|
|
group_counts: dict[str, int] = {}
|
|
|
|
await emit_business_log(
|
|
logger,
|
|
event="collector.celestrak.fallback_groups.start",
|
|
message="CelesTrak fallback group download started",
|
|
category="collector",
|
|
level="warning",
|
|
service="collector",
|
|
module=__name__,
|
|
context={
|
|
"collector_name": self.name,
|
|
"datasource_id": getattr(self, "_datasource_id", None),
|
|
"groups": list(FALLBACK_GROUPS),
|
|
"reason": "active_not_updated_without_cache",
|
|
},
|
|
)
|
|
|
|
try:
|
|
for group in FALLBACK_GROUPS:
|
|
group_url = self._group_url(group)
|
|
try:
|
|
body_path = await self._downloader.download_file(
|
|
client,
|
|
group_url,
|
|
extension=".json",
|
|
accept="application/json",
|
|
validate_existing=self._validate_json_file,
|
|
)
|
|
except DownloadHTTPStatusError as exc:
|
|
if not self._is_not_updated_response(exc):
|
|
raise RuntimeError(f"CelesTrak fallback group '{group}' download failed: {exc}") from exc
|
|
cached_path = self._downloader.get_cached_file(
|
|
group_url,
|
|
".json",
|
|
validate_existing=self._validate_json_file,
|
|
)
|
|
if cached_path is None:
|
|
raise RuntimeError(
|
|
f"CelesTrak fallback group '{group}' has not updated and no local cached copy is available"
|
|
) from exc
|
|
body_path = cached_path
|
|
|
|
group_records = await self._load_downloaded_payload(
|
|
body_path,
|
|
group_url,
|
|
query_group=group,
|
|
constellation_group=group,
|
|
)
|
|
group_counts[group] = len(group_records)
|
|
for item in group_records:
|
|
norad_cat_id = item.get("NORAD_CAT_ID")
|
|
if norad_cat_id is None:
|
|
continue
|
|
records_by_norad.setdefault(str(norad_cat_id), item)
|
|
except Exception as exc:
|
|
await emit_business_log(
|
|
logger,
|
|
event="collector.celestrak.fallback_groups.failed",
|
|
message="CelesTrak fallback group download failed",
|
|
category="collector",
|
|
level="error",
|
|
service="collector",
|
|
module=__name__,
|
|
context=exception_context(
|
|
exc,
|
|
{
|
|
"collector_name": self.name,
|
|
"datasource_id": getattr(self, "_datasource_id", None),
|
|
"groups": list(FALLBACK_GROUPS),
|
|
"completed_groups": list(group_counts),
|
|
"duration_ms": self._duration_ms(started_at),
|
|
},
|
|
),
|
|
)
|
|
raise RuntimeError(
|
|
"CelesTrak active data has not updated since this network's last successful download, "
|
|
"no active cache is available, and fallback group mode failed. Wait until CelesTrak "
|
|
"publishes the next GP update, restore the Planet download cache, or use Space-Track."
|
|
) from active_error
|
|
|
|
records = list(records_by_norad.values())
|
|
if not records:
|
|
raise RuntimeError("CelesTrak fallback group mode produced no satellite records")
|
|
|
|
await emit_business_log(
|
|
logger,
|
|
event="collector.celestrak.fallback_groups.success",
|
|
message="CelesTrak fallback group download completed",
|
|
category="collector",
|
|
level="warning",
|
|
service="collector",
|
|
module=__name__,
|
|
context={
|
|
"collector_name": self.name,
|
|
"datasource_id": getattr(self, "_datasource_id", None),
|
|
"groups": list(FALLBACK_GROUPS),
|
|
"group_counts": group_counts,
|
|
"record_count": len(records),
|
|
"duration_ms": self._duration_ms(started_at),
|
|
},
|
|
)
|
|
return records
|
|
|
|
async def _load_downloaded_payload(
|
|
self,
|
|
body_path: Path,
|
|
url: str,
|
|
*,
|
|
query_group: str = ACTIVE_GROUP,
|
|
constellation_group: str | None = None,
|
|
) -> List[Dict[str, Any]]:
|
|
try:
|
|
data = self._load_active_payload(body_path)
|
|
except RuntimeError as exc:
|
|
await self._log_parse_failure(exc)
|
|
raise
|
|
for item in data:
|
|
item["_celestrak_query_group"] = query_group
|
|
item["_celestrak_source_url"] = url
|
|
if constellation_group:
|
|
item["_celestrak_group"] = constellation_group
|
|
return data
|
|
|
|
@staticmethod
|
|
def _is_not_updated_response(exc: DownloadHTTPStatusError) -> bool:
|
|
return exc.status_code == 403 and CELESTRAK_NOT_UPDATED_MARKER in exc.body
|
|
|
|
@staticmethod
|
|
def _duration_ms(started_at: float) -> int:
|
|
return int((perf_counter() - started_at) * 1000)
|
|
|
|
async def _report_download_progress(self, downloaded: int, total: int | None) -> None:
|
|
if total and total > 0:
|
|
await self.update_phase_progress(
|
|
current=min(downloaded, total),
|
|
total=total,
|
|
unit="bytes",
|
|
message=f"正在下载 CelesTrak active 卫星数据 {downloaded}/{total} bytes",
|
|
commit=True,
|
|
)
|
|
|
|
@staticmethod
|
|
def _validate_json_file(path: Path) -> bool:
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError, UnicodeDecodeError):
|
|
return False
|
|
return isinstance(data, list)
|
|
|
|
async def _log_parse_failure(self, exc: Exception) -> None:
|
|
await emit_business_log(
|
|
logger,
|
|
event="collector.celestrak.parse.failed",
|
|
message="CelesTrak active satellite JSON parsing failed",
|
|
category="collector",
|
|
level="error",
|
|
service="collector",
|
|
module=__name__,
|
|
context=exception_context(
|
|
exc,
|
|
{
|
|
"collector_name": self.name,
|
|
"datasource_id": getattr(self, "_datasource_id", None),
|
|
"group": ACTIVE_GROUP,
|
|
},
|
|
),
|
|
)
|
|
|
|
def _load_active_payload(self, path: Path) -> List[Dict[str, Any]]:
|
|
try:
|
|
raw = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError, UnicodeDecodeError) as exc:
|
|
raise RuntimeError(f"CelesTrak active payload is not valid JSON: {exc}") from exc
|
|
if not isinstance(raw, list):
|
|
raise RuntimeError("CelesTrak active payload is not a JSON array")
|
|
|
|
records: List[Dict[str, Any]] = []
|
|
invalid_count = 0
|
|
for item in raw:
|
|
if isinstance(item, dict) and item.get("NORAD_CAT_ID") is not None:
|
|
records.append(item)
|
|
else:
|
|
invalid_count += 1
|
|
if invalid_count:
|
|
raise RuntimeError(f"CelesTrak active payload contains {invalid_count} invalid record(s)")
|
|
if not records:
|
|
raise RuntimeError("CelesTrak active payload contains no satellite records")
|
|
return records
|
|
|
|
def transform(self, raw_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
transformed = []
|
|
for item in raw_data:
|
|
norad_cat_id = item.get("NORAD_CAT_ID")
|
|
tle_line1, tle_line2 = build_tle_lines_from_elements(
|
|
norad_cat_id=norad_cat_id,
|
|
epoch=item.get("EPOCH"),
|
|
inclination=item.get("INCLINATION"),
|
|
raan=item.get("RA_OF_ASC_NODE"),
|
|
eccentricity=item.get("ECCENTRICITY"),
|
|
arg_of_perigee=item.get("ARG_OF_PERICENTER"),
|
|
mean_anomaly=item.get("MEAN_ANOMALY"),
|
|
mean_motion=item.get("MEAN_MOTION"),
|
|
)
|
|
constellation_group = self._infer_constellation_group(item)
|
|
|
|
transformed.append(
|
|
{
|
|
"source_id": str(norad_cat_id),
|
|
"name": item.get("OBJECT_NAME", "Unknown"),
|
|
"reference_date": item.get("EPOCH", ""),
|
|
"metadata": {
|
|
"constellation_group": constellation_group,
|
|
"celestrak_query_group": item.get("_celestrak_query_group") or ACTIVE_GROUP,
|
|
"celestrak_source_url": item.get("_celestrak_source_url"),
|
|
"norad_cat_id": norad_cat_id,
|
|
"international_designator": item.get("OBJECT_ID"),
|
|
"epoch": item.get("EPOCH"),
|
|
"mean_motion": item.get("MEAN_MOTION"),
|
|
"eccentricity": item.get("ECCENTRICITY"),
|
|
"inclination": item.get("INCLINATION"),
|
|
"raan": item.get("RA_OF_ASC_NODE"),
|
|
"arg_of_perigee": item.get("ARG_OF_PERICENTER"),
|
|
"mean_anomaly": item.get("MEAN_ANOMALY"),
|
|
"classification_type": item.get("CLASSIFICATION_TYPE"),
|
|
"bstar": item.get("BSTAR"),
|
|
"mean_motion_dot": item.get("MEAN_MOTION_DOT"),
|
|
"mean_motion_ddot": item.get("MEAN_MOTION_DDOT"),
|
|
"ephemeris_type": item.get("EPHEMERIS_TYPE"),
|
|
# Prefer the original TLE lines when the source provides them.
|
|
# If they are missing, store a normalized TLE pair built once on the backend.
|
|
"tle_line1": item.get("TLE_LINE1") or tle_line1,
|
|
"tle_line2": item.get("TLE_LINE2") or tle_line2,
|
|
},
|
|
}
|
|
)
|
|
return transformed
|
|
|
|
@staticmethod
|
|
def _infer_constellation_group(item: Dict[str, Any]) -> str | None:
|
|
explicit_group = str(item.get("_celestrak_group") or "").strip().lower()
|
|
if explicit_group and explicit_group != ACTIVE_GROUP:
|
|
return explicit_group
|
|
|
|
name = str(item.get("OBJECT_NAME") or "").strip().upper()
|
|
if name.startswith("STARLINK"):
|
|
return "starlink"
|
|
if name.startswith("IRIDIUM"):
|
|
return "iridium-next"
|
|
return None
|
|
|
|
def _get_sample_data(self) -> List[Dict[str, Any]]:
|
|
return [
|
|
{
|
|
"name": "STARLINK-1000",
|
|
"norad_cat_id": 44720,
|
|
"international_designator": "2019-029AZ",
|
|
"epoch": "2026-03-13T00:00:00Z",
|
|
"mean_motion": 15.79234567,
|
|
"eccentricity": 0.0001234,
|
|
"inclination": 53.0,
|
|
},
|
|
]
|