linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

release: bump version to 0.43.0

Browse Source
This commit is contained in:
linkong
2026-04-28 16:10:17 +08:00
parent 1cd2dab0ee
commit ac69d5d354
69 changed files with 6954 additions and 1141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/app/core/data_sources.py
View File

@@ -31,6 +31,7 @@ COLLECTOR_URL_KEYS = {
"opengeofeed_prefix_geo": "opengeofeed.public_csv_url",
"nro_delegated_prefix_geo": "nro.delegated_stats_url",
"news_live_streams": "news_live_streams.channels_url",
"barentswatch_vessels": "barentswatch_vessels.url",
}

4
backend/app/core/data_sources.yaml
View File

@@ -94,3 +94,7 @@ news_live_streams:
streams_url: "https://iptv-org.github.io/api/streams.json"
# IPTV-org 台标 JSON
logos_url: "https://iptv-org.github.io/api/logos.json"
barentswatch_vessels:
# BarentsWatch Live AIS latest combined endpoint. Requires an AIS bearer token.
url: "https://live.ais.barentswatch.no/v1/latest/combined"

83
backend/app/core/datasource_defaults.py
View File

@@ -4,163 +4,246 @@ DEFAULT_DATASOURCES = {
"top500": {
"id": 1,
"name": "TOP500 Supercomputers",
"display_name": "TOP500 超算榜单",
"module": "L1",
"priority": "P0",
"frequency_minutes": 240,
"is_free": True,
"requires_credentials": False,
},
"epoch_ai_gpu": {
"id": 2,
"name": "Epoch AI GPU Clusters",
"display_name": "Epoch AI GPU 集群",
"module": "L1",
"priority": "P0",
"frequency_minutes": 360,
"is_free": True,
"requires_credentials": False,
},
"huggingface_models": {
"id": 3,
"name": "HuggingFace Models",
"display_name": "Hugging Face 模型",
"module": "L2",
"priority": "P1",
"frequency_minutes": 720,
"is_free": True,
"requires_credentials": False,
},
"huggingface_datasets": {
"id": 4,
"name": "HuggingFace Datasets",
"display_name": "Hugging Face 数据集",
"module": "L2",
"priority": "P1",
"frequency_minutes": 720,
"is_free": True,
"requires_credentials": False,
},
"huggingface_spaces": {
"id": 5,
"name": "HuggingFace Spaces",
"display_name": "Hugging Face Spaces",
"module": "L2",
"priority": "P2",
"frequency_minutes": 1440,
"is_free": True,
"requires_credentials": False,
},
"peeringdb_ixp": {
"id": 6,
"name": "PeeringDB IXP",
"display_name": "PeeringDB 交换中心",
"module": "L2",
"priority": "P1",
"frequency_minutes": 1440,
"is_free": True,
"requires_credentials": False,
},
"peeringdb_network": {
"id": 7,
"name": "PeeringDB Networks",
"display_name": "PeeringDB 网络",
"module": "L2",
"priority": "P2",
"frequency_minutes": 2880,
"is_free": True,
"requires_credentials": False,
},
"peeringdb_facility": {
"id": 8,
"name": "PeeringDB Facilities",
"display_name": "PeeringDB 设施",
"module": "L2",
"priority": "P2",
"frequency_minutes": 2880,
"is_free": True,
"requires_credentials": False,
},
"telegeography_cables": {
"id": 9,
"name": "Submarine Cables",
"display_name": "海底光缆",
"module": "L2",
"priority": "P1",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"telegeography_landing": {
"id": 10,
"name": "Cable Landing Points",
"display_name": "光缆登陆点",
"module": "L2",
"priority": "P2",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"telegeography_systems": {
"id": 11,
"name": "Cable Systems",
"display_name": "光缆系统",
"module": "L2",
"priority": "P2",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"arcgis_cables": {
"id": 15,
"name": "ArcGIS Submarine Cables",
"display_name": "ArcGIS 海底光缆",
"module": "L2",
"priority": "P1",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"arcgis_landing_points": {
"id": 16,
"name": "ArcGIS Landing Points",
"display_name": "ArcGIS 登陆点",
"module": "L2",
"priority": "P1",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"arcgis_cable_landing_relation": {
"id": 17,
"name": "ArcGIS Cable-Landing Relations",
"display_name": "ArcGIS 光缆登陆关系",
"module": "L2",
"priority": "P1",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"fao_landing_points": {
"id": 18,
"name": "FAO Landing Points",
"display_name": "FAO 登陆点",
"module": "L2",
"priority": "P1",
"frequency_minutes": 10080,
"is_free": True,
"requires_credentials": False,
},
"spacetrack_tle": {
"id": 19,
"name": "Space-Track TLE",
"display_name": "Space-Track 轨道根数",
"module": "L3",
"priority": "P2",
"frequency_minutes": 1440,
"is_free": True,
"requires_credentials": True,
"credential_provider": "spacetrack",
"credential_status": "planned",
},
"celestrak_tle": {
"id": 20,
"name": "CelesTrak TLE",
"display_name": "CelesTrak 轨道根数",
"module": "L3",
"priority": "P2",
"frequency_minutes": 1440,
"is_free": True,
"requires_credentials": False,
},
"ris_live_bgp": {
"id": 21,
"name": "RIPE RIS Live BGP",
"display_name": "RIPE RIS 实时 BGP",
"module": "L3",
"priority": "P1",
"frequency_minutes": 15,
"is_free": True,
"requires_credentials": False,
},
"bgpstream_bgp": {
"id": 22,
"name": "CAIDA BGPStream Backfill",
"display_name": "CAIDA BGPStream 回填",
"module": "L3",
"priority": "P1",
"frequency_minutes": 360,
"is_free": True,
"requires_credentials": False,
},
"iptoasn_prefix_geo": {
"id": 23,
"name": "IPtoASN Prefix Geography",
"display_name": "IPtoASN 前缀地理",
"module": "L3",
"priority": "P1",
"frequency_minutes": 1440,
"is_free": True,
"requires_credentials": False,
},
"opengeofeed_prefix_geo": {
"id": 24,
"name": "OpenGeoFeed Prefix Geography",
"display_name": "OpenGeoFeed 前缀地理",
"module": "L3",
"priority": "P1",
"frequency_minutes": 1440,
"is_free": True,
"requires_credentials": False,
},
"nro_delegated_prefix_geo": {
"id": 25,
"name": "NRO Delegated Prefix Geography",
"display_name": "NRO 分配前缀地理",
"module": "L3",
"priority": "P1",
"frequency_minutes": 1440,
"is_free": True,
"requires_credentials": False,
},
"news_live_streams": {
"id": 26,
"name": "News Live Streams",
"display_name": "新闻直播源",
"module": "L4",
"priority": "P2",
"frequency_minutes": 720,
"is_free": True,
"requires_credentials": False,
},
"barentswatch_vessels": {
"id": 27,
"name": "BarentsWatch AIS Vessels",
"display_name": "BarentsWatch AIS 船舶",
"module": "L4",
"priority": "P1",
"frequency_minutes": 1,
"is_free": True,
"requires_credentials": True,
"credential_provider": "barentswatch",
"credential_status": "supported",
},
}

151
backend/app/core/target_schema_registry.py Normal file
View File

@@ -0,0 +1,151 @@
"""Registry of target schemas supported by mapped custom data sources."""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from typing import Any
from pydantic import BaseModel, Field, ValidationError, field_validator
class VesselAISRecord(BaseModel):
mmsi: int = Field(ge=100000000, le=999999999)
lat: float = Field(ge=-90, le=90)
lon: float = Field(ge=-180, le=180)
sog: float | None = None
cog: float | None = Field(default=None, ge=0, le=360)
heading: int | None = Field(default=None, ge=0, le=511)
name: str | None = None
vessel_type: str | int | None = None
received_at: datetime | None = None
class GeoPointRecord(BaseModel):
lat: float = Field(ge=-90, le=90)
lon: float = Field(ge=-180, le=180)
name: str | None = None
type: str | None = None
source_id: str | None = None
observed_at: datetime | None = None
metadata: dict[str, Any] = Field(default_factory=dict)
class GenericRecord(BaseModel):
data: dict[str, Any] = Field(default_factory=dict)
source_id: str | None = None
observed_at: datetime | None = None
@field_validator("data")
@classmethod
def require_payload(cls, value: dict[str, Any]) -> dict[str, Any]:
if not value:
raise ValueError("generic_records requires a non-empty data object")
return value
@dataclass(frozen=True)
class TargetField:
name: str
type: str
required: bool = False
description: str = ""
example: Any = None
def to_dict(self) -> dict[str, Any]:
return {
"name": self.name,
"type": self.type,
"required": self.required,
"description": self.description,
"example": self.example,
}
@dataclass(frozen=True)
class TargetSchema:
key: str
label: str
description: str
fields: tuple[TargetField, ...]
model: type[BaseModel]
destination: str
def to_dict(self) -> dict[str, Any]:
return {
"key": self.key,
"label": self.label,
"description": self.description,
"destination": self.destination,
"fields": [field.to_dict() for field in self.fields],
}
def validate_record(self, record: dict[str, Any]) -> tuple[dict[str, Any] | None, list[str]]:
try:
return self.model.model_validate(record).model_dump(mode="json"), []
except ValidationError as exc:
return None, [
".".join(str(part) for part in error["loc"]) + f": {error['msg']}"
for error in exc.errors()
]
TARGET_SCHEMAS: dict[str, TargetSchema] = {
"vessel_ais": TargetSchema(
key="vessel_ais",
label="船舶 AIS",
description="船只位置、航速、航向、MMSI 等 AIS 数据。",
destination="vessel_position",
model=VesselAISRecord,
fields=(
TargetField("mmsi", "integer", True, "MMSI 九位船舶标识", 257123000),
TargetField("lat", "float", True, "纬度", 59.91),
TargetField("lon", "float", True, "经度", 10.75),
TargetField("sog", "float", False, "对地航速，单位节", 12.4),
TargetField("cog", "float", False, "对地航向，0-360 度", 184.5),
TargetField("heading", "integer", False, "船首向，0-511", 186),
TargetField("name", "string", False, "船名", "OSLO EXPRESS"),
TargetField("vessel_type", "string", False, "船型", "cargo"),
TargetField("received_at", "datetime", False, "数据接收时间", "2026-04-28T00:00:00Z"),
),
),
"geo_points": TargetSchema(
key="geo_points",
label="通用地理点",
description="带经纬度的通用实体或事件点位。",
destination="generic_geo_points",
model=GeoPointRecord,
fields=(
TargetField("lat", "float", True, "纬度", 1.3),
TargetField("lon", "float", True, "经度", 103.8),
TargetField("name", "string", False, "点位名称", "Singapore"),
TargetField("type", "string", False, "点位类型", "datacenter"),
TargetField("source_id", "string", False, "来源侧 ID", "sg-1"),
TargetField("observed_at", "datetime", False, "观测时间", "2026-04-28T00:00:00Z"),
TargetField("metadata", "object", False, "扩展字段", {"provider": "example"}),
),
),
"generic_records": TargetSchema(
key="generic_records",
label="通用结构化记录",
description="未知结构数据沉淀，不直接进入 Earth 图层。",
destination="collected_data",
model=GenericRecord,
fields=(
TargetField("data", "object", True, "结构化记录主体", {"raw": "value"}),
TargetField("source_id", "string", False, "来源侧 ID", "record-1"),
TargetField("observed_at", "datetime", False, "观测时间", "2026-04-28T00:00:00Z"),
),
),
}
def list_target_schemas() -> list[dict[str, Any]]:
return [schema.to_dict() for schema in TARGET_SCHEMAS.values()]
def get_target_schema(key: str) -> TargetSchema:
try:
return TARGET_SCHEMAS[key]
except KeyError as exc:
raise ValueError(f"Unsupported target schema: {key}") from exc