linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix: optimize visualization hot paths and refine bgp brief workspace

Browse Source
This commit is contained in:
rayd1o
2026-04-10 02:12:29 +08:00
parent 83839b8b11
commit 749e6e76b6
18 changed files with 776 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files

289
backend/app/api/v1/visualization.py
View File

@@ -205,42 +205,84 @@ def convert_satellite_to_geojson(records: List[CollectedData]) -> Dict[str, Any]
return {"type": "FeatureCollection", "features": features}
def dedupe_satellite_records(records: List[CollectedData]) -> List[CollectedData]:
"""Keep only the newest record for each satellite identity."""
latest_by_key: Dict[str, CollectedData] = {}
for record in records:
metadata = record.extra_data or {}
norad_id = metadata.get("norad_cat_id")
dedupe_key = (
str(norad_id)
if norad_id not in (None, "")
else str(record.source_id or record.entity_key or record.name or record.id)
)
existing = latest_by_key.get(dedupe_key)
if existing is None or (record.id or 0) > (existing.id or 0):
latest_by_key[dedupe_key] = record
return sorted(latest_by_key.values(), key=lambda item: item.id or 0, reverse=True)
def _current_collected_data_stmt(source: str):
return (
select(CollectedData)
.where(CollectedData.source == source)
.where(CollectedData.is_current.is_(True))
.order_by(CollectedData.id.desc())
)
def dedupe_collected_records(records: List[CollectedData]) -> List[CollectedData]:
"""Keep only the newest record for each collected entity."""
latest_by_key: Dict[str, CollectedData] = {}
async def _load_current_collected_data(
db: AsyncSession,
source: str,
*,
exclude_unknown_name: bool = False,
limit: Optional[int] = None,
) -> List[CollectedData]:
stmt = _current_collected_data_stmt(source)
if exclude_unknown_name:
stmt = stmt.where(CollectedData.name != "Unknown")
if limit is not None:
stmt = stmt.limit(limit)
for record in records:
dedupe_key = str(
record.source_id
or record.entity_key
or record.name
or record.id
)
existing = latest_by_key.get(dedupe_key)
if existing is None or (record.id or 0) > (existing.id or 0):
latest_by_key[dedupe_key] = record
result = await db.execute(stmt)
return list(result.scalars().all())
return sorted(latest_by_key.values(), key=lambda item: item.id or 0, reverse=True)
async def _load_current_collected_data_by_sources(
db: AsyncSession,
sources: List[str],
) -> Dict[str, List[CollectedData]]:
if not sources:
return {}
stmt = (
select(CollectedData)
.where(CollectedData.source.in_(sources))
.where(CollectedData.is_current.is_(True))
.order_by(CollectedData.source.asc(), CollectedData.id.desc())
)
result = await db.execute(stmt)
grouped_records: Dict[str, List[CollectedData]] = {source: [] for source in sources}
for record in result.scalars().all():
grouped_records.setdefault(record.source, []).append(record)
return grouped_records
def _build_landing_point_cable_maps(
relation_records: List[CollectedData],
cable_records: List[CollectedData],
) -> tuple[Dict[int, List[int]], Dict[int, str]]:
city_to_cable_ids_map: Dict[int, List[int]] = {}
for relation_record in relation_records:
if not relation_record.extra_data:
continue
city_id = relation_record.extra_data.get("city_id")
cable_id = relation_record.extra_data.get("cable_id")
if city_id is None or cable_id is None:
continue
city_to_cable_ids_map.setdefault(city_id, [])
if cable_id not in city_to_cable_ids_map[city_id]:
city_to_cable_ids_map[city_id].append(cable_id)
cable_id_to_name_map: Dict[int, str] = {}
for cable_record in cable_records:
if not cable_record.extra_data:
continue
cable_id = cable_record.extra_data.get("cable_id")
cable_name = cable_record.name
if cable_id and cable_name:
cable_id_to_name_map[cable_id] = cable_name
return city_to_cable_ids_map, cable_id_to_name_map
def _filter_known_records(records: List[CollectedData]) -> List[CollectedData]:
return [record for record in records if record.name != "Unknown"]
def convert_supercomputer_to_geojson(records: List[CollectedData]) -> Dict[str, Any]:
@@ -722,9 +764,7 @@ def convert_bgp_incidents_to_geojson(
async def get_cables_geojson(db: AsyncSession = Depends(get_db)):
"""获取海底电缆 GeoJSON 数据 (LineString)"""
try:
stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
result = await db.execute(stmt)
records = dedupe_collected_records(list(result.scalars().all()))
records = await _load_current_collected_data(db, "arcgis_cables")
if not records:
raise HTTPException(
@@ -742,36 +782,14 @@ async def get_cables_geojson(db: AsyncSession = Depends(get_db)):
@router.get("/geo/landing-points")
async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
try:
landing_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
landing_result = await db.execute(landing_stmt)
records = dedupe_collected_records(list(landing_result.scalars().all()))
relation_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cable_landing_relation")
relation_result = await db.execute(relation_stmt)
relation_records = dedupe_collected_records(list(relation_result.scalars().all()))
cable_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
cable_result = await db.execute(cable_stmt)
cable_records = dedupe_collected_records(list(cable_result.scalars().all()))
city_to_cable_ids_map = {}
for rel in relation_records:
if rel.extra_data:
city_id = rel.extra_data.get("city_id")
cable_id = rel.extra_data.get("cable_id")
if city_id is not None and cable_id is not None:
if city_id not in city_to_cable_ids_map:
city_to_cable_ids_map[city_id] = []
if cable_id not in city_to_cable_ids_map[city_id]:
city_to_cable_ids_map[city_id].append(cable_id)
cable_id_to_name_map = {}
for cable in cable_records:
if cable.extra_data:
cable_id = cable.extra_data.get("cable_id")
cable_name = cable.name
if cable_id and cable_name:
cable_id_to_name_map[cable_id] = cable_name
records = await _load_current_collected_data(db, "arcgis_landing_points")
relation_records = await _load_current_collected_data(db, "arcgis_cable_landing_relation")
cable_records = await _load_current_collected_data(db, "arcgis_cables")
city_to_cable_ids_map, cable_id_to_name_map = _build_landing_point_cable_maps(
relation_records,
cable_records,
)
if not records:
raise HTTPException(
@@ -788,36 +806,21 @@ async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
@router.get("/geo/all")
async def get_all_geojson(db: AsyncSession = Depends(get_db)):
cables_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
cables_result = await db.execute(cables_stmt)
cables_records = dedupe_collected_records(list(cables_result.scalars().all()))
points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
points_result = await db.execute(points_stmt)
points_records = dedupe_collected_records(list(points_result.scalars().all()))
relation_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cable_landing_relation")
relation_result = await db.execute(relation_stmt)
relation_records = dedupe_collected_records(list(relation_result.scalars().all()))
city_to_cable_ids_map = {}
for rel in relation_records:
if rel.extra_data:
city_id = rel.extra_data.get("city_id")
cable_id = rel.extra_data.get("cable_id")
if city_id is not None and cable_id is not None:
if city_id not in city_to_cable_ids_map:
city_to_cable_ids_map[city_id] = []
if cable_id not in city_to_cable_ids_map[city_id]:
city_to_cable_ids_map[city_id].append(cable_id)
cable_id_to_name_map = {}
for cable in cables_records:
if cable.extra_data:
cable_id = cable.extra_data.get("cable_id")
cable_name = cable.name
if cable_id and cable_name:
cable_id_to_name_map[cable_id] = cable_name
records_by_source = await _load_current_collected_data_by_sources(
db,
[
"arcgis_cables",
"arcgis_landing_points",
"arcgis_cable_landing_relation",
],
)
cables_records = records_by_source.get("arcgis_cables", [])
points_records = records_by_source.get("arcgis_landing_points", [])
relation_records = records_by_source.get("arcgis_cable_landing_relation", [])
city_to_cable_ids_map, cable_id_to_name_map = _build_landing_point_cable_maps(
relation_records,
cables_records,
)
cables = (
convert_cable_to_geojson(cables_records)
@@ -850,17 +853,12 @@ async def get_satellites_geojson(
db: AsyncSession = Depends(get_db),
):
"""获取卫星 TLE GeoJSON 数据"""
stmt = (
select(CollectedData)
.where(CollectedData.source == "celestrak_tle")
.where(CollectedData.name != "Unknown")
.order_by(CollectedData.id.desc())
records = await _load_current_collected_data(
db,
"celestrak_tle",
exclude_unknown_name=True,
limit=limit,
)
result = await db.execute(stmt)
records = dedupe_satellite_records(list(result.scalars().all()))
if limit is not None:
records = records[:limit]
if not records:
return {"type": "FeatureCollection", "features": [], "count": 0}
@@ -878,15 +876,12 @@ async def get_supercomputers_geojson(
db: AsyncSession = Depends(get_db),
):
"""获取 TOP500 超算中心 GeoJSON 数据"""
stmt = (
select(CollectedData)
.where(CollectedData.source == "top500")
.where(CollectedData.name != "Unknown")
.order_by(CollectedData.id.desc())
records = await _load_current_collected_data(
db,
"top500",
exclude_unknown_name=True,
limit=limit,
)
result = await db.execute(stmt)
records = dedupe_collected_records(list(result.scalars().all()))
records = records[:limit]
if not records:
return {"type": "FeatureCollection", "features": [], "count": 0}
@@ -904,15 +899,12 @@ async def get_gpu_clusters_geojson(
db: AsyncSession = Depends(get_db),
):
"""获取 GPU 集群 GeoJSON 数据"""
stmt = (
select(CollectedData)
.where(CollectedData.source == "epoch_ai_gpu")
.where(CollectedData.name != "Unknown")
.order_by(CollectedData.id.desc())
records = await _load_current_collected_data(
db,
"epoch_ai_gpu",
exclude_unknown_name=True,
limit=limit,
)
result = await db.execute(stmt)
records = dedupe_collected_records(list(result.scalars().all()))
records = records[:limit]
if not records:
return {"type": "FeatureCollection", "features": [], "count": 0}
@@ -990,37 +982,27 @@ async def get_all_visualization_data(db: AsyncSession = Depends(get_db)):
- supercomputers: TOP500 超算
- gpu_clusters: GPU 集群
"""
cables_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
cables_result = await db.execute(cables_stmt)
cables_records = dedupe_collected_records(list(cables_result.scalars().all()))
points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
points_result = await db.execute(points_stmt)
points_records = dedupe_collected_records(list(points_result.scalars().all()))
satellites_stmt = (
select(CollectedData)
.where(CollectedData.source == "celestrak_tle")
.where(CollectedData.name != "Unknown")
records_by_source = await _load_current_collected_data_by_sources(
db,
[
"arcgis_cables",
"arcgis_landing_points",
"celestrak_tle",
"top500",
"epoch_ai_gpu",
],
)
satellites_result = await db.execute(satellites_stmt)
satellites_records = dedupe_satellite_records(list(satellites_result.scalars().all()))
supercomputers_stmt = (
select(CollectedData)
.where(CollectedData.source == "top500")
.where(CollectedData.name != "Unknown")
cables_records = records_by_source.get("arcgis_cables", [])
points_records = records_by_source.get("arcgis_landing_points", [])
satellites_records = _filter_known_records(
records_by_source.get("celestrak_tle", []),
)
supercomputers_result = await db.execute(supercomputers_stmt)
supercomputers_records = dedupe_collected_records(list(supercomputers_result.scalars().all()))
gpu_stmt = (
select(CollectedData)
.where(CollectedData.source == "epoch_ai_gpu")
.where(CollectedData.name != "Unknown")
supercomputers_records = _filter_known_records(
records_by_source.get("top500", []),
)
gpu_records = _filter_known_records(
records_by_source.get("epoch_ai_gpu", []),
)
gpu_result = await db.execute(gpu_stmt)
gpu_records = dedupe_collected_records(list(gpu_result.scalars().all()))
cables = (
convert_cable_to_geojson(cables_records)
@@ -1084,13 +1066,8 @@ async def get_cable_graph(db: AsyncSession) -> CableGraph:
global _cable_graph
if _cable_graph is None:
cables_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
cables_result = await db.execute(cables_stmt)
cables_records = list(cables_result.scalars().all())
points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
points_result = await db.execute(points_stmt)
points_records = list(points_result.scalars().all())
cables_records = await _load_current_collected_data(db, "arcgis_cables")
points_records = await _load_current_collected_data(db, "arcgis_landing_points")
cables_data = convert_cable_to_geojson(cables_records)
points_data = convert_landing_point_to_geojson(points_records)