linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

release: bump version to 0.48.0

Browse Source
This commit is contained in:
linkong
2026-05-07 18:06:06 +08:00
parent 421234301a
commit bb9183b8a4
51 changed files with 4609 additions and 400 deletions
Download Patch File Download Diff File Expand all files Collapse all files

489
backend/app/api/v1/visualization.py
View File

@@ -6,6 +6,7 @@ Returns GeoJSON format compatible with Three.js, CesiumJS, and Unreal Cesium.
from datetime import UTC, datetime, timedelta
import math
import re
import httpx
from fastapi import APIRouter, HTTPException, Depends, Query, Response
from sqlalchemy.ext.asyncio import AsyncSession
@@ -19,14 +20,16 @@ from app.core.time import to_iso8601_utc
from app.db.session import get_db
from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_incident import BGPIncident
from app.models.bgp_observation import BGPObservation
from app.models.collected_data import CollectedData
from app.models.vessel import VesselPosition, VesselStatic
from app.models.vessel import AISSourceHealth, VesselPosition, VesselStatic
from app.services.bgp_collectors import build_bgp_collector_coverage
from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance
from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
from app.services.persistent_logs import record_system_log
from app.services.vessel_ais_aggregation import (
build_field_conflict_candidates,
count_unique_raw_vessel_mmsi,
get_aggregated_vessel,
get_aggregated_vessel_track,
get_aggregated_vessels,
@@ -40,6 +43,7 @@ logger = get_logger(__name__, service="api")
TERRAIN_TILE_URL_TEMPLATE = (
"https://s3.amazonaws.com/elevation-tiles-prod/terrarium/{z}/{x}/{y}.png"
)
VESSEL_NAME_FALLBACK_PATTERN = re.compile(r"^mmsi\s*\d+$", re.IGNORECASE)
# ============== Converter Functions ==============
@@ -281,6 +285,120 @@ async def _load_current_collected_data(
return list(result.scalars().all())
async def _latest_task_id_for_source(
db: AsyncSession,
source: str,
*,
exclude_unknown_name: bool = False,
) -> int | None:
stmt = (
select(
CollectedData.task_id,
func.max(CollectedData.collected_at).label("latest_collected_at"),
func.max(CollectedData.id).label("latest_id"),
)
.where(CollectedData.source == source)
.where(CollectedData.task_id.isnot(None))
.group_by(CollectedData.task_id)
.order_by(func.max(CollectedData.collected_at).desc(), func.max(CollectedData.id).desc())
.limit(1)
)
if exclude_unknown_name:
stmt = stmt.where(CollectedData.name != "Unknown")
result = await db.execute(stmt)
row = result.first()
return int(row.task_id) if row and row.task_id is not None else None
async def _load_current_or_latest_task_data(
db: AsyncSession,
source: str,
*,
exclude_unknown_name: bool = False,
limit: Optional[int] = None,
) -> List[CollectedData]:
records = await _load_current_collected_data(
db,
source,
exclude_unknown_name=exclude_unknown_name,
limit=limit,
)
if records:
return records
latest_task_id = await _latest_task_id_for_source(
db,
source,
exclude_unknown_name=exclude_unknown_name,
)
if latest_task_id is None:
return []
stmt = (
select(CollectedData)
.where(CollectedData.source == source)
.where(CollectedData.task_id == latest_task_id)
.order_by(CollectedData.id.desc())
)
if exclude_unknown_name:
stmt = stmt.where(CollectedData.name != "Unknown")
if limit is not None:
stmt = stmt.limit(limit)
result = await db.execute(stmt)
return list(result.scalars().all())
async def _count_current_or_latest_task_data(
db: AsyncSession,
source: str,
*,
exclude_unknown_name: bool = False,
) -> int:
current_stmt = (
select(func.count(CollectedData.id))
.where(CollectedData.source == source)
.where(CollectedData.is_current.is_(True))
)
if exclude_unknown_name:
current_stmt = current_stmt.where(CollectedData.name != "Unknown")
current_result = await db.execute(current_stmt)
current_scalar = current_result.scalar()
if current_scalar is None and hasattr(current_result, "scalars"):
current_rows = current_result.scalars().all()
current_count = sum(
1
for row in current_rows
if getattr(row, "source", None) == source
and (not exclude_unknown_name or getattr(row, "name", None) != "Unknown")
)
else:
current_count = int(current_scalar or 0)
if current_count > 0:
return current_count
latest_task_id = await _latest_task_id_for_source(
db,
source,
exclude_unknown_name=exclude_unknown_name,
)
if latest_task_id is None:
return 0
latest_stmt = (
select(func.count(CollectedData.id))
.where(CollectedData.source == source)
.where(CollectedData.task_id == latest_task_id)
)
if exclude_unknown_name:
latest_stmt = latest_stmt.where(CollectedData.name != "Unknown")
latest_result = await db.execute(latest_stmt)
return int(latest_result.scalar() or 0)
async def _load_current_collected_data_by_sources(
db: AsyncSession,
sources: List[str],
@@ -636,14 +754,21 @@ VESSEL_TYPE_FILTERS = {
def convert_vessels_to_geojson(rows: List[Any]) -> Dict[str, Any]:
features = []
seen_mmsi: set[int] = set()
for position, static in rows:
if position.lat is None or position.lon is None:
continue
if position.mmsi in seen_mmsi:
continue
seen_mmsi.add(position.mmsi)
props = {
"mmsi": position.mmsi,
"mmsi_display": str(position.mmsi),
"name": getattr(static, "name", None) or f"MMSI {position.mmsi}",
"name_is_fallback": _is_vessel_name_fallback(getattr(static, "name", None), position.mmsi),
"callsign": getattr(static, "callsign", None),
"imo": getattr(static, "imo", None),
"imo_display": str(getattr(static, "imo")) if getattr(static, "imo", None) else None,
"vessel_type": getattr(static, "vessel_type", None),
"vessel_type_name": getattr(static, "vessel_type_name", None) or "Other",
"flag": getattr(static, "flag", None),
@@ -685,9 +810,12 @@ def convert_aggregated_vessels_to_geojson(vessels: List[dict[str, Any]]) -> Dict
}
props = {
"mmsi": vessel["mmsi"],
"mmsi_display": str(vessel["mmsi"]),
"name": vessel.get("name") or f"MMSI {vessel['mmsi']}",
"name_is_fallback": _is_vessel_name_fallback(vessel.get("name"), vessel["mmsi"]),
"callsign": vessel.get("callsign"),
"imo": vessel.get("imo"),
"imo_display": str(vessel.get("imo")) if vessel.get("imo") else None,
"vessel_type": vessel.get("vessel_type"),
"vessel_type_name": vessel.get("vessel_type_name") or "Other",
"flag": vessel.get("flag"),
@@ -704,6 +832,7 @@ def convert_aggregated_vessels_to_geojson(vessels: List[dict[str, Any]]) -> Dict
"source_summary": source_summary,
"quality_flags": vessel.get("quality_flags") or [],
"conflict_count": vessel.get("conflict_count", 0),
"aggregation_strategy_version": vessel.get("aggregation_strategy_version", 0),
"data_type": "vessel",
}
features.append(
@@ -737,6 +866,24 @@ def _parse_bbox(value: Optional[str]) -> tuple[float, float, float, float] | Non
return lon_min, lat_min, lon_max, lat_max
def _is_vessel_name_fallback(name: Any, mmsi: Any) -> bool:
text = str(name or "").strip()
mmsi_text = str(mmsi or "").strip()
if not text:
return True
if mmsi_text and text == mmsi_text:
return True
return bool(VESSEL_NAME_FALLBACK_PATTERN.match(text))
def _requested_vessel_types(value: Optional[str]) -> set[str]:
return {
item.strip().lower()
for item in (value or "").split(",")
if item.strip()
}
def _matches_vessel_type(props: dict[str, Any], requested_types: set[str]) -> bool:
if not requested_types:
return True
@@ -747,6 +894,88 @@ def _matches_vessel_type(props: dict[str, Any], requested_types: set[str]) -> bo
return False
def _feature_mmsi_key(feature: dict[str, Any]) -> str | None:
props = feature.get("properties", {})
mmsi = props.get("mmsi") or feature.get("id")
if mmsi in (None, ""):
return None
return str(mmsi)
def _feature_in_bbox(feature: dict[str, Any], bbox: tuple[float, float, float, float] | None) -> bool:
if bbox is None:
return True
coordinates = feature.get("geometry", {}).get("coordinates") or []
if len(coordinates) < 2:
return False
try:
lon = float(coordinates[0])
lat = float(coordinates[1])
except (TypeError, ValueError):
return False
lon_min, lat_min, lon_max, lat_max = bbox
return lon_min <= lon <= lon_max and lat_min <= lat <= lat_max
def _filter_vessel_features(
features: list[dict[str, Any]],
*,
bbox: tuple[float, float, float, float] | None,
requested_types: set[str],
) -> list[dict[str, Any]]:
return [
feature
for feature in features
if _feature_in_bbox(feature, bbox)
and _matches_vessel_type(feature.get("properties", {}), requested_types)
]
def _merge_vessel_features(
raw_features: list[dict[str, Any]],
legacy_features: list[dict[str, Any]],
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
"""Prefer aggregated raw observations as the canonical source of truth.
Legacy `vessel_position` rows only fill MMSIs that the unified pipeline does
not yet know about, so a vessel never appears twice when both BarentsWatch
and AISStream observe it. Once the legacy table drains, this branch becomes
a no-op.
"""
merged: list[dict[str, Any]] = []
seen: set[str] = set()
raw_keys: set[str] = set()
legacy_keys: set[str] = set()
for feature in raw_features:
key = _feature_mmsi_key(feature)
if key is None or key in seen:
continue
seen.add(key)
raw_keys.add(key)
merged.append(feature)
legacy_added = 0
for feature in legacy_features:
key = _feature_mmsi_key(feature)
if key is None:
continue
legacy_keys.add(key)
if key in seen:
continue
seen.add(key)
legacy_added += 1
merged.append(feature)
return merged, {
"raw_unique_mmsi": len(raw_keys),
"legacy_unique_mmsi": len(legacy_keys),
"legacy_backfilled_mmsi": legacy_added,
"final_unique_mmsi": len(seen),
}
def _build_vessel_stats(features: List[dict[str, Any]]) -> dict[str, Any]:
by_type: dict[str, int] = {}
underway = 0
@@ -1347,7 +1576,7 @@ async def get_satellites_geojson(
db: AsyncSession = Depends(get_db),
):
"""获取卫星 TLE GeoJSON 数据"""
records = await _load_current_collected_data(
records = await _load_current_or_latest_task_data(
db,
"celestrak_tle",
exclude_unknown_name=True,
@@ -1476,27 +1705,30 @@ async def get_vessels_geojson(
):
"""Return latest vessel positions as GeoJSON points."""
parsed_bbox = _parse_bbox(bbox)
aggregated_vessels = await get_aggregated_vessels(db, bbox=parsed_bbox, limit=limit)
if aggregated_vessels:
geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels)
requested_types = {
item.strip().lower()
for item in (type or "").split(",")
if item.strip()
}
if requested_types:
geojson["features"] = [
feature
for feature in geojson.get("features", [])
if _matches_vessel_type(feature.get("properties", {}), requested_types)
]
requested_types = _requested_vessel_types(type)
merged_features, diagnostics = await _load_merged_vessel_features(db)
features = _filter_vessel_features(
merged_features,
bbox=parsed_bbox,
requested_types=requested_types,
)
if limit and limit > 0:
features = features[:limit]
return {
"type": "FeatureCollection",
"features": features,
"count": len(features),
"stats": _build_vessel_stats(features),
"diagnostics": {
**diagnostics,
"filtered_count": len(features),
},
}
features = geojson.get("features", [])
return {
**geojson,
"count": len(features),
"stats": _build_vessel_stats(features),
}
async def _load_merged_vessel_features(db: AsyncSession) -> tuple[list[dict[str, Any]], dict[str, Any]]:
aggregated_vessels = await get_aggregated_vessels(db)
raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels)
latest_times = (
select(
@@ -1516,50 +1748,122 @@ async def get_vessels_geojson(
.outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi)
.order_by(VesselPosition.received_at.desc())
)
if limit and limit > 0:
stmt = stmt.limit(limit)
if parsed_bbox is not None:
lon_min, lat_min, lon_max, lat_max = parsed_bbox
stmt = stmt.where(
VesselPosition.lon >= lon_min,
VesselPosition.lon <= lon_max,
VesselPosition.lat >= lat_min,
VesselPosition.lat <= lat_max,
)
result = await db.execute(stmt)
rows = list(result.all())
geojson = convert_vessels_to_geojson(rows)
requested_types = {
item.strip().lower()
for item in (type or "").split(",")
if item.strip()
legacy_geojson = convert_vessels_to_geojson(rows)
merged_features, diagnostics = _merge_vessel_features(
raw_geojson.get("features", []),
legacy_geojson.get("features", []),
)
return merged_features, {
**diagnostics,
"raw_feature_count": len(raw_geojson.get("features", [])),
"legacy_feature_count": len(legacy_geojson.get("features", [])),
}
if requested_types:
geojson["features"] = [
feature
for feature in geojson.get("features", [])
if _matches_vessel_type(feature.get("properties", {}), requested_types)
]
features = geojson.get("features", [])
@router.get("/vessels/custom-supplements")
async def get_vessel_custom_supplements(db: AsyncSession = Depends(get_db)):
"""Group custom vessel_ais sources by their declared merge target for diagnostics."""
from app.models.datasource_config import DataSourceConfig
result = await db.execute(
select(DataSourceConfig.name, DataSourceConfig.config, DataSourceConfig.is_active)
.where(DataSourceConfig.config["target_schema"].as_string() == "vessel_ais")
)
grouped: dict[str, dict[str, Any]] = {}
for name, config, is_active in result.all():
config = config or {}
merge_target = str(config.get("merge_target_source") or "barentswatch_vessels")
bucket = grouped.setdefault(merge_target, {"merge_target": merge_target, "sources": []})
bucket["sources"].append({"name": name, "is_active": bool(is_active)})
return {"groups": list(grouped.values())}
@router.get("/vessels/name-fallbacks")
async def get_vessel_name_fallbacks(
limit: int = Query(500, ge=0, description="Maximum fallback-name vessels to return. 0 means no limit."),
db: AsyncSession = Depends(get_db),
):
"""Return vessels whose display name still falls back to MMSI."""
aggregated_vessels = await get_aggregated_vessels(db)
raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels)
latest_times = (
select(
VesselPosition.mmsi.label("mmsi"),
func.max(VesselPosition.received_at).label("received_at"),
)
.group_by(VesselPosition.mmsi)
.subquery()
)
result = await db.execute(
select(VesselPosition, VesselStatic)
.join(
latest_times,
(VesselPosition.mmsi == latest_times.c.mmsi)
& (VesselPosition.received_at == latest_times.c.received_at),
)
.outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi)
.order_by(VesselPosition.received_at.desc())
)
legacy_geojson = convert_vessels_to_geojson(list(result.all()))
features, diagnostics = _merge_vessel_features(
raw_geojson.get("features", []),
legacy_geojson.get("features", []),
)
fallback_items = []
for feature in features:
props = feature.get("properties", {})
mmsi = props.get("mmsi")
name = props.get("name")
if not _is_vessel_name_fallback(name, mmsi):
continue
source_summary = props.get("source_summary") or {}
fallback_items.append(
{
"mmsi": str(mmsi),
"display_name": name or f"MMSI {mmsi}",
"reason": "missing_real_name",
"received_at": props.get("received_at"),
"sources": sorted(source_summary.keys()),
"source_summary": source_summary,
"message_types": sorted(
{
message_type
for summary in source_summary.values()
for message_type in (summary.get("message_types") or [])
}
),
"field_sources": props.get("field_sources") or {},
}
)
if limit and limit > 0:
fallback_items = fallback_items[:limit]
return {
**geojson,
"count": len(features),
"stats": _build_vessel_stats(features),
"count": len(fallback_items),
"items": fallback_items,
"diagnostics": diagnostics,
}
@router.get("/vessels/{mmsi}")
async def get_vessel_detail(mmsi: int, db: AsyncSession = Depends(get_db)):
from app.services.vessel_enrichment import get_vessel_enrichment_bundle
aggregated = await get_aggregated_vessel(db, mmsi)
enrichment = await get_vessel_enrichment_bundle(db, mmsi)
if aggregated is not None:
return {
**aggregated,
"received_at": to_iso8601_utc(aggregated.get("received_at")),
"latitude": aggregated["lat"],
"longitude": aggregated["lon"],
"enrichment": enrichment,
}
latest_position_stmt = (
@@ -1578,6 +1882,7 @@ async def get_vessel_detail(mmsi: int, db: AsyncSession = Depends(get_db)):
**(geojson["features"][0]["properties"]),
"latitude": position.lat,
"longitude": position.lon,
"enrichment": enrichment,
}
@@ -1737,31 +2042,16 @@ async def get_bgp_collectors_geojson(db: AsyncSession = Depends(get_db)):
@router.get("/geo/summary")
async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)):
"""Return lightweight Earth HUD counts without loading layer GeoJSON payloads."""
records_by_source = await _load_current_collected_data_by_sources(
cable_count = await _count_current_or_latest_task_data(db, "arcgis_cables")
landing_point_count = await _count_current_or_latest_task_data(db, "arcgis_landing_points")
satellite_count = await _count_current_or_latest_task_data(
db,
[
"arcgis_cables",
"arcgis_landing_points",
"celestrak_tle",
"top500",
"epoch_ai_gpu",
],
"celestrak_tle",
exclude_unknown_name=True,
)
cables = convert_cable_to_geojson(records_by_source.get("arcgis_cables", []))
landing_points = convert_landing_point_to_geojson(
records_by_source.get("arcgis_landing_points", []),
)
satellites = convert_satellite_to_geojson(
_filter_known_records(records_by_source.get("celestrak_tle", [])),
)
compute_centers = convert_compute_centers_to_geojson(
_filter_known_records(
records_by_source.get("top500", [])
+ records_by_source.get("epoch_ai_gpu", []),
),
)
compute_features = compute_centers.get("features", [])
supercomputer_count = await _count_current_or_latest_task_data(db, "top500")
gpu_cluster_count = await _count_current_or_latest_task_data(db, "epoch_ai_gpu")
compute_center_count = supercomputer_count + gpu_cluster_count
active_incident_result = await db.execute(
select(func.count(BGPIncident.id)).where(BGPIncident.status == "active"),
@@ -1771,35 +2061,56 @@ async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)):
)
active_incident_count = int(active_incident_result.scalar() or 0)
active_anomaly_count = int(active_anomaly_result.scalar() or 0)
bgp_collectors = await build_bgp_collector_coverage(
bgp_collector_result = await db.execute(
select(func.count(func.distinct(BGPObservation.collector)))
.where(BGPObservation.collector.isnot(None))
.where(func.length(func.btrim(BGPObservation.collector)) > 0)
.where(BGPObservation.source.in_(("ris_live_bgp", "bgpstream_bgp")))
)
bgp_collector_scalar = bgp_collector_result.scalar()
if bgp_collector_scalar is None:
bgp_collectors = await build_bgp_collector_coverage(
db,
source_filter=("ris_live_bgp", "bgpstream_bgp"),
)
bgp_collector_count = len(
[item for item in bgp_collectors if item.get("collector")]
)
else:
bgp_collector_count = int(bgp_collector_scalar or 0)
raw_unique_window_hours = 24
raw_unique_mmsi = await count_unique_raw_vessel_mmsi(
db,
source_filter=("ris_live_bgp", "bgpstream_bgp"),
observed_since=datetime.now(UTC) - timedelta(hours=raw_unique_window_hours),
)
vessel_count_result = await db.execute(
select(func.count(func.distinct(VesselPosition.mmsi))),
legacy_unique_result = await db.execute(
select(func.count(func.distinct(VesselPosition.mmsi)))
)
vessel_count = int(vessel_count_result.scalar() or 0)
legacy_unique_mmsi = int(legacy_unique_result.scalar() or 0)
vessel_count = max(raw_unique_mmsi, legacy_unique_mmsi)
aisstream_health = await db.get(AISSourceHealth, "aisstream_vessels")
return {
"generated_at": to_iso8601_utc(datetime.now(UTC)),
"stats": {
"cable_count": len(cables.get("features", [])),
"landing_point_count": len(landing_points.get("features", [])),
"satellite_count": len(satellites.get("features", [])),
"compute_center_count": len(compute_features),
"cable_count": cable_count,
"landing_point_count": landing_point_count,
"satellite_count": satellite_count,
"compute_center_count": compute_center_count,
"vessel_count": vessel_count,
"supercomputer_count": sum(
1 for feature in compute_features
if feature.get("properties", {}).get("site_type") == "supercomputer"
),
"gpu_cluster_count": sum(
1 for feature in compute_features
if feature.get("properties", {}).get("site_type") == "gpu_cluster"
),
"vessel_raw_unique_mmsi": raw_unique_mmsi,
"vessel_raw_unique_window_hours": raw_unique_window_hours,
"vessel_legacy_unique_mmsi": legacy_unique_mmsi,
"aisstream_connection_state": aisstream_health.connection_state if aisstream_health else None,
"aisstream_last_seen_at": to_iso8601_utc(aisstream_health.last_seen_at) if aisstream_health else None,
"aisstream_message_rate": aisstream_health.message_rate if aisstream_health else None,
"aisstream_lag_seconds": aisstream_health.lag_seconds if aisstream_health else None,
"supercomputer_count": supercomputer_count,
"gpu_cluster_count": gpu_cluster_count,
"bgp_event_count": active_incident_count or active_anomaly_count,
"bgp_incident_count": active_incident_count,
"bgp_anomaly_count": active_anomaly_count,
"bgp_collector_count": len([item for item in bgp_collectors if item.get("collector")]),
"bgp_collector_count": bgp_collector_count,
},
}