From bb9183b8a426e2fe576d50a9f760597835ccf469 Mon Sep 17 00:00:00 2001 From: linkong Date: Thu, 7 May 2026 18:06:06 +0800 Subject: [PATCH] release: bump version to 0.48.0 --- TODO.md | 5 + VERSION | 2 +- backend/app/api/main.py | 6 + backend/app/api/v1/datasource_config.py | 162 ++++-- backend/app/api/v1/vessel_aggregation.py | 132 +++++ backend/app/api/v1/visualization.py | 489 ++++++++++++++---- backend/app/api/v1/websocket.py | 41 +- backend/app/core/target_schema_registry.py | 8 +- backend/app/core/websocket/broadcaster.py | 2 +- backend/app/core/websocket/manager.py | 46 +- backend/app/db/session.py | 25 + backend/app/models/collected_data.py | 2 + backend/app/models/vessel.py | 1 + backend/app/models/vessel_enrichment.py | 63 +++ backend/app/services/collectors/aisstream.py | 225 +++++++- backend/app/services/collectors/vessel_ais.py | 79 ++- .../app/services/custom_datasource_runtime.py | 391 ++++++++++++++ backend/app/services/datasource_mapping.py | 76 ++- .../services/vessel_aggregation_strategy.py | 198 +++++++ .../app/services/vessel_ais_aggregation.py | 138 ++++- backend/app/services/vessel_enrichment.py | 109 ++++ .../test_custom_datasource_runtime_live.py | 149 ++++++ backend/tests/test_datasource_mapping.py | 129 +++++ .../tests/test_vessel_aggregation_strategy.py | 161 ++++++ backend/tests/test_vessel_enrichment.py | 155 ++++++ backend/tests/test_vessels.py | 202 +++++++- .../test_visualization_compute_centers.py | 10 +- backend/tests/test_websocket_manager.py | 46 ++ docs/CHANGELOG.md | 16 + docs/plans/custom-source-live-mock-plan.md | 384 ++++++++++++++ .../earth-vessel-ais-aggregation-plan.md | 121 ++++- docs/technical/zh/README.md | 1 + .../zh/earth-toolbar-overlay-coordination.md | 94 ++++ docs/version-history.md | 3 +- frontend/package.json | 2 +- frontend/public/earth/index.html | 5 + frontend/public/earth/js/controls.js | 7 +- frontend/public/earth/js/info-card.js | 124 ++++- .../public/earth/js/layer-startup-tasks.js | 2 + frontend/public/earth/js/main.js | 49 ++ frontend/public/earth/js/ui.js | 1 + frontend/public/earth/js/vessels.js | 204 +++++++- frontend/src/hooks/useWebSocket.ts | 34 +- frontend/src/pages/Dashboard/Dashboard.tsx | 9 +- .../src/pages/DataSources/DataSources.tsx | 130 ++--- frontend/src/pages/Settings/Settings.tsx | 406 ++++++++++++++- package.json | 7 + planet.sh | 69 ++- pyproject.toml | 2 +- scripts/mock-ais-ws-server.ts | 285 ++++++++++ uv.lock | 2 +- 51 files changed, 4609 insertions(+), 400 deletions(-) create mode 100644 backend/app/api/v1/vessel_aggregation.py create mode 100644 backend/app/models/vessel_enrichment.py create mode 100644 backend/app/services/custom_datasource_runtime.py create mode 100644 backend/app/services/vessel_aggregation_strategy.py create mode 100644 backend/app/services/vessel_enrichment.py create mode 100644 backend/tests/test_custom_datasource_runtime_live.py create mode 100644 backend/tests/test_vessel_aggregation_strategy.py create mode 100644 backend/tests/test_vessel_enrichment.py create mode 100644 backend/tests/test_websocket_manager.py create mode 100644 docs/plans/custom-source-live-mock-plan.md create mode 100644 docs/technical/zh/earth-toolbar-overlay-coordination.md create mode 100644 package.json create mode 100644 scripts/mock-ais-ws-server.ts diff --git a/TODO.md b/TODO.md index ab1e8fd1..717e8953 100644 --- a/TODO.md +++ b/TODO.md @@ -26,6 +26,11 @@ - [ ] 重写控制台 UI，逐步抛弃 Ant Design，建立自有组件体系，并统一采用 `tabler.io` / Tabler Icons 作为控制台主图标库 - [ ] 把 Earth 态势新闻源从 [earth_news.py](/home/ray/dev/linkong/planet/backend/app/services/earth_news.py) 的硬编码列表抽成可配置目录，优先保持当前“实时聚合”链路不变，只先解决新闻源不可配置的问题 - [ ] 为 Earth 态势新闻设计后续采集器化方案：明确新闻数据模型、去重策略、区域映射、过期清理和 Earth/AI 复用方式，再决定何时把新闻从实时抓取升级成正式 collector +- [ ] AIS v3.1：修复船只聚合完整性，`/geo/vessels` 合并 raw observation 聚合结果与 legacy `vessel_position + vessel_static` 最新结果，确保 BarentsWatch-only 船只不会因为 AISStream 子集存在而消失，并增加 raw/legacy/final unique MMSI 诊断统计 +- [ ] AIS v3.2：把 AISStream 从收满 `max_messages` 后结束的批采集改成长连接 streaming service，持续写入 raw observations，通过内部 `/ws` 的 `vessels` channel 推送新船、位置和航向增量，Earth 前端按 MMSI upsert marker +- [ ] AIS v3.3：修正 AISStream 采集页面状态语义，使用 connecting/streaming/reconnecting/stopped 与 indeterminate 状态，展示运行时长、消息数、unique MMSI、message rate、最近消息和错误，不再用一次性 REST 进度条表示长连接 +- [ ] AIS v3.4：修复船只身份字段和名称聚合，MMSI/IMO/callsign 按字符串显示且不带千分位符；查询并列出所有仍以 MMSI 号码或 `MMSI ` 作为船名的记录，标注来源、最近观测、message types 和缺失原因，并把这批 fallback-name 船只纳入名称聚合修复集合 +- [ ] Earth Live Sync：建立统一态势实时同步链路，新增 `earth_summary` WS channel，任意采集器成功后广播轻量 summary invalidation，前端收到后重新拉 `/api/v1/visualization/geo/summary` 并更新 HUD；同时为 BGP 增加 `bgp` WS channel，使 BGP incidents/anomalies/collectors 在不刷新页面时也能 upsert 图层；卫星采集完成后触发 summary 刷新，必要时按 TLE 版本重新 hydrate 卫星数据 - [ ] AIS v4：开放船只多源聚合策略配置，支持 source priority、字段级规则、freshness 窗口和高级保护开关；保存时校验未知字段、非法模式和危险动态字段锁定，并在聚合接口返回命中的配置版本 - [ ] AIS v5：实现船舶资料 enrichment 与冲突治理，按 `mmsi + imo + name + callsign` 异步补充船型细分、AIS 大类、旗国、尺寸、建造年份、运营方和图片缓存；详情面板展示缓存资料和字段来源，不在实时 AIS 请求链路现场抓第三方页面 - [ ] 为 Earth 地球表面增加一层与基础纹理对齐的材质/纹理 overlay，并在同层叠加国界轮廓参考线；要求国界线与底图稳定对齐，且 hover 到国家轮廓时能高亮当前国家，便于校准地表和增强交互 diff --git a/VERSION b/VERSION index 421ab545..a758a09a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.47.0 +0.48.0 diff --git a/backend/app/api/main.py b/backend/app/api/main.py index 54f4f4b6..ae47610b 100644 --- a/backend/app/api/main.py +++ b/backend/app/api/main.py @@ -12,6 +12,7 @@ from app.api.v1 import ( settings, collected_data, visualization, + vessel_aggregation, bgp, news, system_control, @@ -34,6 +35,11 @@ api_router.include_router(alerts.router, prefix="/alerts", tags=["alerts"]) api_router.include_router(settings.router, prefix="/settings", tags=["settings"]) api_router.include_router(system_control.router, prefix="/system", tags=["system"]) api_router.include_router(visualization.router, prefix="/visualization", tags=["visualization"]) +api_router.include_router( + vessel_aggregation.router, + prefix="/vessel-aggregation", + tags=["vessel-aggregation"], +) api_router.include_router(bgp.router, prefix="/bgp", tags=["bgp"]) api_router.include_router(tv.router, prefix="/tv", tags=["tv"]) api_router.include_router(news.router, prefix="/news", tags=["news"]) diff --git a/backend/app/api/v1/datasource_config.py b/backend/app/api/v1/datasource_config.py index 81b15075..f41146ec 100644 --- a/backend/app/api/v1/datasource_config.py +++ b/backend/app/api/v1/datasource_config.py @@ -5,8 +5,8 @@ from datetime import datetime import base64 import json import re -from fastapi import APIRouter, Depends, HTTPException -from sqlalchemy import select, func +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import delete, select, func from sqlalchemy.ext.asyncio import AsyncSession from pydantic import BaseModel, Field import httpx @@ -17,6 +17,8 @@ from app.db.session import get_db from app.models.user import User from app.models.datasource_config import DataSourceConfig from app.models.datasource_mapping import DataSourceMappingTemplate +from app.models.collected_data import CollectedData +from app.models.vessel import AISRawObservation, AISSourceHealth from app.core.security import get_current_user from app.core.cache import cache from app.core.time import to_iso8601_utc @@ -26,10 +28,19 @@ from app.services.datasource_mapping import ( MappingError, build_heuristic_mapping, execute_mapping, - persist_mapped_records, redact_for_llm, stable_payload_hash, ) +from app.services.custom_datasource_runtime import ( + CustomDatasourceRuntimeError, + fetch_rest_payload, + get_custom_stream_status, + run_mapped_rest_config, + run_mapped_websocket_config, + start_custom_stream, + stop_custom_stream, + test_websocket_config, +) from app.services.datasource_connectivity import ( get_builtin_connection_status, save_connectivity_success, @@ -43,7 +54,7 @@ router = APIRouter() class DataSourceConfigCreate(BaseModel): name: str = Field(..., min_length=1, max_length=100) description: Optional[str] = None - source_type: str = Field(..., description="http, api, database") + source_type: str = Field(..., description="rest, websocket, http, api, database") endpoint: str = Field(..., max_length=500) auth_type: str = Field(default="none", description="none, bearer, api_key, basic") auth_config: dict = Field(default={}) @@ -219,6 +230,8 @@ def _build_query_params(auth_type: str, auth_config: dict, config: dict) -> dict async def fetch_custom_sample_from_config(config: DataSourceConfig, limit_bytes: int) -> Any: + if str(config.source_type or "").lower() in {"websocket", "ws"}: + raise HTTPException(status_code=400, detail="WebSocket sources must use connection test or run-mapped stream.") request_config = config.config or {} method = str(request_config.get("method") or request_config.get("request_method") or "GET").upper() if method not in {"GET", "POST"}: @@ -488,6 +501,8 @@ async def update_config( @router.delete("/configs/{config_id}") async def delete_config( config_id: int, + delete_mappings: bool = Query(False), + delete_source_data: bool = Query(False), current_user: User = Depends(get_current_user), db: AsyncSession = Depends(get_db), ): @@ -498,12 +513,59 @@ async def delete_config( if not config: raise HTTPException(status_code=404, detail="Configuration not found") + deleted_mappings = 0 + deleted_records = { + "collected_data": 0, + "ais_raw_observations": 0, + "ais_source_health": 0, + } + + if delete_source_data: + collected_result = await db.execute( + delete(CollectedData).where(CollectedData.source == config.name) + ) + raw_result = await db.execute( + delete(AISRawObservation).where(AISRawObservation.source == config.name) + ) + health_result = await db.execute( + delete(AISSourceHealth).where(AISSourceHealth.source == config.name) + ) + deleted_records = { + "collected_data": collected_result.rowcount or 0, + "ais_raw_observations": raw_result.rowcount or 0, + "ais_source_health": health_result.rowcount or 0, + } + + if delete_mappings or delete_source_data: + mapping_result = await db.execute( + delete(DataSourceMappingTemplate).where( + DataSourceMappingTemplate.datasource_config_id == config_id + ) + ) + deleted_mappings = mapping_result.rowcount or 0 + await db.delete(config) await db.commit() cache.delete_pattern("datasource_configs:*") - return {"message": "Configuration deleted successfully"} + if delete_source_data and (config.config or {}).get("target_schema") == "vessel_ais": + from app.core.websocket.broadcaster import broadcaster + + await broadcaster.broadcast_custom( + "vessels", + { + "action": "reload", + "source": config.name, + "reason": "custom_source_deleted", + }, + ) + + return { + "message": "Configuration deleted successfully", + "deleted_mappings": deleted_mappings, + "deleted_records": deleted_records, + } @router.post("/configs/{config_id}/test") @@ -520,6 +582,8 @@ async def test_config( raise HTTPException(status_code=404, detail="Configuration not found") try: + if str(config.source_type or "").lower() in {"websocket", "ws"}: + return await test_websocket_config(config) result = await test_endpoint( endpoint=config.endpoint, auth_type=config.auth_type, @@ -550,6 +614,18 @@ async def test_new_config( ): """Test a new data source configuration without saving""" try: + if str(config_data.source_type or "").lower() in {"websocket", "ws"}: + config = DataSourceConfig( + name=config_data.name, + description=config_data.description, + source_type=config_data.source_type, + endpoint=config_data.endpoint, + auth_type=config_data.auth_type, + auth_config=config_data.auth_config, + headers=config_data.headers, + config=config_data.config, + ) + return await test_websocket_config(config) result = await test_endpoint( endpoint=config_data.endpoint, auth_type=config_data.auth_type, @@ -875,6 +951,8 @@ async def update_datasource_mapping( @router.post("/{config_id}/run-mapped") async def run_mapped_datasource( config_id: int, + background: bool = Query(False, description="For WebSocket sources, start a background stream task."), + debug_max_messages: int | None = Query(None, ge=1), current_user: User = Depends(get_current_user), db: AsyncSession = Depends(get_db), ): @@ -883,20 +961,24 @@ async def run_mapped_datasource( if not datasource: raise HTTPException(status_code=404, detail="Configuration not found") - result = await db.execute( - select(DataSourceMappingTemplate) - .where(DataSourceMappingTemplate.datasource_config_id == config_id) - .where(DataSourceMappingTemplate.is_active.is_(True)) - .order_by(DataSourceMappingTemplate.version.desc()) - .limit(1) - ) - mapping = result.scalar_one_or_none() - if not mapping: - raise HTTPException(status_code=404, detail="No active mapping template found") - try: - sample = await fetch_custom_sample_from_config(datasource, 5_000_000) - mapped = execute_mapping(sample, mapping.mapping_json, mapping.target_schema) + if str(datasource.source_type or "").lower() in {"websocket", "ws"}: + if background and debug_max_messages is None: + started = start_custom_stream(config_id) + if not started: + raise HTTPException(status_code=409, detail="Custom WebSocket source is already running") + return { + "status": "started", + "datasource_config_id": config_id, + "stream": get_custom_stream_status(config_id), + } + return await run_mapped_websocket_config( + db, + datasource, + debug_max_messages=debug_max_messages, + ) + + return await run_mapped_rest_config(db, datasource) except httpx.HTTPStatusError as exc: raise HTTPException( status_code=exc.response.status_code, @@ -904,36 +986,26 @@ async def run_mapped_datasource( ) from exc except httpx.HTTPError as exc: raise HTTPException(status_code=502, detail=f"Datasource request failed: {exc}") from exc - except (MappingError, ValueError) as exc: + except (CustomDatasourceRuntimeError, MappingError, ValueError) as exc: raise HTTPException(status_code=400, detail=f"Mapping failed: {exc}") from exc - if mapped["failed_count"] > 0: - return { - "status": "failed", - "datasource_config_id": config_id, - "mapping_id": mapping.id, - "mapping_version": mapping.version, - "target_schema": mapping.target_schema, - "mapped_count": mapped["mapped_count"], - "failed_count": mapped["failed_count"], - "errors": mapped["errors"][:20], - } - written_count = await persist_mapped_records( - db, - datasource_name=datasource.name, - datasource_config_id=datasource.id, - target_schema=mapping.target_schema, - records=mapped["records"], - mapping_version=mapping.version, - ) +@router.post("/{config_id}/stop-mapped") +async def stop_mapped_datasource( + config_id: int, + current_user: User = Depends(get_current_user), +): + stopped = await stop_custom_stream(config_id) return { - "status": "success", + "status": "stopped" if stopped else "not_running", "datasource_config_id": config_id, - "mapping_id": mapping.id, - "mapping_version": mapping.version, - "target_schema": mapping.target_schema, - "fetched_count": mapped["total_items"], - "mapped_count": mapped["mapped_count"], - "written_count": written_count, + "stream": get_custom_stream_status(config_id), } + + +@router.get("/{config_id}/stream-status") +async def get_mapped_stream_status( + config_id: int, + current_user: User = Depends(get_current_user), +): + return get_custom_stream_status(config_id) diff --git a/backend/app/api/v1/vessel_aggregation.py b/backend/app/api/v1/vessel_aggregation.py new file mode 100644 index 00000000..0f685f3f --- /dev/null +++ b/backend/app/api/v1/vessel_aggregation.py @@ -0,0 +1,132 @@ +"""v4 strategy + v5 conflict-promotion + enrichment APIs for vessel_ais.""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.security import get_current_user +from app.db.session import get_db +from app.models.user import User +from app.models.vessel import AISConflictRecord +from app.services.vessel_aggregation_strategy import ( + StrategyValidationError, + load_strategy, + reset_strategy, + save_strategy, +) +from app.services.vessel_enrichment import ( + get_vessel_enrichment_bundle, + upsert_vessel_media_enrichment, + upsert_vessel_profile_enrichment, +) + +router = APIRouter() + + +@router.get("/strategy") +async def get_aggregation_strategy(db: AsyncSession = Depends(get_db)): + return await load_strategy(db) + + +@router.put("/strategy") +async def put_aggregation_strategy( + payload: dict[str, Any], + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + try: + return await save_strategy(db, payload) + except StrategyValidationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.delete("/strategy") +async def reset_aggregation_strategy( + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + return await reset_strategy(db) + + +@router.post("/conflicts/{mmsi}/{field}/promote-to-rule") +async def promote_conflict_to_rule( + mmsi: int, + field: str, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Lift the current conflict resolution into a persistent strategy rule.""" + + result = await db.execute( + select(AISConflictRecord) + .where(AISConflictRecord.target_schema == "vessel_ais") + .where(AISConflictRecord.entity_key == str(mmsi)) + .where(AISConflictRecord.field == field) + .order_by(AISConflictRecord.updated_at.desc(), AISConflictRecord.id.desc()) + .limit(1) + ) + record = result.scalar_one_or_none() + if record is None or not record.selected_source: + raise HTTPException(status_code=404, detail="Conflict record with selected_source not found") + + strategy = await load_strategy(db) + vessel_ais = dict(strategy.get("vessel_ais") or {}) + field_rules = dict(vessel_ais.get("field_rules") or {}) + field_rules[field] = {"mode": "source_priority", "source_priority": [record.selected_source]} + vessel_ais["field_rules"] = field_rules + + incoming = {"version": int(strategy.get("version") or 0), "vessel_ais": vessel_ais} + try: + return await save_strategy(db, incoming) + except StrategyValidationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.delete("/conflicts/{mmsi}/{field}/promote-to-rule") +async def revert_conflict_rule( + mmsi: int, + field: str, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + strategy = await load_strategy(db) + vessel_ais = dict(strategy.get("vessel_ais") or {}) + field_rules = dict(vessel_ais.get("field_rules") or {}) + if field in field_rules: + del field_rules[field] + vessel_ais["field_rules"] = field_rules + + incoming = {"version": int(strategy.get("version") or 0), "vessel_ais": vessel_ais} + try: + return await save_strategy(db, incoming) + except StrategyValidationError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.get("/enrichment/{mmsi}") +async def get_vessel_enrichment(mmsi: int, db: AsyncSession = Depends(get_db)): + return await get_vessel_enrichment_bundle(db, mmsi) + + +@router.put("/enrichment/{mmsi}/profile") +async def put_vessel_profile_enrichment( + mmsi: int, + payload: dict[str, Any], + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + return await upsert_vessel_profile_enrichment(db, mmsi=mmsi, payload=payload) + + +@router.put("/enrichment/{mmsi}/media") +async def put_vessel_media_enrichment( + mmsi: int, + payload: dict[str, Any], + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + return await upsert_vessel_media_enrichment(db, mmsi=mmsi, payload=payload) diff --git a/backend/app/api/v1/visualization.py b/backend/app/api/v1/visualization.py index e551be94..11fc0a72 100644 --- a/backend/app/api/v1/visualization.py +++ b/backend/app/api/v1/visualization.py @@ -6,6 +6,7 @@ Returns GeoJSON format compatible with Three.js, CesiumJS, and Unreal Cesium. from datetime import UTC, datetime, timedelta import math +import re import httpx from fastapi import APIRouter, HTTPException, Depends, Query, Response from sqlalchemy.ext.asyncio import AsyncSession @@ -19,14 +20,16 @@ from app.core.time import to_iso8601_utc from app.db.session import get_db from app.models.bgp_anomaly import BGPAnomaly from app.models.bgp_incident import BGPIncident +from app.models.bgp_observation import BGPObservation from app.models.collected_data import CollectedData -from app.models.vessel import VesselPosition, VesselStatic +from app.models.vessel import AISSourceHealth, VesselPosition, VesselStatic from app.services.bgp_collectors import build_bgp_collector_coverage from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS from app.services.persistent_logs import record_system_log from app.services.vessel_ais_aggregation import ( build_field_conflict_candidates, + count_unique_raw_vessel_mmsi, get_aggregated_vessel, get_aggregated_vessel_track, get_aggregated_vessels, @@ -40,6 +43,7 @@ logger = get_logger(__name__, service="api") TERRAIN_TILE_URL_TEMPLATE = ( "https://s3.amazonaws.com/elevation-tiles-prod/terrarium/{z}/{x}/{y}.png" ) +VESSEL_NAME_FALLBACK_PATTERN = re.compile(r"^mmsi\s*\d+$", re.IGNORECASE) # ============== Converter Functions ============== @@ -281,6 +285,120 @@ async def _load_current_collected_data( return list(result.scalars().all()) +async def _latest_task_id_for_source( + db: AsyncSession, + source: str, + *, + exclude_unknown_name: bool = False, +) -> int | None: + stmt = ( + select( + CollectedData.task_id, + func.max(CollectedData.collected_at).label("latest_collected_at"), + func.max(CollectedData.id).label("latest_id"), + ) + .where(CollectedData.source == source) + .where(CollectedData.task_id.isnot(None)) + .group_by(CollectedData.task_id) + .order_by(func.max(CollectedData.collected_at).desc(), func.max(CollectedData.id).desc()) + .limit(1) + ) + if exclude_unknown_name: + stmt = stmt.where(CollectedData.name != "Unknown") + + result = await db.execute(stmt) + row = result.first() + return int(row.task_id) if row and row.task_id is not None else None + + +async def _load_current_or_latest_task_data( + db: AsyncSession, + source: str, + *, + exclude_unknown_name: bool = False, + limit: Optional[int] = None, +) -> List[CollectedData]: + records = await _load_current_collected_data( + db, + source, + exclude_unknown_name=exclude_unknown_name, + limit=limit, + ) + if records: + return records + + latest_task_id = await _latest_task_id_for_source( + db, + source, + exclude_unknown_name=exclude_unknown_name, + ) + if latest_task_id is None: + return [] + + stmt = ( + select(CollectedData) + .where(CollectedData.source == source) + .where(CollectedData.task_id == latest_task_id) + .order_by(CollectedData.id.desc()) + ) + if exclude_unknown_name: + stmt = stmt.where(CollectedData.name != "Unknown") + if limit is not None: + stmt = stmt.limit(limit) + + result = await db.execute(stmt) + return list(result.scalars().all()) + + +async def _count_current_or_latest_task_data( + db: AsyncSession, + source: str, + *, + exclude_unknown_name: bool = False, +) -> int: + current_stmt = ( + select(func.count(CollectedData.id)) + .where(CollectedData.source == source) + .where(CollectedData.is_current.is_(True)) + ) + if exclude_unknown_name: + current_stmt = current_stmt.where(CollectedData.name != "Unknown") + + current_result = await db.execute(current_stmt) + current_scalar = current_result.scalar() + if current_scalar is None and hasattr(current_result, "scalars"): + current_rows = current_result.scalars().all() + current_count = sum( + 1 + for row in current_rows + if getattr(row, "source", None) == source + and (not exclude_unknown_name or getattr(row, "name", None) != "Unknown") + ) + else: + current_count = int(current_scalar or 0) + if current_count > 0: + return current_count + + latest_task_id = await _latest_task_id_for_source( + db, + source, + exclude_unknown_name=exclude_unknown_name, + ) + if latest_task_id is None: + return 0 + + latest_stmt = ( + select(func.count(CollectedData.id)) + .where(CollectedData.source == source) + .where(CollectedData.task_id == latest_task_id) + ) + if exclude_unknown_name: + latest_stmt = latest_stmt.where(CollectedData.name != "Unknown") + + latest_result = await db.execute(latest_stmt) + return int(latest_result.scalar() or 0) + + async def _load_current_collected_data_by_sources( db: AsyncSession, sources: List[str], @@ -636,14 +754,21 @@ VESSEL_TYPE_FILTERS = { def convert_vessels_to_geojson(rows: List[Any]) -> Dict[str, Any]: features = [] + seen_mmsi: set[int] = set() for position, static in rows: if position.lat is None or position.lon is None: continue + if position.mmsi in seen_mmsi: + continue + seen_mmsi.add(position.mmsi) props = { "mmsi": position.mmsi, + "mmsi_display": str(position.mmsi), "name": getattr(static, "name", None) or f"MMSI {position.mmsi}", + "name_is_fallback": _is_vessel_name_fallback(getattr(static, "name", None), position.mmsi), "callsign": getattr(static, "callsign", None), "imo": getattr(static, "imo", None), + "imo_display": str(getattr(static, "imo")) if getattr(static, "imo", None) else None, "vessel_type": getattr(static, "vessel_type", None), "vessel_type_name": getattr(static, "vessel_type_name", None) or "Other", "flag": getattr(static, "flag", None), @@ -685,9 +810,12 @@ def convert_aggregated_vessels_to_geojson(vessels: List[dict[str, Any]]) -> Dict } props = { "mmsi": vessel["mmsi"], + "mmsi_display": str(vessel["mmsi"]), "name": vessel.get("name") or f"MMSI {vessel['mmsi']}", + "name_is_fallback": _is_vessel_name_fallback(vessel.get("name"), vessel["mmsi"]), "callsign": vessel.get("callsign"), "imo": vessel.get("imo"), + "imo_display": str(vessel.get("imo")) if vessel.get("imo") else None, "vessel_type": vessel.get("vessel_type"), "vessel_type_name": vessel.get("vessel_type_name") or "Other", "flag": vessel.get("flag"), @@ -704,6 +832,7 @@ def convert_aggregated_vessels_to_geojson(vessels: List[dict[str, Any]]) -> Dict "source_summary": source_summary, "quality_flags": vessel.get("quality_flags") or [], "conflict_count": vessel.get("conflict_count", 0), + "aggregation_strategy_version": vessel.get("aggregation_strategy_version", 0), "data_type": "vessel", } features.append( @@ -737,6 +866,24 @@ def _parse_bbox(value: Optional[str]) -> tuple[float, float, float, float] | Non return lon_min, lat_min, lon_max, lat_max +def _is_vessel_name_fallback(name: Any, mmsi: Any) -> bool: + text = str(name or "").strip() + mmsi_text = str(mmsi or "").strip() + if not text: + return True + if mmsi_text and text == mmsi_text: + return True + return bool(VESSEL_NAME_FALLBACK_PATTERN.match(text)) + + +def _requested_vessel_types(value: Optional[str]) -> set[str]: + return { + item.strip().lower() + for item in (value or "").split(",") + if item.strip() + } + + def _matches_vessel_type(props: dict[str, Any], requested_types: set[str]) -> bool: if not requested_types: return True @@ -747,6 +894,88 @@ def _matches_vessel_type(props: dict[str, Any], requested_types: set[str]) -> bo return False +def _feature_mmsi_key(feature: dict[str, Any]) -> str | None: + props = feature.get("properties", {}) + mmsi = props.get("mmsi") or feature.get("id") + if mmsi in (None, ""): + return None + return str(mmsi) + + +def _feature_in_bbox(feature: dict[str, Any], bbox: tuple[float, float, float, float] | None) -> bool: + if bbox is None: + return True + coordinates = feature.get("geometry", {}).get("coordinates") or [] + if len(coordinates) < 2: + return False + try: + lon = float(coordinates[0]) + lat = float(coordinates[1]) + except (TypeError, ValueError): + return False + lon_min, lat_min, lon_max, lat_max = bbox + return lon_min <= lon <= lon_max and lat_min <= lat <= lat_max + + +def _filter_vessel_features( + features: list[dict[str, Any]], + *, + bbox: tuple[float, float, float, float] | None, + requested_types: set[str], +) -> list[dict[str, Any]]: + return [ + feature + for feature in features + if _feature_in_bbox(feature, bbox) + and _matches_vessel_type(feature.get("properties", {}), requested_types) + ] + + +def _merge_vessel_features( + raw_features: list[dict[str, Any]], + legacy_features: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + """Prefer aggregated raw observations as the canonical source of truth. + + Legacy `vessel_position` rows only fill MMSIs that the unified pipeline does + not yet know about, so a vessel never appears twice when both BarentsWatch + and AISStream observe it. Once the legacy table drains, this branch becomes + a no-op. + """ + + merged: list[dict[str, Any]] = [] + seen: set[str] = set() + raw_keys: set[str] = set() + legacy_keys: set[str] = set() + + for feature in raw_features: + key = _feature_mmsi_key(feature) + if key is None or key in seen: + continue + seen.add(key) + raw_keys.add(key) + merged.append(feature) + + legacy_added = 0 + for feature in legacy_features: + key = _feature_mmsi_key(feature) + if key is None: + continue + legacy_keys.add(key) + if key in seen: + continue + seen.add(key) + legacy_added += 1 + merged.append(feature) + + return merged, { + "raw_unique_mmsi": len(raw_keys), + "legacy_unique_mmsi": len(legacy_keys), + "legacy_backfilled_mmsi": legacy_added, + "final_unique_mmsi": len(seen), + } + + def _build_vessel_stats(features: List[dict[str, Any]]) -> dict[str, Any]: by_type: dict[str, int] = {} underway = 0 @@ -1347,7 +1576,7 @@ async def get_satellites_geojson( db: AsyncSession = Depends(get_db), ): """获取卫星 TLE GeoJSON 数据""" - records = await _load_current_collected_data( + records = await _load_current_or_latest_task_data( db, "celestrak_tle", exclude_unknown_name=True, @@ -1476,27 +1705,30 @@ async def get_vessels_geojson( ): """Return latest vessel positions as GeoJSON points.""" parsed_bbox = _parse_bbox(bbox) - aggregated_vessels = await get_aggregated_vessels(db, bbox=parsed_bbox, limit=limit) - if aggregated_vessels: - geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels) - requested_types = { - item.strip().lower() - for item in (type or "").split(",") - if item.strip() - } - if requested_types: - geojson["features"] = [ - feature - for feature in geojson.get("features", []) - if _matches_vessel_type(feature.get("properties", {}), requested_types) - ] + requested_types = _requested_vessel_types(type) + merged_features, diagnostics = await _load_merged_vessel_features(db) + features = _filter_vessel_features( + merged_features, + bbox=parsed_bbox, + requested_types=requested_types, + ) + if limit and limit > 0: + features = features[:limit] + return { + "type": "FeatureCollection", + "features": features, + "count": len(features), + "stats": _build_vessel_stats(features), + "diagnostics": { + **diagnostics, + "filtered_count": len(features), + }, + } - features = geojson.get("features", []) - return { - **geojson, - "count": len(features), - "stats": _build_vessel_stats(features), - } + +async def _load_merged_vessel_features(db: AsyncSession) -> tuple[list[dict[str, Any]], dict[str, Any]]: + aggregated_vessels = await get_aggregated_vessels(db) + raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels) latest_times = ( select( @@ -1516,50 +1748,122 @@ async def get_vessels_geojson( .outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi) .order_by(VesselPosition.received_at.desc()) ) - if limit and limit > 0: - stmt = stmt.limit(limit) - - if parsed_bbox is not None: - lon_min, lat_min, lon_max, lat_max = parsed_bbox - stmt = stmt.where( - VesselPosition.lon >= lon_min, - VesselPosition.lon <= lon_max, - VesselPosition.lat >= lat_min, - VesselPosition.lat <= lat_max, - ) result = await db.execute(stmt) rows = list(result.all()) - geojson = convert_vessels_to_geojson(rows) - requested_types = { - item.strip().lower() - for item in (type or "").split(",") - if item.strip() + legacy_geojson = convert_vessels_to_geojson(rows) + merged_features, diagnostics = _merge_vessel_features( + raw_geojson.get("features", []), + legacy_geojson.get("features", []), + ) + return merged_features, { + **diagnostics, + "raw_feature_count": len(raw_geojson.get("features", [])), + "legacy_feature_count": len(legacy_geojson.get("features", [])), } - if requested_types: - geojson["features"] = [ - feature - for feature in geojson.get("features", []) - if _matches_vessel_type(feature.get("properties", {}), requested_types) - ] - features = geojson.get("features", []) + +@router.get("/vessels/custom-supplements") +async def get_vessel_custom_supplements(db: AsyncSession = Depends(get_db)): + """Group custom vessel_ais sources by their declared merge target for diagnostics.""" + + from app.models.datasource_config import DataSourceConfig + + result = await db.execute( + select(DataSourceConfig.name, DataSourceConfig.config, DataSourceConfig.is_active) + .where(DataSourceConfig.config["target_schema"].as_string() == "vessel_ais") + ) + grouped: dict[str, dict[str, Any]] = {} + for name, config, is_active in result.all(): + config = config or {} + merge_target = str(config.get("merge_target_source") or "barentswatch_vessels") + bucket = grouped.setdefault(merge_target, {"merge_target": merge_target, "sources": []}) + bucket["sources"].append({"name": name, "is_active": bool(is_active)}) + return {"groups": list(grouped.values())} + + +@router.get("/vessels/name-fallbacks") +async def get_vessel_name_fallbacks( + limit: int = Query(500, ge=0, description="Maximum fallback-name vessels to return. 0 means no limit."), + db: AsyncSession = Depends(get_db), +): + """Return vessels whose display name still falls back to MMSI.""" + aggregated_vessels = await get_aggregated_vessels(db) + raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels) + + latest_times = ( + select( + VesselPosition.mmsi.label("mmsi"), + func.max(VesselPosition.received_at).label("received_at"), + ) + .group_by(VesselPosition.mmsi) + .subquery() + ) + result = await db.execute( + select(VesselPosition, VesselStatic) + .join( + latest_times, + (VesselPosition.mmsi == latest_times.c.mmsi) + & (VesselPosition.received_at == latest_times.c.received_at), + ) + .outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi) + .order_by(VesselPosition.received_at.desc()) + ) + legacy_geojson = convert_vessels_to_geojson(list(result.all())) + features, diagnostics = _merge_vessel_features( + raw_geojson.get("features", []), + legacy_geojson.get("features", []), + ) + + fallback_items = [] + for feature in features: + props = feature.get("properties", {}) + mmsi = props.get("mmsi") + name = props.get("name") + if not _is_vessel_name_fallback(name, mmsi): + continue + source_summary = props.get("source_summary") or {} + fallback_items.append( + { + "mmsi": str(mmsi), + "display_name": name or f"MMSI {mmsi}", + "reason": "missing_real_name", + "received_at": props.get("received_at"), + "sources": sorted(source_summary.keys()), + "source_summary": source_summary, + "message_types": sorted( + { + message_type + for summary in source_summary.values() + for message_type in (summary.get("message_types") or []) + } + ), + "field_sources": props.get("field_sources") or {}, + } + ) + + if limit and limit > 0: + fallback_items = fallback_items[:limit] return { - **geojson, - "count": len(features), - "stats": _build_vessel_stats(features), + "count": len(fallback_items), + "items": fallback_items, + "diagnostics": diagnostics, } @router.get("/vessels/{mmsi}") async def get_vessel_detail(mmsi: int, db: AsyncSession = Depends(get_db)): + from app.services.vessel_enrichment import get_vessel_enrichment_bundle + aggregated = await get_aggregated_vessel(db, mmsi) + enrichment = await get_vessel_enrichment_bundle(db, mmsi) if aggregated is not None: return { **aggregated, "received_at": to_iso8601_utc(aggregated.get("received_at")), "latitude": aggregated["lat"], "longitude": aggregated["lon"], + "enrichment": enrichment, } latest_position_stmt = ( @@ -1578,6 +1882,7 @@ async def get_vessel_detail(mmsi: int, db: AsyncSession = Depends(get_db)): **(geojson["features"][0]["properties"]), "latitude": position.lat, "longitude": position.lon, + "enrichment": enrichment, } @@ -1737,31 +2042,16 @@ async def get_bgp_collectors_geojson(db: AsyncSession = Depends(get_db)): @router.get("/geo/summary") async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)): """Return lightweight Earth HUD counts without loading layer GeoJSON payloads.""" - records_by_source = await _load_current_collected_data_by_sources( + cable_count = await _count_current_or_latest_task_data(db, "arcgis_cables") + landing_point_count = await _count_current_or_latest_task_data(db, "arcgis_landing_points") + satellite_count = await _count_current_or_latest_task_data( db, - [ - "arcgis_cables", - "arcgis_landing_points", - "celestrak_tle", - "top500", - "epoch_ai_gpu", - ], + "celestrak_tle", + exclude_unknown_name=True, ) - - cables = convert_cable_to_geojson(records_by_source.get("arcgis_cables", [])) - landing_points = convert_landing_point_to_geojson( - records_by_source.get("arcgis_landing_points", []), - ) - satellites = convert_satellite_to_geojson( - _filter_known_records(records_by_source.get("celestrak_tle", [])), - ) - compute_centers = convert_compute_centers_to_geojson( - _filter_known_records( - records_by_source.get("top500", []) - + records_by_source.get("epoch_ai_gpu", []), - ), - ) - compute_features = compute_centers.get("features", []) + supercomputer_count = await _count_current_or_latest_task_data(db, "top500") + gpu_cluster_count = await _count_current_or_latest_task_data(db, "epoch_ai_gpu") + compute_center_count = supercomputer_count + gpu_cluster_count active_incident_result = await db.execute( select(func.count(BGPIncident.id)).where(BGPIncident.status == "active"), @@ -1771,35 +2061,56 @@ async def get_visualization_geo_summary(db: AsyncSession = Depends(get_db)): ) active_incident_count = int(active_incident_result.scalar() or 0) active_anomaly_count = int(active_anomaly_result.scalar() or 0) - bgp_collectors = await build_bgp_collector_coverage( + bgp_collector_result = await db.execute( + select(func.count(func.distinct(BGPObservation.collector))) + .where(BGPObservation.collector.isnot(None)) + .where(func.length(func.btrim(BGPObservation.collector)) > 0) + .where(BGPObservation.source.in_(("ris_live_bgp", "bgpstream_bgp"))) + ) + bgp_collector_scalar = bgp_collector_result.scalar() + if bgp_collector_scalar is None: + bgp_collectors = await build_bgp_collector_coverage( + db, + source_filter=("ris_live_bgp", "bgpstream_bgp"), + ) + bgp_collector_count = len( + [item for item in bgp_collectors if item.get("collector")] + ) + else: + bgp_collector_count = int(bgp_collector_scalar or 0) + raw_unique_window_hours = 24 + raw_unique_mmsi = await count_unique_raw_vessel_mmsi( db, - source_filter=("ris_live_bgp", "bgpstream_bgp"), + observed_since=datetime.now(UTC) - timedelta(hours=raw_unique_window_hours), ) - vessel_count_result = await db.execute( - select(func.count(func.distinct(VesselPosition.mmsi))), + legacy_unique_result = await db.execute( + select(func.count(func.distinct(VesselPosition.mmsi))) ) - vessel_count = int(vessel_count_result.scalar() or 0) + legacy_unique_mmsi = int(legacy_unique_result.scalar() or 0) + vessel_count = max(raw_unique_mmsi, legacy_unique_mmsi) + aisstream_health = await db.get(AISSourceHealth, "aisstream_vessels") return { "generated_at": to_iso8601_utc(datetime.now(UTC)), "stats": { - "cable_count": len(cables.get("features", [])), - "landing_point_count": len(landing_points.get("features", [])), - "satellite_count": len(satellites.get("features", [])), - "compute_center_count": len(compute_features), + "cable_count": cable_count, + "landing_point_count": landing_point_count, + "satellite_count": satellite_count, + "compute_center_count": compute_center_count, "vessel_count": vessel_count, - "supercomputer_count": sum( - 1 for feature in compute_features - if feature.get("properties", {}).get("site_type") == "supercomputer" - ), - "gpu_cluster_count": sum( - 1 for feature in compute_features - if feature.get("properties", {}).get("site_type") == "gpu_cluster" - ), + "vessel_raw_unique_mmsi": raw_unique_mmsi, + "vessel_raw_unique_window_hours": raw_unique_window_hours, + "vessel_legacy_unique_mmsi": legacy_unique_mmsi, + "aisstream_connection_state": aisstream_health.connection_state if aisstream_health else None, + "aisstream_last_seen_at": to_iso8601_utc(aisstream_health.last_seen_at) if aisstream_health else None, + "aisstream_message_rate": aisstream_health.message_rate if aisstream_health else None, + "aisstream_lag_seconds": aisstream_health.lag_seconds if aisstream_health else None, + "supercomputer_count": supercomputer_count, + "gpu_cluster_count": gpu_cluster_count, "bgp_event_count": active_incident_count or active_anomaly_count, "bgp_incident_count": active_incident_count, "bgp_anomaly_count": active_anomaly_count, - "bgp_collector_count": len([item for item in bgp_collectors if item.get("collector")]), + "bgp_collector_count": bgp_collector_count, }, } diff --git a/backend/app/api/v1/websocket.py b/backend/app/api/v1/websocket.py index cde409f0..b4cbc903 100644 --- a/backend/app/api/v1/websocket.py +++ b/backend/app/api/v1/websocket.py @@ -40,16 +40,16 @@ async def authenticate_token(token: str) -> Optional[dict]: @router.websocket("/ws") async def websocket_endpoint( websocket: WebSocket, - token: str = Query(...), + token: str | None = Query(None), ): """WebSocket endpoint for real-time data""" logger.info_event( "WebSocket connection attempt", event="auth.websocket.connection_attempt", - context={"token_preview": f"{token[:8]}..."}, + context={"token_preview": f"{token[:8]}..." if token else "anonymous"}, ) - payload = await authenticate_token(token) - if payload is None: + payload = await authenticate_token(token) if token else None + if token and payload is None: logger.warning_event( "WebSocket authentication failed, closing connection", event="auth.websocket.connection_rejected", @@ -57,7 +57,17 @@ async def websocket_endpoint( await websocket.close(code=4001) return - user_id = str(payload.get("sub")) + is_anonymous = payload is None + user_id = str(payload.get("sub")) if payload else f"anonymous:{id(websocket)}" + supported_channels = ["vessels"] if is_anonymous else [ + "gpu_clusters", + "submarine_cables", + "ixp_nodes", + "alerts", + "dashboard", + "datasource_tasks", + "vessels", + ] await manager.connect(websocket, user_id) try: @@ -68,14 +78,7 @@ async def websocket_endpoint( "connection_id": f"conn_{user_id}", "server_version": settings.VERSION, "heartbeat_interval": 30, - "supported_channels": [ - "gpu_clusters", - "submarine_cables", - "ixp_nodes", - "alerts", - "dashboard", - "datasource_tasks", - ], + "supported_channels": supported_channels, }, } ) @@ -93,12 +96,24 @@ async def websocket_endpoint( ) elif data.get("type") == "subscribe": channels = data.get("data", {}).get("channels", []) + if is_anonymous: + channels = [channel for channel in channels if channel in supported_channels] + manager.subscribe(websocket, channels) await websocket.send_json( { "type": "subscription_confirmed", "data": {"action": "subscribe", "channels": channels}, } ) + elif data.get("type") == "unsubscribe": + channels = data.get("data", {}).get("channels", []) + manager.unsubscribe(websocket, channels) + await websocket.send_json( + { + "type": "subscription_confirmed", + "data": {"action": "unsubscribe", "channels": channels}, + } + ) elif data.get("type") == "control_frame": await websocket.send_json( {"type": "control_acknowledged", "data": {"received": True}} diff --git a/backend/app/core/target_schema_registry.py b/backend/app/core/target_schema_registry.py index 0b2aba7e..61ade066 100644 --- a/backend/app/core/target_schema_registry.py +++ b/backend/app/core/target_schema_registry.py @@ -16,8 +16,11 @@ class VesselAISRecord(BaseModel): sog: float | None = None cog: float | None = Field(default=None, ge=0, le=360) heading: int | None = Field(default=None, ge=0, le=511) + nav_status: int | None = None name: str | None = None + callsign: str | None = None vessel_type: str | int | None = None + vessel_type_name: str | None = None received_at: datetime | None = None @@ -104,8 +107,11 @@ TARGET_SCHEMAS: dict[str, TargetSchema] = { TargetField("sog", "float", False, "对地航速，单位节", 12.4), TargetField("cog", "float", False, "对地航向，0-360 度", 184.5), TargetField("heading", "integer", False, "船首向，0-511", 186), + TargetField("nav_status", "integer", False, "导航状态码", 0), TargetField("name", "string", False, "船名", "OSLO EXPRESS"), - TargetField("vessel_type", "string", False, "船型", "cargo"), + TargetField("callsign", "string", False, "呼号", "LAAB"), + TargetField("vessel_type", "string", False, "船型代码", 70), + TargetField("vessel_type_name", "string", False, "船型名称", "Cargo"), TargetField("received_at", "datetime", False, "数据接收时间", "2026-04-28T00:00:00Z"), ), ), diff --git a/backend/app/core/websocket/broadcaster.py b/backend/app/core/websocket/broadcaster.py index 0fbc2523..563c23de 100644 --- a/backend/app/core/websocket/broadcaster.py +++ b/backend/app/core/websocket/broadcaster.py @@ -75,7 +75,7 @@ class DataBroadcaster: "timestamp": to_iso8601_utc(datetime.now(UTC)), "payload": data, }, - channel=channel if channel in manager.active_connections else "all", + channel=channel, ) async def broadcast_datasource_task_update(self, data: Dict[str, Any]): diff --git a/backend/app/core/websocket/manager.py b/backend/app/core/websocket/manager.py index a4994119..57bad1ad 100644 --- a/backend/app/core/websocket/manager.py +++ b/backend/app/core/websocket/manager.py @@ -1,9 +1,6 @@ """WebSocket Connection Manager""" -import json -import asyncio from typing import Dict, Set, Optional -from datetime import datetime from fastapi import WebSocket import redis.asyncio as redis @@ -15,6 +12,8 @@ class ConnectionManager: def __init__(self): self.active_connections: Dict[str, Set[WebSocket]] = {} # user_id -> connections + self.channel_subscriptions: Dict[str, Set[WebSocket]] = {} + self.websocket_channels: Dict[WebSocket, Set[str]] = {} self.redis_client: Optional[redis.Redis] = None async def connect(self, websocket: WebSocket, user_id: str): @@ -40,6 +39,39 @@ class ConnectionManager: self.active_connections[user_id].discard(websocket) if not self.active_connections[user_id]: del self.active_connections[user_id] + self.unsubscribe_all(websocket) + + def subscribe(self, websocket: WebSocket, channels: list[str]): + normalized_channels = { + str(channel).strip() + for channel in channels + if str(channel).strip() + } + if not normalized_channels: + return + + socket_channels = self.websocket_channels.setdefault(websocket, set()) + for channel in normalized_channels: + self.channel_subscriptions.setdefault(channel, set()).add(websocket) + socket_channels.add(channel) + + def unsubscribe(self, websocket: WebSocket, channels: list[str]): + for channel in {str(channel).strip() for channel in channels if str(channel).strip()}: + subscribers = self.channel_subscriptions.get(channel) + if subscribers is not None: + subscribers.discard(websocket) + if not subscribers: + del self.channel_subscriptions[channel] + socket_channels = self.websocket_channels.get(websocket) + if socket_channels is not None: + socket_channels.discard(channel) + if not socket_channels: + del self.websocket_channels[websocket] + + def unsubscribe_all(self, websocket: WebSocket): + channels = list(self.websocket_channels.get(websocket, set())) + if channels: + self.unsubscribe(websocket, channels) async def send_personal_message(self, message: dict, user_id: str): if user_id in self.active_connections: @@ -54,13 +86,19 @@ class ConnectionManager: for user_id in self.active_connections: await self.send_personal_message(message, user_id) else: - await self.send_personal_message(message, channel) + for connection in list(self.channel_subscriptions.get(channel, set())): + try: + await connection.send_json(message) + except Exception: + self.unsubscribe_all(connection) async def close_all(self): for user_id in self.active_connections: for connection in self.active_connections[user_id]: await connection.close() self.active_connections.clear() + self.channel_subscriptions.clear() + self.websocket_channels.clear() manager = ConnectionManager() diff --git a/backend/app/db/session.py b/backend/app/db/session.py index 33f8f52a..2483f803 100644 --- a/backend/app/db/session.py +++ b/backend/app/db/session.py @@ -111,6 +111,7 @@ async def init_db(): import app.models.playground_message # noqa: F401 import app.models.system_log # noqa: F401 import app.models.vessel # noqa: F401 + import app.models.vessel_enrichment # noqa: F401 import app.models.datasource_mapping # noqa: F401 logger.warning_event( @@ -163,6 +164,30 @@ async def init_db(): """ ) ) + await conn.execute( + text( + """ + CREATE INDEX IF NOT EXISTS idx_collected_data_source_current_id + ON collected_data (source, is_current, id) + """ + ) + ) + await conn.execute( + text( + """ + CREATE INDEX IF NOT EXISTS idx_collected_data_source_task_id + ON collected_data (source, task_id, id) + """ + ) + ) + await conn.execute( + text( + """ + CREATE INDEX IF NOT EXISTS idx_ais_raw_schema_observed_entity + ON ais_raw_observations (target_schema, observed_at, entity_key) + """ + ) + ) await conn.execute( text( """ diff --git a/backend/app/models/collected_data.py b/backend/app/models/collected_data.py index 438db389..5bb8f195 100644 --- a/backend/app/models/collected_data.py +++ b/backend/app/models/collected_data.py @@ -48,6 +48,8 @@ class CollectedData(Base): # Indexes for common queries __table_args__ = ( Index("idx_collected_data_source_collected", "source", "collected_at"), + Index("idx_collected_data_source_current_id", "source", "is_current", "id"), + Index("idx_collected_data_source_task_id", "source", "task_id", "id"), Index("idx_collected_data_source_type", "source", "data_type"), Index("idx_collected_data_source_source_id", "source", "source_id"), ) diff --git a/backend/app/models/vessel.py b/backend/app/models/vessel.py index c8436c1e..35e93b1c 100644 --- a/backend/app/models/vessel.py +++ b/backend/app/models/vessel.py @@ -97,6 +97,7 @@ class AISRawObservation(Base): __table_args__ = ( Index("idx_ais_raw_entity_observed", "target_schema", "entity_key", "observed_at"), + Index("idx_ais_raw_schema_observed_entity", "target_schema", "observed_at", "entity_key"), Index("idx_ais_raw_source_entity", "source", "entity_key"), ) diff --git a/backend/app/models/vessel_enrichment.py b/backend/app/models/vessel_enrichment.py new file mode 100644 index 00000000..fc73208d --- /dev/null +++ b/backend/app/models/vessel_enrichment.py @@ -0,0 +1,63 @@ +"""Vessel enrichment cache tables (v5). + +Profile and media enrichment are stored separately so cache TTLs can differ +and so the conflict-resolution + display layers can read either independently. +""" + +from sqlalchemy import BigInteger, Column, DateTime, Float, JSON, String +from sqlalchemy.sql import func + +from app.core.time import to_iso8601_utc +from app.db.session import Base + + +class VesselProfileEnrichment(Base): + """Cached static vessel profile (type, flag, dimensions, operator, etc.).""" + + __tablename__ = "vessel_profile_enrichment" + + mmsi = Column(BigInteger, primary_key=True) + source = Column(String(100), nullable=False, default="system") + payload = Column(JSON, nullable=False, default=dict) + fetched_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now()) + expires_at = Column(DateTime(timezone=True), nullable=True) + confidence = Column(Float, nullable=True) + reference_url = Column(String(500), nullable=True) + updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()) + + def to_dict(self) -> dict: + return { + "mmsi": self.mmsi, + "source": self.source, + "payload": self.payload or {}, + "fetched_at": to_iso8601_utc(self.fetched_at), + "expires_at": to_iso8601_utc(self.expires_at), + "confidence": self.confidence, + "reference_url": self.reference_url, + } + + +class VesselMediaEnrichment(Base): + """Cached vessel imagery / external detail references.""" + + __tablename__ = "vessel_media_enrichment" + + mmsi = Column(BigInteger, primary_key=True) + source = Column(String(100), nullable=False, default="system") + payload = Column(JSON, nullable=False, default=dict) + fetched_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now()) + expires_at = Column(DateTime(timezone=True), nullable=True) + confidence = Column(Float, nullable=True) + reference_url = Column(String(500), nullable=True) + updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()) + + def to_dict(self) -> dict: + return { + "mmsi": self.mmsi, + "source": self.source, + "payload": self.payload or {}, + "fetched_at": to_iso8601_utc(self.fetched_at), + "expires_at": to_iso8601_utc(self.expires_at), + "confidence": self.confidence, + "reference_url": self.reference_url, + } diff --git a/backend/app/services/collectors/aisstream.py b/backend/app/services/collectors/aisstream.py index 1c1ff272..3bd24362 100644 --- a/backend/app/services/collectors/aisstream.py +++ b/backend/app/services/collectors/aisstream.py @@ -10,7 +10,10 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.core.data_sources import get_data_sources_config +from app.core.time import to_iso8601_utc +from app.core.websocket.broadcaster import broadcaster from app.models.datasource_config import DataSourceConfig +from app.models.task import CollectionTask from app.services.collectors.base import BaseCollector from app.services.vessel_ais_aggregation import ( AISSTREAM_DELIVERY_MODE, @@ -66,9 +69,20 @@ class AISStreamCollector(BaseCollector): "bounding_boxes": config.get("bounding_boxes") or DEFAULT_BOUNDING_BOXES, "message_types": config.get("message_types") or DEFAULT_MESSAGE_TYPES, "max_messages": int(config.get("max_messages") or 500), + "streaming_enabled": config.get("streaming_enabled", True) is not False, + "streaming_commit_interval": int(config.get("streaming_commit_interval") or 1), + "streaming_max_messages": int(config.get("streaming_max_messages") or 0), + "reconnect_delay_seconds": float(config.get("reconnect_delay_seconds") or 5), "receive_timeout_seconds": float(config.get("receive_timeout_seconds") or 30), } + def _build_subscription(self, config: dict[str, Any]) -> dict[str, Any]: + return { + "APIKey": config["api_key"], + "BoundingBoxes": config["bounding_boxes"], + "FilterMessageTypes": config["message_types"], + } + async def fetch(self) -> list[dict[str, Any]]: config = await self._get_effective_config() if not config["api_key"]: @@ -79,11 +93,7 @@ class AISStreamCollector(BaseCollector): except ImportError as exc: raise RuntimeError("Python package 'websockets' is required for AISStream") from exc - subscription = { - "APIKey": config["api_key"], - "BoundingBoxes": config["bounding_boxes"], - "FilterMessageTypes": config["message_types"], - } + subscription = self._build_subscription(config) messages: list[dict[str, Any]] = [] try: @@ -113,6 +123,157 @@ class AISStreamCollector(BaseCollector): return messages + async def run(self, db: AsyncSession) -> dict[str, Any]: + """Run AISStream as a long-lived streaming collector by default.""" + config = await self._get_effective_config() + if not config.get("streaming_enabled", True): + return await super().run(db) + if not config["api_key"]: + return {"status": "failed", "error": "AISStream API key is not configured"} + + from app.services.collectors.registry import collector_registry + + if not collector_registry.is_active(self.name): + return {"status": "skipped", "reason": "Collector is disabled"} + + try: + import websockets + except ImportError as exc: + return {"status": "failed", "error": "Python package 'websockets' is required for AISStream"} + + start_time = datetime.now(UTC) + task = CollectionTask( + datasource_id=getattr(self, "_datasource_id", 1), + status="running", + phase="connecting", + phase_message="正在连接 AISStream 实时流", + phase_unit="messages", + started_at=start_time, + ) + db.add(task) + await db.commit() + self._current_task = task + self._db_session = db + self._last_broadcast_progress = None + await self.resolve_url(db) + await self._publish_task_update(force=True) + + records_added = 0 + messages_seen = 0 + unique_mmsi: set[str] = set() + reconnect_delay = config["reconnect_delay_seconds"] + + try: + while True: + config = await self._get_effective_config() + subscription = self._build_subscription(config) + try: + await update_ais_source_health( + db, + source=self.name, + connection_state="connecting", + ) + await self.set_phase("connecting", message="正在连接 AISStream 实时流") + await db.commit() + + async with websockets.connect(config["endpoint"]) as websocket: + await websocket.send(json.dumps(subscription)) + await update_ais_source_health( + db, + source=self.name, + connection_state="connected", + last_success_at=datetime.now(UTC), + ) + await self.set_phase( + "streaming", + message="正在接收 AISStream 实时消息", + reset_progress=False, + ) + await db.commit() + + while True: + try: + raw_message = await asyncio.wait_for( + websocket.recv(), + timeout=config["receive_timeout_seconds"], + ) + except TimeoutError: + await update_ais_source_health( + db, + source=self.name, + connection_state="connected", + last_success_at=datetime.now(UTC), + ) + await db.commit() + continue + + payload = json.loads(raw_message) + if not isinstance(payload, dict): + continue + messages_seen += 1 + record = self._normalize_message(payload) + if not record: + continue + unique_mmsi.add(str(record["mmsi"])) + created = await self._save_stream_record(db, record) + if created: + records_added += 1 + + task.records_processed = messages_seen + task.total_records = None + task.progress = None + task.phase = "streaming" + task.phase_message = "正在接收 AISStream 实时消息" + task.phase_current = messages_seen + task.phase_total = None + task.phase_unit = "messages" + await self._publish_task_update(force=True) + + if config["streaming_max_messages"] and messages_seen >= config["streaming_max_messages"]: + task.status = "success" + task.phase = "stopped" + task.phase_message = "AISStream 测试流已停止" + task.completed_at = datetime.now(UTC) + await db.commit() + await self._publish_task_update(force=True) + return { + "status": "success", + "task_id": task.id, + "records_processed": records_added, + "messages_seen": messages_seen, + "unique_mmsi": len(unique_mmsi), + "execution_time_seconds": (datetime.now(UTC) - start_time).total_seconds(), + } + except asyncio.CancelledError: + raise + except Exception as exc: + await update_ais_source_health( + db, + source=self.name, + connection_state="reconnecting", + last_error=f"{exc.__class__.__name__}: {exc}", + ) + task.phase = "reconnecting" + task.phase_message = "AISStream 连接中断，正在重连" + task.error_message = f"{exc.__class__.__name__}: {exc}" + await db.commit() + await self._publish_task_update(force=True) + await asyncio.sleep(reconnect_delay) + except asyncio.CancelledError: + task.status = "cancelled" + task.phase = "stopped" + task.phase_message = "AISStream 实时流已停止" + task.completed_at = datetime.now(UTC) + await update_ais_source_health( + db, + source=self.name, + connection_state="disconnected", + last_error=None, + ) + await db.commit() + await self._publish_task_update(force=True) + raise + def transform(self, raw_data: list[dict[str, Any]]) -> list[dict[str, Any]]: records = [] for item in raw_data: @@ -165,6 +326,60 @@ class AISStreamCollector(BaseCollector): await self.update_progress(records_added, force=True) return records_added + async def _save_stream_record(self, db: AsyncSession, item: dict[str, Any]) -> bool: + now = datetime.now(UTC) + observed_at = item.get("received_at") or now + observation = await record_vessel_ais_observation( + db, + source=self.name, + normalized_payload=item, + raw_payload=item.get("_raw_payload") or item, + delivery_mode=AISSTREAM_DELIVERY_MODE, + transport=AISSTREAM_TRANSPORT, + message_type=item.get("_message_type") or "PositionReport", + source_message_id=item.get("_source_message_id"), + observed_at=observed_at, + collected_at=now, + ) + await update_ais_source_health( + db, + source=self.name, + connection_state="connected", + observed_count=1, + last_seen_at=observed_at if isinstance(observed_at, datetime) else now, + last_success_at=now, + lag_seconds=max((now - observed_at).total_seconds(), 0) if isinstance(observed_at, datetime) else None, + ) + await db.commit() + await self._broadcast_vessel_delta(item, created=observation is not None) + return observation is not None + + async def _broadcast_vessel_delta(self, item: dict[str, Any], *, created: bool) -> None: + await broadcaster.broadcast_custom( + "vessels", + { + "action": "upsert", + "source": self.name, + "created": created, + "vessels": [ + { + "mmsi": item.get("mmsi"), + "mmsi_display": str(item.get("mmsi")) if item.get("mmsi") is not None else None, + "name": item.get("name"), + "lat": item.get("lat"), + "lon": item.get("lon"), + "sog": item.get("sog"), + "cog": item.get("cog"), + "heading": item.get("heading"), + "nav_status": item.get("nav_status"), + "vessel_type": item.get("vessel_type"), + "vessel_type_name": item.get("vessel_type_name"), + "received_at": to_iso8601_utc(item.get("received_at")), + } + ], + }, + ) + def _normalize_message(self, item: dict[str, Any]) -> dict[str, Any] | None: message_type = str(item.get("MessageType") or item.get("message_type") or "") metadata = item.get("MetaData") if isinstance(item.get("MetaData"), dict) else {} diff --git a/backend/app/services/collectors/vessel_ais.py b/backend/app/services/collectors/vessel_ais.py index 95e4e383..810b91c2 100644 --- a/backend/app/services/collectors/vessel_ais.py +++ b/backend/app/services/collectors/vessel_ais.py @@ -1,13 +1,13 @@ """BarentsWatch AIS collector for vessel tracking.""" -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime from typing import Any import httpx -from sqlalchemy import delete from sqlalchemy.ext.asyncio import AsyncSession -from app.models.vessel import VesselPosition, VesselStatic +from app.core.time import to_iso8601_utc +from app.core.websocket.broadcaster import broadcaster from app.services.barentswatch import ( BARENTSWATCH_LATEST_URL, fetch_barentswatch_access_token, @@ -101,40 +101,6 @@ class VesselAISCollector(BaseCollector): observed_at=observed_at, collected_at=now, ) - - static = await db.get(VesselStatic, item["mmsi"]) - if static is None: - static = VesselStatic(mmsi=item["mmsi"]) - db.add(static) - - for field in ( - "name", - "callsign", - "vessel_type", - "vessel_type_name", - "flag", - "length", - "width", - "draught", - "imo", - ): - value = item.get(field) - if value not in (None, ""): - setattr(static, field, value) - static.updated_at = now - - db.add( - VesselPosition( - mmsi=item["mmsi"], - lat=item["lat"], - lon=item["lon"], - sog=item.get("sog"), - cog=item.get("cog"), - heading=item.get("heading"), - nav_status=item.get("nav_status"), - received_at=observed_at, - ) - ) records_added += 1 if (index + 1) % 1000 == 0: @@ -153,13 +119,46 @@ class VesselAISCollector(BaseCollector): last_success_at=now if data else None, lag_seconds=max((now - latest_observed_at).total_seconds(), 0), ) - await db.execute( - delete(VesselPosition).where(VesselPosition.received_at < now - timedelta(hours=24)) - ) await db.commit() + await self._broadcast_vessel_snapshot(data) await self.update_progress(records_added, force=True) return records_added + async def _broadcast_vessel_snapshot(self, data: list[dict[str, Any]]) -> None: + """Push REST collector updates through the same realtime vessel channel.""" + if not data: + return + + batch_size = 500 + for offset in range(0, len(data), batch_size): + batch = data[offset : offset + batch_size] + await broadcaster.broadcast_custom( + "vessels", + { + "action": "upsert", + "source": self.name, + "created": True, + "vessels": [ + { + "mmsi": item.get("mmsi"), + "mmsi_display": str(item.get("mmsi")) if item.get("mmsi") is not None else None, + "name": item.get("name"), + "callsign": item.get("callsign"), + "lat": item.get("lat"), + "lon": item.get("lon"), + "sog": item.get("sog"), + "cog": item.get("cog"), + "heading": item.get("heading"), + "nav_status": item.get("nav_status"), + "vessel_type": item.get("vessel_type"), + "vessel_type_name": item.get("vessel_type_name"), + "received_at": to_iso8601_utc(item.get("received_at")), + } + for item in batch + ], + }, + ) + def _normalize_record(self, item: dict[str, Any]) -> dict[str, Any] | None: mmsi = _as_int(_pick(item, "mmsi", "MMSI", "Mmsi")) lat = _as_float(_pick(item, "lat", "latitude", "Latitude")) diff --git a/backend/app/services/custom_datasource_runtime.py b/backend/app/services/custom_datasource_runtime.py new file mode 100644 index 00000000..ebf41225 --- /dev/null +++ b/backend/app/services/custom_datasource_runtime.py @@ -0,0 +1,391 @@ +"""Runtime helpers for mapped custom data sources.""" + +from __future__ import annotations + +import asyncio +import base64 +import json +from datetime import UTC, datetime +from typing import Any + +import httpx +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.target_schema_registry import TARGET_SCHEMAS +from app.db.session import async_session_factory +from app.models.datasource_config import DataSourceConfig +from app.models.datasource_mapping import DataSourceMappingTemplate +from app.services.datasource_mapping import ( + MappingError, + execute_mapping, + extract_path, + persist_mapped_records, +) + +DEFAULT_MAPPING_TEMPLATES: dict[str, dict[str, Any]] = { + "vessel_ais": { + "source": {"items_path": "$"}, + "fields": { + "mmsi": {"path": "$.mmsi", "type": "integer"}, + "name": {"path": "$.name", "type": "string", "default": None}, + "lat": {"path": "$.lat", "type": "float"}, + "lon": {"path": "$.lon", "type": "float"}, + "sog": {"path": "$.sog", "type": "float", "default": None}, + "cog": {"path": "$.cog", "type": "float", "default": None}, + "heading": {"path": "$.heading", "type": "integer", "default": None}, + "nav_status": {"path": "$.nav_status", "type": "integer", "default": None}, + "callsign": {"path": "$.callsign", "type": "string", "default": None}, + "vessel_type": {"path": "$.vessel_type", "type": "string", "default": None}, + "vessel_type_name": {"path": "$.vessel_type_name", "type": "string", "default": None}, + "received_at": {"path": "$.received_at", "type": "datetime", "default": None}, + }, + "meta": {"generated_by": "default_template", "requires_review": False}, + }, +} + +RUNNING_CUSTOM_STREAM_TASKS: dict[int, asyncio.Task[Any]] = {} + + +class CustomDatasourceRuntimeError(RuntimeError): + """Raised when a custom datasource cannot run.""" + + +def build_request_headers(auth_type: str, auth_config: dict, headers: dict) -> dict[str, str]: + request_headers = {str(key): str(value) for key, value in (headers or {}).items()} + auth_type = str(auth_type or "none").lower() + auth_config = auth_config or {} + + if auth_type == "bearer" and auth_config.get("token"): + request_headers["Authorization"] = f"Bearer {auth_config['token']}" + elif auth_type == "api_key" and auth_config.get("api_key"): + location = str(auth_config.get("in") or auth_config.get("location") or "header").lower() + if location != "query": + key_name = auth_config.get("key_name", "X-API-Key") + request_headers[str(key_name)] = str(auth_config["api_key"]) + elif auth_type == "basic": + username = auth_config.get("username", "") + password = auth_config.get("password", "") + credentials = f"{username}:{password}" + encoded = base64.b64encode(credentials.encode()).decode() + request_headers["Authorization"] = f"Basic {encoded}" + return request_headers + + +def build_query_params(auth_type: str, auth_config: dict, config: dict) -> dict[str, Any]: + params: dict[str, Any] = {} + candidate = (config or {}).get("params") or (config or {}).get("query_params") + if isinstance(candidate, dict): + params.update(candidate) + + auth_type = str(auth_type or "none").lower() + auth_config = auth_config or {} + if auth_type == "api_key" and auth_config.get("api_key"): + location = str(auth_config.get("in") or auth_config.get("location") or "header").lower() + if location == "query": + key_name = auth_config.get("key_name") or auth_config.get("param_name") or "api_key" + params[str(key_name)] = auth_config["api_key"] + return params + + +async def load_active_mapping( + db: AsyncSession, + datasource_config_id: int, +) -> DataSourceMappingTemplate: + result = await db.execute( + select(DataSourceMappingTemplate) + .where(DataSourceMappingTemplate.datasource_config_id == datasource_config_id) + .where(DataSourceMappingTemplate.is_active.is_(True)) + .order_by(DataSourceMappingTemplate.version.desc()) + .limit(1) + ) + mapping = result.scalar_one_or_none() + if mapping is not None: + return mapping + + datasource = await db.get(DataSourceConfig, datasource_config_id) + if datasource is None: + raise CustomDatasourceRuntimeError("Configuration not found") + target_schema = (datasource.config or {}).get("target_schema") + template_body = DEFAULT_MAPPING_TEMPLATES.get(str(target_schema or "")) if target_schema else None + if not template_body or target_schema not in TARGET_SCHEMAS: + raise CustomDatasourceRuntimeError( + "No active mapping template found and no default template available for this target schema" + ) + + mapping = DataSourceMappingTemplate( + datasource_config_id=datasource_config_id, + target_schema=str(target_schema), + mapping_json=template_body, + sample_payload_hash=None, + validation_status="valid", + version=1, + is_active=True, + ) + db.add(mapping) + await db.commit() + await db.refresh(mapping) + return mapping + + +async def fetch_rest_payload(config: DataSourceConfig, limit_bytes: int) -> Any: + request_config = config.config or {} + method = str(request_config.get("method") or request_config.get("request_method") or "GET").upper() + if method not in {"GET", "POST"}: + raise CustomDatasourceRuntimeError("Only GET and POST sample requests are supported.") + + headers = build_request_headers(config.auth_type, config.auth_config or {}, config.headers or {}) + params = build_query_params(config.auth_type, config.auth_config or {}, request_config) + timeout = float(request_config.get("timeout", 30)) + json_body = request_config.get("json_body") + if json_body is None and str(request_config.get("body_type") or "").lower() in {"json", ""}: + candidate = request_config.get("body") + if isinstance(candidate, (dict, list)): + json_body = candidate + + async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client: + response = await client.request( + method, + config.endpoint, + headers=headers, + params=params or None, + json=json_body, + ) + response.raise_for_status() + content = response.content[:limit_bytes] + if "application/json" in response.headers.get("content-type", ""): + return json.loads(content.decode(response.encoding or "utf-8")) + return {"text": content.decode(response.encoding or "utf-8", errors="replace")} + + +async def run_mapped_rest_config( + db: AsyncSession, + datasource: DataSourceConfig, +) -> dict[str, Any]: + mapping = await load_active_mapping(db, datasource.id) + sample = await fetch_rest_payload(datasource, 5_000_000) + mapped = execute_mapping(sample, mapping.mapping_json, mapping.target_schema) + if mapped["failed_count"] > 0: + return { + "status": "failed", + "datasource_config_id": datasource.id, + "mapping_id": mapping.id, + "mapping_version": mapping.version, + "target_schema": mapping.target_schema, + "mapped_count": mapped["mapped_count"], + "failed_count": mapped["failed_count"], + "errors": mapped["errors"][:20], + } + + request_config = datasource.config or {} + written_count = await persist_mapped_records( + db, + datasource_name=datasource.name, + datasource_config_id=datasource.id, + target_schema=mapping.target_schema, + records=mapped["records"], + mapping_version=mapping.version, + delivery_mode=request_config.get("delivery_mode") or "polling", + transport="http", + ) + return { + "status": "success", + "datasource_config_id": datasource.id, + "mapping_id": mapping.id, + "mapping_version": mapping.version, + "target_schema": mapping.target_schema, + "fetched_count": mapped["total_items"], + "mapped_count": mapped["mapped_count"], + "written_count": written_count, + } + + +def _items_from_ws_message(payload: Any, config: dict) -> Any: + message_path = config.get("ws_message_path") + items_path = config.get("ws_items_path") + value = extract_path(payload, message_path) if message_path else payload + return extract_path(value, items_path) if items_path else value + + +async def _connect_websocket(endpoint: str, headers: dict[str, str]): + import websockets + + try: + return await websockets.connect(endpoint, additional_headers=headers or None) + except TypeError: + return await websockets.connect(endpoint, extra_headers=headers or None) + + +async def test_websocket_config(config: DataSourceConfig) -> dict[str, Any]: + if not str(config.endpoint or "").startswith(("ws://", "wss://")): + raise CustomDatasourceRuntimeError("WebSocket datasource endpoint must start with ws:// or wss://") + + runtime_config = config.config or {} + headers = build_request_headers(config.auth_type, config.auth_config or {}, config.headers or {}) + receive_timeout = float(runtime_config.get("receive_timeout_seconds") or runtime_config.get("timeout") or 10) + async with await _connect_websocket(config.endpoint, headers) as websocket: + subscribe_message = runtime_config.get("ws_subscribe_message") + if isinstance(subscribe_message, (dict, list)): + await websocket.send(json.dumps(subscribe_message)) + elif isinstance(subscribe_message, str) and subscribe_message.strip(): + await websocket.send(subscribe_message) + raw_message = await asyncio.wait_for(websocket.recv(), timeout=receive_timeout) + return { + "success": True, + "message_preview": raw_message[:1000] if isinstance(raw_message, str) else str(raw_message)[:1000], + } + + +async def run_mapped_websocket_config( + db: AsyncSession, + datasource: DataSourceConfig, + *, + debug_max_messages: int | None = None, + use_config_debug_max_messages: bool = True, +) -> dict[str, Any]: + if not str(datasource.endpoint or "").startswith(("ws://", "wss://")): + raise CustomDatasourceRuntimeError("WebSocket datasource endpoint must start with ws:// or wss://") + + mapping = await load_active_mapping(db, datasource.id) + runtime_config = datasource.config or {} + max_messages = debug_max_messages + if max_messages is None and use_config_debug_max_messages: + max_messages = runtime_config.get("debug_max_messages") + max_messages = int(max_messages) if max_messages else None + receive_timeout = float(runtime_config.get("receive_timeout_seconds") or runtime_config.get("timeout") or 30) + reconnect = bool(runtime_config.get("ws_reconnect", True)) + reconnect_delay = float(runtime_config.get("reconnect_delay_seconds") or 3) + headers = build_request_headers(datasource.auth_type, datasource.auth_config or {}, datasource.headers or {}) + + messages_seen = 0 + mapped_count = 0 + failed_count = 0 + written_count = 0 + errors: list[dict[str, Any]] = [] + started_at = datetime.now(UTC) + + while True: + try: + async with await _connect_websocket(datasource.endpoint, headers) as websocket: + subscribe_message = runtime_config.get("ws_subscribe_message") + if isinstance(subscribe_message, (dict, list)): + await websocket.send(json.dumps(subscribe_message)) + elif isinstance(subscribe_message, str) and subscribe_message.strip(): + await websocket.send(subscribe_message) + + while True: + raw_message = await asyncio.wait_for(websocket.recv(), timeout=receive_timeout) + messages_seen += 1 + try: + payload = json.loads(raw_message) + except json.JSONDecodeError as exc: + failed_count += 1 + errors.append({"message": "invalid_json", "error": str(exc)}) + continue + + extracted = _items_from_ws_message(payload, runtime_config) + try: + mapped = execute_mapping(extracted, mapping.mapping_json, mapping.target_schema) + except (MappingError, ValueError) as exc: + failed_count += 1 + errors.append({"message": "mapping_failed", "error": str(exc)}) + continue + + mapped_count += mapped["mapped_count"] + failed_count += mapped["failed_count"] + if mapped["errors"]: + errors.extend(mapped["errors"][:5]) + if mapped["records"]: + written_count += await persist_mapped_records( + db, + datasource_name=datasource.name, + datasource_config_id=datasource.id, + target_schema=mapping.target_schema, + records=mapped["records"], + mapping_version=mapping.version, + delivery_mode=runtime_config.get("delivery_mode") or "realtime_stream", + transport="websocket", + ) + + if max_messages and messages_seen >= max_messages: + return { + "status": "success", + "datasource_config_id": datasource.id, + "mapping_id": mapping.id, + "mapping_version": mapping.version, + "target_schema": mapping.target_schema, + "messages_seen": messages_seen, + "mapped_count": mapped_count, + "failed_count": failed_count, + "written_count": written_count, + "errors": errors[:20], + "execution_time_seconds": (datetime.now(UTC) - started_at).total_seconds(), + } + except asyncio.CancelledError: + raise + except Exception as exc: + failed_count += 1 + errors.append({"message": "websocket_error", "error": f"{exc.__class__.__name__}: {exc}"}) + if not reconnect or max_messages: + return { + "status": "failed" if written_count == 0 else "partial", + "datasource_config_id": datasource.id, + "mapping_id": mapping.id, + "mapping_version": mapping.version, + "target_schema": mapping.target_schema, + "messages_seen": messages_seen, + "mapped_count": mapped_count, + "failed_count": failed_count, + "written_count": written_count, + "errors": errors[:20], + } + await asyncio.sleep(reconnect_delay) + + +async def run_custom_stream_by_id(config_id: int) -> dict[str, Any]: + async with async_session_factory() as db: + datasource = await db.get(DataSourceConfig, config_id) + if not datasource: + raise CustomDatasourceRuntimeError("Configuration not found") + return await run_mapped_websocket_config( + db, + datasource, + use_config_debug_max_messages=False, + ) + + +def start_custom_stream(config_id: int) -> bool: + existing = RUNNING_CUSTOM_STREAM_TASKS.get(config_id) + if existing is not None and not existing.done(): + return False + task = asyncio.create_task(run_custom_stream_by_id(config_id), name=f"custom-stream:{config_id}") + RUNNING_CUSTOM_STREAM_TASKS[config_id] = task + + def _cleanup(done_task: asyncio.Task[Any]) -> None: + if RUNNING_CUSTOM_STREAM_TASKS.get(config_id) is done_task: + RUNNING_CUSTOM_STREAM_TASKS.pop(config_id, None) + + task.add_done_callback(_cleanup) + return True + + +async def stop_custom_stream(config_id: int) -> bool: + task = RUNNING_CUSTOM_STREAM_TASKS.get(config_id) + if task is None or task.done(): + RUNNING_CUSTOM_STREAM_TASKS.pop(config_id, None) + return False + task.cancel() + try: + await task + except asyncio.CancelledError: + return True + return task.cancelled() + + +def get_custom_stream_status(config_id: int) -> dict[str, Any]: + task = RUNNING_CUSTOM_STREAM_TASKS.get(config_id) + return { + "config_id": config_id, + "running": bool(task and not task.done()), + "done": bool(task and task.done()), + } diff --git a/backend/app/services/datasource_mapping.py b/backend/app/services/datasource_mapping.py index 6877f266..03dbf297 100644 --- a/backend/app/services/datasource_mapping.py +++ b/backend/app/services/datasource_mapping.py @@ -290,25 +290,77 @@ async def persist_mapped_records( target_schema: str, records: list[dict[str, Any]], mapping_version: int, + delivery_mode: str | None = None, + transport: str | None = None, ) -> int: """Persist validated mapped records to the destination for a target schema.""" if target_schema == "vessel_ais": - from app.models.vessel import VesselPosition + from app.core.time import to_iso8601_utc + from app.core.websocket.broadcaster import broadcaster + from app.services.vessel_ais_aggregation import ( + record_vessel_ais_observation, + update_ais_source_health, + ) + now = datetime.now(UTC) + latest_observed_at = now + written_count = 0 for record in records: - db.add( - VesselPosition( - mmsi=record["mmsi"], - lat=record["lat"], - lon=record["lon"], - sog=record.get("sog"), - cog=record.get("cog"), - heading=record.get("heading"), - received_at=_parse_datetime(record.get("received_at")) or datetime.now(UTC), - ) + observed_at = _parse_datetime(record.get("received_at")) or now + observation = await record_vessel_ais_observation( + db, + source=datasource_name, + normalized_payload=record, + raw_payload=record, + delivery_mode=delivery_mode or "polling", + transport=transport or "http", + message_type="PositionReport", + observed_at=observed_at, + collected_at=now, ) + if observation is not None: + written_count += 1 + if observed_at > latest_observed_at: + latest_observed_at = observed_at + + await update_ais_source_health( + db, + source=datasource_name, + connection_state="connected", + observed_count=len(records), + last_seen_at=latest_observed_at, + last_success_at=now if records else None, + lag_seconds=max((now - latest_observed_at).total_seconds(), 0), + ) await db.commit() - return len(records) + if records: + await broadcaster.broadcast_custom( + "vessels", + { + "action": "upsert", + "source": datasource_name, + "created": True, + "vessels": [ + { + "mmsi": record.get("mmsi"), + "mmsi_display": str(record.get("mmsi")) if record.get("mmsi") is not None else None, + "name": record.get("name"), + "callsign": record.get("callsign"), + "lat": record.get("lat"), + "lon": record.get("lon"), + "sog": record.get("sog"), + "cog": record.get("cog"), + "heading": record.get("heading"), + "nav_status": record.get("nav_status"), + "vessel_type": record.get("vessel_type"), + "vessel_type_name": record.get("vessel_type_name"), + "received_at": to_iso8601_utc(_parse_datetime(record.get("received_at"))), + } + for record in records + ], + }, + ) + return written_count from app.models.collected_data import CollectedData diff --git a/backend/app/services/vessel_aggregation_strategy.py b/backend/app/services/vessel_aggregation_strategy.py new file mode 100644 index 00000000..33c516a2 --- /dev/null +++ b/backend/app/services/vessel_aggregation_strategy.py @@ -0,0 +1,198 @@ +"""Persistence + validation for the v4 vessel_ais aggregation strategy.""" + +from __future__ import annotations + +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.system_setting import SystemSetting + +VESSEL_AGGREGATION_STRATEGY_CATEGORY = "vessel_aggregation_strategy" + +DYNAMIC_FIELDS: tuple[str, ...] = ("lat", "lon", "sog", "cog", "heading", "nav_status") +STATIC_FIELDS: tuple[str, ...] = ( + "name", + "callsign", + "imo", + "flag", + "vessel_type", + "vessel_type_name", + "length", + "width", + "draught", +) +ALLOWED_FIELDS: frozenset[str] = frozenset(DYNAMIC_FIELDS + STATIC_FIELDS) +ALLOWED_DYNAMIC_MODES: frozenset[str] = frozenset({"newest"}) +ALLOWED_STATIC_MODES: frozenset[str] = frozenset({"source_priority", "non_empty", "newest", "locked"}) +ALLOWED_LOCKED_DYNAMIC_MODES: frozenset[str] = frozenset({"newest", "source_priority", "locked"}) + + +DEFAULT_STRATEGY: dict[str, Any] = { + "version": 1, + "vessel_ais": { + "source_priority": ["aisstream_vessels", "barentswatch_vessels"], + "field_rules": {}, + "freshness": { + "realtime_stream_seconds": 900, + "polling_seconds": 3600, + }, + "allow_dynamic_lock": False, + }, +} + + +class StrategyValidationError(ValueError): + """Raised when a saved strategy payload is malformed.""" + + +def _coerce_str_list(value: Any, *, label: str) -> list[str]: + if value is None: + return [] + if not isinstance(value, list): + raise StrategyValidationError(f"{label} must be a list of source names") + out: list[str] = [] + for item in value: + if not isinstance(item, str) or not item.strip(): + raise StrategyValidationError(f"{label} entries must be non-empty strings") + out.append(item.strip()) + return out + + +def validate_strategy(payload: dict[str, Any]) -> dict[str, Any]: + """Validate and normalize a strategy payload. Raise StrategyValidationError on issues.""" + + if not isinstance(payload, dict): + raise StrategyValidationError("strategy payload must be an object") + + vessel_ais = payload.get("vessel_ais") + if not isinstance(vessel_ais, dict): + raise StrategyValidationError("strategy.vessel_ais is required and must be an object") + + allow_dynamic_lock = bool(vessel_ais.get("allow_dynamic_lock", False)) + source_priority = _coerce_str_list( + vessel_ais.get("source_priority"), + label="vessel_ais.source_priority", + ) + + raw_rules = vessel_ais.get("field_rules") or {} + if not isinstance(raw_rules, dict): + raise StrategyValidationError("vessel_ais.field_rules must be an object") + field_rules: dict[str, dict[str, Any]] = {} + for field, rule in raw_rules.items(): + if field not in ALLOWED_FIELDS: + raise StrategyValidationError(f"unknown vessel_ais field: {field}") + if not isinstance(rule, dict): + raise StrategyValidationError(f"field_rules.{field} must be an object") + mode = str(rule.get("mode") or "").strip() + if not mode: + raise StrategyValidationError(f"field_rules.{field}.mode is required") + is_dynamic = field in DYNAMIC_FIELDS + if is_dynamic: + allowed_modes = ALLOWED_LOCKED_DYNAMIC_MODES if allow_dynamic_lock else ALLOWED_DYNAMIC_MODES + if mode not in allowed_modes: + if not allow_dynamic_lock: + raise StrategyValidationError( + f"field_rules.{field}.mode='{mode}' requires allow_dynamic_lock=true" + ) + raise StrategyValidationError( + f"field_rules.{field}.mode must be one of {sorted(allowed_modes)}" + ) + else: + if mode not in ALLOWED_STATIC_MODES: + raise StrategyValidationError( + f"field_rules.{field}.mode must be one of {sorted(ALLOWED_STATIC_MODES)}" + ) + normalized_rule: dict[str, Any] = {"mode": mode} + rule_priority = rule.get("source_priority") + if rule_priority is not None: + normalized_rule["source_priority"] = _coerce_str_list( + rule_priority, + label=f"field_rules.{field}.source_priority", + ) + if mode == "locked": + locked_source = rule.get("locked_source") + if not isinstance(locked_source, str) or not locked_source.strip(): + raise StrategyValidationError( + f"field_rules.{field}.locked_source must be a non-empty string when mode=locked" + ) + normalized_rule["locked_source"] = locked_source.strip() + field_rules[field] = normalized_rule + + raw_freshness = vessel_ais.get("freshness") or {} + if not isinstance(raw_freshness, dict): + raise StrategyValidationError("vessel_ais.freshness must be an object") + freshness: dict[str, int] = {} + for key in ("realtime_stream_seconds", "polling_seconds"): + value = raw_freshness.get(key, DEFAULT_STRATEGY["vessel_ais"]["freshness"][key]) + try: + seconds = int(value) + except (TypeError, ValueError) as exc: + raise StrategyValidationError(f"freshness.{key} must be an integer") from exc + if seconds < 0: + raise StrategyValidationError(f"freshness.{key} must be non-negative") + freshness[key] = seconds + + return { + "version": int(payload.get("version") or 0) + 1, + "vessel_ais": { + "source_priority": source_priority, + "field_rules": field_rules, + "freshness": freshness, + "allow_dynamic_lock": allow_dynamic_lock, + }, + } + + +async def _select_setting(db: AsyncSession) -> SystemSetting | None: + result = await db.execute( + select(SystemSetting).where(SystemSetting.category == VESSEL_AGGREGATION_STRATEGY_CATEGORY) + ) + return result.scalar_one_or_none() + + +def _current_version(setting: SystemSetting | None) -> int: + if setting is None: + return 0 + payload = setting.payload or {} + return int(payload.get("version") or 0) + + +async def load_strategy(db: AsyncSession) -> dict[str, Any]: + setting = await _select_setting(db) + if setting is None or not isinstance(setting.payload, dict): + return DEFAULT_STRATEGY + payload = setting.payload + if "vessel_ais" not in payload: + return DEFAULT_STRATEGY + return payload + + +async def save_strategy(db: AsyncSession, payload: dict[str, Any]) -> dict[str, Any]: + """Validate + persist; bumps version automatically.""" + + existing = await _select_setting(db) + incoming = dict(payload) + incoming.setdefault("version", _current_version(existing)) + validated = validate_strategy(incoming) + + if existing is None: + existing = SystemSetting(category=VESSEL_AGGREGATION_STRATEGY_CATEGORY, payload=validated) + db.add(existing) + else: + existing.payload = validated + await db.commit() + return validated + + +async def reset_strategy(db: AsyncSession) -> dict[str, Any]: + existing = await _select_setting(db) + payload = {**DEFAULT_STRATEGY, "version": _current_version(existing) + 1} + if existing is None: + existing = SystemSetting(category=VESSEL_AGGREGATION_STRATEGY_CATEGORY, payload=payload) + db.add(existing) + else: + existing.payload = payload + await db.commit() + return payload diff --git a/backend/app/services/vessel_ais_aggregation.py b/backend/app/services/vessel_ais_aggregation.py index 8fa6dcff..c8d45904 100644 --- a/backend/app/services/vessel_ais_aggregation.py +++ b/backend/app/services/vessel_ais_aggregation.py @@ -1,6 +1,6 @@ """AIS raw observation and aggregation support for vessel collectors.""" -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from hashlib import sha256 import json from typing import Any, Iterable @@ -9,9 +9,14 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth +from app.services.vessel_aggregation_strategy import ( + DEFAULT_STRATEGY, + load_strategy, +) from app.services.vessel_types import normalize_vessel_type_name VESSEL_AIS_SCHEMA = "vessel_ais" +DEFAULT_AGGREGATION_WINDOW_HOURS = 24 BARENTSWATCH_DELIVERY_MODE = "polling" BARENTSWATCH_TRANSPORT = "http" AISSTREAM_DELIVERY_MODE = "realtime_stream" @@ -171,13 +176,43 @@ def _is_future_observation(observation: AISRawObservation, now: datetime) -> boo return observation.observed_at > now +def _strategy_source_rank( + source: str, + strategy: dict[str, Any], +) -> int: + priority = (strategy.get("vessel_ais") or {}).get("source_priority") or [] + if source in priority: + return len(priority) - priority.index(source) + return 0 + + +def _is_stream_stale( + observation: AISRawObservation, + *, + now: datetime, + strategy: dict[str, Any], +) -> bool: + delivery_mode = str(observation.delivery_mode or "") + freshness = (strategy.get("vessel_ais") or {}).get("freshness") or {} + if delivery_mode == "realtime_stream": + window = int(freshness.get("realtime_stream_seconds", 0) or 0) + else: + window = int(freshness.get("polling_seconds", 0) or 0) + if window <= 0: + return False + return (now - observation.observed_at).total_seconds() > window + + def _select_position_observation( observations: list[AISRawObservation], *, now: datetime, + strategy: dict[str, Any] | None = None, ) -> tuple[AISRawObservation | None, list[str]]: + strategy = strategy or DEFAULT_STRATEGY rejected_flags: list[str] = [] - candidates = [] + fresh_candidates: list[AISRawObservation] = [] + stale_candidates: list[AISRawObservation] = [] for observation in observations: payload = observation.normalized_payload or {} if not _has_valid_position(payload): @@ -186,8 +221,13 @@ def _select_position_observation( if _is_future_observation(observation, now): rejected_flags.append("future_timestamp") continue - candidates.append(observation) + if _is_stream_stale(observation, now=now, strategy=strategy): + stale_candidates.append(observation) + rejected_flags.append("freshness_fallback") + continue + fresh_candidates.append(observation) + candidates = fresh_candidates or stale_candidates if not candidates: return None, sorted(set(rejected_flags)) @@ -195,6 +235,7 @@ def _select_position_observation( key=lambda item: ( item.observed_at, _delivery_priority(item), + _strategy_source_rank(item.source, strategy), item.collected_at, item.id or 0, ), @@ -206,7 +247,9 @@ def _select_position_observation( def _select_static_field( observations: list[AISRawObservation], field: str, + strategy: dict[str, Any] | None = None, ) -> tuple[Any, str | None, str | None]: + strategy = strategy or DEFAULT_STRATEGY candidates = [] for observation in observations: value = _payload_value(observation.normalized_payload or {}, field) @@ -219,6 +262,39 @@ def _select_static_field( if not candidates: return None, None, None + field_rules = (strategy.get("vessel_ais") or {}).get("field_rules") or {} + rule = field_rules.get(field) or {"mode": "source_priority"} + mode = rule.get("mode") + + if mode == "locked": + locked_source = rule.get("locked_source") + for observation, value in candidates: + if observation.source == locked_source: + return value, observation.source, "locked" + + if mode in ("source_priority", "locked"): + priority = rule.get("source_priority") or (strategy.get("vessel_ais") or {}).get("source_priority") or [] + ranked = sorted( + candidates, + key=lambda item: ( + priority.index(item[0].source) if item[0].source in priority else len(priority) + 1, + -_delivery_priority(item[0]), + -(item[0].observed_at.timestamp() if item[0].observed_at else 0), + ), + ) + observation, value = ranked[0] + return value, observation.source, "source_priority" + + if mode == "newest": + ranked = sorted( + candidates, + key=lambda item: (item[0].observed_at, _delivery_priority(item[0]), item[0].id or 0), + reverse=True, + ) + observation, value = ranked[0] + return value, observation.source, "newest_observation" + + # default / non_empty: prefer delivery mode priority, then newest candidates.sort( key=lambda item: ( _delivery_priority(item[0]), @@ -261,8 +337,12 @@ def _build_aggregated_vessel( observations: list[AISRawObservation], *, now: datetime, + strategy: dict[str, Any] | None = None, ) -> dict[str, Any] | None: - position_observation, rejected_flags = _select_position_observation(observations, now=now) + strategy = strategy or DEFAULT_STRATEGY + position_observation, rejected_flags = _select_position_observation( + observations, now=now, strategy=strategy + ) if position_observation is None: return None @@ -279,6 +359,7 @@ def _build_aggregated_vessel( "quality_flags": sorted( set((position_observation.quality_flags or []) + rejected_flags) ), + "aggregation_strategy_version": int(strategy.get("version") or 0), } for field in DYNAMIC_FIELDS: @@ -289,7 +370,9 @@ def _build_aggregated_vessel( result["selected_reasons"][field] = "newest_observation" for field in CONFLICT_FIELDS: - selected_value, selected_source, reason = _select_static_field(observations, field) + selected_value, selected_source, reason = _select_static_field( + observations, field, strategy=strategy + ) if selected_value is None: continue result[field] = selected_value @@ -408,12 +491,16 @@ async def aggregate_vessel_observations( db: AsyncSession, observations: Iterable[AISRawObservation], *, - write_conflicts: bool = True, + write_conflicts: bool = False, + strategy: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: + strategy = strategy if strategy is not None else await _safe_load_strategy(db) now = datetime.now(UTC) vessels = [] for entity_key, entity_observations in _group_observations(observations).items(): - aggregated = _build_aggregated_vessel(entity_key, entity_observations, now=now) + aggregated = _build_aggregated_vessel( + entity_key, entity_observations, now=now, strategy=strategy + ) if aggregated is None: continue if write_conflicts: @@ -431,15 +518,28 @@ async def aggregate_vessel_observations( return vessels +async def _safe_load_strategy(db: AsyncSession) -> dict[str, Any]: + """Tolerate fake test sessions where load_strategy may misbehave.""" + try: + return await load_strategy(db) + except Exception: + return DEFAULT_STRATEGY + + async def get_aggregated_vessels( db: AsyncSession, *, bbox: tuple[float, float, float, float] | None = None, limit: int | None = None, + observed_since: datetime | None = None, ) -> list[dict[str, Any]]: + observed_since = observed_since or ( + datetime.now(UTC) - timedelta(hours=DEFAULT_AGGREGATION_WINDOW_HOURS) + ) stmt = ( select(AISRawObservation) .where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA) + .where(AISRawObservation.observed_at >= observed_since) .order_by(AISRawObservation.observed_at.desc(), AISRawObservation.id.desc()) ) if limit and limit > 0: @@ -545,6 +645,30 @@ async def update_ais_source_health( return health +async def count_unique_raw_vessel_mmsi( + db: AsyncSession, + *, + observed_since: datetime | None = None, +) -> int: + """Count unique raw vessel MMSI values for HUD counts; never aggregates.""" + from sqlalchemy import func as sa_func + + unique_mmsi_stmt = ( + select(AISRawObservation.entity_key) + .where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA) + .distinct() + ) + if observed_since is not None: + unique_mmsi_stmt = unique_mmsi_stmt.where( + AISRawObservation.observed_at >= observed_since, + ) + + result = await db.execute( + select(sa_func.count()).select_from(unique_mmsi_stmt.subquery()), + ) + return int(result.scalar() or 0) + + async def get_vessel_raw_observations( db: AsyncSession, mmsi: int, diff --git a/backend/app/services/vessel_enrichment.py b/backend/app/services/vessel_enrichment.py new file mode 100644 index 00000000..ef4d6512 --- /dev/null +++ b/backend/app/services/vessel_enrichment.py @@ -0,0 +1,109 @@ +"""v5 vessel enrichment service. + +Read-only side: `get_vessel_enrichment_bundle` is the only path the +aggregation/detail endpoints use. It never reaches out to third parties; it +just returns whatever the upsert side has already cached. Expired rows are +filtered out so old data never leaks back into the live UI. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.vessel_enrichment import VesselMediaEnrichment, VesselProfileEnrichment + + +def _coerce_datetime(value: Any) -> datetime | None: + if value in (None, ""): + return None + if isinstance(value, datetime): + return value if value.tzinfo else value.replace(tzinfo=UTC) + if isinstance(value, (int, float)): + ts = float(value) + if ts > 10_000_000_000: + ts /= 1000 + return datetime.fromtimestamp(ts, UTC) + if isinstance(value, str): + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) + except ValueError: + return None + return None + + +def _build_payload(record, *, now: datetime) -> dict[str, Any] | None: + if record is None: + return None + expires_at = record.expires_at + if isinstance(expires_at, datetime): + if expires_at.tzinfo is None: + expires_at = expires_at.replace(tzinfo=UTC) + if expires_at < now: + return None + return record.to_dict() + + +async def get_vessel_enrichment_bundle(db: AsyncSession, mmsi: int) -> dict[str, Any]: + now = datetime.now(UTC) + profile = await db.get(VesselProfileEnrichment, mmsi) + media = await db.get(VesselMediaEnrichment, mmsi) + return { + "mmsi": mmsi, + "profile": _build_payload(profile, now=now), + "media": _build_payload(media, now=now), + } + + +async def upsert_vessel_profile_enrichment( + db: AsyncSession, + *, + mmsi: int, + payload: dict[str, Any], +) -> dict[str, Any]: + record = await db.get(VesselProfileEnrichment, mmsi) + if record is None: + record = VesselProfileEnrichment(mmsi=mmsi) + db.add(record) + return _apply_upsert(record, payload) + + +async def upsert_vessel_media_enrichment( + db: AsyncSession, + *, + mmsi: int, + payload: dict[str, Any], +) -> dict[str, Any]: + record = await db.get(VesselMediaEnrichment, mmsi) + if record is None: + record = VesselMediaEnrichment(mmsi=mmsi) + db.add(record) + return _apply_upsert(record, payload) + + +def _apply_upsert(record, payload: dict[str, Any]) -> dict[str, Any]: + if not isinstance(payload, dict): + raise ValueError("enrichment payload must be an object") + body = payload.get("payload") + if body is not None and not isinstance(body, dict): + raise ValueError("payload.payload must be an object") + if body is not None: + record.payload = body + if "source" in payload and isinstance(payload["source"], str) and payload["source"].strip(): + record.source = payload["source"].strip() + fetched_at = _coerce_datetime(payload.get("fetched_at")) + record.fetched_at = fetched_at or datetime.now(UTC) + record.expires_at = _coerce_datetime(payload.get("expires_at")) + confidence = payload.get("confidence") + if confidence is not None: + try: + record.confidence = float(confidence) + except (TypeError, ValueError): + record.confidence = None + if "reference_url" in payload: + ref = payload.get("reference_url") + record.reference_url = str(ref) if ref else None + return record.to_dict() diff --git a/backend/tests/test_custom_datasource_runtime_live.py b/backend/tests/test_custom_datasource_runtime_live.py new file mode 100644 index 00000000..dd036e23 --- /dev/null +++ b/backend/tests/test_custom_datasource_runtime_live.py @@ -0,0 +1,149 @@ +"""End-to-end integration test for the custom WebSocket datasource runner. + +Boots an in-process WebSocket server that mimics the bun mock AIS server +(`scripts/mock-ais-ws-server.ts`) and runs the real +`run_mapped_websocket_config` against it. Catches regressions where the +runner stops connecting, fails to extract the configured message path, +or quietly drops mapped records before broadcasting. +""" + +from __future__ import annotations + +import asyncio +import json +from contextlib import asynccontextmanager +from datetime import UTC, datetime +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +import websockets + +from app.models.datasource_config import DataSourceConfig +from app.services import custom_datasource_runtime +from app.services.custom_datasource_runtime import run_mapped_websocket_config + + +def _make_payload(seq: int) -> str: + return json.dumps( + { + "type": "vessel", + "sequence": seq, + "data": { + "mmsi": str(999_000_000 + seq), + "name": f"MOCK VESSEL {seq:03d}", + "lat": 36.20 + seq * 0.001, + "lon": 14.20 + seq * 0.001, + "sog": 12.0, + "cog": 90.0, + "heading": 90, + "vessel_type": 70, + "vessel_type_name": "Cargo", + "received_at": datetime.now(UTC).isoformat(), + }, + } + ) + + +@asynccontextmanager +async def _mock_ais_server(emit_count: int): + received_subscribe: list[str] = [] + + async def handler(ws): + try: + try: + msg = await asyncio.wait_for(ws.recv(), timeout=0.5) + received_subscribe.append(msg) + except (asyncio.TimeoutError, websockets.ConnectionClosed): + pass + for seq in range(1, emit_count + 1): + await ws.send(_make_payload(seq)) + await asyncio.sleep(0.01) + # keep the socket open briefly so the runner observes the messages + await asyncio.sleep(0.05) + except websockets.ConnectionClosed: + return + + async with websockets.serve(handler, "127.0.0.1", 0) as server: + port = next(iter(server.sockets)).getsockname()[1] + yield port, received_subscribe + + +@pytest.mark.asyncio +async def test_websocket_runner_streams_from_live_mock(monkeypatch): + mapping = SimpleNamespace( + id=11, + version=3, + target_schema="vessel_ais", + mapping_json={ + "source": {"items_path": "$"}, + "fields": { + "mmsi": {"path": "$.mmsi", "type": "integer"}, + "lat": {"path": "$.lat", "type": "float"}, + "lon": {"path": "$.lon", "type": "float"}, + "name": {"path": "$.name", "type": "string"}, + "vessel_type": {"path": "$.vessel_type", "type": "integer", "default": None}, + "vessel_type_name": {"path": "$.vessel_type_name", "type": "string", "default": None}, + "sog": {"path": "$.sog", "type": "float", "default": None}, + "cog": {"path": "$.cog", "type": "float", "default": None}, + "heading": {"path": "$.heading", "type": "integer", "default": None}, + "received_at": {"path": "$.received_at", "type": "datetime"}, + }, + }, + ) + + class FakeResult: + def scalar_one_or_none(self): + return mapping + + class FakeDB: + async def execute(self, _stmt): + return FakeResult() + + persist = AsyncMock(return_value=1) + monkeypatch.setattr(custom_datasource_runtime, "persist_mapped_records", persist) + + async with _mock_ais_server(emit_count=3) as (port, received_subscribe): + result = await run_mapped_websocket_config( + FakeDB(), + DataSourceConfig( + id=99, + name="mock_ais_ws", + source_type="websocket", + endpoint=f"ws://127.0.0.1:{port}", + auth_type="none", + headers={}, + config={ + "ws_message_path": "$.data", + "ws_subscribe_message": { + "type": "subscribe", + "anchor": {"lat": 36.2, "lon": 14.2}, + "spread_km": 50, + "rate_hz": 1, + }, + "debug_max_messages": 2, + "delivery_mode": "realtime_stream", + "ws_reconnect": False, + }, + ), + use_config_debug_max_messages=True, + ) + + assert result["status"] == "success" + assert result["messages_seen"] == 2 + assert result["written_count"] == 2 + assert result["mapped_count"] == 2 + assert result["target_schema"] == "vessel_ais" + # subscribe message must reach the server unchanged + assert received_subscribe, "runner did not forward ws_subscribe_message" + parsed = json.loads(received_subscribe[0]) + assert parsed["type"] == "subscribe" + assert parsed["anchor"] == {"lat": 36.2, "lon": 14.2} + assert parsed["rate_hz"] == 1 + # mapped records carry the real MMSIs from the mock stream + persisted_records = [] + for call in persist.await_args_list: + persisted_records.extend(call.kwargs["records"]) + assert {record["mmsi"] for record in persisted_records} == {999_000_001, 999_000_002} + assert all(record["vessel_type"] == 70 for record in persisted_records) + assert all(record["vessel_type_name"] == "Cargo" for record in persisted_records) diff --git a/backend/tests/test_datasource_mapping.py b/backend/tests/test_datasource_mapping.py index 9ccb8c4e..aca3d040 100644 --- a/backend/tests/test_datasource_mapping.py +++ b/backend/tests/test_datasource_mapping.py @@ -1,13 +1,18 @@ from types import SimpleNamespace import pytest +from unittest.mock import AsyncMock from httpx import ASGITransport, AsyncClient from app.api.v1.datasource_config import get_ai_provider_client +from app.core.websocket import broadcaster as broadcaster_module from app.core.security import get_current_user from app.core.target_schema_registry import get_target_schema, list_target_schemas from app.main import app from app.models.user import User +from app.models.datasource_config import DataSourceConfig +from app.services import custom_datasource_runtime +from app.services.custom_datasource_runtime import run_mapped_websocket_config from app.services.datasource_mapping import execute_mapping, persist_mapped_records, redact_for_llm @@ -106,6 +111,130 @@ async def test_persist_mapped_records_writes_generic_records(): assert db.added[0].extra_data["mapping_version"] == 3 +@pytest.mark.asyncio +async def test_persist_mapped_vessel_records_writes_raw_and_broadcasts(monkeypatch): + record_observation = AsyncMock(return_value=object()) + update_health = AsyncMock() + broadcast_custom = AsyncMock() + monkeypatch.setattr( + "app.services.vessel_ais_aggregation.record_vessel_ais_observation", + record_observation, + ) + monkeypatch.setattr( + "app.services.vessel_ais_aggregation.update_ais_source_health", + update_health, + ) + monkeypatch.setattr(broadcaster_module, "broadcast_custom", broadcast_custom) + + class FakeDB: + def __init__(self): + self.committed = False + + async def commit(self): + self.committed = True + + db = FakeDB() + + count = await persist_mapped_records( + db, + datasource_name="mock_ais_ws", + datasource_config_id=42, + target_schema="vessel_ais", + records=[ + { + "mmsi": 999000001, + "lat": 31.2, + "lon": 121.4, + "name": "MOCK VESSEL 001", + "received_at": "2026-05-01T00:00:00Z", + } + ], + mapping_version=1, + delivery_mode="realtime_stream", + transport="websocket", + ) + + assert count == 1 + assert db.committed is True + record_observation.assert_awaited_once() + assert record_observation.await_args.kwargs["source"] == "mock_ais_ws" + assert record_observation.await_args.kwargs["delivery_mode"] == "realtime_stream" + assert record_observation.await_args.kwargs["transport"] == "websocket" + update_health.assert_awaited_once() + broadcast_custom.assert_awaited_once() + assert broadcast_custom.await_args.args[0] == "vessels" + assert broadcast_custom.await_args.args[1]["vessels"][0]["mmsi_display"] == "999000001" + + +@pytest.mark.asyncio +async def test_custom_websocket_runner_maps_and_persists_vessel_records(monkeypatch): + mapping = SimpleNamespace( + id=7, + version=2, + target_schema="vessel_ais", + mapping_json={ + "source": {"items_path": "$"}, + "fields": { + "mmsi": {"path": "$.mmsi", "type": "integer"}, + "lat": {"path": "$.lat", "type": "float"}, + "lon": {"path": "$.lon", "type": "float"}, + "name": {"path": "$.name", "type": "string"}, + "received_at": {"path": "$.received_at", "type": "datetime"}, + }, + }, + ) + + class FakeResult: + def scalar_one_or_none(self): + return mapping + + class FakeDB: + async def execute(self, _stmt): + return FakeResult() + + class FakeWebSocket: + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + return None + + async def send(self, _message): + return None + + async def recv(self): + return ( + '{"type":"vessel","data":{"mmsi":"999000001","name":"MOCK VESSEL 001",' + '"lat":31.2,"lon":121.4,"received_at":"2026-05-01T00:00:00Z"}}' + ) + + persist = AsyncMock(return_value=1) + monkeypatch.setattr(custom_datasource_runtime, "_connect_websocket", AsyncMock(return_value=FakeWebSocket())) + monkeypatch.setattr(custom_datasource_runtime, "persist_mapped_records", persist) + + result = await run_mapped_websocket_config( + FakeDB(), + DataSourceConfig( + id=42, + name="mock_ais_ws", + source_type="websocket", + endpoint="ws://localhost:8787/ais", + auth_type="none", + headers={}, + config={"ws_message_path": "$.data", "debug_max_messages": 1}, + ), + ) + + assert result["status"] == "success" + assert result["messages_seen"] == 1 + assert result["written_count"] == 1 + persist.assert_awaited_once() + assert persist.await_args.kwargs["datasource_name"] == "mock_ais_ws" + assert persist.await_args.kwargs["records"][0]["mmsi"] == 999000001 + assert persist.await_args.kwargs["delivery_mode"] == "realtime_stream" + assert persist.await_args.kwargs["transport"] == "websocket" + + @pytest.mark.asyncio async def test_mapping_preview_api_uses_deterministic_engine(): def override_get_current_user(): diff --git a/backend/tests/test_vessel_aggregation_strategy.py b/backend/tests/test_vessel_aggregation_strategy.py new file mode 100644 index 00000000..a849c348 --- /dev/null +++ b/backend/tests/test_vessel_aggregation_strategy.py @@ -0,0 +1,161 @@ +"""Tests for the v4 vessel_ais aggregation strategy.""" + +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock + +import pytest + +from app.models.vessel import AISRawObservation +from app.services.vessel_aggregation_strategy import ( + DEFAULT_STRATEGY, + StrategyValidationError, + validate_strategy, +) +from app.services.vessel_ais_aggregation import aggregate_vessel_observations + + +def _obs(*, source: str, mmsi: int, observed_at: datetime, **payload) -> AISRawObservation: + payload = {"mmsi": mmsi, "lat": 50.0, "lon": 10.0, **payload} + delivery_mode = "realtime_stream" if source == "aisstream_vessels" else "polling" + transport = "websocket" if source == "aisstream_vessels" else "http" + return AISRawObservation( + target_schema="vessel_ais", + source=source, + entity_key=str(mmsi), + delivery_mode=delivery_mode, + transport=transport, + message_type="PositionReport", + observation_hash=f"{source}:{mmsi}:{observed_at.isoformat()}", + observed_at=observed_at, + collected_at=observed_at, + normalized_payload=payload, + raw_payload=payload, + quality_flags=[], + ) + + +def test_validate_rejects_unknown_field(): + with pytest.raises(StrategyValidationError, match="unknown vessel_ais field"): + validate_strategy({"vessel_ais": {"field_rules": {"definitely_not_a_field": {"mode": "newest"}}}}) + + +def test_validate_rejects_dynamic_lock_without_flag(): + with pytest.raises(StrategyValidationError, match="allow_dynamic_lock"): + validate_strategy( + { + "vessel_ais": { + "field_rules": {"lat": {"mode": "source_priority"}}, + "allow_dynamic_lock": False, + } + } + ) + + +def test_validate_allows_dynamic_lock_with_flag(): + normalized = validate_strategy( + { + "version": 0, + "vessel_ais": { + "field_rules": {"lat": {"mode": "source_priority", "source_priority": ["barentswatch_vessels"]}}, + "allow_dynamic_lock": True, + }, + } + ) + assert normalized["vessel_ais"]["field_rules"]["lat"]["mode"] == "source_priority" + assert normalized["version"] == 1 + + +def test_validate_increments_version(): + first = validate_strategy({"version": 5, "vessel_ais": {}}) + assert first["version"] == 6 + + +@pytest.mark.asyncio +async def test_strategy_field_rule_promotes_specific_source(monkeypatch): + now = datetime(2026, 5, 4, 12, 0, tzinfo=timezone.utc) + + obs_a = _obs( + source="aisstream_vessels", + mmsi=257123000, + observed_at=now, + name="AISSTREAM ONE", + vessel_type_name="Cargo", + ) + obs_b = _obs( + source="barentswatch_vessels", + mmsi=257123000, + observed_at=now - timedelta(seconds=1), + name="BARENTSWATCH ONE", + vessel_type_name="Cargo", + ) + + strategy = { + "version": 7, + "vessel_ais": { + "source_priority": [], + "field_rules": { + "name": {"mode": "source_priority", "source_priority": ["barentswatch_vessels", "aisstream_vessels"]}, + }, + "freshness": {"realtime_stream_seconds": 0, "polling_seconds": 0}, + "allow_dynamic_lock": False, + }, + } + + db = AsyncMock() + vessels = await aggregate_vessel_observations( + db, + [obs_a, obs_b], + write_conflicts=False, + strategy=strategy, + ) + assert len(vessels) == 1 + vessel = vessels[0] + assert vessel["name"] == "BARENTSWATCH ONE" + assert vessel["field_sources"]["name"] == "barentswatch_vessels" + assert vessel["selected_reasons"]["name"] == "source_priority" + assert vessel["aggregation_strategy_version"] == 7 + + +@pytest.mark.asyncio +async def test_strategy_freshness_falls_back_to_polling_when_realtime_stale(): + now = datetime(2026, 5, 4, 12, 0, tzinfo=timezone.utc) + + stale_realtime = _obs( + source="aisstream_vessels", + mmsi=257123000, + observed_at=now - timedelta(hours=1), + lat=58.0, + lon=10.0, + ) + fresh_polling = _obs( + source="barentswatch_vessels", + mmsi=257123000, + observed_at=now - timedelta(seconds=30), + lat=60.0, + lon=11.0, + ) + + strategy = { + "version": 1, + "vessel_ais": { + "source_priority": ["aisstream_vessels", "barentswatch_vessels"], + "field_rules": {}, + "freshness": {"realtime_stream_seconds": 900, "polling_seconds": 7200}, + "allow_dynamic_lock": False, + }, + } + + db = AsyncMock() + vessels = await aggregate_vessel_observations( + db, + [stale_realtime, fresh_polling], + write_conflicts=False, + strategy=strategy, + ) + assert vessels[0]["field_sources"]["lat"] == "barentswatch_vessels" + assert vessels[0]["lat"] == 60.0 + + +def test_default_strategy_is_stable(): + assert DEFAULT_STRATEGY["vessel_ais"]["allow_dynamic_lock"] is False + assert "freshness" in DEFAULT_STRATEGY["vessel_ais"] diff --git a/backend/tests/test_vessel_enrichment.py b/backend/tests/test_vessel_enrichment.py new file mode 100644 index 00000000..74f9e1ce --- /dev/null +++ b/backend/tests/test_vessel_enrichment.py @@ -0,0 +1,155 @@ +"""Tests for v5 enrichment + conflict promote-to-rule.""" + +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock + +import pytest + +from app.models.vessel import AISConflictRecord, AISRawObservation +from app.models.vessel_enrichment import VesselMediaEnrichment, VesselProfileEnrichment +from app.services.vessel_ais_aggregation import aggregate_vessel_observations +from app.services.vessel_enrichment import ( + _apply_upsert, + get_vessel_enrichment_bundle, +) + + +class _StoreSession: + """Minimal AsyncSession stand-in that tracks mmsi-keyed enrichment + a strategy.""" + + def __init__(self, *, profile=None, media=None, conflicts=None): + self.profile = profile + self.media = media + self.conflicts = list(conflicts or []) + self.added: list = [] + self.committed = False + + async def get(self, model, key): + if model is VesselProfileEnrichment: + return self.profile if self.profile and self.profile.mmsi == key else None + if model is VesselMediaEnrichment: + return self.media if self.media and self.media.mmsi == key else None + return None + + +@pytest.mark.asyncio +async def test_enrichment_bundle_filters_expired_records(): + now = datetime.now(timezone.utc) + fresh = VesselProfileEnrichment( + mmsi=257123000, + source="local_cache", + payload={"vessel_subtype": "Container"}, + fetched_at=now - timedelta(hours=1), + expires_at=now + timedelta(days=7), + confidence=0.9, + ) + expired_media = VesselMediaEnrichment( + mmsi=257123000, + source="vesselfinder", + payload={"images": ["https://example.com/a.jpg"]}, + fetched_at=now - timedelta(days=30), + expires_at=now - timedelta(days=1), + ) + db = _StoreSession(profile=fresh, media=expired_media) + + bundle = await get_vessel_enrichment_bundle(db, 257123000) + + assert bundle["profile"]["payload"]["vessel_subtype"] == "Container" + assert bundle["media"] is None + + +def test_apply_upsert_preserves_payload_and_metadata(): + record = VesselProfileEnrichment(mmsi=257123000) + out = _apply_upsert( + record, + { + "source": "vesselfinder", + "payload": {"vessel_subtype": "Container", "operator": "Maersk"}, + "expires_at": "2026-12-31T00:00:00Z", + "confidence": 0.85, + "reference_url": "https://www.vesselfinder.com/vessels/257123000", + }, + ) + assert out["payload"]["operator"] == "Maersk" + assert out["confidence"] == 0.85 + assert record.reference_url == "https://www.vesselfinder.com/vessels/257123000" + assert record.expires_at is not None + assert record.expires_at.year == 2026 + + +def _obs(*, source: str, mmsi: int, observed_at, **payload) -> AISRawObservation: + payload = {"mmsi": mmsi, "lat": 60.0, "lon": 5.0, **payload} + delivery_mode = "realtime_stream" if source == "aisstream_vessels" else "polling" + transport = "websocket" if source == "aisstream_vessels" else "http" + return AISRawObservation( + target_schema="vessel_ais", + source=source, + entity_key=str(mmsi), + delivery_mode=delivery_mode, + transport=transport, + message_type="PositionReport", + observation_hash=f"{source}:{mmsi}:{observed_at.isoformat()}", + observed_at=observed_at, + collected_at=observed_at, + normalized_payload=payload, + raw_payload=payload, + quality_flags=[], + ) + + +@pytest.mark.asyncio +async def test_promoted_rule_wins_during_aggregation(): + """Simulate the strategy that conflict-promote-to-rule writes.""" + now = datetime.now(timezone.utc) + obs_a = _obs( + source="aisstream_vessels", + mmsi=257111000, + observed_at=now, + name="STREAM NAME", + vessel_type_name="Cargo", + ) + obs_b = _obs( + source="barentswatch_vessels", + mmsi=257111000, + observed_at=now - timedelta(seconds=1), + name="REST NAME", + vessel_type_name="Cargo", + ) + promoted_strategy = { + "version": 99, + "vessel_ais": { + "source_priority": [], + "field_rules": { + "name": {"mode": "source_priority", "source_priority": ["barentswatch_vessels"]} + }, + "freshness": {"realtime_stream_seconds": 0, "polling_seconds": 0}, + "allow_dynamic_lock": False, + }, + } + + db = AsyncMock() + vessels = await aggregate_vessel_observations( + db, + [obs_a, obs_b], + write_conflicts=False, + strategy=promoted_strategy, + ) + assert vessels[0]["name"] == "REST NAME" + assert vessels[0]["selected_reasons"]["name"] == "source_priority" + assert vessels[0]["aggregation_strategy_version"] == 99 + + +def test_conflict_record_holds_selected_source(): + """Sanity: the promote-to-rule API reads selected_source from this column.""" + record = AISConflictRecord( + target_schema="vessel_ais", + entity_key="257111000", + field="name", + candidates={"a": "X", "b": "Y"}, + selected_source="barentswatch_vessels", + selected_value="Y", + selected_reason="delivery_mode_priority", + ) + serialized = record.to_dict() + assert serialized["selected_source"] == "barentswatch_vessels" + assert serialized["field"] == "name" diff --git a/backend/tests/test_vessels.py b/backend/tests/test_vessels.py index 76b92573..6f37efac 100644 --- a/backend/tests/test_vessels.py +++ b/backend/tests/test_vessels.py @@ -4,6 +4,7 @@ from unittest.mock import AsyncMock import pytest from httpx import ASGITransport, AsyncClient +from app.api.v1 import visualization from app.api.v1.visualization import convert_vessels_to_geojson from app.db.session import get_db from app.main import app @@ -200,11 +201,12 @@ async def test_aggregate_vessel_observations_prefers_realtime_and_records_confli @pytest.mark.asyncio -async def test_vessel_collector_writes_raw_observations_without_changing_position_save(monkeypatch): +async def test_vessel_collector_writes_raw_observations_only(monkeypatch): collector = VesselAISCollector() collector.update_progress = AsyncMock() record_observation = AsyncMock() update_health = AsyncMock() + broadcast_custom = AsyncMock() monkeypatch.setattr( "app.services.collectors.vessel_ais.record_vessel_ais_observation", record_observation, @@ -213,6 +215,10 @@ async def test_vessel_collector_writes_raw_observations_without_changing_positio "app.services.collectors.vessel_ais.update_ais_source_health", update_health, ) + monkeypatch.setattr( + "app.services.collectors.vessel_ais.broadcaster.broadcast_custom", + broadcast_custom, + ) class _Session: def __init__(self): @@ -249,12 +255,17 @@ async def test_vessel_collector_writes_raw_observations_without_changing_positio assert saved == 1 assert db.committed is True - assert any(isinstance(item, VesselStatic) for item in db.added) - assert any(isinstance(item, VesselPosition) for item in db.added) + # BarentsWatch must funnel through the unified AIS pipeline only — no legacy writes. + assert not any(isinstance(item, VesselStatic) for item in db.added) + assert not any(isinstance(item, VesselPosition) for item in db.added) record_observation.assert_awaited_once() assert record_observation.await_args.kwargs["source"] == "barentswatch_vessels" assert record_observation.await_args.kwargs["normalized_payload"]["mmsi"] == 257123000 update_health.assert_awaited_once() + broadcast_custom.assert_awaited_once() + assert broadcast_custom.await_args.args[0] == "vessels" + assert broadcast_custom.await_args.args[1]["action"] == "upsert" + assert broadcast_custom.await_args.args[1]["vessels"][0]["mmsi_display"] == "257123000" def test_aisstream_collector_normalizes_position_report(): @@ -365,6 +376,48 @@ async def test_aisstream_collector_writes_only_raw_observations(monkeypatch): update_health.assert_awaited_once() +@pytest.mark.asyncio +async def test_aisstream_stream_record_broadcasts_vessel_delta(monkeypatch): + collector = AISStreamCollector() + record_observation = AsyncMock(return_value=object()) + update_health = AsyncMock() + broadcast_custom = AsyncMock() + monkeypatch.setattr( + "app.services.collectors.aisstream.record_vessel_ais_observation", + record_observation, + ) + monkeypatch.setattr( + "app.services.collectors.aisstream.update_ais_source_health", + update_health, + ) + monkeypatch.setattr( + "app.services.collectors.aisstream.broadcaster.broadcast_custom", + broadcast_custom, + ) + + class _Session: + async def commit(self): + pass + + created = await collector._save_stream_record( + _Session(), + { + "mmsi": 257123000, + "lat": 59.91, + "lon": 10.73, + "cog": 214, + "received_at": datetime(2026, 4, 30, 12, 0, tzinfo=timezone.utc), + }, + ) + + assert created is True + record_observation.assert_awaited_once() + broadcast_custom.assert_awaited_once() + assert broadcast_custom.await_args.args[0] == "vessels" + assert broadcast_custom.await_args.args[1]["action"] == "upsert" + assert broadcast_custom.await_args.args[1]["vessels"][0]["mmsi_display"] == "257123000" + + def test_barentswatch_reads_credentials_from_zshrc(tmp_path): zshrc = tmp_path / ".zshrc" zshrc.write_text( @@ -436,6 +489,39 @@ def test_convert_vessels_to_geojson(): assert payload["features"][0]["properties"]["vessel_type_name"] == "Cargo" +def test_convert_vessels_to_geojson_dedupes_mmsi_rows(): + first = VesselPosition( + mmsi=257123000, + lat=59.91, + lon=10.73, + received_at=datetime(2026, 4, 28, 1, 0, tzinfo=timezone.utc), + ) + duplicate = VesselPosition( + mmsi=257123000, + lat=60.01, + lon=10.83, + received_at=datetime(2026, 4, 28, 1, 0, tzinfo=timezone.utc), + ) + other = VesselPosition( + mmsi=257456000, + lat=60.3, + lon=5.3, + received_at=datetime(2026, 4, 28, 0, 59, tzinfo=timezone.utc), + ) + + payload = convert_vessels_to_geojson( + [ + (first, VesselStatic(mmsi=257123000, name="OSLO TRADER")), + (duplicate, VesselStatic(mmsi=257123000, name="OSLO TRADER DUP")), + (other, VesselStatic(mmsi=257456000, name="BERGEN FERRY")), + ] + ) + + mmsis = [feature["properties"]["mmsi"] for feature in payload["features"]] + assert mmsis == [257123000, 257456000] + assert payload["features"][0]["geometry"]["coordinates"] == [10.73, 59.91] + + @pytest.mark.asyncio async def test_vessels_geojson_endpoint_filters_type_and_bbox(): now = datetime(2026, 4, 28, 1, 0, tzinfo=timezone.utc) @@ -477,3 +563,113 @@ async def test_vessels_geojson_endpoint_filters_type_and_bbox(): assert data["stats"]["by_type"]["Cargo"] == 1 finally: app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_vessels_geojson_merges_raw_and_legacy_sources(monkeypatch): + now = datetime(2026, 4, 28, 1, 0, tzinfo=timezone.utc) + monkeypatch.setattr( + visualization, + "get_aggregated_vessels", + AsyncMock( + return_value=[ + { + "mmsi": 1, + "lat": 59.9, + "lon": 10.7, + "received_at": now, + "name": "AISSTREAM SHIP", + "vessel_type_name": "Cargo", + "source_summary": {"aisstream_vessels": {"message_types": ["PositionReport"]}}, + } + ] + ), + ) + rows = [ + ( + VesselPosition(mmsi=1, lat=60.0, lon=10.8, received_at=now), + VesselStatic(mmsi=1, name="LEGACY DUP", vessel_type_name="Cargo"), + ), + ( + VesselPosition(mmsi=2, lat=60.3, lon=5.3, received_at=now), + VesselStatic(mmsi=2, name="BARENTSWATCH ONLY", vessel_type_name="Passenger"), + ), + ] + + class _Result: + def all(self): + return rows + + class _FakeSession: + async def execute(self, _query): + return _Result() + + async def override_get_db(): + yield _FakeSession() + + app.dependency_overrides[get_db] = override_get_db + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/v1/visualization/geo/vessels") + + assert response.status_code == 200 + data = response.json() + names = {feature["properties"]["mmsi"]: feature["properties"]["name"] for feature in data["features"]} + assert data["count"] == 2 + assert names == {1: "AISSTREAM SHIP", 2: "BARENTSWATCH ONLY"} + assert data["diagnostics"]["legacy_backfilled_mmsi"] == 1 + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_vessel_name_fallbacks_reports_mmsi_display_names(monkeypatch): + now = datetime(2026, 4, 28, 1, 0, tzinfo=timezone.utc) + monkeypatch.setattr( + visualization, + "get_aggregated_vessels", + AsyncMock( + return_value=[ + { + "mmsi": 257123000, + "lat": 59.9, + "lon": 10.7, + "received_at": now, + "name": "MMSI 257123000", + "vessel_type_name": "Other", + "source_summary": { + "aisstream_vessels": { + "latest_observed_at": now, + "message_types": ["PositionReport"], + } + }, + } + ] + ), + ) + + class _Result: + def all(self): + return [] + + class _FakeSession: + async def execute(self, _query): + return _Result() + + async def override_get_db(): + yield _FakeSession() + + app.dependency_overrides[get_db] = override_get_db + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/v1/visualization/vessels/name-fallbacks") + + assert response.status_code == 200 + data = response.json() + assert data["count"] == 1 + assert data["items"][0]["mmsi"] == "257123000" + assert data["items"][0]["message_types"] == ["PositionReport"] + finally: + app.dependency_overrides.clear() diff --git a/backend/tests/test_visualization_compute_centers.py b/backend/tests/test_visualization_compute_centers.py index 85c5f7ec..5253e9c7 100644 --- a/backend/tests/test_visualization_compute_centers.py +++ b/backend/tests/test_visualization_compute_centers.py @@ -292,6 +292,9 @@ async def test_visualization_geo_summary_returns_counts(monkeypatch): def scalar(self): return self._scalar_value + def all(self): + return list(self._rows) + def scalars(self): class _Scalars: def __init__(self, rows): @@ -304,13 +307,18 @@ async def test_visualization_geo_summary_returns_counts(monkeypatch): class _FakeSession: async def execute(self, query): - query_text = str(query) + query_text = str(query).lower() if "bgp_incidents" in query_text: return _ScalarResult(scalar_value=2) if "bgp_anomalies" in query_text: return _ScalarResult(scalar_value=3) + if "ais_raw_observations" in query_text or "vessel_position" in query_text: + return _ScalarResult(rows=[]) return _ScalarResult(rows=records) + async def get(self, *_args, **_kwargs): + return None + async def override_get_db(): yield _FakeSession() diff --git a/backend/tests/test_websocket_manager.py b/backend/tests/test_websocket_manager.py new file mode 100644 index 00000000..498447a9 --- /dev/null +++ b/backend/tests/test_websocket_manager.py @@ -0,0 +1,46 @@ +import pytest + +from app.core.websocket.manager import ConnectionManager + + +class FakeWebSocket: + def __init__(self): + self.accepted = False + self.sent = [] + self.closed = False + + async def accept(self): + self.accepted = True + + async def send_json(self, message): + self.sent.append(message) + + async def close(self): + self.closed = True + + +@pytest.mark.asyncio +async def test_channel_subscribers_receive_channel_broadcasts(): + manager = ConnectionManager() + socket = FakeWebSocket() + + await manager.connect(socket, "user-1") + manager.subscribe(socket, ["dashboard"]) + await manager.broadcast({"type": "data_frame", "channel": "dashboard"}, channel="dashboard") + + assert socket.accepted is True + assert socket.sent == [{"type": "data_frame", "channel": "dashboard"}] + + +@pytest.mark.asyncio +async def test_disconnect_removes_channel_subscriptions(): + manager = ConnectionManager() + socket = FakeWebSocket() + + await manager.connect(socket, "user-1") + manager.subscribe(socket, ["dashboard"]) + manager.disconnect(socket, "user-1") + await manager.broadcast({"type": "data_frame", "channel": "dashboard"}, channel="dashboard") + + assert socket.sent == [] + assert "dashboard" not in manager.channel_subscriptions diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 61ec5bd7..0affca98 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -8,6 +8,22 @@ This project follows the repository versioning rule: - `improvement` -> `+0.0.1`（bugfix + 小功能混合） - `bugfix` -> `+0.0.1` +## [0.48.0] — 2026-05-07 + +Released: 2026-05-07 + +### ✨ Highlights +- 自定义数据源新增 REST / WebSocket 映射运行时，并提供本地 AIS mock WebSocket，用于实时船只 upsert 链路验证。 +- AIS 原始观测、聚合策略、字段来源、冲突记录与船舶 enrichment 继续完善，Earth 船只实时展示链路更接近生产数据形态。 +- Earth 全球态势 summary 改为轻量 SQL 聚合，并在卫星 current 异常时回退到最近有效 TLE 批次，避免统计接口被大规模明细读取拖慢。 + +### 🔧 Improvements +- 修复 `/geo/summary` 与 `/geo/satellites` 在大表下加载慢或超时的问题，并补充 `collected_data` 与 AIS raw 相关索引。 +- WebSocket 管理器支持匿名连接、频道订阅清理和更稳的连接生命周期测试，前端 WebSocket candidates / fallback 更可靠。 +- `planet.sh` 强化端口释放、端口诊断和前端启动流程，mock AIS server 提供 Bun 脚本入口。 + +--- + ## [0.47.0] — 2026-04-30 Released: 2026-04-30 diff --git a/docs/plans/custom-source-live-mock-plan.md b/docs/plans/custom-source-live-mock-plan.md new file mode 100644 index 00000000..1e6fb231 --- /dev/null +++ b/docs/plans/custom-source-live-mock-plan.md @@ -0,0 +1,384 @@ +# Custom Source Live Mock 计划 + +**状态**：实施中 +**创建日期**：2026-05-01 +**任务名**：`Custom Source Live Mock` +**核心目标**：把自定义源升级为同时支持 REST 与 WebSocket 的可映射采集入口，并提供本地 AIS mock WebSocket 服务，用于验证 Earth 船只实时新增与 upsert 链路。 + +## 背景 + +真实 AIS 接口变化频率不可控，无法稳定验证 Earth 页面“不刷新也能看到新船只”的实时链路。当前系统已经有自定义源基础设施： + +- `datasource_configs` 保存 endpoint、auth、headers、config。 +- `datasource_mapping_templates` 保存目标 schema 的确定性映射模板。 +- `run-mapped` 支持保存后的自定义 REST 源通过 active mapping 写入目标数据。 + +但现有能力主要面向 REST sample 和批量 mapping，缺少以下能力： + +- 自定义源不能明确选择 `REST` 或 `WebSocket` 采集模式。 +- WebSocket 长连接、订阅消息、重连、消息路径提取还没有通用 runtime。 +- `vessel_ais` 自定义数据写入后需要进入 AIS raw observation 和 `vessels` WS channel，才能真实验证 Earth 实时 upsert。 +- 删除自定义源时没有清晰的数据清理选项。 +- 设置中心里“采集调度 / 凭证 / 自定义源”入口混杂，用户很难判断该在哪里配置。 + +## 已确认决策 + +| 项目 | 决策 | +|-----|------| +| 计划名称 | `Custom Source Live Mock` | +| 自定义源传输类型 | 支持 `REST` 与 `WebSocket` | +| 采集写入方式 | 先映射到目标 schema，再由 destination handler 写入 | +| AIS mock 目标 | 优先打通 `vessel_ais`，验证 Earth 船只实时新增和同 MMSI upsert | +| mock 服务 runtime | 使用 `bun` 启动本地 mock WS 服务 | +| 凭证配置 | 支持 headers、bearer、api key、basic，并保留 query/header API key 位置配置 | +| 删除策略 | 删除自定义源时允许选择是否删除该源写入的数据 | +| 合并语义 | 自定义源必须选择“合并到哪个内置数据”，作为内置源的补充数据进入同一聚合链路 | +| UI 方向 | 自定义源创建和维护放在“配置中心 > 采集器设置”的采集器下拉框内联入口；数据源页保留总览与运行控制 | + +## 范围 + +### 本阶段要做 + +- 自定义源可选择 `REST` 或 `WebSocket`。 +- 自定义源支持请求头、凭证、query params、body、WS subscribe message。 +- WebSocket 自定义源支持长连接、重连、消息解析、mapping、写入。 +- `vessel_ais` 自定义源写入 AIS raw observations，并广播 `vessels` channel。 +- 提供 mock AIS WS 服务，持续发送新增 MMSI 和位置变更。 +- 删除自定义源时提供“是否删除该源数据”的选项。 +- 梳理设置中心信息架构，明确后续 UI 重构方向。 + +### 暂不做 + +- 不新增任意动态数据库表。 +- 不允许用户提交可执行脚本作为 mapping。 +- 不让 LLM 进入正式采集链路。 +- 不把 mock 数据直接写 legacy `vessel_position`，优先写 AIS raw observations，保持可追踪和可删除。 +- 不在本阶段完成完整 `Earth Live Sync`，但要为后续 summary invalidation 留出 hook。 + +## 现状入口 + +| 能力 | 当前位置 | +|-----|----------| +| 自定义源配置模型 | `backend/app/models/datasource_config.py` | +| 自定义源 mapping 模型 | `backend/app/models/datasource_mapping.py` | +| 自定义源 API | `backend/app/api/v1/datasource_config.py` | +| 目标 schema registry | `backend/app/core/target_schema_registry.py` | +| mapping engine | `backend/app/services/datasource_mapping.py` | +| 数据源总览 UI | `frontend/src/pages/DataSources/DataSources.tsx` | +| 采集器设置 UI | `frontend/src/pages/Settings/Settings.tsx` | + +## 目标架构 + +```mermaid +flowchart LR + A[Custom Source Config] --> B{source_type} + B -->|rest| C[Mapped REST Runner] + B -->|websocket| D[Mapped WS Runner] + C --> E[Mapping Engine] + D --> E + E --> F[Target Schema Validator] + F --> G{Destination Handler} + G -->|vessel_ais| H[AIS Raw Observations] + H --> I[AIS Aggregation] + H --> J[vessels WS Channel] + J --> K[Earth Vessel Upsert] +``` + +## 数据配置设计 + +短期可以继续复用 `DataSourceConfig`，避免大迁移。语义约定如下： + +| 字段 | 用途 | +|-----|------| +| `name` | 自定义源唯一名称，例如 `mock_ais_ws` | +| `source_type` | `rest` 或 `websocket` | +| `endpoint` | `http(s)://...` 或 `ws(s)://...` | +| `auth_type` | `none`、`bearer`、`api_key`、`basic` | +| `auth_config` | token、api_key、key name、basic username/password 等 | +| `headers` | 静态请求头 | +| `config` | method、params、body、timeout、retry、WS 订阅消息、重连策略、消息路径等 | + +建议 `config` 结构： + +```json +{ + "transport": "websocket", + "delivery_mode": "realtime_stream", + "merge_target_source": "barentswatch_vessels", + "target_schema": "vessel_ais", + "method": "GET", + "params": {}, + "body": null, + "timeout": 30, + "retry": 3, + "ws_subscribe_message": {"type": "subscribe", "channel": "vessels"}, + "ws_message_path": "$.data", + "ws_items_path": "$.vessels[*]", + "ws_reconnect": true, + "reconnect_delay_seconds": 3, + "debug_max_messages": null, + "delete_policy": "config_only" +} +``` + +## 后端实施计划 + +### Phase 1 — 自定义源类型与连接测试 + +- 允许 `source_type` 为 `rest` 或 `websocket`。 +- REST 连接测试保留现有 HTTP 请求逻辑。 +- WebSocket 连接测试新增： + - 校验 endpoint 必须是 `ws://` 或 `wss://`。 + - 注入 headers 和 auth。 + - 连接后可选发送 `ws_subscribe_message`。 + - 读取一条消息或超时返回诊断。 + +### Phase 2 — Mapped REST Runner 补齐 + +现有 `run-mapped` 继续作为 REST 一次性采集入口，补齐： + +- `GET/POST` method。 +- query params。 +- JSON body。 +- headers 和 auth 注入。 +- sample limit 与响应大小限制。 +- `vessel_ais` destination handler。 + +### Phase 3 — Mapped WebSocket Runner + +新增通用 WebSocket runner，读取 `DataSourceConfig + active mapping`： + +- 建立长连接。 +- 发送可选订阅消息。 +- 循环接收消息。 +- JSON parse。 +- 按 `ws_message_path/ws_items_path` 提取 item 或 list。 +- 使用 mapping engine 转换。 +- 使用 target schema validator 校验。 +- 调用 destination handler 写入。 +- 更新采集任务状态： + - `connecting` + - `streaming` + - `reconnecting` + - `stopped` +- 维护运行指标： + - `messages_seen` + - `records_written` + - `unique_entities` + - `last_message_at` + - `last_error` +- 后台长连接不读取 `config.debug_max_messages`；该字段只用于显式的一次性调试运行，避免正式 WS 流被测试上限截断。 + +### Phase 4 — Destination Handler + +为 target schema 建立明确写入处理器。 + +`vessel_ais` handler： + +- 写入 `AISRawObservation`。 +- `source = datasource.name`。 +- `delivery_mode` 来自 config，默认 WS 为 `realtime_stream`、REST 为 `polling`。 +- `transport` 来自 `source_type`。 +- 生成幂等 observation hash。 +- 更新 AIS source health。 +- 广播 `vessels` channel，payload 使用当前 Earth 已支持的 upsert 格式。 + +`generic_records` handler： + +- 写入通用 collected data 或后续 generic store。 +- 不直接进入 Earth。 + +### Phase 5 — 删除与数据清理 + +删除自定义源时新增清理策略： + +| 选项 | 行为 | +|-----|------| +| 只删除配置 | 删除 `datasource_configs`，保留 mapping 和历史数据需要另行处理 | +| 删除配置和 mapping | 删除配置及对应 `datasource_mapping_templates` | +| 删除配置、mapping 和该源数据 | 同时删除该源写入的数据 | + +数据删除范围： + +- `collected_data.source == datasource.name` +- `ais_raw_observations.source == datasource.name` +- `ais_source_health.source == datasource.name` + +不建议直接删除 legacy `vessel_position`，因为当前 legacy 表不带 source，无法安全归因。自定义 AIS 源应优先只写 raw observations。 + +删除数据后应触发： + +- `vessels` channel 的 reload/invalidation 事件，提示 Earth 重新拉船只聚合。 +- 后续接入 `Earth Live Sync` 后，触发 `earth_summary` invalidation。 + +### Phase 6 — Mock AIS WebSocket 服务 + +新增脚本： + +`scripts/mock-ais-ws-server.ts` + +运行方式建议： + +```bash +bun run mock:ais-ws +``` + +服务行为： + +- 监听 `ws://localhost:8787/ais`。 +- 接受任意客户端连接。 +- 可记录收到的 subscribe message。 +- 每 1-2 秒发送一条 AIS-like JSON。 +- 每隔 N 条生成新 MMSI，验证船只数量增长。 +- 已存在 MMSI 随时间改变 `lat/lon/cog/heading`，验证同 MMSI upsert。 +- 支持固定 seed，保证测试可复现。 + +示例 payload： + +```json +{ + "type": "vessel", + "data": { + "mmsi": "999000001", + "name": "MOCK VESSEL 001", + "lat": 31.23, + "lon": 121.47, + "sog": 12.4, + "cog": 86, + "heading": 90, + "received_at": "2026-05-01T00:00:00Z" + } +} +``` + +## 前端实施计划 + +### 信息架构调整 + +自定义源不作为割裂的新入口，而是作为内置采集器的补充源，直接纳入“配置中心 > 采集器设置”的采集器选择器： + +- 采集器下拉框同时展示内置采集器和自定义补充源。 +- 下拉框右侧提供加号按钮，用于添加自定义源。 +- 新建自定义源时必须选择“合并到内置数据”，例如合并到 `barentswatch_vessels`。 +- 选择自定义源后，右侧基础配置区域沿用正常采集器配置形态，支持连接测试、保存、endpoint、headers、auth、高级 JSON。 +- 自定义源比内置源多一个“删除自定义源”按钮。 +- 删除时弹出确认框，可勾选“同时删除该自定义源生成的所有数据”。 + +数据源页保留： + +- 内置源总览。 +- 内置源最近状态。 +- 内置源手动触发。 +- 不展示自定义源管理入口；自定义源创建、维护、删除统一在采集器设置中完成。 + +### 自定义源表单 + +新增或重构自定义源表单： + +- 源名称。 +- 类型：`REST` / `WebSocket`。 +- 合并到内置数据：必选，用于声明该源补充哪个内置数据域。 +- endpoint。 +- method/body/params，仅 REST 显示。 +- subscribe message/message path/items path，仅 WS 显示。 +- auth type。 +- headers。 +- target schema。 +- sample/test 按钮。 +- mapping assistant/preview。 +- 保存并运行。 + +### 删除确认 + +删除自定义源时弹出确认： + +- 默认只删除配置。 +- 可勾选删除 mapping。 +- 可勾选删除该源写入的数据。 +- 显示将删除的数据范围和不可恢复提示。 + +## 验证方案 + +### Mock WS 验证路径 + +1. 启动 mock 服务： + +```bash +bun run mock:ais-ws +``` + +2. 新建自定义源： + +| 字段 | 值 | +|-----|----| +| name | `mock_ais_ws` | +| source_type | `websocket` | +| endpoint | `ws://localhost:8787/ais` | +| merge_target_source | `barentswatch_vessels` | +| target_schema | `vessel_ais` | +| ws_message_path | `$.data` | + +3. 保存 active mapping： + +```json +{ + "source": { + "items_path": "$" + }, + "fields": { + "mmsi": {"path": "$.mmsi", "type": "integer"}, + "name": {"path": "$.name", "type": "string"}, + "lat": {"path": "$.lat", "type": "float"}, + "lon": {"path": "$.lon", "type": "float"}, + "sog": {"path": "$.sog", "type": "float", "default": null}, + "cog": {"path": "$.cog", "type": "float", "default": null}, + "heading": {"path": "$.heading", "type": "integer", "default": null}, + "received_at": {"path": "$.received_at", "type": "datetime", "default": null} + } +} +``` + +4. 启动自定义源。 + +5. 打开 Earth 船只图层，不刷新页面观察： + +- `vessels` WS channel 收到 `source = mock_ais_ws`。 +- HUD 船只数在新 MMSI 到达时增加。 +- 地球出现 `MOCK VESSEL`。 +- 同 MMSI 后续消息更新位置和航向，不重复叠加。 + +### 自动化测试 + +后端测试： + +- WebSocket 自定义源连接测试。 +- WS message path 和 items path 提取。 +- mapping 到 `vessel_ais`。 +- 写入 AIS raw observation。 +- 广播 `vessels` channel。 +- 删除自定义源时按策略删除 mapping 和源数据。 + +前端测试： + +- REST/WS 表单条件显示。 +- 删除确认选项。 +- mock 源配置保存 payload。 +- mapping preview 展示错误和成功记录。 + +## 风险与约束 + +- WebSocket 自定义源是长连接，不能沿用一次性 REST 进度条。 +- 如果 mock 源写 legacy vessel 表，删除会变得不安全，因此先只写 raw observations。 +- 自定义 WS 可能消息量很大，必须有 backpressure、日志限流和任务取消能力。 +- 任意外部 WS 不能信任 payload，必须经过 mapping 和 schema validation。 +- headers/auth 不能进入 LLM mapping prompt。 + +## 交付顺序 + +1. Mock AIS WS 服务。 +2. 后端自定义 WS runner。 +3. `vessel_ais` destination handler 和 `vessels` broadcast。 +4. 删除自定义源及数据清理。 +5. 设置中心采集器下拉框内联自定义源 UI。 +6. 配置中心信息架构重整。 +7. 与 `Earth Live Sync` 对接 summary invalidation。 diff --git a/docs/plans/earth-vessel-ais-aggregation-plan.md b/docs/plans/earth-vessel-ais-aggregation-plan.md index aa877198..c46f2ac5 100644 --- a/docs/plans/earth-vessel-ais-aggregation-plan.md +++ b/docs/plans/earth-vessel-ais-aggregation-plan.md @@ -1,6 +1,6 @@ # AIS 多源采集、冲突记录与聚合接口计划 -**状态**：v0-v3 已实现，v4+ 规划中 +**状态**：v0-v3 已实现，v3.1-v3.4 为 v4/v5 前置稳定化任务，v4 / v5 已落最小可用子集 **创建日期**：2026-04-30 **核心原则**：采集器只写原始观测；去重、合并、冲突解释放在聚合接口中完成 @@ -16,6 +16,7 @@ | 过期保护 | 实时流源断流超过 freshness 窗口后，不能仅凭“实时源”身份压过更新的轮询数据 | | 源健康状态 | 聚合时必须参考采集器健康状态，不能只看配置中的理论优先级 | | 媒体富化 | 船只图片等媒体信息不进入 AIS 实时聚合主链路，后续单独做 enrichment | +| v4/v5 顺序 | 在聚合完整性、AISStream 实时链路、采集状态语义和基础身份信息显示修好之前，不进入策略配置和 enrichment UI | ## 背景 @@ -300,7 +301,7 @@ VesselFinder 等服务里的船只图片不属于 AIS 实时数据本身。图 ## 版本拆分 -计划按 5 个版本推进： +计划先按 v0-v3 建立基础能力，再用 v3.1-v3.4 修复当前稳定性缺口，最后进入 v4/v5： ### v0 — 聚合基础设施（已实现） @@ -343,27 +344,109 @@ VesselFinder 等服务里的船只图片不属于 AIS 实时数据本身。图 5. 船名标准化会读取 AISStream `MetaData.ShipName`；船型展示会从 `vessel_type_name` 和 AIS 数字 `vessel_type` 共同归一化，保证 marker 颜色、详情卡、hover 和搜索结果一致。 6. `/geo/vessels` 不再默认限制 5000 艘；不传 `limit` 或传 `limit=0` 表示全量返回，前端默认也不再二次裁剪到 5000。 -### v4 — 策略配置 +### v3.1 — 聚合完整性修复（v4 前置） + +目标是先保证“所有已采集到的船都能显示”，BarentsWatch 不因为接入 AISStream 而被 raw observation 聚合结果遮蔽。 + +当前风险是 `/geo/vessels` 只要 raw observation 聚合返回非空，就直接使用 raw 聚合结果，不再补读兼容层 `vessel_position + vessel_static`。如果 raw observation 中只存在 AISStream 的几百艘船，或 BarentsWatch 历史数据没有完整回填到 raw 层，最终 Earth 就会只显示 AISStream 子集。 + +1. `/geo/vessels` 必须合并 raw observation 聚合结果和 legacy latest position 结果。 +2. raw 与 legacy 同一 MMSI 同时存在时只显示一艘，优先使用 raw 聚合结果及其 `field_sources` / `selected_reasons`。 +3. raw 中不存在的 BarentsWatch-only MMSI 必须从 `vessel_position + vessel_static` 补齐。 +4. `bbox`、`type`、`limit` 过滤必须作用在合并后的最终集合上；不传 `limit` 或 `limit=0` 仍表示全量返回。 +5. 增加诊断统计，至少能看到 raw AISStream unique MMSI、raw BarentsWatch unique MMSI、legacy unique MMSI、final merged unique MMSI 和被 legacy 补齐的数量。 +6. 为 raw 只有 AISStream 子集、legacy 有更多 BarentsWatch 船只的场景补回归测试。 + +### v3.2 — AISStream 真实时链路（v4 前置） + +目标是把 AISStream 从“一次 collector 收一批消息后结束”改成真正的 WebSocket 长连接实时数据源，并把实时变化推送到 Earth。 + +当前 `aisstream_vessels` 只在 collector `fetch()` 中连接 `wss://stream.aisstream.io/v0/stream`，默认收 `max_messages = 500` 条后结束。这不符合 WebSocket 流式数据源的运行语义，也不能保证新船、位置变化和航向变化实时出现在前端。 + +1. 为 AISStream 增加 streaming service / long-running runner，不再依赖单次 `fetch -> transform -> save -> completed` 表达实时采集。 +2. 外部 AISStream WebSocket 保持长连接，断线后指数退避重连，并持续更新 `AISSourceHealth`。 +3. 每条或小批量 AIS 消息标准化后写入 `ais_raw_observations`，按时间或数量短周期 commit，避免长事务堆积。 +4. 将新增船只、位置变化、航向变化和静态字段补充转换成 vessel delta。 +5. 通过应用内部 `/ws` 的 `vessels` channel 广播 delta，复用 `DataBroadcaster.broadcast_custom("vessels", payload)`。 +6. Earth 前端订阅 `vessels` channel，`vessels.js` 支持按 MMSI upsert marker，而不是每次全量 reload。 +7. 船只改变航向时，前端必须更新 course bin / marker bucket，避免 marker 方向滞后。 +8. freshness 超时或 AISStream 健康异常时，动态字段可回退到 BarentsWatch 最新可用观测。 + +### v3.3 — Streaming 采集状态语义（v4 前置） + +目标是让采集页面正确表达 AISStream 这类长连接数据源，不再使用一次性 REST collector 的完成型进度条。 + +REST collector 的自然状态是 `fetch -> transform -> save -> progress 0..100 -> completed`。AISStream 的自然状态应是 `connecting -> streaming -> reconnecting -> stopped/failed`，没有固定总量，也不应在收到一批消息后显示“采集完成”。 + +1. AISStream 采集状态使用 indeterminate / streaming 状态，而不是百分比完成进度条。 +2. 设置页运行状态卡展示连接状态、已运行时长、本轮消息数、新增观测数、unique MMSI、message rate、最近消息时间、延迟和最近错误。 +3. `phase_message` 使用“正在接收 AISStream 实时消息”“重连中”“已停止”等长连接语义。 +4. 停止、重连和配置变更要有明确操作入口；配置变化后必须安全重订阅。 +5. 后端任务状态不能因为没有 `total_records` 就长期显示 `0%` 或误判失败。 +6. WebSocket 健康状态和 collector task 状态要分离：上游短暂断线是 `reconnecting`，不是普通采集任务完成或失败。 + +### v3.4 — 船只身份字段和名称聚合修复（v4 前置） + +目标是把 MMSI、IMO、callsign 这类身份编号按字符串显示，并把仍然使用 MMSI 作为船名的记录视为信息聚合未完成，而不是正常船名。 + +1. 前端详情卡、hover、搜索结果和日志中的 `mmsi`、`imo`、`callsign` 必须作为 identifier 字段展示，禁止走 `toLocaleString()` 或数字千分位格式。 +2. GeoJSON 可增加 `mmsi_display` / `imo_display` 等字符串字段，但前端仍必须对 identifier key 做兜底格式保护。 +3. 聚合服务生成船名时，不能把 `MMSI 257123000` 当成真实 `name` 的成功结果；它只能作为 display fallback。 +4. 增加诊断查询，列出所有当前仍以 MMSI 号码或 `MMSI ` 作为船只名称的记录，包括： + - `vessel_static.name` 为空或等于 MMSI fallback 的 MMSI； + - raw observation 中没有任何非空 `name` / `MetaData.ShipName` / `ShipStaticData.Name` 的 MMSI； + - 聚合结果最终 `name` 仍为 fallback 的 MMSI； + - 每个 MMSI 的可用来源、最近观测时间、message types 和缺失原因。 +5. 对这些 fallback-name 船只建立待修复集合，优先通过 AISStream `ShipStaticData`、BarentsWatch 静态字段和后续 enrichment 缓存补齐。 +6. 船只详情面板需要区分“真实船名”和“显示兜底”：真实船名缺失时展示 `MMSI ` 可以继续作为标题，但字段来源应标注为 `fallback`，避免误以为聚合成功。 +7. 为 MMSI 千分位格式、fallback-name 诊断和名称来源解释补回归测试。 + +### v4 — 策略配置（v0 可用） 目标是开放系统级配置，但仍以安全默认值兜底。 -1. 接入系统设置中的聚合策略配置。 -2. 支持 source priority、字段级规则、freshness 窗口和高级保护开关。 -3. 保存配置时校验未知字段、非法模式和危险动态字段锁定。 -4. 聚合接口返回当前命中的配置版本，方便排查。 +已落地的最小子集： -### v5 — 船舶资料 enrichment 与冲突治理 +1. 策略持久化在 `system_settings.category = 'vessel_aggregation_strategy'`，保存时自动版本递增。 +2. `app/services/vessel_aggregation_strategy.py` 暴露 `load_strategy / save_strategy / reset_strategy / validate_strategy`，并维护 `DEFAULT_STRATEGY` 兜底。 +3. 校验规则： + - 未知 `field_rules.` → `400 unknown vessel_ais field`； + - 未知 mode → `400 mode must be one of ...`； + - 动态字段（`lat/lon/sog/cog/heading/nav_status`）使用非 `newest` mode 时必须显式 `allow_dynamic_lock=true`，否则拒绝； + - `freshness.realtime_stream_seconds` / `polling_seconds` 必须为非负整数； + - `mode=locked` 必须带非空 `locked_source`。 +4. 聚合服务 `vessel_ais_aggregation.py` 在 `_select_position_observation` 中按 `freshness` 把过期实时流降级到 stale 候选；在 `_select_static_field` 中按 `field_rules.mode = source_priority / locked / newest / non_empty` 选源。 +5. 聚合输出每条 vessel 携带 `aggregation_strategy_version`，并在 `/geo/vessels` GeoJSON properties + `/vessels/{mmsi}` 详情中暴露。 +6. API： + - `GET /api/v1/vessel-aggregation/strategy` + - `PUT /api/v1/vessel-aggregation/strategy`（校验失败 400） + - `DELETE /api/v1/vessel-aggregation/strategy`（恢复默认并 bump version） + +未做项（留给 v4 后续）： + +- 系统设置 UI 中的策略编辑器尚未做，目前直接调 API； +- `transport_priority`、`quality_flags` 级别的策略尚未引入； +- `source_priority` 中的未知 source 不强校验，留给后续 warn-only 提示。 + +### v5 — 船舶资料 enrichment 与冲突治理（v0 可用） 目标是把 AIS 实时流里不稳定或低频出现的静态信息，补成可缓存、可审计的船舶资料层，同时把冲突解释变成可操作能力。 -1. 做冲突治理 UI。 -2. 支持把人工选择沉淀成字段级规则。 -3. 支持恢复默认策略。 -4. 设计 `vessel_profile_enrichment`，按 `mmsi + imo + name + callsign` 异步补充船名、船型细分、AIS 大类、旗国、尺寸、建造年份、运营方等静态资料。 -5. 设计 `vessel_media_enrichment`，异步补充船只图片和外部详情缓存。 -6. enrichment 结果必须带 `source`、`fetched_at`、`expires_at`、`confidence` 和原始引用，不覆盖 AIS 原始观测。 -7. 聚合接口只读取已缓存 enrichment；请求链路不现场抓取第三方页面，避免慢请求和授权风险。 -8. 前端船只详情面板展示已缓存资料和媒体，并标注字段来源，不阻塞 AIS 实时链路。 +已落地的最小子集： + +1. 新增模型 `app/models/vessel_enrichment.py::VesselProfileEnrichment` + `VesselMediaEnrichment`：以 `mmsi` 为主键，记录 `source / payload / fetched_at / expires_at / confidence / reference_url`；通过 `Base.metadata.create_all` 在 `init_db` 中建表。 +2. 服务 `app/services/vessel_enrichment.py` 提供 `upsert_vessel_profile_enrichment` / `upsert_vessel_media_enrichment` / `get_vessel_enrichment_bundle`；读路径只读缓存，过期记录（`expires_at < now`）直接过滤为 `None`，永不联网。 +3. 聚合接口在 `/api/v1/visualization/vessels/{mmsi}` 响应中追加 `enrichment.profile` 与 `enrichment.media` 字段（含 `source / fetched_at / expires_at / confidence / reference_url`）；命中失败时返回 `null`，不阻塞 AIS 实时链路。 +4. 冲突治理 API： + - `POST /api/v1/vessel-aggregation/conflicts/{mmsi}/{field}/promote-to-rule` 读取最近 `AISConflictRecord.selected_source`，写入 `field_rules[field] = {mode: source_priority, source_priority: []}` 并 bump version； + - `DELETE` 对应路径移除该 field 的覆盖，恢复默认。 +5. 前端 Earth `info-card.js` 渲染 `船舶资料` 区块：profile.payload 标量字段平铺、媒体 `images` 数组缩略图、来源 / 更新时间 / 置信度元数据；缓存命中失败回退到 `资料缓存中`；常规字段在 `field_sources` 命中时附带来源 tag。 + +未做项（留给 v5 后续）： + +- 没有真正的异步 enrichment 抓取作业；当前依赖外部脚本/管理 API 写入缓存； +- 冲突治理 UI 还没接入设置中心，目前只暴露 API； +- enrichment 命中状态尚未广播到 `vessels` channel，详情面板首次打开时按需请求即可。 ## 测试计划 @@ -377,6 +460,12 @@ VesselFinder 等服务里的船只图片不属于 AIS 实时数据本身。图 - 明显异常位置不会进入默认展示轨迹，并会留下 `quality_flags`。 - 同一时间窗口内多来源相近轨迹点只展示一个点。 - AISStream 重连或回放导致的重复消息不会重复进入聚合结果。 +- raw observation 聚合结果和 legacy latest position 结果会按 MMSI 合并，BarentsWatch-only 船只不会因为 AISStream 子集存在而消失。 +- 不传 `limit` 或传 `limit=0` 时，`/geo/vessels` 全量返回合并后的船只集合。 +- AISStream 长连接收到新船、位置变化和航向变化后，会通过内部 `/ws` 的 `vessels` channel 推送增量。 +- AISStream streaming 状态不会显示成固定百分比完成进度条，也不会在收到一批消息后误报采集完成。 +- `mmsi`、`imo`、`callsign` 等身份编号在前端不显示千分位符。 +- 聚合结果中仍以 MMSI fallback 作为船名的记录可以被诊断查询完整列出，并带来源和缺失原因。 - 字段级配置可以覆盖默认来源优先级。 - 聚合接口在没有冲突表时仍可返回兼容 GeoJSON。 diff --git a/docs/technical/zh/README.md b/docs/technical/zh/README.md index 4399a64a..716fb833 100644 --- a/docs/technical/zh/README.md +++ b/docs/technical/zh/README.md @@ -25,6 +25,7 @@ - [Planet 使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/manual.md)：控制台、`planet.sh`、Earth 和 Docs 的完整使用手册 - [数据源、采集器设置与连接验证](/home/ray/dev/linkong/planet/docs/technical/zh/datasource-collector-settings-connectivity.md)：数据源目录、采集器设置、连接验证、BarentsWatch 凭证链路 - [Earth 可交互图标接入](/home/ray/dev/linkong/planet/docs/technical/zh/earth-interactable-usage.md)：Earth 地表可交互图标 `Interactable` 的接口、生命周期和接入示例 +- [Earth 工具栏与浮层协同](/home/ray/dev/linkong/planet/docs/technical/zh/earth-toolbar-overlay-coordination.md)：工具栏按钮与搜索 / 设置 / 新闻 / 图层浮层之间的关闭矩阵和接入规则 不适合放入这里的内容： diff --git a/docs/technical/zh/earth-toolbar-overlay-coordination.md b/docs/technical/zh/earth-toolbar-overlay-coordination.md new file mode 100644 index 00000000..59618cb0 --- /dev/null +++ b/docs/technical/zh/earth-toolbar-overlay-coordination.md @@ -0,0 +1,94 @@ +# Earth 工具栏与浮层协同 + +本文件描述 Earth 大屏右侧工具栏按钮，以及搜索面板、设置弹窗、新闻直播面板、图层面板这几个浮层之间当前的协同规则。改交互、加按钮、调整面板时按这个表对齐，避免出现「点 A 把不该关的 B 也关了」之类的协同冲突。 + +相关入口： + +- [Earth 前端结构](/home/ray/dev/linkong/planet/docs/technical/zh/earth-frontend-context.md) +- [前端布局指南](/home/ray/dev/linkong/planet/docs/technical/zh/frontend-layout-guidelines.md) + +## 工具栏按钮目录 + +工具栏在 [index.html](/home/ray/dev/linkong/planet/frontend/public/earth/index.html) 中以 `.earth-toolbar-btn` 标识，按钮列表： + +| ID | 标题 | 类型 | 触发的浮层/动作 | +|----|------|------|------------------| +| `layer-action` | 图层 | 浮层切换 | HUD 面板 `layer-toggles`（桌面）/ 移动端抽屉 `layers` 卡 | +| `search-action` | 搜索 | 浮层切换 | 搜索面板（桌面）/ 移动端抽屉 `search` 卡 | +| `rotate-toggle` | 自动旋转 | 独立开关 | 不打开任何浮层 | +| `toggle-tv` | 新闻直播 | 浮层切换 | 媒体面板 `media-panel`（含 TV/News 两个 tab） | +| `reload-data` | 重新加载数据 | 独立动作 | 不打开任何浮层 | +| `zoom-trigger` | 缩放控制 | 浮动菜单 | 缩放 floating menu | +| `settings-trigger` | 设置 | 浮层切换 | 设置弹窗（桌面）/ 移动端抽屉 `settings` 卡 | +| `reset-view` | 重置视角 | 独立动作 | 不打开任何浮层 | +| `layout-toggle` | 最大化布局 | 独立开关 | 不打开任何浮层 | + +## 浮层协同的统一入口 + +[controls.js::closeTransientMobileOverlays](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) 是「打开 X 时该关谁」的统一协调函数。 + +调用约定：每个会进入 fullscreen-style 浮层的开启路径调用 `closeTransientMobileOverlays({ except })`，告诉协调函数「除了我这一类，其他互斥浮层一律关掉」。 + +```js +closeTransientMobileOverlays({ except: "search" }); // 搜索打开 +closeTransientMobileOverlays({ except: "settings" }); // 设置打开 +closeTransientMobileOverlays({ except: "media" }); // 新闻直播打开 +closeTransientMobileOverlays({ except: "layer-toggles" }); // 图层抽屉（移动端） +``` + +`except` 当前可取的值：`"search"`、`"settings"`、`"media"`、`"layer-toggles"`，或省略表示「全部关闭」。 + +## 关闭矩阵 + +下表描述「打开 X」时其它浮层的命运。`✓` = 关闭，`—` = 保留。 + +| 触发动作 → | 关搜索 | 关设置 | 关图层抽屉（移动端） | 关新闻/直播 | +|-----------|:------:|:------:|:--------------------:|:-----------:| +| 打开搜索 (`except: "search"`) | （自身）| ✓ | ✓ | — | +| 打开设置 (`except: "settings"`) | ✓ | （自身）| ✓ | — | +| 打开新闻/直播 (`except: "media"`) | ✓ | ✓ | ✓ | （自身）| +| 打开图层抽屉 (`except: "layer-toggles"`) | ✓ | ✓ | （自身）| ✓ | +| 全部关闭 (`except: null`) | ✓ | ✓ | ✓ | ✓ | + +读法举例： + +- 点工具栏「设置」，搜索面板和图层抽屉会被关掉，新闻/直播面板保持原状。 +- 点工具栏「图层」（移动端打开 `layers` 抽屉），搜索 / 设置 / 新闻 全关。 +- 点工具栏「新闻直播」，搜索 / 设置 / 图层抽屉全关，新闻面板自身切换为打开。 + +## 设计原则 + +下面是当前矩阵背后的几条不变量。新增浮层或调整规则时按它们对齐： + +1. **`zoom-trigger` 等浮动菜单不属于浮层。** 它们走 `bindFloatingMenu`，由 `closeFloatingMenus()` 单独管理；任何浮层打开都会先调一次 `closeFloatingMenus()`。 +2. **桌面 `layer-toggles` 是常驻 HUD 面板，不是浮层。** `closeTransientMobileOverlays` 中只有 `activeMobileDrawerId === "layer-toggles"`（移动端抽屉态）才会被关掉。所以桌面打开搜索/设置/新闻不会动图层面板，符合「桌面屏幕大、可共存」的预期。 +3. **新闻/直播面板独立于设置。** 用户切到设置改采集器时，常常想边看新闻边改配置，所以打开设置时不关新闻面板。这条是 2026-05 的协同补丁后建立的不变量；改设置打开路径时不要再去主动关 `media-panel`。 +4. **搜索和新闻面板视为「主信息浮层」，互相独立。** 搜索打开不关新闻、新闻打开不关搜索：两者面向不同任务（搜索定位 / 浏览态势新闻），允许同屏共存。如果未来 UX 上希望它们互斥，要在 `closeTransientMobileOverlays` 中**同时**改两边的规则，避免单边修改导致非对称的关闭逻辑。 +5. **移动端抽屉是 fullscreen 级别的状态。** 一旦进入移动端抽屉，无论是 `layers` / `search` / `settings` 哪一类，都会通过 `setMobileDrawerState` 关闭其它浮层。这是 mobile 单一焦点 UX 的要求。 +6. **`Escape` 键有固定的关闭顺序。** 见 [controls.js::setupKeyboardControls](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js)：搜索 → 设置 → 移动端抽屉 → 浮动菜单 → 工具栏 hub → 锁定对象。新增浮层要决定它在这个顺序中的位置。 + +## 新加按钮 / 浮层时怎么接 + +按下面的清单走，规则就不会乱： + +1. 按钮加在 [index.html](/home/ray/dev/linkong/planet/frontend/public/earth/index.html) 的 `.earth-toolbar` 容器里，class 跟齐 `floating-btn liquid-glass-surface earth-toolbar-btn`。 +2. 决定它属于哪一类： + - **独立动作**（reload / reset / rotate / layout）：直接 `bindListener`，不调任何 `closeTransientMobileOverlays`。 + - **浮动菜单**（zoom 这种 dropdown）：用 `bindFloatingMenu`，不进协同矩阵。 + - **互斥浮层**：进矩阵。 +3. 互斥浮层要做两件事： + - 在打开路径调用 `closeTransientMobileOverlays({ except: "" })`，让其他浮层主动让位。 + - 在 `closeTransientMobileOverlays` 函数体内补一条 `if (except !== "" && isYourPanelVisible()) closeYourPanel();` 让别的浮层打开时关掉自己。 +4. 如果新浮层和某个现有浮层（例如新闻面板）应当共存，参考第 3 条规则：在自己的关闭判断里 `&& except !== ""` 把对方排除掉。**不要**只单边改一处，否则关闭逻辑会非对称。 +5. 新浮层应该有 `Escape` 关闭路径，加在 `setupKeyboardControls` 中合适的位置。 +6. 移动端如果应进入抽屉态，使用 `setMobileDrawerState({ open: true, card: "" })` 而不是直接 toggle 面板。 + +## 当前实现位置 + +- 协调入口：[controls.js::closeTransientMobileOverlays](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) +- 设置浮层：[controls.js::openSettingsModal / closeSettingsModal](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) +- 搜索浮层：[controls.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js)（导入自 search 模块） +- 新闻/直播浮层：[tv.js::setTVPanelVisible](/home/ray/dev/linkong/planet/frontend/public/earth/js/tv.js)、新闻 tab 在 [news.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/news.js) +- 图层抽屉（移动端）：[controls.js::setMobileDrawerState](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) +- 浮动菜单：[controls.js::bindFloatingMenu](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js) +- 工具栏 DOM：[index.html](/home/ray/dev/linkong/planet/frontend/public/earth/index.html) diff --git a/docs/version-history.md b/docs/version-history.md index 8b69c42e..b784aa51 100644 --- a/docs/version-history.md +++ b/docs/version-history.md @@ -16,12 +16,13 @@ ## Current Version - `main` 当前主线历史推导到：`0.16.5` -- `dev` 当前开发分支历史推导到：`0.47.0` +- `dev` 当前开发分支历史推导到：`0.48.0` ## Timeline | Version | Type | Branch | Commit | Summary | | --- | --- | --- | --- | --- | +| `0.48.0` | feature | `dev` | `pending` | 新增自定义源 REST/WebSocket 实时 mock 链路，完善 AIS 多源聚合/船舶 enrichment，并将 Earth 全球态势统计改为轻量 SQL 聚合 | | `0.47.0` | feature | `dev` | `pending` | 新增 AISStream WebSocket 船只采集器、多源 AIS 原始观测聚合、采集器状态配置、船型显示修正和文档规则解耦 | | `0.46.3` | bugfix | `dev` | `pending` | 优化 Starlink footprint 拖拽性能，避免旋转地球时重复重建覆盖网格，并恢复线缆点击呼吸动画 | | `0.46.2` | bugfix | `dev` | `pending` | 修复 Earth 启动加载顺序、图层 localStorage 恢复、国界线底图语义、媒体面板、船只轨迹和 Iridium footprint 显示问题，并补充 AIS 聚合计划 | diff --git a/frontend/package.json b/frontend/package.json index 7a7728fa..141cc612 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "planet-frontend", - "version": "0.47.0", + "version": "0.48.0", "private": true, "packageManager": "bun@1", "dependencies": { diff --git a/frontend/public/earth/index.html b/frontend/public/earth/index.html index c229c566..f9e85b54 100644 --- a/frontend/public/earth/index.html +++ b/frontend/public/earth/index.html @@ -477,6 +477,10 @@ 暂无观测数据 +
+ + AISStream 未连接 +
@@ -708,6 +712,7 @@
BGP 状态
暂无观测数据
+
AISStream 未连接
diff --git a/frontend/public/earth/js/controls.js b/frontend/public/earth/js/controls.js index ba2b8019..80b9db54 100644 --- a/frontend/public/earth/js/controls.js +++ b/frontend/public/earth/js/controls.js @@ -271,7 +271,12 @@ function closeTransientMobileOverlays({ except = null } = {}) { setMobileDrawerOpen("layer-toggles", false); } - if (except !== "media" && except !== "search" && isTVPanelVisible()) { + if ( + except !== "media" + && except !== "search" + && except !== "settings" + && isTVPanelVisible() + ) { setTVPanelVisible(false); } } diff --git a/frontend/public/earth/js/info-card.js b/frontend/public/earth/js/info-card.js index 2c4d8987..70c0909e 100644 --- a/frontend/public/earth/js/info-card.js +++ b/frontend/public/earth/js/info-card.js @@ -8,6 +8,30 @@ let typewriterToken = 0; let pendingMobileDetailState = null; let mobileDetailsListenerBound = false; let renderedMobileDetailKey = null; +const IDENTIFIER_FIELD_KEYS = new Set([ + 'mmsi', + 'mmsi_display', + 'imo', + 'imo_display', + 'callsign', +]); +const MAX_VESSEL_MEDIA_TILES = 4; + +function formatInfoCardValue(field, rawValue) { + if (rawValue === undefined || rawValue === null || rawValue === '') { + return '-'; + } + let value = rawValue; + if (IDENTIFIER_FIELD_KEYS.has(field.key)) { + value = String(value); + } else if (typeof value === 'number') { + value = value.toLocaleString(); + } + if (field.unit && value !== '-') { + value = value + ' ' + field.unit; + } + return value; +} function getNewsSummaryText(data) { return (data?.summary || data?.title || '').trim() || '暂无摘要'; @@ -109,13 +133,7 @@ function renderMobileDetailContent(type, config, data) { let html = ''; for (const field of config.fields) { - let value = data[field.key]; - if (value === undefined || value === null || value === '') { - value = '-'; - } else if (typeof value === 'number') { - value = value.toLocaleString(); - } - if (field.unit && value !== '-') value = value + ' ' + field.unit; + const value = formatInfoCardValue(field, data[field.key]); html += `
${field.label} @@ -166,29 +184,95 @@ function ensureMobileDetailsListener() { function renderDefaultCardContent(content, config, data) { let html = ''; for (const field of config.fields) { - let value = data[field.key]; - - if (value === undefined || value === null || value === '') { - value = '-'; - } else if (typeof value === 'number') { - value = value.toLocaleString(); - } - - if (field.unit && value !== '-') { - value = value + ' ' + field.unit; - } - + const value = formatInfoCardValue(field, data[field.key]); + const sourceLabel = getFieldSourceLabel(data, field.key); html += `
${field.label} - ${value} + ${value}${sourceLabel}
`; } + if (config.className === 'vessel') { + html += renderVesselEnrichmentSection(data?.enrichment); + } + content.innerHTML = html; } +function getFieldSourceLabel(data, fieldKey) { + const sources = data && typeof data === 'object' ? data.field_sources : null; + if (!sources || typeof sources !== 'object') return ''; + const source = sources[fieldKey]; + if (!source) return ''; + return ` ${source}`; +} + +function renderVesselEnrichmentSection(enrichment) { + if (!enrichment || typeof enrichment !== 'object') return ''; + const profile = enrichment.profile; + const media = enrichment.media; + if (!profile && !media) { + return ` +
+
船舶资料
+
资料缓存中
+
+ `; + } + let inner = ''; + if (profile?.payload && typeof profile.payload === 'object') { + inner += renderEnrichmentPayloadRows(profile.payload); + inner += renderEnrichmentMeta('资料', profile); + } + if (media?.payload && typeof media.payload === 'object') { + if (Array.isArray(media.payload.images) && media.payload.images.length > 0) { + const tiles = media.payload.images + .slice(0, MAX_VESSEL_MEDIA_TILES) + .map((url) => `vessel media`) + .join(''); + inner += `
${tiles}
`; + } + inner += renderEnrichmentMeta('媒体', media); + } + if (!inner) { + inner = '
资料缓存中
'; + } + return ` +
+
船舶资料
+ ${inner} +
+ `; +} + +function renderEnrichmentPayloadRows(payload) { + let rows = ''; + for (const [key, value] of Object.entries(payload)) { + if (value === null || value === undefined || value === '') continue; + if (typeof value === 'object') continue; + rows += ` +
+ ${key} + ${String(value)} +
+ `; + } + return rows; +} + +function renderEnrichmentMeta(label, record) { + const parts = []; + if (record.source) parts.push(`来源 ${record.source}`); + if (record.fetched_at) parts.push(`更新 ${record.fetched_at}`); + if (record.confidence !== null && record.confidence !== undefined) { + parts.push(`置信 ${Number(record.confidence).toFixed(2)}`); + } + if (!parts.length) return ''; + return `
${label}：${parts.join(' · ')}
`; +} + // ── Mobile popup ───────────────────────────────────────────── function getMobilePopupTitle(type, data) { diff --git a/frontend/public/earth/js/layer-startup-tasks.js b/frontend/public/earth/js/layer-startup-tasks.js index baefc430..46960039 100644 --- a/frontend/public/earth/js/layer-startup-tasks.js +++ b/frontend/public/earth/js/layer-startup-tasks.js @@ -98,6 +98,8 @@ function registerBuiltinLayerStartupTasks() { function registerVesselStartupTask() { registerLayerStartupTask("vessels", (context) => async (layer) => { + if (!context.getShowVessels()) return; + context.setLoadingMessage( resolveStartupMessage(layer, "load", "正在加载船只..."), ); diff --git a/frontend/public/earth/js/main.js b/frontend/public/earth/js/main.js index e7db7ec2..1cc62ff4 100644 --- a/frontend/public/earth/js/main.js +++ b/frontend/public/earth/js/main.js @@ -180,9 +180,12 @@ import { getVesselLegendItems, getVesselMarkers, getVesselPointerIntersections as getVesselIconPointerIntersections, + getVesselRealtimeStats, loadVessels, setVesselMarkerState, showVesselTrack, + startVesselRealtime, + stopVesselRealtime, toggleVessels, updateVesselVisualState, } from "./vessels.js"; @@ -1489,6 +1492,13 @@ async function loadEarthStatsSummary({ shouldApply = () => true } = {}) { satelliteCount: toCount(stats.satellite_count), computeCenterCount: toCount(stats.compute_center_count), vesselCount: toCount(stats.vessel_count), + vesselRawUniqueMmsi: toCount(stats.vessel_raw_unique_mmsi), + vesselLegacyUniqueMmsi: toCount(stats.vessel_legacy_unique_mmsi), + aisstreamConnectionState: stats.aisstream_connection_state || null, + aisstreamLastSeenAt: stats.aisstream_last_seen_at || null, + aisstreamLagSeconds: Number.isFinite(Number(stats.aisstream_lag_seconds)) + ? Number(stats.aisstream_lag_seconds) + : null, bgpEventCount: toCount(stats.bgp_event_count), bgpIncidentCount: toCount(stats.bgp_incident_count), bgpAnomalyCount: toCount(stats.bgp_anomaly_count), @@ -2154,6 +2164,36 @@ function updateVesselHud(result = {}) { setEarthStatValue("vessel-count", `${count} 艘`); } +function formatRelativeTime(value) { + const date = value instanceof Date ? value : value ? new Date(value) : null; + if (!date || Number.isNaN(date.getTime())) return null; + const elapsedSeconds = Math.max(0, Math.round((Date.now() - date.getTime()) / 1000)); + if (elapsedSeconds < 5) return "刚刚"; + if (elapsedSeconds < 60) return `${elapsedSeconds} 秒前`; + const elapsedMinutes = Math.round(elapsedSeconds / 60); + if (elapsedMinutes < 60) return `${elapsedMinutes} 分钟前`; + return date.toLocaleTimeString("zh-CN", { hour: "2-digit", minute: "2-digit" }); +} + +function formatVesselLiveSummary() { + const stream = getVesselRealtimeStats(); + if (stream.connected) { + const lastUpdate = formatRelativeTime(stream.lastUpdateAt); + if (stream.updates > 0) { + return `AISStream 实时已连接 · ${stream.updates} 次更新${lastUpdate ? ` · ${lastUpdate}` : ""}`; + } + return "AISStream 实时已连接 · 等待首批更新"; + } + const state = earthStatsSummary?.aisstreamConnectionState; + if (state === "connected") { + const lastSeen = formatRelativeTime(earthStatsSummary?.aisstreamLastSeenAt); + return `AISStream 后台已连接${lastSeen ? ` · 最近 ${lastSeen}` : ""}`; + } + if (state === "reconnecting") return "AISStream 正在重连"; + if (state === "connecting") return "AISStream 正在连接"; + return "AISStream 未连接"; +} + function updateVesselToggleUi(enabled, vesselCount = getVesselCount()) { const vesselBtn = document.getElementById("toggle-vessels"); if (vesselBtn) { @@ -2164,6 +2204,7 @@ function updateVesselToggleUi(enabled, vesselCount = getVesselCount()) { }); } setEarthStatValue("vessel-count", `${vesselCount || 0} 艘`); + setEarthStatValue("vessel-live-summary", formatVesselLiveSummary()); } function updateCableToggleUi(enabled) { @@ -2292,6 +2333,12 @@ async function ensureVesselsEnabled() { vesselsEnabled = true; const result = await loadVessels(scene, earth); toggleVessels(true); + startVesselRealtime(earth, { + onUpdate: ({ totalCount }) => { + updateVesselToggleUi(true, totalCount); + updateStatsSummary(); + }, + }); updateVesselToggleUi(true, result.totalCount); setLegendItems("vessels", getVesselLegendItems()); refreshLegend(); @@ -2300,6 +2347,7 @@ async function ensureVesselsEnabled() { function disableVessels() { vesselsEnabled = false; + stopVesselRealtime(); toggleVessels(false); clearVesselSelection(); updateVesselToggleUi(false, 0); @@ -2334,6 +2382,7 @@ function updateStatsSummary() { landingPointCount: `${landingPointCount}个`, satelliteCount: `${satelliteCount} 颗`, vesselCount: `${vesselCount} 艘`, + vesselLiveSummary: formatVesselLiveSummary(), computeCenterCount: `${computeCenterCount} 个`, bgpAnomalyCount: `${bgpEventCount} 起`, bgpCollectorCount: `${bgpCollectorCount} 个`, diff --git a/frontend/public/earth/js/ui.js b/frontend/public/earth/js/ui.js index 03147dd9..fe6d829c 100644 --- a/frontend/public/earth/js/ui.js +++ b/frontend/public/earth/js/ui.js @@ -218,6 +218,7 @@ export function updateEarthStats(stats) { setEarthStatValue("compute-center-count", String(stats.computeCenterCount || 0)); } if (has("vesselCount")) setEarthStatValue("vessel-count", String(stats.vesselCount || 0)); + if (has("vesselLiveSummary")) setEarthStatValue("vessel-live-summary", stats.vesselLiveSummary || "-"); if (has("bgpAnomalyCount")) setEarthStatValue("bgp-anomaly-count", String(stats.bgpAnomalyCount || 0)); if (has("bgpCollectorCount")) { setEarthStatValue("bgp-collector-count", String(stats.bgpCollectorCount || 0)); diff --git a/frontend/public/earth/js/vessels.js b/frontend/public/earth/js/vessels.js index fa2974e6..ed5b8a6f 100644 --- a/frontend/public/earth/js/vessels.js +++ b/frontend/public/earth/js/vessels.js @@ -6,6 +6,15 @@ import { latLonToVector3 } from "./utils.js"; let showVessels = false; let activeTrackLine = null; +let vesselStreamSocket = null; +let vesselStreamReconnectTimer = null; +let vesselDataByKey = new Map(); +let vesselRealtimeStats = { + connected: false, + updates: 0, + lastUpdateAt: null, + lastBatchSize: 0, +}; const VESSEL_RENDER_ORDER = 4.4; const VESSEL_POINT_SIZE = 34; @@ -13,6 +22,95 @@ const VESSEL_ATLAS_CELL_SIZE = 128; const VESSEL_COURSE_BINS = 32; const VESSEL_TRACK_ENDPOINT_EPSILON = 0.001; +function getVesselDedupeKey(feature, markerData) { + const props = feature?.properties || {}; + const mmsi = props.mmsi ?? feature?.id ?? markerData?.mmsi; + if (mmsi !== undefined && mmsi !== null && String(mmsi).trim() !== "") { + return `mmsi:${String(mmsi).trim()}`; + } + return [ + "position", + Number(markerData.latitude).toFixed(5), + Number(markerData.longitude).toFixed(5), + String(props.name || markerData.name || "").trim().toLowerCase(), + ].join(":"); +} + +function dedupeVesselFeatures(features) { + const seen = new Set(); + const markerData = []; + features.forEach((feature) => { + const marker = buildVesselMarkerData(feature); + if (!marker) return; + const key = getVesselDedupeKey(feature, marker); + if (seen.has(key)) return; + seen.add(key); + markerData.push(marker); + }); + return markerData; +} + +function markerDataToDedupeKey(item) { + const mmsi = item?.mmsi; + if (mmsi !== undefined && mmsi !== null && String(mmsi).trim() !== "") { + return `mmsi:${String(mmsi).trim()}`; + } + return [ + "position", + Number(item.latitude).toFixed(5), + Number(item.longitude).toFixed(5), + String(item.name || "").trim().toLowerCase(), + ].join(":"); +} + +function buildVesselFeatureFromDelta(item) { + const lat = Number(item?.lat ?? item?.latitude); + const lon = Number(item?.lon ?? item?.lng ?? item?.longitude); + if (!Number.isFinite(lat) || !Number.isFinite(lon)) return null; + return { + type: "Feature", + id: item.mmsi, + geometry: { + type: "Point", + coordinates: [lon, lat], + }, + properties: { + ...item, + mmsi: item.mmsi, + mmsi_display: item.mmsi_display || (item.mmsi !== undefined && item.mmsi !== null ? String(item.mmsi) : undefined), + }, + }; +} + +function rebuildVesselLayerFromCache(earth) { + if (!earth) return; + vesselIconLayer.setData(Array.from(vesselDataByKey.values())); + vesselIconLayer.attach(earth); + vesselIconLayer.setVisible(showVessels); +} + +function applyVesselDeltas(earth, vessels = []) { + let changed = false; + vessels.forEach((item) => { + const feature = buildVesselFeatureFromDelta(item); + if (!feature) return; + const marker = buildVesselMarkerData(feature); + if (!marker) return; + vesselDataByKey.set(markerDataToDedupeKey(marker), marker); + changed = true; + }); + if (changed) { + rebuildVesselLayerFromCache(earth); + } + return changed; +} + +function getVesselStreamUrl() { + if (typeof window === "undefined") return "ws://localhost:8000/ws"; + const protocol = window.location.protocol === "https:" ? "wss:" : "ws:"; + return `${protocol}//${window.location.host}/ws`; +} + function normalizeVesselType(value, code) { const type = String(value || "").trim().toLowerCase(); const numericCode = Number(code); @@ -67,9 +165,14 @@ function buildVesselMarkerData(feature) { const navStatus = Number(props.nav_status); const speed = Number(props.sog); const anchored = navStatus === 1 || navStatus === 5 || (Number.isFinite(speed) && speed < 0.5); + const mmsiString = props.mmsi !== undefined && props.mmsi !== null && String(props.mmsi).trim() !== "" + ? String(props.mmsi) + : null; return { ...props, + mmsi: mmsiString, + mmsi_display: props.mmsi_display ? String(props.mmsi_display) : mmsiString, latitude, longitude, type, @@ -163,6 +266,10 @@ export function getVesselCount() { return vesselIconLayer.getCount(); } +export function getVesselRealtimeStats() { + return { ...vesselRealtimeStats }; +} + export function getShowVessels() { return showVessels; } @@ -199,6 +306,7 @@ export function getVesselPointerIntersections(options) { export function clearVesselData(earth) { clearVesselSelection(); + vesselDataByKey.clear(); vesselIconLayer.clearData(earth); } @@ -216,12 +324,11 @@ export async function loadVessels(_scene, earth, options = {}) { const features = Array.isArray(payload?.features) ? payload.features : []; clearVesselData(earth); - let markerData = features - .map((feature) => buildVesselMarkerData(feature)) - .filter(Boolean); + let markerData = dedupeVesselFeatures(features); if (Number.isFinite(requestedLimit) && requestedLimit > 0) { markerData = markerData.slice(0, requestedLimit); } + vesselDataByKey = new Map(markerData.map((item) => [markerDataToDedupeKey(item), item])); vesselIconLayer.setData(markerData); vesselIconLayer.attach(earth); @@ -233,6 +340,97 @@ export async function loadVessels(_scene, earth, options = {}) { }; } +export function startVesselRealtime(earth, { onUpdate } = {}) { + if (vesselStreamSocket || typeof WebSocket === "undefined") return; + const connect = () => { + if (!showVessels || vesselStreamSocket) return; + const socket = new WebSocket(getVesselStreamUrl()); + vesselStreamSocket = socket; + socket.onopen = () => { + vesselRealtimeStats = { + ...vesselRealtimeStats, + connected: true, + }; + onUpdate?.({ totalCount: getVesselCount(), stream: getVesselRealtimeStats() }); + socket.send(JSON.stringify({ type: "subscribe", data: { channels: ["vessels"] } })); + }; + socket.onmessage = (event) => { + let message; + try { + message = JSON.parse(event.data); + } catch { + return; + } + if (message.type === "heartbeat" && message.data?.action === "ping") { + socket.send(JSON.stringify({ type: "heartbeat" })); + return; + } + if (message.type !== "data_frame" || message.channel !== "vessels") return; + const payload = message.payload || {}; + if (payload.action === "reload") { + loadVessels(null, earth) + .then((result) => { + vesselRealtimeStats = { + ...vesselRealtimeStats, + connected: true, + updates: vesselRealtimeStats.updates + 1, + lastUpdateAt: new Date(), + lastBatchSize: 0, + }; + onUpdate?.({ totalCount: result?.totalCount ?? getVesselCount(), payload, stream: getVesselRealtimeStats() }); + }) + .catch(() => {}); + return; + } + if (payload.action !== "upsert" || !Array.isArray(payload.vessels)) return; + if (applyVesselDeltas(earth, payload.vessels)) { + vesselRealtimeStats = { + connected: true, + updates: vesselRealtimeStats.updates + 1, + lastUpdateAt: new Date(), + lastBatchSize: payload.vessels.length, + }; + onUpdate?.({ totalCount: getVesselCount(), payload, stream: getVesselRealtimeStats() }); + } + }; + socket.onclose = () => { + if (vesselStreamSocket === socket) { + vesselStreamSocket = null; + } + vesselRealtimeStats = { + ...vesselRealtimeStats, + connected: false, + }; + onUpdate?.({ totalCount: getVesselCount(), stream: getVesselRealtimeStats() }); + if (showVessels) { + vesselStreamReconnectTimer = window.setTimeout(connect, 3000); + } + }; + socket.onerror = () => { + socket.close(); + }; + }; + connect(); +} + +export function stopVesselRealtime() { + if (vesselStreamReconnectTimer) { + window.clearTimeout(vesselStreamReconnectTimer); + vesselStreamReconnectTimer = null; + } + if (vesselStreamSocket) { + const socket = vesselStreamSocket; + vesselStreamSocket = null; + socket.close(); + } + vesselRealtimeStats = { + connected: false, + updates: 0, + lastUpdateAt: null, + lastBatchSize: 0, + }; +} + export async function showVesselTrack(marker, earth) { clearVesselTrack(); if (!marker?.userData?.mmsi || !earth) return null; diff --git a/frontend/src/hooks/useWebSocket.ts b/frontend/src/hooks/useWebSocket.ts index ec96810e..70be063a 100644 --- a/frontend/src/hooks/useWebSocket.ts +++ b/frontend/src/hooks/useWebSocket.ts @@ -54,6 +54,8 @@ interface UseWebSocketOptions { interface UseWebSocketReturn { connected: boolean + connecting: boolean + status: 'connecting' | 'connected' | 'disconnected' lastMessage: WebSocketMessage | null sendMessage: (message: Record) => void subscribe: (channels: string[]) => void @@ -65,6 +67,7 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet const { autoConnect = true, autoSubscribe = [], + heartbeatInterval = 25000, onMessage, onConnect, onDisconnect, @@ -75,6 +78,7 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet const wsRef = useRef(null) const [connected, setConnected] = useState(false) + const [connecting, setConnecting] = useState(false) const [lastMessage, setLastMessage] = useState(null) const reconnectTimeoutRef = useRef | null>(null) const heartbeatTimerRef = useRef | null>(null) @@ -97,17 +101,21 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet const connect = useCallback(() => { if (!token) { + setConnected(false) + setConnecting(false) return } intentionalCloseRef.current = false + setConnected(false) + setConnecting(true) const candidates = buildWebSocketCandidates() let candidateIndex = 0 let opened = false const tryConnect = () => { const baseUrl = candidates[candidateIndex] - const wsUrl = `${baseUrl}?token=${token}` + const wsUrl = `${baseUrl}?token=${encodeURIComponent(token)}` activeWsUrlRef.current = baseUrl const ws = new WebSocket(wsUrl) @@ -119,15 +127,27 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet } opened = true setConnected(true) + setConnecting(false) if (autoSubscribeRef.current.length > 0) { ws.send(JSON.stringify({ type: 'subscribe', data: { channels: autoSubscribeRef.current } })) } + if (heartbeatTimerRef.current) { + clearInterval(heartbeatTimerRef.current) + } + heartbeatTimerRef.current = setInterval(() => { + if (ws.readyState === WebSocket.OPEN) { + ws.send(JSON.stringify({ type: 'heartbeat' })) + } + }, heartbeatInterval) onConnectRef.current?.() } ws.onmessage = (event) => { try { const message: WebSocketMessage = JSON.parse(event.data) + if (message.type === 'heartbeat' && message.data?.action === 'ping' && ws.readyState === WebSocket.OPEN) { + ws.send(JSON.stringify({ type: 'heartbeat' })) + } setLastMessage(message) onMessageRef.current?.(message) } catch { @@ -150,10 +170,13 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet if (!opened && candidateIndex < candidates.length - 1) { candidateIndex += 1 + setConnecting(true) tryConnect() return } + setConnecting(false) + if (intentionalCloseRef.current) { return } @@ -169,6 +192,9 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet ws.onerror = (error) => { setConnected(false) + if (opened || candidateIndex >= candidates.length - 1) { + setConnecting(false) + } if (intentionalCloseRef.current || ws.readyState === WebSocket.CLOSING || ws.readyState === WebSocket.CLOSED) { return } @@ -185,6 +211,7 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet tryConnect() } catch (error) { setConnected(false) + setConnecting(false) console.warn('[WebSocket] Failed to initialize connection', { url: activeWsUrlRef.current, error }) if (autoConnect && token) { reconnectTimeoutRef.current = setTimeout(() => { @@ -192,7 +219,7 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet }, 3000) } } - }, [token, autoConnect]) + }, [token, autoConnect, heartbeatInterval]) const disconnect = useCallback(() => { intentionalCloseRef.current = true @@ -213,6 +240,7 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet } } setConnected(false) + setConnecting(false) }, []) const sendMessage = useCallback((message: Record) => { @@ -237,6 +265,8 @@ export function useWebSocket(options: UseWebSocketOptions = {}): UseWebSocketRet return { connected, + connecting, + status: connected ? 'connected' : connecting ? 'connecting' : 'disconnected', lastMessage, sendMessage, subscribe, diff --git a/frontend/src/pages/Dashboard/Dashboard.tsx b/frontend/src/pages/Dashboard/Dashboard.tsx index a90136b3..740db16f 100644 --- a/frontend/src/pages/Dashboard/Dashboard.tsx +++ b/frontend/src/pages/Dashboard/Dashboard.tsx @@ -9,6 +9,7 @@ import { WifiOutlined, DisconnectOutlined, ReloadOutlined, + LoadingOutlined, } from '@ant-design/icons' import { Link } from 'react-router-dom' import axios from 'axios' @@ -137,6 +138,7 @@ function Dashboard() { const [stats, setStats] = useState(cachedDashboardStats) const [loading, setLoading] = useState(cachedDashboardStats === null) const [wsConnected, setWsConnected] = useState(false) + const [wsConnecting, setWsConnecting] = useState(false) const [error, setError] = useState(null) const [restartModalOpen, setRestartModalOpen] = useState(false) const [restartSubmitting, setRestartSubmitting] = useState(false) @@ -180,7 +182,7 @@ function Dashboard() { fetchStats() }, [token, clearAuth]) - const { connected: dashboardSocketConnected } = useWebSocket({ + const { connected: dashboardSocketConnected, connecting: dashboardSocketConnecting } = useWebSocket({ autoConnect: true, autoSubscribe: ['dashboard'], onMessage: (message) => { @@ -194,7 +196,8 @@ function Dashboard() { useEffect(() => { setWsConnected(dashboardSocketConnected) - }, [dashboardSocketConnected]) + setWsConnecting(dashboardSocketConnecting) + }, [dashboardSocketConnected, dashboardSocketConnecting]) const handleRetry = () => { window.location.reload() @@ -406,6 +409,8 @@ function Dashboard() { {wsConnected ? ( } color="success">实时连接 + ) : wsConnecting ? ( + } color="processing">正在连接 ) : ( } color="default">离线 )} diff --git a/frontend/src/pages/DataSources/DataSources.tsx b/frontend/src/pages/DataSources/DataSources.tsx index 4f75f6a4..2a38d5b5 100644 --- a/frontend/src/pages/DataSources/DataSources.tsx +++ b/frontend/src/pages/DataSources/DataSources.tsx @@ -36,8 +36,6 @@ import { formatPhaseMetric, getPhaseDisplay, getPhaseSummary } from '../../utils const { Text } = Typography const COLLECTION_REFRESH_DELAY_MS = 800 -type SourceKind = 'builtin' | 'custom' - interface BuiltInDataSource { id: number source: string @@ -69,7 +67,7 @@ interface BuiltInDataSource { credential_status?: string } -interface CustomDataSource { +interface CustomDataSourceOverride { id: number name: string description: string | null @@ -96,7 +94,6 @@ interface EditableDataSourceConfig { interface UnifiedDataSource { key: string - kind: SourceKind id: number name: string display_name: string @@ -172,7 +169,6 @@ type DatasourceTaskStatus = { function normalizeBuiltin(source: BuiltInDataSource): UnifiedDataSource { return { key: `builtin:${source.id}`, - kind: 'builtin', id: source.id, name: source.name, display_name: source.display_name || source.name, @@ -203,32 +199,12 @@ function normalizeBuiltin(source: BuiltInDataSource): UnifiedDataSource { } } -function normalizeCustom(source: CustomDataSource): UnifiedDataSource { - return { - key: `custom:${source.id}`, - kind: 'custom', - id: source.id, - name: source.name, - display_name: source.name, - source: source.name, - source_type: source.source_type, - endpoint: source.endpoint, - auth_type: source.auth_type, - is_active: source.is_active, - created_at: source.created_at, - updated_at: source.updated_at, - description: source.description, - headers: {}, - config: {}, - } -} - function DataSources() { const [messageApi, contextHolder] = message.useMessage() const navigate = useNavigate() const [modal, modalContextHolder] = Modal.useModal() const [builtInSources, setBuiltInSources] = useState([]) - const [customSources, setCustomSources] = useState([]) + const [customOverrides, setCustomOverrides] = useState([]) const [loading, setLoading] = useState(false) const [triggerAllLoading, setTriggerAllLoading] = useState(false) const [forceTriggerAll, setForceTriggerAll] = useState(false) @@ -239,13 +215,7 @@ function DataSources() { const [tableHeight, setTableHeight] = useState(360) const tableRegionRef = useRef(null) - const allSources = useMemo( - () => [ - ...builtInSources.map(normalizeBuiltin), - ...customSources.map(normalizeCustom), - ], - [builtInSources, customSources], - ) + const allSources = useMemo(() => builtInSources.map(normalizeBuiltin), [builtInSources]) const activeBuiltInCount = builtInSources.filter((source) => source.is_active).length const runningBuiltInSources = builtInSources.filter((source) => source.is_running) @@ -266,7 +236,7 @@ function DataSources() { axios.get('/api/v1/datasources/configs'), ]) setBuiltInSources(builtinRes.data.data || []) - setCustomSources(customRes.data.data || []) + setCustomOverrides(customRes.data.data || []) } catch (error) { console.error('Failed to fetch data:', error) messageApi.error('获取数据源列表失败') @@ -396,8 +366,8 @@ function DataSources() { const handleViewSource = async (source: UnifiedDataSource) => { try { - if (source.kind === 'builtin') { - const override = customSources.find((item) => item.name === source.source) + { + const override = customOverrides.find((item) => item.name === source.source) const [detailRes, statsRes, overrideDetail] = await Promise.all([ axios.get(`/api/v1/datasources/${source.id}`), axios.get(`/api/v1/datasources/${source.id}/stats`), @@ -422,17 +392,6 @@ function DataSources() { credential_status: data.credential_status, }) setRecordCount(statsRes.data.total_records || 0) - } else { - const detail = await axios.get(`/api/v1/datasources/configs/${source.id}`).then((res) => res.data) - setViewingSource({ - ...source, - description: detail.description, - endpoint: detail.endpoint, - auth_type: detail.auth_type, - headers: detail.headers || {}, - config: detail.config || {}, - }) - setRecordCount(null) } setViewDrawerVisible(true) } catch (error) { @@ -468,16 +427,15 @@ function DataSources() { }, { title: '类型', - dataIndex: 'kind', key: 'kind', width: 100, - render: (kind: SourceKind) => {kind === 'builtin' ? '内置' : '自定义'}, + render: () => 内置, }, { title: '层级/类型', key: 'module', width: 120, - render: (_: unknown, record: UnifiedDataSource) => record.kind === 'builtin' ? {record.module} : {record.source_type || 'api'}, + render: (_: unknown, record: UnifiedDataSource) => {record.module}, }, { title: '频率', @@ -498,9 +456,6 @@ function DataSources() { key: 'status', width: 180, render: (_: unknown, record: UnifiedDataSource) => { - if (record.kind === 'custom') { - return {record.is_active ? '启用' : '禁用'} - } if (record.is_running) { return ( @@ -517,7 +472,7 @@ function DataSources() { key: 'action', fixed: 'right' as const, width: 190, - render: (_: unknown, record: UnifiedDataSource) => record.kind === 'builtin' ? ( + render: (_: unknown, record: UnifiedDataSource) => ( - ) : 设置中维护, + ), }, ] @@ -565,10 +520,6 @@ function DataSources() { 内置 {builtInSources.length}
-
- 自定义 - {customSources.length} -
已启用内置 {activeBuiltInCount} @@ -678,7 +629,7 @@ function DataSources() { - {viewingSource.kind === 'builtin' ? '内置数据源' : '自定义数据源'} + 内置数据源 {viewingSource.is_active ? '启用' : '禁用'} @@ -690,45 +641,26 @@ function DataSources() {
标识
- {viewingSource.kind === 'builtin' ? ( - <> - -
模块
- - - -
优先级
- - - -
频率
- - - -
数据量
- - - -
采集器
- - - - ) : ( - <> - -
源类型
- - - -
认证方式
- - - -
描述
- - - - )} + +
模块
+ + + +
优先级
+ + + +
频率
+ + + +
数据量
+ + + +
采集器
+ + diff --git a/frontend/src/pages/Settings/Settings.tsx b/frontend/src/pages/Settings/Settings.tsx index 05de53ae..37f1132c 100644 --- a/frontend/src/pages/Settings/Settings.tsx +++ b/frontend/src/pages/Settings/Settings.tsx @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState, type ReactNode } from 'react' +import { useEffect, useMemo, useRef, useState, type ReactNode } from 'react' import { useCollapsedActions } from '../../hooks' import { TableActions, actionCellProps } from '../../components/TableActions/TableActions' import { @@ -7,9 +7,11 @@ import { CheckCircleOutlined, DeleteOutlined, EditOutlined, + PlayCircleOutlined, PlusOutlined, ReloadOutlined, RobotOutlined, + StopOutlined, SyncOutlined, } from '@ant-design/icons' import { @@ -42,6 +44,7 @@ import { useSearchParams } from 'react-router-dom' const { Title, Text } = Typography const ANTHROPIC_MESSAGES_MAX_TOKENS = 1200 const DEFAULT_PROVIDER_MAX_TOKENS = 4096 +const CUSTOM_STREAM_STATUS_POLL_MS = 5000 interface SystemSettings { system_name: string @@ -83,6 +86,7 @@ interface CollectorSettings { credential_provider?: string | null credential_status?: string ais_health?: AISSourceHealth | null + is_custom?: boolean } interface AISSourceHealth { @@ -171,6 +175,7 @@ interface CredentialGuide { } interface CollectorConfigOption { + id?: number name: string default_url: string endpoint: string @@ -184,6 +189,7 @@ interface CollectorConfigOption { config: Record config_id: number | null description: string + is_custom?: boolean } const AISSTREAM_BBOX_PRESETS = [ @@ -310,6 +316,7 @@ function Settings() { const [loading, setLoading] = useState(true) const [savingCollectorId, setSavingCollectorId] = useState(null) const [collectors, setCollectors] = useState([]) + const [customSourceConfigs, setCustomSourceConfigs] = useState([]) const [systemSettings, setSystemSettings] = useState(null) const [notificationSettings, setNotificationSettings] = useState(null) const [securitySettings, setSecuritySettings] = useState(null) @@ -338,10 +345,33 @@ function Settings() { const [securityForm] = Form.useForm() const [integrationForm] = Form.useForm() const [collectorConfigForm] = Form.useForm() + const [customSourceForm] = Form.useForm() const [tvEditForm] = Form.useForm() const selectedAiProvider = Form.useWatch(['ai_provider', 'provider'], integrationForm) - const selectedCollector = collectors.find((collector) => collector.source === selectedCollectorSource) - const selectedCollectorConfig = collectorConfigs.find((config) => config.name === selectedCollectorSource) + const customCollectors: CollectorSettings[] = useMemo(() => customSourceConfigs.map((config) => ({ + id: -(config.config_id || 0), + name: config.name, + display_name: config.description || config.name, + source: config.name, + module: 'CUSTOM', + priority: 'P2', + frequency_minutes: Number(config.config?.frequency_minutes || 0), + frequency: 'custom', + is_active: config.is_active, + last_run_at: null, + last_status: null, + next_run_at: null, + is_free: true, + requires_credentials: config.auth_type !== 'none', + credential_provider: null, + credential_status: 'custom', + is_custom: true, + })), [customSourceConfigs]) + const collectorOptions = useMemo(() => [...collectors, ...customCollectors], [collectors, customCollectors]) + const selectedCollector = collectorOptions.find((collector) => collector.source === selectedCollectorSource) + const selectedCollectorConfig = [...collectorConfigs, ...customSourceConfigs].find((config) => config.name === selectedCollectorSource) + const [customStreamStatus, setCustomStreamStatus] = useState<{ running: boolean; done: boolean } | null>(null) + const [customStreamBusy, setCustomStreamBusy] = useState(false) const selectedCollectorHealth = selectedCollector ? collectorHealthStatus[selectedCollector.source] : undefined @@ -374,10 +404,11 @@ function Settings() { const fetchSettings = async () => { try { setLoading(true) - const [response, presetsResponse, collectorConfigsResponse] = await Promise.all([ + const [response, presetsResponse, collectorConfigsResponse, customConfigsResponse] = await Promise.all([ axios.get('/api/v1/settings'), axios.get('/api/v1/settings/integrations/ai-provider/presets'), axios.get('/api/v1/datasources/configs/all'), + axios.get('/api/v1/datasources/configs'), ]) setSystemSettings(response.data.system) setNotificationSettings(response.data.notifications) @@ -386,7 +417,12 @@ function Settings() { setIntegrations(response.data.integrations || null) setCollectors(response.data.collectors || []) setAiProviderPresets(presetsResponse.data.data || []) - setCollectorConfigs(collectorConfigsResponse.data.data || []) + const builtinConfigs = collectorConfigsResponse.data.data || [] + setCollectorConfigs(builtinConfigs) + const builtinNames = new Set((response.data.collectors || []).map((collector: CollectorSettings) => collector.source)) + setCustomSourceConfigs((customConfigsResponse.data.data || []) + .filter((config: CollectorConfigOption) => !builtinNames.has(config.name)) + .map((config: CollectorConfigOption) => ({ ...config, is_custom: true, config_id: config.id ?? config.config_id }))) } catch (error) { message.error('获取系统配置失败') console.error(error) @@ -449,6 +485,24 @@ function Settings() { if (loading || !selectedCollectorConfig) return const config = selectedCollectorConfig.config || {} const boundingBoxes = config.bounding_boxes ?? [[[-90, -180], [90, 180]]] + if (selectedCollector?.is_custom) { + collectorConfigForm.setFieldsValue({ + endpoint: selectedCollectorConfig.endpoint, + source_type: selectedCollectorConfig.source_type || 'websocket', + auth_type: selectedCollectorConfig.auth_type || 'none', + merge_target_source: config.merge_target_source || 'barentswatch_vessels', + target_schema: config.target_schema || 'vessel_ais', + auth_config: { + api_key: selectedCollectorConfig.auth_configured?.api_key ? '••••••••' : '', + }, + headers: Object.entries(selectedCollectorConfig.headers || {}).map(([key, value]) => ({ key, value })), + config: { + ...config, + advanced_json: JSON.stringify(config, null, 2), + }, + }) + return + } collectorConfigForm.setFieldsValue({ endpoint: selectedCollectorConfig.endpoint, auth_config: { @@ -465,12 +519,12 @@ function Settings() { bounding_boxes_json: stringifyBoundingBoxes(boundingBoxes), }, }) - }, [collectorConfigForm, loading, selectedCollectorConfig]) + }, [collectorConfigForm, loading, selectedCollector, selectedCollectorConfig]) useEffect(() => { - if (!requestedCollector || !collectors.some((collector) => collector.source === requestedCollector)) return + if (!requestedCollector || !collectorOptions.some((collector) => collector.source === requestedCollector)) return setSelectedCollectorSource(requestedCollector) - }, [collectors, requestedCollector]) + }, [collectorOptions, requestedCollector]) useEffect(() => { const updateTableHeight = () => { @@ -516,6 +570,16 @@ function Settings() { }, {}) ) + const parseJsonObjectField = (value: string | undefined, fallback: Record = {}) => { + const text = String(value || '').trim() + if (!text) return fallback + const parsed = JSON.parse(text) + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error('高级配置必须是 JSON object') + } + return parsed + } + const applyAisstreamBboxPreset = (presetValue: string) => { const preset = AISSTREAM_BBOX_PRESETS.find((item) => item.value === presetValue) if (!preset) return @@ -571,6 +635,34 @@ function Settings() { const baseValues = collectorConfigForm.getFieldsValue(true) const headers = headersListToMap(baseValues.headers) + if (selectedCollector.is_custom) { + const configValues = { + ...parseJsonObjectField(baseValues.config?.advanced_json, baseValues.config || {}), + merge_target_source: baseValues.merge_target_source, + target_schema: baseValues.target_schema || 'vessel_ais', + } + delete configValues.advanced_json + const payload: Record = { + name: selectedCollector.source, + description: selectedCollector.name, + source_type: baseValues.source_type || selectedCollectorConfig.source_type || 'websocket', + endpoint: baseValues.endpoint, + auth_type: baseValues.auth_type || selectedCollectorConfig.auth_type || 'none', + headers, + config: configValues, + } + const apiKey = String(baseValues.auth_config?.api_key || '').trim() + if (payload.auth_type === 'api_key' && apiKey && !apiKey.startsWith('••••')) { + payload.auth_config = { api_key: apiKey } + } else { + payload.auth_config = {} + } + await axios.put(`/api/v1/datasources/configs/${selectedCollectorConfig.config_id}`, payload) + message.success('自定义源设置已保存') + await fetchSettings() + return + } + if (selectedCollector.source === 'barentswatch_vessels') { const integrationValues = integrationForm.getFieldsValue(true) await saveIntegrations({ @@ -632,6 +724,184 @@ function Settings() { } } + const createCustomSourceFromSettings = async () => { + try { + const values = await customSourceForm.validateFields() + const config = { + ...(parseJsonObjectField(values.advanced_json, {})), + merge_target_source: values.merge_target_source, + target_schema: values.target_schema || 'vessel_ais', + } + await axios.post('/api/v1/datasources/configs', { + name: values.name, + description: values.description || values.name, + source_type: values.source_type || 'websocket', + endpoint: values.endpoint, + auth_type: values.auth_type || 'none', + auth_config: {}, + headers: {}, + config, + }) + message.success('自定义源已创建') + customSourceForm.resetFields() + await fetchSettings() + setSelectedCollectorSource(values.name) + } catch (error: unknown) { + const err = error as { response?: { data?: { detail?: string } }; message?: string } + message.error(err.response?.data?.detail || err.message || '创建自定义源失败') + } + } + + const confirmCreateCustomSource = () => { + customSourceForm.setFieldsValue({ + source_type: 'websocket', + auth_type: 'none', + merge_target_source: 'barentswatch_vessels', + target_schema: 'vessel_ais', + endpoint: 'ws://localhost:8787/ais', + advanced_json: JSON.stringify({ + ws_message_path: '$.data', + ws_reconnect: true, + delivery_mode: 'realtime_stream', + }, null, 2), + }) + Modal.confirm({ + title: '添加自定义源', + width: 720, + icon: null, + content: ( +
+
+ + + + + + + + + + + + + + + + + - - - + {!selectedCollector?.is_custom ? ( + + + + ) : null} {(fields, { add, remove }) => ( @@ -1706,16 +2036,52 @@ function Settings() { ) : null} + {selectedCollector?.is_custom ? ( + + + + ) : null} - + + + {selectedCollector?.is_custom && selectedCollectorConfig?.source_type === 'websocket' ? ( + <> + + + + {customStreamStatus?.running ? 'streaming' : customStreamStatus?.done ? 'stopped' : '未运行'} + + + ) : null} + {selectedCollector?.is_custom ? ( + + ) : null} + ), diff --git a/package.json b/package.json new file mode 100644 index 00000000..e846e24b --- /dev/null +++ b/package.json @@ -0,0 +1,7 @@ +{ + "private": true, + "packageManager": "bun@1", + "scripts": { + "mock:ais-ws": "bun scripts/mock-ais-ws-server.ts" + } +} diff --git a/planet.sh b/planet.sh index 7a492d20..e9cd56e1 100755 --- a/planet.sh +++ b/planet.sh @@ -1153,6 +1153,7 @@ fail_unreleased_port() { if [ -z "$(collect_port_pids "$port" || true)" ]; then log_error "端口 ${port} 当前环境内未发现占用进程，但端口仍不可用，请检查宿主机或外部环境占用" + print_port_listener_details "$port" else log_error "端口 ${port} 清理失败，请检查占用进程" if command -v lsof >/dev/null 2>&1; then @@ -1160,6 +1161,7 @@ fail_unreleased_port() { elif command -v ss >/dev/null 2>&1; then ss -ltnp "( sport = :${port} )" 2>/dev/null || true fi + print_windows_port_listener_details "$port" || true fi exit 1 @@ -1473,27 +1475,25 @@ terminate_process_tree() { can_bind_port() { local port="$1" - if command -v ss >/dev/null 2>&1; then - ! ss -tlnH 2>/dev/null | awk '{print $4}' | grep -qE ":${port}$" - return - fi - if command -v lsof >/dev/null 2>&1; then - [ -z "$(lsof -tiTCP:"${port}" -sTCP:LISTEN 2>/dev/null)" ] - return - fi - python3 - "$port" <<'PY' >/dev/null 2>&1 + + if command -v python3 >/dev/null 2>&1; then + python3 - "$port" <<'PY' >/dev/null 2>&1 import socket import sys port = int(sys.argv[1]) sockets = [] try: - for family, host in ((socket.AF_INET, "127.0.0.1"), (socket.AF_INET6, "::1")): + for family, host in ((socket.AF_INET, "0.0.0.0"), (socket.AF_INET6, "::")): try: sock = socket.socket(family) + if family == socket.AF_INET6 and hasattr(socket, "IPV6_V6ONLY"): + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1) sock.bind((host, port)) sockets.append(sock) - except OSError: + except OSError as exc: + if getattr(exc, "errno", None) in (socket.EAFNOSUPPORT, getattr(socket, "EADDRNOTAVAIL", -1)): + continue raise finally: for sock in sockets: @@ -1502,6 +1502,17 @@ finally: except OSError: pass PY + return + fi + + if command -v ss >/dev/null 2>&1; then + ! ss -tlnH 2>/dev/null | awk '{print $4}' | grep -qE ":${port}$" + return + fi + if command -v lsof >/dev/null 2>&1; then + [ -z "$(lsof -tiTCP:"${port}" -sTCP:LISTEN 2>/dev/null)" ] + return + fi } wait_for_port_release() { @@ -1574,6 +1585,38 @@ backend_log_indicates_port_conflict() { grep -Eiq "Address already in use|Errno 98" "$log_file" 2>/dev/null } +print_windows_port_listener_details() { + local port="$1" + local output="" + + command -v powershell.exe >/dev/null 2>&1 || return 1 + + output="$( + powershell.exe -NoProfile -Command " +\$ErrorActionPreference = 'SilentlyContinue' +\$port = [int]${port} +\$connections = Get-NetTCPConnection -LocalPort \$port -State Listen +foreach (\$connection in \$connections) { + \$owningProcessId = \$connection.OwningProcess + \$process = Get-CimInstance Win32_Process -Filter \"ProcessId=\$owningProcessId\" + \$services = Get-CimInstance Win32_Service | Where-Object { \$_.ProcessId -eq \$owningProcessId } | Select-Object -ExpandProperty Name + \$processName = if (\$process.Name) { \$process.Name } else { 'unknown' } + \$serviceText = if (\$services) { ' services=' + (\$services -join ',') } else { '' } + 'Windows listener: {0}:{1} pid={2} process={3}{4}' -f \$connection.LocalAddress, \$connection.LocalPort, \$owningProcessId, \$processName, \$serviceText +} +" 2>/dev/null | tr -d '\r' + )" + + [ -n "$output" ] || return 1 + while IFS= read -r line; do + [ -n "$line" ] || continue + printf "${DIM} %s${NC}\n" "$line" + done < | null +} + +const REGION_PRESETS: Record = { + mediterranean: { lat: 36.2, lon: 14.2 }, + shanghai: { lat: 31.1, lon: 121.25 }, + east_asia: { lat: 31.1, lon: 121.25 }, + north_sea: { lat: 56.2, lon: 3.2 }, + norway: { lat: 59.9, lon: 10.7 }, +} + +const port = Number(Bun.env.MOCK_AIS_WS_PORT || 8787) +const baseMmsi = Number(Bun.env.MOCK_AIS_BASE_MMSI || 999000000) +const region = String(Bun.env.MOCK_AIS_REGION || "mediterranean").toLowerCase() +const defaultAnchor: Anchor = REGION_PRESETS[region] ?? REGION_PRESETS.mediterranean +const defaultIntervalMs = clamp(Number(Bun.env.MOCK_AIS_WS_INTERVAL_MS || 1500), 200, 30_000) +const defaultSpreadKm = clamp(Number(Bun.env.MOCK_AIS_SPREAD_KM || 60), 1, 5_000) +const defaultSogKn = clamp(Number(Bun.env.MOCK_AIS_SOG_KN || 12), 0, 60) +const defaultMaxVessels = clamp(Number(Bun.env.MOCK_AIS_MAX_VESSELS || 12), 1, 200) + +const KN_TO_DEG_LAT_PER_SEC = 1 / 60 / 60 // 1 nautical mile = 1/60 degree of latitude; per second +const VESSEL_TYPE_BANK: Array<{ type: number; name: string }> = [ + { type: 70, name: "Cargo" }, + { type: 80, name: "Tanker" }, + { type: 60, name: "Passenger" }, + { type: 30, name: "Fishing" }, + { type: 35, name: "Military" }, +] + +function clamp(value: number, min: number, max: number): number { + if (!Number.isFinite(value)) return min + return Math.min(Math.max(value, min), max) +} + +function defaultConfig(): StreamConfig { + return { + anchor: { ...defaultAnchor }, + spreadKm: defaultSpreadKm, + rateMs: defaultIntervalMs, + sogKn: defaultSogKn, + maxVessels: defaultMaxVessels, + } +} + +function applyOverrides(base: StreamConfig, overrides: Record): StreamConfig { + const next: StreamConfig = { + anchor: { ...base.anchor }, + spreadKm: base.spreadKm, + rateMs: base.rateMs, + sogKn: base.sogKn, + maxVessels: base.maxVessels, + } + const anchor = overrides?.anchor + if (anchor && typeof anchor === "object") { + const lat = Number((anchor as Record).lat) + const lon = Number((anchor as Record).lon) + if (Number.isFinite(lat) && Number.isFinite(lon) && lat >= -90 && lat <= 90 && lon >= -180 && lon <= 180) { + next.anchor = { lat, lon } + } + } + if (Number.isFinite(Number(overrides.lat)) && Number.isFinite(Number(overrides.lon))) { + const lat = Number(overrides.lat) + const lon = Number(overrides.lon) + if (lat >= -90 && lat <= 90 && lon >= -180 && lon <= 180) { + next.anchor = { lat, lon } + } + } + if (Number.isFinite(Number(overrides.spread_km))) { + next.spreadKm = clamp(Number(overrides.spread_km), 1, 5_000) + } + if (Number.isFinite(Number(overrides.rate_hz)) && Number(overrides.rate_hz) > 0) { + next.rateMs = clamp(1000 / Number(overrides.rate_hz), 100, 30_000) + } + if (Number.isFinite(Number(overrides.rate_ms))) { + next.rateMs = clamp(Number(overrides.rate_ms), 100, 30_000) + } + if (Number.isFinite(Number(overrides.sog_kn))) { + next.sogKn = clamp(Number(overrides.sog_kn), 0, 60) + } + if (Number.isFinite(Number(overrides.max_vessels))) { + next.maxVessels = clamp(Number(overrides.max_vessels), 1, 200) + } + return next +} + +function spawnVessel(state: SocketState, index: number): VesselState { + const { anchor, spreadKm, sogKn } = state.config + const angle = Math.random() * Math.PI * 2 + const radius = Math.random() * spreadKm + const dLat = (radius * Math.cos(angle)) / 111 + const dLon = (radius * Math.sin(angle)) / (111 * Math.cos((anchor.lat * Math.PI) / 180) || 1) + const cog = Math.random() * 360 + const profile = VESSEL_TYPE_BANK[index % VESSEL_TYPE_BANK.length] + return { + mmsi: baseMmsi + index + 1, + name: `MOCK VESSEL ${String(index + 1).padStart(3, "0")}`, + lat: clamp(anchor.lat + dLat, -89.999, 89.999), + lon: ((anchor.lon + dLon + 540) % 360) - 180, + sog: clamp(sogKn * (0.6 + Math.random() * 0.6), 0, 60), + cog, + heading: Math.round(cog), + vesselType: profile.type, + vesselTypeName: profile.name, + } +} + +function advanceVessel(vessel: VesselState, dtSeconds: number) { + const radians = (vessel.cog * Math.PI) / 180 + const speedDegPerSec = vessel.sog * KN_TO_DEG_LAT_PER_SEC + const dLat = speedDegPerSec * Math.cos(radians) * dtSeconds + const dLon = (speedDegPerSec * Math.sin(radians) * dtSeconds) / (Math.cos((vessel.lat * Math.PI) / 180) || 1) + vessel.lat = clamp(vessel.lat + dLat, -89.999, 89.999) + vessel.lon = ((vessel.lon + dLon + 540) % 360) - 180 + // small course wander so the path isn't a straight line + vessel.cog = (vessel.cog + (Math.random() - 0.5) * 4 + 360) % 360 + vessel.heading = Math.round(vessel.cog) +} + +function pickNextVessel(state: SocketState): VesselState { + state.sequence += 1 + if (state.vessels.length === 0 || (state.sequence % 4 === 0 && state.vessels.length < state.config.maxVessels)) { + const vessel = spawnVessel(state, state.vessels.length) + state.vessels.push(vessel) + return vessel + } + const vessel = state.vessels[state.sequence % state.vessels.length] + advanceVessel(vessel, state.config.rateMs / 1000) + return vessel +} + +function buildPayload(state: SocketState, vessel: VesselState) { + return { + type: "vessel", + sequence: state.sequence, + config: { + anchor: state.config.anchor, + spread_km: state.config.spreadKm, + rate_ms: state.config.rateMs, + sog_kn: state.config.sogKn, + max_vessels: state.config.maxVessels, + }, + data: { + mmsi: String(vessel.mmsi), + name: vessel.name, + lat: Number(vessel.lat.toFixed(6)), + lon: Number(vessel.lon.toFixed(6)), + sog: Number(vessel.sog.toFixed(2)), + cog: Number(vessel.cog.toFixed(1)), + heading: vessel.heading, + vessel_type: vessel.vesselType, + vessel_type_name: vessel.vesselTypeName, + source_note: `mock:${region}`, + received_at: new Date().toISOString(), + }, + } +} + +const sockets = new Map, SocketState>() + +function startTimer(socket: ServerWebSocket, state: SocketState) { + if (state.timer) clearInterval(state.timer) + state.timer = setInterval(() => { + const vessel = pickNextVessel(state) + const payload = buildPayload(state, vessel) + try { + socket.send(JSON.stringify(payload)) + } catch { + // socket already closed; cleanup happens in close() + } + }, state.config.rateMs) +} + +const server = Bun.serve({ + port, + fetch(request, server) { + const url = new URL(request.url) + if (url.pathname !== "/ais") { + return new Response("Mock AIS WS server. Connect to /ais.", { status: 200 }) + } + if (server.upgrade(request)) { + return undefined + } + return new Response("WebSocket upgrade failed", { status: 400 }) + }, + websocket: { + open(socket) { + const state: SocketState = { + config: defaultConfig(), + vessels: [], + sequence: 0, + timer: null, + } + sockets.set(socket, state) + socket.send( + JSON.stringify({ + type: "hello", + source: "mock_ais_ws", + config: { + anchor: state.config.anchor, + spread_km: state.config.spreadKm, + rate_ms: state.config.rateMs, + sog_kn: state.config.sogKn, + max_vessels: state.config.maxVessels, + }, + subscribe_hint: + "Send { type: 'subscribe', anchor: { lat, lon }, spread_km, rate_hz, sog_kn, max_vessels } to retarget.", + }), + ) + startTimer(socket, state) + }, + message(socket, message) { + const state = sockets.get(socket) + if (!state) return + let parsed: Record = {} + try { + parsed = JSON.parse(typeof message === "string" ? message : message.toString()) + } catch { + return + } + if (!parsed || typeof parsed !== "object") return + const overrides = (parsed as { type?: unknown }).type === "subscribe" ? parsed : parsed + const before = state.config + state.config = applyOverrides(state.config, overrides) + const anchorChanged = + before.anchor.lat !== state.config.anchor.lat || before.anchor.lon !== state.config.anchor.lon + if (anchorChanged) { + // throw away old vessels so new ones spawn at the new anchor + state.vessels = [] + state.sequence = 0 + } + console.info("[mock-ais-ws] config update", state.config) + startTimer(socket, state) + socket.send( + JSON.stringify({ + type: "subscribe_ack", + config: { + anchor: state.config.anchor, + spread_km: state.config.spreadKm, + rate_ms: state.config.rateMs, + sog_kn: state.config.sogKn, + max_vessels: state.config.maxVessels, + }, + }), + ) + }, + close(socket) { + const state = sockets.get(socket) + if (state?.timer) clearInterval(state.timer) + sockets.delete(socket) + }, + }, +}) + +console.info(`[mock-ais-ws] listening on ws://localhost:${server.port}/ais`) +console.info( + `[mock-ais-ws] defaults region=${region} anchor=${JSON.stringify(defaultAnchor)} spread_km=${defaultSpreadKm} rate_ms=${defaultIntervalMs} sog_kn=${defaultSogKn} max_vessels=${defaultMaxVessels}`, +) diff --git a/uv.lock b/uv.lock index dbfb67d0..bedbdca7 100644 --- a/uv.lock +++ b/uv.lock @@ -475,7 +475,7 @@ wheels = [ [[package]] name = "planet" -version = "0.47.0" +version = "0.48.0" source = { virtual = "." } dependencies = [ { name = "aiofiles" },