release: bump version to 0.48.0
This commit is contained in:
linkong
@@ -10,7 +10,10 @@ from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
from app.models.task import CollectionTask
|
||||
from app.services.collectors.base import BaseCollector
|
||||
from app.services.vessel_ais_aggregation import (
|
||||
AISSTREAM_DELIVERY_MODE,
|
||||
@@ -66,9 +69,20 @@ class AISStreamCollector(BaseCollector):
|
||||
"bounding_boxes": config.get("bounding_boxes") or DEFAULT_BOUNDING_BOXES,
|
||||
"message_types": config.get("message_types") or DEFAULT_MESSAGE_TYPES,
|
||||
"max_messages": int(config.get("max_messages") or 500),
|
||||
"streaming_enabled": config.get("streaming_enabled", True) is not False,
|
||||
"streaming_commit_interval": int(config.get("streaming_commit_interval") or 1),
|
||||
"streaming_max_messages": int(config.get("streaming_max_messages") or 0),
|
||||
"reconnect_delay_seconds": float(config.get("reconnect_delay_seconds") or 5),
|
||||
"receive_timeout_seconds": float(config.get("receive_timeout_seconds") or 30),
|
||||
}
|
||||
|
||||
def _build_subscription(self, config: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"APIKey": config["api_key"],
|
||||
"BoundingBoxes": config["bounding_boxes"],
|
||||
"FilterMessageTypes": config["message_types"],
|
||||
}
|
||||
|
||||
async def fetch(self) -> list[dict[str, Any]]:
|
||||
config = await self._get_effective_config()
|
||||
if not config["api_key"]:
|
||||
@@ -79,11 +93,7 @@ class AISStreamCollector(BaseCollector):
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("Python package 'websockets' is required for AISStream") from exc
|
||||
|
||||
subscription = {
|
||||
"APIKey": config["api_key"],
|
||||
"BoundingBoxes": config["bounding_boxes"],
|
||||
"FilterMessageTypes": config["message_types"],
|
||||
}
|
||||
subscription = self._build_subscription(config)
|
||||
|
||||
messages: list[dict[str, Any]] = []
|
||||
try:
|
||||
@@ -113,6 +123,157 @@ class AISStreamCollector(BaseCollector):
|
||||
|
||||
return messages
|
||||
|
||||
async def run(self, db: AsyncSession) -> dict[str, Any]:
|
||||
"""Run AISStream as a long-lived streaming collector by default."""
|
||||
config = await self._get_effective_config()
|
||||
if not config.get("streaming_enabled", True):
|
||||
return await super().run(db)
|
||||
if not config["api_key"]:
|
||||
return {"status": "failed", "error": "AISStream API key is not configured"}
|
||||
|
||||
from app.services.collectors.registry import collector_registry
|
||||
|
||||
if not collector_registry.is_active(self.name):
|
||||
return {"status": "skipped", "reason": "Collector is disabled"}
|
||||
|
||||
try:
|
||||
import websockets
|
||||
except ImportError as exc:
|
||||
return {"status": "failed", "error": "Python package 'websockets' is required for AISStream"}
|
||||
|
||||
start_time = datetime.now(UTC)
|
||||
task = CollectionTask(
|
||||
datasource_id=getattr(self, "_datasource_id", 1),
|
||||
status="running",
|
||||
phase="connecting",
|
||||
phase_message="正在连接 AISStream 实时流",
|
||||
phase_unit="messages",
|
||||
started_at=start_time,
|
||||
)
|
||||
db.add(task)
|
||||
await db.commit()
|
||||
self._current_task = task
|
||||
self._db_session = db
|
||||
self._last_broadcast_progress = None
|
||||
await self.resolve_url(db)
|
||||
await self._publish_task_update(force=True)
|
||||
|
||||
records_added = 0
|
||||
messages_seen = 0
|
||||
unique_mmsi: set[str] = set()
|
||||
reconnect_delay = config["reconnect_delay_seconds"]
|
||||
|
||||
try:
|
||||
while True:
|
||||
config = await self._get_effective_config()
|
||||
subscription = self._build_subscription(config)
|
||||
try:
|
||||
await update_ais_source_health(
|
||||
db,
|
||||
source=self.name,
|
||||
connection_state="connecting",
|
||||
)
|
||||
await self.set_phase("connecting", message="正在连接 AISStream 实时流")
|
||||
await db.commit()
|
||||
|
||||
async with websockets.connect(config["endpoint"]) as websocket:
|
||||
await websocket.send(json.dumps(subscription))
|
||||
await update_ais_source_health(
|
||||
db,
|
||||
source=self.name,
|
||||
connection_state="connected",
|
||||
last_success_at=datetime.now(UTC),
|
||||
)
|
||||
await self.set_phase(
|
||||
"streaming",
|
||||
message="正在接收 AISStream 实时消息",
|
||||
reset_progress=False,
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
while True:
|
||||
try:
|
||||
raw_message = await asyncio.wait_for(
|
||||
websocket.recv(),
|
||||
timeout=config["receive_timeout_seconds"],
|
||||
)
|
||||
except TimeoutError:
|
||||
await update_ais_source_health(
|
||||
db,
|
||||
source=self.name,
|
||||
connection_state="connected",
|
||||
last_success_at=datetime.now(UTC),
|
||||
)
|
||||
await db.commit()
|
||||
continue
|
||||
|
||||
payload = json.loads(raw_message)
|
||||
if not isinstance(payload, dict):
|
||||
continue
|
||||
messages_seen += 1
|
||||
record = self._normalize_message(payload)
|
||||
if not record:
|
||||
continue
|
||||
unique_mmsi.add(str(record["mmsi"]))
|
||||
created = await self._save_stream_record(db, record)
|
||||
if created:
|
||||
records_added += 1
|
||||
|
||||
task.records_processed = messages_seen
|
||||
task.total_records = None
|
||||
task.progress = None
|
||||
task.phase = "streaming"
|
||||
task.phase_message = "正在接收 AISStream 实时消息"
|
||||
task.phase_current = messages_seen
|
||||
task.phase_total = None
|
||||
task.phase_unit = "messages"
|
||||
await self._publish_task_update(force=True)
|
||||
|
||||
if config["streaming_max_messages"] and messages_seen >= config["streaming_max_messages"]:
|
||||
task.status = "success"
|
||||
task.phase = "stopped"
|
||||
task.phase_message = "AISStream 测试流已停止"
|
||||
task.completed_at = datetime.now(UTC)
|
||||
await db.commit()
|
||||
await self._publish_task_update(force=True)
|
||||
return {
|
||||
"status": "success",
|
||||
"task_id": task.id,
|
||||
"records_processed": records_added,
|
||||
"messages_seen": messages_seen,
|
||||
"unique_mmsi": len(unique_mmsi),
|
||||
"execution_time_seconds": (datetime.now(UTC) - start_time).total_seconds(),
|
||||
}
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
await update_ais_source_health(
|
||||
db,
|
||||
source=self.name,
|
||||
connection_state="reconnecting",
|
||||
last_error=f"{exc.__class__.__name__}: {exc}",
|
||||
)
|
||||
task.phase = "reconnecting"
|
||||
task.phase_message = "AISStream 连接中断，正在重连"
|
||||
task.error_message = f"{exc.__class__.__name__}: {exc}"
|
||||
await db.commit()
|
||||
await self._publish_task_update(force=True)
|
||||
await asyncio.sleep(reconnect_delay)
|
||||
except asyncio.CancelledError:
|
||||
task.status = "cancelled"
|
||||
task.phase = "stopped"
|
||||
task.phase_message = "AISStream 实时流已停止"
|
||||
task.completed_at = datetime.now(UTC)
|
||||
await update_ais_source_health(
|
||||
db,
|
||||
source=self.name,
|
||||
connection_state="disconnected",
|
||||
last_error=None,
|
||||
)
|
||||
await db.commit()
|
||||
await self._publish_task_update(force=True)
|
||||
raise
|
||||
|
||||
def transform(self, raw_data: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
records = []
|
||||
for item in raw_data:
|
||||
@@ -165,6 +326,60 @@ class AISStreamCollector(BaseCollector):
|
||||
await self.update_progress(records_added, force=True)
|
||||
return records_added
|
||||
|
||||
async def _save_stream_record(self, db: AsyncSession, item: dict[str, Any]) -> bool:
|
||||
now = datetime.now(UTC)
|
||||
observed_at = item.get("received_at") or now
|
||||
observation = await record_vessel_ais_observation(
|
||||
db,
|
||||
source=self.name,
|
||||
normalized_payload=item,
|
||||
raw_payload=item.get("_raw_payload") or item,
|
||||
delivery_mode=AISSTREAM_DELIVERY_MODE,
|
||||
transport=AISSTREAM_TRANSPORT,
|
||||
message_type=item.get("_message_type") or "PositionReport",
|
||||
source_message_id=item.get("_source_message_id"),
|
||||
observed_at=observed_at,
|
||||
collected_at=now,
|
||||
)
|
||||
await update_ais_source_health(
|
||||
db,
|
||||
source=self.name,
|
||||
connection_state="connected",
|
||||
observed_count=1,
|
||||
last_seen_at=observed_at if isinstance(observed_at, datetime) else now,
|
||||
last_success_at=now,
|
||||
lag_seconds=max((now - observed_at).total_seconds(), 0) if isinstance(observed_at, datetime) else None,
|
||||
)
|
||||
await db.commit()
|
||||
await self._broadcast_vessel_delta(item, created=observation is not None)
|
||||
return observation is not None
|
||||
|
||||
async def _broadcast_vessel_delta(self, item: dict[str, Any], *, created: bool) -> None:
|
||||
await broadcaster.broadcast_custom(
|
||||
"vessels",
|
||||
{
|
||||
"action": "upsert",
|
||||
"source": self.name,
|
||||
"created": created,
|
||||
"vessels": [
|
||||
{
|
||||
"mmsi": item.get("mmsi"),
|
||||
"mmsi_display": str(item.get("mmsi")) if item.get("mmsi") is not None else None,
|
||||
"name": item.get("name"),
|
||||
"lat": item.get("lat"),
|
||||
"lon": item.get("lon"),
|
||||
"sog": item.get("sog"),
|
||||
"cog": item.get("cog"),
|
||||
"heading": item.get("heading"),
|
||||
"nav_status": item.get("nav_status"),
|
||||
"vessel_type": item.get("vessel_type"),
|
||||
"vessel_type_name": item.get("vessel_type_name"),
|
||||
"received_at": to_iso8601_utc(item.get("received_at")),
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
def _normalize_message(self, item: dict[str, Any]) -> dict[str, Any] | None:
|
||||
message_type = str(item.get("MessageType") or item.get("message_type") or "")
|
||||
metadata = item.get("MetaData") if isinstance(item.get("MetaData"), dict) else {}
|
||||
|
||||
Reference in New Issue
Block a user