release: bump version to 0.68.0

backend/app/api/v1/datasources.py

@@ -19,7 +19,6 @@ from app.models.datasource_config import DataSourceConfig
 from app.models.task import CollectionTask
 from app.models.user import User
 from app.models.vessel import AISRawObservation
-from app.services.vessel_ais_aggregation import VESSEL_AIS_SCHEMA
 from app.services.scheduler import (
     sync_datasource_job,
 )
@@ -165,6 +164,8 @@ async def _load_latest_tasks(
 async def _load_collected_record_counts(
     db: AsyncSession,
     sources: list[str],
+    *,
+    exact_vessel_counts: bool = False,
 ) -> dict[str, int]:
     if not sources:
         return {}
@@ -185,14 +186,46 @@ async def _load_collected_record_counts(
         or "ais" in source
     ]
     if vessel_sources:
-        raw_result = await db.execute(
-            select(AISRawObservation.source, func.count(AISRawObservation.id))
-            .where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
-            .where(AISRawObservation.source.in_(vessel_sources))
-            .group_by(AISRawObservation.source)
+        if exact_vessel_counts:
+            exact_result = await db.execute(
+                select(AISRawObservation.source, func.count(AISRawObservation.id))
+                .where(AISRawObservation.source.in_(vessel_sources))
+                .group_by(AISRawObservation.source)
+            )
+            for source, count in exact_result.all():
+                counts[source] = max(counts.get(source, 0), int(count or 0))
+            return counts
+
+        # AIS raw observations can be tens of millions of rows. Use planner
+        # statistics for the datasource list instead of blocking page load on
+        # source-level count(*) scans.
+        stats_result = await db.execute(
+            text(
+                """
+                SELECT
+                    COALESCE(pg_class.reltuples, 0)::bigint AS total_rows,
+                    pg_stats.most_common_vals::text AS source_values,
+                    pg_stats.most_common_freqs::text AS source_freqs
+                FROM pg_class
+                LEFT JOIN pg_stats
+                  ON pg_stats.schemaname = 'public'
+                 AND pg_stats.tablename = 'ais_raw_observations'
+                 AND pg_stats.attname = 'source'
+                WHERE pg_class.relname = 'ais_raw_observations'
+                LIMIT 1
+                """
+            )
         )
-        for source, count in raw_result.all():
-            counts[source] = max(counts.get(source, 0), int(count or 0))
+        stats = stats_result.mappings().first()
+        if stats:
+            total_rows = int(stats["total_rows"] or 0)
+            values = str(stats["source_values"] or "").strip("{}")
+            freqs = str(stats["source_freqs"] or "").strip("{}")
+            source_values = [value.strip('"') for value in values.split(",") if value]
+            source_freqs = [float(value) for value in freqs.split(",") if value]
+            for source, freq in zip(source_values, source_freqs):
+                if source in vessel_sources:
+                    counts[source] = max(counts.get(source, 0), int(round(total_rows * freq)))
 
     return counts
 
@@ -929,7 +962,7 @@ async def get_datasource_row(
         [datasource],
         include_endpoint=include_endpoint,
     )
-    record_counts = await _load_collected_record_counts(db, [datasource.source])
+    record_counts = await _load_collected_record_counts(db, [datasource.source], exact_vessel_counts=True)
     return {
         "data": serialize_datasource_row(
             datasource,

backend/app/api/v1/tasks.py

@@ -29,7 +29,8 @@ async def list_tasks(
         SELECT ct.id, ct.datasource_id, ds.name as datasource_name, ct.status,
                ct.started_at, ct.completed_at, ct.records_processed, ct.error_message,
                ct.phase, ct.phase_progress, ct.phase_message, ct.phase_current,
-               ct.phase_total, ct.phase_unit, ct.total_records, ct.progress
+               ct.phase_total, ct.phase_unit, ct.total_records, ct.progress,
+               ct.task_type, ct.source, ds.source as datasource_source
         FROM collection_tasks ct
         JOIN data_sources ds ON ct.datasource_id = ds.id
         WHERE 1=1
@@ -39,12 +40,19 @@ async def list_tasks(
 
     if datasource_id:
         query += " AND ct.datasource_id = :datasource_id"
-        count_query += " WHERE ct.datasource_id = :datasource_id"
+        count_query += " AND ct.datasource_id = :datasource_id"
         params["datasource_id"] = datasource_id
     if status:
-        query += " AND ct.status = :status"
-        count_query += " AND ct.status = :status"
-        params["status"] = status
+        statuses = [item.strip() for item in status.split(",") if item.strip()]
+        if len(statuses) > 1:
+            placeholders = ", ".join(f":status_{index}" for index, _item in enumerate(statuses))
+            query += f" AND ct.status IN ({placeholders})"
+            count_query += f" AND ct.status IN ({placeholders})"
+            params.update({f"status_{index}": item for index, item in enumerate(statuses)})
+        else:
+            query += " AND ct.status = :status"
+            count_query += " AND ct.status = :status"
+            params["status"] = statuses[0] if statuses else status
 
     query += f" ORDER BY ct.created_at DESC LIMIT {page_size} OFFSET {offset}"
 
@@ -76,6 +84,9 @@ async def list_tasks(
                 "phase_unit": t[13],
                 "total_records": t[14],
                 "progress": t[15],
+                "task_type": t[16],
+                "source": t[17] or t[18],
+                "datasource_source": t[18],
             }
             for t in tasks
         ],

backend/app/services/collectors/aisstream.py

@@ -322,6 +322,21 @@ class AISStreamCollector(BaseCollector):
             last_success_at=now if data else None,
             lag_seconds=max((now - latest_observed_at).total_seconds(), 0),
         )
+        if snapshot_id is not None:
+            from app.models.data_snapshot import DataSnapshot
+
+            snapshot = await db.get(DataSnapshot, snapshot_id)
+            if snapshot:
+                snapshot.record_count = records_added
+                snapshot.status = "success"
+                snapshot.completed_at = now
+                snapshot.summary = {
+                    "created": records_added,
+                    "updated": 0,
+                    "unchanged": 0,
+                    "deleted": 0,
+                    "storage": "ais_raw_observations",
+                }
         await db.commit()
         await self.update_progress(records_added, force=True)
         return records_added

backend/app/services/collectors/vessel_ais.py

@@ -119,6 +119,21 @@ class VesselAISCollector(BaseCollector):
             last_success_at=now if data else None,
             lag_seconds=max((now - latest_observed_at).total_seconds(), 0),
         )
+        if snapshot_id is not None:
+            from app.models.data_snapshot import DataSnapshot
+
+            snapshot = await db.get(DataSnapshot, snapshot_id)
+            if snapshot:
+                snapshot.record_count = records_added
+                snapshot.status = "success"
+                snapshot.completed_at = now
+                snapshot.summary = {
+                    "created": records_added,
+                    "updated": 0,
+                    "unchanged": 0,
+                    "deleted": 0,
+                    "storage": "ais_raw_observations",
+                }
         await db.commit()
         await self._broadcast_vessel_snapshot(data)
         await self.update_progress(records_added, force=True)

backend/app/services/data_jobs.py

@@ -54,6 +54,9 @@ DATA_WRITE_JOB_TYPES = (JOB_TYPE_COLLECT, JOB_TYPE_CLEAR_DATA, JOB_TYPE_CLEAR_CA
 SOURCE_LOCK_JOB_STATUSES = (JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING)
 QUEUE_POLL_SECONDS = 0.35
 JOB_STALE_LOCK_MINUTES = 90
+ORPHAN_CANCELLING_GRACE_SECONDS = 30
+JOB_RECOVERY_SWEEP_SECONDS = 15
+DATA_DELETE_BATCH_SIZE = 50_000
 DEFAULT_WORKER_CONCURRENCY = 2
 
 RUNNING_DATA_JOB_TASKS: dict[int, asyncio.Task[Any]] = {}
@@ -281,6 +284,7 @@ class DataJobWorker:
         self._task: asyncio.Task[None] | None = None
         self._stop_event: asyncio.Event | None = None
         self._running: set[asyncio.Task[Any]] = set()
+        self._last_recovery_sweep_at: datetime | None = None
 
     def start(self) -> None:
         if self._task and not self._task.done():
@@ -303,6 +307,11 @@ class DataJobWorker:
         await self._recover_stale_running_jobs()
         while not self._stop_event.is_set():
             self._running = {task for task in self._running if not task.done()}
+            if (
+                self._last_recovery_sweep_at is None
+                or (_utcnow() - self._last_recovery_sweep_at).total_seconds() >= JOB_RECOVERY_SWEEP_SECONDS
+            ):
+                await self._recover_stale_running_jobs()
             if len(self._running) >= self.concurrency:
                 await asyncio.sleep(QUEUE_POLL_SECONDS)
                 continue
@@ -316,7 +325,9 @@ class DataJobWorker:
             self._running.add(runner)
 
     async def _recover_stale_running_jobs(self) -> None:
+        self._last_recovery_sweep_at = _utcnow()
         cutoff = _utcnow() - timedelta(minutes=JOB_STALE_LOCK_MINUTES)
+        orphan_cancelling_cutoff = _utcnow() - timedelta(seconds=ORPHAN_CANCELLING_GRACE_SECONDS)
         async with async_session_factory() as db:
             result = await db.execute(
                 select(CollectionTask)
@@ -332,6 +343,21 @@ class DataJobWorker:
                 job.error_message = "Marked failed after stale data job lock timeout"
             if stale_jobs:
                 await db.commit()
+            orphan_result = await db.execute(
+                select(CollectionTask)
+                .where(CollectionTask.status == JOB_STATUS_CANCELLING)
+                .where(CollectionTask.locked_at.is_(None))
+                .where(CollectionTask.requested_cancel_at.is_not(None))
+                .where(CollectionTask.requested_cancel_at < orphan_cancelling_cutoff)
+            )
+            for job in orphan_result.scalars().all():
+                if job.id in RUNNING_DATA_JOB_TASKS:
+                    continue
+                await _cancel_task_without_runner(
+                    db,
+                    job,
+                    reason=job.cancel_reason or "cancelled_after_orphaned_runner",
+                )
 
     async def _claim_next_job(self) -> int | None:
         async with async_session_factory() as db:
@@ -477,15 +503,22 @@ async def _run_clear_data_job(db: AsyncSession, task: CollectionTask) -> None:
     await db.commit()
     await _broadcast_task_update(task)
 
-    count_result = await db.execute(
-        select(CollectedData.id).where(CollectedData.source == source)
+    deleted_count = await _delete_table_rows_by_source(
+        db,
+        task,
+        table_name="collected_data",
+        source_column="source",
+        source=source,
+    )
+    derived_deleted_counts = await _clear_derived_datasource_data_in_batches(
+        db,
+        task,
+        source,
+        progress_offset=deleted_count,
     )
-    collected_ids = [row[0] for row in count_result.all()]
-    derived_deleted_counts = await clear_derived_datasource_data(db, source)
-    if collected_ids:
-        await db.execute(CollectedData.__table__.delete().where(CollectedData.id.in_(collected_ids)))
-    deleted_count = len(collected_ids)
     derived_deleted_count = sum(derived_deleted_counts.values())
+    if any(key.startswith("ais_") for key in derived_deleted_counts):
+        await db.execute(text("ANALYZE ais_raw_observations"))
 
     task.records_processed = deleted_count + derived_deleted_count
     task.total_records = task.records_processed
@@ -504,10 +537,99 @@ async def _run_clear_data_job(db: AsyncSession, task: CollectionTask) -> None:
     task.phase = "completed"
     task.phase_message = "数据库数据已清理"
     task.completed_at = _utcnow()
+    datasource = await db.get(DataSource, task.datasource_id)
+    if datasource is not None:
+        datasource.last_status = JOB_STATUS_SUCCESS
+        datasource.last_run_at = task.completed_at
+    await db.execute(
+        DataSnapshot.__table__.update()
+        .where(DataSnapshot.source == source)
+        .values(is_current=False)
+    )
     await db.commit()
     await _broadcast_task_update(task)
 
 
+async def _delete_table_rows_by_source(
+    db: AsyncSession,
+    task: CollectionTask,
+    *,
+    table_name: str,
+    source_column: str,
+    source: str,
+    progress_offset: int = 0,
+) -> int:
+    deleted = 0
+    while True:
+        result = await db.execute(
+            text(
+                f"""
+                WITH doomed AS (
+                    SELECT ctid
+                    FROM {table_name}
+                    WHERE {source_column} = :source
+                    LIMIT :batch_size
+                ),
+                deleted_rows AS (
+                    DELETE FROM {table_name}
+                    USING doomed
+                    WHERE {table_name}.ctid = doomed.ctid
+                    RETURNING 1
+                )
+                SELECT COUNT(*) FROM deleted_rows
+                """
+            ),
+            {"source": source, "batch_size": DATA_DELETE_BATCH_SIZE},
+        )
+        batch_deleted = max(int(result.scalar_one() or 0), 0)
+        if batch_deleted <= 0:
+            break
+        deleted += batch_deleted
+        task.records_processed = progress_offset + deleted
+        task.phase_current = task.records_processed
+        task.phase_unit = "records"
+        task.phase_message = f"正在删除数据：{task.records_processed} 条"
+        await db.commit()
+        await _broadcast_task_update(task)
+    return deleted
+
+
+async def _clear_derived_datasource_data_in_batches(
+    db: AsyncSession,
+    task: CollectionTask,
+    source: str,
+    progress_offset: int = 0,
+) -> dict[str, int]:
+    deleted_counts: dict[str, int] = {}
+    if source in {"barentswatch_vessels", "aisstream_vessels"}:
+        deleted_counts["ais_conflict_records"] = await _delete_table_rows_by_source(
+            db,
+            task,
+            table_name="ais_conflict_records",
+            source_column="selected_source",
+            source=source,
+            progress_offset=progress_offset + sum(deleted_counts.values()),
+        )
+        deleted_counts["ais_source_health"] = await _delete_table_rows_by_source(
+            db,
+            task,
+            table_name="ais_source_health",
+            source_column="source",
+            source=source,
+            progress_offset=progress_offset + sum(deleted_counts.values()),
+        )
+        deleted_counts["ais_raw_observations"] = await _delete_table_rows_by_source(
+            db,
+            task,
+            table_name="ais_raw_observations",
+            source_column="source",
+            source=source,
+            progress_offset=progress_offset + sum(deleted_counts.values()),
+        )
+        return deleted_counts
+    return await clear_derived_datasource_data(db, source)
+
+
 async def _run_clear_cache_job(db: AsyncSession, task: CollectionTask) -> None:
     source = str(task.source or (task.payload or {}).get("source") or "").strip()
     if not source:

backend/app/services/earth_layer_adapters.py

@@ -24,6 +24,7 @@ EARTH_LAYER_ADAPTERS: tuple[EarthLayerAdapter, ...] = (
         tables=frozenset({"vessel_static", "vessel_position", "ais_raw_observations", "ais_source_health"}),
         layers=("vessels",),
         cache_patterns=("vessels*", "summary*"),
+        derived_models=("ais_raw_observations", "ais_conflict_records", "ais_source_health"),
     ),
     EarthLayerAdapter(
         sources=frozenset(
@@ -163,17 +164,24 @@ async def clear_derived_datasource_data(db: AsyncSession, source: str) -> dict[s
     from app.models.bgp_anomaly import BGPAnomaly
     from app.models.bgp_incident import BGPIncident
     from app.models.bgp_observation import BGPObservation
+    from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth
 
     model_by_key: dict[str, Any] = {
         "bgp_observations": BGPObservation,
         "bgp_anomalies": BGPAnomaly,
         "bgp_incidents": BGPIncident,
+        "ais_raw_observations": AISRawObservation,
+        "ais_conflict_records": AISConflictRecord,
+        "ais_source_health": AISSourceHealth,
     }
     deleted_counts: dict[str, int] = {}
     for key in adapter.derived_models:
         model = model_by_key.get(key)
         if model is None:
             continue
-        result = await db.execute(model.__table__.delete().where(model.source == source))
+        if key == "ais_conflict_records":
+            result = await db.execute(model.__table__.delete().where(model.selected_source == source))
+        else:
+            result = await db.execute(model.__table__.delete().where(model.source == source))
         deleted_counts[key] = int(result.rowcount or 0)
     return deleted_counts