Add force rerun recovery and polish CLI startup

backend/app/services/scheduler.py

@@ -19,6 +19,30 @@ logger = logging.getLogger(__name__)
 
 scheduler = AsyncIOScheduler()
 RUNNING_TASK_GUARD_TIMEOUT_MINUTES = 90
+RUNNING_COLLECTOR_TASKS: dict[str, asyncio.Task[Any]] = {}
+
+
+def _collector_task_name(collector_name: str) -> str:
+    return f"collector:{collector_name}"
+
+
+def get_running_collector_task(collector_name: str) -> asyncio.Task[Any] | None:
+    task = RUNNING_COLLECTOR_TASKS.get(collector_name)
+    if task is not None and not task.done():
+        return task
+
+    if task is not None and task.done():
+        RUNNING_COLLECTOR_TASKS.pop(collector_name, None)
+
+    target_name = _collector_task_name(collector_name)
+    for candidate in asyncio.all_tasks():
+        if candidate.done():
+            continue
+        if candidate.get_name() == target_name:
+            RUNNING_COLLECTOR_TASKS[collector_name] = candidate
+            return candidate
+
+    return None
 
 
 async def _update_next_run_at(datasource: DataSource, session) -> None:
@@ -133,6 +157,12 @@ async def run_collector_task(collector_name: str):
             datasource.last_status = task_result.get("status")
             await _update_next_run_at(datasource, db)
             logger.info("Collector %s completed: %s", collector_name, task_result)
+        except asyncio.CancelledError:
+            datasource.last_run_at = datetime.now(UTC)
+            datasource.last_status = "cancelled"
+            await db.commit()
+            logger.warning("Collector %s cancelled by operator", collector_name)
+            raise
         except Exception as exc:
             datasource.last_run_at = datetime.now(UTC)
             datasource.last_status = "failed"
@@ -245,9 +275,31 @@ def run_collector_now(collector_name: str) -> bool:
         return False
 
     try:
-        asyncio.create_task(run_collector_task(collector_name))
+        task = asyncio.create_task(run_collector_task(collector_name), name=_collector_task_name(collector_name))
+        RUNNING_COLLECTOR_TASKS[collector_name] = task
+
+        def _cleanup_task(done_task: asyncio.Task[Any]) -> None:
+            current = RUNNING_COLLECTOR_TASKS.get(collector_name)
+            if current is done_task:
+                RUNNING_COLLECTOR_TASKS.pop(collector_name, None)
+
+        task.add_done_callback(_cleanup_task)
         logger.info("Triggered collector: %s", collector_name)
         return True
     except Exception as exc:
         logger.error("Failed to trigger collector %s: %s", collector_name, exc)
-        return False
+        return False
+
+
+async def cancel_running_collector_now(collector_name: str) -> bool:
+    task = get_running_collector_task(collector_name)
+    if task is None or task.done():
+        RUNNING_COLLECTOR_TASKS.pop(collector_name, None)
+        return False
+
+    task.cancel()
+    try:
+        await task
+    except asyncio.CancelledError:
+        return True
+    return task.cancelled()