feat(bgp): improve prefix-geo pipeline and collector reliability
This commit is contained in:
@@ -18,6 +18,7 @@ from app.services.collectors.registry import collector_registry
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
scheduler = AsyncIOScheduler()
|
||||
RUNNING_TASK_GUARD_TIMEOUT_MINUTES = 90
|
||||
|
||||
|
||||
async def _update_next_run_at(datasource: DataSource, session) -> None:
|
||||
@@ -76,6 +77,54 @@ async def run_collector_task(collector_name: str):
|
||||
logger.info("Skipping disabled collector: %s", collector_name)
|
||||
return
|
||||
|
||||
running_result = await db.execute(
|
||||
select(CollectionTask)
|
||||
.where(
|
||||
CollectionTask.datasource_id == datasource.id,
|
||||
CollectionTask.status == "running",
|
||||
)
|
||||
.order_by(CollectionTask.started_at.desc(), CollectionTask.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
existing_running = running_result.scalar_one_or_none()
|
||||
if existing_running is not None:
|
||||
now = datetime.now(UTC)
|
||||
started_at = existing_running.started_at
|
||||
if started_at is not None and started_at.tzinfo is None:
|
||||
started_at = started_at.replace(tzinfo=UTC)
|
||||
|
||||
is_stale = (
|
||||
started_at is not None
|
||||
and (now - started_at) > timedelta(minutes=RUNNING_TASK_GUARD_TIMEOUT_MINUTES)
|
||||
)
|
||||
if not is_stale:
|
||||
logger.warning(
|
||||
"Skipping collector %s trigger because task %s is already running",
|
||||
collector_name,
|
||||
existing_running.id,
|
||||
)
|
||||
return
|
||||
|
||||
existing_error = (existing_running.error_message or "").strip()
|
||||
stale_reason = (
|
||||
f"Marked failed automatically after stale running timeout "
|
||||
f"({RUNNING_TASK_GUARD_TIMEOUT_MINUTES}m) in scheduler guard"
|
||||
)
|
||||
existing_running.status = "failed"
|
||||
existing_running.phase = "failed"
|
||||
existing_running.completed_at = now
|
||||
existing_running.error_message = (
|
||||
f"{existing_error}\n{stale_reason}".strip()
|
||||
if existing_error
|
||||
else stale_reason
|
||||
)
|
||||
await db.commit()
|
||||
logger.warning(
|
||||
"Marked stale running task %s as failed before rerun of %s",
|
||||
existing_running.id,
|
||||
collector_name,
|
||||
)
|
||||
|
||||
try:
|
||||
collector._datasource_id = datasource.id
|
||||
logger.info("Running collector: %s (datasource_id=%s)", collector_name, datasource.id)
|
||||
|
||||
Reference in New Issue
Block a user