feat(bgp): improve prefix-geo pipeline and collector reliability

This commit is contained in:
linkong
2026-04-07 15:23:26 +08:00
parent 5b9ef0223d
commit c439e91d12
11 changed files with 621 additions and 62 deletions

View File

@@ -18,6 +18,7 @@ from app.services.collectors.registry import collector_registry
logger = logging.getLogger(__name__)
scheduler = AsyncIOScheduler()
RUNNING_TASK_GUARD_TIMEOUT_MINUTES = 90
async def _update_next_run_at(datasource: DataSource, session) -> None:
@@ -76,6 +77,54 @@ async def run_collector_task(collector_name: str):
logger.info("Skipping disabled collector: %s", collector_name)
return
running_result = await db.execute(
select(CollectionTask)
.where(
CollectionTask.datasource_id == datasource.id,
CollectionTask.status == "running",
)
.order_by(CollectionTask.started_at.desc(), CollectionTask.id.desc())
.limit(1)
)
existing_running = running_result.scalar_one_or_none()
if existing_running is not None:
now = datetime.now(UTC)
started_at = existing_running.started_at
if started_at is not None and started_at.tzinfo is None:
started_at = started_at.replace(tzinfo=UTC)
is_stale = (
started_at is not None
and (now - started_at) > timedelta(minutes=RUNNING_TASK_GUARD_TIMEOUT_MINUTES)
)
if not is_stale:
logger.warning(
"Skipping collector %s trigger because task %s is already running",
collector_name,
existing_running.id,
)
return
existing_error = (existing_running.error_message or "").strip()
stale_reason = (
f"Marked failed automatically after stale running timeout "
f"({RUNNING_TASK_GUARD_TIMEOUT_MINUTES}m) in scheduler guard"
)
existing_running.status = "failed"
existing_running.phase = "failed"
existing_running.completed_at = now
existing_running.error_message = (
f"{existing_error}\n{stale_reason}".strip()
if existing_error
else stale_reason
)
await db.commit()
logger.warning(
"Marked stale running task %s as failed before rerun of %s",
existing_running.id,
collector_name,
)
try:
collector._datasource_id = datasource.id
logger.info("Running collector: %s (datasource_id=%s)", collector_name, datasource.id)