release: bump version to 0.45.0

This commit is contained in:
rayd1o
2026-04-29 23:43:54 +08:00
parent 9dafbf4f6e
commit ba54545ac7
32 changed files with 602 additions and 29 deletions

View File

@@ -54,6 +54,11 @@ class BaseCollector(ABC):
"task_id": self._current_task.id,
"status": self._current_task.status,
"phase": self._current_task.phase,
"phase_progress": self._current_task.phase_progress,
"phase_message": self._current_task.phase_message,
"phase_current": self._current_task.phase_current,
"phase_total": self._current_task.phase_total,
"phase_unit": self._current_task.phase_unit,
"progress": progress,
"records_processed": self._current_task.records_processed,
"total_records": self._current_task.total_records,
@@ -80,12 +85,52 @@ class BaseCollector(ABC):
await self._publish_task_update(force=force)
async def set_phase(self, phase: str):
async def set_phase(self, phase: str, *, message: str | None = None, reset_progress: bool = True):
if self._current_task and self._db_session:
self._current_task.phase = phase
self._current_task.phase_message = message
if reset_progress:
self._current_task.phase_progress = None
self._current_task.phase_current = None
self._current_task.phase_total = None
self._current_task.phase_unit = None
await self._db_session.commit()
await self._publish_task_update(force=True)
async def update_phase_progress(
self,
*,
current: int | None = None,
total: int | None = None,
unit: str | None = None,
message: str | None = None,
progress: float | None = None,
commit: bool = False,
force: bool = False,
):
"""Update progress for the current phase without changing task totals."""
if not self._current_task or not self._db_session:
return
if progress is None and current is not None and total and total > 0:
progress = (current / total) * 100
if progress is not None:
self._current_task.phase_progress = max(0.0, min(float(progress), 100.0))
if current is not None:
self._current_task.phase_current = max(0, int(current))
if total is not None:
self._current_task.phase_total = max(0, int(total))
if unit is not None:
self._current_task.phase_unit = unit
if message is not None:
self._current_task.phase_message = message
if commit:
await self._db_session.commit()
await self._publish_task_update(force=force)
@abstractmethod
async def fetch(self) -> List[Dict[str, Any]]:
"""Fetch raw data from source"""
@@ -251,7 +296,7 @@ class BaseCollector(ABC):
await self._publish_task_update(force=True)
try:
await self.set_phase("fetching")
await self.set_phase("fetching", message="正在拉取原始数据")
raw_data = await self.fetch()
task.total_records = len(raw_data)
await db.commit()
@@ -260,15 +305,20 @@ class BaseCollector(ABC):
if self.fail_on_empty and not raw_data:
raise RuntimeError(f"Collector {self.name} returned no data")
await self.set_phase("transforming")
await self.set_phase("transforming", message="正在转换采集数据")
data = self.transform(raw_data)
snapshot_id = await self._create_snapshot(db, task_id, data, start_time)
await self.set_phase("saving")
await self.set_phase("saving", message="正在保存采集数据")
records_count = await self._save_data(db, data, task_id=task_id, snapshot_id=snapshot_id)
task.status = "success"
task.phase = "completed"
task.phase_progress = 100.0
task.phase_message = "采集完成"
task.phase_current = records_count
task.phase_total = records_count
task.phase_unit = "records"
task.records_processed = records_count
task.progress = 100.0
task.completed_at = datetime.now(UTC)
@@ -285,6 +335,7 @@ class BaseCollector(ABC):
await db.rollback()
task.status = "cancelled"
task.phase = "cancelled"
task.phase_message = "采集已取消"
task.error_message = "Collection cancelled by operator and rolled back"
task.completed_at = datetime.now(UTC)
if snapshot_id is not None:
@@ -301,6 +352,7 @@ class BaseCollector(ABC):
await db.rollback()
task.status = "failed"
task.phase = "failed"
task.phase_message = str(e)
task.error_message = str(e)
task.completed_at = datetime.now(UTC)
if snapshot_id is not None:

View File

@@ -108,6 +108,11 @@ class IPtoASNPrefixGeoCollector(BaseCollector):
self._current_task.total_records = total_expected
self._current_task.records_processed = 0
self._current_task.progress = 0.0
self._current_task.phase_progress = 0.0
self._current_task.phase_message = "正在下载 IPtoASN 数据"
self._current_task.phase_current = 0
self._current_task.phase_total = total_expected
self._current_task.phase_unit = "bytes"
await self._db_session.commit()
await self._publish_task_update(force=True)
@@ -135,7 +140,14 @@ class IPtoASNPrefixGeoCollector(BaseCollector):
return
last_emit["value"] = aggregated
last_emit["t"] = now
await self.update_progress(min(aggregated, total_expected), commit=True)
current = min(aggregated, total_expected)
await self.update_phase_progress(
current=current,
total=total_expected,
unit="bytes",
message="正在下载 IPtoASN 数据",
)
await self.update_progress(current, commit=True)
batches = await asyncio.gather(
*(
@@ -148,6 +160,12 @@ class IPtoASNPrefixGeoCollector(BaseCollector):
)
)
if total_expected > 0:
await self.update_phase_progress(
current=total_expected,
total=total_expected,
unit="bytes",
message="IPtoASN 数据下载完成",
)
await self.update_progress(total_expected, commit=True, force=True)
rows: list[dict[str, Any]] = []

View File

@@ -39,12 +39,23 @@ class NRODelegatedPrefixGeoCollector(BaseCollector):
self._current_task.total_records = total_expected
self._current_task.records_processed = 0
self._current_task.progress = 0.0
self._current_task.phase_progress = 0.0
self._current_task.phase_message = "正在下载 NRO delegated 数据"
self._current_task.phase_current = 0
self._current_task.phase_total = total_expected
self._current_task.phase_unit = "bytes"
await self._db_session.commit()
await self._publish_task_update(force=True)
async def on_progress(downloaded: int, total: int | None) -> None:
if not total or total <= 0:
return
await self.update_phase_progress(
current=min(downloaded, total),
total=total,
unit="bytes",
message="正在下载 NRO delegated 数据",
)
await self.update_progress(min(downloaded, total), commit=True)
body_path = await self._downloader.download_file(

View File

@@ -40,12 +40,23 @@ class OpenGeoFeedPrefixGeoCollector(BaseCollector):
self._current_task.total_records = total_expected
self._current_task.records_processed = 0
self._current_task.progress = 0.0
self._current_task.phase_progress = 0.0
self._current_task.phase_message = "正在下载 OpenGeoFeed 数据"
self._current_task.phase_current = 0
self._current_task.phase_total = total_expected
self._current_task.phase_unit = "bytes"
await self._db_session.commit()
await self._publish_task_update(force=True)
async def on_progress(downloaded: int, total: int | None) -> None:
if not total or total <= 0:
return
await self.update_phase_progress(
current=min(downloaded, total),
total=total,
unit="bytes",
message="正在下载 OpenGeoFeed 数据",
)
await self.update_progress(min(downloaded, total), commit=True)
body_path = await self._downloader.download_file(