fix: optimize visualization hot paths and refine bgp brief workspace
This commit is contained in:
@@ -101,25 +101,44 @@ async def get_alert_stats(
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
critical_query = select(func.count(Alert.id)).where(
|
||||
Alert.severity == AlertSeverity.CRITICAL,
|
||||
Alert.status == AlertStatus.ACTIVE,
|
||||
result = await db.execute(
|
||||
select(
|
||||
func.sum(
|
||||
case(
|
||||
(
|
||||
(Alert.severity == AlertSeverity.CRITICAL)
|
||||
& (Alert.status == AlertStatus.ACTIVE),
|
||||
1,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
).label("critical"),
|
||||
func.sum(
|
||||
case(
|
||||
(
|
||||
(Alert.severity == AlertSeverity.WARNING)
|
||||
& (Alert.status == AlertStatus.ACTIVE),
|
||||
1,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
).label("warning"),
|
||||
func.sum(
|
||||
case(
|
||||
(
|
||||
(Alert.severity == AlertSeverity.INFO)
|
||||
& (Alert.status == AlertStatus.ACTIVE),
|
||||
1,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
).label("info"),
|
||||
)
|
||||
)
|
||||
warning_query = select(func.count(Alert.id)).where(
|
||||
Alert.severity == AlertSeverity.WARNING,
|
||||
Alert.status == AlertStatus.ACTIVE,
|
||||
)
|
||||
info_query = select(func.count(Alert.id)).where(
|
||||
Alert.severity == AlertSeverity.INFO,
|
||||
Alert.status == AlertStatus.ACTIVE,
|
||||
)
|
||||
|
||||
critical_result = await db.execute(critical_query)
|
||||
warning_result = await db.execute(warning_query)
|
||||
info_result = await db.execute(info_query)
|
||||
row = result.one()
|
||||
|
||||
return {
|
||||
"critical": critical_result.scalar() or 0,
|
||||
"warning": warning_result.scalar() or 0,
|
||||
"info": info_result.scalar() or 0,
|
||||
"critical": row.critical or 0,
|
||||
"warning": row.warning or 0,
|
||||
"info": row.info or 0,
|
||||
}
|
||||
|
||||
@@ -118,58 +118,77 @@ async def get_stats(
|
||||
built_in_count = len(COLLECTOR_INFO)
|
||||
built_in_active = built_in_count # Built-in are always "active" for counting purposes
|
||||
|
||||
# Count custom configs from database
|
||||
result = await db.execute(select(func.count(DataSourceConfig.id)))
|
||||
custom_count = result.scalar() or 0
|
||||
|
||||
result = await db.execute(
|
||||
select(func.count(DataSourceConfig.id)).where(DataSourceConfig.is_active == True)
|
||||
select(
|
||||
func.count(DataSourceConfig.id).label("custom_count"),
|
||||
func.sum(
|
||||
case((DataSourceConfig.is_active == True, 1), else_=0)
|
||||
).label("custom_active"),
|
||||
)
|
||||
)
|
||||
custom_active = result.scalar() or 0
|
||||
datasource_stats = result.one()
|
||||
custom_count = datasource_stats.custom_count or 0
|
||||
custom_active = datasource_stats.custom_active or 0
|
||||
|
||||
# Total datasources
|
||||
total_datasources = built_in_count + custom_count
|
||||
active_datasources = built_in_active + custom_active
|
||||
|
||||
# Tasks today (from database)
|
||||
result = await db.execute(
|
||||
select(func.count(CollectionTask.id)).where(CollectionTask.started_at >= today_start)
|
||||
)
|
||||
tasks_today = result.scalar() or 0
|
||||
|
||||
result = await db.execute(
|
||||
select(func.count(CollectionTask.id)).where(
|
||||
CollectionTask.status == "success",
|
||||
CollectionTask.started_at >= today_start,
|
||||
select(
|
||||
func.count(CollectionTask.id).label("tasks_today"),
|
||||
func.sum(
|
||||
case(
|
||||
(CollectionTask.status == "success", 1),
|
||||
else_=0,
|
||||
)
|
||||
).label("success_tasks"),
|
||||
)
|
||||
.where(CollectionTask.started_at >= today_start)
|
||||
)
|
||||
success_tasks = result.scalar() or 0
|
||||
task_stats = result.one()
|
||||
tasks_today = task_stats.tasks_today or 0
|
||||
success_tasks = task_stats.success_tasks or 0
|
||||
success_rate = (success_tasks / tasks_today * 100) if tasks_today > 0 else 0
|
||||
|
||||
# Alerts
|
||||
result = await db.execute(
|
||||
select(func.count(Alert.id)).where(
|
||||
Alert.severity == AlertSeverity.CRITICAL,
|
||||
Alert.status == "active",
|
||||
select(
|
||||
func.sum(
|
||||
case(
|
||||
(
|
||||
(Alert.severity == AlertSeverity.CRITICAL)
|
||||
& (Alert.status == "active"),
|
||||
1,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
).label("critical_alerts"),
|
||||
func.sum(
|
||||
case(
|
||||
(
|
||||
(Alert.severity == AlertSeverity.WARNING)
|
||||
& (Alert.status == "active"),
|
||||
1,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
).label("warning_alerts"),
|
||||
func.sum(
|
||||
case(
|
||||
(
|
||||
(Alert.severity == AlertSeverity.INFO)
|
||||
& (Alert.status == "active"),
|
||||
1,
|
||||
),
|
||||
else_=0,
|
||||
)
|
||||
).label("info_alerts"),
|
||||
)
|
||||
)
|
||||
critical_alerts = result.scalar() or 0
|
||||
|
||||
result = await db.execute(
|
||||
select(func.count(Alert.id)).where(
|
||||
Alert.severity == AlertSeverity.WARNING,
|
||||
Alert.status == "active",
|
||||
)
|
||||
)
|
||||
warning_alerts = result.scalar() or 0
|
||||
|
||||
result = await db.execute(
|
||||
select(func.count(Alert.id)).where(
|
||||
Alert.severity == AlertSeverity.INFO,
|
||||
Alert.status == "active",
|
||||
)
|
||||
)
|
||||
info_alerts = result.scalar() or 0
|
||||
alert_stats = result.one()
|
||||
critical_alerts = alert_stats.critical_alerts or 0
|
||||
warning_alerts = alert_stats.warning_alerts or 0
|
||||
info_alerts = alert_stats.info_alerts or 0
|
||||
|
||||
response = {
|
||||
"total_datasources": total_datasources,
|
||||
|
||||
@@ -13,6 +13,7 @@ from app.db.session import get_db
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.models.data_snapshot import DataSnapshot
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
from app.models.task import CollectionTask
|
||||
from app.models.user import User
|
||||
from app.services.scheduler import (
|
||||
@@ -40,6 +41,156 @@ def is_due_for_collection(datasource: DataSource, now: datetime) -> bool:
|
||||
return datasource.last_run_at + timedelta(minutes=datasource.frequency_minutes) <= now
|
||||
|
||||
|
||||
def _task_rank_column(order_column):
|
||||
return func.row_number().over(
|
||||
partition_by=CollectionTask.datasource_id,
|
||||
order_by=(order_column.desc().nullslast(), CollectionTask.id.desc()),
|
||||
).label("row_num")
|
||||
|
||||
|
||||
async def _load_latest_running_tasks(
|
||||
db: AsyncSession,
|
||||
datasource_ids: list[int],
|
||||
) -> dict[int, CollectionTask]:
|
||||
if not datasource_ids:
|
||||
return {}
|
||||
|
||||
ranked_tasks = (
|
||||
select(
|
||||
CollectionTask.id.label("task_id"),
|
||||
_task_rank_column(CollectionTask.started_at),
|
||||
)
|
||||
.where(CollectionTask.datasource_id.in_(datasource_ids))
|
||||
.where(CollectionTask.status == "running")
|
||||
.subquery()
|
||||
)
|
||||
result = await db.execute(
|
||||
select(CollectionTask)
|
||||
.join(ranked_tasks, CollectionTask.id == ranked_tasks.c.task_id)
|
||||
.where(ranked_tasks.c.row_num == 1)
|
||||
)
|
||||
return {task.datasource_id: task for task in result.scalars().all()}
|
||||
|
||||
|
||||
async def _load_latest_completed_tasks(
|
||||
db: AsyncSession,
|
||||
datasource_ids: list[int],
|
||||
) -> dict[int, CollectionTask]:
|
||||
if not datasource_ids:
|
||||
return {}
|
||||
|
||||
ranked_tasks = (
|
||||
select(
|
||||
CollectionTask.id.label("task_id"),
|
||||
_task_rank_column(CollectionTask.completed_at),
|
||||
)
|
||||
.where(CollectionTask.datasource_id.in_(datasource_ids))
|
||||
.where(CollectionTask.completed_at.isnot(None))
|
||||
.where(CollectionTask.status.in_(("success", "failed", "cancelled")))
|
||||
.subquery()
|
||||
)
|
||||
result = await db.execute(
|
||||
select(CollectionTask)
|
||||
.join(ranked_tasks, CollectionTask.id == ranked_tasks.c.task_id)
|
||||
.where(ranked_tasks.c.row_num == 1)
|
||||
)
|
||||
return {task.datasource_id: task for task in result.scalars().all()}
|
||||
|
||||
|
||||
async def _load_latest_task_ids(
|
||||
db: AsyncSession,
|
||||
datasource_ids: list[int],
|
||||
) -> dict[int, int]:
|
||||
if not datasource_ids:
|
||||
return {}
|
||||
|
||||
ranked_tasks = (
|
||||
select(
|
||||
CollectionTask.id.label("task_id"),
|
||||
CollectionTask.datasource_id.label("datasource_id"),
|
||||
func.row_number().over(
|
||||
partition_by=CollectionTask.datasource_id,
|
||||
order_by=CollectionTask.id.desc(),
|
||||
).label("row_num"),
|
||||
)
|
||||
.where(CollectionTask.datasource_id.in_(datasource_ids))
|
||||
.subquery()
|
||||
)
|
||||
result = await db.execute(
|
||||
select(ranked_tasks.c.datasource_id, ranked_tasks.c.task_id)
|
||||
.where(ranked_tasks.c.row_num == 1)
|
||||
)
|
||||
return {datasource_id: task_id for datasource_id, task_id in result.all()}
|
||||
|
||||
|
||||
async def _load_datasource_data_counts(
|
||||
db: AsyncSession,
|
||||
sources: list[str],
|
||||
) -> dict[str, int]:
|
||||
if not sources:
|
||||
return {}
|
||||
|
||||
result = await db.execute(
|
||||
select(CollectedData.source, func.count(CollectedData.id))
|
||||
.where(CollectedData.source.in_(sources))
|
||||
.group_by(CollectedData.source)
|
||||
)
|
||||
return {source: count for source, count in result.all()}
|
||||
|
||||
|
||||
async def _load_datasource_endpoint_overrides(
|
||||
db: AsyncSession,
|
||||
sources: list[str],
|
||||
) -> dict[str, str]:
|
||||
if not sources:
|
||||
return {}
|
||||
|
||||
result = await db.execute(
|
||||
select(DataSourceConfig.name, DataSourceConfig.endpoint)
|
||||
.where(DataSourceConfig.name.in_(sources))
|
||||
.where(DataSourceConfig.is_active.is_(True))
|
||||
.where(DataSourceConfig.endpoint.isnot(None))
|
||||
)
|
||||
return {
|
||||
name: endpoint
|
||||
for name, endpoint in result.all()
|
||||
if endpoint
|
||||
}
|
||||
|
||||
|
||||
async def _load_datasource_list_context(
|
||||
db: AsyncSession,
|
||||
datasources: list[DataSource],
|
||||
) -> tuple[dict[int, CollectionTask], dict[int, CollectionTask], dict[str, int], dict[str, str]]:
|
||||
datasource_ids = [datasource.id for datasource in datasources]
|
||||
sources = [datasource.source for datasource in datasources]
|
||||
|
||||
running_tasks = await _load_latest_running_tasks(db, datasource_ids)
|
||||
datasource_by_id = {datasource.id: datasource for datasource in datasources}
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
stale_datasource_ids: list[int] = []
|
||||
for datasource_id, task in running_tasks.items():
|
||||
started_at = task.started_at
|
||||
if started_at is None:
|
||||
continue
|
||||
if started_at.tzinfo is None:
|
||||
started_at = started_at.replace(tzinfo=timezone.utc)
|
||||
if now - started_at > timedelta(minutes=STALE_RUNNING_TASK_TIMEOUT_MINUTES):
|
||||
datasource = datasource_by_id.get(datasource_id)
|
||||
if datasource is not None:
|
||||
await fail_and_rollback_stale_running_task(db, datasource, task)
|
||||
stale_datasource_ids.append(datasource_id)
|
||||
|
||||
if stale_datasource_ids:
|
||||
running_tasks = await _load_latest_running_tasks(db, datasource_ids)
|
||||
|
||||
completed_tasks = await _load_latest_completed_tasks(db, datasource_ids)
|
||||
data_counts = await _load_datasource_data_counts(db, sources)
|
||||
endpoint_overrides = await _load_datasource_endpoint_overrides(db, sources)
|
||||
return running_tasks, completed_tasks, data_counts, endpoint_overrides
|
||||
|
||||
|
||||
async def get_datasource_record(db: AsyncSession, source_id: str) -> Optional[DataSource]:
|
||||
datasource = None
|
||||
try:
|
||||
@@ -58,18 +209,6 @@ async def get_datasource_record(db: AsyncSession, source_id: str) -> Optional[Da
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def get_last_completed_task(db: AsyncSession, datasource_id: int) -> Optional[CollectionTask]:
|
||||
result = await db.execute(
|
||||
select(CollectionTask)
|
||||
.where(CollectionTask.datasource_id == datasource_id)
|
||||
.where(CollectionTask.completed_at.isnot(None))
|
||||
.where(CollectionTask.status.in_(("success", "failed", "cancelled")))
|
||||
.order_by(CollectionTask.completed_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def get_running_task(db: AsyncSession, datasource_id: int) -> Optional[CollectionTask]:
|
||||
result = await db.execute(
|
||||
select(CollectionTask)
|
||||
@@ -262,14 +401,17 @@ async def list_datasources(
|
||||
|
||||
collector_list = []
|
||||
config = get_data_sources_config()
|
||||
running_tasks, completed_tasks, data_counts, endpoint_overrides = await _load_datasource_list_context(
|
||||
db,
|
||||
datasources,
|
||||
)
|
||||
for datasource in datasources:
|
||||
running_task = await get_running_task(db, datasource.id)
|
||||
last_task = await get_last_completed_task(db, datasource.id)
|
||||
endpoint = await config.get_url(datasource.source, db)
|
||||
data_count_result = await db.execute(
|
||||
select(func.count(CollectedData.id)).where(CollectedData.source == datasource.source)
|
||||
running_task = running_tasks.get(datasource.id)
|
||||
last_task = completed_tasks.get(datasource.id)
|
||||
endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(
|
||||
datasource.source,
|
||||
)
|
||||
data_count = data_count_result.scalar() or 0
|
||||
data_count = data_counts.get(datasource.source, 0)
|
||||
|
||||
last_run_at = datasource.last_run_at or (last_task.completed_at if last_task else None)
|
||||
last_run = to_iso8601_utc(last_run_at)
|
||||
@@ -330,9 +472,13 @@ async def trigger_all_datasources(
|
||||
skipped_sources: list[dict] = []
|
||||
failed_sources: list[dict] = []
|
||||
now = datetime.now(timezone.utc)
|
||||
running_tasks = await _load_latest_running_tasks(
|
||||
db,
|
||||
[datasource.id for datasource in datasources],
|
||||
)
|
||||
|
||||
for datasource in datasources:
|
||||
running_task = await get_running_task(db, datasource.id)
|
||||
running_task = running_tasks.get(datasource.id)
|
||||
if running_task is not None:
|
||||
skipped_sources.append(
|
||||
{
|
||||
@@ -360,7 +506,7 @@ async def trigger_all_datasources(
|
||||
)
|
||||
continue
|
||||
|
||||
previous_task_ids[datasource.id] = await get_latest_task_id_for_datasource(datasource.id)
|
||||
previous_task_ids[datasource.id] = None
|
||||
success = run_collector_now(datasource.source)
|
||||
if not success:
|
||||
failed_sources.append(
|
||||
@@ -382,13 +528,24 @@ async def trigger_all_datasources(
|
||||
}
|
||||
)
|
||||
|
||||
latest_task_ids = await _load_latest_task_ids(
|
||||
db,
|
||||
[datasource.id for datasource in datasources],
|
||||
)
|
||||
for datasource_id in previous_task_ids:
|
||||
previous_task_ids[datasource_id] = latest_task_ids.get(datasource_id)
|
||||
|
||||
for _ in range(20):
|
||||
await asyncio.sleep(0.1)
|
||||
pending = [item for item in triggered_sources if item["task_id"] is None]
|
||||
if not pending:
|
||||
break
|
||||
latest_task_ids = await _load_latest_task_ids(
|
||||
db,
|
||||
[item["id"] for item in pending],
|
||||
)
|
||||
for item in pending:
|
||||
task_id = await get_latest_task_id_for_datasource(item["id"])
|
||||
task_id = latest_task_ids.get(item["id"])
|
||||
if task_id is not None and task_id != previous_task_ids.get(item["id"]):
|
||||
item["task_id"] = task_id
|
||||
|
||||
|
||||
@@ -79,6 +79,26 @@ async def get_setting_record(db: AsyncSession, category: str) -> Optional[System
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def get_setting_payloads(db: AsyncSession, categories: list[str]) -> dict[str, dict]:
|
||||
if not categories:
|
||||
return {}
|
||||
|
||||
result = await db.execute(
|
||||
select(SystemSetting).where(SystemSetting.category.in_(categories))
|
||||
)
|
||||
records_by_category = {
|
||||
record.category: record
|
||||
for record in result.scalars().all()
|
||||
}
|
||||
return {
|
||||
category: merge_with_defaults(
|
||||
category,
|
||||
records_by_category.get(category).payload if records_by_category.get(category) else None,
|
||||
)
|
||||
for category in categories
|
||||
}
|
||||
|
||||
|
||||
async def get_setting_payload(db: AsyncSession, category: str) -> dict:
|
||||
record = await get_setting_record(db, category)
|
||||
return merge_with_defaults(category, record.payload if record else None)
|
||||
@@ -212,10 +232,14 @@ async def get_all_settings(
|
||||
):
|
||||
result = await db.execute(select(DataSource).order_by(DataSource.module, DataSource.id))
|
||||
datasources = result.scalars().all()
|
||||
setting_payloads = await get_setting_payloads(
|
||||
db,
|
||||
["system", "notifications", "security"],
|
||||
)
|
||||
return {
|
||||
"system": await get_setting_payload(db, "system"),
|
||||
"notifications": await get_setting_payload(db, "notifications"),
|
||||
"security": await get_setting_payload(db, "security"),
|
||||
"system": setting_payloads["system"],
|
||||
"notifications": setting_payloads["notifications"],
|
||||
"security": setting_payloads["security"],
|
||||
"collectors": [serialize_collector(datasource) for datasource in datasources],
|
||||
"generated_at": to_iso8601_utc(datetime.now(UTC)),
|
||||
}
|
||||
|
||||
@@ -205,42 +205,84 @@ def convert_satellite_to_geojson(records: List[CollectedData]) -> Dict[str, Any]
|
||||
return {"type": "FeatureCollection", "features": features}
|
||||
|
||||
|
||||
def dedupe_satellite_records(records: List[CollectedData]) -> List[CollectedData]:
|
||||
"""Keep only the newest record for each satellite identity."""
|
||||
latest_by_key: Dict[str, CollectedData] = {}
|
||||
|
||||
for record in records:
|
||||
metadata = record.extra_data or {}
|
||||
norad_id = metadata.get("norad_cat_id")
|
||||
dedupe_key = (
|
||||
str(norad_id)
|
||||
if norad_id not in (None, "")
|
||||
else str(record.source_id or record.entity_key or record.name or record.id)
|
||||
)
|
||||
|
||||
existing = latest_by_key.get(dedupe_key)
|
||||
if existing is None or (record.id or 0) > (existing.id or 0):
|
||||
latest_by_key[dedupe_key] = record
|
||||
|
||||
return sorted(latest_by_key.values(), key=lambda item: item.id or 0, reverse=True)
|
||||
def _current_collected_data_stmt(source: str):
|
||||
return (
|
||||
select(CollectedData)
|
||||
.where(CollectedData.source == source)
|
||||
.where(CollectedData.is_current.is_(True))
|
||||
.order_by(CollectedData.id.desc())
|
||||
)
|
||||
|
||||
|
||||
def dedupe_collected_records(records: List[CollectedData]) -> List[CollectedData]:
|
||||
"""Keep only the newest record for each collected entity."""
|
||||
latest_by_key: Dict[str, CollectedData] = {}
|
||||
async def _load_current_collected_data(
|
||||
db: AsyncSession,
|
||||
source: str,
|
||||
*,
|
||||
exclude_unknown_name: bool = False,
|
||||
limit: Optional[int] = None,
|
||||
) -> List[CollectedData]:
|
||||
stmt = _current_collected_data_stmt(source)
|
||||
if exclude_unknown_name:
|
||||
stmt = stmt.where(CollectedData.name != "Unknown")
|
||||
if limit is not None:
|
||||
stmt = stmt.limit(limit)
|
||||
|
||||
for record in records:
|
||||
dedupe_key = str(
|
||||
record.source_id
|
||||
or record.entity_key
|
||||
or record.name
|
||||
or record.id
|
||||
)
|
||||
existing = latest_by_key.get(dedupe_key)
|
||||
if existing is None or (record.id or 0) > (existing.id or 0):
|
||||
latest_by_key[dedupe_key] = record
|
||||
result = await db.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
|
||||
return sorted(latest_by_key.values(), key=lambda item: item.id or 0, reverse=True)
|
||||
|
||||
async def _load_current_collected_data_by_sources(
|
||||
db: AsyncSession,
|
||||
sources: List[str],
|
||||
) -> Dict[str, List[CollectedData]]:
|
||||
if not sources:
|
||||
return {}
|
||||
|
||||
stmt = (
|
||||
select(CollectedData)
|
||||
.where(CollectedData.source.in_(sources))
|
||||
.where(CollectedData.is_current.is_(True))
|
||||
.order_by(CollectedData.source.asc(), CollectedData.id.desc())
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
|
||||
grouped_records: Dict[str, List[CollectedData]] = {source: [] for source in sources}
|
||||
for record in result.scalars().all():
|
||||
grouped_records.setdefault(record.source, []).append(record)
|
||||
|
||||
return grouped_records
|
||||
|
||||
|
||||
def _build_landing_point_cable_maps(
|
||||
relation_records: List[CollectedData],
|
||||
cable_records: List[CollectedData],
|
||||
) -> tuple[Dict[int, List[int]], Dict[int, str]]:
|
||||
city_to_cable_ids_map: Dict[int, List[int]] = {}
|
||||
for relation_record in relation_records:
|
||||
if not relation_record.extra_data:
|
||||
continue
|
||||
city_id = relation_record.extra_data.get("city_id")
|
||||
cable_id = relation_record.extra_data.get("cable_id")
|
||||
if city_id is None or cable_id is None:
|
||||
continue
|
||||
city_to_cable_ids_map.setdefault(city_id, [])
|
||||
if cable_id not in city_to_cable_ids_map[city_id]:
|
||||
city_to_cable_ids_map[city_id].append(cable_id)
|
||||
|
||||
cable_id_to_name_map: Dict[int, str] = {}
|
||||
for cable_record in cable_records:
|
||||
if not cable_record.extra_data:
|
||||
continue
|
||||
cable_id = cable_record.extra_data.get("cable_id")
|
||||
cable_name = cable_record.name
|
||||
if cable_id and cable_name:
|
||||
cable_id_to_name_map[cable_id] = cable_name
|
||||
|
||||
return city_to_cable_ids_map, cable_id_to_name_map
|
||||
|
||||
|
||||
def _filter_known_records(records: List[CollectedData]) -> List[CollectedData]:
|
||||
return [record for record in records if record.name != "Unknown"]
|
||||
|
||||
|
||||
def convert_supercomputer_to_geojson(records: List[CollectedData]) -> Dict[str, Any]:
|
||||
@@ -722,9 +764,7 @@ def convert_bgp_incidents_to_geojson(
|
||||
async def get_cables_geojson(db: AsyncSession = Depends(get_db)):
|
||||
"""获取海底电缆 GeoJSON 数据 (LineString)"""
|
||||
try:
|
||||
stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
|
||||
result = await db.execute(stmt)
|
||||
records = dedupe_collected_records(list(result.scalars().all()))
|
||||
records = await _load_current_collected_data(db, "arcgis_cables")
|
||||
|
||||
if not records:
|
||||
raise HTTPException(
|
||||
@@ -742,36 +782,14 @@ async def get_cables_geojson(db: AsyncSession = Depends(get_db)):
|
||||
@router.get("/geo/landing-points")
|
||||
async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
|
||||
try:
|
||||
landing_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
|
||||
landing_result = await db.execute(landing_stmt)
|
||||
records = dedupe_collected_records(list(landing_result.scalars().all()))
|
||||
|
||||
relation_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cable_landing_relation")
|
||||
relation_result = await db.execute(relation_stmt)
|
||||
relation_records = dedupe_collected_records(list(relation_result.scalars().all()))
|
||||
|
||||
cable_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
|
||||
cable_result = await db.execute(cable_stmt)
|
||||
cable_records = dedupe_collected_records(list(cable_result.scalars().all()))
|
||||
|
||||
city_to_cable_ids_map = {}
|
||||
for rel in relation_records:
|
||||
if rel.extra_data:
|
||||
city_id = rel.extra_data.get("city_id")
|
||||
cable_id = rel.extra_data.get("cable_id")
|
||||
if city_id is not None and cable_id is not None:
|
||||
if city_id not in city_to_cable_ids_map:
|
||||
city_to_cable_ids_map[city_id] = []
|
||||
if cable_id not in city_to_cable_ids_map[city_id]:
|
||||
city_to_cable_ids_map[city_id].append(cable_id)
|
||||
|
||||
cable_id_to_name_map = {}
|
||||
for cable in cable_records:
|
||||
if cable.extra_data:
|
||||
cable_id = cable.extra_data.get("cable_id")
|
||||
cable_name = cable.name
|
||||
if cable_id and cable_name:
|
||||
cable_id_to_name_map[cable_id] = cable_name
|
||||
records = await _load_current_collected_data(db, "arcgis_landing_points")
|
||||
relation_records = await _load_current_collected_data(db, "arcgis_cable_landing_relation")
|
||||
cable_records = await _load_current_collected_data(db, "arcgis_cables")
|
||||
|
||||
city_to_cable_ids_map, cable_id_to_name_map = _build_landing_point_cable_maps(
|
||||
relation_records,
|
||||
cable_records,
|
||||
)
|
||||
|
||||
if not records:
|
||||
raise HTTPException(
|
||||
@@ -788,36 +806,21 @@ async def get_landing_points_geojson(db: AsyncSession = Depends(get_db)):
|
||||
|
||||
@router.get("/geo/all")
|
||||
async def get_all_geojson(db: AsyncSession = Depends(get_db)):
|
||||
cables_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
|
||||
cables_result = await db.execute(cables_stmt)
|
||||
cables_records = dedupe_collected_records(list(cables_result.scalars().all()))
|
||||
|
||||
points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
|
||||
points_result = await db.execute(points_stmt)
|
||||
points_records = dedupe_collected_records(list(points_result.scalars().all()))
|
||||
|
||||
relation_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cable_landing_relation")
|
||||
relation_result = await db.execute(relation_stmt)
|
||||
relation_records = dedupe_collected_records(list(relation_result.scalars().all()))
|
||||
|
||||
city_to_cable_ids_map = {}
|
||||
for rel in relation_records:
|
||||
if rel.extra_data:
|
||||
city_id = rel.extra_data.get("city_id")
|
||||
cable_id = rel.extra_data.get("cable_id")
|
||||
if city_id is not None and cable_id is not None:
|
||||
if city_id not in city_to_cable_ids_map:
|
||||
city_to_cable_ids_map[city_id] = []
|
||||
if cable_id not in city_to_cable_ids_map[city_id]:
|
||||
city_to_cable_ids_map[city_id].append(cable_id)
|
||||
|
||||
cable_id_to_name_map = {}
|
||||
for cable in cables_records:
|
||||
if cable.extra_data:
|
||||
cable_id = cable.extra_data.get("cable_id")
|
||||
cable_name = cable.name
|
||||
if cable_id and cable_name:
|
||||
cable_id_to_name_map[cable_id] = cable_name
|
||||
records_by_source = await _load_current_collected_data_by_sources(
|
||||
db,
|
||||
[
|
||||
"arcgis_cables",
|
||||
"arcgis_landing_points",
|
||||
"arcgis_cable_landing_relation",
|
||||
],
|
||||
)
|
||||
cables_records = records_by_source.get("arcgis_cables", [])
|
||||
points_records = records_by_source.get("arcgis_landing_points", [])
|
||||
relation_records = records_by_source.get("arcgis_cable_landing_relation", [])
|
||||
city_to_cable_ids_map, cable_id_to_name_map = _build_landing_point_cable_maps(
|
||||
relation_records,
|
||||
cables_records,
|
||||
)
|
||||
|
||||
cables = (
|
||||
convert_cable_to_geojson(cables_records)
|
||||
@@ -850,17 +853,12 @@ async def get_satellites_geojson(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""获取卫星 TLE GeoJSON 数据"""
|
||||
stmt = (
|
||||
select(CollectedData)
|
||||
.where(CollectedData.source == "celestrak_tle")
|
||||
.where(CollectedData.name != "Unknown")
|
||||
.order_by(CollectedData.id.desc())
|
||||
records = await _load_current_collected_data(
|
||||
db,
|
||||
"celestrak_tle",
|
||||
exclude_unknown_name=True,
|
||||
limit=limit,
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
records = dedupe_satellite_records(list(result.scalars().all()))
|
||||
|
||||
if limit is not None:
|
||||
records = records[:limit]
|
||||
|
||||
if not records:
|
||||
return {"type": "FeatureCollection", "features": [], "count": 0}
|
||||
@@ -878,15 +876,12 @@ async def get_supercomputers_geojson(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""获取 TOP500 超算中心 GeoJSON 数据"""
|
||||
stmt = (
|
||||
select(CollectedData)
|
||||
.where(CollectedData.source == "top500")
|
||||
.where(CollectedData.name != "Unknown")
|
||||
.order_by(CollectedData.id.desc())
|
||||
records = await _load_current_collected_data(
|
||||
db,
|
||||
"top500",
|
||||
exclude_unknown_name=True,
|
||||
limit=limit,
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
records = dedupe_collected_records(list(result.scalars().all()))
|
||||
records = records[:limit]
|
||||
|
||||
if not records:
|
||||
return {"type": "FeatureCollection", "features": [], "count": 0}
|
||||
@@ -904,15 +899,12 @@ async def get_gpu_clusters_geojson(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""获取 GPU 集群 GeoJSON 数据"""
|
||||
stmt = (
|
||||
select(CollectedData)
|
||||
.where(CollectedData.source == "epoch_ai_gpu")
|
||||
.where(CollectedData.name != "Unknown")
|
||||
.order_by(CollectedData.id.desc())
|
||||
records = await _load_current_collected_data(
|
||||
db,
|
||||
"epoch_ai_gpu",
|
||||
exclude_unknown_name=True,
|
||||
limit=limit,
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
records = dedupe_collected_records(list(result.scalars().all()))
|
||||
records = records[:limit]
|
||||
|
||||
if not records:
|
||||
return {"type": "FeatureCollection", "features": [], "count": 0}
|
||||
@@ -990,37 +982,27 @@ async def get_all_visualization_data(db: AsyncSession = Depends(get_db)):
|
||||
- supercomputers: TOP500 超算
|
||||
- gpu_clusters: GPU 集群
|
||||
"""
|
||||
cables_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
|
||||
cables_result = await db.execute(cables_stmt)
|
||||
cables_records = dedupe_collected_records(list(cables_result.scalars().all()))
|
||||
|
||||
points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
|
||||
points_result = await db.execute(points_stmt)
|
||||
points_records = dedupe_collected_records(list(points_result.scalars().all()))
|
||||
|
||||
satellites_stmt = (
|
||||
select(CollectedData)
|
||||
.where(CollectedData.source == "celestrak_tle")
|
||||
.where(CollectedData.name != "Unknown")
|
||||
records_by_source = await _load_current_collected_data_by_sources(
|
||||
db,
|
||||
[
|
||||
"arcgis_cables",
|
||||
"arcgis_landing_points",
|
||||
"celestrak_tle",
|
||||
"top500",
|
||||
"epoch_ai_gpu",
|
||||
],
|
||||
)
|
||||
satellites_result = await db.execute(satellites_stmt)
|
||||
satellites_records = dedupe_satellite_records(list(satellites_result.scalars().all()))
|
||||
|
||||
supercomputers_stmt = (
|
||||
select(CollectedData)
|
||||
.where(CollectedData.source == "top500")
|
||||
.where(CollectedData.name != "Unknown")
|
||||
cables_records = records_by_source.get("arcgis_cables", [])
|
||||
points_records = records_by_source.get("arcgis_landing_points", [])
|
||||
satellites_records = _filter_known_records(
|
||||
records_by_source.get("celestrak_tle", []),
|
||||
)
|
||||
supercomputers_result = await db.execute(supercomputers_stmt)
|
||||
supercomputers_records = dedupe_collected_records(list(supercomputers_result.scalars().all()))
|
||||
|
||||
gpu_stmt = (
|
||||
select(CollectedData)
|
||||
.where(CollectedData.source == "epoch_ai_gpu")
|
||||
.where(CollectedData.name != "Unknown")
|
||||
supercomputers_records = _filter_known_records(
|
||||
records_by_source.get("top500", []),
|
||||
)
|
||||
gpu_records = _filter_known_records(
|
||||
records_by_source.get("epoch_ai_gpu", []),
|
||||
)
|
||||
gpu_result = await db.execute(gpu_stmt)
|
||||
gpu_records = dedupe_collected_records(list(gpu_result.scalars().all()))
|
||||
|
||||
cables = (
|
||||
convert_cable_to_geojson(cables_records)
|
||||
@@ -1084,13 +1066,8 @@ async def get_cable_graph(db: AsyncSession) -> CableGraph:
|
||||
global _cable_graph
|
||||
|
||||
if _cable_graph is None:
|
||||
cables_stmt = select(CollectedData).where(CollectedData.source == "arcgis_cables")
|
||||
cables_result = await db.execute(cables_stmt)
|
||||
cables_records = list(cables_result.scalars().all())
|
||||
|
||||
points_stmt = select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
|
||||
points_result = await db.execute(points_stmt)
|
||||
points_records = list(points_result.scalars().all())
|
||||
cables_records = await _load_current_collected_data(db, "arcgis_cables")
|
||||
points_records = await _load_current_collected_data(db, "arcgis_landing_points")
|
||||
|
||||
cables_data = convert_cable_to_geojson(cables_records)
|
||||
points_data = convert_landing_point_to_geojson(points_records)
|
||||
|
||||
@@ -35,8 +35,8 @@ async def build_bgp_collector_coverage(
|
||||
recent_7d_threshold = now - timedelta(days=7)
|
||||
|
||||
filters = _collector_base_filters(source_filter)
|
||||
country_expr = func.nullif(BGPObservation.collector_geo["country"].astext, "")
|
||||
city_expr = func.nullif(BGPObservation.collector_geo["city"].astext, "")
|
||||
country_expr = func.nullif(BGPObservation.collector_geo["country"].as_string(), "")
|
||||
city_expr = func.nullif(BGPObservation.collector_geo["city"].as_string(), "")
|
||||
|
||||
aggregate_stmt = (
|
||||
select(
|
||||
|
||||
@@ -7,7 +7,7 @@ from collections import defaultdict
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy import Integer, cast, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.countries import get_country_centroid, normalize_country
|
||||
@@ -261,29 +261,40 @@ async def enrich_bgp_events_for_batch(
|
||||
historical_prefix_baseline: dict[str, dict[str, Any]] = {}
|
||||
if prefix_values:
|
||||
previous_result = await db.execute(
|
||||
select(BGPObservation).where(
|
||||
select(
|
||||
BGPObservation.prefix,
|
||||
BGPObservation.origin_asn,
|
||||
BGPObservation.collector,
|
||||
BGPObservation.collector_geo,
|
||||
).where(
|
||||
BGPObservation.source == source,
|
||||
BGPObservation.prefix.in_(prefix_values),
|
||||
)
|
||||
)
|
||||
by_prefix: defaultdict[str, list[BGPObservation]] = defaultdict(list)
|
||||
for observation in previous_result.scalars().all():
|
||||
if observation.prefix:
|
||||
by_prefix[observation.prefix].append(observation)
|
||||
by_prefix: defaultdict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for prefix, origin_asn, collector, collector_geo in previous_result.all():
|
||||
if prefix:
|
||||
by_prefix[str(prefix)].append(
|
||||
{
|
||||
"origin_asn": origin_asn,
|
||||
"collector": collector,
|
||||
"collector_geo": collector_geo or {},
|
||||
}
|
||||
)
|
||||
|
||||
for prefix, observations in by_prefix.items():
|
||||
unique_origins = sorted(
|
||||
{
|
||||
observation.origin_asn
|
||||
observation["origin_asn"]
|
||||
for observation in observations
|
||||
if observation.origin_asn is not None
|
||||
if observation["origin_asn"] is not None
|
||||
}
|
||||
)
|
||||
unique_collectors = sorted(
|
||||
{
|
||||
observation.collector
|
||||
observation["collector"]
|
||||
for observation in observations
|
||||
if observation.collector
|
||||
if observation["collector"]
|
||||
}
|
||||
)
|
||||
historical_prefix_baseline[prefix] = {
|
||||
@@ -292,9 +303,9 @@ async def enrich_bgp_events_for_batch(
|
||||
"historical_observation_count": len(observations),
|
||||
"historical_regions": _compact_locations(
|
||||
[
|
||||
observation.collector_geo or {}
|
||||
observation["collector_geo"] or {}
|
||||
for observation in observations
|
||||
if observation.collector_geo
|
||||
if observation["collector_geo"]
|
||||
]
|
||||
),
|
||||
}
|
||||
@@ -303,7 +314,13 @@ async def enrich_bgp_events_for_batch(
|
||||
prefix_geographies = await _lookup_prefix_geography(db, prefix_values) if prefix_values else {}
|
||||
if origin_asns:
|
||||
peeringdb_result = await db.execute(
|
||||
select(CollectedData).where(CollectedData.source == "peeringdb_network")
|
||||
select(CollectedData)
|
||||
.where(CollectedData.source == "peeringdb_network")
|
||||
.where(CollectedData.is_current.is_(True))
|
||||
.where(
|
||||
cast(CollectedData.extra_data["asn"].as_string(), Integer).in_(origin_asns),
|
||||
)
|
||||
.order_by(CollectedData.id.desc())
|
||||
)
|
||||
for record in peeringdb_result.scalars().all():
|
||||
metadata = record.extra_data or {}
|
||||
|
||||
@@ -48,14 +48,36 @@ def _collector_regions_from_anomaly(anomaly: BGPAnomaly) -> list[dict]:
|
||||
return collected
|
||||
|
||||
|
||||
def _dedupe_collected_records(records: list[CollectedData]) -> list[CollectedData]:
|
||||
latest_by_key: dict[str, CollectedData] = {}
|
||||
for record in records:
|
||||
dedupe_key = str(record.source_id or record.entity_key or record.name or record.id)
|
||||
existing = latest_by_key.get(dedupe_key)
|
||||
if existing is None or (record.id or 0) > (existing.id or 0):
|
||||
latest_by_key[dedupe_key] = record
|
||||
return list(latest_by_key.values())
|
||||
async def _load_current_infrastructure_records(
|
||||
db: AsyncSession,
|
||||
) -> tuple[list[CollectedData], list[CollectedData], list[CollectedData]]:
|
||||
result = await db.execute(
|
||||
select(CollectedData)
|
||||
.where(
|
||||
CollectedData.source.in_(
|
||||
(
|
||||
"arcgis_landing_points",
|
||||
"arcgis_cable_landing_relation",
|
||||
"arcgis_cables",
|
||||
)
|
||||
)
|
||||
)
|
||||
.where(CollectedData.is_current.is_(True))
|
||||
.order_by(CollectedData.source.asc(), CollectedData.id.desc())
|
||||
)
|
||||
grouped_records = {
|
||||
"arcgis_landing_points": [],
|
||||
"arcgis_cable_landing_relation": [],
|
||||
"arcgis_cables": [],
|
||||
}
|
||||
for record in result.scalars().all():
|
||||
grouped_records.setdefault(record.source, []).append(record)
|
||||
|
||||
return (
|
||||
grouped_records["arcgis_landing_points"],
|
||||
grouped_records["arcgis_cable_landing_relation"],
|
||||
grouped_records["arcgis_cables"],
|
||||
)
|
||||
|
||||
|
||||
async def infer_related_infrastructure(
|
||||
@@ -75,19 +97,9 @@ async def infer_related_infrastructure(
|
||||
if not valid_regions:
|
||||
return {"related_cables": [], "related_ixps": []}
|
||||
|
||||
landing_result = await db.execute(
|
||||
select(CollectedData).where(CollectedData.source == "arcgis_landing_points")
|
||||
landing_records, relation_records, cable_records = await _load_current_infrastructure_records(
|
||||
db,
|
||||
)
|
||||
relation_result = await db.execute(
|
||||
select(CollectedData).where(CollectedData.source == "arcgis_cable_landing_relation")
|
||||
)
|
||||
cable_result = await db.execute(
|
||||
select(CollectedData).where(CollectedData.source == "arcgis_cables")
|
||||
)
|
||||
|
||||
landing_records = _dedupe_collected_records(list(landing_result.scalars().all()))
|
||||
relation_records = _dedupe_collected_records(list(relation_result.scalars().all()))
|
||||
cable_records = _dedupe_collected_records(list(cable_result.scalars().all()))
|
||||
|
||||
city_to_cable_ids: dict[int, list[int]] = {}
|
||||
for relation in relation_records:
|
||||
|
||||
@@ -36,6 +36,28 @@ Released: 2026-04-10
|
||||
- Fixed [backend/app/api/v1/bgp.py](/home/ray/dev/linkong/planet/backend/app/api/v1/bgp.py) so `events`, `anomalies`, and `incidents` no longer fetch whole tables into Python just to apply filtering, pagination, and counting.
|
||||
- Fixed the repository workflow around saved BGP brief artifacts by ignoring [data/ai/bgp-briefs/](/home/ray/dev/linkong/planet/data/ai/bgp-briefs/) in [.gitignore](/home/ray/dev/linkong/planet/.gitignore) instead of leaving runtime Markdown output to pollute git status during normal operator use.
|
||||
|
||||
## 0.24.6
|
||||
|
||||
Released: 2026-04-10
|
||||
|
||||
### Highlights
|
||||
|
||||
- Tightened several backend hot paths outside the original BGP page fixes, stabilized BGP collector coverage after the recent query refactors, and rebuilt the BGP AI brief tab so saved Markdown briefs render and scroll like a proper operator workspace instead of collapsing inside the shared table layout.
|
||||
|
||||
### Improved
|
||||
|
||||
- Improved [backend/app/api/v1/datasources.py](/home/ray/dev/linkong/planet/backend/app/api/v1/datasources.py) by replacing per-datasource task, count, and endpoint lookups with batched prefetch helpers, reducing the worst `1 + N` behavior on the datasource list and `trigger-all` flow.
|
||||
- Improved [backend/app/api/v1/visualization.py](/home/ray/dev/linkong/planet/backend/app/api/v1/visualization.py) by switching the main Earth-facing `CollectedData` endpoints to `is_current` records, batching multi-source loads for aggregate endpoints, and removing stale Python-side dedupe paths from the hot route.
|
||||
- Improved [backend/app/services/bgp_incidents.py](/home/ray/dev/linkong/planet/backend/app/services/bgp_incidents.py) and [backend/app/services/bgp_enrichment.py](/home/ray/dev/linkong/planet/backend/app/services/bgp_enrichment.py) by avoiding historical full-table infrastructure scans, narrowing observation baseline payloads to required columns, and pushing more ASN filtering into the database.
|
||||
- Improved [backend/app/api/v1/alerts.py](/home/ray/dev/linkong/planet/backend/app/api/v1/alerts.py), [backend/app/api/v1/dashboard.py](/home/ray/dev/linkong/planet/backend/app/api/v1/dashboard.py), and [backend/app/api/v1/settings.py](/home/ray/dev/linkong/planet/backend/app/api/v1/settings.py) by collapsing several repeated count and settings queries into fewer aggregate or batched reads.
|
||||
- Improved [frontend/src/pages/BGP/BGP.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/BGP/BGP.tsx), [frontend/src/index.css](/home/ray/dev/linkong/planet/frontend/src/index.css), and [frontend/src/components/MarkdownRenderer/MarkdownRenderer.tsx](/home/ray/dev/linkong/planet/frontend/src/components/MarkdownRenderer/MarkdownRenderer.tsx) by rebuilding the `AI 简报` tab layout, fixing saved brief scrolling behavior, and extending the renderer to handle tables, separators, and stored metadata comments more gracefully.
|
||||
- Improved [docs/ai-playground-development-plan.md](/home/ray/dev/linkong/planet/docs/ai-playground-development-plan.md) by explicitly recording that the current BGP brief is only the first-stage summary flow and that regional prefix-geography analysis remains a planned Phase B follow-up.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fixed [backend/app/services/bgp_collectors.py](/home/ray/dev/linkong/planet/backend/app/services/bgp_collectors.py) and [backend/app/services/bgp_enrichment.py](/home/ray/dev/linkong/planet/backend/app/services/bgp_enrichment.py) so JSON field extraction no longer depends on the less portable `.astext` path that could break BGP collector endpoints in local environments.
|
||||
- Fixed the BGP AI brief tab in [frontend/src/index.css](/home/ray/dev/linkong/planet/frontend/src/index.css) and [frontend/src/pages/BGP/BGP.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/BGP/BGP.tsx) so long saved briefs are no longer compressed into a tiny clipped viewport by the shared tab/table overflow rules.
|
||||
|
||||
## 0.24.4
|
||||
|
||||
Released: 2026-04-09
|
||||
|
||||
@@ -150,6 +150,7 @@
|
||||
|
||||
- 不再依赖手工填写“观察项”
|
||||
- 让系统自动把真实 BGP 数据注入 AI
|
||||
- 让 BGP 页面逐步从“摘要汇总”升级为“证据驱动的区域态势分析”
|
||||
|
||||
建议实现:
|
||||
|
||||
@@ -163,10 +164,37 @@
|
||||
- 后端将结构化事实注入 `context / observations`
|
||||
- 前端在 BGP 页面增加“生成 AI 简报”
|
||||
|
||||
当前阶段说明:
|
||||
|
||||
- 第一版 `BGP AI 简报` 允许先落地为“值班摘要生成器”
|
||||
- 也就是先把 incidents / anomalies / events / collector coverage 自动注入
|
||||
- 允许模型先做事实摘要、风险归纳、建议动作
|
||||
|
||||
但这不应被视为 Phase B 的最终形态。
|
||||
|
||||
Phase B 后续还需要补齐:
|
||||
|
||||
- prefix geography 证据注入
|
||||
- `iptoasn`
|
||||
- `opengeofeed`
|
||||
- `nro_delegated`
|
||||
- 基于 `affected_regions` 与 prefix geography 的区域聚合
|
||||
- 区分“真实区域热度”与“collector coverage 偏差”
|
||||
- 对高风险 prefix / ASN 给出更明确的国家、城市、运营商归属线索
|
||||
- 让 AI 输出明确回答:
|
||||
- 哪些区域正在异常升温
|
||||
- 哪些结论只是观测站偏差
|
||||
- 当前还缺哪些区域证据
|
||||
|
||||
完成标准:
|
||||
|
||||
- 用户不需要手工录入 BGP 观察项
|
||||
- AI 输出能明确区分“事实”和“研判”
|
||||
- AI 不只是复述总量和最近几条事件,还能利用 prefix geography 与 affected regions 做区域态势判断
|
||||
- 输出中能明确指出:
|
||||
- 高风险区域
|
||||
- 区域证据来源
|
||||
- collector coverage 偏差对判断的影响
|
||||
|
||||
### Phase C: 告警 / 数据源健康 AI 简报
|
||||
|
||||
@@ -315,4 +343,19 @@
|
||||
1. 稳住 `Playground` 当前布局,不再大幅重做
|
||||
2. 在 `BGP` 页面新增专用 “AI 简报” 入口
|
||||
3. 后端新增 `BGP brief` 专用接口,自动注入真实数据
|
||||
4. 把 AI 输出逐步从自由文本升级为结构化 assessment
|
||||
4. 补齐 `BGP brief` 的区域态势证据层
|
||||
5. 把 AI 输出逐步从自由文本升级为结构化 assessment
|
||||
|
||||
### BGP Brief 后续子项
|
||||
|
||||
为避免把“已有 AI 简报”误判成“区域分析已完成”,这里单独记录 `BGP brief` 的后续 backlog:
|
||||
|
||||
1. 把高风险 prefix 命中的 `iptoasn / opengeofeed / nro_delegated` 结果注入 brief context
|
||||
2. 按国家/城市聚合 active incidents、anomalies、affected prefixes,生成区域热点事实层
|
||||
3. 把 collector coverage 与区域热点并排注入,避免模型把观测偏差误判成区域风险
|
||||
4. 对高风险 ASN / prefix 追加归属线索,如国家、城市、可能运营商或注册区域
|
||||
5. 在输出结构中单独增加:
|
||||
- 区域态势
|
||||
- 证据来源
|
||||
- 观测偏差说明
|
||||
- 缺失区域证据
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
## Current Version
|
||||
|
||||
- `main` 当前主线历史推导到:`0.16.5`
|
||||
- `dev` 当前开发分支历史推导到:`0.24.5`
|
||||
- `dev` 当前开发分支历史推导到:`0.24.6`
|
||||
|
||||
## Timeline
|
||||
|
||||
@@ -78,6 +78,7 @@
|
||||
| `0.24.3` | bugfix | `dev` | `pending` | expand Playground diagnostics presets and result inspection, and make `planet.sh` rebuild changed AI Provider images with explicit Compose fallback reporting |
|
||||
| `0.24.4` | bugfix | `dev` | `pending` | polish `planet.sh` AI Provider rebuild stage boundaries, hide raw Compose build logs on success, and add explicit image-build completion feedback |
|
||||
| `0.24.5` | bugfix | `dev` | `pending` | add persistent BGP AI briefs with Markdown history, lazy-load BGP tabs, and move BGP hot-path filtering and aggregation back into the database |
|
||||
| `0.24.6` | bugfix | `dev` | `pending` | batch datasource and visualization hot-path queries, fix BGP collector JSON extraction, and rebuild the BGP AI brief tab layout and markdown rendering |
|
||||
|
||||
## Maintenance Commits Not Counted as Version Bumps
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "planet-frontend",
|
||||
"version": "0.24.5",
|
||||
"version": "0.24.6",
|
||||
"private": true,
|
||||
"packageManager": "bun@1",
|
||||
"dependencies": {
|
||||
|
||||
@@ -76,6 +76,11 @@ export default function MarkdownRenderer({ markdown, className }: MarkdownRender
|
||||
continue
|
||||
}
|
||||
|
||||
if (trimmed.startsWith('<!--') && trimmed.endsWith('-->')) {
|
||||
index += 1
|
||||
continue
|
||||
}
|
||||
|
||||
if (trimmed.startsWith('```')) {
|
||||
const codeLines: string[] = []
|
||||
index += 1
|
||||
@@ -94,6 +99,12 @@ export default function MarkdownRenderer({ markdown, className }: MarkdownRender
|
||||
continue
|
||||
}
|
||||
|
||||
if (/^(-{3,}|\*{3,}|_{3,})$/.test(trimmed)) {
|
||||
nodes.push(<hr key={`block-${index}`} />)
|
||||
index += 1
|
||||
continue
|
||||
}
|
||||
|
||||
const headingMatch = trimmed.match(/^(#{1,6})\s+(.+)$/)
|
||||
if (headingMatch) {
|
||||
const level = headingMatch[1].length
|
||||
@@ -155,6 +166,50 @@ export default function MarkdownRenderer({ markdown, className }: MarkdownRender
|
||||
continue
|
||||
}
|
||||
|
||||
const tableHeaderCells = parseTableRow(trimmed)
|
||||
const tableDividerCells = lines[index + 1] ? parseTableDivider(lines[index + 1].trim()) : null
|
||||
if (
|
||||
tableHeaderCells &&
|
||||
tableHeaderCells.length > 0 &&
|
||||
tableDividerCells &&
|
||||
tableDividerCells.length === tableHeaderCells.length
|
||||
) {
|
||||
const bodyRows: string[][] = []
|
||||
index += 2
|
||||
while (index < lines.length) {
|
||||
const rowCells = parseTableRow(lines[index].trim())
|
||||
if (!rowCells || rowCells.length !== tableHeaderCells.length) {
|
||||
break
|
||||
}
|
||||
bodyRows.push(rowCells)
|
||||
index += 1
|
||||
}
|
||||
|
||||
nodes.push(
|
||||
<div key={`block-${index}`} className="markdown-renderer__table-wrap">
|
||||
<table className="markdown-renderer__table">
|
||||
<thead>
|
||||
<tr>
|
||||
{tableHeaderCells.map((cell, cellIndex) => (
|
||||
<th key={`head-${cellIndex}`}>{renderInlineMarkdown(cell)}</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{bodyRows.map((row, rowIndex) => (
|
||||
<tr key={`row-${rowIndex}`}>
|
||||
{row.map((cell, cellIndex) => (
|
||||
<td key={`cell-${rowIndex}-${cellIndex}`}>{renderInlineMarkdown(cell)}</td>
|
||||
))}
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>,
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
const paragraphLines: string[] = []
|
||||
while (index < lines.length && lines[index].trim()) {
|
||||
paragraphLines.push(lines[index].trim())
|
||||
@@ -165,3 +220,23 @@ export default function MarkdownRenderer({ markdown, className }: MarkdownRender
|
||||
|
||||
return <div className={className ? `markdown-renderer ${className}` : 'markdown-renderer'}>{nodes}</div>
|
||||
}
|
||||
|
||||
function parseTableRow(line: string): string[] | null {
|
||||
if (!line.includes('|')) {
|
||||
return null
|
||||
}
|
||||
|
||||
const normalized = line.replace(/^\|/, '').replace(/\|$/, '')
|
||||
const cells = normalized.split('|').map((cell) => cell.trim())
|
||||
return cells.length > 0 ? cells : null
|
||||
}
|
||||
|
||||
function parseTableDivider(line: string): string[] | null {
|
||||
const cells = parseTableRow(line)
|
||||
if (!cells || cells.length === 0) {
|
||||
return null
|
||||
}
|
||||
|
||||
const isDivider = cells.every((cell) => /^:?-{3,}:?$/.test(cell))
|
||||
return isDivider ? cells : null
|
||||
}
|
||||
|
||||
@@ -1006,11 +1006,45 @@ body {
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.bgp-page__brief-tabpane {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
min-height: 0;
|
||||
overflow: auto;
|
||||
padding-right: 4px;
|
||||
scrollbar-width: thin;
|
||||
scrollbar-color: rgba(148, 163, 184, 0.88) transparent;
|
||||
}
|
||||
|
||||
.bgp-page__brief-tabpane::-webkit-scrollbar {
|
||||
width: 8px;
|
||||
}
|
||||
|
||||
.bgp-page__brief-tabpane::-webkit-scrollbar-thumb {
|
||||
border-radius: 999px;
|
||||
background: rgba(148, 163, 184, 0.88);
|
||||
}
|
||||
|
||||
.bgp-page__brief-tabpane::-webkit-scrollbar-track {
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
.bgp-page__brief-card {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 14px;
|
||||
min-height: 100%;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.bgp-page__brief-body {
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.bgp-page__brief-content {
|
||||
min-height: 360px;
|
||||
overflow: visible;
|
||||
padding: 14px 16px;
|
||||
border-radius: 14px;
|
||||
background: linear-gradient(180deg, rgba(255, 255, 255, 0.98), rgba(246, 248, 251, 0.98));
|
||||
border: 1px solid rgba(5, 5, 5, 0.08);
|
||||
}
|
||||
|
||||
.bgp-page__brief-head {
|
||||
@@ -1045,24 +1079,16 @@ body {
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.bgp-page__brief-card > * + * {
|
||||
margin-top: 14px;
|
||||
}
|
||||
|
||||
.bgp-page__brief-meta .ant-descriptions-view {
|
||||
background: #f7f8fa;
|
||||
border-radius: 12px;
|
||||
padding: 8px 12px;
|
||||
}
|
||||
|
||||
.bgp-page__brief-content {
|
||||
flex: 1 1 auto;
|
||||
min-height: 0;
|
||||
overflow: auto;
|
||||
padding: 14px 16px;
|
||||
border-radius: 14px;
|
||||
background: linear-gradient(180deg, rgba(255, 255, 255, 0.98), rgba(246, 248, 251, 0.98));
|
||||
border: 1px solid rgba(5, 5, 5, 0.08);
|
||||
scrollbar-width: thin;
|
||||
scrollbar-color: rgba(148, 163, 184, 0.88) transparent;
|
||||
}
|
||||
|
||||
.markdown-renderer {
|
||||
color: #262626;
|
||||
font-size: 13px;
|
||||
@@ -1103,7 +1129,9 @@ body {
|
||||
.markdown-renderer ul,
|
||||
.markdown-renderer ol,
|
||||
.markdown-renderer blockquote,
|
||||
.markdown-renderer pre {
|
||||
.markdown-renderer pre,
|
||||
.markdown-renderer hr,
|
||||
.markdown-renderer__table-wrap {
|
||||
margin: 0 0 0.9em;
|
||||
}
|
||||
|
||||
@@ -1132,6 +1160,43 @@ body {
|
||||
color: #e2e8f0;
|
||||
}
|
||||
|
||||
.markdown-renderer hr {
|
||||
border: 0;
|
||||
border-top: 1px solid rgba(15, 23, 42, 0.12);
|
||||
}
|
||||
|
||||
.markdown-renderer__table-wrap {
|
||||
overflow-x: auto;
|
||||
}
|
||||
|
||||
.markdown-renderer__table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
min-width: 520px;
|
||||
font-size: 13px;
|
||||
background: rgba(255, 255, 255, 0.96);
|
||||
border: 1px solid rgba(15, 23, 42, 0.08);
|
||||
border-radius: 10px;
|
||||
}
|
||||
|
||||
.markdown-renderer__table th,
|
||||
.markdown-renderer__table td {
|
||||
padding: 10px 12px;
|
||||
text-align: left;
|
||||
vertical-align: top;
|
||||
border-bottom: 1px solid rgba(15, 23, 42, 0.08);
|
||||
}
|
||||
|
||||
.markdown-renderer__table th {
|
||||
font-weight: 600;
|
||||
color: #111827;
|
||||
background: rgba(248, 250, 252, 0.95);
|
||||
}
|
||||
|
||||
.markdown-renderer__table tbody tr:last-child td {
|
||||
border-bottom: 0;
|
||||
}
|
||||
|
||||
.markdown-renderer code {
|
||||
padding: 0.08em 0.32em;
|
||||
border-radius: 6px;
|
||||
|
||||
@@ -519,60 +519,62 @@ function BGP() {
|
||||
}
|
||||
|
||||
const briefTabContent = (
|
||||
<div className="bgp-page__brief-card">
|
||||
<div className="bgp-page__brief-head">
|
||||
<div>
|
||||
<Text strong>BGP AI 简报</Text>
|
||||
<div className="bgp-page__brief-subtitle">
|
||||
每次生成都会保存为 Markdown。打开页面时默认展示最近一次简报,也可以从历史列表切换查看。
|
||||
<div className="bgp-page__brief-tabpane">
|
||||
<div className="bgp-page__brief-card">
|
||||
<div className="bgp-page__brief-head">
|
||||
<div>
|
||||
<Text strong>BGP AI 简报</Text>
|
||||
<div className="bgp-page__brief-subtitle">
|
||||
每次生成都会保存为 Markdown。打开页面时默认展示最近一次简报,也可以从历史列表切换查看。
|
||||
</div>
|
||||
</div>
|
||||
<div className="bgp-page__brief-actions">
|
||||
<Select
|
||||
className="bgp-page__brief-select"
|
||||
placeholder="选择历史简报"
|
||||
value={selectedBriefId || undefined}
|
||||
options={formatBriefOptions(briefOptions)}
|
||||
onChange={(value) => void handleBriefSelectionChange(value)}
|
||||
disabled={briefLoading || briefDetailLoading || briefOptions.length === 0}
|
||||
/>
|
||||
<Button
|
||||
type="primary"
|
||||
icon={<ReloadOutlined />}
|
||||
loading={briefLoading}
|
||||
onClick={() => void handleGenerateBrief()}
|
||||
>
|
||||
生成 AI 简报
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
<div className="bgp-page__brief-actions">
|
||||
<Select
|
||||
className="bgp-page__brief-select"
|
||||
placeholder="选择历史简报"
|
||||
value={selectedBriefId || undefined}
|
||||
options={formatBriefOptions(briefOptions)}
|
||||
onChange={(value) => void handleBriefSelectionChange(value)}
|
||||
disabled={briefLoading || briefDetailLoading || briefOptions.length === 0}
|
||||
/>
|
||||
<Button
|
||||
type="primary"
|
||||
icon={<ReloadOutlined />}
|
||||
loading={briefLoading}
|
||||
onClick={() => void handleGenerateBrief()}
|
||||
>
|
||||
生成 AI 简报
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{briefLoading || briefDetailLoading ? (
|
||||
<div className="bgp-page__brief-loading">
|
||||
<Spin />
|
||||
<Text type="secondary">
|
||||
{briefLoading ? '正在整理 BGP 事实并生成简报...' : '正在加载已保存的 BGP 简报...'}
|
||||
</Text>
|
||||
</div>
|
||||
) : brief ? (
|
||||
<Space direction="vertical" size={12} style={{ width: '100%' }}>
|
||||
<Descriptions size="small" column={compactViewport ? 1 : 3} className="bgp-page__brief-meta">
|
||||
<Descriptions.Item label="Provider">{brief.provider || '-'}</Descriptions.Item>
|
||||
<Descriptions.Item label="模型">{brief.model || '-'}</Descriptions.Item>
|
||||
<Descriptions.Item label="请求ID">{brief.request_id || '-'}</Descriptions.Item>
|
||||
<Descriptions.Item label="生成时间" span={compactViewport ? 1 : 3}>
|
||||
{formatDateTimeZhCN(brief.generated_at)}
|
||||
</Descriptions.Item>
|
||||
</Descriptions>
|
||||
<div className="bgp-page__brief-content">
|
||||
<MarkdownRenderer markdown={brief.content_markdown} />
|
||||
{briefLoading || briefDetailLoading ? (
|
||||
<div className="bgp-page__brief-loading">
|
||||
<Spin />
|
||||
<Text type="secondary">
|
||||
{briefLoading ? '正在整理 BGP 事实并生成简报...' : '正在加载已保存的 BGP 简报...'}
|
||||
</Text>
|
||||
</div>
|
||||
</Space>
|
||||
) : (
|
||||
<div className="bgp-page__brief-empty">
|
||||
<Text type="secondary">还没有生成简报。点击右侧按钮,用当前 BGP 真实数据生成并保存第一份 Markdown 简报。</Text>
|
||||
</div>
|
||||
)}
|
||||
) : brief ? (
|
||||
<div className="bgp-page__brief-body">
|
||||
<Descriptions size="small" column={compactViewport ? 1 : 3} className="bgp-page__brief-meta">
|
||||
<Descriptions.Item label="Provider">{brief.provider || '-'}</Descriptions.Item>
|
||||
<Descriptions.Item label="模型">{brief.model || '-'}</Descriptions.Item>
|
||||
<Descriptions.Item label="请求ID">{brief.request_id || '-'}</Descriptions.Item>
|
||||
<Descriptions.Item label="生成时间" span={compactViewport ? 1 : 3}>
|
||||
{formatDateTimeZhCN(brief.generated_at)}
|
||||
</Descriptions.Item>
|
||||
</Descriptions>
|
||||
<div className="bgp-page__brief-content">
|
||||
<MarkdownRenderer markdown={brief.content_markdown} />
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="bgp-page__brief-empty">
|
||||
<Text type="secondary">还没有生成简报。点击右侧按钮,用当前 BGP 真实数据生成并保存第一份 Markdown 简报。</Text>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "planet"
|
||||
version = "0.24.5"
|
||||
version = "0.24.6"
|
||||
description = "智能星球计划 - 态势感知系统"
|
||||
requires-python = ">=3.14"
|
||||
dependencies = [
|
||||
|
||||
Reference in New Issue
Block a user