Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5bf5c73ca0 | ||
| e65267fe21 | |||
| ae982e51cd |
6
TODO.md
6
TODO.md
@@ -7,9 +7,6 @@ This file is the active backlog only. Completed history belongs in `docs/CHANGEL
|
||||
- [ ] Earth AI command entry: merge natural-language and speech-triggered LLM commands into the existing Earth search panel as described in [Agent Runtime, Earth LLM Command, And Speech Entry Plan](/home/ray/dev/linkong/planet/docs/plans/agents-earth-command-runtime-plan.md).
|
||||
- [ ] Earth action executor: implement safe visualization actions for layer toggles, batch highlights, filters, focus, result panels, and clear-highlight behavior.
|
||||
- [ ] Earth entity matching: support stable entity ids and batch matching for Beidou satellites, mainland China compute centers, BGP, news, vessels, and cables.
|
||||
- [x] High-precision country boundary tile framework: implement the static vector tile builder, versioned seed output, frontend bbox tile loader, debounce, in-flight dedupe, and LRU cache described in [Earth High Precision Boundary Tiles Plan](/home/ray/dev/linkong/planet/docs/plans/earth-high-precision-boundary-tiles-plan.md).
|
||||
- [x] Add the `pmtiles-mvt` frontend tile provider contract, MVT decoder dependencies, static PMTiles Nginx handling, collector artifact registration, production readiness check, and user operation docs for Earth boundaries.
|
||||
- [x] Split Earth boundary ingestion into standard source collectors (`earth_admin0_boundaries`, `earth_coastline`, `earth_claim_lines`) plus the downstream `earth_boundary_tiles` PMTiles builder.
|
||||
- [ ] Replace debug GeoJSON boundary tiles with the real `earth-boundaries-china-pov-v1.pmtiles` production artifact after audited admin-0 / coastline / claim-line sources and the PMTiles toolchain are available.
|
||||
- [ ] Import authoritative China POV / coastline / claim-line source packages through the three standard Earth boundary source collectors, then rebuild a versioned PMTiles artifact so highest zoom `8-10` preserves trusted source geometry instead of seed data.
|
||||
- [ ] Earth boundary data: acquire or generate auditable China POV geometry for Zangnan, Aksai Chin, Taiwan/Penghu, Diaoyu Dao and affiliated islands, Chiwei Yu, South China Sea islands, Kosovo, Gaza, and the official dashed maritime claim line before implementing final visual changes.
|
||||
@@ -61,6 +58,9 @@ Archived items stay here so old context is not lost. Completed items remain chec
|
||||
|
||||
### Completed
|
||||
|
||||
- [x] Implemented the high-precision country boundary tile framework from [Earth High Precision Boundary Tiles Plan](/home/ray/dev/linkong/planet/docs/plans/earth-high-precision-boundary-tiles-plan.md): static vector tile builder, versioned seed output, frontend bbox tile loader, debounce, in-flight dedupe, and LRU cache.
|
||||
- [x] Added the `pmtiles-mvt` frontend tile provider contract, MVT decoder dependencies, static PMTiles Nginx handling, collector artifact registration, production readiness check, and user operation docs for Earth boundaries.
|
||||
- [x] Split Earth boundary ingestion into standard source collectors (`earth_admin0_boundaries`, `earth_coastline`, `earth_claim_lines`) plus the downstream `earth_boundary_tiles` PMTiles builder.
|
||||
- [x] Refined BGP observer and anomaly `hover/click` feel.
|
||||
- [x] Added BGP anomaly relationship display with cables / regions.
|
||||
- [x] Added the Earth BGP activity layer so the map still feels alive when incident density is low.
|
||||
|
||||
@@ -2,10 +2,14 @@
|
||||
|
||||
ARG PYTHON_IMAGE=python:3.14-slim
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:latest
|
||||
ARG AI_PROVIDER_BUILD_FINGERPRINT=unknown
|
||||
|
||||
FROM ${UV_IMAGE} AS uv
|
||||
FROM ${PYTHON_IMAGE}
|
||||
|
||||
ARG AI_PROVIDER_BUILD_FINGERPRINT
|
||||
LABEL planet.aiprovider.build-fingerprint="${AI_PROVIDER_BUILD_FINGERPRINT}"
|
||||
|
||||
COPY --from=uv /uv /uvx /bin/
|
||||
|
||||
WORKDIR /app
|
||||
@@ -15,12 +19,15 @@ ENV PYTHONUNBUFFERED=1
|
||||
ENV UV_COMPILE_BYTECODE=1
|
||||
ENV UV_LINK_MODE=copy
|
||||
|
||||
RUN mkdir -p /root/.config/uv
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY pyproject.toml uv.lock /app/
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=type=secret,id=planet_uv_config,target=/root/.config/uv/uv.toml,required=false \
|
||||
uv sync --frozen --no-dev
|
||||
|
||||
COPY aiprovider /app/aiprovider
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
ARG PYTHON_IMAGE=python:3.14-slim
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:latest
|
||||
|
||||
@@ -14,12 +16,16 @@ ENV UV_COMPILE_BYTECODE=1
|
||||
ENV UV_LINK_MODE=copy
|
||||
ENV PYTHONPATH=/app/backend
|
||||
|
||||
RUN mkdir -p /root/.config/uv
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY pyproject.toml uv.lock /app/
|
||||
RUN uv sync --frozen --no-dev
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=type=secret,id=planet_uv_config,target=/root/.config/uv/uv.toml,required=false \
|
||||
uv sync --frozen --no-dev
|
||||
|
||||
COPY backend /app/backend
|
||||
COPY VERSION /app/VERSION
|
||||
|
||||
@@ -19,6 +19,7 @@ from app.api.v1 import (
|
||||
vessels,
|
||||
bgp,
|
||||
news,
|
||||
interactables,
|
||||
realtime_sources,
|
||||
system_control,
|
||||
tv,
|
||||
@@ -53,4 +54,5 @@ api_router.include_router(vessels.router, prefix="/vessels", tags=["vessels"])
|
||||
api_router.include_router(bgp.router, prefix="/bgp", tags=["bgp"])
|
||||
api_router.include_router(tv.router, prefix="/tv", tags=["tv"])
|
||||
api_router.include_router(news.router, prefix="/news", tags=["news"])
|
||||
api_router.include_router(interactables.router, prefix="/interactables", tags=["interactables"])
|
||||
api_router.include_router(realtime_sources.router, prefix="/realtime-sources", tags=["realtime-sources"])
|
||||
|
||||
@@ -3,6 +3,7 @@ from uuid import uuid4
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.core.security import get_current_user
|
||||
from app.db.session import get_db
|
||||
from app.models.user import User
|
||||
@@ -47,8 +48,10 @@ from app.services.playground_chat_service import (
|
||||
stop_message,
|
||||
)
|
||||
from app.services.situational_alert_ai_brief import build_situational_alert_brief_request
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
|
||||
router = APIRouter()
|
||||
logger = get_logger(__name__, service="api")
|
||||
|
||||
|
||||
@router.get("/provider/status", response_model=AIProviderStatusResponse)
|
||||
@@ -122,6 +125,16 @@ async def create_playground_message(
|
||||
provider_client: AIProviderClient = Depends(get_ai_provider_client),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.playground.message.create",
|
||||
message="Playground message creation requested",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={"session_key": payload.session_key, "preset": payload.selected_preset_key},
|
||||
)
|
||||
return await create_turn(
|
||||
db,
|
||||
user_id=current_user.id,
|
||||
@@ -136,6 +149,16 @@ async def stop_playground_message(
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.playground.message.stop",
|
||||
message="Playground message stop requested",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={"session_key": payload.session_key, "message_id": payload.message_id},
|
||||
)
|
||||
return await stop_message(
|
||||
db,
|
||||
user_id=current_user.id,
|
||||
@@ -150,6 +173,16 @@ async def resend_playground_message(
|
||||
provider_client: AIProviderClient = Depends(get_ai_provider_client),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.playground.message.resend",
|
||||
message="Playground message resend requested",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={"session_key": payload.session_key, "user_message_id": payload.user_message_id},
|
||||
)
|
||||
return await resend_turn(
|
||||
db,
|
||||
user_id=current_user.id,
|
||||
@@ -214,16 +247,70 @@ async def analyze_bgp_brief(
|
||||
anomaly_limit=payload.anomaly_limit,
|
||||
collector_limit=payload.collector_limit,
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.bgp.facts_collected",
|
||||
message="BGP brief facts collected",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context={
|
||||
"incident_limit": payload.incident_limit,
|
||||
"anomaly_limit": payload.anomaly_limit,
|
||||
"collector_limit": payload.collector_limit,
|
||||
"fact_count": len(facts or []),
|
||||
},
|
||||
)
|
||||
brief_request.preferred_model = payload.preferred_model
|
||||
brief_request.thinking = payload.thinking
|
||||
|
||||
analysis = await provider_client.analyze(brief_request, request_id=request_id)
|
||||
return save_bgp_brief_record(
|
||||
analysis,
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.bgp.start",
|
||||
message="BGP brief AI analysis started",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
facts=facts,
|
||||
context=context,
|
||||
user_id=current_user.id,
|
||||
context={"preferred_model": payload.preferred_model},
|
||||
)
|
||||
try:
|
||||
analysis = await provider_client.analyze(brief_request, request_id=request_id)
|
||||
record = save_bgp_brief_record(
|
||||
analysis,
|
||||
request_id=request_id,
|
||||
facts=facts,
|
||||
context=context,
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.bgp.completed",
|
||||
message="BGP brief AI analysis saved",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context={"provider": analysis.provider, "model": analysis.model, "brief_id": record.id},
|
||||
)
|
||||
return record
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.bgp.failed",
|
||||
message="BGP brief AI analysis failed",
|
||||
category="ai",
|
||||
level="error",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"preferred_model": payload.preferred_model}),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@router.post("/alerts/brief", response_model=AlertBriefResponse)
|
||||
@@ -242,17 +329,65 @@ async def analyze_alert_brief(
|
||||
db,
|
||||
alert_limit=payload.alert_limit,
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.alerts.facts_collected",
|
||||
message="Alert brief facts collected",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context={"alert_limit": payload.alert_limit, "fact_count": len(facts or [])},
|
||||
)
|
||||
brief_request.preferred_model = payload.preferred_model
|
||||
brief_request.thinking = payload.thinking
|
||||
|
||||
analysis = await provider_client.analyze(brief_request, request_id=request_id)
|
||||
return AlertBriefResponse(
|
||||
**analysis.model_dump(),
|
||||
title=brief_request.title,
|
||||
objective=brief_request.objective,
|
||||
facts=facts,
|
||||
context=context,
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.alerts.start",
|
||||
message="Alert brief AI analysis started",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context={"preferred_model": payload.preferred_model},
|
||||
)
|
||||
try:
|
||||
analysis = await provider_client.analyze(brief_request, request_id=request_id)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.alerts.completed",
|
||||
message="Alert brief AI analysis completed",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context={"provider": analysis.provider, "model": analysis.model},
|
||||
)
|
||||
return AlertBriefResponse(
|
||||
**analysis.model_dump(),
|
||||
title=brief_request.title,
|
||||
objective=brief_request.objective,
|
||||
facts=facts,
|
||||
context=context,
|
||||
)
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.alerts.failed",
|
||||
message="Alert brief AI analysis failed",
|
||||
category="ai",
|
||||
level="error",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"preferred_model": payload.preferred_model}),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@router.post("/situational-alerts/brief", response_model=SituationalAlertBriefResponse)
|
||||
@@ -268,14 +403,62 @@ async def analyze_situational_alert_brief(
|
||||
response.headers["X-Request-ID"] = request_id
|
||||
|
||||
brief_request, facts, context = await build_situational_alert_brief_request(db)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.situational_alerts.facts_collected",
|
||||
message="Situational alert brief facts collected",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context={"fact_count": len(facts or [])},
|
||||
)
|
||||
brief_request.preferred_model = payload.preferred_model
|
||||
brief_request.thinking = payload.thinking
|
||||
|
||||
analysis = await provider_client.analyze(brief_request, request_id=request_id)
|
||||
return SituationalAlertBriefResponse(
|
||||
**analysis.model_dump(),
|
||||
title=brief_request.title,
|
||||
objective=brief_request.objective,
|
||||
facts=facts,
|
||||
context=context,
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.situational_alerts.start",
|
||||
message="Situational alert brief AI analysis started",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context={"preferred_model": payload.preferred_model},
|
||||
)
|
||||
try:
|
||||
analysis = await provider_client.analyze(brief_request, request_id=request_id)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.situational_alerts.completed",
|
||||
message="Situational alert brief AI analysis completed",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context={"provider": analysis.provider, "model": analysis.model},
|
||||
)
|
||||
return SituationalAlertBriefResponse(
|
||||
**analysis.model_dump(),
|
||||
title=brief_request.title,
|
||||
objective=brief_request.objective,
|
||||
facts=facts,
|
||||
context=context,
|
||||
)
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.brief.situational_alerts.failed",
|
||||
message="Situational alert brief AI analysis failed",
|
||||
category="ai",
|
||||
level="error",
|
||||
service="api",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"preferred_model": payload.preferred_model}),
|
||||
)
|
||||
raise
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import asyncio
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Optional
|
||||
|
||||
@@ -7,7 +6,7 @@ from pydantic import BaseModel, Field
|
||||
from sqlalchemy import func, or_, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.cache import cache
|
||||
from app.core.logging import get_logger
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.security import get_current_user
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
@@ -22,16 +21,26 @@ from app.models.user import User
|
||||
from app.models.vessel import AISRawObservation
|
||||
from app.services.vessel_ais_aggregation import VESSEL_AIS_SCHEMA
|
||||
from app.services.scheduler import (
|
||||
cancel_running_collector_now,
|
||||
get_latest_task_id_for_datasource,
|
||||
run_collector_now,
|
||||
sync_datasource_job,
|
||||
)
|
||||
from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
|
||||
from app.services.data_jobs import (
|
||||
JOB_STATUS_CANCELLING,
|
||||
JOB_STATUS_QUEUED,
|
||||
JOB_STATUS_RUNNING,
|
||||
JOB_TYPE_CLEAR_CACHE,
|
||||
JOB_TYPE_CLEAR_DATA,
|
||||
JOB_TYPE_COLLECT,
|
||||
enqueue_datasource_job,
|
||||
get_active_datasource_job,
|
||||
request_cancel_datasource_task,
|
||||
)
|
||||
from app.services.business_logs import emit_business_log
|
||||
|
||||
router = APIRouter()
|
||||
logger = get_logger(__name__, service="api")
|
||||
STALE_RUNNING_TASK_TIMEOUT_MINUTES = 90
|
||||
|
||||
|
||||
PRODUCT_SOURCE_KEYWORDS: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||||
("vessels", ("vessel", "ais")),
|
||||
("cables", ("cable", "landing", "telegeography", "arcgis", "fao")),
|
||||
@@ -115,7 +124,7 @@ async def _load_latest_running_tasks(
|
||||
_task_rank_column(CollectionTask.started_at),
|
||||
)
|
||||
.where(CollectionTask.datasource_id.in_(datasource_ids))
|
||||
.where(CollectionTask.status == "running")
|
||||
.where(CollectionTask.status.in_((JOB_STATUS_QUEUED, JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING)))
|
||||
.subquery()
|
||||
)
|
||||
result = await db.execute(
|
||||
@@ -126,32 +135,6 @@ async def _load_latest_running_tasks(
|
||||
return {task.datasource_id: task for task in result.scalars().all()}
|
||||
|
||||
|
||||
async def _load_latest_task_ids(
|
||||
db: AsyncSession,
|
||||
datasource_ids: list[int],
|
||||
) -> dict[int, int]:
|
||||
if not datasource_ids:
|
||||
return {}
|
||||
|
||||
ranked_tasks = (
|
||||
select(
|
||||
CollectionTask.id.label("task_id"),
|
||||
CollectionTask.datasource_id.label("datasource_id"),
|
||||
func.row_number().over(
|
||||
partition_by=CollectionTask.datasource_id,
|
||||
order_by=CollectionTask.id.desc(),
|
||||
).label("row_num"),
|
||||
)
|
||||
.where(CollectionTask.datasource_id.in_(datasource_ids))
|
||||
.subquery()
|
||||
)
|
||||
result = await db.execute(
|
||||
select(ranked_tasks.c.datasource_id, ranked_tasks.c.task_id)
|
||||
.where(ranked_tasks.c.row_num == 1)
|
||||
)
|
||||
return {datasource_id: task_id for datasource_id, task_id in result.all()}
|
||||
|
||||
|
||||
async def _load_latest_tasks(
|
||||
db: AsyncSession,
|
||||
datasource_ids: list[int],
|
||||
@@ -303,8 +286,11 @@ def serialize_datasource_row(
|
||||
"last_run": to_iso8601_utc(last_run_at),
|
||||
"last_run_at": to_iso8601_utc(last_run_at),
|
||||
"last_status": last_status,
|
||||
"is_running": running_task is not None,
|
||||
"is_running": running_task is not None and running_task.task_type == JOB_TYPE_COLLECT,
|
||||
"is_task_active": running_task is not None,
|
||||
"task_status": running_task.status if running_task else None,
|
||||
"task_id": display_task.id if display_task else None,
|
||||
"task_type": display_task.task_type if display_task else None,
|
||||
"progress": display_task.progress if display_task else None,
|
||||
"phase": display_task.phase if display_task else None,
|
||||
"phase_progress": display_task.phase_progress if display_task else None,
|
||||
@@ -400,9 +386,25 @@ async def _trigger_datasource_batch(
|
||||
datasources: list[DataSource],
|
||||
*,
|
||||
force: bool,
|
||||
actor_id: int | None = None,
|
||||
trigger_kind: str = "batch",
|
||||
) -> dict:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event=f"collector.trigger.{trigger_kind}.start",
|
||||
message="Datasource batch trigger started",
|
||||
category="collector",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=actor_id,
|
||||
context={
|
||||
"trigger_kind": trigger_kind,
|
||||
"force": force,
|
||||
"requested_count": len(datasources),
|
||||
},
|
||||
)
|
||||
if not datasources:
|
||||
return {
|
||||
result = {
|
||||
"status": "noop",
|
||||
"message": "No matching data sources to trigger",
|
||||
"force": force,
|
||||
@@ -410,8 +412,18 @@ async def _trigger_datasource_batch(
|
||||
"skipped": [],
|
||||
"failed": [],
|
||||
}
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event=f"collector.trigger.{trigger_kind}.completed",
|
||||
message="Datasource batch trigger completed with no matching sources",
|
||||
category="collector",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=actor_id,
|
||||
context={"trigger_kind": trigger_kind, "force": force, "status": "noop", "triggered_count": 0},
|
||||
)
|
||||
return result
|
||||
|
||||
previous_task_ids: dict[int, Optional[int]] = {}
|
||||
triggered_sources: list[dict] = []
|
||||
skipped_sources: list[dict] = []
|
||||
failed_sources: list[dict] = []
|
||||
@@ -446,9 +458,11 @@ async def _trigger_datasource_batch(
|
||||
}
|
||||
)
|
||||
continue
|
||||
cancelled = await cancel_running_collector_now(datasource.source)
|
||||
if not cancelled:
|
||||
await rollback_orphaned_running_task(db, datasource, running_task)
|
||||
await request_cancel_datasource_task(
|
||||
db,
|
||||
running_task,
|
||||
reason="superseded_by_forced_collection",
|
||||
)
|
||||
|
||||
if not force and not is_due_for_collection(datasource, now):
|
||||
skipped_sources.append(
|
||||
@@ -465,57 +479,51 @@ async def _trigger_datasource_batch(
|
||||
)
|
||||
continue
|
||||
|
||||
previous_task_ids[datasource.id] = None
|
||||
success = run_collector_now(datasource.source)
|
||||
if not success:
|
||||
failed_sources.append(
|
||||
{
|
||||
"id": datasource.id,
|
||||
"source": datasource.source,
|
||||
"name": datasource.name,
|
||||
"reason": "trigger_failed",
|
||||
}
|
||||
)
|
||||
continue
|
||||
task = await enqueue_datasource_job(
|
||||
db,
|
||||
datasource,
|
||||
JOB_TYPE_COLLECT,
|
||||
payload={"force": force, "trigger": "batch"},
|
||||
)
|
||||
|
||||
triggered_sources.append(
|
||||
{
|
||||
"id": datasource.id,
|
||||
"source": datasource.source,
|
||||
"name": datasource.name,
|
||||
"task_id": None,
|
||||
"task_id": task.id,
|
||||
}
|
||||
)
|
||||
|
||||
latest_task_ids = await _load_latest_task_ids(
|
||||
db,
|
||||
[datasource.id for datasource in datasources],
|
||||
)
|
||||
for datasource_id in previous_task_ids:
|
||||
previous_task_ids[datasource_id] = latest_task_ids.get(datasource_id)
|
||||
|
||||
for _ in range(20):
|
||||
await asyncio.sleep(0.1)
|
||||
pending = [item for item in triggered_sources if item["task_id"] is None]
|
||||
if not pending:
|
||||
break
|
||||
latest_task_ids = await _load_latest_task_ids(
|
||||
db,
|
||||
[item["id"] for item in pending],
|
||||
)
|
||||
for item in pending:
|
||||
task_id = latest_task_ids.get(item["id"])
|
||||
if task_id is not None and task_id != previous_task_ids.get(item["id"]):
|
||||
item["task_id"] = task_id
|
||||
|
||||
return {
|
||||
"status": "triggered" if triggered_sources else "partial",
|
||||
"message": f"Triggered {len(triggered_sources)} data sources",
|
||||
result = {
|
||||
"status": "queued" if triggered_sources else "partial",
|
||||
"message": f"Queued {len(triggered_sources)} data source jobs",
|
||||
"force": force,
|
||||
"triggered": triggered_sources,
|
||||
"skipped": skipped_sources,
|
||||
"failed": failed_sources,
|
||||
}
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event=f"collector.trigger.{trigger_kind}.completed",
|
||||
message="Datasource batch trigger completed",
|
||||
category="collector",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=actor_id,
|
||||
context={
|
||||
"trigger_kind": trigger_kind,
|
||||
"force": force,
|
||||
"status": result["status"],
|
||||
"requested_count": len(datasources),
|
||||
"triggered_count": len(triggered_sources),
|
||||
"skipped_count": len(skipped_sources),
|
||||
"failed_count": len(failed_sources),
|
||||
"triggered_sources": [item["source"] for item in triggered_sources],
|
||||
"skipped_reasons": [item["reason"] for item in skipped_sources],
|
||||
},
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def get_datasource_record(db: AsyncSession, source_id: str) -> Optional[DataSource]:
|
||||
@@ -780,7 +788,13 @@ async def trigger_all_datasources(
|
||||
.order_by(DataSource.module, DataSource.id)
|
||||
)
|
||||
datasources = result.scalars().all()
|
||||
return await _trigger_datasource_batch(db, datasources, force=force)
|
||||
return await _trigger_datasource_batch(
|
||||
db,
|
||||
datasources,
|
||||
force=force,
|
||||
actor_id=current_user.id,
|
||||
trigger_kind="all",
|
||||
)
|
||||
|
||||
|
||||
@router.post("/trigger-batch")
|
||||
@@ -816,7 +830,13 @@ async def trigger_datasource_batch(
|
||||
collected=None if payload.source_ids else payload.collected,
|
||||
credential_status=None if payload.source_ids else payload.credential_status,
|
||||
)
|
||||
return await _trigger_datasource_batch(db, datasources, force=payload.force)
|
||||
return await _trigger_datasource_batch(
|
||||
db,
|
||||
datasources,
|
||||
force=payload.force,
|
||||
actor_id=current_user.id,
|
||||
trigger_kind="batch",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/snapshots")
|
||||
@@ -992,8 +1012,24 @@ async def trigger_datasource(
|
||||
if not datasource.is_active:
|
||||
raise HTTPException(status_code=400, detail="Data source is disabled")
|
||||
|
||||
running_task = await get_running_task(db, datasource.id)
|
||||
running_task = await get_active_datasource_job(db, datasource.id, task_types=(JOB_TYPE_COLLECT,))
|
||||
if running_task is not None and not force:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.trigger.single.skipped_already_running",
|
||||
message="Datasource trigger skipped because a task is already running",
|
||||
category="collector",
|
||||
level="warning",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={
|
||||
"collector_name": datasource.source,
|
||||
"datasource_id": datasource.id,
|
||||
"task_id": running_task.id,
|
||||
"status": "skipped",
|
||||
},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
@@ -1013,31 +1049,42 @@ async def trigger_datasource(
|
||||
)
|
||||
|
||||
if running_task is not None and force:
|
||||
cancelled = await cancel_running_collector_now(datasource.source)
|
||||
if not cancelled:
|
||||
await rollback_orphaned_running_task(db, datasource, running_task)
|
||||
await request_cancel_datasource_task(
|
||||
db,
|
||||
running_task,
|
||||
reason="superseded_by_forced_collection",
|
||||
)
|
||||
|
||||
previous_task_id = await get_latest_task_id_for_datasource(datasource.id)
|
||||
success = run_collector_now(datasource.source)
|
||||
if not success:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to trigger collector '{datasource.source}'")
|
||||
|
||||
task_id = None
|
||||
for _ in range(20):
|
||||
await asyncio.sleep(0.1)
|
||||
task_id = await get_latest_task_id_for_datasource(datasource.id)
|
||||
if task_id is not None and task_id != previous_task_id:
|
||||
break
|
||||
if task_id == previous_task_id:
|
||||
task_id = None
|
||||
task = await enqueue_datasource_job(
|
||||
db,
|
||||
datasource,
|
||||
JOB_TYPE_COLLECT,
|
||||
payload={"force": force, "trigger": "single"},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.trigger.single.completed",
|
||||
message="Datasource trigger queued",
|
||||
category="collector",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={
|
||||
"collector_name": datasource.source,
|
||||
"datasource_id": datasource.id,
|
||||
"task_id": task.id,
|
||||
"force": force,
|
||||
"status": "queued",
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "triggered",
|
||||
"status": "queued",
|
||||
"source_id": datasource.id,
|
||||
"task_id": task_id,
|
||||
"task_id": task.id,
|
||||
"collector_name": datasource.source,
|
||||
"force": force,
|
||||
"message": f"Collector '{datasource.source}' has been triggered",
|
||||
"message": f"Collector '{datasource.source}' has been queued",
|
||||
}
|
||||
|
||||
|
||||
@@ -1051,22 +1098,30 @@ async def clear_datasource_data(
|
||||
if not datasource:
|
||||
raise HTTPException(status_code=404, detail="Data source not found")
|
||||
|
||||
result = await db.execute(
|
||||
select(func.count(CollectedData.id)).where(CollectedData.source == datasource.source)
|
||||
active_task = await get_active_datasource_job(db, datasource.id)
|
||||
if active_task is not None:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
"reason": "datasource_job_in_progress",
|
||||
"message": "当前数据源已有任务在执行,请等待完成或先取消任务。",
|
||||
"task_id": active_task.id,
|
||||
"task_type": active_task.task_type,
|
||||
"status": active_task.status,
|
||||
},
|
||||
)
|
||||
task = await enqueue_datasource_job(
|
||||
db,
|
||||
datasource,
|
||||
JOB_TYPE_CLEAR_DATA,
|
||||
payload={"source": datasource.source},
|
||||
)
|
||||
count = result.scalar() or 0
|
||||
|
||||
if count == 0:
|
||||
return {"status": "success", "message": "No data to clear", "deleted_count": 0}
|
||||
|
||||
delete_query = CollectedData.__table__.delete().where(CollectedData.source == datasource.source)
|
||||
await db.execute(delete_query)
|
||||
await db.commit()
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Cleared {count} records for data source '{datasource.name}'",
|
||||
"deleted_count": count,
|
||||
"status": "queued",
|
||||
"message": f"Queued data clearing for data source '{datasource.name}'",
|
||||
"task_id": task.id,
|
||||
"deleted_count": None,
|
||||
}
|
||||
|
||||
|
||||
@@ -1080,16 +1135,44 @@ async def clear_datasource_cache(
|
||||
if not datasource:
|
||||
raise HTTPException(status_code=404, detail="Data source not found")
|
||||
|
||||
earth_deleted_count = invalidate_earth_layer_cache_for_source(datasource.source)
|
||||
dashboard_deleted_count = int(cache.delete("dashboard:stats")) + int(cache.delete("dashboard:summary"))
|
||||
deleted_count = earth_deleted_count + dashboard_deleted_count
|
||||
task = await enqueue_datasource_job(
|
||||
db,
|
||||
datasource,
|
||||
JOB_TYPE_CLEAR_CACHE,
|
||||
payload={"source": datasource.source},
|
||||
dedupe_key=f"clear_cache:{datasource.source}",
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Cleared {deleted_count} cache keys for data source '{datasource.name}'",
|
||||
"deleted_count": deleted_count,
|
||||
"earth_layer_deleted_count": earth_deleted_count,
|
||||
"dashboard_deleted_count": dashboard_deleted_count,
|
||||
"status": "queued",
|
||||
"message": f"Queued cache clearing for data source '{datasource.name}'",
|
||||
"task_id": task.id,
|
||||
"deleted_count": None,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{source_id}/tasks/{task_id}/cancel")
|
||||
async def cancel_datasource_task(
|
||||
source_id: str,
|
||||
task_id: int,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
datasource = await get_datasource_record(db, source_id)
|
||||
if not datasource:
|
||||
raise HTTPException(status_code=404, detail="Data source not found")
|
||||
|
||||
task = await db.get(CollectionTask, task_id)
|
||||
if not task or task.datasource_id != datasource.id:
|
||||
raise HTTPException(status_code=404, detail="Task not found")
|
||||
|
||||
task = await request_cancel_datasource_task(db, task)
|
||||
return {
|
||||
"status": "cancelled" if task.completed_at else "cancelling",
|
||||
"task_id": task.id,
|
||||
"task_type": task.task_type,
|
||||
"phase": task.phase,
|
||||
"requested_cancel_at": to_iso8601_utc(task.requested_cancel_at),
|
||||
}
|
||||
|
||||
|
||||
@@ -1109,7 +1192,7 @@ async def get_task_status(
|
||||
if not task or task.datasource_id != datasource.id:
|
||||
raise HTTPException(status_code=404, detail="Task not found")
|
||||
else:
|
||||
task = await get_running_task(db, datasource.id)
|
||||
task = await get_active_datasource_job(db, datasource.id)
|
||||
if task is None:
|
||||
result = await db.execute(
|
||||
select(CollectionTask)
|
||||
@@ -1134,8 +1217,12 @@ async def get_task_status(
|
||||
}
|
||||
|
||||
return {
|
||||
"is_running": task.status == "running",
|
||||
"is_running": task.status in {JOB_STATUS_QUEUED, JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING}
|
||||
and task.task_type == JOB_TYPE_COLLECT,
|
||||
"is_task_active": task.status in {JOB_STATUS_QUEUED, JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING},
|
||||
"task_id": task.id,
|
||||
"task_type": task.task_type,
|
||||
"task_status": task.status,
|
||||
"progress": task.progress,
|
||||
"phase": task.phase,
|
||||
"phase_progress": task.phase_progress,
|
||||
@@ -1146,5 +1233,6 @@ async def get_task_status(
|
||||
"records_processed": task.records_processed,
|
||||
"total_records": task.total_records,
|
||||
"status": task.status,
|
||||
"requested_cancel_at": to_iso8601_utc(task.requested_cancel_at),
|
||||
"error_message": task.error_message,
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ from pydantic import BaseModel, Field
|
||||
from sqlalchemy import delete, func, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import settings as app_settings
|
||||
from app.core.security import decode_token, get_current_user, redis_client
|
||||
from app.db.session import get_db
|
||||
from app.models.collected_data import CollectedData
|
||||
@@ -36,9 +37,16 @@ EARTH_BRAND_ASSET_DIR = REPO_ROOT / "data" / "earth-brand"
|
||||
EARTH_BRAND_ASSET_URL_PREFIX = "/earth-brand-assets"
|
||||
EARTH_BRAND_CATEGORY = "earth_brand"
|
||||
EARTH_ABOUT_CATEGORY = "earth_about"
|
||||
SYSTEM_SETTINGS_CATEGORY = "system"
|
||||
MAX_EARTH_BRAND_ASSET_BYTES = 3 * 1024 * 1024
|
||||
ALLOWED_EARTH_BRAND_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp", ".svg"}
|
||||
|
||||
|
||||
def _app_version_label() -> str:
|
||||
version = str(app_settings.VERSION or "").strip() or "0.0.0"
|
||||
return version if version.startswith("v") else f"v{version}"
|
||||
|
||||
|
||||
DEFAULT_EARTH_BRAND = {
|
||||
"logo_src": "/earth/assets/brand/earth-logo.png",
|
||||
"title_src": "/earth/assets/brand/title-zh.png",
|
||||
@@ -53,14 +61,15 @@ DEFAULT_EARTH_ABOUT = {
|
||||
"logo_src": "/earth/assets/brand/lim-logo.png",
|
||||
"kicker": "About",
|
||||
"title": "智能星球计划",
|
||||
"version": "v0.64.0",
|
||||
"version": _app_version_label(),
|
||||
"description": "面向临空场景下的智能媒体研究、全球态势感知与多源开放数据巡航,提供可视化观测、事件聚合与交互式探索能力。",
|
||||
"meta": [
|
||||
{"label": "出品方", "value": "浙江大学临空智能媒体研究院"},
|
||||
{"label": "策划人", "value": "黄柳青"},
|
||||
{"label": "策划人", "value": "方兴东、黄柳青"},
|
||||
{"label": "产品兼开发者", "value": "钱坤、张鸽、齐鹏"},
|
||||
],
|
||||
}
|
||||
EARTH_ABOUT_LEGACY_PLANNER_VALUE = "黄柳青"
|
||||
|
||||
|
||||
class EarthBoundaryConfigPayload(BaseModel):
|
||||
@@ -116,11 +125,12 @@ def _normalize_earth_about_payload(payload: dict[str, Any] | None) -> dict[str,
|
||||
}
|
||||
raw_meta = DEFAULT_EARTH_ABOUT["meta"]
|
||||
if payload:
|
||||
for key in ("logo_src", "kicker", "title", "version", "description"):
|
||||
for key in ("logo_src", "kicker", "title", "description"):
|
||||
value = payload.get(key)
|
||||
if value is not None:
|
||||
merged[key] = str(value).strip()
|
||||
raw_meta = payload.get("meta") if isinstance(payload.get("meta"), list) else raw_meta
|
||||
merged["version"] = _app_version_label()
|
||||
|
||||
for key, default_value in DEFAULT_EARTH_ABOUT.items():
|
||||
if key == "meta":
|
||||
@@ -134,6 +144,8 @@ def _normalize_earth_about_payload(payload: dict[str, Any] | None) -> dict[str,
|
||||
continue
|
||||
label = str(item.get("label") or "").strip()
|
||||
value = str(item.get("value") or "").strip()
|
||||
if label == "策划人" and value == EARTH_ABOUT_LEGACY_PLANNER_VALUE:
|
||||
value = "方兴东、黄柳青"
|
||||
if label or value:
|
||||
normalized_meta.append({"label": label, "value": value})
|
||||
if not normalized_meta:
|
||||
@@ -142,6 +154,10 @@ def _normalize_earth_about_payload(payload: dict[str, Any] | None) -> dict[str,
|
||||
return merged
|
||||
|
||||
|
||||
def _is_demo_mode_enabled(payload: Any) -> bool:
|
||||
return bool(payload.get("demo_mode")) if isinstance(payload, dict) else False
|
||||
|
||||
|
||||
async def _get_earth_brand_record(db: AsyncSession) -> SystemSetting | None:
|
||||
result = await db.execute(
|
||||
select(SystemSetting).where(SystemSetting.category == EARTH_BRAND_CATEGORY)
|
||||
@@ -317,6 +333,11 @@ async def get_earth_oobe_status(
|
||||
select(func.count(CollectedData.id)).where(CollectedData.is_current.is_(True))
|
||||
)
|
||||
current_record_count = int(current_count_result.scalar() or 0)
|
||||
system_result = await db.execute(
|
||||
select(SystemSetting).where(SystemSetting.category == SYSTEM_SETTINGS_CATEGORY)
|
||||
)
|
||||
system_record = system_result.scalar_one_or_none()
|
||||
demo_mode = _is_demo_mode_enabled(system_record.payload if system_record else None)
|
||||
|
||||
datasource_count_result = await db.execute(select(func.count(DataSource.id)))
|
||||
datasource_count = int(datasource_count_result.scalar() or 0)
|
||||
@@ -337,6 +358,8 @@ async def get_earth_oobe_status(
|
||||
ready = has_collected_data
|
||||
|
||||
suggestions: list[str] = []
|
||||
if demo_mode:
|
||||
suggestions.append("演示模式已开启")
|
||||
if not current_user:
|
||||
suggestions.append("登录控制台")
|
||||
if not has_collected_data:
|
||||
@@ -348,8 +371,9 @@ async def get_earth_oobe_status(
|
||||
|
||||
return {
|
||||
"ready": ready,
|
||||
"demo_mode": demo_mode,
|
||||
"authenticated": current_user is not None,
|
||||
"needs_login": current_user is None and not ready,
|
||||
"needs_login": current_user is None and not ready and not demo_mode,
|
||||
"has_collected_data": has_collected_data,
|
||||
"has_tv_sources": tv_source_count > 0,
|
||||
"has_core_layers": has_core_layers,
|
||||
|
||||
190
backend/app/api/v1/interactables.py
Normal file
190
backend/app/api/v1/interactables.py
Normal file
@@ -0,0 +1,190 @@
|
||||
"""CRUD APIs for persistent Earth interactables."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Response, status
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.security import get_current_user
|
||||
from app.db.session import get_db
|
||||
from app.models.earth_interactable import EarthInteractable
|
||||
from app.models.user import User
|
||||
from app.services.earth_interactables import (
|
||||
build_interactable_event,
|
||||
interactables_to_geojson,
|
||||
invalidate_interactable_cache,
|
||||
list_interactables,
|
||||
normalize_interactable_id,
|
||||
publish_interactable_event,
|
||||
serialize_interactable,
|
||||
)
|
||||
from app.services.earth_layer_cache import (
|
||||
EarthLayerCachePolicy,
|
||||
earth_layer_cache,
|
||||
get_or_build_layer_payload,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
INTERACTABLE_CACHE_POLICY = EarthLayerCachePolicy(
|
||||
fresh_ttl_seconds=60,
|
||||
stale_ttl_seconds=10 * 60,
|
||||
max_features=5000,
|
||||
)
|
||||
|
||||
|
||||
class InteractableCreate(BaseModel):
|
||||
id: str | None = Field(default=None, max_length=160)
|
||||
layer: str = Field(default="default", min_length=1, max_length=80)
|
||||
kind: str = Field(default="default", min_length=1, max_length=80)
|
||||
label: str = Field(default="", max_length=255)
|
||||
description: str = Field(default="", max_length=4000)
|
||||
latitude: float = Field(ge=-90, le=90)
|
||||
longitude: float = Field(ge=-180, le=180)
|
||||
altitude: float | None = None
|
||||
properties: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("layer", "kind")
|
||||
@classmethod
|
||||
def normalize_key(cls, value: str) -> str:
|
||||
normalized = str(value or "").strip()
|
||||
if not normalized:
|
||||
raise ValueError("must not be empty")
|
||||
return normalized
|
||||
|
||||
|
||||
class InteractableUpdate(BaseModel):
|
||||
layer: str | None = Field(default=None, min_length=1, max_length=80)
|
||||
kind: str | None = Field(default=None, min_length=1, max_length=80)
|
||||
label: str | None = Field(default=None, max_length=255)
|
||||
description: str | None = Field(default=None, max_length=4000)
|
||||
latitude: float | None = Field(default=None, ge=-90, le=90)
|
||||
longitude: float | None = Field(default=None, ge=-180, le=180)
|
||||
altitude: float | None = None
|
||||
properties: dict[str, Any] | None = None
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def get_interactables(
|
||||
response: Response,
|
||||
layer: str | None = Query(default=None),
|
||||
include_deleted: bool = Query(default=False),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
items = await list_interactables(db, layer=layer, include_deleted=include_deleted)
|
||||
response.headers["X-Planet-Interactables-Count"] = str(len(items))
|
||||
return {"items": [serialize_interactable(item) for item in items]}
|
||||
|
||||
|
||||
@router.get("/geojson")
|
||||
async def get_interactables_geojson(
|
||||
response: Response,
|
||||
layer: str | None = Query(default=None),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
async def build_payload() -> dict[str, Any]:
|
||||
items = await list_interactables(db, layer=layer)
|
||||
return interactables_to_geojson(items)
|
||||
|
||||
payload = await get_or_build_layer_payload(
|
||||
key=earth_layer_cache.key("interactables", layer=layer or "all"),
|
||||
policy=INTERACTABLE_CACHE_POLICY,
|
||||
builder=build_payload,
|
||||
response=response,
|
||||
)
|
||||
response.headers["X-Planet-Interactables-Count"] = str(len(payload.get("features") or []))
|
||||
return payload
|
||||
|
||||
|
||||
@router.post("", status_code=status.HTTP_201_CREATED)
|
||||
async def create_interactable(
|
||||
payload: InteractableCreate,
|
||||
_current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
record_id = normalize_interactable_id(payload.id)
|
||||
existing = await db.get(EarthInteractable, record_id)
|
||||
if existing and not existing.is_deleted:
|
||||
raise HTTPException(status_code=409, detail="Interactable already exists")
|
||||
|
||||
if existing is None:
|
||||
record = EarthInteractable(id=record_id)
|
||||
db.add(record)
|
||||
else:
|
||||
record = existing
|
||||
record.is_deleted = False
|
||||
record.deleted_at = None
|
||||
record.revision += 1
|
||||
|
||||
record.layer = payload.layer
|
||||
record.kind = payload.kind
|
||||
record.label = payload.label
|
||||
record.description = payload.description
|
||||
record.latitude = payload.latitude
|
||||
record.longitude = payload.longitude
|
||||
record.altitude = payload.altitude
|
||||
record.properties = payload.properties
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(record)
|
||||
invalidate_interactable_cache(record.layer)
|
||||
await publish_interactable_event("created", record)
|
||||
return {"item": serialize_interactable(record)}
|
||||
|
||||
|
||||
@router.get("/{interactable_id}")
|
||||
async def get_interactable(interactable_id: str, db: AsyncSession = Depends(get_db)):
|
||||
record = await db.get(EarthInteractable, interactable_id)
|
||||
if record is None or record.is_deleted:
|
||||
raise HTTPException(status_code=404, detail="Interactable not found")
|
||||
return {"item": serialize_interactable(record)}
|
||||
|
||||
|
||||
@router.patch("/{interactable_id}")
|
||||
async def update_interactable(
|
||||
interactable_id: str,
|
||||
payload: InteractableUpdate,
|
||||
_current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
record = await db.get(EarthInteractable, interactable_id)
|
||||
if record is None or record.is_deleted:
|
||||
raise HTTPException(status_code=404, detail="Interactable not found")
|
||||
|
||||
previous_layer = record.layer
|
||||
patch = payload.model_dump(exclude_unset=True)
|
||||
for key, value in patch.items():
|
||||
setattr(record, key, value)
|
||||
record.revision += 1
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(record)
|
||||
invalidate_interactable_cache(previous_layer)
|
||||
if record.layer != previous_layer:
|
||||
invalidate_interactable_cache(record.layer)
|
||||
await publish_interactable_event("updated", record)
|
||||
return {"item": serialize_interactable(record)}
|
||||
|
||||
|
||||
@router.delete("/{interactable_id}")
|
||||
async def delete_interactable(
|
||||
interactable_id: str,
|
||||
_current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
record = await db.get(EarthInteractable, interactable_id)
|
||||
if record is None or record.is_deleted:
|
||||
raise HTTPException(status_code=404, detail="Interactable not found")
|
||||
|
||||
record.is_deleted = True
|
||||
record.deleted_at = datetime.now(UTC)
|
||||
record.revision += 1
|
||||
await db.commit()
|
||||
await db.refresh(record)
|
||||
invalidate_interactable_cache(record.layer)
|
||||
await publish_interactable_event("deleted", record)
|
||||
event = build_interactable_event(action="deleted", record=record, include_item=True)
|
||||
return {"deleted": True, "event": event}
|
||||
@@ -11,6 +11,7 @@ from dotenv import dotenv_values
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.core.security import get_current_user
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.config import settings as app_settings
|
||||
@@ -66,8 +67,10 @@ from app.services.llm_provider_catalog import (
|
||||
from app.services.scheduler import sync_datasource_job
|
||||
from app.services.tv_streams import DEFAULT_TV_SETTINGS, get_tv_settings_payload, normalize_tv_settings
|
||||
from app.services.persistent_logs import record_audit_log
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
|
||||
router = APIRouter()
|
||||
logger = get_logger(__name__, service="api")
|
||||
AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS = 5
|
||||
AI_CONNECTION_TEST_PROMPT_KEY = "ai.connection_test"
|
||||
SECRET_REVEAL_ROLES = {"admin", "super_admin"}
|
||||
@@ -79,6 +82,7 @@ DEFAULT_SETTINGS = {
|
||||
"auto_refresh": True,
|
||||
"data_retention_days": 30,
|
||||
"max_concurrent_tasks": 5,
|
||||
"demo_mode": False,
|
||||
},
|
||||
"notifications": {
|
||||
"email_enabled": False,
|
||||
@@ -203,6 +207,7 @@ class SystemSettingsUpdate(BaseModel):
|
||||
auto_refresh: bool = True
|
||||
data_retention_days: int = Field(default=30, ge=1, le=3650)
|
||||
max_concurrent_tasks: int = Field(default=5, ge=1, le=50)
|
||||
demo_mode: bool = False
|
||||
|
||||
|
||||
class NotificationSettingsUpdate(BaseModel):
|
||||
@@ -699,6 +704,18 @@ async def _validate_ai_provider_full_connection(ai_payload: dict) -> dict:
|
||||
retry_attempts=runtime_config["retry_attempts"],
|
||||
llm_config=runtime_config.get("llm_config") or {},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.ai_provider.full_connection.start",
|
||||
message="AI provider full connection validation started",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
context={
|
||||
"provider": runtime_config.get("llm_config", {}).get("provider"),
|
||||
"model": runtime_config.get("llm_config", {}).get("model"),
|
||||
},
|
||||
)
|
||||
status_result = await client.get_status()
|
||||
if not status_result.configured:
|
||||
raise HTTPException(
|
||||
@@ -715,6 +732,19 @@ async def _validate_ai_provider_full_connection(ai_payload: dict) -> dict:
|
||||
constraints=["回复尽量简短。"],
|
||||
)
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.ai_provider.full_connection.success",
|
||||
message="AI provider full connection validation completed",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
context={
|
||||
"provider": analysis_result.provider,
|
||||
"model": analysis_result.model,
|
||||
"configured": status_result.configured,
|
||||
},
|
||||
)
|
||||
return {
|
||||
"status": status_result.model_dump(),
|
||||
"provider": analysis_result.provider,
|
||||
@@ -1612,9 +1642,34 @@ async def connect_ai_provider_integration(
|
||||
llm_config=quick_llm_config,
|
||||
)
|
||||
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.ai_provider.connect.start",
|
||||
message="AI provider connection test started",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={
|
||||
"provider": payload.provider,
|
||||
"model": payload.model,
|
||||
"timeout_seconds": min(int(runtime_config["timeout_seconds"] or 60), AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS),
|
||||
},
|
||||
)
|
||||
try:
|
||||
status_result = await client.get_status()
|
||||
if not status_result.configured:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.ai_provider.connect.failed",
|
||||
message="AI provider connection test failed because provider is incomplete",
|
||||
category="ai",
|
||||
level="warning",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={"provider": payload.provider, "model": payload.model, "configured": False},
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
@@ -1628,6 +1683,21 @@ async def connect_ai_provider_integration(
|
||||
AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS,
|
||||
),
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.ai_provider.connect.success",
|
||||
message="AI provider connection test completed",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={
|
||||
"provider": payload.provider,
|
||||
"model": payload.model,
|
||||
"configured": True,
|
||||
"lightweight_status": lightweight_result.get("status"),
|
||||
},
|
||||
)
|
||||
return {
|
||||
**lightweight_result,
|
||||
"status": status_result.model_dump(),
|
||||
@@ -1639,6 +1709,17 @@ async def connect_ai_provider_integration(
|
||||
"message": str(exc.detail),
|
||||
}
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.ai_provider.connect.failed",
|
||||
message="AI provider connection test failed",
|
||||
category="ai",
|
||||
level="error",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"provider": payload.provider, "model": payload.model}),
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
@@ -1786,8 +1867,28 @@ async def connect_web_search_integration(
|
||||
runtime_config = _runtime_config_from_web_search_payload(draft_web_search_payload)
|
||||
client = WebSearchClient(runtime_config)
|
||||
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.web_search.connect.start",
|
||||
message="WebSearch connection test started",
|
||||
category="ai_tool",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={"provider": runtime_config.default_provider},
|
||||
)
|
||||
try:
|
||||
results = await client.test_connection()
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.web_search.connect.success",
|
||||
message="WebSearch connection test completed",
|
||||
category="ai_tool",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={"provider": runtime_config.default_provider, "result_count": len(results)},
|
||||
)
|
||||
return {
|
||||
"success": True,
|
||||
"connected": True,
|
||||
@@ -1796,18 +1897,51 @@ async def connect_web_search_integration(
|
||||
"results": [item.model_dump(mode="json") for item in results[:3]],
|
||||
}
|
||||
except WebSearchConfigurationError as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.web_search.connect.failed",
|
||||
message="WebSearch connection test failed because configuration is incomplete",
|
||||
category="ai_tool",
|
||||
level="warning",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"provider": runtime_config.default_provider}),
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": str(exc),
|
||||
}
|
||||
except WebSearchError as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.web_search.connect.failed",
|
||||
message="WebSearch connection test failed",
|
||||
category="ai_tool",
|
||||
level="error",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"provider": runtime_config.default_provider}),
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
"message": str(exc),
|
||||
}
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.web_search.connect.failed",
|
||||
message="WebSearch connection test failed",
|
||||
category="ai_tool",
|
||||
level="error",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"provider": runtime_config.default_provider}),
|
||||
)
|
||||
return {
|
||||
"success": False,
|
||||
"connected": False,
|
||||
@@ -1835,17 +1969,60 @@ async def generate_provider_credential_guide(
|
||||
ai_client: AIProviderClient = Depends(get_ai_provider_client),
|
||||
):
|
||||
try:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.credential_guide.generate.start",
|
||||
message="Credential guide generation started",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={"provider": provider},
|
||||
)
|
||||
web_search_client = await get_web_search_client(db)
|
||||
return {
|
||||
"guide": await generate_credential_guide(
|
||||
guide = await generate_credential_guide(
|
||||
db,
|
||||
provider,
|
||||
ai_client,
|
||||
web_search_client,
|
||||
)
|
||||
}
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.credential_guide.generate.success",
|
||||
message="Credential guide generation completed",
|
||||
category="ai",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context={"provider": provider},
|
||||
)
|
||||
return {"guide": guide}
|
||||
except ValueError as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.credential_guide.generate.failed",
|
||||
message="Credential guide generation failed",
|
||||
category="ai",
|
||||
level="warning",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"provider": provider}),
|
||||
)
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="settings.credential_guide.generate.failed",
|
||||
message="Credential guide generation failed",
|
||||
category="ai",
|
||||
level="error",
|
||||
service="api",
|
||||
module=__name__,
|
||||
user_id=current_user.id,
|
||||
context=exception_context(exc, {"provider": provider}),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
@router.post("/credential-guides/{provider}/reset")
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
@@ -44,6 +45,42 @@ from app.services.earth_layer_cache import earth_layer_cache
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _compact_log_context(context: dict | None) -> str:
|
||||
if not context:
|
||||
return ""
|
||||
allowed = {
|
||||
key: value
|
||||
for key, value in (context or {}).items()
|
||||
if key
|
||||
in {
|
||||
"status",
|
||||
"duration_ms",
|
||||
"provider",
|
||||
"model",
|
||||
"result_provider",
|
||||
"result_model",
|
||||
"collector_name",
|
||||
"datasource_id",
|
||||
"task_id",
|
||||
"snapshot_id",
|
||||
"raw_count",
|
||||
"transformed_count",
|
||||
"saved_count",
|
||||
"created",
|
||||
"updated",
|
||||
"unchanged",
|
||||
"deleted",
|
||||
"result_count",
|
||||
"status_code",
|
||||
"error_type",
|
||||
"error",
|
||||
}
|
||||
}
|
||||
if not allowed:
|
||||
return ""
|
||||
return json.dumps(allowed, ensure_ascii=False, sort_keys=True)
|
||||
|
||||
|
||||
class RestartTaskCreate(BaseModel):
|
||||
action: str
|
||||
|
||||
@@ -374,8 +411,11 @@ async def read_database_log_snapshot(
|
||||
record.occurred_at.isoformat() if record.occurred_at else "",
|
||||
record_level.upper(),
|
||||
record.source,
|
||||
record.category or "",
|
||||
record.event or "",
|
||||
f"request_id={record.request_id}" if record.request_id else "",
|
||||
record.message,
|
||||
_compact_log_context(record.context),
|
||||
]
|
||||
if part
|
||||
)
|
||||
|
||||
@@ -38,7 +38,7 @@ from app.services.compute_center_locations import (
|
||||
upsert_compute_center_location,
|
||||
)
|
||||
from app.services.ai_client import get_ai_provider_client
|
||||
from app.api.v1.settings import get_web_search_client
|
||||
from app.api.v1.settings import get_runtime_web_search_config, get_web_search_client
|
||||
from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
|
||||
from app.services.location.llm_fallback import (
|
||||
collect_llm_location_fallback_candidate,
|
||||
@@ -105,8 +105,8 @@ LANDING_POINT_CACHE_POLICY = EarthLayerCachePolicy(
|
||||
SATELLITE_CACHE_POLICY = EarthLayerCachePolicy(
|
||||
SATELLITE_CACHE_FRESH_SECONDS,
|
||||
SATELLITE_CACHE_STALE_SECONDS,
|
||||
max_features=8000,
|
||||
max_bytes=10 * BYTES_PER_MIB,
|
||||
max_features=25000,
|
||||
max_bytes=32 * BYTES_PER_MIB,
|
||||
)
|
||||
COMPUTE_CENTER_CACHE_POLICY = EarthLayerCachePolicy(
|
||||
COMPUTE_CENTER_CACHE_FRESH_SECONDS,
|
||||
@@ -1563,15 +1563,7 @@ async def _build_cables_geojson(db: AsyncSession) -> dict[str, Any]:
|
||||
try:
|
||||
records = await _load_current_collected_data(db, "arcgis_cables")
|
||||
|
||||
if not records:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="No cable data found. Please run the arcgis_cables collector first.",
|
||||
)
|
||||
|
||||
return convert_cable_to_geojson(records)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception_event(
|
||||
"Failed to build cables GeoJSON response",
|
||||
@@ -1625,16 +1617,8 @@ async def _build_landing_points_geojson(db: AsyncSession) -> dict[str, Any]:
|
||||
relation_records,
|
||||
cable_records,
|
||||
)
|
||||
|
||||
if not records:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="No landing point data found. Please run the arcgis_landing_points collector first.",
|
||||
)
|
||||
|
||||
|
||||
return convert_landing_point_to_geojson(records, city_to_cable_ids_map, cable_id_to_name_map)
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.exception_event(
|
||||
"Failed to build landing points GeoJSON response",
|
||||
@@ -2071,6 +2055,44 @@ class SaveComputeCenterLocationRequest(BaseModel):
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
async def _compute_center_location_web_search_capability(db: AsyncSession) -> Dict[str, Any]:
|
||||
try:
|
||||
config = await get_runtime_web_search_config(db)
|
||||
except Exception as exc:
|
||||
return {
|
||||
"enabled": False,
|
||||
"provider": None,
|
||||
"reason": f"WebSearch 配置读取失败:{exc}",
|
||||
}
|
||||
provider_config = config.active_provider_config
|
||||
has_api_key = bool((provider_config.api_key or "").strip())
|
||||
if not config.enabled:
|
||||
return {
|
||||
"enabled": False,
|
||||
"provider": config.default_provider,
|
||||
"reason": "WebSearch 未开启,无法进行事实核查定位。",
|
||||
}
|
||||
if not has_api_key:
|
||||
return {
|
||||
"enabled": False,
|
||||
"provider": config.default_provider,
|
||||
"reason": f"WebSearch Provider {config.default_provider} 未配置 API Key。",
|
||||
}
|
||||
return {
|
||||
"enabled": True,
|
||||
"provider": config.default_provider,
|
||||
"reason": "",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/compute-centers/location-capability")
|
||||
async def get_compute_center_location_capability(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Return whether fact-checked compute-center location collection can run."""
|
||||
return await _compute_center_location_web_search_capability(db)
|
||||
|
||||
|
||||
@router.post("/compute-centers/{source_id}/collect-location")
|
||||
async def collect_compute_center_location(
|
||||
source_id: str,
|
||||
@@ -2089,6 +2111,9 @@ async def collect_compute_center_location(
|
||||
"""
|
||||
if not source_id or not source_id.strip():
|
||||
raise HTTPException(status_code=400, detail="source_id is required")
|
||||
capability = await _compute_center_location_web_search_capability(db)
|
||||
if not capability.get("enabled"):
|
||||
raise HTTPException(status_code=409, detail=capability)
|
||||
|
||||
record = await _load_compute_center_record(db, source_id)
|
||||
name = payload.name or (record.name if record else None)
|
||||
@@ -2696,6 +2721,8 @@ async def _build_bgp_collectors_geojson(db: AsyncSession) -> dict[str, Any]:
|
||||
db,
|
||||
source_filter=("ris_live_bgp", "bgpstream_bgp"),
|
||||
)
|
||||
if not any(int(item.get("observation_count") or 0) > 0 for item in coverage):
|
||||
return {"type": "FeatureCollection", "features": [], "count": 0}
|
||||
coverage_by_collector = {
|
||||
item["collector"]: item
|
||||
for item in coverage
|
||||
|
||||
@@ -150,7 +150,7 @@ class DataBroadcaster:
|
||||
"timestamp": to_iso8601_utc(datetime.now(UTC)),
|
||||
"payload": data,
|
||||
},
|
||||
channel="all",
|
||||
channel="datasource_tasks",
|
||||
)
|
||||
|
||||
def start(self):
|
||||
|
||||
@@ -203,6 +203,7 @@ async def init_db():
|
||||
import app.models.vessel_enrichment # noqa: F401
|
||||
import app.models.datasource_mapping # noqa: F401
|
||||
import app.models.earth_news # noqa: F401
|
||||
import app.models.earth_interactable # noqa: F401
|
||||
|
||||
logger.warning_event(
|
||||
"Database pool settings active",
|
||||
@@ -258,6 +259,406 @@ async def init_db():
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS earth_data_change_events (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
table_name VARCHAR(128) NOT NULL,
|
||||
operation VARCHAR(16) NOT NULL,
|
||||
source VARCHAR(128),
|
||||
entity_key VARCHAR(255),
|
||||
payload JSONB NOT NULL,
|
||||
occurred_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
consumed_at TIMESTAMPTZ
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_earth_data_change_events_unconsumed
|
||||
ON earth_data_change_events (consumed_at, id)
|
||||
WHERE consumed_at IS NULL
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE OR REPLACE FUNCTION planet_emit_earth_data_changed_statement(
|
||||
change_table TEXT,
|
||||
change_operation TEXT,
|
||||
change_source TEXT,
|
||||
source_record_count INTEGER,
|
||||
source_entity_keys TEXT[]
|
||||
)
|
||||
RETURNS VOID AS $$
|
||||
DECLARE
|
||||
change_event_id BIGINT;
|
||||
change_payload JSONB;
|
||||
BEGIN
|
||||
change_payload := jsonb_build_object(
|
||||
'event', 'earth.layer.changed',
|
||||
'table', change_table,
|
||||
'operation', change_operation,
|
||||
'source', change_source,
|
||||
'entity_key', NULL,
|
||||
'entity_keys', COALESCE(to_jsonb(source_entity_keys), '[]'::jsonb),
|
||||
'records_processed', COALESCE(source_record_count, 0),
|
||||
'occurred_at', NOW()
|
||||
);
|
||||
|
||||
INSERT INTO earth_data_change_events (
|
||||
table_name,
|
||||
operation,
|
||||
source,
|
||||
entity_key,
|
||||
payload,
|
||||
occurred_at
|
||||
) VALUES (
|
||||
change_table,
|
||||
change_operation,
|
||||
change_source,
|
||||
NULL,
|
||||
change_payload,
|
||||
NOW()
|
||||
)
|
||||
RETURNING id INTO change_event_id;
|
||||
|
||||
change_payload := change_payload || jsonb_build_object(
|
||||
'event_id', change_event_id
|
||||
);
|
||||
|
||||
UPDATE earth_data_change_events
|
||||
SET payload = change_payload
|
||||
WHERE id = change_event_id;
|
||||
|
||||
PERFORM pg_notify(
|
||||
'planet_earth_data_changes',
|
||||
change_payload::text
|
||||
);
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE OR REPLACE FUNCTION planet_emit_collected_data_changed_statement(
|
||||
change_operation TEXT,
|
||||
change_source TEXT,
|
||||
source_record_count INTEGER,
|
||||
source_entity_keys TEXT[]
|
||||
)
|
||||
RETURNS VOID AS $$
|
||||
BEGIN
|
||||
PERFORM planet_emit_earth_data_changed_statement(
|
||||
'collected_data',
|
||||
change_operation,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE OR REPLACE FUNCTION planet_notify_earth_table_changed_statement()
|
||||
RETURNS trigger AS $$
|
||||
DECLARE
|
||||
change_source TEXT;
|
||||
source_record_count INTEGER;
|
||||
source_entity_keys TEXT[];
|
||||
BEGIN
|
||||
IF TG_OP = 'INSERT' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME)
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM new_rows AS t) changed_rows
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(
|
||||
NULLIF(row_data->>'entity_key', ''),
|
||||
NULLIF(row_data->>'source_id', ''),
|
||||
NULLIF(row_data->>'incident_key', ''),
|
||||
NULLIF(row_data->>'id', ''),
|
||||
NULLIF(row_data->>'mmsi', '')
|
||||
)
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM new_rows AS t) rows_for_keys
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM new_rows AS t) rows_for_count
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source;
|
||||
|
||||
PERFORM planet_emit_earth_data_changed_statement(
|
||||
TG_TABLE_NAME,
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
ELSIF TG_OP = 'DELETE' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME)
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM old_rows AS t) changed_rows
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(
|
||||
NULLIF(row_data->>'entity_key', ''),
|
||||
NULLIF(row_data->>'source_id', ''),
|
||||
NULLIF(row_data->>'incident_key', ''),
|
||||
NULLIF(row_data->>'id', ''),
|
||||
NULLIF(row_data->>'mmsi', '')
|
||||
)
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM old_rows AS t) rows_for_keys
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM (SELECT to_jsonb(t) AS row_data FROM old_rows AS t) rows_for_count
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source;
|
||||
|
||||
PERFORM planet_emit_earth_data_changed_statement(
|
||||
TG_TABLE_NAME,
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
ELSIF TG_OP = 'UPDATE' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME)
|
||||
FROM (
|
||||
SELECT to_jsonb(t) AS row_data FROM new_rows AS t
|
||||
UNION ALL
|
||||
SELECT to_jsonb(t) AS row_data FROM old_rows AS t
|
||||
) changed_rows
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(
|
||||
NULLIF(row_data->>'entity_key', ''),
|
||||
NULLIF(row_data->>'source_id', ''),
|
||||
NULLIF(row_data->>'incident_key', ''),
|
||||
NULLIF(row_data->>'id', ''),
|
||||
NULLIF(row_data->>'mmsi', '')
|
||||
)
|
||||
FROM (
|
||||
SELECT to_jsonb(t) AS row_data FROM new_rows AS t
|
||||
UNION ALL
|
||||
SELECT to_jsonb(t) AS row_data FROM old_rows AS t
|
||||
) rows_for_keys
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM (
|
||||
SELECT to_jsonb(t) AS row_data FROM new_rows AS t
|
||||
UNION ALL
|
||||
SELECT to_jsonb(t) AS row_data FROM old_rows AS t
|
||||
) rows_for_count
|
||||
WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source;
|
||||
|
||||
PERFORM planet_emit_earth_data_changed_statement(
|
||||
TG_TABLE_NAME,
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
END IF;
|
||||
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE OR REPLACE FUNCTION planet_notify_collected_data_changed_statement()
|
||||
RETURNS trigger AS $$
|
||||
DECLARE
|
||||
change_source TEXT;
|
||||
source_record_count INTEGER;
|
||||
source_entity_keys TEXT[];
|
||||
BEGIN
|
||||
IF TG_OP = 'INSERT' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT source FROM new_rows WHERE source IS NOT NULL
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(entity_key, source_id, id::text)
|
||||
FROM new_rows
|
||||
WHERE source = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM new_rows
|
||||
WHERE source = change_source;
|
||||
|
||||
PERFORM planet_emit_collected_data_changed_statement(
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
ELSIF TG_OP = 'DELETE' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT source FROM old_rows WHERE source IS NOT NULL
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(entity_key, source_id, id::text)
|
||||
FROM old_rows
|
||||
WHERE source = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM old_rows
|
||||
WHERE source = change_source;
|
||||
|
||||
PERFORM planet_emit_collected_data_changed_statement(
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
ELSIF TG_OP = 'UPDATE' THEN
|
||||
FOR change_source IN
|
||||
SELECT DISTINCT source FROM (
|
||||
SELECT source FROM new_rows
|
||||
UNION
|
||||
SELECT source FROM old_rows
|
||||
) changed_sources
|
||||
WHERE source IS NOT NULL
|
||||
LOOP
|
||||
SELECT
|
||||
COUNT(*),
|
||||
ARRAY(
|
||||
SELECT DISTINCT COALESCE(entity_key, source_id, id::text)
|
||||
FROM (
|
||||
SELECT id, source_id, entity_key, source FROM new_rows
|
||||
UNION ALL
|
||||
SELECT id, source_id, entity_key, source FROM old_rows
|
||||
) changed_rows
|
||||
WHERE source = change_source
|
||||
LIMIT 20
|
||||
)
|
||||
INTO source_record_count, source_entity_keys
|
||||
FROM (
|
||||
SELECT id, source_id, entity_key, source FROM new_rows
|
||||
UNION ALL
|
||||
SELECT id, source_id, entity_key, source FROM old_rows
|
||||
) changed_rows
|
||||
WHERE source = change_source;
|
||||
|
||||
PERFORM planet_emit_collected_data_changed_statement(
|
||||
TG_OP,
|
||||
change_source,
|
||||
source_record_count,
|
||||
source_entity_keys
|
||||
);
|
||||
END LOOP;
|
||||
END IF;
|
||||
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
"""
|
||||
)
|
||||
)
|
||||
for statement in (
|
||||
"DROP TRIGGER IF EXISTS tr_planet_collected_data_changed ON collected_data",
|
||||
"DROP TRIGGER IF EXISTS tr_planet_collected_data_changed_insert ON collected_data",
|
||||
"DROP TRIGGER IF EXISTS tr_planet_collected_data_changed_update ON collected_data",
|
||||
"DROP TRIGGER IF EXISTS tr_planet_collected_data_changed_delete ON collected_data",
|
||||
"DROP FUNCTION IF EXISTS planet_notify_collected_data_changed()",
|
||||
"""
|
||||
CREATE TRIGGER tr_planet_collected_data_changed_insert
|
||||
AFTER INSERT ON collected_data
|
||||
REFERENCING NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_collected_data_changed_statement()
|
||||
""",
|
||||
"""
|
||||
CREATE TRIGGER tr_planet_collected_data_changed_update
|
||||
AFTER UPDATE ON collected_data
|
||||
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_collected_data_changed_statement()
|
||||
""",
|
||||
"""
|
||||
CREATE TRIGGER tr_planet_collected_data_changed_delete
|
||||
AFTER DELETE ON collected_data
|
||||
REFERENCING OLD TABLE AS old_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_collected_data_changed_statement()
|
||||
""",
|
||||
):
|
||||
await conn.execute(text(statement))
|
||||
for table_name in (
|
||||
"bgp_observations",
|
||||
"bgp_anomalies",
|
||||
"bgp_incidents",
|
||||
"bgp_collector_locations",
|
||||
"vessel_static",
|
||||
"vessel_position",
|
||||
"ais_raw_observations",
|
||||
"ais_source_health",
|
||||
"compute_center_locations",
|
||||
"earth_interactables",
|
||||
"earth_news_items",
|
||||
):
|
||||
for statement in (
|
||||
f"DROP TRIGGER IF EXISTS tr_planet_{table_name}_changed_insert ON {table_name}",
|
||||
f"DROP TRIGGER IF EXISTS tr_planet_{table_name}_changed_update ON {table_name}",
|
||||
f"DROP TRIGGER IF EXISTS tr_planet_{table_name}_changed_delete ON {table_name}",
|
||||
f"""
|
||||
CREATE TRIGGER tr_planet_{table_name}_changed_insert
|
||||
AFTER INSERT ON {table_name}
|
||||
REFERENCING NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_earth_table_changed_statement()
|
||||
""",
|
||||
f"""
|
||||
CREATE TRIGGER tr_planet_{table_name}_changed_update
|
||||
AFTER UPDATE ON {table_name}
|
||||
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_earth_table_changed_statement()
|
||||
""",
|
||||
f"""
|
||||
CREATE TRIGGER tr_planet_{table_name}_changed_delete
|
||||
AFTER DELETE ON {table_name}
|
||||
REFERENCING OLD TABLE AS old_rows
|
||||
FOR EACH STATEMENT
|
||||
EXECUTE FUNCTION planet_notify_earth_table_changed_statement()
|
||||
""",
|
||||
):
|
||||
await conn.execute(text(statement))
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
@@ -267,7 +668,16 @@ async def init_db():
|
||||
ADD COLUMN IF NOT EXISTS phase_message VARCHAR(255),
|
||||
ADD COLUMN IF NOT EXISTS phase_current BIGINT,
|
||||
ADD COLUMN IF NOT EXISTS phase_total BIGINT,
|
||||
ADD COLUMN IF NOT EXISTS phase_unit VARCHAR(30)
|
||||
ADD COLUMN IF NOT EXISTS phase_unit VARCHAR(30),
|
||||
ADD COLUMN IF NOT EXISTS source VARCHAR(100),
|
||||
ADD COLUMN IF NOT EXISTS task_type VARCHAR(30) NOT NULL DEFAULT 'collect',
|
||||
ADD COLUMN IF NOT EXISTS payload JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
ADD COLUMN IF NOT EXISTS rollback_policy VARCHAR(40) NOT NULL DEFAULT 'keep_committed_batches',
|
||||
ADD COLUMN IF NOT EXISTS dedupe_key VARCHAR(180),
|
||||
ADD COLUMN IF NOT EXISTS worker_id VARCHAR(120),
|
||||
ADD COLUMN IF NOT EXISTS locked_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS requested_cancel_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS cancel_reason TEXT
|
||||
"""
|
||||
)
|
||||
)
|
||||
@@ -283,6 +693,17 @@ async def init_db():
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
ALTER TABLE earth_interactables
|
||||
ADD COLUMN IF NOT EXISTS altitude DOUBLE PRECISION,
|
||||
ADD COLUMN IF NOT EXISTS revision INTEGER NOT NULL DEFAULT 1,
|
||||
ADD COLUMN IF NOT EXISTS is_deleted BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
@@ -307,6 +728,48 @@ async def init_db():
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_earth_interactables_layer_deleted
|
||||
ON earth_interactables (layer, is_deleted)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_earth_interactables_updated_at
|
||||
ON earth_interactables (updated_at)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_collection_tasks_source_status
|
||||
ON collection_tasks (source, status)
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_collection_tasks_queue
|
||||
ON collection_tasks (status, created_at, id)
|
||||
WHERE status = 'queued'
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE INDEX IF NOT EXISTS idx_collection_tasks_dedupe
|
||||
ON collection_tasks (dedupe_key)
|
||||
WHERE dedupe_key IS NOT NULL
|
||||
"""
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
text(
|
||||
"""
|
||||
|
||||
@@ -24,6 +24,11 @@ from app.services.earth_news_worker import (
|
||||
start_earth_news_target_worker,
|
||||
stop_earth_news_target_worker,
|
||||
)
|
||||
from app.services.earth_db_change_listener import (
|
||||
start_earth_db_change_listener,
|
||||
stop_earth_db_change_listener,
|
||||
)
|
||||
from app.services.data_jobs import start_data_job_worker, stop_data_job_worker
|
||||
|
||||
|
||||
configure_logging()
|
||||
@@ -59,9 +64,13 @@ async def lifespan(app: FastAPI):
|
||||
start_scheduler()
|
||||
await sync_scheduler_with_datasources()
|
||||
broadcaster.start()
|
||||
start_data_job_worker()
|
||||
start_earth_db_change_listener()
|
||||
start_earth_news_target_worker()
|
||||
yield
|
||||
await stop_earth_news_target_worker()
|
||||
await stop_earth_db_change_listener()
|
||||
await stop_data_job_worker()
|
||||
broadcaster.stop()
|
||||
stop_scheduler()
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ from app.models.system_log import SystemLog, AuditLog
|
||||
from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth, VesselPosition, VesselStatic
|
||||
from app.models.datasource_mapping import DataSourceMappingTemplate
|
||||
from app.models.earth_news import EarthNewsItem
|
||||
from app.models.earth_interactable import EarthInteractable
|
||||
|
||||
__all__ = [
|
||||
"User",
|
||||
@@ -45,4 +46,5 @@ __all__ = [
|
||||
"AISSourceHealth",
|
||||
"DataSourceMappingTemplate",
|
||||
"EarthNewsItem",
|
||||
"EarthInteractable",
|
||||
]
|
||||
|
||||
30
backend/app/models/earth_interactable.py
Normal file
30
backend/app/models/earth_interactable.py
Normal file
@@ -0,0 +1,30 @@
|
||||
"""Persistent Earth interactable objects."""
|
||||
|
||||
from sqlalchemy import Boolean, Column, DateTime, Float, Index, Integer, JSON, String, Text
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from app.db.session import Base
|
||||
|
||||
|
||||
class EarthInteractable(Base):
|
||||
__tablename__ = "earth_interactables"
|
||||
|
||||
id = Column(String(160), primary_key=True)
|
||||
layer = Column(String(80), nullable=False, default="interactables", index=True)
|
||||
kind = Column(String(80), nullable=False, default="default", index=True)
|
||||
label = Column(String(255), nullable=False, default="")
|
||||
description = Column(Text, nullable=False, default="")
|
||||
latitude = Column(Float, nullable=False)
|
||||
longitude = Column(Float, nullable=False)
|
||||
altitude = Column(Float, nullable=True)
|
||||
revision = Column(Integer, nullable=False, default=1)
|
||||
properties = Column(JSON, nullable=False, default=dict)
|
||||
is_deleted = Column(Boolean, nullable=False, default=False, index=True)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
|
||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
deleted_at = Column(DateTime(timezone=True), nullable=True, index=True)
|
||||
|
||||
__table_args__ = (
|
||||
Index("idx_earth_interactables_layer_deleted", "layer", "is_deleted"),
|
||||
Index("idx_earth_interactables_updated_at", "updated_at"),
|
||||
)
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Collection Task model"""
|
||||
"""Datasource job model."""
|
||||
|
||||
from sqlalchemy import BigInteger, Column, DateTime, Integer, String, Text, Float
|
||||
from sqlalchemy import BigInteger, Column, DateTime, Float, Integer, JSON, String, Text
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from app.db.session import Base
|
||||
@@ -11,7 +11,9 @@ class CollectionTask(Base):
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
datasource_id = Column(Integer, nullable=False, index=True)
|
||||
status = Column(String(20), nullable=False) # pending, running, success, failed, cancelled
|
||||
source = Column(String(100), nullable=True, index=True)
|
||||
task_type = Column(String(30), nullable=False, default="collect", index=True)
|
||||
status = Column(String(20), nullable=False) # queued, running, cancelling, success, failed, cancelled
|
||||
phase = Column(String(30), default="queued")
|
||||
phase_progress = Column(Float)
|
||||
phase_message = Column(String(255))
|
||||
@@ -24,6 +26,13 @@ class CollectionTask(Base):
|
||||
total_records = Column(Integer, default=0) # Total records to process
|
||||
progress = Column(Float, default=0.0) # Progress percentage (0-100)
|
||||
error_message = Column(Text)
|
||||
payload = Column(JSON, default=dict)
|
||||
rollback_policy = Column(String(40), nullable=False, default="keep_committed_batches")
|
||||
dedupe_key = Column(String(180), nullable=True, index=True)
|
||||
worker_id = Column(String(120), nullable=True, index=True)
|
||||
locked_at = Column(DateTime(timezone=True), nullable=True, index=True)
|
||||
requested_cancel_at = Column(DateTime(timezone=True), nullable=True)
|
||||
cancel_reason = Column(Text)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
def __repr__(self):
|
||||
|
||||
@@ -2,18 +2,24 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from time import perf_counter
|
||||
|
||||
import httpx
|
||||
from fastapi import Depends, HTTPException, status
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.logging import get_logger
|
||||
from app.db.session import get_db
|
||||
from app.schemas.ai import (
|
||||
AIProviderStatusResponse,
|
||||
SituationalAnalysisRequest,
|
||||
SituationalAnalysisResponse,
|
||||
)
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
|
||||
|
||||
logger = get_logger(__name__, service="ai")
|
||||
|
||||
|
||||
class AIProviderClient:
|
||||
@@ -64,7 +70,19 @@ class AIProviderClient:
|
||||
return headers
|
||||
|
||||
async def get_status(self, request_id: str | None = None) -> AIProviderStatusResponse:
|
||||
context = self._base_log_context(operation="status")
|
||||
if not self.service_url:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.provider.status.failed",
|
||||
message="AI provider status skipped because service URL is not configured",
|
||||
category="ai",
|
||||
level="warning",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context={**context, "status": "unconfigured"},
|
||||
)
|
||||
return AIProviderStatusResponse(
|
||||
provider="unconfigured",
|
||||
enabled=False,
|
||||
@@ -73,27 +91,133 @@ class AIProviderClient:
|
||||
base_url=None,
|
||||
)
|
||||
|
||||
data = await self._request("GET", "/v1/provider/status", request_id=request_id)
|
||||
return AIProviderStatusResponse.model_validate(data)
|
||||
started_at = perf_counter()
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.provider.status.start",
|
||||
message="AI provider status request started",
|
||||
category="ai",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context=context,
|
||||
)
|
||||
try:
|
||||
data = await self._request("GET", "/v1/provider/status", request_id=request_id, operation="status")
|
||||
result = AIProviderStatusResponse.model_validate(data)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.provider.status.success",
|
||||
message="AI provider status request completed",
|
||||
category="ai",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context={
|
||||
**context,
|
||||
"status": "success",
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
"result_provider": result.provider,
|
||||
"result_model": result.model,
|
||||
"configured": result.configured,
|
||||
"enabled": result.enabled,
|
||||
},
|
||||
)
|
||||
return result
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.provider.status.failed",
|
||||
message="AI provider status request failed",
|
||||
category="ai",
|
||||
level="error",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context=exception_context(exc, {**context, "status": "failed", "duration_ms": self._duration_ms(started_at)}),
|
||||
)
|
||||
raise
|
||||
|
||||
async def analyze(
|
||||
self,
|
||||
payload: SituationalAnalysisRequest,
|
||||
request_id: str | None = None,
|
||||
) -> SituationalAnalysisResponse:
|
||||
context = self._base_log_context(
|
||||
operation="analyze",
|
||||
preferred_model=payload.preferred_model,
|
||||
input_summary=self._summarize_analysis_payload(payload),
|
||||
)
|
||||
if not self.service_url:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.provider.analyze.failed",
|
||||
message="AI provider analyze skipped because service URL is not configured",
|
||||
category="ai",
|
||||
level="warning",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context={**context, "status": "unconfigured"},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="AI provider service URL is not configured.",
|
||||
)
|
||||
|
||||
data = await self._request(
|
||||
"POST",
|
||||
"/v1/analyze",
|
||||
json=payload.model_dump(),
|
||||
started_at = perf_counter()
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.provider.analyze.start",
|
||||
message="AI provider analyze request started",
|
||||
category="ai",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context=context,
|
||||
)
|
||||
return SituationalAnalysisResponse.model_validate(data)
|
||||
try:
|
||||
data = await self._request(
|
||||
"POST",
|
||||
"/v1/analyze",
|
||||
json=payload.model_dump(),
|
||||
request_id=request_id,
|
||||
operation="analyze",
|
||||
payload_summary=context["input_summary"],
|
||||
)
|
||||
result = SituationalAnalysisResponse.model_validate(data)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.provider.analyze.success",
|
||||
message="AI provider analyze request completed",
|
||||
category="ai",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context={
|
||||
**context,
|
||||
"status": "success",
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
"result_provider": result.provider,
|
||||
"result_model": result.model,
|
||||
"content_block_count": len(result.content_blocks or []),
|
||||
"thinking_block_count": len(result.thinking_blocks or []),
|
||||
},
|
||||
)
|
||||
return result
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.provider.analyze.failed",
|
||||
message="AI provider analyze request failed",
|
||||
category="ai",
|
||||
level="error",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context=exception_context(exc, {**context, "status": "failed", "duration_ms": self._duration_ms(started_at)}),
|
||||
)
|
||||
raise
|
||||
|
||||
async def _request(
|
||||
self,
|
||||
@@ -101,9 +225,12 @@ class AIProviderClient:
|
||||
path: str,
|
||||
json: dict | None = None,
|
||||
request_id: str | None = None,
|
||||
operation: str = "request",
|
||||
payload_summary: dict | None = None,
|
||||
) -> dict:
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(1, self.retry_attempts + 1):
|
||||
attempt_started_at = perf_counter()
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
response = await client.request(
|
||||
@@ -117,6 +244,15 @@ class AIProviderClient:
|
||||
except httpx.HTTPStatusError as exc:
|
||||
last_error = exc
|
||||
if attempt < self.retry_attempts and exc.response.status_code >= 500:
|
||||
await self._log_retry(
|
||||
operation=operation,
|
||||
request_id=request_id,
|
||||
attempt=attempt,
|
||||
status_code=exc.response.status_code,
|
||||
duration_ms=self._duration_ms(attempt_started_at),
|
||||
error=exc,
|
||||
payload_summary=payload_summary,
|
||||
)
|
||||
await asyncio.sleep(0.3 * attempt)
|
||||
continue
|
||||
detail = exc.response.text or "AI provider service returned an error"
|
||||
@@ -127,6 +263,14 @@ class AIProviderClient:
|
||||
except httpx.HTTPError as exc:
|
||||
last_error = exc
|
||||
if attempt < self.retry_attempts:
|
||||
await self._log_retry(
|
||||
operation=operation,
|
||||
request_id=request_id,
|
||||
attempt=attempt,
|
||||
duration_ms=self._duration_ms(attempt_started_at),
|
||||
error=exc,
|
||||
payload_summary=payload_summary,
|
||||
)
|
||||
await asyncio.sleep(0.3 * attempt)
|
||||
continue
|
||||
raise HTTPException(
|
||||
@@ -139,6 +283,71 @@ class AIProviderClient:
|
||||
detail=f"AI provider service request failed: {last_error}",
|
||||
)
|
||||
|
||||
def _base_log_context(self, **extra: object) -> dict:
|
||||
llm_provider_apis = self.llm_config.get("model_provider_apis")
|
||||
return {
|
||||
"provider": self.llm_config.get("provider") or "",
|
||||
"provider_api": self.llm_config.get("provider_api") or "",
|
||||
"model": self.llm_config.get("model") or "",
|
||||
"base_url_configured": bool(self.llm_config.get("base_url")),
|
||||
"service_url_configured": bool(self.service_url),
|
||||
"timeout_seconds": self.timeout,
|
||||
"retry_attempts": self.retry_attempts,
|
||||
"model_provider_api_count": len(llm_provider_apis or {}) if isinstance(llm_provider_apis, dict) else 0,
|
||||
**extra,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _duration_ms(started_at: float) -> int:
|
||||
return int((perf_counter() - started_at) * 1000)
|
||||
|
||||
@staticmethod
|
||||
def _summarize_analysis_payload(payload: SituationalAnalysisRequest) -> dict:
|
||||
context = payload.context if isinstance(payload.context, dict) else {}
|
||||
thinking = payload.thinking if isinstance(payload.thinking, dict) else payload.thinking
|
||||
return {
|
||||
"title_length": len(payload.title or ""),
|
||||
"objective_length": len(payload.objective or ""),
|
||||
"observation_count": len(payload.observations or []),
|
||||
"constraint_count": len(payload.constraints or []),
|
||||
"has_system_prompt": bool(payload.system_prompt),
|
||||
"thinking_enabled": bool(thinking),
|
||||
"context_keys": sorted(str(key) for key in context.keys()),
|
||||
}
|
||||
|
||||
async def _log_retry(
|
||||
self,
|
||||
*,
|
||||
operation: str,
|
||||
request_id: str | None,
|
||||
attempt: int,
|
||||
duration_ms: int,
|
||||
error: BaseException,
|
||||
status_code: int | None = None,
|
||||
payload_summary: dict | None = None,
|
||||
) -> None:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event=f"ai.provider.{operation}.retry",
|
||||
message="AI provider request will retry",
|
||||
category="ai",
|
||||
level="warning",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
context=exception_context(
|
||||
error,
|
||||
{
|
||||
**self._base_log_context(operation=operation),
|
||||
"attempt": attempt,
|
||||
"next_attempt": attempt + 1,
|
||||
"status_code": status_code,
|
||||
"duration_ms": duration_ms,
|
||||
"input_summary": payload_summary,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def get_ai_provider_client(db: AsyncSession = Depends(get_db)) -> AIProviderClient:
|
||||
from app.api.v1.settings import get_runtime_ai_provider_config
|
||||
|
||||
@@ -1,11 +1,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from time import perf_counter
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.services.ai_tools.schemas import FetchedEvidence
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
|
||||
|
||||
logger = get_logger(__name__, service="ai_tool")
|
||||
|
||||
|
||||
class WebFetchError(RuntimeError):
|
||||
@@ -29,8 +36,33 @@ async def fetch_url_evidence(
|
||||
timeout_seconds: int = 20,
|
||||
max_bytes: int = 1_500_000,
|
||||
) -> FetchedEvidence:
|
||||
started_at = perf_counter()
|
||||
if not url:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_fetch.failed",
|
||||
message="WebFetch failed because URL is empty",
|
||||
category="ai_tool",
|
||||
level="warning",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context={"reason": "empty_url"},
|
||||
)
|
||||
raise WebFetchError("url is required")
|
||||
request_host = urlparse(url).netloc
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_fetch.start",
|
||||
message="WebFetch request started",
|
||||
category="ai_tool",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context={
|
||||
"url_host": request_host,
|
||||
"timeout_seconds": timeout_seconds,
|
||||
"max_bytes": max_bytes,
|
||||
},
|
||||
)
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=timeout_seconds,
|
||||
@@ -41,10 +73,45 @@ async def fetch_url_evidence(
|
||||
response.raise_for_status()
|
||||
content = response.content[:max_bytes]
|
||||
except httpx.HTTPError as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_fetch.failed",
|
||||
message="WebFetch request failed",
|
||||
category="ai_tool",
|
||||
level="error",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context=exception_context(
|
||||
exc,
|
||||
{
|
||||
"url_host": request_host,
|
||||
"status": "failed",
|
||||
"duration_ms": int((perf_counter() - started_at) * 1000),
|
||||
},
|
||||
),
|
||||
)
|
||||
raise WebFetchError(f"failed to fetch page: {exc}") from exc
|
||||
|
||||
title, text = _extract_title_and_text(content.decode(response.encoding or "utf-8", errors="ignore"))
|
||||
content_hash = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_fetch.success",
|
||||
message="WebFetch request completed",
|
||||
category="ai_tool",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context={
|
||||
"url_host": request_host,
|
||||
"final_url_host": urlparse(str(response.url)).netloc,
|
||||
"status": "success",
|
||||
"status_code": response.status_code,
|
||||
"bytes_read": len(content),
|
||||
"content_hash": content_hash,
|
||||
"duration_ms": int((perf_counter() - started_at) * 1000),
|
||||
"extractor": "beautifulsoup_basic",
|
||||
},
|
||||
)
|
||||
return FetchedEvidence(
|
||||
url=url,
|
||||
final_url=str(response.url),
|
||||
@@ -53,4 +120,3 @@ async def fetch_url_evidence(
|
||||
content_hash=content_hash,
|
||||
extractor="beautifulsoup_basic",
|
||||
)
|
||||
|
||||
|
||||
@@ -1,11 +1,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
import hashlib
|
||||
from time import perf_counter
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.services.ai_tools.schemas import SearchEvidence, WebSearchConfig, WebSearchProviderConfig
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
|
||||
|
||||
logger = get_logger(__name__, service="ai_tool")
|
||||
|
||||
|
||||
WEB_SEARCH_PROVIDER_PRESETS: dict[str, dict[str, Any]] = {
|
||||
@@ -120,29 +127,108 @@ class WebSearchClient:
|
||||
domains: list[str] | None = None,
|
||||
freshness_days: int | None = None,
|
||||
) -> list[SearchEvidence]:
|
||||
started_at = perf_counter()
|
||||
if not self.config.enabled:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_search.unavailable",
|
||||
message="WebSearch skipped because integration is disabled",
|
||||
category="ai_tool",
|
||||
level="warning",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context={"provider": self.config.default_provider, "reason": "disabled"},
|
||||
)
|
||||
raise WebSearchConfigurationError("WebSearch is disabled.")
|
||||
provider_config = self.config.active_provider_config
|
||||
provider = normalize_web_search_provider(provider_config.provider)
|
||||
if provider != "searxng" and not provider_config.api_key:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_search.unavailable",
|
||||
message="WebSearch skipped because API key is not configured",
|
||||
category="ai_tool",
|
||||
level="warning",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context={"provider": provider, "reason": "missing_api_key"},
|
||||
)
|
||||
raise WebSearchConfigurationError(f"{provider} API key is not configured.")
|
||||
query = " ".join(str(query or "").split())
|
||||
if not query:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_search.failed",
|
||||
message="WebSearch failed because query is empty",
|
||||
category="ai_tool",
|
||||
level="warning",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context={"provider": provider, "reason": "empty_query"},
|
||||
)
|
||||
raise WebSearchConfigurationError("search query is required.")
|
||||
limit = max_results or provider_config.max_results
|
||||
if provider == "tavily":
|
||||
return await self._search_tavily(provider_config, query, limit, domains, freshness_days)
|
||||
if provider == "brave":
|
||||
return await self._search_brave(provider_config, query, limit, domains)
|
||||
if provider == "serpapi":
|
||||
return await self._search_serpapi(provider_config, query, limit)
|
||||
if provider == "exa":
|
||||
return await self._search_exa(provider_config, query, limit, domains)
|
||||
if provider == "firecrawl":
|
||||
return await self._search_firecrawl(provider_config, query, limit)
|
||||
if provider == "searxng":
|
||||
return await self._search_searxng(provider_config, query, limit, domains)
|
||||
raise WebSearchConfigurationError(f"Unsupported web search provider: {provider}")
|
||||
context = {
|
||||
"provider": provider,
|
||||
"query_hash": hashlib.sha256(query.encode("utf-8")).hexdigest(),
|
||||
"query_length": len(query),
|
||||
"max_results": limit,
|
||||
"domain_count": len(domains or []),
|
||||
"freshness_days": freshness_days,
|
||||
}
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_search.start",
|
||||
message="WebSearch request started",
|
||||
category="ai_tool",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context=context,
|
||||
)
|
||||
try:
|
||||
if provider == "tavily":
|
||||
results = await self._search_tavily(provider_config, query, limit, domains, freshness_days)
|
||||
elif provider == "brave":
|
||||
results = await self._search_brave(provider_config, query, limit, domains)
|
||||
elif provider == "serpapi":
|
||||
results = await self._search_serpapi(provider_config, query, limit)
|
||||
elif provider == "exa":
|
||||
results = await self._search_exa(provider_config, query, limit, domains)
|
||||
elif provider == "firecrawl":
|
||||
results = await self._search_firecrawl(provider_config, query, limit)
|
||||
elif provider == "searxng":
|
||||
results = await self._search_searxng(provider_config, query, limit, domains)
|
||||
else:
|
||||
raise WebSearchConfigurationError(f"Unsupported web search provider: {provider}")
|
||||
event = "ai_tool.web_search.success" if results else "ai_tool.web_search.empty"
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event=event,
|
||||
message="WebSearch request completed" if results else "WebSearch returned no results",
|
||||
category="ai_tool",
|
||||
level="info" if results else "warning",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context={
|
||||
**context,
|
||||
"status": "success" if results else "empty",
|
||||
"result_count": len(results),
|
||||
"duration_ms": int((perf_counter() - started_at) * 1000),
|
||||
},
|
||||
)
|
||||
return results
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai_tool.web_search.failed",
|
||||
message="WebSearch request failed",
|
||||
category="ai_tool",
|
||||
level="error",
|
||||
service="ai_tool",
|
||||
module=__name__,
|
||||
context=exception_context(exc, {**context, "status": "failed", "duration_ms": int((perf_counter() - started_at) * 1000)}),
|
||||
)
|
||||
raise
|
||||
|
||||
async def test_connection(self) -> list[SearchEvidence]:
|
||||
return await self.search("Planet WebSearch connectivity test", max_results=1)
|
||||
@@ -388,4 +474,3 @@ def _float_or_none(value: Any) -> float | None:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
117
backend/app/services/business_logs.py
Normal file
117
backend/app/services/business_logs.py
Normal file
@@ -0,0 +1,117 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from app.core.logging import PlanetLoggerAdapter, sanitize_log_value
|
||||
from app.core.request_context import get_request_id
|
||||
from app.services.persistent_logs import record_system_log
|
||||
|
||||
|
||||
LEVEL_METHODS = {
|
||||
"debug": "debug_event",
|
||||
"info": "info_event",
|
||||
"warning": "warning_event",
|
||||
"error": "error_event",
|
||||
}
|
||||
|
||||
|
||||
def normalize_business_level(level: str | None) -> str:
|
||||
normalized = str(level or "info").strip().lower()
|
||||
if normalized in {"warn", "warning"}:
|
||||
return "warning"
|
||||
if normalized in {"err", "error", "critical", "fatal"}:
|
||||
return "error"
|
||||
if normalized == "debug":
|
||||
return "debug"
|
||||
return "info"
|
||||
|
||||
|
||||
def build_business_context(
|
||||
context: Mapping[str, Any] | None = None,
|
||||
**fields: Any,
|
||||
) -> dict[str, Any]:
|
||||
payload = dict(context or {})
|
||||
for key, value in fields.items():
|
||||
if value is not None:
|
||||
payload[key] = value
|
||||
return sanitize_log_value(payload)
|
||||
|
||||
|
||||
async def emit_business_log(
|
||||
logger: PlanetLoggerAdapter,
|
||||
*,
|
||||
event: str,
|
||||
message: str,
|
||||
category: str,
|
||||
level: str = "info",
|
||||
source: str = "backend",
|
||||
service: str | None = None,
|
||||
module: str | None = None,
|
||||
request_id: str | None = None,
|
||||
user_id: int | None = None,
|
||||
context: Mapping[str, Any] | None = None,
|
||||
) -> None:
|
||||
normalized_level = normalize_business_level(level)
|
||||
safe_context = build_business_context(context)
|
||||
log_method = getattr(logger, LEVEL_METHODS[normalized_level])
|
||||
log_method(message, event=event, context=safe_context)
|
||||
await record_system_log(
|
||||
source=source,
|
||||
level=normalized_level,
|
||||
message=message,
|
||||
service=service,
|
||||
module=module,
|
||||
event=event,
|
||||
request_id=request_id or get_request_id(),
|
||||
user_id=user_id,
|
||||
category=category,
|
||||
context=safe_context,
|
||||
)
|
||||
|
||||
|
||||
def emit_business_log_background(
|
||||
logger: PlanetLoggerAdapter,
|
||||
*,
|
||||
event: str,
|
||||
message: str,
|
||||
category: str,
|
||||
level: str = "info",
|
||||
source: str = "backend",
|
||||
service: str | None = None,
|
||||
module: str | None = None,
|
||||
request_id: str | None = None,
|
||||
user_id: int | None = None,
|
||||
context: Mapping[str, Any] | None = None,
|
||||
) -> None:
|
||||
normalized_level = normalize_business_level(level)
|
||||
safe_context = build_business_context(context)
|
||||
log_method = getattr(logger, LEVEL_METHODS[normalized_level])
|
||||
log_method(message, event=event, context=safe_context)
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return
|
||||
loop.create_task(
|
||||
record_system_log(
|
||||
source=source,
|
||||
level=normalized_level,
|
||||
message=message,
|
||||
service=service,
|
||||
module=module,
|
||||
event=event,
|
||||
request_id=request_id or get_request_id(),
|
||||
user_id=user_id,
|
||||
category=category,
|
||||
context=safe_context,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def exception_context(exc: BaseException, context: Mapping[str, Any] | None = None) -> dict[str, Any]:
|
||||
return build_business_context(
|
||||
context,
|
||||
error_type=type(exc).__name__,
|
||||
error=str(exc),
|
||||
)
|
||||
@@ -4,42 +4,22 @@ import asyncio
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, List, Any, Optional
|
||||
from datetime import UTC, datetime
|
||||
from time import perf_counter
|
||||
from urllib.parse import urlparse
|
||||
import httpx
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.collected_data_fields import build_dynamic_metadata, get_record_field
|
||||
from app.core.countries import normalize_country
|
||||
from app.core.logging import get_logger
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
|
||||
|
||||
EARTH_UPDATE_LAYER_HINTS: dict[str, list[str]] = {
|
||||
"ris_live_bgp": ["bgp"],
|
||||
"bgpstream_bgp": ["bgp"],
|
||||
"top500_supercomputers": ["computeCenters"],
|
||||
"epoch_ai_gpu": ["computeCenters"],
|
||||
"huggingface_models": ["computeCenters"],
|
||||
"huggingface_datasets": ["computeCenters"],
|
||||
"huggingface_spaces": ["computeCenters"],
|
||||
"telegeography_cables": ["cables"],
|
||||
"telegeography_landing_points": ["cables"],
|
||||
"telegeography_cable_systems": ["cables"],
|
||||
"arcgis_cables": ["cables"],
|
||||
"fao_landing_points": ["cables"],
|
||||
"arcgis_landing_points": ["cables"],
|
||||
"arcgis_cable_landing_relations": ["cables"],
|
||||
"spacetrack_tle": ["satellites"],
|
||||
"celestrak_tle": ["satellites"],
|
||||
"barentswatch_vessels": ["vessels"],
|
||||
"aisstream_vessels": ["vessels"],
|
||||
"news_live_streams": ["media"],
|
||||
"media_news_archive": ["news"],
|
||||
}
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
from app.services.earth_layer_adapters import get_earth_update_layers_for_source
|
||||
|
||||
|
||||
def get_earth_update_layers_for_source(source: str) -> list[str]:
|
||||
return EARTH_UPDATE_LAYER_HINTS.get(source, [])
|
||||
logger = get_logger(__name__, service="collector")
|
||||
|
||||
|
||||
class BaseCollector(ABC):
|
||||
@@ -58,6 +38,7 @@ class BaseCollector(ABC):
|
||||
self._datasource_id = 1
|
||||
self._resolved_url: Optional[str] = None
|
||||
self._last_broadcast_progress: Optional[int] = None
|
||||
self._last_save_summary: dict[str, int] = {}
|
||||
|
||||
async def resolve_url(self, db: AsyncSession) -> None:
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
@@ -96,29 +77,6 @@ class BaseCollector(ABC):
|
||||
)
|
||||
self._last_broadcast_progress = rounded_progress
|
||||
|
||||
async def _publish_earth_update(
|
||||
self,
|
||||
*,
|
||||
action: str,
|
||||
records_processed: int,
|
||||
task_id: int | None = None,
|
||||
) -> None:
|
||||
layers = get_earth_update_layers_for_source(self.name)
|
||||
if not layers:
|
||||
return
|
||||
await broadcaster.broadcast_earth_update(
|
||||
{
|
||||
"action": action,
|
||||
"source": self.name,
|
||||
"data_type": self.data_type,
|
||||
"layers": layers,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"task_id": task_id,
|
||||
"records_processed": records_processed,
|
||||
"timestamp": to_iso8601_utc(datetime.now(UTC)),
|
||||
}
|
||||
)
|
||||
|
||||
async def update_progress(self, records_processed: int, *, commit: bool = False, force: bool = False):
|
||||
"""Update task progress - call this during data processing"""
|
||||
if self._current_task and self._db_session:
|
||||
@@ -322,19 +280,39 @@ class BaseCollector(ABC):
|
||||
from app.models.data_snapshot import DataSnapshot
|
||||
|
||||
start_time = datetime.now(UTC)
|
||||
started_at = perf_counter()
|
||||
datasource_id = getattr(self, "_datasource_id", 1)
|
||||
snapshot_id: Optional[int] = None
|
||||
|
||||
if not collector_registry.is_active(self.name):
|
||||
await self._log_collection_event(
|
||||
"collector.run.skipped_disabled",
|
||||
"Collector skipped because it is disabled",
|
||||
level="info",
|
||||
context={"status": "skipped", "reason": "disabled"},
|
||||
)
|
||||
return {"status": "skipped", "reason": "Collector is disabled"}
|
||||
|
||||
task = CollectionTask(
|
||||
datasource_id=datasource_id,
|
||||
status="running",
|
||||
phase="queued",
|
||||
started_at=start_time,
|
||||
)
|
||||
db.add(task)
|
||||
task = self._current_task if isinstance(self._current_task, CollectionTask) else None
|
||||
if task is None:
|
||||
task = CollectionTask(
|
||||
datasource_id=datasource_id,
|
||||
source=self.name,
|
||||
task_type="collect",
|
||||
status="running",
|
||||
phase="queued",
|
||||
started_at=start_time,
|
||||
)
|
||||
db.add(task)
|
||||
else:
|
||||
task.datasource_id = datasource_id
|
||||
task.source = task.source or self.name
|
||||
task.task_type = task.task_type or "collect"
|
||||
task.status = "running"
|
||||
task.phase = "queued"
|
||||
task.started_at = task.started_at or start_time
|
||||
task.completed_at = None
|
||||
task.error_message = None
|
||||
await db.commit()
|
||||
task_id = task.id
|
||||
|
||||
@@ -344,23 +322,76 @@ class BaseCollector(ABC):
|
||||
|
||||
await self.resolve_url(db)
|
||||
await self._publish_task_update(force=True)
|
||||
await self._log_collection_event(
|
||||
"collector.run.started",
|
||||
"Collector run started",
|
||||
context={"status": "running", "task_id": task_id},
|
||||
)
|
||||
|
||||
try:
|
||||
phase_started_at = perf_counter()
|
||||
await self.set_phase("fetching", message="正在拉取原始数据")
|
||||
await self._log_collection_event(
|
||||
"collector.phase.fetching.start",
|
||||
"Collector fetch phase started",
|
||||
context={"task_id": task_id, "snapshot_id": snapshot_id},
|
||||
)
|
||||
raw_data = await self.fetch()
|
||||
task.total_records = len(raw_data)
|
||||
await db.commit()
|
||||
await self._publish_task_update(force=True)
|
||||
await self._log_collection_event(
|
||||
"collector.phase.fetching.success",
|
||||
"Collector fetch phase completed",
|
||||
context={
|
||||
"task_id": task_id,
|
||||
"raw_count": len(raw_data),
|
||||
"duration_ms": self._duration_ms(phase_started_at),
|
||||
},
|
||||
)
|
||||
|
||||
if self.fail_on_empty and not raw_data:
|
||||
raise RuntimeError(f"Collector {self.name} returned no data")
|
||||
|
||||
phase_started_at = perf_counter()
|
||||
await self.set_phase("transforming", message="正在转换采集数据")
|
||||
await self._log_collection_event(
|
||||
"collector.phase.transforming.start",
|
||||
"Collector transform phase started",
|
||||
context={"task_id": task_id, "raw_count": len(raw_data)},
|
||||
)
|
||||
data = self.transform(raw_data)
|
||||
await self._log_collection_event(
|
||||
"collector.phase.transforming.success",
|
||||
"Collector transform phase completed",
|
||||
context={
|
||||
"task_id": task_id,
|
||||
"raw_count": len(raw_data),
|
||||
"transformed_count": len(data),
|
||||
"duration_ms": self._duration_ms(phase_started_at),
|
||||
},
|
||||
)
|
||||
snapshot_id = await self._create_snapshot(db, task_id, data, start_time)
|
||||
|
||||
phase_started_at = perf_counter()
|
||||
await self.set_phase("saving", message="正在保存采集数据")
|
||||
await self._log_collection_event(
|
||||
"collector.phase.saving.start",
|
||||
"Collector save phase started",
|
||||
context={"task_id": task_id, "snapshot_id": snapshot_id, "transformed_count": len(data)},
|
||||
)
|
||||
records_count = await self._save_data(db, data, task_id=task_id, snapshot_id=snapshot_id)
|
||||
await self._log_collection_event(
|
||||
"collector.phase.saving.success",
|
||||
"Collector save phase completed",
|
||||
context={
|
||||
"task_id": task_id,
|
||||
"snapshot_id": snapshot_id,
|
||||
"saved_count": records_count,
|
||||
**self._last_save_summary,
|
||||
"duration_ms": self._duration_ms(phase_started_at),
|
||||
},
|
||||
)
|
||||
|
||||
task.status = "success"
|
||||
task.phase = "completed"
|
||||
@@ -374,10 +405,19 @@ class BaseCollector(ABC):
|
||||
task.completed_at = datetime.now(UTC)
|
||||
await db.commit()
|
||||
await self._publish_task_update(force=True)
|
||||
await self._publish_earth_update(
|
||||
action="collector_completed",
|
||||
records_processed=records_count,
|
||||
task_id=task_id,
|
||||
await self._log_collection_event(
|
||||
"collector.run.completed",
|
||||
"Collector run completed",
|
||||
context={
|
||||
"status": "success",
|
||||
"task_id": task_id,
|
||||
"snapshot_id": snapshot_id,
|
||||
"raw_count": len(raw_data),
|
||||
"transformed_count": len(data),
|
||||
"saved_count": records_count,
|
||||
**self._last_save_summary,
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
@@ -402,6 +442,17 @@ class BaseCollector(ABC):
|
||||
)
|
||||
await db.commit()
|
||||
await self._publish_task_update(force=True)
|
||||
await self._log_collection_event(
|
||||
"collector.run.cancelled",
|
||||
"Collector run cancelled",
|
||||
level="warning",
|
||||
context={
|
||||
"status": "cancelled",
|
||||
"task_id": task_id,
|
||||
"snapshot_id": snapshot_id,
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
},
|
||||
)
|
||||
raise
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
@@ -418,6 +469,20 @@ class BaseCollector(ABC):
|
||||
snapshot.summary = {"error": str(e)}
|
||||
await db.commit()
|
||||
await self._publish_task_update(force=True)
|
||||
await self._log_collection_event(
|
||||
"collector.run.failed",
|
||||
"Collector run failed",
|
||||
level="error",
|
||||
context=exception_context(
|
||||
e,
|
||||
{
|
||||
"status": "failed",
|
||||
"task_id": task_id,
|
||||
"snapshot_id": snapshot_id,
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "failed",
|
||||
@@ -438,6 +503,7 @@ class BaseCollector(ABC):
|
||||
from app.models.data_snapshot import DataSnapshot
|
||||
|
||||
if not data:
|
||||
self._last_save_summary = {"created": 0, "updated": 0, "unchanged": 0, "deleted": 0}
|
||||
if snapshot_id is not None:
|
||||
snapshot = await db.get(DataSnapshot, snapshot_id)
|
||||
if snapshot:
|
||||
@@ -584,12 +650,51 @@ class BaseCollector(ABC):
|
||||
"unchanged": unchanged_count,
|
||||
"deleted": len(deleted_keys),
|
||||
}
|
||||
self._last_save_summary = {
|
||||
"created": created_count,
|
||||
"updated": updated_count,
|
||||
"unchanged": unchanged_count,
|
||||
"deleted": len(deleted_keys),
|
||||
}
|
||||
else:
|
||||
self._last_save_summary = {
|
||||
"created": created_count,
|
||||
"updated": updated_count,
|
||||
"unchanged": unchanged_count,
|
||||
"deleted": 0,
|
||||
}
|
||||
|
||||
await db.commit()
|
||||
invalidate_earth_layer_cache_for_source(self.name)
|
||||
await self.update_progress(len(data), force=True)
|
||||
return records_added
|
||||
|
||||
@staticmethod
|
||||
def _duration_ms(started_at: float) -> int:
|
||||
return int((perf_counter() - started_at) * 1000)
|
||||
|
||||
async def _log_collection_event(
|
||||
self,
|
||||
event: str,
|
||||
message: str,
|
||||
*,
|
||||
level: str = "info",
|
||||
context: Dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event=event,
|
||||
message=message,
|
||||
category="collector",
|
||||
level=level,
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
**(context or {}),
|
||||
},
|
||||
)
|
||||
|
||||
async def save(self, db: AsyncSession, data: List[Dict[str, Any]]) -> int:
|
||||
"""Save data to database (legacy method, use _save_data instead)"""
|
||||
return await self._save_data(db, data)
|
||||
@@ -602,10 +707,65 @@ class HTTPCollector(BaseCollector):
|
||||
headers: Dict[str, str] = {}
|
||||
|
||||
async def fetch(self) -> List[Dict[str, Any]]:
|
||||
started_at = perf_counter()
|
||||
request_host = urlparse(self.base_url).netloc
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.http.fetch.start",
|
||||
message="Collector HTTP request started",
|
||||
category="collector",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"url_host": request_host,
|
||||
},
|
||||
)
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
response = await client.get(self.base_url, headers=self.headers)
|
||||
response.raise_for_status()
|
||||
return self.parse_response(response.json())
|
||||
try:
|
||||
response = await client.get(self.base_url, headers=self.headers)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
parsed = self.parse_response(payload)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.http.fetch.success",
|
||||
message="Collector HTTP request completed",
|
||||
category="collector",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"url_host": request_host,
|
||||
"status_code": response.status_code,
|
||||
"response_bytes": len(response.content or b""),
|
||||
"parsed_count": len(parsed),
|
||||
"duration_ms": BaseCollector._duration_ms(started_at),
|
||||
},
|
||||
)
|
||||
return parsed
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.http.fetch.failed",
|
||||
message="Collector HTTP request failed",
|
||||
category="collector",
|
||||
level="error",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context=exception_context(
|
||||
exc,
|
||||
{
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"url_host": request_host,
|
||||
"duration_ms": BaseCollector._duration_ms(started_at),
|
||||
},
|
||||
),
|
||||
)
|
||||
raise
|
||||
|
||||
@abstractmethod
|
||||
def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
|
||||
@@ -1,15 +1,28 @@
|
||||
"""CelesTrak TLE Collector
|
||||
"""CelesTrak TLE Collector.
|
||||
|
||||
Collects satellite TLE (Two-Line Element) data from CelesTrak.org.
|
||||
Free, no authentication required.
|
||||
Collects the full active satellite GP element set from CelesTrak.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Dict, Any, List
|
||||
from pathlib import Path
|
||||
from time import perf_counter
|
||||
from typing import Any, Dict, List
|
||||
from urllib.parse import urlencode, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.core.satellite_tle import build_tle_lines_from_elements
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
from app.services.collectors.base import BaseCollector
|
||||
from app.services.collectors.downloads import ResumableFileDownloader
|
||||
|
||||
|
||||
logger = get_logger(__name__, service="collector")
|
||||
ACTIVE_GROUP = "active"
|
||||
FETCH_RETRY_ATTEMPTS = 3
|
||||
FETCH_RETRY_BASE_DELAY_SECONDS = 0.8
|
||||
|
||||
|
||||
class CelesTrakTLECollector(BaseCollector):
|
||||
@@ -18,55 +31,179 @@ class CelesTrakTLECollector(BaseCollector):
|
||||
module = "L3"
|
||||
frequency_hours = 24
|
||||
data_type = "satellite_tle"
|
||||
_downloader = ResumableFileDownloader(
|
||||
cache_namespace="celestrak",
|
||||
default_accept="application/json",
|
||||
)
|
||||
|
||||
@property
|
||||
def base_url(self) -> str:
|
||||
return self._resolved_url or ""
|
||||
|
||||
def _active_url(self) -> str:
|
||||
if not self.base_url:
|
||||
raise RuntimeError("CelesTrak base URL is not configured")
|
||||
return f"{self.base_url}?{urlencode({'GROUP': ACTIVE_GROUP, 'FORMAT': 'json'})}"
|
||||
|
||||
async def fetch(self) -> List[Dict[str, Any]]:
|
||||
satellite_groups = [
|
||||
"starlink",
|
||||
"gps-ops",
|
||||
"galileo",
|
||||
"glonass",
|
||||
"beidou",
|
||||
"leo",
|
||||
"geo",
|
||||
"iridium-next",
|
||||
]
|
||||
url = self._active_url()
|
||||
last_error: Exception | None = None
|
||||
|
||||
all_satellites = []
|
||||
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
for group in satellite_groups:
|
||||
async with httpx.AsyncClient(timeout=180.0, follow_redirects=True) as client:
|
||||
for attempt in range(1, FETCH_RETRY_ATTEMPTS + 1):
|
||||
started_at = perf_counter()
|
||||
try:
|
||||
url = f"{self.base_url}?GROUP={group}&FORMAT=json"
|
||||
response = await client.get(url)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.celestrak.download.start",
|
||||
message="CelesTrak active satellite download started",
|
||||
category="collector",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"group": ACTIVE_GROUP,
|
||||
"attempt": attempt,
|
||||
"url_host": urlparse(url).netloc,
|
||||
},
|
||||
)
|
||||
body_path = await self._downloader.download_file(
|
||||
client,
|
||||
url,
|
||||
extension=".json",
|
||||
accept="application/json",
|
||||
progress_callback=self._report_download_progress,
|
||||
validate_existing=self._validate_json_file,
|
||||
)
|
||||
try:
|
||||
data = self._load_active_payload(body_path)
|
||||
except RuntimeError as exc:
|
||||
await self._log_parse_failure(exc)
|
||||
raise
|
||||
for item in data:
|
||||
item["_celestrak_query_group"] = ACTIVE_GROUP
|
||||
item["_celestrak_source_url"] = url
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.celestrak.download.success",
|
||||
message="CelesTrak active satellite download completed",
|
||||
category="collector",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"group": ACTIVE_GROUP,
|
||||
"attempt": attempt,
|
||||
"record_count": len(data),
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
},
|
||||
)
|
||||
return data
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
is_final_attempt = attempt >= FETCH_RETRY_ATTEMPTS
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event=(
|
||||
"collector.celestrak.download.failed"
|
||||
if is_final_attempt
|
||||
else "collector.celestrak.download.retry"
|
||||
),
|
||||
message=(
|
||||
"CelesTrak active satellite download failed"
|
||||
if is_final_attempt
|
||||
else "CelesTrak active satellite download will retry"
|
||||
),
|
||||
category="collector",
|
||||
level="error" if is_final_attempt else "warning",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context=exception_context(
|
||||
exc,
|
||||
{
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"group": ACTIVE_GROUP,
|
||||
"attempt": attempt,
|
||||
"duration_ms": self._duration_ms(started_at),
|
||||
},
|
||||
),
|
||||
)
|
||||
if not is_final_attempt:
|
||||
await asyncio.sleep(FETCH_RETRY_BASE_DELAY_SECONDS * attempt)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if isinstance(data, list):
|
||||
for item in data:
|
||||
if isinstance(item, dict):
|
||||
item["_celestrak_group"] = group
|
||||
all_satellites.extend(data)
|
||||
print(f"CelesTrak: Fetched {len(data)} satellites from group '{group}'")
|
||||
except Exception as e:
|
||||
print(f"CelesTrak: Error fetching group '{group}': {e}")
|
||||
raise RuntimeError(f"CelesTrak active satellite download failed after retries: {last_error}")
|
||||
|
||||
if not all_satellites:
|
||||
return self._get_sample_data()
|
||||
@staticmethod
|
||||
def _duration_ms(started_at: float) -> int:
|
||||
return int((perf_counter() - started_at) * 1000)
|
||||
|
||||
print(f"CelesTrak: Total satellites fetched: {len(all_satellites)}")
|
||||
async def _report_download_progress(self, downloaded: int, total: int | None) -> None:
|
||||
if total and total > 0:
|
||||
await self.update_phase_progress(
|
||||
current=min(downloaded, total),
|
||||
total=total,
|
||||
unit="bytes",
|
||||
message=f"正在下载 CelesTrak active 卫星数据 {downloaded}/{total} bytes",
|
||||
commit=True,
|
||||
)
|
||||
|
||||
# Return raw data - base.run() will call transform()
|
||||
return all_satellites
|
||||
@staticmethod
|
||||
def _validate_json_file(path: Path) -> bool:
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError, UnicodeDecodeError):
|
||||
return False
|
||||
return isinstance(data, list)
|
||||
|
||||
async def _log_parse_failure(self, exc: Exception) -> None:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.celestrak.parse.failed",
|
||||
message="CelesTrak active satellite JSON parsing failed",
|
||||
category="collector",
|
||||
level="error",
|
||||
service="collector",
|
||||
module=__name__,
|
||||
context=exception_context(
|
||||
exc,
|
||||
{
|
||||
"collector_name": self.name,
|
||||
"datasource_id": getattr(self, "_datasource_id", None),
|
||||
"group": ACTIVE_GROUP,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
def _load_active_payload(self, path: Path) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError, UnicodeDecodeError) as exc:
|
||||
raise RuntimeError(f"CelesTrak active payload is not valid JSON: {exc}") from exc
|
||||
if not isinstance(raw, list):
|
||||
raise RuntimeError("CelesTrak active payload is not a JSON array")
|
||||
|
||||
records: List[Dict[str, Any]] = []
|
||||
invalid_count = 0
|
||||
for item in raw:
|
||||
if isinstance(item, dict) and item.get("NORAD_CAT_ID") is not None:
|
||||
records.append(item)
|
||||
else:
|
||||
invalid_count += 1
|
||||
if invalid_count:
|
||||
raise RuntimeError(f"CelesTrak active payload contains {invalid_count} invalid record(s)")
|
||||
if not records:
|
||||
raise RuntimeError("CelesTrak active payload contains no satellite records")
|
||||
return records
|
||||
|
||||
def transform(self, raw_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
transformed = []
|
||||
for item in raw_data:
|
||||
norad_cat_id = item.get("NORAD_CAT_ID")
|
||||
tle_line1, tle_line2 = build_tle_lines_from_elements(
|
||||
norad_cat_id=item.get("NORAD_CAT_ID"),
|
||||
norad_cat_id=norad_cat_id,
|
||||
epoch=item.get("EPOCH"),
|
||||
inclination=item.get("INCLINATION"),
|
||||
raan=item.get("RA_OF_ASC_NODE"),
|
||||
@@ -75,14 +212,18 @@ class CelesTrakTLECollector(BaseCollector):
|
||||
mean_anomaly=item.get("MEAN_ANOMALY"),
|
||||
mean_motion=item.get("MEAN_MOTION"),
|
||||
)
|
||||
constellation_group = self._infer_constellation_group(item)
|
||||
|
||||
transformed.append(
|
||||
{
|
||||
"source_id": str(norad_cat_id),
|
||||
"name": item.get("OBJECT_NAME", "Unknown"),
|
||||
"reference_date": item.get("EPOCH", ""),
|
||||
"metadata": {
|
||||
"constellation_group": item.get("_celestrak_group"),
|
||||
"norad_cat_id": item.get("NORAD_CAT_ID"),
|
||||
"constellation_group": constellation_group,
|
||||
"celestrak_query_group": item.get("_celestrak_query_group") or ACTIVE_GROUP,
|
||||
"celestrak_source_url": item.get("_celestrak_source_url"),
|
||||
"norad_cat_id": norad_cat_id,
|
||||
"international_designator": item.get("OBJECT_ID"),
|
||||
"epoch": item.get("EPOCH"),
|
||||
"mean_motion": item.get("MEAN_MOTION"),
|
||||
@@ -105,6 +246,19 @@ class CelesTrakTLECollector(BaseCollector):
|
||||
)
|
||||
return transformed
|
||||
|
||||
@staticmethod
|
||||
def _infer_constellation_group(item: Dict[str, Any]) -> str | None:
|
||||
explicit_group = str(item.get("_celestrak_group") or "").strip().lower()
|
||||
if explicit_group and explicit_group != ACTIVE_GROUP:
|
||||
return explicit_group
|
||||
|
||||
name = str(item.get("OBJECT_NAME") or "").strip().upper()
|
||||
if name.startswith("STARLINK"):
|
||||
return "starlink"
|
||||
if name.startswith("IRIDIUM"):
|
||||
return "iridium-next"
|
||||
return None
|
||||
|
||||
def _get_sample_data(self) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
|
||||
@@ -163,32 +163,64 @@ class TeleGeographyLandingPointCollector(BaseCollector):
|
||||
data_type = "landing_point"
|
||||
|
||||
async def fetch(self) -> List[Dict[str, Any]]:
|
||||
"""Fetch landing point data from GitHub mirror"""
|
||||
url = self._resolved_url or ""
|
||||
"""Fetch landing point data, falling back when the old mirror disappears."""
|
||||
config = get_data_sources_config()
|
||||
sources = [
|
||||
self._resolved_url or "",
|
||||
str(config.get_yaml_value("telegeography.landing_point_url") or ""),
|
||||
str(config.get_yaml_value("arcgis.landing_point_url") or ""),
|
||||
]
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
response = await client.get(url)
|
||||
response.raise_for_status()
|
||||
return self.parse_response(response.json())
|
||||
last_error: Exception | None = None
|
||||
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
|
||||
for url in dict.fromkeys(source for source in sources if source):
|
||||
try:
|
||||
params = (
|
||||
{"where": "1=1", "outFields": "*", "returnGeometry": "true", "f": "geojson"}
|
||||
if "FeatureServer" in url or url.endswith("/query")
|
||||
else None
|
||||
)
|
||||
response = await client.get(url, params=params)
|
||||
response.raise_for_status()
|
||||
records = self.parse_response(response.json())
|
||||
if records:
|
||||
return records
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
continue
|
||||
|
||||
def parse_response(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||
if last_error:
|
||||
raise last_error
|
||||
return self._get_sample_data()
|
||||
|
||||
def parse_response(self, data: List[Dict[str, Any]] | Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""Parse landing point data"""
|
||||
result = []
|
||||
items = data.get("features", []) if isinstance(data, dict) else data
|
||||
|
||||
for item in data:
|
||||
for item in items:
|
||||
props = item.get("properties", {}) if isinstance(item, dict) else {}
|
||||
geometry = item.get("geometry", {}) if isinstance(item, dict) else {}
|
||||
source = props or item
|
||||
coords = geometry.get("coordinates", []) if isinstance(geometry, dict) else []
|
||||
longitude = coords[0] if len(coords) > 0 else source.get("longitude")
|
||||
latitude = coords[1] if len(coords) > 1 else source.get("latitude")
|
||||
source_id = source.get("id") or source.get("OBJECTID") or source.get("city_id") or ""
|
||||
try:
|
||||
entry = {
|
||||
"source_id": f"telegeo_lp_{item.get('id', '')}",
|
||||
"name": item.get("name", "Unknown"),
|
||||
"country": item.get("country", "Unknown"),
|
||||
"city": item.get("city", item.get("name", "")),
|
||||
"latitude": str(item.get("latitude", "")),
|
||||
"longitude": str(item.get("longitude", "")),
|
||||
"source_id": f"telegeo_lp_{source_id}",
|
||||
"name": source.get("name", source.get("Name", "Unknown")),
|
||||
"country": source.get("country", "Unknown"),
|
||||
"city": source.get("city", source.get("Name", source.get("name", ""))),
|
||||
"latitude": str(latitude or ""),
|
||||
"longitude": str(longitude or ""),
|
||||
"value": "",
|
||||
"unit": "",
|
||||
"metadata": {
|
||||
"cable_count": len(item.get("cables", [])),
|
||||
"url": item.get("url"),
|
||||
"cable_count": len(source.get("cables", [])),
|
||||
"url": source.get("url"),
|
||||
"objectid": source.get("OBJECTID"),
|
||||
"city_id": source.get("city_id"),
|
||||
},
|
||||
"reference_date": datetime.now(UTC).strftime("%Y-%m-%d"),
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ startup, so it must stay local and deterministic.
|
||||
|
||||
For the full design and the reason behind the abstraction (compute centers,
|
||||
BGP collectors, BGP events, and future entities all share one pipeline),
|
||||
see ``docs/plans/location-resolver-shared-pipeline-plan.md``.
|
||||
see ``docs/technical/zh/location-pipeline-development.md``.
|
||||
|
||||
The ``ComputeCenterLocation`` dataclass and the public function signatures are
|
||||
preserved verbatim so existing callers and tests do not need to change.
|
||||
|
||||
597
backend/app/services/data_jobs.py
Normal file
597
backend/app/services/data_jobs.py
Normal file
@@ -0,0 +1,597 @@
|
||||
"""Kafka-ready datasource job queue backed by PostgreSQL for v1."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from uuid import uuid4
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import bindparam, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.cache import cache
|
||||
from app.core.config import settings
|
||||
from app.core.logging import get_logger
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
from app.db.session import async_session_factory
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.models.data_snapshot import DataSnapshot
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.task import CollectionTask
|
||||
from app.services.collectors.registry import collector_registry
|
||||
from app.services.datasource_connectivity import (
|
||||
build_builtin_connectivity_checksum,
|
||||
get_builtin_effective_candidate,
|
||||
save_connectivity_success,
|
||||
)
|
||||
from app.services.earth_layer_adapters import (
|
||||
clear_derived_datasource_data,
|
||||
get_earth_refresh_strategy_for_change,
|
||||
get_earth_update_layers_for_source,
|
||||
)
|
||||
from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
|
||||
from app.services.scheduler import sync_datasource_job
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
JOB_TYPE_COLLECT = "collect"
|
||||
JOB_TYPE_CLEAR_DATA = "clear_data"
|
||||
JOB_TYPE_CLEAR_CACHE = "clear_cache"
|
||||
JOB_TYPE_EARTH_REFRESH = "earth_refresh"
|
||||
|
||||
JOB_STATUS_QUEUED = "queued"
|
||||
JOB_STATUS_RUNNING = "running"
|
||||
JOB_STATUS_CANCELLING = "cancelling"
|
||||
JOB_STATUS_SUCCESS = "success"
|
||||
JOB_STATUS_FAILED = "failed"
|
||||
JOB_STATUS_CANCELLED = "cancelled"
|
||||
|
||||
ACTIVE_JOB_STATUSES = (JOB_STATUS_QUEUED, JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING)
|
||||
TERMINAL_JOB_STATUSES = (JOB_STATUS_SUCCESS, JOB_STATUS_FAILED, JOB_STATUS_CANCELLED)
|
||||
DATA_WRITE_JOB_TYPES = (JOB_TYPE_COLLECT, JOB_TYPE_CLEAR_DATA, JOB_TYPE_CLEAR_CACHE)
|
||||
SOURCE_LOCK_JOB_STATUSES = (JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING)
|
||||
QUEUE_POLL_SECONDS = 0.35
|
||||
JOB_STALE_LOCK_MINUTES = 90
|
||||
DEFAULT_WORKER_CONCURRENCY = 2
|
||||
|
||||
RUNNING_DATA_JOB_TASKS: dict[int, asyncio.Task[Any]] = {}
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _job_worker_id() -> str:
|
||||
return f"{settings.PROJECT_NAME}:data-job-worker:{uuid4().hex[:8]}"
|
||||
|
||||
|
||||
def is_terminal_job_status(status: str | None) -> bool:
|
||||
return status in TERMINAL_JOB_STATUSES
|
||||
|
||||
|
||||
async def enqueue_datasource_job(
|
||||
db: AsyncSession,
|
||||
datasource: DataSource,
|
||||
task_type: str,
|
||||
*,
|
||||
payload: dict[str, Any] | None = None,
|
||||
rollback_policy: str = "keep_committed_batches",
|
||||
dedupe_key: str | None = None,
|
||||
) -> CollectionTask:
|
||||
if dedupe_key:
|
||||
existing = await _get_active_job_by_dedupe_key(db, dedupe_key)
|
||||
if existing is not None:
|
||||
return existing
|
||||
|
||||
task = CollectionTask(
|
||||
datasource_id=datasource.id,
|
||||
source=datasource.source,
|
||||
task_type=task_type,
|
||||
status=JOB_STATUS_QUEUED,
|
||||
phase="queued",
|
||||
phase_message="任务已进入队列",
|
||||
payload=payload or {},
|
||||
rollback_policy=rollback_policy,
|
||||
dedupe_key=dedupe_key,
|
||||
)
|
||||
db.add(task)
|
||||
await db.commit()
|
||||
await db.refresh(task)
|
||||
await _broadcast_task_update(task)
|
||||
return task
|
||||
|
||||
|
||||
async def enqueue_earth_refresh_job(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
source: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
) -> CollectionTask | None:
|
||||
layers = list((payload or {}).get("layers") or get_earth_update_layers_for_source(source))
|
||||
if not layers:
|
||||
return None
|
||||
|
||||
datasource = await _get_or_create_virtual_datasource(db, source)
|
||||
refresh_payload = {
|
||||
"source": source,
|
||||
"layers": layers,
|
||||
"refresh_strategy": (payload or {}).get("refresh_strategy")
|
||||
or get_earth_refresh_strategy_for_change((payload or {}).get("table"), source)
|
||||
or "clear_then_reload",
|
||||
**(payload or {}),
|
||||
}
|
||||
return await enqueue_datasource_job(
|
||||
db,
|
||||
datasource,
|
||||
JOB_TYPE_EARTH_REFRESH,
|
||||
payload=refresh_payload,
|
||||
dedupe_key=f"earth_refresh:{source}",
|
||||
)
|
||||
|
||||
|
||||
async def enqueue_earth_refresh_from_update(payload: dict[str, Any]) -> None:
|
||||
source = str(payload.get("source") or "").strip()
|
||||
if not source:
|
||||
return
|
||||
async with async_session_factory() as db:
|
||||
await enqueue_earth_refresh_job(db, source=source, payload=payload)
|
||||
|
||||
|
||||
async def request_cancel_datasource_task(
|
||||
db: AsyncSession,
|
||||
task: CollectionTask,
|
||||
*,
|
||||
reason: str = "cancelled_by_operator",
|
||||
) -> CollectionTask:
|
||||
if is_terminal_job_status(task.status):
|
||||
return task
|
||||
|
||||
running_task = RUNNING_DATA_JOB_TASKS.get(task.id)
|
||||
if task.status == JOB_STATUS_QUEUED or (
|
||||
running_task is None
|
||||
and (
|
||||
task.status == JOB_STATUS_CANCELLING
|
||||
or (task.status == JOB_STATUS_RUNNING and task.task_type != JOB_TYPE_COLLECT)
|
||||
)
|
||||
):
|
||||
return await _cancel_task_without_runner(db, task, reason=reason)
|
||||
|
||||
task.status = JOB_STATUS_CANCELLING
|
||||
task.phase = JOB_STATUS_CANCELLING
|
||||
task.phase_message = "正在停止任务"
|
||||
task.requested_cancel_at = _utcnow()
|
||||
task.cancel_reason = reason
|
||||
await db.commit()
|
||||
await db.refresh(task)
|
||||
|
||||
if running_task is not None and not running_task.done():
|
||||
running_task.cancel()
|
||||
|
||||
await _broadcast_task_update(task)
|
||||
return task
|
||||
|
||||
|
||||
async def _cancel_task_without_runner(
|
||||
db: AsyncSession,
|
||||
task: CollectionTask,
|
||||
*,
|
||||
reason: str,
|
||||
) -> CollectionTask:
|
||||
if task.task_type == JOB_TYPE_COLLECT:
|
||||
await db.execute(CollectedData.__table__.delete().where(CollectedData.task_id == task.id))
|
||||
snapshot_result = await db.execute(select(DataSnapshot).where(DataSnapshot.task_id == task.id))
|
||||
for snapshot in snapshot_result.scalars().all():
|
||||
snapshot.status = JOB_STATUS_CANCELLED
|
||||
snapshot.completed_at = _utcnow()
|
||||
snapshot.error_message = "Cancelled after operator stop request; no active worker handle remained"
|
||||
datasource = await db.get(DataSource, task.datasource_id)
|
||||
if datasource is not None:
|
||||
datasource.last_status = JOB_STATUS_CANCELLED
|
||||
|
||||
task.status = JOB_STATUS_CANCELLED
|
||||
task.phase = JOB_STATUS_CANCELLED
|
||||
task.phase_message = "任务已停止"
|
||||
task.completed_at = _utcnow()
|
||||
task.requested_cancel_at = task.requested_cancel_at or _utcnow()
|
||||
task.cancel_reason = reason
|
||||
await db.commit()
|
||||
await db.refresh(task)
|
||||
await _broadcast_task_update(task)
|
||||
return task
|
||||
|
||||
|
||||
async def get_active_datasource_job(
|
||||
db: AsyncSession,
|
||||
datasource_id: int,
|
||||
*,
|
||||
task_types: tuple[str, ...] = DATA_WRITE_JOB_TYPES,
|
||||
) -> CollectionTask | None:
|
||||
result = await db.execute(
|
||||
select(CollectionTask)
|
||||
.where(CollectionTask.datasource_id == datasource_id)
|
||||
.where(CollectionTask.task_type.in_(task_types))
|
||||
.where(CollectionTask.status.in_(ACTIVE_JOB_STATUSES))
|
||||
.order_by(CollectionTask.created_at.desc().nullslast(), CollectionTask.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _get_active_job_by_dedupe_key(db: AsyncSession, dedupe_key: str) -> CollectionTask | None:
|
||||
result = await db.execute(
|
||||
select(CollectionTask)
|
||||
.where(CollectionTask.dedupe_key == dedupe_key)
|
||||
.where(CollectionTask.status.in_(ACTIVE_JOB_STATUSES))
|
||||
.order_by(CollectionTask.created_at.desc().nullslast(), CollectionTask.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def _get_or_create_virtual_datasource(db: AsyncSession, source: str) -> DataSource:
|
||||
result = await db.execute(select(DataSource).where(DataSource.source == source))
|
||||
datasource = result.scalar_one_or_none()
|
||||
if datasource is not None:
|
||||
return datasource
|
||||
|
||||
datasource = DataSource(
|
||||
name=f"Earth refresh: {source}",
|
||||
source=source,
|
||||
module="SYS",
|
||||
collector_class="EarthRefreshJob",
|
||||
is_active=True,
|
||||
)
|
||||
db.add(datasource)
|
||||
await db.commit()
|
||||
await db.refresh(datasource)
|
||||
return datasource
|
||||
|
||||
|
||||
async def _broadcast_task_update(task: CollectionTask) -> None:
|
||||
await broadcaster.broadcast_datasource_task_update(
|
||||
{
|
||||
"datasource_id": task.datasource_id,
|
||||
"collector_name": task.source,
|
||||
"task_id": task.id,
|
||||
"task_type": task.task_type,
|
||||
"status": task.status,
|
||||
"phase": task.phase,
|
||||
"phase_progress": task.phase_progress,
|
||||
"phase_message": task.phase_message,
|
||||
"phase_current": task.phase_current,
|
||||
"phase_total": task.phase_total,
|
||||
"phase_unit": task.phase_unit,
|
||||
"progress": task.progress,
|
||||
"records_processed": task.records_processed,
|
||||
"total_records": task.total_records,
|
||||
"started_at": to_iso8601_utc(task.started_at),
|
||||
"completed_at": to_iso8601_utc(task.completed_at),
|
||||
"requested_cancel_at": to_iso8601_utc(task.requested_cancel_at),
|
||||
"error_message": task.error_message,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class DataJobWorker:
|
||||
def __init__(self, *, concurrency: int = DEFAULT_WORKER_CONCURRENCY) -> None:
|
||||
self.worker_id = _job_worker_id()
|
||||
self.concurrency = max(1, concurrency)
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._stop_event: asyncio.Event | None = None
|
||||
self._running: set[asyncio.Task[Any]] = set()
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task and not self._task.done():
|
||||
return
|
||||
self._stop_event = asyncio.Event()
|
||||
self._task = asyncio.create_task(self._run(), name="data-job-worker")
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._stop_event:
|
||||
self._stop_event.set()
|
||||
for task in list(self._running):
|
||||
task.cancel()
|
||||
if self._task:
|
||||
await asyncio.gather(self._task, return_exceptions=True)
|
||||
if self._running:
|
||||
await asyncio.gather(*self._running, return_exceptions=True)
|
||||
|
||||
async def _run(self) -> None:
|
||||
assert self._stop_event is not None
|
||||
await self._recover_stale_running_jobs()
|
||||
while not self._stop_event.is_set():
|
||||
self._running = {task for task in self._running if not task.done()}
|
||||
if len(self._running) >= self.concurrency:
|
||||
await asyncio.sleep(QUEUE_POLL_SECONDS)
|
||||
continue
|
||||
|
||||
task_id = await self._claim_next_job()
|
||||
if task_id is None:
|
||||
await asyncio.sleep(QUEUE_POLL_SECONDS)
|
||||
continue
|
||||
|
||||
runner = asyncio.create_task(self._run_claimed_job(task_id), name=f"data-job:{task_id}")
|
||||
self._running.add(runner)
|
||||
|
||||
async def _recover_stale_running_jobs(self) -> None:
|
||||
cutoff = _utcnow() - timedelta(minutes=JOB_STALE_LOCK_MINUTES)
|
||||
async with async_session_factory() as db:
|
||||
result = await db.execute(
|
||||
select(CollectionTask)
|
||||
.where(CollectionTask.status.in_((JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING)))
|
||||
.where(CollectionTask.locked_at.is_not(None))
|
||||
.where(CollectionTask.locked_at < cutoff)
|
||||
)
|
||||
stale_jobs = list(result.scalars().all())
|
||||
for job in stale_jobs:
|
||||
job.status = JOB_STATUS_FAILED
|
||||
job.phase = JOB_STATUS_FAILED
|
||||
job.completed_at = _utcnow()
|
||||
job.error_message = "Marked failed after stale data job lock timeout"
|
||||
if stale_jobs:
|
||||
await db.commit()
|
||||
|
||||
async def _claim_next_job(self) -> int | None:
|
||||
async with async_session_factory() as db:
|
||||
row = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT queued.id
|
||||
FROM collection_tasks AS queued
|
||||
WHERE queued.status = :queued_status
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM collection_tasks AS active
|
||||
WHERE active.source = queued.source
|
||||
AND active.id <> queued.id
|
||||
AND active.status IN :active_statuses
|
||||
)
|
||||
ORDER BY queued.created_at ASC NULLS FIRST, queued.id ASC
|
||||
LIMIT 1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
"""
|
||||
).bindparams(bindparam("active_statuses", expanding=True)),
|
||||
{
|
||||
"queued_status": JOB_STATUS_QUEUED,
|
||||
"active_statuses": SOURCE_LOCK_JOB_STATUSES,
|
||||
},
|
||||
)
|
||||
task_id = row.scalar_one_or_none()
|
||||
if task_id is None:
|
||||
return None
|
||||
|
||||
task = await db.get(CollectionTask, int(task_id))
|
||||
if task is None:
|
||||
return None
|
||||
task.status = JOB_STATUS_RUNNING
|
||||
task.phase = "starting"
|
||||
task.phase_message = "任务开始执行"
|
||||
task.started_at = task.started_at or _utcnow()
|
||||
task.worker_id = self.worker_id
|
||||
task.locked_at = _utcnow()
|
||||
await db.commit()
|
||||
await _broadcast_task_update(task)
|
||||
return int(task_id)
|
||||
|
||||
async def _run_claimed_job(self, task_id: int) -> None:
|
||||
current_task = asyncio.current_task()
|
||||
if current_task is not None:
|
||||
RUNNING_DATA_JOB_TASKS[task_id] = current_task
|
||||
try:
|
||||
async with async_session_factory() as db:
|
||||
task = await db.get(CollectionTask, task_id)
|
||||
if task is None:
|
||||
return
|
||||
await self._execute_job(db, task)
|
||||
except asyncio.CancelledError:
|
||||
async with async_session_factory() as db:
|
||||
task = await db.get(CollectionTask, task_id)
|
||||
if task is not None and not is_terminal_job_status(task.status):
|
||||
task.status = JOB_STATUS_CANCELLED
|
||||
task.phase = JOB_STATUS_CANCELLED
|
||||
task.phase_message = "任务已停止"
|
||||
task.completed_at = _utcnow()
|
||||
await db.commit()
|
||||
await _broadcast_task_update(task)
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.exception_event(
|
||||
"Data job failed",
|
||||
event="data_jobs.job_failed",
|
||||
context={"task_id": task_id, "error": str(exc)},
|
||||
)
|
||||
async with async_session_factory() as db:
|
||||
task = await db.get(CollectionTask, task_id)
|
||||
if task is not None:
|
||||
task.status = JOB_STATUS_FAILED
|
||||
task.phase = JOB_STATUS_FAILED
|
||||
task.phase_message = str(exc)
|
||||
task.error_message = str(exc)
|
||||
task.completed_at = _utcnow()
|
||||
await db.commit()
|
||||
await _broadcast_task_update(task)
|
||||
finally:
|
||||
RUNNING_DATA_JOB_TASKS.pop(task_id, None)
|
||||
|
||||
async def _execute_job(self, db: AsyncSession, task: CollectionTask) -> None:
|
||||
if task.task_type == JOB_TYPE_COLLECT:
|
||||
await _run_collect_job(db, task)
|
||||
elif task.task_type == JOB_TYPE_CLEAR_DATA:
|
||||
await _run_clear_data_job(db, task)
|
||||
elif task.task_type == JOB_TYPE_CLEAR_CACHE:
|
||||
await _run_clear_cache_job(db, task)
|
||||
elif task.task_type == JOB_TYPE_EARTH_REFRESH:
|
||||
await _run_earth_refresh_job(db, task)
|
||||
else:
|
||||
raise RuntimeError(f"Unsupported data job type: {task.task_type}")
|
||||
|
||||
|
||||
async def _run_collect_job(db: AsyncSession, task: CollectionTask) -> None:
|
||||
datasource = await db.get(DataSource, task.datasource_id)
|
||||
if datasource is None:
|
||||
raise RuntimeError("Data source not found")
|
||||
|
||||
collector = collector_registry.get(datasource.source)
|
||||
if collector is None:
|
||||
raise RuntimeError(f"Collector '{datasource.source}' not found")
|
||||
if not datasource.is_active:
|
||||
raise RuntimeError("Data source is disabled")
|
||||
|
||||
collector._datasource_id = datasource.id
|
||||
collector._current_task = task
|
||||
collector._db_session = db
|
||||
result = await collector.run(db)
|
||||
|
||||
datasource.last_run_at = _utcnow()
|
||||
datasource.last_status = result.get("status")
|
||||
if datasource.last_status == JOB_STATUS_SUCCESS:
|
||||
effective_candidate = await get_builtin_effective_candidate(db, datasource.source)
|
||||
checksum, _credential_context = await build_builtin_connectivity_checksum(
|
||||
datasource.source,
|
||||
effective_candidate["endpoint"],
|
||||
effective_candidate["auth_type"],
|
||||
effective_candidate["headers"],
|
||||
effective_candidate["config"],
|
||||
db,
|
||||
)
|
||||
await save_connectivity_success(
|
||||
db,
|
||||
datasource.source,
|
||||
checksum,
|
||||
{"status_code": None},
|
||||
connected_by="collection",
|
||||
)
|
||||
await db.commit()
|
||||
await sync_datasource_job(datasource.id)
|
||||
|
||||
|
||||
async def _run_clear_data_job(db: AsyncSession, task: CollectionTask) -> None:
|
||||
source = str(task.source or (task.payload or {}).get("source") or "").strip()
|
||||
if not source:
|
||||
raise RuntimeError("Clear data job has no source")
|
||||
|
||||
task.phase = "clearing_data"
|
||||
task.phase_message = "正在删除数据库数据"
|
||||
await db.commit()
|
||||
await _broadcast_task_update(task)
|
||||
|
||||
count_result = await db.execute(
|
||||
select(CollectedData.id).where(CollectedData.source == source)
|
||||
)
|
||||
collected_ids = [row[0] for row in count_result.all()]
|
||||
derived_deleted_counts = await clear_derived_datasource_data(db, source)
|
||||
if collected_ids:
|
||||
await db.execute(CollectedData.__table__.delete().where(CollectedData.id.in_(collected_ids)))
|
||||
deleted_count = len(collected_ids)
|
||||
derived_deleted_count = sum(derived_deleted_counts.values())
|
||||
|
||||
task.records_processed = deleted_count + derived_deleted_count
|
||||
task.total_records = task.records_processed
|
||||
task.progress = 100.0
|
||||
task.phase_progress = 100.0
|
||||
task.phase_current = task.records_processed
|
||||
task.phase_total = task.records_processed
|
||||
task.phase_unit = "records"
|
||||
task.payload = {
|
||||
**(task.payload or {}),
|
||||
"deleted_count": deleted_count,
|
||||
"derived_deleted_count": derived_deleted_count,
|
||||
"derived_deleted_counts": derived_deleted_counts,
|
||||
}
|
||||
task.status = JOB_STATUS_SUCCESS
|
||||
task.phase = "completed"
|
||||
task.phase_message = "数据库数据已清理"
|
||||
task.completed_at = _utcnow()
|
||||
await db.commit()
|
||||
await _broadcast_task_update(task)
|
||||
|
||||
|
||||
async def _run_clear_cache_job(db: AsyncSession, task: CollectionTask) -> None:
|
||||
source = str(task.source or (task.payload or {}).get("source") or "").strip()
|
||||
if not source:
|
||||
raise RuntimeError("Clear cache job has no source")
|
||||
|
||||
earth_deleted_count = invalidate_earth_layer_cache_for_source(source)
|
||||
dashboard_deleted_count = int(cache.delete("dashboard:stats")) + int(cache.delete("dashboard:summary"))
|
||||
deleted_count = earth_deleted_count + dashboard_deleted_count
|
||||
|
||||
task.records_processed = deleted_count
|
||||
task.total_records = deleted_count
|
||||
task.progress = 100.0
|
||||
task.phase_progress = 100.0
|
||||
task.phase = "completed"
|
||||
task.phase_message = "缓存已清理"
|
||||
task.payload = {
|
||||
**(task.payload or {}),
|
||||
"earth_layer_deleted_count": earth_deleted_count,
|
||||
"dashboard_deleted_count": dashboard_deleted_count,
|
||||
}
|
||||
task.status = JOB_STATUS_SUCCESS
|
||||
task.completed_at = _utcnow()
|
||||
await db.commit()
|
||||
await _broadcast_task_update(task)
|
||||
await enqueue_earth_refresh_job(db, source=source, payload={"operation": "CACHE_INVALIDATED"})
|
||||
|
||||
|
||||
async def _run_earth_refresh_job(db: AsyncSession, task: CollectionTask) -> None:
|
||||
payload = task.payload or {}
|
||||
source = str(payload.get("source") or task.source or "").strip()
|
||||
layers = list(payload.get("layers") or get_earth_update_layers_for_source(source))
|
||||
if not source or not layers:
|
||||
task.status = JOB_STATUS_SUCCESS
|
||||
task.phase = "completed"
|
||||
task.phase_message = "没有需要刷新的 Earth 图层"
|
||||
task.completed_at = _utcnow()
|
||||
await db.commit()
|
||||
await _broadcast_task_update(task)
|
||||
return
|
||||
|
||||
deleted_cache_entries = invalidate_earth_layer_cache_for_source(source)
|
||||
update_payload = {
|
||||
"event": "earth.layer.changed",
|
||||
"action": "database_changed",
|
||||
"source": source,
|
||||
"table": payload.get("table"),
|
||||
"data_type": source,
|
||||
"layers": layers,
|
||||
"refresh_strategy": payload.get("refresh_strategy") or "clear_then_reload",
|
||||
"records_processed": payload.get("records_processed", 0),
|
||||
"operations": payload.get("operations") or [payload.get("operation") or "CHANGE"],
|
||||
"operation": payload.get("operation"),
|
||||
"cache_entries_invalidated": deleted_cache_entries,
|
||||
"timestamp": to_iso8601_utc(_utcnow()),
|
||||
}
|
||||
if payload.get("entity") == "interactable":
|
||||
update_payload.update(
|
||||
{
|
||||
"entity": "interactable",
|
||||
"action": payload.get("action") or "changed",
|
||||
"ids": payload.get("ids") or payload.get("entity_keys") or [],
|
||||
"item": payload.get("item"),
|
||||
}
|
||||
)
|
||||
await broadcaster.broadcast_earth_update(update_payload)
|
||||
|
||||
task.records_processed = int(payload.get("records_processed") or 0)
|
||||
task.progress = 100.0
|
||||
task.phase_progress = 100.0
|
||||
task.phase = "completed"
|
||||
task.phase_message = "Earth 图层刷新通知已发送"
|
||||
task.status = JOB_STATUS_SUCCESS
|
||||
task.completed_at = _utcnow()
|
||||
task.payload = {**payload, "cache_entries_invalidated": deleted_cache_entries}
|
||||
await db.commit()
|
||||
await _broadcast_task_update(task)
|
||||
|
||||
|
||||
_worker = DataJobWorker()
|
||||
|
||||
|
||||
def start_data_job_worker() -> None:
|
||||
_worker.start()
|
||||
|
||||
|
||||
async def stop_data_job_worker() -> None:
|
||||
await _worker.stop()
|
||||
@@ -32,28 +32,29 @@ class DocsMetadata:
|
||||
|
||||
DOCS_METADATA: tuple[DocsMetadata, ...] = (
|
||||
DocsMetadata(DOCS_README_FILENAME, DEFAULT_DOCS_SLUG, "public", "Overview", 0, "技术文档", "Technical Docs"),
|
||||
DocsMetadata("quickstart.md", "quickstart", "public", "Manual", 1, "快速开始", "Quickstart"),
|
||||
DocsMetadata("manual.md", "manual", "public", "Manual", 2, "Planet 使用手册", "Planet Manual"),
|
||||
DocsMetadata("manual.md", "manual", "public", "Manual", 1, "Planet 使用手册", "Planet Manual"),
|
||||
DocsMetadata("quickstart.md", "quickstart", "public", "Manual", 2, "快速开始", "Quickstart"),
|
||||
DocsMetadata("faq.md", "faq", "public", "Manual", 3, "常见问题", "FAQ"),
|
||||
DocsMetadata("location-pipeline-user.md", "location-pipeline-user", "public", "Manual", 4, "Earth 位置候选采集使用手册", "Earth Location Candidate Collection User Guide"),
|
||||
DocsMetadata("platform-data-flows.md", "platform-data-flows", "docs_developer", "Architecture", 5, "业务架构与数据流转", "Business Architecture and Data Flows"),
|
||||
DocsMetadata("naming-glossary.md", "naming-glossary", "docs_developer", "Architecture", 6, "命名与术语对照", "Naming Glossary"),
|
||||
DocsMetadata("earth-frontend-context.md", "earth-frontend-context", "docs_developer", "Earth", 10, "Earth 前端结构", "Earth Frontend Context"),
|
||||
DocsMetadata("earth-layer-style-reference.md", "earth-layer-style-reference", "docs_developer", "Earth", 11, "Earth 图层样式属性索引", "Earth Layer Style Reference"),
|
||||
DocsMetadata("earth-render-layer-order.md", "earth-render-layer-order", "docs_developer", "Earth", 12, "Earth 渲染图层顺序", "Earth Render Layer Order"),
|
||||
DocsMetadata("earth-satellite-footprint-policy.md", "earth-satellite-footprint-policy", "docs_developer", "Earth", 13, "Earth 卫星覆盖策略", "Earth Satellite Footprint Policy"),
|
||||
DocsMetadata("earth-bgp-context.md", "earth-bgp-context", "docs_developer", "Earth", 14, "BGP 态势上下文", "BGP Context"),
|
||||
DocsMetadata("earth-news-live-streams-collector-format.md", "earth-news-live-streams-collector-format", "docs_developer", "Earth", 15, "新闻直播采集格式", "News Live Streams Collector Format"),
|
||||
DocsMetadata("earth-interactable-usage.md", "earth-interactable-usage", "docs_developer", "Earth", 16, "Earth 可交互图标接入", "Earth Interactable Usage"),
|
||||
DocsMetadata("earth-toolbar-overlay-coordination.md", "earth-toolbar-overlay-coordination", "docs_developer", "Earth", 17, "Earth 工具栏与浮层协同", "Earth Toolbar and Overlay Coordination"),
|
||||
DocsMetadata("frontend-admin-frontend-context.md", "frontend-admin-frontend-context", "docs_developer", "Frontend", 20, "控制台前端结构", "Admin Frontend Context"),
|
||||
DocsMetadata("frontend-layout-guidelines.md", "frontend-layout-guidelines", "docs_developer", "Frontend", 21, "前端布局指南", "Frontend Layout Guidelines"),
|
||||
DocsMetadata("docs-gatekeeper-development.md", "docs-gatekeeper-development", "docs_developer", "Frontend", 22, "Docs Gatekeeper 开发说明", "Docs Gatekeeper Development Guide"),
|
||||
DocsMetadata("naming-glossary.md", "naming-glossary", "docs_developer", "Frontend", 23, "命名与术语对照", "Naming Glossary"),
|
||||
DocsMetadata("tactile-ui-components.md", "tactile-ui-components", "docs_developer", "Frontend", 24, "Tactile UI 组件库", "Tactile UI Components"),
|
||||
DocsMetadata("backend-collectors.md", "backend-collectors", "docs_developer", "Backend", 30, "数据采集系统", "Data Collectors"),
|
||||
DocsMetadata("backend-system-service-control.md", "backend-system-service-control", "docs_admin", "Backend", 31, "系统服务控制", "System Service Control"),
|
||||
DocsMetadata("datasource-collector-settings-connectivity.md", "datasource-collector-settings-connectivity", "docs_developer", "Backend", 32, "数据源、采集器设置与连接验证", "Datasource Collector Settings and Connectivity"),
|
||||
DocsMetadata("backend-datasources-api-performance.md", "backend-datasources-api-performance", "docs_developer", "Backend", 33, "数据源 API 性能", "Datasource API Performance"),
|
||||
DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 34, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"),
|
||||
DocsMetadata("data-job-earth-sync-architecture.md", "data-job-earth-sync-architecture", "docs_developer", "Backend", 34, "数据作业与 Outbox 技术架构", "Data Jobs and Outbox Architecture"),
|
||||
DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 35, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"),
|
||||
DocsMetadata("earth-news-live-streams-collector-format.md", "earth-news-live-streams-collector-format", "docs_developer", "Backend", 36, "新闻直播采集格式", "News Live Streams Collector Format"),
|
||||
DocsMetadata("docs-gatekeeper-development.md", "docs-gatekeeper-development", "docs_developer", "Backend", 37, "Docs Gatekeeper 开发说明", "Docs Gatekeeper Development Guide"),
|
||||
DocsMetadata("agents-aiprovider.md", "agents-aiprovider", "docs_developer", "Agents", 40, "AI Provider 指南", "AI Provider Guide"),
|
||||
DocsMetadata("ops-runbook.md", "ops-runbook", "docs_admin", "Ops", 49, "Planet 运维手册", "Planet Ops Runbook"),
|
||||
DocsMetadata("ops-docker-compose-buildx-upgrade.md", "ops-docker-compose-buildx-upgrade", "docs_admin", "Ops", 50, "Docker + Compose + Buildx 升级", "Docker + Compose + Buildx Upgrade"),
|
||||
|
||||
482
backend/app/services/earth_db_change_listener.py
Normal file
482
backend/app/services/earth_db_change_listener.py
Normal file
@@ -0,0 +1,482 @@
|
||||
"""PostgreSQL LISTEN/NOTIFY bridge for Earth layer refresh events."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from time import monotonic
|
||||
from typing import Any, Awaitable, Callable
|
||||
|
||||
import asyncpg
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.logging import get_logger
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
from app.services.earth_layer_adapters import (
|
||||
get_earth_refresh_strategy_for_change,
|
||||
get_earth_update_layers_for_change,
|
||||
get_earth_update_layers_for_source,
|
||||
)
|
||||
from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
EARTH_DATA_CHANGES_CHANNEL = "planet_earth_data_changes"
|
||||
DEFAULT_DEBOUNCE_SECONDS = 0.25
|
||||
DEFAULT_MAX_WAIT_SECONDS = 1.5
|
||||
DELETE_FAST_FLUSH_SECONDS = 0.05
|
||||
LISTEN_KEEPALIVE_SECONDS = 5.0
|
||||
OUTBOX_POLL_LIMIT = 5000
|
||||
MAX_ENTITY_KEY_SAMPLES = 20
|
||||
MAX_SEEN_EVENT_IDS = 20000
|
||||
|
||||
BroadcastFn = Callable[[dict[str, Any]], Awaitable[None]]
|
||||
InvalidateFn = Callable[[str], int]
|
||||
|
||||
|
||||
def normalize_asyncpg_dsn(dsn: str) -> str:
|
||||
"""Convert SQLAlchemy asyncpg URLs into asyncpg-compatible URLs."""
|
||||
return dsn.replace("postgresql+asyncpg://", "postgresql://", 1)
|
||||
|
||||
|
||||
def build_earth_update_from_db_payload(payload: dict[str, Any]) -> dict[str, Any] | None:
|
||||
table = payload.get("table")
|
||||
source = str(payload.get("source") or "").strip()
|
||||
table_name = str(table or "").strip()
|
||||
if not source and not table_name:
|
||||
return None
|
||||
layers = get_earth_update_layers_for_change(table_name, source)
|
||||
if not layers:
|
||||
return None
|
||||
refresh_strategy = get_earth_refresh_strategy_for_change(table_name, source) or "clear_then_reload"
|
||||
source_has_adapter = bool(get_earth_update_layers_for_source(source))
|
||||
effective_source = source if source_has_adapter else (table_name if table_name else source)
|
||||
operation = payload.get("operation")
|
||||
update: dict[str, Any] = {
|
||||
"event": "earth.layer.changed",
|
||||
"action": "database_changed",
|
||||
"source": effective_source,
|
||||
"original_source": source or None,
|
||||
"table": table_name or None,
|
||||
"data_type": effective_source,
|
||||
"layers": layers,
|
||||
"refresh_strategy": refresh_strategy,
|
||||
"operation": operation,
|
||||
"entity_key": payload.get("entity_key"),
|
||||
"timestamp": to_iso8601_utc(datetime.now(UTC)),
|
||||
}
|
||||
if table_name == "earth_interactables" and refresh_strategy == "delta":
|
||||
ids = payload.get("entity_keys")
|
||||
if not isinstance(ids, list):
|
||||
ids = [payload.get("entity_key")] if payload.get("entity_key") else []
|
||||
update.update(
|
||||
{
|
||||
"entity": "interactable",
|
||||
"action": "deleted" if operation == "DELETE" else "changed",
|
||||
"ids": [str(item) for item in ids if item],
|
||||
"item": None,
|
||||
}
|
||||
)
|
||||
return {
|
||||
**update,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class PendingEarthDbChange:
|
||||
source: str
|
||||
layers: list[str]
|
||||
table: str | None = None
|
||||
refresh_strategy: str = "clear_then_reload"
|
||||
entity: str | None = None
|
||||
action: str = "database_changed"
|
||||
records_processed: int = 0
|
||||
operations: set[str] = field(default_factory=set)
|
||||
entity_keys: list[str] = field(default_factory=list)
|
||||
first_occurred_at: str | None = None
|
||||
last_occurred_at: str | None = None
|
||||
first_seen_monotonic: float = field(default_factory=monotonic)
|
||||
last_seen_monotonic: float = field(default_factory=monotonic)
|
||||
|
||||
def add(self, payload: dict[str, Any]) -> None:
|
||||
self.last_seen_monotonic = monotonic()
|
||||
records_processed = payload.get("records_processed", 1)
|
||||
try:
|
||||
records_processed = int(records_processed)
|
||||
except (TypeError, ValueError):
|
||||
records_processed = 1
|
||||
self.records_processed += max(records_processed, 1)
|
||||
operation = payload.get("operation")
|
||||
if operation:
|
||||
self.operations.add(str(operation))
|
||||
entity_keys = payload.get("entity_keys")
|
||||
if not isinstance(entity_keys, list):
|
||||
entity_key = payload.get("entity_key")
|
||||
entity_keys = [entity_key] if entity_key else []
|
||||
for entity_key in entity_keys:
|
||||
if entity_key and len(self.entity_keys) < MAX_ENTITY_KEY_SAMPLES:
|
||||
self.entity_keys.append(str(entity_key))
|
||||
occurred_at = payload.get("occurred_at")
|
||||
if occurred_at:
|
||||
occurred_at = str(occurred_at)
|
||||
self.first_occurred_at = self.first_occurred_at or occurred_at
|
||||
self.last_occurred_at = occurred_at
|
||||
|
||||
|
||||
class EarthDbChangeDispatcher:
|
||||
"""Debounces database notifications and broadcasts Earth refresh hints."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
broadcast_earth_update: BroadcastFn | None = None,
|
||||
invalidate_cache: InvalidateFn | None = None,
|
||||
debounce_seconds: float = DEFAULT_DEBOUNCE_SECONDS,
|
||||
max_wait_seconds: float = DEFAULT_MAX_WAIT_SECONDS,
|
||||
) -> None:
|
||||
self._broadcast_earth_update = broadcast_earth_update or broadcaster.broadcast_earth_update
|
||||
self._invalidate_cache = invalidate_cache or invalidate_earth_layer_cache_for_source
|
||||
self._debounce_seconds = debounce_seconds
|
||||
self._max_wait_seconds = max(max_wait_seconds, debounce_seconds)
|
||||
self._pending: dict[str, PendingEarthDbChange] = {}
|
||||
self._flush_tasks: dict[str, asyncio.Task[None]] = {}
|
||||
self._seen_event_ids: set[int] = set()
|
||||
self._seen_event_order: deque[int] = deque()
|
||||
|
||||
def handle_notification(self, payload_text: str) -> bool:
|
||||
try:
|
||||
payload = json.loads(payload_text)
|
||||
except json.JSONDecodeError:
|
||||
logger.warning_event(
|
||||
"Ignoring malformed Earth database change notification",
|
||||
event="earth.db_changes.notification_malformed",
|
||||
)
|
||||
return False
|
||||
if not isinstance(payload, dict):
|
||||
return False
|
||||
return self.handle_payload(payload)
|
||||
|
||||
def handle_payload(self, payload: dict[str, Any]) -> bool:
|
||||
event_id = payload.get("event_id")
|
||||
if event_id is not None:
|
||||
try:
|
||||
normalized_event_id = int(event_id)
|
||||
except (TypeError, ValueError):
|
||||
normalized_event_id = None
|
||||
if normalized_event_id is not None:
|
||||
if normalized_event_id in self._seen_event_ids:
|
||||
return False
|
||||
self._remember_event_id(normalized_event_id)
|
||||
|
||||
update = build_earth_update_from_db_payload(payload)
|
||||
if not update:
|
||||
return False
|
||||
|
||||
source = update["source"]
|
||||
pending = self._pending.get(source)
|
||||
if pending is None:
|
||||
pending = PendingEarthDbChange(
|
||||
source=source,
|
||||
layers=list(update["layers"]),
|
||||
table=update.get("table"),
|
||||
refresh_strategy=str(update.get("refresh_strategy") or "clear_then_reload"),
|
||||
entity=update.get("entity"),
|
||||
action=str(update.get("action") or "database_changed"),
|
||||
)
|
||||
self._pending[source] = pending
|
||||
pending.add(payload)
|
||||
|
||||
task = self._flush_tasks.pop(source, None)
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
self._flush_tasks[source] = asyncio.create_task(
|
||||
self._flush_later(source, delay_seconds=self._next_flush_delay(pending))
|
||||
)
|
||||
return True
|
||||
|
||||
def _next_flush_delay(self, pending: PendingEarthDbChange) -> float:
|
||||
if "DELETE" in pending.operations and pending.refresh_strategy == "clear_then_reload":
|
||||
return DELETE_FAST_FLUSH_SECONDS
|
||||
elapsed = max(0.0, monotonic() - pending.first_seen_monotonic)
|
||||
remaining = self._max_wait_seconds - elapsed
|
||||
if remaining <= 0:
|
||||
return 0.0
|
||||
return min(self._debounce_seconds, remaining)
|
||||
|
||||
def _remember_event_id(self, event_id: int) -> None:
|
||||
self._seen_event_ids.add(event_id)
|
||||
self._seen_event_order.append(event_id)
|
||||
while len(self._seen_event_order) > MAX_SEEN_EVENT_IDS:
|
||||
expired_event_id = self._seen_event_order.popleft()
|
||||
self._seen_event_ids.discard(expired_event_id)
|
||||
|
||||
async def _flush_later(self, source: str, *, delay_seconds: float) -> None:
|
||||
try:
|
||||
if delay_seconds > 0:
|
||||
await asyncio.sleep(delay_seconds)
|
||||
await self.flush_source(source)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.exception_event(
|
||||
"Failed to broadcast debounced Earth database change",
|
||||
event="earth.db_changes.flush_failed",
|
||||
context={"source": source, "error": str(exc)},
|
||||
)
|
||||
finally:
|
||||
current = self._flush_tasks.get(source)
|
||||
if current is asyncio.current_task():
|
||||
self._flush_tasks.pop(source, None)
|
||||
|
||||
async def flush_source(self, source: str) -> None:
|
||||
pending = self._pending.get(source)
|
||||
if pending is None:
|
||||
return
|
||||
|
||||
flushed_at = datetime.now(UTC)
|
||||
deleted_cache_entries = self._invalidate_cache(source)
|
||||
operations = sorted(pending.operations)
|
||||
payload: dict[str, Any] = {
|
||||
"event": "earth.layer.changed",
|
||||
"action": "database_changed",
|
||||
"source": source,
|
||||
"table": pending.table,
|
||||
"data_type": source,
|
||||
"layers": pending.layers,
|
||||
"refresh_strategy": pending.refresh_strategy,
|
||||
"records_processed": pending.records_processed,
|
||||
"operations": operations,
|
||||
"operation": operations[-1] if len(operations) == 1 else None,
|
||||
"entity_keys": pending.entity_keys,
|
||||
"entity_key_sample_size": len(pending.entity_keys),
|
||||
"cache_entries_invalidated": deleted_cache_entries,
|
||||
"first_occurred_at": pending.first_occurred_at,
|
||||
"last_occurred_at": pending.last_occurred_at,
|
||||
"debounce_ms": int((monotonic() - pending.first_seen_monotonic) * 1000),
|
||||
"total_latency_ms": self._total_latency_ms(pending, flushed_at),
|
||||
"timestamp": to_iso8601_utc(flushed_at),
|
||||
}
|
||||
if pending.entity == "interactable":
|
||||
payload.update(
|
||||
{
|
||||
"entity": "interactable",
|
||||
"action": "deleted" if "DELETE" in pending.operations else "changed",
|
||||
"ids": pending.entity_keys,
|
||||
"item": None,
|
||||
}
|
||||
)
|
||||
await self._broadcast_earth_update(payload)
|
||||
self._pending.pop(source, None)
|
||||
logger.info_event(
|
||||
"Broadcasted Earth database change",
|
||||
event="earth.db_changes.broadcasted",
|
||||
context={
|
||||
"source": source,
|
||||
"layers": pending.layers,
|
||||
"records_processed": pending.records_processed,
|
||||
"cache_entries_invalidated": deleted_cache_entries,
|
||||
"debounce_ms": int((monotonic() - pending.first_seen_monotonic) * 1000),
|
||||
"total_latency_ms": payload["total_latency_ms"],
|
||||
},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _total_latency_ms(pending: PendingEarthDbChange, flushed_at: datetime) -> int | None:
|
||||
occurred_at = pending.first_occurred_at
|
||||
if not occurred_at:
|
||||
return None
|
||||
try:
|
||||
normalized = occurred_at.replace("Z", "+00:00")
|
||||
occurred = datetime.fromisoformat(normalized)
|
||||
if occurred.tzinfo is None:
|
||||
occurred = occurred.replace(tzinfo=UTC)
|
||||
return max(0, int((flushed_at - occurred.astimezone(UTC)).total_seconds() * 1000))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
async def flush_all(self) -> None:
|
||||
sources = list(self._pending)
|
||||
for source in sources:
|
||||
task = self._flush_tasks.pop(source, None)
|
||||
if task and not task.done():
|
||||
task.cancel()
|
||||
await self.flush_source(source)
|
||||
|
||||
async def stop(self) -> None:
|
||||
tasks = [task for task in self._flush_tasks.values() if not task.done()]
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
self._flush_tasks.clear()
|
||||
await self.flush_all()
|
||||
|
||||
|
||||
class EarthDbChangeListener:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
dsn: str,
|
||||
dispatcher: EarthDbChangeDispatcher,
|
||||
channel: str = EARTH_DATA_CHANGES_CHANNEL,
|
||||
) -> None:
|
||||
self._dsn = normalize_asyncpg_dsn(dsn)
|
||||
self._dispatcher = dispatcher
|
||||
self._channel = channel
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._stop_event: asyncio.Event | None = None
|
||||
self._connection: asyncpg.Connection | None = None
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task and not self._task.done():
|
||||
return
|
||||
self._loop = asyncio.get_running_loop()
|
||||
self._stop_event = asyncio.Event()
|
||||
self._task = asyncio.create_task(self._run())
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._stop_event:
|
||||
self._stop_event.set()
|
||||
if self._connection:
|
||||
await self._connection.close()
|
||||
if self._task:
|
||||
await asyncio.gather(self._task, return_exceptions=True)
|
||||
await self._dispatcher.stop()
|
||||
|
||||
async def _run(self) -> None:
|
||||
backoff_seconds = 1.0
|
||||
assert self._stop_event is not None
|
||||
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
self._connection = await asyncpg.connect(self._dsn)
|
||||
await self._connection.add_listener(self._channel, self._on_notification)
|
||||
logger.info_event(
|
||||
"Earth database change listener connected",
|
||||
event="earth.db_changes.connected",
|
||||
context={"channel": self._channel},
|
||||
)
|
||||
backoff_seconds = 1.0
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._stop_event.wait(),
|
||||
timeout=LISTEN_KEEPALIVE_SECONDS,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
await self._poll_outbox()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.exception_event(
|
||||
"Earth database change listener failed",
|
||||
event="earth.db_changes.listener_failed",
|
||||
context={"channel": self._channel, "error": str(exc)},
|
||||
)
|
||||
try:
|
||||
await asyncio.wait_for(self._stop_event.wait(), timeout=backoff_seconds)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
backoff_seconds = min(backoff_seconds * 2, 30.0)
|
||||
finally:
|
||||
if self._connection:
|
||||
try:
|
||||
await self._connection.remove_listener(self._channel, self._on_notification)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await self._connection.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._connection = None
|
||||
|
||||
async def _poll_outbox(self) -> None:
|
||||
if self._connection is None:
|
||||
return
|
||||
|
||||
rows = await self._connection.fetch(
|
||||
"""
|
||||
SELECT id, payload
|
||||
FROM earth_data_change_events
|
||||
WHERE consumed_at IS NULL
|
||||
ORDER BY id
|
||||
LIMIT $1
|
||||
""",
|
||||
OUTBOX_POLL_LIMIT,
|
||||
)
|
||||
accepted_count = 0
|
||||
consumed_ids: list[int] = []
|
||||
for row in rows:
|
||||
payload = row["payload"]
|
||||
if isinstance(payload, str):
|
||||
try:
|
||||
payload = json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
consumed_ids.append(int(row["id"]))
|
||||
continue
|
||||
if isinstance(payload, dict):
|
||||
if self._dispatcher.handle_payload(payload):
|
||||
accepted_count += 1
|
||||
consumed_ids.append(int(row["id"]))
|
||||
else:
|
||||
consumed_ids.append(int(row["id"]))
|
||||
if consumed_ids:
|
||||
await self._dispatcher.flush_all()
|
||||
if consumed_ids:
|
||||
await self._connection.execute(
|
||||
"""
|
||||
UPDATE earth_data_change_events
|
||||
SET consumed_at = NOW()
|
||||
WHERE id = ANY($1::bigint[])
|
||||
AND consumed_at IS NULL
|
||||
""",
|
||||
consumed_ids,
|
||||
)
|
||||
if rows:
|
||||
logger.info_event(
|
||||
"Polled Earth database change outbox",
|
||||
event="earth.db_changes.outbox_polled",
|
||||
context={"events": len(rows), "accepted": accepted_count},
|
||||
)
|
||||
|
||||
def _on_notification(
|
||||
self,
|
||||
_connection: asyncpg.Connection,
|
||||
_pid: int,
|
||||
_channel: str,
|
||||
payload: str,
|
||||
) -> None:
|
||||
if self._loop and self._loop.is_running():
|
||||
self._loop.call_soon_threadsafe(self._dispatcher.handle_notification, payload)
|
||||
return
|
||||
self._dispatcher.handle_notification(payload)
|
||||
|
||||
|
||||
_dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcaster.broadcast_earth_update,
|
||||
invalidate_cache=invalidate_earth_layer_cache_for_source,
|
||||
)
|
||||
_listener: EarthDbChangeListener | None = None
|
||||
|
||||
|
||||
def start_earth_db_change_listener() -> None:
|
||||
global _listener
|
||||
if _listener is not None:
|
||||
return
|
||||
_listener = EarthDbChangeListener(dsn=settings.DATABASE_URL, dispatcher=_dispatcher)
|
||||
_listener.start()
|
||||
|
||||
|
||||
async def stop_earth_db_change_listener() -> None:
|
||||
global _listener
|
||||
if _listener is None:
|
||||
await _dispatcher.stop()
|
||||
return
|
||||
listener = _listener
|
||||
_listener = None
|
||||
await listener.stop()
|
||||
113
backend/app/services/earth_interactables.py
Normal file
113
backend/app/services/earth_interactables.py
Normal file
@@ -0,0 +1,113 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
from app.models.earth_interactable import EarthInteractable
|
||||
from app.services.earth_layer_cache import EARTH_LAYER_CACHE_PREFIX, earth_layer_cache
|
||||
|
||||
INTERACTABLE_ENTITY = "interactable"
|
||||
INTERACTABLE_LAYER = "interactables"
|
||||
|
||||
|
||||
def normalize_interactable_id(value: str | None = None) -> str:
|
||||
raw = str(value or "").strip()
|
||||
return raw or f"interactable-{uuid4().hex}"
|
||||
|
||||
|
||||
def serialize_interactable(record: EarthInteractable) -> dict[str, Any]:
|
||||
return {
|
||||
"id": record.id,
|
||||
"layer": record.layer,
|
||||
"kind": record.kind,
|
||||
"label": record.label,
|
||||
"description": record.description,
|
||||
"latitude": record.latitude,
|
||||
"longitude": record.longitude,
|
||||
"altitude": record.altitude,
|
||||
"revision": record.revision,
|
||||
"properties": record.properties or {},
|
||||
"is_deleted": bool(record.is_deleted),
|
||||
"created_at": to_iso8601_utc(record.created_at),
|
||||
"updated_at": to_iso8601_utc(record.updated_at),
|
||||
"deleted_at": to_iso8601_utc(record.deleted_at),
|
||||
}
|
||||
|
||||
|
||||
def interactables_to_geojson(items: list[EarthInteractable]) -> dict[str, Any]:
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": item.id,
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [item.longitude, item.latitude],
|
||||
},
|
||||
"properties": serialize_interactable(item),
|
||||
}
|
||||
for item in items
|
||||
if not item.is_deleted
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def invalidate_interactable_cache(layer: str | None = None) -> int:
|
||||
layer_key = str(layer or "*").strip() or "*"
|
||||
deleted = earth_layer_cache.delete_pattern(
|
||||
f"{EARTH_LAYER_CACHE_PREFIX}:interactables:layer:{layer_key}*"
|
||||
)
|
||||
if layer_key != "all":
|
||||
deleted += earth_layer_cache.delete_pattern(
|
||||
f"{EARTH_LAYER_CACHE_PREFIX}:interactables:layer:all*"
|
||||
)
|
||||
return deleted
|
||||
|
||||
|
||||
def build_interactable_event(
|
||||
*,
|
||||
action: str,
|
||||
record: EarthInteractable,
|
||||
include_item: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
item = serialize_interactable(record)
|
||||
return {
|
||||
"entity": INTERACTABLE_ENTITY,
|
||||
"action": action,
|
||||
"layer": record.layer,
|
||||
"layers": [INTERACTABLE_LAYER],
|
||||
"ids": [record.id],
|
||||
"revision": record.revision,
|
||||
"changed_at": item["deleted_at"] or item["updated_at"] or to_iso8601_utc(datetime.now(UTC)),
|
||||
"item": item if include_item else None,
|
||||
"source": "earth_interactables",
|
||||
}
|
||||
|
||||
|
||||
async def publish_interactable_event(action: str, record: EarthInteractable, *, include_item: bool = True) -> None:
|
||||
await broadcaster.broadcast_earth_update(
|
||||
build_interactable_event(action=action, record=record, include_item=include_item)
|
||||
)
|
||||
|
||||
|
||||
async def list_interactables(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
layer: str | None = None,
|
||||
include_deleted: bool = False,
|
||||
) -> list[EarthInteractable]:
|
||||
stmt = select(EarthInteractable)
|
||||
if layer:
|
||||
stmt = stmt.where(EarthInteractable.layer == layer)
|
||||
if not include_deleted:
|
||||
stmt = stmt.where(EarthInteractable.is_deleted.is_(False))
|
||||
stmt = stmt.order_by(EarthInteractable.updated_at.desc(), EarthInteractable.id.asc())
|
||||
result = await db.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
179
backend/app/services/earth_layer_adapters.py
Normal file
179
backend/app/services/earth_layer_adapters.py
Normal file
@@ -0,0 +1,179 @@
|
||||
"""Earth layer adapter registry for datasource-backed refresh behavior."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EarthLayerAdapter:
|
||||
sources: frozenset[str]
|
||||
layers: tuple[str, ...]
|
||||
cache_patterns: tuple[str, ...]
|
||||
tables: frozenset[str] = field(default_factory=frozenset)
|
||||
derived_models: tuple[str, ...] = field(default_factory=tuple)
|
||||
refresh_strategy: str = "clear_then_reload"
|
||||
|
||||
|
||||
EARTH_LAYER_ADAPTERS: tuple[EarthLayerAdapter, ...] = (
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset({"barentswatch_vessels", "aisstream_vessels", "vessel_static", "vessel_position", "ais_raw_observations", "ais_source_health"}),
|
||||
tables=frozenset({"vessel_static", "vessel_position", "ais_raw_observations", "ais_source_health"}),
|
||||
layers=("vessels",),
|
||||
cache_patterns=("vessels*", "summary*"),
|
||||
),
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset(
|
||||
{
|
||||
"telegeography_cables",
|
||||
"telegeography_landing",
|
||||
"telegeography_landing_points",
|
||||
"telegeography_systems",
|
||||
"telegeography_cable_systems",
|
||||
"arcgis_cables",
|
||||
"arcgis_landing_points",
|
||||
"arcgis_cable_landing_relation",
|
||||
"arcgis_cable_landing_relations",
|
||||
"fao_landing_points",
|
||||
}
|
||||
),
|
||||
tables=frozenset({"collected_data"}),
|
||||
layers=("cables",),
|
||||
cache_patterns=("cables*", "landing-points*", "summary*"),
|
||||
),
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset({"celestrak_tle", "spacetrack_tle"}),
|
||||
tables=frozenset({"collected_data"}),
|
||||
layers=("satellites",),
|
||||
cache_patterns=("satellites*", "summary*"),
|
||||
),
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset(
|
||||
{
|
||||
"top500",
|
||||
"top500_supercomputers",
|
||||
"epoch_ai_gpu",
|
||||
"huggingface_models",
|
||||
"huggingface_datasets",
|
||||
"huggingface_spaces",
|
||||
"compute_center_locations",
|
||||
}
|
||||
),
|
||||
tables=frozenset({"compute_center_locations"}),
|
||||
layers=("computeCenters",),
|
||||
cache_patterns=("compute-centers*", "summary*"),
|
||||
refresh_strategy="reload",
|
||||
),
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset(
|
||||
{
|
||||
"ris_live_bgp",
|
||||
"bgpstream_bgp",
|
||||
"iptoasn_prefix_geo",
|
||||
"opengeofeed_prefix_geo",
|
||||
"nro_delegated_prefix_geo",
|
||||
"bgp_observations",
|
||||
"bgp_anomalies",
|
||||
"bgp_incidents",
|
||||
"bgp_collector_locations",
|
||||
}
|
||||
),
|
||||
tables=frozenset({"bgp_observations", "bgp_anomalies", "bgp_incidents", "bgp_collector_locations"}),
|
||||
layers=("bgp",),
|
||||
cache_patterns=("bgp*", "summary*"),
|
||||
derived_models=("bgp_observations", "bgp_anomalies", "bgp_incidents"),
|
||||
),
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset({"news_live_streams"}),
|
||||
tables=frozenset({"collected_data"}),
|
||||
layers=("media",),
|
||||
cache_patterns=("summary*",),
|
||||
refresh_strategy="reload",
|
||||
),
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset({"media_news_archive", "earth_news_items"}),
|
||||
tables=frozenset({"earth_news_items"}),
|
||||
layers=("news",),
|
||||
cache_patterns=("summary*",),
|
||||
refresh_strategy="reload",
|
||||
),
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset({"earth_interactables"}),
|
||||
tables=frozenset({"earth_interactables"}),
|
||||
layers=("interactables",),
|
||||
cache_patterns=("interactables*", "summary*"),
|
||||
refresh_strategy="delta",
|
||||
),
|
||||
)
|
||||
|
||||
_ADAPTERS_BY_SOURCE = {
|
||||
source: adapter
|
||||
for adapter in EARTH_LAYER_ADAPTERS
|
||||
for source in adapter.sources
|
||||
}
|
||||
_ADAPTERS_BY_TABLE = {
|
||||
table: adapter
|
||||
for adapter in EARTH_LAYER_ADAPTERS
|
||||
for table in adapter.tables
|
||||
}
|
||||
|
||||
|
||||
def get_earth_layer_adapter_for_source(source: str | None) -> EarthLayerAdapter | None:
|
||||
return _ADAPTERS_BY_SOURCE.get(str(source or "").strip())
|
||||
|
||||
|
||||
def get_earth_layer_adapter_for_change(table: str | None, source: str | None) -> EarthLayerAdapter | None:
|
||||
table_key = str(table or "").strip()
|
||||
source_key = str(source or "").strip()
|
||||
if table_key and table_key != "collected_data":
|
||||
adapter = _ADAPTERS_BY_TABLE.get(table_key)
|
||||
if adapter is not None:
|
||||
return adapter
|
||||
return get_earth_layer_adapter_for_source(source_key)
|
||||
|
||||
|
||||
def get_earth_update_layers_for_source(source: str | None) -> list[str]:
|
||||
adapter = get_earth_layer_adapter_for_source(source)
|
||||
return list(adapter.layers) if adapter else []
|
||||
|
||||
|
||||
def get_earth_update_layers_for_change(table: str | None, source: str | None) -> list[str]:
|
||||
adapter = get_earth_layer_adapter_for_change(table, source)
|
||||
return list(adapter.layers) if adapter else []
|
||||
|
||||
|
||||
def get_earth_refresh_strategy_for_change(table: str | None, source: str | None) -> str | None:
|
||||
adapter = get_earth_layer_adapter_for_change(table, source)
|
||||
return adapter.refresh_strategy if adapter else None
|
||||
|
||||
|
||||
def get_earth_cache_patterns_for_source(source: str | None) -> list[str]:
|
||||
adapter = get_earth_layer_adapter_for_source(source)
|
||||
return list(adapter.cache_patterns) if adapter else []
|
||||
|
||||
|
||||
async def clear_derived_datasource_data(db: AsyncSession, source: str) -> dict[str, int]:
|
||||
adapter = get_earth_layer_adapter_for_source(source)
|
||||
if adapter is None or not adapter.derived_models:
|
||||
return {}
|
||||
|
||||
from app.models.bgp_anomaly import BGPAnomaly
|
||||
from app.models.bgp_incident import BGPIncident
|
||||
from app.models.bgp_observation import BGPObservation
|
||||
|
||||
model_by_key: dict[str, Any] = {
|
||||
"bgp_observations": BGPObservation,
|
||||
"bgp_anomalies": BGPAnomaly,
|
||||
"bgp_incidents": BGPIncident,
|
||||
}
|
||||
deleted_counts: dict[str, int] = {}
|
||||
for key in adapter.derived_models:
|
||||
model = model_by_key.get(key)
|
||||
if model is None:
|
||||
continue
|
||||
result = await db.execute(model.__table__.delete().where(model.source == source))
|
||||
deleted_counts[key] = int(result.rowcount or 0)
|
||||
return deleted_counts
|
||||
@@ -268,34 +268,10 @@ def apply_payload_budget(payload: dict[str, Any], policy: EarthLayerCachePolicy)
|
||||
|
||||
|
||||
def invalidate_earth_layer_cache_for_source(source: str) -> int:
|
||||
from app.services.earth_layer_adapters import get_earth_cache_patterns_for_source
|
||||
|
||||
source_key = str(source or "").strip()
|
||||
patterns = {
|
||||
"barentswatch_vessels": ["vessels*", "summary*"],
|
||||
"aisstream_vessels": ["vessels*", "summary*"],
|
||||
"telegeography_cables": ["cables*", "landing-points*", "summary*"],
|
||||
"telegeography_landing": ["landing-points*", "summary*"],
|
||||
"telegeography_landing_points": ["landing-points*", "summary*"],
|
||||
"telegeography_systems": ["cables*", "summary*"],
|
||||
"telegeography_cable_systems": ["cables*", "summary*"],
|
||||
"arcgis_cables": ["cables*", "landing-points*", "summary*"],
|
||||
"arcgis_landing_points": ["landing-points*", "summary*"],
|
||||
"arcgis_cable_landing_relation": ["landing-points*", "summary*"],
|
||||
"arcgis_cable_landing_relations": ["landing-points*", "summary*"],
|
||||
"fao_landing_points": ["landing-points*", "summary*"],
|
||||
"celestrak_tle": ["satellites*", "summary*"],
|
||||
"spacetrack_tle": ["satellites*", "summary*"],
|
||||
"top500": ["compute-centers*", "summary*"],
|
||||
"top500_supercomputers": ["compute-centers*", "summary*"],
|
||||
"epoch_ai_gpu": ["compute-centers*", "summary*"],
|
||||
"huggingface_models": ["compute-centers*", "summary*"],
|
||||
"huggingface_datasets": ["compute-centers*", "summary*"],
|
||||
"huggingface_spaces": ["compute-centers*", "summary*"],
|
||||
"ris_live_bgp": ["bgp*", "summary*"],
|
||||
"bgpstream_bgp": ["bgp*", "summary*"],
|
||||
"iptoasn_prefix_geo": ["bgp*", "summary*"],
|
||||
"opengeofeed_prefix_geo": ["bgp*", "summary*"],
|
||||
"nro_delegated_prefix_geo": ["bgp*", "summary*"],
|
||||
}.get(source_key, [])
|
||||
patterns = get_earth_cache_patterns_for_source(source_key)
|
||||
deleted = 0
|
||||
for layer_pattern in patterns:
|
||||
deleted += earth_layer_cache.delete_pattern(f"{EARTH_LAYER_CACHE_PREFIX}:{layer_pattern}")
|
||||
|
||||
@@ -1063,6 +1063,7 @@ def _build_payload(
|
||||
lon: float | None,
|
||||
active_region: str,
|
||||
items: list[ParsedNewsItem],
|
||||
cruise_items: list[ParsedNewsItem] | None = None,
|
||||
sources: list[NewsFeedSource],
|
||||
errors: list[str],
|
||||
stale: bool,
|
||||
@@ -1082,6 +1083,10 @@ def _build_payload(
|
||||
},
|
||||
"sources": _serialize_sources(sources),
|
||||
"items": [_serialize_item(item, active_region=active_region) for item in items],
|
||||
"cruise_items": [
|
||||
_serialize_item(item, active_region=active_region)
|
||||
for item in (cruise_items if cruise_items is not None else items)
|
||||
],
|
||||
"errors": errors,
|
||||
"stale": stale,
|
||||
}
|
||||
@@ -1295,6 +1300,7 @@ async def get_earth_news_payload(
|
||||
|
||||
from app.services.earth_news_store import (
|
||||
get_earth_news_freshness,
|
||||
list_earth_news_cruise_items,
|
||||
list_earth_news_items,
|
||||
upsert_earth_news_items,
|
||||
)
|
||||
@@ -1312,6 +1318,13 @@ async def get_earth_news_payload(
|
||||
active_region=active_region,
|
||||
limit=MAX_ITEMS_TOTAL,
|
||||
)
|
||||
if hasattr(db, "execute"):
|
||||
cruise_items = await list_earth_news_cruise_items(
|
||||
db,
|
||||
limit=MAX_ITEMS_TOTAL * len(REGION_ANCHORS),
|
||||
)
|
||||
else:
|
||||
cruise_items = items
|
||||
await _enqueue_unverified_locations(items)
|
||||
stale = bool(errors and items)
|
||||
|
||||
@@ -1320,6 +1333,7 @@ async def get_earth_news_payload(
|
||||
lon=lon,
|
||||
active_region=active_region,
|
||||
items=items,
|
||||
cruise_items=cruise_items,
|
||||
sources=sources,
|
||||
errors=errors,
|
||||
stale=stale,
|
||||
|
||||
@@ -78,6 +78,24 @@ async def list_earth_news_items(
|
||||
return [record_to_parsed_news_item(record) for record in result.scalars().all()]
|
||||
|
||||
|
||||
async def list_earth_news_cruise_items(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
limit: int,
|
||||
) -> list[ParsedNewsItem]:
|
||||
result = await db.execute(
|
||||
select(EarthNewsItem)
|
||||
.order_by(
|
||||
EarthNewsItem.region.asc(),
|
||||
EarthNewsItem.published_at.desc().nullslast(),
|
||||
EarthNewsItem.last_seen_at.desc(),
|
||||
EarthNewsItem.feed_name.asc(),
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
return [record_to_parsed_news_item(record) for record in result.scalars().all()]
|
||||
|
||||
|
||||
async def get_earth_news_freshness(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
|
||||
@@ -10,6 +10,7 @@ from fastapi import HTTPException, status
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.db.session import async_session_factory
|
||||
from app.models.playground_message import PlaygroundMessage
|
||||
from app.models.playground_session import PlaygroundSession
|
||||
@@ -27,9 +28,11 @@ from app.schemas.ai import (
|
||||
SituationalAnalysisRequest,
|
||||
)
|
||||
from app.services.ai_client import AIProviderClient
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
from app.services.playground_session_store import _to_response as session_to_response
|
||||
from app.services.playground_session_store import upsert_playground_session
|
||||
|
||||
logger = get_logger(__name__, service="ai")
|
||||
STREAM_CHUNK_SIZE = 24
|
||||
STREAM_INTERVAL_SECONDS = 0.08
|
||||
THINKING_PREVIEW_SECONDS = 2.6
|
||||
@@ -624,7 +627,42 @@ async def _run_assistant_message(
|
||||
thinking={"type": "enabled"},
|
||||
)
|
||||
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.playground.run.start",
|
||||
message="Playground AI run started",
|
||||
category="ai",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=user_id,
|
||||
context={
|
||||
"session_id": session_id,
|
||||
"session_key": session_key,
|
||||
"user_message_id": user_message_id,
|
||||
"assistant_message_id": assistant_message_id,
|
||||
"preset": payload.selected_preset_key,
|
||||
},
|
||||
)
|
||||
analysis = await provider_client.analyze(request_payload, request_id=request_id)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.playground.run.success",
|
||||
message="Playground AI run completed",
|
||||
category="ai",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=user_id,
|
||||
context={
|
||||
"session_id": session_id,
|
||||
"session_key": session_key,
|
||||
"provider": analysis.provider,
|
||||
"model": analysis.model,
|
||||
"content_block_count": len(analysis.content_blocks or []),
|
||||
"thinking_block_count": len(analysis.thinking_blocks or []),
|
||||
},
|
||||
)
|
||||
|
||||
async with async_session_factory() as db:
|
||||
assistant_message = await _mark_message_state(
|
||||
@@ -704,6 +742,23 @@ async def _run_assistant_message(
|
||||
await db.flush()
|
||||
await db.commit()
|
||||
except asyncio.CancelledError:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.playground.run.cancelled",
|
||||
message="Playground AI run cancelled",
|
||||
category="ai",
|
||||
level="warning",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=user_id,
|
||||
context={
|
||||
"session_id": session_id,
|
||||
"session_key": session_key,
|
||||
"assistant_message_id": assistant_message_id,
|
||||
"duration_ms": round((perf_counter() - started_at) * 1000),
|
||||
},
|
||||
)
|
||||
async with async_session_factory() as db:
|
||||
result = await db.execute(select(PlaygroundMessage).where(PlaygroundMessage.id == assistant_message_id))
|
||||
message = result.scalar_one_or_none()
|
||||
@@ -715,6 +770,26 @@ async def _run_assistant_message(
|
||||
await db.commit()
|
||||
raise
|
||||
except Exception as exc:
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="ai.playground.run.failed",
|
||||
message="Playground AI run failed",
|
||||
category="ai",
|
||||
level="error",
|
||||
service="ai",
|
||||
module=__name__,
|
||||
request_id=request_id,
|
||||
user_id=user_id,
|
||||
context=exception_context(
|
||||
exc,
|
||||
{
|
||||
"session_id": session_id,
|
||||
"session_key": session_key,
|
||||
"assistant_message_id": assistant_message_id,
|
||||
"duration_ms": round((perf_counter() - started_at) * 1000),
|
||||
},
|
||||
),
|
||||
)
|
||||
error_message = _format_run_exception(exc)
|
||||
async with async_session_factory() as db:
|
||||
result = await db.execute(select(PlaygroundMessage).where(PlaygroundMessage.id == assistant_message_id))
|
||||
|
||||
@@ -13,6 +13,7 @@ from app.db.session import async_session_factory
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.task import CollectionTask
|
||||
from app.services.business_logs import emit_business_log, emit_business_log_background, exception_context
|
||||
from app.services.collectors.registry import collector_registry
|
||||
from app.services.datasource_connectivity import (
|
||||
build_builtin_connectivity_checksum,
|
||||
@@ -124,6 +125,15 @@ async def run_collector_task(collector_name: str):
|
||||
event="collector.run.skipped_disabled",
|
||||
context={"collector_name": collector_name},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.run.skipped_disabled",
|
||||
message="Skipping disabled collector",
|
||||
category="collector",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context={"collector_name": collector_name, "datasource_id": datasource.id, "status": "skipped"},
|
||||
)
|
||||
return
|
||||
|
||||
running_result = await db.execute(
|
||||
@@ -152,6 +162,21 @@ async def run_collector_task(collector_name: str):
|
||||
event="collector.run.skipped_already_running",
|
||||
context={"collector_name": collector_name, "task_id": existing_running.id},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.run.skipped_already_running",
|
||||
message="Skipping collector trigger because task is already running",
|
||||
category="collector",
|
||||
level="warning",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": collector_name,
|
||||
"datasource_id": datasource.id,
|
||||
"task_id": existing_running.id,
|
||||
"status": "skipped",
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
existing_error = (existing_running.error_message or "").strip()
|
||||
@@ -173,6 +198,21 @@ async def run_collector_task(collector_name: str):
|
||||
event="collector.run.stale_task_failed",
|
||||
context={"collector_name": collector_name, "task_id": existing_running.id},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.run.stale_task_failed",
|
||||
message="Marked stale running task as failed before rerun",
|
||||
category="collector",
|
||||
level="warning",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": collector_name,
|
||||
"datasource_id": datasource.id,
|
||||
"task_id": existing_running.id,
|
||||
"status": "failed",
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
datasource_id = datasource.id
|
||||
@@ -183,6 +223,15 @@ async def run_collector_task(collector_name: str):
|
||||
event="collector.run.started",
|
||||
context={"collector_name": collector_name, "datasource_id": datasource_id},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.run.scheduled_started",
|
||||
message="Scheduler started collector run",
|
||||
category="collector",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context={"collector_name": collector_name, "datasource_id": datasource_id, "status": "running"},
|
||||
)
|
||||
task_result = await collector.run(db)
|
||||
datasource = await db.get(DataSource, datasource_id)
|
||||
if datasource is None:
|
||||
@@ -217,6 +266,20 @@ async def run_collector_task(collector_name: str):
|
||||
event="collector.run.completed",
|
||||
context={"collector_name": collector_name, "datasource_id": datasource_id, "result": task_result},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.run.scheduled_completed",
|
||||
message="Scheduler completed collector run",
|
||||
category="collector",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context={
|
||||
"collector_name": collector_name,
|
||||
"datasource_id": datasource_id,
|
||||
"status": task_result.get("status"),
|
||||
"result": task_result,
|
||||
},
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
await db.rollback()
|
||||
datasource = await db.get(DataSource, datasource_id)
|
||||
@@ -228,6 +291,16 @@ async def run_collector_task(collector_name: str):
|
||||
event="collector.run.cancelled",
|
||||
context={"collector_name": collector_name, "datasource_id": datasource.id},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.run.cancelled",
|
||||
message="Collector cancelled by operator",
|
||||
category="collector",
|
||||
level="warning",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context={"collector_name": collector_name, "datasource_id": datasource.id, "status": "cancelled"},
|
||||
)
|
||||
raise
|
||||
except Exception as exc:
|
||||
await db.rollback()
|
||||
@@ -240,6 +313,19 @@ async def run_collector_task(collector_name: str):
|
||||
event="collector.run.failed",
|
||||
context={"collector_name": collector_name, "datasource_id": datasource.id, "error": str(exc)},
|
||||
)
|
||||
await emit_business_log(
|
||||
logger,
|
||||
event="collector.run.failed",
|
||||
message="Collector failed",
|
||||
category="collector",
|
||||
level="error",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context=exception_context(
|
||||
exc,
|
||||
{"collector_name": collector_name, "datasource_id": datasource.id, "status": "failed"},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def cleanup_stale_running_tasks(max_age_hours: int = 2) -> int:
|
||||
@@ -361,6 +447,16 @@ def run_collector_now(collector_name: str) -> bool:
|
||||
event="collector.trigger.skipped_already_running",
|
||||
context={"collector_name": collector_name},
|
||||
)
|
||||
emit_business_log_background(
|
||||
logger,
|
||||
event="collector.trigger.skipped_already_running",
|
||||
message="Collector is already running in-memory; skipping duplicate trigger",
|
||||
category="collector",
|
||||
level="warning",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context={"collector_name": collector_name, "status": "skipped"},
|
||||
)
|
||||
return False
|
||||
|
||||
try:
|
||||
@@ -378,6 +474,15 @@ def run_collector_now(collector_name: str) -> bool:
|
||||
event="collector.trigger.started",
|
||||
context={"collector_name": collector_name},
|
||||
)
|
||||
emit_business_log_background(
|
||||
logger,
|
||||
event="collector.trigger.started",
|
||||
message="Triggered collector",
|
||||
category="collector",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context={"collector_name": collector_name, "status": "queued"},
|
||||
)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.error_event(
|
||||
@@ -385,6 +490,16 @@ def run_collector_now(collector_name: str) -> bool:
|
||||
event="collector.trigger.failed",
|
||||
context={"collector_name": collector_name, "error": str(exc)},
|
||||
)
|
||||
emit_business_log_background(
|
||||
logger,
|
||||
event="collector.trigger.failed",
|
||||
message="Failed to trigger collector",
|
||||
category="collector",
|
||||
level="error",
|
||||
service="scheduler",
|
||||
module=__name__,
|
||||
context=exception_context(exc, {"collector_name": collector_name, "status": "failed"}),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
|
||||
128
backend/tests/test_ai_observability.py
Normal file
128
backend/tests/test_ai_observability.py
Normal file
@@ -0,0 +1,128 @@
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.services.ai_tools import web_search as web_search_module
|
||||
from app.services.ai_tools.schemas import SearchEvidence, WebSearchConfig, WebSearchProviderConfig
|
||||
from app.services.ai_tools.web_search import WebSearchClient
|
||||
from app.services import ai_client as ai_client_module
|
||||
from app.services.ai_client import AIProviderClient
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ai_client_analyze_logs_summary_without_prompt(monkeypatch):
|
||||
events = []
|
||||
|
||||
async def fake_emit_business_log(_logger, **payload):
|
||||
events.append(payload)
|
||||
|
||||
async def fake_request(self, method, path, json=None, request_id=None, operation="request", payload_summary=None):
|
||||
return {
|
||||
"provider": "test-provider",
|
||||
"model": "test-model",
|
||||
"content": "ok",
|
||||
"content_blocks": [],
|
||||
"text_blocks": ["ok"],
|
||||
"thinking_blocks": [],
|
||||
"raw_response": {},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(ai_client_module, "emit_business_log", fake_emit_business_log)
|
||||
monkeypatch.setattr(AIProviderClient, "_request", fake_request)
|
||||
|
||||
client = AIProviderClient(
|
||||
service_url="http://provider.test",
|
||||
llm_config={"provider": "openai", "provider_api": "openai-completions", "model": "gpt-test", "api_key": "sk-secret"},
|
||||
)
|
||||
result = await client.analyze(
|
||||
SituationalAnalysisRequest(
|
||||
title="Sensitive title",
|
||||
objective="Do not store this full prompt",
|
||||
observations=["secret observation"],
|
||||
constraints=["secret constraint"],
|
||||
context={"source": "test", "private": "value"},
|
||||
),
|
||||
request_id="req-ai-test",
|
||||
)
|
||||
|
||||
assert result.model == "test-model"
|
||||
assert [event["event"] for event in events] == [
|
||||
"ai.provider.analyze.start",
|
||||
"ai.provider.analyze.success",
|
||||
]
|
||||
serialized = str(events)
|
||||
assert "Do not store this full prompt" not in serialized
|
||||
assert "secret observation" not in serialized
|
||||
assert "sk-secret" not in serialized
|
||||
start_context = events[0]["context"]
|
||||
assert start_context["model"] == "gpt-test"
|
||||
assert start_context["input_summary"]["objective_length"] == len("Do not store this full prompt")
|
||||
assert start_context["input_summary"]["observation_count"] == 1
|
||||
assert start_context["input_summary"]["context_keys"] == ["private", "source"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ai_client_analyze_logs_failure(monkeypatch):
|
||||
events = []
|
||||
|
||||
async def fake_emit_business_log(_logger, **payload):
|
||||
events.append(payload)
|
||||
|
||||
async def fake_request(self, method, path, json=None, request_id=None, operation="request", payload_summary=None):
|
||||
raise HTTPException(status_code=502, detail="provider failed")
|
||||
|
||||
monkeypatch.setattr(ai_client_module, "emit_business_log", fake_emit_business_log)
|
||||
monkeypatch.setattr(AIProviderClient, "_request", fake_request)
|
||||
|
||||
client = AIProviderClient(service_url="http://provider.test", llm_config={"provider": "openai", "model": "gpt-test"})
|
||||
|
||||
with pytest.raises(HTTPException):
|
||||
await client.analyze(
|
||||
SituationalAnalysisRequest(title="T", objective="O", observations=["one"]),
|
||||
request_id="req-ai-fail",
|
||||
)
|
||||
|
||||
assert events[-1]["event"] == "ai.provider.analyze.failed"
|
||||
assert events[-1]["level"] == "error"
|
||||
assert events[-1]["context"]["error_type"] == "HTTPException"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_web_search_logs_query_hash_without_query(monkeypatch):
|
||||
events = []
|
||||
|
||||
async def fake_emit_business_log(_logger, **payload):
|
||||
events.append(payload)
|
||||
|
||||
async def fake_search_tavily(self, config, query, max_results, domains, freshness_days):
|
||||
return [
|
||||
SearchEvidence(
|
||||
title="Example",
|
||||
url="https://example.test",
|
||||
snippet="result",
|
||||
source_provider="tavily",
|
||||
)
|
||||
]
|
||||
|
||||
monkeypatch.setattr(web_search_module, "emit_business_log", fake_emit_business_log)
|
||||
monkeypatch.setattr(WebSearchClient, "_search_tavily", fake_search_tavily)
|
||||
|
||||
client = WebSearchClient(
|
||||
WebSearchConfig(
|
||||
enabled=True,
|
||||
default_provider="tavily",
|
||||
providers={"tavily": WebSearchProviderConfig(provider="tavily", api_key="secret-key")},
|
||||
)
|
||||
)
|
||||
results = await client.search("secret query text", max_results=1)
|
||||
|
||||
assert len(results) == 1
|
||||
assert [event["event"] for event in events] == [
|
||||
"ai_tool.web_search.start",
|
||||
"ai_tool.web_search.success",
|
||||
]
|
||||
serialized = str(events)
|
||||
assert "secret query text" not in serialized
|
||||
assert "secret-key" not in serialized
|
||||
assert events[0]["context"]["query_length"] == len("secret query text")
|
||||
assert events[1]["context"]["result_count"] == 1
|
||||
@@ -1,9 +1,12 @@
|
||||
"""Unit tests for data collectors"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
||||
from app.services.collectors.celestrak import CelesTrakTLECollector
|
||||
from app.services.credential_guides import DEFAULT_CREDENTIAL_GUIDES
|
||||
from app.services.collectors.top500 import TOP500Collector
|
||||
from app.services.collectors.registry import collector_registry
|
||||
@@ -149,6 +152,77 @@ class TestHTTPCollector:
|
||||
assert callable(collector.parse_response)
|
||||
|
||||
|
||||
class TestCelesTrakTLECollector:
|
||||
def test_transform_uses_norad_as_source_id_and_preserves_starlink_group(self):
|
||||
collector = CelesTrakTLECollector()
|
||||
result = collector.transform([
|
||||
{
|
||||
"NORAD_CAT_ID": 44720,
|
||||
"OBJECT_NAME": "STARLINK-1000",
|
||||
"OBJECT_ID": "2019-029AZ",
|
||||
"EPOCH": "2026-03-13T00:00:00Z",
|
||||
"MEAN_MOTION": 15.79234567,
|
||||
"ECCENTRICITY": 0.0001234,
|
||||
"INCLINATION": 53.0,
|
||||
"RA_OF_ASC_NODE": 10.0,
|
||||
"ARG_OF_PERICENTER": 20.0,
|
||||
"MEAN_ANOMALY": 30.0,
|
||||
"_celestrak_query_group": "active",
|
||||
"_celestrak_source_url": "https://celestrak.example/gp.php?GROUP=active&FORMAT=json",
|
||||
}
|
||||
])
|
||||
|
||||
assert result[0]["source_id"] == "44720"
|
||||
assert result[0]["metadata"]["constellation_group"] == "starlink"
|
||||
assert result[0]["metadata"]["celestrak_query_group"] == "active"
|
||||
assert result[0]["metadata"]["norad_cat_id"] == 44720
|
||||
assert result[0]["metadata"]["tle_line1"]
|
||||
assert result[0]["metadata"]["tle_line2"]
|
||||
|
||||
def test_load_active_payload_rejects_invalid_records(self, tmp_path):
|
||||
collector = CelesTrakTLECollector()
|
||||
payload_path = tmp_path / "active.json"
|
||||
payload_path.write_text(json.dumps([{"OBJECT_NAME": "missing norad"}]), encoding="utf-8")
|
||||
|
||||
with pytest.raises(RuntimeError, match="invalid record"):
|
||||
collector._load_active_payload(payload_path)
|
||||
|
||||
def test_load_active_payload_accepts_complete_array(self, tmp_path):
|
||||
collector = CelesTrakTLECollector()
|
||||
payload_path = tmp_path / "active.json"
|
||||
payload_path.write_text(
|
||||
json.dumps([{"NORAD_CAT_ID": 25544, "OBJECT_NAME": "ISS (ZARYA)"}]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
records = collector._load_active_payload(payload_path)
|
||||
|
||||
assert records == [{"NORAD_CAT_ID": 25544, "OBJECT_NAME": "ISS (ZARYA)"}]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fetch_retries_and_raises_instead_of_returning_partial_data(self, monkeypatch, tmp_path):
|
||||
collector = CelesTrakTLECollector()
|
||||
collector._resolved_url = "https://celestrak.example/NORAD/elements/gp.php"
|
||||
attempts = 0
|
||||
|
||||
async def fake_download_file(*args, **kwargs):
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
raise RuntimeError("network interrupted")
|
||||
|
||||
async def fake_emit_business_log(*args, **kwargs):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(collector._downloader, "download_file", fake_download_file)
|
||||
monkeypatch.setattr("app.services.collectors.celestrak.emit_business_log", fake_emit_business_log)
|
||||
monkeypatch.setattr("app.services.collectors.celestrak.asyncio.sleep", AsyncMock())
|
||||
|
||||
with pytest.raises(RuntimeError, match="failed after retries"):
|
||||
await collector.fetch()
|
||||
|
||||
assert attempts == 3
|
||||
|
||||
|
||||
def test_aisstream_collector_is_registered():
|
||||
collector = collector_registry.get("aisstream_vessels")
|
||||
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import asyncio
|
||||
import inspect
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from app.api.v1 import datasources as datasources_api
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.task import CollectionTask
|
||||
from app.services import data_jobs
|
||||
from app.services import earth_layer_cache as earth_cache
|
||||
|
||||
|
||||
@@ -100,6 +102,71 @@ def test_serialize_datasource_row_includes_endpoint_when_requested():
|
||||
assert row["endpoint"] == "https://example.test/arcgis_landing_points"
|
||||
|
||||
|
||||
def test_serialize_datasource_row_separates_collector_running_from_delete_task():
|
||||
datasource = make_datasource(9, "top500", last_status="success")
|
||||
task = CollectionTask(
|
||||
id=44,
|
||||
datasource_id=9,
|
||||
source="top500",
|
||||
task_type="clear_data",
|
||||
status="running",
|
||||
phase="clearing_data",
|
||||
)
|
||||
|
||||
row = datasources_api.serialize_datasource_row(
|
||||
datasource,
|
||||
running_tasks={9: task},
|
||||
latest_tasks={9: task},
|
||||
record_counts={"top500": 500},
|
||||
endpoint_overrides={},
|
||||
config=object(),
|
||||
include_endpoint=False,
|
||||
)
|
||||
|
||||
assert row["is_task_active"] is True
|
||||
assert row["is_running"] is False
|
||||
assert row["task_type"] == "clear_data"
|
||||
assert row["task_status"] == "running"
|
||||
|
||||
|
||||
def test_cancel_queued_delete_task_finishes_immediately(monkeypatch):
|
||||
task = CollectionTask(
|
||||
id=45,
|
||||
datasource_id=9,
|
||||
source="top500",
|
||||
task_type="clear_data",
|
||||
status="queued",
|
||||
phase="queued",
|
||||
)
|
||||
|
||||
class FakeDb:
|
||||
async def commit(self):
|
||||
return None
|
||||
|
||||
async def refresh(self, _task):
|
||||
return None
|
||||
|
||||
async def fake_broadcast(_task):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(data_jobs, "_broadcast_task_update", fake_broadcast)
|
||||
|
||||
async def run():
|
||||
return await data_jobs.request_cancel_datasource_task(FakeDb(), task)
|
||||
|
||||
result = asyncio.run(run())
|
||||
|
||||
assert result.status == "cancelled"
|
||||
assert result.phase == "cancelled"
|
||||
assert result.completed_at is not None
|
||||
|
||||
|
||||
def test_clear_data_job_relies_on_db_outbox_instead_of_extra_earth_refresh_task():
|
||||
source = inspect.getsource(data_jobs._run_clear_data_job)
|
||||
|
||||
assert "enqueue_earth_refresh_job" not in source
|
||||
|
||||
|
||||
def test_invalidate_earth_layer_cache_for_source_covers_datasource_aliases(monkeypatch):
|
||||
patterns: list[str] = []
|
||||
|
||||
@@ -119,34 +186,36 @@ def test_invalidate_earth_layer_cache_for_source_covers_datasource_aliases(monke
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trigger_datasource_batch_skips_disabled_and_frequency_window(monkeypatch):
|
||||
now = datetime.now(timezone.utc)
|
||||
disabled = make_datasource(1, "aisstream_vessels", is_active=False)
|
||||
not_due = make_datasource(2, "telegeography_cables", last_run_at=now, frequency_minutes=120)
|
||||
due = make_datasource(3, "ris_live_bgp", last_run_at=now - timedelta(hours=2))
|
||||
triggered_sources: list[str] = []
|
||||
def test_trigger_datasource_batch_skips_disabled_and_frequency_window(monkeypatch):
|
||||
async def run():
|
||||
now = datetime.now(timezone.utc)
|
||||
disabled = make_datasource(1, "aisstream_vessels", is_active=False)
|
||||
not_due = make_datasource(2, "telegeography_cables", last_run_at=now, frequency_minutes=120)
|
||||
due = make_datasource(3, "ris_live_bgp", last_run_at=now - timedelta(hours=2))
|
||||
queued_sources: list[str] = []
|
||||
|
||||
async def fake_running_tasks(_db, _ids):
|
||||
return {}
|
||||
async def fake_running_tasks(_db, _ids):
|
||||
return {}
|
||||
|
||||
async def fake_latest_task_ids(_db, _ids):
|
||||
return {}
|
||||
async def fake_enqueue(_db, datasource, task_type, **_kwargs):
|
||||
queued_sources.append(datasource.source)
|
||||
return CollectionTask(id=100 + datasource.id, datasource_id=datasource.id, source=datasource.source, task_type=task_type, status="queued")
|
||||
|
||||
monkeypatch.setattr(datasources_api, "_load_latest_running_tasks", fake_running_tasks)
|
||||
monkeypatch.setattr(datasources_api, "_load_latest_task_ids", fake_latest_task_ids)
|
||||
monkeypatch.setattr(
|
||||
datasources_api,
|
||||
"run_collector_now",
|
||||
lambda source: triggered_sources.append(source) or True,
|
||||
)
|
||||
monkeypatch.setattr(datasources_api, "_load_latest_running_tasks", fake_running_tasks)
|
||||
monkeypatch.setattr(
|
||||
datasources_api,
|
||||
"enqueue_datasource_job",
|
||||
fake_enqueue,
|
||||
)
|
||||
|
||||
result = await datasources_api._trigger_datasource_batch(
|
||||
object(),
|
||||
[disabled, not_due, due],
|
||||
force=False,
|
||||
)
|
||||
result = await datasources_api._trigger_datasource_batch(
|
||||
object(),
|
||||
[disabled, not_due, due],
|
||||
force=False,
|
||||
)
|
||||
|
||||
assert [item["source"] for item in result["triggered"]] == ["ris_live_bgp"]
|
||||
assert {item["reason"] for item in result["skipped"]} == {"disabled", "within_frequency_window"}
|
||||
assert triggered_sources == ["ris_live_bgp"]
|
||||
assert [item["source"] for item in result["triggered"]] == ["ris_live_bgp"]
|
||||
assert {item["reason"] for item in result["skipped"]} == {"disabled", "within_frequency_window"}
|
||||
assert queued_sources == ["ris_live_bgp"]
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
"""Docs Gatekeeper API tests."""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.api.v1 import docs as docs_api
|
||||
from app.main import app
|
||||
from app.models.user import User
|
||||
from app.services.docs_gatekeeper import DOCS_METADATA
|
||||
|
||||
|
||||
def make_user(role: str = "viewer", groups: list[str] | None = None) -> User:
|
||||
@@ -43,17 +47,17 @@ async def test_public_catalog_only_for_anonymous_user():
|
||||
assert response.status_code == 200
|
||||
items = response.json()["items"]
|
||||
assert {item["access"] for item in items} == {"public"}
|
||||
assert {item["slug"] for item in items if item["lang"] == "zh"} == {
|
||||
zh_items = [item for item in items if item["lang"] == "zh"]
|
||||
assert [item["slug"] for item in zh_items] == [
|
||||
"overview",
|
||||
"quickstart",
|
||||
"manual",
|
||||
"quickstart",
|
||||
"faq",
|
||||
"location-pipeline-user",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_developer_catalog_includes_frontend_reference_docs():
|
||||
async def test_developer_catalog_includes_architecture_and_frontend_reference_docs():
|
||||
response = await get_json(
|
||||
"/api/v1/docs/catalog",
|
||||
make_user(role="viewer", groups=["docs_developer"]),
|
||||
@@ -61,6 +65,10 @@ async def test_developer_catalog_includes_frontend_reference_docs():
|
||||
|
||||
assert response.status_code == 200
|
||||
zh_slugs = {item["slug"] for item in response.json()["items"] if item["lang"] == "zh"}
|
||||
zh_items = [item for item in response.json()["items"] if item["lang"] == "zh"]
|
||||
assert [item["group"] for item in zh_items[:4]] == ["Overview", "Manual", "Manual", "Manual"]
|
||||
assert zh_items[4]["group"] == "Architecture"
|
||||
assert "platform-data-flows" in zh_slugs
|
||||
assert "naming-glossary" in zh_slugs
|
||||
assert "tactile-ui-components" in zh_slugs
|
||||
|
||||
@@ -68,10 +76,16 @@ async def test_developer_catalog_includes_frontend_reference_docs():
|
||||
@pytest.mark.asyncio
|
||||
async def test_anonymous_can_read_public_doc():
|
||||
response = await get_json("/api/v1/docs/zh/quickstart")
|
||||
manual_response = await get_json("/api/v1/docs/zh/manual")
|
||||
overview_response = await get_json("/api/v1/docs/zh/overview")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["access"] == "public"
|
||||
assert "快速开始" in response.json()["markdown"]
|
||||
assert manual_response.status_code == 200
|
||||
assert manual_response.json()["access"] == "public"
|
||||
assert overview_response.status_code == 200
|
||||
assert overview_response.json()["access"] == "public"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -133,3 +147,29 @@ async def test_unknown_language_slug_and_path_traversal_do_not_read_files():
|
||||
assert bad_lang.status_code == 404
|
||||
assert bad_slug.status_code == 404
|
||||
assert traversal.status_code == 404
|
||||
|
||||
|
||||
def test_public_docs_markdown_links_do_not_create_missing_docs_routes():
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
technical_root = repo_root / "docs" / "technical"
|
||||
registered_filenames = {entry.filename for entry in DOCS_METADATA}
|
||||
problems: list[str] = []
|
||||
|
||||
for markdown_path in sorted(technical_root.glob("*/*.md")):
|
||||
markdown = markdown_path.read_text(encoding="utf-8")
|
||||
for match in re.finditer(r"\[([^\]]+)]\(([^)]+\.md(?:#[^)]+)?)\)", markdown):
|
||||
label, href = match.group(1), match.group(2)
|
||||
if href.startswith(("http://", "https://", "mailto:")):
|
||||
continue
|
||||
|
||||
href_without_hash = href.split("#", 1)[0].replace("\\", "/")
|
||||
filename = Path(href_without_hash).name
|
||||
if "/docs/technical/" in href_without_hash:
|
||||
if filename not in registered_filenames:
|
||||
problems.append(f"{markdown_path.relative_to(repo_root)} links unregistered public doc {href!r} ({label})")
|
||||
continue
|
||||
|
||||
if href_without_hash.endswith(".md"):
|
||||
problems.append(f"{markdown_path.relative_to(repo_root)} links non-public markdown {href!r} ({label})")
|
||||
|
||||
assert problems == []
|
||||
|
||||
454
backend/tests/test_earth_db_change_listener.py
Normal file
454
backend/tests/test_earth_db_change_listener.py
Normal file
@@ -0,0 +1,454 @@
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from app.services.earth_db_change_listener import (
|
||||
EarthDbChangeDispatcher,
|
||||
EarthDbChangeListener,
|
||||
build_earth_update_from_db_payload,
|
||||
normalize_asyncpg_dsn,
|
||||
)
|
||||
|
||||
|
||||
def test_normalize_asyncpg_dsn_strips_sqlalchemy_driver():
|
||||
assert (
|
||||
normalize_asyncpg_dsn("postgresql+asyncpg://postgres:postgres@localhost:5432/planet_db")
|
||||
== "postgresql://postgres:postgres@localhost:5432/planet_db"
|
||||
)
|
||||
|
||||
|
||||
def test_build_earth_update_maps_known_sources_to_layers():
|
||||
satellite_update = build_earth_update_from_db_payload(
|
||||
{"source": "celestrak_tle", "operation": "DELETE", "entity_key": "25544"}
|
||||
)
|
||||
cable_update = build_earth_update_from_db_payload(
|
||||
{"source": "arcgis_cables", "operation": "UPDATE", "entity_key": "cable-1"}
|
||||
)
|
||||
compute_update = build_earth_update_from_db_payload(
|
||||
{"source": "top500", "operation": "DELETE", "entity_key": None}
|
||||
)
|
||||
landing_update = build_earth_update_from_db_payload(
|
||||
{"source": "telegeography_landing", "operation": "DELETE", "entity_key": None}
|
||||
)
|
||||
|
||||
assert satellite_update is not None
|
||||
assert satellite_update["layers"] == ["satellites"]
|
||||
assert cable_update is not None
|
||||
assert cable_update["layers"] == ["cables"]
|
||||
assert compute_update is not None
|
||||
assert compute_update["layers"] == ["computeCenters"]
|
||||
assert landing_update is not None
|
||||
assert landing_update["layers"] == ["cables"]
|
||||
assert landing_update["refresh_strategy"] == "clear_then_reload"
|
||||
|
||||
|
||||
def test_build_earth_update_maps_derived_tables_to_layers():
|
||||
bgp_update = build_earth_update_from_db_payload(
|
||||
{"table": "bgp_anomalies", "source": "ris_live_bgp", "operation": "DELETE", "entity_key": "a1"}
|
||||
)
|
||||
compute_update = build_earth_update_from_db_payload(
|
||||
{"table": "compute_center_locations", "source": "top500", "operation": "UPDATE", "entity_key": "top500_1"}
|
||||
)
|
||||
vessel_update = build_earth_update_from_db_payload(
|
||||
{"table": "vessel_position", "operation": "DELETE", "entity_key": "123456789"}
|
||||
)
|
||||
|
||||
assert bgp_update is not None
|
||||
assert bgp_update["source"] == "ris_live_bgp"
|
||||
assert bgp_update["table"] == "bgp_anomalies"
|
||||
assert bgp_update["layers"] == ["bgp"]
|
||||
assert bgp_update["refresh_strategy"] == "clear_then_reload"
|
||||
assert compute_update is not None
|
||||
assert compute_update["layers"] == ["computeCenters"]
|
||||
assert compute_update["refresh_strategy"] == "reload"
|
||||
assert vessel_update is not None
|
||||
assert vessel_update["source"] == "vessel_position"
|
||||
assert vessel_update["layers"] == ["vessels"]
|
||||
|
||||
|
||||
def test_build_earth_update_maps_interactable_delete_to_delta():
|
||||
update = build_earth_update_from_db_payload(
|
||||
{
|
||||
"table": "earth_interactables",
|
||||
"operation": "DELETE",
|
||||
"entity_keys": ["note-1"],
|
||||
"records_processed": 1,
|
||||
}
|
||||
)
|
||||
|
||||
assert update is not None
|
||||
assert update["entity"] == "interactable"
|
||||
assert update["action"] == "deleted"
|
||||
assert update["ids"] == ["note-1"]
|
||||
assert update["layers"] == ["interactables"]
|
||||
assert update["refresh_strategy"] == "delta"
|
||||
|
||||
|
||||
def test_build_earth_update_ignores_unmapped_sources():
|
||||
assert build_earth_update_from_db_payload({"source": "not_for_earth"}) is None
|
||||
|
||||
|
||||
def test_dispatcher_debounces_same_source_notifications():
|
||||
async def run():
|
||||
broadcasts = []
|
||||
invalidated = []
|
||||
|
||||
async def broadcast(payload):
|
||||
broadcasts.append(payload)
|
||||
|
||||
def invalidate(source):
|
||||
invalidated.append(source)
|
||||
return 2
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=invalidate,
|
||||
debounce_seconds=10,
|
||||
)
|
||||
dispatcher.handle_payload(
|
||||
{
|
||||
"table": "collected_data",
|
||||
"operation": "INSERT",
|
||||
"source": "arcgis_cables",
|
||||
"entity_key": "cable-1",
|
||||
"occurred_at": "2026-05-22T00:00:00Z",
|
||||
}
|
||||
)
|
||||
dispatcher.handle_payload(
|
||||
{
|
||||
"table": "collected_data",
|
||||
"operation": "UPDATE",
|
||||
"source": "arcgis_cables",
|
||||
"entity_key": "cable-2",
|
||||
"occurred_at": "2026-05-22T00:00:01Z",
|
||||
}
|
||||
)
|
||||
|
||||
await dispatcher.flush_all()
|
||||
|
||||
assert invalidated == ["arcgis_cables"]
|
||||
assert len(broadcasts) == 1
|
||||
assert broadcasts[0]["action"] == "database_changed"
|
||||
assert broadcasts[0]["source"] == "arcgis_cables"
|
||||
assert broadcasts[0]["layers"] == ["cables"]
|
||||
assert broadcasts[0]["records_processed"] == 2
|
||||
assert broadcasts[0]["operations"] == ["INSERT", "UPDATE"]
|
||||
assert broadcasts[0]["entity_keys"] == ["cable-1", "cable-2"]
|
||||
assert broadcasts[0]["cache_entries_invalidated"] == 2
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_dispatcher_handles_delete_notification_as_earth_update():
|
||||
async def run():
|
||||
broadcasts = []
|
||||
|
||||
async def broadcast(payload):
|
||||
broadcasts.append(payload)
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=lambda source: 1,
|
||||
debounce_seconds=10,
|
||||
)
|
||||
accepted = dispatcher.handle_notification(
|
||||
json.dumps(
|
||||
{
|
||||
"table": "collected_data",
|
||||
"operation": "DELETE",
|
||||
"source": "celestrak_tle",
|
||||
"entity_key": "sat:25544",
|
||||
"occurred_at": "2026-05-22T00:00:00Z",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
await dispatcher.flush_all()
|
||||
|
||||
assert accepted is True
|
||||
assert len(broadcasts) == 1
|
||||
assert broadcasts[0]["action"] == "database_changed"
|
||||
assert broadcasts[0]["source"] == "celestrak_tle"
|
||||
assert broadcasts[0]["layers"] == ["satellites"]
|
||||
assert broadcasts[0]["operation"] == "DELETE"
|
||||
assert broadcasts[0]["refresh_strategy"] == "clear_then_reload"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_dispatcher_debounces_bgp_derived_table_events_by_source():
|
||||
async def run():
|
||||
broadcasts = []
|
||||
|
||||
async def broadcast(payload):
|
||||
broadcasts.append(payload)
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=lambda source: 0,
|
||||
debounce_seconds=10,
|
||||
)
|
||||
dispatcher.handle_payload(
|
||||
{
|
||||
"event_id": 200,
|
||||
"table": "bgp_observations",
|
||||
"operation": "DELETE",
|
||||
"source": "ris_live_bgp",
|
||||
"records_processed": 2,
|
||||
}
|
||||
)
|
||||
dispatcher.handle_payload(
|
||||
{
|
||||
"event_id": 201,
|
||||
"table": "bgp_anomalies",
|
||||
"operation": "DELETE",
|
||||
"source": "ris_live_bgp",
|
||||
"records_processed": 3,
|
||||
}
|
||||
)
|
||||
|
||||
await dispatcher.flush_all()
|
||||
|
||||
assert len(broadcasts) == 1
|
||||
assert broadcasts[0]["source"] == "ris_live_bgp"
|
||||
assert broadcasts[0]["layers"] == ["bgp"]
|
||||
assert broadcasts[0]["records_processed"] == 5
|
||||
assert broadcasts[0]["refresh_strategy"] == "clear_then_reload"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_dispatcher_deduplicates_notify_and_outbox_by_event_id():
|
||||
async def run():
|
||||
broadcasts = []
|
||||
|
||||
async def broadcast(payload):
|
||||
broadcasts.append(payload)
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=lambda source: 0,
|
||||
debounce_seconds=10,
|
||||
)
|
||||
payload = {
|
||||
"event_id": 42,
|
||||
"table": "collected_data",
|
||||
"operation": "INSERT",
|
||||
"source": "celestrak_tle",
|
||||
"entity_key": "sat:42",
|
||||
"occurred_at": "2026-05-23T00:00:00Z",
|
||||
}
|
||||
|
||||
assert dispatcher.handle_payload(payload) is True
|
||||
assert dispatcher.handle_payload(dict(payload)) is False
|
||||
await dispatcher.flush_all()
|
||||
|
||||
assert len(broadcasts) == 1
|
||||
assert broadcasts[0]["records_processed"] == 1
|
||||
assert broadcasts[0]["entity_keys"] == ["sat:42"]
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_dispatcher_flushes_continuous_events_at_max_wait():
|
||||
async def run():
|
||||
broadcasts = []
|
||||
sleeps = []
|
||||
|
||||
async def broadcast(payload):
|
||||
broadcasts.append(payload)
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=lambda source: 0,
|
||||
debounce_seconds=10,
|
||||
max_wait_seconds=0.01,
|
||||
)
|
||||
dispatcher.handle_payload(
|
||||
{
|
||||
"event_id": 300,
|
||||
"table": "collected_data",
|
||||
"operation": "INSERT",
|
||||
"source": "arcgis_cables",
|
||||
"records_processed": 1,
|
||||
}
|
||||
)
|
||||
await asyncio.sleep(0.02)
|
||||
dispatcher.handle_payload(
|
||||
{
|
||||
"event_id": 301,
|
||||
"table": "collected_data",
|
||||
"operation": "UPDATE",
|
||||
"source": "arcgis_cables",
|
||||
"records_processed": 1,
|
||||
}
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
await dispatcher.flush_all()
|
||||
|
||||
assert len(broadcasts) == 1
|
||||
assert broadcasts[0]["records_processed"] == 2
|
||||
assert broadcasts[0]["debounce_ms"] >= 0
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_dispatcher_fast_flushes_delete_clear_then_reload():
|
||||
async def run():
|
||||
broadcasts = []
|
||||
|
||||
async def broadcast(payload):
|
||||
broadcasts.append(payload)
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=lambda source: 0,
|
||||
debounce_seconds=10,
|
||||
max_wait_seconds=10,
|
||||
)
|
||||
dispatcher.handle_payload(
|
||||
{
|
||||
"event_id": 310,
|
||||
"table": "collected_data",
|
||||
"operation": "DELETE",
|
||||
"source": "celestrak_tle",
|
||||
"records_processed": 10,
|
||||
}
|
||||
)
|
||||
await asyncio.sleep(0.08)
|
||||
|
||||
assert len(broadcasts) == 1
|
||||
assert broadcasts[0]["source"] == "celestrak_tle"
|
||||
assert broadcasts[0]["refresh_strategy"] == "clear_then_reload"
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_outbox_rows_are_consumed_after_successful_flush():
|
||||
async def run():
|
||||
class FakeConnection:
|
||||
def __init__(self):
|
||||
self.consumed_ids = []
|
||||
|
||||
async def fetch(self, _query, _limit):
|
||||
return [
|
||||
{
|
||||
"id": 501,
|
||||
"payload": {
|
||||
"event_id": 501,
|
||||
"table": "collected_data",
|
||||
"operation": "DELETE",
|
||||
"source": "celestrak_tle",
|
||||
"records_processed": 1,
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
async def execute(self, _query, consumed_ids):
|
||||
self.consumed_ids.extend(consumed_ids)
|
||||
|
||||
broadcasts = []
|
||||
|
||||
async def broadcast(payload):
|
||||
broadcasts.append(payload)
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=lambda source: 0,
|
||||
debounce_seconds=10,
|
||||
)
|
||||
listener = EarthDbChangeListener(
|
||||
dsn="postgresql://example/db",
|
||||
dispatcher=dispatcher,
|
||||
)
|
||||
connection = FakeConnection()
|
||||
listener._connection = connection
|
||||
|
||||
await listener._poll_outbox()
|
||||
|
||||
assert connection.consumed_ids == [501]
|
||||
assert len(broadcasts) == 1
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_outbox_rows_remain_unconsumed_when_flush_fails():
|
||||
async def run():
|
||||
class FakeConnection:
|
||||
def __init__(self):
|
||||
self.consumed_ids = []
|
||||
|
||||
async def fetch(self, _query, _limit):
|
||||
return [
|
||||
{
|
||||
"id": 601,
|
||||
"payload": {
|
||||
"event_id": 601,
|
||||
"table": "collected_data",
|
||||
"operation": "DELETE",
|
||||
"source": "celestrak_tle",
|
||||
"records_processed": 1,
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
async def execute(self, _query, consumed_ids):
|
||||
self.consumed_ids.extend(consumed_ids)
|
||||
|
||||
async def broadcast(_payload):
|
||||
raise RuntimeError("ws down")
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=lambda source: 0,
|
||||
debounce_seconds=10,
|
||||
)
|
||||
listener = EarthDbChangeListener(
|
||||
dsn="postgresql://example/db",
|
||||
dispatcher=dispatcher,
|
||||
)
|
||||
connection = FakeConnection()
|
||||
listener._connection = connection
|
||||
|
||||
try:
|
||||
await listener._poll_outbox()
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
assert connection.consumed_ids == []
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
def test_dispatcher_uses_aggregated_statement_record_count():
|
||||
async def run():
|
||||
broadcasts = []
|
||||
|
||||
async def broadcast(payload):
|
||||
broadcasts.append(payload)
|
||||
|
||||
dispatcher = EarthDbChangeDispatcher(
|
||||
broadcast_earth_update=broadcast,
|
||||
invalidate_cache=lambda source: 0,
|
||||
debounce_seconds=10,
|
||||
)
|
||||
dispatcher.handle_payload(
|
||||
{
|
||||
"event_id": 99,
|
||||
"table": "collected_data",
|
||||
"operation": "DELETE",
|
||||
"source": "top500",
|
||||
"records_processed": 100,
|
||||
"entity_keys": ["top500:1", "top500:2"],
|
||||
"occurred_at": "2026-05-23T00:00:00Z",
|
||||
}
|
||||
)
|
||||
|
||||
await dispatcher.flush_all()
|
||||
|
||||
assert len(broadcasts) == 1
|
||||
assert broadcasts[0]["source"] == "top500"
|
||||
assert broadcasts[0]["layers"] == ["computeCenters"]
|
||||
assert broadcasts[0]["records_processed"] == 100
|
||||
assert broadcasts[0]["entity_keys"] == ["top500:1", "top500:2"]
|
||||
|
||||
asyncio.run(run())
|
||||
79
backend/tests/test_earth_interactables.py
Normal file
79
backend/tests/test_earth_interactables.py
Normal file
@@ -0,0 +1,79 @@
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.models.earth_interactable import EarthInteractable
|
||||
from app.services import earth_interactables
|
||||
|
||||
|
||||
def make_interactable(**overrides):
|
||||
values = {
|
||||
"id": "poi-1",
|
||||
"layer": "places",
|
||||
"kind": "note",
|
||||
"label": "Test POI",
|
||||
"description": "A point on Earth",
|
||||
"latitude": 30.25,
|
||||
"longitude": 120.15,
|
||||
"altitude": None,
|
||||
"revision": 3,
|
||||
"properties": {"owner": "test"},
|
||||
"is_deleted": False,
|
||||
"created_at": datetime(2026, 5, 22, 1, 0, tzinfo=UTC),
|
||||
"updated_at": datetime(2026, 5, 22, 1, 5, tzinfo=UTC),
|
||||
"deleted_at": None,
|
||||
}
|
||||
values.update(overrides)
|
||||
return EarthInteractable(**values)
|
||||
|
||||
|
||||
def test_interactable_event_uses_object_delta_contract():
|
||||
record = make_interactable()
|
||||
|
||||
event = earth_interactables.build_interactable_event(
|
||||
action="updated",
|
||||
record=record,
|
||||
)
|
||||
|
||||
assert event["entity"] == "interactable"
|
||||
assert event["action"] == "updated"
|
||||
assert event["layer"] == "places"
|
||||
assert event["layers"] == ["interactables"]
|
||||
assert event["ids"] == ["poi-1"]
|
||||
assert event["revision"] == 3
|
||||
assert event["item"]["latitude"] == 30.25
|
||||
assert event["item"]["properties"] == {"owner": "test"}
|
||||
|
||||
|
||||
def test_interactable_geojson_omits_deleted_records():
|
||||
active = make_interactable(id="active")
|
||||
deleted = make_interactable(id="deleted", is_deleted=True)
|
||||
|
||||
payload = earth_interactables.interactables_to_geojson([active, deleted])
|
||||
|
||||
assert payload["type"] == "FeatureCollection"
|
||||
assert [feature["id"] for feature in payload["features"]] == ["active"]
|
||||
assert payload["features"][0]["geometry"] == {
|
||||
"type": "Point",
|
||||
"coordinates": [120.15, 30.25],
|
||||
}
|
||||
|
||||
|
||||
def test_interactable_cache_invalidation_clears_layer_and_all(monkeypatch):
|
||||
patterns = []
|
||||
|
||||
def fake_delete_pattern(pattern):
|
||||
patterns.append(pattern)
|
||||
return 1
|
||||
|
||||
monkeypatch.setattr(
|
||||
earth_interactables.earth_layer_cache,
|
||||
"delete_pattern",
|
||||
fake_delete_pattern,
|
||||
)
|
||||
|
||||
deleted = earth_interactables.invalidate_interactable_cache("places")
|
||||
|
||||
assert deleted == 2
|
||||
assert patterns == [
|
||||
"earth:layer:v1:interactables:layer:places*",
|
||||
"earth:layer:v1:interactables:layer:all*",
|
||||
]
|
||||
@@ -421,6 +421,62 @@ async def test_earth_news_payload_uses_fresh_database_items_without_rss(monkeypa
|
||||
assert payload["stale"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_earth_news_payload_keeps_current_items_and_all_cruise_items(monkeypatch):
|
||||
class FakeDb:
|
||||
execute = object()
|
||||
|
||||
current_item = ParsedNewsItem(
|
||||
id="db:current",
|
||||
title="Current region story",
|
||||
summary="Current summary",
|
||||
url="https://example.com/current",
|
||||
source="Stored Source",
|
||||
feed_name="Stored Feed",
|
||||
feed_region="americas",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 15, 3, 0, tzinfo=UTC),
|
||||
)
|
||||
cruise_item = ParsedNewsItem(
|
||||
id="db:apac",
|
||||
title="APAC story",
|
||||
summary="APAC summary",
|
||||
url="https://example.com/apac",
|
||||
source="Stored Source",
|
||||
feed_name="Stored Feed",
|
||||
feed_region="asia-pacific",
|
||||
homepage_url="https://example.com",
|
||||
published_at=datetime(2026, 5, 15, 4, 0, tzinfo=UTC),
|
||||
)
|
||||
|
||||
async def fake_get_earth_news_freshness(_db, *, active_region):
|
||||
return 12, datetime.now(UTC)
|
||||
|
||||
async def fake_list_earth_news_items(_db, *, active_region, limit):
|
||||
return [current_item]
|
||||
|
||||
async def fake_list_earth_news_cruise_items(_db, *, limit):
|
||||
return [current_item, cruise_item]
|
||||
|
||||
async def fake_enqueue_target_location_job(_payload, **_kwargs):
|
||||
return True
|
||||
|
||||
async def fail_fetch(_sources):
|
||||
raise AssertionError("fresh database items should not fetch RSS")
|
||||
|
||||
monkeypatch.setattr("app.services.earth_news_store.get_earth_news_freshness", fake_get_earth_news_freshness)
|
||||
monkeypatch.setattr("app.services.earth_news_store.list_earth_news_items", fake_list_earth_news_items)
|
||||
monkeypatch.setattr("app.services.earth_news_store.list_earth_news_cruise_items", fake_list_earth_news_cruise_items)
|
||||
monkeypatch.setattr("app.services.earth_news_queue.enqueue_target_location_job", fake_enqueue_target_location_job)
|
||||
monkeypatch.setattr("app.services.earth_news._fetch_rss_items_for_sources", fail_fetch)
|
||||
|
||||
payload = await get_earth_news_payload(lat=35.0, lon=-100.0, db=FakeDb())
|
||||
|
||||
assert [item["id"] for item in payload["items"]] == ["db:current"]
|
||||
assert [item["id"] for item in payload["cruise_items"]] == ["db:current", "db:apac"]
|
||||
assert payload["cruise_items"][1]["region"] == "asia-pacific"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_earth_news_payload_initializes_empty_database_from_rss(monkeypatch):
|
||||
db = object()
|
||||
|
||||
@@ -4,8 +4,11 @@ import logging
|
||||
|
||||
from io import StringIO
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.logging import PlanetContextFilter, PlanetFormatter, get_logger
|
||||
from app.core.request_context import set_request_id
|
||||
from app.services import business_logs
|
||||
|
||||
|
||||
def _capture_output(callback):
|
||||
@@ -76,3 +79,43 @@ def test_structured_logger_redacts_sensitive_text_and_context():
|
||||
assert "hunter2" not in output
|
||||
assert "[REDACTED]" in output
|
||||
assert '"safe": "visible"' in output
|
||||
|
||||
|
||||
def test_business_context_redacts_nested_sensitive_values():
|
||||
context = business_logs.build_business_context(
|
||||
{
|
||||
"provider": "openai",
|
||||
"api_key": "sk-secret",
|
||||
"nested": {
|
||||
"token": "plain-token",
|
||||
"safe": "visible",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert context["api_key"] == "[REDACTED]"
|
||||
assert context["nested"]["token"] == "[REDACTED]"
|
||||
assert context["nested"]["safe"] == "visible"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_emit_business_log_persists_sanitized_system_event(monkeypatch):
|
||||
events = []
|
||||
|
||||
async def fake_record_system_log(**payload):
|
||||
events.append(payload)
|
||||
|
||||
monkeypatch.setattr(business_logs, "record_system_log", fake_record_system_log)
|
||||
|
||||
await business_logs.emit_business_log(
|
||||
get_logger("tests.business"),
|
||||
event="ai.provider.analyze.success",
|
||||
message="AI request completed",
|
||||
category="ai",
|
||||
context={"model": "gpt-test", "api_key": "sk-secret"},
|
||||
)
|
||||
|
||||
assert events[0]["event"] == "ai.provider.analyze.success"
|
||||
assert events[0]["category"] == "ai"
|
||||
assert events[0]["context"]["model"] == "gpt-test"
|
||||
assert events[0]["context"]["api_key"] == "[REDACTED]"
|
||||
|
||||
@@ -816,6 +816,27 @@ async def test_visualization_geo_summary_returns_counts(monkeypatch):
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_cable_layers_return_empty_feature_collections():
|
||||
class _ScalarResult:
|
||||
def scalars(self):
|
||||
class _Scalars:
|
||||
def all(self):
|
||||
return []
|
||||
|
||||
return _Scalars()
|
||||
|
||||
class _FakeSession:
|
||||
async def execute(self, _query):
|
||||
return _ScalarResult()
|
||||
|
||||
cables = await visualization_api._build_cables_geojson(_FakeSession())
|
||||
landing_points = await visualization_api._build_landing_points_geojson(_FakeSession())
|
||||
|
||||
assert cables == {"type": "FeatureCollection", "features": []}
|
||||
assert landing_points == {"type": "FeatureCollection", "features": []}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collect_location_endpoint_returns_candidates_for_known_record(monkeypatch):
|
||||
def _fake_ror(query):
|
||||
|
||||
@@ -15,12 +15,16 @@ services:
|
||||
retries: 10
|
||||
|
||||
aiprovider:
|
||||
image: ${AI_PROVIDER_IMAGE_NAME:-planet-aiprovider:latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: aiprovider/Dockerfile
|
||||
args:
|
||||
PYTHON_IMAGE: ${PYTHON_IMAGE:-python:3.14-slim}
|
||||
UV_IMAGE: ${UV_IMAGE:-ghcr.io/astral-sh/uv:latest}
|
||||
AI_PROVIDER_BUILD_FINGERPRINT: ${AI_PROVIDER_BUILD_FINGERPRINT:-unknown}
|
||||
secrets:
|
||||
- planet_uv_config
|
||||
container_name: planet_aiprovider
|
||||
ports:
|
||||
- "8010:8010"
|
||||
@@ -43,3 +47,7 @@ services:
|
||||
|
||||
volumes:
|
||||
ollama_data:
|
||||
|
||||
secrets:
|
||||
planet_uv_config:
|
||||
file: ${PLANET_UV_CONFIG_FILE:-/dev/null}
|
||||
|
||||
@@ -2,12 +2,16 @@ version: '3.8'
|
||||
|
||||
services:
|
||||
aiprovider:
|
||||
image: ${AI_PROVIDER_IMAGE_NAME:-planet-aiprovider:latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: aiprovider/Dockerfile
|
||||
args:
|
||||
PYTHON_IMAGE: ${PYTHON_IMAGE:-python:3.14-slim}
|
||||
UV_IMAGE: ${UV_IMAGE:-ghcr.io/astral-sh/uv:latest}
|
||||
AI_PROVIDER_BUILD_FINGERPRINT: ${AI_PROVIDER_BUILD_FINGERPRINT:-unknown}
|
||||
secrets:
|
||||
- planet_uv_config
|
||||
env_file:
|
||||
- ./aiprovider/.env
|
||||
- ${PLANET_AI_PROVIDER_RUNTIME_ENV_FILE:-./aiprovider/.env}
|
||||
@@ -43,3 +47,7 @@ services:
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
|
||||
secrets:
|
||||
planet_uv_config:
|
||||
file: ${PLANET_UV_CONFIG_FILE:-/dev/null}
|
||||
|
||||
@@ -2,12 +2,16 @@ version: '3.8'
|
||||
|
||||
services:
|
||||
aiprovider:
|
||||
image: ${AI_PROVIDER_IMAGE_NAME:-planet-aiprovider:latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: aiprovider/Dockerfile
|
||||
args:
|
||||
PYTHON_IMAGE: ${PYTHON_IMAGE:-python:3.14-slim}
|
||||
UV_IMAGE: ${UV_IMAGE:-ghcr.io/astral-sh/uv:latest}
|
||||
AI_PROVIDER_BUILD_FINGERPRINT: ${AI_PROVIDER_BUILD_FINGERPRINT:-unknown}
|
||||
secrets:
|
||||
- planet_uv_config
|
||||
env_file:
|
||||
- ./aiprovider/.env
|
||||
- ${PLANET_AI_PROVIDER_RUNTIME_ENV_FILE:-./aiprovider/.env}
|
||||
@@ -53,3 +57,7 @@ services:
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
|
||||
secrets:
|
||||
planet_uv_config:
|
||||
file: ${PLANET_UV_CONFIG_FILE:-/dev/null}
|
||||
|
||||
@@ -8,6 +8,56 @@ This project follows the repository versioning rule:
|
||||
- `improvement` -> `+0.0.1`(bugfix + 小功能混合)
|
||||
- `bugfix` -> `+0.0.1`
|
||||
|
||||
## [0.66.0] — 2026-05-26
|
||||
|
||||
Released: 2026-05-26
|
||||
|
||||
### Highlights
|
||||
- 将 Admin 正式化为唯一控制台入口,移除旧 AntD 后台、Admin Next 路由痕迹和相关依赖。
|
||||
- 引入 PostgreSQL 数据作业队列、Earth outbox 同步和可交互对象管线,让采集、清理和 Earth 刷新进入可追踪异步链路。
|
||||
- 强化 AI、AI 工具、设置连接测试和采集任务的结构化业务日志,关键事件可在系统日志中检索。
|
||||
- CelesTrak 轨道根数采集改为完整 `active` 目录下载、续传和重试,避免部分分组失败时保存不完整卫星数据。
|
||||
|
||||
### Added / Fixed / Improved
|
||||
- Admin 数据源页修复内置源调度启停按钮,使用 `is_active` 判断 `/enable` 与 `/disable`,避免采集状态误导调度开关。
|
||||
- Earth 新增通用 interactables 层和平台数据流文档,支持对象级 delta 同步与后续小图层扩展。
|
||||
- `planet.sh init/start/destroy` 补齐 uv 镜像回退、Bun unzip 依赖、HTTPS/LAN 跳转协议和 OOBE 清理语义。
|
||||
- Markdown code block、数据源队列、演示模式、About 版本展示、TV 链接协议和文档索引同步完成。
|
||||
|
||||
---
|
||||
|
||||
## [0.65.2] — 2026-05-22
|
||||
|
||||
Released: 2026-05-22
|
||||
|
||||
### Highlights
|
||||
- Stabilize AI Provider rebuild detection so unchanged source no longer rebuilds just because file timestamps or local cache state changed.
|
||||
- Keep `uv.lock` on the official registry while still allowing local and Docker builds to use the user's `uv.toml`.
|
||||
- Prevent local startup and bootstrap commands from rewriting `uv.lock` when a user-level mirror is configured.
|
||||
|
||||
### Added / Fixed / Improved
|
||||
- AI Provider fingerprints now use content hashes, base image inputs, dependency metadata, and an image label instead of file mtimes.
|
||||
- `planet.sh` and `scripts/bootstrap-dev.sh` now use `uv sync --frozen`, and backend startup uses `uv run --frozen`.
|
||||
- Docker Compose passes the AI Provider fingerprint into the image build so future starts can inspect the image label directly.
|
||||
|
||||
---
|
||||
|
||||
## [0.65.1] — 2026-05-22
|
||||
|
||||
Released: 2026-05-22
|
||||
|
||||
### Highlights
|
||||
- 修复 `planet.sh` 与 Docker Compose 对 AI Provider 镜像名不一致导致的重复构建问题。
|
||||
- 让本地 `uv` 与 Docker build 统一使用用户机器上的 `uv.toml`,避免把镜像源写入 `uv.lock`。
|
||||
- 通过 BuildKit secret 向容器构建传入 uv 配置,保留用户源选择且不把配置写入镜像层。
|
||||
|
||||
### Added / Fixed / Improved
|
||||
- `planet.sh` 自动解析 `UV_CONFIG_FILE`、仓库 `uv.toml` 与用户级 `~/.config/uv/uv.toml`,并清理会覆盖配置的 `UV_INDEX_URL` 环境变量。
|
||||
- AI Provider 和后端 Dockerfile 在 `uv sync` 阶段挂载 uv 配置 secret,并复用 BuildKit uv cache。
|
||||
- Docker Compose 三套配置统一声明 `planet-aiprovider:latest` 镜像名和 `planet_uv_config` build secret。
|
||||
|
||||
---
|
||||
|
||||
## [0.65.0] — 2026-05-21
|
||||
|
||||
Released: 2026-05-21
|
||||
|
||||
@@ -16,7 +16,24 @@
|
||||
- 已被正式实现替代、继续放在 `docs/` 根目录会误导后续开发的计划,归档
|
||||
- 仍然指导未来开发、尚未完成或仍有明确执行价值的文档,继续保留在 `docs/`
|
||||
|
||||
当前替代入口:
|
||||
|
||||
- 业务和数据产品链路见 [业务架构与数据流转](/home/ray/dev/linkong/planet/docs/technical/zh/platform-data-flows.md)。
|
||||
- 当前后端作业、outbox 和 Earth 同步实现见 [数据作业与 Outbox 技术架构](/home/ray/dev/linkong/planet/docs/technical/zh/data-job-earth-sync-architecture.md)。
|
||||
- 用户操作流程见 [Planet 使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/manual.md)。
|
||||
- 当前代码结构见 [技术文档索引](/home/ray/dev/linkong/planet/docs/technical/zh/README.md)。
|
||||
|
||||
补充说明:
|
||||
|
||||
- 一部分归档文档来自外部或临时工作流草案,例如 sisyphus 生成的初稿
|
||||
- 这类文档如果有可用内容,应先吸收到 `docs/plans/` 或 `docs/technical/`,再归档保留来源记录
|
||||
|
||||
## 近期归档
|
||||
|
||||
- [Docs Gatekeeper 鉴权系统计划](/home/ray/dev/linkong/planet/docs/deprecated/docs-gatekeeper-auth-plan.md):已落地,当前实现见技术文档。
|
||||
- [Location Resolver 共享管线计划](/home/ray/dev/linkong/planet/docs/deprecated/location-resolver-shared-pipeline-plan.md):已落地,当前实现见技术文档。
|
||||
- [Earth Surface Hover Info Plan](/home/ray/dev/linkong/planet/docs/deprecated/earth-surface-hover-info-plan.md):已实现,保留为历史记录。
|
||||
- [Admin Next Dual Track Full Migration Plan](/home/ray/dev/linkong/planet/docs/deprecated/admin-next-dual-track-full-migration-plan.md):已被当前 `frontend/src/admin/` 控制台结构替代。
|
||||
- [Admin Next Soft Glass Goal Driven Plan](/home/ray/dev/linkong/planet/docs/deprecated/admin-next-soft-glass-goal-driven-plan.md):已被当前 `frontend/src/admin/` 控制台结构替代。
|
||||
- [Admin Next Parity Checklist](/home/ray/dev/linkong/planet/docs/deprecated/admin-next-parity-checklist.md):旧 `/admin-next/*` 验收资料,保留为迁移历史。
|
||||
- [Admin Next Parity Audit Closeout](/home/ray/dev/linkong/planet/docs/deprecated/admin-next-parity-audit-closeout-plan.md):旧 `/admin-next/*` 审计资料,保留为迁移历史。
|
||||
|
||||
@@ -104,7 +104,7 @@ The old AntD page itself treated these Earth content tabs as placeholder-level c
|
||||
- `models_3d`
|
||||
- `news_anchor_strategy`
|
||||
|
||||
If backend endpoints are later added, these items must be promoted into `docs/plans/admin-next-parity-checklist.md` with concrete API and UI acceptance criteria.
|
||||
If backend endpoints are later added, these items must be promoted into a new active plan under `docs/plans/` with concrete API and UI acceptance criteria.
|
||||
|
||||
## Final Gate
|
||||
|
||||
@@ -114,7 +114,7 @@ After the route promotion, the final gate is no longer “switch old routes.”
|
||||
2. Run the static checks:
|
||||
- `rg "map: \\(\\) => \\[\\]|暂不支持保存|placeholder" frontend/src/admin-next`
|
||||
- `rg "ShadowPage|FeatureConsole|GlassPanel|InspectorDrawer" frontend/src/admin-next`
|
||||
3. Manually verify every official route listed in `docs/plans/admin-next-parity-checklist.md`.
|
||||
3. Manually verify every official route listed in `docs/deprecated/admin-next-parity-checklist.md`.
|
||||
4. Confirm `/legacy/admin/*` still opens old AntD pages during the validation window.
|
||||
5. Delete old AntD pages and remove AntD dependencies only as a separate final cleanup task after explicit confirmation.
|
||||
|
||||
@@ -8,7 +8,7 @@ The redesign must cover desktop and mobile. Data display, icon semantics, table
|
||||
|
||||
## Criteria For Success
|
||||
|
||||
- This plan exists at `docs/plans/admin-next-soft-glass-goal-driven-plan.md`.
|
||||
- This archived plan exists at `docs/deprecated/admin-next-soft-glass-goal-driven-plan.md`.
|
||||
- `/admin-next/*` has real pages for every route; route usage of `ShadowPage` is removed.
|
||||
- Admin Next supports `system`, `light`, and `dark` theme modes using the same persistence and system-theme idea as Docs.
|
||||
- The visual language reads as soft-glass / light-neumorphic instead of an AntD reskin: translucent panels, fine borders, subtle glow, cool backgrounds, restrained accent colors, crisp icons, and tactile controls.
|
||||
@@ -214,4 +214,3 @@ Manual viewport checks:
|
||||
- low height;
|
||||
- 125% / 150% browser zoom;
|
||||
- light / dark / system theme modes.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Location Resolver Shared Pipeline Plan
|
||||
|
||||
**状态**:已实现,当前用户流程见 [Earth 位置候选采集使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-user.md),开发接口见 [通用位置估算管线开发说明](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-development.md)。
|
||||
**状态**:已实现,当前用户流程见 [Planet 使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/manual.md) 的 Earth 位置候选采集章节,开发接口见 [通用位置估算管线开发说明](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-development.md)。
|
||||
|
||||
## Goal
|
||||
|
||||
@@ -37,7 +37,7 @@ If the same action has both a UI and a CLI path (e.g. user creation), describe t
|
||||
- a `credential_provider` in `backend/app/core/datasource_defaults.py`;
|
||||
- a default credential guide in `backend/app/services/credential_guides.py`;
|
||||
- a supported connectivity provider in `backend/app/services/datasource_connectivity.py`;
|
||||
- settings UI guidance or a credential form in `frontend/src/pages/Settings/Settings.tsx`;
|
||||
- settings UI guidance or a credential form in `frontend/src/admin/pages/PlainResourcePages.tsx`;
|
||||
- a regression test that fails if the guide/provider is missing.
|
||||
|
||||
## Recommended Checks
|
||||
|
||||
@@ -7,6 +7,12 @@
|
||||
- 分几期做
|
||||
- 当前差距和下一步是什么
|
||||
|
||||
当前实现、业务架构和数据链路不放在这里。它们分别进入:
|
||||
|
||||
- [业务架构与数据流转](/home/ray/dev/linkong/planet/docs/technical/zh/platform-data-flows.md)
|
||||
- [数据作业与 Outbox 技术架构](/home/ray/dev/linkong/planet/docs/technical/zh/data-job-earth-sync-architecture.md)
|
||||
- [技术文档索引](/home/ray/dev/linkong/planet/docs/technical/zh/README.md)
|
||||
|
||||
适合放入这里的内容:
|
||||
|
||||
- Earth / BGP / 地形 / 天球实施方案
|
||||
@@ -16,38 +22,40 @@
|
||||
|
||||
当前重点入口:
|
||||
|
||||
- [earth-mobile-drawer-ui-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-mobile-drawer-ui-plan.md)
|
||||
- [earth-compute-center-bgp-style-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-compute-center-bgp-style-plan.md)
|
||||
- [earth-renderer-architecture-separation-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-renderer-architecture-separation-plan.md)
|
||||
- [Earth Mobile Drawer UI Plan](/home/ray/dev/linkong/planet/docs/plans/earth-mobile-drawer-ui-plan.md)
|
||||
- [Earth Compute Center BGP Style Plan](/home/ray/dev/linkong/planet/docs/plans/earth-compute-center-bgp-style-plan.md)
|
||||
- [Earth Renderer Architecture Separation Plan](/home/ray/dev/linkong/planet/docs/plans/earth-renderer-architecture-separation-plan.md)
|
||||
- [Earth 高精度国界静态瓦片计划](/home/ray/dev/linkong/planet/docs/plans/earth-high-precision-boundary-tiles-plan.md)
|
||||
- [earth-predicted-orbit-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-predicted-orbit-plan.md)
|
||||
- [earth-webgl-instancing-satellites-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-webgl-instancing-satellites-plan.md)
|
||||
- [earth-real-terrain-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-real-terrain-plan.md)
|
||||
- [earth-news-source-configuration-and-collector-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-news-source-configuration-and-collector-plan.md)
|
||||
- [earth-news-cruise-summary-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-news-cruise-summary-plan.md)
|
||||
- [Earth Predicted Orbit Plan](/home/ray/dev/linkong/planet/docs/plans/earth-predicted-orbit-plan.md)
|
||||
- [Earth WebGL Instancing Satellites Plan](/home/ray/dev/linkong/planet/docs/plans/earth-webgl-instancing-satellites-plan.md)
|
||||
- [Earth Real Terrain Plan](/home/ray/dev/linkong/planet/docs/plans/earth-real-terrain-plan.md)
|
||||
- [Earth News Source Configuration And Collector Plan](/home/ray/dev/linkong/planet/docs/plans/earth-news-source-configuration-and-collector-plan.md)
|
||||
- [Earth News Cruise Summary Plan](/home/ray/dev/linkong/planet/docs/plans/earth-news-cruise-summary-plan.md)
|
||||
- [Earth 动作捕捉手势控制计划](/home/ray/dev/linkong/planet/docs/plans/earth-motion-capture-gesture-control-plan.md)
|
||||
- [Earth 动捕交互语义 V2 计划](/home/ray/dev/linkong/planet/docs/plans/earth-motion-gesture-interaction-v2-plan.md)
|
||||
- [Earth Presentation 解耦架构计划](/home/ray/dev/linkong/planet/docs/plans/earth-presentation-decoupled-architecture-plan.md)
|
||||
- [earth-vessel-rendering-performance-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-vessel-rendering-performance-plan.md)
|
||||
- [Earth Vessel Rendering Performance Plan](/home/ray/dev/linkong/planet/docs/plans/earth-vessel-rendering-performance-plan.md)
|
||||
- [AIS 多源采集、冲突记录与聚合接口计划](/home/ray/dev/linkong/planet/docs/plans/earth-vessel-ais-aggregation-plan.md)
|
||||
- [earth-interactable-layer-plan.md](/home/ray/dev/linkong/planet/docs/plans/earth-interactable-layer-plan.md)
|
||||
- [frontend-public-docs-site-plan.md](/home/ray/dev/linkong/planet/docs/plans/frontend-public-docs-site-plan.md)
|
||||
- [Docs Gatekeeper 鉴权系统计划](/home/ray/dev/linkong/planet/docs/plans/docs-gatekeeper-auth-plan.md)
|
||||
- [Location Resolver 共享管线计划](/home/ray/dev/linkong/planet/docs/plans/location-resolver-shared-pipeline-plan.md)
|
||||
- [frontend-ai-playground-development-plan.md](/home/ray/dev/linkong/planet/docs/plans/frontend-ai-playground-development-plan.md)
|
||||
- [Earth Interactable Layer Plan](/home/ray/dev/linkong/planet/docs/plans/earth-interactable-layer-plan.md)
|
||||
- [Frontend Public Docs Site Plan](/home/ray/dev/linkong/planet/docs/plans/frontend-public-docs-site-plan.md)
|
||||
- [Frontend AI Playground Development Plan](/home/ray/dev/linkong/planet/docs/plans/frontend-ai-playground-development-plan.md)
|
||||
- [AI Provider OpenClaw-Style Routing Plan](/home/ray/dev/linkong/planet/docs/plans/ai-provider-openclaw-style-routing-plan.md)
|
||||
- [统一集成配置 Schema 系统计划](/home/ray/dev/linkong/planet/docs/plans/integration-config-schema-system-plan.md)
|
||||
- [Lightweight Agent Orchestrator 与 WebSearch 证据层计划](/home/ray/dev/linkong/planet/docs/plans/agents-light-orchestrator-websearch-plan.md)
|
||||
- [Admin Next Parity Checklist](/home/ray/dev/linkong/planet/docs/plans/admin-next-parity-checklist.md)
|
||||
- [Admin Next Parity Audit Closeout](/home/ray/dev/linkong/planet/docs/plans/admin-next-parity-audit-closeout-plan.md)
|
||||
- [ue5-mvp-fused-plan.md](/home/ray/dev/linkong/planet/docs/plans/ue5-mvp-fused-plan.md)
|
||||
- [UE5 MVP Fused Plan](/home/ray/dev/linkong/planet/docs/plans/ue5-mvp-fused-plan.md)
|
||||
|
||||
历史计划入口:
|
||||
|
||||
- 已完成、已替代或只作为历史决策背景保留的文档,统一放在 [Deprecated Docs](/home/ray/dev/linkong/planet/docs/deprecated/README.md)。
|
||||
|
||||
不适合放入这里的内容:
|
||||
|
||||
- 当前代码结构说明
|
||||
- 组件现状和实现入口
|
||||
- 已经落地的技术上下文说明
|
||||
- 已经成为当前行为的“架构说明”
|
||||
|
||||
这些应放入:
|
||||
|
||||
- [docs/technical/README.md](/home/ray/dev/linkong/planet/docs/technical/README.md)
|
||||
- [技术文档索引](/home/ray/dev/linkong/planet/docs/technical/zh/README.md)
|
||||
- [Technical Docs Index](/home/ray/dev/linkong/planet/docs/technical/en/README.md)
|
||||
|
||||
@@ -178,5 +178,5 @@ Runtime 选择规则:
|
||||
- `backend/app/services/ai_client.py`
|
||||
- `aiprovider/main.py`
|
||||
- `aiprovider/provider_service.py`
|
||||
- `frontend/src/admin-next/pages/PlainResourcePages.tsx`
|
||||
- `docs/plans/admin-next-parity-audit-closeout-plan.md`
|
||||
- `frontend/src/admin/pages/PlainResourcePages.tsx`
|
||||
- `docs/deprecated/admin-next-parity-audit-closeout-plan.md`
|
||||
|
||||
@@ -1,46 +1,52 @@
|
||||
# Technical Docs
|
||||
|
||||
This directory holds "current implementation and current structure" documentation, focusing on:
|
||||
This is the current Planet documentation entry point. Docs are organized by reader path: start with business architecture to understand data products, then move into user manuals or implementation references.
|
||||
|
||||
- How the code is organized right now
|
||||
- Where the current entry points are
|
||||
- How state and components work
|
||||
- Which implementation boundaries future changes should follow
|
||||
## Business Architecture
|
||||
|
||||
What belongs here:
|
||||
- [Business Architecture and Data Flows](/home/ray/dev/linkong/planet/docs/technical/en/platform-data-flows.md): purpose, collection flow, fact tables, derived tables, cache, and WebSocket broadcast path for each Earth data product
|
||||
- [Naming Glossary](/home/ray/dev/linkong/planet/docs/technical/en/naming-glossary.md): English/Chinese terms used across the console, Earth, backend, and docs
|
||||
|
||||
- Quickstart and user manual
|
||||
- Frontend context
|
||||
- Earth frontend structure
|
||||
- Earth satellite footprint policy
|
||||
- Earth render layer order
|
||||
- Earth layer style property index
|
||||
- Backend runtime control
|
||||
- Collector status
|
||||
- Collector settings and connectivity validation
|
||||
- Earth Interactable integration
|
||||
- Collection format conventions
|
||||
## Manual
|
||||
|
||||
## Entry Points
|
||||
- [Quickstart](/home/ray/dev/linkong/planet/docs/technical/en/quickstart.md): shortest path to getting Planet running from scratch
|
||||
- [Planet Manual](/home/ray/dev/linkong/planet/docs/technical/en/manual.md): user workflows for the console, Earth, Docs, and common features
|
||||
- [FAQ](/home/ray/dev/linkong/planet/docs/technical/en/faq.md): troubleshooting for Windows / WSL, ports, dependencies, motion capture, credentials, and Docs permissions
|
||||
|
||||
- [Quickstart](/home/ray/dev/linkong/planet/docs/technical/en/quickstart.md): The shortest path to getting Planet running from scratch
|
||||
- [Planet Manual](/home/ray/dev/linkong/planet/docs/technical/en/manual.md): Complete usage guide for the console, `planet.sh`, Earth, and Docs
|
||||
- [FAQ](/home/ray/dev/linkong/planet/docs/technical/en/faq.md): Central troubleshooting entry for Windows / WSL, ports, dependencies, motion capture, credentials, and Docs permissions
|
||||
- [Earth Location Candidate Collection User Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-user.md): Collect and preview coordinate candidates for compute centers and BGP collectors on Earth
|
||||
- [Collector Settings and Connectivity Validation](/home/ray/dev/linkong/planet/docs/technical/en/datasource-collector-settings-connectivity.md): Data source catalog, collector settings, connectivity validation, and BarentsWatch credentials
|
||||
- [Shared Location Resolution Pipeline Development Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-development.md): Backend location resolver / pipeline interfaces, registries, and extension points
|
||||
- [Docs Gatekeeper Development Guide](/home/ray/dev/linkong/planet/docs/technical/en/docs-gatekeeper-development.md): Backend Docs catalog, Markdown content loading, and Gatekeeper permission groups
|
||||
- [Naming Glossary](/home/ray/dev/linkong/planet/docs/technical/en/naming-glossary.md): English/Chinese term mapping for the console, Earth, backend, and docs
|
||||
- [Earth Interactable Usage](/home/ray/dev/linkong/planet/docs/technical/en/earth-interactable-usage.md): API, lifecycle, and integration examples for Earth surface icon Interactable
|
||||
- [Earth Toolbar and Overlay Coordination](/home/ray/dev/linkong/planet/docs/technical/en/earth-toolbar-overlay-coordination.md): Closing matrix and integration rules for toolbar buttons, search, settings, news, and layer overlays
|
||||
- [Tactile UI Components](/home/ray/dev/linkong/planet/docs/technical/en/tactile-ui-components.md): Portable button, switch, tooltip, and scrollbar APIs, theme tokens, and migration rules
|
||||
## Earth Implementation
|
||||
|
||||
What does not belong here:
|
||||
- [Earth Frontend Context](/home/ray/dev/linkong/planet/docs/technical/en/earth-frontend-context.md): Earth modules, state, WebSocket refresh, and layer lifecycle
|
||||
- [Earth Layer Style Reference](/home/ray/dev/linkong/planet/docs/technical/en/earth-layer-style-reference.md): layer colors, symbols, materials, and visual parameters
|
||||
- [Earth Render Layer Order](/home/ray/dev/linkong/planet/docs/technical/en/earth-render-layer-order.md): renderOrder, depth strategy, picking, and collision avoidance
|
||||
- [Earth Satellite Footprint Policy](/home/ray/dev/linkong/planet/docs/technical/en/earth-satellite-footprint-policy.md): satellite footprint display boundaries and strategy
|
||||
- [BGP Context](/home/ray/dev/linkong/planet/docs/technical/en/earth-bgp-context.md): BGP rendering, aggregation, and collector implementation in Earth
|
||||
- [Earth Interactable Usage](/home/ray/dev/linkong/planet/docs/technical/en/earth-interactable-usage.md): `Interactable` API, lifecycle, and integration examples
|
||||
- [Earth Toolbar and Overlay Coordination](/home/ray/dev/linkong/planet/docs/technical/en/earth-toolbar-overlay-coordination.md): close matrix for toolbar buttons, search, settings, news, and layer overlays
|
||||
|
||||
- Incomplete roadmaps
|
||||
- Future iteration plans
|
||||
- Large-scale refactor proposals
|
||||
## Frontend Implementation
|
||||
|
||||
Those belong in:
|
||||
- [Admin Frontend Context](/home/ray/dev/linkong/planet/docs/technical/en/frontend-admin-frontend-context.md): console page structure, state boundaries, and lazy loading
|
||||
- [Frontend Layout Guidelines](/home/ray/dev/linkong/planet/docs/technical/en/frontend-layout-guidelines.md): layout, tables, panels, and responsive constraints
|
||||
- [Tactile UI Components](/home/ray/dev/linkong/planet/docs/technical/en/tactile-ui-components.md): button, switch, tooltip, scrollbar APIs, and theme tokens
|
||||
|
||||
- [Plans Index](/home/ray/dev/linkong/planet/docs/plans/README.md)
|
||||
## Backend Implementation
|
||||
|
||||
- [Data Collectors](/home/ray/dev/linkong/planet/docs/technical/en/backend-collectors.md): collectors, task types, save layer, and status updates
|
||||
- [System Service Control](/home/ray/dev/linkong/planet/docs/technical/en/backend-system-service-control.md): backend service control and system operation APIs
|
||||
- [Datasource Collector Settings and Connectivity](/home/ray/dev/linkong/planet/docs/technical/en/datasource-collector-settings-connectivity.md): datasource catalog, collector settings, connectivity validation, and credentials
|
||||
- [Datasource API Performance](/home/ray/dev/linkong/planet/docs/technical/en/backend-datasources-api-performance.md): DataSources list API performance and caching
|
||||
- [Data Jobs and Outbox Architecture](/home/ray/dev/linkong/planet/docs/technical/en/data-job-earth-sync-architecture.md): PostgreSQL job queue, outbox, listener, and Kafka / Spark evolution boundaries
|
||||
- [Shared Location Resolution Pipeline Development Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-development.md): location resolver / pipeline interfaces, registries, and extension points
|
||||
- [News Live Streams Collector Format](/home/ray/dev/linkong/planet/docs/technical/en/earth-news-live-streams-collector-format.md): news, live stream, and media collection payload conventions
|
||||
- [Docs Gatekeeper Development Guide](/home/ray/dev/linkong/planet/docs/technical/en/docs-gatekeeper-development.md): backend Docs catalog, Markdown content loading, and Gatekeeper permission groups
|
||||
|
||||
## Agents and Operations
|
||||
|
||||
- [AI Provider Guide](/home/ray/dev/linkong/planet/docs/technical/en/agents-aiprovider.md): model provider adapters, task prompts, and invocation boundaries
|
||||
- [Planet Ops Runbook](/home/ray/dev/linkong/planet/docs/technical/en/ops-runbook.md): deployment, startup, troubleshooting, and sensitive operations
|
||||
- [Docker + Compose + Buildx Upgrade](/home/ray/dev/linkong/planet/docs/technical/en/ops-docker-compose-buildx-upgrade.md): Docker toolchain upgrade steps
|
||||
- [planet.sh Startup](/home/ray/dev/linkong/planet/docs/technical/en/ops-planet-sh-startup.md): startup script, health checks, and performance optimization
|
||||
|
||||
## Plans and History
|
||||
|
||||
Incomplete roadmaps, large refactor proposals, and future plans live in the repository path `docs/plans/README.md`. Completed, outdated, or replaced designs live in `docs/deprecated/README.md`.
|
||||
|
||||
@@ -101,7 +101,7 @@ The AI settings page uses:
|
||||
|
||||
These endpoints require an authenticated user. The `secrets` endpoint is only used when the settings page reveals a key or token; hiding the field restores the masked preview.
|
||||
|
||||
Admin Next keeps the AI page aligned with the legacy information architecture:
|
||||
Admin keeps the AI page aligned with the legacy information architecture:
|
||||
|
||||
- `Model Providers`
|
||||
- Manages provider, wire adapter, default model, LLM API key, proxy URL, proxy token, model refresh, set-as-default, and lightweight connectivity testing.
|
||||
@@ -272,7 +272,7 @@ Each provider has its own key slot. Resolution order is:
|
||||
|
||||
`.env` is only a fallback. After the settings page saves successfully, or after the connection test succeeds, PostgreSQL becomes the global default source.
|
||||
|
||||
Admin Next must compute key status per provider or tool:
|
||||
Admin must compute key status per provider or tool:
|
||||
|
||||
- If the database has a key for the current provider/tool, show `configured`.
|
||||
- If the database has no key but the fallback provider, model, or tool matches the current item, show the fallback masked preview.
|
||||
@@ -283,7 +283,7 @@ Tool keys follow the same rule. WebSearch and OCR must match the current tool an
|
||||
|
||||
### Lightweight Connectivity Testing
|
||||
|
||||
The Admin Next plug button performs a lightweight connectivity check and does not save configuration. Common API-platform practice is two-tiered:
|
||||
The Admin plug button performs a lightweight connectivity check and does not save configuration. Common API-platform practice is two-tiered:
|
||||
|
||||
- Check a provider catalog or low-cost endpoint to validate base URL, authentication, and model reachability.
|
||||
- Send full model requests only when the user explicitly runs Playground or a business task.
|
||||
|
||||
@@ -73,6 +73,8 @@ async def run(self, db):
|
||||
|
||||
**Core file**: `backend/app/services/collectors/base.py`
|
||||
|
||||
Manual trigger, data clearing, and cache clearing now enter the PostgreSQL data job queue. `collection_tasks` remains the task ledger. Collectors only own `fetch -> transform -> save`; the `data_jobs.py` worker claims `collect` / `clear_data` / `clear_cache` / `earth_refresh` jobs and writes progress back. Earth layer refresh relationships live in `earth_layer_adapters.py`; do not hand-code cache invalidation or WebSocket broadcasts inside individual collectors or buttons.
|
||||
|
||||
## III. Collector List
|
||||
|
||||
| Collector | Data type | Content | Frequency |
|
||||
@@ -93,7 +95,7 @@ Earth boundaries are no longer data collectors. They are Earth static rendering
|
||||
|
||||
TOP500 and Epoch AI compute sources do not always provide usable coordinates. The unified Earth compute-center endpoint uses only valid source-provided coordinates or `compute_center_locations` dimension-table coordinates during the main map startup path; records without coordinates are returned as `unresolved` instead of being rendered from a local registry, country centroid, or guessed city. When users manually collect candidates, the backend queries ROR and Nominatim/OpenStreetMap from source fields; accepted candidates are saved into `compute_center_locations` and rendered from that table on the next layer refresh.
|
||||
|
||||
Admin Next collection management follows the business hierarchy instead of flattening every endpoint into one table:
|
||||
Admin collection management follows the business hierarchy instead of flattening every endpoint into one table:
|
||||
|
||||
- `Collectors`: endpoint, authentication, headers, base parameters, enabled state, and credential guides.
|
||||
- `Collection Schedule`: scheduler state and task controls.
|
||||
|
||||
@@ -96,4 +96,4 @@ if (res.data.task_id) {
|
||||
## Related Files
|
||||
|
||||
- [datasources.py](/home/ray/dev/linkong/planet/backend/app/api/v1/datasources.py): `_load_datasource_list_context`, `list_datasources`
|
||||
- [DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataSources/DataSources.tsx): `BuiltInDataSource`, `triggerDatasource`
|
||||
- [DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx): `BuiltInDataSource`, `triggerDatasource`
|
||||
|
||||
141
docs/technical/en/data-job-earth-sync-architecture.md
Normal file
141
docs/technical/en/data-job-earth-sync-architecture.md
Normal file
@@ -0,0 +1,141 @@
|
||||
# Data Jobs and Outbox Architecture
|
||||
|
||||
This document records the technical boundary for Planet v1 data jobs, database outbox, and Earth refresh. For each data product's business purpose and end-to-end flow, see [Business Architecture and Data Flows](/home/ray/dev/linkong/planet/docs/technical/en/platform-data-flows.md).
|
||||
|
||||
## Architecture Boundary
|
||||
|
||||
- PostgreSQL is the durable v1 job ledger and outbox. Kafka, Celery, and RQ are intentionally not part of v1.
|
||||
- `collection_tasks` records collection, data clearing, cache clearing, and non-database Earth refresh jobs.
|
||||
- `earth_data_change_events` records fact-table or derived-table changes and is the reliable source for Earth sync.
|
||||
- `LISTEN/NOTIFY` is only the low-latency wakeup path; the listener still polls unconsumed outbox rows.
|
||||
- Redis is mainly cache, auth helper, OTP / rate limit, temporary logs, and WebSocket support. It is not the durable queue.
|
||||
|
||||
## Database Change Sync
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Write["Fact or derived table write"] --> Trigger["PostgreSQL trigger"]
|
||||
Trigger --> Outbox["earth_data_change_events"]
|
||||
Trigger --> Notify["planet_earth_data_changes"]
|
||||
Outbox --> Listener["earth_db_change_listener"]
|
||||
Notify --> Listener
|
||||
Listener --> Adapter["earth_layer_adapters"]
|
||||
Adapter --> Cache["cache invalidation"]
|
||||
Cache --> WS["earth_updates"]
|
||||
```
|
||||
|
||||
1. Collection, clearing, location resolution, or projection jobs write fact or derived tables.
|
||||
2. Statement triggers write outbox rows for `INSERT` / `UPDATE` / `DELETE`.
|
||||
3. The listener wakes through notify or finds pending rows through polling.
|
||||
4. The listener maps `table + source` to Earth layers, refresh strategy, and cache patterns through `earth_layer_adapters.py`.
|
||||
5. The listener merges short-window same-layer events, invalidates cache, and broadcasts `earth_updates`.
|
||||
6. The outbox row is marked consumed only after successful broadcast; failed rows stay retryable.
|
||||
|
||||
DB changes no longer create default `earth_refresh` jobs, so they are not blocked by long same-source collection or clearing jobs. `earth_refresh` remains for manual cache clearing and non-DB refresh hints.
|
||||
|
||||
## Data Job Queue
|
||||
|
||||
`collection_tasks` is the unified job ledger. Workers claim `queued` jobs with PostgreSQL `FOR UPDATE SKIP LOCKED`; write jobs for the same `source` run serially, while different sources may run in parallel.
|
||||
|
||||
| task_type | Purpose |
|
||||
| --- | --- |
|
||||
| `collect` | Run a built-in datasource collector |
|
||||
| `clear_data` | Delete collected rows and declared derived rows for the source |
|
||||
| `clear_cache` | Delete Earth / dashboard cache for the source |
|
||||
| `earth_refresh` | Invalidate Earth layer cache and broadcast a refresh hint for non-DB changes |
|
||||
|
||||
API handlers only create jobs and return `task_id`. Execution, progress, cancellation, and terminal state are written back by workers and pushed to the frontend through the `datasource_tasks` channel.
|
||||
|
||||
Cancellation means “keep committed batches”: clicking stop marks the job as `cancelling` and cancels the in-memory coroutine. Already committed batches remain; unfinished batches follow the collector or cleanup rollback path.
|
||||
|
||||
## Earth Sync Event Model
|
||||
|
||||
The unified event model is `earth.layer.changed`:
|
||||
|
||||
```json
|
||||
{
|
||||
"event": "earth.layer.changed",
|
||||
"action": "database_changed",
|
||||
"source": "celestrak_tle",
|
||||
"table": "collected_data",
|
||||
"operation": "DELETE",
|
||||
"layers": ["satellites"],
|
||||
"refresh_strategy": "clear_then_reload",
|
||||
"records_processed": 11125,
|
||||
"occurred_at": "2026-05-25T10:20:30Z"
|
||||
}
|
||||
```
|
||||
|
||||
| strategy | Purpose |
|
||||
| --- | --- |
|
||||
| `clear_then_reload` | Clear local frontend layer objects first, then force a refetch. Prefer this for deletes. |
|
||||
| `reload` | Keep old objects until fresh data returns. Use it for location, metadata, or non-destructive updates. |
|
||||
| `delta` | Used only for `earth_interactables`; upsert or remove objects by id. |
|
||||
|
||||
APIs must return HTTP 200 with an empty collection for real zero-data states; 5xx is reserved for real endpoint failures. After a delete event, if refetch fails, the frontend should keep the cleared state and show a lightweight error instead of restoring stale objects.
|
||||
|
||||
## Layer Adapter Contract
|
||||
|
||||
`earth_layer_adapters.py` is the single registry for sources, derived tables, Earth layers, cache patterns, and refresh strategy. New layers should be added through an adapter entry, not through one-off button handlers, collector branches, or frontend special cases.
|
||||
|
||||
Each adapter must declare:
|
||||
|
||||
- Which source or table feeds which Earth layer.
|
||||
- Which Earth cache key patterns must be invalidated.
|
||||
- Which owned derived tables must be removed during `clear_data`.
|
||||
- The default refresh strategy for that layer.
|
||||
|
||||
When a source is cleared, the `clear_data` job first deletes `collected_data.source = <source>`, then deletes adapter-owned derived rows. Direct derived-table edits also trigger the outbox, so background jobs, admin APIs, and SQL repair scripts reach Earth as long as they mutate fact or derived tables.
|
||||
|
||||
## Operations and Troubleshooting
|
||||
|
||||
Check whether outbox rows are piling up:
|
||||
|
||||
```sql
|
||||
SELECT id, table_name, operation, source, occurred_at
|
||||
FROM earth_data_change_events
|
||||
WHERE consumed_at IS NULL
|
||||
ORDER BY id
|
||||
LIMIT 20;
|
||||
```
|
||||
|
||||
Check that triggers exist:
|
||||
|
||||
```sql
|
||||
SELECT tgname, tgrelid::regclass
|
||||
FROM pg_trigger
|
||||
WHERE tgname LIKE 'tr_planet_%_changed_%'
|
||||
ORDER BY 2, 1;
|
||||
```
|
||||
|
||||
Useful log events:
|
||||
|
||||
- `earth.db_changes.connected`: the listener connected to PostgreSQL and started listening.
|
||||
- `earth.db_changes.outbox_polled`: polling found unconsumed outbox rows.
|
||||
- `earth.db_changes.broadcasted`: an Earth refresh broadcast was produced.
|
||||
- `data_job.started` / `data_job.completed`: job execution state.
|
||||
|
||||
If Earth does not update, check in order: fact table changed, outbox was consumed, adapter covers the `source/table`, the listener is online, frontend WebSocket is connected, and the visualization API returns HTTP 200 with either an empty collection or fresh data.
|
||||
|
||||
## Kafka-ready Boundaries
|
||||
|
||||
Business code avoids depending on a concrete queue implementation by preserving these boundaries:
|
||||
|
||||
- `JobQueue`: submit, claim, cancel, and complete data jobs.
|
||||
- `DataChangeBus`: publish database fact changes.
|
||||
- `EarthLayerAdapterRegistry`: declare source, layer, cache, and derived-data relationships.
|
||||
|
||||
Kafka becomes appropriate when:
|
||||
|
||||
- Several independent services must consume the same data-change stream.
|
||||
- AIS, BGP, or sensor streams become sustained high-throughput inputs.
|
||||
- Consumer groups, replay, and service decoupling are required.
|
||||
|
||||
Spark becomes appropriate when:
|
||||
|
||||
- Historical data reaches tens or hundreds of millions of rows and PostgreSQL aggregation becomes expensive.
|
||||
- Cross-source, long-window, spatiotemporal analysis is needed.
|
||||
- Raw data lands in Parquet / Iceberg / Delta and the system starts producing offline derived data products.
|
||||
|
||||
For second-level continuous stream processing, evaluate Flink first. Spark is a better fit for batch or micro-batch analytics.
|
||||
|
||||
@@ -32,8 +32,8 @@ If endpoint, headers, base configuration, or credential fingerprint changes afte
|
||||
|
||||
Files:
|
||||
|
||||
- [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/pages/PlainResourcePages.tsx)
|
||||
- [AdminNextRoutes.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/AdminNextRoutes.tsx)
|
||||
- [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx)
|
||||
- [AdminRoutes.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/AdminRoutes.tsx)
|
||||
|
||||
Current behavior:
|
||||
|
||||
@@ -57,7 +57,7 @@ Current behavior:
|
||||
|
||||
File:
|
||||
|
||||
- [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/pages/PlainResourcePages.tsx)
|
||||
- [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx)
|
||||
|
||||
Current behavior:
|
||||
|
||||
|
||||
@@ -83,7 +83,7 @@ The user API:
|
||||
- Validates group names on update: only `docs_user`, `docs_developer`, and `docs_admin` are accepted.
|
||||
- Allows only `super_admin` to modify Gatekeeper groups.
|
||||
|
||||
Frontend [Users.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Users/Users.tsx) displays group tags and provides a multi-select in the edit form. Non-`super_admin` users see the field disabled, and submission removes `gatekeeper_groups` before sending.
|
||||
Frontend [Users.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/Users.tsx) displays group tags and provides a multi-select in the edit form. Non-`super_admin` users see the field disabled, and submission removes `gatekeeper_groups` before sending.
|
||||
|
||||
## Frontend Docs Loading
|
||||
|
||||
|
||||
@@ -188,7 +188,7 @@ Current reality:
|
||||
- that is expected, because incidents are aggregated and de-noised
|
||||
- but incident-first rendering makes the Earth view look too quiet unless there is another always-available activity layer
|
||||
|
||||
Implementation detail for the recommended `activity layer` is expanded in the [BGP Region Aggregation Plan](/home/ray/dev/linkong/planet/docs/plans/earth-bgp-region-aggregation-plan.md).
|
||||
Implementation detail for the recommended `activity layer` is kept in the repository path `docs/plans/earth-bgp-region-aggregation-plan.md`.
|
||||
|
||||
So the immediate next milestone is:
|
||||
|
||||
@@ -259,7 +259,7 @@ Reference inspiration:
|
||||
|
||||
Relevant page:
|
||||
|
||||
- `frontend/src/pages/BGP/BGP.tsx`
|
||||
- `frontend/src/admin/pages/PlainResourcePages.tsx`
|
||||
|
||||
Current BGP console page has three levels:
|
||||
|
||||
@@ -321,7 +321,7 @@ Backend:
|
||||
|
||||
Frontend:
|
||||
|
||||
- `frontend/src/pages/BGP/BGP.tsx`
|
||||
- `frontend/src/admin/pages/PlainResourcePages.tsx`
|
||||
- `frontend/public/earth/js/bgp.js`
|
||||
- `frontend/public/earth/js/main.js`
|
||||
- `frontend/public/earth/js/info-card.js`
|
||||
|
||||
@@ -4,7 +4,7 @@ This document describes the current real structure of the Earth display frontend
|
||||
|
||||
Related references:
|
||||
|
||||
- [Project Rules](/home/ray/dev/linkong/planet/rules.md)
|
||||
- Repository root `rules.md`
|
||||
- [Frontend Layout Guidelines](/home/ray/dev/linkong/planet/docs/technical/en/frontend-layout-guidelines.md)
|
||||
|
||||
## Current Goal
|
||||
@@ -56,6 +56,8 @@ Responsibilities:
|
||||
- Layer module integration
|
||||
- Earth-level state synchronization
|
||||
|
||||
Earth treats `/ws` `earth_updates` as refresh hints only; real data is fetched again from `/api/v1/visualization/...`. Backend database-driven refresh now has the listener clear cache and broadcast directly instead of going through the default `earth_refresh` job queue. After `database_changed`, the frontend applies the per-layer `clear_then_reload`, `reload`, or `delta` strategy. `clear_then_reload` must clear Three.js objects before a no-store refetch, and summary is only a consistency check, not a reason to skip a layer reload when the count is `0`. See [Data Jobs and Outbox Architecture](/home/ray/dev/linkong/planet/docs/technical/en/data-job-earth-sync-architecture.md) for the technical pipeline and [Business Architecture and Data Flows](/home/ray/dev/linkong/planet/docs/technical/en/platform-data-flows.md) for business data flows.
|
||||
|
||||
### 3. Earth Control Layer
|
||||
|
||||
- [controls.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js)
|
||||
@@ -151,11 +153,11 @@ Each module is responsible for its own:
|
||||
|
||||
`brand.js` manages Earth HUD brand resources. Static assets provide the default brand; runtime overrides come from `/api/v1/earth/brand`, and uploaded images are served from `/earth-brand-assets/...`. The frontend must treat logo/title images and text fallback separately: if an image fails, show the text title; if text fields are empty, rely on backend defaults so the HUD brand area never renders blank. The console Earth Content page owns saving and resetting brand configuration; the Earth frontend only consumes it.
|
||||
|
||||
`about.js` manages the About card inside Earth settings. Frontend defaults remain as a fallback, while runtime content is loaded from `/api/v1/earth/about`. If the request fails or fields are missing, the renderer must fall back per field so the settings page never renders an empty card. Admin Next exposes an Earth Content `About` tab; saving uses `PUT /api/v1/earth/about`, and restoring defaults uses `DELETE /api/v1/earth/about`.
|
||||
`about.js` manages the About card inside Earth settings. Frontend defaults remain as a fallback, while runtime content is loaded from `/api/v1/earth/about`. If the request fails or fields are missing, the renderer must fall back per field so the settings page never renders an empty card. Admin exposes an Earth Content `About` tab; saving uses `PUT /api/v1/earth/about`, and restoring defaults uses `DELETE /api/v1/earth/about`.
|
||||
|
||||
`oobe.js` manages the first-run Earth initialization guide. OOBE visibility must be driven by `/api/v1/earth/oobe-status` and its `ready` field, not by `localStorage`. `localStorage` may only store a short-lived "skip on this browser" flag; if the backend reports `ready: true`, logout, cleared browser storage, or a different browser must not show OOBE again. Desktop uses a dark starfield scrim and glass startup panel, while mobile uses a bottom sheet and respects `prefers-reduced-motion`.
|
||||
|
||||
The Admin Next Earth Content page must preserve runtime semantics:
|
||||
The Admin Earth Content page must preserve runtime semantics:
|
||||
|
||||
- `Brand`: brand preview should use the same dark starfield background, size, spacing, logo/title rendering, and text fallback as the Earth HUD top-left brand block, not a generic form preview.
|
||||
- `About`: configures the About card in Earth settings, including logo, kicker, title, version, description, and metadata items. Earth runtime reads `/earth/about` and falls back to defaults on failure.
|
||||
|
||||
@@ -17,26 +17,15 @@ Related context:
|
||||
|
||||
## Current Local Categories
|
||||
|
||||
Current CelesTrak satellite groups in [backend/app/services/collectors/celestrak.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/celestrak.py) include:
|
||||
The CelesTrak collector now downloads the complete active satellite catalog from `GROUP=active&FORMAT=json` instead of fetching several smaller groups and merging them. This prevents one failed CelesTrak group request from being saved as a successful but incomplete batch. The collector only proceeds when the downloaded JSON is a parseable array and records include `NORAD_CAT_ID`; network, resume, or parsing failures are retried, and final failure preserves the previous current dataset.
|
||||
|
||||
- `starlink`
|
||||
- `gps-ops`
|
||||
- `galileo`
|
||||
- `glonass`
|
||||
- `beidou`
|
||||
- `leo`
|
||||
- `geo`
|
||||
- `iridium-next`
|
||||
The collector still provides `metadata.constellation_group` to the frontend, but the value now comes from executable inference:
|
||||
|
||||
Non-Starlink categories:
|
||||
- `OBJECT_NAME` starting with `STARLINK` is marked as `starlink`
|
||||
- `OBJECT_NAME` starting with `IRIDIUM` is marked as `iridium-next`
|
||||
- Other active satellites are not forced into the old CelesTrak small-group labels, because a broad source group is not an exact constellation
|
||||
|
||||
- `gps-ops`
|
||||
- `galileo`
|
||||
- `glonass`
|
||||
- `beidou`
|
||||
- `leo`
|
||||
- `geo`
|
||||
- `iridium-next`
|
||||
The product policy therefore still discusses GNSS/RNSS, GEO, generic LEO, and Iridium NEXT semantics, but code should no longer assume that saved CelesTrak rows carry the old `gps-ops`, `galileo`, `glonass`, `beidou`, `leo`, or `geo` group labels.
|
||||
|
||||
## Research Conclusions
|
||||
|
||||
@@ -136,7 +125,8 @@ This implementation only does the minimum executable version and does not change
|
||||
|
||||
1. Backend passes constellation group and footprint policy hint to the frontend
|
||||
|
||||
- CelesTrak collector stores `GROUP` in `metadata.constellation_group`
|
||||
- CelesTrak collector stores the source query in `metadata.celestrak_query_group = active`
|
||||
- `metadata.constellation_group` only stores inferred business constellations such as `starlink` and `iridium-next`
|
||||
- Visualization API outputs:
|
||||
- `properties.constellation_group`
|
||||
- `properties.footprint_policy`
|
||||
|
||||
@@ -4,7 +4,7 @@ This document describes the current real structure of the console frontend. The
|
||||
|
||||
Related references:
|
||||
|
||||
- [Project Rules](/home/ray/dev/linkong/planet/rules.md)
|
||||
- Repository root `rules.md`
|
||||
- [Frontend Layout Guidelines](/home/ray/dev/linkong/planet/docs/technical/en/frontend-layout-guidelines.md)
|
||||
|
||||
## Current Goal
|
||||
@@ -22,7 +22,7 @@ Main entry point:
|
||||
|
||||
- [App.tsx](/home/ray/dev/linkong/planet/frontend/src/App.tsx)
|
||||
|
||||
Admin Next now owns the official admin routes:
|
||||
Admin now owns the official admin routes:
|
||||
|
||||
- `/admin`
|
||||
- `/users`
|
||||
@@ -37,23 +37,7 @@ Admin Next now owns the official admin routes:
|
||||
- `/collection-management`
|
||||
- `/settings`
|
||||
|
||||
These routes render [AdminNextRoutes.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/AdminNextRoutes.tsx). Page metadata and menu entries come from [manifest.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/routes/manifest.tsx). `/admin-next/*` remains only as a compatibility entry and redirects to the official route; it is no longer a parallel primary entry.
|
||||
|
||||
The old AntD console remains available under `/legacy/admin/*` for comparison and rollback:
|
||||
|
||||
- `/legacy/admin`
|
||||
- `/legacy/admin/datasources`
|
||||
- `/legacy/admin/data`
|
||||
- `/legacy/admin/collection-management`
|
||||
- `/legacy/admin/earth-content`
|
||||
- `/legacy/admin/ai`
|
||||
- `/legacy/admin/logs`
|
||||
- `/legacy/admin/settings`
|
||||
- `/legacy/admin/users`
|
||||
- `/legacy/admin/bgp`
|
||||
- `/legacy/admin/alerts/*`
|
||||
|
||||
Legacy pages, `AppLayout`, `antd`, and `@ant-design/icons` stay in place during the legacy validation window. Do not remove them before Admin Next parity is accepted.
|
||||
These routes render [AdminRoutes.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/AdminRoutes.tsx). Page metadata and menu entries come from [manifest.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/routes/manifest.tsx). Admin is the only console entry point; there is no parallel console or rollback route.
|
||||
|
||||
`/earth` is a standalone display page and is not part of the console shell.
|
||||
|
||||
@@ -61,7 +45,7 @@ Legacy pages, `AppLayout`, `antd`, and `@ant-design/icons` stay in place during
|
||||
|
||||
The official admin shell is at:
|
||||
|
||||
- [AdminNextLayout.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/components/layout/AdminNextLayout.tsx)
|
||||
- [AdminLayout.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/components/layout/AdminLayout.tsx)
|
||||
|
||||
Responsibilities:
|
||||
|
||||
@@ -69,38 +53,13 @@ Responsibilities:
|
||||
- Current account, version, logout, and theme switching
|
||||
- Top search, breadcrumbs, and page shortcuts
|
||||
- Single-screen content-area height closure
|
||||
- Coordination for Admin Next internal scrolling, tables, detail panels, and mobile detail views
|
||||
- Coordination for Admin internal scrolling, tables, detail panels, and mobile detail views
|
||||
|
||||
The old AntD legacy shell remains at:
|
||||
Future official admin pages should adapt to `AdminLayout` and Admin page patterns; do not reintroduce a parallel admin shell.
|
||||
|
||||
- [AppLayout.tsx](/home/ray/dev/linkong/planet/frontend/src/components/AppLayout/AppLayout.tsx)
|
||||
## Admin Section Loading
|
||||
|
||||
Legacy responsibilities:
|
||||
|
||||
- Left-side navigation
|
||||
- Collapse and expand
|
||||
- Current account / version information
|
||||
- Content area height closure
|
||||
- Site-wide unified sidebar scrollbar
|
||||
|
||||
Current structure:
|
||||
|
||||
```tsx
|
||||
<Layout className="dashboard-layout">
|
||||
<Sider className="dashboard-sider">...</Sider>
|
||||
<Layout>
|
||||
<Content className="dashboard-content">
|
||||
<div className="dashboard-content-inner">{children}</div>
|
||||
</Content>
|
||||
</Layout>
|
||||
</Layout>
|
||||
```
|
||||
|
||||
Future official admin pages should adapt to `AdminNextLayout` and Admin Next page patterns rather than adding new capability to the old `AppLayout`. Only `/legacy/admin/*` maintenance should change the old shell.
|
||||
|
||||
## Admin Next Section Loading
|
||||
|
||||
Multi-tab pages are currently coordinated by [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/pages/PlainResourcePages.tsx), which hosts the current management and information workbench patterns. Section loading follows these rules:
|
||||
Multi-tab pages are currently coordinated by [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx), which hosts the current management and information workbench patterns. Section loading follows these rules:
|
||||
|
||||
- Initial page load requests only the active tab; it does not prefetch every tab endpoint.
|
||||
- Switching tabs lazily loads that tab. Loaded tabs stay cached in local `states`, so returning to a tab reuses the previous data.
|
||||
@@ -112,20 +71,22 @@ This keeps Earth, AI, collection management, and other multi-section pages from
|
||||
|
||||
## Datasource Collection Queue
|
||||
|
||||
The Admin Next datasource page routes single-source trigger, table-selected trigger, and trigger-all into a browser-download-list style collection queue:
|
||||
The Admin datasource page routes single-source trigger, table-selected trigger, and trigger-all into a browser-download-list style collection queue:
|
||||
|
||||
- Queue state is managed by [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/pages/PlainResourcePages.tsx). It is a current-session visibility layer and does not fake task history in `localStorage`.
|
||||
- Queue state is managed by [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx). It is a current-session visibility layer and does not fake task history in `localStorage`.
|
||||
- Progress first consumes the `/ws` `datasource_tasks` channel. If the socket is unavailable or stale, the page polls `/api/v1/datasources/{id}/task-status`.
|
||||
- Trigger responses immediately insert `triggered`, `skipped`, and `failed` items. After a refresh, the queue restores only real backend rows that are still `running`, `pending`, or `queued`.
|
||||
- The `Built-in Sources` section uses the table selection column for selected-source triggering. With no rows selected, the primary button is `Trigger All`; after selection, the same button becomes `Trigger Selected N`, replacing the old manual-ID batch button.
|
||||
- The expanded queue no longer lives in the page content flow, so trigger-all cannot squeeze the table and detail panel. The top-right actions area uses the existing `Button` styling; the empty state shows a `ListChecks` icon, and active queues show only a pure circular total-progress indicator. Clicking it opens a floating panel grouped by running, failed, completed, and skipped.
|
||||
- Queue rows can jump to the datasource detail panel, and failed rows can retry. The detail panel's task summary only reports the selected source's latest task; it does not save configuration.
|
||||
- While a single-source collection is running, the action button becomes `Stop Collection`; unfinished queue rows expose a cancel action on the right. Cancellation calls `/api/v1/datasources/{source_id}/tasks/{task_id}/cancel`; the backend keeps committed batches and rolls back unfinished work.
|
||||
- Clearing database data and clearing display cache enter the same queue. The frontend shows task state and does not assume the API completed synchronously.
|
||||
|
||||
This queue is a user-perception layer. Backend task status remains the only source of truth for running, completion, failure, and skipped decisions.
|
||||
|
||||
## Admin Next Theme Slider
|
||||
## Admin Theme Slider
|
||||
|
||||
The Admin Next sidebar theme switcher still reuses shared [SegmentedControl.tsx](/home/ray/dev/linkong/planet/frontend/src/components/SegmentedControl/SegmentedControl.tsx), while [styles.css](/home/ray/dev/linkong/planet/frontend/src/admin-next/styles.css) overrides the segment variables by `data-theme`:
|
||||
The Admin sidebar theme switcher still reuses shared [SegmentedControl.tsx](/home/ray/dev/linkong/planet/frontend/src/components/SegmentedControl/SegmentedControl.tsx), while [styles.css](/home/ray/dev/linkong/planet/frontend/src/admin/styles.css) overrides the segment variables by `data-theme`:
|
||||
|
||||
- Light mode uses `--d-segment-bg: #eef3f9`, a white slider, and a light external shadow.
|
||||
- Dark mode uses the same dark base, `#202938` slider, and dark external shadow semantics as Docs.
|
||||
@@ -164,7 +125,7 @@ Purpose:
|
||||
|
||||
Current usage:
|
||||
|
||||
- Admin Next data sources, collected data, collection management, logs, alerts, and BGP pages
|
||||
- Admin data sources, collected data, collection management, logs, alerts, and BGP pages
|
||||
- Old AntD legacy pages continue using shared scrolling behavior through compatibility wrappers
|
||||
|
||||
### 3. `TableScrollRegion`
|
||||
@@ -188,7 +149,7 @@ Files:
|
||||
|
||||
Purpose:
|
||||
|
||||
- Admin Next global tool buttons and detail-panel toolbars
|
||||
- Admin global tool buttons and detail-panel toolbars
|
||||
- Icon-only ordinary actions with tooltips
|
||||
- Strong-intent actions such as save, create, confirm, delete, and stop
|
||||
- Compact switches aligned with the Docs theme slider
|
||||
@@ -246,39 +207,19 @@ Current constraints:
|
||||
- Internal document links should be converted to `/docs/:slug` through `transformLink`
|
||||
- Heading anchors are injected through `getHeadingId`, keeping route state outside the renderer
|
||||
|
||||
### 7. `ConnectionTestInput`
|
||||
### 7. Admin UI Primitives
|
||||
|
||||
File:
|
||||
Files:
|
||||
|
||||
- [ConnectionTestInput.tsx](/home/ray/dev/linkong/planet/frontend/src/components/ConnectionTestInput/ConnectionTestInput.tsx)
|
||||
- [button.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/components/ui/button.tsx)
|
||||
- [dialog.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/components/ui/dialog.tsx)
|
||||
- [switch.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/components/ui/switch.tsx)
|
||||
|
||||
Purpose:
|
||||
Use:
|
||||
|
||||
- Console form fields that combine an endpoint/Base URL value with a connection check
|
||||
- Connection-test entry points for AI Provider and WebSearch
|
||||
- Future collector configuration fields should reuse it when the test action belongs inside the input
|
||||
|
||||
Current constraints:
|
||||
|
||||
- The input suffix shows a single plug/connector icon, not an adjacent text button
|
||||
- Disabled integrations must grey out both the input and its connection-test action
|
||||
- The component only combines the input and action; callers still own form state, loading, disabled state, and the request itself
|
||||
|
||||
### 8. `TableActions`
|
||||
|
||||
File:
|
||||
|
||||
- [TableActions.tsx](/home/ray/dev/linkong/planet/frontend/src/components/TableActions/TableActions.tsx)
|
||||
|
||||
Purpose:
|
||||
|
||||
- Shared action entry for table operation columns
|
||||
- Shows inline actions when expanded
|
||||
- Uses a more-actions dropdown when collapsed
|
||||
|
||||
Companion export:
|
||||
|
||||
- `actionCellProps`: for action-column `onCell`, preventing action buttons from being ellipsized or wrapped
|
||||
- Global tool buttons, detail actions, confirmation dialogs, and binary settings.
|
||||
- Aligned with Tactile UI tokens so Admin controls keep consistent size, hover, disabled, and dark-mode behavior.
|
||||
- Row actions should prefer icon buttons plus tooltip/title; do not reintroduce a separate action-menu component.
|
||||
|
||||
## Current State Sources
|
||||
|
||||
@@ -301,7 +242,7 @@ Responsibilities:
|
||||
|
||||
File:
|
||||
|
||||
- [AISettings.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/AISettings/AISettings.tsx)
|
||||
- [AISettings.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx)
|
||||
|
||||
Responsibilities:
|
||||
|
||||
@@ -337,7 +278,7 @@ Constraints:
|
||||
|
||||
Example:
|
||||
|
||||
- [Dashboard.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Dashboard/Dashboard.tsx)
|
||||
- [Dashboard.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/Dashboard.tsx)
|
||||
|
||||
Priority goals:
|
||||
|
||||
@@ -349,10 +290,10 @@ Priority goals:
|
||||
|
||||
Examples:
|
||||
|
||||
- [DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataSources/DataSources.tsx)
|
||||
- [DataList.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataList/DataList.tsx)
|
||||
- [Users.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Users/Users.tsx)
|
||||
- [Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Settings/Settings.tsx)
|
||||
- [DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx)
|
||||
- [DataList.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/DataList.tsx)
|
||||
- [Users.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/Users.tsx)
|
||||
- [Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx)
|
||||
|
||||
Constraints:
|
||||
|
||||
@@ -362,7 +303,7 @@ Constraints:
|
||||
|
||||
### Datasource Directory Page
|
||||
|
||||
[DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataSources/DataSources.tsx) is the datasource directory and collection operation page. It should not grow back into a configuration editor.
|
||||
[DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx) is the datasource directory and collection operation page. It should not grow back into a configuration editor.
|
||||
|
||||
Current page boundary:
|
||||
|
||||
@@ -376,7 +317,9 @@ Keep this boundary: do not put custom datasource editing, built-in endpoint over
|
||||
|
||||
### Collectors Page
|
||||
|
||||
[Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Settings/Settings.tsx) has three route modes: `/settings` for System Settings, `/earth-content` for Earth Content, and `/collection-management` for Collection Management. The `collector_credentials` tab is shown as `Collectors` under `/collection-management`.
|
||||
[Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx) has three route modes: `/settings` for System Settings, `/earth-content` for Earth Content, and `/collection-management` for Collection Management. The `collector_credentials` tab is shown as `Collectors` under `/collection-management`.
|
||||
|
||||
The `System Display` section under `/settings` includes the `Demo Mode` switch. When enabled, Earth OOBE ignores existing current collected data and the local `browse first` temporary skip state, then opens the initialization guide directly. This switch is only for demos and acceptance checks; it does not change datasources, collection queues, or Earth content resources.
|
||||
|
||||
Current boundary:
|
||||
|
||||
@@ -388,7 +331,7 @@ Current boundary:
|
||||
|
||||
### Earth Content Page
|
||||
|
||||
`/earth-content` reuses the same single-screen tab container from [Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Settings/Settings.tsx), but its ownership is separate from System Settings:
|
||||
`/earth-content` reuses the same single-screen tab container from [Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx), but its ownership is separate from System Settings:
|
||||
|
||||
- `TV Livestream` owns the Earth media-panel source configuration.
|
||||
- `Boundary Precision` owns the Earth static boundary asset state: provider, low-precision fallback, high-precision manifest/PMTiles, source JSON, and build action.
|
||||
@@ -400,8 +343,8 @@ Do not add Earth experience resources or collection-lifecycle tabs back into `/s
|
||||
|
||||
Examples:
|
||||
|
||||
- [BGP.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/BGP/BGP.tsx)
|
||||
- [Playground.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Playground/Playground.tsx)
|
||||
- [BGP.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx)
|
||||
- [Playground.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx)
|
||||
|
||||
Constraints:
|
||||
|
||||
|
||||
@@ -8,7 +8,8 @@ Admin pages in this project default to a "single-screen workspace" layout standa
|
||||
|
||||
Current recommended reference implementations:
|
||||
|
||||
- [frontend/src/pages/BGP/BGP.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/BGP/BGP.tsx)
|
||||
- [frontend/src/admin/pages/PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx)
|
||||
- [frontend/src/admin/pages/DataList.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/DataList.tsx)
|
||||
- [frontend/src/index.css](/home/ray/dev/linkong/planet/frontend/src/index.css)
|
||||
|
||||
## Core Principles
|
||||
@@ -23,15 +24,15 @@ Admin pages default to:
|
||||
Recommended structure:
|
||||
|
||||
```tsx
|
||||
<AppLayout>
|
||||
<AdminLayout>
|
||||
<div className="page-shell">
|
||||
<div className="page-shell__header">...</div>
|
||||
<div className="page-shell__body">...</div>
|
||||
</div>
|
||||
</AppLayout>
|
||||
</AdminLayout>
|
||||
```
|
||||
|
||||
Total page height should be bounded within the `AppLayout` content area, not allowed to grow naturally downward without limit.
|
||||
Total page height should be bounded within the `AdminLayout` content area, not allowed to grow naturally downward without limit.
|
||||
|
||||
### 2. Scrolling Should Happen Inside Modules
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
`backend/app/services/location/` is the shared abstraction for any "given a record, decide its lat/lon" workflow. Compute centers, BGP collectors, and BGP events now run on this pipeline. Future entities such as satellite ground stations, user-claimed points, and IXP facilities should plug in here instead of creating another geocoding path.
|
||||
|
||||
For the user workflow, see [Earth Location Candidate Collection User Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-user.md).
|
||||
For the user workflow, see the Earth coordinate-candidate section in [Planet Manual](/home/ray/dev/linkong/planet/docs/technical/en/manual.md).
|
||||
|
||||
## Design Goals
|
||||
|
||||
|
||||
@@ -280,7 +280,17 @@ Search finds cables, landing points, satellites, compute centers, BGP events, BG
|
||||
|
||||
Compute center and BGP observer detail cards support automatic coordinate-candidate collection. Click the object then use "Collect Coordinate Candidates" or "Re-collect Coordinates". The backend assembles candidates from source coordinates, public-org registry APIs, and online geocoders. When regular sources have no candidate, the current default AI Provider runs one LLM factcheck fallback. BGP observers' stored coordinates only fill query context; they are not returned as candidates.
|
||||
|
||||
Candidates preview on Earth directly. Saving a compute-center candidate writes to the `compute_center_locations` dimension table and refreshes the layer immediately. The notification badge at the top-left of the compute-center layer shows the unresolved count; clicking it opens the queue, supports single collection, or "Adopt All" to save the top-confidence candidates from top to bottom. Records without candidates stay in the queue rather than being faked to country centroids. See [Earth Location Candidate Collection User Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-user.md).
|
||||
Candidates preview on Earth directly. Saving a compute-center candidate writes to the `compute_center_locations` dimension table and refreshes the layer immediately. The notification badge at the top-left of the compute-center layer shows the unresolved count; clicking it opens the queue, supports single collection, or "Adopt All" to save the top-confidence candidates from top to bottom. Records without candidates stay in the queue rather than being faked to country centroids.
|
||||
|
||||
Recommended single-object flow:
|
||||
|
||||
1. Open a compute center or BGP observer detail card.
|
||||
2. Click "Collect Coordinate Candidates".
|
||||
3. Wait for candidates; entries that depend on WebSearch / AI factcheck show collection state.
|
||||
4. Preview candidate positions on Earth.
|
||||
5. Save the candidate when it is credible; closing the card does not lose the current task state.
|
||||
|
||||
Adopt All is for batch processing the compute-center unresolved queue. It starts from the top and adopts the highest-confidence candidate. Records without factual support remain in the queue. When WebSearch is disabled, single locate and Adopt All are disabled because location validation depends on factual lookup.
|
||||
|
||||
### Settings
|
||||
|
||||
@@ -351,7 +361,7 @@ Mobile uses a drawer layout: layer control moves into a drawer; search/settings/
|
||||
|
||||
Docs at `http://localhost:3000/docs` are served by the backend with access control, not bundled into the frontend build.
|
||||
|
||||
Anonymous visitors see only `public` docs: README, Quickstart, Manual, FAQ, Earth Location Candidate Collection User Guide. Authenticated users with Gatekeeper groups see more:
|
||||
Anonymous visitors see only `public` docs: README, Quickstart, Manual, and FAQ. Authenticated users with Gatekeeper groups see more:
|
||||
|
||||
- `docs_user`: end-user operational docs
|
||||
- `docs_developer`: Earth, frontend, backend, collectors, AI Provider development docs
|
||||
@@ -365,5 +375,4 @@ Docs supports: category navigation, Markdown rendering, tables and code blocks,
|
||||
|
||||
- [Quickstart](/home/ray/dev/linkong/planet/docs/technical/en/quickstart.md)
|
||||
- [FAQ](/home/ray/dev/linkong/planet/docs/technical/en/faq.md)
|
||||
- [Earth Location Candidate Collection User Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-user.md)
|
||||
- [Planet Ops Runbook](/home/ray/dev/linkong/planet/docs/technical/en/ops-runbook.md)
|
||||
|
||||
@@ -15,7 +15,7 @@ This document standardizes terms used across the Planet console, Earth, backend
|
||||
| English / Key | Chinese Display Name | Usage |
|
||||
| --- | --- | --- |
|
||||
| Planet | Planet | Product name |
|
||||
| Admin Next | 新控制台 | New admin console |
|
||||
| Admin | 控制台 | Admin console context |
|
||||
| Earth | Earth | Visualization product name |
|
||||
| datasource | 数据源 | APIs, lists, filters |
|
||||
| collector | 采集器 | Collection jobs and credential configuration |
|
||||
|
||||
@@ -136,6 +136,20 @@ Useful for:
|
||||
- Demoing Earth from a phone or tablet
|
||||
- Other LAN machines reaching the same dev instance
|
||||
|
||||
On Windows, the repository-root `planet.cmd` can be used as a one-click entrypoint. It requests Administrator privileges, enters the `Ubuntu` WSL distribution at `/home/linkong/planet`, runs `./planet.sh restart --allow-lan`, opens `http://localhost:3000/earth` after a successful restart, and leaves the terminal inside a WSL shell for log inspection. If the local WSL distribution name or checkout path differs, adjust the `wsl.exe -d ... --cd ...` arguments in `planet.cmd` first.
|
||||
|
||||
On a new Windows machine, check the WSL generation first:
|
||||
|
||||
```powershell
|
||||
wsl -l -v
|
||||
```
|
||||
|
||||
Planet development should use WSL2. WSL1 has different networking, filesystem, and process behavior, and can surface as Bun package-manager commands returning only `An unknown error occurred (Unexpected)`, unstable port release, or LAN behavior that does not match the script's assumptions. Convert the distribution if it still runs as WSL1:
|
||||
|
||||
```powershell
|
||||
wsl --set-version Ubuntu 2
|
||||
```
|
||||
|
||||
`--allow-lan` directly exposes the frontend, backend, and AI Provider from the development machine: frontend `3000`, backend `8000`, and AI Provider `8010`. Before startup, the script checks all three ports. If WSL/Linux cannot release a port and a Windows-side listener or stale `portproxy` rule owns it, the script requests Administrator PowerShell cleanup. When Planet runs in WSL, Windows can usually reach it through `localhost`; other LAN machines reaching the Windows LAN IP still need Windows Firewall allow rules.
|
||||
|
||||
Diagnose in this order:
|
||||
@@ -200,13 +214,32 @@ The AI Provider image only rebuilds when code, Dockerfile, Compose config, or Py
|
||||
./planet.sh restart -a
|
||||
```
|
||||
|
||||
Rebuild detection is based on a content fingerprint rather than only file mtimes. `planet.sh` hashes the `aiprovider/` files, `aiprovider/Dockerfile`, `pyproject.toml`, `uv.lock`, `PYTHON_IMAGE`, `UV_IMAGE`, and the dependency fingerprint into `AI_PROVIDER_BUILD_FINGERPRINT`; Docker writes it into the image label `planet.aiprovider.build-fingerprint`. If the existing `planet-aiprovider:latest` image has a matching label, the script skips rebuild and refreshes the local stamp. Older images without the label fall back to the state/cache stamp.
|
||||
|
||||
Docker builds use `uv sync --frozen`. To make container builds reuse the host uv mirror configuration, the script resolves the first config file in this order and mounts it into the build as a BuildKit secret at `/root/.config/uv/uv.toml`:
|
||||
|
||||
1. The current `UV_CONFIG_FILE`
|
||||
2. Repository-root `uv.toml`
|
||||
3. `${XDG_CONFIG_HOME:-~/.config}/uv/uv.toml`
|
||||
4. `~/.uv/uv.toml`
|
||||
|
||||
If none exists, the script creates an empty state file for the secret so Compose does not fail on a missing file. Before Docker build it unsets `UV_DEFAULT_INDEX`, `UV_INDEX_URL`, and `UV_EXTRA_INDEX_URL`, keeping the build tied to the explicit `UV_CONFIG_FILE`. For a temporary Tsinghua mirror, place this in repository-root `uv.toml`:
|
||||
|
||||
```toml
|
||||
[[index]]
|
||||
name = "tsinghua"
|
||||
url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple/"
|
||||
default = true
|
||||
```
|
||||
|
||||
Diagnose slow builds:
|
||||
|
||||
| Symptom | Common cause | Fix |
|
||||
| --- | --- | --- |
|
||||
| Large `transferring context` | build context includes unrelated frontend / data files | `.dockerignore` ships only required files |
|
||||
| `uv sync` is slow | first build or cold cache | wait for the first build; later runs reuse BuildKit cache |
|
||||
| `uv sync --frozen` is slow | first build, cold cache, or missing uv mirror config | wait for the first build; later runs reuse BuildKit cache; configure `uv.toml` when needed |
|
||||
| Old keys still in effect after edit | container not restarted | `./planet.sh restart -a` |
|
||||
| Code changed but the image did not rebuild | fingerprint still matches the image label | Confirm the change is under `aiprovider/`, Dockerfile, or Python dependency inputs; delete `planet-aiprovider:latest` and retry if needed |
|
||||
|
||||
## SMTP Email (Required for Public Registration)
|
||||
|
||||
@@ -233,6 +266,15 @@ One-time codes are stored in Redis under `otp:{purpose}:{email}` with a 600-seco
|
||||
|
||||
## Development Command Conventions
|
||||
|
||||
Backend setup and script initialization use the lockfile:
|
||||
|
||||
```bash
|
||||
uv python install 3.14
|
||||
uv sync --frozen --group dev
|
||||
```
|
||||
|
||||
`--frozen` rejects implicit `uv.lock` rewrites, which is the desired behavior on new machines, CI, and Docker builds. Dependency upgrades should explicitly update `pyproject.toml` / `uv.lock` on a development machine and commit the lockfile.
|
||||
|
||||
Frontend must use Bun:
|
||||
|
||||
```bash
|
||||
|
||||
160
docs/technical/en/platform-data-flows.md
Normal file
160
docs/technical/en/platform-data-flows.md
Normal file
@@ -0,0 +1,160 @@
|
||||
# Business Architecture and Data Flows
|
||||
|
||||
This document is the business entry point for Planet data products. It explains why each Earth data type exists, where it is collected from, which fact or derived tables it uses, and how cache invalidation plus WebSocket hints reach Earth. Frontend, backend, and Earth docs should focus on implementation details; start here when you need the cross-system data flow.
|
||||
|
||||
## Overview
|
||||
|
||||
Planet data moves through three stages:
|
||||
|
||||
1. **Collect and normalize**: built-in collectors, admin actions, or the location pipeline write PostgreSQL. Generic raw output lands in `collected_data`; layer-ready projections land in derived tables.
|
||||
2. **Project and broadcast**: database triggers write fact changes to the `earth_data_change_events` outbox and wake the backend listener with `LISTEN/NOTIFY`. The listener maps the change through layer adapters, invalidates cache, and broadcasts `earth_updates`.
|
||||
3. **Reload and render in Earth**: Earth receives layer-level refresh hints, applies `clear_then_reload`, `reload`, or `delta`, then refetches `/api/v1/visualization/...`.
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Source["External source / admin action"] --> Collector["Collector or data job"]
|
||||
Collector --> Facts["collected_data"]
|
||||
Collector --> Derived["Derived tables"]
|
||||
Facts --> Outbox["earth_data_change_events"]
|
||||
Derived --> Outbox
|
||||
Outbox --> Listener["DB change listener"]
|
||||
Listener --> Cache["Earth cache invalidation"]
|
||||
Listener --> WS["earth_updates"]
|
||||
WS --> Earth["Earth layer reload"]
|
||||
Earth --> API["Visualization APIs"]
|
||||
```
|
||||
|
||||
`LISTEN/NOTIFY` is only the low-latency wakeup path. The reliable source is the outbox. A real zero-data state is valid and APIs should return HTTP 200 with an empty collection; only actual endpoint failures should return 5xx.
|
||||
|
||||
## Data Products
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
Satellites["Satellite TLE"] --> SatelliteLayer["satellites layer"]
|
||||
Cables["Cables + landing points"] --> CableLayer["cables layer"]
|
||||
Compute["TOP500 / AI GPU / HF"] --> ComputeLocations["compute_center_locations"]
|
||||
ComputeLocations --> ComputeLayer["computeCenters layer"]
|
||||
BgpRaw["RIS Live / BGPStream / Prefix"] --> BgpDerived["bgp_observations / anomalies / incidents"]
|
||||
BgpDerived --> BgpLayer["bgp layer"]
|
||||
VesselRaw["AIS / BarentsWatch"] --> VesselDerived["vessel_static / vessel_position"]
|
||||
VesselDerived --> VesselLayer["vessels layer"]
|
||||
Interactables["earth_interactables"] --> InteractableLayer["interactables layer"]
|
||||
NewsRaw["RSS / Live / News"] --> NewsItems["earth_news_items"]
|
||||
NewsItems --> NewsLayer["news / media layer"]
|
||||
```
|
||||
|
||||
| Data product | Business use | Fact sources | Derived / dimension tables | Earth layer | Strategy |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| Satellites | In-orbit objects, tracks, coverage, and cruise targets | `celestrak_tle`, `spacetrack_tle` | No long-lived projection; APIs convert TLE on demand | `satellites` | `clear_then_reload` |
|
||||
| Cables and landing points | Submarine connectivity, landing cities, cable detail, and search | `arcgis_cables`, `arcgis_landing_points`, TeleGeography / FAO landing sources | Cable relation and landing aggregation data | `cables` | `clear_then_reload` |
|
||||
| Compute centers | TOP500, AI GPU, model platform facilities, and location-completion state | `top500`, `epoch_ai_gpu`, HuggingFace sources | `compute_center_locations` | `computeCenters` | `reload` |
|
||||
| BGP context | Collectors, anomalies, incidents, route events, and regional context | `ris_live_bgp`, `bgpstream_bgp`, prefix geography sources | `bgp_observations`, `bgp_anomalies`, `bgp_incidents`, `bgp_collector_locations` | `bgp` | `clear_then_reload` |
|
||||
| Vessels | AIS vessels, positions, tracks, legend, and source health | AIS sources, `barentswatch_vessels` | `vessel_static`, `vessel_position`, `ais_raw_observations`, `ais_source_health` | `vessels` | `clear_then_reload` |
|
||||
| Interactables | Generic surface icons, manual objects, and future small layers | `earth_interactables` | None | `interactables` | `delta` |
|
||||
| News and media | Earth news, live streams, cruise summaries, and situation content | news sources | `earth_news_items` | `news` / `media` | `reload` |
|
||||
|
||||
## Satellites
|
||||
|
||||
Satellite data powers online satellite markers, tracks, footprint policy, and cruise lists. CelesTrak or Space-Track collectors fetch TLE and write `collected_data`. The visualization API converts TLE into current positions and trails on request.
|
||||
|
||||
- **Collection entry**: CelesTrak TLE, Space-Track TLE.
|
||||
- **Fact table**: `collected_data.source IN ('celestrak_tle', 'spacetrack_tle')`.
|
||||
- **API**: satellite visualization API reads TLE and builds the Earth payload.
|
||||
- **Delete semantics**: deleting the source broadcasts `satellites` with `clear_then_reload`; the frontend clears satellite markers and trails before refetching. With no TLE, the API returns an empty list.
|
||||
- **Common failure**: summary is zero but Earth still shows satellites. Usually WS did not fire, the adapter did not cover the source, or the frontend did not clear existing Three.js objects.
|
||||
|
||||
## Cables and Landing Points
|
||||
|
||||
Cables and landing points show global network connectivity, landing cities, cable details, and search targets. Cable lines and landing points are one business layer; deleting either side must refresh `cables`.
|
||||
|
||||
- **Collection entry**: ArcGIS cables, ArcGIS landing points, and TeleGeography / FAO landing related sources.
|
||||
- **Fact table**: cable and landing point sources in `collected_data`.
|
||||
- **Derived data**: cable relation tables, landing point aggregation, interface cache.
|
||||
- **API**: cable visualization API returns cables, landing points, and relation data.
|
||||
- **Delete semantics**: deleting cable or landing source clears owned projections and broadcasts `cables` with `clear_then_reload`. Empty data returns HTTP 200, not 404.
|
||||
- **Common failure**: cables disappear for a second and return. Usually a collection replacement or cache refresh window reuses old cache; check duplicate broadcasts and cache patterns.
|
||||
|
||||
## Compute Centers
|
||||
|
||||
Compute centers represent supercomputers, AI GPU sites, model-platform facilities, and unresolved-location state. Raw sources often only provide organization, country, site, or fuzzy location, so Earth rendering depends on `compute_center_locations`.
|
||||
|
||||
- **Collection entry**: TOP500, Epoch AI GPU, HuggingFace related sources.
|
||||
- **Fact table**: compute sources in `collected_data`.
|
||||
- **Dimension table**: `compute_center_locations` stores adopted coordinate candidates.
|
||||
- **API**: compute center visualization API merges raw records with location rows.
|
||||
- **Delete semantics**: deleting TOP500 or similar sources refreshes `computeCenters`; changing the location table also refreshes the layer. The default is `reload`, because location updates do not always need an immediate clear.
|
||||
- **Common failure**: collection finished but Earth count did not change. Check visualization cache, location rows, and adapter coverage.
|
||||
|
||||
## BGP
|
||||
|
||||
BGP data shows route collectors, anomalies, incidents, diffusion rings, and situation summaries. Earth does not render raw BGP rows directly; it renders projected observations, anomalies, and incidents. This is the most common path where deleting raw data can leave old objects visible.
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant UI as Admin delete button
|
||||
participant Job as clear_data job
|
||||
participant DB as PostgreSQL
|
||||
participant Outbox as earth_data_change_events
|
||||
participant Listener as DB change listener
|
||||
participant Earth as Earth frontend
|
||||
|
||||
UI->>Job: Submit delete for ris_live_bgp / bgpstream_bgp
|
||||
Job->>DB: Delete collected_data raw rows
|
||||
Job->>DB: Delete BGP-owned derived tables
|
||||
DB->>Outbox: trigger writes bgp layer changed
|
||||
DB-->>Listener: LISTEN/NOTIFY wakeup
|
||||
Listener->>Listener: merge events and clear bgp cache
|
||||
Listener-->>Earth: broadcast earth_updates clear_then_reload
|
||||
Earth->>Earth: clear BGP objects
|
||||
Earth->>DB: refetch projections through visualization API
|
||||
```
|
||||
|
||||
- **Collection entry**: RIPE RIS Live BGP, CAIDA BGPStream Backfill, IPtoASN / OpenGeoFeed / NRO prefix geography.
|
||||
- **Fact table**: BGP and prefix sources in `collected_data`.
|
||||
- **Derived tables**: `bgp_observations`, `bgp_anomalies`, `bgp_incidents`, `bgp_collector_locations`.
|
||||
- **API**: BGP visualization API reads projections and combines stored or candidate locations.
|
||||
- **Delete semantics**: deleting RIS Live or BGPStream raw sources must also clear BGP-owned projections and broadcast `bgp` with `clear_then_reload`. Direct derived-table deletes also write outbox events.
|
||||
- **Common failure**: collectors or incidents remain after raw rows are removed. Check derived tables first, not only `collected_data`.
|
||||
|
||||
## Vessels
|
||||
|
||||
Vessel data shows AIS vessels, navigation state, vessel-type legend, and source health. Earth rendering uses position snapshots and static vessel information; it should not render every raw AIS observation.
|
||||
|
||||
- **Collection entry**: AIS sources, BarentsWatch vessels.
|
||||
- **Fact table**: `collected_data` or AIS raw observation tables.
|
||||
- **Derived tables**: `vessel_static`, `vessel_position`, `ais_raw_observations`, `ais_source_health`.
|
||||
- **API**: vessel visualization API returns current vessel markers and detail fields.
|
||||
- **Delete semantics**: deleting any vessel source lets owned projections broadcast `vessels` with `clear_then_reload`.
|
||||
- **Common failure**: the count panel changes but vessels remain. Summary and layer data are separate; Earth should clear objects based on layer updates.
|
||||
|
||||
## Interactables
|
||||
|
||||
`earth_interactables` is the generic surface-icon capability for manual objects, extensions, and future small layers. Unlike most layers, it keeps object-level delta.
|
||||
|
||||
- **Fact table**: `earth_interactables`.
|
||||
- **API**: interactable API and Earth generic icon interface.
|
||||
- **Strategy**: upsert for create/update and removeItem for delete; no full layer reload.
|
||||
- **Delete semantics**: delete payloads must include a stable id so the frontend can remove the object.
|
||||
- **Common failure**: an object cannot be removed. Usually the stable id is missing, the delta handler did not match the object type, or an older duplicate render path still exists.
|
||||
|
||||
## News and Media
|
||||
|
||||
News and media support the Earth news ticker, live stream panel, news cruise, and situation summaries. They are content refresh paths rather than stable geographic object layers, so they default to `reload`.
|
||||
|
||||
- **Collection entry**: RSS, live streams, news sources.
|
||||
- **Fact table**: news source rows in `collected_data`.
|
||||
- **Derived table**: `earth_news_items`.
|
||||
- **API**: news, live stream, and media content APIs.
|
||||
- **Delete semantics**: deleting news sources or `earth_news_items` broadcasts `news` / `media` reload; empty responses hide the corresponding content.
|
||||
- **Common failure**: the live panel shows stale content. Usually the media component ignored the layer update or the content API cache was not invalidated.
|
||||
|
||||
## Adding a New Layer
|
||||
|
||||
Add a new Earth data product in this order:
|
||||
|
||||
1. Define the business purpose and Earth layer name.
|
||||
2. Identify fact sources, fact tables, and derived tables.
|
||||
3. Register source/table coverage, cache patterns, owned derived cleanup, and default strategy in the backend layer adapter.
|
||||
4. Ensure the visualization API returns HTTP 200 with an empty collection for zero data.
|
||||
5. Keep the Earth frontend driven only by layer and strategy, not database table names.
|
||||
6. Update this document with the data product flow, then document implementation details in the relevant frontend, backend, or Earth technical doc.
|
||||
@@ -64,5 +64,5 @@ Open `/forgot-password`, enter your email, receive a code, then enter the code p
|
||||
|
||||
- Full UI walkthrough: [Planet Manual](/home/ray/dev/linkong/planet/docs/technical/en/manual.md)
|
||||
- Troubleshooting and configuration questions: [FAQ](/home/ray/dev/linkong/planet/docs/technical/en/faq.md)
|
||||
- Detailed Earth coordinate candidate flow: [Earth Location Candidate Collection User Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-user.md)
|
||||
- Detailed Earth coordinate candidate flow: see the Earth section in [Planet Manual](/home/ray/dev/linkong/planet/docs/technical/en/manual.md)
|
||||
- Deployment / operations commands: [Planet Ops Runbook](/home/ray/dev/linkong/planet/docs/technical/en/ops-runbook.md)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Tactile UI Components
|
||||
|
||||
Tactile UI is Planet's portable React control layer. It was extracted from the Admin Next button, switch, scrollbar, and tooltip work, but the components themselves do not depend on Admin Next, AntD, Radix, Tailwind, or `an-*` classes. The immediate goal is stable in-repo usage; the structure is intentionally close to something that can later be published as an npm package.
|
||||
Tactile UI is Planet's portable React control layer. It was extracted from the Admin button, switch, scrollbar, and tooltip work, but the components themselves do not depend on Admin, AntD, Radix, Tailwind, or `an-*` classes. The immediate goal is stable in-repo usage; the structure is intentionally close to something that can later be published as an npm package.
|
||||
|
||||
## Design Goals
|
||||
|
||||
@@ -51,19 +51,19 @@ Most controls also accept a `tactile` prop for local width, height, radius, back
|
||||
|
||||
## Portals and Dark Theme
|
||||
|
||||
`TactileTooltip` and the Admin Next `Dialog`, `Select`, and toast controls that use Tactile UI may render through a portal attached to `document.body`. Those nodes are not descendants of `.admin-next-theme-root[data-theme='dark']`, so dark tokens cannot rely only on an ancestor selector inside the Admin Next root.
|
||||
`TactileTooltip` and the Admin `Dialog`, `Select`, and toast controls that use Tactile UI may render through a portal attached to `document.body`. Those nodes are not descendants of `.admin-theme-root[data-theme='dark']`, so dark tokens cannot rely only on an ancestor selector inside the Admin root.
|
||||
|
||||
The Admin Next theme provider mirrors the active theme to `body[data-admin-next-theme]`. Shared styles need to support both selector paths:
|
||||
The Admin theme provider mirrors the active theme to `body[data-admin-theme]`. Shared styles need to support both selector paths:
|
||||
|
||||
```css
|
||||
[data-theme='dark'] .tui-button,
|
||||
body[data-admin-next-theme='dark'] .tui-button {
|
||||
body[data-admin-theme='dark'] .tui-button {
|
||||
--tui-surface: #172033;
|
||||
--tui-text: #e5edf8;
|
||||
}
|
||||
```
|
||||
|
||||
When adding a portal-based control, first check whether it renders into body. If it does, add a `body[data-admin-next-theme='dark']` branch in that component's style entry, or reuse the already covered `--tui-*` / `--an-*` tokens. Avoid hard-coding a one-off dark modal style, because the same contrast problem can reappear in dropdowns, tooltips, toasts, and confirmation dialogs.
|
||||
When adding a portal-based control, first check whether it renders into body. If it does, add a `body[data-admin-theme='dark']` branch in that component's style entry, or reuse the already covered `--tui-*` / `--an-*` tokens. Avoid hard-coding a one-off dark modal style, because the same contrast problem can reappear in dropdowns, tooltips, toasts, and confirmation dialogs.
|
||||
|
||||
## `TactileButton`
|
||||
|
||||
@@ -195,7 +195,7 @@ Do not force overlay scrollbars onto textareas. Text selection and the resize gr
|
||||
|
||||
## `TableScrollRegion`
|
||||
|
||||
`TableScrollRegion` is a convenience wrapper for table scroll areas. The default target selector is `.tui-scroll-target`; Admin Next passes its table viewport selector explicitly so package code does not contain product-specific names.
|
||||
`TableScrollRegion` is a convenience wrapper for table scroll areas. The default target selector is `.tui-scroll-target`; Admin passes its table viewport selector explicitly so package code does not contain product-specific names.
|
||||
|
||||
```tsx
|
||||
<TableScrollRegion targetSelector=".table-viewport">
|
||||
@@ -210,9 +210,9 @@ Do not force overlay scrollbars onto textareas. Text selection and the resize gr
|
||||
- Tooltip text is explanatory only; state must still be represented by text, badges, or `aria-*` attributes.
|
||||
- Disabled and loading states set `aria-disabled`; real `button` elements also receive `disabled`.
|
||||
|
||||
## Admin Next Migration Rules
|
||||
## Admin Migration Rules
|
||||
|
||||
Admin Next should use Tactile UI for global tool buttons, detail-panel toolbars, and list footer actions:
|
||||
Admin should use Tactile UI for global tool buttons, detail-panel toolbars, and list footer actions:
|
||||
|
||||
- Unambiguous actions: `TactileButton iconOnly tooltip`
|
||||
- Save/create/confirm: `TactileButton variant="primary"`, usually with text
|
||||
|
||||
@@ -1,44 +1,52 @@
|
||||
# 技术文档
|
||||
|
||||
这里放“当前实现和当前结构”的文档,重点回答:
|
||||
这里是 Planet 当前文档入口。文档按读者和问题类型分层:先看业务架构理解数据产品,再进入使用手册或技术实现文档。
|
||||
|
||||
- 现在代码是怎么组织的
|
||||
- 当前入口在哪
|
||||
- 状态和组件如何工作
|
||||
## 业务架构
|
||||
|
||||
适合放入这里的内容:
|
||||
- [业务架构与数据流转](/home/ray/dev/linkong/planet/docs/technical/zh/platform-data-flows.md):每类 Earth 数据的用途、采集链路、事实表、派生表、缓存和 WebSocket 广播链路
|
||||
- [命名与术语对照](/home/ray/dev/linkong/planet/docs/technical/zh/naming-glossary.md):控制台、Earth、后端和文档常见名词的中英对照
|
||||
|
||||
- 快速开始和使用手册
|
||||
- 前端上下文
|
||||
- Earth 前端结构
|
||||
- Earth 卫星覆盖策略
|
||||
- Earth 渲染图层顺序
|
||||
- Earth 图层样式属性索引
|
||||
- 后端运行控制
|
||||
- 采集器现状
|
||||
- 采集器设置与连接验证
|
||||
- 采集格式约定
|
||||
|
||||
## 使用入口
|
||||
## 使用手册
|
||||
|
||||
- [快速开始](/home/ray/dev/linkong/planet/docs/technical/zh/quickstart.md):从零启动 Planet 的最短路径
|
||||
- [Planet 使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/manual.md):控制台、`planet.sh`、Earth 和 Docs 的完整使用手册
|
||||
- [常见问题](/home/ray/dev/linkong/planet/docs/technical/zh/faq.md):Windows / WSL、端口、依赖、动捕、凭证和 Docs 权限的集中排障入口
|
||||
- [Earth 位置候选采集使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-user.md):在 Earth 上为算力中心和 BGP 观测站采集、预览坐标候选
|
||||
- [数据源、采集器设置与连接验证](/home/ray/dev/linkong/planet/docs/technical/zh/datasource-collector-settings-connectivity.md):数据源目录、采集器设置、连接验证、BarentsWatch 凭证链路
|
||||
- [通用位置估算管线开发说明](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-development.md):后端 location resolver / pipeline 的接口、注册表和扩展方式
|
||||
- [Planet 使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/manual.md):控制台、Earth、Docs 和常用功能的用户操作说明
|
||||
- [常见问题](/home/ray/dev/linkong/planet/docs/technical/zh/faq.md):Windows / WSL、端口、依赖、动捕、凭证和 Docs 权限排障
|
||||
|
||||
## Earth 技术实现
|
||||
|
||||
- [Earth 前端结构](/home/ray/dev/linkong/planet/docs/technical/zh/earth-frontend-context.md):Earth 页面模块、状态、WebSocket 刷新和图层生命周期
|
||||
- [Earth 图层样式属性索引](/home/ray/dev/linkong/planet/docs/technical/zh/earth-layer-style-reference.md):图层颜色、符号、材质和视觉参数
|
||||
- [Earth 渲染图层顺序](/home/ray/dev/linkong/planet/docs/technical/zh/earth-render-layer-order.md):renderOrder、深度策略、拾取和同坐标避让
|
||||
- [Earth 卫星覆盖策略](/home/ray/dev/linkong/planet/docs/technical/zh/earth-satellite-footprint-policy.md):卫星 footprint 的显示边界和策略
|
||||
- [BGP 态势上下文](/home/ray/dev/linkong/planet/docs/technical/zh/earth-bgp-context.md):BGP 在 Earth 中的渲染、聚合和观测站实现
|
||||
- [Earth 可交互图标接入](/home/ray/dev/linkong/planet/docs/technical/zh/earth-interactable-usage.md):`Interactable` 的接口、生命周期和接入示例
|
||||
- [Earth 工具栏与浮层协同](/home/ray/dev/linkong/planet/docs/technical/zh/earth-toolbar-overlay-coordination.md):工具栏按钮与搜索、设置、新闻、图层浮层的关闭矩阵
|
||||
|
||||
## 前端技术实现
|
||||
|
||||
- [控制台前端结构](/home/ray/dev/linkong/planet/docs/technical/zh/frontend-admin-frontend-context.md):控制台页面结构、状态边界和懒加载策略
|
||||
- [前端布局指南](/home/ray/dev/linkong/planet/docs/technical/zh/frontend-layout-guidelines.md):布局、表格、面板和响应式约束
|
||||
- [Tactile UI 组件库](/home/ray/dev/linkong/planet/docs/technical/zh/tactile-ui-components.md):按钮、开关、tooltip、滚动条组件 API 和主题 token
|
||||
|
||||
## 后端技术实现
|
||||
|
||||
- [数据采集系统](/home/ray/dev/linkong/planet/docs/technical/zh/backend-collectors.md):采集器、任务类型、保存层和状态回写
|
||||
- [系统服务控制](/home/ray/dev/linkong/planet/docs/technical/zh/backend-system-service-control.md):后端服务控制和系统操作接口
|
||||
- [数据源、采集器设置与连接验证](/home/ray/dev/linkong/planet/docs/technical/zh/datasource-collector-settings-connectivity.md):数据源目录、采集器设置、连接验证和凭证链路
|
||||
- [数据源 API 性能](/home/ray/dev/linkong/planet/docs/technical/zh/backend-datasources-api-performance.md):DataSources 列表接口性能和缓存策略
|
||||
- [数据作业与 Outbox 技术架构](/home/ray/dev/linkong/planet/docs/technical/zh/data-job-earth-sync-architecture.md):PostgreSQL 作业队列、outbox、listener 和 Kafka / Spark 演进边界
|
||||
- [通用位置估算管线开发说明](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-development.md):location resolver / pipeline 的接口、注册表和扩展方式
|
||||
- [新闻直播采集格式](/home/ray/dev/linkong/planet/docs/technical/zh/earth-news-live-streams-collector-format.md):新闻、直播和媒体采集 payload 约定
|
||||
- [Docs Gatekeeper 开发说明](/home/ray/dev/linkong/planet/docs/technical/zh/docs-gatekeeper-development.md):后端 Docs 目录、正文读取和 Gatekeeper 权限组实现
|
||||
- [命名与术语对照](/home/ray/dev/linkong/planet/docs/technical/zh/naming-glossary.md):控制台、Earth、后端和文档常见名词的中英对照
|
||||
- [Earth 可交互图标接入](/home/ray/dev/linkong/planet/docs/technical/zh/earth-interactable-usage.md):Earth 地表可交互图标 `Interactable` 的接口、生命周期和接入示例
|
||||
- [Earth 工具栏与浮层协同](/home/ray/dev/linkong/planet/docs/technical/zh/earth-toolbar-overlay-coordination.md):工具栏按钮与搜索 / 设置 / 新闻 / 图层浮层之间的关闭矩阵和接入规则
|
||||
- [Tactile UI 组件库](/home/ray/dev/linkong/planet/docs/technical/zh/tactile-ui-components.md):可移植按钮、开关、tooltip 和滚动条组件的 API、主题 token 与迁移约定
|
||||
|
||||
不适合放入这里的内容:
|
||||
## 智能体与运维
|
||||
|
||||
- 尚未完成的路线图
|
||||
- 未来迭代方案
|
||||
- 大范围重构计划
|
||||
- [AI Provider 指南](/home/ray/dev/linkong/planet/docs/technical/zh/agents-aiprovider.md):模型供应商适配、任务 prompt 和调用边界
|
||||
- [Planet 运维手册](/home/ray/dev/linkong/planet/docs/technical/zh/ops-runbook.md):部署、启动、排障和敏感操作
|
||||
- [Docker + Compose + Buildx 升级](/home/ray/dev/linkong/planet/docs/technical/zh/ops-docker-compose-buildx-upgrade.md):Docker 工具链升级步骤
|
||||
- [planet.sh 启动机制](/home/ray/dev/linkong/planet/docs/technical/zh/ops-planet-sh-startup.md):启动脚本、健康检查和性能优化
|
||||
|
||||
这些应放入:
|
||||
## 计划与历史
|
||||
|
||||
- [计划文档索引](/home/ray/dev/linkong/planet/docs/plans/README.md)
|
||||
尚未完成的路线图、大范围重构方案和未来计划放在仓库路径 `docs/plans/README.md`。已完成、过时或被新实现替代的方案放在 `docs/deprecated/README.md`。
|
||||
|
||||
@@ -101,7 +101,7 @@ AI 配置页使用的接口:
|
||||
|
||||
这些接口都需要用户登录。`secrets` 接口只用于配置页点击显示 key/token 时取回明文,隐藏时前端恢复为脱敏预览。
|
||||
|
||||
Admin Next 的 AI 页面按旧版信息架构组织为:
|
||||
Admin 的 AI 页面按业务信息架构组织为:
|
||||
|
||||
- `模型供应商`
|
||||
- 管理 provider、协议适配、默认模型、LLM API Key、代理地址、代理 token、模型列表刷新、设为默认和轻量连通性测试。
|
||||
@@ -272,7 +272,7 @@ AI 配置仍保存在 PostgreSQL,不写入 JSON 文件。核心结构如下:
|
||||
|
||||
`.env` 只是兜底。配置页保存或测试连接成功后,PostgreSQL 中的配置会成为全局默认。
|
||||
|
||||
Admin Next 的密钥状态必须按 provider / tool 精确判断:
|
||||
Admin 的密钥状态必须按 provider / tool 精确判断:
|
||||
|
||||
- 数据库中当前 provider/tool 有密钥时,显示为“已配置”。
|
||||
- 数据库没有密钥,但 fallback provider、model 或 tool 与当前项匹配时,可以显示 fallback 的脱敏预览。
|
||||
@@ -283,7 +283,7 @@ Admin Next 的密钥状态必须按 provider / tool 精确判断:
|
||||
|
||||
### 轻量连通性测试
|
||||
|
||||
Admin Next 的插头按钮走轻量连通性测试,不承担保存职责。业界常见做法是分两层:
|
||||
Admin 的插头按钮走轻量连通性测试,不承担保存职责。业界常见做法是分两层:
|
||||
|
||||
- 快速检查 provider 目录或低成本 endpoint,确认 base URL、鉴权和当前模型是否可达。
|
||||
- 只有在用户明确运行 Playground 或业务任务时才发完整模型请求。
|
||||
|
||||
@@ -73,6 +73,8 @@ async def run(self, db):
|
||||
|
||||
**核心文件**: `backend/app/services/collectors/base.py`
|
||||
|
||||
手动触发、删除数据、清理缓存现在统一进入 PostgreSQL 数据作业队列,任务账本仍是 `collection_tasks`。采集器只负责 `fetch -> transform -> save`,由 `data_jobs.py` worker 领取 `collect` / `clear_data` / `clear_cache` / `earth_refresh` 任务并回写进度。Earth 图层刷新关系集中在 `earth_layer_adapters.py`,不要再在单个采集器或按钮里手写缓存失效和 WebSocket 广播。
|
||||
|
||||
## 三、采集器列表
|
||||
|
||||
| 采集器 | 数据类型 | 数据内容 | 采集频率 |
|
||||
@@ -230,7 +232,7 @@ if datasource.last_status == "success":
|
||||
|
||||
### 采集管理与快照
|
||||
|
||||
Admin Next 的采集管理入口按业务层级组织:
|
||||
Admin 的采集管理入口按业务层级组织:
|
||||
|
||||
- `采集器`:配置 endpoint、认证方式、请求头、基础参数、启用状态和凭证教程。
|
||||
- `采集调度`:查看和调整调度状态,触发、停止或刷新采集任务。
|
||||
|
||||
@@ -96,4 +96,4 @@ if (res.data.task_id) {
|
||||
## 相关文件
|
||||
|
||||
- `backend/app/api/v1/datasources.py` — `_load_datasource_list_context`、`list_datasources`
|
||||
- `frontend/src/pages/DataSources/DataSources.tsx` — `BuiltInDataSource` interface、`triggerDatasource`
|
||||
- `frontend/src/admin/pages/PlainResourcePages.tsx` — `BuiltInDataSource` interface、`triggerDatasource`
|
||||
|
||||
141
docs/technical/zh/data-job-earth-sync-architecture.md
Normal file
141
docs/technical/zh/data-job-earth-sync-architecture.md
Normal file
@@ -0,0 +1,141 @@
|
||||
# 数据作业与 Outbox 技术架构
|
||||
|
||||
本文记录 Planet v1 的任务队列、数据库 outbox 和 Earth 刷新技术边界。每类数据产品的业务用途和端到端链路见 [业务架构与数据流转](/home/ray/dev/linkong/planet/docs/technical/zh/platform-data-flows.md)。
|
||||
|
||||
## 架构边界
|
||||
|
||||
- PostgreSQL 是 v1 的可靠任务账本和 outbox,不引入 Kafka、Celery 或 RQ。
|
||||
- `collection_tasks` 记录采集、删除、清缓存和非数据库触发的 Earth refresh 任务。
|
||||
- `earth_data_change_events` 记录事实表或派生表变化,是 Earth 同步的可靠来源。
|
||||
- `LISTEN/NOTIFY` 只做低延迟唤醒;listener 仍会轮询未消费 outbox。
|
||||
- Redis 主要用于缓存、认证辅助、OTP / rate limit、临时日志和 WebSocket 辅助,不是可靠队列。
|
||||
|
||||
## 数据库变化同步
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Write["Fact or derived table write"] --> Trigger["PostgreSQL trigger"]
|
||||
Trigger --> Outbox["earth_data_change_events"]
|
||||
Trigger --> Notify["planet_earth_data_changes"]
|
||||
Outbox --> Listener["earth_db_change_listener"]
|
||||
Notify --> Listener
|
||||
Listener --> Adapter["earth_layer_adapters"]
|
||||
Adapter --> Cache["cache invalidation"]
|
||||
Cache --> WS["earth_updates"]
|
||||
```
|
||||
|
||||
1. 采集、删除、定位或派生任务写入事实表或派生表。
|
||||
2. statement trigger 为 `INSERT` / `UPDATE` / `DELETE` 写入 outbox。
|
||||
3. listener 被 notify 唤醒,或通过轮询发现未消费事件。
|
||||
4. listener 使用 `earth_layer_adapters.py` 把 `table + source` 映射成 Earth layer、刷新策略和 cache pattern。
|
||||
5. listener 短窗口合并同 layer 事件,失效缓存并广播 `earth_updates`。
|
||||
6. 广播成功后标记 outbox consumed;失败时保留待重试。
|
||||
|
||||
DB 变化不再默认创建 `earth_refresh` 任务,因此不会被同 source 的长采集或删除任务阻塞。`earth_refresh` 只保留给手动清缓存和非 DB 变化刷新提示。
|
||||
|
||||
## 数据作业队列
|
||||
|
||||
`collection_tasks` 是统一 job ledger。worker 使用 PostgreSQL `FOR UPDATE SKIP LOCKED` 领取 `queued` 任务;同一 `source` 的写任务串行,不同 source 可并行。
|
||||
|
||||
| task_type | 作用 |
|
||||
| --- | --- |
|
||||
| `collect` | 执行内置 datasource 采集 |
|
||||
| `clear_data` | 删除该 source 的采集数据和声明过的派生数据 |
|
||||
| `clear_cache` | 删除该 source 对应的 Earth / dashboard 缓存 |
|
||||
| `earth_refresh` | 非 DB 变化场景下失效 Earth 图层缓存并广播刷新提示 |
|
||||
|
||||
接口只创建任务并返回 `task_id`。任务执行、进度、取消和终态由 worker 写回 `collection_tasks`,并通过 `datasource_tasks` channel 通知前端。
|
||||
|
||||
取消语义是“保留已提交批次”:点击停止后,后端把任务标记为 `cancelling` 并取消内存中的执行协程;已经提交的批次保留,未完成批次按采集器或清理任务的回滚逻辑处理。
|
||||
|
||||
## Earth 同步事件模型
|
||||
|
||||
统一事件模型是 `earth.layer.changed`:
|
||||
|
||||
```json
|
||||
{
|
||||
"event": "earth.layer.changed",
|
||||
"action": "database_changed",
|
||||
"source": "celestrak_tle",
|
||||
"table": "collected_data",
|
||||
"operation": "DELETE",
|
||||
"layers": ["satellites"],
|
||||
"refresh_strategy": "clear_then_reload",
|
||||
"records_processed": 11125,
|
||||
"occurred_at": "2026-05-25T10:20:30Z"
|
||||
}
|
||||
```
|
||||
|
||||
| strategy | 用途 |
|
||||
| --- | --- |
|
||||
| `clear_then_reload` | 先清前端本地图层对象,再强制重拉接口。删除数据时优先使用。 |
|
||||
| `reload` | 保留旧对象直到新数据返回,适合定位、元数据或非破坏性更新。 |
|
||||
| `delta` | 只用于 `earth_interactables`,按 id upsert 或 remove。 |
|
||||
|
||||
接口在真实 0 数据时必须返回 200 和空集合;只有真实接口异常才返回 5xx。前端收到删除事件后,如果重拉失败,应保持已清空状态并显示轻量错误,不恢复旧对象。
|
||||
|
||||
## Layer Adapter 约定
|
||||
|
||||
`earth_layer_adapters.py` 是 source、派生表、Earth layer、缓存和刷新策略的唯一注册表。新图层只应新增 adapter,不应在按钮 handler、采集器或前端分支里手写同步逻辑。
|
||||
|
||||
Adapter 必须声明:
|
||||
|
||||
- source 或 table 由哪个 Earth layer 消费。
|
||||
- 需要清理哪些 Earth cache key pattern。
|
||||
- `clear_data` 删除 source 时是否需要同时删除 owned 派生表。
|
||||
- 该 layer 的默认刷新策略。
|
||||
|
||||
删除 source 时,`clear_data` 作业先删除 `collected_data.source = <source>`,再根据 adapter 删除 owned 派生表。直接修改派生表也会触发 outbox,所以后台任务、管理接口和 SQL 修复脚本只要落到事实表或派生表,Earth 都能感知变化。
|
||||
|
||||
## 运维排障
|
||||
|
||||
检查 outbox 是否堆积:
|
||||
|
||||
```sql
|
||||
SELECT id, table_name, operation, source, occurred_at
|
||||
FROM earth_data_change_events
|
||||
WHERE consumed_at IS NULL
|
||||
ORDER BY id
|
||||
LIMIT 20;
|
||||
```
|
||||
|
||||
检查触发器是否存在:
|
||||
|
||||
```sql
|
||||
SELECT tgname, tgrelid::regclass
|
||||
FROM pg_trigger
|
||||
WHERE tgname LIKE 'tr_planet_%_changed_%'
|
||||
ORDER BY 2, 1;
|
||||
```
|
||||
|
||||
常用日志事件:
|
||||
|
||||
- `earth.db_changes.connected`:listener 已连接 PostgreSQL 并开始监听。
|
||||
- `earth.db_changes.outbox_polled`:轮询到了未消费 outbox。
|
||||
- `earth.db_changes.broadcasted`:已产生 Earth 刷新广播。
|
||||
- `data_job.started` / `data_job.completed`:任务执行状态。
|
||||
|
||||
如果 Earth 没更新,按顺序检查:事实表是否变化、outbox 是否消费、adapter 是否覆盖对应 `source/table`、listener 是否在线、前端 WebSocket 是否连接、visualization 接口是否返回 200 空集合或新数据。
|
||||
|
||||
## Kafka-ready 边界
|
||||
|
||||
业务代码不直接依赖具体队列实现,而是通过这些边界组织:
|
||||
|
||||
- `JobQueue`:提交、领取、取消、完成数据作业。
|
||||
- `DataChangeBus`:发布数据库事实变化。
|
||||
- `EarthLayerAdapterRegistry`:声明 source、layer、缓存和派生数据关系。
|
||||
|
||||
需要 Kafka 的信号:
|
||||
|
||||
- 多个独立服务需要消费同一批数据变化。
|
||||
- AIS、BGP 或传感器流达到持续高吞吐。
|
||||
- 需要 consumer group、事件回放、跨服务解耦。
|
||||
|
||||
需要 Spark 的信号:
|
||||
|
||||
- 历史数据到千万或亿级,PostgreSQL 聚合开始吃力。
|
||||
- 需要跨源、长时间窗口、空间时间关联分析。
|
||||
- 原始数据进入 Parquet / Iceberg / Delta 等湖仓,并开始生产离线派生数据产品。
|
||||
|
||||
若目标是秒级连续流计算,优先评估 Flink;Spark 更适合批量或微批分析。
|
||||
|
||||
@@ -32,8 +32,8 @@
|
||||
|
||||
文件:
|
||||
|
||||
- [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/pages/PlainResourcePages.tsx)
|
||||
- [AdminNextRoutes.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/AdminNextRoutes.tsx)
|
||||
- [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx)
|
||||
- [AdminRoutes.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/AdminRoutes.tsx)
|
||||
|
||||
当前行为:
|
||||
|
||||
@@ -57,11 +57,11 @@
|
||||
|
||||
文件:
|
||||
|
||||
- [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/pages/PlainResourcePages.tsx)
|
||||
- [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx)
|
||||
|
||||
当前行为:
|
||||
|
||||
- `/collection-management` 按旧版层级收敛为 `采集器`、`采集调度`、`采集历史 / 快照`。
|
||||
- `/collection-management` 按业务层级收敛为 `采集器`、`采集调度`、`采集历史 / 快照`。
|
||||
- `采集器` 是配置页,左侧列表展示采集器配置;右侧表单编辑 endpoint、认证、请求头、采集参数和启用状态。
|
||||
- 新增采集器和目标 Schema 使用草稿详情页,不再用透明 JSON 弹窗;保存后才固化到列表,取消会销毁草稿。
|
||||
- 连接按钮只做连通性测试,不保存配置;保存按钮只持久化表单。
|
||||
@@ -80,7 +80,7 @@
|
||||
- `目标 Schema` 维护可写入目标结构。
|
||||
- `run-mapped`、`stop-mapped`、`stream-status` 负责运行映射后的自定义采集器。
|
||||
|
||||
这些入口在新版中需要表单化,只有高级字段才折叠为 JSON。不要把模板、Schema、运行状态和采集器配置拍平成同一张表。
|
||||
这些入口在 Admin 中需要表单化,只有高级字段才折叠为 JSON。不要把模板、Schema、运行状态和采集器配置拍平成同一张表。
|
||||
|
||||
## 后端接口
|
||||
|
||||
|
||||
@@ -83,7 +83,7 @@ DocsMetadata(
|
||||
- 更新用户时校验组名只能是 `docs_user`、`docs_developer`、`docs_admin`。
|
||||
- 只有 `super_admin` 能修改 Gatekeeper 权限组。
|
||||
|
||||
前端 [Users.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Users/Users.tsx) 展示权限组标签,并在编辑表单中提供多选框。非 `super_admin` 打开的表单会禁用该字段,并在提交前移除 `gatekeeper_groups`。
|
||||
前端 [Users.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/Users.tsx) 展示权限组标签,并在编辑表单中提供多选框。非 `super_admin` 打开的表单会禁用该字段,并在提交前移除 `gatekeeper_groups`。
|
||||
|
||||
## 前端 Docs 加载
|
||||
|
||||
|
||||
@@ -188,7 +188,7 @@ Earth info-card 策略:
|
||||
- 这是预期行为,因为 incident 是聚合和去噪后的结果
|
||||
- 但 incident-first 渲染会让 Earth 显得过于安静,除非有另一层始终可用的 activity layer
|
||||
|
||||
推荐 `activity layer` 的实现细节在 [BGP 区域聚合计划](/home/ray/dev/linkong/planet/docs/plans/earth-bgp-region-aggregation-plan.md) 中展开。
|
||||
推荐 `activity layer` 的未来实现细节保存在仓库路径 `docs/plans/earth-bgp-region-aggregation-plan.md`。
|
||||
|
||||
因此最近的里程碑是:
|
||||
|
||||
@@ -259,7 +259,7 @@ Earth 的 `incident` 层不应该像一大片发光区域,而应该像紧凑
|
||||
|
||||
相关页面:
|
||||
|
||||
- `frontend/src/pages/BGP/BGP.tsx`
|
||||
- `frontend/src/admin/pages/PlainResourcePages.tsx`
|
||||
|
||||
当前 BGP 控制台页面有三层:
|
||||
|
||||
@@ -321,7 +321,7 @@ BGP 专项测试位于:
|
||||
|
||||
前端:
|
||||
|
||||
- `frontend/src/pages/BGP/BGP.tsx`
|
||||
- `frontend/src/admin/pages/PlainResourcePages.tsx`
|
||||
- `frontend/public/earth/js/bgp.js`
|
||||
- `frontend/public/earth/js/main.js`
|
||||
- `frontend/public/earth/js/info-card.js`
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
相关规则建议一起参考:
|
||||
|
||||
- [项目规则](/home/ray/dev/linkong/planet/rules.md)
|
||||
- 仓库根目录 `rules.md`
|
||||
- [前端布局指南](/home/ray/dev/linkong/planet/docs/technical/zh/frontend-layout-guidelines.md)
|
||||
|
||||
## 当前目标
|
||||
@@ -56,6 +56,8 @@ React 路由入口:
|
||||
- 各图层集成
|
||||
- Earth 级别状态同步
|
||||
|
||||
Earth 收到 `/ws` 的 `earth_updates` 时只把它当作刷新提示,真实数据仍通过 `/api/v1/visualization/...` 接口重新 GET。数据库驱动的刷新由后端 listener 直接清理缓存再广播,不再默认经过 `earth_refresh` 作业队列;前端收到 `database_changed` 后会按 layer 读取 `clear_then_reload`、`reload` 或 `delta` 策略。`clear_then_reload` 必须先清 Three.js 对象再 no-store 重拉,summary 只做一致性校验,不能用 `0` 作为跳过图层重拉的理由。技术链路见 [数据作业与 Outbox 技术架构](/home/ray/dev/linkong/planet/docs/technical/zh/data-job-earth-sync-architecture.md),业务数据流见 [业务架构与数据流转](/home/ray/dev/linkong/planet/docs/technical/zh/platform-data-flows.md)。
|
||||
|
||||
### 3. 地球控制层
|
||||
|
||||
- [controls.js](/home/ray/dev/linkong/planet/frontend/public/earth/js/controls.js)
|
||||
@@ -162,11 +164,11 @@ Earth 地表是多层近似同心球,不是单一 mesh。`earth.js` 的基座
|
||||
|
||||
`brand.js` 管理 Earth HUD 品牌资源。默认品牌来自静态资源,运行时覆盖值来自 `/api/v1/earth/brand`,上传的图片通过 `/earth-brand-assets/...` 读取。前端必须把 logo/title 图片和文本 fallback 分开处理:图片加载失败时显示文本标题,文本字段为空时使用后端默认值,避免 HUD 品牌区空白。控制台的 Earth 内容页负责保存和重置品牌配置,Earth 前端只消费结果。
|
||||
|
||||
`about.js` 管理 Earth 设置里的“关于”卡片。默认内容仍保留在前端作为兜底,运行时优先读取 `/api/v1/earth/about`。接口失败或字段缺失时必须回退默认值,避免设置页出现空白。Admin Next 的 Earth 内容页提供“关于”tab,保存走 `PUT /api/v1/earth/about`,恢复默认走 `DELETE /api/v1/earth/about`。
|
||||
`about.js` 管理 Earth 设置里的“关于”卡片。默认内容仍保留在前端作为兜底,运行时优先读取 `/api/v1/earth/about`。接口失败或字段缺失时必须回退默认值,避免设置页出现空白。Admin 的 Earth 内容页提供“关于”tab,保存走 `PUT /api/v1/earth/about`,恢复默认走 `DELETE /api/v1/earth/about`。
|
||||
|
||||
`oobe.js` 管理 Earth 首次初始化引导。是否显示 OOBE 必须由 `/api/v1/earth/oobe-status` 的 `ready` 字段决定,不能依赖 `localStorage` 判断系统是否初始化。`localStorage` 只允许记录“本浏览器暂时跳过”的短时状态;如果后端已经认为 `ready: true`,退出登录、清空本地缓存或换浏览器都不应再次弹出 OOBE。桌面端使用深色星空遮罩和毛玻璃启动面板,移动端改为底部 sheet,并尊重 `prefers-reduced-motion`。
|
||||
|
||||
Admin Next 的 Earth 内容页必须按运行时语义组织这些配置:
|
||||
Admin 的 Earth 内容页必须按运行时语义组织这些配置:
|
||||
|
||||
- `品牌标识`:品牌预览应使用与 Earth HUD 左上角一致的深色星空背景、尺寸、间距、logo/title 渲染和文本 fallback,而不是普通表单预览。
|
||||
- `关于`:配置 Earth 设置里的 About 卡片,包括 logo、眉标、标题、版本、描述和元信息条目;Earth 运行时从 `/earth/about` 读取,失败时回退默认内容。
|
||||
@@ -197,9 +199,7 @@ TV 预览需要尽量复用 Earth 运行时的直播卡片结构和状态标签
|
||||
|
||||
当前 BGP 巡航只是这套能力的一个调用方,不应再把“按队列巡航”和“BGP 事件展示”混写在同一个状态机里。
|
||||
|
||||
新闻巡航摘要计划见:
|
||||
|
||||
- [Earth 新闻巡航摘要计划](/home/ray/dev/linkong/planet/docs/plans/earth-news-cruise-summary-plan.md)
|
||||
新闻巡航摘要的未来计划保存在仓库路径 `docs/plans/earth-news-cruise-summary-plan.md`,不作为公开 Docs 页面入口。
|
||||
|
||||
## 当前样式分层
|
||||
|
||||
|
||||
@@ -17,26 +17,15 @@
|
||||
|
||||
## 本地实际类别
|
||||
|
||||
当前 CelesTrak 卫星分组在 [backend/app/services/collectors/celestrak.py](/home/ray/dev/linkong/planet/backend/app/services/collectors/celestrak.py) 中包括:
|
||||
当前 CelesTrak 采集器从 `GROUP=active&FORMAT=json` 拉取完整活跃卫星目录,而不是逐个小分组拉取后合并。这样可以避免某个 CelesTrak 分组请求失败时仍把不完整结果保存为成功批次。采集器只在完整 JSON 数组可解析、且记录含 `NORAD_CAT_ID` 时进入转换和保存;网络、续传或解析失败会重试,最终失败时保留上一批 current 数据。
|
||||
|
||||
- `starlink`
|
||||
- `gps-ops`
|
||||
- `galileo`
|
||||
- `glonass`
|
||||
- `beidou`
|
||||
- `leo`
|
||||
- `geo`
|
||||
- `iridium-next`
|
||||
采集结果仍会给前端提供 `metadata.constellation_group`,但该字段现在来自可执行推断:
|
||||
|
||||
其中非 Starlink 类别是:
|
||||
- `OBJECT_NAME` 以 `STARLINK` 开头时标记为 `starlink`
|
||||
- `OBJECT_NAME` 以 `IRIDIUM` 开头时标记为 `iridium-next`
|
||||
- 其它活跃卫星不强行归入旧 CelesTrak 小分组,避免把泛化类别当成精确星座
|
||||
|
||||
- `gps-ops`
|
||||
- `galileo`
|
||||
- `glonass`
|
||||
- `beidou`
|
||||
- `leo`
|
||||
- `geo`
|
||||
- `iridium-next`
|
||||
因此,非 Starlink 类别在产品策略中仍包括 GNSS/RNSS、GEO、generic LEO 和 Iridium NEXT 等语义,但不能再假设采集器保存了旧的 `gps-ops`、`galileo`、`glonass`、`beidou`、`leo`、`geo` 分组标签。
|
||||
|
||||
## 资料结论
|
||||
|
||||
@@ -136,7 +125,8 @@
|
||||
|
||||
1. 后端把星座分组和 footprint 策略提示透给前端
|
||||
|
||||
- CelesTrak collector 会把 `GROUP` 记入 `metadata.constellation_group`
|
||||
- CelesTrak collector 会把原始查询来源记入 `metadata.celestrak_query_group = active`
|
||||
- `metadata.constellation_group` 只保存可推断的业务星座,例如 `starlink` 和 `iridium-next`
|
||||
- Visualization API 会输出:
|
||||
- `properties.constellation_group`
|
||||
- `properties.footprint_policy`
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
相关规则建议一起参考:
|
||||
|
||||
- [项目规则](/home/ray/dev/linkong/planet/rules.md)
|
||||
- 仓库根目录 `rules.md`
|
||||
- [前端布局指南](/home/ray/dev/linkong/planet/docs/technical/zh/frontend-layout-guidelines.md)
|
||||
|
||||
## 当前目标
|
||||
@@ -22,7 +22,7 @@
|
||||
|
||||
- [App.tsx](/home/ray/dev/linkong/planet/frontend/src/App.tsx)
|
||||
|
||||
当前正式后台路由已经由 Admin Next 接管:
|
||||
当前正式后台路由已经由 Admin 接管:
|
||||
|
||||
- `/admin`
|
||||
- `/users`
|
||||
@@ -37,23 +37,7 @@
|
||||
- `/collection-management`
|
||||
- `/settings`
|
||||
|
||||
这些路径渲染 [AdminNextRoutes.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/AdminNextRoutes.tsx),页面清单和菜单元信息来自 [manifest.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/routes/manifest.tsx)。`/admin-next/*` 仅作为兼容入口存在,会重定向到上面的正式路径,不再作为并行主入口。
|
||||
|
||||
旧 AntD 控制台保留在 `/legacy/admin/*`,用于对照和回退:
|
||||
|
||||
- `/legacy/admin`
|
||||
- `/legacy/admin/datasources`
|
||||
- `/legacy/admin/data`
|
||||
- `/legacy/admin/collection-management`
|
||||
- `/legacy/admin/earth-content`
|
||||
- `/legacy/admin/ai`
|
||||
- `/legacy/admin/logs`
|
||||
- `/legacy/admin/settings`
|
||||
- `/legacy/admin/users`
|
||||
- `/legacy/admin/bgp`
|
||||
- `/legacy/admin/alerts/*`
|
||||
|
||||
旧页面、`AppLayout`、`antd` 和 `@ant-design/icons` 在 legacy 验收期继续保留。不要在新版 parity 验收前删除这些文件或依赖。
|
||||
这些路径渲染 [AdminRoutes.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/AdminRoutes.tsx),页面清单和菜单元信息来自 [manifest.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/routes/manifest.tsx)。Admin 是唯一后台控制台入口,不再维护并行控制台或回退路由。
|
||||
|
||||
`/earth` 是独立展示页,不属于控制台骨架。
|
||||
|
||||
@@ -61,7 +45,7 @@
|
||||
|
||||
正式后台公共壳层在:
|
||||
|
||||
- [AdminNextLayout.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/components/layout/AdminNextLayout.tsx)
|
||||
- [AdminLayout.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/components/layout/AdminLayout.tsx)
|
||||
|
||||
职责:
|
||||
|
||||
@@ -69,38 +53,13 @@
|
||||
- 当前账号、版本、退出登录和主题切换
|
||||
- 顶部搜索、面包屑和页面快捷入口
|
||||
- 内容区单屏高度闭合
|
||||
- Admin Next 内部滚动、表格、详情面板和移动端详情视图协调
|
||||
- Admin 内部滚动、表格、详情面板和移动端详情视图协调
|
||||
|
||||
旧 AntD legacy 壳层仍在:
|
||||
后续正式控制台页面应适配 `AdminLayout` 和 Admin 页面模式;不要重新引入并行后台壳层。
|
||||
|
||||
- [AppLayout.tsx](/home/ray/dev/linkong/planet/frontend/src/components/AppLayout/AppLayout.tsx)
|
||||
## Admin 分区加载策略
|
||||
|
||||
legacy 职责:
|
||||
|
||||
- 左侧导航
|
||||
- 折叠与展开
|
||||
- 当前账号/版本信息
|
||||
- 内容区高度闭合
|
||||
- 全站统一侧边栏滚动条
|
||||
|
||||
当前结构是:
|
||||
|
||||
```tsx
|
||||
<Layout className="dashboard-layout">
|
||||
<Sider className="dashboard-sider">...</Sider>
|
||||
<Layout>
|
||||
<Content className="dashboard-content">
|
||||
<div className="dashboard-content-inner">{children}</div>
|
||||
</Content>
|
||||
</Layout>
|
||||
</Layout>
|
||||
```
|
||||
|
||||
后续正式控制台页面应优先适配 `AdminNextLayout` 和 Admin Next 页面模式,而不是继续往旧 `AppLayout` 增加新能力。只有维护 `/legacy/admin/*` 时才应修改旧壳层。
|
||||
|
||||
## Admin Next 分区加载策略
|
||||
|
||||
多 tab 页面由 [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/pages/PlainResourcePages.tsx) 统一承载当前的管理型和信息型工作台。分区加载规则是:
|
||||
多 tab 页面由 [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx) 统一承载当前的管理型和信息型工作台。分区加载规则是:
|
||||
|
||||
- 初次进入页面只请求当前 active tab,不预先拉取所有 tab 的接口。
|
||||
- 用户切换 tab 时懒加载该 tab;已经加载过的 tab 保留在本地 `states` 缓存中,切回时直接复用。
|
||||
@@ -112,20 +71,22 @@ legacy 职责:
|
||||
|
||||
## 数据源采集队列
|
||||
|
||||
Admin Next 的数据源页把单源触发、表格勾选触发和触发全部统一接入浏览器下载列表式采集队列:
|
||||
Admin 的数据源页把单源触发、表格勾选触发和触发全部统一接入浏览器下载列表式采集队列:
|
||||
|
||||
- 队列状态由 [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin-next/pages/PlainResourcePages.tsx) 管理,只保存当前会话中的可见任务,不用 `localStorage` 伪造历史。
|
||||
- 队列状态由 [PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx) 管理,只保存当前会话中的可见任务,不用 `localStorage` 伪造历史。
|
||||
- 任务进度优先消费 `/ws` 的 `datasource_tasks` channel;如果 WebSocket 未连接或没有及时返回,则轮询 `/api/v1/datasources/{id}/task-status`。
|
||||
- 触发接口返回的 `triggered`、`skipped`、`failed` 会立即进入队列;刷新页面后只根据后端当前仍在 `running/pending/queued` 的数据源恢复队列。
|
||||
- `内置源` 分区通过表格选择列收敛批量触发。没有勾选时主按钮是“触发全部”;勾选后同一个主按钮变成“触发已选 N”,不再提供手填 ID 的独立批量按钮。
|
||||
- 页面内容流不再承载展开队列,避免全量触发后挤压列表和详情面板。右上角 actions 区的队列按钮沿用现有 `Button` 样式;空态使用 `ListChecks` 图标,有任务时只显示纯圆环总进度。点击后打开浮层,按运行中、失败、完成、跳过分组。
|
||||
- 队列项可以跳转到对应数据源详情,失败项可以重试。详情页内的“采集任务”摘要只展示当前数据源最近任务,不承担保存配置职责。
|
||||
- 运行中的单源采集按钮显示为“停止采集”;队列中未完成项右侧提供取消按钮。取消调用 `/api/v1/datasources/{source_id}/tasks/{task_id}/cancel`,后端语义是保留已提交批次并回滚未完成批次。
|
||||
- 删除数据库数据和清理展示缓存也会进入同一队列,前端只展示任务状态,不假定接口同步完成。
|
||||
|
||||
这个队列是用户感知层,不替代后端调度状态。后端仍然是任务是否运行、完成、失败或跳过的唯一事实来源。
|
||||
|
||||
## Admin Next 主题滑块
|
||||
## Admin 主题滑块
|
||||
|
||||
Admin Next 侧栏底部主题切换继续复用共享 [SegmentedControl.tsx](/home/ray/dev/linkong/planet/frontend/src/components/SegmentedControl/SegmentedControl.tsx),但主题变量在 [styles.css](/home/ray/dev/linkong/planet/frontend/src/admin-next/styles.css) 内跟随 `data-theme` 覆盖:
|
||||
Admin 侧栏底部主题切换继续复用共享 [SegmentedControl.tsx](/home/ray/dev/linkong/planet/frontend/src/components/SegmentedControl/SegmentedControl.tsx),但主题变量在 [styles.css](/home/ray/dev/linkong/planet/frontend/src/admin/styles.css) 内跟随 `data-theme` 覆盖:
|
||||
|
||||
- light 下使用 `--d-segment-bg: #eef3f9`、白色 slider 和轻投影。
|
||||
- dark 下使用与 Docs 一致的深色底座、`#202938` slider 和深色外投影。
|
||||
@@ -164,8 +125,7 @@ Admin Next 侧栏底部主题切换继续复用共享 [SegmentedControl.tsx](/ho
|
||||
|
||||
当前使用场景:
|
||||
|
||||
- Admin Next 数据源、采集数据、采集管理、日志、告警和 BGP 页面
|
||||
- 旧 AntD legacy 页面通过兼容封装继续使用共享滚动能力
|
||||
- Admin 数据源、采集数据、采集管理、日志、告警和 BGP 页面
|
||||
|
||||
### 3. `TableScrollRegion`
|
||||
|
||||
@@ -188,7 +148,7 @@ Admin Next 侧栏底部主题切换继续复用共享 [SegmentedControl.tsx](/ho
|
||||
|
||||
用途:
|
||||
|
||||
- Admin Next 全局工具按钮和详情页工具按钮
|
||||
- Admin 全局工具按钮和详情页工具按钮
|
||||
- icon-only + tooltip 的普通操作
|
||||
- 保存、创建、确认、删除、停止等强意图操作
|
||||
- 与 Docs 主题滑块一致的紧凑开关
|
||||
@@ -246,39 +206,19 @@ Admin Next 侧栏底部主题切换继续复用共享 [SegmentedControl.tsx](/ho
|
||||
- 文档内部链接应通过 `transformLink` 转成 `/docs/:slug`
|
||||
- 标题锚点由 `getHeadingId` 注入,避免渲染器自己理解路由状态
|
||||
|
||||
### 7. `ConnectionTestInput`
|
||||
### 7. Admin UI primitives
|
||||
|
||||
文件:
|
||||
|
||||
- [ConnectionTestInput.tsx](/home/ray/dev/linkong/planet/frontend/src/components/ConnectionTestInput/ConnectionTestInput.tsx)
|
||||
- [button.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/components/ui/button.tsx)
|
||||
- [dialog.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/components/ui/dialog.tsx)
|
||||
- [switch.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/components/ui/switch.tsx)
|
||||
|
||||
用途:
|
||||
|
||||
- Endpoint、Base URL 这类“输入值 + 连接验证”的控制台表单项
|
||||
- AI Provider 和 WebSearch 的连接测试入口
|
||||
- 后续采集器配置如果把连接测试放进输入框,也应复用它
|
||||
|
||||
当前约束:
|
||||
|
||||
- 输入框末端只显示一个插头/连接器图标,不再并排放“测试连接”文字按钮
|
||||
- 禁用的集成能力必须同时置灰输入框和连接测试按钮
|
||||
- 组件只负责输入框与测试入口组合,不保存业务状态;调用方仍负责表单值、loading、disabled 和连接请求
|
||||
|
||||
### 8. `TableActions`
|
||||
|
||||
文件:
|
||||
|
||||
- [TableActions.tsx](/home/ray/dev/linkong/planet/frontend/src/components/TableActions/TableActions.tsx)
|
||||
|
||||
用途:
|
||||
|
||||
- 表格操作列的统一操作入口
|
||||
- 展开状态下直接展示按钮
|
||||
- 收起状态下用更多菜单承载操作
|
||||
|
||||
配套导出:
|
||||
|
||||
- `actionCellProps`:用于操作列 `onCell`,防止操作按钮被省略号截断或换行
|
||||
- 全局工具按钮、详情页动作、确认弹窗和二元设置。
|
||||
- 与 Tactile UI token 对齐,保持 Admin 内部控件尺寸、hover、disabled 和 dark mode 一致。
|
||||
- 表格行内动作优先使用 icon button + tooltip/title,不重新引入独立操作菜单组件。
|
||||
|
||||
## 当前状态来源
|
||||
|
||||
@@ -301,7 +241,7 @@ Admin Next 侧栏底部主题切换继续复用共享 [SegmentedControl.tsx](/ho
|
||||
|
||||
文件:
|
||||
|
||||
- [AISettings.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/AISettings/AISettings.tsx)
|
||||
- [AISettings.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx)
|
||||
|
||||
职责:
|
||||
|
||||
@@ -337,7 +277,7 @@ Admin Next 侧栏底部主题切换继续复用共享 [SegmentedControl.tsx](/ho
|
||||
|
||||
例如:
|
||||
|
||||
- [Dashboard.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Dashboard/Dashboard.tsx)
|
||||
- [Dashboard.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/Dashboard.tsx)
|
||||
|
||||
优先目标:
|
||||
|
||||
@@ -349,10 +289,10 @@ Admin Next 侧栏底部主题切换继续复用共享 [SegmentedControl.tsx](/ho
|
||||
|
||||
例如:
|
||||
|
||||
- [DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataSources/DataSources.tsx)
|
||||
- [DataList.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataList/DataList.tsx)
|
||||
- [Users.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Users/Users.tsx)
|
||||
- [Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Settings/Settings.tsx)
|
||||
- [DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx)
|
||||
- [DataList.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/DataList.tsx)
|
||||
- [Users.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/Users.tsx)
|
||||
- [Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx)
|
||||
|
||||
约束:
|
||||
|
||||
@@ -362,7 +302,7 @@ Admin Next 侧栏底部主题切换继续复用共享 [SegmentedControl.tsx](/ho
|
||||
|
||||
### 数据源目录页
|
||||
|
||||
[DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/DataSources/DataSources.tsx) 当前不再承担配置编辑职责,而是数据源目录和采集操作页。
|
||||
[DataSources.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx) 当前不再承担配置编辑职责,而是数据源目录和采集操作页。
|
||||
|
||||
当前页面边界:
|
||||
|
||||
@@ -386,7 +326,9 @@ Admin Next 侧栏底部主题切换继续复用共享 [SegmentedControl.tsx](/ho
|
||||
|
||||
### 采集器设置页
|
||||
|
||||
[Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Settings/Settings.tsx) 会按路由进入三种模式:`/settings` 是系统设置,`/earth-content` 是 Earth 内容,`/collection-management` 是采集管理。`collector_credentials` tab 当前在 `/collection-management` 下显示为“采集器”。
|
||||
[Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx) 会按路由进入三种模式:`/settings` 是系统设置,`/earth-content` 是 Earth 内容,`/collection-management` 是采集管理。`collector_credentials` tab 当前在 `/collection-management` 下显示为“采集器”。
|
||||
|
||||
`/settings` 的“系统显示”分区包含 `演示模式` 开关。开启后,Earth 的 OOBE 会忽略“已有当前采集数据”和本地“先浏览”临时跳过状态,直接展示初始化引导;该开关仅用于演示/验收流程,不改变数据源、采集队列或 Earth 内容资源配置。
|
||||
|
||||
当前页面边界:
|
||||
|
||||
@@ -419,7 +361,7 @@ Admin Next 侧栏底部主题切换继续复用共享 [SegmentedControl.tsx](/ho
|
||||
|
||||
### Earth 内容页
|
||||
|
||||
`/earth-content` 复用 [Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Settings/Settings.tsx) 的单屏 tab 容器,但页面责任与系统设置分离:
|
||||
`/earth-content` 复用 [Settings.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx) 的单屏 tab 容器,但页面责任与系统设置分离:
|
||||
|
||||
- `电视直播` 迁移原直播源配置,继续管理 Earth 媒体面板内容源。
|
||||
- `国界精度` 管理 Earth 静态国界资产:provider 状态、低精 fallback、高精 manifest/PMTiles、源配置 JSON 和构建动作。
|
||||
@@ -431,8 +373,8 @@ Admin Next 侧栏底部主题切换继续复用共享 [SegmentedControl.tsx](/ho
|
||||
|
||||
例如:
|
||||
|
||||
- [BGP.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/BGP/BGP.tsx)
|
||||
- [Playground.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/Playground/Playground.tsx)
|
||||
- [BGP.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx)
|
||||
- [Playground.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx)
|
||||
|
||||
约束:
|
||||
|
||||
|
||||
@@ -8,7 +8,8 @@
|
||||
|
||||
当前推荐参考实现:
|
||||
|
||||
- [frontend/src/pages/BGP/BGP.tsx](/home/ray/dev/linkong/planet/frontend/src/pages/BGP/BGP.tsx)
|
||||
- [frontend/src/admin/pages/PlainResourcePages.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/PlainResourcePages.tsx)
|
||||
- [frontend/src/admin/pages/DataList.tsx](/home/ray/dev/linkong/planet/frontend/src/admin/pages/DataList.tsx)
|
||||
- [frontend/src/index.css](/home/ray/dev/linkong/planet/frontend/src/index.css)
|
||||
|
||||
## 核心原则
|
||||
@@ -23,15 +24,15 @@
|
||||
推荐结构:
|
||||
|
||||
```tsx
|
||||
<AppLayout>
|
||||
<AdminLayout>
|
||||
<div className="page-shell">
|
||||
<div className="page-shell__header">...</div>
|
||||
<div className="page-shell__body">...</div>
|
||||
</div>
|
||||
</AppLayout>
|
||||
</AdminLayout>
|
||||
```
|
||||
|
||||
页面总高度应被限制在 `AppLayout` 内容区内,而不是继续让整个页面自然向下增长。
|
||||
页面总高度应被限制在 `AdminLayout` 内容区内,而不是继续让整个页面自然向下增长。
|
||||
|
||||
### 2. 滚动优先发生在模块内部
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
`backend/app/services/location/` 是所有“给定一条记录,决定它的 lat/lon”业务的共享抽象。算力中心、BGP 观测站、BGP 事件目前都跑在这条管线上。未来需要位置估算的实体,例如卫星地面站、用户认领点位、IXP 设施,也应接入这里,而不是各自再写地理解析逻辑。
|
||||
|
||||
用户侧流程见 [Earth 位置候选采集使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-user.md)。
|
||||
用户侧流程见 [Planet 使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/manual.md) 的 Earth 位置候选采集章节。
|
||||
|
||||
## 设计目标
|
||||
|
||||
|
||||
@@ -283,7 +283,17 @@ AIS 船只图例按船型显示颜色:货轮、油轮、客船、渔船、军
|
||||
|
||||
算力中心和 BGP 观测站详情卡支持自动采集坐标候选。点击对象后用"自动采集坐标候选"或"重新自动采集坐标"按钮,后端会从源坐标、开放组织注册 API 和在线地理编码中整理候选;常规来源没有候选时使用当前默认 AI Provider 做 LLM factcheck 兜底。BGP 观测站的已存储位置只用于补齐查询上下文,不会作为候选直接返回。
|
||||
|
||||
候选可以直接在 Earth 预览。算力中心候选点击"保存"后写入 `compute_center_locations` 维表并刷新图层。算力中心图层左上角的通知气泡显示无法渲染的待定位数量;点击查看列表,单条采集候选,或用"一键采用"从上到下保存最高置信候选。没有可用候选的记录会留在列表中,不会被国家中心点或硬编码 hint 伪造位置。详细流程见 [Earth 位置候选采集使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-user.md)。
|
||||
候选可以直接在 Earth 预览。算力中心候选点击"保存"后写入 `compute_center_locations` 维表并刷新图层。算力中心图层左上角的通知气泡显示无法渲染的待定位数量;点击查看列表,单条采集候选,或用"一键采用"从上到下保存最高置信候选。没有可用候选的记录会留在列表中,不会被国家中心点或硬编码 hint 伪造位置。
|
||||
|
||||
单个对象的推荐流程:
|
||||
|
||||
1. 打开算力中心或 BGP 观测站详情卡。
|
||||
2. 点击"自动采集坐标候选"。
|
||||
3. 等待候选列表返回;有 WebSearch / AI factcheck 依赖的候选会显示采集中状态。
|
||||
4. 在 Earth 上预览候选位置。
|
||||
5. 确认可用候选后点击"保存";不确定时关闭卡片不会丢失当前任务状态。
|
||||
|
||||
一键定位用于批量处理算力中心待定位队列。它会从列表顶部开始采用最高置信候选;仍没有事实依据的记录会保留在队列中。未开启 WebSearch 时,单个定位和一键定位会置灰,因为位置核验依赖事实查询。
|
||||
|
||||
### 设置
|
||||
|
||||
@@ -354,7 +364,7 @@ Earth 预留了动作捕捉控制入口。实时链路两种输入源:
|
||||
|
||||
文档站 `http://localhost:3000/docs` 由后端按权限读取,不再把全部 Markdown 直接打进前端构建产物。
|
||||
|
||||
未登录访客默认只能看到 `public` 文档:首页、快速开始、使用手册、常见问题、Earth 位置候选采集使用手册。登录用户被分配 Gatekeeper 权限组后可以看到更多技术文档:
|
||||
未登录访客默认只能看到 `public` 文档:首页、快速开始、使用手册、常见问题。登录用户被分配 Gatekeeper 权限组后可以看到更多技术文档:
|
||||
|
||||
- `docs_user`:用户操作类文档
|
||||
- `docs_developer`:Earth、前端、后端、采集器和 AI Provider 等开发文档
|
||||
@@ -368,5 +378,4 @@ Docs 支持分类导航、Markdown 渲染、表格和代码块、文档内目录
|
||||
|
||||
- [快速开始](/home/ray/dev/linkong/planet/docs/technical/zh/quickstart.md)
|
||||
- [常见问题](/home/ray/dev/linkong/planet/docs/technical/zh/faq.md)
|
||||
- [Earth 位置候选采集使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-user.md)
|
||||
- [Planet 运维手册](/home/ray/dev/linkong/planet/docs/technical/zh/ops-runbook.md)
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
| English / Key | 中文显示名 | 使用场景 |
|
||||
| --- | --- | --- |
|
||||
| Planet | Planet | 产品名,保留英文 |
|
||||
| Admin Next | 新控制台 | 新版管理端上下文 |
|
||||
| Admin | 控制台 | 管理端上下文 |
|
||||
| Earth | Earth | 地球可视化产品名,保留英文 |
|
||||
| datasource | 数据源 | API、列表、筛选 |
|
||||
| collector | 采集器 | 采集任务、凭证配置 |
|
||||
|
||||
@@ -136,6 +136,20 @@
|
||||
- 手机或平板演示 Earth
|
||||
- 局域网其他机器访问同一开发实例
|
||||
|
||||
Windows 侧可以使用仓库根目录的 `planet.cmd` 作为一键入口。它会请求管理员权限,进入 `Ubuntu` WSL 发行版的 `/home/linkong/planet`,执行 `./planet.sh restart --allow-lan`,成功后打开 `http://localhost:3000/earth`,并把终端停留在 WSL shell 中便于继续排查日志。若本机发行版名称或项目路径不同,需要先按实际环境调整 `planet.cmd` 中的 `wsl.exe -d ... --cd ...` 参数。
|
||||
|
||||
新机器优先确认 WSL 版本:
|
||||
|
||||
```powershell
|
||||
wsl -l -v
|
||||
```
|
||||
|
||||
Planet 开发环境建议使用 WSL2。WSL1 下网络、文件系统和进程模型与 Linux 差异更大,可能表现为 Bun 包管理命令只返回 `An unknown error occurred (Unexpected)`、端口释放不稳定,或局域网访问行为与脚本预期不一致。若发行版仍是 WSL1,可转换:
|
||||
|
||||
```powershell
|
||||
wsl --set-version Ubuntu 2
|
||||
```
|
||||
|
||||
`--allow-lan` 会让前端、后端和 AI Provider 直接对开发机开放:前端 `3000`、后端 `8000`、AI Provider `8010`。脚本启动前会检查这三个端口;如果 WSL/Linux 侧无法释放端口,并检测到 Windows 侧 listener 或旧 `portproxy`,会请求管理员 PowerShell 清理。WSL 中运行时,Windows 本机一般可以通过 `localhost` 访问,局域网其他机器访问 Windows 局域网 IP 时还需要 Windows 防火墙放行。
|
||||
|
||||
建议按顺序排查:
|
||||
@@ -200,13 +214,32 @@ AI Provider 镜像只在代码、Dockerfile、Compose 配置或相关 Python 依
|
||||
./planet.sh restart -a
|
||||
```
|
||||
|
||||
重建判断使用内容 fingerprint,而不是只看文件 mtime。`planet.sh` 会把 `aiprovider/` 文件、`aiprovider/Dockerfile`、`pyproject.toml`、`uv.lock`、`PYTHON_IMAGE`、`UV_IMAGE` 和依赖指纹合成 `AI_PROVIDER_BUILD_FINGERPRINT`,构建时写入镜像 label `planet.aiprovider.build-fingerprint`。如果现有 `planet-aiprovider:latest` 镜像的 label 与当前 fingerprint 一致,脚本会跳过 rebuild 并刷新本地 stamp;旧镜像没有 label 时才回退到 state/cache 里的 stamp 判断。
|
||||
|
||||
Docker 构建统一使用 `uv sync --frozen`。为了让容器构建也能复用本机 uv 镜像源配置,脚本会解析以下顺序中的第一个配置文件,并通过 BuildKit secret 挂到容器内 `/root/.config/uv/uv.toml`:
|
||||
|
||||
1. 当前环境的 `UV_CONFIG_FILE`
|
||||
2. 仓库根目录 `uv.toml`
|
||||
3. `${XDG_CONFIG_HOME:-~/.config}/uv/uv.toml`
|
||||
4. `~/.uv/uv.toml`
|
||||
|
||||
如果都不存在,脚本会创建一个空的 state 文件作为 secret,避免 Compose 的 secret file 缺失。进入 Docker 构建前会清掉 `UV_DEFAULT_INDEX`、`UV_INDEX_URL`、`UV_EXTRA_INDEX_URL` 这类环境变量,只保留明确的 `UV_CONFIG_FILE`,让本地和容器里的依赖解析更可复现。需要临时使用清华源时,可在仓库根目录准备:
|
||||
|
||||
```toml
|
||||
[[index]]
|
||||
name = "tsinghua"
|
||||
url = "https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple/"
|
||||
default = true
|
||||
```
|
||||
|
||||
构建较慢时按层排查:
|
||||
|
||||
| 现象 | 常见原因 | 处理方式 |
|
||||
| --- | --- | --- |
|
||||
| `transferring context` 很大 | build context 含前端资源等无关文件 | `.dockerignore` 只发送必需文件 |
|
||||
| `uv sync` 下载较慢 | 首次构建或缓存为空 | 等待首次完成,后续复用 BuildKit 缓存 |
|
||||
| `uv sync --frozen` 下载较慢 | 首次构建、缓存为空或 uv 镜像源未配置 | 等待首次完成,后续复用 BuildKit 缓存;必要时配置 `uv.toml` |
|
||||
| 改密钥后仍是旧配置 | 容器未重启 | `./planet.sh restart -a` |
|
||||
| 修改代码但镜像未重建 | fingerprint 与镜像 label 一致 | 确认改动是否进入 `aiprovider/`、Dockerfile 或 Python 依赖;必要时删除 `planet-aiprovider:latest` 后重试 |
|
||||
|
||||
## SMTP 邮件(公开注册依赖)
|
||||
|
||||
@@ -233,6 +266,15 @@ OTP 一次性验证码走 Redis,key 格式 `otp:{purpose}:{email}`,TTL 600
|
||||
|
||||
## 开发命令约定
|
||||
|
||||
后端和脚本初始化统一使用锁文件:
|
||||
|
||||
```bash
|
||||
uv python install 3.14
|
||||
uv sync --frozen --group dev
|
||||
```
|
||||
|
||||
`--frozen` 会拒绝隐式改写 `uv.lock`,适合新机器、CI 和 Docker 构建。需要升级依赖时,应先在开发机明确更新 `pyproject.toml` / `uv.lock`,再提交锁文件。
|
||||
|
||||
前端必须使用 Bun:
|
||||
|
||||
```bash
|
||||
|
||||
160
docs/technical/zh/platform-data-flows.md
Normal file
160
docs/technical/zh/platform-data-flows.md
Normal file
@@ -0,0 +1,160 @@
|
||||
# 业务架构与数据流转
|
||||
|
||||
本文是 Planet 数据产品的业务入口。它解释每类 Earth 数据为什么存在、从哪里采集、落到哪些事实表或派生表、如何通过缓存和 WebSocket 反映到 Earth。前端、后端和 Earth 技术文档只记录实现细节;跨端理解数据链路时优先从这里开始。
|
||||
|
||||
## 总览
|
||||
|
||||
Planet 的核心数据链路分三段:
|
||||
|
||||
1. **采集与整理**:内置采集器、后台操作或定位管线写入 PostgreSQL。通用原始结果进入 `collected_data`,图层需要的二次结果进入派生表。
|
||||
2. **投影与广播**:数据库触发器把事实变化写入 `earth_data_change_events` outbox,并用 `LISTEN/NOTIFY` 唤醒后端 listener。listener 通过 layer adapter 找到 Earth 图层,失效缓存并广播 `earth_updates`。
|
||||
3. **Earth 重拉与呈现**:Earth 前端收到 layer 级刷新提示后,按 `clear_then_reload`、`reload` 或 `delta` 策略清理本地图层对象,再从 `/api/v1/visualization/...` 重拉数据。
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
Source["外部数据源 / 后台操作"] --> Collector["采集器或数据作业"]
|
||||
Collector --> Facts["collected_data"]
|
||||
Collector --> Derived["派生表"]
|
||||
Facts --> Outbox["earth_data_change_events"]
|
||||
Derived --> Outbox
|
||||
Outbox --> Listener["DB change listener"]
|
||||
Listener --> Cache["Earth cache invalidation"]
|
||||
Listener --> WS["earth_updates"]
|
||||
WS --> Earth["Earth layer reload"]
|
||||
Earth --> API["Visualization APIs"]
|
||||
```
|
||||
|
||||
`LISTEN/NOTIFY` 只负责低延迟唤醒,可靠来源是 outbox。真实 0 数据是正常状态,接口应返回 200 和空集合;只有接口异常才返回 5xx。
|
||||
|
||||
## 数据产品清单
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
Satellites["卫星 TLE"] --> SatelliteLayer["satellites 图层"]
|
||||
Cables["海缆 + 登陆点"] --> CableLayer["cables 图层"]
|
||||
Compute["TOP500 / AI GPU / HF"] --> ComputeLocations["compute_center_locations"]
|
||||
ComputeLocations --> ComputeLayer["computeCenters 图层"]
|
||||
BgpRaw["RIS Live / BGPStream / Prefix"] --> BgpDerived["bgp_observations / anomalies / incidents"]
|
||||
BgpDerived --> BgpLayer["bgp 图层"]
|
||||
VesselRaw["AIS / BarentsWatch"] --> VesselDerived["vessel_static / vessel_position"]
|
||||
VesselDerived --> VesselLayer["vessels 图层"]
|
||||
Interactables["earth_interactables"] --> InteractableLayer["interactables 图层"]
|
||||
NewsRaw["RSS / Live / News"] --> NewsItems["earth_news_items"]
|
||||
NewsItems --> NewsLayer["news / media 图层"]
|
||||
```
|
||||
|
||||
| 数据产品 | 业务用途 | 事实来源 | 派生 / 维表 | Earth 图层 | 刷新策略 |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 卫星 | 展示在轨目标、轨迹、覆盖和巡航目标 | `celestrak_tle`、`spacetrack_tle` | 无稳定独立派生表,TLE 由接口实时转换 | `satellites` | `clear_then_reload` |
|
||||
| 海缆与登陆点 | 展示跨洋连接、登陆点和 cable 详情 | `arcgis_cables`、`arcgis_landing_points`、TeleGeography / FAO landing sources | 海缆关系和登陆点聚合数据 | `cables` | `clear_then_reload` |
|
||||
| 算力中心 | 展示 TOP500、AI GPU、HuggingFace 等算力节点 | `top500`、`epoch_ai_gpu`、HuggingFace sources | `compute_center_locations` | `computeCenters` | `reload` |
|
||||
| BGP 态势 | 展示观测站、异常事件、路由事件和区域态势 | `ris_live_bgp`、`bgpstream_bgp`、prefix geography sources | `bgp_observations`、`bgp_anomalies`、`bgp_incidents`、`bgp_collector_locations` | `bgp` | `clear_then_reload` |
|
||||
| 船舶 | 展示 AIS 船只、位置、轨迹和源健康 | AIS sources、`barentswatch_vessels` | `vessel_static`、`vessel_position`、`ais_raw_observations`、`ais_source_health` | `vessels` | `clear_then_reload` |
|
||||
| 可交互对象 | 支撑通用地表图标、人工点位和未来扩展对象 | `earth_interactables` | 无 | `interactables` | `delta` |
|
||||
| 新闻与媒体 | 支撑 Earth 新闻、直播和巡航摘要 | news sources | `earth_news_items` | `news` / `media` | `reload` |
|
||||
|
||||
## 卫星链路
|
||||
|
||||
卫星数据用于 Earth 的在线卫星点、轨迹线、覆盖策略和巡航列表。采集器从 CelesTrak 或 Space-Track 拉取 TLE,写入 `collected_data`。可视化接口按请求把 TLE 转成当前时刻的位置和轨迹,不依赖长期派生表。
|
||||
|
||||
- **采集入口**:CelesTrak TLE、Space-Track TLE。
|
||||
- **事实表**:`collected_data.source IN ('celestrak_tle', 'spacetrack_tle')`。
|
||||
- **接口**:卫星 visualization API 读取 TLE 并生成 Earth payload。
|
||||
- **删除语义**:删除对应 source 后,listener 广播 `satellites` 的 `clear_then_reload`,前端先清卫星点和轨迹,再重拉接口。接口若无 TLE,应返回空列表。
|
||||
- **常见异常**:summary 已变 0 但 Earth 仍显示,通常是 WS 未触发、adapter 未覆盖 source,或前端没有在 `clear_then_reload` 中清掉已有 Three.js 对象。
|
||||
|
||||
## 海缆与登陆点链路
|
||||
|
||||
海缆和登陆点用于展示跨海网络连接、登陆城市、线路详情和搜索对象。海缆线与登陆点都属于同一个业务图层;删除任一侧数据都必须刷新 `cables`,否则会出现线消失但点残留,或点消失但线残留。
|
||||
|
||||
- **采集入口**:ArcGIS cables、ArcGIS landing points,以及 TeleGeography / FAO landing 相关 source。
|
||||
- **事实表**:`collected_data` 中的 cable 和 landing point source。
|
||||
- **派生数据**:海缆关系表、landing point 聚合结果、接口缓存。
|
||||
- **接口**:海缆 visualization API 返回 cables、landing points 和关系数据。
|
||||
- **删除语义**:删除海缆或登陆点 source 后,adapter 清理 owned 派生数据并广播 `cables` 的 `clear_then_reload`。接口没有数据时返回 200 空集合,不返回 404。
|
||||
- **常见异常**:海缆短暂变 0 又回来,多半是采集替换或缓存刷新窗口内旧缓存被重新命中,需要检查数据作业是否重复广播或 cache pattern 是否覆盖完整。
|
||||
|
||||
## 算力中心链路
|
||||
|
||||
算力中心用于展示超算、AI GPU、模型平台相关设施和位置补全状态。原始 source 通常只有机构、国家、站点名或模糊位置,Earth 渲染依赖 `compute_center_locations` 维表提供可用坐标。
|
||||
|
||||
- **采集入口**:TOP500、Epoch AI GPU、HuggingFace 相关 source。
|
||||
- **事实表**:`collected_data` 中的算力 source。
|
||||
- **维表**:`compute_center_locations` 保存人工或自动采集到的坐标候选采用结果。
|
||||
- **接口**:算力中心 visualization API 合并原始记录和位置维表。
|
||||
- **删除语义**:删除 TOP500 等 source 后必须刷新 `computeCenters`;删除位置维表则也要刷新该图层。通常用 `reload`,因为位置更新不一定需要先清空。
|
||||
- **常见异常**:采集已完成但 Earth 数量不变,通常是接口使用缓存、位置维表未更新,或 source 删除没有触发 adapter。
|
||||
|
||||
## BGP 链路
|
||||
|
||||
BGP 数据用于展示路由观测站、异常事件、事件扩散圈和态势摘要。Earth 不直接展示原始 BGP 行,而是展示聚合后的观测、异常和事件。因此 BGP 是最容易出现“原始数据删了但 Earth 还在”的链路。
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant UI as 后台删除按钮
|
||||
participant Job as clear_data 作业
|
||||
participant DB as PostgreSQL
|
||||
participant Outbox as earth_data_change_events
|
||||
participant Listener as DB change listener
|
||||
participant Earth as Earth 前端
|
||||
|
||||
UI->>Job: 提交删除 ris_live_bgp / bgpstream_bgp
|
||||
Job->>DB: 删除 collected_data 原始记录
|
||||
Job->>DB: 删除 BGP owned 派生表
|
||||
DB->>Outbox: trigger 写入 bgp layer changed
|
||||
DB-->>Listener: LISTEN/NOTIFY 唤醒
|
||||
Listener->>Listener: 合并事件并清理 bgp cache
|
||||
Listener-->>Earth: broadcast earth_updates clear_then_reload
|
||||
Earth->>Earth: 清空 BGP 对象
|
||||
Earth->>DB: 通过 visualization API 重拉派生结果
|
||||
```
|
||||
|
||||
- **采集入口**:RIPE RIS Live BGP、CAIDA BGPStream Backfill、IPtoASN / OpenGeoFeed / NRO prefix geography。
|
||||
- **事实表**:`collected_data` 中的 BGP 和 prefix source。
|
||||
- **派生表**:`bgp_observations`、`bgp_anomalies`、`bgp_incidents`、`bgp_collector_locations`。
|
||||
- **接口**:BGP visualization API 读取派生表,并结合定位候选或已保存位置。
|
||||
- **删除语义**:删除 RIS Live 或 BGPStream 原始 source 时,必须同步清理 BGP owned 派生表并广播 `bgp` 的 `clear_then_reload`。直接删除派生表也要触发 outbox。
|
||||
- **常见异常**:观测站或异常事件没有消失,优先查派生表是否还保留旧记录,而不是只看 `collected_data`。
|
||||
|
||||
## 船舶链路
|
||||
|
||||
船舶数据用于展示 AIS 船只、航行状态、船型图例和源健康。Earth 渲染使用位置快照和静态船舶信息,不应依赖原始 AIS 记录逐条渲染。
|
||||
|
||||
- **采集入口**:AIS sources、BarentsWatch vessels。
|
||||
- **事实表**:`collected_data` 或 AIS 原始观测表。
|
||||
- **派生表**:`vessel_static`、`vessel_position`、`ais_raw_observations`、`ais_source_health`。
|
||||
- **接口**:vessels visualization API 返回当前船只 marker 和必要详情。
|
||||
- **删除语义**:删除任一船舶 source 后,owned 派生表变化会广播 `vessels` 的 `clear_then_reload`。
|
||||
- **常见异常**:数量面板变化但船只仍在,多半是 summary 和图层数据分离,前端应以 layer update 为准清空对象。
|
||||
|
||||
## 可交互对象链路
|
||||
|
||||
`earth_interactables` 是通用地表图标能力,用于人工对象、扩展对象和未来小型图层。它和大多数图层不同,保留对象级 delta。
|
||||
|
||||
- **事实表**:`earth_interactables`。
|
||||
- **接口**:interactable API 和 Earth 通用图标接口。
|
||||
- **刷新策略**:新增或更新用 upsert,删除用 removeItem,不重拉整个图层。
|
||||
- **删除语义**:删除一条 interactable 后,WS payload 必须包含稳定 id,让前端移除对应对象。
|
||||
- **常见异常**:对象删不掉,通常是缺少稳定 id、前端 delta handler 没有命中 object type,或旧图层还有重复渲染路径。
|
||||
|
||||
## 新闻与媒体链路
|
||||
|
||||
新闻与媒体数据用于 Earth 顶部新闻条、直播面板、新闻巡航和态势摘要。它们的视觉状态比地理对象更偏内容刷新,因此默认使用 `reload`。
|
||||
|
||||
- **采集入口**:RSS、直播源、新闻 source。
|
||||
- **事实表**:新闻 source 的 `collected_data`。
|
||||
- **派生表**:`earth_news_items`。
|
||||
- **接口**:新闻、直播和媒体 visualization / content API。
|
||||
- **删除语义**:删除新闻 source 或 `earth_news_items` 后广播 `news` / `media` reload;前端重拉后列表为空即隐藏对应内容。
|
||||
- **常见异常**:直播面板仍显示旧内容,通常是媒体组件本地状态没有响应 layer update,或内容接口缓存未失效。
|
||||
|
||||
## 扩展新图层
|
||||
|
||||
新增 Earth 数据产品时,按这个顺序接入:
|
||||
|
||||
1. 定义业务用途和 Earth 图层名。
|
||||
2. 明确事实 source、事实表和派生表。
|
||||
3. 在后端 layer adapter 注册 source/table、cache pattern、owned derived cleanup 和默认刷新策略。
|
||||
4. 确保 visualization API 对空数据返回 200 空集合。
|
||||
5. 让 Earth 前端只按 layer 和 strategy 刷新,不理解数据库表名。
|
||||
6. 在本文补充该数据产品的链路,再到对应前端、后端或 Earth 技术文档记录实现细节。
|
||||
@@ -64,5 +64,5 @@
|
||||
|
||||
- 完整 UI 操作说明:[Planet 使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/manual.md)
|
||||
- 排障与配置疑问:[常见问题](/home/ray/dev/linkong/planet/docs/technical/zh/faq.md)
|
||||
- Earth 坐标候选采集详细流程:[Earth 位置候选采集使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-user.md)
|
||||
- Earth 坐标候选采集流程见 [Planet 使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/manual.md) 的 Earth 章节
|
||||
- 部署 / 运维相关命令:[Planet 运维手册](/home/ray/dev/linkong/planet/docs/technical/zh/ops-runbook.md)
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
# Tactile UI 组件库
|
||||
|
||||
Tactile UI 是 Planet 内部抽出的可移植 React 控件层。它来自 Admin Next 的按钮、开关、滚动条和 tooltip 收口,但组件本身不依赖 Admin Next、AntD、Radix、Tailwind 或 `an-*` class。目标是先在仓库内稳定使用,后续可以作为独立 npm 包发布。
|
||||
Tactile UI 是 Planet 内部抽出的可移植 React 控件层。它来自 Admin 的按钮、开关、滚动条和 tooltip 收口,但组件本身不依赖 Admin、AntD、Radix、Tailwind 或 `an-*` class。目标是先在仓库内稳定使用,后续可以作为独立 npm 包发布。
|
||||
|
||||
## 设计目标
|
||||
|
||||
- **轻触感**:默认控件使用白色或主题表面、细边框和外部投影,接近 Docs 主题滑块的轻微立体感,不使用大色块或发光效果。
|
||||
- **可移植**:组件 class 使用 `tui-*` 前缀,样式集中在 `frontend/src/components/tactile-ui/styles.css`。
|
||||
- **低依赖**:组件只假设 React/React DOM;图标预设当前使用 `lucide-react`,调用方也可以传自定义 React 节点。
|
||||
- **主题友好**:默认样式通过 CSS variables 暴露,Planet 可以在 Admin Next 或其它页面按主题覆盖 token。
|
||||
- **主题友好**:默认样式通过 CSS variables 暴露,Planet 可以在 Admin 或其它页面按主题覆盖 token。
|
||||
- **语义清楚**:无歧义操作优先 icon-only + tooltip;保存、确认、创建、执行这类强意图操作可以保留文字。
|
||||
|
||||
## 导入
|
||||
@@ -51,19 +51,19 @@ import '@planet/tactile-ui/styles.css'
|
||||
|
||||
## Portal 与深色主题
|
||||
|
||||
`TactileTooltip` 和使用 Tactile UI 的 Admin Next `Dialog`、`Select`、Toast 都可能通过 portal 挂到 `document.body`。这类节点不在 `.admin-next-theme-root[data-theme='dark']` 下面,不能只依赖局部祖先选择器读取深色 token。
|
||||
`TactileTooltip` 和使用 Tactile UI 的 Admin `Dialog`、`Select`、Toast 都可能通过 portal 挂到 `document.body`。这类节点不在 `.admin-theme-root[data-theme='dark']` 下面,不能只依赖局部祖先选择器读取深色 token。
|
||||
|
||||
Admin Next 的主题 provider 会把当前主题同步到 `body[data-admin-next-theme]`。共享样式必须同时支持两类选择器:
|
||||
Admin 的主题 provider 会把当前主题同步到 `body[data-admin-theme]`。共享样式必须同时支持两类选择器:
|
||||
|
||||
```css
|
||||
[data-theme='dark'] .tui-button,
|
||||
body[data-admin-next-theme='dark'] .tui-button {
|
||||
body[data-admin-theme='dark'] .tui-button {
|
||||
--tui-surface: #172033;
|
||||
--tui-text: #e5edf8;
|
||||
}
|
||||
```
|
||||
|
||||
新增 portal 控件时,先确认它是否渲染到 body。如果是,就要在组件自己的样式入口补 `body[data-admin-next-theme='dark']` 分支,或复用已经覆盖过的 `--tui-*` / `--an-*` token。不要在单个弹窗里手写固定深色,因为同一问题会在下拉菜单、tooltip、toast 和确认弹窗里重复出现。
|
||||
新增 portal 控件时,先确认它是否渲染到 body。如果是,就要在组件自己的样式入口补 `body[data-admin-theme='dark']` 分支,或复用已经覆盖过的 `--tui-*` / `--an-*` token。不要在单个弹窗里手写固定深色,因为同一问题会在下拉菜单、tooltip、toast 和确认弹窗里重复出现。
|
||||
|
||||
## `TactileButton`
|
||||
|
||||
@@ -195,7 +195,7 @@ Textarea 不建议强行套 overlay 滚动条,因为浏览器 resize grip 和
|
||||
|
||||
## `TableScrollRegion`
|
||||
|
||||
`TableScrollRegion` 是表格滚动区域的便利封装。默认目标选择器是 `.tui-scroll-target`,Admin Next 会显式传自己的 table viewport selector,避免组件库里出现业务命名。
|
||||
`TableScrollRegion` 是表格滚动区域的便利封装。默认目标选择器是 `.tui-scroll-target`,Admin 会显式传自己的 table viewport selector,避免组件库里出现业务命名。
|
||||
|
||||
```tsx
|
||||
<TableScrollRegion targetSelector=".table-viewport">
|
||||
@@ -210,9 +210,9 @@ Textarea 不建议强行套 overlay 滚动条,因为浏览器 resize grip 和
|
||||
- tooltip 只是说明,不承担唯一状态表达;状态仍应通过文本、badge 或 `aria-*` 呈现。
|
||||
- 禁用和 loading 状态会设置 `aria-disabled`,button 元素会同步 `disabled`。
|
||||
|
||||
## Admin Next 迁移约定
|
||||
## Admin 迁移约定
|
||||
|
||||
Admin Next 中所有全局工具按钮、详情页工具按钮和列表底部动作应使用 Tactile UI:
|
||||
Admin 中所有全局工具按钮、详情页工具按钮和列表底部动作应使用 Tactile UI:
|
||||
|
||||
- 无歧义动作:`TactileButton iconOnly tooltip`
|
||||
- 保存/创建/确认:`TactileButton variant="primary"`,通常保留文字
|
||||
|
||||
@@ -16,12 +16,15 @@
|
||||
## Current Version
|
||||
|
||||
- `main` 当前主线历史推导到:`0.16.5`
|
||||
- `dev` 当前开发分支历史推导到:`0.65.0`
|
||||
- `dev` 当前开发分支历史推导到:`0.66.0`
|
||||
|
||||
## Timeline
|
||||
|
||||
| Version | Type | Branch | Commit | Summary |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `0.66.0` | feature | `dev` | `pending` | Admin 正式化为唯一控制台,新增数据作业/outbox 与 Earth interactables 管线,补齐 AI/采集日志,修复 CelesTrak 完整 active 目录采集和内置源启停判断 |
|
||||
| `0.65.2` | bugfix | `dev` | `pending` | AI Provider 镜像重建判定改为内容 fingerprint 与镜像 label,启动链路改用 frozen uv,避免用户级镜像源污染 `uv.lock`,并加入 Windows 一键启动脚本 |
|
||||
| `0.65.1` | bugfix | `dev` | `pending` | 统一 `planet.sh` 与 Compose 的 AI Provider 镜像名,并让本地和 Docker build 通过用户级 `uv.toml` 共享 uv 源配置,避免镜像源污染 `uv.lock` |
|
||||
| `0.65.0` | feature | `dev` | `pending` | Admin Next 数据源触发入口收敛为“触发全部/触发已选 N”,队列改为右上角浮层按钮,并强化 `planet.sh destroy` 的 OOBE 数据清理 |
|
||||
| `0.64.0` | feature | `dev` | `pending` | Earth 新增后端状态驱动 OOBE 与 About 配置,Admin Next 数据源页新增采集队列,补齐深色主题滑块和用户/技术文档 |
|
||||
| `0.63.1` | bugfix | `dev` | `pending` | 补上被 `lib/` ignore 规则漏提交的 Admin Next utility module,修复新设备初始化后 AdminNextRoutes 动态导入 500 |
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
"": {
|
||||
"name": "planet-frontend",
|
||||
"dependencies": {
|
||||
"@ant-design/icons": "^5.2.6",
|
||||
"@hookform/resolvers": "^5.2.2",
|
||||
"@mapbox/vector-tile": "^2.0.4",
|
||||
"@radix-ui/react-dialog": "^1.1.15",
|
||||
@@ -16,7 +15,6 @@
|
||||
"@radix-ui/react-toast": "^1.2.15",
|
||||
"@radix-ui/react-tooltip": "^1.2.8",
|
||||
"@tanstack/react-table": "^8.21.3",
|
||||
"antd": "^5.12.5",
|
||||
"autoprefixer": "^10.5.0",
|
||||
"axios": "^1.6.2",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
@@ -24,6 +22,7 @@
|
||||
"dayjs": "^1.11.10",
|
||||
"echarts": "^6.0.0",
|
||||
"lucide-react": "^1.16.0",
|
||||
"mermaid": "^11.15.0",
|
||||
"pbf": "^4.0.1",
|
||||
"pmtiles": "^4.4.1",
|
||||
"postcss": "^8.5.14",
|
||||
@@ -51,19 +50,7 @@
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@ant-design/colors": ["@ant-design/colors@7.2.1", "", { "dependencies": { "@ant-design/fast-color": "^2.0.6" } }, "sha512-lCHDcEzieu4GA3n8ELeZ5VQ8pKQAWcGGLRTQ50aQM2iqPpq2evTxER84jfdPvsPAtEcZ7m44NI45edFMo8oOYQ=="],
|
||||
|
||||
"@ant-design/cssinjs": ["@ant-design/cssinjs@1.24.0", "", { "dependencies": { "@babel/runtime": "^7.11.1", "@emotion/hash": "^0.8.0", "@emotion/unitless": "^0.7.5", "classnames": "^2.3.1", "csstype": "^3.1.3", "rc-util": "^5.35.0", "stylis": "^4.3.4" }, "peerDependencies": { "react": ">=16.0.0", "react-dom": ">=16.0.0" } }, "sha512-K4cYrJBsgvL+IoozUXYjbT6LHHNt+19a9zkvpBPxLjFHas1UpPM2A5MlhROb0BT8N8WoavM5VsP9MeSeNK/3mg=="],
|
||||
|
||||
"@ant-design/cssinjs-utils": ["@ant-design/cssinjs-utils@1.1.3", "", { "dependencies": { "@ant-design/cssinjs": "^1.21.0", "@babel/runtime": "^7.23.2", "rc-util": "^5.38.0" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-nOoQMLW1l+xR1Co8NFVYiP8pZp3VjIIzqV6D6ShYF2ljtdwWJn5WSsH+7kvCktXL/yhEtWURKOfH5Xz/gzlwsg=="],
|
||||
|
||||
"@ant-design/fast-color": ["@ant-design/fast-color@2.0.6", "", { "dependencies": { "@babel/runtime": "^7.24.7" } }, "sha512-y2217gk4NqL35giHl72o6Zzqji9O7vHh9YmhUVkPtAOpoTCH4uWxo/pr4VE8t0+ChEPs0qo4eJRC5Q1eXWo3vA=="],
|
||||
|
||||
"@ant-design/icons": ["@ant-design/icons@5.6.1", "", { "dependencies": { "@ant-design/colors": "^7.0.0", "@ant-design/icons-svg": "^4.4.0", "@babel/runtime": "^7.24.8", "classnames": "^2.2.6", "rc-util": "^5.31.1" }, "peerDependencies": { "react": ">=16.0.0", "react-dom": ">=16.0.0" } }, "sha512-0/xS39c91WjPAZOWsvi1//zjx6kAp4kxWwctR6kuU6p133w8RU0D2dSCvZC19uQyharg/sAvYxGYWl01BbZZfg=="],
|
||||
|
||||
"@ant-design/icons-svg": ["@ant-design/icons-svg@4.4.2", "", {}, "sha512-vHbT+zJEVzllwP+CM+ul7reTEfBR0vgxFe7+lREAsAA7YGsYpboiq2sQNeQeRvh09GfQgs/GyFEvZpJ9cLXpXA=="],
|
||||
|
||||
"@ant-design/react-slick": ["@ant-design/react-slick@1.1.2", "", { "dependencies": { "@babel/runtime": "^7.10.4", "classnames": "^2.2.5", "json2mq": "^0.2.0", "resize-observer-polyfill": "^1.5.1", "throttle-debounce": "^5.0.0" }, "peerDependencies": { "react": ">=16.9.0" } }, "sha512-EzlvzE6xQUBrZuuhSAFTdsr4P2bBBHGZwKFemEfq8gIGyIQCxalYfZW/T2ORbtQx5rU69o+WycP3exY/7T1hGA=="],
|
||||
"@antfu/install-pkg": ["@antfu/install-pkg@1.1.0", "", { "dependencies": { "package-manager-detector": "^1.3.0", "tinyexec": "^1.0.1" } }, "sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ=="],
|
||||
|
||||
"@babel/code-frame": ["@babel/code-frame@7.29.0", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.28.5", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" } }, "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw=="],
|
||||
|
||||
@@ -97,17 +84,15 @@
|
||||
|
||||
"@babel/plugin-transform-react-jsx-source": ["@babel/plugin-transform-react-jsx-source@7.27.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.27.1" }, "peerDependencies": { "@babel/core": "^7.0.0-0" } }, "sha512-zbwoTsBruTeKB9hSq73ha66iFeJHuaFkUbwvqElnygoNbj/jHRsSeokowZFN3CZ64IvEqcmmkVe89OPXc7ldAw=="],
|
||||
|
||||
"@babel/runtime": ["@babel/runtime@7.29.2", "", {}, "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g=="],
|
||||
|
||||
"@babel/template": ["@babel/template@7.28.6", "", { "dependencies": { "@babel/code-frame": "^7.28.6", "@babel/parser": "^7.28.6", "@babel/types": "^7.28.6" } }, "sha512-YA6Ma2KsCdGb+WC6UpBVFJGXL58MDA6oyONbjyF/+5sBgxY/dwkhLogbMT2GXXyU84/IhRw/2D1Os1B/giz+BQ=="],
|
||||
|
||||
"@babel/traverse": ["@babel/traverse@7.29.0", "", { "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", "@babel/helper-globals": "^7.28.0", "@babel/parser": "^7.29.0", "@babel/template": "^7.28.6", "@babel/types": "^7.29.0", "debug": "^4.3.1" } }, "sha512-4HPiQr0X7+waHfyXPZpWPfWL/J7dcN1mx9gL6WdQVMbPnF3+ZhSMs8tCxN7oHddJE9fhNE7+lxdnlyemKfJRuA=="],
|
||||
|
||||
"@babel/types": ["@babel/types@7.29.0", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A=="],
|
||||
|
||||
"@emotion/hash": ["@emotion/hash@0.8.0", "", {}, "sha512-kBJtf7PH6aWwZ6fka3zQ0p6SBYzx4fl1LoZXE2RrnYST9Xljm7WfKJrU4g/Xr3Beg72MLrp1AWNUmuYJTL7Cow=="],
|
||||
"@braintree/sanitize-url": ["@braintree/sanitize-url@7.1.2", "", {}, "sha512-jigsZK+sMF/cuiB7sERuo9V7N9jx+dhmHHnQyDSVdpZwVutaBu7WvNYqMDLSgFgfB30n452TP3vjDAvFC973mA=="],
|
||||
|
||||
"@emotion/unitless": ["@emotion/unitless@0.7.5", "", {}, "sha512-OWORNpfjMsSSUBVrRBVGECkhWcULOAJz9ZW8uK9qgxD+87M7jHRcvh/A96XXNhXTLmKcoYSQtBEX7lHMO7YRwg=="],
|
||||
"@chevrotain/types": ["@chevrotain/types@11.1.2", "", {}, "sha512-U+HFai5+zmJCkK86QsaJtoITlboZHBqrVketcO2ROv865xfCMSFpELQoz1GkX5GzME8pTa+3kbKrZHQtI0gdbw=="],
|
||||
|
||||
"@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.21.5", "", { "os": "aix", "cpu": "ppc64" }, "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ=="],
|
||||
|
||||
@@ -165,6 +150,10 @@
|
||||
|
||||
"@hookform/resolvers": ["@hookform/resolvers@5.2.2", "", { "dependencies": { "@standard-schema/utils": "^0.3.0" }, "peerDependencies": { "react-hook-form": "^7.55.0" } }, "sha512-A/IxlMLShx3KjV/HeTcTfaMxdwy690+L/ZADoeaTltLx+CVuzkeVIPuybK3jrRfw7YZnmdKsVVHAlEPIAEUNlA=="],
|
||||
|
||||
"@iconify/types": ["@iconify/types@2.0.0", "", {}, "sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg=="],
|
||||
|
||||
"@iconify/utils": ["@iconify/utils@3.1.3", "", { "dependencies": { "@antfu/install-pkg": "^1.1.0", "@iconify/types": "^2.0.0", "import-meta-resolve": "^4.2.0" } }, "sha512-LPKOXPn/zV+zis1oOfGWogaXVpqUybF3ZS6SCZIsz8vg0ivVp9+fVqyYB7xq0aiST/VhUQYGO1qo6uoYSiEJqw=="],
|
||||
|
||||
"@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="],
|
||||
|
||||
"@jridgewell/remapping": ["@jridgewell/remapping@2.3.5", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ=="],
|
||||
@@ -179,6 +168,8 @@
|
||||
|
||||
"@mapbox/vector-tile": ["@mapbox/vector-tile@2.0.4", "", { "dependencies": { "@mapbox/point-geometry": "~1.1.0", "@types/geojson": "^7946.0.16", "pbf": "^4.0.1" } }, "sha512-AkOLcbgGTdXScosBWwmmD7cDlvOjkg/DetGva26pIRiZPdeJYjYKarIlb4uxVzi6bwHO6EWH82eZ5Nuv4T5DUg=="],
|
||||
|
||||
"@mermaid-js/parser": ["@mermaid-js/parser@1.1.1", "", { "dependencies": { "@chevrotain/types": "~11.1.1" } }, "sha512-VuHdsYMK1bT6X2JbcAaWAhugTRvRBRyuZgd+c22swUeI9g/ntaxF7CY7dYarhZovofCbUNO0G7JesfmNtjYOCw=="],
|
||||
|
||||
"@radix-ui/number": ["@radix-ui/number@1.1.1", "", {}, "sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g=="],
|
||||
|
||||
"@radix-ui/primitive": ["@radix-ui/primitive@1.1.3", "", {}, "sha512-JTF99U/6XIjCBo0wqkU5sK10glYe27MRRsfwoiq5zzOEZLHU3A3KCMa5X/azekYRCJ0HlwI0crAXS/5dEHTzDg=="],
|
||||
@@ -247,24 +238,6 @@
|
||||
|
||||
"@radix-ui/rect": ["@radix-ui/rect@1.1.1", "", {}, "sha512-HPwpGIzkl28mWyZqG52jiqDJ12waP11Pa1lGoiyUkIEuMLBP0oeK/C89esbXrxsky5we7dfd8U58nm0SgAWpVw=="],
|
||||
|
||||
"@rc-component/async-validator": ["@rc-component/async-validator@5.1.0", "", { "dependencies": { "@babel/runtime": "^7.24.4" } }, "sha512-n4HcR5siNUXRX23nDizbZBQPO0ZM/5oTtmKZ6/eqL0L2bo747cklFdZGRN2f+c9qWGICwDzrhW0H7tE9PptdcA=="],
|
||||
|
||||
"@rc-component/color-picker": ["@rc-component/color-picker@2.0.1", "", { "dependencies": { "@ant-design/fast-color": "^2.0.6", "@babel/runtime": "^7.23.6", "classnames": "^2.2.6", "rc-util": "^5.38.1" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-WcZYwAThV/b2GISQ8F+7650r5ZZJ043E57aVBFkQ+kSY4C6wdofXgB0hBx+GPGpIU0Z81eETNoDUJMr7oy/P8Q=="],
|
||||
|
||||
"@rc-component/context": ["@rc-component/context@1.4.0", "", { "dependencies": { "@babel/runtime": "^7.10.1", "rc-util": "^5.27.0" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-kFcNxg9oLRMoL3qki0OMxK+7g5mypjgaaJp/pkOis/6rVxma9nJBF/8kCIuTYHUQNr0ii7MxqE33wirPZLJQ2w=="],
|
||||
|
||||
"@rc-component/mini-decimal": ["@rc-component/mini-decimal@1.1.3", "", { "dependencies": { "@babel/runtime": "^7.18.0" } }, "sha512-bk/FJ09fLf+NLODMAFll6CfYrHPBioTedhW6lxDBuuWucJEqFUd4l/D/5JgIi3dina6sYahB8iuPAZTNz2pMxw=="],
|
||||
|
||||
"@rc-component/mutate-observer": ["@rc-component/mutate-observer@1.1.0", "", { "dependencies": { "@babel/runtime": "^7.18.0", "classnames": "^2.3.2", "rc-util": "^5.24.4" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-QjrOsDXQusNwGZPf4/qRQasg7UFEj06XiCJ8iuiq/Io7CrHrgVi6Uuetw60WAMG1799v+aM8kyc+1L/GBbHSlw=="],
|
||||
|
||||
"@rc-component/portal": ["@rc-component/portal@1.1.2", "", { "dependencies": { "@babel/runtime": "^7.18.0", "classnames": "^2.3.2", "rc-util": "^5.24.4" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-6f813C0IsasTZms08kfA8kPAGxbbkYToa8ALaiDIGGECU4i9hj8Plgbx0sNJDrey3EtHO30hmdaxtT0138xZcg=="],
|
||||
|
||||
"@rc-component/qrcode": ["@rc-component/qrcode@1.1.1", "", { "dependencies": { "@babel/runtime": "^7.24.7" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-LfLGNymzKdUPjXUbRP+xOhIWY4jQ+YMj5MmWAcgcAq1Ij8XP7tRmAXqyuv96XvLUBE/5cA8hLFl9eO1JQMujrA=="],
|
||||
|
||||
"@rc-component/tour": ["@rc-component/tour@1.15.1", "", { "dependencies": { "@babel/runtime": "^7.18.0", "@rc-component/portal": "^1.0.0-9", "@rc-component/trigger": "^2.0.0", "classnames": "^2.3.2", "rc-util": "^5.24.4" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-Tr2t7J1DKZUpfJuDZWHxyxWpfmj8EZrqSgyMZ+BCdvKZ6r1UDsfU46M/iWAAFBy961Ssfom2kv5f3UcjIL2CmQ=="],
|
||||
|
||||
"@rc-component/trigger": ["@rc-component/trigger@2.3.1", "", { "dependencies": { "@babel/runtime": "^7.23.2", "@rc-component/portal": "^1.1.0", "classnames": "^2.3.2", "rc-motion": "^2.0.0", "rc-resize-observer": "^1.3.1", "rc-util": "^5.44.0" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-ORENF39PeXTzM+gQEshuk460Z8N4+6DkjpxlpE7Q3gYy1iBpLrx0FOJz3h62ryrJZ/3zCAUIkT1Pb/8hHWpb3A=="],
|
||||
|
||||
"@remix-run/router": ["@remix-run/router@1.23.2", "", {}, "sha512-Ic6m2U/rMjTkhERIa/0ZtXJP17QUi2CbWE7cqx4J58M8aA3QTfW+2UlQ4psvTX9IO1RfNVhK3pcpdjej7L+t2w=="],
|
||||
|
||||
"@rolldown/pluginutils": ["@rolldown/pluginutils@1.0.0-beta.27", "", {}, "sha512-+d0F4MKMCbeVUJwG96uQ4SgAznZNSq93I3V+9NHA4OpvqG8mRCpGdKmK8l/dl02h2CCDHwW2FqilnTyDcAnqjA=="],
|
||||
@@ -335,6 +308,68 @@
|
||||
|
||||
"@types/babel__traverse": ["@types/babel__traverse@7.28.0", "", { "dependencies": { "@babel/types": "^7.28.2" } }, "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q=="],
|
||||
|
||||
"@types/d3": ["@types/d3@7.4.3", "", { "dependencies": { "@types/d3-array": "*", "@types/d3-axis": "*", "@types/d3-brush": "*", "@types/d3-chord": "*", "@types/d3-color": "*", "@types/d3-contour": "*", "@types/d3-delaunay": "*", "@types/d3-dispatch": "*", "@types/d3-drag": "*", "@types/d3-dsv": "*", "@types/d3-ease": "*", "@types/d3-fetch": "*", "@types/d3-force": "*", "@types/d3-format": "*", "@types/d3-geo": "*", "@types/d3-hierarchy": "*", "@types/d3-interpolate": "*", "@types/d3-path": "*", "@types/d3-polygon": "*", "@types/d3-quadtree": "*", "@types/d3-random": "*", "@types/d3-scale": "*", "@types/d3-scale-chromatic": "*", "@types/d3-selection": "*", "@types/d3-shape": "*", "@types/d3-time": "*", "@types/d3-time-format": "*", "@types/d3-timer": "*", "@types/d3-transition": "*", "@types/d3-zoom": "*" } }, "sha512-lZXZ9ckh5R8uiFVt8ogUNf+pIrK4EsWrx2Np75WvF/eTpJ0FMHNhjXk8CKEx/+gpHbNQyJWehbFaTvqmHWB3ww=="],
|
||||
|
||||
"@types/d3-array": ["@types/d3-array@3.2.2", "", {}, "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw=="],
|
||||
|
||||
"@types/d3-axis": ["@types/d3-axis@3.0.6", "", { "dependencies": { "@types/d3-selection": "*" } }, "sha512-pYeijfZuBd87T0hGn0FO1vQ/cgLk6E1ALJjfkC0oJ8cbwkZl3TpgS8bVBLZN+2jjGgg38epgxb2zmoGtSfvgMw=="],
|
||||
|
||||
"@types/d3-brush": ["@types/d3-brush@3.0.6", "", { "dependencies": { "@types/d3-selection": "*" } }, "sha512-nH60IZNNxEcrh6L1ZSMNA28rj27ut/2ZmI3r96Zd+1jrZD++zD3LsMIjWlvg4AYrHn/Pqz4CF3veCxGjtbqt7A=="],
|
||||
|
||||
"@types/d3-chord": ["@types/d3-chord@3.0.6", "", {}, "sha512-LFYWWd8nwfwEmTZG9PfQxd17HbNPksHBiJHaKuY1XeqscXacsS2tyoo6OdRsjf+NQYeB6XrNL3a25E3gH69lcg=="],
|
||||
|
||||
"@types/d3-color": ["@types/d3-color@3.1.3", "", {}, "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A=="],
|
||||
|
||||
"@types/d3-contour": ["@types/d3-contour@3.0.6", "", { "dependencies": { "@types/d3-array": "*", "@types/geojson": "*" } }, "sha512-BjzLgXGnCWjUSYGfH1cpdo41/hgdWETu4YxpezoztawmqsvCeep+8QGfiY6YbDvfgHz/DkjeIkkZVJavB4a3rg=="],
|
||||
|
||||
"@types/d3-delaunay": ["@types/d3-delaunay@6.0.4", "", {}, "sha512-ZMaSKu4THYCU6sV64Lhg6qjf1orxBthaC161plr5KuPHo3CNm8DTHiLw/5Eq2b6TsNP0W0iJrUOFscY6Q450Hw=="],
|
||||
|
||||
"@types/d3-dispatch": ["@types/d3-dispatch@3.0.7", "", {}, "sha512-5o9OIAdKkhN1QItV2oqaE5KMIiXAvDWBDPrD85e58Qlz1c1kI/J0NcqbEG88CoTwJrYe7ntUCVfeUl2UJKbWgA=="],
|
||||
|
||||
"@types/d3-drag": ["@types/d3-drag@3.0.7", "", { "dependencies": { "@types/d3-selection": "*" } }, "sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ=="],
|
||||
|
||||
"@types/d3-dsv": ["@types/d3-dsv@3.0.7", "", {}, "sha512-n6QBF9/+XASqcKK6waudgL0pf/S5XHPPI8APyMLLUHd8NqouBGLsU8MgtO7NINGtPBtk9Kko/W4ea0oAspwh9g=="],
|
||||
|
||||
"@types/d3-ease": ["@types/d3-ease@3.0.2", "", {}, "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA=="],
|
||||
|
||||
"@types/d3-fetch": ["@types/d3-fetch@3.0.7", "", { "dependencies": { "@types/d3-dsv": "*" } }, "sha512-fTAfNmxSb9SOWNB9IoG5c8Hg6R+AzUHDRlsXsDZsNp6sxAEOP0tkP3gKkNSO/qmHPoBFTxNrjDprVHDQDvo5aA=="],
|
||||
|
||||
"@types/d3-force": ["@types/d3-force@3.0.10", "", {}, "sha512-ZYeSaCF3p73RdOKcjj+swRlZfnYpK1EbaDiYICEEp5Q6sUiqFaFQ9qgoshp5CzIyyb/yD09kD9o2zEltCexlgw=="],
|
||||
|
||||
"@types/d3-format": ["@types/d3-format@3.0.4", "", {}, "sha512-fALi2aI6shfg7vM5KiR1wNJnZ7r6UuggVqtDA+xiEdPZQwy/trcQaHnwShLuLdta2rTymCNpxYTiMZX/e09F4g=="],
|
||||
|
||||
"@types/d3-geo": ["@types/d3-geo@3.1.0", "", { "dependencies": { "@types/geojson": "*" } }, "sha512-856sckF0oP/diXtS4jNsiQw/UuK5fQG8l/a9VVLeSouf1/PPbBE1i1W852zVwKwYCBkFJJB7nCFTbk6UMEXBOQ=="],
|
||||
|
||||
"@types/d3-hierarchy": ["@types/d3-hierarchy@3.1.7", "", {}, "sha512-tJFtNoYBtRtkNysX1Xq4sxtjK8YgoWUNpIiUee0/jHGRwqvzYxkq0hGVbbOGSz+JgFxxRu4K8nb3YpG3CMARtg=="],
|
||||
|
||||
"@types/d3-interpolate": ["@types/d3-interpolate@3.0.4", "", { "dependencies": { "@types/d3-color": "*" } }, "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA=="],
|
||||
|
||||
"@types/d3-path": ["@types/d3-path@3.1.1", "", {}, "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg=="],
|
||||
|
||||
"@types/d3-polygon": ["@types/d3-polygon@3.0.2", "", {}, "sha512-ZuWOtMaHCkN9xoeEMr1ubW2nGWsp4nIql+OPQRstu4ypeZ+zk3YKqQT0CXVe/PYqrKpZAi+J9mTs05TKwjXSRA=="],
|
||||
|
||||
"@types/d3-quadtree": ["@types/d3-quadtree@3.0.6", "", {}, "sha512-oUzyO1/Zm6rsxKRHA1vH0NEDG58HrT5icx/azi9MF1TWdtttWl0UIUsjEQBBh+SIkrpd21ZjEv7ptxWys1ncsg=="],
|
||||
|
||||
"@types/d3-random": ["@types/d3-random@3.0.3", "", {}, "sha512-Imagg1vJ3y76Y2ea0871wpabqp613+8/r0mCLEBfdtqC7xMSfj9idOnmBYyMoULfHePJyxMAw3nWhJxzc+LFwQ=="],
|
||||
|
||||
"@types/d3-scale": ["@types/d3-scale@4.0.9", "", { "dependencies": { "@types/d3-time": "*" } }, "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw=="],
|
||||
|
||||
"@types/d3-scale-chromatic": ["@types/d3-scale-chromatic@3.1.0", "", {}, "sha512-iWMJgwkK7yTRmWqRB5plb1kadXyQ5Sj8V/zYlFGMUBbIPKQScw+Dku9cAAMgJG+z5GYDoMjWGLVOvjghDEFnKQ=="],
|
||||
|
||||
"@types/d3-selection": ["@types/d3-selection@3.0.11", "", {}, "sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w=="],
|
||||
|
||||
"@types/d3-shape": ["@types/d3-shape@3.1.8", "", { "dependencies": { "@types/d3-path": "*" } }, "sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w=="],
|
||||
|
||||
"@types/d3-time": ["@types/d3-time@3.0.4", "", {}, "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g=="],
|
||||
|
||||
"@types/d3-time-format": ["@types/d3-time-format@4.0.3", "", {}, "sha512-5xg9rC+wWL8kdDj153qZcsJ0FWiFt0J5RB6LYUNZjwSnesfblqrI/bJ1wBdJ8OQfncgbJG5+2F+qfqnqyzYxyg=="],
|
||||
|
||||
"@types/d3-timer": ["@types/d3-timer@3.0.2", "", {}, "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw=="],
|
||||
|
||||
"@types/d3-transition": ["@types/d3-transition@3.0.9", "", { "dependencies": { "@types/d3-selection": "*" } }, "sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg=="],
|
||||
|
||||
"@types/d3-zoom": ["@types/d3-zoom@3.0.8", "", { "dependencies": { "@types/d3-interpolate": "*", "@types/d3-selection": "*" } }, "sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw=="],
|
||||
|
||||
"@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="],
|
||||
|
||||
"@types/geojson": ["@types/geojson@7946.0.16", "", {}, "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg=="],
|
||||
@@ -347,9 +382,11 @@
|
||||
|
||||
"@types/react-dom": ["@types/react-dom@18.3.7", "", { "peerDependencies": { "@types/react": "^18.0.0" } }, "sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ=="],
|
||||
|
||||
"@vitejs/plugin-react": ["@vitejs/plugin-react@4.7.0", "", { "dependencies": { "@babel/core": "^7.28.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-beta.27", "@types/babel__core": "^7.20.5", "react-refresh": "^0.17.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" } }, "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA=="],
|
||||
"@types/trusted-types": ["@types/trusted-types@2.0.7", "", {}, "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw=="],
|
||||
|
||||
"antd": ["antd@5.29.3", "", { "dependencies": { "@ant-design/colors": "^7.2.1", "@ant-design/cssinjs": "^1.23.0", "@ant-design/cssinjs-utils": "^1.1.3", "@ant-design/fast-color": "^2.0.6", "@ant-design/icons": "^5.6.1", "@ant-design/react-slick": "~1.1.2", "@babel/runtime": "^7.26.0", "@rc-component/color-picker": "~2.0.1", "@rc-component/mutate-observer": "^1.1.0", "@rc-component/qrcode": "~1.1.0", "@rc-component/tour": "~1.15.1", "@rc-component/trigger": "^2.3.0", "classnames": "^2.5.1", "copy-to-clipboard": "^3.3.3", "dayjs": "^1.11.11", "rc-cascader": "~3.34.0", "rc-checkbox": "~3.5.0", "rc-collapse": "~3.9.0", "rc-dialog": "~9.6.0", "rc-drawer": "~7.3.0", "rc-dropdown": "~4.2.1", "rc-field-form": "~2.7.1", "rc-image": "~7.12.0", "rc-input": "~1.8.0", "rc-input-number": "~9.5.0", "rc-mentions": "~2.20.0", "rc-menu": "~9.16.1", "rc-motion": "^2.9.5", "rc-notification": "~5.6.4", "rc-pagination": "~5.1.0", "rc-picker": "~4.11.3", "rc-progress": "~4.0.0", "rc-rate": "~2.13.1", "rc-resize-observer": "^1.4.3", "rc-segmented": "~2.7.0", "rc-select": "~14.16.8", "rc-slider": "~11.1.9", "rc-steps": "~6.0.1", "rc-switch": "~4.1.0", "rc-table": "~7.54.0", "rc-tabs": "~15.7.0", "rc-textarea": "~1.10.2", "rc-tooltip": "~6.4.0", "rc-tree": "~5.13.1", "rc-tree-select": "~5.27.0", "rc-upload": "~4.11.0", "rc-util": "^5.44.4", "scroll-into-view-if-needed": "^3.1.0", "throttle-debounce": "^5.0.2" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-3DdbGCa9tWAJGcCJ6rzR8EJFsv2CtyEbkVabZE14pfgUHfCicWCj0/QzQVLDYg8CPfQk9BH7fHCoTXHTy7MP/A=="],
|
||||
"@upsetjs/venn.js": ["@upsetjs/venn.js@2.0.0", "", { "optionalDependencies": { "d3-selection": "^3.0.0", "d3-transition": "^3.0.1" } }, "sha512-WbBhLrooyePuQ1VZxrJjtLvTc4NVfpOyKx0sKqioq9bX1C1m7Jgykkn8gLrtwumBioXIqam8DLxp88Adbue6Hw=="],
|
||||
|
||||
"@vitejs/plugin-react": ["@vitejs/plugin-react@4.7.0", "", { "dependencies": { "@babel/core": "^7.28.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-beta.27", "@types/babel__core": "^7.20.5", "react-refresh": "^0.17.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" } }, "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA=="],
|
||||
|
||||
"aria-hidden": ["aria-hidden@1.2.6", "", { "dependencies": { "tslib": "^2.0.0" } }, "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA=="],
|
||||
|
||||
@@ -369,28 +406,102 @@
|
||||
|
||||
"class-variance-authority": ["class-variance-authority@0.7.1", "", { "dependencies": { "clsx": "^2.1.1" } }, "sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg=="],
|
||||
|
||||
"classnames": ["classnames@2.5.1", "", {}, "sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow=="],
|
||||
|
||||
"clsx": ["clsx@2.1.1", "", {}, "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=="],
|
||||
|
||||
"combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="],
|
||||
|
||||
"compute-scroll-into-view": ["compute-scroll-into-view@3.1.1", "", {}, "sha512-VRhuHOLoKYOy4UbilLbUzbYg93XLjv2PncJC50EuTWPA3gaja1UjBsUP/D/9/juV3vQFr6XBEzn9KCAHdUvOHw=="],
|
||||
"commander": ["commander@8.3.0", "", {}, "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww=="],
|
||||
|
||||
"convert-source-map": ["convert-source-map@2.0.0", "", {}, "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="],
|
||||
|
||||
"copy-to-clipboard": ["copy-to-clipboard@3.3.3", "", { "dependencies": { "toggle-selection": "^1.0.6" } }, "sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA=="],
|
||||
"cose-base": ["cose-base@1.0.3", "", { "dependencies": { "layout-base": "^1.0.0" } }, "sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg=="],
|
||||
|
||||
"csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="],
|
||||
|
||||
"cytoscape": ["cytoscape@3.33.4", "", {}, "sha512-HIN5Pmd9MrX9BkV7tDwnOcEJCSFvCpc8X97h3f508J6I5FsqAY65wKOCvgH2CuP42CaahWaz4tuh32SOOIH7ww=="],
|
||||
|
||||
"cytoscape-cose-bilkent": ["cytoscape-cose-bilkent@4.1.0", "", { "dependencies": { "cose-base": "^1.0.0" }, "peerDependencies": { "cytoscape": "^3.2.0" } }, "sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ=="],
|
||||
|
||||
"cytoscape-fcose": ["cytoscape-fcose@2.2.0", "", { "dependencies": { "cose-base": "^2.2.0" }, "peerDependencies": { "cytoscape": "^3.2.0" } }, "sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ=="],
|
||||
|
||||
"d3": ["d3@7.9.0", "", { "dependencies": { "d3-array": "3", "d3-axis": "3", "d3-brush": "3", "d3-chord": "3", "d3-color": "3", "d3-contour": "4", "d3-delaunay": "6", "d3-dispatch": "3", "d3-drag": "3", "d3-dsv": "3", "d3-ease": "3", "d3-fetch": "3", "d3-force": "3", "d3-format": "3", "d3-geo": "3", "d3-hierarchy": "3", "d3-interpolate": "3", "d3-path": "3", "d3-polygon": "3", "d3-quadtree": "3", "d3-random": "3", "d3-scale": "4", "d3-scale-chromatic": "3", "d3-selection": "3", "d3-shape": "3", "d3-time": "3", "d3-time-format": "4", "d3-timer": "3", "d3-transition": "3", "d3-zoom": "3" } }, "sha512-e1U46jVP+w7Iut8Jt8ri1YsPOvFpg46k+K8TpCb0P+zjCkjkPnV7WzfDJzMHy1LnA+wj5pLT1wjO901gLXeEhA=="],
|
||||
|
||||
"d3-array": ["d3-array@3.2.4", "", { "dependencies": { "internmap": "1 - 2" } }, "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg=="],
|
||||
|
||||
"d3-axis": ["d3-axis@3.0.0", "", {}, "sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw=="],
|
||||
|
||||
"d3-brush": ["d3-brush@3.0.0", "", { "dependencies": { "d3-dispatch": "1 - 3", "d3-drag": "2 - 3", "d3-interpolate": "1 - 3", "d3-selection": "3", "d3-transition": "3" } }, "sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ=="],
|
||||
|
||||
"d3-chord": ["d3-chord@3.0.1", "", { "dependencies": { "d3-path": "1 - 3" } }, "sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g=="],
|
||||
|
||||
"d3-color": ["d3-color@3.1.0", "", {}, "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA=="],
|
||||
|
||||
"d3-contour": ["d3-contour@4.0.2", "", { "dependencies": { "d3-array": "^3.2.0" } }, "sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA=="],
|
||||
|
||||
"d3-delaunay": ["d3-delaunay@6.0.4", "", { "dependencies": { "delaunator": "5" } }, "sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A=="],
|
||||
|
||||
"d3-dispatch": ["d3-dispatch@3.0.1", "", {}, "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg=="],
|
||||
|
||||
"d3-drag": ["d3-drag@3.0.0", "", { "dependencies": { "d3-dispatch": "1 - 3", "d3-selection": "3" } }, "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg=="],
|
||||
|
||||
"d3-dsv": ["d3-dsv@3.0.1", "", { "dependencies": { "commander": "7", "iconv-lite": "0.6", "rw": "1" }, "bin": { "csv2json": "bin/dsv2json.js", "csv2tsv": "bin/dsv2dsv.js", "dsv2dsv": "bin/dsv2dsv.js", "dsv2json": "bin/dsv2json.js", "json2csv": "bin/json2dsv.js", "json2dsv": "bin/json2dsv.js", "json2tsv": "bin/json2dsv.js", "tsv2csv": "bin/dsv2dsv.js", "tsv2json": "bin/dsv2json.js" } }, "sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q=="],
|
||||
|
||||
"d3-ease": ["d3-ease@3.0.1", "", {}, "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w=="],
|
||||
|
||||
"d3-fetch": ["d3-fetch@3.0.1", "", { "dependencies": { "d3-dsv": "1 - 3" } }, "sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw=="],
|
||||
|
||||
"d3-force": ["d3-force@3.0.0", "", { "dependencies": { "d3-dispatch": "1 - 3", "d3-quadtree": "1 - 3", "d3-timer": "1 - 3" } }, "sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg=="],
|
||||
|
||||
"d3-format": ["d3-format@3.1.2", "", {}, "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg=="],
|
||||
|
||||
"d3-geo": ["d3-geo@3.1.1", "", { "dependencies": { "d3-array": "2.5.0 - 3" } }, "sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q=="],
|
||||
|
||||
"d3-hierarchy": ["d3-hierarchy@3.1.2", "", {}, "sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA=="],
|
||||
|
||||
"d3-interpolate": ["d3-interpolate@3.0.1", "", { "dependencies": { "d3-color": "1 - 3" } }, "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g=="],
|
||||
|
||||
"d3-path": ["d3-path@3.1.0", "", {}, "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ=="],
|
||||
|
||||
"d3-polygon": ["d3-polygon@3.0.1", "", {}, "sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg=="],
|
||||
|
||||
"d3-quadtree": ["d3-quadtree@3.0.1", "", {}, "sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw=="],
|
||||
|
||||
"d3-random": ["d3-random@3.0.1", "", {}, "sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ=="],
|
||||
|
||||
"d3-sankey": ["d3-sankey@0.12.3", "", { "dependencies": { "d3-array": "1 - 2", "d3-shape": "^1.2.0" } }, "sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ=="],
|
||||
|
||||
"d3-scale": ["d3-scale@4.0.2", "", { "dependencies": { "d3-array": "2.10.0 - 3", "d3-format": "1 - 3", "d3-interpolate": "1.2.0 - 3", "d3-time": "2.1.1 - 3", "d3-time-format": "2 - 4" } }, "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ=="],
|
||||
|
||||
"d3-scale-chromatic": ["d3-scale-chromatic@3.1.0", "", { "dependencies": { "d3-color": "1 - 3", "d3-interpolate": "1 - 3" } }, "sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ=="],
|
||||
|
||||
"d3-selection": ["d3-selection@3.0.0", "", {}, "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ=="],
|
||||
|
||||
"d3-shape": ["d3-shape@3.2.0", "", { "dependencies": { "d3-path": "^3.1.0" } }, "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA=="],
|
||||
|
||||
"d3-time": ["d3-time@3.1.0", "", { "dependencies": { "d3-array": "2 - 3" } }, "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q=="],
|
||||
|
||||
"d3-time-format": ["d3-time-format@4.1.0", "", { "dependencies": { "d3-time": "1 - 3" } }, "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg=="],
|
||||
|
||||
"d3-timer": ["d3-timer@3.0.1", "", {}, "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA=="],
|
||||
|
||||
"d3-transition": ["d3-transition@3.0.1", "", { "dependencies": { "d3-color": "1 - 3", "d3-dispatch": "1 - 3", "d3-ease": "1 - 3", "d3-interpolate": "1 - 3", "d3-timer": "1 - 3" }, "peerDependencies": { "d3-selection": "2 - 3" } }, "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w=="],
|
||||
|
||||
"d3-zoom": ["d3-zoom@3.0.0", "", { "dependencies": { "d3-dispatch": "1 - 3", "d3-drag": "2 - 3", "d3-interpolate": "1 - 3", "d3-selection": "2 - 3", "d3-transition": "2 - 3" } }, "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw=="],
|
||||
|
||||
"dagre-d3-es": ["dagre-d3-es@7.0.14", "", { "dependencies": { "d3": "^7.9.0", "lodash-es": "^4.17.21" } }, "sha512-P4rFMVq9ESWqmOgK+dlXvOtLwYg0i7u0HBGJER0LZDJT2VHIPAMZ/riPxqJceWMStH5+E61QxFra9kIS3AqdMg=="],
|
||||
|
||||
"dayjs": ["dayjs@1.11.19", "", {}, "sha512-t5EcLVS6QPBNqM2z8fakk/NKel+Xzshgt8FFKAn+qwlD1pzZWxh0nVCrvFK7ZDb6XucZeF9z8C7CBWTRIVApAw=="],
|
||||
|
||||
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
|
||||
|
||||
"delaunator": ["delaunator@5.1.0", "", { "dependencies": { "robust-predicates": "^3.0.2" } }, "sha512-AGrQ4QSgssa1NGmWmLPqN5NY2KajF5MqxetNEO+o0n3ZwZZeTmt7bBnvzHWrmkZFxGgr4HdyFgelzgi06otLuQ=="],
|
||||
|
||||
"delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="],
|
||||
|
||||
"detect-node-es": ["detect-node-es@1.1.0", "", {}, "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ=="],
|
||||
|
||||
"dompurify": ["dompurify@3.4.5", "", { "optionalDependencies": { "@types/trusted-types": "^2.0.7" } }, "sha512-OrwIBKsdNSVEeubdJ1HBv/wNENRM9ytAVCv7YXt//A3vPdVMNuACRqK9mXCGCBW2ln7BT/A4X0jXHo2Gu89miA=="],
|
||||
|
||||
"dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
|
||||
|
||||
"echarts": ["echarts@6.0.0", "", { "dependencies": { "tslib": "2.3.0", "zrender": "6.0.0" } }, "sha512-Tte/grDQRiETQP4xz3iZWSvoHrkCQtwqd6hs+mifXcjrCuo2iKWbajFObuLJVBlDIJlOzgQPd1hsaKt/3+OMkQ=="],
|
||||
@@ -409,6 +520,8 @@
|
||||
|
||||
"es-set-tostringtag": ["es-set-tostringtag@2.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", "hasown": "^2.0.2" } }, "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA=="],
|
||||
|
||||
"es-toolkit": ["es-toolkit@1.47.0", "", {}, "sha512-n1GuoD0WEQZMBk5tttoZSqwgyLx01oqa5XsBmCHwPyNe1S9jPBEmtR2pSgp2kJuWE3ciFZ6yRHmY4pM4C3OOkw=="],
|
||||
|
||||
"esbuild": ["esbuild@0.21.5", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.21.5", "@esbuild/android-arm": "0.21.5", "@esbuild/android-arm64": "0.21.5", "@esbuild/android-x64": "0.21.5", "@esbuild/darwin-arm64": "0.21.5", "@esbuild/darwin-x64": "0.21.5", "@esbuild/freebsd-arm64": "0.21.5", "@esbuild/freebsd-x64": "0.21.5", "@esbuild/linux-arm": "0.21.5", "@esbuild/linux-arm64": "0.21.5", "@esbuild/linux-ia32": "0.21.5", "@esbuild/linux-loong64": "0.21.5", "@esbuild/linux-mips64el": "0.21.5", "@esbuild/linux-ppc64": "0.21.5", "@esbuild/linux-riscv64": "0.21.5", "@esbuild/linux-s390x": "0.21.5", "@esbuild/linux-x64": "0.21.5", "@esbuild/netbsd-x64": "0.21.5", "@esbuild/openbsd-x64": "0.21.5", "@esbuild/sunos-x64": "0.21.5", "@esbuild/win32-arm64": "0.21.5", "@esbuild/win32-ia32": "0.21.5", "@esbuild/win32-x64": "0.21.5" }, "bin": "bin/esbuild" }, "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw=="],
|
||||
|
||||
"escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="],
|
||||
@@ -435,28 +548,46 @@
|
||||
|
||||
"gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="],
|
||||
|
||||
"hachure-fill": ["hachure-fill@0.5.2", "", {}, "sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg=="],
|
||||
|
||||
"has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="],
|
||||
|
||||
"has-tostringtag": ["has-tostringtag@1.0.2", "", { "dependencies": { "has-symbols": "^1.0.3" } }, "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw=="],
|
||||
|
||||
"hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
|
||||
|
||||
"iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="],
|
||||
|
||||
"import-meta-resolve": ["import-meta-resolve@4.2.0", "", {}, "sha512-Iqv2fzaTQN28s/FwZAoFq0ZSs/7hMAHJVX+w8PZl3cY19Pxk6jFFalxQoIfW2826i/fDLXv8IiEZRIT0lDuWcg=="],
|
||||
|
||||
"internmap": ["internmap@2.0.3", "", {}, "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg=="],
|
||||
|
||||
"js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="],
|
||||
|
||||
"jsesc": ["jsesc@3.1.0", "", { "bin": "bin/jsesc" }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="],
|
||||
|
||||
"json2mq": ["json2mq@0.2.0", "", { "dependencies": { "string-convert": "^0.2.0" } }, "sha512-SzoRg7ux5DWTII9J2qkrZrqV1gt+rTaoufMxEzXbS26Uid0NwaJd123HcoB80TgubEppxxIGdNxCx50fEoEWQA=="],
|
||||
|
||||
"json5": ["json5@2.2.3", "", { "bin": "lib/cli.js" }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="],
|
||||
|
||||
"katex": ["katex@0.16.47", "", { "dependencies": { "commander": "^8.3.0" }, "bin": { "katex": "cli.js" } }, "sha512-Eeo8Ys1doU1z+x8AZsPpQu+p/QcZBI5PeOo7QGQdy2x2m0MU/hYagBbGOmXwr5KVbEfVuWv9LpnQWeehogurjg=="],
|
||||
|
||||
"khroma": ["khroma@2.1.0", "", {}, "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw=="],
|
||||
|
||||
"layout-base": ["layout-base@1.0.2", "", {}, "sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg=="],
|
||||
|
||||
"lodash-es": ["lodash-es@4.18.1", "", {}, "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A=="],
|
||||
|
||||
"loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": "cli.js" }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="],
|
||||
|
||||
"lru-cache": ["lru-cache@5.1.1", "", { "dependencies": { "yallist": "^3.0.2" } }, "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w=="],
|
||||
|
||||
"lucide-react": ["lucide-react@1.16.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-dYwyPzb4MEKpGUmNYk3WKWPnMrHs3FKM+q94kAnJrcDIqqn1hq2xY8scaS2ovsOCM5D51ey2gaRG3PBb1vgoYQ=="],
|
||||
|
||||
"marked": ["marked@16.4.2", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-TI3V8YYWvkVf3KJe1dRkpnjs68JUPyEa5vjKrp1XEEJUAOaQc+Qj+L1qWbPd0SJuAdQkFU0h73sXXqwDYxsiDA=="],
|
||||
|
||||
"math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="],
|
||||
|
||||
"mermaid": ["mermaid@11.15.0", "", { "dependencies": { "@braintree/sanitize-url": "^7.1.1", "@iconify/utils": "^3.0.2", "@mermaid-js/parser": "^1.1.1", "@types/d3": "^7.4.3", "@upsetjs/venn.js": "^2.0.0", "cytoscape": "^3.33.1", "cytoscape-cose-bilkent": "^4.1.0", "cytoscape-fcose": "^2.2.0", "d3": "^7.9.0", "d3-sankey": "^0.12.3", "dagre-d3-es": "7.0.14", "dayjs": "^1.11.19", "dompurify": "^3.3.1", "es-toolkit": "^1.45.1", "katex": "^0.16.25", "khroma": "^2.1.0", "marked": "^16.3.0", "roughjs": "^4.6.6", "stylis": "^4.3.6", "ts-dedent": "^2.2.0", "uuid": "^11.1.0 || ^12 || ^13 || ^14.0.0" } }, "sha512-pTMbcf3rWdtLiYGpmoTjHEpeY8seiy6sR+9nD7LOs8KfUbHE4lOUAprTRqRAcWSQ6MQpdX+YEsxShtGsINtPtw=="],
|
||||
|
||||
"mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="],
|
||||
|
||||
"mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="],
|
||||
@@ -469,12 +600,20 @@
|
||||
|
||||
"object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="],
|
||||
|
||||
"package-manager-detector": ["package-manager-detector@1.6.0", "", {}, "sha512-61A5ThoTiDG/C8s8UMZwSorAGwMJ0ERVGj2OjoW5pAalsNOg15+iQiPzrLJ4jhZ1HJzmC2PIHT2oEiH3R5fzNA=="],
|
||||
|
||||
"path-data-parser": ["path-data-parser@0.1.0", "", {}, "sha512-NOnmBpt5Y2RWbuv0LMzsayp3lVylAHLPUTut412ZA3l+C4uw4ZVkQbjShYCQ8TCpUMdPapr4YjUqLYD6v68j+w=="],
|
||||
|
||||
"pbf": ["pbf@4.0.1", "", { "dependencies": { "resolve-protobuf-schema": "^2.1.0" }, "bin": { "pbf": "bin/pbf" } }, "sha512-SuLdBvS42z33m8ejRbInMapQe8n0D3vN/Xd5fmWM3tufNgRQFBpaW2YVJxQZV4iPNqb0vEFvssMEo5w9c6BTIA=="],
|
||||
|
||||
"picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="],
|
||||
|
||||
"pmtiles": ["pmtiles@4.4.1", "", { "dependencies": { "fflate": "^0.8.2" } }, "sha512-5oTeQc/yX/ft1evbpIlnoCZugQuug/iYIAj/ZTqIqzdGek4uZEho99En890EE6NOSI3JTI3IG8R7r8+SltphxA=="],
|
||||
|
||||
"points-on-curve": ["points-on-curve@0.2.0", "", {}, "sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A=="],
|
||||
|
||||
"points-on-path": ["points-on-path@0.2.1", "", { "dependencies": { "path-data-parser": "0.1.0", "points-on-curve": "0.2.0" } }, "sha512-25ClnWWuw7JbWZcgqY/gJ4FQWadKxGWk+3kR/7kD0tCaDtPPMj7oHu2ToLaVhfpnHrZzYby2w6tUA0eOIuUg8g=="],
|
||||
|
||||
"postcss": ["postcss@8.5.14", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg=="],
|
||||
|
||||
"postcss-value-parser": ["postcss-value-parser@4.2.0", "", {}, "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ=="],
|
||||
@@ -485,74 +624,6 @@
|
||||
|
||||
"proxy-from-env": ["proxy-from-env@1.1.0", "", {}, "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="],
|
||||
|
||||
"rc-cascader": ["rc-cascader@3.34.0", "", { "dependencies": { "@babel/runtime": "^7.25.7", "classnames": "^2.3.1", "rc-select": "~14.16.2", "rc-tree": "~5.13.0", "rc-util": "^5.43.0" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-KpXypcvju9ptjW9FaN2NFcA2QH9E9LHKq169Y0eWtH4e/wHQ5Wh5qZakAgvb8EKZ736WZ3B0zLLOBsrsja5Dag=="],
|
||||
|
||||
"rc-checkbox": ["rc-checkbox@3.5.0", "", { "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.3.2", "rc-util": "^5.25.2" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-aOAQc3E98HteIIsSqm6Xk2FPKIER6+5vyEFMZfo73TqM+VVAIqOkHoPjgKLqSNtVLWScoaM7vY2ZrGEheI79yg=="],
|
||||
|
||||
"rc-collapse": ["rc-collapse@3.9.0", "", { "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "2.x", "rc-motion": "^2.3.4", "rc-util": "^5.27.0" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-swDdz4QZ4dFTo4RAUMLL50qP0EY62N2kvmk2We5xYdRwcRn8WcYtuetCJpwpaCbUfUt5+huLpVxhvmnK+PHrkA=="],
|
||||
|
||||
"rc-dialog": ["rc-dialog@9.6.0", "", { "dependencies": { "@babel/runtime": "^7.10.1", "@rc-component/portal": "^1.0.0-8", "classnames": "^2.2.6", "rc-motion": "^2.3.0", "rc-util": "^5.21.0" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-ApoVi9Z8PaCQg6FsUzS8yvBEQy0ZL2PkuvAgrmohPkN3okps5WZ5WQWPc1RNuiOKaAYv8B97ACdsFU5LizzCqg=="],
|
||||
|
||||
"rc-drawer": ["rc-drawer@7.3.0", "", { "dependencies": { "@babel/runtime": "^7.23.9", "@rc-component/portal": "^1.1.1", "classnames": "^2.2.6", "rc-motion": "^2.6.1", "rc-util": "^5.38.1" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-DX6CIgiBWNpJIMGFO8BAISFkxiuKitoizooj4BDyee8/SnBn0zwO2FHrNDpqqepj0E/TFTDpmEBCyFuTgC7MOg=="],
|
||||
|
||||
"rc-dropdown": ["rc-dropdown@4.2.1", "", { "dependencies": { "@babel/runtime": "^7.18.3", "@rc-component/trigger": "^2.0.0", "classnames": "^2.2.6", "rc-util": "^5.44.1" }, "peerDependencies": { "react": ">=16.11.0", "react-dom": ">=16.11.0" } }, "sha512-YDAlXsPv3I1n42dv1JpdM7wJ+gSUBfeyPK59ZpBD9jQhK9jVuxpjj3NmWQHOBceA1zEPVX84T2wbdb2SD0UjmA=="],
|
||||
|
||||
"rc-field-form": ["rc-field-form@2.7.1", "", { "dependencies": { "@babel/runtime": "^7.18.0", "@rc-component/async-validator": "^5.0.3", "rc-util": "^5.32.2" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-vKeSifSJ6HoLaAB+B8aq/Qgm8a3dyxROzCtKNCsBQgiverpc4kWDQihoUwzUj+zNWJOykwSY4dNX3QrGwtVb9A=="],
|
||||
|
||||
"rc-image": ["rc-image@7.12.0", "", { "dependencies": { "@babel/runtime": "^7.11.2", "@rc-component/portal": "^1.0.2", "classnames": "^2.2.6", "rc-dialog": "~9.6.0", "rc-motion": "^2.6.2", "rc-util": "^5.34.1" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-cZ3HTyyckPnNnUb9/DRqduqzLfrQRyi+CdHjdqgsyDpI3Ln5UX1kXnAhPBSJj9pVRzwRFgqkN7p9b6HBDjmu/Q=="],
|
||||
|
||||
"rc-input": ["rc-input@1.8.0", "", { "dependencies": { "@babel/runtime": "^7.11.1", "classnames": "^2.2.1", "rc-util": "^5.18.1" }, "peerDependencies": { "react": ">=16.0.0", "react-dom": ">=16.0.0" } }, "sha512-KXvaTbX+7ha8a/k+eg6SYRVERK0NddX8QX7a7AnRvUa/rEH0CNMlpcBzBkhI0wp2C8C4HlMoYl8TImSN+fuHKA=="],
|
||||
|
||||
"rc-input-number": ["rc-input-number@9.5.0", "", { "dependencies": { "@babel/runtime": "^7.10.1", "@rc-component/mini-decimal": "^1.0.1", "classnames": "^2.2.5", "rc-input": "~1.8.0", "rc-util": "^5.40.1" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-bKaEvB5tHebUURAEXw35LDcnRZLq3x1k7GxfAqBMzmpHkDGzjAtnUL8y4y5N15rIFIg5IJgwr211jInl3cipag=="],
|
||||
|
||||
"rc-mentions": ["rc-mentions@2.20.0", "", { "dependencies": { "@babel/runtime": "^7.22.5", "@rc-component/trigger": "^2.0.0", "classnames": "^2.2.6", "rc-input": "~1.8.0", "rc-menu": "~9.16.0", "rc-textarea": "~1.10.0", "rc-util": "^5.34.1" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-w8HCMZEh3f0nR8ZEd466ATqmXFCMGMN5UFCzEUL0bM/nGw/wOS2GgRzKBcm19K++jDyuWCOJOdgcKGXU3fXfbQ=="],
|
||||
|
||||
"rc-menu": ["rc-menu@9.16.1", "", { "dependencies": { "@babel/runtime": "^7.10.1", "@rc-component/trigger": "^2.0.0", "classnames": "2.x", "rc-motion": "^2.4.3", "rc-overflow": "^1.3.1", "rc-util": "^5.27.0" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-ghHx6/6Dvp+fw8CJhDUHFHDJ84hJE3BXNCzSgLdmNiFErWSOaZNsihDAsKq9ByTALo/xkNIwtDFGIl6r+RPXBg=="],
|
||||
|
||||
"rc-motion": ["rc-motion@2.9.5", "", { "dependencies": { "@babel/runtime": "^7.11.1", "classnames": "^2.2.1", "rc-util": "^5.44.0" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-w+XTUrfh7ArbYEd2582uDrEhmBHwK1ZENJiSJVb7uRxdE7qJSYjbO2eksRXmndqyKqKoYPc9ClpPh5242mV1vA=="],
|
||||
|
||||
"rc-notification": ["rc-notification@5.6.4", "", { "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "2.x", "rc-motion": "^2.9.0", "rc-util": "^5.20.1" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-KcS4O6B4qzM3KH7lkwOB7ooLPZ4b6J+VMmQgT51VZCeEcmghdeR4IrMcFq0LG+RPdnbe/ArT086tGM8Snimgiw=="],
|
||||
|
||||
"rc-overflow": ["rc-overflow@1.5.0", "", { "dependencies": { "@babel/runtime": "^7.11.1", "classnames": "^2.2.1", "rc-resize-observer": "^1.0.0", "rc-util": "^5.37.0" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-Lm/v9h0LymeUYJf0x39OveU52InkdRXqnn2aYXfWmo8WdOonIKB2kfau+GF0fWq6jPgtdO9yMqveGcK6aIhJmg=="],
|
||||
|
||||
"rc-pagination": ["rc-pagination@5.1.0", "", { "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.3.2", "rc-util": "^5.38.0" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-8416Yip/+eclTFdHXLKTxZvn70duYVGTvUUWbckCCZoIl3jagqke3GLsFrMs0bsQBikiYpZLD9206Ej4SOdOXQ=="],
|
||||
|
||||
"rc-picker": ["rc-picker@4.11.3", "", { "dependencies": { "@babel/runtime": "^7.24.7", "@rc-component/trigger": "^2.0.0", "classnames": "^2.2.1", "rc-overflow": "^1.3.2", "rc-resize-observer": "^1.4.0", "rc-util": "^5.43.0" }, "peerDependencies": { "date-fns": ">= 2.x", "dayjs": ">= 1.x", "luxon": ">= 3.x", "moment": ">= 2.x", "react": ">=16.9.0", "react-dom": ">=16.9.0" }, "optionalPeers": ["date-fns", "dayjs", "luxon", "moment"] }, "sha512-MJ5teb7FlNE0NFHTncxXQ62Y5lytq6sh5nUw0iH8OkHL/TjARSEvSHpr940pWgjGANpjCwyMdvsEV55l5tYNSg=="],
|
||||
|
||||
"rc-progress": ["rc-progress@4.0.0", "", { "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.6", "rc-util": "^5.16.1" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-oofVMMafOCokIUIBnZLNcOZFsABaUw8PPrf1/y0ZBvKZNpOiu5h4AO9vv11Sw0p4Hb3D0yGWuEattcQGtNJ/aw=="],
|
||||
|
||||
"rc-rate": ["rc-rate@2.13.1", "", { "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.5", "rc-util": "^5.0.1" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-QUhQ9ivQ8Gy7mtMZPAjLbxBt5y9GRp65VcUyGUMF3N3fhiftivPHdpuDIaWIMOTEprAjZPC08bls1dQB+I1F2Q=="],
|
||||
|
||||
"rc-resize-observer": ["rc-resize-observer@1.4.3", "", { "dependencies": { "@babel/runtime": "^7.20.7", "classnames": "^2.2.1", "rc-util": "^5.44.1", "resize-observer-polyfill": "^1.5.1" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-YZLjUbyIWox8E9i9C3Tm7ia+W7euPItNWSPX5sCcQTYbnwDb5uNpnLHQCG1f22oZWUhLw4Mv2tFmeWe68CDQRQ=="],
|
||||
|
||||
"rc-segmented": ["rc-segmented@2.7.1", "", { "dependencies": { "@babel/runtime": "^7.11.1", "classnames": "^2.2.1", "rc-motion": "^2.4.4", "rc-util": "^5.17.0" }, "peerDependencies": { "react": ">=16.0.0", "react-dom": ">=16.0.0" } }, "sha512-izj1Nw/Dw2Vb7EVr+D/E9lUTkBe+kKC+SAFSU9zqr7WV2W5Ktaa9Gc7cB2jTqgk8GROJayltaec+DBlYKc6d+g=="],
|
||||
|
||||
"rc-select": ["rc-select@14.16.8", "", { "dependencies": { "@babel/runtime": "^7.10.1", "@rc-component/trigger": "^2.1.1", "classnames": "2.x", "rc-motion": "^2.0.1", "rc-overflow": "^1.3.1", "rc-util": "^5.16.1", "rc-virtual-list": "^3.5.2" }, "peerDependencies": { "react": "*", "react-dom": "*" } }, "sha512-NOV5BZa1wZrsdkKaiK7LHRuo5ZjZYMDxPP6/1+09+FB4KoNi8jcG1ZqLE3AVCxEsYMBe65OBx71wFoHRTP3LRg=="],
|
||||
|
||||
"rc-slider": ["rc-slider@11.1.9", "", { "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.5", "rc-util": "^5.36.0" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-h8IknhzSh3FEM9u8ivkskh+Ef4Yo4JRIY2nj7MrH6GQmrwV6mcpJf5/4KgH5JaVI1H3E52yCdpOlVyGZIeph5A=="],
|
||||
|
||||
"rc-steps": ["rc-steps@6.0.1", "", { "dependencies": { "@babel/runtime": "^7.16.7", "classnames": "^2.2.3", "rc-util": "^5.16.1" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-lKHL+Sny0SeHkQKKDJlAjV5oZ8DwCdS2hFhAkIjuQt1/pB81M0cA0ErVFdHq9+jmPmFw1vJB2F5NBzFXLJxV+g=="],
|
||||
|
||||
"rc-switch": ["rc-switch@4.1.0", "", { "dependencies": { "@babel/runtime": "^7.21.0", "classnames": "^2.2.1", "rc-util": "^5.30.0" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-TI8ufP2Az9oEbvyCeVE4+90PDSljGyuwix3fV58p7HV2o4wBnVToEyomJRVyTaZeqNPAp+vqeo4Wnj5u0ZZQBg=="],
|
||||
|
||||
"rc-table": ["rc-table@7.54.0", "", { "dependencies": { "@babel/runtime": "^7.10.1", "@rc-component/context": "^1.4.0", "classnames": "^2.2.5", "rc-resize-observer": "^1.1.0", "rc-util": "^5.44.3", "rc-virtual-list": "^3.14.2" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-/wDTkki6wBTjwylwAGjpLKYklKo9YgjZwAU77+7ME5mBoS32Q4nAwoqhA2lSge6fobLW3Tap6uc5xfwaL2p0Sw=="],
|
||||
|
||||
"rc-tabs": ["rc-tabs@15.7.0", "", { "dependencies": { "@babel/runtime": "^7.11.2", "classnames": "2.x", "rc-dropdown": "~4.2.0", "rc-menu": "~9.16.0", "rc-motion": "^2.6.2", "rc-resize-observer": "^1.0.0", "rc-util": "^5.34.1" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-ZepiE+6fmozYdWf/9gVp7k56PKHB1YYoDsKeQA1CBlJ/POIhjkcYiv0AGP0w2Jhzftd3AVvZP/K+V+Lpi2ankA=="],
|
||||
|
||||
"rc-textarea": ["rc-textarea@1.10.2", "", { "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "^2.2.1", "rc-input": "~1.8.0", "rc-resize-observer": "^1.0.0", "rc-util": "^5.27.0" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-HfaeXiaSlpiSp0I/pvWpecFEHpVysZ9tpDLNkxQbMvMz6gsr7aVZ7FpWP9kt4t7DB+jJXesYS0us1uPZnlRnwQ=="],
|
||||
|
||||
"rc-tooltip": ["rc-tooltip@6.4.0", "", { "dependencies": { "@babel/runtime": "^7.11.2", "@rc-component/trigger": "^2.0.0", "classnames": "^2.3.1", "rc-util": "^5.44.3" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-kqyivim5cp8I5RkHmpsp1Nn/Wk+1oeloMv9c7LXNgDxUpGm+RbXJGL+OPvDlcRnx9DBeOe4wyOIl4OKUERyH1g=="],
|
||||
|
||||
"rc-tree": ["rc-tree@5.13.1", "", { "dependencies": { "@babel/runtime": "^7.10.1", "classnames": "2.x", "rc-motion": "^2.0.1", "rc-util": "^5.16.1", "rc-virtual-list": "^3.5.1" }, "peerDependencies": { "react": "*", "react-dom": "*" } }, "sha512-FNhIefhftobCdUJshO7M8uZTA9F4OPGVXqGfZkkD/5soDeOhwO06T/aKTrg0WD8gRg/pyfq+ql3aMymLHCTC4A=="],
|
||||
|
||||
"rc-tree-select": ["rc-tree-select@5.27.0", "", { "dependencies": { "@babel/runtime": "^7.25.7", "classnames": "2.x", "rc-select": "~14.16.2", "rc-tree": "~5.13.0", "rc-util": "^5.43.0" }, "peerDependencies": { "react": "*", "react-dom": "*" } }, "sha512-2qTBTzwIT7LRI1o7zLyrCzmo5tQanmyGbSaGTIf7sYimCklAToVVfpMC6OAldSKolcnjorBYPNSKQqJmN3TCww=="],
|
||||
|
||||
"rc-upload": ["rc-upload@4.11.0", "", { "dependencies": { "@babel/runtime": "^7.18.3", "classnames": "^2.2.5", "rc-util": "^5.2.0" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-ZUyT//2JAehfHzjWowqROcwYJKnZkIUGWaTE/VogVrepSl7AFNbQf4+zGfX4zl9Vrj/Jm8scLO0R6UlPDKK4wA=="],
|
||||
|
||||
"rc-util": ["rc-util@5.44.4", "", { "dependencies": { "@babel/runtime": "^7.18.3", "react-is": "^18.2.0" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-resueRJzmHG9Q6rI/DfK6Kdv9/Lfls05vzMs1Sk3M2P+3cJa+MakaZyWY8IPfehVuhPJFKrIY1IK4GqbiaiY5w=="],
|
||||
|
||||
"rc-virtual-list": ["rc-virtual-list@3.19.2", "", { "dependencies": { "@babel/runtime": "^7.20.0", "classnames": "^2.2.6", "rc-resize-observer": "^1.0.0", "rc-util": "^5.36.0" }, "peerDependencies": { "react": ">=16.9.0", "react-dom": ">=16.9.0" } }, "sha512-Ys6NcjwGkuwkeaWBDqfI3xWuZ7rDiQXlH1o2zLfFzATfEgXcqpk8CkgMfbJD81McqjcJVez25a3kPxCR807evA=="],
|
||||
|
||||
"react": ["react@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ=="],
|
||||
|
||||
"react-dom": ["react-dom@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" }, "peerDependencies": { "react": "^18.3.1" } }, "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw=="],
|
||||
@@ -561,7 +632,7 @@
|
||||
|
||||
"react-hook-form": ["react-hook-form@7.76.0", "", { "peerDependencies": { "react": "^16.8.0 || ^17 || ^18 || ^19" } }, "sha512-eKtLGgFeSgkHqQD8J59AMZ9a4uD1D83iSIzt4YlTGD7liDen5rrjcUO1rVIGd9yC1gofryjtHbv+4ny4hkLWlw=="],
|
||||
|
||||
"react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="],
|
||||
"react-is": ["react-is@16.13.1", "", {}, "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="],
|
||||
|
||||
"react-refresh": ["react-refresh@0.17.0", "", {}, "sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ=="],
|
||||
|
||||
@@ -577,15 +648,19 @@
|
||||
|
||||
"react-style-singleton": ["react-style-singleton@2.2.3", "", { "dependencies": { "get-nonce": "^1.0.0", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ=="],
|
||||
|
||||
"resize-observer-polyfill": ["resize-observer-polyfill@1.5.1", "", {}, "sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg=="],
|
||||
|
||||
"resolve-protobuf-schema": ["resolve-protobuf-schema@2.1.0", "", { "dependencies": { "protocol-buffers-schema": "^3.3.1" } }, "sha512-kI5ffTiZWmJaS/huM8wZfEMer1eRd7oJQhDuxeCLe3t7N7mX3z94CN0xPxBQxFYQTSNz9T0i+v6inKqSdK8xrQ=="],
|
||||
|
||||
"robust-predicates": ["robust-predicates@3.0.3", "", {}, "sha512-NS3levdsRIUOmiJ8FZWCP7LG3QpJyrs/TE0Zpf1yvZu8cAJJ6QMW92H1c7kWpdIHo8RvmLxN/o2JXTKHp74lUA=="],
|
||||
|
||||
"rollup": ["rollup@4.57.1", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.57.1", "@rollup/rollup-android-arm64": "4.57.1", "@rollup/rollup-darwin-arm64": "4.57.1", "@rollup/rollup-darwin-x64": "4.57.1", "@rollup/rollup-freebsd-arm64": "4.57.1", "@rollup/rollup-freebsd-x64": "4.57.1", "@rollup/rollup-linux-arm-gnueabihf": "4.57.1", "@rollup/rollup-linux-arm-musleabihf": "4.57.1", "@rollup/rollup-linux-arm64-gnu": "4.57.1", "@rollup/rollup-linux-arm64-musl": "4.57.1", "@rollup/rollup-linux-loong64-gnu": "4.57.1", "@rollup/rollup-linux-loong64-musl": "4.57.1", "@rollup/rollup-linux-ppc64-gnu": "4.57.1", "@rollup/rollup-linux-ppc64-musl": "4.57.1", "@rollup/rollup-linux-riscv64-gnu": "4.57.1", "@rollup/rollup-linux-riscv64-musl": "4.57.1", "@rollup/rollup-linux-s390x-gnu": "4.57.1", "@rollup/rollup-linux-x64-gnu": "4.57.1", "@rollup/rollup-linux-x64-musl": "4.57.1", "@rollup/rollup-openbsd-x64": "4.57.1", "@rollup/rollup-openharmony-arm64": "4.57.1", "@rollup/rollup-win32-arm64-msvc": "4.57.1", "@rollup/rollup-win32-ia32-msvc": "4.57.1", "@rollup/rollup-win32-x64-gnu": "4.57.1", "@rollup/rollup-win32-x64-msvc": "4.57.1", "fsevents": "~2.3.2" }, "bin": "dist/bin/rollup" }, "sha512-oQL6lgK3e2QZeQ7gcgIkS2YZPg5slw37hYufJ3edKlfQSGGm8ICoxswK15ntSzF/a8+h7ekRy7k7oWc3BQ7y8A=="],
|
||||
|
||||
"scheduler": ["scheduler@0.23.2", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ=="],
|
||||
"roughjs": ["roughjs@4.6.6", "", { "dependencies": { "hachure-fill": "^0.5.2", "path-data-parser": "^0.1.0", "points-on-curve": "^0.2.0", "points-on-path": "^0.2.1" } }, "sha512-ZUz/69+SYpFN/g/lUlo2FXcIjRkSu3nDarreVdGGndHEBJ6cXPdKguS8JGxwj5HA5xIbVKSmLgr5b3AWxtRfvQ=="],
|
||||
|
||||
"scroll-into-view-if-needed": ["scroll-into-view-if-needed@3.1.0", "", { "dependencies": { "compute-scroll-into-view": "^3.0.2" } }, "sha512-49oNpRjWRvnU8NyGVmUaYG4jtTkNonFZI86MmGRDqBphEK2EXT9gdEUoQPZhuBM8yWHxCWbobltqYO5M4XrUvQ=="],
|
||||
"rw": ["rw@1.3.3", "", {}, "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ=="],
|
||||
|
||||
"safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="],
|
||||
|
||||
"scheduler": ["scheduler@0.23.2", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ=="],
|
||||
|
||||
"semver": ["semver@6.3.1", "", { "bin": "bin/semver.js" }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="],
|
||||
|
||||
@@ -597,8 +672,6 @@
|
||||
|
||||
"source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="],
|
||||
|
||||
"string-convert": ["string-convert@0.2.1", "", {}, "sha512-u/1tdPl4yQnPBjnVrmdLo9gtuLvELKsAoRapekWggdiQNvvvum+jYF329d84NAa660KQw7pB2n36KrIKVoXa3A=="],
|
||||
|
||||
"stylis": ["stylis@4.4.0", "", {}, "sha512-5Z9ZpRzfuH6l/UAvCPAPUo3665Nk2wLaZU3x+TLHKVzIz33+sbJqbtrYoC3KD4/uVOr2Zp+L0LySezP9OHV9yA=="],
|
||||
|
||||
"tailwind-merge": ["tailwind-merge@3.6.0", "", {}, "sha512-uxL7qAVQriqRQPAyK3pj66VqskWqoZ37PW94jwOTwNfq/z9oyu1V+eqrZqtR2+fCiXdYOZe/Modt8GtvqNzu+w=="],
|
||||
@@ -607,9 +680,9 @@
|
||||
|
||||
"three": ["three@0.160.1", "", {}, "sha512-Bgl2wPJypDOZ1stAxwfWAcJ0WQf7QzlptsxkjYiURPz+n5k4RBDLsq+6f9Y75TYxn6aHLcWz+JNmwTOXWrQTBQ=="],
|
||||
|
||||
"throttle-debounce": ["throttle-debounce@5.0.2", "", {}, "sha512-B71/4oyj61iNH0KeCamLuE2rmKuTO5byTOSVwECM5FA7TiAiAW+UqTKZ9ERueC4qvgSttUhdmq1mXC3kJqGX7A=="],
|
||||
"tinyexec": ["tinyexec@1.2.2", "", {}, "sha512-M/Q0B2cp4K7kynaT/vnED1j8TlLY+Pp7C6Wl2bl/7u/F0mUVwdyOpwomQb8JpYLitHUssAJRmLZdMCGsrx7i+g=="],
|
||||
|
||||
"toggle-selection": ["toggle-selection@1.0.6", "", {}, "sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ=="],
|
||||
"ts-dedent": ["ts-dedent@2.2.0", "", {}, "sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ=="],
|
||||
|
||||
"tslib": ["tslib@2.3.0", "", {}, "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg=="],
|
||||
|
||||
@@ -625,6 +698,8 @@
|
||||
|
||||
"use-sync-external-store": ["use-sync-external-store@1.6.0", "", { "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w=="],
|
||||
|
||||
"uuid": ["uuid@14.0.0", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg=="],
|
||||
|
||||
"vite": ["vite@5.4.21", "", { "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", "rollup": "^4.20.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || >=20.0.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" }, "optionalPeers": ["@types/node", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser"], "bin": "bin/vite.js" }, "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw=="],
|
||||
|
||||
"ws": ["ws@8.18.3", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg=="],
|
||||
@@ -653,7 +728,13 @@
|
||||
|
||||
"@radix-ui/react-tooltip/@radix-ui/react-slot": ["@radix-ui/react-slot@1.2.3", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.2" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-aeNmHnBxbi2St0au6VBVC7JXFlhLlOnvIIlePNniyUNAClzmtAUEY8/pBiK3iHjufOlwA+c20/8jngo7xcrg8A=="],
|
||||
|
||||
"prop-types/react-is": ["react-is@16.13.1", "", {}, "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="],
|
||||
"cytoscape-fcose/cose-base": ["cose-base@2.2.0", "", { "dependencies": { "layout-base": "^2.0.0" } }, "sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g=="],
|
||||
|
||||
"d3-dsv/commander": ["commander@7.2.0", "", {}, "sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw=="],
|
||||
|
||||
"d3-sankey/d3-array": ["d3-array@2.12.1", "", { "dependencies": { "internmap": "^1.0.0" } }, "sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ=="],
|
||||
|
||||
"d3-sankey/d3-shape": ["d3-shape@1.3.7", "", { "dependencies": { "d3-path": "1" } }, "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw=="],
|
||||
|
||||
"vite/postcss": ["postcss@8.5.6", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg=="],
|
||||
|
||||
@@ -664,5 +745,11 @@
|
||||
"@babel/helper-compilation-targets/browserslist/electron-to-chromium": ["electron-to-chromium@1.5.286", "", {}, "sha512-9tfDXhJ4RKFNerfjdCcZfufu49vg620741MNs26a9+bhLThdB+plgMeou98CAaHu/WATj2iHOOHTp1hWtABj2A=="],
|
||||
|
||||
"@babel/helper-compilation-targets/browserslist/node-releases": ["node-releases@2.0.27", "", {}, "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA=="],
|
||||
|
||||
"cytoscape-fcose/cose-base/layout-base": ["layout-base@2.0.1", "", {}, "sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg=="],
|
||||
|
||||
"d3-sankey/d3-array/internmap": ["internmap@1.0.1", "", {}, "sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw=="],
|
||||
|
||||
"d3-sankey/d3-shape/d3-path": ["d3-path@1.0.9", "", {}, "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg=="],
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user