release: bump version to 0.66.0

backend/app/api/main.py

@@ -19,6 +19,7 @@ from app.api.v1 import (
     vessels,
     bgp,
     news,
+    interactables,
     realtime_sources,
     system_control,
     tv,
@@ -53,4 +54,5 @@ api_router.include_router(vessels.router, prefix="/vessels", tags=["vessels"])
 api_router.include_router(bgp.router, prefix="/bgp", tags=["bgp"])
 api_router.include_router(tv.router, prefix="/tv", tags=["tv"])
 api_router.include_router(news.router, prefix="/news", tags=["news"])
+api_router.include_router(interactables.router, prefix="/interactables", tags=["interactables"])
 api_router.include_router(realtime_sources.router, prefix="/realtime-sources", tags=["realtime-sources"])

backend/app/api/v1/ai.py

@@ -3,6 +3,7 @@ from uuid import uuid4
 from fastapi import APIRouter, Depends, HTTPException, Request, Response
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app.core.logging import get_logger
 from app.core.security import get_current_user
 from app.db.session import get_db
 from app.models.user import User
@@ -47,8 +48,10 @@ from app.services.playground_chat_service import (
     stop_message,
 )
 from app.services.situational_alert_ai_brief import build_situational_alert_brief_request
+from app.services.business_logs import emit_business_log, exception_context
 
 router = APIRouter()
+logger = get_logger(__name__, service="api")
 
 
 @router.get("/provider/status", response_model=AIProviderStatusResponse)
@@ -122,6 +125,16 @@ async def create_playground_message(
     provider_client: AIProviderClient = Depends(get_ai_provider_client),
     db: AsyncSession = Depends(get_db),
 ):
+    await emit_business_log(
+        logger,
+        event="ai.playground.message.create",
+        message="Playground message creation requested",
+        category="ai",
+        service="api",
+        module=__name__,
+        user_id=current_user.id,
+        context={"session_key": payload.session_key, "preset": payload.selected_preset_key},
+    )
     return await create_turn(
         db,
         user_id=current_user.id,
@@ -136,6 +149,16 @@ async def stop_playground_message(
     current_user: User = Depends(get_current_user),
     db: AsyncSession = Depends(get_db),
 ):
+    await emit_business_log(
+        logger,
+        event="ai.playground.message.stop",
+        message="Playground message stop requested",
+        category="ai",
+        service="api",
+        module=__name__,
+        user_id=current_user.id,
+        context={"session_key": payload.session_key, "message_id": payload.message_id},
+    )
     return await stop_message(
         db,
         user_id=current_user.id,
@@ -150,6 +173,16 @@ async def resend_playground_message(
     provider_client: AIProviderClient = Depends(get_ai_provider_client),
     db: AsyncSession = Depends(get_db),
 ):
+    await emit_business_log(
+        logger,
+        event="ai.playground.message.resend",
+        message="Playground message resend requested",
+        category="ai",
+        service="api",
+        module=__name__,
+        user_id=current_user.id,
+        context={"session_key": payload.session_key, "user_message_id": payload.user_message_id},
+    )
     return await resend_turn(
         db,
         user_id=current_user.id,
@@ -214,16 +247,70 @@ async def analyze_bgp_brief(
         anomaly_limit=payload.anomaly_limit,
         collector_limit=payload.collector_limit,
     )
+    await emit_business_log(
+        logger,
+        event="ai.brief.bgp.facts_collected",
+        message="BGP brief facts collected",
+        category="ai",
+        service="api",
+        module=__name__,
+        request_id=request_id,
+        user_id=current_user.id,
+        context={
+            "incident_limit": payload.incident_limit,
+            "anomaly_limit": payload.anomaly_limit,
+            "collector_limit": payload.collector_limit,
+            "fact_count": len(facts or []),
+        },
+    )
     brief_request.preferred_model = payload.preferred_model
     brief_request.thinking = payload.thinking
 
-    analysis = await provider_client.analyze(brief_request, request_id=request_id)
-    return save_bgp_brief_record(
-        analysis,
+    await emit_business_log(
+        logger,
+        event="ai.brief.bgp.start",
+        message="BGP brief AI analysis started",
+        category="ai",
+        service="api",
+        module=__name__,
         request_id=request_id,
-        facts=facts,
-        context=context,
+        user_id=current_user.id,
+        context={"preferred_model": payload.preferred_model},
     )
+    try:
+        analysis = await provider_client.analyze(brief_request, request_id=request_id)
+        record = save_bgp_brief_record(
+            analysis,
+            request_id=request_id,
+            facts=facts,
+            context=context,
+        )
+        await emit_business_log(
+            logger,
+            event="ai.brief.bgp.completed",
+            message="BGP brief AI analysis saved",
+            category="ai",
+            service="api",
+            module=__name__,
+            request_id=request_id,
+            user_id=current_user.id,
+            context={"provider": analysis.provider, "model": analysis.model, "brief_id": record.id},
+        )
+        return record
+    except Exception as exc:
+        await emit_business_log(
+            logger,
+            event="ai.brief.bgp.failed",
+            message="BGP brief AI analysis failed",
+            category="ai",
+            level="error",
+            service="api",
+            module=__name__,
+            request_id=request_id,
+            user_id=current_user.id,
+            context=exception_context(exc, {"preferred_model": payload.preferred_model}),
+        )
+        raise
 
 
 @router.post("/alerts/brief", response_model=AlertBriefResponse)
@@ -242,17 +329,65 @@ async def analyze_alert_brief(
         db,
         alert_limit=payload.alert_limit,
     )
+    await emit_business_log(
+        logger,
+        event="ai.brief.alerts.facts_collected",
+        message="Alert brief facts collected",
+        category="ai",
+        service="api",
+        module=__name__,
+        request_id=request_id,
+        user_id=current_user.id,
+        context={"alert_limit": payload.alert_limit, "fact_count": len(facts or [])},
+    )
     brief_request.preferred_model = payload.preferred_model
     brief_request.thinking = payload.thinking
 
-    analysis = await provider_client.analyze(brief_request, request_id=request_id)
-    return AlertBriefResponse(
-        **analysis.model_dump(),
-        title=brief_request.title,
-        objective=brief_request.objective,
-        facts=facts,
-        context=context,
+    await emit_business_log(
+        logger,
+        event="ai.brief.alerts.start",
+        message="Alert brief AI analysis started",
+        category="ai",
+        service="api",
+        module=__name__,
+        request_id=request_id,
+        user_id=current_user.id,
+        context={"preferred_model": payload.preferred_model},
     )
+    try:
+        analysis = await provider_client.analyze(brief_request, request_id=request_id)
+        await emit_business_log(
+            logger,
+            event="ai.brief.alerts.completed",
+            message="Alert brief AI analysis completed",
+            category="ai",
+            service="api",
+            module=__name__,
+            request_id=request_id,
+            user_id=current_user.id,
+            context={"provider": analysis.provider, "model": analysis.model},
+        )
+        return AlertBriefResponse(
+            **analysis.model_dump(),
+            title=brief_request.title,
+            objective=brief_request.objective,
+            facts=facts,
+            context=context,
+        )
+    except Exception as exc:
+        await emit_business_log(
+            logger,
+            event="ai.brief.alerts.failed",
+            message="Alert brief AI analysis failed",
+            category="ai",
+            level="error",
+            service="api",
+            module=__name__,
+            request_id=request_id,
+            user_id=current_user.id,
+            context=exception_context(exc, {"preferred_model": payload.preferred_model}),
+        )
+        raise
 
 
 @router.post("/situational-alerts/brief", response_model=SituationalAlertBriefResponse)
@@ -268,14 +403,62 @@ async def analyze_situational_alert_brief(
     response.headers["X-Request-ID"] = request_id
 
     brief_request, facts, context = await build_situational_alert_brief_request(db)
+    await emit_business_log(
+        logger,
+        event="ai.brief.situational_alerts.facts_collected",
+        message="Situational alert brief facts collected",
+        category="ai",
+        service="api",
+        module=__name__,
+        request_id=request_id,
+        user_id=current_user.id,
+        context={"fact_count": len(facts or [])},
+    )
     brief_request.preferred_model = payload.preferred_model
     brief_request.thinking = payload.thinking
 
-    analysis = await provider_client.analyze(brief_request, request_id=request_id)
-    return SituationalAlertBriefResponse(
-        **analysis.model_dump(),
-        title=brief_request.title,
-        objective=brief_request.objective,
-        facts=facts,
-        context=context,
+    await emit_business_log(
+        logger,
+        event="ai.brief.situational_alerts.start",
+        message="Situational alert brief AI analysis started",
+        category="ai",
+        service="api",
+        module=__name__,
+        request_id=request_id,
+        user_id=current_user.id,
+        context={"preferred_model": payload.preferred_model},
     )
+    try:
+        analysis = await provider_client.analyze(brief_request, request_id=request_id)
+        await emit_business_log(
+            logger,
+            event="ai.brief.situational_alerts.completed",
+            message="Situational alert brief AI analysis completed",
+            category="ai",
+            service="api",
+            module=__name__,
+            request_id=request_id,
+            user_id=current_user.id,
+            context={"provider": analysis.provider, "model": analysis.model},
+        )
+        return SituationalAlertBriefResponse(
+            **analysis.model_dump(),
+            title=brief_request.title,
+            objective=brief_request.objective,
+            facts=facts,
+            context=context,
+        )
+    except Exception as exc:
+        await emit_business_log(
+            logger,
+            event="ai.brief.situational_alerts.failed",
+            message="Situational alert brief AI analysis failed",
+            category="ai",
+            level="error",
+            service="api",
+            module=__name__,
+            request_id=request_id,
+            user_id=current_user.id,
+            context=exception_context(exc, {"preferred_model": payload.preferred_model}),
+        )
+        raise

backend/app/api/v1/datasources.py

@@ -1,4 +1,3 @@
-import asyncio
 from datetime import datetime, timedelta, timezone
 from typing import Optional
 
@@ -7,7 +6,7 @@ from pydantic import BaseModel, Field
 from sqlalchemy import func, or_, select, text
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.core.cache import cache
+from app.core.logging import get_logger
 from app.core.time import to_iso8601_utc
 from app.core.security import get_current_user
 from app.core.data_sources import get_data_sources_config
@@ -22,16 +21,26 @@ from app.models.user import User
 from app.models.vessel import AISRawObservation
 from app.services.vessel_ais_aggregation import VESSEL_AIS_SCHEMA
 from app.services.scheduler import (
-    cancel_running_collector_now,
-    get_latest_task_id_for_datasource,
-    run_collector_now,
     sync_datasource_job,
 )
-from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
+from app.services.data_jobs import (
+    JOB_STATUS_CANCELLING,
+    JOB_STATUS_QUEUED,
+    JOB_STATUS_RUNNING,
+    JOB_TYPE_CLEAR_CACHE,
+    JOB_TYPE_CLEAR_DATA,
+    JOB_TYPE_COLLECT,
+    enqueue_datasource_job,
+    get_active_datasource_job,
+    request_cancel_datasource_task,
+)
+from app.services.business_logs import emit_business_log
 
 router = APIRouter()
+logger = get_logger(__name__, service="api")
 STALE_RUNNING_TASK_TIMEOUT_MINUTES = 90
 
+
 PRODUCT_SOURCE_KEYWORDS: tuple[tuple[str, tuple[str, ...]], ...] = (
     ("vessels", ("vessel", "ais")),
     ("cables", ("cable", "landing", "telegeography", "arcgis", "fao")),
@@ -115,7 +124,7 @@ async def _load_latest_running_tasks(
             _task_rank_column(CollectionTask.started_at),
         )
         .where(CollectionTask.datasource_id.in_(datasource_ids))
-        .where(CollectionTask.status == "running")
+        .where(CollectionTask.status.in_((JOB_STATUS_QUEUED, JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING)))
         .subquery()
     )
     result = await db.execute(
@@ -126,32 +135,6 @@ async def _load_latest_running_tasks(
     return {task.datasource_id: task for task in result.scalars().all()}
 
 
-async def _load_latest_task_ids(
-    db: AsyncSession,
-    datasource_ids: list[int],
-) -> dict[int, int]:
-    if not datasource_ids:
-        return {}
-
-    ranked_tasks = (
-        select(
-            CollectionTask.id.label("task_id"),
-            CollectionTask.datasource_id.label("datasource_id"),
-            func.row_number().over(
-                partition_by=CollectionTask.datasource_id,
-                order_by=CollectionTask.id.desc(),
-            ).label("row_num"),
-        )
-        .where(CollectionTask.datasource_id.in_(datasource_ids))
-        .subquery()
-    )
-    result = await db.execute(
-        select(ranked_tasks.c.datasource_id, ranked_tasks.c.task_id)
-        .where(ranked_tasks.c.row_num == 1)
-    )
-    return {datasource_id: task_id for datasource_id, task_id in result.all()}
-
-
 async def _load_latest_tasks(
     db: AsyncSession,
     datasource_ids: list[int],
@@ -303,8 +286,11 @@ def serialize_datasource_row(
         "last_run": to_iso8601_utc(last_run_at),
         "last_run_at": to_iso8601_utc(last_run_at),
         "last_status": last_status,
-        "is_running": running_task is not None,
+        "is_running": running_task is not None and running_task.task_type == JOB_TYPE_COLLECT,
+        "is_task_active": running_task is not None,
+        "task_status": running_task.status if running_task else None,
         "task_id": display_task.id if display_task else None,
+        "task_type": display_task.task_type if display_task else None,
         "progress": display_task.progress if display_task else None,
         "phase": display_task.phase if display_task else None,
         "phase_progress": display_task.phase_progress if display_task else None,
@@ -400,9 +386,25 @@ async def _trigger_datasource_batch(
     datasources: list[DataSource],
     *,
     force: bool,
+    actor_id: int | None = None,
+    trigger_kind: str = "batch",
 ) -> dict:
+    await emit_business_log(
+        logger,
+        event=f"collector.trigger.{trigger_kind}.start",
+        message="Datasource batch trigger started",
+        category="collector",
+        service="api",
+        module=__name__,
+        user_id=actor_id,
+        context={
+            "trigger_kind": trigger_kind,
+            "force": force,
+            "requested_count": len(datasources),
+        },
+    )
     if not datasources:
-        return {
+        result = {
             "status": "noop",
             "message": "No matching data sources to trigger",
             "force": force,
@@ -410,8 +412,18 @@ async def _trigger_datasource_batch(
             "skipped": [],
             "failed": [],
         }
+        await emit_business_log(
+            logger,
+            event=f"collector.trigger.{trigger_kind}.completed",
+            message="Datasource batch trigger completed with no matching sources",
+            category="collector",
+            service="api",
+            module=__name__,
+            user_id=actor_id,
+            context={"trigger_kind": trigger_kind, "force": force, "status": "noop", "triggered_count": 0},
+        )
+        return result
 
-    previous_task_ids: dict[int, Optional[int]] = {}
     triggered_sources: list[dict] = []
     skipped_sources: list[dict] = []
     failed_sources: list[dict] = []
@@ -446,9 +458,11 @@ async def _trigger_datasource_batch(
                     }
                 )
                 continue
-            cancelled = await cancel_running_collector_now(datasource.source)
-            if not cancelled:
-                await rollback_orphaned_running_task(db, datasource, running_task)
+            await request_cancel_datasource_task(
+                db,
+                running_task,
+                reason="superseded_by_forced_collection",
+            )
 
         if not force and not is_due_for_collection(datasource, now):
             skipped_sources.append(
@@ -465,57 +479,51 @@ async def _trigger_datasource_batch(
             )
             continue
 
-        previous_task_ids[datasource.id] = None
-        success = run_collector_now(datasource.source)
-        if not success:
-            failed_sources.append(
-                {
-                    "id": datasource.id,
-                    "source": datasource.source,
-                    "name": datasource.name,
-                    "reason": "trigger_failed",
-                }
-            )
-            continue
+        task = await enqueue_datasource_job(
+            db,
+            datasource,
+            JOB_TYPE_COLLECT,
+            payload={"force": force, "trigger": "batch"},
+        )
 
         triggered_sources.append(
             {
                 "id": datasource.id,
                 "source": datasource.source,
                 "name": datasource.name,
-                "task_id": None,
+                "task_id": task.id,
             }
         )
 
-    latest_task_ids = await _load_latest_task_ids(
-        db,
-        [datasource.id for datasource in datasources],
-    )
-    for datasource_id in previous_task_ids:
-        previous_task_ids[datasource_id] = latest_task_ids.get(datasource_id)
-
-    for _ in range(20):
-        await asyncio.sleep(0.1)
-        pending = [item for item in triggered_sources if item["task_id"] is None]
-        if not pending:
-            break
-        latest_task_ids = await _load_latest_task_ids(
-            db,
-            [item["id"] for item in pending],
-        )
-        for item in pending:
-            task_id = latest_task_ids.get(item["id"])
-            if task_id is not None and task_id != previous_task_ids.get(item["id"]):
-                item["task_id"] = task_id
-
-    return {
-        "status": "triggered" if triggered_sources else "partial",
-        "message": f"Triggered {len(triggered_sources)} data sources",
+    result = {
+        "status": "queued" if triggered_sources else "partial",
+        "message": f"Queued {len(triggered_sources)} data source jobs",
         "force": force,
         "triggered": triggered_sources,
         "skipped": skipped_sources,
         "failed": failed_sources,
     }
+    await emit_business_log(
+        logger,
+        event=f"collector.trigger.{trigger_kind}.completed",
+        message="Datasource batch trigger completed",
+        category="collector",
+        service="api",
+        module=__name__,
+        user_id=actor_id,
+        context={
+            "trigger_kind": trigger_kind,
+            "force": force,
+            "status": result["status"],
+            "requested_count": len(datasources),
+            "triggered_count": len(triggered_sources),
+            "skipped_count": len(skipped_sources),
+            "failed_count": len(failed_sources),
+            "triggered_sources": [item["source"] for item in triggered_sources],
+            "skipped_reasons": [item["reason"] for item in skipped_sources],
+        },
+    )
+    return result
 
 
 async def get_datasource_record(db: AsyncSession, source_id: str) -> Optional[DataSource]:
@@ -780,7 +788,13 @@ async def trigger_all_datasources(
         .order_by(DataSource.module, DataSource.id)
     )
     datasources = result.scalars().all()
-    return await _trigger_datasource_batch(db, datasources, force=force)
+    return await _trigger_datasource_batch(
+        db,
+        datasources,
+        force=force,
+        actor_id=current_user.id,
+        trigger_kind="all",
+    )
 
 
 @router.post("/trigger-batch")
@@ -816,7 +830,13 @@ async def trigger_datasource_batch(
         collected=None if payload.source_ids else payload.collected,
         credential_status=None if payload.source_ids else payload.credential_status,
     )
-    return await _trigger_datasource_batch(db, datasources, force=payload.force)
+    return await _trigger_datasource_batch(
+        db,
+        datasources,
+        force=payload.force,
+        actor_id=current_user.id,
+        trigger_kind="batch",
+    )
 
 
 @router.get("/snapshots")
@@ -992,8 +1012,24 @@ async def trigger_datasource(
     if not datasource.is_active:
         raise HTTPException(status_code=400, detail="Data source is disabled")
 
-    running_task = await get_running_task(db, datasource.id)
+    running_task = await get_active_datasource_job(db, datasource.id, task_types=(JOB_TYPE_COLLECT,))
     if running_task is not None and not force:
+        await emit_business_log(
+            logger,
+            event="collector.trigger.single.skipped_already_running",
+            message="Datasource trigger skipped because a task is already running",
+            category="collector",
+            level="warning",
+            service="api",
+            module=__name__,
+            user_id=current_user.id,
+            context={
+                "collector_name": datasource.source,
+                "datasource_id": datasource.id,
+                "task_id": running_task.id,
+                "status": "skipped",
+            },
+        )
         raise HTTPException(
             status_code=409,
             detail={
@@ -1013,31 +1049,42 @@ async def trigger_datasource(
         )
 
     if running_task is not None and force:
-        cancelled = await cancel_running_collector_now(datasource.source)
-        if not cancelled:
-            await rollback_orphaned_running_task(db, datasource, running_task)
+        await request_cancel_datasource_task(
+            db,
+            running_task,
+            reason="superseded_by_forced_collection",
+        )
 
-    previous_task_id = await get_latest_task_id_for_datasource(datasource.id)
-    success = run_collector_now(datasource.source)
-    if not success:
-        raise HTTPException(status_code=500, detail=f"Failed to trigger collector '{datasource.source}'")
-
-    task_id = None
-    for _ in range(20):
-        await asyncio.sleep(0.1)
-        task_id = await get_latest_task_id_for_datasource(datasource.id)
-        if task_id is not None and task_id != previous_task_id:
-            break
-    if task_id == previous_task_id:
-        task_id = None
+    task = await enqueue_datasource_job(
+        db,
+        datasource,
+        JOB_TYPE_COLLECT,
+        payload={"force": force, "trigger": "single"},
+    )
+    await emit_business_log(
+        logger,
+        event="collector.trigger.single.completed",
+        message="Datasource trigger queued",
+        category="collector",
+        service="api",
+        module=__name__,
+        user_id=current_user.id,
+        context={
+            "collector_name": datasource.source,
+            "datasource_id": datasource.id,
+            "task_id": task.id,
+            "force": force,
+            "status": "queued",
+        },
+    )
 
     return {
-        "status": "triggered",
+        "status": "queued",
         "source_id": datasource.id,
-        "task_id": task_id,
+        "task_id": task.id,
         "collector_name": datasource.source,
         "force": force,
-        "message": f"Collector '{datasource.source}' has been triggered",
+        "message": f"Collector '{datasource.source}' has been queued",
     }
 
 
@@ -1051,22 +1098,30 @@ async def clear_datasource_data(
     if not datasource:
         raise HTTPException(status_code=404, detail="Data source not found")
 
-    result = await db.execute(
-        select(func.count(CollectedData.id)).where(CollectedData.source == datasource.source)
+    active_task = await get_active_datasource_job(db, datasource.id)
+    if active_task is not None:
+        raise HTTPException(
+            status_code=409,
+            detail={
+                "reason": "datasource_job_in_progress",
+                "message": "当前数据源已有任务在执行，请等待完成或先取消任务。",
+                "task_id": active_task.id,
+                "task_type": active_task.task_type,
+                "status": active_task.status,
+            },
+        )
+    task = await enqueue_datasource_job(
+        db,
+        datasource,
+        JOB_TYPE_CLEAR_DATA,
+        payload={"source": datasource.source},
     )
-    count = result.scalar() or 0
-
-    if count == 0:
-        return {"status": "success", "message": "No data to clear", "deleted_count": 0}
-
-    delete_query = CollectedData.__table__.delete().where(CollectedData.source == datasource.source)
-    await db.execute(delete_query)
-    await db.commit()
 
     return {
-        "status": "success",
-        "message": f"Cleared {count} records for data source '{datasource.name}'",
-        "deleted_count": count,
+        "status": "queued",
+        "message": f"Queued data clearing for data source '{datasource.name}'",
+        "task_id": task.id,
+        "deleted_count": None,
     }
 
 
@@ -1080,16 +1135,44 @@ async def clear_datasource_cache(
     if not datasource:
         raise HTTPException(status_code=404, detail="Data source not found")
 
-    earth_deleted_count = invalidate_earth_layer_cache_for_source(datasource.source)
-    dashboard_deleted_count = int(cache.delete("dashboard:stats")) + int(cache.delete("dashboard:summary"))
-    deleted_count = earth_deleted_count + dashboard_deleted_count
+    task = await enqueue_datasource_job(
+        db,
+        datasource,
+        JOB_TYPE_CLEAR_CACHE,
+        payload={"source": datasource.source},
+        dedupe_key=f"clear_cache:{datasource.source}",
+    )
 
     return {
-        "status": "success",
-        "message": f"Cleared {deleted_count} cache keys for data source '{datasource.name}'",
-        "deleted_count": deleted_count,
-        "earth_layer_deleted_count": earth_deleted_count,
-        "dashboard_deleted_count": dashboard_deleted_count,
+        "status": "queued",
+        "message": f"Queued cache clearing for data source '{datasource.name}'",
+        "task_id": task.id,
+        "deleted_count": None,
+    }
+
+
+@router.post("/{source_id}/tasks/{task_id}/cancel")
+async def cancel_datasource_task(
+    source_id: str,
+    task_id: int,
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    datasource = await get_datasource_record(db, source_id)
+    if not datasource:
+        raise HTTPException(status_code=404, detail="Data source not found")
+
+    task = await db.get(CollectionTask, task_id)
+    if not task or task.datasource_id != datasource.id:
+        raise HTTPException(status_code=404, detail="Task not found")
+
+    task = await request_cancel_datasource_task(db, task)
+    return {
+        "status": "cancelled" if task.completed_at else "cancelling",
+        "task_id": task.id,
+        "task_type": task.task_type,
+        "phase": task.phase,
+        "requested_cancel_at": to_iso8601_utc(task.requested_cancel_at),
     }
 
 
@@ -1109,7 +1192,7 @@ async def get_task_status(
         if not task or task.datasource_id != datasource.id:
             raise HTTPException(status_code=404, detail="Task not found")
     else:
-        task = await get_running_task(db, datasource.id)
+        task = await get_active_datasource_job(db, datasource.id)
         if task is None:
             result = await db.execute(
                 select(CollectionTask)
@@ -1134,8 +1217,12 @@ async def get_task_status(
         }
 
     return {
-        "is_running": task.status == "running",
+        "is_running": task.status in {JOB_STATUS_QUEUED, JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING}
+        and task.task_type == JOB_TYPE_COLLECT,
+        "is_task_active": task.status in {JOB_STATUS_QUEUED, JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING},
         "task_id": task.id,
+        "task_type": task.task_type,
+        "task_status": task.status,
         "progress": task.progress,
         "phase": task.phase,
         "phase_progress": task.phase_progress,
@@ -1146,5 +1233,6 @@ async def get_task_status(
         "records_processed": task.records_processed,
         "total_records": task.total_records,
         "status": task.status,
+        "requested_cancel_at": to_iso8601_utc(task.requested_cancel_at),
         "error_message": task.error_message,
     }

backend/app/api/v1/earth.py

@@ -12,6 +12,7 @@ from pydantic import BaseModel, Field
 from sqlalchemy import delete, func, select, text
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app.core.config import settings as app_settings
 from app.core.security import decode_token, get_current_user, redis_client
 from app.db.session import get_db
 from app.models.collected_data import CollectedData
@@ -36,9 +37,16 @@ EARTH_BRAND_ASSET_DIR = REPO_ROOT / "data" / "earth-brand"
 EARTH_BRAND_ASSET_URL_PREFIX = "/earth-brand-assets"
 EARTH_BRAND_CATEGORY = "earth_brand"
 EARTH_ABOUT_CATEGORY = "earth_about"
+SYSTEM_SETTINGS_CATEGORY = "system"
 MAX_EARTH_BRAND_ASSET_BYTES = 3 * 1024 * 1024
 ALLOWED_EARTH_BRAND_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp", ".svg"}
 
+
+def _app_version_label() -> str:
+    version = str(app_settings.VERSION or "").strip() or "0.0.0"
+    return version if version.startswith("v") else f"v{version}"
+
+
 DEFAULT_EARTH_BRAND = {
     "logo_src": "/earth/assets/brand/earth-logo.png",
     "title_src": "/earth/assets/brand/title-zh.png",
@@ -53,14 +61,15 @@ DEFAULT_EARTH_ABOUT = {
     "logo_src": "/earth/assets/brand/lim-logo.png",
     "kicker": "About",
     "title": "智能星球计划",
-    "version": "v0.64.0",
+    "version": _app_version_label(),
     "description": "面向临空场景下的智能媒体研究、全球态势感知与多源开放数据巡航，提供可视化观测、事件聚合与交互式探索能力。",
     "meta": [
         {"label": "出品方", "value": "浙江大学临空智能媒体研究院"},
-        {"label": "策划人", "value": "黄柳青"},
+        {"label": "策划人", "value": "方兴东、黄柳青"},
         {"label": "产品兼开发者", "value": "钱坤、张鸽、齐鹏"},
     ],
 }
+EARTH_ABOUT_LEGACY_PLANNER_VALUE = "黄柳青"
 
 
 class EarthBoundaryConfigPayload(BaseModel):
@@ -116,11 +125,12 @@ def _normalize_earth_about_payload(payload: dict[str, Any] | None) -> dict[str,
     }
     raw_meta = DEFAULT_EARTH_ABOUT["meta"]
     if payload:
-        for key in ("logo_src", "kicker", "title", "version", "description"):
+        for key in ("logo_src", "kicker", "title", "description"):
             value = payload.get(key)
             if value is not None:
                 merged[key] = str(value).strip()
         raw_meta = payload.get("meta") if isinstance(payload.get("meta"), list) else raw_meta
+    merged["version"] = _app_version_label()
 
     for key, default_value in DEFAULT_EARTH_ABOUT.items():
         if key == "meta":
@@ -134,6 +144,8 @@ def _normalize_earth_about_payload(payload: dict[str, Any] | None) -> dict[str,
             continue
         label = str(item.get("label") or "").strip()
         value = str(item.get("value") or "").strip()
+        if label == "策划人" and value == EARTH_ABOUT_LEGACY_PLANNER_VALUE:
+            value = "方兴东、黄柳青"
         if label or value:
             normalized_meta.append({"label": label, "value": value})
     if not normalized_meta:
@@ -142,6 +154,10 @@ def _normalize_earth_about_payload(payload: dict[str, Any] | None) -> dict[str,
     return merged
 
 
+def _is_demo_mode_enabled(payload: Any) -> bool:
+    return bool(payload.get("demo_mode")) if isinstance(payload, dict) else False
+
+
 async def _get_earth_brand_record(db: AsyncSession) -> SystemSetting | None:
     result = await db.execute(
         select(SystemSetting).where(SystemSetting.category == EARTH_BRAND_CATEGORY)
@@ -317,6 +333,11 @@ async def get_earth_oobe_status(
         select(func.count(CollectedData.id)).where(CollectedData.is_current.is_(True))
     )
     current_record_count = int(current_count_result.scalar() or 0)
+    system_result = await db.execute(
+        select(SystemSetting).where(SystemSetting.category == SYSTEM_SETTINGS_CATEGORY)
+    )
+    system_record = system_result.scalar_one_or_none()
+    demo_mode = _is_demo_mode_enabled(system_record.payload if system_record else None)
 
     datasource_count_result = await db.execute(select(func.count(DataSource.id)))
     datasource_count = int(datasource_count_result.scalar() or 0)
@@ -337,6 +358,8 @@ async def get_earth_oobe_status(
     ready = has_collected_data
 
     suggestions: list[str] = []
+    if demo_mode:
+        suggestions.append("演示模式已开启")
     if not current_user:
         suggestions.append("登录控制台")
     if not has_collected_data:
@@ -348,8 +371,9 @@ async def get_earth_oobe_status(
 
     return {
         "ready": ready,
+        "demo_mode": demo_mode,
         "authenticated": current_user is not None,
-        "needs_login": current_user is None and not ready,
+        "needs_login": current_user is None and not ready and not demo_mode,
         "has_collected_data": has_collected_data,
         "has_tv_sources": tv_source_count > 0,
         "has_core_layers": has_core_layers,

backend/app/api/v1/interactables.py

@@ -0,0 +1,190 @@
+"""CRUD APIs for persistent Earth interactables."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Response, status
+from pydantic import BaseModel, Field, field_validator
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.security import get_current_user
+from app.db.session import get_db
+from app.models.earth_interactable import EarthInteractable
+from app.models.user import User
+from app.services.earth_interactables import (
+    build_interactable_event,
+    interactables_to_geojson,
+    invalidate_interactable_cache,
+    list_interactables,
+    normalize_interactable_id,
+    publish_interactable_event,
+    serialize_interactable,
+)
+from app.services.earth_layer_cache import (
+    EarthLayerCachePolicy,
+    earth_layer_cache,
+    get_or_build_layer_payload,
+)
+
+router = APIRouter()
+INTERACTABLE_CACHE_POLICY = EarthLayerCachePolicy(
+    fresh_ttl_seconds=60,
+    stale_ttl_seconds=10 * 60,
+    max_features=5000,
+)
+
+
+class InteractableCreate(BaseModel):
+    id: str | None = Field(default=None, max_length=160)
+    layer: str = Field(default="default", min_length=1, max_length=80)
+    kind: str = Field(default="default", min_length=1, max_length=80)
+    label: str = Field(default="", max_length=255)
+    description: str = Field(default="", max_length=4000)
+    latitude: float = Field(ge=-90, le=90)
+    longitude: float = Field(ge=-180, le=180)
+    altitude: float | None = None
+    properties: dict[str, Any] = Field(default_factory=dict)
+
+    @field_validator("layer", "kind")
+    @classmethod
+    def normalize_key(cls, value: str) -> str:
+        normalized = str(value or "").strip()
+        if not normalized:
+            raise ValueError("must not be empty")
+        return normalized
+
+
+class InteractableUpdate(BaseModel):
+    layer: str | None = Field(default=None, min_length=1, max_length=80)
+    kind: str | None = Field(default=None, min_length=1, max_length=80)
+    label: str | None = Field(default=None, max_length=255)
+    description: str | None = Field(default=None, max_length=4000)
+    latitude: float | None = Field(default=None, ge=-90, le=90)
+    longitude: float | None = Field(default=None, ge=-180, le=180)
+    altitude: float | None = None
+    properties: dict[str, Any] | None = None
+
+
+@router.get("")
+async def get_interactables(
+    response: Response,
+    layer: str | None = Query(default=None),
+    include_deleted: bool = Query(default=False),
+    db: AsyncSession = Depends(get_db),
+):
+    items = await list_interactables(db, layer=layer, include_deleted=include_deleted)
+    response.headers["X-Planet-Interactables-Count"] = str(len(items))
+    return {"items": [serialize_interactable(item) for item in items]}
+
+
+@router.get("/geojson")
+async def get_interactables_geojson(
+    response: Response,
+    layer: str | None = Query(default=None),
+    db: AsyncSession = Depends(get_db),
+):
+    async def build_payload() -> dict[str, Any]:
+        items = await list_interactables(db, layer=layer)
+        return interactables_to_geojson(items)
+
+    payload = await get_or_build_layer_payload(
+        key=earth_layer_cache.key("interactables", layer=layer or "all"),
+        policy=INTERACTABLE_CACHE_POLICY,
+        builder=build_payload,
+        response=response,
+    )
+    response.headers["X-Planet-Interactables-Count"] = str(len(payload.get("features") or []))
+    return payload
+
+
+@router.post("", status_code=status.HTTP_201_CREATED)
+async def create_interactable(
+    payload: InteractableCreate,
+    _current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    record_id = normalize_interactable_id(payload.id)
+    existing = await db.get(EarthInteractable, record_id)
+    if existing and not existing.is_deleted:
+        raise HTTPException(status_code=409, detail="Interactable already exists")
+
+    if existing is None:
+        record = EarthInteractable(id=record_id)
+        db.add(record)
+    else:
+        record = existing
+        record.is_deleted = False
+        record.deleted_at = None
+        record.revision += 1
+
+    record.layer = payload.layer
+    record.kind = payload.kind
+    record.label = payload.label
+    record.description = payload.description
+    record.latitude = payload.latitude
+    record.longitude = payload.longitude
+    record.altitude = payload.altitude
+    record.properties = payload.properties
+
+    await db.commit()
+    await db.refresh(record)
+    invalidate_interactable_cache(record.layer)
+    await publish_interactable_event("created", record)
+    return {"item": serialize_interactable(record)}
+
+
+@router.get("/{interactable_id}")
+async def get_interactable(interactable_id: str, db: AsyncSession = Depends(get_db)):
+    record = await db.get(EarthInteractable, interactable_id)
+    if record is None or record.is_deleted:
+        raise HTTPException(status_code=404, detail="Interactable not found")
+    return {"item": serialize_interactable(record)}
+
+
+@router.patch("/{interactable_id}")
+async def update_interactable(
+    interactable_id: str,
+    payload: InteractableUpdate,
+    _current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    record = await db.get(EarthInteractable, interactable_id)
+    if record is None or record.is_deleted:
+        raise HTTPException(status_code=404, detail="Interactable not found")
+
+    previous_layer = record.layer
+    patch = payload.model_dump(exclude_unset=True)
+    for key, value in patch.items():
+        setattr(record, key, value)
+    record.revision += 1
+
+    await db.commit()
+    await db.refresh(record)
+    invalidate_interactable_cache(previous_layer)
+    if record.layer != previous_layer:
+        invalidate_interactable_cache(record.layer)
+    await publish_interactable_event("updated", record)
+    return {"item": serialize_interactable(record)}
+
+
+@router.delete("/{interactable_id}")
+async def delete_interactable(
+    interactable_id: str,
+    _current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    record = await db.get(EarthInteractable, interactable_id)
+    if record is None or record.is_deleted:
+        raise HTTPException(status_code=404, detail="Interactable not found")
+
+    record.is_deleted = True
+    record.deleted_at = datetime.now(UTC)
+    record.revision += 1
+    await db.commit()
+    await db.refresh(record)
+    invalidate_interactable_cache(record.layer)
+    await publish_interactable_event("deleted", record)
+    event = build_interactable_event(action="deleted", record=record, include_item=True)
+    return {"deleted": True, "event": event}

backend/app/api/v1/settings.py

@@ -11,6 +11,7 @@ from dotenv import dotenv_values
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app.core.logging import get_logger
 from app.core.security import get_current_user
 from app.core.time import to_iso8601_utc
 from app.core.config import settings as app_settings
@@ -66,8 +67,10 @@ from app.services.llm_provider_catalog import (
 from app.services.scheduler import sync_datasource_job
 from app.services.tv_streams import DEFAULT_TV_SETTINGS, get_tv_settings_payload, normalize_tv_settings
 from app.services.persistent_logs import record_audit_log
+from app.services.business_logs import emit_business_log, exception_context
 
 router = APIRouter()
+logger = get_logger(__name__, service="api")
 AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS = 5
 AI_CONNECTION_TEST_PROMPT_KEY = "ai.connection_test"
 SECRET_REVEAL_ROLES = {"admin", "super_admin"}
@@ -79,6 +82,7 @@ DEFAULT_SETTINGS = {
         "auto_refresh": True,
         "data_retention_days": 30,
         "max_concurrent_tasks": 5,
+        "demo_mode": False,
     },
     "notifications": {
         "email_enabled": False,
@@ -203,6 +207,7 @@ class SystemSettingsUpdate(BaseModel):
     auto_refresh: bool = True
     data_retention_days: int = Field(default=30, ge=1, le=3650)
     max_concurrent_tasks: int = Field(default=5, ge=1, le=50)
+    demo_mode: bool = False
 
 
 class NotificationSettingsUpdate(BaseModel):
@@ -699,6 +704,18 @@ async def _validate_ai_provider_full_connection(ai_payload: dict) -> dict:
         retry_attempts=runtime_config["retry_attempts"],
         llm_config=runtime_config.get("llm_config") or {},
     )
+    await emit_business_log(
+        logger,
+        event="settings.ai_provider.full_connection.start",
+        message="AI provider full connection validation started",
+        category="ai",
+        service="api",
+        module=__name__,
+        context={
+            "provider": runtime_config.get("llm_config", {}).get("provider"),
+            "model": runtime_config.get("llm_config", {}).get("model"),
+        },
+    )
     status_result = await client.get_status()
     if not status_result.configured:
         raise HTTPException(
@@ -715,6 +732,19 @@ async def _validate_ai_provider_full_connection(ai_payload: dict) -> dict:
             constraints=["回复尽量简短。"],
         )
     )
+    await emit_business_log(
+        logger,
+        event="settings.ai_provider.full_connection.success",
+        message="AI provider full connection validation completed",
+        category="ai",
+        service="api",
+        module=__name__,
+        context={
+            "provider": analysis_result.provider,
+            "model": analysis_result.model,
+            "configured": status_result.configured,
+        },
+    )
     return {
         "status": status_result.model_dump(),
         "provider": analysis_result.provider,
@@ -1612,9 +1642,34 @@ async def connect_ai_provider_integration(
         llm_config=quick_llm_config,
     )
 
+    await emit_business_log(
+        logger,
+        event="settings.ai_provider.connect.start",
+        message="AI provider connection test started",
+        category="ai",
+        service="api",
+        module=__name__,
+        user_id=current_user.id,
+        context={
+            "provider": payload.provider,
+            "model": payload.model,
+            "timeout_seconds": min(int(runtime_config["timeout_seconds"] or 60), AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS),
+        },
+    )
     try:
         status_result = await client.get_status()
         if not status_result.configured:
+            await emit_business_log(
+                logger,
+                event="settings.ai_provider.connect.failed",
+                message="AI provider connection test failed because provider is incomplete",
+                category="ai",
+                level="warning",
+                service="api",
+                module=__name__,
+                user_id=current_user.id,
+                context={"provider": payload.provider, "model": payload.model, "configured": False},
+            )
             return {
                 "success": False,
                 "connected": False,
@@ -1628,6 +1683,21 @@ async def connect_ai_provider_integration(
                 AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS,
             ),
         )
+        await emit_business_log(
+            logger,
+            event="settings.ai_provider.connect.success",
+            message="AI provider connection test completed",
+            category="ai",
+            service="api",
+            module=__name__,
+            user_id=current_user.id,
+            context={
+                "provider": payload.provider,
+                "model": payload.model,
+                "configured": True,
+                "lightweight_status": lightweight_result.get("status"),
+            },
+        )
         return {
             **lightweight_result,
             "status": status_result.model_dump(),
@@ -1639,6 +1709,17 @@ async def connect_ai_provider_integration(
             "message": str(exc.detail),
         }
     except Exception as exc:
+        await emit_business_log(
+            logger,
+            event="settings.ai_provider.connect.failed",
+            message="AI provider connection test failed",
+            category="ai",
+            level="error",
+            service="api",
+            module=__name__,
+            user_id=current_user.id,
+            context=exception_context(exc, {"provider": payload.provider, "model": payload.model}),
+        )
         return {
             "success": False,
             "connected": False,
@@ -1786,8 +1867,28 @@ async def connect_web_search_integration(
     runtime_config = _runtime_config_from_web_search_payload(draft_web_search_payload)
     client = WebSearchClient(runtime_config)
 
+    await emit_business_log(
+        logger,
+        event="settings.web_search.connect.start",
+        message="WebSearch connection test started",
+        category="ai_tool",
+        service="api",
+        module=__name__,
+        user_id=current_user.id,
+        context={"provider": runtime_config.default_provider},
+    )
     try:
         results = await client.test_connection()
+        await emit_business_log(
+            logger,
+            event="settings.web_search.connect.success",
+            message="WebSearch connection test completed",
+            category="ai_tool",
+            service="api",
+            module=__name__,
+            user_id=current_user.id,
+            context={"provider": runtime_config.default_provider, "result_count": len(results)},
+        )
         return {
             "success": True,
             "connected": True,
@@ -1796,18 +1897,51 @@ async def connect_web_search_integration(
             "results": [item.model_dump(mode="json") for item in results[:3]],
         }
     except WebSearchConfigurationError as exc:
+        await emit_business_log(
+            logger,
+            event="settings.web_search.connect.failed",
+            message="WebSearch connection test failed because configuration is incomplete",
+            category="ai_tool",
+            level="warning",
+            service="api",
+            module=__name__,
+            user_id=current_user.id,
+            context=exception_context(exc, {"provider": runtime_config.default_provider}),
+        )
         return {
             "success": False,
             "connected": False,
             "message": str(exc),
         }
     except WebSearchError as exc:
+        await emit_business_log(
+            logger,
+            event="settings.web_search.connect.failed",
+            message="WebSearch connection test failed",
+            category="ai_tool",
+            level="error",
+            service="api",
+            module=__name__,
+            user_id=current_user.id,
+            context=exception_context(exc, {"provider": runtime_config.default_provider}),
+        )
         return {
             "success": False,
             "connected": False,
             "message": str(exc),
         }
     except Exception as exc:
+        await emit_business_log(
+            logger,
+            event="settings.web_search.connect.failed",
+            message="WebSearch connection test failed",
+            category="ai_tool",
+            level="error",
+            service="api",
+            module=__name__,
+            user_id=current_user.id,
+            context=exception_context(exc, {"provider": runtime_config.default_provider}),
+        )
         return {
             "success": False,
             "connected": False,
@@ -1835,17 +1969,60 @@ async def generate_provider_credential_guide(
     ai_client: AIProviderClient = Depends(get_ai_provider_client),
 ):
     try:
+        await emit_business_log(
+            logger,
+            event="settings.credential_guide.generate.start",
+            message="Credential guide generation started",
+            category="ai",
+            service="api",
+            module=__name__,
+            user_id=current_user.id,
+            context={"provider": provider},
+        )
         web_search_client = await get_web_search_client(db)
-        return {
-            "guide": await generate_credential_guide(
+        guide = await generate_credential_guide(
                 db,
                 provider,
                 ai_client,
                 web_search_client,
             )
-        }
+        await emit_business_log(
+            logger,
+            event="settings.credential_guide.generate.success",
+            message="Credential guide generation completed",
+            category="ai",
+            service="api",
+            module=__name__,
+            user_id=current_user.id,
+            context={"provider": provider},
+        )
+        return {"guide": guide}
     except ValueError as exc:
+        await emit_business_log(
+            logger,
+            event="settings.credential_guide.generate.failed",
+            message="Credential guide generation failed",
+            category="ai",
+            level="warning",
+            service="api",
+            module=__name__,
+            user_id=current_user.id,
+            context=exception_context(exc, {"provider": provider}),
+        )
         raise HTTPException(status_code=404, detail=str(exc)) from exc
+    except Exception as exc:
+        await emit_business_log(
+            logger,
+            event="settings.credential_guide.generate.failed",
+            message="Credential guide generation failed",
+            category="ai",
+            level="error",
+            service="api",
+            module=__name__,
+            user_id=current_user.id,
+            context=exception_context(exc, {"provider": provider}),
+        )
+        raise
 
 
 @router.post("/credential-guides/{provider}/reset")

backend/app/api/v1/system_control.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+import json
 import subprocess
 import sys
 
@@ -44,6 +45,42 @@ from app.services.earth_layer_cache import earth_layer_cache
 router = APIRouter()
 
 
+def _compact_log_context(context: dict | None) -> str:
+    if not context:
+        return ""
+    allowed = {
+        key: value
+        for key, value in (context or {}).items()
+        if key
+        in {
+            "status",
+            "duration_ms",
+            "provider",
+            "model",
+            "result_provider",
+            "result_model",
+            "collector_name",
+            "datasource_id",
+            "task_id",
+            "snapshot_id",
+            "raw_count",
+            "transformed_count",
+            "saved_count",
+            "created",
+            "updated",
+            "unchanged",
+            "deleted",
+            "result_count",
+            "status_code",
+            "error_type",
+            "error",
+        }
+    }
+    if not allowed:
+        return ""
+    return json.dumps(allowed, ensure_ascii=False, sort_keys=True)
+
+
 class RestartTaskCreate(BaseModel):
     action: str
 
@@ -374,8 +411,11 @@ async def read_database_log_snapshot(
                     record.occurred_at.isoformat() if record.occurred_at else "",
                     record_level.upper(),
                     record.source,
+                    record.category or "",
                     record.event or "",
+                    f"request_id={record.request_id}" if record.request_id else "",
                     record.message,
+                    _compact_log_context(record.context),
                 ]
                 if part
             )

backend/app/api/v1/visualization.py

@@ -38,7 +38,7 @@ from app.services.compute_center_locations import (
     upsert_compute_center_location,
 )
 from app.services.ai_client import get_ai_provider_client
-from app.api.v1.settings import get_web_search_client
+from app.api.v1.settings import get_runtime_web_search_config, get_web_search_client
 from app.services.collectors.bgp_common import RIPE_RIS_COLLECTOR_COORDS
 from app.services.location.llm_fallback import (
     collect_llm_location_fallback_candidate,
@@ -105,8 +105,8 @@ LANDING_POINT_CACHE_POLICY = EarthLayerCachePolicy(
 SATELLITE_CACHE_POLICY = EarthLayerCachePolicy(
     SATELLITE_CACHE_FRESH_SECONDS,
     SATELLITE_CACHE_STALE_SECONDS,
-    max_features=8000,
-    max_bytes=10 * BYTES_PER_MIB,
+    max_features=25000,
+    max_bytes=32 * BYTES_PER_MIB,
 )
 COMPUTE_CENTER_CACHE_POLICY = EarthLayerCachePolicy(
     COMPUTE_CENTER_CACHE_FRESH_SECONDS,
@@ -1563,15 +1563,7 @@ async def _build_cables_geojson(db: AsyncSession) -> dict[str, Any]:
     try:
         records = await _load_current_collected_data(db, "arcgis_cables")
 
-        if not records:
-            raise HTTPException(
-                status_code=404,
-                detail="No cable data found. Please run the arcgis_cables collector first.",
-            )
-
         return convert_cable_to_geojson(records)
-    except HTTPException:
-        raise
     except Exception as e:
         logger.exception_event(
             "Failed to build cables GeoJSON response",
@@ -1625,16 +1617,8 @@ async def _build_landing_points_geojson(db: AsyncSession) -> dict[str, Any]:
             relation_records,
             cable_records,
         )
-        
-        if not records:
-            raise HTTPException(
-                status_code=404,
-                detail="No landing point data found. Please run the arcgis_landing_points collector first.",
-            )
-        
+
         return convert_landing_point_to_geojson(records, city_to_cable_ids_map, cable_id_to_name_map)
-    except HTTPException:
-        raise
     except Exception as e:
         logger.exception_event(
             "Failed to build landing points GeoJSON response",
@@ -2071,6 +2055,44 @@ class SaveComputeCenterLocationRequest(BaseModel):
     model_config = {"populate_by_name": True}
 
 
+async def _compute_center_location_web_search_capability(db: AsyncSession) -> Dict[str, Any]:
+    try:
+        config = await get_runtime_web_search_config(db)
+    except Exception as exc:
+        return {
+            "enabled": False,
+            "provider": None,
+            "reason": f"WebSearch 配置读取失败：{exc}",
+        }
+    provider_config = config.active_provider_config
+    has_api_key = bool((provider_config.api_key or "").strip())
+    if not config.enabled:
+        return {
+            "enabled": False,
+            "provider": config.default_provider,
+            "reason": "WebSearch 未开启，无法进行事实核查定位。",
+        }
+    if not has_api_key:
+        return {
+            "enabled": False,
+            "provider": config.default_provider,
+            "reason": f"WebSearch Provider {config.default_provider} 未配置 API Key。",
+        }
+    return {
+        "enabled": True,
+        "provider": config.default_provider,
+        "reason": "",
+    }
+
+
+@router.get("/compute-centers/location-capability")
+async def get_compute_center_location_capability(
+    db: AsyncSession = Depends(get_db),
+):
+    """Return whether fact-checked compute-center location collection can run."""
+    return await _compute_center_location_web_search_capability(db)
+
+
 @router.post("/compute-centers/{source_id}/collect-location")
 async def collect_compute_center_location(
     source_id: str,
@@ -2089,6 +2111,9 @@ async def collect_compute_center_location(
     """
     if not source_id or not source_id.strip():
         raise HTTPException(status_code=400, detail="source_id is required")
+    capability = await _compute_center_location_web_search_capability(db)
+    if not capability.get("enabled"):
+        raise HTTPException(status_code=409, detail=capability)
 
     record = await _load_compute_center_record(db, source_id)
     name = payload.name or (record.name if record else None)
@@ -2696,6 +2721,8 @@ async def _build_bgp_collectors_geojson(db: AsyncSession) -> dict[str, Any]:
         db,
         source_filter=("ris_live_bgp", "bgpstream_bgp"),
     )
+    if not any(int(item.get("observation_count") or 0) > 0 for item in coverage):
+        return {"type": "FeatureCollection", "features": [], "count": 0}
     coverage_by_collector = {
         item["collector"]: item
         for item in coverage

backend/app/core/websocket/broadcaster.py

@@ -150,7 +150,7 @@ class DataBroadcaster:
                 "timestamp": to_iso8601_utc(datetime.now(UTC)),
                 "payload": data,
             },
-            channel="all",
+            channel="datasource_tasks",
         )
 
     def start(self):

backend/app/db/session.py

@@ -203,6 +203,7 @@ async def init_db():
     import app.models.vessel_enrichment  # noqa: F401
     import app.models.datasource_mapping  # noqa: F401
     import app.models.earth_news  # noqa: F401
+    import app.models.earth_interactable  # noqa: F401
 
     logger.warning_event(
         "Database pool settings active",
@@ -258,6 +259,406 @@ async def init_db():
                 """
             )
         )
+        await conn.execute(
+            text(
+                """
+                CREATE TABLE IF NOT EXISTS earth_data_change_events (
+                    id BIGSERIAL PRIMARY KEY,
+                    table_name VARCHAR(128) NOT NULL,
+                    operation VARCHAR(16) NOT NULL,
+                    source VARCHAR(128),
+                    entity_key VARCHAR(255),
+                    payload JSONB NOT NULL,
+                    occurred_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+                    consumed_at TIMESTAMPTZ
+                )
+                """
+            )
+        )
+        await conn.execute(
+            text(
+                """
+                CREATE INDEX IF NOT EXISTS idx_earth_data_change_events_unconsumed
+                ON earth_data_change_events (consumed_at, id)
+                WHERE consumed_at IS NULL
+                """
+            )
+        )
+        await conn.execute(
+            text(
+                """
+                CREATE OR REPLACE FUNCTION planet_emit_earth_data_changed_statement(
+                    change_table TEXT,
+                    change_operation TEXT,
+                    change_source TEXT,
+                    source_record_count INTEGER,
+                    source_entity_keys TEXT[]
+                )
+                RETURNS VOID AS $$
+                DECLARE
+                    change_event_id BIGINT;
+                    change_payload JSONB;
+                BEGIN
+                    change_payload := jsonb_build_object(
+                        'event', 'earth.layer.changed',
+                        'table', change_table,
+                        'operation', change_operation,
+                        'source', change_source,
+                        'entity_key', NULL,
+                        'entity_keys', COALESCE(to_jsonb(source_entity_keys), '[]'::jsonb),
+                        'records_processed', COALESCE(source_record_count, 0),
+                        'occurred_at', NOW()
+                    );
+
+                    INSERT INTO earth_data_change_events (
+                        table_name,
+                        operation,
+                        source,
+                        entity_key,
+                        payload,
+                        occurred_at
+                    ) VALUES (
+                        change_table,
+                        change_operation,
+                        change_source,
+                        NULL,
+                        change_payload,
+                        NOW()
+                    )
+                    RETURNING id INTO change_event_id;
+
+                    change_payload := change_payload || jsonb_build_object(
+                        'event_id', change_event_id
+                    );
+
+                    UPDATE earth_data_change_events
+                    SET payload = change_payload
+                    WHERE id = change_event_id;
+
+                    PERFORM pg_notify(
+                        'planet_earth_data_changes',
+                        change_payload::text
+                    );
+                END;
+                $$ LANGUAGE plpgsql;
+                """
+            )
+        )
+        await conn.execute(
+            text(
+                """
+                CREATE OR REPLACE FUNCTION planet_emit_collected_data_changed_statement(
+                    change_operation TEXT,
+                    change_source TEXT,
+                    source_record_count INTEGER,
+                    source_entity_keys TEXT[]
+                )
+                RETURNS VOID AS $$
+                BEGIN
+                    PERFORM planet_emit_earth_data_changed_statement(
+                        'collected_data',
+                        change_operation,
+                        change_source,
+                        source_record_count,
+                        source_entity_keys
+                    );
+                END;
+                $$ LANGUAGE plpgsql;
+                """
+            )
+        )
+        await conn.execute(
+            text(
+                """
+                CREATE OR REPLACE FUNCTION planet_notify_earth_table_changed_statement()
+                RETURNS trigger AS $$
+                DECLARE
+                    change_source TEXT;
+                    source_record_count INTEGER;
+                    source_entity_keys TEXT[];
+                BEGIN
+                    IF TG_OP = 'INSERT' THEN
+                        FOR change_source IN
+                            SELECT DISTINCT COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME)
+                            FROM (SELECT to_jsonb(t) AS row_data FROM new_rows AS t) changed_rows
+                        LOOP
+                            SELECT
+                                COUNT(*),
+                                ARRAY(
+                                    SELECT DISTINCT COALESCE(
+                                        NULLIF(row_data->>'entity_key', ''),
+                                        NULLIF(row_data->>'source_id', ''),
+                                        NULLIF(row_data->>'incident_key', ''),
+                                        NULLIF(row_data->>'id', ''),
+                                        NULLIF(row_data->>'mmsi', '')
+                                    )
+                                    FROM (SELECT to_jsonb(t) AS row_data FROM new_rows AS t) rows_for_keys
+                                    WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source
+                                    LIMIT 20
+                                )
+                            INTO source_record_count, source_entity_keys
+                            FROM (SELECT to_jsonb(t) AS row_data FROM new_rows AS t) rows_for_count
+                            WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source;
+
+                            PERFORM planet_emit_earth_data_changed_statement(
+                                TG_TABLE_NAME,
+                                TG_OP,
+                                change_source,
+                                source_record_count,
+                                source_entity_keys
+                            );
+                        END LOOP;
+                    ELSIF TG_OP = 'DELETE' THEN
+                        FOR change_source IN
+                            SELECT DISTINCT COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME)
+                            FROM (SELECT to_jsonb(t) AS row_data FROM old_rows AS t) changed_rows
+                        LOOP
+                            SELECT
+                                COUNT(*),
+                                ARRAY(
+                                    SELECT DISTINCT COALESCE(
+                                        NULLIF(row_data->>'entity_key', ''),
+                                        NULLIF(row_data->>'source_id', ''),
+                                        NULLIF(row_data->>'incident_key', ''),
+                                        NULLIF(row_data->>'id', ''),
+                                        NULLIF(row_data->>'mmsi', '')
+                                    )
+                                    FROM (SELECT to_jsonb(t) AS row_data FROM old_rows AS t) rows_for_keys
+                                    WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source
+                                    LIMIT 20
+                                )
+                            INTO source_record_count, source_entity_keys
+                            FROM (SELECT to_jsonb(t) AS row_data FROM old_rows AS t) rows_for_count
+                            WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source;
+
+                            PERFORM planet_emit_earth_data_changed_statement(
+                                TG_TABLE_NAME,
+                                TG_OP,
+                                change_source,
+                                source_record_count,
+                                source_entity_keys
+                            );
+                        END LOOP;
+                    ELSIF TG_OP = 'UPDATE' THEN
+                        FOR change_source IN
+                            SELECT DISTINCT COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME)
+                            FROM (
+                                SELECT to_jsonb(t) AS row_data FROM new_rows AS t
+                                UNION ALL
+                                SELECT to_jsonb(t) AS row_data FROM old_rows AS t
+                            ) changed_rows
+                        LOOP
+                            SELECT
+                                COUNT(*),
+                                ARRAY(
+                                    SELECT DISTINCT COALESCE(
+                                        NULLIF(row_data->>'entity_key', ''),
+                                        NULLIF(row_data->>'source_id', ''),
+                                        NULLIF(row_data->>'incident_key', ''),
+                                        NULLIF(row_data->>'id', ''),
+                                        NULLIF(row_data->>'mmsi', '')
+                                    )
+                                    FROM (
+                                        SELECT to_jsonb(t) AS row_data FROM new_rows AS t
+                                        UNION ALL
+                                        SELECT to_jsonb(t) AS row_data FROM old_rows AS t
+                                    ) rows_for_keys
+                                    WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source
+                                    LIMIT 20
+                                )
+                            INTO source_record_count, source_entity_keys
+                            FROM (
+                                SELECT to_jsonb(t) AS row_data FROM new_rows AS t
+                                UNION ALL
+                                SELECT to_jsonb(t) AS row_data FROM old_rows AS t
+                            ) rows_for_count
+                            WHERE COALESCE(NULLIF(row_data->>'source', ''), TG_TABLE_NAME) = change_source;
+
+                            PERFORM planet_emit_earth_data_changed_statement(
+                                TG_TABLE_NAME,
+                                TG_OP,
+                                change_source,
+                                source_record_count,
+                                source_entity_keys
+                            );
+                        END LOOP;
+                    END IF;
+
+                    RETURN NULL;
+                END;
+                $$ LANGUAGE plpgsql;
+                """
+            )
+        )
+        await conn.execute(
+            text(
+                """
+                CREATE OR REPLACE FUNCTION planet_notify_collected_data_changed_statement()
+                RETURNS trigger AS $$
+                DECLARE
+                    change_source TEXT;
+                    source_record_count INTEGER;
+                    source_entity_keys TEXT[];
+                BEGIN
+                    IF TG_OP = 'INSERT' THEN
+                        FOR change_source IN
+                            SELECT DISTINCT source FROM new_rows WHERE source IS NOT NULL
+                        LOOP
+                            SELECT
+                                COUNT(*),
+                                ARRAY(
+                                    SELECT DISTINCT COALESCE(entity_key, source_id, id::text)
+                                    FROM new_rows
+                                    WHERE source = change_source
+                                    LIMIT 20
+                                )
+                            INTO source_record_count, source_entity_keys
+                            FROM new_rows
+                            WHERE source = change_source;
+
+                            PERFORM planet_emit_collected_data_changed_statement(
+                                TG_OP,
+                                change_source,
+                                source_record_count,
+                                source_entity_keys
+                            );
+                        END LOOP;
+                    ELSIF TG_OP = 'DELETE' THEN
+                        FOR change_source IN
+                            SELECT DISTINCT source FROM old_rows WHERE source IS NOT NULL
+                        LOOP
+                            SELECT
+                                COUNT(*),
+                                ARRAY(
+                                    SELECT DISTINCT COALESCE(entity_key, source_id, id::text)
+                                    FROM old_rows
+                                    WHERE source = change_source
+                                    LIMIT 20
+                                )
+                            INTO source_record_count, source_entity_keys
+                            FROM old_rows
+                            WHERE source = change_source;
+
+                            PERFORM planet_emit_collected_data_changed_statement(
+                                TG_OP,
+                                change_source,
+                                source_record_count,
+                                source_entity_keys
+                            );
+                        END LOOP;
+                    ELSIF TG_OP = 'UPDATE' THEN
+                        FOR change_source IN
+                            SELECT DISTINCT source FROM (
+                                SELECT source FROM new_rows
+                                UNION
+                                SELECT source FROM old_rows
+                            ) changed_sources
+                            WHERE source IS NOT NULL
+                        LOOP
+                            SELECT
+                                COUNT(*),
+                                ARRAY(
+                                    SELECT DISTINCT COALESCE(entity_key, source_id, id::text)
+                                    FROM (
+                                        SELECT id, source_id, entity_key, source FROM new_rows
+                                        UNION ALL
+                                        SELECT id, source_id, entity_key, source FROM old_rows
+                                    ) changed_rows
+                                    WHERE source = change_source
+                                    LIMIT 20
+                                )
+                            INTO source_record_count, source_entity_keys
+                            FROM (
+                                SELECT id, source_id, entity_key, source FROM new_rows
+                                UNION ALL
+                                SELECT id, source_id, entity_key, source FROM old_rows
+                            ) changed_rows
+                            WHERE source = change_source;
+
+                            PERFORM planet_emit_collected_data_changed_statement(
+                                TG_OP,
+                                change_source,
+                                source_record_count,
+                                source_entity_keys
+                            );
+                        END LOOP;
+                    END IF;
+
+                    RETURN NULL;
+                END;
+                $$ LANGUAGE plpgsql;
+                """
+            )
+        )
+        for statement in (
+            "DROP TRIGGER IF EXISTS tr_planet_collected_data_changed ON collected_data",
+            "DROP TRIGGER IF EXISTS tr_planet_collected_data_changed_insert ON collected_data",
+            "DROP TRIGGER IF EXISTS tr_planet_collected_data_changed_update ON collected_data",
+            "DROP TRIGGER IF EXISTS tr_planet_collected_data_changed_delete ON collected_data",
+            "DROP FUNCTION IF EXISTS planet_notify_collected_data_changed()",
+            """
+            CREATE TRIGGER tr_planet_collected_data_changed_insert
+            AFTER INSERT ON collected_data
+            REFERENCING NEW TABLE AS new_rows
+            FOR EACH STATEMENT
+            EXECUTE FUNCTION planet_notify_collected_data_changed_statement()
+            """,
+            """
+            CREATE TRIGGER tr_planet_collected_data_changed_update
+            AFTER UPDATE ON collected_data
+            REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
+            FOR EACH STATEMENT
+            EXECUTE FUNCTION planet_notify_collected_data_changed_statement()
+            """,
+            """
+            CREATE TRIGGER tr_planet_collected_data_changed_delete
+            AFTER DELETE ON collected_data
+            REFERENCING OLD TABLE AS old_rows
+            FOR EACH STATEMENT
+            EXECUTE FUNCTION planet_notify_collected_data_changed_statement()
+            """,
+        ):
+            await conn.execute(text(statement))
+        for table_name in (
+            "bgp_observations",
+            "bgp_anomalies",
+            "bgp_incidents",
+            "bgp_collector_locations",
+            "vessel_static",
+            "vessel_position",
+            "ais_raw_observations",
+            "ais_source_health",
+            "compute_center_locations",
+            "earth_interactables",
+            "earth_news_items",
+        ):
+            for statement in (
+                f"DROP TRIGGER IF EXISTS tr_planet_{table_name}_changed_insert ON {table_name}",
+                f"DROP TRIGGER IF EXISTS tr_planet_{table_name}_changed_update ON {table_name}",
+                f"DROP TRIGGER IF EXISTS tr_planet_{table_name}_changed_delete ON {table_name}",
+                f"""
+                CREATE TRIGGER tr_planet_{table_name}_changed_insert
+                AFTER INSERT ON {table_name}
+                REFERENCING NEW TABLE AS new_rows
+                FOR EACH STATEMENT
+                EXECUTE FUNCTION planet_notify_earth_table_changed_statement()
+                """,
+                f"""
+                CREATE TRIGGER tr_planet_{table_name}_changed_update
+                AFTER UPDATE ON {table_name}
+                REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
+                FOR EACH STATEMENT
+                EXECUTE FUNCTION planet_notify_earth_table_changed_statement()
+                """,
+                f"""
+                CREATE TRIGGER tr_planet_{table_name}_changed_delete
+                AFTER DELETE ON {table_name}
+                REFERENCING OLD TABLE AS old_rows
+                FOR EACH STATEMENT
+                EXECUTE FUNCTION planet_notify_earth_table_changed_statement()
+                """,
+            ):
+                await conn.execute(text(statement))
         await conn.execute(
             text(
                 """
@@ -267,7 +668,16 @@ async def init_db():
                 ADD COLUMN IF NOT EXISTS phase_message VARCHAR(255),
                 ADD COLUMN IF NOT EXISTS phase_current BIGINT,
                 ADD COLUMN IF NOT EXISTS phase_total BIGINT,
-                ADD COLUMN IF NOT EXISTS phase_unit VARCHAR(30)
+                ADD COLUMN IF NOT EXISTS phase_unit VARCHAR(30),
+                ADD COLUMN IF NOT EXISTS source VARCHAR(100),
+                ADD COLUMN IF NOT EXISTS task_type VARCHAR(30) NOT NULL DEFAULT 'collect',
+                ADD COLUMN IF NOT EXISTS payload JSONB NOT NULL DEFAULT '{}'::jsonb,
+                ADD COLUMN IF NOT EXISTS rollback_policy VARCHAR(40) NOT NULL DEFAULT 'keep_committed_batches',
+                ADD COLUMN IF NOT EXISTS dedupe_key VARCHAR(180),
+                ADD COLUMN IF NOT EXISTS worker_id VARCHAR(120),
+                ADD COLUMN IF NOT EXISTS locked_at TIMESTAMPTZ,
+                ADD COLUMN IF NOT EXISTS requested_cancel_at TIMESTAMPTZ,
+                ADD COLUMN IF NOT EXISTS cancel_reason TEXT
                 """
             )
         )
@@ -283,6 +693,17 @@ async def init_db():
                 """
             )
         )
+        await conn.execute(
+            text(
+                """
+                ALTER TABLE earth_interactables
+                ADD COLUMN IF NOT EXISTS altitude DOUBLE PRECISION,
+                ADD COLUMN IF NOT EXISTS revision INTEGER NOT NULL DEFAULT 1,
+                ADD COLUMN IF NOT EXISTS is_deleted BOOLEAN NOT NULL DEFAULT FALSE,
+                ADD COLUMN IF NOT EXISTS deleted_at TIMESTAMPTZ
+                """
+            )
+        )
         await conn.execute(
             text(
                 """
@@ -307,6 +728,48 @@ async def init_db():
                 """
             )
         )
+        await conn.execute(
+            text(
+                """
+                CREATE INDEX IF NOT EXISTS idx_earth_interactables_layer_deleted
+                ON earth_interactables (layer, is_deleted)
+                """
+            )
+        )
+        await conn.execute(
+            text(
+                """
+                CREATE INDEX IF NOT EXISTS idx_earth_interactables_updated_at
+                ON earth_interactables (updated_at)
+                """
+            )
+        )
+        await conn.execute(
+            text(
+                """
+                CREATE INDEX IF NOT EXISTS idx_collection_tasks_source_status
+                ON collection_tasks (source, status)
+                """
+            )
+        )
+        await conn.execute(
+            text(
+                """
+                CREATE INDEX IF NOT EXISTS idx_collection_tasks_queue
+                ON collection_tasks (status, created_at, id)
+                WHERE status = 'queued'
+                """
+            )
+        )
+        await conn.execute(
+            text(
+                """
+                CREATE INDEX IF NOT EXISTS idx_collection_tasks_dedupe
+                ON collection_tasks (dedupe_key)
+                WHERE dedupe_key IS NOT NULL
+                """
+            )
+        )
         await conn.execute(
             text(
                 """

backend/app/main.py

@@ -24,6 +24,11 @@ from app.services.earth_news_worker import (
     start_earth_news_target_worker,
     stop_earth_news_target_worker,
 )
+from app.services.earth_db_change_listener import (
+    start_earth_db_change_listener,
+    stop_earth_db_change_listener,
+)
+from app.services.data_jobs import start_data_job_worker, stop_data_job_worker
 
 
 configure_logging()
@@ -59,9 +64,13 @@ async def lifespan(app: FastAPI):
     start_scheduler()
     await sync_scheduler_with_datasources()
     broadcaster.start()
+    start_data_job_worker()
+    start_earth_db_change_listener()
     start_earth_news_target_worker()
     yield
     await stop_earth_news_target_worker()
+    await stop_earth_db_change_listener()
+    await stop_data_job_worker()
     broadcaster.stop()
     stop_scheduler()
 

backend/app/models/__init__.py

@@ -17,6 +17,7 @@ from app.models.system_log import SystemLog, AuditLog
 from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth, VesselPosition, VesselStatic
 from app.models.datasource_mapping import DataSourceMappingTemplate
 from app.models.earth_news import EarthNewsItem
+from app.models.earth_interactable import EarthInteractable
 
 __all__ = [
     "User",
@@ -45,4 +46,5 @@ __all__ = [
     "AISSourceHealth",
     "DataSourceMappingTemplate",
     "EarthNewsItem",
+    "EarthInteractable",
 ]

backend/app/models/earth_interactable.py

@@ -0,0 +1,30 @@
+"""Persistent Earth interactable objects."""
+
+from sqlalchemy import Boolean, Column, DateTime, Float, Index, Integer, JSON, String, Text
+from sqlalchemy.sql import func
+
+from app.db.session import Base
+
+
+class EarthInteractable(Base):
+    __tablename__ = "earth_interactables"
+
+    id = Column(String(160), primary_key=True)
+    layer = Column(String(80), nullable=False, default="interactables", index=True)
+    kind = Column(String(80), nullable=False, default="default", index=True)
+    label = Column(String(255), nullable=False, default="")
+    description = Column(Text, nullable=False, default="")
+    latitude = Column(Float, nullable=False)
+    longitude = Column(Float, nullable=False)
+    altitude = Column(Float, nullable=True)
+    revision = Column(Integer, nullable=False, default=1)
+    properties = Column(JSON, nullable=False, default=dict)
+    is_deleted = Column(Boolean, nullable=False, default=False, index=True)
+    created_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False)
+    updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+    deleted_at = Column(DateTime(timezone=True), nullable=True, index=True)
+
+    __table_args__ = (
+        Index("idx_earth_interactables_layer_deleted", "layer", "is_deleted"),
+        Index("idx_earth_interactables_updated_at", "updated_at"),
+    )

backend/app/models/task.py

@@ -1,6 +1,6 @@
-"""Collection Task model"""
+"""Datasource job model."""
 
-from sqlalchemy import BigInteger, Column, DateTime, Integer, String, Text, Float
+from sqlalchemy import BigInteger, Column, DateTime, Float, Integer, JSON, String, Text
 from sqlalchemy.sql import func
 
 from app.db.session import Base
@@ -11,7 +11,9 @@ class CollectionTask(Base):
 
     id = Column(Integer, primary_key=True, autoincrement=True)
     datasource_id = Column(Integer, nullable=False, index=True)
-    status = Column(String(20), nullable=False)  # pending, running, success, failed, cancelled
+    source = Column(String(100), nullable=True, index=True)
+    task_type = Column(String(30), nullable=False, default="collect", index=True)
+    status = Column(String(20), nullable=False)  # queued, running, cancelling, success, failed, cancelled
     phase = Column(String(30), default="queued")
     phase_progress = Column(Float)
     phase_message = Column(String(255))
@@ -24,6 +26,13 @@ class CollectionTask(Base):
     total_records = Column(Integer, default=0)  # Total records to process
     progress = Column(Float, default=0.0)  # Progress percentage (0-100)
     error_message = Column(Text)
+    payload = Column(JSON, default=dict)
+    rollback_policy = Column(String(40), nullable=False, default="keep_committed_batches")
+    dedupe_key = Column(String(180), nullable=True, index=True)
+    worker_id = Column(String(120), nullable=True, index=True)
+    locked_at = Column(DateTime(timezone=True), nullable=True, index=True)
+    requested_cancel_at = Column(DateTime(timezone=True), nullable=True)
+    cancel_reason = Column(Text)
     created_at = Column(DateTime(timezone=True), server_default=func.now())
 
     def __repr__(self):

backend/app/services/ai_client.py

@@ -2,18 +2,24 @@ from __future__ import annotations
 
 import asyncio
 import json
+from time import perf_counter
 
 import httpx
 from fastapi import Depends, HTTPException, status
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.core.config import settings
+from app.core.logging import get_logger
 from app.db.session import get_db
 from app.schemas.ai import (
     AIProviderStatusResponse,
     SituationalAnalysisRequest,
     SituationalAnalysisResponse,
 )
+from app.services.business_logs import emit_business_log, exception_context
+
+
+logger = get_logger(__name__, service="ai")
 
 
 class AIProviderClient:
@@ -64,7 +70,19 @@ class AIProviderClient:
         return headers
 
     async def get_status(self, request_id: str | None = None) -> AIProviderStatusResponse:
+        context = self._base_log_context(operation="status")
         if not self.service_url:
+            await emit_business_log(
+                logger,
+                event="ai.provider.status.failed",
+                message="AI provider status skipped because service URL is not configured",
+                category="ai",
+                level="warning",
+                service="ai",
+                module=__name__,
+                request_id=request_id,
+                context={**context, "status": "unconfigured"},
+            )
             return AIProviderStatusResponse(
                 provider="unconfigured",
                 enabled=False,
@@ -73,27 +91,133 @@ class AIProviderClient:
                 base_url=None,
             )
 
-        data = await self._request("GET", "/v1/provider/status", request_id=request_id)
-        return AIProviderStatusResponse.model_validate(data)
+        started_at = perf_counter()
+        await emit_business_log(
+            logger,
+            event="ai.provider.status.start",
+            message="AI provider status request started",
+            category="ai",
+            service="ai",
+            module=__name__,
+            request_id=request_id,
+            context=context,
+        )
+        try:
+            data = await self._request("GET", "/v1/provider/status", request_id=request_id, operation="status")
+            result = AIProviderStatusResponse.model_validate(data)
+            await emit_business_log(
+                logger,
+                event="ai.provider.status.success",
+                message="AI provider status request completed",
+                category="ai",
+                service="ai",
+                module=__name__,
+                request_id=request_id,
+                context={
+                    **context,
+                    "status": "success",
+                    "duration_ms": self._duration_ms(started_at),
+                    "result_provider": result.provider,
+                    "result_model": result.model,
+                    "configured": result.configured,
+                    "enabled": result.enabled,
+                },
+            )
+            return result
+        except Exception as exc:
+            await emit_business_log(
+                logger,
+                event="ai.provider.status.failed",
+                message="AI provider status request failed",
+                category="ai",
+                level="error",
+                service="ai",
+                module=__name__,
+                request_id=request_id,
+                context=exception_context(exc, {**context, "status": "failed", "duration_ms": self._duration_ms(started_at)}),
+            )
+            raise
 
     async def analyze(
         self,
         payload: SituationalAnalysisRequest,
         request_id: str | None = None,
     ) -> SituationalAnalysisResponse:
+        context = self._base_log_context(
+            operation="analyze",
+            preferred_model=payload.preferred_model,
+            input_summary=self._summarize_analysis_payload(payload),
+        )
         if not self.service_url:
+            await emit_business_log(
+                logger,
+                event="ai.provider.analyze.failed",
+                message="AI provider analyze skipped because service URL is not configured",
+                category="ai",
+                level="warning",
+                service="ai",
+                module=__name__,
+                request_id=request_id,
+                context={**context, "status": "unconfigured"},
+            )
             raise HTTPException(
                 status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
                 detail="AI provider service URL is not configured.",
             )
 
-        data = await self._request(
-            "POST",
-            "/v1/analyze",
-            json=payload.model_dump(),
+        started_at = perf_counter()
+        await emit_business_log(
+            logger,
+            event="ai.provider.analyze.start",
+            message="AI provider analyze request started",
+            category="ai",
+            service="ai",
+            module=__name__,
             request_id=request_id,
+            context=context,
         )
-        return SituationalAnalysisResponse.model_validate(data)
+        try:
+            data = await self._request(
+                "POST",
+                "/v1/analyze",
+                json=payload.model_dump(),
+                request_id=request_id,
+                operation="analyze",
+                payload_summary=context["input_summary"],
+            )
+            result = SituationalAnalysisResponse.model_validate(data)
+            await emit_business_log(
+                logger,
+                event="ai.provider.analyze.success",
+                message="AI provider analyze request completed",
+                category="ai",
+                service="ai",
+                module=__name__,
+                request_id=request_id,
+                context={
+                    **context,
+                    "status": "success",
+                    "duration_ms": self._duration_ms(started_at),
+                    "result_provider": result.provider,
+                    "result_model": result.model,
+                    "content_block_count": len(result.content_blocks or []),
+                    "thinking_block_count": len(result.thinking_blocks or []),
+                },
+            )
+            return result
+        except Exception as exc:
+            await emit_business_log(
+                logger,
+                event="ai.provider.analyze.failed",
+                message="AI provider analyze request failed",
+                category="ai",
+                level="error",
+                service="ai",
+                module=__name__,
+                request_id=request_id,
+                context=exception_context(exc, {**context, "status": "failed", "duration_ms": self._duration_ms(started_at)}),
+            )
+            raise
 
     async def _request(
         self,
@@ -101,9 +225,12 @@ class AIProviderClient:
         path: str,
         json: dict | None = None,
         request_id: str | None = None,
+        operation: str = "request",
+        payload_summary: dict | None = None,
     ) -> dict:
         last_error: Exception | None = None
         for attempt in range(1, self.retry_attempts + 1):
+            attempt_started_at = perf_counter()
             try:
                 async with httpx.AsyncClient(timeout=self.timeout) as client:
                     response = await client.request(
@@ -117,6 +244,15 @@ class AIProviderClient:
             except httpx.HTTPStatusError as exc:
                 last_error = exc
                 if attempt < self.retry_attempts and exc.response.status_code >= 500:
+                    await self._log_retry(
+                        operation=operation,
+                        request_id=request_id,
+                        attempt=attempt,
+                        status_code=exc.response.status_code,
+                        duration_ms=self._duration_ms(attempt_started_at),
+                        error=exc,
+                        payload_summary=payload_summary,
+                    )
                     await asyncio.sleep(0.3 * attempt)
                     continue
                 detail = exc.response.text or "AI provider service returned an error"
@@ -127,6 +263,14 @@ class AIProviderClient:
             except httpx.HTTPError as exc:
                 last_error = exc
                 if attempt < self.retry_attempts:
+                    await self._log_retry(
+                        operation=operation,
+                        request_id=request_id,
+                        attempt=attempt,
+                        duration_ms=self._duration_ms(attempt_started_at),
+                        error=exc,
+                        payload_summary=payload_summary,
+                    )
                     await asyncio.sleep(0.3 * attempt)
                     continue
                 raise HTTPException(
@@ -139,6 +283,71 @@ class AIProviderClient:
             detail=f"AI provider service request failed: {last_error}",
         )
 
+    def _base_log_context(self, **extra: object) -> dict:
+        llm_provider_apis = self.llm_config.get("model_provider_apis")
+        return {
+            "provider": self.llm_config.get("provider") or "",
+            "provider_api": self.llm_config.get("provider_api") or "",
+            "model": self.llm_config.get("model") or "",
+            "base_url_configured": bool(self.llm_config.get("base_url")),
+            "service_url_configured": bool(self.service_url),
+            "timeout_seconds": self.timeout,
+            "retry_attempts": self.retry_attempts,
+            "model_provider_api_count": len(llm_provider_apis or {}) if isinstance(llm_provider_apis, dict) else 0,
+            **extra,
+        }
+
+    @staticmethod
+    def _duration_ms(started_at: float) -> int:
+        return int((perf_counter() - started_at) * 1000)
+
+    @staticmethod
+    def _summarize_analysis_payload(payload: SituationalAnalysisRequest) -> dict:
+        context = payload.context if isinstance(payload.context, dict) else {}
+        thinking = payload.thinking if isinstance(payload.thinking, dict) else payload.thinking
+        return {
+            "title_length": len(payload.title or ""),
+            "objective_length": len(payload.objective or ""),
+            "observation_count": len(payload.observations or []),
+            "constraint_count": len(payload.constraints or []),
+            "has_system_prompt": bool(payload.system_prompt),
+            "thinking_enabled": bool(thinking),
+            "context_keys": sorted(str(key) for key in context.keys()),
+        }
+
+    async def _log_retry(
+        self,
+        *,
+        operation: str,
+        request_id: str | None,
+        attempt: int,
+        duration_ms: int,
+        error: BaseException,
+        status_code: int | None = None,
+        payload_summary: dict | None = None,
+    ) -> None:
+        await emit_business_log(
+            logger,
+            event=f"ai.provider.{operation}.retry",
+            message="AI provider request will retry",
+            category="ai",
+            level="warning",
+            service="ai",
+            module=__name__,
+            request_id=request_id,
+            context=exception_context(
+                error,
+                {
+                    **self._base_log_context(operation=operation),
+                    "attempt": attempt,
+                    "next_attempt": attempt + 1,
+                    "status_code": status_code,
+                    "duration_ms": duration_ms,
+                    "input_summary": payload_summary,
+                },
+            ),
+        )
+
 
 async def get_ai_provider_client(db: AsyncSession = Depends(get_db)) -> AIProviderClient:
     from app.api.v1.settings import get_runtime_ai_provider_config

backend/app/services/ai_tools/web_fetch.py

@@ -1,11 +1,18 @@
 from __future__ import annotations
 
 import hashlib
+from time import perf_counter
+from urllib.parse import urlparse
 
 import httpx
 from bs4 import BeautifulSoup
 
+from app.core.logging import get_logger
 from app.services.ai_tools.schemas import FetchedEvidence
+from app.services.business_logs import emit_business_log, exception_context
+
+
+logger = get_logger(__name__, service="ai_tool")
 
 
 class WebFetchError(RuntimeError):
@@ -29,8 +36,33 @@ async def fetch_url_evidence(
     timeout_seconds: int = 20,
     max_bytes: int = 1_500_000,
 ) -> FetchedEvidence:
+    started_at = perf_counter()
     if not url:
+        await emit_business_log(
+            logger,
+            event="ai_tool.web_fetch.failed",
+            message="WebFetch failed because URL is empty",
+            category="ai_tool",
+            level="warning",
+            service="ai_tool",
+            module=__name__,
+            context={"reason": "empty_url"},
+        )
         raise WebFetchError("url is required")
+    request_host = urlparse(url).netloc
+    await emit_business_log(
+        logger,
+        event="ai_tool.web_fetch.start",
+        message="WebFetch request started",
+        category="ai_tool",
+        service="ai_tool",
+        module=__name__,
+        context={
+            "url_host": request_host,
+            "timeout_seconds": timeout_seconds,
+            "max_bytes": max_bytes,
+        },
+    )
     try:
         async with httpx.AsyncClient(
             timeout=timeout_seconds,
@@ -41,10 +73,45 @@ async def fetch_url_evidence(
             response.raise_for_status()
             content = response.content[:max_bytes]
     except httpx.HTTPError as exc:
+        await emit_business_log(
+            logger,
+            event="ai_tool.web_fetch.failed",
+            message="WebFetch request failed",
+            category="ai_tool",
+            level="error",
+            service="ai_tool",
+            module=__name__,
+            context=exception_context(
+                exc,
+                {
+                    "url_host": request_host,
+                    "status": "failed",
+                    "duration_ms": int((perf_counter() - started_at) * 1000),
+                },
+            ),
+        )
         raise WebFetchError(f"failed to fetch page: {exc}") from exc
 
     title, text = _extract_title_and_text(content.decode(response.encoding or "utf-8", errors="ignore"))
     content_hash = hashlib.sha256(text.encode("utf-8")).hexdigest()
+    await emit_business_log(
+        logger,
+        event="ai_tool.web_fetch.success",
+        message="WebFetch request completed",
+        category="ai_tool",
+        service="ai_tool",
+        module=__name__,
+        context={
+            "url_host": request_host,
+            "final_url_host": urlparse(str(response.url)).netloc,
+            "status": "success",
+            "status_code": response.status_code,
+            "bytes_read": len(content),
+            "content_hash": content_hash,
+            "duration_ms": int((perf_counter() - started_at) * 1000),
+            "extractor": "beautifulsoup_basic",
+        },
+    )
     return FetchedEvidence(
         url=url,
         final_url=str(response.url),
@@ -53,4 +120,3 @@ async def fetch_url_evidence(
         content_hash=content_hash,
         extractor="beautifulsoup_basic",
     )
-

backend/app/services/ai_tools/web_search.py

@@ -1,11 +1,18 @@
 from __future__ import annotations
 
 from copy import deepcopy
+import hashlib
+from time import perf_counter
 from typing import Any
 
 import httpx
 
+from app.core.logging import get_logger
 from app.services.ai_tools.schemas import SearchEvidence, WebSearchConfig, WebSearchProviderConfig
+from app.services.business_logs import emit_business_log, exception_context
+
+
+logger = get_logger(__name__, service="ai_tool")
 
 
 WEB_SEARCH_PROVIDER_PRESETS: dict[str, dict[str, Any]] = {
@@ -120,29 +127,108 @@ class WebSearchClient:
         domains: list[str] | None = None,
         freshness_days: int | None = None,
     ) -> list[SearchEvidence]:
+        started_at = perf_counter()
         if not self.config.enabled:
+            await emit_business_log(
+                logger,
+                event="ai_tool.web_search.unavailable",
+                message="WebSearch skipped because integration is disabled",
+                category="ai_tool",
+                level="warning",
+                service="ai_tool",
+                module=__name__,
+                context={"provider": self.config.default_provider, "reason": "disabled"},
+            )
             raise WebSearchConfigurationError("WebSearch is disabled.")
         provider_config = self.config.active_provider_config
         provider = normalize_web_search_provider(provider_config.provider)
         if provider != "searxng" and not provider_config.api_key:
+            await emit_business_log(
+                logger,
+                event="ai_tool.web_search.unavailable",
+                message="WebSearch skipped because API key is not configured",
+                category="ai_tool",
+                level="warning",
+                service="ai_tool",
+                module=__name__,
+                context={"provider": provider, "reason": "missing_api_key"},
+            )
             raise WebSearchConfigurationError(f"{provider} API key is not configured.")
         query = " ".join(str(query or "").split())
         if not query:
+            await emit_business_log(
+                logger,
+                event="ai_tool.web_search.failed",
+                message="WebSearch failed because query is empty",
+                category="ai_tool",
+                level="warning",
+                service="ai_tool",
+                module=__name__,
+                context={"provider": provider, "reason": "empty_query"},
+            )
             raise WebSearchConfigurationError("search query is required.")
         limit = max_results or provider_config.max_results
-        if provider == "tavily":
-            return await self._search_tavily(provider_config, query, limit, domains, freshness_days)
-        if provider == "brave":
-            return await self._search_brave(provider_config, query, limit, domains)
-        if provider == "serpapi":
-            return await self._search_serpapi(provider_config, query, limit)
-        if provider == "exa":
-            return await self._search_exa(provider_config, query, limit, domains)
-        if provider == "firecrawl":
-            return await self._search_firecrawl(provider_config, query, limit)
-        if provider == "searxng":
-            return await self._search_searxng(provider_config, query, limit, domains)
-        raise WebSearchConfigurationError(f"Unsupported web search provider: {provider}")
+        context = {
+            "provider": provider,
+            "query_hash": hashlib.sha256(query.encode("utf-8")).hexdigest(),
+            "query_length": len(query),
+            "max_results": limit,
+            "domain_count": len(domains or []),
+            "freshness_days": freshness_days,
+        }
+        await emit_business_log(
+            logger,
+            event="ai_tool.web_search.start",
+            message="WebSearch request started",
+            category="ai_tool",
+            service="ai_tool",
+            module=__name__,
+            context=context,
+        )
+        try:
+            if provider == "tavily":
+                results = await self._search_tavily(provider_config, query, limit, domains, freshness_days)
+            elif provider == "brave":
+                results = await self._search_brave(provider_config, query, limit, domains)
+            elif provider == "serpapi":
+                results = await self._search_serpapi(provider_config, query, limit)
+            elif provider == "exa":
+                results = await self._search_exa(provider_config, query, limit, domains)
+            elif provider == "firecrawl":
+                results = await self._search_firecrawl(provider_config, query, limit)
+            elif provider == "searxng":
+                results = await self._search_searxng(provider_config, query, limit, domains)
+            else:
+                raise WebSearchConfigurationError(f"Unsupported web search provider: {provider}")
+            event = "ai_tool.web_search.success" if results else "ai_tool.web_search.empty"
+            await emit_business_log(
+                logger,
+                event=event,
+                message="WebSearch request completed" if results else "WebSearch returned no results",
+                category="ai_tool",
+                level="info" if results else "warning",
+                service="ai_tool",
+                module=__name__,
+                context={
+                    **context,
+                    "status": "success" if results else "empty",
+                    "result_count": len(results),
+                    "duration_ms": int((perf_counter() - started_at) * 1000),
+                },
+            )
+            return results
+        except Exception as exc:
+            await emit_business_log(
+                logger,
+                event="ai_tool.web_search.failed",
+                message="WebSearch request failed",
+                category="ai_tool",
+                level="error",
+                service="ai_tool",
+                module=__name__,
+                context=exception_context(exc, {**context, "status": "failed", "duration_ms": int((perf_counter() - started_at) * 1000)}),
+            )
+            raise
 
     async def test_connection(self) -> list[SearchEvidence]:
         return await self.search("Planet WebSearch connectivity test", max_results=1)
@@ -388,4 +474,3 @@ def _float_or_none(value: Any) -> float | None:
         return float(value)
     except (TypeError, ValueError):
         return None
-

backend/app/services/business_logs.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Mapping
+from typing import Any
+
+from app.core.logging import PlanetLoggerAdapter, sanitize_log_value
+from app.core.request_context import get_request_id
+from app.services.persistent_logs import record_system_log
+
+
+LEVEL_METHODS = {
+    "debug": "debug_event",
+    "info": "info_event",
+    "warning": "warning_event",
+    "error": "error_event",
+}
+
+
+def normalize_business_level(level: str | None) -> str:
+    normalized = str(level or "info").strip().lower()
+    if normalized in {"warn", "warning"}:
+        return "warning"
+    if normalized in {"err", "error", "critical", "fatal"}:
+        return "error"
+    if normalized == "debug":
+        return "debug"
+    return "info"
+
+
+def build_business_context(
+    context: Mapping[str, Any] | None = None,
+    **fields: Any,
+) -> dict[str, Any]:
+    payload = dict(context or {})
+    for key, value in fields.items():
+        if value is not None:
+            payload[key] = value
+    return sanitize_log_value(payload)
+
+
+async def emit_business_log(
+    logger: PlanetLoggerAdapter,
+    *,
+    event: str,
+    message: str,
+    category: str,
+    level: str = "info",
+    source: str = "backend",
+    service: str | None = None,
+    module: str | None = None,
+    request_id: str | None = None,
+    user_id: int | None = None,
+    context: Mapping[str, Any] | None = None,
+) -> None:
+    normalized_level = normalize_business_level(level)
+    safe_context = build_business_context(context)
+    log_method = getattr(logger, LEVEL_METHODS[normalized_level])
+    log_method(message, event=event, context=safe_context)
+    await record_system_log(
+        source=source,
+        level=normalized_level,
+        message=message,
+        service=service,
+        module=module,
+        event=event,
+        request_id=request_id or get_request_id(),
+        user_id=user_id,
+        category=category,
+        context=safe_context,
+    )
+
+
+def emit_business_log_background(
+    logger: PlanetLoggerAdapter,
+    *,
+    event: str,
+    message: str,
+    category: str,
+    level: str = "info",
+    source: str = "backend",
+    service: str | None = None,
+    module: str | None = None,
+    request_id: str | None = None,
+    user_id: int | None = None,
+    context: Mapping[str, Any] | None = None,
+) -> None:
+    normalized_level = normalize_business_level(level)
+    safe_context = build_business_context(context)
+    log_method = getattr(logger, LEVEL_METHODS[normalized_level])
+    log_method(message, event=event, context=safe_context)
+    try:
+        loop = asyncio.get_running_loop()
+    except RuntimeError:
+        return
+    loop.create_task(
+        record_system_log(
+            source=source,
+            level=normalized_level,
+            message=message,
+            service=service,
+            module=module,
+            event=event,
+            request_id=request_id or get_request_id(),
+            user_id=user_id,
+            category=category,
+            context=safe_context,
+        )
+    )
+
+
+def exception_context(exc: BaseException, context: Mapping[str, Any] | None = None) -> dict[str, Any]:
+    return build_business_context(
+        context,
+        error_type=type(exc).__name__,
+        error=str(exc),
+    )

backend/app/services/collectors/base.py

@@ -4,42 +4,22 @@ import asyncio
 from abc import ABC, abstractmethod
 from typing import Dict, List, Any, Optional
 from datetime import UTC, datetime
+from time import perf_counter
+from urllib.parse import urlparse
 import httpx
 from sqlalchemy import select, text
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.core.collected_data_fields import build_dynamic_metadata, get_record_field
 from app.core.countries import normalize_country
+from app.core.logging import get_logger
 from app.core.time import to_iso8601_utc
 from app.core.websocket.broadcaster import broadcaster
-from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
-
-EARTH_UPDATE_LAYER_HINTS: dict[str, list[str]] = {
-    "ris_live_bgp": ["bgp"],
-    "bgpstream_bgp": ["bgp"],
-    "top500_supercomputers": ["computeCenters"],
-    "epoch_ai_gpu": ["computeCenters"],
-    "huggingface_models": ["computeCenters"],
-    "huggingface_datasets": ["computeCenters"],
-    "huggingface_spaces": ["computeCenters"],
-    "telegeography_cables": ["cables"],
-    "telegeography_landing_points": ["cables"],
-    "telegeography_cable_systems": ["cables"],
-    "arcgis_cables": ["cables"],
-    "fao_landing_points": ["cables"],
-    "arcgis_landing_points": ["cables"],
-    "arcgis_cable_landing_relations": ["cables"],
-    "spacetrack_tle": ["satellites"],
-    "celestrak_tle": ["satellites"],
-    "barentswatch_vessels": ["vessels"],
-    "aisstream_vessels": ["vessels"],
-    "news_live_streams": ["media"],
-    "media_news_archive": ["news"],
-}
+from app.services.business_logs import emit_business_log, exception_context
+from app.services.earth_layer_adapters import get_earth_update_layers_for_source
 
 
-def get_earth_update_layers_for_source(source: str) -> list[str]:
-    return EARTH_UPDATE_LAYER_HINTS.get(source, [])
+logger = get_logger(__name__, service="collector")
 
 
 class BaseCollector(ABC):
@@ -58,6 +38,7 @@ class BaseCollector(ABC):
         self._datasource_id = 1
         self._resolved_url: Optional[str] = None
         self._last_broadcast_progress: Optional[int] = None
+        self._last_save_summary: dict[str, int] = {}
 
     async def resolve_url(self, db: AsyncSession) -> None:
         from app.core.data_sources import get_data_sources_config
@@ -96,29 +77,6 @@ class BaseCollector(ABC):
         )
         self._last_broadcast_progress = rounded_progress
 
-    async def _publish_earth_update(
-        self,
-        *,
-        action: str,
-        records_processed: int,
-        task_id: int | None = None,
-    ) -> None:
-        layers = get_earth_update_layers_for_source(self.name)
-        if not layers:
-            return
-        await broadcaster.broadcast_earth_update(
-            {
-                "action": action,
-                "source": self.name,
-                "data_type": self.data_type,
-                "layers": layers,
-                "datasource_id": getattr(self, "_datasource_id", None),
-                "task_id": task_id,
-                "records_processed": records_processed,
-                "timestamp": to_iso8601_utc(datetime.now(UTC)),
-            }
-        )
-
     async def update_progress(self, records_processed: int, *, commit: bool = False, force: bool = False):
         """Update task progress - call this during data processing"""
         if self._current_task and self._db_session:
@@ -322,19 +280,39 @@ class BaseCollector(ABC):
         from app.models.data_snapshot import DataSnapshot
 
         start_time = datetime.now(UTC)
+        started_at = perf_counter()
         datasource_id = getattr(self, "_datasource_id", 1)
         snapshot_id: Optional[int] = None
 
         if not collector_registry.is_active(self.name):
+            await self._log_collection_event(
+                "collector.run.skipped_disabled",
+                "Collector skipped because it is disabled",
+                level="info",
+                context={"status": "skipped", "reason": "disabled"},
+            )
             return {"status": "skipped", "reason": "Collector is disabled"}
 
-        task = CollectionTask(
-            datasource_id=datasource_id,
-            status="running",
-            phase="queued",
-            started_at=start_time,
-        )
-        db.add(task)
+        task = self._current_task if isinstance(self._current_task, CollectionTask) else None
+        if task is None:
+            task = CollectionTask(
+                datasource_id=datasource_id,
+                source=self.name,
+                task_type="collect",
+                status="running",
+                phase="queued",
+                started_at=start_time,
+            )
+            db.add(task)
+        else:
+            task.datasource_id = datasource_id
+            task.source = task.source or self.name
+            task.task_type = task.task_type or "collect"
+            task.status = "running"
+            task.phase = "queued"
+            task.started_at = task.started_at or start_time
+            task.completed_at = None
+            task.error_message = None
         await db.commit()
         task_id = task.id
 
@@ -344,23 +322,76 @@ class BaseCollector(ABC):
 
         await self.resolve_url(db)
         await self._publish_task_update(force=True)
+        await self._log_collection_event(
+            "collector.run.started",
+            "Collector run started",
+            context={"status": "running", "task_id": task_id},
+        )
 
         try:
+            phase_started_at = perf_counter()
             await self.set_phase("fetching", message="正在拉取原始数据")
+            await self._log_collection_event(
+                "collector.phase.fetching.start",
+                "Collector fetch phase started",
+                context={"task_id": task_id, "snapshot_id": snapshot_id},
+            )
             raw_data = await self.fetch()
             task.total_records = len(raw_data)
             await db.commit()
             await self._publish_task_update(force=True)
+            await self._log_collection_event(
+                "collector.phase.fetching.success",
+                "Collector fetch phase completed",
+                context={
+                    "task_id": task_id,
+                    "raw_count": len(raw_data),
+                    "duration_ms": self._duration_ms(phase_started_at),
+                },
+            )
 
             if self.fail_on_empty and not raw_data:
                 raise RuntimeError(f"Collector {self.name} returned no data")
 
+            phase_started_at = perf_counter()
             await self.set_phase("transforming", message="正在转换采集数据")
+            await self._log_collection_event(
+                "collector.phase.transforming.start",
+                "Collector transform phase started",
+                context={"task_id": task_id, "raw_count": len(raw_data)},
+            )
             data = self.transform(raw_data)
+            await self._log_collection_event(
+                "collector.phase.transforming.success",
+                "Collector transform phase completed",
+                context={
+                    "task_id": task_id,
+                    "raw_count": len(raw_data),
+                    "transformed_count": len(data),
+                    "duration_ms": self._duration_ms(phase_started_at),
+                },
+            )
             snapshot_id = await self._create_snapshot(db, task_id, data, start_time)
 
+            phase_started_at = perf_counter()
             await self.set_phase("saving", message="正在保存采集数据")
+            await self._log_collection_event(
+                "collector.phase.saving.start",
+                "Collector save phase started",
+                context={"task_id": task_id, "snapshot_id": snapshot_id, "transformed_count": len(data)},
+            )
             records_count = await self._save_data(db, data, task_id=task_id, snapshot_id=snapshot_id)
+            await self._log_collection_event(
+                "collector.phase.saving.success",
+                "Collector save phase completed",
+                context={
+                    "task_id": task_id,
+                    "snapshot_id": snapshot_id,
+                    "saved_count": records_count,
+                    **self._last_save_summary,
+                    "duration_ms": self._duration_ms(phase_started_at),
+                },
+            )
 
             task.status = "success"
             task.phase = "completed"
@@ -374,10 +405,19 @@ class BaseCollector(ABC):
             task.completed_at = datetime.now(UTC)
             await db.commit()
             await self._publish_task_update(force=True)
-            await self._publish_earth_update(
-                action="collector_completed",
-                records_processed=records_count,
-                task_id=task_id,
+            await self._log_collection_event(
+                "collector.run.completed",
+                "Collector run completed",
+                context={
+                    "status": "success",
+                    "task_id": task_id,
+                    "snapshot_id": snapshot_id,
+                    "raw_count": len(raw_data),
+                    "transformed_count": len(data),
+                    "saved_count": records_count,
+                    **self._last_save_summary,
+                    "duration_ms": self._duration_ms(started_at),
+                },
             )
 
             return {
@@ -402,6 +442,17 @@ class BaseCollector(ABC):
                 )
             await db.commit()
             await self._publish_task_update(force=True)
+            await self._log_collection_event(
+                "collector.run.cancelled",
+                "Collector run cancelled",
+                level="warning",
+                context={
+                    "status": "cancelled",
+                    "task_id": task_id,
+                    "snapshot_id": snapshot_id,
+                    "duration_ms": self._duration_ms(started_at),
+                },
+            )
             raise
         except Exception as e:
             await db.rollback()
@@ -418,6 +469,20 @@ class BaseCollector(ABC):
                     snapshot.summary = {"error": str(e)}
             await db.commit()
             await self._publish_task_update(force=True)
+            await self._log_collection_event(
+                "collector.run.failed",
+                "Collector run failed",
+                level="error",
+                context=exception_context(
+                    e,
+                    {
+                        "status": "failed",
+                        "task_id": task_id,
+                        "snapshot_id": snapshot_id,
+                        "duration_ms": self._duration_ms(started_at),
+                    },
+                ),
+            )
 
             return {
                 "status": "failed",
@@ -438,6 +503,7 @@ class BaseCollector(ABC):
         from app.models.data_snapshot import DataSnapshot
 
         if not data:
+            self._last_save_summary = {"created": 0, "updated": 0, "unchanged": 0, "deleted": 0}
             if snapshot_id is not None:
                 snapshot = await db.get(DataSnapshot, snapshot_id)
                 if snapshot:
@@ -584,12 +650,51 @@ class BaseCollector(ABC):
                     "unchanged": unchanged_count,
                     "deleted": len(deleted_keys),
                 }
+            self._last_save_summary = {
+                "created": created_count,
+                "updated": updated_count,
+                "unchanged": unchanged_count,
+                "deleted": len(deleted_keys),
+            }
+        else:
+            self._last_save_summary = {
+                "created": created_count,
+                "updated": updated_count,
+                "unchanged": unchanged_count,
+                "deleted": 0,
+            }
 
         await db.commit()
-        invalidate_earth_layer_cache_for_source(self.name)
         await self.update_progress(len(data), force=True)
         return records_added
 
+    @staticmethod
+    def _duration_ms(started_at: float) -> int:
+        return int((perf_counter() - started_at) * 1000)
+
+    async def _log_collection_event(
+        self,
+        event: str,
+        message: str,
+        *,
+        level: str = "info",
+        context: Dict[str, Any] | None = None,
+    ) -> None:
+        await emit_business_log(
+            logger,
+            event=event,
+            message=message,
+            category="collector",
+            level=level,
+            service="collector",
+            module=__name__,
+            context={
+                "collector_name": self.name,
+                "datasource_id": getattr(self, "_datasource_id", None),
+                **(context or {}),
+            },
+        )
+
     async def save(self, db: AsyncSession, data: List[Dict[str, Any]]) -> int:
         """Save data to database (legacy method, use _save_data instead)"""
         return await self._save_data(db, data)
@@ -602,10 +707,65 @@ class HTTPCollector(BaseCollector):
     headers: Dict[str, str] = {}
 
     async def fetch(self) -> List[Dict[str, Any]]:
+        started_at = perf_counter()
+        request_host = urlparse(self.base_url).netloc
+        await emit_business_log(
+            logger,
+            event="collector.http.fetch.start",
+            message="Collector HTTP request started",
+            category="collector",
+            service="collector",
+            module=__name__,
+            context={
+                "collector_name": self.name,
+                "datasource_id": getattr(self, "_datasource_id", None),
+                "url_host": request_host,
+            },
+        )
         async with httpx.AsyncClient(timeout=60.0) as client:
-            response = await client.get(self.base_url, headers=self.headers)
-            response.raise_for_status()
-            return self.parse_response(response.json())
+            try:
+                response = await client.get(self.base_url, headers=self.headers)
+                response.raise_for_status()
+                payload = response.json()
+                parsed = self.parse_response(payload)
+                await emit_business_log(
+                    logger,
+                    event="collector.http.fetch.success",
+                    message="Collector HTTP request completed",
+                    category="collector",
+                    service="collector",
+                    module=__name__,
+                    context={
+                        "collector_name": self.name,
+                        "datasource_id": getattr(self, "_datasource_id", None),
+                        "url_host": request_host,
+                        "status_code": response.status_code,
+                        "response_bytes": len(response.content or b""),
+                        "parsed_count": len(parsed),
+                        "duration_ms": BaseCollector._duration_ms(started_at),
+                    },
+                )
+                return parsed
+            except Exception as exc:
+                await emit_business_log(
+                    logger,
+                    event="collector.http.fetch.failed",
+                    message="Collector HTTP request failed",
+                    category="collector",
+                    level="error",
+                    service="collector",
+                    module=__name__,
+                    context=exception_context(
+                        exc,
+                        {
+                            "collector_name": self.name,
+                            "datasource_id": getattr(self, "_datasource_id", None),
+                            "url_host": request_host,
+                            "duration_ms": BaseCollector._duration_ms(started_at),
+                        },
+                    ),
+                )
+                raise
 
     @abstractmethod
     def parse_response(self, response: Dict[str, Any]) -> List[Dict[str, Any]]:

backend/app/services/collectors/celestrak.py

@@ -1,15 +1,28 @@
-"""CelesTrak TLE Collector
+"""CelesTrak TLE Collector.
 
-Collects satellite TLE (Two-Line Element) data from CelesTrak.org.
-Free, no authentication required.
+Collects the full active satellite GP element set from CelesTrak.
 """
 
+import asyncio
 import json
-from typing import Dict, Any, List
+from pathlib import Path
+from time import perf_counter
+from typing import Any, Dict, List
+from urllib.parse import urlencode, urlparse
+
 import httpx
 
+from app.core.logging import get_logger
 from app.core.satellite_tle import build_tle_lines_from_elements
+from app.services.business_logs import emit_business_log, exception_context
 from app.services.collectors.base import BaseCollector
+from app.services.collectors.downloads import ResumableFileDownloader
+
+
+logger = get_logger(__name__, service="collector")
+ACTIVE_GROUP = "active"
+FETCH_RETRY_ATTEMPTS = 3
+FETCH_RETRY_BASE_DELAY_SECONDS = 0.8
 
 
 class CelesTrakTLECollector(BaseCollector):
@@ -18,55 +31,179 @@ class CelesTrakTLECollector(BaseCollector):
     module = "L3"
     frequency_hours = 24
     data_type = "satellite_tle"
+    _downloader = ResumableFileDownloader(
+        cache_namespace="celestrak",
+        default_accept="application/json",
+    )
 
     @property
     def base_url(self) -> str:
         return self._resolved_url or ""
 
+    def _active_url(self) -> str:
+        if not self.base_url:
+            raise RuntimeError("CelesTrak base URL is not configured")
+        return f"{self.base_url}?{urlencode({'GROUP': ACTIVE_GROUP, 'FORMAT': 'json'})}"
+
     async def fetch(self) -> List[Dict[str, Any]]:
-        satellite_groups = [
-            "starlink",
-            "gps-ops",
-            "galileo",
-            "glonass",
-            "beidou",
-            "leo",
-            "geo",
-            "iridium-next",
-        ]
+        url = self._active_url()
+        last_error: Exception | None = None
 
-        all_satellites = []
-
-        async with httpx.AsyncClient(timeout=120.0) as client:
-            for group in satellite_groups:
+        async with httpx.AsyncClient(timeout=180.0, follow_redirects=True) as client:
+            for attempt in range(1, FETCH_RETRY_ATTEMPTS + 1):
+                started_at = perf_counter()
                 try:
-                    url = f"{self.base_url}?GROUP={group}&FORMAT=json"
-                    response = await client.get(url)
+                    await emit_business_log(
+                        logger,
+                        event="collector.celestrak.download.start",
+                        message="CelesTrak active satellite download started",
+                        category="collector",
+                        service="collector",
+                        module=__name__,
+                        context={
+                            "collector_name": self.name,
+                            "datasource_id": getattr(self, "_datasource_id", None),
+                            "group": ACTIVE_GROUP,
+                            "attempt": attempt,
+                            "url_host": urlparse(url).netloc,
+                        },
+                    )
+                    body_path = await self._downloader.download_file(
+                        client,
+                        url,
+                        extension=".json",
+                        accept="application/json",
+                        progress_callback=self._report_download_progress,
+                        validate_existing=self._validate_json_file,
+                    )
+                    try:
+                        data = self._load_active_payload(body_path)
+                    except RuntimeError as exc:
+                        await self._log_parse_failure(exc)
+                        raise
+                    for item in data:
+                        item["_celestrak_query_group"] = ACTIVE_GROUP
+                        item["_celestrak_source_url"] = url
+                    await emit_business_log(
+                        logger,
+                        event="collector.celestrak.download.success",
+                        message="CelesTrak active satellite download completed",
+                        category="collector",
+                        service="collector",
+                        module=__name__,
+                        context={
+                            "collector_name": self.name,
+                            "datasource_id": getattr(self, "_datasource_id", None),
+                            "group": ACTIVE_GROUP,
+                            "attempt": attempt,
+                            "record_count": len(data),
+                            "duration_ms": self._duration_ms(started_at),
+                        },
+                    )
+                    return data
+                except Exception as exc:
+                    last_error = exc
+                    is_final_attempt = attempt >= FETCH_RETRY_ATTEMPTS
+                    await emit_business_log(
+                        logger,
+                        event=(
+                            "collector.celestrak.download.failed"
+                            if is_final_attempt
+                            else "collector.celestrak.download.retry"
+                        ),
+                        message=(
+                            "CelesTrak active satellite download failed"
+                            if is_final_attempt
+                            else "CelesTrak active satellite download will retry"
+                        ),
+                        category="collector",
+                        level="error" if is_final_attempt else "warning",
+                        service="collector",
+                        module=__name__,
+                        context=exception_context(
+                            exc,
+                            {
+                                "collector_name": self.name,
+                                "datasource_id": getattr(self, "_datasource_id", None),
+                                "group": ACTIVE_GROUP,
+                                "attempt": attempt,
+                                "duration_ms": self._duration_ms(started_at),
+                            },
+                        ),
+                    )
+                    if not is_final_attempt:
+                        await asyncio.sleep(FETCH_RETRY_BASE_DELAY_SECONDS * attempt)
 
-                    if response.status_code == 200:
-                        data = response.json()
-                        if isinstance(data, list):
-                            for item in data:
-                                if isinstance(item, dict):
-                                    item["_celestrak_group"] = group
-                            all_satellites.extend(data)
-                            print(f"CelesTrak: Fetched {len(data)} satellites from group '{group}'")
-                except Exception as e:
-                    print(f"CelesTrak: Error fetching group '{group}': {e}")
+        raise RuntimeError(f"CelesTrak active satellite download failed after retries: {last_error}")
 
-        if not all_satellites:
-            return self._get_sample_data()
+    @staticmethod
+    def _duration_ms(started_at: float) -> int:
+        return int((perf_counter() - started_at) * 1000)
 
-        print(f"CelesTrak: Total satellites fetched: {len(all_satellites)}")
+    async def _report_download_progress(self, downloaded: int, total: int | None) -> None:
+        if total and total > 0:
+            await self.update_phase_progress(
+                current=min(downloaded, total),
+                total=total,
+                unit="bytes",
+                message=f"正在下载 CelesTrak active 卫星数据 {downloaded}/{total} bytes",
+                commit=True,
+            )
 
-        # Return raw data - base.run() will call transform()
-        return all_satellites
+    @staticmethod
+    def _validate_json_file(path: Path) -> bool:
+        try:
+            data = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError, UnicodeDecodeError):
+            return False
+        return isinstance(data, list)
+
+    async def _log_parse_failure(self, exc: Exception) -> None:
+        await emit_business_log(
+            logger,
+            event="collector.celestrak.parse.failed",
+            message="CelesTrak active satellite JSON parsing failed",
+            category="collector",
+            level="error",
+            service="collector",
+            module=__name__,
+            context=exception_context(
+                exc,
+                {
+                    "collector_name": self.name,
+                    "datasource_id": getattr(self, "_datasource_id", None),
+                    "group": ACTIVE_GROUP,
+                },
+            ),
+        )
+
+    def _load_active_payload(self, path: Path) -> List[Dict[str, Any]]:
+        try:
+            raw = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError, UnicodeDecodeError) as exc:
+            raise RuntimeError(f"CelesTrak active payload is not valid JSON: {exc}") from exc
+        if not isinstance(raw, list):
+            raise RuntimeError("CelesTrak active payload is not a JSON array")
+
+        records: List[Dict[str, Any]] = []
+        invalid_count = 0
+        for item in raw:
+            if isinstance(item, dict) and item.get("NORAD_CAT_ID") is not None:
+                records.append(item)
+            else:
+                invalid_count += 1
+        if invalid_count:
+            raise RuntimeError(f"CelesTrak active payload contains {invalid_count} invalid record(s)")
+        if not records:
+            raise RuntimeError("CelesTrak active payload contains no satellite records")
+        return records
 
     def transform(self, raw_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
         transformed = []
         for item in raw_data:
+            norad_cat_id = item.get("NORAD_CAT_ID")
             tle_line1, tle_line2 = build_tle_lines_from_elements(
-                norad_cat_id=item.get("NORAD_CAT_ID"),
+                norad_cat_id=norad_cat_id,
                 epoch=item.get("EPOCH"),
                 inclination=item.get("INCLINATION"),
                 raan=item.get("RA_OF_ASC_NODE"),
@@ -75,14 +212,18 @@ class CelesTrakTLECollector(BaseCollector):
                 mean_anomaly=item.get("MEAN_ANOMALY"),
                 mean_motion=item.get("MEAN_MOTION"),
             )
+            constellation_group = self._infer_constellation_group(item)
 
             transformed.append(
                 {
+                    "source_id": str(norad_cat_id),
                     "name": item.get("OBJECT_NAME", "Unknown"),
                     "reference_date": item.get("EPOCH", ""),
                     "metadata": {
-                        "constellation_group": item.get("_celestrak_group"),
-                        "norad_cat_id": item.get("NORAD_CAT_ID"),
+                        "constellation_group": constellation_group,
+                        "celestrak_query_group": item.get("_celestrak_query_group") or ACTIVE_GROUP,
+                        "celestrak_source_url": item.get("_celestrak_source_url"),
+                        "norad_cat_id": norad_cat_id,
                         "international_designator": item.get("OBJECT_ID"),
                         "epoch": item.get("EPOCH"),
                         "mean_motion": item.get("MEAN_MOTION"),
@@ -105,6 +246,19 @@ class CelesTrakTLECollector(BaseCollector):
             )
         return transformed
 
+    @staticmethod
+    def _infer_constellation_group(item: Dict[str, Any]) -> str | None:
+        explicit_group = str(item.get("_celestrak_group") or "").strip().lower()
+        if explicit_group and explicit_group != ACTIVE_GROUP:
+            return explicit_group
+
+        name = str(item.get("OBJECT_NAME") or "").strip().upper()
+        if name.startswith("STARLINK"):
+            return "starlink"
+        if name.startswith("IRIDIUM"):
+            return "iridium-next"
+        return None
+
     def _get_sample_data(self) -> List[Dict[str, Any]]:
         return [
             {

backend/app/services/collectors/telegeography.py

@@ -163,32 +163,64 @@ class TeleGeographyLandingPointCollector(BaseCollector):
     data_type = "landing_point"
 
     async def fetch(self) -> List[Dict[str, Any]]:
-        """Fetch landing point data from GitHub mirror"""
-        url = self._resolved_url or ""
+        """Fetch landing point data, falling back when the old mirror disappears."""
+        config = get_data_sources_config()
+        sources = [
+            self._resolved_url or "",
+            str(config.get_yaml_value("telegeography.landing_point_url") or ""),
+            str(config.get_yaml_value("arcgis.landing_point_url") or ""),
+        ]
 
-        async with httpx.AsyncClient(timeout=60.0) as client:
-            response = await client.get(url)
-            response.raise_for_status()
-            return self.parse_response(response.json())
+        last_error: Exception | None = None
+        async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
+            for url in dict.fromkeys(source for source in sources if source):
+                try:
+                    params = (
+                        {"where": "1=1", "outFields": "*", "returnGeometry": "true", "f": "geojson"}
+                        if "FeatureServer" in url or url.endswith("/query")
+                        else None
+                    )
+                    response = await client.get(url, params=params)
+                    response.raise_for_status()
+                    records = self.parse_response(response.json())
+                    if records:
+                        return records
+                except Exception as exc:
+                    last_error = exc
+                    continue
 
-    def parse_response(self, data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        if last_error:
+            raise last_error
+        return self._get_sample_data()
+
+    def parse_response(self, data: List[Dict[str, Any]] | Dict[str, Any]) -> List[Dict[str, Any]]:
         """Parse landing point data"""
         result = []
+        items = data.get("features", []) if isinstance(data, dict) else data
 
-        for item in data:
+        for item in items:
+            props = item.get("properties", {}) if isinstance(item, dict) else {}
+            geometry = item.get("geometry", {}) if isinstance(item, dict) else {}
+            source = props or item
+            coords = geometry.get("coordinates", []) if isinstance(geometry, dict) else []
+            longitude = coords[0] if len(coords) > 0 else source.get("longitude")
+            latitude = coords[1] if len(coords) > 1 else source.get("latitude")
+            source_id = source.get("id") or source.get("OBJECTID") or source.get("city_id") or ""
             try:
                 entry = {
-                    "source_id": f"telegeo_lp_{item.get('id', '')}",
-                    "name": item.get("name", "Unknown"),
-                    "country": item.get("country", "Unknown"),
-                    "city": item.get("city", item.get("name", "")),
-                    "latitude": str(item.get("latitude", "")),
-                    "longitude": str(item.get("longitude", "")),
+                    "source_id": f"telegeo_lp_{source_id}",
+                    "name": source.get("name", source.get("Name", "Unknown")),
+                    "country": source.get("country", "Unknown"),
+                    "city": source.get("city", source.get("Name", source.get("name", ""))),
+                    "latitude": str(latitude or ""),
+                    "longitude": str(longitude or ""),
                     "value": "",
                     "unit": "",
                     "metadata": {
-                        "cable_count": len(item.get("cables", [])),
-                        "url": item.get("url"),
+                        "cable_count": len(source.get("cables", [])),
+                        "url": source.get("url"),
+                        "objectid": source.get("OBJECTID"),
+                        "city_id": source.get("city_id"),
                     },
                     "reference_date": datetime.now(UTC).strftime("%Y-%m-%d"),
                 }

backend/app/services/compute_center_locations.py

@@ -11,7 +11,7 @@ startup, so it must stay local and deterministic.
 
 For the full design and the reason behind the abstraction (compute centers,
 BGP collectors, BGP events, and future entities all share one pipeline),
-see ``docs/plans/location-resolver-shared-pipeline-plan.md``.
+see ``docs/technical/zh/location-pipeline-development.md``.
 
 The ``ComputeCenterLocation`` dataclass and the public function signatures are
 preserved verbatim so existing callers and tests do not need to change.

backend/app/services/data_jobs.py

@@ -0,0 +1,597 @@
+"""Kafka-ready datasource job queue backed by PostgreSQL for v1."""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import UTC, datetime, timedelta
+from uuid import uuid4
+from typing import Any
+
+from sqlalchemy import bindparam, select, text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.cache import cache
+from app.core.config import settings
+from app.core.logging import get_logger
+from app.core.time import to_iso8601_utc
+from app.core.websocket.broadcaster import broadcaster
+from app.db.session import async_session_factory
+from app.models.collected_data import CollectedData
+from app.models.data_snapshot import DataSnapshot
+from app.models.datasource import DataSource
+from app.models.task import CollectionTask
+from app.services.collectors.registry import collector_registry
+from app.services.datasource_connectivity import (
+    build_builtin_connectivity_checksum,
+    get_builtin_effective_candidate,
+    save_connectivity_success,
+)
+from app.services.earth_layer_adapters import (
+    clear_derived_datasource_data,
+    get_earth_refresh_strategy_for_change,
+    get_earth_update_layers_for_source,
+)
+from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
+from app.services.scheduler import sync_datasource_job
+
+logger = get_logger(__name__)
+
+JOB_TYPE_COLLECT = "collect"
+JOB_TYPE_CLEAR_DATA = "clear_data"
+JOB_TYPE_CLEAR_CACHE = "clear_cache"
+JOB_TYPE_EARTH_REFRESH = "earth_refresh"
+
+JOB_STATUS_QUEUED = "queued"
+JOB_STATUS_RUNNING = "running"
+JOB_STATUS_CANCELLING = "cancelling"
+JOB_STATUS_SUCCESS = "success"
+JOB_STATUS_FAILED = "failed"
+JOB_STATUS_CANCELLED = "cancelled"
+
+ACTIVE_JOB_STATUSES = (JOB_STATUS_QUEUED, JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING)
+TERMINAL_JOB_STATUSES = (JOB_STATUS_SUCCESS, JOB_STATUS_FAILED, JOB_STATUS_CANCELLED)
+DATA_WRITE_JOB_TYPES = (JOB_TYPE_COLLECT, JOB_TYPE_CLEAR_DATA, JOB_TYPE_CLEAR_CACHE)
+SOURCE_LOCK_JOB_STATUSES = (JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING)
+QUEUE_POLL_SECONDS = 0.35
+JOB_STALE_LOCK_MINUTES = 90
+DEFAULT_WORKER_CONCURRENCY = 2
+
+RUNNING_DATA_JOB_TASKS: dict[int, asyncio.Task[Any]] = {}
+
+
+def _utcnow() -> datetime:
+    return datetime.now(UTC)
+
+
+def _job_worker_id() -> str:
+    return f"{settings.PROJECT_NAME}:data-job-worker:{uuid4().hex[:8]}"
+
+
+def is_terminal_job_status(status: str | None) -> bool:
+    return status in TERMINAL_JOB_STATUSES
+
+
+async def enqueue_datasource_job(
+    db: AsyncSession,
+    datasource: DataSource,
+    task_type: str,
+    *,
+    payload: dict[str, Any] | None = None,
+    rollback_policy: str = "keep_committed_batches",
+    dedupe_key: str | None = None,
+) -> CollectionTask:
+    if dedupe_key:
+        existing = await _get_active_job_by_dedupe_key(db, dedupe_key)
+        if existing is not None:
+            return existing
+
+    task = CollectionTask(
+        datasource_id=datasource.id,
+        source=datasource.source,
+        task_type=task_type,
+        status=JOB_STATUS_QUEUED,
+        phase="queued",
+        phase_message="任务已进入队列",
+        payload=payload or {},
+        rollback_policy=rollback_policy,
+        dedupe_key=dedupe_key,
+    )
+    db.add(task)
+    await db.commit()
+    await db.refresh(task)
+    await _broadcast_task_update(task)
+    return task
+
+
+async def enqueue_earth_refresh_job(
+    db: AsyncSession,
+    *,
+    source: str,
+    payload: dict[str, Any] | None = None,
+) -> CollectionTask | None:
+    layers = list((payload or {}).get("layers") or get_earth_update_layers_for_source(source))
+    if not layers:
+        return None
+
+    datasource = await _get_or_create_virtual_datasource(db, source)
+    refresh_payload = {
+        "source": source,
+        "layers": layers,
+        "refresh_strategy": (payload or {}).get("refresh_strategy")
+        or get_earth_refresh_strategy_for_change((payload or {}).get("table"), source)
+        or "clear_then_reload",
+        **(payload or {}),
+    }
+    return await enqueue_datasource_job(
+        db,
+        datasource,
+        JOB_TYPE_EARTH_REFRESH,
+        payload=refresh_payload,
+        dedupe_key=f"earth_refresh:{source}",
+    )
+
+
+async def enqueue_earth_refresh_from_update(payload: dict[str, Any]) -> None:
+    source = str(payload.get("source") or "").strip()
+    if not source:
+        return
+    async with async_session_factory() as db:
+        await enqueue_earth_refresh_job(db, source=source, payload=payload)
+
+
+async def request_cancel_datasource_task(
+    db: AsyncSession,
+    task: CollectionTask,
+    *,
+    reason: str = "cancelled_by_operator",
+) -> CollectionTask:
+    if is_terminal_job_status(task.status):
+        return task
+
+    running_task = RUNNING_DATA_JOB_TASKS.get(task.id)
+    if task.status == JOB_STATUS_QUEUED or (
+        running_task is None
+        and (
+            task.status == JOB_STATUS_CANCELLING
+            or (task.status == JOB_STATUS_RUNNING and task.task_type != JOB_TYPE_COLLECT)
+        )
+    ):
+        return await _cancel_task_without_runner(db, task, reason=reason)
+
+    task.status = JOB_STATUS_CANCELLING
+    task.phase = JOB_STATUS_CANCELLING
+    task.phase_message = "正在停止任务"
+    task.requested_cancel_at = _utcnow()
+    task.cancel_reason = reason
+    await db.commit()
+    await db.refresh(task)
+
+    if running_task is not None and not running_task.done():
+        running_task.cancel()
+
+    await _broadcast_task_update(task)
+    return task
+
+
+async def _cancel_task_without_runner(
+    db: AsyncSession,
+    task: CollectionTask,
+    *,
+    reason: str,
+) -> CollectionTask:
+    if task.task_type == JOB_TYPE_COLLECT:
+        await db.execute(CollectedData.__table__.delete().where(CollectedData.task_id == task.id))
+        snapshot_result = await db.execute(select(DataSnapshot).where(DataSnapshot.task_id == task.id))
+        for snapshot in snapshot_result.scalars().all():
+            snapshot.status = JOB_STATUS_CANCELLED
+            snapshot.completed_at = _utcnow()
+            snapshot.error_message = "Cancelled after operator stop request; no active worker handle remained"
+        datasource = await db.get(DataSource, task.datasource_id)
+        if datasource is not None:
+            datasource.last_status = JOB_STATUS_CANCELLED
+
+    task.status = JOB_STATUS_CANCELLED
+    task.phase = JOB_STATUS_CANCELLED
+    task.phase_message = "任务已停止"
+    task.completed_at = _utcnow()
+    task.requested_cancel_at = task.requested_cancel_at or _utcnow()
+    task.cancel_reason = reason
+    await db.commit()
+    await db.refresh(task)
+    await _broadcast_task_update(task)
+    return task
+
+
+async def get_active_datasource_job(
+    db: AsyncSession,
+    datasource_id: int,
+    *,
+    task_types: tuple[str, ...] = DATA_WRITE_JOB_TYPES,
+) -> CollectionTask | None:
+    result = await db.execute(
+        select(CollectionTask)
+        .where(CollectionTask.datasource_id == datasource_id)
+        .where(CollectionTask.task_type.in_(task_types))
+        .where(CollectionTask.status.in_(ACTIVE_JOB_STATUSES))
+        .order_by(CollectionTask.created_at.desc().nullslast(), CollectionTask.id.desc())
+        .limit(1)
+    )
+    return result.scalar_one_or_none()
+
+
+async def _get_active_job_by_dedupe_key(db: AsyncSession, dedupe_key: str) -> CollectionTask | None:
+    result = await db.execute(
+        select(CollectionTask)
+        .where(CollectionTask.dedupe_key == dedupe_key)
+        .where(CollectionTask.status.in_(ACTIVE_JOB_STATUSES))
+        .order_by(CollectionTask.created_at.desc().nullslast(), CollectionTask.id.desc())
+        .limit(1)
+    )
+    return result.scalar_one_or_none()
+
+
+async def _get_or_create_virtual_datasource(db: AsyncSession, source: str) -> DataSource:
+    result = await db.execute(select(DataSource).where(DataSource.source == source))
+    datasource = result.scalar_one_or_none()
+    if datasource is not None:
+        return datasource
+
+    datasource = DataSource(
+        name=f"Earth refresh: {source}",
+        source=source,
+        module="SYS",
+        collector_class="EarthRefreshJob",
+        is_active=True,
+    )
+    db.add(datasource)
+    await db.commit()
+    await db.refresh(datasource)
+    return datasource
+
+
+async def _broadcast_task_update(task: CollectionTask) -> None:
+    await broadcaster.broadcast_datasource_task_update(
+        {
+            "datasource_id": task.datasource_id,
+            "collector_name": task.source,
+            "task_id": task.id,
+            "task_type": task.task_type,
+            "status": task.status,
+            "phase": task.phase,
+            "phase_progress": task.phase_progress,
+            "phase_message": task.phase_message,
+            "phase_current": task.phase_current,
+            "phase_total": task.phase_total,
+            "phase_unit": task.phase_unit,
+            "progress": task.progress,
+            "records_processed": task.records_processed,
+            "total_records": task.total_records,
+            "started_at": to_iso8601_utc(task.started_at),
+            "completed_at": to_iso8601_utc(task.completed_at),
+            "requested_cancel_at": to_iso8601_utc(task.requested_cancel_at),
+            "error_message": task.error_message,
+        }
+    )
+
+
+class DataJobWorker:
+    def __init__(self, *, concurrency: int = DEFAULT_WORKER_CONCURRENCY) -> None:
+        self.worker_id = _job_worker_id()
+        self.concurrency = max(1, concurrency)
+        self._task: asyncio.Task[None] | None = None
+        self._stop_event: asyncio.Event | None = None
+        self._running: set[asyncio.Task[Any]] = set()
+
+    def start(self) -> None:
+        if self._task and not self._task.done():
+            return
+        self._stop_event = asyncio.Event()
+        self._task = asyncio.create_task(self._run(), name="data-job-worker")
+
+    async def stop(self) -> None:
+        if self._stop_event:
+            self._stop_event.set()
+        for task in list(self._running):
+            task.cancel()
+        if self._task:
+            await asyncio.gather(self._task, return_exceptions=True)
+        if self._running:
+            await asyncio.gather(*self._running, return_exceptions=True)
+
+    async def _run(self) -> None:
+        assert self._stop_event is not None
+        await self._recover_stale_running_jobs()
+        while not self._stop_event.is_set():
+            self._running = {task for task in self._running if not task.done()}
+            if len(self._running) >= self.concurrency:
+                await asyncio.sleep(QUEUE_POLL_SECONDS)
+                continue
+
+            task_id = await self._claim_next_job()
+            if task_id is None:
+                await asyncio.sleep(QUEUE_POLL_SECONDS)
+                continue
+
+            runner = asyncio.create_task(self._run_claimed_job(task_id), name=f"data-job:{task_id}")
+            self._running.add(runner)
+
+    async def _recover_stale_running_jobs(self) -> None:
+        cutoff = _utcnow() - timedelta(minutes=JOB_STALE_LOCK_MINUTES)
+        async with async_session_factory() as db:
+            result = await db.execute(
+                select(CollectionTask)
+                .where(CollectionTask.status.in_((JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING)))
+                .where(CollectionTask.locked_at.is_not(None))
+                .where(CollectionTask.locked_at < cutoff)
+            )
+            stale_jobs = list(result.scalars().all())
+            for job in stale_jobs:
+                job.status = JOB_STATUS_FAILED
+                job.phase = JOB_STATUS_FAILED
+                job.completed_at = _utcnow()
+                job.error_message = "Marked failed after stale data job lock timeout"
+            if stale_jobs:
+                await db.commit()
+
+    async def _claim_next_job(self) -> int | None:
+        async with async_session_factory() as db:
+            row = await db.execute(
+                text(
+                    """
+                    SELECT queued.id
+                    FROM collection_tasks AS queued
+                    WHERE queued.status = :queued_status
+                      AND NOT EXISTS (
+                        SELECT 1
+                        FROM collection_tasks AS active
+                        WHERE active.source = queued.source
+                          AND active.id <> queued.id
+                          AND active.status IN :active_statuses
+                      )
+                    ORDER BY queued.created_at ASC NULLS FIRST, queued.id ASC
+                    LIMIT 1
+                    FOR UPDATE SKIP LOCKED
+                    """
+                ).bindparams(bindparam("active_statuses", expanding=True)),
+                {
+                    "queued_status": JOB_STATUS_QUEUED,
+                    "active_statuses": SOURCE_LOCK_JOB_STATUSES,
+                },
+            )
+            task_id = row.scalar_one_or_none()
+            if task_id is None:
+                return None
+
+            task = await db.get(CollectionTask, int(task_id))
+            if task is None:
+                return None
+            task.status = JOB_STATUS_RUNNING
+            task.phase = "starting"
+            task.phase_message = "任务开始执行"
+            task.started_at = task.started_at or _utcnow()
+            task.worker_id = self.worker_id
+            task.locked_at = _utcnow()
+            await db.commit()
+            await _broadcast_task_update(task)
+            return int(task_id)
+
+    async def _run_claimed_job(self, task_id: int) -> None:
+        current_task = asyncio.current_task()
+        if current_task is not None:
+            RUNNING_DATA_JOB_TASKS[task_id] = current_task
+        try:
+            async with async_session_factory() as db:
+                task = await db.get(CollectionTask, task_id)
+                if task is None:
+                    return
+                await self._execute_job(db, task)
+        except asyncio.CancelledError:
+            async with async_session_factory() as db:
+                task = await db.get(CollectionTask, task_id)
+                if task is not None and not is_terminal_job_status(task.status):
+                    task.status = JOB_STATUS_CANCELLED
+                    task.phase = JOB_STATUS_CANCELLED
+                    task.phase_message = "任务已停止"
+                    task.completed_at = _utcnow()
+                    await db.commit()
+                    await _broadcast_task_update(task)
+            raise
+        except Exception as exc:
+            logger.exception_event(
+                "Data job failed",
+                event="data_jobs.job_failed",
+                context={"task_id": task_id, "error": str(exc)},
+            )
+            async with async_session_factory() as db:
+                task = await db.get(CollectionTask, task_id)
+                if task is not None:
+                    task.status = JOB_STATUS_FAILED
+                    task.phase = JOB_STATUS_FAILED
+                    task.phase_message = str(exc)
+                    task.error_message = str(exc)
+                    task.completed_at = _utcnow()
+                    await db.commit()
+                    await _broadcast_task_update(task)
+        finally:
+            RUNNING_DATA_JOB_TASKS.pop(task_id, None)
+
+    async def _execute_job(self, db: AsyncSession, task: CollectionTask) -> None:
+        if task.task_type == JOB_TYPE_COLLECT:
+            await _run_collect_job(db, task)
+        elif task.task_type == JOB_TYPE_CLEAR_DATA:
+            await _run_clear_data_job(db, task)
+        elif task.task_type == JOB_TYPE_CLEAR_CACHE:
+            await _run_clear_cache_job(db, task)
+        elif task.task_type == JOB_TYPE_EARTH_REFRESH:
+            await _run_earth_refresh_job(db, task)
+        else:
+            raise RuntimeError(f"Unsupported data job type: {task.task_type}")
+
+
+async def _run_collect_job(db: AsyncSession, task: CollectionTask) -> None:
+    datasource = await db.get(DataSource, task.datasource_id)
+    if datasource is None:
+        raise RuntimeError("Data source not found")
+
+    collector = collector_registry.get(datasource.source)
+    if collector is None:
+        raise RuntimeError(f"Collector '{datasource.source}' not found")
+    if not datasource.is_active:
+        raise RuntimeError("Data source is disabled")
+
+    collector._datasource_id = datasource.id
+    collector._current_task = task
+    collector._db_session = db
+    result = await collector.run(db)
+
+    datasource.last_run_at = _utcnow()
+    datasource.last_status = result.get("status")
+    if datasource.last_status == JOB_STATUS_SUCCESS:
+        effective_candidate = await get_builtin_effective_candidate(db, datasource.source)
+        checksum, _credential_context = await build_builtin_connectivity_checksum(
+            datasource.source,
+            effective_candidate["endpoint"],
+            effective_candidate["auth_type"],
+            effective_candidate["headers"],
+            effective_candidate["config"],
+            db,
+        )
+        await save_connectivity_success(
+            db,
+            datasource.source,
+            checksum,
+            {"status_code": None},
+            connected_by="collection",
+        )
+    await db.commit()
+    await sync_datasource_job(datasource.id)
+
+
+async def _run_clear_data_job(db: AsyncSession, task: CollectionTask) -> None:
+    source = str(task.source or (task.payload or {}).get("source") or "").strip()
+    if not source:
+        raise RuntimeError("Clear data job has no source")
+
+    task.phase = "clearing_data"
+    task.phase_message = "正在删除数据库数据"
+    await db.commit()
+    await _broadcast_task_update(task)
+
+    count_result = await db.execute(
+        select(CollectedData.id).where(CollectedData.source == source)
+    )
+    collected_ids = [row[0] for row in count_result.all()]
+    derived_deleted_counts = await clear_derived_datasource_data(db, source)
+    if collected_ids:
+        await db.execute(CollectedData.__table__.delete().where(CollectedData.id.in_(collected_ids)))
+    deleted_count = len(collected_ids)
+    derived_deleted_count = sum(derived_deleted_counts.values())
+
+    task.records_processed = deleted_count + derived_deleted_count
+    task.total_records = task.records_processed
+    task.progress = 100.0
+    task.phase_progress = 100.0
+    task.phase_current = task.records_processed
+    task.phase_total = task.records_processed
+    task.phase_unit = "records"
+    task.payload = {
+        **(task.payload or {}),
+        "deleted_count": deleted_count,
+        "derived_deleted_count": derived_deleted_count,
+        "derived_deleted_counts": derived_deleted_counts,
+    }
+    task.status = JOB_STATUS_SUCCESS
+    task.phase = "completed"
+    task.phase_message = "数据库数据已清理"
+    task.completed_at = _utcnow()
+    await db.commit()
+    await _broadcast_task_update(task)
+
+
+async def _run_clear_cache_job(db: AsyncSession, task: CollectionTask) -> None:
+    source = str(task.source or (task.payload or {}).get("source") or "").strip()
+    if not source:
+        raise RuntimeError("Clear cache job has no source")
+
+    earth_deleted_count = invalidate_earth_layer_cache_for_source(source)
+    dashboard_deleted_count = int(cache.delete("dashboard:stats")) + int(cache.delete("dashboard:summary"))
+    deleted_count = earth_deleted_count + dashboard_deleted_count
+
+    task.records_processed = deleted_count
+    task.total_records = deleted_count
+    task.progress = 100.0
+    task.phase_progress = 100.0
+    task.phase = "completed"
+    task.phase_message = "缓存已清理"
+    task.payload = {
+        **(task.payload or {}),
+        "earth_layer_deleted_count": earth_deleted_count,
+        "dashboard_deleted_count": dashboard_deleted_count,
+    }
+    task.status = JOB_STATUS_SUCCESS
+    task.completed_at = _utcnow()
+    await db.commit()
+    await _broadcast_task_update(task)
+    await enqueue_earth_refresh_job(db, source=source, payload={"operation": "CACHE_INVALIDATED"})
+
+
+async def _run_earth_refresh_job(db: AsyncSession, task: CollectionTask) -> None:
+    payload = task.payload or {}
+    source = str(payload.get("source") or task.source or "").strip()
+    layers = list(payload.get("layers") or get_earth_update_layers_for_source(source))
+    if not source or not layers:
+        task.status = JOB_STATUS_SUCCESS
+        task.phase = "completed"
+        task.phase_message = "没有需要刷新的 Earth 图层"
+        task.completed_at = _utcnow()
+        await db.commit()
+        await _broadcast_task_update(task)
+        return
+
+    deleted_cache_entries = invalidate_earth_layer_cache_for_source(source)
+    update_payload = {
+        "event": "earth.layer.changed",
+        "action": "database_changed",
+        "source": source,
+        "table": payload.get("table"),
+        "data_type": source,
+        "layers": layers,
+        "refresh_strategy": payload.get("refresh_strategy") or "clear_then_reload",
+        "records_processed": payload.get("records_processed", 0),
+        "operations": payload.get("operations") or [payload.get("operation") or "CHANGE"],
+        "operation": payload.get("operation"),
+        "cache_entries_invalidated": deleted_cache_entries,
+        "timestamp": to_iso8601_utc(_utcnow()),
+    }
+    if payload.get("entity") == "interactable":
+        update_payload.update(
+            {
+                "entity": "interactable",
+                "action": payload.get("action") or "changed",
+                "ids": payload.get("ids") or payload.get("entity_keys") or [],
+                "item": payload.get("item"),
+            }
+        )
+    await broadcaster.broadcast_earth_update(update_payload)
+
+    task.records_processed = int(payload.get("records_processed") or 0)
+    task.progress = 100.0
+    task.phase_progress = 100.0
+    task.phase = "completed"
+    task.phase_message = "Earth 图层刷新通知已发送"
+    task.status = JOB_STATUS_SUCCESS
+    task.completed_at = _utcnow()
+    task.payload = {**payload, "cache_entries_invalidated": deleted_cache_entries}
+    await db.commit()
+    await _broadcast_task_update(task)
+
+
+_worker = DataJobWorker()
+
+
+def start_data_job_worker() -> None:
+    _worker.start()
+
+
+async def stop_data_job_worker() -> None:
+    await _worker.stop()

backend/app/services/docs_gatekeeper.py

@@ -32,28 +32,29 @@ class DocsMetadata:
 
 DOCS_METADATA: tuple[DocsMetadata, ...] = (
     DocsMetadata(DOCS_README_FILENAME, DEFAULT_DOCS_SLUG, "public", "Overview", 0, "技术文档", "Technical Docs"),
-    DocsMetadata("quickstart.md", "quickstart", "public", "Manual", 1, "快速开始", "Quickstart"),
-    DocsMetadata("manual.md", "manual", "public", "Manual", 2, "Planet 使用手册", "Planet Manual"),
+    DocsMetadata("manual.md", "manual", "public", "Manual", 1, "Planet 使用手册", "Planet Manual"),
+    DocsMetadata("quickstart.md", "quickstart", "public", "Manual", 2, "快速开始", "Quickstart"),
     DocsMetadata("faq.md", "faq", "public", "Manual", 3, "常见问题", "FAQ"),
-    DocsMetadata("location-pipeline-user.md", "location-pipeline-user", "public", "Manual", 4, "Earth 位置候选采集使用手册", "Earth Location Candidate Collection User Guide"),
+    DocsMetadata("platform-data-flows.md", "platform-data-flows", "docs_developer", "Architecture", 5, "业务架构与数据流转", "Business Architecture and Data Flows"),
+    DocsMetadata("naming-glossary.md", "naming-glossary", "docs_developer", "Architecture", 6, "命名与术语对照", "Naming Glossary"),
     DocsMetadata("earth-frontend-context.md", "earth-frontend-context", "docs_developer", "Earth", 10, "Earth 前端结构", "Earth Frontend Context"),
     DocsMetadata("earth-layer-style-reference.md", "earth-layer-style-reference", "docs_developer", "Earth", 11, "Earth 图层样式属性索引", "Earth Layer Style Reference"),
     DocsMetadata("earth-render-layer-order.md", "earth-render-layer-order", "docs_developer", "Earth", 12, "Earth 渲染图层顺序", "Earth Render Layer Order"),
     DocsMetadata("earth-satellite-footprint-policy.md", "earth-satellite-footprint-policy", "docs_developer", "Earth", 13, "Earth 卫星覆盖策略", "Earth Satellite Footprint Policy"),
     DocsMetadata("earth-bgp-context.md", "earth-bgp-context", "docs_developer", "Earth", 14, "BGP 态势上下文", "BGP Context"),
-    DocsMetadata("earth-news-live-streams-collector-format.md", "earth-news-live-streams-collector-format", "docs_developer", "Earth", 15, "新闻直播采集格式", "News Live Streams Collector Format"),
     DocsMetadata("earth-interactable-usage.md", "earth-interactable-usage", "docs_developer", "Earth", 16, "Earth 可交互图标接入", "Earth Interactable Usage"),
     DocsMetadata("earth-toolbar-overlay-coordination.md", "earth-toolbar-overlay-coordination", "docs_developer", "Earth", 17, "Earth 工具栏与浮层协同", "Earth Toolbar and Overlay Coordination"),
     DocsMetadata("frontend-admin-frontend-context.md", "frontend-admin-frontend-context", "docs_developer", "Frontend", 20, "控制台前端结构", "Admin Frontend Context"),
     DocsMetadata("frontend-layout-guidelines.md", "frontend-layout-guidelines", "docs_developer", "Frontend", 21, "前端布局指南", "Frontend Layout Guidelines"),
-    DocsMetadata("docs-gatekeeper-development.md", "docs-gatekeeper-development", "docs_developer", "Frontend", 22, "Docs Gatekeeper 开发说明", "Docs Gatekeeper Development Guide"),
-    DocsMetadata("naming-glossary.md", "naming-glossary", "docs_developer", "Frontend", 23, "命名与术语对照", "Naming Glossary"),
     DocsMetadata("tactile-ui-components.md", "tactile-ui-components", "docs_developer", "Frontend", 24, "Tactile UI 组件库", "Tactile UI Components"),
     DocsMetadata("backend-collectors.md", "backend-collectors", "docs_developer", "Backend", 30, "数据采集系统", "Data Collectors"),
     DocsMetadata("backend-system-service-control.md", "backend-system-service-control", "docs_admin", "Backend", 31, "系统服务控制", "System Service Control"),
     DocsMetadata("datasource-collector-settings-connectivity.md", "datasource-collector-settings-connectivity", "docs_developer", "Backend", 32, "数据源、采集器设置与连接验证", "Datasource Collector Settings and Connectivity"),
     DocsMetadata("backend-datasources-api-performance.md", "backend-datasources-api-performance", "docs_developer", "Backend", 33, "数据源 API 性能", "Datasource API Performance"),
-    DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 34, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"),
+    DocsMetadata("data-job-earth-sync-architecture.md", "data-job-earth-sync-architecture", "docs_developer", "Backend", 34, "数据作业与 Outbox 技术架构", "Data Jobs and Outbox Architecture"),
+    DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 35, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"),
+    DocsMetadata("earth-news-live-streams-collector-format.md", "earth-news-live-streams-collector-format", "docs_developer", "Backend", 36, "新闻直播采集格式", "News Live Streams Collector Format"),
+    DocsMetadata("docs-gatekeeper-development.md", "docs-gatekeeper-development", "docs_developer", "Backend", 37, "Docs Gatekeeper 开发说明", "Docs Gatekeeper Development Guide"),
     DocsMetadata("agents-aiprovider.md", "agents-aiprovider", "docs_developer", "Agents", 40, "AI Provider 指南", "AI Provider Guide"),
     DocsMetadata("ops-runbook.md", "ops-runbook", "docs_admin", "Ops", 49, "Planet 运维手册", "Planet Ops Runbook"),
     DocsMetadata("ops-docker-compose-buildx-upgrade.md", "ops-docker-compose-buildx-upgrade", "docs_admin", "Ops", 50, "Docker + Compose + Buildx 升级", "Docker + Compose + Buildx Upgrade"),

backend/app/services/earth_db_change_listener.py

@@ -0,0 +1,482 @@
+"""PostgreSQL LISTEN/NOTIFY bridge for Earth layer refresh events."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from collections import deque
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from time import monotonic
+from typing import Any, Awaitable, Callable
+
+import asyncpg
+
+from app.core.config import settings
+from app.core.logging import get_logger
+from app.core.time import to_iso8601_utc
+from app.core.websocket.broadcaster import broadcaster
+from app.services.earth_layer_adapters import (
+    get_earth_refresh_strategy_for_change,
+    get_earth_update_layers_for_change,
+    get_earth_update_layers_for_source,
+)
+from app.services.earth_layer_cache import invalidate_earth_layer_cache_for_source
+
+logger = get_logger(__name__)
+
+EARTH_DATA_CHANGES_CHANNEL = "planet_earth_data_changes"
+DEFAULT_DEBOUNCE_SECONDS = 0.25
+DEFAULT_MAX_WAIT_SECONDS = 1.5
+DELETE_FAST_FLUSH_SECONDS = 0.05
+LISTEN_KEEPALIVE_SECONDS = 5.0
+OUTBOX_POLL_LIMIT = 5000
+MAX_ENTITY_KEY_SAMPLES = 20
+MAX_SEEN_EVENT_IDS = 20000
+
+BroadcastFn = Callable[[dict[str, Any]], Awaitable[None]]
+InvalidateFn = Callable[[str], int]
+
+
+def normalize_asyncpg_dsn(dsn: str) -> str:
+    """Convert SQLAlchemy asyncpg URLs into asyncpg-compatible URLs."""
+    return dsn.replace("postgresql+asyncpg://", "postgresql://", 1)
+
+
+def build_earth_update_from_db_payload(payload: dict[str, Any]) -> dict[str, Any] | None:
+    table = payload.get("table")
+    source = str(payload.get("source") or "").strip()
+    table_name = str(table or "").strip()
+    if not source and not table_name:
+        return None
+    layers = get_earth_update_layers_for_change(table_name, source)
+    if not layers:
+        return None
+    refresh_strategy = get_earth_refresh_strategy_for_change(table_name, source) or "clear_then_reload"
+    source_has_adapter = bool(get_earth_update_layers_for_source(source))
+    effective_source = source if source_has_adapter else (table_name if table_name else source)
+    operation = payload.get("operation")
+    update: dict[str, Any] = {
+        "event": "earth.layer.changed",
+        "action": "database_changed",
+        "source": effective_source,
+        "original_source": source or None,
+        "table": table_name or None,
+        "data_type": effective_source,
+        "layers": layers,
+        "refresh_strategy": refresh_strategy,
+        "operation": operation,
+        "entity_key": payload.get("entity_key"),
+        "timestamp": to_iso8601_utc(datetime.now(UTC)),
+    }
+    if table_name == "earth_interactables" and refresh_strategy == "delta":
+        ids = payload.get("entity_keys")
+        if not isinstance(ids, list):
+            ids = [payload.get("entity_key")] if payload.get("entity_key") else []
+        update.update(
+            {
+                "entity": "interactable",
+                "action": "deleted" if operation == "DELETE" else "changed",
+                "ids": [str(item) for item in ids if item],
+                "item": None,
+            }
+        )
+    return {
+        **update,
+    }
+
+
+@dataclass
+class PendingEarthDbChange:
+    source: str
+    layers: list[str]
+    table: str | None = None
+    refresh_strategy: str = "clear_then_reload"
+    entity: str | None = None
+    action: str = "database_changed"
+    records_processed: int = 0
+    operations: set[str] = field(default_factory=set)
+    entity_keys: list[str] = field(default_factory=list)
+    first_occurred_at: str | None = None
+    last_occurred_at: str | None = None
+    first_seen_monotonic: float = field(default_factory=monotonic)
+    last_seen_monotonic: float = field(default_factory=monotonic)
+
+    def add(self, payload: dict[str, Any]) -> None:
+        self.last_seen_monotonic = monotonic()
+        records_processed = payload.get("records_processed", 1)
+        try:
+            records_processed = int(records_processed)
+        except (TypeError, ValueError):
+            records_processed = 1
+        self.records_processed += max(records_processed, 1)
+        operation = payload.get("operation")
+        if operation:
+            self.operations.add(str(operation))
+        entity_keys = payload.get("entity_keys")
+        if not isinstance(entity_keys, list):
+            entity_key = payload.get("entity_key")
+            entity_keys = [entity_key] if entity_key else []
+        for entity_key in entity_keys:
+            if entity_key and len(self.entity_keys) < MAX_ENTITY_KEY_SAMPLES:
+                self.entity_keys.append(str(entity_key))
+        occurred_at = payload.get("occurred_at")
+        if occurred_at:
+            occurred_at = str(occurred_at)
+            self.first_occurred_at = self.first_occurred_at or occurred_at
+            self.last_occurred_at = occurred_at
+
+
+class EarthDbChangeDispatcher:
+    """Debounces database notifications and broadcasts Earth refresh hints."""
+
+    def __init__(
+        self,
+        *,
+        broadcast_earth_update: BroadcastFn | None = None,
+        invalidate_cache: InvalidateFn | None = None,
+        debounce_seconds: float = DEFAULT_DEBOUNCE_SECONDS,
+        max_wait_seconds: float = DEFAULT_MAX_WAIT_SECONDS,
+    ) -> None:
+        self._broadcast_earth_update = broadcast_earth_update or broadcaster.broadcast_earth_update
+        self._invalidate_cache = invalidate_cache or invalidate_earth_layer_cache_for_source
+        self._debounce_seconds = debounce_seconds
+        self._max_wait_seconds = max(max_wait_seconds, debounce_seconds)
+        self._pending: dict[str, PendingEarthDbChange] = {}
+        self._flush_tasks: dict[str, asyncio.Task[None]] = {}
+        self._seen_event_ids: set[int] = set()
+        self._seen_event_order: deque[int] = deque()
+
+    def handle_notification(self, payload_text: str) -> bool:
+        try:
+            payload = json.loads(payload_text)
+        except json.JSONDecodeError:
+            logger.warning_event(
+                "Ignoring malformed Earth database change notification",
+                event="earth.db_changes.notification_malformed",
+            )
+            return False
+        if not isinstance(payload, dict):
+            return False
+        return self.handle_payload(payload)
+
+    def handle_payload(self, payload: dict[str, Any]) -> bool:
+        event_id = payload.get("event_id")
+        if event_id is not None:
+            try:
+                normalized_event_id = int(event_id)
+            except (TypeError, ValueError):
+                normalized_event_id = None
+            if normalized_event_id is not None:
+                if normalized_event_id in self._seen_event_ids:
+                    return False
+                self._remember_event_id(normalized_event_id)
+
+        update = build_earth_update_from_db_payload(payload)
+        if not update:
+            return False
+
+        source = update["source"]
+        pending = self._pending.get(source)
+        if pending is None:
+            pending = PendingEarthDbChange(
+                source=source,
+                layers=list(update["layers"]),
+                table=update.get("table"),
+                refresh_strategy=str(update.get("refresh_strategy") or "clear_then_reload"),
+                entity=update.get("entity"),
+                action=str(update.get("action") or "database_changed"),
+            )
+            self._pending[source] = pending
+        pending.add(payload)
+
+        task = self._flush_tasks.pop(source, None)
+        if task and not task.done():
+            task.cancel()
+        self._flush_tasks[source] = asyncio.create_task(
+            self._flush_later(source, delay_seconds=self._next_flush_delay(pending))
+        )
+        return True
+
+    def _next_flush_delay(self, pending: PendingEarthDbChange) -> float:
+        if "DELETE" in pending.operations and pending.refresh_strategy == "clear_then_reload":
+            return DELETE_FAST_FLUSH_SECONDS
+        elapsed = max(0.0, monotonic() - pending.first_seen_monotonic)
+        remaining = self._max_wait_seconds - elapsed
+        if remaining <= 0:
+            return 0.0
+        return min(self._debounce_seconds, remaining)
+
+    def _remember_event_id(self, event_id: int) -> None:
+        self._seen_event_ids.add(event_id)
+        self._seen_event_order.append(event_id)
+        while len(self._seen_event_order) > MAX_SEEN_EVENT_IDS:
+            expired_event_id = self._seen_event_order.popleft()
+            self._seen_event_ids.discard(expired_event_id)
+
+    async def _flush_later(self, source: str, *, delay_seconds: float) -> None:
+        try:
+            if delay_seconds > 0:
+                await asyncio.sleep(delay_seconds)
+            await self.flush_source(source)
+        except asyncio.CancelledError:
+            raise
+        except Exception as exc:
+            logger.exception_event(
+                "Failed to broadcast debounced Earth database change",
+                event="earth.db_changes.flush_failed",
+                context={"source": source, "error": str(exc)},
+            )
+        finally:
+            current = self._flush_tasks.get(source)
+            if current is asyncio.current_task():
+                self._flush_tasks.pop(source, None)
+
+    async def flush_source(self, source: str) -> None:
+        pending = self._pending.get(source)
+        if pending is None:
+            return
+
+        flushed_at = datetime.now(UTC)
+        deleted_cache_entries = self._invalidate_cache(source)
+        operations = sorted(pending.operations)
+        payload: dict[str, Any] = {
+            "event": "earth.layer.changed",
+            "action": "database_changed",
+            "source": source,
+            "table": pending.table,
+            "data_type": source,
+            "layers": pending.layers,
+            "refresh_strategy": pending.refresh_strategy,
+            "records_processed": pending.records_processed,
+            "operations": operations,
+            "operation": operations[-1] if len(operations) == 1 else None,
+            "entity_keys": pending.entity_keys,
+            "entity_key_sample_size": len(pending.entity_keys),
+            "cache_entries_invalidated": deleted_cache_entries,
+            "first_occurred_at": pending.first_occurred_at,
+            "last_occurred_at": pending.last_occurred_at,
+            "debounce_ms": int((monotonic() - pending.first_seen_monotonic) * 1000),
+            "total_latency_ms": self._total_latency_ms(pending, flushed_at),
+            "timestamp": to_iso8601_utc(flushed_at),
+        }
+        if pending.entity == "interactable":
+            payload.update(
+                {
+                    "entity": "interactable",
+                    "action": "deleted" if "DELETE" in pending.operations else "changed",
+                    "ids": pending.entity_keys,
+                    "item": None,
+                }
+            )
+        await self._broadcast_earth_update(payload)
+        self._pending.pop(source, None)
+        logger.info_event(
+            "Broadcasted Earth database change",
+            event="earth.db_changes.broadcasted",
+            context={
+                "source": source,
+                "layers": pending.layers,
+                "records_processed": pending.records_processed,
+                "cache_entries_invalidated": deleted_cache_entries,
+                "debounce_ms": int((monotonic() - pending.first_seen_monotonic) * 1000),
+                "total_latency_ms": payload["total_latency_ms"],
+            },
+        )
+
+    @staticmethod
+    def _total_latency_ms(pending: PendingEarthDbChange, flushed_at: datetime) -> int | None:
+        occurred_at = pending.first_occurred_at
+        if not occurred_at:
+            return None
+        try:
+            normalized = occurred_at.replace("Z", "+00:00")
+            occurred = datetime.fromisoformat(normalized)
+            if occurred.tzinfo is None:
+                occurred = occurred.replace(tzinfo=UTC)
+            return max(0, int((flushed_at - occurred.astimezone(UTC)).total_seconds() * 1000))
+        except ValueError:
+            return None
+
+    async def flush_all(self) -> None:
+        sources = list(self._pending)
+        for source in sources:
+            task = self._flush_tasks.pop(source, None)
+            if task and not task.done():
+                task.cancel()
+            await self.flush_source(source)
+
+    async def stop(self) -> None:
+        tasks = [task for task in self._flush_tasks.values() if not task.done()]
+        for task in tasks:
+            task.cancel()
+        if tasks:
+            await asyncio.gather(*tasks, return_exceptions=True)
+        self._flush_tasks.clear()
+        await self.flush_all()
+
+
+class EarthDbChangeListener:
+    def __init__(
+        self,
+        *,
+        dsn: str,
+        dispatcher: EarthDbChangeDispatcher,
+        channel: str = EARTH_DATA_CHANGES_CHANNEL,
+    ) -> None:
+        self._dsn = normalize_asyncpg_dsn(dsn)
+        self._dispatcher = dispatcher
+        self._channel = channel
+        self._task: asyncio.Task[None] | None = None
+        self._stop_event: asyncio.Event | None = None
+        self._connection: asyncpg.Connection | None = None
+        self._loop: asyncio.AbstractEventLoop | None = None
+
+    def start(self) -> None:
+        if self._task and not self._task.done():
+            return
+        self._loop = asyncio.get_running_loop()
+        self._stop_event = asyncio.Event()
+        self._task = asyncio.create_task(self._run())
+
+    async def stop(self) -> None:
+        if self._stop_event:
+            self._stop_event.set()
+        if self._connection:
+            await self._connection.close()
+        if self._task:
+            await asyncio.gather(self._task, return_exceptions=True)
+        await self._dispatcher.stop()
+
+    async def _run(self) -> None:
+        backoff_seconds = 1.0
+        assert self._stop_event is not None
+
+        while not self._stop_event.is_set():
+            try:
+                self._connection = await asyncpg.connect(self._dsn)
+                await self._connection.add_listener(self._channel, self._on_notification)
+                logger.info_event(
+                    "Earth database change listener connected",
+                    event="earth.db_changes.connected",
+                    context={"channel": self._channel},
+                )
+                backoff_seconds = 1.0
+                while not self._stop_event.is_set():
+                    try:
+                        await asyncio.wait_for(
+                            self._stop_event.wait(),
+                            timeout=LISTEN_KEEPALIVE_SECONDS,
+                        )
+                    except asyncio.TimeoutError:
+                        await self._poll_outbox()
+            except asyncio.CancelledError:
+                raise
+            except Exception as exc:
+                logger.exception_event(
+                    "Earth database change listener failed",
+                    event="earth.db_changes.listener_failed",
+                    context={"channel": self._channel, "error": str(exc)},
+                )
+                try:
+                    await asyncio.wait_for(self._stop_event.wait(), timeout=backoff_seconds)
+                except asyncio.TimeoutError:
+                    pass
+                backoff_seconds = min(backoff_seconds * 2, 30.0)
+            finally:
+                if self._connection:
+                    try:
+                        await self._connection.remove_listener(self._channel, self._on_notification)
+                    except Exception:
+                        pass
+                    try:
+                        await self._connection.close()
+                    except Exception:
+                        pass
+                    self._connection = None
+
+    async def _poll_outbox(self) -> None:
+        if self._connection is None:
+            return
+
+        rows = await self._connection.fetch(
+            """
+            SELECT id, payload
+            FROM earth_data_change_events
+            WHERE consumed_at IS NULL
+            ORDER BY id
+            LIMIT $1
+            """,
+            OUTBOX_POLL_LIMIT,
+        )
+        accepted_count = 0
+        consumed_ids: list[int] = []
+        for row in rows:
+            payload = row["payload"]
+            if isinstance(payload, str):
+                try:
+                    payload = json.loads(payload)
+                except json.JSONDecodeError:
+                    consumed_ids.append(int(row["id"]))
+                    continue
+            if isinstance(payload, dict):
+                if self._dispatcher.handle_payload(payload):
+                    accepted_count += 1
+                consumed_ids.append(int(row["id"]))
+            else:
+                consumed_ids.append(int(row["id"]))
+        if consumed_ids:
+            await self._dispatcher.flush_all()
+        if consumed_ids:
+            await self._connection.execute(
+                """
+                UPDATE earth_data_change_events
+                SET consumed_at = NOW()
+                WHERE id = ANY($1::bigint[])
+                  AND consumed_at IS NULL
+                """,
+                consumed_ids,
+            )
+        if rows:
+            logger.info_event(
+                "Polled Earth database change outbox",
+                event="earth.db_changes.outbox_polled",
+                context={"events": len(rows), "accepted": accepted_count},
+            )
+
+    def _on_notification(
+        self,
+        _connection: asyncpg.Connection,
+        _pid: int,
+        _channel: str,
+        payload: str,
+    ) -> None:
+        if self._loop and self._loop.is_running():
+            self._loop.call_soon_threadsafe(self._dispatcher.handle_notification, payload)
+            return
+        self._dispatcher.handle_notification(payload)
+
+
+_dispatcher = EarthDbChangeDispatcher(
+    broadcast_earth_update=broadcaster.broadcast_earth_update,
+    invalidate_cache=invalidate_earth_layer_cache_for_source,
+)
+_listener: EarthDbChangeListener | None = None
+
+
+def start_earth_db_change_listener() -> None:
+    global _listener
+    if _listener is not None:
+        return
+    _listener = EarthDbChangeListener(dsn=settings.DATABASE_URL, dispatcher=_dispatcher)
+    _listener.start()
+
+
+async def stop_earth_db_change_listener() -> None:
+    global _listener
+    if _listener is None:
+        await _dispatcher.stop()
+        return
+    listener = _listener
+    _listener = None
+    await listener.stop()

backend/app/services/earth_interactables.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any
+from uuid import uuid4
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.time import to_iso8601_utc
+from app.core.websocket.broadcaster import broadcaster
+from app.models.earth_interactable import EarthInteractable
+from app.services.earth_layer_cache import EARTH_LAYER_CACHE_PREFIX, earth_layer_cache
+
+INTERACTABLE_ENTITY = "interactable"
+INTERACTABLE_LAYER = "interactables"
+
+
+def normalize_interactable_id(value: str | None = None) -> str:
+    raw = str(value or "").strip()
+    return raw or f"interactable-{uuid4().hex}"
+
+
+def serialize_interactable(record: EarthInteractable) -> dict[str, Any]:
+    return {
+        "id": record.id,
+        "layer": record.layer,
+        "kind": record.kind,
+        "label": record.label,
+        "description": record.description,
+        "latitude": record.latitude,
+        "longitude": record.longitude,
+        "altitude": record.altitude,
+        "revision": record.revision,
+        "properties": record.properties or {},
+        "is_deleted": bool(record.is_deleted),
+        "created_at": to_iso8601_utc(record.created_at),
+        "updated_at": to_iso8601_utc(record.updated_at),
+        "deleted_at": to_iso8601_utc(record.deleted_at),
+    }
+
+
+def interactables_to_geojson(items: list[EarthInteractable]) -> dict[str, Any]:
+    return {
+        "type": "FeatureCollection",
+        "features": [
+            {
+                "type": "Feature",
+                "id": item.id,
+                "geometry": {
+                    "type": "Point",
+                    "coordinates": [item.longitude, item.latitude],
+                },
+                "properties": serialize_interactable(item),
+            }
+            for item in items
+            if not item.is_deleted
+        ],
+    }
+
+
+def invalidate_interactable_cache(layer: str | None = None) -> int:
+    layer_key = str(layer or "*").strip() or "*"
+    deleted = earth_layer_cache.delete_pattern(
+        f"{EARTH_LAYER_CACHE_PREFIX}:interactables:layer:{layer_key}*"
+    )
+    if layer_key != "all":
+        deleted += earth_layer_cache.delete_pattern(
+            f"{EARTH_LAYER_CACHE_PREFIX}:interactables:layer:all*"
+        )
+    return deleted
+
+
+def build_interactable_event(
+    *,
+    action: str,
+    record: EarthInteractable,
+    include_item: bool = True,
+) -> dict[str, Any]:
+    item = serialize_interactable(record)
+    return {
+        "entity": INTERACTABLE_ENTITY,
+        "action": action,
+        "layer": record.layer,
+        "layers": [INTERACTABLE_LAYER],
+        "ids": [record.id],
+        "revision": record.revision,
+        "changed_at": item["deleted_at"] or item["updated_at"] or to_iso8601_utc(datetime.now(UTC)),
+        "item": item if include_item else None,
+        "source": "earth_interactables",
+    }
+
+
+async def publish_interactable_event(action: str, record: EarthInteractable, *, include_item: bool = True) -> None:
+    await broadcaster.broadcast_earth_update(
+        build_interactable_event(action=action, record=record, include_item=include_item)
+    )
+
+
+async def list_interactables(
+    db: AsyncSession,
+    *,
+    layer: str | None = None,
+    include_deleted: bool = False,
+) -> list[EarthInteractable]:
+    stmt = select(EarthInteractable)
+    if layer:
+        stmt = stmt.where(EarthInteractable.layer == layer)
+    if not include_deleted:
+        stmt = stmt.where(EarthInteractable.is_deleted.is_(False))
+    stmt = stmt.order_by(EarthInteractable.updated_at.desc(), EarthInteractable.id.asc())
+    result = await db.execute(stmt)
+    return list(result.scalars().all())

backend/app/services/earth_layer_adapters.py

@@ -0,0 +1,179 @@
+"""Earth layer adapter registry for datasource-backed refresh behavior."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+
+@dataclass(frozen=True)
+class EarthLayerAdapter:
+    sources: frozenset[str]
+    layers: tuple[str, ...]
+    cache_patterns: tuple[str, ...]
+    tables: frozenset[str] = field(default_factory=frozenset)
+    derived_models: tuple[str, ...] = field(default_factory=tuple)
+    refresh_strategy: str = "clear_then_reload"
+
+
+EARTH_LAYER_ADAPTERS: tuple[EarthLayerAdapter, ...] = (
+    EarthLayerAdapter(
+        sources=frozenset({"barentswatch_vessels", "aisstream_vessels", "vessel_static", "vessel_position", "ais_raw_observations", "ais_source_health"}),
+        tables=frozenset({"vessel_static", "vessel_position", "ais_raw_observations", "ais_source_health"}),
+        layers=("vessels",),
+        cache_patterns=("vessels*", "summary*"),
+    ),
+    EarthLayerAdapter(
+        sources=frozenset(
+            {
+                "telegeography_cables",
+                "telegeography_landing",
+                "telegeography_landing_points",
+                "telegeography_systems",
+                "telegeography_cable_systems",
+                "arcgis_cables",
+                "arcgis_landing_points",
+                "arcgis_cable_landing_relation",
+                "arcgis_cable_landing_relations",
+                "fao_landing_points",
+            }
+        ),
+        tables=frozenset({"collected_data"}),
+        layers=("cables",),
+        cache_patterns=("cables*", "landing-points*", "summary*"),
+    ),
+    EarthLayerAdapter(
+        sources=frozenset({"celestrak_tle", "spacetrack_tle"}),
+        tables=frozenset({"collected_data"}),
+        layers=("satellites",),
+        cache_patterns=("satellites*", "summary*"),
+    ),
+    EarthLayerAdapter(
+        sources=frozenset(
+            {
+                "top500",
+                "top500_supercomputers",
+                "epoch_ai_gpu",
+                "huggingface_models",
+                "huggingface_datasets",
+                "huggingface_spaces",
+                "compute_center_locations",
+            }
+        ),
+        tables=frozenset({"compute_center_locations"}),
+        layers=("computeCenters",),
+        cache_patterns=("compute-centers*", "summary*"),
+        refresh_strategy="reload",
+    ),
+    EarthLayerAdapter(
+        sources=frozenset(
+            {
+                "ris_live_bgp",
+                "bgpstream_bgp",
+                "iptoasn_prefix_geo",
+                "opengeofeed_prefix_geo",
+                "nro_delegated_prefix_geo",
+                "bgp_observations",
+                "bgp_anomalies",
+                "bgp_incidents",
+                "bgp_collector_locations",
+            }
+        ),
+        tables=frozenset({"bgp_observations", "bgp_anomalies", "bgp_incidents", "bgp_collector_locations"}),
+        layers=("bgp",),
+        cache_patterns=("bgp*", "summary*"),
+        derived_models=("bgp_observations", "bgp_anomalies", "bgp_incidents"),
+    ),
+    EarthLayerAdapter(
+        sources=frozenset({"news_live_streams"}),
+        tables=frozenset({"collected_data"}),
+        layers=("media",),
+        cache_patterns=("summary*",),
+        refresh_strategy="reload",
+    ),
+    EarthLayerAdapter(
+        sources=frozenset({"media_news_archive", "earth_news_items"}),
+        tables=frozenset({"earth_news_items"}),
+        layers=("news",),
+        cache_patterns=("summary*",),
+        refresh_strategy="reload",
+    ),
+    EarthLayerAdapter(
+        sources=frozenset({"earth_interactables"}),
+        tables=frozenset({"earth_interactables"}),
+        layers=("interactables",),
+        cache_patterns=("interactables*", "summary*"),
+        refresh_strategy="delta",
+    ),
+)
+
+_ADAPTERS_BY_SOURCE = {
+    source: adapter
+    for adapter in EARTH_LAYER_ADAPTERS
+    for source in adapter.sources
+}
+_ADAPTERS_BY_TABLE = {
+    table: adapter
+    for adapter in EARTH_LAYER_ADAPTERS
+    for table in adapter.tables
+}
+
+
+def get_earth_layer_adapter_for_source(source: str | None) -> EarthLayerAdapter | None:
+    return _ADAPTERS_BY_SOURCE.get(str(source or "").strip())
+
+
+def get_earth_layer_adapter_for_change(table: str | None, source: str | None) -> EarthLayerAdapter | None:
+    table_key = str(table or "").strip()
+    source_key = str(source or "").strip()
+    if table_key and table_key != "collected_data":
+        adapter = _ADAPTERS_BY_TABLE.get(table_key)
+        if adapter is not None:
+            return adapter
+    return get_earth_layer_adapter_for_source(source_key)
+
+
+def get_earth_update_layers_for_source(source: str | None) -> list[str]:
+    adapter = get_earth_layer_adapter_for_source(source)
+    return list(adapter.layers) if adapter else []
+
+
+def get_earth_update_layers_for_change(table: str | None, source: str | None) -> list[str]:
+    adapter = get_earth_layer_adapter_for_change(table, source)
+    return list(adapter.layers) if adapter else []
+
+
+def get_earth_refresh_strategy_for_change(table: str | None, source: str | None) -> str | None:
+    adapter = get_earth_layer_adapter_for_change(table, source)
+    return adapter.refresh_strategy if adapter else None
+
+
+def get_earth_cache_patterns_for_source(source: str | None) -> list[str]:
+    adapter = get_earth_layer_adapter_for_source(source)
+    return list(adapter.cache_patterns) if adapter else []
+
+
+async def clear_derived_datasource_data(db: AsyncSession, source: str) -> dict[str, int]:
+    adapter = get_earth_layer_adapter_for_source(source)
+    if adapter is None or not adapter.derived_models:
+        return {}
+
+    from app.models.bgp_anomaly import BGPAnomaly
+    from app.models.bgp_incident import BGPIncident
+    from app.models.bgp_observation import BGPObservation
+
+    model_by_key: dict[str, Any] = {
+        "bgp_observations": BGPObservation,
+        "bgp_anomalies": BGPAnomaly,
+        "bgp_incidents": BGPIncident,
+    }
+    deleted_counts: dict[str, int] = {}
+    for key in adapter.derived_models:
+        model = model_by_key.get(key)
+        if model is None:
+            continue
+        result = await db.execute(model.__table__.delete().where(model.source == source))
+        deleted_counts[key] = int(result.rowcount or 0)
+    return deleted_counts

backend/app/services/earth_layer_cache.py

@@ -268,34 +268,10 @@ def apply_payload_budget(payload: dict[str, Any], policy: EarthLayerCachePolicy)
 
 
 def invalidate_earth_layer_cache_for_source(source: str) -> int:
+    from app.services.earth_layer_adapters import get_earth_cache_patterns_for_source
+
     source_key = str(source or "").strip()
-    patterns = {
-        "barentswatch_vessels": ["vessels*", "summary*"],
-        "aisstream_vessels": ["vessels*", "summary*"],
-        "telegeography_cables": ["cables*", "landing-points*", "summary*"],
-        "telegeography_landing": ["landing-points*", "summary*"],
-        "telegeography_landing_points": ["landing-points*", "summary*"],
-        "telegeography_systems": ["cables*", "summary*"],
-        "telegeography_cable_systems": ["cables*", "summary*"],
-        "arcgis_cables": ["cables*", "landing-points*", "summary*"],
-        "arcgis_landing_points": ["landing-points*", "summary*"],
-        "arcgis_cable_landing_relation": ["landing-points*", "summary*"],
-        "arcgis_cable_landing_relations": ["landing-points*", "summary*"],
-        "fao_landing_points": ["landing-points*", "summary*"],
-        "celestrak_tle": ["satellites*", "summary*"],
-        "spacetrack_tle": ["satellites*", "summary*"],
-        "top500": ["compute-centers*", "summary*"],
-        "top500_supercomputers": ["compute-centers*", "summary*"],
-        "epoch_ai_gpu": ["compute-centers*", "summary*"],
-        "huggingface_models": ["compute-centers*", "summary*"],
-        "huggingface_datasets": ["compute-centers*", "summary*"],
-        "huggingface_spaces": ["compute-centers*", "summary*"],
-        "ris_live_bgp": ["bgp*", "summary*"],
-        "bgpstream_bgp": ["bgp*", "summary*"],
-        "iptoasn_prefix_geo": ["bgp*", "summary*"],
-        "opengeofeed_prefix_geo": ["bgp*", "summary*"],
-        "nro_delegated_prefix_geo": ["bgp*", "summary*"],
-    }.get(source_key, [])
+    patterns = get_earth_cache_patterns_for_source(source_key)
     deleted = 0
     for layer_pattern in patterns:
         deleted += earth_layer_cache.delete_pattern(f"{EARTH_LAYER_CACHE_PREFIX}:{layer_pattern}")

backend/app/services/earth_news.py

@@ -1063,6 +1063,7 @@ def _build_payload(
     lon: float | None,
     active_region: str,
     items: list[ParsedNewsItem],
+    cruise_items: list[ParsedNewsItem] | None = None,
     sources: list[NewsFeedSource],
     errors: list[str],
     stale: bool,
@@ -1082,6 +1083,10 @@ def _build_payload(
         },
         "sources": _serialize_sources(sources),
         "items": [_serialize_item(item, active_region=active_region) for item in items],
+        "cruise_items": [
+            _serialize_item(item, active_region=active_region)
+            for item in (cruise_items if cruise_items is not None else items)
+        ],
         "errors": errors,
         "stale": stale,
     }
@@ -1295,6 +1300,7 @@ async def get_earth_news_payload(
 
     from app.services.earth_news_store import (
         get_earth_news_freshness,
+        list_earth_news_cruise_items,
         list_earth_news_items,
         upsert_earth_news_items,
     )
@@ -1312,6 +1318,13 @@ async def get_earth_news_payload(
         active_region=active_region,
         limit=MAX_ITEMS_TOTAL,
     )
+    if hasattr(db, "execute"):
+        cruise_items = await list_earth_news_cruise_items(
+            db,
+            limit=MAX_ITEMS_TOTAL * len(REGION_ANCHORS),
+        )
+    else:
+        cruise_items = items
     await _enqueue_unverified_locations(items)
     stale = bool(errors and items)
 
@@ -1320,6 +1333,7 @@ async def get_earth_news_payload(
         lon=lon,
         active_region=active_region,
         items=items,
+        cruise_items=cruise_items,
         sources=sources,
         errors=errors,
         stale=stale,

backend/app/services/earth_news_store.py

@@ -78,6 +78,24 @@ async def list_earth_news_items(
     return [record_to_parsed_news_item(record) for record in result.scalars().all()]
 
 
+async def list_earth_news_cruise_items(
+    db: AsyncSession,
+    *,
+    limit: int,
+) -> list[ParsedNewsItem]:
+    result = await db.execute(
+        select(EarthNewsItem)
+        .order_by(
+            EarthNewsItem.region.asc(),
+            EarthNewsItem.published_at.desc().nullslast(),
+            EarthNewsItem.last_seen_at.desc(),
+            EarthNewsItem.feed_name.asc(),
+        )
+        .limit(limit)
+    )
+    return [record_to_parsed_news_item(record) for record in result.scalars().all()]
+
+
 async def get_earth_news_freshness(
     db: AsyncSession,
     *,

backend/app/services/playground_chat_service.py

@@ -10,6 +10,7 @@ from fastapi import HTTPException, status
 from sqlalchemy import func, select, update
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app.core.logging import get_logger
 from app.db.session import async_session_factory
 from app.models.playground_message import PlaygroundMessage
 from app.models.playground_session import PlaygroundSession
@@ -27,9 +28,11 @@ from app.schemas.ai import (
     SituationalAnalysisRequest,
 )
 from app.services.ai_client import AIProviderClient
+from app.services.business_logs import emit_business_log, exception_context
 from app.services.playground_session_store import _to_response as session_to_response
 from app.services.playground_session_store import upsert_playground_session
 
+logger = get_logger(__name__, service="ai")
 STREAM_CHUNK_SIZE = 24
 STREAM_INTERVAL_SECONDS = 0.08
 THINKING_PREVIEW_SECONDS = 2.6
@@ -624,7 +627,42 @@ async def _run_assistant_message(
                 thinking={"type": "enabled"},
             )
 
+        await emit_business_log(
+            logger,
+            event="ai.playground.run.start",
+            message="Playground AI run started",
+            category="ai",
+            service="ai",
+            module=__name__,
+            request_id=request_id,
+            user_id=user_id,
+            context={
+                "session_id": session_id,
+                "session_key": session_key,
+                "user_message_id": user_message_id,
+                "assistant_message_id": assistant_message_id,
+                "preset": payload.selected_preset_key,
+            },
+        )
         analysis = await provider_client.analyze(request_payload, request_id=request_id)
+        await emit_business_log(
+            logger,
+            event="ai.playground.run.success",
+            message="Playground AI run completed",
+            category="ai",
+            service="ai",
+            module=__name__,
+            request_id=request_id,
+            user_id=user_id,
+            context={
+                "session_id": session_id,
+                "session_key": session_key,
+                "provider": analysis.provider,
+                "model": analysis.model,
+                "content_block_count": len(analysis.content_blocks or []),
+                "thinking_block_count": len(analysis.thinking_blocks or []),
+            },
+        )
 
         async with async_session_factory() as db:
             assistant_message = await _mark_message_state(
@@ -704,6 +742,23 @@ async def _run_assistant_message(
                 await db.flush()
             await db.commit()
     except asyncio.CancelledError:
+        await emit_business_log(
+            logger,
+            event="ai.playground.run.cancelled",
+            message="Playground AI run cancelled",
+            category="ai",
+            level="warning",
+            service="ai",
+            module=__name__,
+            request_id=request_id,
+            user_id=user_id,
+            context={
+                "session_id": session_id,
+                "session_key": session_key,
+                "assistant_message_id": assistant_message_id,
+                "duration_ms": round((perf_counter() - started_at) * 1000),
+            },
+        )
         async with async_session_factory() as db:
             result = await db.execute(select(PlaygroundMessage).where(PlaygroundMessage.id == assistant_message_id))
             message = result.scalar_one_or_none()
@@ -715,6 +770,26 @@ async def _run_assistant_message(
                 await db.commit()
         raise
     except Exception as exc:
+        await emit_business_log(
+            logger,
+            event="ai.playground.run.failed",
+            message="Playground AI run failed",
+            category="ai",
+            level="error",
+            service="ai",
+            module=__name__,
+            request_id=request_id,
+            user_id=user_id,
+            context=exception_context(
+                exc,
+                {
+                    "session_id": session_id,
+                    "session_key": session_key,
+                    "assistant_message_id": assistant_message_id,
+                    "duration_ms": round((perf_counter() - started_at) * 1000),
+                },
+            ),
+        )
         error_message = _format_run_exception(exc)
         async with async_session_factory() as db:
             result = await db.execute(select(PlaygroundMessage).where(PlaygroundMessage.id == assistant_message_id))

backend/app/services/scheduler.py

@@ -13,6 +13,7 @@ from app.db.session import async_session_factory
 from app.core.time import to_iso8601_utc
 from app.models.datasource import DataSource
 from app.models.task import CollectionTask
+from app.services.business_logs import emit_business_log, emit_business_log_background, exception_context
 from app.services.collectors.registry import collector_registry
 from app.services.datasource_connectivity import (
     build_builtin_connectivity_checksum,
@@ -124,6 +125,15 @@ async def run_collector_task(collector_name: str):
                 event="collector.run.skipped_disabled",
                 context={"collector_name": collector_name},
             )
+            await emit_business_log(
+                logger,
+                event="collector.run.skipped_disabled",
+                message="Skipping disabled collector",
+                category="collector",
+                service="scheduler",
+                module=__name__,
+                context={"collector_name": collector_name, "datasource_id": datasource.id, "status": "skipped"},
+            )
             return
 
         running_result = await db.execute(
@@ -152,6 +162,21 @@ async def run_collector_task(collector_name: str):
                     event="collector.run.skipped_already_running",
                     context={"collector_name": collector_name, "task_id": existing_running.id},
                 )
+                await emit_business_log(
+                    logger,
+                    event="collector.run.skipped_already_running",
+                    message="Skipping collector trigger because task is already running",
+                    category="collector",
+                    level="warning",
+                    service="scheduler",
+                    module=__name__,
+                    context={
+                        "collector_name": collector_name,
+                        "datasource_id": datasource.id,
+                        "task_id": existing_running.id,
+                        "status": "skipped",
+                    },
+                )
                 return
 
             existing_error = (existing_running.error_message or "").strip()
@@ -173,6 +198,21 @@ async def run_collector_task(collector_name: str):
                 event="collector.run.stale_task_failed",
                 context={"collector_name": collector_name, "task_id": existing_running.id},
             )
+            await emit_business_log(
+                logger,
+                event="collector.run.stale_task_failed",
+                message="Marked stale running task as failed before rerun",
+                category="collector",
+                level="warning",
+                service="scheduler",
+                module=__name__,
+                context={
+                    "collector_name": collector_name,
+                    "datasource_id": datasource.id,
+                    "task_id": existing_running.id,
+                    "status": "failed",
+                },
+            )
 
         try:
             datasource_id = datasource.id
@@ -183,6 +223,15 @@ async def run_collector_task(collector_name: str):
                 event="collector.run.started",
                 context={"collector_name": collector_name, "datasource_id": datasource_id},
             )
+            await emit_business_log(
+                logger,
+                event="collector.run.scheduled_started",
+                message="Scheduler started collector run",
+                category="collector",
+                service="scheduler",
+                module=__name__,
+                context={"collector_name": collector_name, "datasource_id": datasource_id, "status": "running"},
+            )
             task_result = await collector.run(db)
             datasource = await db.get(DataSource, datasource_id)
             if datasource is None:
@@ -217,6 +266,20 @@ async def run_collector_task(collector_name: str):
                 event="collector.run.completed",
                 context={"collector_name": collector_name, "datasource_id": datasource_id, "result": task_result},
             )
+            await emit_business_log(
+                logger,
+                event="collector.run.scheduled_completed",
+                message="Scheduler completed collector run",
+                category="collector",
+                service="scheduler",
+                module=__name__,
+                context={
+                    "collector_name": collector_name,
+                    "datasource_id": datasource_id,
+                    "status": task_result.get("status"),
+                    "result": task_result,
+                },
+            )
         except asyncio.CancelledError:
             await db.rollback()
             datasource = await db.get(DataSource, datasource_id)
@@ -228,6 +291,16 @@ async def run_collector_task(collector_name: str):
                 event="collector.run.cancelled",
                 context={"collector_name": collector_name, "datasource_id": datasource.id},
             )
+            await emit_business_log(
+                logger,
+                event="collector.run.cancelled",
+                message="Collector cancelled by operator",
+                category="collector",
+                level="warning",
+                service="scheduler",
+                module=__name__,
+                context={"collector_name": collector_name, "datasource_id": datasource.id, "status": "cancelled"},
+            )
             raise
         except Exception as exc:
             await db.rollback()
@@ -240,6 +313,19 @@ async def run_collector_task(collector_name: str):
                 event="collector.run.failed",
                 context={"collector_name": collector_name, "datasource_id": datasource.id, "error": str(exc)},
             )
+            await emit_business_log(
+                logger,
+                event="collector.run.failed",
+                message="Collector failed",
+                category="collector",
+                level="error",
+                service="scheduler",
+                module=__name__,
+                context=exception_context(
+                    exc,
+                    {"collector_name": collector_name, "datasource_id": datasource.id, "status": "failed"},
+                ),
+            )
 
 
 async def cleanup_stale_running_tasks(max_age_hours: int = 2) -> int:
@@ -361,6 +447,16 @@ def run_collector_now(collector_name: str) -> bool:
             event="collector.trigger.skipped_already_running",
             context={"collector_name": collector_name},
         )
+        emit_business_log_background(
+            logger,
+            event="collector.trigger.skipped_already_running",
+            message="Collector is already running in-memory; skipping duplicate trigger",
+            category="collector",
+            level="warning",
+            service="scheduler",
+            module=__name__,
+            context={"collector_name": collector_name, "status": "skipped"},
+        )
         return False
 
     try:
@@ -378,6 +474,15 @@ def run_collector_now(collector_name: str) -> bool:
             event="collector.trigger.started",
             context={"collector_name": collector_name},
         )
+        emit_business_log_background(
+            logger,
+            event="collector.trigger.started",
+            message="Triggered collector",
+            category="collector",
+            service="scheduler",
+            module=__name__,
+            context={"collector_name": collector_name, "status": "queued"},
+        )
         return True
     except Exception as exc:
         logger.error_event(
@@ -385,6 +490,16 @@ def run_collector_now(collector_name: str) -> bool:
             event="collector.trigger.failed",
             context={"collector_name": collector_name, "error": str(exc)},
         )
+        emit_business_log_background(
+            logger,
+            event="collector.trigger.failed",
+            message="Failed to trigger collector",
+            category="collector",
+            level="error",
+            service="scheduler",
+            module=__name__,
+            context=exception_context(exc, {"collector_name": collector_name, "status": "failed"}),
+        )
         return False