release: bump version to 0.52.0

backend/app/api/v1/auth.py

@@ -1,26 +1,85 @@
-from datetime import timedelta
-
 from fastapi import APIRouter, Depends, HTTPException, status
 from fastapi.security import OAuth2PasswordRequestForm
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy import text
 
 from app.core.config import settings
+from app.core.logging import get_logger
 from app.core.security import (
     create_access_token,
     create_refresh_token,
-    blacklist_token,
     get_current_user,
+    get_password_hash,
     verify_password,
 )
 from app.db.session import get_db
 from app.models.user import User
 from app.schemas.token import Token
-from app.schemas.user import UserCreate, UserResponse
+from app.schemas.user import (
+    ForgotPasswordRequest,
+    ResendCodeRequest,
+    ResetPasswordRequest,
+    UserRegister,
+    UserResponse,
+    VerifyEmailRequest,
+)
+from app.services import otp
+from app.services.email import (
+    EmailError,
+    EmailNotConfiguredError,
+    send_verification_email,
+)
+
+logger = get_logger(__name__)
 
 router = APIRouter()
 
 
+def _token_response(user: User) -> dict:
+    access_token = create_access_token(data={"sub": user.id})
+    refresh = create_refresh_token(data={"sub": user.id})
+    expires_in = (
+        settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60
+        if settings.ACCESS_TOKEN_EXPIRE_MINUTES > 0
+        else None
+    )
+    return {
+        "access_token": access_token,
+        "token_type": "bearer",
+        "expires_in": expires_in,
+        "refresh_token": refresh,
+        "user": {
+            "id": user.id,
+            "username": user.username,
+            "role": user.role,
+            "gatekeeper_groups": user.gatekeeper_groups or [],
+        },
+    }
+
+
+async def _load_user_by_email(db: AsyncSession, email: str) -> User | None:
+    result = await db.execute(
+        text(
+            "SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups, email_verified "
+            "FROM users WHERE email = :email"
+        ),
+        {"email": email},
+    )
+    row = result.fetchone()
+    if row is None:
+        return None
+    user = User()
+    user.id = row[0]
+    user.username = row[1]
+    user.email = row[2]
+    user.password_hash = row[3]
+    user.role = row[4]
+    user.is_active = row[5]
+    user.gatekeeper_groups = row[6] or []
+    user.email_verified = bool(row[7])
+    return user
+
+
 @router.post("/login", response_model=Token)
 async def login(
     form_data: OAuth2PasswordRequestForm = Depends(),
@@ -28,7 +87,8 @@ async def login(
 ):
     result = await db.execute(
         text(
-            "SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE username = :username"
+            "SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups, email_verified "
+            "FROM users WHERE username = :username"
         ),
         {"username": form_data.username},
     )
@@ -47,6 +107,7 @@ async def login(
     user.role = row[4]
     user.is_active = row[5]
     user.gatekeeper_groups = row[6] or []
+    user.email_verified = bool(row[7])
 
     if not verify_password(form_data.password, user.password_hash):
         raise HTTPException(
@@ -58,25 +119,13 @@ async def login(
             status_code=status.HTTP_401_UNAUTHORIZED,
             detail="User is inactive",
         )
+    if not user.email_verified:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail={"code": "EMAIL_NOT_VERIFIED", "email": user.email},
+        )
 
-    access_token = create_access_token(data={"sub": user.id})
-    refresh_token = create_refresh_token(data={"sub": user.id})
-
-    expires_in = None
-    if settings.ACCESS_TOKEN_EXPIRE_MINUTES > 0:
-        expires_in = settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60
-
-    return {
-        "access_token": access_token,
-        "token_type": "bearer",
-        "expires_in": expires_in,
-        "user": {
-            "id": user.id,
-            "username": user.username,
-            "role": user.role,
-            "gatekeeper_groups": user.gatekeeper_groups or [],
-        },
-    }
+    return _token_response(user)
 
 
 @router.post("/refresh", response_model=Token)
@@ -116,5 +165,179 @@ async def get_me(current_user: User = Depends(get_current_user)):
         "role": current_user.role,
         "gatekeeper_groups": current_user.gatekeeper_groups or [],
         "is_active": current_user.is_active,
+        "email_verified": getattr(current_user, "email_verified", True),
         "created_at": current_user.created_at,
     }
+
+
+async def _send_code_or_raise(db: AsyncSession, email: str, code: str, purpose: str) -> None:
+    try:
+        await send_verification_email(db, to=email, code=code, purpose=purpose)
+    except EmailNotConfiguredError as exc:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail={"code": exc.code, "message": str(exc)},
+        ) from exc
+    except EmailError as exc:
+        logger.warning_event(
+            "SMTP send failed",
+            event="auth.email.send_failed",
+            context={"email": email, "purpose": purpose, "error": str(exc)},
+        )
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail={"code": exc.code, "message": str(exc)},
+        ) from exc
+
+
+@router.post("/register", status_code=status.HTTP_201_CREATED)
+async def register(payload: UserRegister, db: AsyncSession = Depends(get_db)):
+    existing = await db.execute(
+        text("SELECT id, email_verified FROM users WHERE username = :u OR email = :e"),
+        {"u": payload.username, "e": payload.email},
+    )
+    row = existing.fetchone()
+    if row is not None:
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail={"code": "USER_ALREADY_EXISTS", "message": "Username or email already in use"},
+        )
+
+    user = User(
+        username=payload.username,
+        email=payload.email,
+        password_hash=get_password_hash(payload.password),
+        role="viewer",
+        is_active=True,
+        email_verified=False,
+    )
+    db.add(user)
+    await db.commit()
+
+    try:
+        code = otp.issue_code(payload.email, "register")
+    except otp.OtpResendRateLimited as exc:
+        raise HTTPException(
+            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+            detail={"code": exc.code, "retry_after_seconds": exc.retry_after_seconds},
+        ) from exc
+    await _send_code_or_raise(db, payload.email, code, "register")
+    return {"status": "pending_verification", "email": payload.email}
+
+
+@router.post("/verify-email", response_model=Token)
+async def verify_email(payload: VerifyEmailRequest, db: AsyncSession = Depends(get_db)):
+    user = await _load_user_by_email(db, payload.email)
+    if user is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail={"code": "USER_NOT_FOUND"},
+        )
+    try:
+        otp.verify_code(payload.email, "register", payload.code)
+    except otp.OtpExpired as exc:
+        raise HTTPException(
+            status_code=status.HTTP_410_GONE,
+            detail={"code": exc.code, "message": str(exc)},
+        ) from exc
+    except otp.OtpAttemptsExceeded as exc:
+        raise HTTPException(
+            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+            detail={"code": exc.code, "message": str(exc)},
+        ) from exc
+    except otp.OtpInvalid as exc:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail={"code": exc.code, "message": str(exc)},
+        ) from exc
+
+    await db.execute(
+        text("UPDATE users SET email_verified = TRUE WHERE id = :id"),
+        {"id": user.id},
+    )
+    await db.commit()
+    user.email_verified = True
+    return _token_response(user)
+
+
+@router.post("/resend-code")
+async def resend_code(payload: ResendCodeRequest, db: AsyncSession = Depends(get_db)):
+    user = await _load_user_by_email(db, payload.email)
+    if user is None:
+        # Avoid email enumeration; pretend success.
+        return {"status": "ok"}
+    if payload.purpose == "register" and user.email_verified:
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail={"code": "ALREADY_VERIFIED"},
+        )
+    try:
+        code = otp.issue_code(payload.email, payload.purpose)
+    except otp.OtpResendRateLimited as exc:
+        raise HTTPException(
+            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+            detail={"code": exc.code, "retry_after_seconds": exc.retry_after_seconds},
+        ) from exc
+    await _send_code_or_raise(db, payload.email, code, payload.purpose)
+    return {"status": "ok"}
+
+
+@router.post("/forgot-password")
+async def forgot_password(payload: ForgotPasswordRequest, db: AsyncSession = Depends(get_db)):
+    user = await _load_user_by_email(db, payload.email)
+    if user is None:
+        # Don't leak whether an email is registered.
+        return {"status": "ok"}
+    try:
+        code = otp.issue_code(payload.email, "reset_password")
+    except otp.OtpResendRateLimited:
+        # Silently accept; the user can retry after the cooldown.
+        return {"status": "ok"}
+    try:
+        await send_verification_email(db, to=payload.email, code=code, purpose="reset_password")
+    except EmailNotConfiguredError as exc:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail={"code": exc.code, "message": str(exc)},
+        ) from exc
+    except EmailError as exc:
+        logger.warning_event(
+            "SMTP send failed",
+            event="auth.email.send_failed",
+            context={"email": payload.email, "purpose": "reset_password", "error": str(exc)},
+        )
+    return {"status": "ok"}
+
+
+@router.post("/reset-password")
+async def reset_password(payload: ResetPasswordRequest, db: AsyncSession = Depends(get_db)):
+    user = await _load_user_by_email(db, payload.email)
+    if user is None:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail={"code": "OTP_INVALID"},
+        )
+    try:
+        otp.verify_code(payload.email, "reset_password", payload.code)
+    except otp.OtpExpired as exc:
+        raise HTTPException(
+            status_code=status.HTTP_410_GONE,
+            detail={"code": exc.code, "message": str(exc)},
+        ) from exc
+    except otp.OtpAttemptsExceeded as exc:
+        raise HTTPException(
+            status_code=status.HTTP_429_TOO_MANY_REQUESTS,
+            detail={"code": exc.code, "message": str(exc)},
+        ) from exc
+    except otp.OtpInvalid as exc:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail={"code": exc.code, "message": str(exc)},
+        ) from exc
+
+    await db.execute(
+        text("UPDATE users SET password_hash = :p, email_verified = TRUE WHERE id = :id"),
+        {"p": get_password_hash(payload.new_password), "id": user.id},
+    )
+    await db.commit()
+    return {"status": "ok"}

backend/app/api/v1/data_products.py

@@ -0,0 +1,98 @@
+from datetime import UTC, datetime
+
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.v1.visualization import get_visualization_geo_summary
+from app.core.time import to_iso8601_utc
+from app.db.session import get_db
+
+router = APIRouter()
+
+PRODUCT_DEFINITIONS: dict[str, dict] = {
+    "vessels": {
+        "name": "船只",
+        "sources": ["aisstream_vessels", "barentswatch_vessels"],
+        "primary_stat_key": "vessel_count",
+        "stat_keys": ["vessel_count", "vessel_raw_unique_mmsi", "vessel_legacy_unique_mmsi"],
+    },
+    "cables": {
+        "name": "海底光缆",
+        "sources": [
+            "arcgis_cables",
+            "arcgis_landing_points",
+            "arcgis_cable_landing_relation",
+            "telegeography_cables",
+            "telegeography_landing",
+            "telegeography_systems",
+            "fao_landing_points",
+        ],
+        "primary_stat_key": "cable_count",
+        "stat_keys": ["cable_count", "landing_point_count"],
+    },
+    "satellites": {
+        "name": "卫星",
+        "sources": ["celestrak_tle", "spacetrack_tle"],
+        "primary_stat_key": "satellite_count",
+        "stat_keys": ["satellite_count"],
+    },
+    "bgp": {
+        "name": "BGP",
+        "sources": [
+            "ris_live_bgp",
+            "bgpstream_bgp",
+            "iptoasn_prefix_geo",
+            "opengeofeed_prefix_geo",
+            "nro_delegated_prefix_geo",
+        ],
+        "primary_stat_key": "bgp_event_count",
+        "stat_keys": ["bgp_event_count", "bgp_incident_count", "bgp_anomaly_count", "bgp_collector_count"],
+    },
+    "compute": {
+        "name": "算力",
+        "sources": ["top500", "epoch_ai_gpu"],
+        "primary_stat_key": "compute_center_count",
+        "stat_keys": ["compute_center_count", "supercomputer_count", "gpu_cluster_count"],
+    },
+}
+
+
+def _build_product_status(product_id: str, summary: dict) -> dict:
+    definition = PRODUCT_DEFINITIONS[product_id]
+    stats = summary.get("stats", {})
+    product_stats = {key: stats.get(key, 0) for key in definition["stat_keys"]}
+    total_count = int(product_stats.get(definition["primary_stat_key"]) or 0)
+    return {
+        "product_id": product_id,
+        "name": definition["name"],
+        "sources": definition["sources"],
+        "generated_at": summary.get("generated_at") or to_iso8601_utc(datetime.now(UTC)),
+        "total_count": total_count,
+        "stats": product_stats,
+        "build_state": "ready",
+        "stats_scope": "global",
+        "stats_freshness": "cached_or_indexed",
+    }
+
+
+@router.get("")
+async def list_data_products(db: AsyncSession = Depends(get_db)):
+    summary = await get_visualization_geo_summary(db)
+    return {
+        "generated_at": summary.get("generated_at"),
+        "data": [
+            _build_product_status(product_id, summary)
+            for product_id in PRODUCT_DEFINITIONS
+        ],
+    }
+
+
+@router.get("/{product_id}/status")
+async def get_data_product_status(
+    product_id: str,
+    db: AsyncSession = Depends(get_db),
+):
+    if product_id not in PRODUCT_DEFINITIONS:
+        raise HTTPException(status_code=404, detail="Unknown data product")
+    summary = await get_visualization_geo_summary(db)
+    return _build_product_status(product_id, summary)

backend/app/api/v1/datasources.py

@@ -3,7 +3,8 @@ from datetime import datetime, timedelta, timezone
 from typing import Optional
 
 from fastapi import APIRouter, Depends, HTTPException, Query
-from sqlalchemy import func, select, text
+from pydantic import BaseModel, Field
+from sqlalchemy import func, or_, select, text
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.core.time import to_iso8601_utc
@@ -27,6 +28,29 @@ from app.services.scheduler import (
 router = APIRouter()
 STALE_RUNNING_TASK_TIMEOUT_MINUTES = 90
 
+PRODUCT_SOURCE_KEYWORDS: tuple[tuple[str, tuple[str, ...]], ...] = (
+    ("vessels", ("vessel", "ais")),
+    ("cables", ("cable", "landing", "telegeography", "arcgis", "fao")),
+    ("satellites", ("tle", "satellite", "spacetrack", "celestrak")),
+    ("bgp", ("bgp", "asn", "prefix_geo", "opengeofeed", "nro")),
+    ("compute", ("top500", "gpu", "supercomputer", "compute")),
+    ("ai", ("huggingface", "epoch_ai")),
+    ("media", ("news", "tv", "live_stream")),
+)
+
+
+class DatasourceBatchTriggerRequest(BaseModel):
+    source_ids: list[int] = Field(default_factory=list)
+    force: bool = False
+    module: Optional[str] = None
+    product: Optional[str] = None
+    is_active: Optional[bool] = None
+    priority: Optional[str] = None
+    run_status: Optional[str] = None
+    collected: Optional[bool] = None
+    credential_status: Optional[str] = None
+    q: Optional[str] = None
+
 
 def format_frequency_label(minutes: int) -> str:
     if minutes % 1440 == 0:
@@ -47,6 +71,20 @@ def datasource_metadata(source: str) -> dict:
     }
 
 
+def datasource_product_key(datasource: DataSource) -> str:
+    haystack = " ".join(
+        [
+            datasource.source or "",
+            datasource.name or "",
+            datasource.collector_class or "",
+        ]
+    ).lower()
+    for product, keywords in PRODUCT_SOURCE_KEYWORDS:
+        if any(keyword in haystack for keyword in keywords):
+            return product
+    return "other"
+
+
 def is_due_for_collection(datasource: DataSource, now: datetime) -> bool:
     if datasource.last_run_at is None:
         return True
@@ -110,6 +148,22 @@ async def _load_latest_task_ids(
     return {datasource_id: task_id for datasource_id, task_id in result.all()}
 
 
+async def _load_collected_record_counts(
+    db: AsyncSession,
+    sources: list[str],
+) -> dict[str, int]:
+    if not sources:
+        return {}
+
+    result = await db.execute(
+        select(CollectedData.source, func.count(CollectedData.id))
+        .where(CollectedData.source.in_(sources))
+        .where(CollectedData.is_current.is_(True))
+        .group_by(CollectedData.source)
+    )
+    return {source: int(count or 0) for source, count in result.all()}
+
+
 async def _load_datasource_endpoint_overrides(
     db: AsyncSession,
     sources: list[str],
@@ -161,6 +215,192 @@ async def _load_datasource_list_context(
     return running_tasks, endpoint_overrides
 
 
+def _apply_datasource_query_filters(
+    query,
+    *,
+    module: Optional[str] = None,
+    is_active: Optional[bool] = None,
+    priority: Optional[str] = None,
+    run_status: Optional[str] = None,
+    q: Optional[str] = None,
+) -> object:
+    if module:
+        query = query.where(DataSource.module == module)
+    if is_active is not None:
+        query = query.where(DataSource.is_active == is_active)
+    if priority:
+        query = query.where(DataSource.priority == priority)
+    if run_status and run_status not in {"running", "collected", "uncollected"}:
+        if run_status == "not_run":
+            query = query.where(DataSource.last_status.is_(None))
+        else:
+            query = query.where(DataSource.last_status == run_status)
+    if q:
+        like_value = f"%{q.strip()}%"
+        query = query.where(
+            or_(
+                DataSource.name.ilike(like_value),
+                DataSource.source.ilike(like_value),
+                DataSource.collector_class.ilike(like_value),
+            )
+        )
+    return query
+
+
+def _filter_datasources_in_memory(
+    datasources: list[DataSource],
+    *,
+    running_tasks: dict[int, CollectionTask],
+    record_counts: dict[str, int],
+    product: Optional[str] = None,
+    run_status: Optional[str] = None,
+    collected: Optional[bool] = None,
+    credential_status: Optional[str] = None,
+) -> list[DataSource]:
+    filtered: list[DataSource] = []
+    for datasource in datasources:
+        record_count = record_counts.get(datasource.source, 0)
+        if product and datasource_product_key(datasource) != product:
+            continue
+        if collected is not None and (record_count > 0) != collected:
+            continue
+        if credential_status:
+            metadata = datasource_metadata(datasource.source)
+            if metadata["credential_status"] != credential_status:
+                continue
+        if run_status == "running" and datasource.id not in running_tasks:
+            continue
+        if run_status == "collected" and record_count <= 0:
+            continue
+        if run_status == "uncollected" and record_count > 0:
+            continue
+        filtered.append(datasource)
+    return filtered
+
+
+async def _trigger_datasource_batch(
+    db: AsyncSession,
+    datasources: list[DataSource],
+    *,
+    force: bool,
+) -> dict:
+    if not datasources:
+        return {
+            "status": "noop",
+            "message": "No matching data sources to trigger",
+            "force": force,
+            "triggered": [],
+            "skipped": [],
+            "failed": [],
+        }
+
+    previous_task_ids: dict[int, Optional[int]] = {}
+    triggered_sources: list[dict] = []
+    skipped_sources: list[dict] = []
+    failed_sources: list[dict] = []
+    now = datetime.now(timezone.utc)
+    running_tasks = await _load_latest_running_tasks(
+        db,
+        [datasource.id for datasource in datasources],
+    )
+
+    for datasource in datasources:
+        if not datasource.is_active:
+            skipped_sources.append(
+                {
+                    "id": datasource.id,
+                    "source": datasource.source,
+                    "name": datasource.name,
+                    "reason": "disabled",
+                }
+            )
+            continue
+
+        running_task = running_tasks.get(datasource.id)
+        if running_task is not None:
+            if not force:
+                skipped_sources.append(
+                    {
+                        "id": datasource.id,
+                        "source": datasource.source,
+                        "name": datasource.name,
+                        "reason": "already_running",
+                        "task_id": running_task.id,
+                    }
+                )
+                continue
+            cancelled = await cancel_running_collector_now(datasource.source)
+            if not cancelled:
+                await rollback_orphaned_running_task(db, datasource, running_task)
+
+        if not force and not is_due_for_collection(datasource, now):
+            skipped_sources.append(
+                {
+                    "id": datasource.id,
+                    "source": datasource.source,
+                    "name": datasource.name,
+                    "reason": "within_frequency_window",
+                    "last_run_at": to_iso8601_utc(datasource.last_run_at),
+                    "next_run_at": to_iso8601_utc(
+                        datasource.last_run_at + timedelta(minutes=datasource.frequency_minutes)
+                    ),
+                }
+            )
+            continue
+
+        previous_task_ids[datasource.id] = None
+        success = run_collector_now(datasource.source)
+        if not success:
+            failed_sources.append(
+                {
+                    "id": datasource.id,
+                    "source": datasource.source,
+                    "name": datasource.name,
+                    "reason": "trigger_failed",
+                }
+            )
+            continue
+
+        triggered_sources.append(
+            {
+                "id": datasource.id,
+                "source": datasource.source,
+                "name": datasource.name,
+                "task_id": None,
+            }
+        )
+
+    latest_task_ids = await _load_latest_task_ids(
+        db,
+        [datasource.id for datasource in datasources],
+    )
+    for datasource_id in previous_task_ids:
+        previous_task_ids[datasource_id] = latest_task_ids.get(datasource_id)
+
+    for _ in range(20):
+        await asyncio.sleep(0.1)
+        pending = [item for item in triggered_sources if item["task_id"] is None]
+        if not pending:
+            break
+        latest_task_ids = await _load_latest_task_ids(
+            db,
+            [item["id"] for item in pending],
+        )
+        for item in pending:
+            task_id = latest_task_ids.get(item["id"])
+            if task_id is not None and task_id != previous_task_ids.get(item["id"]):
+                item["task_id"] = task_id
+
+    return {
+        "status": "triggered" if triggered_sources else "partial",
+        "message": f"Triggered {len(triggered_sources)} data sources",
+        "force": force,
+        "triggered": triggered_sources,
+        "skipped": skipped_sources,
+        "failed": failed_sources,
+    }
+
+
 async def get_datasource_record(db: AsyncSession, source_id: str) -> Optional[DataSource]:
     datasource = None
     try:
@@ -355,16 +595,23 @@ async def list_datasources(
     module: Optional[str] = None,
     is_active: Optional[bool] = None,
     priority: Optional[str] = None,
+    product: Optional[str] = None,
+    run_status: Optional[str] = None,
+    collected: Optional[bool] = None,
+    credential_status: Optional[str] = None,
+    q: Optional[str] = None,
     current_user: User = Depends(get_current_user),
     db: AsyncSession = Depends(get_db),
 ):
     query = select(DataSource).order_by(DataSource.module, DataSource.id)
-    if module:
-        query = query.where(DataSource.module == module)
-    if is_active is not None:
-        query = query.where(DataSource.is_active == is_active)
-    if priority:
-        query = query.where(DataSource.priority == priority)
+    query = _apply_datasource_query_filters(
+        query,
+        module=module,
+        is_active=is_active,
+        priority=priority,
+        run_status=run_status,
+        q=q,
+    )
 
     result = await db.execute(query)
     datasources = result.scalars().all()
@@ -372,11 +619,22 @@ async def list_datasources(
     collector_list = []
     config = get_data_sources_config()
     running_tasks, endpoint_overrides = await _load_datasource_list_context(db, datasources)
+    record_counts = await _load_collected_record_counts(db, [datasource.source for datasource in datasources])
+    datasources = _filter_datasources_in_memory(
+        datasources,
+        running_tasks=running_tasks,
+        record_counts=record_counts,
+        product=product,
+        run_status=run_status,
+        collected=collected,
+        credential_status=credential_status,
+    )
     for datasource in datasources:
         running_task = running_tasks.get(datasource.id)
         endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
         last_run_at = datasource.last_run_at
         last_status = datasource.last_status
+        collected_records = record_counts.get(datasource.source, 0)
 
         collector_list.append(
             {
@@ -384,6 +642,7 @@ async def list_datasources(
                 "source": datasource.source,
                 "name": datasource.name,
                 **datasource_metadata(datasource.source),
+                "product": datasource_product_key(datasource),
                 "module": datasource.module,
                 "priority": datasource.priority,
                 "frequency": format_frequency_label(datasource.frequency_minutes),
@@ -405,6 +664,8 @@ async def list_datasources(
                 "phase_unit": running_task.phase_unit if running_task else None,
                 "records_processed": running_task.records_processed if running_task else None,
                 "total_records": running_task.total_records if running_task else None,
+                "collected_records": collected_records,
+                "has_collected_data": collected_records > 0,
             }
         )
 
@@ -419,110 +680,46 @@ async def trigger_all_datasources(
 ):
     result = await db.execute(
         select(DataSource)
-        .where(DataSource.is_active == True)
+        .where(DataSource.is_active.is_(True))
         .order_by(DataSource.module, DataSource.id)
     )
     datasources = result.scalars().all()
+    return await _trigger_datasource_batch(db, datasources, force=force)
 
-    if not datasources:
-        return {
-            "status": "noop",
-            "message": "No active data sources to trigger",
-            "triggered": [],
-            "skipped": [],
-            "failed": [],
-        }
 
-    previous_task_ids: dict[int, Optional[int]] = {}
-    triggered_sources: list[dict] = []
-    skipped_sources: list[dict] = []
-    failed_sources: list[dict] = []
-    now = datetime.now(timezone.utc)
-    running_tasks = await _load_latest_running_tasks(
-        db,
-        [datasource.id for datasource in datasources],
-    )
-
-    for datasource in datasources:
-        running_task = running_tasks.get(datasource.id)
-        if running_task is not None:
-            skipped_sources.append(
-                {
-                    "id": datasource.id,
-                    "source": datasource.source,
-                    "name": datasource.name,
-                    "reason": "already_running",
-                    "task_id": running_task.id,
-                }
-            )
-            continue
-
-        if not force and not is_due_for_collection(datasource, now):
-            skipped_sources.append(
-                {
-                    "id": datasource.id,
-                    "source": datasource.source,
-                    "name": datasource.name,
-                    "reason": "within_frequency_window",
-                    "last_run_at": to_iso8601_utc(datasource.last_run_at),
-                    "next_run_at": to_iso8601_utc(
-                        datasource.last_run_at + timedelta(minutes=datasource.frequency_minutes)
-                    ),
-                }
-            )
-            continue
-
-        previous_task_ids[datasource.id] = None
-        success = run_collector_now(datasource.source)
-        if not success:
-            failed_sources.append(
-                {
-                    "id": datasource.id,
-                    "source": datasource.source,
-                    "name": datasource.name,
-                    "reason": "trigger_failed",
-                }
-            )
-            continue
-
-        triggered_sources.append(
-            {
-                "id": datasource.id,
-                "source": datasource.source,
-                "name": datasource.name,
-                "task_id": None,
-            }
+@router.post("/trigger-batch")
+async def trigger_datasource_batch(
+    payload: DatasourceBatchTriggerRequest,
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    query = select(DataSource).order_by(DataSource.module, DataSource.id)
+    if payload.source_ids:
+        query = query.where(DataSource.id.in_(payload.source_ids))
+    else:
+        query = _apply_datasource_query_filters(
+            query,
+            module=payload.module,
+            is_active=payload.is_active,
+            priority=payload.priority,
+            run_status=payload.run_status,
+            q=payload.q,
         )
 
-    latest_task_ids = await _load_latest_task_ids(
-        db,
-        [datasource.id for datasource in datasources],
+    result = await db.execute(query)
+    datasources = result.scalars().all()
+    running_tasks, _ = await _load_datasource_list_context(db, datasources)
+    record_counts = await _load_collected_record_counts(db, [datasource.source for datasource in datasources])
+    datasources = _filter_datasources_in_memory(
+        datasources,
+        running_tasks=running_tasks,
+        record_counts=record_counts,
+        product=None if payload.source_ids else payload.product,
+        run_status=None if payload.source_ids else payload.run_status,
+        collected=None if payload.source_ids else payload.collected,
+        credential_status=None if payload.source_ids else payload.credential_status,
     )
-    for datasource_id in previous_task_ids:
-        previous_task_ids[datasource_id] = latest_task_ids.get(datasource_id)
-
-    for _ in range(20):
-        await asyncio.sleep(0.1)
-        pending = [item for item in triggered_sources if item["task_id"] is None]
-        if not pending:
-            break
-        latest_task_ids = await _load_latest_task_ids(
-            db,
-            [item["id"] for item in pending],
-        )
-        for item in pending:
-            task_id = latest_task_ids.get(item["id"])
-            if task_id is not None and task_id != previous_task_ids.get(item["id"]):
-                item["task_id"] = task_id
-
-    return {
-        "status": "triggered" if triggered_sources else "partial",
-        "message": f"Triggered {len(triggered_sources)} data sources",
-        "force": force,
-        "triggered": triggered_sources,
-        "skipped": skipped_sources,
-        "failed": failed_sources,
-    }
+    return await _trigger_datasource_batch(db, datasources, force=payload.force)
 
 
 @router.get("/{source_id}")

backend/app/api/v1/layers.py

@@ -0,0 +1,229 @@
+from typing import Any, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.v1.visualization import (
+    _parse_bbox,
+    get_bgp_anomalies_geojson,
+    get_bgp_collectors_geojson,
+    get_bgp_incidents_geojson,
+    get_cables_geojson,
+    get_landing_points_geojson,
+    get_satellites_geojson,
+)
+from app.api.v1.vessels import build_vessel_snapshot_response
+from app.db.session import get_db
+
+router = APIRouter()
+
+DEFAULT_LAYER_LIMIT = 1000
+MAX_LAYER_LIMIT = 5000
+LOW_ZOOM_FEATURE_LIMIT = 500
+
+
+def _clamp_limit(limit: int, zoom: int) -> tuple[int, bool]:
+    clamped = min(max(limit, 1), MAX_LAYER_LIMIT)
+    if zoom <= 3:
+        return min(clamped, LOW_ZOOM_FEATURE_LIMIT), clamped != limit or clamped > LOW_ZOOM_FEATURE_LIMIT
+    return clamped, clamped != limit
+
+
+def _coordinate_in_bbox(coord: Any, bbox: tuple[float, float, float, float]) -> bool:
+    if not isinstance(coord, (list, tuple)) or len(coord) < 2:
+        return False
+    try:
+        lon = float(coord[0])
+        lat = float(coord[1])
+    except (TypeError, ValueError):
+        return False
+    lon_min, lat_min, lon_max, lat_max = bbox
+    return lon_min <= lon <= lon_max and lat_min <= lat <= lat_max
+
+
+def _geometry_intersects_bbox(geometry: dict, bbox: tuple[float, float, float, float]) -> bool:
+    coordinates = geometry.get("coordinates")
+    geometry_type = geometry.get("type")
+    if geometry_type == "Point":
+        return _coordinate_in_bbox(coordinates, bbox)
+    if geometry_type in {"LineString", "MultiPoint"}:
+        return any(_coordinate_in_bbox(coord, bbox) for coord in coordinates or [])
+    if geometry_type in {"Polygon", "MultiLineString"}:
+        return any(
+            _coordinate_in_bbox(coord, bbox)
+            for line in coordinates or []
+            for coord in line
+        )
+    if geometry_type == "MultiPolygon":
+        return any(
+            _coordinate_in_bbox(coord, bbox)
+            for polygon in coordinates or []
+            for line in polygon
+            for coord in line
+        )
+    return False
+
+
+def _guard_geojson_layer(
+    geojson: dict,
+    *,
+    bbox: tuple[float, float, float, float],
+    zoom: int,
+    limit: int,
+) -> dict:
+    bounded_limit, limit_clamped = _clamp_limit(limit, zoom)
+    features = [
+        feature
+        for feature in geojson.get("features", [])
+        if _geometry_intersects_bbox(feature.get("geometry") or {}, bbox)
+    ]
+    visible_count = len(features)
+    returned_features = features[:bounded_limit]
+    return {
+        **geojson,
+        "features": returned_features,
+        "visible_count": visible_count,
+        "returned_count": len(returned_features),
+        "diagnostics": {
+            "bbox_limited": True,
+            "limit": bounded_limit,
+            "limit_clamped": limit_clamped,
+            "truncated": visible_count > len(returned_features),
+            "degraded": zoom <= 3 or visible_count > len(returned_features),
+            "stats_scope": "viewport",
+        },
+    }
+
+
+def _parse_layer_bbox(bbox: str) -> tuple[float, float, float, float]:
+    parsed = _parse_bbox(bbox)
+    if parsed is None:
+        raise HTTPException(status_code=400, detail="bbox is required")
+    return parsed
+
+
+@router.get("/vessels/snapshot")
+async def get_vessel_layer_snapshot(
+    bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"),
+    zoom: int = Query(..., ge=1, le=20),
+    limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1),
+    vessel_type: Optional[str] = Query(None, alias="type"),
+    since_minutes: int = Query(60, ge=1, le=1440),
+    db: AsyncSession = Depends(get_db),
+):
+    parsed_bbox = _parse_layer_bbox(bbox)
+    return await build_vessel_snapshot_response(
+        db,
+        bbox=parsed_bbox,
+        zoom=zoom,
+        limit=limit,
+        vessel_type=vessel_type,
+        since_minutes=since_minutes,
+    )
+
+
+@router.get("/cables")
+async def get_cable_layer(
+    bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"),
+    zoom: int = Query(..., ge=1, le=20),
+    limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1),
+    db: AsyncSession = Depends(get_db),
+):
+    return _guard_geojson_layer(
+        await get_cables_geojson(db),
+        bbox=_parse_layer_bbox(bbox),
+        zoom=zoom,
+        limit=limit,
+    )
+
+
+@router.get("/landing-points")
+async def get_landing_point_layer(
+    bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"),
+    zoom: int = Query(..., ge=1, le=20),
+    limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1),
+    db: AsyncSession = Depends(get_db),
+):
+    return _guard_geojson_layer(
+        await get_landing_points_geojson(db),
+        bbox=_parse_layer_bbox(bbox),
+        zoom=zoom,
+        limit=limit,
+    )
+
+
+@router.get("/satellites")
+async def get_satellite_layer(
+    bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"),
+    zoom: int = Query(..., ge=1, le=20),
+    limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1),
+    db: AsyncSession = Depends(get_db),
+):
+    bounded_limit, _ = _clamp_limit(limit, zoom)
+    return _guard_geojson_layer(
+        await get_satellites_geojson(limit=bounded_limit, db=db),
+        bbox=_parse_layer_bbox(bbox),
+        zoom=zoom,
+        limit=limit,
+    )
+
+
+@router.get("/bgp/anomalies")
+async def get_bgp_anomaly_layer(
+    bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"),
+    zoom: int = Query(..., ge=1, le=20),
+    limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1),
+    severity: Optional[str] = Query(None),
+    status: Optional[str] = Query("active"),
+    db: AsyncSession = Depends(get_db),
+):
+    bounded_limit, _ = _clamp_limit(limit, zoom)
+    return _guard_geojson_layer(
+        await get_bgp_anomalies_geojson(
+            severity=severity,
+            status=status,
+            limit=bounded_limit,
+            db=db,
+        ),
+        bbox=_parse_layer_bbox(bbox),
+        zoom=zoom,
+        limit=limit,
+    )
+
+
+@router.get("/bgp/incidents")
+async def get_bgp_incident_layer(
+    bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"),
+    zoom: int = Query(..., ge=1, le=20),
+    limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1),
+    severity: Optional[str] = Query(None),
+    status: Optional[str] = Query("active"),
+    db: AsyncSession = Depends(get_db),
+):
+    bounded_limit, _ = _clamp_limit(limit, zoom)
+    return _guard_geojson_layer(
+        await get_bgp_incidents_geojson(
+            severity=severity,
+            status=status,
+            limit=min(bounded_limit, 500),
+            db=db,
+        ),
+        bbox=_parse_layer_bbox(bbox),
+        zoom=zoom,
+        limit=limit,
+    )
+
+
+@router.get("/bgp/collectors")
+async def get_bgp_collector_layer(
+    bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"),
+    zoom: int = Query(..., ge=1, le=20),
+    limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1),
+    db: AsyncSession = Depends(get_db),
+):
+    return _guard_geojson_layer(
+        await get_bgp_collectors_geojson(db),
+        bbox=_parse_layer_bbox(bbox),
+        zoom=zoom,
+        limit=limit,
+    )

backend/app/api/v1/settings.py

@@ -81,6 +81,17 @@ DEFAULT_SETTINGS = {
         "password_policy": "medium",
     },
     "tv": DEFAULT_TV_SETTINGS,
+    "smtp": {
+        "host": "",
+        "port": 587,
+        "username": "",
+        "password": "",
+        "from_address": "",
+        "from_name": "Planet",
+        "use_tls": False,
+        "use_starttls": True,
+        "timeout_seconds": 20,
+    },
     "external_integrations": {
         "ai_provider": {
             "service_url": "",
@@ -95,6 +106,17 @@ DEFAULT_SETTINGS = {
             "default_provider": "tavily",
             "providers": {},
         },
+        "ocr": {
+            "enabled": False,
+            "provider": "paddleocr",
+            "base_url": "",
+            "api_key": "",
+            "model": "",
+            "languages": ["zh", "en"],
+            "timeout_seconds": 30,
+            "max_file_size_mb": 20,
+            "output_format": "markdown",
+        },
     },
 }
 
@@ -153,6 +175,24 @@ class TVSettingsUpdate(BaseModel):
     sources: list[TVStreamSourceUpdate] = Field(default_factory=list)
 
 
+class SMTPSettingsUpdate(BaseModel):
+    host: str = Field(default="", max_length=255)
+    port: int = Field(default=587, ge=1, le=65535)
+    username: str = Field(default="", max_length=255)
+    password: Optional[str] = None
+    clear_password: bool = False
+    from_address: str = Field(default="", max_length=255)
+    from_name: str = Field(default="Planet", max_length=120)
+    use_tls: bool = False
+    use_starttls: bool = True
+    timeout_seconds: int = Field(default=20, ge=3, le=300)
+
+
+class SMTPTestRequest(BaseModel):
+    to: EmailStr
+    settings: Optional[SMTPSettingsUpdate] = None
+
+
 class AIProviderIntegrationUpdate(BaseModel):
     service_url: str = ""
     service_token: Optional[str] = None
@@ -198,10 +238,23 @@ class WebSearchIntegrationUpdate(BaseModel):
     scrape_formats: list[str] = Field(default_factory=lambda: ["markdown"])
 
 
+class OCRIntegrationUpdate(BaseModel):
+    enabled: bool = False
+    provider: str = Field(default="paddleocr", max_length=80)
+    base_url: str = Field(default="", max_length=500)
+    api_key: Optional[str] = None
+    model: str = Field(default="", max_length=200)
+    languages: list[str] = Field(default_factory=lambda: ["zh", "en"])
+    timeout_seconds: int = Field(default=30, ge=3, le=300)
+    max_file_size_mb: int = Field(default=20, ge=1, le=200)
+    output_format: str = Field(default="markdown", pattern="^(markdown|text|json)$")
+
+
 class ExternalIntegrationsUpdate(BaseModel):
     ai_provider: AIProviderIntegrationUpdate
     barentswatch: BarentsWatchIntegrationUpdate
     web_search: WebSearchIntegrationUpdate | None = None
+    ocr: OCRIntegrationUpdate | None = None
 
 
 def merge_with_defaults(category: str, payload: Optional[dict]) -> dict:
@@ -649,6 +702,59 @@ def _runtime_config_from_web_search_payload(web_search_payload: dict) -> WebSear
     )
 
 
+def _normalize_ocr_payload(ocr_payload: dict | None) -> dict:
+    raw = dict(ocr_payload or {})
+    languages = raw.get("languages")
+    if not isinstance(languages, list) or not languages:
+        languages = ["zh", "en"]
+    return {
+        "enabled": bool(raw.get("enabled", False)),
+        "provider": str(raw.get("provider") or "paddleocr").strip().lower() or "paddleocr",
+        "base_url": str(raw.get("base_url") or "").strip(),
+        "api_key": str(raw.get("api_key") or "").strip(),
+        "model": str(raw.get("model") or "").strip(),
+        "languages": [str(item).strip() for item in languages if str(item).strip()],
+        "timeout_seconds": int(raw.get("timeout_seconds") or 30),
+        "max_file_size_mb": int(raw.get("max_file_size_mb") or 20),
+        "output_format": str(raw.get("output_format") or "markdown").strip() or "markdown",
+    }
+
+
+def _resolve_ocr_api_key(ocr_config: dict) -> tuple[str, str]:
+    saved_key = ocr_config.get("api_key") or ""
+    if saved_key:
+        return str(saved_key), "runtime"
+    return _resolve_web_search_env_secret("OCR_API_KEY")
+
+
+def _build_ocr_payload(
+    current_payload: dict,
+    update: OCRIntegrationUpdate | None,
+) -> dict:
+    current_ocr = _normalize_ocr_payload(current_payload.get("ocr") or {})
+    if update is None:
+        return current_ocr
+    current_key, current_key_source = _resolve_ocr_api_key(current_ocr)
+    current_key_preview = _mask_secret(current_key, current_key_source)["preview"]
+    ocr_payload = {
+        "enabled": update.enabled,
+        "provider": update.provider.strip().lower() or current_ocr.get("provider") or "paddleocr",
+        "base_url": update.base_url.strip(),
+        "model": update.model.strip(),
+        "languages": [item.strip() for item in update.languages if item.strip()] or ["zh", "en"],
+        "timeout_seconds": update.timeout_seconds,
+        "max_file_size_mb": update.max_file_size_mb,
+        "output_format": update.output_format.strip() or "markdown",
+    }
+    if not _is_secret_placeholder(update.api_key, current_key_preview):
+        ocr_payload["api_key"] = str(update.api_key).strip()
+    elif current_ocr.get("api_key"):
+        ocr_payload["api_key"] = current_ocr.get("api_key") or ""
+    else:
+        ocr_payload["api_key"] = ""
+    return ocr_payload
+
+
 async def get_runtime_web_search_config(db: AsyncSession) -> WebSearchConfig:
     runtime_record = await get_setting_record(db, "external_integrations")
     payload = merge_with_defaults(
@@ -684,6 +790,7 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
     )
     normalized_ai = _normalize_ai_provider_payload(raw_payload.get("ai_provider") or {})
     normalized_web_search = _normalize_web_search_payload(raw_payload.get("web_search") or {})
+    normalized_ocr = _normalize_ocr_payload(raw_payload.get("ocr") or {})
     default_provider = normalized_ai["default_provider"]
     providers_payload: dict[str, dict] = {}
     for provider in sorted({
@@ -731,6 +838,7 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
         web_search_providers_payload.get(normalized_web_search["default_provider"])
         or _web_search_provider_defaults(normalized_web_search["default_provider"])
     )
+    ocr_api_key, ocr_api_key_source = _resolve_ocr_api_key(normalized_ocr)
     barentswatch_record = await get_barentswatch_config_record(db)
     barentswatch_auth = barentswatch_record.auth_config if barentswatch_record else {}
     barentswatch_auth = barentswatch_auth or {}
@@ -782,6 +890,18 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
             "scrape_formats": display_web_search_config.get("scrape_formats") or ["markdown"],
             "source": "runtime" if runtime_setting else "env",
         },
+        "ocr": {
+            "enabled": normalized_ocr["enabled"],
+            "provider": normalized_ocr["provider"],
+            "base_url": normalized_ocr["base_url"],
+            "api_key": _mask_secret(ocr_api_key, ocr_api_key_source),
+            "model": normalized_ocr["model"],
+            "languages": normalized_ocr["languages"],
+            "timeout_seconds": normalized_ocr["timeout_seconds"],
+            "max_file_size_mb": normalized_ocr["max_file_size_mb"],
+            "output_format": normalized_ocr["output_format"],
+            "source": "runtime" if normalized_ocr.get("api_key") else (ocr_api_key_source or "default"),
+        },
     }
 
 
@@ -792,11 +912,12 @@ async def save_external_integrations_payload(
     current_payload = await get_setting_payload(db, "external_integrations")
     ai_payload = _build_ai_provider_payload(current_payload, update.ai_provider)
     web_search_payload = _build_web_search_payload(current_payload, update.web_search)
+    ocr_payload = _build_ocr_payload(current_payload, update.ocr)
 
     await save_setting_payload(
         db,
         "external_integrations",
-        {"ai_provider": ai_payload, "web_search": web_search_payload},
+        {"ai_provider": ai_payload, "web_search": web_search_payload, "ocr": ocr_payload},
     )
 
     default_endpoint = get_data_sources_config().get_yaml_url("barentswatch_vessels")
@@ -919,6 +1040,97 @@ async def update_security_settings(
     return {"status": "updated", "security": payload}
 
 
+def _serialize_smtp_payload(payload: dict) -> dict:
+    password = str(payload.get("password") or "")
+    return {
+        "host": payload.get("host") or "",
+        "port": int(payload.get("port") or 587),
+        "username": payload.get("username") or "",
+        "password": _mask_secret(password, "runtime" if password else ""),
+        "from_address": payload.get("from_address") or "",
+        "from_name": payload.get("from_name") or "Planet",
+        "use_tls": bool(payload.get("use_tls", False)),
+        "use_starttls": bool(payload.get("use_starttls", True)),
+        "timeout_seconds": int(payload.get("timeout_seconds") or 20),
+        "configured": bool(payload.get("host") and payload.get("from_address")),
+    }
+
+
+def _build_smtp_payload(current_payload: dict, update: SMTPSettingsUpdate) -> dict:
+    current_password = str(current_payload.get("password") or "")
+    current_preview = _mask_secret(current_password, "runtime" if current_password else "")["preview"]
+    if update.clear_password:
+        password = ""
+    elif _is_secret_placeholder(update.password, current_preview):
+        password = current_password
+    else:
+        password = str(update.password).strip()
+    return {
+        "host": update.host.strip(),
+        "port": update.port,
+        "username": update.username.strip(),
+        "password": password,
+        "from_address": update.from_address.strip(),
+        "from_name": update.from_name.strip() or "Planet",
+        "use_tls": update.use_tls,
+        "use_starttls": update.use_starttls,
+        "timeout_seconds": update.timeout_seconds,
+    }
+
+
+@router.get("/smtp")
+async def get_smtp_settings(
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    return {"smtp": _serialize_smtp_payload(await get_setting_payload(db, "smtp"))}
+
+
+@router.put("/smtp")
+async def update_smtp_settings(
+    payload: SMTPSettingsUpdate,
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    if current_user.role not in ("admin", "super_admin"):
+        raise HTTPException(status_code=403, detail="Only administrators can change SMTP settings")
+    current = await get_setting_payload(db, "smtp")
+    merged = _build_smtp_payload(current, payload)
+    saved = await save_setting_payload(db, "smtp", merged)
+    return {"status": "updated", "smtp": _serialize_smtp_payload(saved)}
+
+
+@router.post("/smtp/test")
+async def test_smtp_settings(
+    payload: SMTPTestRequest,
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    if current_user.role not in ("admin", "super_admin"):
+        raise HTTPException(status_code=403, detail="Only administrators can test SMTP settings")
+    from app.services.email import EmailError, send_email
+
+    current = await get_setting_payload(db, "smtp")
+    config = _build_smtp_payload(current, payload.settings) if payload.settings else current
+    if not config.get("host") or not config.get("from_address"):
+        raise HTTPException(
+            status_code=400,
+            detail="Host and from_address are required to send a test email",
+        )
+    try:
+        await send_email(
+            db,
+            to=payload.to,
+            subject="Planet SMTP test",
+            text_body="This is a test email from Planet SMTP settings.",
+            html_body="<p>This is a test email from Planet SMTP settings.</p>",
+            config=config,
+        )
+    except EmailError as exc:
+        return {"success": False, "message": str(exc), "code": exc.code}
+    return {"success": True, "message": "Test email sent"}
+
+
 @router.get("/tv")
 async def get_tv_settings(
     current_user: User = Depends(get_current_user),
@@ -1033,10 +1245,11 @@ async def connect_ai_provider_integration(
             )
         )
         current_web_search = _normalize_web_search_payload(current_payload.get("web_search") or {})
+        current_ocr = _normalize_ocr_payload(current_payload.get("ocr") or {})
         await save_setting_payload(
             db,
             "external_integrations",
-            {"ai_provider": draft_ai_payload, "web_search": current_web_search},
+            {"ai_provider": draft_ai_payload, "web_search": current_web_search, "ocr": current_ocr},
         )
         return {
             "success": True,
@@ -1110,6 +1323,21 @@ async def reveal_web_search_secrets(
     }
 
 
+@router.get("/integrations/ocr/secrets")
+async def reveal_ocr_secrets(
+    current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
+):
+    current_payload = await get_setting_payload(db, "external_integrations")
+    ocr_payload = _normalize_ocr_payload(current_payload.get("ocr") or {})
+    api_key, api_key_source = _resolve_ocr_api_key(ocr_payload)
+    return {
+        "provider": ocr_payload["provider"],
+        "api_key": api_key,
+        "api_key_source": api_key_source,
+    }
+
+
 @router.post("/integrations/web-search/connect")
 async def connect_web_search_integration(
     payload: WebSearchIntegrationUpdate,

backend/app/api/v1/vessels.py

@@ -0,0 +1,39 @@
+"""Bounded vessel snapshot APIs for viewport-first consumers."""
+
+from typing import Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.api.v1.visualization import _parse_bbox, build_vessel_snapshot_response
+from app.db.session import get_db
+from app.services.vessel_ais_aggregation import MAX_SNAPSHOT_LIMIT
+
+router = APIRouter()
+
+
+@router.get("/snapshot")
+async def get_vessel_snapshot(
+    bbox: Optional[str] = Query(None, description="Viewport bbox as lon_min,lat_min,lon_max,lat_max"),
+    zoom: int = Query(..., ge=1, le=20, description="Current map zoom level"),
+    type: Optional[str] = Query(
+        None,
+        description="Comma-separated vessel types: cargo,tanker,passenger,fishing,military,other",
+    ),
+    limit: int = Query(1000, ge=1, le=MAX_SNAPSHOT_LIMIT),
+    since_minutes: int = Query(60, ge=1, le=1440),
+    db: AsyncSession = Depends(get_db),
+):
+    if not bbox:
+        raise HTTPException(status_code=400, detail="bbox is required")
+    parsed_bbox = _parse_bbox(bbox)
+    if parsed_bbox is None:
+        raise HTTPException(status_code=400, detail="bbox is required")
+    return await build_vessel_snapshot_response(
+        db,
+        bbox=parsed_bbox,
+        zoom=zoom,
+        type_filter=type,
+        limit=limit,
+        since_minutes=since_minutes,
+    )

backend/app/api/v1/visualization.py

@@ -51,8 +51,10 @@ from app.services.vessel_ais_aggregation import (
     get_aggregated_vessel,
     get_aggregated_vessel_track,
     get_aggregated_vessels,
+    get_aggregated_vessels_snapshot,
     get_vessel_conflict_records,
     get_vessel_raw_observations,
+    MAX_SNAPSHOT_LIMIT,
 )
 from app.core.logging import get_logger
 
@@ -960,6 +962,52 @@ def _build_vessel_stats(features: List[dict[str, Any]]) -> dict[str, Any]:
     }
 
 
+def _safe_vessel_limit(value: int | None, *, default: int = 1000) -> int:
+    if value is None or value <= 0:
+        return default
+    return min(value, MAX_SNAPSHOT_LIMIT)
+
+
+async def build_vessel_snapshot_response(
+    db: AsyncSession,
+    *,
+    bbox: tuple[float, float, float, float] | None,
+    zoom: int | None,
+    type_filter: str | None,
+    limit: int | None,
+    since_minutes: int = 60,
+) -> dict[str, Any]:
+    requested_types = _requested_vessel_types(type_filter)
+    safe_limit = _safe_vessel_limit(limit)
+    safe_since_minutes = min(max(int(since_minutes or 60), 1), 1440)
+    observed_since = datetime.now(UTC) - timedelta(minutes=safe_since_minutes)
+    features, diagnostics = await _load_raw_vessel_snapshot_features(
+        db,
+        bbox=bbox,
+        limit=safe_limit,
+        observed_since=observed_since,
+    )
+    features = _filter_vessel_features(
+        features,
+        bbox=bbox,
+        requested_types=requested_types,
+    )[:safe_limit]
+    return {
+        "type": "FeatureCollection",
+        "features": features,
+        "count": len(features),
+        "stats": _build_vessel_stats(features),
+        "diagnostics": {
+            **diagnostics,
+            "filtered_count": len(features),
+            "bbox_applied": bbox is not None,
+            "zoom": zoom,
+            "limit": safe_limit,
+            "since_minutes": safe_since_minutes,
+        },
+    }
+
+
 def convert_bgp_anomalies_to_geojson(
     records: List[BGPAnomaly],
     geography_hints: Optional[Dict[str, Dict[str, Any]]] = None,
@@ -2042,66 +2090,58 @@ async def get_vessels_geojson(
     ),
     db: AsyncSession = Depends(get_db),
 ):
-    """Return latest vessel positions as GeoJSON points."""
-    parsed_bbox = _parse_bbox(bbox)
-    requested_types = _requested_vessel_types(type)
-    merged_features, diagnostics = await _load_merged_vessel_features(db)
-    features = _filter_vessel_features(
-        merged_features,
-        bbox=parsed_bbox,
-        requested_types=requested_types,
+    """Legacy vessel endpoint removed in favor of /api/v1/vessels/snapshot."""
+    raise HTTPException(
+        status_code=410,
+        detail=(
+            "Legacy vessel GeoJSON endpoint has been removed. "
+            "Use /api/v1/vessels/snapshot with bbox, zoom, and limit."
+        ),
     )
-    if limit and limit > 0:
-        features = features[:limit]
-    return {
-        "type": "FeatureCollection",
-        "features": features,
-        "count": len(features),
-        "stats": _build_vessel_stats(features),
-        "diagnostics": {
-            **diagnostics,
-            "filtered_count": len(features),
-        },
-    }
 
 
-async def _load_merged_vessel_features(db: AsyncSession) -> tuple[list[dict[str, Any]], dict[str, Any]]:
-    aggregated_vessels = await get_aggregated_vessels(db)
+async def _load_raw_vessel_snapshot_features(
+    db: AsyncSession,
+    *,
+    bbox: tuple[float, float, float, float] | None,
+    limit: int,
+    observed_since: datetime,
+) -> tuple[list[dict[str, Any]], dict[str, Any]]:
+    if bbox is None:
+        aggregated_vessels = await get_aggregated_vessels(
+            db,
+            limit=limit,
+            observed_since=observed_since,
+        )
+    else:
+        aggregated_vessels = await get_aggregated_vessels_snapshot(
+            db,
+            bbox=bbox,
+            limit=limit,
+            observed_since=observed_since,
+        )
     raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels)
-
-    latest_times = (
-        select(
-            VesselPosition.mmsi.label("mmsi"),
-            func.max(VesselPosition.received_at).label("received_at"),
-        )
-        .group_by(VesselPosition.mmsi)
-        .subquery()
-    )
-    stmt = (
-        select(VesselPosition, VesselStatic)
-        .join(
-            latest_times,
-            (VesselPosition.mmsi == latest_times.c.mmsi)
-            & (VesselPosition.received_at == latest_times.c.received_at),
-        )
-        .outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi)
-        .order_by(VesselPosition.received_at.desc())
-    )
-
-    result = await db.execute(stmt)
-    rows = list(result.all())
-    legacy_geojson = convert_vessels_to_geojson(rows)
-    merged_features, diagnostics = _merge_vessel_features(
-        raw_geojson.get("features", []),
-        legacy_geojson.get("features", []),
-    )
-    return merged_features, {
-        **diagnostics,
-        "raw_feature_count": len(raw_geojson.get("features", [])),
-        "legacy_feature_count": len(legacy_geojson.get("features", [])),
+    features = raw_geojson.get("features", [])
+    return features, {
+        "raw_feature_count": len(features),
+        "raw_unique_mmsi": len(
+            {
+                key
+                for key in (_feature_mmsi_key(feature) for feature in features)
+                if key is not None
+            }
+        ),
+        "legacy_feature_count": 0,
+        "legacy_backfilled_mmsi": 0,
+        "final_unique_mmsi": len(
+            {
+                key
+                for key in (_feature_mmsi_key(feature) for feature in features)
+                if key is not None
+            }
+        ),
     }
 
-
 @router.get("/vessels/custom-supplements")
 async def get_vessel_custom_supplements(db: AsyncSession = Depends(get_db)):
     """Group custom vessel_ais sources by their declared merge target for diagnostics."""

backend/app/api/v1/websocket.py

@@ -1,7 +1,6 @@
 """WebSocket API endpoints"""
 
 import asyncio
-import json
 from datetime import UTC, datetime
 from typing import Optional
 
@@ -95,14 +94,44 @@ async def websocket_endpoint(
                         }
                     )
                 elif data.get("type") == "subscribe":
-                    channels = data.get("data", {}).get("channels", [])
+                    payload_data = data.get("data", {})
+                    if not isinstance(payload_data, dict):
+                        payload_data = {}
+                    channels = payload_data.get("channels", [])
+                    if isinstance(channels, str):
+                        channels = [channels]
+                    elif not isinstance(channels, list):
+                        channels = []
+                    channel = payload_data.get("channel")
+                    if channel and channel not in channels:
+                        channels = [*channels, channel]
                     if is_anonymous:
                         channels = [channel for channel in channels if channel in supported_channels]
+                    vessel_subscription = None
+                    if "vessels" in channels and "bbox" in payload_data:
+                        try:
+                            vessel_subscription = manager.subscribe_vessels(websocket, payload_data)
+                        except ValueError as exc:
+                            await websocket.send_json(
+                                {
+                                    "type": "subscription_error",
+                                    "data": {"channel": "vessels", "detail": str(exc)},
+                                }
+                            )
+                            continue
+                        channels = [channel for channel in channels if channel != "vessels"]
                     manager.subscribe(websocket, channels)
                     await websocket.send_json(
                         {
                             "type": "subscription_confirmed",
-                            "data": {"action": "subscribe", "channels": channels},
+                            "data": {
+                                "action": "subscribe",
+                                "channels": [
+                                    *channels,
+                                    *(["vessels"] if vessel_subscription else []),
+                                ],
+                                "vessels": vessel_subscription,
+                            },
                         }
                     )
                 elif data.get("type") == "unsubscribe":