From b87cb310fd7f2ac4cc5e9f00a6faaa2637da2595 Mon Sep 17 00:00:00 2001 From: linkong Date: Tue, 12 May 2026 17:15:02 +0800 Subject: [PATCH] release: bump version to 0.52.0 --- .claude/commands/docs.md | 10 + VERSION | 2 +- backend/app/api/main.py | 7 +- backend/app/api/v1/auth.py | 269 +++++- backend/app/api/v1/data_products.py | 98 ++ backend/app/api/v1/datasources.py | 403 ++++++--- backend/app/api/v1/layers.py | 229 +++++ backend/app/api/v1/settings.py | 232 ++++- backend/app/api/v1/vessels.py | 39 + backend/app/api/v1/visualization.py | 148 +-- backend/app/api/v1/websocket.py | 35 +- backend/app/core/websocket/broadcaster.py | 61 +- backend/app/core/websocket/manager.py | 105 ++- backend/app/db/session.py | 40 +- backend/app/models/user.py | 2 + backend/app/schemas/user.py | 27 + backend/app/services/collectors/aisstream.py | 2 +- backend/app/services/docs_gatekeeper.py | 1 + backend/app/services/email.py | 123 +++ backend/app/services/otp.py | 100 +++ .../app/services/vessel_ais_aggregation.py | 45 + backend/tests/test_datasources_batch.py | 86 ++ backend/tests/test_layers.py | 44 + backend/tests/test_otp_service.py | 106 +++ backend/tests/test_settings_ai_provider.py | 40 + backend/tests/test_settings_smtp.py | 86 ++ backend/tests/test_vessels.py | 191 ++-- backend/tests/test_websocket_manager.py | 92 ++ docs/CHANGELOG.md | 16 + docs/documentation-coverage-rules.md | 10 + ...data-products-layer-guard-redesign-plan.md | 146 +++ docs/plans/docs-audience-split-plan.md | 116 +++ .../earth-vessel-ais-aggregation-plan.md | 20 +- ...earth-vessel-rendering-performance-plan.md | 2 +- ...er-registration-email-verification-plan.md | 155 ++++ docs/technical/en/backend-collectors.md | 49 +- ...asource-collector-settings-connectivity.md | 10 + docs/technical/en/earth-frontend-context.md | 6 +- .../en/frontend-admin-frontend-context.md | 27 +- docs/technical/en/manual.md | 816 +++++------------ docs/technical/en/ops-runbook.md | 244 +++++ docs/technical/en/quickstart.md | 248 +---- docs/technical/zh/backend-collectors.md | 49 +- ...asource-collector-settings-connectivity.md | 10 + docs/technical/zh/earth-frontend-context.md | 9 +- .../zh/frontend-admin-frontend-context.md | 27 +- docs/technical/zh/manual.md | 846 +++++------------- docs/technical/zh/ops-runbook.md | 244 +++++ docs/technical/zh/quickstart.md | 248 +---- docs/version-history.md | 3 +- frontend/package.json | 2 +- frontend/public/earth/js/constants.js | 2 +- frontend/public/earth/js/vessels.js | 23 +- frontend/src/App.tsx | 17 +- .../ConnectionTestInput.tsx | 107 +++ frontend/src/index.css | 54 +- frontend/src/pages/AISettings/AISettings.tsx | 419 ++++++--- .../src/pages/DataSources/DataSources.tsx | 194 +++- frontend/src/pages/Docs/docs-content.ts | 4 + .../pages/ForgotPassword/ForgotPassword.tsx | 144 +++ frontend/src/pages/Login/Login.tsx | 29 +- frontend/src/pages/Playground/Playground.tsx | 42 +- frontend/src/pages/Register/Register.tsx | 195 ++++ frontend/src/pages/Settings/Settings.tsx | 219 ++--- frontend/src/pages/Settings/SmtpPanel.tsx | 208 +++++ .../src/pages/VerifyEmail/VerifyEmail.tsx | 121 +++ pyproject.toml | 3 +- rules.md | 1 + uv.lock | 13 +- websocket_protocol.md | 55 +- 70 files changed, 5589 insertions(+), 2187 deletions(-) create mode 100644 backend/app/api/v1/data_products.py create mode 100644 backend/app/api/v1/layers.py create mode 100644 backend/app/api/v1/vessels.py create mode 100644 backend/app/services/email.py create mode 100644 backend/app/services/otp.py create mode 100644 backend/tests/test_datasources_batch.py create mode 100644 backend/tests/test_layers.py create mode 100644 backend/tests/test_otp_service.py create mode 100644 backend/tests/test_settings_smtp.py create mode 100644 docs/plans/data-products-layer-guard-redesign-plan.md create mode 100644 docs/plans/docs-audience-split-plan.md create mode 100644 docs/plans/user-registration-email-verification-plan.md create mode 100644 docs/technical/en/ops-runbook.md create mode 100644 docs/technical/zh/ops-runbook.md create mode 100644 frontend/src/components/ConnectionTestInput/ConnectionTestInput.tsx create mode 100644 frontend/src/pages/ForgotPassword/ForgotPassword.tsx create mode 100644 frontend/src/pages/Register/Register.tsx create mode 100644 frontend/src/pages/Settings/SmtpPanel.tsx create mode 100644 frontend/src/pages/VerifyEmail/VerifyEmail.tsx diff --git a/.claude/commands/docs.md b/.claude/commands/docs.md index e67c2b97..647bddb1 100644 --- a/.claude/commands/docs.md +++ b/.claude/commands/docs.md @@ -46,6 +46,16 @@ rg -n "class |def |function |export |router|@router|interface |type " - Keep filenames lowercase and hyphenated. - Apply the repository-specific rules file before writing. +#### Document Audience Routing (Planet) + +In this repository, classify the action's performer before picking a target file: + +- Browser/UI end user → `docs/technical/{zh,en}/manual.md` or `quickstart.md`. +- Shell / Docker / log paths / `planet.sh` / SMTP fallbacks / port forwarding → `docs/technical/{zh,en}/ops-runbook.md` (or an existing `ops-*.md`). +- Second-party developers → existing `*-context.md` / `backend-*.md` / `earth-*.md` files. + +Never put shell commands, log paths, or Docker operations into `manual.md` / `quickstart.md`. Never put UI button labels or screenshots into `ops-*.md`. When the same action has both a UI and a CLI path, write each in its own home and cross-link them with one sentence. + For ambiguous or large documentation changes, briefly state the intended doc plan before editing. For clear small changes, proceed directly. ### Step 3 — Write diff --git a/VERSION b/VERSION index 1e0c609c..4f9b378b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.51.1 +0.52.0 diff --git a/backend/app/api/main.py b/backend/app/api/main.py index 26fb73b4..8e30980e 100644 --- a/backend/app/api/main.py +++ b/backend/app/api/main.py @@ -8,12 +8,14 @@ from app.api.v1 import ( docs, tasks, dashboard, - websocket, alerts, settings, collected_data, + data_products, + layers, visualization, vessel_aggregation, + vessels, bgp, news, system_control, @@ -36,12 +38,15 @@ api_router.include_router(dashboard.router, prefix="/dashboard", tags=["dashboar api_router.include_router(alerts.router, prefix="/alerts", tags=["alerts"]) api_router.include_router(settings.router, prefix="/settings", tags=["settings"]) api_router.include_router(system_control.router, prefix="/system", tags=["system"]) +api_router.include_router(data_products.router, prefix="/data-products", tags=["data-products"]) +api_router.include_router(layers.router, prefix="/layers", tags=["layers"]) api_router.include_router(visualization.router, prefix="/visualization", tags=["visualization"]) api_router.include_router( vessel_aggregation.router, prefix="/vessel-aggregation", tags=["vessel-aggregation"], ) +api_router.include_router(vessels.router, prefix="/vessels", tags=["vessels"]) api_router.include_router(bgp.router, prefix="/bgp", tags=["bgp"]) api_router.include_router(tv.router, prefix="/tv", tags=["tv"]) api_router.include_router(news.router, prefix="/news", tags=["news"]) diff --git a/backend/app/api/v1/auth.py b/backend/app/api/v1/auth.py index 379e4853..ebd23d8e 100644 --- a/backend/app/api/v1/auth.py +++ b/backend/app/api/v1/auth.py @@ -1,26 +1,85 @@ -from datetime import timedelta - from fastapi import APIRouter, Depends, HTTPException, status from fastapi.security import OAuth2PasswordRequestForm from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import text from app.core.config import settings +from app.core.logging import get_logger from app.core.security import ( create_access_token, create_refresh_token, - blacklist_token, get_current_user, + get_password_hash, verify_password, ) from app.db.session import get_db from app.models.user import User from app.schemas.token import Token -from app.schemas.user import UserCreate, UserResponse +from app.schemas.user import ( + ForgotPasswordRequest, + ResendCodeRequest, + ResetPasswordRequest, + UserRegister, + UserResponse, + VerifyEmailRequest, +) +from app.services import otp +from app.services.email import ( + EmailError, + EmailNotConfiguredError, + send_verification_email, +) + +logger = get_logger(__name__) router = APIRouter() +def _token_response(user: User) -> dict: + access_token = create_access_token(data={"sub": user.id}) + refresh = create_refresh_token(data={"sub": user.id}) + expires_in = ( + settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60 + if settings.ACCESS_TOKEN_EXPIRE_MINUTES > 0 + else None + ) + return { + "access_token": access_token, + "token_type": "bearer", + "expires_in": expires_in, + "refresh_token": refresh, + "user": { + "id": user.id, + "username": user.username, + "role": user.role, + "gatekeeper_groups": user.gatekeeper_groups or [], + }, + } + + +async def _load_user_by_email(db: AsyncSession, email: str) -> User | None: + result = await db.execute( + text( + "SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups, email_verified " + "FROM users WHERE email = :email" + ), + {"email": email}, + ) + row = result.fetchone() + if row is None: + return None + user = User() + user.id = row[0] + user.username = row[1] + user.email = row[2] + user.password_hash = row[3] + user.role = row[4] + user.is_active = row[5] + user.gatekeeper_groups = row[6] or [] + user.email_verified = bool(row[7]) + return user + + @router.post("/login", response_model=Token) async def login( form_data: OAuth2PasswordRequestForm = Depends(), @@ -28,7 +87,8 @@ async def login( ): result = await db.execute( text( - "SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE username = :username" + "SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups, email_verified " + "FROM users WHERE username = :username" ), {"username": form_data.username}, ) @@ -47,6 +107,7 @@ async def login( user.role = row[4] user.is_active = row[5] user.gatekeeper_groups = row[6] or [] + user.email_verified = bool(row[7]) if not verify_password(form_data.password, user.password_hash): raise HTTPException( @@ -58,25 +119,13 @@ async def login( status_code=status.HTTP_401_UNAUTHORIZED, detail="User is inactive", ) + if not user.email_verified: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail={"code": "EMAIL_NOT_VERIFIED", "email": user.email}, + ) - access_token = create_access_token(data={"sub": user.id}) - refresh_token = create_refresh_token(data={"sub": user.id}) - - expires_in = None - if settings.ACCESS_TOKEN_EXPIRE_MINUTES > 0: - expires_in = settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60 - - return { - "access_token": access_token, - "token_type": "bearer", - "expires_in": expires_in, - "user": { - "id": user.id, - "username": user.username, - "role": user.role, - "gatekeeper_groups": user.gatekeeper_groups or [], - }, - } + return _token_response(user) @router.post("/refresh", response_model=Token) @@ -116,5 +165,179 @@ async def get_me(current_user: User = Depends(get_current_user)): "role": current_user.role, "gatekeeper_groups": current_user.gatekeeper_groups or [], "is_active": current_user.is_active, + "email_verified": getattr(current_user, "email_verified", True), "created_at": current_user.created_at, } + + +async def _send_code_or_raise(db: AsyncSession, email: str, code: str, purpose: str) -> None: + try: + await send_verification_email(db, to=email, code=code, purpose=purpose) + except EmailNotConfiguredError as exc: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail={"code": exc.code, "message": str(exc)}, + ) from exc + except EmailError as exc: + logger.warning_event( + "SMTP send failed", + event="auth.email.send_failed", + context={"email": email, "purpose": purpose, "error": str(exc)}, + ) + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail={"code": exc.code, "message": str(exc)}, + ) from exc + + +@router.post("/register", status_code=status.HTTP_201_CREATED) +async def register(payload: UserRegister, db: AsyncSession = Depends(get_db)): + existing = await db.execute( + text("SELECT id, email_verified FROM users WHERE username = :u OR email = :e"), + {"u": payload.username, "e": payload.email}, + ) + row = existing.fetchone() + if row is not None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={"code": "USER_ALREADY_EXISTS", "message": "Username or email already in use"}, + ) + + user = User( + username=payload.username, + email=payload.email, + password_hash=get_password_hash(payload.password), + role="viewer", + is_active=True, + email_verified=False, + ) + db.add(user) + await db.commit() + + try: + code = otp.issue_code(payload.email, "register") + except otp.OtpResendRateLimited as exc: + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail={"code": exc.code, "retry_after_seconds": exc.retry_after_seconds}, + ) from exc + await _send_code_or_raise(db, payload.email, code, "register") + return {"status": "pending_verification", "email": payload.email} + + +@router.post("/verify-email", response_model=Token) +async def verify_email(payload: VerifyEmailRequest, db: AsyncSession = Depends(get_db)): + user = await _load_user_by_email(db, payload.email) + if user is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"code": "USER_NOT_FOUND"}, + ) + try: + otp.verify_code(payload.email, "register", payload.code) + except otp.OtpExpired as exc: + raise HTTPException( + status_code=status.HTTP_410_GONE, + detail={"code": exc.code, "message": str(exc)}, + ) from exc + except otp.OtpAttemptsExceeded as exc: + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail={"code": exc.code, "message": str(exc)}, + ) from exc + except otp.OtpInvalid as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": exc.code, "message": str(exc)}, + ) from exc + + await db.execute( + text("UPDATE users SET email_verified = TRUE WHERE id = :id"), + {"id": user.id}, + ) + await db.commit() + user.email_verified = True + return _token_response(user) + + +@router.post("/resend-code") +async def resend_code(payload: ResendCodeRequest, db: AsyncSession = Depends(get_db)): + user = await _load_user_by_email(db, payload.email) + if user is None: + # Avoid email enumeration; pretend success. + return {"status": "ok"} + if payload.purpose == "register" and user.email_verified: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={"code": "ALREADY_VERIFIED"}, + ) + try: + code = otp.issue_code(payload.email, payload.purpose) + except otp.OtpResendRateLimited as exc: + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail={"code": exc.code, "retry_after_seconds": exc.retry_after_seconds}, + ) from exc + await _send_code_or_raise(db, payload.email, code, payload.purpose) + return {"status": "ok"} + + +@router.post("/forgot-password") +async def forgot_password(payload: ForgotPasswordRequest, db: AsyncSession = Depends(get_db)): + user = await _load_user_by_email(db, payload.email) + if user is None: + # Don't leak whether an email is registered. + return {"status": "ok"} + try: + code = otp.issue_code(payload.email, "reset_password") + except otp.OtpResendRateLimited: + # Silently accept; the user can retry after the cooldown. + return {"status": "ok"} + try: + await send_verification_email(db, to=payload.email, code=code, purpose="reset_password") + except EmailNotConfiguredError as exc: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail={"code": exc.code, "message": str(exc)}, + ) from exc + except EmailError as exc: + logger.warning_event( + "SMTP send failed", + event="auth.email.send_failed", + context={"email": payload.email, "purpose": "reset_password", "error": str(exc)}, + ) + return {"status": "ok"} + + +@router.post("/reset-password") +async def reset_password(payload: ResetPasswordRequest, db: AsyncSession = Depends(get_db)): + user = await _load_user_by_email(db, payload.email) + if user is None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": "OTP_INVALID"}, + ) + try: + otp.verify_code(payload.email, "reset_password", payload.code) + except otp.OtpExpired as exc: + raise HTTPException( + status_code=status.HTTP_410_GONE, + detail={"code": exc.code, "message": str(exc)}, + ) from exc + except otp.OtpAttemptsExceeded as exc: + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail={"code": exc.code, "message": str(exc)}, + ) from exc + except otp.OtpInvalid as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": exc.code, "message": str(exc)}, + ) from exc + + await db.execute( + text("UPDATE users SET password_hash = :p, email_verified = TRUE WHERE id = :id"), + {"p": get_password_hash(payload.new_password), "id": user.id}, + ) + await db.commit() + return {"status": "ok"} diff --git a/backend/app/api/v1/data_products.py b/backend/app/api/v1/data_products.py new file mode 100644 index 00000000..4af43b66 --- /dev/null +++ b/backend/app/api/v1/data_products.py @@ -0,0 +1,98 @@ +from datetime import UTC, datetime + +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.v1.visualization import get_visualization_geo_summary +from app.core.time import to_iso8601_utc +from app.db.session import get_db + +router = APIRouter() + +PRODUCT_DEFINITIONS: dict[str, dict] = { + "vessels": { + "name": "船只", + "sources": ["aisstream_vessels", "barentswatch_vessels"], + "primary_stat_key": "vessel_count", + "stat_keys": ["vessel_count", "vessel_raw_unique_mmsi", "vessel_legacy_unique_mmsi"], + }, + "cables": { + "name": "海底光缆", + "sources": [ + "arcgis_cables", + "arcgis_landing_points", + "arcgis_cable_landing_relation", + "telegeography_cables", + "telegeography_landing", + "telegeography_systems", + "fao_landing_points", + ], + "primary_stat_key": "cable_count", + "stat_keys": ["cable_count", "landing_point_count"], + }, + "satellites": { + "name": "卫星", + "sources": ["celestrak_tle", "spacetrack_tle"], + "primary_stat_key": "satellite_count", + "stat_keys": ["satellite_count"], + }, + "bgp": { + "name": "BGP", + "sources": [ + "ris_live_bgp", + "bgpstream_bgp", + "iptoasn_prefix_geo", + "opengeofeed_prefix_geo", + "nro_delegated_prefix_geo", + ], + "primary_stat_key": "bgp_event_count", + "stat_keys": ["bgp_event_count", "bgp_incident_count", "bgp_anomaly_count", "bgp_collector_count"], + }, + "compute": { + "name": "算力", + "sources": ["top500", "epoch_ai_gpu"], + "primary_stat_key": "compute_center_count", + "stat_keys": ["compute_center_count", "supercomputer_count", "gpu_cluster_count"], + }, +} + + +def _build_product_status(product_id: str, summary: dict) -> dict: + definition = PRODUCT_DEFINITIONS[product_id] + stats = summary.get("stats", {}) + product_stats = {key: stats.get(key, 0) for key in definition["stat_keys"]} + total_count = int(product_stats.get(definition["primary_stat_key"]) or 0) + return { + "product_id": product_id, + "name": definition["name"], + "sources": definition["sources"], + "generated_at": summary.get("generated_at") or to_iso8601_utc(datetime.now(UTC)), + "total_count": total_count, + "stats": product_stats, + "build_state": "ready", + "stats_scope": "global", + "stats_freshness": "cached_or_indexed", + } + + +@router.get("") +async def list_data_products(db: AsyncSession = Depends(get_db)): + summary = await get_visualization_geo_summary(db) + return { + "generated_at": summary.get("generated_at"), + "data": [ + _build_product_status(product_id, summary) + for product_id in PRODUCT_DEFINITIONS + ], + } + + +@router.get("/{product_id}/status") +async def get_data_product_status( + product_id: str, + db: AsyncSession = Depends(get_db), +): + if product_id not in PRODUCT_DEFINITIONS: + raise HTTPException(status_code=404, detail="Unknown data product") + summary = await get_visualization_geo_summary(db) + return _build_product_status(product_id, summary) diff --git a/backend/app/api/v1/datasources.py b/backend/app/api/v1/datasources.py index 16d0239d..7c03a90a 100644 --- a/backend/app/api/v1/datasources.py +++ b/backend/app/api/v1/datasources.py @@ -3,7 +3,8 @@ from datetime import datetime, timedelta, timezone from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Query -from sqlalchemy import func, select, text +from pydantic import BaseModel, Field +from sqlalchemy import func, or_, select, text from sqlalchemy.ext.asyncio import AsyncSession from app.core.time import to_iso8601_utc @@ -27,6 +28,29 @@ from app.services.scheduler import ( router = APIRouter() STALE_RUNNING_TASK_TIMEOUT_MINUTES = 90 +PRODUCT_SOURCE_KEYWORDS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("vessels", ("vessel", "ais")), + ("cables", ("cable", "landing", "telegeography", "arcgis", "fao")), + ("satellites", ("tle", "satellite", "spacetrack", "celestrak")), + ("bgp", ("bgp", "asn", "prefix_geo", "opengeofeed", "nro")), + ("compute", ("top500", "gpu", "supercomputer", "compute")), + ("ai", ("huggingface", "epoch_ai")), + ("media", ("news", "tv", "live_stream")), +) + + +class DatasourceBatchTriggerRequest(BaseModel): + source_ids: list[int] = Field(default_factory=list) + force: bool = False + module: Optional[str] = None + product: Optional[str] = None + is_active: Optional[bool] = None + priority: Optional[str] = None + run_status: Optional[str] = None + collected: Optional[bool] = None + credential_status: Optional[str] = None + q: Optional[str] = None + def format_frequency_label(minutes: int) -> str: if minutes % 1440 == 0: @@ -47,6 +71,20 @@ def datasource_metadata(source: str) -> dict: } +def datasource_product_key(datasource: DataSource) -> str: + haystack = " ".join( + [ + datasource.source or "", + datasource.name or "", + datasource.collector_class or "", + ] + ).lower() + for product, keywords in PRODUCT_SOURCE_KEYWORDS: + if any(keyword in haystack for keyword in keywords): + return product + return "other" + + def is_due_for_collection(datasource: DataSource, now: datetime) -> bool: if datasource.last_run_at is None: return True @@ -110,6 +148,22 @@ async def _load_latest_task_ids( return {datasource_id: task_id for datasource_id, task_id in result.all()} +async def _load_collected_record_counts( + db: AsyncSession, + sources: list[str], +) -> dict[str, int]: + if not sources: + return {} + + result = await db.execute( + select(CollectedData.source, func.count(CollectedData.id)) + .where(CollectedData.source.in_(sources)) + .where(CollectedData.is_current.is_(True)) + .group_by(CollectedData.source) + ) + return {source: int(count or 0) for source, count in result.all()} + + async def _load_datasource_endpoint_overrides( db: AsyncSession, sources: list[str], @@ -161,6 +215,192 @@ async def _load_datasource_list_context( return running_tasks, endpoint_overrides +def _apply_datasource_query_filters( + query, + *, + module: Optional[str] = None, + is_active: Optional[bool] = None, + priority: Optional[str] = None, + run_status: Optional[str] = None, + q: Optional[str] = None, +) -> object: + if module: + query = query.where(DataSource.module == module) + if is_active is not None: + query = query.where(DataSource.is_active == is_active) + if priority: + query = query.where(DataSource.priority == priority) + if run_status and run_status not in {"running", "collected", "uncollected"}: + if run_status == "not_run": + query = query.where(DataSource.last_status.is_(None)) + else: + query = query.where(DataSource.last_status == run_status) + if q: + like_value = f"%{q.strip()}%" + query = query.where( + or_( + DataSource.name.ilike(like_value), + DataSource.source.ilike(like_value), + DataSource.collector_class.ilike(like_value), + ) + ) + return query + + +def _filter_datasources_in_memory( + datasources: list[DataSource], + *, + running_tasks: dict[int, CollectionTask], + record_counts: dict[str, int], + product: Optional[str] = None, + run_status: Optional[str] = None, + collected: Optional[bool] = None, + credential_status: Optional[str] = None, +) -> list[DataSource]: + filtered: list[DataSource] = [] + for datasource in datasources: + record_count = record_counts.get(datasource.source, 0) + if product and datasource_product_key(datasource) != product: + continue + if collected is not None and (record_count > 0) != collected: + continue + if credential_status: + metadata = datasource_metadata(datasource.source) + if metadata["credential_status"] != credential_status: + continue + if run_status == "running" and datasource.id not in running_tasks: + continue + if run_status == "collected" and record_count <= 0: + continue + if run_status == "uncollected" and record_count > 0: + continue + filtered.append(datasource) + return filtered + + +async def _trigger_datasource_batch( + db: AsyncSession, + datasources: list[DataSource], + *, + force: bool, +) -> dict: + if not datasources: + return { + "status": "noop", + "message": "No matching data sources to trigger", + "force": force, + "triggered": [], + "skipped": [], + "failed": [], + } + + previous_task_ids: dict[int, Optional[int]] = {} + triggered_sources: list[dict] = [] + skipped_sources: list[dict] = [] + failed_sources: list[dict] = [] + now = datetime.now(timezone.utc) + running_tasks = await _load_latest_running_tasks( + db, + [datasource.id for datasource in datasources], + ) + + for datasource in datasources: + if not datasource.is_active: + skipped_sources.append( + { + "id": datasource.id, + "source": datasource.source, + "name": datasource.name, + "reason": "disabled", + } + ) + continue + + running_task = running_tasks.get(datasource.id) + if running_task is not None: + if not force: + skipped_sources.append( + { + "id": datasource.id, + "source": datasource.source, + "name": datasource.name, + "reason": "already_running", + "task_id": running_task.id, + } + ) + continue + cancelled = await cancel_running_collector_now(datasource.source) + if not cancelled: + await rollback_orphaned_running_task(db, datasource, running_task) + + if not force and not is_due_for_collection(datasource, now): + skipped_sources.append( + { + "id": datasource.id, + "source": datasource.source, + "name": datasource.name, + "reason": "within_frequency_window", + "last_run_at": to_iso8601_utc(datasource.last_run_at), + "next_run_at": to_iso8601_utc( + datasource.last_run_at + timedelta(minutes=datasource.frequency_minutes) + ), + } + ) + continue + + previous_task_ids[datasource.id] = None + success = run_collector_now(datasource.source) + if not success: + failed_sources.append( + { + "id": datasource.id, + "source": datasource.source, + "name": datasource.name, + "reason": "trigger_failed", + } + ) + continue + + triggered_sources.append( + { + "id": datasource.id, + "source": datasource.source, + "name": datasource.name, + "task_id": None, + } + ) + + latest_task_ids = await _load_latest_task_ids( + db, + [datasource.id for datasource in datasources], + ) + for datasource_id in previous_task_ids: + previous_task_ids[datasource_id] = latest_task_ids.get(datasource_id) + + for _ in range(20): + await asyncio.sleep(0.1) + pending = [item for item in triggered_sources if item["task_id"] is None] + if not pending: + break + latest_task_ids = await _load_latest_task_ids( + db, + [item["id"] for item in pending], + ) + for item in pending: + task_id = latest_task_ids.get(item["id"]) + if task_id is not None and task_id != previous_task_ids.get(item["id"]): + item["task_id"] = task_id + + return { + "status": "triggered" if triggered_sources else "partial", + "message": f"Triggered {len(triggered_sources)} data sources", + "force": force, + "triggered": triggered_sources, + "skipped": skipped_sources, + "failed": failed_sources, + } + + async def get_datasource_record(db: AsyncSession, source_id: str) -> Optional[DataSource]: datasource = None try: @@ -355,16 +595,23 @@ async def list_datasources( module: Optional[str] = None, is_active: Optional[bool] = None, priority: Optional[str] = None, + product: Optional[str] = None, + run_status: Optional[str] = None, + collected: Optional[bool] = None, + credential_status: Optional[str] = None, + q: Optional[str] = None, current_user: User = Depends(get_current_user), db: AsyncSession = Depends(get_db), ): query = select(DataSource).order_by(DataSource.module, DataSource.id) - if module: - query = query.where(DataSource.module == module) - if is_active is not None: - query = query.where(DataSource.is_active == is_active) - if priority: - query = query.where(DataSource.priority == priority) + query = _apply_datasource_query_filters( + query, + module=module, + is_active=is_active, + priority=priority, + run_status=run_status, + q=q, + ) result = await db.execute(query) datasources = result.scalars().all() @@ -372,11 +619,22 @@ async def list_datasources( collector_list = [] config = get_data_sources_config() running_tasks, endpoint_overrides = await _load_datasource_list_context(db, datasources) + record_counts = await _load_collected_record_counts(db, [datasource.source for datasource in datasources]) + datasources = _filter_datasources_in_memory( + datasources, + running_tasks=running_tasks, + record_counts=record_counts, + product=product, + run_status=run_status, + collected=collected, + credential_status=credential_status, + ) for datasource in datasources: running_task = running_tasks.get(datasource.id) endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source) last_run_at = datasource.last_run_at last_status = datasource.last_status + collected_records = record_counts.get(datasource.source, 0) collector_list.append( { @@ -384,6 +642,7 @@ async def list_datasources( "source": datasource.source, "name": datasource.name, **datasource_metadata(datasource.source), + "product": datasource_product_key(datasource), "module": datasource.module, "priority": datasource.priority, "frequency": format_frequency_label(datasource.frequency_minutes), @@ -405,6 +664,8 @@ async def list_datasources( "phase_unit": running_task.phase_unit if running_task else None, "records_processed": running_task.records_processed if running_task else None, "total_records": running_task.total_records if running_task else None, + "collected_records": collected_records, + "has_collected_data": collected_records > 0, } ) @@ -419,110 +680,46 @@ async def trigger_all_datasources( ): result = await db.execute( select(DataSource) - .where(DataSource.is_active == True) + .where(DataSource.is_active.is_(True)) .order_by(DataSource.module, DataSource.id) ) datasources = result.scalars().all() + return await _trigger_datasource_batch(db, datasources, force=force) - if not datasources: - return { - "status": "noop", - "message": "No active data sources to trigger", - "triggered": [], - "skipped": [], - "failed": [], - } - previous_task_ids: dict[int, Optional[int]] = {} - triggered_sources: list[dict] = [] - skipped_sources: list[dict] = [] - failed_sources: list[dict] = [] - now = datetime.now(timezone.utc) - running_tasks = await _load_latest_running_tasks( - db, - [datasource.id for datasource in datasources], - ) - - for datasource in datasources: - running_task = running_tasks.get(datasource.id) - if running_task is not None: - skipped_sources.append( - { - "id": datasource.id, - "source": datasource.source, - "name": datasource.name, - "reason": "already_running", - "task_id": running_task.id, - } - ) - continue - - if not force and not is_due_for_collection(datasource, now): - skipped_sources.append( - { - "id": datasource.id, - "source": datasource.source, - "name": datasource.name, - "reason": "within_frequency_window", - "last_run_at": to_iso8601_utc(datasource.last_run_at), - "next_run_at": to_iso8601_utc( - datasource.last_run_at + timedelta(minutes=datasource.frequency_minutes) - ), - } - ) - continue - - previous_task_ids[datasource.id] = None - success = run_collector_now(datasource.source) - if not success: - failed_sources.append( - { - "id": datasource.id, - "source": datasource.source, - "name": datasource.name, - "reason": "trigger_failed", - } - ) - continue - - triggered_sources.append( - { - "id": datasource.id, - "source": datasource.source, - "name": datasource.name, - "task_id": None, - } +@router.post("/trigger-batch") +async def trigger_datasource_batch( + payload: DatasourceBatchTriggerRequest, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + query = select(DataSource).order_by(DataSource.module, DataSource.id) + if payload.source_ids: + query = query.where(DataSource.id.in_(payload.source_ids)) + else: + query = _apply_datasource_query_filters( + query, + module=payload.module, + is_active=payload.is_active, + priority=payload.priority, + run_status=payload.run_status, + q=payload.q, ) - latest_task_ids = await _load_latest_task_ids( - db, - [datasource.id for datasource in datasources], + result = await db.execute(query) + datasources = result.scalars().all() + running_tasks, _ = await _load_datasource_list_context(db, datasources) + record_counts = await _load_collected_record_counts(db, [datasource.source for datasource in datasources]) + datasources = _filter_datasources_in_memory( + datasources, + running_tasks=running_tasks, + record_counts=record_counts, + product=None if payload.source_ids else payload.product, + run_status=None if payload.source_ids else payload.run_status, + collected=None if payload.source_ids else payload.collected, + credential_status=None if payload.source_ids else payload.credential_status, ) - for datasource_id in previous_task_ids: - previous_task_ids[datasource_id] = latest_task_ids.get(datasource_id) - - for _ in range(20): - await asyncio.sleep(0.1) - pending = [item for item in triggered_sources if item["task_id"] is None] - if not pending: - break - latest_task_ids = await _load_latest_task_ids( - db, - [item["id"] for item in pending], - ) - for item in pending: - task_id = latest_task_ids.get(item["id"]) - if task_id is not None and task_id != previous_task_ids.get(item["id"]): - item["task_id"] = task_id - - return { - "status": "triggered" if triggered_sources else "partial", - "message": f"Triggered {len(triggered_sources)} data sources", - "force": force, - "triggered": triggered_sources, - "skipped": skipped_sources, - "failed": failed_sources, - } + return await _trigger_datasource_batch(db, datasources, force=payload.force) @router.get("/{source_id}") diff --git a/backend/app/api/v1/layers.py b/backend/app/api/v1/layers.py new file mode 100644 index 00000000..5d6ab164 --- /dev/null +++ b/backend/app/api/v1/layers.py @@ -0,0 +1,229 @@ +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.v1.visualization import ( + _parse_bbox, + get_bgp_anomalies_geojson, + get_bgp_collectors_geojson, + get_bgp_incidents_geojson, + get_cables_geojson, + get_landing_points_geojson, + get_satellites_geojson, +) +from app.api.v1.vessels import build_vessel_snapshot_response +from app.db.session import get_db + +router = APIRouter() + +DEFAULT_LAYER_LIMIT = 1000 +MAX_LAYER_LIMIT = 5000 +LOW_ZOOM_FEATURE_LIMIT = 500 + + +def _clamp_limit(limit: int, zoom: int) -> tuple[int, bool]: + clamped = min(max(limit, 1), MAX_LAYER_LIMIT) + if zoom <= 3: + return min(clamped, LOW_ZOOM_FEATURE_LIMIT), clamped != limit or clamped > LOW_ZOOM_FEATURE_LIMIT + return clamped, clamped != limit + + +def _coordinate_in_bbox(coord: Any, bbox: tuple[float, float, float, float]) -> bool: + if not isinstance(coord, (list, tuple)) or len(coord) < 2: + return False + try: + lon = float(coord[0]) + lat = float(coord[1]) + except (TypeError, ValueError): + return False + lon_min, lat_min, lon_max, lat_max = bbox + return lon_min <= lon <= lon_max and lat_min <= lat <= lat_max + + +def _geometry_intersects_bbox(geometry: dict, bbox: tuple[float, float, float, float]) -> bool: + coordinates = geometry.get("coordinates") + geometry_type = geometry.get("type") + if geometry_type == "Point": + return _coordinate_in_bbox(coordinates, bbox) + if geometry_type in {"LineString", "MultiPoint"}: + return any(_coordinate_in_bbox(coord, bbox) for coord in coordinates or []) + if geometry_type in {"Polygon", "MultiLineString"}: + return any( + _coordinate_in_bbox(coord, bbox) + for line in coordinates or [] + for coord in line + ) + if geometry_type == "MultiPolygon": + return any( + _coordinate_in_bbox(coord, bbox) + for polygon in coordinates or [] + for line in polygon + for coord in line + ) + return False + + +def _guard_geojson_layer( + geojson: dict, + *, + bbox: tuple[float, float, float, float], + zoom: int, + limit: int, +) -> dict: + bounded_limit, limit_clamped = _clamp_limit(limit, zoom) + features = [ + feature + for feature in geojson.get("features", []) + if _geometry_intersects_bbox(feature.get("geometry") or {}, bbox) + ] + visible_count = len(features) + returned_features = features[:bounded_limit] + return { + **geojson, + "features": returned_features, + "visible_count": visible_count, + "returned_count": len(returned_features), + "diagnostics": { + "bbox_limited": True, + "limit": bounded_limit, + "limit_clamped": limit_clamped, + "truncated": visible_count > len(returned_features), + "degraded": zoom <= 3 or visible_count > len(returned_features), + "stats_scope": "viewport", + }, + } + + +def _parse_layer_bbox(bbox: str) -> tuple[float, float, float, float]: + parsed = _parse_bbox(bbox) + if parsed is None: + raise HTTPException(status_code=400, detail="bbox is required") + return parsed + + +@router.get("/vessels/snapshot") +async def get_vessel_layer_snapshot( + bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"), + zoom: int = Query(..., ge=1, le=20), + limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1), + vessel_type: Optional[str] = Query(None, alias="type"), + since_minutes: int = Query(60, ge=1, le=1440), + db: AsyncSession = Depends(get_db), +): + parsed_bbox = _parse_layer_bbox(bbox) + return await build_vessel_snapshot_response( + db, + bbox=parsed_bbox, + zoom=zoom, + limit=limit, + vessel_type=vessel_type, + since_minutes=since_minutes, + ) + + +@router.get("/cables") +async def get_cable_layer( + bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"), + zoom: int = Query(..., ge=1, le=20), + limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1), + db: AsyncSession = Depends(get_db), +): + return _guard_geojson_layer( + await get_cables_geojson(db), + bbox=_parse_layer_bbox(bbox), + zoom=zoom, + limit=limit, + ) + + +@router.get("/landing-points") +async def get_landing_point_layer( + bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"), + zoom: int = Query(..., ge=1, le=20), + limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1), + db: AsyncSession = Depends(get_db), +): + return _guard_geojson_layer( + await get_landing_points_geojson(db), + bbox=_parse_layer_bbox(bbox), + zoom=zoom, + limit=limit, + ) + + +@router.get("/satellites") +async def get_satellite_layer( + bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"), + zoom: int = Query(..., ge=1, le=20), + limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1), + db: AsyncSession = Depends(get_db), +): + bounded_limit, _ = _clamp_limit(limit, zoom) + return _guard_geojson_layer( + await get_satellites_geojson(limit=bounded_limit, db=db), + bbox=_parse_layer_bbox(bbox), + zoom=zoom, + limit=limit, + ) + + +@router.get("/bgp/anomalies") +async def get_bgp_anomaly_layer( + bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"), + zoom: int = Query(..., ge=1, le=20), + limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1), + severity: Optional[str] = Query(None), + status: Optional[str] = Query("active"), + db: AsyncSession = Depends(get_db), +): + bounded_limit, _ = _clamp_limit(limit, zoom) + return _guard_geojson_layer( + await get_bgp_anomalies_geojson( + severity=severity, + status=status, + limit=bounded_limit, + db=db, + ), + bbox=_parse_layer_bbox(bbox), + zoom=zoom, + limit=limit, + ) + + +@router.get("/bgp/incidents") +async def get_bgp_incident_layer( + bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"), + zoom: int = Query(..., ge=1, le=20), + limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1), + severity: Optional[str] = Query(None), + status: Optional[str] = Query("active"), + db: AsyncSession = Depends(get_db), +): + bounded_limit, _ = _clamp_limit(limit, zoom) + return _guard_geojson_layer( + await get_bgp_incidents_geojson( + severity=severity, + status=status, + limit=min(bounded_limit, 500), + db=db, + ), + bbox=_parse_layer_bbox(bbox), + zoom=zoom, + limit=limit, + ) + + +@router.get("/bgp/collectors") +async def get_bgp_collector_layer( + bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"), + zoom: int = Query(..., ge=1, le=20), + limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1), + db: AsyncSession = Depends(get_db), +): + return _guard_geojson_layer( + await get_bgp_collectors_geojson(db), + bbox=_parse_layer_bbox(bbox), + zoom=zoom, + limit=limit, + ) diff --git a/backend/app/api/v1/settings.py b/backend/app/api/v1/settings.py index f65e8a6d..d21d4fd0 100644 --- a/backend/app/api/v1/settings.py +++ b/backend/app/api/v1/settings.py @@ -81,6 +81,17 @@ DEFAULT_SETTINGS = { "password_policy": "medium", }, "tv": DEFAULT_TV_SETTINGS, + "smtp": { + "host": "", + "port": 587, + "username": "", + "password": "", + "from_address": "", + "from_name": "Planet", + "use_tls": False, + "use_starttls": True, + "timeout_seconds": 20, + }, "external_integrations": { "ai_provider": { "service_url": "", @@ -95,6 +106,17 @@ DEFAULT_SETTINGS = { "default_provider": "tavily", "providers": {}, }, + "ocr": { + "enabled": False, + "provider": "paddleocr", + "base_url": "", + "api_key": "", + "model": "", + "languages": ["zh", "en"], + "timeout_seconds": 30, + "max_file_size_mb": 20, + "output_format": "markdown", + }, }, } @@ -153,6 +175,24 @@ class TVSettingsUpdate(BaseModel): sources: list[TVStreamSourceUpdate] = Field(default_factory=list) +class SMTPSettingsUpdate(BaseModel): + host: str = Field(default="", max_length=255) + port: int = Field(default=587, ge=1, le=65535) + username: str = Field(default="", max_length=255) + password: Optional[str] = None + clear_password: bool = False + from_address: str = Field(default="", max_length=255) + from_name: str = Field(default="Planet", max_length=120) + use_tls: bool = False + use_starttls: bool = True + timeout_seconds: int = Field(default=20, ge=3, le=300) + + +class SMTPTestRequest(BaseModel): + to: EmailStr + settings: Optional[SMTPSettingsUpdate] = None + + class AIProviderIntegrationUpdate(BaseModel): service_url: str = "" service_token: Optional[str] = None @@ -198,10 +238,23 @@ class WebSearchIntegrationUpdate(BaseModel): scrape_formats: list[str] = Field(default_factory=lambda: ["markdown"]) +class OCRIntegrationUpdate(BaseModel): + enabled: bool = False + provider: str = Field(default="paddleocr", max_length=80) + base_url: str = Field(default="", max_length=500) + api_key: Optional[str] = None + model: str = Field(default="", max_length=200) + languages: list[str] = Field(default_factory=lambda: ["zh", "en"]) + timeout_seconds: int = Field(default=30, ge=3, le=300) + max_file_size_mb: int = Field(default=20, ge=1, le=200) + output_format: str = Field(default="markdown", pattern="^(markdown|text|json)$") + + class ExternalIntegrationsUpdate(BaseModel): ai_provider: AIProviderIntegrationUpdate barentswatch: BarentsWatchIntegrationUpdate web_search: WebSearchIntegrationUpdate | None = None + ocr: OCRIntegrationUpdate | None = None def merge_with_defaults(category: str, payload: Optional[dict]) -> dict: @@ -649,6 +702,59 @@ def _runtime_config_from_web_search_payload(web_search_payload: dict) -> WebSear ) +def _normalize_ocr_payload(ocr_payload: dict | None) -> dict: + raw = dict(ocr_payload or {}) + languages = raw.get("languages") + if not isinstance(languages, list) or not languages: + languages = ["zh", "en"] + return { + "enabled": bool(raw.get("enabled", False)), + "provider": str(raw.get("provider") or "paddleocr").strip().lower() or "paddleocr", + "base_url": str(raw.get("base_url") or "").strip(), + "api_key": str(raw.get("api_key") or "").strip(), + "model": str(raw.get("model") or "").strip(), + "languages": [str(item).strip() for item in languages if str(item).strip()], + "timeout_seconds": int(raw.get("timeout_seconds") or 30), + "max_file_size_mb": int(raw.get("max_file_size_mb") or 20), + "output_format": str(raw.get("output_format") or "markdown").strip() or "markdown", + } + + +def _resolve_ocr_api_key(ocr_config: dict) -> tuple[str, str]: + saved_key = ocr_config.get("api_key") or "" + if saved_key: + return str(saved_key), "runtime" + return _resolve_web_search_env_secret("OCR_API_KEY") + + +def _build_ocr_payload( + current_payload: dict, + update: OCRIntegrationUpdate | None, +) -> dict: + current_ocr = _normalize_ocr_payload(current_payload.get("ocr") or {}) + if update is None: + return current_ocr + current_key, current_key_source = _resolve_ocr_api_key(current_ocr) + current_key_preview = _mask_secret(current_key, current_key_source)["preview"] + ocr_payload = { + "enabled": update.enabled, + "provider": update.provider.strip().lower() or current_ocr.get("provider") or "paddleocr", + "base_url": update.base_url.strip(), + "model": update.model.strip(), + "languages": [item.strip() for item in update.languages if item.strip()] or ["zh", "en"], + "timeout_seconds": update.timeout_seconds, + "max_file_size_mb": update.max_file_size_mb, + "output_format": update.output_format.strip() or "markdown", + } + if not _is_secret_placeholder(update.api_key, current_key_preview): + ocr_payload["api_key"] = str(update.api_key).strip() + elif current_ocr.get("api_key"): + ocr_payload["api_key"] = current_ocr.get("api_key") or "" + else: + ocr_payload["api_key"] = "" + return ocr_payload + + async def get_runtime_web_search_config(db: AsyncSession) -> WebSearchConfig: runtime_record = await get_setting_record(db, "external_integrations") payload = merge_with_defaults( @@ -684,6 +790,7 @@ async def serialize_external_integrations(db: AsyncSession) -> dict: ) normalized_ai = _normalize_ai_provider_payload(raw_payload.get("ai_provider") or {}) normalized_web_search = _normalize_web_search_payload(raw_payload.get("web_search") or {}) + normalized_ocr = _normalize_ocr_payload(raw_payload.get("ocr") or {}) default_provider = normalized_ai["default_provider"] providers_payload: dict[str, dict] = {} for provider in sorted({ @@ -731,6 +838,7 @@ async def serialize_external_integrations(db: AsyncSession) -> dict: web_search_providers_payload.get(normalized_web_search["default_provider"]) or _web_search_provider_defaults(normalized_web_search["default_provider"]) ) + ocr_api_key, ocr_api_key_source = _resolve_ocr_api_key(normalized_ocr) barentswatch_record = await get_barentswatch_config_record(db) barentswatch_auth = barentswatch_record.auth_config if barentswatch_record else {} barentswatch_auth = barentswatch_auth or {} @@ -782,6 +890,18 @@ async def serialize_external_integrations(db: AsyncSession) -> dict: "scrape_formats": display_web_search_config.get("scrape_formats") or ["markdown"], "source": "runtime" if runtime_setting else "env", }, + "ocr": { + "enabled": normalized_ocr["enabled"], + "provider": normalized_ocr["provider"], + "base_url": normalized_ocr["base_url"], + "api_key": _mask_secret(ocr_api_key, ocr_api_key_source), + "model": normalized_ocr["model"], + "languages": normalized_ocr["languages"], + "timeout_seconds": normalized_ocr["timeout_seconds"], + "max_file_size_mb": normalized_ocr["max_file_size_mb"], + "output_format": normalized_ocr["output_format"], + "source": "runtime" if normalized_ocr.get("api_key") else (ocr_api_key_source or "default"), + }, } @@ -792,11 +912,12 @@ async def save_external_integrations_payload( current_payload = await get_setting_payload(db, "external_integrations") ai_payload = _build_ai_provider_payload(current_payload, update.ai_provider) web_search_payload = _build_web_search_payload(current_payload, update.web_search) + ocr_payload = _build_ocr_payload(current_payload, update.ocr) await save_setting_payload( db, "external_integrations", - {"ai_provider": ai_payload, "web_search": web_search_payload}, + {"ai_provider": ai_payload, "web_search": web_search_payload, "ocr": ocr_payload}, ) default_endpoint = get_data_sources_config().get_yaml_url("barentswatch_vessels") @@ -919,6 +1040,97 @@ async def update_security_settings( return {"status": "updated", "security": payload} +def _serialize_smtp_payload(payload: dict) -> dict: + password = str(payload.get("password") or "") + return { + "host": payload.get("host") or "", + "port": int(payload.get("port") or 587), + "username": payload.get("username") or "", + "password": _mask_secret(password, "runtime" if password else ""), + "from_address": payload.get("from_address") or "", + "from_name": payload.get("from_name") or "Planet", + "use_tls": bool(payload.get("use_tls", False)), + "use_starttls": bool(payload.get("use_starttls", True)), + "timeout_seconds": int(payload.get("timeout_seconds") or 20), + "configured": bool(payload.get("host") and payload.get("from_address")), + } + + +def _build_smtp_payload(current_payload: dict, update: SMTPSettingsUpdate) -> dict: + current_password = str(current_payload.get("password") or "") + current_preview = _mask_secret(current_password, "runtime" if current_password else "")["preview"] + if update.clear_password: + password = "" + elif _is_secret_placeholder(update.password, current_preview): + password = current_password + else: + password = str(update.password).strip() + return { + "host": update.host.strip(), + "port": update.port, + "username": update.username.strip(), + "password": password, + "from_address": update.from_address.strip(), + "from_name": update.from_name.strip() or "Planet", + "use_tls": update.use_tls, + "use_starttls": update.use_starttls, + "timeout_seconds": update.timeout_seconds, + } + + +@router.get("/smtp") +async def get_smtp_settings( + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + return {"smtp": _serialize_smtp_payload(await get_setting_payload(db, "smtp"))} + + +@router.put("/smtp") +async def update_smtp_settings( + payload: SMTPSettingsUpdate, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + if current_user.role not in ("admin", "super_admin"): + raise HTTPException(status_code=403, detail="Only administrators can change SMTP settings") + current = await get_setting_payload(db, "smtp") + merged = _build_smtp_payload(current, payload) + saved = await save_setting_payload(db, "smtp", merged) + return {"status": "updated", "smtp": _serialize_smtp_payload(saved)} + + +@router.post("/smtp/test") +async def test_smtp_settings( + payload: SMTPTestRequest, + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + if current_user.role not in ("admin", "super_admin"): + raise HTTPException(status_code=403, detail="Only administrators can test SMTP settings") + from app.services.email import EmailError, send_email + + current = await get_setting_payload(db, "smtp") + config = _build_smtp_payload(current, payload.settings) if payload.settings else current + if not config.get("host") or not config.get("from_address"): + raise HTTPException( + status_code=400, + detail="Host and from_address are required to send a test email", + ) + try: + await send_email( + db, + to=payload.to, + subject="Planet SMTP test", + text_body="This is a test email from Planet SMTP settings.", + html_body="

This is a test email from Planet SMTP settings.

", + config=config, + ) + except EmailError as exc: + return {"success": False, "message": str(exc), "code": exc.code} + return {"success": True, "message": "Test email sent"} + + @router.get("/tv") async def get_tv_settings( current_user: User = Depends(get_current_user), @@ -1033,10 +1245,11 @@ async def connect_ai_provider_integration( ) ) current_web_search = _normalize_web_search_payload(current_payload.get("web_search") or {}) + current_ocr = _normalize_ocr_payload(current_payload.get("ocr") or {}) await save_setting_payload( db, "external_integrations", - {"ai_provider": draft_ai_payload, "web_search": current_web_search}, + {"ai_provider": draft_ai_payload, "web_search": current_web_search, "ocr": current_ocr}, ) return { "success": True, @@ -1110,6 +1323,21 @@ async def reveal_web_search_secrets( } +@router.get("/integrations/ocr/secrets") +async def reveal_ocr_secrets( + current_user: User = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + current_payload = await get_setting_payload(db, "external_integrations") + ocr_payload = _normalize_ocr_payload(current_payload.get("ocr") or {}) + api_key, api_key_source = _resolve_ocr_api_key(ocr_payload) + return { + "provider": ocr_payload["provider"], + "api_key": api_key, + "api_key_source": api_key_source, + } + + @router.post("/integrations/web-search/connect") async def connect_web_search_integration( payload: WebSearchIntegrationUpdate, diff --git a/backend/app/api/v1/vessels.py b/backend/app/api/v1/vessels.py new file mode 100644 index 00000000..091ca283 --- /dev/null +++ b/backend/app/api/v1/vessels.py @@ -0,0 +1,39 @@ +"""Bounded vessel snapshot APIs for viewport-first consumers.""" + +from typing import Optional + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.v1.visualization import _parse_bbox, build_vessel_snapshot_response +from app.db.session import get_db +from app.services.vessel_ais_aggregation import MAX_SNAPSHOT_LIMIT + +router = APIRouter() + + +@router.get("/snapshot") +async def get_vessel_snapshot( + bbox: Optional[str] = Query(None, description="Viewport bbox as lon_min,lat_min,lon_max,lat_max"), + zoom: int = Query(..., ge=1, le=20, description="Current map zoom level"), + type: Optional[str] = Query( + None, + description="Comma-separated vessel types: cargo,tanker,passenger,fishing,military,other", + ), + limit: int = Query(1000, ge=1, le=MAX_SNAPSHOT_LIMIT), + since_minutes: int = Query(60, ge=1, le=1440), + db: AsyncSession = Depends(get_db), +): + if not bbox: + raise HTTPException(status_code=400, detail="bbox is required") + parsed_bbox = _parse_bbox(bbox) + if parsed_bbox is None: + raise HTTPException(status_code=400, detail="bbox is required") + return await build_vessel_snapshot_response( + db, + bbox=parsed_bbox, + zoom=zoom, + type_filter=type, + limit=limit, + since_minutes=since_minutes, + ) diff --git a/backend/app/api/v1/visualization.py b/backend/app/api/v1/visualization.py index 79a0a669..2fa6e65d 100644 --- a/backend/app/api/v1/visualization.py +++ b/backend/app/api/v1/visualization.py @@ -51,8 +51,10 @@ from app.services.vessel_ais_aggregation import ( get_aggregated_vessel, get_aggregated_vessel_track, get_aggregated_vessels, + get_aggregated_vessels_snapshot, get_vessel_conflict_records, get_vessel_raw_observations, + MAX_SNAPSHOT_LIMIT, ) from app.core.logging import get_logger @@ -960,6 +962,52 @@ def _build_vessel_stats(features: List[dict[str, Any]]) -> dict[str, Any]: } +def _safe_vessel_limit(value: int | None, *, default: int = 1000) -> int: + if value is None or value <= 0: + return default + return min(value, MAX_SNAPSHOT_LIMIT) + + +async def build_vessel_snapshot_response( + db: AsyncSession, + *, + bbox: tuple[float, float, float, float] | None, + zoom: int | None, + type_filter: str | None, + limit: int | None, + since_minutes: int = 60, +) -> dict[str, Any]: + requested_types = _requested_vessel_types(type_filter) + safe_limit = _safe_vessel_limit(limit) + safe_since_minutes = min(max(int(since_minutes or 60), 1), 1440) + observed_since = datetime.now(UTC) - timedelta(minutes=safe_since_minutes) + features, diagnostics = await _load_raw_vessel_snapshot_features( + db, + bbox=bbox, + limit=safe_limit, + observed_since=observed_since, + ) + features = _filter_vessel_features( + features, + bbox=bbox, + requested_types=requested_types, + )[:safe_limit] + return { + "type": "FeatureCollection", + "features": features, + "count": len(features), + "stats": _build_vessel_stats(features), + "diagnostics": { + **diagnostics, + "filtered_count": len(features), + "bbox_applied": bbox is not None, + "zoom": zoom, + "limit": safe_limit, + "since_minutes": safe_since_minutes, + }, + } + + def convert_bgp_anomalies_to_geojson( records: List[BGPAnomaly], geography_hints: Optional[Dict[str, Dict[str, Any]]] = None, @@ -2042,66 +2090,58 @@ async def get_vessels_geojson( ), db: AsyncSession = Depends(get_db), ): - """Return latest vessel positions as GeoJSON points.""" - parsed_bbox = _parse_bbox(bbox) - requested_types = _requested_vessel_types(type) - merged_features, diagnostics = await _load_merged_vessel_features(db) - features = _filter_vessel_features( - merged_features, - bbox=parsed_bbox, - requested_types=requested_types, + """Legacy vessel endpoint removed in favor of /api/v1/vessels/snapshot.""" + raise HTTPException( + status_code=410, + detail=( + "Legacy vessel GeoJSON endpoint has been removed. " + "Use /api/v1/vessels/snapshot with bbox, zoom, and limit." + ), ) - if limit and limit > 0: - features = features[:limit] - return { - "type": "FeatureCollection", - "features": features, - "count": len(features), - "stats": _build_vessel_stats(features), - "diagnostics": { - **diagnostics, - "filtered_count": len(features), - }, - } -async def _load_merged_vessel_features(db: AsyncSession) -> tuple[list[dict[str, Any]], dict[str, Any]]: - aggregated_vessels = await get_aggregated_vessels(db) +async def _load_raw_vessel_snapshot_features( + db: AsyncSession, + *, + bbox: tuple[float, float, float, float] | None, + limit: int, + observed_since: datetime, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + if bbox is None: + aggregated_vessels = await get_aggregated_vessels( + db, + limit=limit, + observed_since=observed_since, + ) + else: + aggregated_vessels = await get_aggregated_vessels_snapshot( + db, + bbox=bbox, + limit=limit, + observed_since=observed_since, + ) raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels) - - latest_times = ( - select( - VesselPosition.mmsi.label("mmsi"), - func.max(VesselPosition.received_at).label("received_at"), - ) - .group_by(VesselPosition.mmsi) - .subquery() - ) - stmt = ( - select(VesselPosition, VesselStatic) - .join( - latest_times, - (VesselPosition.mmsi == latest_times.c.mmsi) - & (VesselPosition.received_at == latest_times.c.received_at), - ) - .outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi) - .order_by(VesselPosition.received_at.desc()) - ) - - result = await db.execute(stmt) - rows = list(result.all()) - legacy_geojson = convert_vessels_to_geojson(rows) - merged_features, diagnostics = _merge_vessel_features( - raw_geojson.get("features", []), - legacy_geojson.get("features", []), - ) - return merged_features, { - **diagnostics, - "raw_feature_count": len(raw_geojson.get("features", [])), - "legacy_feature_count": len(legacy_geojson.get("features", [])), + features = raw_geojson.get("features", []) + return features, { + "raw_feature_count": len(features), + "raw_unique_mmsi": len( + { + key + for key in (_feature_mmsi_key(feature) for feature in features) + if key is not None + } + ), + "legacy_feature_count": 0, + "legacy_backfilled_mmsi": 0, + "final_unique_mmsi": len( + { + key + for key in (_feature_mmsi_key(feature) for feature in features) + if key is not None + } + ), } - @router.get("/vessels/custom-supplements") async def get_vessel_custom_supplements(db: AsyncSession = Depends(get_db)): """Group custom vessel_ais sources by their declared merge target for diagnostics.""" diff --git a/backend/app/api/v1/websocket.py b/backend/app/api/v1/websocket.py index b4cbc903..d6a8bf94 100644 --- a/backend/app/api/v1/websocket.py +++ b/backend/app/api/v1/websocket.py @@ -1,7 +1,6 @@ """WebSocket API endpoints""" import asyncio -import json from datetime import UTC, datetime from typing import Optional @@ -95,14 +94,44 @@ async def websocket_endpoint( } ) elif data.get("type") == "subscribe": - channels = data.get("data", {}).get("channels", []) + payload_data = data.get("data", {}) + if not isinstance(payload_data, dict): + payload_data = {} + channels = payload_data.get("channels", []) + if isinstance(channels, str): + channels = [channels] + elif not isinstance(channels, list): + channels = [] + channel = payload_data.get("channel") + if channel and channel not in channels: + channels = [*channels, channel] if is_anonymous: channels = [channel for channel in channels if channel in supported_channels] + vessel_subscription = None + if "vessels" in channels and "bbox" in payload_data: + try: + vessel_subscription = manager.subscribe_vessels(websocket, payload_data) + except ValueError as exc: + await websocket.send_json( + { + "type": "subscription_error", + "data": {"channel": "vessels", "detail": str(exc)}, + } + ) + continue + channels = [channel for channel in channels if channel != "vessels"] manager.subscribe(websocket, channels) await websocket.send_json( { "type": "subscription_confirmed", - "data": {"action": "subscribe", "channels": channels}, + "data": { + "action": "subscribe", + "channels": [ + *channels, + *(["vessels"] if vessel_subscription else []), + ], + "vessels": vessel_subscription, + }, } ) elif data.get("type") == "unsubscribe": diff --git a/backend/app/core/websocket/broadcaster.py b/backend/app/core/websocket/broadcaster.py index 563c23de..4e295d5c 100644 --- a/backend/app/core/websocket/broadcaster.py +++ b/backend/app/core/websocket/broadcaster.py @@ -2,7 +2,7 @@ import asyncio from datetime import UTC, datetime -from typing import Dict, Any, Optional +from typing import Dict, Any from app.core.time import to_iso8601_utc from app.core.websocket.manager import manager @@ -15,6 +15,8 @@ class DataBroadcaster: def __init__(self): self.running = False self.tasks: Dict[str, asyncio.Task] = {} + self._pending_vessel_updates: Dict[str, Dict[str, Any]] = {} + self._vessel_flush_interval = 1.0 async def get_dashboard_stats(self) -> Dict[str, Any]: """Get dashboard statistics""" @@ -68,6 +70,9 @@ class DataBroadcaster: async def broadcast_custom(self, channel: str, data: Dict[str, Any]): """Broadcast custom data to a specific channel""" + if channel == "vessels": + self.enqueue_vessel_update(data) + return await manager.broadcast( { "type": "data_frame", @@ -78,6 +83,58 @@ class DataBroadcaster: channel=channel, ) + def enqueue_vessel_update(self, data: Dict[str, Any]): + vessels = data.get("vessels") if isinstance(data, dict) else None + if not isinstance(vessels, list): + return + source = data.get("source") + action = data.get("action") or "upsert" + created = data.get("created") + for vessel in vessels: + if not isinstance(vessel, dict): + continue + mmsi = vessel.get("mmsi") + if mmsi in (None, ""): + continue + self._pending_vessel_updates[str(mmsi)] = { + **vessel, + "_source": source, + "_action": action, + "_created": created, + } + + async def flush_vessel_updates(self): + if not self._pending_vessel_updates: + return + pending = self._pending_vessel_updates + self._pending_vessel_updates = {} + vessels = [] + for item in pending.values(): + vessel = dict(item) + source = vessel.pop("_source", None) + action = vessel.pop("_action", "upsert") + created = vessel.pop("_created", None) + vessel["source"] = source + vessel["action"] = action + vessel["created"] = created + vessels.append(vessel) + await manager.broadcast_vessels( + { + "action": "upsert", + "source": "mixed", + "created": None, + "vessels": vessels, + } + ) + + async def broadcast_vessels_periodically(self): + while self.running: + try: + await self.flush_vessel_updates() + except Exception: + pass + await asyncio.sleep(self._vessel_flush_interval) + async def broadcast_datasource_task_update(self, data: Dict[str, Any]): """Broadcast datasource task progress updates to connected clients.""" await manager.broadcast( @@ -95,6 +152,7 @@ class DataBroadcaster: if not self.running: self.running = True self.tasks["dashboard"] = asyncio.create_task(self.broadcast_stats(5)) + self.tasks["vessels"] = asyncio.create_task(self.broadcast_vessels_periodically()) def stop(self): """Stop all broadcasters""" @@ -102,6 +160,7 @@ class DataBroadcaster: for task in self.tasks.values(): task.cancel() self.tasks.clear() + self._pending_vessel_updates.clear() broadcaster = DataBroadcaster() diff --git a/backend/app/core/websocket/manager.py b/backend/app/core/websocket/manager.py index 57bad1ad..7d884f32 100644 --- a/backend/app/core/websocket/manager.py +++ b/backend/app/core/websocket/manager.py @@ -1,11 +1,16 @@ """WebSocket Connection Manager""" -from typing import Dict, Set, Optional +from datetime import UTC, datetime +from typing import Any, Dict, Set, Optional from fastapi import WebSocket import redis.asyncio as redis from app.core.config import settings +MAX_VESSEL_SUBSCRIPTION_LIMIT = 5000 +MAX_VESSEL_WS_MESSAGE_ITEMS = 1000 +MAX_VESSEL_BBOX_AREA = 2500.0 + class ConnectionManager: """Manages WebSocket connections""" @@ -14,6 +19,7 @@ class ConnectionManager: self.active_connections: Dict[str, Set[WebSocket]] = {} # user_id -> connections self.channel_subscriptions: Dict[str, Set[WebSocket]] = {} self.websocket_channels: Dict[WebSocket, Set[str]] = {} + self.vessel_subscriptions: Dict[WebSocket, dict[str, Any]] = {} self.redis_client: Optional[redis.Redis] = None async def connect(self, websocket: WebSocket, user_id: str): @@ -72,6 +78,50 @@ class ConnectionManager: channels = list(self.websocket_channels.get(websocket, set())) if channels: self.unsubscribe(websocket, channels) + self.vessel_subscriptions.pop(websocket, None) + + def subscribe_vessels(self, websocket: WebSocket, config: dict[str, Any]) -> dict[str, Any]: + subscription = self._normalize_vessel_subscription(config) + self.channel_subscriptions.setdefault("vessels", set()).add(websocket) + self.websocket_channels.setdefault(websocket, set()).add("vessels") + self.vessel_subscriptions[websocket] = subscription + return subscription + + def _normalize_vessel_subscription(self, config: dict[str, Any]) -> dict[str, Any]: + bbox = config.get("bbox") + if not isinstance(bbox, (list, tuple)) or len(bbox) != 4: + raise ValueError("vessels subscription requires bbox=[lon_min,lat_min,lon_max,lat_max]") + try: + lon_min, lat_min, lon_max, lat_max = [float(value) for value in bbox] + except (TypeError, ValueError) as exc: + raise ValueError("bbox values must be numbers") from exc + if lat_min > lat_max: + lat_min, lat_max = lat_max, lat_min + if lon_min > lon_max: + lon_min, lon_max = lon_max, lon_min + if not (-180 <= lon_min <= 180 and -180 <= lon_max <= 180): + raise ValueError("bbox longitude values must be between -180 and 180") + if not (-90 <= lat_min <= 90 and -90 <= lat_max <= 90): + raise ValueError("bbox latitude values must be between -90 and 90") + if (lon_max - lon_min) * (lat_max - lat_min) > MAX_VESSEL_BBOX_AREA: + raise ValueError("bbox is too large; zoom in or request a smaller viewport") + + zoom = int(config.get("zoom") or 1) + if zoom < 1 or zoom > 20: + raise ValueError("zoom must be between 1 and 20") + limit = min(max(int(config.get("limit") or 1000), 1), MAX_VESSEL_SUBSCRIPTION_LIMIT) + vessel_types = { + str(item).strip().lower() + for item in str(config.get("type") or "").split(",") + if str(item).strip() + } + return { + "bbox": (lon_min, lat_min, lon_max, lat_max), + "zoom": zoom, + "limit": limit, + "type": vessel_types, + "last_sent_at": None, + } async def send_personal_message(self, message: dict, user_id: str): if user_id in self.active_connections: @@ -92,6 +142,58 @@ class ConnectionManager: except Exception: self.unsubscribe_all(connection) + async def broadcast_vessels(self, data: dict[str, Any]): + vessels = data.get("vessels") if isinstance(data, dict) else None + if not isinstance(vessels, list) or not vessels: + return + + for connection, subscription in list(self.vessel_subscriptions.items()): + matched = [ + vessel + for vessel in vessels + if self._vessel_matches_subscription(vessel, subscription) + ][: min(subscription["limit"], MAX_VESSEL_WS_MESSAGE_ITEMS)] + if not matched: + continue + subscription["last_sent_at"] = datetime.now(UTC) + message = { + "type": "data_frame", + "channel": "vessels", + "timestamp": subscription["last_sent_at"].isoformat(), + "payload": { + **data, + "vessels": matched, + "subscription": { + "bbox": list(subscription["bbox"]), + "zoom": subscription["zoom"], + "limit": subscription["limit"], + }, + }, + } + try: + await connection.send_json(message) + except Exception: + self.unsubscribe_all(connection) + + def _vessel_matches_subscription( + self, + vessel: dict[str, Any], + subscription: dict[str, Any], + ) -> bool: + try: + lon = float(vessel.get("lon")) + lat = float(vessel.get("lat")) + except (TypeError, ValueError): + return False + lon_min, lat_min, lon_max, lat_max = subscription["bbox"] + if not (lon_min <= lon <= lon_max and lat_min <= lat <= lat_max): + return False + requested_types = subscription.get("type") or set() + if not requested_types: + return True + type_name = str(vessel.get("vessel_type_name") or "").lower() + return any(requested_type in type_name for requested_type in requested_types) + async def close_all(self): for user_id in self.active_connections: for connection in self.active_connections[user_id]: @@ -99,6 +201,7 @@ class ConnectionManager: self.active_connections.clear() self.channel_subscriptions.clear() self.websocket_channels.clear() + self.vessel_subscriptions.clear() manager = ConnectionManager() diff --git a/backend/app/db/session.py b/backend/app/db/session.py index 8fa9b3b4..6e9286bd 100644 --- a/backend/app/db/session.py +++ b/backend/app/db/session.py @@ -107,6 +107,7 @@ async def ensure_default_admin_user(session: AsyncSession): password_hash=get_password_hash(default_user["password"]), role=default_user["role"], is_active=True, + email_verified=True, ) ) await session.commit() @@ -148,14 +149,31 @@ async def init_db(): async with engine.begin() as conn: await conn.run_sync(Base.metadata.create_all) + users_email_verified_existed = ( + await conn.execute( + text( + """ + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'users' AND column_name = 'email_verified' + """ + ) + ) + ).fetchone() is not None await conn.execute( text( """ ALTER TABLE users - ADD COLUMN IF NOT EXISTS gatekeeper_groups JSONB DEFAULT '[]'::jsonb + ADD COLUMN IF NOT EXISTS gatekeeper_groups JSONB DEFAULT '[]'::jsonb, + ADD COLUMN IF NOT EXISTS email_verified BOOLEAN NOT NULL DEFAULT FALSE, + ADD COLUMN IF NOT EXISTS pending_email VARCHAR(255) """ ) ) + if not users_email_verified_existed: + await conn.execute( + text("UPDATE users SET email_verified = TRUE WHERE email_verified = FALSE") + ) await conn.execute( text( """ @@ -216,6 +234,26 @@ async def init_db(): """ ) ) + await conn.execute( + text( + """ + CREATE INDEX IF NOT EXISTS idx_ais_raw_schema_observed_desc + ON ais_raw_observations (target_schema, observed_at DESC) + """ + ) + ) + await conn.execute( + text( + """ + CREATE INDEX IF NOT EXISTS idx_ais_raw_payload_lon_lat + ON ais_raw_observations ( + ((normalized_payload->>'lon')::double precision), + ((normalized_payload->>'lat')::double precision) + ) + WHERE target_schema = 'vessel_ais' + """ + ) + ) await conn.execute( text( """ diff --git a/backend/app/models/user.py b/backend/app/models/user.py index b16f4c49..e06407b7 100644 --- a/backend/app/models/user.py +++ b/backend/app/models/user.py @@ -14,6 +14,8 @@ class User(Base): role = Column(String(20), default="viewer") gatekeeper_groups = Column(JSON, default=list) is_active = Column(Boolean, default=True) + email_verified = Column(Boolean, default=False, nullable=False) + pending_email = Column(String(255), nullable=True) last_login_at = Column(DateTime(timezone=True)) created_at = Column(DateTime(timezone=True), server_default=func.now()) updated_at = Column( diff --git a/backend/app/schemas/user.py b/backend/app/schemas/user.py index 9148d561..eaa0c272 100644 --- a/backend/app/schemas/user.py +++ b/backend/app/schemas/user.py @@ -39,7 +39,34 @@ class UserResponse(UserBase): role: str gatekeeper_groups: list[str] = Field(default_factory=list) is_active: bool + email_verified: bool = False created_at: datetime class Config: from_attributes = True + + +class UserRegister(BaseModel): + username: str = Field(..., min_length=3, max_length=50) + email: EmailStr + password: str = Field(..., min_length=8, max_length=128) + + +class VerifyEmailRequest(BaseModel): + email: EmailStr + code: str = Field(..., min_length=6, max_length=6) + + +class ResendCodeRequest(BaseModel): + email: EmailStr + purpose: str = Field(default="register", pattern="^(register|verify_email|reset_password)$") + + +class ForgotPasswordRequest(BaseModel): + email: EmailStr + + +class ResetPasswordRequest(BaseModel): + email: EmailStr + code: str = Field(..., min_length=6, max_length=6) + new_password: str = Field(..., min_length=8, max_length=128) diff --git a/backend/app/services/collectors/aisstream.py b/backend/app/services/collectors/aisstream.py index 3bd24362..eb9956f1 100644 --- a/backend/app/services/collectors/aisstream.py +++ b/backend/app/services/collectors/aisstream.py @@ -138,7 +138,7 @@ class AISStreamCollector(BaseCollector): try: import websockets - except ImportError as exc: + except ImportError: return {"status": "failed", "error": "Python package 'websockets' is required for AISStream"} start_time = datetime.now(UTC) diff --git a/backend/app/services/docs_gatekeeper.py b/backend/app/services/docs_gatekeeper.py index 807e5878..7fb69293 100644 --- a/backend/app/services/docs_gatekeeper.py +++ b/backend/app/services/docs_gatekeeper.py @@ -53,6 +53,7 @@ DOCS_METADATA: tuple[DocsMetadata, ...] = ( DocsMetadata("backend-datasources-api-performance.md", "backend-datasources-api-performance", "docs_developer", "Backend", 33, "数据源 API 性能", "Datasource API Performance"), DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 34, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"), DocsMetadata("agents-aiprovider.md", "agents-aiprovider", "docs_developer", "Agents", 40, "AI Provider 指南", "AI Provider Guide"), + DocsMetadata("ops-runbook.md", "ops-runbook", "docs_admin", "Ops", 49, "Planet 运维手册", "Planet Ops Runbook"), DocsMetadata("ops-docker-compose-buildx-upgrade.md", "ops-docker-compose-buildx-upgrade", "docs_admin", "Ops", 50, "Docker + Compose + Buildx 升级", "Docker + Compose + Buildx Upgrade"), DocsMetadata("ops-planet-sh-startup.md", "ops-planet-sh-startup", "docs_admin", "Ops", 51, "planet.sh 启动机制", "planet.sh Startup"), ) diff --git a/backend/app/services/email.py b/backend/app/services/email.py new file mode 100644 index 00000000..6db4fe32 --- /dev/null +++ b/backend/app/services/email.py @@ -0,0 +1,123 @@ +"""SMTP-backed email sender. + +Generic primitive used by registration/verification today, reusable for alert +digests and other notifications later. Configuration lives in the `smtp` row of +`system_settings` and is loaded once per send (small surface, no caching layer +yet to keep behavior obvious after settings changes). +""" + +from __future__ import annotations + +from email.message import EmailMessage +from typing import Literal, Optional + +import aiosmtplib +from sqlalchemy.ext.asyncio import AsyncSession + +OtpPurpose = Literal["register", "verify_email", "reset_password"] + + +class EmailError(Exception): + code: str = "EMAIL_ERROR" + + +class EmailNotConfiguredError(EmailError): + code = "EMAIL_PROVIDER_NOT_CONFIGURED" + + +class EmailSendError(EmailError): + code = "EMAIL_SEND_FAILED" + + +async def _load_smtp_config(db: AsyncSession) -> dict: + from app.api.v1.settings import get_setting_payload # local import avoids cycle + + payload = await get_setting_payload(db, "smtp") + if not payload.get("host") or not payload.get("from_address"): + raise EmailNotConfiguredError("SMTP host/from_address not set") + return payload + + +async def send_email( + db: AsyncSession, + *, + to: str, + subject: str, + text_body: str, + html_body: Optional[str] = None, + config: Optional[dict] = None, +) -> None: + cfg = config or await _load_smtp_config(db) + + message = EmailMessage() + from_name = (cfg.get("from_name") or "").strip() + from_address = cfg["from_address"] + message["From"] = f"{from_name} <{from_address}>" if from_name else from_address + message["To"] = to + message["Subject"] = subject + message.set_content(text_body) + if html_body: + message.add_alternative(html_body, subtype="html") + + use_tls = bool(cfg.get("use_tls", True)) + use_starttls = bool(cfg.get("use_starttls", False)) + port = int(cfg.get("port") or (465 if use_tls else 587)) + + try: + await aiosmtplib.send( + message, + hostname=cfg["host"], + port=port, + username=cfg.get("username") or None, + password=cfg.get("password") or None, + use_tls=use_tls and not use_starttls, + start_tls=use_starttls, + timeout=int(cfg.get("timeout_seconds") or 20), + ) + except aiosmtplib.SMTPException as exc: + raise EmailSendError(str(exc)) from exc + except OSError as exc: + raise EmailSendError(str(exc)) from exc + + +_SUBJECTS: dict[OtpPurpose, str] = { + "register": "Confirm your Planet account", + "verify_email": "Verify your Planet email", + "reset_password": "Reset your Planet password", +} + +_HEADLINES: dict[OtpPurpose, str] = { + "register": "Welcome to Planet — confirm your email to activate your account.", + "verify_email": "Confirm your new email address to keep your Planet account active.", + "reset_password": "Use this code to set a new password for your Planet account.", +} + + +async def send_verification_email( + db: AsyncSession, + *, + to: str, + code: str, + purpose: OtpPurpose, + config: Optional[dict] = None, +) -> None: + subject = _SUBJECTS[purpose] + headline = _HEADLINES[purpose] + text_body = ( + f"{headline}\n\n" + f"Your verification code: {code}\n" + "This code expires in 10 minutes. If you did not request it, ignore this email.\n" + ) + html_body = ( + f"

{headline}

" + f"

{code}

" + "

This code expires in 10 minutes. If you did not request it, ignore this email.

" + ) + await send_email( + db, + to=to, + subject=subject, + text_body=text_body, + html_body=html_body, + config=config, + ) diff --git a/backend/app/services/otp.py b/backend/app/services/otp.py new file mode 100644 index 00000000..4d9cafcf --- /dev/null +++ b/backend/app/services/otp.py @@ -0,0 +1,100 @@ +"""One-time verification codes backed by Redis. + +Reusable primitive for register/verify-email/reset-password (and any future 2FA or +phone-number verification). Codes are bcrypt-hashed before storage so a Redis dump +does not leak active codes. +""" + +from __future__ import annotations + +import json +import secrets +from typing import Literal + +import bcrypt + +from app.core.security import redis_client + +OtpPurpose = Literal["register", "verify_email", "reset_password"] + +CODE_TTL_SECONDS = 600 # 10 minutes +RESEND_COOLDOWN_SECONDS = 60 +MAX_ATTEMPTS = 5 +CODE_LENGTH = 6 + + +class OtpError(Exception): + code: str = "OTP_ERROR" + + +class OtpResendRateLimited(OtpError): + code = "OTP_RESEND_RATE_LIMITED" + + def __init__(self, retry_after_seconds: int) -> None: + super().__init__(f"Resend allowed in {retry_after_seconds}s") + self.retry_after_seconds = retry_after_seconds + + +class OtpInvalid(OtpError): + code = "OTP_INVALID" + + +class OtpExpired(OtpError): + code = "OTP_EXPIRED" + + +class OtpAttemptsExceeded(OtpError): + code = "OTP_ATTEMPTS_EXCEEDED" + + +def _code_key(email: str, purpose: OtpPurpose) -> str: + return f"otp:{purpose}:{email.lower()}" + + +def _rate_key(email: str, purpose: OtpPurpose) -> str: + return f"otp_rate:{purpose}:{email.lower()}" + + +def _generate_code() -> str: + # secrets.randbelow gives uniform 0..10**CODE_LENGTH-1 without modulo bias + return f"{secrets.randbelow(10 ** CODE_LENGTH):0{CODE_LENGTH}d}" + + +def check_resend_allowed(email: str, purpose: OtpPurpose) -> None: + ttl = redis_client.ttl(_rate_key(email, purpose)) + if ttl and ttl > 0: + raise OtpResendRateLimited(ttl) + + +def issue_code(email: str, purpose: OtpPurpose) -> str: + """Generate a new code, persist its hash, and start the resend cooldown. + + Caller is responsible for delivering the returned plaintext (e.g. via email). + Any pre-existing code for the same (purpose, email) is overwritten. + """ + check_resend_allowed(email, purpose) + code = _generate_code() + hashed = bcrypt.hashpw(code.encode(), bcrypt.gensalt()).decode() + payload = json.dumps({"hash": hashed, "attempts": 0}) + redis_client.set(_code_key(email, purpose), payload, ex=CODE_TTL_SECONDS) + redis_client.set(_rate_key(email, purpose), "1", ex=RESEND_COOLDOWN_SECONDS) + return code + + +def verify_code(email: str, purpose: OtpPurpose, code: str) -> None: + """Validate and consume a code. Raises subclasses of OtpError on failure.""" + key = _code_key(email, purpose) + raw = redis_client.get(key) + if raw is None: + raise OtpExpired("Code expired or never issued") + record = json.loads(raw) + attempts = int(record.get("attempts", 0)) + if attempts >= MAX_ATTEMPTS: + redis_client.delete(key) + raise OtpAttemptsExceeded("Too many invalid attempts") + if not bcrypt.checkpw(code.encode(), record["hash"].encode()): + record["attempts"] = attempts + 1 + ttl = redis_client.ttl(key) + redis_client.set(key, json.dumps(record), ex=max(ttl, 1)) + raise OtpInvalid("Incorrect code") + redis_client.delete(key) diff --git a/backend/app/services/vessel_ais_aggregation.py b/backend/app/services/vessel_ais_aggregation.py index c8d45904..bb6f2c60 100644 --- a/backend/app/services/vessel_ais_aggregation.py +++ b/backend/app/services/vessel_ais_aggregation.py @@ -6,6 +6,7 @@ import json from typing import Any, Iterable from sqlalchemy import select +from sqlalchemy import Float from sqlalchemy.ext.asyncio import AsyncSession from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth @@ -17,6 +18,10 @@ from app.services.vessel_types import normalize_vessel_type_name VESSEL_AIS_SCHEMA = "vessel_ais" DEFAULT_AGGREGATION_WINDOW_HOURS = 24 +DEFAULT_SNAPSHOT_WINDOW_MINUTES = 60 +MAX_SNAPSHOT_LIMIT = 5000 +MAX_SNAPSHOT_CANDIDATE_MULTIPLIER = 20 +MAX_SNAPSHOT_CANDIDATE_OBSERVATIONS = 100_000 BARENTSWATCH_DELIVERY_MODE = "polling" BARENTSWATCH_TRANSPORT = "http" AISSTREAM_DELIVERY_MODE = "realtime_stream" @@ -564,6 +569,46 @@ async def get_aggregated_vessels( return vessels +async def get_aggregated_vessels_snapshot( + db: AsyncSession, + *, + bbox: tuple[float, float, float, float], + limit: int = 1000, + observed_since: datetime | None = None, +) -> list[dict[str, Any]]: + """Return a bounded viewport snapshot without loading the global AIS window.""" + + observed_since = observed_since or ( + datetime.now(UTC) - timedelta(minutes=DEFAULT_SNAPSHOT_WINDOW_MINUTES) + ) + safe_limit = min(max(int(limit or 1000), 1), MAX_SNAPSHOT_LIMIT) + candidate_limit = min( + max(safe_limit * MAX_SNAPSHOT_CANDIDATE_MULTIPLIER, safe_limit), + MAX_SNAPSHOT_CANDIDATE_OBSERVATIONS, + ) + lon_min, lat_min, lon_max, lat_max = bbox + payload_lon = AISRawObservation.normalized_payload["lon"].as_string().cast(Float) + payload_lat = AISRawObservation.normalized_payload["lat"].as_string().cast(Float) + + stmt = ( + select(AISRawObservation) + .where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA) + .where(AISRawObservation.observed_at >= observed_since) + .where(payload_lon >= lon_min) + .where(payload_lon <= lon_max) + .where(payload_lat >= lat_min) + .where(payload_lat <= lat_max) + .order_by(AISRawObservation.observed_at.desc(), AISRawObservation.id.desc()) + .limit(candidate_limit) + ) + + result = await db.execute(stmt) + if not hasattr(result, "scalars"): + return [] + vessels = await aggregate_vessel_observations(db, result.scalars().all()) + return vessels[:safe_limit] + + async def get_aggregated_vessel(db: AsyncSession, mmsi: int) -> dict[str, Any] | None: observations = await get_vessel_raw_observations(db, mmsi, limit=1000) vessels = await aggregate_vessel_observations(db, observations) diff --git a/backend/tests/test_datasources_batch.py b/backend/tests/test_datasources_batch.py new file mode 100644 index 00000000..5f1ecf0f --- /dev/null +++ b/backend/tests/test_datasources_batch.py @@ -0,0 +1,86 @@ +from datetime import datetime, timedelta, timezone + +import pytest + +from app.api.v1 import datasources as datasources_api +from app.models.datasource import DataSource + + +def make_datasource( + datasource_id: int, + source: str, + *, + name: str | None = None, + module: str = "L4", + is_active: bool = True, + last_status: str | None = None, + last_run_at: datetime | None = None, + frequency_minutes: int = 60, +) -> DataSource: + return DataSource( + id=datasource_id, + name=name or source, + source=source, + module=module, + priority="P1", + frequency_minutes=frequency_minutes, + collector_class=source, + is_active=is_active, + last_status=last_status, + last_run_at=last_run_at, + ) + + +def test_datasource_product_key_groups_domain_specific_sources(): + assert datasources_api.datasource_product_key(make_datasource(1, "aisstream_vessels")) == "vessels" + assert datasources_api.datasource_product_key(make_datasource(2, "telegeography_cables")) == "cables" + assert datasources_api.datasource_product_key(make_datasource(3, "celestrak_tle")) == "satellites" + assert datasources_api.datasource_product_key(make_datasource(4, "ris_live_bgp")) == "bgp" + + +def test_filter_datasources_by_product_status_and_collected_state(): + vessels = make_datasource(1, "aisstream_vessels", last_status="success") + cables = make_datasource(2, "telegeography_cables", last_status="failed") + filtered = datasources_api._filter_datasources_in_memory( + [vessels, cables], + running_tasks={}, + record_counts={"aisstream_vessels": 12, "telegeography_cables": 0}, + product="vessels", + run_status="success", + collected=True, + ) + + assert filtered == [vessels] + + +@pytest.mark.asyncio +async def test_trigger_datasource_batch_skips_disabled_and_frequency_window(monkeypatch): + now = datetime.now(timezone.utc) + disabled = make_datasource(1, "aisstream_vessels", is_active=False) + not_due = make_datasource(2, "telegeography_cables", last_run_at=now, frequency_minutes=120) + due = make_datasource(3, "ris_live_bgp", last_run_at=now - timedelta(hours=2)) + triggered_sources: list[str] = [] + + async def fake_running_tasks(_db, _ids): + return {} + + async def fake_latest_task_ids(_db, _ids): + return {} + + monkeypatch.setattr(datasources_api, "_load_latest_running_tasks", fake_running_tasks) + monkeypatch.setattr(datasources_api, "_load_latest_task_ids", fake_latest_task_ids) + monkeypatch.setattr( + datasources_api, + "run_collector_now", + lambda source: triggered_sources.append(source) or True, + ) + + result = await datasources_api._trigger_datasource_batch( + object(), + [disabled, not_due, due], + force=False, + ) + + assert [item["source"] for item in result["triggered"]] == ["ris_live_bgp"] + assert {item["reason"] for item in result["skipped"]} == {"disabled", "within_frequency_window"} + assert triggered_sources == ["ris_live_bgp"] diff --git a/backend/tests/test_layers.py b/backend/tests/test_layers.py new file mode 100644 index 00000000..ab8d9417 --- /dev/null +++ b/backend/tests/test_layers.py @@ -0,0 +1,44 @@ +from fastapi import HTTPException + +from app.api.v1 import layers + + +def test_layer_guard_requires_bbox(): + try: + layers._parse_layer_bbox("") + except HTTPException as exc: + assert exc.status_code == 400 + else: + raise AssertionError("Expected missing bbox to fail") + + +def test_layer_guard_filters_bbox_and_clamps_low_zoom_limit(): + geojson = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [121.0, 31.0]}, + "properties": {"id": "inside"}, + }, + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [10.0, 10.0]}, + "properties": {"id": "outside"}, + }, + ], + } + + result = layers._guard_geojson_layer( + geojson, + bbox=(120.0, 30.0, 122.0, 32.0), + zoom=2, + limit=6000, + ) + + assert result["returned_count"] == 1 + assert result["visible_count"] == 1 + assert result["features"][0]["properties"]["id"] == "inside" + assert result["diagnostics"]["limit"] == layers.LOW_ZOOM_FEATURE_LIMIT + assert result["diagnostics"]["limit_clamped"] is True + assert result["diagnostics"]["degraded"] is True diff --git a/backend/tests/test_otp_service.py b/backend/tests/test_otp_service.py new file mode 100644 index 00000000..e856eba7 --- /dev/null +++ b/backend/tests/test_otp_service.py @@ -0,0 +1,106 @@ +"""Unit tests for app.services.otp using an in-memory Redis fake.""" + +from __future__ import annotations + +import time +from typing import Any + +import pytest + +from app.services import otp + + +class FakeRedis: + """Minimal subset of redis-py used by services.otp.""" + + def __init__(self) -> None: + self._store: dict[str, tuple[Any, float | None]] = {} + + def _expired(self, key: str) -> bool: + item = self._store.get(key) + if item is None: + return True + _, expires = item + if expires is not None and expires <= time.time(): + self._store.pop(key, None) + return True + return False + + def set(self, key: str, value: Any, ex: int | None = None) -> None: + expires = time.time() + ex if ex else None + self._store[key] = (value, expires) + + def get(self, key: str) -> Any: + if self._expired(key): + return None + return self._store[key][0] + + def ttl(self, key: str) -> int: + if self._expired(key): + return -2 + _, expires = self._store[key] + if expires is None: + return -1 + return max(int(expires - time.time()), 0) + + def delete(self, key: str) -> None: + self._store.pop(key, None) + + +@pytest.fixture +def fake_redis(monkeypatch): + fake = FakeRedis() + monkeypatch.setattr(otp, "redis_client", fake) + return fake + + +def test_issue_code_returns_six_digits(fake_redis): + code = otp.issue_code("alice@example.com", "register") + assert len(code) == 6 + assert code.isdigit() + + +def test_verify_code_succeeds_and_consumes(fake_redis): + code = otp.issue_code("alice@example.com", "register") + otp.verify_code("alice@example.com", "register", code) + with pytest.raises(otp.OtpExpired): + otp.verify_code("alice@example.com", "register", code) + + +def test_verify_code_rejects_wrong_code(fake_redis): + otp.issue_code("alice@example.com", "register") + with pytest.raises(otp.OtpInvalid): + otp.verify_code("alice@example.com", "register", "000000") + + +def test_verify_code_locks_after_max_attempts(fake_redis): + code = otp.issue_code("alice@example.com", "register") + for _ in range(otp.MAX_ATTEMPTS): + with pytest.raises(otp.OtpInvalid): + otp.verify_code("alice@example.com", "register", "000000") + # After max attempts the next call should raise OtpAttemptsExceeded and clear the code. + with pytest.raises(otp.OtpAttemptsExceeded): + otp.verify_code("alice@example.com", "register", code) + with pytest.raises(otp.OtpExpired): + otp.verify_code("alice@example.com", "register", code) + + +def test_issue_code_enforces_resend_cooldown(fake_redis): + otp.issue_code("alice@example.com", "register") + with pytest.raises(otp.OtpResendRateLimited) as excinfo: + otp.issue_code("alice@example.com", "register") + assert excinfo.value.retry_after_seconds > 0 + + +def test_issue_code_emails_are_case_insensitive(fake_redis): + code = otp.issue_code("Alice@Example.com", "register") + otp.verify_code("alice@example.com", "register", code) + + +def test_purposes_are_isolated(fake_redis): + register_code = otp.issue_code("alice@example.com", "register") + reset_code = otp.issue_code("alice@example.com", "reset_password") + assert register_code != reset_code + otp.verify_code("alice@example.com", "register", register_code) + # Reset code should still be valid after consuming the register code. + otp.verify_code("alice@example.com", "reset_password", reset_code) diff --git a/backend/tests/test_settings_ai_provider.py b/backend/tests/test_settings_ai_provider.py index dc9042c8..96808a0f 100644 --- a/backend/tests/test_settings_ai_provider.py +++ b/backend/tests/test_settings_ai_provider.py @@ -5,9 +5,12 @@ import pytest from app.api.v1 import settings as settings_api from app.api.v1.settings import ( AIProviderIntegrationUpdate, + OCRIntegrationUpdate, _build_ai_provider_payload, + _build_ocr_payload, _mask_secret, _normalize_ai_provider_payload, + _normalize_ocr_payload, _resolve_provider_api_key, get_runtime_ai_provider_config, ) @@ -130,6 +133,43 @@ def test_build_payload_keeps_saved_key_when_preview_submitted(): assert payload["providers"]["openai"]["api_key"] == "sk-old-secret" +def test_normalize_ocr_payload_adds_defaults(): + payload = _normalize_ocr_payload({}) + + assert payload["enabled"] is False + assert payload["provider"] == "paddleocr" + assert payload["languages"] == ["zh", "en"] + assert payload["output_format"] == "markdown" + + +def test_build_ocr_payload_keeps_saved_key_when_preview_submitted(): + current = { + "ocr": { + "enabled": True, + "provider": "custom", + "base_url": "http://localhost:8020", + "api_key": "ocr-old-secret", + } + } + update = OCRIntegrationUpdate( + enabled=True, + provider="custom", + base_url="http://localhost:8020", + api_key="**************", + model="ocr-model", + languages=["zh", "en"], + timeout_seconds=45, + max_file_size_mb=50, + output_format="json", + ) + + payload = _build_ocr_payload(current, update) + + assert payload["api_key"] == "ocr-old-secret" + assert payload["model"] == "ocr-model" + assert payload["output_format"] == "json" + + @pytest.mark.asyncio async def test_runtime_config_uses_default_provider_specific_key(monkeypatch): record = SimpleNamespace( diff --git a/backend/tests/test_settings_smtp.py b/backend/tests/test_settings_smtp.py new file mode 100644 index 00000000..2da9001a --- /dev/null +++ b/backend/tests/test_settings_smtp.py @@ -0,0 +1,86 @@ +"""Unit tests for SMTP settings helpers in app.api.v1.settings.""" + +from app.api.v1.settings import ( + SMTPSettingsUpdate, + _build_smtp_payload, + _serialize_smtp_payload, +) + + +def test_serialize_masks_password_and_reports_configured(): + serialized = _serialize_smtp_payload( + { + "host": "smtp.example.com", + "port": 587, + "username": "noreply@example.com", + "password": "super-secret", + "from_address": "noreply@example.com", + "from_name": "Planet", + "use_tls": False, + "use_starttls": True, + "timeout_seconds": 20, + } + ) + assert serialized["configured"] is True + assert serialized["password"]["configured"] is True + assert "secret" not in serialized["password"]["preview"] + + +def test_serialize_marks_unconfigured_when_host_missing(): + serialized = _serialize_smtp_payload( + { + "host": "", + "port": 587, + "from_address": "", + } + ) + assert serialized["configured"] is False + assert serialized["password"]["configured"] is False + + +def test_build_payload_preserves_password_when_placeholder_submitted(): + current = { + "host": "smtp.example.com", + "port": 587, + "username": "noreply@example.com", + "password": "super-secret", + "from_address": "noreply@example.com", + "from_name": "Planet", + "use_tls": False, + "use_starttls": True, + "timeout_seconds": 20, + } + preview = _serialize_smtp_payload(current)["password"]["preview"] + update = SMTPSettingsUpdate( + host="smtp.example.com", + port=587, + username="noreply@example.com", + password=preview, + from_address="noreply@example.com", + ) + merged = _build_smtp_payload(current, update) + assert merged["password"] == "super-secret" + + +def test_build_payload_replaces_password_when_new_value_submitted(): + current = {"password": "old", "host": "", "port": 587, "from_address": ""} + update = SMTPSettingsUpdate( + host="smtp.example.com", + port=587, + password="new-secret", + from_address="noreply@example.com", + ) + merged = _build_smtp_payload(current, update) + assert merged["password"] == "new-secret" + + +def test_build_payload_clears_password_when_requested(): + current = {"password": "old"} + update = SMTPSettingsUpdate( + host="smtp.example.com", + port=587, + from_address="noreply@example.com", + clear_password=True, + ) + merged = _build_smtp_payload(current, update) + assert merged["password"] == "" diff --git a/backend/tests/test_vessels.py b/backend/tests/test_vessels.py index 6f37efac..d5d09907 100644 --- a/backend/tests/test_vessels.py +++ b/backend/tests/test_vessels.py @@ -523,22 +523,115 @@ def test_convert_vessels_to_geojson_dedupes_mmsi_rows(): @pytest.mark.asyncio -async def test_vessels_geojson_endpoint_filters_type_and_bbox(): +async def test_vessel_snapshot_filters_type_and_bbox(monkeypatch): now = datetime(2026, 4, 28, 1, 0, tzinfo=timezone.utc) - rows = [ - ( - VesselPosition(mmsi=1, lat=59.9, lon=10.7, received_at=now), - VesselStatic(mmsi=1, name="Cargo Ship", vessel_type=70, vessel_type_name="Cargo"), + monkeypatch.setattr( + visualization, + "get_aggregated_vessels_snapshot", + AsyncMock( + return_value=[ + { + "mmsi": 1, + "lat": 59.9, + "lon": 10.7, + "received_at": now, + "name": "Cargo Ship", + "vessel_type": 70, + "vessel_type_name": "Cargo", + }, + { + "mmsi": 2, + "lat": 60.3, + "lon": 5.3, + "received_at": now - timedelta(minutes=1), + "name": "Passenger Ship", + "vessel_type": 60, + "vessel_type_name": "Passenger", + }, + ] ), - ( - VesselPosition(mmsi=2, lat=60.3, lon=5.3, received_at=now - timedelta(minutes=1)), - VesselStatic(mmsi=2, name="Passenger Ship", vessel_type=60, vessel_type_name="Passenger"), - ), - ] + ) + + async def override_get_db(): + yield object() + + app.dependency_overrides[get_db] = override_get_db + transport = ASGITransport(app=app) + try: + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get( + "/api/v1/vessels/snapshot", + params={"bbox": "0,50,20,70", "zoom": 12, "type": "cargo", "limit": 1000}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["count"] == 1 + assert data["features"][0]["properties"]["name"] == "Cargo Ship" + assert data["stats"]["by_type"]["Cargo"] == 1 + finally: + app.dependency_overrides.clear() + + +@pytest.mark.asyncio +async def test_legacy_vessels_geojson_endpoint_is_gone(): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/v1/visualization/geo/vessels") + + assert response.status_code == 410 + assert "/api/v1/vessels/snapshot" in response.json()["detail"] + + +@pytest.mark.asyncio +async def test_vessel_snapshot_requires_bbox(): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/v1/vessels/snapshot", params={"zoom": 12}) + + assert response.status_code == 400 + assert response.json()["detail"] == "bbox is required" + + +@pytest.mark.asyncio +async def test_vessel_snapshot_filters_bbox_and_caps_limit(monkeypatch): + now = datetime(2026, 4, 28, 1, 0, tzinfo=timezone.utc) + captured = {} + + async def fake_get_aggregated_vessels_snapshot(db, *, bbox, limit, observed_since): + captured["bbox"] = bbox + captured["limit"] = limit + captured["observed_since"] = observed_since + return [ + { + "mmsi": 1, + "lat": 59.9, + "lon": 10.7, + "received_at": now, + "name": "Cargo Ship", + "vessel_type": 70, + "vessel_type_name": "Cargo", + }, + { + "mmsi": 2, + "lat": 60.3, + "lon": 5.3, + "received_at": now, + "name": "Passenger Ship", + "vessel_type": 60, + "vessel_type_name": "Passenger", + }, + ] + + monkeypatch.setattr( + visualization, + "get_aggregated_vessels_snapshot", + fake_get_aggregated_vessels_snapshot, + ) class _Result: def all(self): - return rows + return [] class _FakeSession: async def execute(self, _query): @@ -552,75 +645,39 @@ async def test_vessels_geojson_endpoint_filters_type_and_bbox(): try: async with AsyncClient(transport=transport, base_url="http://test") as client: response = await client.get( - "/api/v1/visualization/geo/vessels", - params={"bbox": "0,50,20,70", "type": "cargo", "limit": 0}, + "/api/v1/vessels/snapshot", + params={ + "bbox": "10,59,11,60", + "zoom": 12, + "type": "cargo", + "limit": 5000, + "since_minutes": 30, + }, ) assert response.status_code == 200 data = response.json() assert data["count"] == 1 assert data["features"][0]["properties"]["name"] == "Cargo Ship" - assert data["stats"]["by_type"]["Cargo"] == 1 + assert captured["bbox"] == (10.0, 59.0, 11.0, 60.0) + assert captured["limit"] == 5000 + assert data["diagnostics"]["bbox_applied"] is True + assert data["diagnostics"]["legacy_feature_count"] == 0 + assert data["diagnostics"]["legacy_backfilled_mmsi"] == 0 finally: app.dependency_overrides.clear() @pytest.mark.asyncio -async def test_vessels_geojson_merges_raw_and_legacy_sources(monkeypatch): - now = datetime(2026, 4, 28, 1, 0, tzinfo=timezone.utc) - monkeypatch.setattr( - visualization, - "get_aggregated_vessels", - AsyncMock( - return_value=[ - { - "mmsi": 1, - "lat": 59.9, - "lon": 10.7, - "received_at": now, - "name": "AISSTREAM SHIP", - "vessel_type_name": "Cargo", - "source_summary": {"aisstream_vessels": {"message_types": ["PositionReport"]}}, - } - ] - ), - ) - rows = [ - ( - VesselPosition(mmsi=1, lat=60.0, lon=10.8, received_at=now), - VesselStatic(mmsi=1, name="LEGACY DUP", vessel_type_name="Cargo"), - ), - ( - VesselPosition(mmsi=2, lat=60.3, lon=5.3, received_at=now), - VesselStatic(mmsi=2, name="BARENTSWATCH ONLY", vessel_type_name="Passenger"), - ), - ] - - class _Result: - def all(self): - return rows - - class _FakeSession: - async def execute(self, _query): - return _Result() - - async def override_get_db(): - yield _FakeSession() - - app.dependency_overrides[get_db] = override_get_db +async def test_legacy_vessels_geojson_rejects_even_with_bbox(): transport = ASGITransport(app=app) - try: - async with AsyncClient(transport=transport, base_url="http://test") as client: - response = await client.get("/api/v1/visualization/geo/vessels") + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get( + "/api/v1/visualization/geo/vessels", + params={"bbox": "10,59,11,60", "type": "cargo", "limit": 1000}, + ) - assert response.status_code == 200 - data = response.json() - names = {feature["properties"]["mmsi"]: feature["properties"]["name"] for feature in data["features"]} - assert data["count"] == 2 - assert names == {1: "AISSTREAM SHIP", 2: "BARENTSWATCH ONLY"} - assert data["diagnostics"]["legacy_backfilled_mmsi"] == 1 - finally: - app.dependency_overrides.clear() + assert response.status_code == 410 @pytest.mark.asyncio diff --git a/backend/tests/test_websocket_manager.py b/backend/tests/test_websocket_manager.py index 498447a9..aa8ac02b 100644 --- a/backend/tests/test_websocket_manager.py +++ b/backend/tests/test_websocket_manager.py @@ -1,6 +1,8 @@ import pytest +import importlib from app.core.websocket.manager import ConnectionManager +from app.core.websocket.broadcaster import DataBroadcaster class FakeWebSocket: @@ -44,3 +46,93 @@ async def test_disconnect_removes_channel_subscriptions(): assert socket.sent == [] assert "dashboard" not in manager.channel_subscriptions + + +@pytest.mark.asyncio +async def test_vessel_subscribers_receive_only_matching_bbox_updates(): + manager = ConnectionManager() + oslo_socket = FakeWebSocket() + bergen_socket = FakeWebSocket() + + await manager.connect(oslo_socket, "user-1") + await manager.connect(bergen_socket, "user-2") + manager.subscribe_vessels( + oslo_socket, + {"bbox": [10, 59, 11, 60], "zoom": 12, "limit": 1000}, + ) + manager.subscribe_vessels( + bergen_socket, + {"bbox": [5, 60, 6, 61], "zoom": 12, "limit": 1000}, + ) + + await manager.broadcast_vessels( + { + "action": "upsert", + "vessels": [ + {"mmsi": 1, "lat": 59.9, "lon": 10.7}, + {"mmsi": 2, "lat": 60.3, "lon": 5.3}, + ], + } + ) + + assert oslo_socket.sent[0]["payload"]["vessels"] == [{"mmsi": 1, "lat": 59.9, "lon": 10.7}] + assert bergen_socket.sent[0]["payload"]["vessels"] == [{"mmsi": 2, "lat": 60.3, "lon": 5.3}] + + +@pytest.mark.asyncio +async def test_vessel_broadcast_removes_slow_connections(): + manager = ConnectionManager() + + class BrokenWebSocket(FakeWebSocket): + async def send_json(self, message): + raise RuntimeError("client is gone") + + socket = BrokenWebSocket() + await manager.connect(socket, "user-1") + manager.subscribe_vessels(socket, {"bbox": [10, 59, 11, 60], "zoom": 12}) + + await manager.broadcast_vessels({"vessels": [{"mmsi": 1, "lat": 59.9, "lon": 10.7}]}) + + assert socket not in manager.vessel_subscriptions + + +def test_vessel_subscription_rejects_large_bbox(): + manager = ConnectionManager() + + with pytest.raises(ValueError, match="bbox is too large"): + manager.subscribe_vessels(FakeWebSocket(), {"bbox": [-180, -90, 180, 90], "zoom": 1}) + + +@pytest.mark.asyncio +async def test_vessel_broadcaster_keeps_latest_update_per_mmsi(monkeypatch): + sent = [] + + async def fake_broadcast_vessels(payload): + sent.append(payload) + + broadcaster_module = importlib.import_module("app.core.websocket.broadcaster") + monkeypatch.setattr(broadcaster_module.manager, "broadcast_vessels", fake_broadcast_vessels) + broadcaster = DataBroadcaster() + broadcaster.enqueue_vessel_update( + { + "source": "aisstream_vessels", + "vessels": [ + {"mmsi": 1, "lat": 59.0, "lon": 10.0}, + {"mmsi": 1, "lat": 59.1, "lon": 10.1}, + ], + } + ) + + await broadcaster.flush_vessel_updates() + + assert len(sent) == 1 + assert sent[0]["vessels"] == [ + { + "mmsi": 1, + "lat": 59.1, + "lon": 10.1, + "source": "aisstream_vessels", + "action": "upsert", + "created": None, + } + ] diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d9f5b3b0..8e6c5e7a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -8,6 +8,22 @@ This project follows the repository versioning rule: - `improvement` -> `+0.0.1`（bugfix + 小功能混合） - `bugfix` -> `+0.0.1` +## [0.52.0] — 2026-05-12 + +Released: 2026-05-12 + +### Highlights +- 新增邮箱验证码注册/找回密码链路、SMTP 设置面板和连接测试组件，补齐公开账号自助入口。 +- 重构 AIS 船只实时链路：新增受控船只 snapshot、WebSocket vessels 订阅、AISStream 长连接状态与节流广播。 +- 新增数据产品统计接口、受控 `/layers/*` 图层接口骨架，以及数据源产品域筛选和批量采集。 + +### Added / Fixed / Improved +- 新增 `/api/v1/data-products`、`/api/v1/layers/*` 和 `/api/v1/datasources/trigger-batch`，拆分全量统计与地图渲染数据。 +- 将 Earth 船只层迁移到 `/api/v1/vessels/snapshot`，并修正 vessels WebSocket 订阅 payload。 +- 更新中英文手册、采集器文档、运维手册和计划状态，覆盖注册、SMTP、AISStream、数据产品和图层保护流程。 + +--- + ## [0.51.1] — 2026-05-11 Released: 2026-05-11 diff --git a/docs/documentation-coverage-rules.md b/docs/documentation-coverage-rules.md index a3b517bf..5ed0fca4 100644 --- a/docs/documentation-coverage-rules.md +++ b/docs/documentation-coverage-rules.md @@ -2,6 +2,16 @@ This file contains Planet-specific documentation coverage rules. Documentation skills and agents should read this file before deciding which docs to update. Keep tool-specific workflow in skills; keep product and repository rules here. +## Audience Routing (mandatory) + +Before deciding scope, classify the change by who performs the action: + +- **Browser/UI end user** (login, account settings, configuring collectors or AI via UI, using Earth/Console pages): update `docs/technical/{zh,en}/manual.md` and `quickstart.md` only. Never put shell commands, log file paths, `planet.sh`, Docker operations, or `netsh portproxy` rules into these files. +- **Operations / deployment / on-call** (`planet.sh`, log paths, SMTP fallbacks like `createuser`, LAN/portproxy, env-var tuning, troubleshooting order, Bun build conventions): update `docs/technical/{zh,en}/ops-runbook.md` (or an existing `ops-*.md`). Never put UI button labels or screenshots into these files. +- **Second-party developers** (component context, render order, internal pipelines): update the existing `*-context.md` / `backend-*.md` / `earth-*.md` files. + +If the same action has both a UI and a CLI path (e.g. user creation), describe the UI path in `manual.md` and the CLI path in `ops-runbook.md`, and cross-link them with a single sentence each. + ## Scope Rules - User-visible workflow changes must update `docs/technical/zh/manual.md` and usually `docs/technical/zh/quickstart.md`. diff --git a/docs/plans/data-products-layer-guard-redesign-plan.md b/docs/plans/data-products-layer-guard-redesign-plan.md new file mode 100644 index 00000000..021cdd56 --- /dev/null +++ b/docs/plans/data-products-layer-guard-redesign-plan.md @@ -0,0 +1,146 @@ +# 数据产品流水线、数据源批量运维与抗击穿图层接口计划 + +## Summary + +前端展示分两类数据： + +- **图层数据**：按 viewport、bbox、zoom、limit 返回，可降级、截断、缓存，用来保护服务器。 +- **聚合面板统计**：必须是全量统计，不受当前 viewport 限制，但不能实时扫全表；通过产品状态表或预计算统计提供。 + +也就是说：地图上低 zoom 可以只画摘要或局部数据，但面板里的“总船只数、总海缆数、BGP 活跃事件数、卫星数”等应该代表全局数据产品状态。 + +## Implementation Status + +- 已新增 `POST /api/v1/datasources/trigger-batch`，支持按选中 `source_ids` 或筛选条件批量触发，并返回 `triggered/skipped/failed`。 +- 已改造 `/datasources` 页面，支持产品域、层级、启用状态、最近执行状态、是否已有数据和关键词筛选，并支持复选框批量采集。 +- 已新增 `/api/v1/data-products` 和 `/api/v1/data-products/{product_id}/status`，聚合面板可以读取全量/全局统计口径。 +- 已新增 `/api/v1/layers/*` 受控图层接口骨架，要求 `bbox/zoom/limit`，响应包含 `visible_count/returned_count/diagnostics`。 +- 非船只图层当前先复用已有 GeoJSON 转换再做保护层；下一步应把 cables/BGP/satellites 的 bbox 过滤继续下推到各自产品查询，避免转换前仍加载过多候选。 + +## Key Changes + +- 新增数据产品状态/统计层： + - 每个产品维护全量统计：总实体数、活跃数、最近更新时间、使用源、缺失源、冲突数、构建状态。 + - 统计在采集成功或产品投影完成后更新，不在用户打开页面时临时全表聚合。 + - 前端聚合面板统一读取产品统计接口，而不是从图层返回量推断总数。 +- 新增接口： + - `GET /api/v1/data-products` + - `GET /api/v1/data-products/{product_id}/status` + - `GET /api/v1/layers/{product}/...` + - `POST /api/v1/datasources/trigger-batch` +- `/layers/*` 只负责可视化数据： + - 支持 bbox、zoom、limit、since。 + - 可以返回 `degraded`、`truncated`、`cache_hit`。 + - 返回 `visible_count` 和 `returned_count`，但不作为全量统计来源。 +- `/data-products/*/status` 负责全量统计： + - 返回 `total_count`、`active_count`、`source_counts`、`last_built_at`、`health`。 + - 数据来自预计算状态或轻量索引统计。 + - 即使图层降级，统计也保持全量口径。 + +## Product Processing + +- 船只： + - 图层：bbox snapshot + WS 聚合流，受限返回。 + - 统计：全量唯一 MMSI、最近窗口活跃 MMSI、AISStream/BarentsWatch/source counts。 +- 海缆： + - 图层：viewport 内 cable segments/landing points，低 zoom 可简化路线。 + - 统计：全量 cable count、landing point count、relation count、graph 构建状态。 + - 处理：专用 cable graph assembler，区分路线源、登陆点源、关系源、补充源，不使用统一字段融合函数。 +- BGP： + - 图层：active incidents/anomalies/collectors，按窗口和 limit 返回。 + - 统计：全量活跃事件、最近 24h/7d 事件数、collector 数、incident/anomaly 分布。 + - 处理：专用事件流水线，区分 observation、anomaly、incident、geo hint、infrastructure inference。 +- 卫星： + - 图层：可见卫星或受控 limit。 + - 统计：全量卫星数、最新 TLE epoch、源覆盖情况。 + - 处理：按 NORAD id 生成轨道快照，TLE epoch 最新优先。 + +## Data Source Page + +- 增加筛选： + - 产品类型、启用/禁用、最近成功/失败/运行中/未执行、已采集/未采集、凭证状态、文本搜索。 +- 增加复选框批量操作： + - 批量启用、禁用、采集、强制采集。 + - 一键采集改为：采集全部启用源、采集筛选结果、采集选中源。 +- 后端 batch 逻辑： + - 禁用源 skipped。 + - 运行中源按 force 处理。 + - 单个失败不影响其他源。 + - 返回 `triggered`、`skipped`、`failed`，并包含每个 source 的原因和 task_id。 + +## Protection Rules + +- 所有 `/layers/*` 接口必须有保护层： + - limit clamp。 + - bbox/zoom 校验。 + - 低 zoom 降级。 + - 短 TTL 缓存。 + - 慢查询超时。 + - diagnostics 返回降级原因。 +- 全量统计不走图层查询： + - 不允许为了面板统计在请求时 `.all()` 全量加载。 + - 统计由采集/投影任务异步更新。 + - 统计缺失时返回 `unknown` 或 `stale`，不触发重型实时计算。 +- 缓存失效规则： + - 采集成功后失效对应产品缓存。 + - 海缆 graph cache 在路线、登陆点或关系源成功采集后失效。 + - BGP incident/anomaly 生成后失效 BGP layer cache。 + - 船只实时流使用短 TTL 或 viewport 级缓存，不清全局缓存。 +- 接口观测： + - 记录每个 layer endpoint 的耗时、返回数量、是否降级、是否缓存命中、limit 是否被 clamp。 + - 对高频 viewport 请求增加简单 per-IP 或 per-user rate limit。 + +## Frontend UX + +- 数据源页： + - 顶部统计可作为快捷筛选入口：全部、启用、禁用、运行中、失败、未采集。 + - 表格左侧增加复选框。 + - 工具栏显示“已选择 N 个”，并提供批量按钮。 + - 筛选结果和选中结果分清楚，避免误触发全部源。 + - 批量采集完成后弹出摘要：触发、跳过、失败数量，可展开查看原因。 +- 设置/配置页： + - “采集器设置”改为“数据产品配置”。 + - 产品内按源角色分组展示，而不是简单列出 collector。 + - 海缆显示路线源、登陆点源、关系源、补充源。 + - BGP 显示实时观测源、历史回填源、地理 hint 源、检测输出。 + - 船只显示实时 AIS、轮询 AIS、自定义补充源。 +- Earth 图层交互： + - 聚合面板统计读取 `/data-products/*/status`，保持全量口径。 + - 图层面板展示当前图层是否降级、截断、缓存命中。 + - 对象详情展示来源证据： + - 船只：字段来源、冲突。 + - 海缆：路线源、登陆点源、关系源。 + - BGP：事件证据、分组依据、地理推断依据。 + - 产品 degraded 时仍显示可用部分，并提示缺失源角色。 + +## Test Plan + +- 图层接口： + - 大 limit 被 clamp。 + - 低 zoom 降级。 + - 大数据集不全量内存过滤。 + - diagnostics 正确说明截断、缓存、降级。 +- 全量统计： + - 面板统计不受 bbox 影响。 + - 图层返回 1000 条时，产品统计仍显示全量总数。 + - 统计陈旧时返回 `stale=true` 和 `last_built_at`。 + - 采集成功后对应产品统计刷新。 +- 数据源批量： + - 筛选、选中、批量采集行为正确。 + - skipped/failed/triggered 分组正确。 + - 禁用源在 batch 中被 skipped。 + - 运行中源按 force 参数处理。 + - batch 单源失败不阻断整体。 +- 产品处理： + - 海缆缺 relation 时产品状态 degraded，但 cable layer 可用。 + - BGP observation 不直接变成前端 marker，必须经过 anomaly/incident 投影。 + - 船只 bbox snapshot 和 WS 节流继续有效。 + - 卫星列表不返回无限轨道点。 + +## Assumptions + +- 前端聚合面板以后只读 `/data-products/*/status`。 +- 地图图层只读 `/layers/*`。 +- 统计可以短暂 stale，但不能因实时全量统计击穿服务器。 +- 保留现有 collector，不为了重构而删除 BarentsWatch 或其他源。 +- 当前开发阶段允许前端从旧 `/visualization/geo/*` 迁移到 `/layers/*`。 diff --git a/docs/plans/docs-audience-split-plan.md b/docs/plans/docs-audience-split-plan.md new file mode 100644 index 00000000..299ce690 --- /dev/null +++ b/docs/plans/docs-audience-split-plan.md @@ -0,0 +1,116 @@ +# 文档受众分层重构计划 + +**状态**：待实施 +**创建日期**：2026-05-12 +**核心目标**：把 `docs/technical/{zh,en}/manual.md` 拆成"纯客户视角"的使用手册，把 `planet.sh`、日志、LAN、故障排查这类运维内容迁到独立 `ops-runbook.md`，并把分层规则写进 `documentation-coverage-rules.md` 和 `.claude/commands/docs.md`，让以后写文档时自动按受众归档。 + +## 背景 + +当前 `manual.md` 把客户实际使用和开发/运维操作混在一份文档里：开头 200 多行讲的是 `planet.sh start/stop/restart/log/health/createuser/--allow-lan`、AI Provider 镜像构建、`netsh portproxy` 和故障排查顺序，后面才进入 Earth、控制台、AI、Docs 这些客户真正会用到的功能。 + +客户读到一半会被 shell 命令吓住，开发者想找运维细节又要在大段 UI 操作里翻。`documentation-coverage-rules.md` 现在也没有受众分层规则，未来文档继续混着写。 + +本计划假定客户已经能拿到账号登录使用 — 注册/验证流程本身见 [用户公开注册与邮箱验证计划](/home/ray/dev/linkong/planet/docs/plans/user-registration-email-verification-plan.md)。 + +## 新的文档地形 + +| 文档 | 受众 | Gatekeeper 组 | 范围 | +| --- | --- | --- | --- | +| `manual.md` (zh+en) | 纯客户/最终用户 | `public` | 注册、登录、账户、设置 UI、collector 配置、AI 配置、Console 页面、Earth、Docs 浏览 | +| `quickstart.md` (zh+en) | 纯客户 | `public` | "我刚拿到 Planet 怎么开始用" — 打开 URL → 注册 → 验证 → 登录 → 第一次配置 | +| `ops-runbook.md` (zh+en, **新增**) | 运维/部署人员 | `docs_admin` | `planet.sh` 完整命令、健康检查、日志位置、LAN/portproxy、故障排查顺序、createuser CLI、Bun 构建约定 | +| `ops-planet-sh-startup.md` (已存在) | 运维 | `docs_admin` | 启动性能、AI Provider 镜像、`PLANET_LOAD_ZSHRC_ENV` 深度调优 — 保持不动 | +| 现有 `*-context.md` / `backend-*.md` | 二次开发者 | `docs_developer` | 保持现状 | + +`backend-system-service-control.md` 偏后端服务控制原理，**不**和 `ops-runbook.md` 重复 — runbook 讲"运维要敲什么命令"，service-control 讲"后端怎么实现服务管控"。 + +## manual.md 重写后的章节顺序（客户旅程） + +1. **欢迎与入口** — Planet 是什么、几个入口（Earth 公开 / Console 需登录 / Docs / API） +2. **注册账户** — 打开 `/login` → 点"注册" → 填用户名/邮箱/密码 → 收邮件 → 输入 6 位验证码 → 登录 +3. **登录与找回密码** — 登录页、忘记密码流程 +4. **账户设置** — 修改密码、修改邮箱（需重新验证）、查看权限组、登出 +5. **Console 总览** — 左侧菜单结构、各路由用途 +6. **配置数据采集器** — `/settings?tab=collector_credentials`：选择 collector、连接测试、保存凭证；BarentsWatch / AISStream 两个典型例子 +7. **配置 AI 凭证** — `/ai?tab=providers`：默认 provider、模型、Base URL、API Key、本地代理；工具 tab（WebSearch、OCR） +8. **系统设置** — `/settings` 其他子 tab（系统设置、电视直播源、SMTP 邮件） +9. **用户管理（管理员）** — `/users`：创建、删除、改角色、Gatekeeper 权限组 +10. **数据探索** — `/datasources`、`/data`、`/bgp`、`/alerts/*` +11. **AI 测试台** — `/ai?tab=playground` +12. **Earth 公开页面** — 现 manual.md 的 Earth 章节原样保留（图层、图例、搜索、位置候选、设置、视角、动捕、巡航、移动端） +13. **Docs 文档站** — 当前 Docs 章节保留（权限组说明） + +不再出现：`planet.sh`、`./planet.sh log`、`netsh portproxy`、`source ~/.zshrc && bun run build`、"故障排查顺序"、"开发命令约定"。 + +## quickstart.md 重写 + +当前 quickstart 假设读者会自己 `git clone` 然后 `./planet.sh start`，这是给开发者看的。改为： + +- 打开管理员给你的 URL +- 注册账号 + 邮箱验证 +- 登录后第一次做什么（建议先到 `/settings?tab=collector_credentials` 配一个 collector，再到 `/ai` 配模型） +- 看 Earth + +部署/开发的 quickstart 内容并入 `ops-runbook.md` 的"首次部署"小节，**不**再单独出 `ops-quickstart.md`，避免新增维护点。 + +## ops-runbook.md 内容大纲 + +抽自现 manual.md，重新组织： + +1. 首次启动 — `./planet.sh start`、默认账号（`admin/admin123`、`linkong/12345678`，引用 `b15d097b` 引入的 `DEFAULT_LOGIN_USERS`） +2. 启停与按模块重启 — `start/stop/restart` 及 `-b -f -a -d` +3. 健康检查 — `./planet.sh health` +4. 日志 — `./planet.sh log` 及 `-f -b -a`，日志文件路径 +5. 创建用户（CLI 兜底）— `./planet.sh createuser`；说明这是公开注册不可用（SMTP 未配置）时的兜底 +6. 局域网/WSL 访问 — `--allow-lan`、`netsh portproxy`、防火墙 +7. AI Provider 环境变量与构建 — `aiprovider/.env`、`~/.zshrc`、`PLANET_LOAD_ZSHRC_ENV` +8. 故障排查顺序 — 现 manual 末尾那段，原样搬来 +9. 开发命令约定 — Bun、`bun run build`、为什么不用 npm + +## documentation-coverage-rules.md 增量 + +在现有"覆盖清单"末尾新增一段： + +> **受众分层（强制）** +> +> - 客户/最终用户能在浏览器里完成的操作 → 只写到 `manual.md` / `quickstart.md` +> - 需要 SSH/shell/Docker/`planet.sh`/日志文件路径/端口转发 → 只写到 `ops-runbook.md`（或现有 `ops-*.md`），**禁止**出现在 manual/quickstart +> - 同一动作两种入口（如"创建用户"既能 UI 也能 CLI）→ UI 路径写 manual.md，CLI 路径写 ops-runbook.md，互相用一句话相互引用 +> - 新增客户可见 UI 流 → 同时更新 `manual.md` zh+en 与 `docs-content.ts` +> - 新增 ops 命令或脚本 → 只更新 `ops-runbook.md` zh+en + +## .claude/commands/docs.md 增量 + +在 "Step 2 — Decide Scope" 后插一段： + +> **Document Audience Routing (Planet)** +> +> 在 Planet 仓库内，写文档前先判断动作的执行者： +> +> - 浏览器 UI 用户 → `docs/technical/{zh,en}/manual.md` / `quickstart.md` +> - shell/容器/运维 → `docs/technical/{zh,en}/ops-runbook.md` 或现有 `ops-*.md` +> - 二次开发者 → 现有 `*-context.md` / `backend-*.md` +> +> 永远不要把 shell 命令、日志路径、Docker 操作写进 manual/quickstart；永远不要把 UI 截图/按钮路径写进 ops-*。 + +## 关键文件清单 + +- `docs/technical/zh/manual.md` & `en/manual.md` — 重写 +- `docs/technical/zh/quickstart.md` & `en/quickstart.md` — 重写 +- `docs/technical/zh/ops-runbook.md` & `en/ops-runbook.md` *(新)* +- `docs/documentation-coverage-rules.md` — 加受众分层段 +- `.claude/commands/docs.md` — 加 Document Audience Routing 段 +- `frontend/src/pages/Docs/docs-content.ts` — 注册 `ops-runbook` 到 `DOCS_METADATA`（`docs_admin` 组） + +## 依赖 + +manual.md 的"注册账户"和"登录与找回密码"两章需要前后端注册/验证流程已经落地，否则文档会描述不存在的功能。注册功能本身见 [用户公开注册与邮箱验证计划](/home/ray/dev/linkong/planet/docs/plans/user-registration-email-verification-plan.md)。建议先实现注册再重写 manual，避免文档与代码错位。 + +## 验证 + +- `rg -n 'planet\.sh' docs/technical/zh/manual.md docs/technical/en/manual.md docs/technical/zh/quickstart.md docs/technical/en/quickstart.md` 应该为空 +- `rg -n '注册账户|register|邮箱验证' docs/technical/zh/manual.md docs/technical/en/manual.md` 应该有命中 +- `rg -n 'planet\.sh' docs/technical/zh/ops-runbook.md docs/technical/en/ops-runbook.md` 应该有命中 +- `frontend/src/pages/Docs/docs-content.ts` 中 `ops-runbook` 出现且分组为 `docs_admin` +- zh/en manual 章节标题对齐（按 `documentation-coverage-rules.md` 现有要求） +- Docs 站点访问：未登录看 manual/quickstart 正常；非 `docs_admin` 用户看不到 `ops-runbook`；`admin` 能看到 diff --git a/docs/plans/earth-vessel-ais-aggregation-plan.md b/docs/plans/earth-vessel-ais-aggregation-plan.md index c46f2ac5..07456a41 100644 --- a/docs/plans/earth-vessel-ais-aggregation-plan.md +++ b/docs/plans/earth-vessel-ais-aggregation-plan.md @@ -176,10 +176,12 @@ freshness: ## 聚合接口 +状态更新：开发期已直接切换到新船只快照接口。旧 `/api/v1/visualization/geo/vessels` 不再兼容返回数据，而是返回 `410 Gone`；新的 Earth 船只首屏应调用 `/api/v1/vessels/snapshot`，实时更新走 `/ws` 的 `vessels` 订阅。 + 现有展示接口应逐步改为消费聚合服务，而不是自己直接拼 `VesselPosition + VesselStatic`。 ```text -GET /api/v1/visualization/geo/vessels +GET /api/v1/vessels/snapshot?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 GET /api/v1/visualization/vessels/{mmsi} GET /api/v1/visualization/vessels/{mmsi}/track GET /api/v1/visualization/vessels/{mmsi}/conflicts @@ -317,7 +319,7 @@ VesselFinder 等服务里的船只图片不属于 AIS 实时数据本身。图 目标是让展示接口开始消费聚合结果，但前端形状保持兼容。 1. 实现 AIS 聚合服务，先兼容读取现有表，再逐步切换到原始观测层。 -2. 将 `/geo/vessels` 和 `/vessels/{mmsi}` 改为走聚合服务。 +2. 将船只列表迁移到 `/api/v1/vessels/snapshot`，并让 `/vessels/{mmsi}` 走聚合服务。 3. 将 `/vessels/{mmsi}/track` 改为走轨迹聚合逻辑。 4. 返回 `field_sources`、`selected_reasons`、`quality_flags`、`conflict_count`。 5. 加入 freshness fallback 和异常位置保护。 @@ -342,13 +344,13 @@ VesselFinder 等服务里的船只图片不属于 AIS 实时数据本身。图 3. 聚合结果返回 `source_summary`，展示每艘船的来源、观测数量、最新观测时间、传输模式和消息类型。 4. 保留 `field_sources` 和 `selected_reasons`，用于解释动态字段来自实时流、静态字段来自可用非空来源。 5. 船名标准化会读取 AISStream `MetaData.ShipName`；船型展示会从 `vessel_type_name` 和 AIS 数字 `vessel_type` 共同归一化，保证 marker 颜色、详情卡、hover 和搜索结果一致。 -6. `/geo/vessels` 不再默认限制 5000 艘；不传 `limit` 或传 `limit=0` 表示全量返回，前端默认也不再二次裁剪到 5000。 +6. 当前实现已转向 `/api/v1/vessels/snapshot`：必须带 bbox / zoom，默认 `limit=1000`，最大 `limit=5000`，不再支持旧 `/geo/vessels` 全量返回。 -### v3.1 — 聚合完整性修复（v4 前置） +### v3.1 — 聚合完整性修复（已被新快照接口取代） -目标是先保证“所有已采集到的船都能显示”，BarentsWatch 不因为接入 AISStream 而被 raw observation 聚合结果遮蔽。 +原目标是先保证“所有已采集到的船都能显示”，BarentsWatch 不因为接入 AISStream 而被 raw observation 聚合结果遮蔽。开发期产品尚未上线后，决策调整为直接淘汰 legacy 船只表兜底：船只快照只读取 `ais_raw_observations` 聚合结果，旧 `vessel_position + vessel_static` 不再合并进 `/api/v1/vessels/snapshot`。 -当前风险是 `/geo/vessels` 只要 raw observation 聚合返回非空，就直接使用 raw 聚合结果，不再补读兼容层 `vessel_position + vessel_static`。如果 raw observation 中只存在 AISStream 的几百艘船，或 BarentsWatch 历史数据没有完整回填到 raw 层，最终 Earth 就会只显示 AISStream 子集。 +因此以下 legacy merge 要求作废，保留在文档中只作为历史决策记录： 1. `/geo/vessels` 必须合并 raw observation 聚合结果和 legacy latest position 结果。 2. raw 与 legacy 同一 MMSI 同时存在时只显示一艘，优先使用 raw 聚合结果及其 `field_sources` / `selected_reasons`。 @@ -416,7 +418,7 @@ REST collector 的自然状态是 `fetch -> transform -> save -> progress 0..100 - `freshness.realtime_stream_seconds` / `polling_seconds` 必须为非负整数； - `mode=locked` 必须带非空 `locked_source`。 4. 聚合服务 `vessel_ais_aggregation.py` 在 `_select_position_observation` 中按 `freshness` 把过期实时流降级到 stale 候选；在 `_select_static_field` 中按 `field_rules.mode = source_priority / locked / newest / non_empty` 选源。 -5. 聚合输出每条 vessel 携带 `aggregation_strategy_version`，并在 `/geo/vessels` GeoJSON properties + `/vessels/{mmsi}` 详情中暴露。 +5. 聚合输出每条 vessel 携带 `aggregation_strategy_version`，并在 `/api/v1/vessels/snapshot` GeoJSON properties + `/vessels/{mmsi}` 详情中暴露。 6. API： - `GET /api/v1/vessel-aggregation/strategy` - `PUT /api/v1/vessel-aggregation/strategy`（校验失败 400） @@ -460,8 +462,8 @@ REST collector 的自然状态是 `fetch -> transform -> save -> progress 0..100 - 明显异常位置不会进入默认展示轨迹，并会留下 `quality_flags`。 - 同一时间窗口内多来源相近轨迹点只展示一个点。 - AISStream 重连或回放导致的重复消息不会重复进入聚合结果。 -- raw observation 聚合结果和 legacy latest position 结果会按 MMSI 合并，BarentsWatch-only 船只不会因为 AISStream 子集存在而消失。 -- 不传 `limit` 或传 `limit=0` 时，`/geo/vessels` 全量返回合并后的船只集合。 +- `/api/v1/vessels/snapshot` 只读取 AIS raw observation 聚合结果；legacy latest position 不再参与船只快照。 +- `/api/v1/visualization/geo/vessels` 返回 `410 Gone`，客户端必须迁移到新 snapshot API。 - AISStream 长连接收到新船、位置变化和航向变化后，会通过内部 `/ws` 的 `vessels` channel 推送增量。 - AISStream streaming 状态不会显示成固定百分比完成进度条，也不会在收到一批消息后误报采集完成。 - `mmsi`、`imo`、`callsign` 等身份编号在前端不显示千分位符。 diff --git a/docs/plans/earth-vessel-rendering-performance-plan.md b/docs/plans/earth-vessel-rendering-performance-plan.md index 7db14adc..7d560def 100644 --- a/docs/plans/earth-vessel-rendering-performance-plan.md +++ b/docs/plans/earth-vessel-rendering-performance-plan.md @@ -216,7 +216,7 @@ hover、locked、dimmed 可通过更新少量 instance attribute 实现，不再 ### 1. 请求视口范围 -前端请求 `/api/v1/visualization/geo/vessels` 时带上当前视口 `bbox`，减少无关船只。 +前端请求 `/api/v1/vessels/snapshot` 时必须带上当前视口 `bbox`、`zoom` 和受控 `limit`，减少无关船只。旧 `/api/v1/visualization/geo/vessels` 已下线并返回 `410 Gone`。 ### 2. 后端排序策略 diff --git a/docs/plans/user-registration-email-verification-plan.md b/docs/plans/user-registration-email-verification-plan.md new file mode 100644 index 00000000..6b1814b9 --- /dev/null +++ b/docs/plans/user-registration-email-verification-plan.md @@ -0,0 +1,155 @@ +# 用户公开注册与邮箱验证计划 + +**状态**：待实施 +**创建日期**：2026-05-12 +**核心目标**：给 Planet 增加公开注册流程 + 邮箱验证码 + 忘记密码，让客户无需管理员介入就能开通账号；同时把 SMTP 邮件作为可复用基础服务接入 `/settings`。 + +## 背景 + +当前认证只暴露 `/auth/login`、`/auth/refresh`、`/auth/logout`、`/auth/me`（`backend/app/api/v1/auth.py`），账号只能由 `super_admin` 在 `/users` 后台创建。User 模型 `backend/app/models/user.py` 没有 `email_verified` 字段，仓库也没有任何 SMTP/邮件发送基础设施。 + +客户旅程想从"打开浏览器→注册→验证→登录"开始走（见 [文档受众分层重构计划](/home/ray/dev/linkong/planet/docs/plans/docs-audience-split-plan.md)），就必须先把这条链路在代码里跑通。 + +注册策略（已确认）： + +- 开放公开注册，任何人可在 `/register` 自助开通 +- 默认角色 `viewer` +- 邮箱验证后立即可登录（无需管理员审批） +- 验证仅走 SMTP 邮件，6 位数字码，10 分钟 TTL + +## 数据模型 + +`backend/app/models/user.py` 加两列： + +```python +email_verified = Column(Boolean, default=False, nullable=False) +pending_email = Column(String(255), nullable=True) # 改邮箱时临时落地待验证地址 +``` + +迁移路径：仓库目前没看到 alembic 目录，沿用 `backend/app/db/session.py` 的初始化风格在启动时跑 `ALTER TABLE users ADD COLUMN IF NOT EXISTS ...`。先确认是否存在 alembic，若有则正规迁移。 + +不另建 `verification_codes` 表 — OTP 走 **Redis**（系统已有 Redis，token blacklist 也走 Redis）： + +``` +key: otp:{purpose}:{email} purpose ∈ {register, verify_email, reset_password} +value: { code_hash: bcrypt, attempts: int, issued_at: ts } +TTL: 600 秒 +``` + +`{purpose}:{email}` 同时配一个限流键 `otp_rate:{purpose}:{email}`，TTL 60 秒，用于"60 秒内禁止重发"。 + +## 服务拆分 + +按项目 `services/` 单职责风格拆两个： + +**`backend/app/services/otp.py`**（通用 OTP 原语，未来 2FA / 手机号验证可直接复用）： + +```python +async def issue_code(email: str, purpose: OtpPurpose) -> str # 生成 6 位、写 Redis、返回明码（调用方负责送达） +async def verify_code(email: str, purpose: OtpPurpose, code: str) -> bool # 校验并消耗 +async def check_resend_allowed(email: str, purpose: OtpPurpose) -> None # 抛 RateLimited 异常 +``` + +- 6 位数字，密码学随机 +- Redis 存 `bcrypt(code)`，不存明码 +- 校验失败计数 ≥ 5 直接失效该 key +- 重发触发即失效旧 code + +**`backend/app/services/email.py`**（通用 SMTP 发送，告警/摘要等后续可复用）： + +```python +async def send_email(to: str, subject: str, html: str, text: str | None = None) -> None +async def send_verification_email(to: str, code: str, purpose: OtpPurpose) -> None # 模板封装 +``` + +- 用 `aiosmtplib` 异步发送 +- 从 `system_settings` 的 `smtp` 命名空间读配置（host/port/username/password/from/use_tls） +- 未配置抛 `EmailNotConfiguredError` +- 模板用简单 HTML + 纯文本双段，按 `purpose` 切换文案 + +编排（"签码 → 发邮件"）在 `api/v1/auth.py` 端点里调两个服务，不在 service 内互相调用，保持单测可单独 mock。 + +## 后端端点 + +新增到 `backend/app/api/v1/auth.py`： + +| 端点 | 入参 | 行为 | +| --- | --- | --- | +| `POST /auth/register` | `username, email, password` | 用户名/邮箱查重 → 写 User `is_active=True, email_verified=False, role="viewer"` → 调 `otp.issue_code(email, "register")` → 调 `email.send_verification_email` | +| `POST /auth/verify-email` | `email, code` | `otp.verify_code` → 置 `email_verified=True` → 直接返回 access/refresh token | +| `POST /auth/resend-code` | `email, purpose` | `check_resend_allowed` → `issue_code` → `send_verification_email` | +| `POST /auth/forgot-password` | `email` | 即便邮箱不存在也返回 200（防枚举）；存在则签 `reset_password` 码并发邮件 | +| `POST /auth/reset-password` | `email, code, new_password` | `verify_code(..., "reset_password")` → `user.set_password(new_password)` | + +`/auth/login` 改造：邮箱未验证用户登录返回 `403 { code: "EMAIL_NOT_VERIFIED", email }`，前端拿到后跳验证页。 + +## SMTP 设置 + +复用 `backend/app/api/v1/settings.py` 现有 setting store，新增 `smtp` 命名空间： + +- `smtp_host`、`smtp_port`、`smtp_username`、`smtp_password`、`smtp_from`、`smtp_from_name`、`smtp_use_tls` +- 密码走与 collector 凭证相同的加密路径（看 `backend/app/services/` 是否已有 `credentials_encryption` 之类工具，若有直接复用） +- `POST /settings/smtp/test` — 用当前未保存的入参试发一封到指定地址，不落库 + +未配置 SMTP 时 `/auth/register` 应返回明确错误 `503 { code: "EMAIL_PROVIDER_NOT_CONFIGURED" }`，提示管理员先去 `/settings` 配 SMTP 或用 `./planet.sh createuser` 兜底。 + +## 前端 + +**新页面**： + +- `frontend/src/pages/Register/Register.tsx` — 两步表单：(1) 用户名/邮箱/密码 (2) 6 位验证码；60s 重发冷却；验证成功写 token，自动跳 `/admin` +- `frontend/src/pages/VerifyEmail/VerifyEmail.tsx` — 给登录拦截 `EMAIL_NOT_VERIFIED` 时落地的页，仅"输码 + 重发" +- `frontend/src/pages/ForgotPassword/ForgotPassword.tsx` — 两步：(1) 输邮箱 (2) 输码 + 新密码 + +**改动**： + +- `frontend/src/pages/Login/Login.tsx` — 表单下加"注册账号"、"忘记密码"链接；接 `EMAIL_NOT_VERIFIED` 跳 `/verify-email` +- `frontend/src/pages/Settings/Settings.tsx` — 新增 SMTP 子 tab（host/port/username/password/from/TLS + 测试发送按钮），用工作区里新建的 `frontend/src/components/ConnectionTestInput/` 做连通测试输入 +- 路由表（`frontend/src/App.tsx` 或 `frontend/src/router/*`）— 加 `/register`、`/forgot-password`、`/verify-email` + +## 关键文件清单 + +后端： + +- `backend/app/models/user.py` — 加字段 +- `backend/app/schemas/user.py` — 新增 `UserRegister`、`VerifyCode`、`ResetPasswordRequest` schema +- `backend/app/api/v1/auth.py` — 新端点 + 登录校验 +- `backend/app/services/otp.py` *(新)* +- `backend/app/services/email.py` *(新)* +- `backend/app/api/v1/settings.py` — SMTP 命名空间 + 测试发送 +- `backend/app/core/config.py` — SMTP 默认值/特性开关 +- `backend/app/db/session.py` — DDL 兜底（若无 alembic） + +前端： + +- `frontend/src/pages/Register/Register.tsx` *(新)* +- `frontend/src/pages/VerifyEmail/VerifyEmail.tsx` *(新)* +- `frontend/src/pages/ForgotPassword/ForgotPassword.tsx` *(新)* +- `frontend/src/pages/Login/Login.tsx` +- `frontend/src/pages/Settings/Settings.tsx` +- 路由文件 + +## 实施顺序 + +1. 后端：User 模型字段 + DDL 兜底 +2. 后端：`services/otp.py`（先纯单测跑通） +3. 后端：`services/email.py`（用 MailHog 本地试发） +4. 后端：`/auth/register` + `/auth/verify-email` + `/auth/resend-code` + 登录拦截 +5. 后端：`/auth/forgot-password` + `/auth/reset-password` +6. 后端：`/settings/smtp` 命名空间 + 测试发送 +7. 前端：`Settings.tsx` 加 SMTP 子 tab +8. 前端：Register / VerifyEmail / ForgotPassword 页 + Login 入口 + +文档同步在 [文档受众分层重构计划](/home/ray/dev/linkong/planet/docs/plans/docs-audience-split-plan.md) 落地。 + +## 验证 + +- **后端单测**（仿 `backend/tests/test_settings_ai_provider.py`）： + - 注册端点用户名/邮箱查重 + - OTP 过期、错码计数、重发限流 + - 邮箱未验证用户登录返回 `EMAIL_NOT_VERIFIED` + - SMTP 未配置时注册端点返回 `EMAIL_PROVIDER_NOT_CONFIGURED` + - 忘记密码对不存在邮箱仍返回 200 +- **后端集测**：本机起 MailHog 或 Mailtrap，把 SMTP 指到上面，跑 register → 收码 → verify → login 一遍 +- **前端**：`source ~/.zshrc && bun run build`；启 dev server 走 `/register` → `/verify-email` → `/admin` 全流程，再试 `/forgot-password` +- **手测**：新邮箱注册 → 收码 → 输错 → 重发 → 输对 → 登录 → 改密码 → 用新密码再登；管理员在 `/settings` 改 SMTP → 测试发送 diff --git a/docs/technical/en/backend-collectors.md b/docs/technical/en/backend-collectors.md index edbea3b2..7fa662d2 100644 --- a/docs/technical/en/backend-collectors.md +++ b/docs/technical/en/backend-collectors.md @@ -309,17 +309,50 @@ AIS observations do not directly replace final vessel records. They are first sa - Dynamic fields such as position, speed, and course are selected by freshness and source priority. - Static fields prefer non-empty values; conflicting candidates are recorded for detail and diagnostics views. -Earth still reads vessel data from: +Earth vessel rendering now consumes the bounded snapshot endpoint and realtime delta channel: ```http -GET /api/v1/visualization/geo/vessels +GET /api/v1/vessels/snapshot?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 GET /api/v1/visualization/vessels/{mmsi} GET /api/v1/visualization/vessels/{mmsi}/track GET /api/v1/visualization/vessels/{mmsi}/conflicts -GET /api/v1/visualization/vessels/aggregation/diagnostics ``` -`/geo/vessels` merges raw observation aggregation with the legacy BarentsWatch latest-position tables so adding AISStream does not hide historical BarentsWatch-only vessels. +`/api/v1/vessels/snapshot` requires `bbox` and `zoom`, defaults to `limit=1000`, and caps `limit` at `5000`. It reads only aggregated `ais_raw_observations`; it no longer merges legacy `vessel_position` / `vessel_static` rows. The old `/api/v1/visualization/geo/vessels` endpoint has been removed and returns `410 Gone`. + +Realtime deltas are sent through the `/ws` `vessels` channel. Clients must subscribe with the current viewport: + +```json +{ + "type": "subscribe", + "data": { + "channel": "vessels", + "bbox": [120.8, 30.7, 122.1, 31.8], + "zoom": 12, + "limit": 1000 + } +} +``` + +The backend stores lightweight subscription filters per connection and only sends vessel updates that match the subscriber bbox. Collector broadcasts enter a 1-second throttle queue; within each flush window, only the latest update per MMSI is retained. + +### Layer APIs And Global Stats + +Earth is moving to two API families: + +```http +GET /api/v1/data-products +GET /api/v1/data-products/{product_id}/status +GET /api/v1/layers/vessels/snapshot?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 +GET /api/v1/layers/cables?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 +GET /api/v1/layers/landing-points?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 +GET /api/v1/layers/satellites?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 +GET /api/v1/layers/bgp/anomalies?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 +GET /api/v1/layers/bgp/incidents?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 +GET /api/v1/layers/bgp/collectors?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 +``` + +`/api/v1/data-products/*` is for aggregate panels and keeps a global statistics scope independent of the map bbox. `/api/v1/layers/*` is for map rendering, requires `bbox` and `zoom`, defaults to `limit=1000`, and caps `limit` at `5000`; low zoom falls back to a smaller response cap and reports `degraded`, `truncated`, `limit_clamped`, and `stats_scope=viewport` in `diagnostics`. Non-vessel layers currently reuse the existing GeoJSON converters before the guard layer; future product-specific queries can push bbox filtering deeper. ## X. Collector Settings And Connectivity Validation @@ -390,4 +423,12 @@ curl -X POST http://localhost:8000/api/v1/datasources/1/trigger \ -H "Authorization: Bearer " ``` +Batch collection uses: + +```http +POST /api/v1/datasources/trigger-batch +``` + +The request body may pass `source_ids` for selected rows. Without `source_ids`, the backend filters by `product`, `module`, `is_active`, `run_status`, `collected`, `credential_status`, and `q`. The endpoint skips disabled sources, sources already running without `force`, and sources still inside their frequency window, then returns `triggered`, `skipped`, and `failed` groups. + **Core file**: `backend/app/api/v1/datasources.py` diff --git a/docs/technical/en/datasource-collector-settings-connectivity.md b/docs/technical/en/datasource-collector-settings-connectivity.md index 40193768..77d857bc 100644 --- a/docs/technical/en/datasource-collector-settings-connectivity.md +++ b/docs/technical/en/datasource-collector-settings-connectivity.md @@ -288,6 +288,16 @@ Normalization: Connectivity validation reads saved configuration, environment variables, and `AISSTREAM_API_KEY` from `~/.zshrc`. For actual collection, prefer saving the API key in collector settings. If the key only lives in `~/.zshrc`, confirm that the backend process inherited it; otherwise validation may pass while the collector runtime cannot read the key. +Connectivity validation and actual collection are separate actions. A banner such as `AISStream credentials configured, WebSocket endpoint format valid` only means the saved settings can be used for a connection attempt; runtime status may still be `disconnected`. Global AIS data is written locally only while the `aisstream_vessels` collector is `streaming` / `connected` and its message count plus `last_seen_at` keep advancing. + +The new vessel list entry point is no longer the legacy `/api/v1/visualization/geo/vessels` route. Earth initial state should call: + +```http +GET /api/v1/vessels/snapshot?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 +``` + +That endpoint reads local aggregated `ais_raw_observations` only. Realtime updates use the `/ws` `vessels` channel; subscriptions must include `bbox`, `zoom`, and `limit`. The server filters updates per connection and merges collector broadcasts every second, keeping only the latest position per MMSI. + ## Custom REST / WebSocket Mapping Runtime Files: diff --git a/docs/technical/en/earth-frontend-context.md b/docs/technical/en/earth-frontend-context.md index 7928d3da..5e7a4811 100644 --- a/docs/technical/en/earth-frontend-context.md +++ b/docs/technical/en/earth-frontend-context.md @@ -152,7 +152,11 @@ The `earth:compute-center-location-saved` reconciliation pipeline is deliberatel ### AIS Vessel Layer -The vessel layer fetches `/api/v1/visualization/geo/vessels` and renders the aggregated AIS GeoJSON through `createInteractableLayer()`. By default it does not send a `limit` parameter, and `VESSEL_CONFIG.maxRenderedMarkers = 0` means the frontend does not clip the result to 5000 vessels. A positive `options.limit` or positive `maxRenderedMarkers` can still be used as an explicit temporary cap. +The vessel layer now uses `/api/v1/vessels/snapshot` for the initial viewport snapshot and the `/ws` `vessels` channel for realtime deltas. Snapshot requests must include `bbox`, `zoom`, and a bounded `limit`; the backend defaults to `limit=1000` and caps it at `5000`. WebSocket subscriptions must include the same viewport fields so the server can filter updates per connection. + +The legacy `/api/v1/visualization/geo/vessels` endpoint has been removed and returns `410 Gone`. Frontend code should fetch a snapshot for the current viewport when the layer opens, then subscribe to `vessels` deltas. After map pan or zoom, reload the snapshot and send a fresh vessels subscription. The backend no longer merges legacy `vessel_position` / `vessel_static` rows into vessel snapshots, so the frontend must not depend on old BarentsWatch-only fallback rows. + +The new layer API family is `/api/v1/layers/*`, which separates map rendering payloads from aggregate panel statistics. Layer requests must include `bbox`, `zoom`, and a bounded `limit`; responses include `visible_count`, `returned_count`, and `diagnostics`, where `degraded`, `truncated`, and `limit_clamped` are the frontend signals for fallback UI. Right-side aggregate panels should not sum the layer response. They should read `/api/v1/data-products` or `/api/v1/data-products/{product_id}/status`, because those statistics stay global and do not change with the viewport. Vessel color and vessel type text must use the same normalized classification. `vessels.js` derives `type` from both `vessel_type_name` and the AIS numeric `vessel_type` code; that `type` drives marker color. It also derives `vessel_type_display`, which `main.js` uses for the info card, hover summary, and search result subtitle. Do not make the info card read only the raw `vessel_type_name`, because AISStream can provide a numeric type while the raw name is still `Other`. diff --git a/docs/technical/en/frontend-admin-frontend-context.md b/docs/technical/en/frontend-admin-frontend-context.md index b7e153f8..dcc87602 100644 --- a/docs/technical/en/frontend-admin-frontend-context.md +++ b/docs/technical/en/frontend-admin-frontend-context.md @@ -163,7 +163,25 @@ Current constraints: - Internal document links should be converted to `/docs/:slug` through `transformLink` - Heading anchors are injected through `getHeadingId`, keeping route state outside the renderer -### 6. `TableActions` +### 6. `ConnectionTestInput` + +File: + +- [ConnectionTestInput.tsx](/home/ray/dev/linkong/planet/frontend/src/components/ConnectionTestInput/ConnectionTestInput.tsx) + +Purpose: + +- Console form fields that combine an endpoint/Base URL value with a connection check +- Connection-test entry points for AI Provider and WebSearch +- Future collector configuration fields should reuse it when the test action belongs inside the input + +Current constraints: + +- The input suffix shows a single plug/connector icon, not an adjacent text button +- Disabled integrations must grey out both the input and its connection-test action +- The component only combines the input and action; callers still own form state, loading, disabled state, and the request itself + +### 7. `TableActions` File: @@ -205,10 +223,13 @@ File: Responsibilities: - `/ai` now owns LLM Provider, AI Tool configuration, and the testbench instead of nesting them under `/settings` -- The `模型供应商` tab manages default provider, model, base URL, provider key, local `aiprovider` proxy, and connection test -- The `工具` tab manages WebSearch provider, search key, base URL, timeout, result count, and advanced provider options +- The `模型供应商` tab manages default provider, model, base URL, provider key, local `aiprovider` proxy, and connection test; provider and model fields use editable comboboxes so users can manually enter new providers/models if the models.dev catalog stops updating +- The `工具` tab first selects a tool from a dropdown menu, then renders that tool's configuration; it currently includes WebSearch and OCR +- WebSearch configuration includes provider, search key, base URL, timeout, result count, and advanced provider options +- OCR configuration includes provider, Base URL, API key, model/engine, languages, timeout, file-size limit, and output format - The `测试台` tab embeds the former Playground real session, preset prompts, and AI Provider status debugging - The page reuses the Settings single-screen tabs, panel card, and internal scrolling style +- AI Provider and WebSearch connection tests use `ConnectionTestInput`, with the connector icon fixed at the end of the Base URL input; when WebSearch is disabled, every configuration field and the test entry point are greyed out except the switch Legacy `/settings?tab=ai` should redirect to `/ai?tab=providers`. Legacy `/playground` should redirect to `/ai?tab=playground`. diff --git a/docs/technical/en/manual.md b/docs/technical/en/manual.md index 132b6f0f..e323680e 100644 --- a/docs/technical/en/manual.md +++ b/docs/technical/en/manual.md @@ -1,666 +1,332 @@ # Planet Manual -This manual is for daily use, demos, development integration, and local operations. It covers four core entry points: +This manual is for Planet end users. Starting from the browser, it covers account registration, login, configuring collectors, configuring AI, using Earth and the console, and reading the docs site. Every action happens in a browser. -- `planet.sh`: local start, stop, restart, health check, and log access -- Earth: public 3D situational awareness page -- Console: admin backend (login required) -- Docs: backend Gatekeeper-controlled documentation; basic usage docs are public, while developer and operations docs require permission groups - -For the shortest path to getting started, see [Quickstart](/home/ray/dev/linkong/planet/docs/technical/en/quickstart.md). For common troubleshooting, see the [FAQ](/home/ray/dev/linkong/planet/docs/technical/en/faq.md). +If you are responsible for deployment or on-call duty, read the [Planet Ops Runbook](/home/ray/dev/linkong/planet/docs/technical/en/ops-runbook.md) instead — it covers shell commands, log paths, and CLI fallbacks for user creation. ## Entry Overview -After a default startup, the common URLs are: - | Name | URL | Login Required | Description | | --- | --- | --- | --- | -| Earth | `http://localhost:3000/earth` | No | 3D globe, layers, BGP, satellites, cables, news situational awareness | -| Docs | `http://localhost:3000/docs` | Partly | Usage docs are public; developer, backend, and operations docs require Gatekeeper groups | -| FAQ | `http://localhost:3000/docs/faq` | No | Windows / WSL, ports, dependencies, motion capture, credentials, and permission troubleshooting | -| Console | `http://localhost:3000/admin` | Yes | Data, config, alerts, logs, and situational observation | -| AI | `http://localhost:3000/ai` | Yes | Model providers, AI tools, and testbench | -| Backend API Docs | `http://localhost:8000/docs` | Depends on endpoint | FastAPI / OpenAPI documentation | +| Earth | `http:///earth` | No | Public 3D situational awareness page | +| Docs | `http:///docs` | Partly | Public docs need no login; developer/ops docs need Gatekeeper groups | +| Register / Login / Forgot Password | `/register`, `/login`, `/forgot-password` | No | Self-serve account creation and recovery | +| Console | `http:///admin` | Yes | Data, collectors, alerts, AI, users, settings | +| AI | `http:///ai` | Yes | Model providers, tools, testbench | +| Backend API Docs | `http://:8000/docs` | Depends | FastAPI / OpenAPI | -## planet.sh +URLs below use the local default `http://localhost:3000`. Replace the prefix with your deployment URL in production. -`planet.sh` is the main control script for local development and demos. Use it to manage services rather than manually starting frontend, backend, database, and AI Provider separately. +## Register an Account -### Start +1. Open `http://localhost:3000/login` and click "Register" under the form. +2. On `/register`, fill in: + - **Username**: 3–50 characters, used to log in + - **Email**: receives the verification code; editable later in account settings + - **Password**: at least 8 characters +3. After submission you are taken to the verify page. A 6-digit code is sent to your email. It expires in 10 minutes. +4. Enter the code and click "Verify and Sign In". On success the system stores a session and sends you to the console. -```bash -./planet.sh start -``` +If no email arrives within 60 seconds: -Default behavior: +- Check spam, promotions, and any enterprise mail gateway +- The "Resend Code" button shows a 60-second countdown; you can resend after it ends +- After 5 wrong attempts the code is invalidated; you must resend a new one -- Starts PostgreSQL and Redis -- Starts AI Provider -- Starts the backend API -- Starts the frontend Vite dev server -- Outputs Earth, console, Playground, and backend API doc URLs +If you see "Email service not configured", the administrator has not yet set up SMTP. Ask the administrator to fill SMTP at `/settings -> SMTP Email`. -Specify custom ports: +The default role for a self-registered user is `viewer`, which can sign in and view public content. To see collector / user / settings pages, ask an `admin` or `super_admin` to promote your role at `/users`. -```bash -./planet.sh start -b 8001 -f 3001 -a 8101 -``` +## Sign In and Recover Password -Parameters: +### Sign In -| Flag | Meaning | -| --- | --- | -| `-b ` | Backend port | -| `-f ` | Frontend port | -| `-a ` | AI Provider port | -| `--allow-lan` | Enable LAN access | -| `--verbose` | Show more command output during execution | +Open `/login`, enter username and password. On success you are taken to `/admin`. -### AI Provider Environment and Builds +If you see "Email not verified", the page automatically redirects to `/verify-email` — follow the prompts to enter the code. -AI Provider runtime configuration can live in `aiprovider/.env` or in matching variables in `~/.zshrc`. `planet.sh` reads simple `export AI_...=...` / `AI_...=...` lines and passes them to the container at startup. +### Forgot Password -Changing model, API key, or base URL does not rebuild the image. Restart only AI Provider to pick up runtime configuration changes: +1. On `/login`, click "Forgot Password?", or open `/forgot-password` directly. +2. Enter your registered email and click "Send Code". The same confirmation is shown regardless of whether the email is registered (to avoid enumeration). +3. After receiving the code, enter it together with a new password (≥ 8 characters) and click "Reset Password". +4. The system sends you back to `/login` — sign in with the new password. -```bash -./planet.sh restart -a -``` +## Account Settings -For complex shell expansion in `~/.zshrc`, opt in explicitly: +Click your username at the top-right of the console to open account settings: -```bash -PLANET_LOAD_ZSHRC_ENV=source ./planet.sh start -a -``` +- Change password: enter current password + new password +- Change email: the system sends a verification code to the new address; the change applies only after verification +- View Gatekeeper groups: lists current groups (`docs_user` / `docs_developer` / `docs_admin`) +- Log out: clears the current session -To ignore `~/.zshrc` during troubleshooting: +## Console Overview -```bash -PLANET_LOAD_ZSHRC_ENV=0 ./planet.sh start -a -``` - -The AI Provider Docker build context is intentionally limited to the files required by the service, and `uv sync` uses a BuildKit cache mount so dependency downloads are reused after the first build. - -### Stop - -```bash -./planet.sh stop -``` - -Stops: - -- Backend -- AI Provider -- Frontend -- PostgreSQL -- Redis - -### Restart - -Full restart: - -```bash -./planet.sh restart -``` - -Per-module restart: - -```bash -./planet.sh restart -b -./planet.sh restart -f -./planet.sh restart -a -./planet.sh restart -d -``` - -| Flag | Effect | -| --- | --- | -| `-b` | Backend only | -| `-f` | Frontend only | -| `-a` | AI Provider only | -| `-d` | Database only | - -Per-module restarts are preferred during development — they avoid interrupting unrelated services. - -### Create User - -```bash -./planet.sh createuser -``` - -Used to create a console login account before first use. The script interactively prompts for username, password, and role. - -### Health Check - -```bash -./planet.sh health -``` - -Checks: - -- `planet_*` container status -- Backend `/health` -- AI Provider `/health` -- Frontend reachability - -If something shows offline, check the corresponding logs first. - -### Logs - -Recent logs: - -```bash -./planet.sh log -``` - -Follow logs: - -```bash -./planet.sh log -f -./planet.sh log -b -./planet.sh log -a -``` - -| Flag | Log source | -| --- | --- | -| `-f` / `--frontend` | `/tmp/planet_frontend.log` | -| `-b` / `--backend` | `/tmp/planet_backend.log` | -| `-a` / `--ai-provider` | `planet_aiprovider` container logs | - -### LAN Access - -```bash -./planet.sh start --allow-lan -``` - -Useful for: - -- Starting in WSL, accessing from Windows browser -- Demos on phone or tablet -- Another machine on the same LAN accessing the same dev instance - -`--allow-lan` only makes the frontend and backend listen on `0.0.0.0`. When Planet runs in WSL, Windows can usually reach it through `localhost`, but access from a phone or another computer through `http://:3000` still depends on Windows port forwarding and firewall rules. - -Use this order to diagnose: - -```bash -# From WSL or the shell running Planet -curl http://localhost:3000 -curl http://localhost:8000/health -ss -ltnp | grep -E ':3000|:8000' -``` - -If this shows `0.0.0.0:3000` and `0.0.0.0:8000`, but the LAN IP still fails, configure Windows from an elevated PowerShell: - -```powershell -netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=3000 connectaddress=127.0.0.1 connectport=3000 -netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=8000 connectaddress=127.0.0.1 connectport=8000 - -New-NetFirewallRule -DisplayName "WSL Planet 3000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 3000 -New-NetFirewallRule -DisplayName "WSL Planet 8000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 8000 -``` - -## Earth - -Earth is the public 3D situational awareness page, accessed at: - -```text -http://localhost:3000/earth -``` - -It is a standalone frontend. The actual page lives at: - -- `frontend/public/earth/index.html` -- `frontend/public/earth/js/` -- `frontend/public/earth/css/` - -The React route `/earth` simply hosts it in an iframe. - -### Main Uses - -Earth is used to observe in a single globe view: - -- BGP events, anomalies, and situational posture -- Satellites and orbital trails -- Submarine cables and landing points -- Compute centers -- AIS vessels -- Border lines, grid lines, HD texture, cloud layer, terrain -- Live news streams and situational news -- Search and focused object details - -### Layer Control - -The right-side layer panel toggles visualization layers on or off. - -Common layers include: - -- Grid lines -- Border lines -- HD texture -- Atmospheric cloud layer -- Submarine cables -- Compute centers -- BGP observation -- AIS vessels -- Satellites -- Orbital trails -- Terrain - -Some layers have dependencies: - -- Terrain requires HD texture -- Trails require Satellites -- When HD texture is off, the globe shows the base map and edge glow effect - -### Legend - -The lower-left legend follows the currently focused or enabled layer. - -Current legend modes include: - -- Cables -- Satellites -- Border lines -- Compute centers -- BGP -- AIS vessels - -AIS vessel legend entries are grouped by vessel type: cargo, tanker, passenger, fishing, military, anchored/slow, and other. Triangle markers represent moving vessels; dots represent anchored or slow vessels. - -### Search - -Earth search finds current globe objects, such as: - -- Submarine cables -- Landing points -- Satellites -- Compute centers -- BGP events -- BGP collectors - -Search results can be used to quickly locate objects and open their details. - -### Location Candidate Collection - -Compute-center and BGP collector detail cards can collect candidate coordinates automatically. After clicking an object, use `自动采集坐标候选` or `重新自动采集坐标`; the backend ranks source coordinates, open organization lookups, and Nominatim online search results. If those regular sources return no candidates, the current default AI Provider is used once as an LLM factcheck fallback. Stored BGP collector locations are used as query context only and are not emitted as candidates. - -Candidates can be previewed directly on Earth. Compute-center candidates can be saved into the `compute_center_locations` dimension table from the detail card, then the layer refreshes immediately. The notification badge on the compute-center layer row shows unresolved records that cannot be rendered; clicking it opens the queue, where users can collect individual candidates or use `一键采用` to save the highest-confidence candidate top-to-bottom. Records without candidates stay in the queue and are not replaced by country centroids or hard-coded hints. See [Earth Location Candidate Collection User Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-user.md) for the full workflow. - -### Settings - -The settings panel contains: - -- Rotation mode / cruise mode / motion mode -- Cruise modules: BGP, News, Compute Centers, Vessels, Cables, Satellites -- View settings: satellite display style, day/night mode, panel visibility -- Motion Debug Mode, Motion Input Source, skeleton-only debug view -- Globe default size -- Terrain opacity -- Reset settings - -These settings are stored in browser local storage. They revert to defaults if you switch browsers or clear site data. - -### View Controls - -Earth supports mouse, touchpad, and touchscreen interaction. - -Common controls: - -| Action | Result | -| --- | --- | -| Left-button drag | Rotates the globe | -| One-finger drag | Rotates the globe on touch devices | -| Mouse wheel | Zooms the view in or out | -| Two-finger pinch | Zooms the view on touch devices | -| Zoom buttons | Adjust zoom in fixed steps | -| Click the zoom percent | Resets to the default zoom | - -When zooming, the top capsule briefly shows the current zoom level, for example `Zoom 180%`. This indicates view zoom only, not data loading progress. Loading status takes priority and will not be interrupted by zoom feedback. - -Drag sensitivity adjusts automatically based on the current zoom. Around the default view it keeps the normal rotation feel; when zoomed in, dragging becomes progressively finer for inspecting a region, vessel, satellite, or BGP event; when zoomed out, dragging is slightly faster for global browsing. - -### Motion Capture Controls - -Earth has a motion-capture control entry point for large-screen and future 3D displays. There are two realtime input sources: the default `Browser Camera` source uses webpage `getUserMedia` and recognizes gestures locally in the browser; the advanced `Motion Agent` source uses `camera/RTSP/HTTP -> local Agent -> local WebSocket -> Earth page`. Neither path sends camera frames or realtime gesture decisions to the cloud, and neither path reuses the news/RSS aggregation APIs. - -It is disabled by default. Enable `Motion Debug Mode` in settings, open Earth with `?motion=1`, or set `planet-earth-motion-control-enabled=true` in browser local storage to start the selected source. The default source is `Browser Camera`; it requires HTTPS or localhost and a granted browser camera permission, but does not require installing an app. For dual cameras, USB indexes, phone/network camera streams, client integration, or edge devices, switch the setting to `Motion Agent`. The default Agent URL is `ws://127.0.0.1:8765/ws/gestures`; the `motionAgent` URL parameter can override it. - -URL parameters can also force the source: `?motion=1&motionProvider=browser` uses the browser camera, `?motion=1&motionProvider=agent` uses Motion Agent, and providing `motionAgent=ws://...` automatically selects Motion Agent. - -Current gesture semantics: - -| Gesture event | Result | -| --- | --- | -| `rotate_left` | Rotates the globe left | -| `rotate_right` | Rotates the globe right | -| `rotate_up` | Rotates the globe upward | -| `rotate_down` | Rotates the globe downward | -| `zoom_in` | Zooms in | -| `zoom_out` | Zooms out | -| `focus_prev` / `focus_next` | Switches targets within the current motion layer | -| `layer_prev` / `layer_next` | Switches the motion candidate layer and cruises to the nearest target in that layer | -| `confirm` | Confirms the currently selected target; browser recognition currently keeps the two-hands-up confirm gesture disabled | - -The settings panel also includes `Motion Debug Mode`, which opens the debug panel. With the Browser Camera source, the panel shows a local live preview and draws joints and bones over it. With the Motion Agent source, the Agent sends normalized skeleton events only and does not send raw video frames. The `Skeleton Only` switch hides the video preview and keeps the dark canvas plus skeleton; `Stop Matching Gestures` pauses gesture execution while preview and skeleton drawing can continue for debugging. Unmatched skeletons are red; once a gesture matches, the skeleton turns green and the matched gesture name is shown. Both this entry and the `Motion Input Source` control already carry Gatekeeper permission markers for future authorization control. - -### Cruise Mode - -Cruise mode makes Earth automatically cycle through focus targets. - -Current cruise modules: - -- BGP -- News -- Compute Centers -- Vessels -- Cables -- Satellites - -Suitable for demos, monitoring displays, or unattended presentations. - -### Mobile - -Earth has a mobile drawer layout. On small screens: - -- Layer controls open in a mobile drawer -- Search, settings, and details use mobile panels -- Main interactions remain centered on globe object clicks, search, and layer toggles - -### Common Issues - -#### Earth Won't Open - -Check whether the frontend is online: - -```bash -./planet.sh health -./planet.sh log -f -``` - -If the frontend port is not `3000`, use the actual port shown at startup. - -#### Layer Has No Data - -Check the backend and data sources: - -```bash -./planet.sh health -./planet.sh log -b -``` - -Then open the console and check: - -- `/datasources` -- `/data` -- `/bgp` - -#### Satellites, BGP, or Cables Load Slowly - -These layers may depend on backend APIs, external data sources, or first-run collection tasks. Wait for startup tasks to finish before checking logs and console data source status. - -## Console - -Console entry point: - -```text -http://localhost:3000/admin -``` - -The console requires login. Create a user first if this is your first time: - -```bash -./planet.sh createuser -``` - -### Page Structure - -The console uses React + Ant Design, with a left-side menu organized by work domain. - -Common pages: +The console at `http://localhost:3000/admin` is built with React + Ant Design. The left menu is organized by work domain. | Page | Route | Purpose | | --- | --- | --- | | Dashboard | `/admin` | System overview | -| Earth | `/earth` | Opens the public Earth page | -| Data Sources | `/datasources` | View data sources and trigger collection | -| Collected Data | `/data` | View collected data | -| BGP Observation | `/bgp` | BGP situational data | +| Earth | `/earth` | Open the public Earth page | +| Datasources | `/datasources` | Source directory and collection triggers | +| Collected Data | `/data` | Data already ingested | +| BGP | `/bgp` | BGP situational view | | System Alerts | `/alerts/system` | System-level alerts | | BGP Alerts | `/alerts/bgp` | BGP-related alerts | -| Situational Alerts | `/alerts/situational` | Situational assessment alerts | -| AI | `/ai` | Model providers, WebSearch-style tools, and testbench | -| System Logs | `/logs` | View system logs (typically super admin only) | -| Users | `/users` | User management | -| Settings | `/settings` | System config and TV live stream sources | +| Situational Alerts | `/alerts/situational` | Situational analysis alerts | +| AI | `/ai` | Model providers, tools, testbench | +| Logs | `/logs` | Usually visible only to super admin | +| Users | `/users` | Create/delete users, change roles/groups | +| Settings | `/settings` | System, SMTP, TV, collectors | -### Data Sources +Menu items hide automatically when you lack permission. If a menu is missing, check your role and Gatekeeper groups. -`/datasources` shows collection sources and triggers collection. It is now a data source directory that lists built-in and custom sources in one table. +## Configure Data Collectors -Common operations: +`/settings?tab=collector_credentials` is the "Collector Settings" page. It manages connection configuration for every collector, not just credentials. -- View data source status -- Trigger collection -- View recent collection tasks -- Open the read-only detail drawer for endpoint, headers, runtime config, and built-in/custom source type +Steps: -If a category of objects is missing on Earth, start here to confirm the data source is available. +1. Pick a collector in the dropdown. +2. Inspect status tags: + - `No credentials` / `Credentials required` + - Owning module + - `Enabled` / `Disabled` + - `Unchecked` / `Reachable` / `Unreachable` +3. Click the plug icon next to the dropdown to run a health check. On success the status becomes `Reachable`. +4. Edit endpoint, headers, timeout, retries; click save. -The data source name opens an information drawer only. Endpoint, credentials, headers, and custom source configuration are maintained under `/settings` collector settings. +For free collectors without credentials, the health check hits the endpoint directly. For credential-bearing collectors it runs the credential flow. If credentials or endpoint changed since the last successful check, click connect again. -When collection tasks are running, the progress area shows a clickable `Collecting N` pill. Clicking it opens a modal with each running task's phase, progress, and processed count. +"Connected" means either: data was successfully collected with the current config, or the connect button passed validation with the current config. -### Collected Data +### BarentsWatch AIS Credentials -`/data` shows the collected data table. - -Useful for diagnosing: - -- Whether data has entered the system -- Whether data update times match expectations -- Whether a data source produced valid records - -### BGP Observation - -`/bgp` is the BGP-focused page. - -It complements the BGP layer on Earth: - -- Earth emphasizes spatial posture and visual focus -- The console BGP page emphasizes lists, status, details, and assessment - -### Alerts - -Alert entry points: - -- `/alerts/system` -- `/alerts/bgp` -- `/alerts/situational` - -Used to view system, network, and situational alerts. - -### System Settings - -`/settings` manages system-level configuration. - -Current common uses: - -- System settings -- TV live stream source configuration -- Collector settings - -### AI - -`/ai` manages the AI runtime chain and is now separate from system settings. Legacy `/playground` redirects to `/ai?tab=playground`. - -It currently contains: - -- `模型供应商`: default LLM provider, model, base URL, API key, local `aiprovider` proxy, and connection test -- `工具`: WebSearch provider, search API key, base URL, max results, timeout, and advanced provider options -- `测试台`: AI Provider status, preset prompts, and real analysis-chain debugging - -Legacy `/settings?tab=ai` redirects to `/ai?tab=providers`. - -Available configuration depends on the current user's role. - -#### Collector Settings - -`/settings?tab=collector_credentials` is currently displayed as Collector Settings. It manages connection settings for all collectors, not only credentials. - -Use it to: - -1. Select a collector from the dropdown. -2. Review tags such as `Requires credentials`, module, enabled state, and `Unchecked` / `Available` / `Unavailable`. -3. Click the plug icon next to the selector to run a health check. -4. Edit endpoint, request headers, timeout, and retry settings. -5. Save the collector settings. - -Free collectors are checked by requesting their endpoint directly. Credentialed collectors use their credential provider. If endpoint or credential fingerprint changes after the last successful validation, the collector must be checked again. - -The system treats a collector as connected when the current configuration has either collected data successfully or passed the manual connection check. - -#### BarentsWatch AIS Credentials - -`BarentsWatch AIS` is a credentialed built-in collector. Its credential card appears above the basic configuration card. - -Configured fields: +`BarentsWatch AIS` is a credential-required built-in collector. Selecting it surfaces the credential section above the base configuration: - `Client ID` - `Client Secret` - `Endpoint` -If a secret is already configured, the input shows a masked preview. Keeping that preview unchanged preserves the stored secret; entering a new value replaces it. +If a secret was saved previously, the input shows a masked preview. Saving while keeping the preview unchanged preserves the original secret; entering a new secret overwrites it. -BarentsWatch AIS credentials can be read from: +When the connection fails, the page opens a credential guide. You can: -1. Collector settings saved in the console. -2. Backend environment variables: - - `BARENTSWATCH_CLIENT_ID` - - `BARENTSWATCH_CLIENT_SECRET` - - historical spellings: `BARRENTSWATCH_CLIENT_ID`, `BARRENTSWATCH_CLIENT_SECRET` -3. matching `export` lines in `~/.zshrc`. +- View the default guide +- Click "Guide not helpful" to ask AI Provider to regenerate from the default prompt +- Click "Reset" to restore the default guide -If connection fails, the page opens the credential guide. The guide can be regenerated through AI Provider or reset to the default guide. The default guide points users to the official BarentsWatch tutorial and emphasizes selecting `AIS - API`, not the regular `BarentsWatch - API`. +The default guide follows the BarentsWatch official tutorial and reminds you to choose `AIS - API` for Live AIS. -### System Logs +### AISStream Realtime Vessels -`/logs` views system logs. If the menu item is not visible, the current user likely lacks the required role. +`AISStream Realtime Vessels` is the global AIS WebSocket collector. A passing connection test only confirms API key + endpoint format. Actual global vessel data requires the backend `aisstream_vessels` collector to stay connected and write to `ais_raw_observations`. -Common troubleshooting sequence: +Steps: -```bash -./planet.sh health -./planet.sh log -``` +1. Open `/settings?tab=collector_credentials` and select `AISStream Realtime Vessels : aisstream_vessels` +2. Fill the AISStream API Key +3. Keep the default endpoint `wss://stream.aisstream.io/v0/stream` +4. Click the plug icon to test; confirm it reports `Reachable` +5. Save collector settings +6. Trigger the `aisstream_vessels` collector from the collection scheduler +7. Watch the `AISStream Runtime` panel: + - `streaming` / `connected` means the live stream is being consumed + - `messages this round` should keep growing + - `disconnected` with `ConnectionResetError` means the upstream or network dropped; re-trigger or wait for reconnect -Then open `/logs` for more structured runtime information. +## Configure AI Credentials -## Docs +`/ai?tab=providers` is the AI management entry. Two key sub-tabs: -Documentation site: +- `Model Providers`: default LLM provider, model, base URL, API key, local `aiprovider` proxy, connection test +- `Tools`: a dropdown for specific tools — currently WebSearch and OCR -```text -http://localhost:3000/docs -``` +### Model Providers -Docs content is read through backend APIs by permission. The frontend no longer bundles all Markdown files directly. Source files still live in: +Providers and models accept presets or arbitrary custom IDs. Common fields: -```text -docs/technical/zh/ (Chinese) -docs/technical/en/ (English) -``` +- Provider: e.g. `minimax`, `openai`, `anthropic`, `ollama` +- Protocol: `OpenAI Chat Completions` / `Anthropic Messages` / `Ollama Generate` +- Base URL: model API URL +- Default Model: e.g. `gpt-5.1`, `MiniMax-M2.7` +- API Key: stored on save; displayed masked afterwards +- Max Tokens, Anthropic Version: keep defaults if unsure +- Timeout / Retry: timeout and retry attempts -Anonymous visitors only see `public` docs such as the overview, quickstart, and manual. Logged-in users can see more technical docs when assigned Gatekeeper groups: +The plug icon at the end of the Base URL input runs a connection test. A passing test echoes the model's short reply. -- `docs_user`: user-operation docs. -- `docs_developer`: Earth, frontend, backend, collector, and AI Provider development docs. -- `docs_admin`: service control, operations, environment variable, and sensitive-operation docs. +### Tools -`admin` receives admin-doc access by default, and `super_admin` can read all Docs content. Gatekeeper groups are configured in the console Users page. +- **WebSearch**: provider, API key, base URL, max results, timeout, advanced provider parameters. While disabled, all fields except the enable switch are greyed out +- **OCR**: provider, base URL, API key, model/engine, recognition languages, timeout, max file size, output format -Docs supports: +The legacy link `/settings?tab=ai` redirects to `/ai?tab=providers`. -- Category navigation -- Markdown rendering -- Tables and code blocks -- In-document table of contents -- Search across currently visible docs -- Internal links between technical documents +## System Settings -When adding a new technical document, check: +`/settings` manages system-level configuration. Sub-tabs: -- Does it have a clear top-level heading -- Does it need to be added to backend Docs metadata for category and ordering -- Should it be classified as `public`, `docs_user`, `docs_developer`, or `docs_admin` +- **System Display**: name, refresh interval, retention, max concurrent tasks +- **Notifications**: alert email switch, recipient, critical/warning/daily summary +- **Security**: session timeout, max login attempts, password policy +- **SMTP Email**: outgoing email used by registration and password reset (visible to `admin` / `super_admin` only) +- **TV Livestream**: TV source management +- **AI / WebSearch / OCR**: see above -## Development Command Conventions +### SMTP Email Settings -Frontend commands must use Bun: +Public registration and verification codes depend on this section. An `admin` or `super_admin` opens `/settings -> SMTP Email` and fills: -```bash -cd frontend -bun install -bun run dev -bun run build -``` +- SMTP host, port +- Username, password +- From address (required), from name +- STARTTLS (typical for port 587) or implicit TLS (port 465) +- Timeout in seconds -Do not use `npm run ...`. The project uses Bun in WSL / Windows mixed environments to avoid Node/npm path compatibility issues. +Save, then click "Send Test Email" and enter a recipient address to verify delivery. Once that works, regular users can self-register at `/register`. -Verify the frontend build: +Leaving the masked password preview unchanged keeps the original password. Enter a new value to replace it. -```bash -source ~/.zshrc && bun run build -``` +## User Management (Admins) -## Troubleshooting Order +`/users` is `super_admin`-only for create/delete. The page supports: -When something goes wrong, follow this sequence: +- Listing users (username, email, role, active, email verified) +- Creating users (equivalent to public registration but skips email verification — administrator vouching) +- Changing roles: `viewer` / `operator` / `admin` / `super_admin` +- Editing Gatekeeper groups: `docs_user` / `docs_developer` / `docs_admin`, controlling which docs are visible +- Disabling / enabling accounts -1. Check service status: +To let a regular user read developer or operations docs, add `docs_developer` or `docs_admin` at `/users`. -```bash -./planet.sh health -``` +## Data Exploration -2. Check recent logs: +- `/datasources`: source directory. It can be filtered by product domain, layer/module, enabled state, last run status, whether collected records exist, and search text. Selecting rows triggers only those sources; with no selected rows, `Collect current filter` triggers the filtered scope. Clicking a name opens an info drawer showing endpoint, headers, base config, and built-in flag; endpoint/credentials editing happens at `/settings -> Collector Settings`. The `Collecting N` tag under the overall progress can be clicked to expand the current collection task list +- `/data`: collected data table — used to verify "did data arrive", "is the freshness right", "does a source emit valid records" +- `/bgp`: BGP detail page with list + detail + analysis; complements the BGP layer on Earth +- `/alerts/system`, `/alerts/bgp`, `/alerts/situational`: system, BGP, and situational alerts -```bash -./planet.sh log -``` +## AI Testbench -3. Check per-module logs: +`/ai?tab=playground` is for real-pipeline debugging: -```bash -./planet.sh log -f -./planet.sh log -b -./planet.sh log -a -``` +- Pick the active provider +- Run preset requests or custom prompts +- Watch AI Provider status and response -4. Restart only the affected module: +The legacy link `/playground` redirects here. -```bash -./planet.sh restart -f -./planet.sh restart -b -./planet.sh restart -a -``` +## Earth Public Page -5. If database or cache is abnormal, restart the database: +Earth at `http://localhost:3000/earth` is the public 3D situational page; no login required. The React route `/earth` wraps a standalone frontend (in `frontend/public/earth/`) via iframe. -```bash -./planet.sh restart -d -``` +### Primary Uses -6. If still unrecovered, do a full restart: +A single globe view of: BGP events and observations, satellites and tracks, cables and landing points, compute centers, country boundaries / graticules / high-res tiles / clouds / terrain, news live streams and situational news, search and focus details. -```bash -./planet.sh restart -``` +### Layer Control + +The right-side layer panel toggles layers. Common layers: graticule, country boundaries, high-res tiles, atmospheric clouds, cables, compute centers, BGP, satellites, AIS vessels, tracks, terrain. + +Dependencies: + +- Terrain depends on high-res tiles +- Tracks depend on satellites +- With high-res tiles disabled, the globe shows the base map with edge highlighting + +### Legend + +The bottom-left legend follows the focused or enabled layer. Covered: cables, satellites, country boundaries, compute centers, BGP, AIS vessels. + +AIS vessel legend colors by type: cargo, tanker, passenger, fishing, military, moored/slow, other. Triangles indicate moving vessels; dots indicate moored or slow targets. + +### Search + +Search finds cables, landing points, satellites, compute centers, BGP events, BGP observers. Results jump to and focus the object. + +### Coordinate Candidate Collection + +Compute center and BGP observer detail cards support automatic coordinate-candidate collection. Click the object then use "Collect Coordinate Candidates" or "Re-collect Coordinates". The backend assembles candidates from source coordinates, public-org registry APIs, and online geocoders. When regular sources have no candidate, the current default AI Provider runs one LLM factcheck fallback. BGP observers' stored coordinates only fill query context; they are not returned as candidates. + +Candidates preview on Earth directly. Saving a compute-center candidate writes to the `compute_center_locations` dimension table and refreshes the layer immediately. The notification badge at the top-left of the compute-center layer shows the unresolved count; clicking it opens the queue, supports single collection, or "Adopt All" to save the top-confidence candidates from top to bottom. Records without candidates stay in the queue rather than being faked to country centroids. See [Earth Location Candidate Collection User Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-user.md). + +### Settings + +The settings panel covers: rotate / cruise / motion mode, cruise modules (BGP/news/compute centers/vessels/cables/satellites), view (satellite display style, day-night mode, panel toggles), motion debug mode / input source / skeleton-only, default globe size, terrain opacity, reset. + +These settings live in browser local storage; switching browsers or clearing site data resets them. + +### View Controls + +| Action | Effect | +| --- | --- | +| Mouse drag | Rotate the globe | +| Single-finger drag | Touch rotate | +| Mouse wheel | Zoom in/out | +| Pinch | Touch zoom | +| Zoom button | Stepped zoom | +| Click zoom percentage | Reset to default zoom | + +A small pill at the top briefly shows the current zoom while zooming. This is not a data loading indicator; if data is loading, the loading state takes precedence. + +Drag sensitivity adjusts to zoom: near the default it is normal; zoomed in it is finer for targeted inspection; zoomed out it is slightly faster for global browsing. + +### Motion Capture + +Earth supports motion capture. Two live inputs: + +- **Browser Camera** (default): uses `getUserMedia` in the page. No install needed but the page must run on HTTPS or localhost, and the user must grant camera permission +- **Motion Agent**: camera/RTSP/HTTP → local agent → local WebSocket → Earth. Used for dual cameras, USB index, phone/IP camera streams + +Enable via the settings toggle "Motion Debug Mode", or with URL parameter `?motion=1`. Motion Agent defaults to `ws://127.0.0.1:8765/ws/gestures`; override with `motionAgent`. You can also pin the input with `?motion=1&motionProvider=browser` or `?motion=1&motionProvider=agent`. + +Neither mode uploads camera frames or live gestures; neither reuses the news/RSS aggregation API. + +Gesture semantics: + +| Event | Effect | +| --- | --- | +| `rotate_left/right/up/down` | Rotate accordingly | +| `zoom_in/out` | Zoom | +| `focus_prev/next` | Cycle focusable targets within the current layer | +| `layer_prev/next` | Switch layer and pan to nearest target | +| `confirm` | Confirm the current selection | + +In the debug panel: the browser camera input shows the live preview with skeleton overlay; Motion Agent sends only normalized skeleton events, never raw frames. "Skeleton Only" hides the video and keeps just the skeleton; "Stop Matching" pauses gesture firing while keeping preview and skeleton. Unmatched skeleton is red; matched turns green and shows the action name. + +### Cruise Mode + +Cruise mode auto-rotates focused targets. Current modules: BGP, news, compute centers, vessels, cables, satellites. Suitable for demos, control rooms, and unattended displays. + +### Mobile + +Mobile uses a drawer layout: layer control moves into a drawer; search/settings/details use mobile panels. Main interaction is still object tap, search, and layer toggles. + +### Common Issues + +- **Earth does not open**: confirm the frontend is online; if not on port `3000`, use the port printed by the startup log +- **Layers have no data**: open `/datasources` to check source status, collected-record state, and the latest run result; then `/data` or `/bgp` for records +- **Satellites / BGP / cables load slowly**: those layers depend on backend APIs and external data sources; the first load waits for startup tasks + +## Docs Site + +Docs at `http://localhost:3000/docs` are served by the backend with access control, not bundled into the frontend build. + +Anonymous visitors see only `public` docs: README, Quickstart, Manual, FAQ, Earth Location Candidate Collection User Guide. Authenticated users with Gatekeeper groups see more: + +- `docs_user`: end-user operational docs +- `docs_developer`: Earth, frontend, backend, collectors, AI Provider development docs +- `docs_admin`: service control, operations, environment variables, sensitive operations (including the Ops Runbook) + +`admin` has `docs_admin` by default; `super_admin` has all docs permissions. Gatekeeper groups are managed at `/users`. + +Docs supports: category navigation, Markdown rendering, tables and code blocks, in-doc table of contents, search across currently visible docs, internal links between technical docs. ## Related Docs - [Quickstart](/home/ray/dev/linkong/planet/docs/technical/en/quickstart.md) -- [Admin Frontend Context](/home/ray/dev/linkong/planet/docs/technical/en/frontend-admin-frontend-context.md) -- [Earth Frontend Context](/home/ray/dev/linkong/planet/docs/technical/en/earth-frontend-context.md) -- [Earth Layer Style Reference](/home/ray/dev/linkong/planet/docs/technical/en/earth-layer-style-reference.md) +- [FAQ](/home/ray/dev/linkong/planet/docs/technical/en/faq.md) - [Earth Location Candidate Collection User Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-user.md) -- [System Service Control](/home/ray/dev/linkong/planet/docs/technical/en/backend-system-service-control.md) -- [Backend Collectors](/home/ray/dev/linkong/planet/docs/technical/en/backend-collectors.md) +- [Planet Ops Runbook](/home/ray/dev/linkong/planet/docs/technical/en/ops-runbook.md) diff --git a/docs/technical/en/ops-runbook.md b/docs/technical/en/ops-runbook.md new file mode 100644 index 00000000..396d26fe --- /dev/null +++ b/docs/technical/en/ops-runbook.md @@ -0,0 +1,244 @@ +# Planet Ops Runbook + +This runbook is for deployment, on-call, and maintenance engineers. End-user UI flows live in the [Planet Manual](/home/ray/dev/linkong/planet/docs/technical/en/manual.md); this document only covers shell, Docker, logs, environment variables, and troubleshooting. + +## First Startup + +```bash +./planet.sh start +``` + +Default behavior: + +- Starts PostgreSQL and Redis +- Starts AI Provider +- Starts the backend API +- Starts the frontend Vite dev server +- Prints Earth, console, Playground, and backend API doc URLs + +First startup seeds two default accounts (see `DEFAULT_LOGIN_USERS` in `backend/app/db/session.py`): + +| Username | Password | Role | +| --- | --- | --- | +| `admin` | `admin123` | `super_admin` | +| `linkong` | `12345678` | `super_admin` | + +Both seed accounts are created with `email_verified = TRUE` and can log into the console immediately. Any other account must either go through the public registration flow described in the Manual, or be created via `./planet.sh createuser`. + +Specify custom ports: + +```bash +./planet.sh start -b 8001 -f 3001 -a 8101 +``` + +| Flag | Meaning | +| --- | --- | +| `-b ` | Backend port | +| `-f ` | Frontend port | +| `-a ` | AI Provider port | +| `--allow-lan` | Enable LAN access | +| `--verbose` | Show extra command output | + +## Stop and Per-Module Restart + +Stop everything: + +```bash +./planet.sh stop +``` + +Stops backend, AI Provider, frontend, PostgreSQL, Redis. + +Per-module restart: + +```bash +./planet.sh restart # full +./planet.sh restart -b # backend +./planet.sh restart -f # frontend +./planet.sh restart -a # AI Provider +./planet.sh restart -d # database +``` + +Per-module restart is preferred during development to avoid interrupting unrelated services. + +## Health Check + +```bash +./planet.sh health +``` + +Checks: + +- `planet_*` container status +- Backend `/health` +- AI Provider `/health` +- Frontend reachability + +If anything reports offline, check the corresponding logs first. + +## Logs + +Recent logs: + +```bash +./planet.sh log +``` + +Follow: + +```bash +./planet.sh log -f # frontend: /tmp/planet_frontend.log +./planet.sh log -b # backend: /tmp/planet_backend.log +./planet.sh log -a # AI Provider: planet_aiprovider container logs +``` + +## CLI User Creation + +```bash +./planet.sh createuser +``` + +Interactively prompts for username, password, and role; writes the user with `email_verified = TRUE` directly. + +Use when: + +- SMTP is not yet configured but an admin account is needed now +- Pre-seeding internal test accounts +- Public registration is unavailable for any reason and a fallback is required + +For ordinary user onboarding, configure SMTP at `/settings -> SMTP Email` first and let users self-register at `/register`. + +## LAN / WSL Access + +```bash +./planet.sh start --allow-lan +``` + +Useful for: + +- Starting in WSL, accessing from Windows browser +- Demoing Earth from a phone or tablet +- Other LAN machines reaching the same dev instance + +`--allow-lan` only makes the frontend and backend listen on `0.0.0.0`. When Planet runs in WSL, Windows can usually reach it through `localhost`, but other LAN machines hitting `http://:3000` still need Windows port forwarding and firewall rules. + +Diagnose in this order: + +```bash +# From the shell running Planet +curl http://localhost:3000 +curl http://localhost:8000/health +ss -ltnp | grep -E ':3000|:8000' +``` + +If WSL shows `0.0.0.0:3000` / `0.0.0.0:8000` but the LAN IP still fails, configure Windows from an elevated PowerShell: + +```powershell +netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=3000 connectaddress=127.0.0.1 connectport=3000 +netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=8000 connectaddress=127.0.0.1 connectport=8000 + +New-NetFirewallRule -DisplayName "WSL Planet 3000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 3000 +New-NetFirewallRule -DisplayName "WSL Planet 8000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 8000 +``` + +## AI Provider Environment and Builds + +AI Provider runtime configuration lives in two places: + +| Location | Best for | Notes | +| --- | --- | --- | +| `aiprovider/.env` | Team-shared local defaults | Read by Docker Compose as `env_file` | +| `~/.zshrc` | Personal provider/model/key/proxy | `planet.sh` reads common `AI_*`, `SERVICE_*`, `PYTHON_IMAGE`, `UV_IMAGE` lines | + +Recommended form: + +```bash +export AI_PROVIDER=minimax +export AI_PROVIDER_API=anthropic-messages +export AI_BASE_URL=https://api.example.com/anthropic +export AI_API_KEY=sk-change-me +export AI_MODEL=MiniMax-M2.7 +export AI_PROVIDER_SERVICE_TOKEN=change_me +``` + +By default `planet.sh` only statically parses simple `export KEY=value` lines from `~/.zshrc`. When complex shell expansion is required, opt in explicitly: + +```bash +PLANET_LOAD_ZSHRC_ENV=source ./planet.sh start -a +``` + +To ignore `~/.zshrc` entirely: + +```bash +PLANET_LOAD_ZSHRC_ENV=0 ./planet.sh start -a +``` + +The AI Provider image only rebuilds when code, Dockerfile, Compose config, or Python dependencies change. After changing keys or base URL, restarting the container is enough: + +```bash +./planet.sh restart -a +``` + +Diagnose slow builds: + +| Symptom | Common cause | Fix | +| --- | --- | --- | +| Large `transferring context` | build context includes unrelated frontend / data files | `.dockerignore` ships only required files | +| `uv sync` is slow | first build or cold cache | wait for the first build; later runs reuse BuildKit cache | +| Old keys still in effect after edit | container not restarted | `./planet.sh restart -a` | + +## SMTP Email (Required for Public Registration) + +Public registration and email verification depend on SMTP. Administrators configure host, port, username, password, from-address, and TLS mode at `/settings -> SMTP Email` in the console, then use the "Send Test Email" button to verify. Settings are persisted in the `system_settings.smtp` row. + +When SMTP is unset, `POST /api/v1/auth/register` returns `503 EMAIL_PROVIDER_NOT_CONFIGURED` and the frontend surfaces a clear error. The operational fallback is `./planet.sh createuser`. + +One-time codes are stored in Redis under `otp:{purpose}:{email}` with a 600-second TTL. The key is invalidated after 5 invalid attempts. Resend cooldown is 60 seconds, enforced via `otp_rate:{purpose}:{email}`. + +## Troubleshooting Order + +```bash +./planet.sh health # 1. service state +./planet.sh log # 2. recent logs +./planet.sh log -f # 3. per-module logs +./planet.sh log -b +./planet.sh log -a +./planet.sh restart -f # 4. restart only the affected module +./planet.sh restart -b +./planet.sh restart -a +./planet.sh restart -d # 5. database / cache issues +./planet.sh restart # 6. full restart if still broken +``` + +## Development Command Conventions + +Frontend must use Bun: + +```bash +cd frontend +bun install +bun run dev +bun run build +``` + +Do not use `npm run ...`. In the WSL / Windows mixed environment Bun avoids Node/npm path inconsistencies. + +Validate the frontend build: + +```bash +source ~/.zshrc && bun run build +``` + +Backend dependencies are managed with uv: + +```bash +uv sync +uv run pytest backend/tests/test_otp_service.py +``` + +## Related Docs + +- [planet.sh Startup Mechanism](/home/ray/dev/linkong/planet/docs/technical/en/ops-planet-sh-startup.md) +- [System Service Control](/home/ray/dev/linkong/planet/docs/technical/en/backend-system-service-control.md) +- [Docker + Compose + Buildx Upgrade](/home/ray/dev/linkong/planet/docs/technical/en/ops-docker-compose-buildx-upgrade.md) +- [Data Collectors](/home/ray/dev/linkong/planet/docs/technical/en/backend-collectors.md) diff --git a/docs/technical/en/quickstart.md b/docs/technical/en/quickstart.md index 347cffa7..a5baee29 100644 --- a/docs/technical/en/quickstart.md +++ b/docs/technical/en/quickstart.md @@ -1,236 +1,66 @@ # Quickstart -This guide is for developers or demo operators starting Planet for the first time. The goal is to get services running via the shortest path and know which URLs to open. +This quickstart is for Planet end users who just received an access URL and need the shortest path from "open the browser" to "first useful configuration done". Every action happens in the browser. -If you run into port conflicts, Windows / WSL LAN access, `uv` / `bun`, camera, or Docs permission issues, start with the [FAQ](/home/ray/dev/linkong/planet/docs/technical/en/faq.md). +If you are responsible for deployment or operations, read the [Planet Ops Runbook](/home/ray/dev/linkong/planet/docs/technical/en/ops-runbook.md) instead. -## Prerequisites +## 1. Open the URL -Recommended: run in a WSL / Linux shell. +Open the URL your administrator gave you, e.g. `http://planet.example.com`. A local demo is usually `http://localhost:3000`. -You need: +Entry points are split in two: -- Docker / Docker Compose available -- `uv` and `bun` accessible in the current shell -- Repository cloned locally +- Public: `/earth` (3D situational view), `/docs` (public documentation) +- Login required: `/admin` (console), `/ai` (AI), `/settings` (system configuration) -On a new machine, run the bootstrap script first: +## 2. Register -```bash -./scripts/bootstrap-dev.sh -``` +1. Open `/login` and click "Register" under the form. +2. On `/register`, fill in username, email, password (≥ 8 characters). +3. After submission, check your inbox for a 6-digit verification code (valid for 10 minutes). +4. Enter the code on the verify page and click "Verify and Sign In". You are taken to the console automatically. -This script checks and syncs common dependencies, and generates if missing: +If the email does not arrive: -- `backend/.env` -- `aiprovider/.env` -- `frontend/.env.local` +- Check spam and your enterprise mail gateway +- The "Resend Code" button has a countdown; you can resend once it ends +- A "Email service not configured" message means your administrator has not yet set up SMTP — please ping them -Personal AI Provider configuration can also live in `~/.zshrc`. `planet.sh` reads simple `export AI_...=...` / `AI_...=...` lines and passes them to the AI Provider container. After changing model, key, or base URL, restart only AI Provider: +The default role is `viewer`: you can sign in but only see public pages. For collectors, user management, or system settings, ask the admin to promote your role or add Gatekeeper groups. -```bash -./planet.sh restart -a -``` +## 3. First Sign-In Checklist -Collector credentials such as AISStream and BarentsWatch can also start in `~/.zshrc` for connectivity validation: +After landing on the `/admin` dashboard, here's a recommended walk-through: -```bash -export AISSTREAM_API_KEY="..." -export BARENTSWATCH_CLIENT_ID="..." -export BARENTSWATCH_CLIENT_SECRET="..." -``` +1. `/settings?tab=collector_credentials`: pick a collector and click the plug icon to test connectivity. Free collectors (e.g. open BGP) usually work right away; credential-bearing ones like `AISStream` or `BarentsWatch` need an API key / client secret first +2. `/ai?tab=providers`: fill an LLM provider (e.g. `minimax` / `openai`), model, base URL, API key, and click the plug at the end of the base URL to test. WebSearch / OCR tools are optional +3. `/datasources` or `/data`: check whether the collectors have produced data +4. `/alerts/system`: verify system alerts look right +5. `/users` (super_admin only): open accounts for teammates or adjust their groups -For actual collection, prefer saving credentials in `Settings -> Collector Settings`, especially for AISStream's long-lived WebSocket collector. That keeps connectivity validation, backend collection tasks, and Earth realtime vessel aggregation on the same configuration source. +## 4. Open Earth -## 1. Start Services - -From the repository root: - -```bash -./planet.sh start -``` - -After startup, the key URLs are: - -| Entry | Default URL | Purpose | -| --- | --- | --- | -| Earth | `http://localhost:3000/earth` | Public 3D Earth visualization | -| Console | `http://localhost:3000/admin` | Admin console (login required) | -| Docs | `http://localhost:3000/docs` | Usage docs are public; developer and operations docs require Gatekeeper groups | -| AI | `http://localhost:3000/ai` | Model provider, tool, and testbench entry (login required) | -| Backend API Docs | `http://localhost:8000/docs` | FastAPI / OpenAPI interface docs | - -If the default ports are taken, specify custom ports: - -```bash -./planet.sh start -f 3001 -b 8001 -a 8101 -``` - -If backend port `8000` is occupied by a Windows listener or an old portproxy rule, follow the [FAQ](/home/ray/dev/linkong/planet/docs/technical/en/faq.md) troubleshooting order. - -## 2. Create a Login User - -The console requires login. For first-time use: - -```bash -./planet.sh createuser -``` - -Follow the prompts to enter username, password, and role. - -To read developer or operations docs, log in as `super_admin` and assign Gatekeeper groups from the Users page. Use `docs_developer` for development docs and `docs_admin` for service-control and operations docs. - -## 3. Open Earth - -Visit: - -```text -http://localhost:3000/earth -``` - -Earth is a public page — no login required. +Visit `/earth`. This is a public page — no login required. Once in, verify: -- The globe renders correctly -- The right-side layer panel can toggle layers on/off -- Search can find cables, satellites, compute centers, BGP events -- Compute-center and BGP collector detail cards can collect and preview coordinate candidates; when regular sources have no candidate, the current default AI Provider runs one LLM factcheck fallback; the compute-center unresolved badge can open the queue and save candidates -- Mouse drag, wheel zoom, and zoom percent feedback work correctly -- Settings panel can switch rotate / cruise / motion mode, day/night mode, and satellite display style; Motion Debug Mode can show the local Browser Camera preview plus skeleton overlay +- The globe renders, and the right-side layer panel can toggle layers +- Search finds cables, satellites, compute centers, BGP events +- Compute-center and BGP collector detail cards can collect coordinate candidates and preview them on Earth +- Mouse drag, wheel zoom, and the zoom percentage indicator work +- The settings panel can switch rotate / cruise / motion modes -## 4. Open the Console +## 5. Recover a Lost Password -Visit: +Open `/forgot-password`, enter your email, receive a code, then enter the code plus a new password. The same confirmation is shown for unknown emails (to avoid enumeration). -```text -http://localhost:3000/admin -``` +## 6. Read the Docs -The console manages data sources, collected data, situational observation, alerts, system logs, and configuration. - -First-time inspection checklist: - -- `/datasources`: data source directory and collection triggers; endpoint, headers, and credentials are configured under `/settings` collector settings -- `/data`: collected data -- `/bgp`: BGP situational view -- `/ai`: AI page for model providers, WebSearch-style tools, and the testbench -- `/alerts/system`: system alerts -- `/settings`: system configuration - -## 5. Check Service Health - -```bash -./planet.sh health -``` - -This shows container status and checks: - -- Backend -- AI Provider -- Frontend - -## 6. View Logs - -Recent logs: - -```bash -./planet.sh log -``` - -Follow a specific service: - -```bash -./planet.sh log -f -./planet.sh log -b -./planet.sh log -a -``` - -Flags: - -- `-f`: frontend logs -- `-b`: backend logs -- `-a`: AI Provider logs - -## 7. Common Restarts - -Frontend only: - -```bash -./planet.sh restart -f -``` - -Backend only: - -```bash -./planet.sh restart -b -``` - -AI Provider only: - -```bash -./planet.sh restart -a -``` - -Database only: - -```bash -./planet.sh restart -d -``` - -Full restart: - -```bash -./planet.sh restart -``` - -## 8. LAN Access - -To allow a Windows browser, phone, or another device on the same network: - -```bash -./planet.sh start --allow-lan -``` - -This makes the frontend and backend listen on a LAN-accessible address. - -Note: `--allow-lan` only makes Planet listen on `0.0.0.0`; it does not automatically expose WSL services through the Windows LAN IP. A common pattern is: - -- `localhost:3000` / `localhost:8000` works inside WSL -- `localhost:3000` / `localhost:8000` works on Windows -- `http://:3000` fails from a phone or another computer - -That usually means Windows still needs port forwarding or firewall rules. - -If access fails, check from the shell running Planet: - -```bash -curl http://localhost:3000 -curl http://localhost:8000/health -ss -ltnp | grep -E ':3000|:8000' -``` - -If WSL is listening on `0.0.0.0:3000` and `0.0.0.0:8000` but the LAN IP still fails, configure Windows forwarding and firewall rules from an elevated PowerShell: - -```powershell -netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=3000 connectaddress=127.0.0.1 connectport=3000 -netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=8000 connectaddress=127.0.0.1 connectport=8000 - -New-NetFirewallRule -DisplayName "WSL Planet 3000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 3000 -New-NetFirewallRule -DisplayName "WSL Planet 8000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 8000 -``` - -## 9. Stop Services - -```bash -./planet.sh stop -``` - -This shuts down the frontend, backend, AI Provider, PostgreSQL, and Redis. +`/docs` is the docs site. Without login you can read: this Quickstart, the Manual, the FAQ. Authenticated users with `docs_user` / `docs_developer` / `docs_admin` groups see additional technical documents. ## Next Steps -- Full usage guide: [Planet Manual](/home/ray/dev/linkong/planet/docs/technical/en/manual.md) -- Console structure: [Admin Frontend Context](/home/ray/dev/linkong/planet/docs/technical/en/frontend-admin-frontend-context.md) -- Earth structure: [Earth Frontend Context](/home/ray/dev/linkong/planet/docs/technical/en/earth-frontend-context.md) -- Backend collectors: [Backend Collectors](/home/ray/dev/linkong/planet/docs/technical/en/backend-collectors.md) +- Full UI walkthrough: [Planet Manual](/home/ray/dev/linkong/planet/docs/technical/en/manual.md) +- Troubleshooting and configuration questions: [FAQ](/home/ray/dev/linkong/planet/docs/technical/en/faq.md) +- Detailed Earth coordinate candidate flow: [Earth Location Candidate Collection User Guide](/home/ray/dev/linkong/planet/docs/technical/en/location-pipeline-user.md) +- Deployment / operations commands: [Planet Ops Runbook](/home/ray/dev/linkong/planet/docs/technical/en/ops-runbook.md) diff --git a/docs/technical/zh/backend-collectors.md b/docs/technical/zh/backend-collectors.md index eb60fd95..6afd1b66 100644 --- a/docs/technical/zh/backend-collectors.md +++ b/docs/technical/zh/backend-collectors.md @@ -336,17 +336,50 @@ AIS 观测写入后不会直接替换最终船只记录，而是先保存为 raw - 位置、速度、航向等动态字段会按 freshness 和来源优先级选择。 - 静态字段优先保留非空值；冲突候选会记录到详情接口，便于排查多源差异。 -Earth 使用的接口仍是： +Earth 船只展示现在使用受控快照接口和实时增量通道： ```http -GET /api/v1/visualization/geo/vessels +GET /api/v1/vessels/snapshot?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 GET /api/v1/visualization/vessels/{mmsi} GET /api/v1/visualization/vessels/{mmsi}/track GET /api/v1/visualization/vessels/{mmsi}/conflicts -GET /api/v1/visualization/vessels/aggregation/diagnostics ``` -`/geo/vessels` 会合并 raw observation 聚合结果和 legacy BarentsWatch latest position 结果，避免只接入 AISStream 后把历史 BarentsWatch 船只遮蔽掉。 +`/api/v1/vessels/snapshot` 必须携带 `bbox` 和 `zoom`，默认 `limit=1000`，最大 `limit=5000`。它只消费 `ais_raw_observations` 聚合结果，不再读取 legacy `vessel_position` / `vessel_static` 作为兜底。旧 `/api/v1/visualization/geo/vessels` 已下线并返回 `410 Gone`。 + +实时增量通过 `/ws` 的 `vessels` channel 推送。客户端订阅时必须带当前视口： + +```json +{ + "type": "subscribe", + "data": { + "channel": "vessels", + "bbox": [120.8, 30.7, 122.1, 31.8], + "zoom": 12, + "limit": 1000 + } +} +``` + +后端按连接保存轻量订阅条件，只向 bbox 命中的连接发送船只更新。collector 广播会先进入 1 秒节流队列，同一 MMSI 在一个 flush 周期内只保留最新位置，避免高频实时流拖垮 WebSocket。 + +### 图层接口与全量统计分离 + +Earth 后续迁移到两类接口： + +```http +GET /api/v1/data-products +GET /api/v1/data-products/{product_id}/status +GET /api/v1/layers/vessels/snapshot?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 +GET /api/v1/layers/cables?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 +GET /api/v1/layers/landing-points?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 +GET /api/v1/layers/satellites?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 +GET /api/v1/layers/bgp/anomalies?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 +GET /api/v1/layers/bgp/incidents?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 +GET /api/v1/layers/bgp/collectors?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 +``` + +`/api/v1/data-products/*` 面向聚合面板，统计口径是全量/全局，不受地图 bbox 影响。`/api/v1/layers/*` 面向地图渲染，必须携带 `bbox` 和 `zoom`，默认 `limit=1000`，最大 `limit=5000`；低 zoom 会降级到更小的返回上限，并在 `diagnostics` 中暴露 `degraded`、`truncated`、`limit_clamped` 和 `stats_scope=viewport`。当前版本的非船只图层先复用已有 GeoJSON 转换再做保护层，后续可继续把 bbox 下推到各产品专用查询。 ## 十、采集器设置与连接验证 @@ -418,4 +451,12 @@ curl -X POST http://localhost:8000/api/v1/datasources/1/trigger \ -H "Authorization: Bearer " ``` +批量采集使用： + +```http +POST /api/v1/datasources/trigger-batch +``` + +请求体可以传 `source_ids` 精确触发选中项；如果不传 `source_ids`，后端按 `product`、`module`、`is_active`、`run_status`、`collected`、`credential_status` 和 `q` 过滤后触发。接口会跳过禁用源、正在运行且未 `force` 的源，以及未到频率窗口的源，并返回 `triggered`、`skipped`、`failed` 三组结果。 + **核心文件**: `backend/app/api/v1/datasources.py` diff --git a/docs/technical/zh/datasource-collector-settings-connectivity.md b/docs/technical/zh/datasource-collector-settings-connectivity.md index 09af46e9..708bed24 100644 --- a/docs/technical/zh/datasource-collector-settings-connectivity.md +++ b/docs/technical/zh/datasource-collector-settings-connectivity.md @@ -290,6 +290,16 @@ AISStream 使用 WebSocket 实时流，采集器只写入 `ais_raw_observations` 连接验证会读取保存配置、环境变量和 `~/.zshrc` 中的 `AISSTREAM_API_KEY`。正式采集时，推荐把 API Key 保存到采集器设置；如果只写在 `~/.zshrc`，需要确认后端进程实际继承了该变量，否则连接验证可能可用但 collector 运行时拿不到 key。 +连接验证和正式采集是两个不同动作。设置页出现 `AISStream 凭证已配置，WebSocket endpoint 格式有效` 只说明配置可以用于连接；运行状态仍可能是 `disconnected`。只有 `aisstream_vessels` collector 任务处于 `streaming` / `connected`，并且 `本轮消息数`、`last_seen_at` 持续更新时，全球 AIS 数据才会不断写入本地库。 + +新版本不再使用 legacy `/api/v1/visualization/geo/vessels` 作为船只列表入口。Earth 初始状态应调用： + +```http +GET /api/v1/vessels/snapshot?bbox=lon_min,lat_min,lon_max,lat_max&zoom=12&limit=1000 +``` + +该接口只查询本地 `ais_raw_observations` 聚合结果。实时更新走 `/ws` 的 `vessels` channel，订阅时必须提供 `bbox`、`zoom` 和 `limit`。服务端按连接过滤 bbox，并对 collector 广播做 1 秒合并，同一 MMSI 只推送最新位置。 + ## 自定义 REST / WebSocket 映射运行时 文件： diff --git a/docs/technical/zh/earth-frontend-context.md b/docs/technical/zh/earth-frontend-context.md index 0dfb9db6..3cf13d9c 100644 --- a/docs/technical/zh/earth-frontend-context.md +++ b/docs/technical/zh/earth-frontend-context.md @@ -286,8 +286,9 @@ AIS 船只图层入口： 船只图层当前负责： -- 请求 `/api/v1/visualization/geo/vessels` -- 将聚合后的 AIS GeoJSON 转为地球局部坐标 marker 数据；请求默认不传 `limit`，后端和前端都不再默认裁剪到 5000 艘 +- 请求 `/api/v1/vessels/snapshot` 获取当前视口初始快照；请求必须携带 `bbox`、`zoom`，并传入受控 `limit` +- 通过 `/ws` 的 `vessels` channel 订阅后续增量；订阅 payload 同样必须携带当前视口 `bbox`、`zoom`、`limit` +- 将聚合后的 AIS GeoJSON 转为地球局部坐标 marker 数据；后端默认 `limit=1000`，最大 `limit=5000` - 通过 `createInteractableLayer()` 注册 Interactable 图标层 - 用按航向分桶的 `THREE.Points` 批量渲染普通船只 marker - 按船型映射颜色；`vessels.js` 会用 `vessel_type_name` 和 AIS `vessel_type` 数字共同归一化船型 @@ -310,6 +311,10 @@ AIS 船只图层入口： AISStream 的 `PositionReport` 常带实时位置和 `MetaData.ShipName`，但船型通常来自低频 `ShipStaticData.Type`。后端会把 `MetaData.ShipName` 补进船名，并将类型码映射为 Cargo / Tanker / Passenger / Fishing / Military；仍缺失的船型需要等待静态 AIS 消息或后续船舶资料 enrichment，不能在前端凭颜色之外的信息臆造细分类。 +旧 `/api/v1/visualization/geo/vessels` 已下线并返回 `410 Gone`。前端打开船只图层时应先按当前视口拉一次 `/api/v1/vessels/snapshot`，再用 WebSocket 接收同一视口内的 upsert 增量；地图拖动或缩放后应重新拉取 snapshot 并重发 vessels 订阅。后端不再把 legacy `vessel_position` / `vessel_static` 合并进船只快照，前端也不应依赖旧表里的 BarentsWatch-only 兜底数据。 + +新的图层接口族是 `/api/v1/layers/*`，用于把地图渲染数据和聚合面板统计分开。地图层请求必须带 `bbox`、`zoom` 和受控 `limit`，响应会返回 `visible_count`、`returned_count` 和 `diagnostics`，其中 `degraded/truncated/limit_clamped` 用于前端提示降级。右侧聚合统计不要从图层响应累加，应读取 `/api/v1/data-products` 或 `/api/v1/data-products/{product_id}/status`，因为这些统计保持全量/全局口径，不随当前视口变化。 + 船只 hover / click 也不再对渲染对象做 `raycaster.intersectObjects()`。`main.js` 只负责传入当前 Earth、camera、pointer 和命中半径，实际命中计算由 `interactable.js` 的图标层接口完成： 1. 拖动地球或惯性旋转时跳过 hover picking。 diff --git a/docs/technical/zh/frontend-admin-frontend-context.md b/docs/technical/zh/frontend-admin-frontend-context.md index a8ed40fa..1bb61b47 100644 --- a/docs/technical/zh/frontend-admin-frontend-context.md +++ b/docs/technical/zh/frontend-admin-frontend-context.md @@ -163,7 +163,25 @@ - 文档内部链接应通过 `transformLink` 转成 `/docs/:slug` - 标题锚点由 `getHeadingId` 注入，避免渲染器自己理解路由状态 -### 6. `TableActions` +### 6. `ConnectionTestInput` + +文件： + +- [ConnectionTestInput.tsx](/home/ray/dev/linkong/planet/frontend/src/components/ConnectionTestInput/ConnectionTestInput.tsx) + +用途： + +- Endpoint、Base URL 这类“输入值 + 连接验证”的控制台表单项 +- AI Provider 和 WebSearch 的连接测试入口 +- 后续采集器配置如果把连接测试放进输入框，也应复用它 + +当前约束： + +- 输入框末端只显示一个插头/连接器图标，不再并排放“测试连接”文字按钮 +- 禁用的集成能力必须同时置灰输入框和连接测试按钮 +- 组件只负责输入框与测试入口组合，不保存业务状态；调用方仍负责表单值、loading、disabled 和连接请求 + +### 7. `TableActions` 文件： @@ -205,10 +223,13 @@ 职责： - `/ai` 独立承载 LLM Provider、AI Tool 配置和测试台，不再放在 `/settings` 的系统配置 tabs 中 -- `模型供应商` tab 管理默认 provider、模型、base URL、provider key、本地 `aiprovider` 代理和连接测试 -- `工具` tab 管理 WebSearch provider、搜索 key、base URL、超时、结果数和高级 provider 参数 +- `模型供应商` tab 管理默认 provider、模型、base URL、provider key、本地 `aiprovider` 代理和连接测试；provider 和模型输入使用可输入组合框，models.dev 目录停更时用户仍可手动填新 provider/model +- `工具` tab 先通过下拉菜单选择工具，再管理对应配置；当前包含 WebSearch 和 OCR +- WebSearch 配置包含 provider、搜索 key、base URL、超时、结果数和高级 provider 参数 +- OCR 配置包含 provider、Base URL、API Key、模型/engine、语言、超时、文件大小上限和输出格式 - `测试台` tab 嵌入原 Playground 的真实会话、预设请求和 AI Provider 状态调试 - 页面复用 Settings 的单屏 tabs、panel card 和内部滚动样式 +- AI Provider / WebSearch 的连接测试使用 `ConnectionTestInput`，连接器图标固定在 Base URL 输入框末端；WebSearch 未启用时，除开关外的配置项和测试入口都置灰 旧的 `/settings?tab=ai` 应跳转到 `/ai?tab=providers`。 旧的 `/playground` 应跳转到 `/ai?tab=playground`。 diff --git a/docs/technical/zh/manual.md b/docs/technical/zh/manual.md index fff5f31c..5cc96134 100644 --- a/docs/technical/zh/manual.md +++ b/docs/technical/zh/manual.md @@ -1,543 +1,92 @@ # Planet 使用手册 -这份手册面向日常使用、演示、开发联调和本地运维。它覆盖四个核心入口： +这份手册面向 Planet 的最终用户。从打开浏览器开始，覆盖注册账号、登录、配置数据采集器、配置 AI、使用 Earth 和控制台、阅读文档站。所有操作都在浏览器里完成。 -- `planet.sh`：本地启动、停止、重启、健康检查和日志入口 -- Earth：公开 3D 地球态势页面 -- 控制台：登录后的管理后台 -- Docs：后端 Gatekeeper 受控的文档站，基础使用文档公开，开发/运维文档按权限组开放 - -快速启动路径见 [快速开始](/home/ray/dev/linkong/planet/docs/technical/zh/quickstart.md)。常见排障见 [常见问题](/home/ray/dev/linkong/planet/docs/technical/zh/faq.md)。 +如果你是负责部署或值班的运维，请改读 [Planet 运维手册](/home/ray/dev/linkong/planet/docs/technical/zh/ops-runbook.md)，里面是 shell 命令、日志位置、SMTP 兜底创建用户等内容。 ## 入口总览 -默认启动后，常用地址如下： - | 名称 | 地址 | 是否需要登录 | 说明 | | --- | --- | --- | --- | -| Earth | `http://localhost:3000/earth` | 否 | 3D 地球、图层、BGP、卫星、海缆、新闻态势 | -| Docs | `http://localhost:3000/docs` | 部分需要 | 使用手册公开；开发、后端、运维文档按 Gatekeeper 权限组开放 | -| FAQ | `http://localhost:3000/docs/faq` | 否 | Windows / WSL、端口、依赖、动捕、凭证和权限排障 | -| 控制台 | `http://localhost:3000/admin` | 是 | 数据、配置、告警、日志和专题观测 | -| AI | `http://localhost:3000/ai` | 是 | 模型供应商、AI 工具和测试台 | -| 后端 API 文档 | `http://localhost:8000/docs` | 视接口而定 | FastAPI / OpenAPI 文档 | +| Earth | `http://<域名>/earth` | 否 | 公开 3D 地球态势页面 | +| Docs | `http://<域名>/docs` | 部分需要 | 公共文档免登录，开发/运维文档按 Gatekeeper 权限组开放 | +| 注册 / 登录 / 找回密码 | `/register`、`/login`、`/forgot-password` | 否 | 自助开通和恢复账号 | +| 控制台 | `http://<域名>/admin` | 是 | 数据、采集器、告警、AI、用户、设置 | +| AI | `http://<域名>/ai` | 是 | 模型供应商、工具和测试台 | +| 后端 API 文档 | `http://<域名>:8000/docs` | 视接口而定 | FastAPI / OpenAPI | -## planet.sh +下面所有 URL 都基于本机演示的默认地址 `http://localhost:3000`，部署到正式环境时把前缀换成你们的访问域名即可。 -`planet.sh` 是本地开发和演示的主控脚本。优先使用它管理服务，而不是手动分别启动前端、后端、数据库和 AI Provider。 +## 注册账号 -### 启动 +1. 打开 `http://localhost:3000/login`，点击表单下方"注册账户"。 +2. 在 `/register` 填写： + - **用户名**：3–50 位字符，登录时使用 + - **邮箱**：用于接收验证码，可在账户设置中修改 + - **密码**：至少 8 位 +3. 提交后会跳到验证页，已将 6 位验证码发到你的邮箱。10 分钟内有效。 +4. 输入验证码，点击"验证并登录"。验证通过后系统会自动写入登录态并跳到控制台。 -```bash -./planet.sh start -``` +如果 60 秒内没收到邮件： -默认行为： +- 检查垃圾邮件、订阅推广、企业邮件网关 +- 验证页右下角的"重新发送验证码"会显示 60 秒倒计时，倒计时结束后可重发 +- 连续输错 5 次后该验证码会失效，需要重发新码 -- 启动 PostgreSQL 和 Redis -- 启动 AI Provider -- 启动后端 API -- 启动前端 Vite dev server -- 输出 Earth、控制台、Playground 和后端 API 文档入口 +如果系统提示"邮件服务尚未配置"，说明管理员还没填 SMTP，请联系管理员开通 SMTP 或在控制台 `/settings -> SMTP 邮件` 完成配置。 -可指定端口： +默认注册角色为 `viewer`，可以登录控制台浏览公共内容。要看采集器、用户管理、系统设置等管理类页面，需要 `admin` 或 `super_admin` 在 `/users` 给你升角色。 -```bash -./planet.sh start -b 8001 -f 3001 -a 8101 -``` +## 登录与找回密码 -参数含义： +### 登录 -| 参数 | 含义 | -| --- | --- | -| `-b ` | 后端端口 | -| `-f ` | 前端端口 | -| `-a ` | AI Provider 端口 | -| `--allow-lan` | 允许局域网访问 | -| `--verbose` | 在执行过程中显示更多命令输出 | +打开 `/login`，输入用户名和密码即可。登录成功后跳到 `/admin`。 -### AI Provider 环境变量和构建 +如果提示"邮箱未验证"，页面会自动跳到 `/verify-email`，按提示输入验证码完成验证。 -AI Provider 的运行期配置可以放在两处： +### 忘记密码 -| 位置 | 适合内容 | 说明 | -| --- | --- | --- | -| `aiprovider/.env` | 团队约定的本地默认配置 | Docker Compose 会作为 `env_file` 读取 | -| `~/.zshrc` | 个人机器上的 provider、模型、密钥和代理变量 | `planet.sh` 启动时会读取常见的 `AI_*`、`SERVICE_*`、`PYTHON_IMAGE`、`UV_IMAGE`、代理变量 | +1. 在 `/login` 点击"忘记密码？"，或直接打开 `/forgot-password`。 +2. 输入注册邮箱，点击"发送验证码"。无论邮箱是否注册，页面都会显示同一句提示（避免账号枚举）。 +3. 收到验证码后，在下一步填入验证码 + 新密码（至少 8 位），点击"重置密码"。 +4. 系统会跳回 `/login`，用新密码登录即可。 -推荐写法： +## 账户设置 -```bash -export AI_PROVIDER=minimax -export AI_PROVIDER_API=anthropic-messages -export AI_BASE_URL=https://api.example.com/anthropic -export AI_API_KEY=sk-change-me -export AI_MODEL=MiniMax-M2.7 -export AI_PROVIDER_SERVICE_TOKEN=change_me -``` +控制台右上角点击你的用户名进入账户设置，可以： -默认情况下，`planet.sh` 只静态解析 `~/.zshrc` 中简单的 `export KEY=value` 或 `KEY=value` 行，避免 shell 主题、插件或交互初始化拖慢启动。如果变量依赖复杂 shell 展开，可以显式启用 source 模式： +- 修改密码：输入当前密码 + 新密码 +- 修改邮箱：输入新邮箱后系统会发验证码到新地址，验证通过后才生效 +- 查看权限组：列出你目前拥有的 Gatekeeper 权限组（`docs_user` / `docs_developer` / `docs_admin`） +- 登出：清除当前会话 -```bash -PLANET_LOAD_ZSHRC_ENV=source ./planet.sh start -a -``` +## 控制台总览 -如需完全忽略 `~/.zshrc`： - -```bash -PLANET_LOAD_ZSHRC_ENV=0 ./planet.sh start -a -``` - -AI Provider 镜像只在代码、Dockerfile、Compose 配置或相关 Python 依赖变化时重建。修改 `aiprovider/.env` 或 `~/.zshrc` 中的模型、密钥、Base URL 不会触发镜像重建；重启 AI Provider 即可让容器读取新配置： - -```bash -./planet.sh restart -a -``` - -构建较慢时，优先判断当前卡在哪一层： - -| 现象 | 常见原因 | 处理方式 | -| --- | --- | --- | -| `transferring context` 很大 | Docker build context 包含前端资源、PDF、数据目录等无关文件 | 当前仓库通过 `.dockerignore` 只发送 AI Provider 必需文件 | -| `uv sync` 下载依赖较慢 | 首次构建或缓存为空，网络访问 Python 包较慢 | 等待首次构建完成；后续会复用 BuildKit 的 uv 下载缓存 | -| 改密钥后仍显示旧配置 | 容器尚未重启 | 执行 `./planet.sh restart -a` | - -### 停止 - -```bash -./planet.sh stop -``` - -会停止： - -- 后端 -- AI Provider -- 前端 -- PostgreSQL -- Redis - -### 重启 - -全量重启： - -```bash -./planet.sh restart -``` - -按模块重启： - -```bash -./planet.sh restart -b -./planet.sh restart -f -./planet.sh restart -a -./planet.sh restart -d -``` - -| 参数 | 作用 | -| --- | --- | -| `-b` | 只重启后端 | -| `-f` | 只重启前端 | -| `-a` | 只重启 AI Provider | -| `-d` | 只重启数据库 | - -按模块重启适合日常开发，能避免无关服务被打断。 - -### 创建用户 - -```bash -./planet.sh createuser -``` - -用于首次进入控制台前创建登录账号。脚本会交互式提示用户名、密码和角色。 - -### 健康检查 - -```bash -./planet.sh health -``` - -会检查： - -- `planet_*` 容器状态 -- 后端 `/health` -- AI Provider `/health` -- 前端页面可达性 - -如果某项显示 offline，优先查看对应日志。 - -### 日志 - -最近日志： - -```bash -./planet.sh log -``` - -持续跟随日志： - -```bash -./planet.sh log -f -./planet.sh log -b -./planet.sh log -a -``` - -| 参数 | 日志来源 | -| --- | --- | -| `-f` / `--frontend` | `/tmp/planet_frontend.log` | -| `-b` / `--backend` | `/tmp/planet_backend.log` | -| `-a` / `--ai-provider` | `planet_aiprovider` 容器日志 | - -### 局域网访问 - -```bash -./planet.sh start --allow-lan -``` - -适合： - -- WSL 中启动，Windows 浏览器访问 -- 手机或平板演示 Earth -- 局域网其他机器访问同一个开发实例 - -`--allow-lan` 只负责让前端和后端监听 `0.0.0.0`。如果服务运行在 WSL 中，Windows 本机通常可以通过 `localhost` 访问，但手机或其他电脑访问 `http://:3000` 还依赖 Windows 端口转发和防火墙放行。 - -推荐按顺序判断： - -```bash -# 在 WSL 或运行 Planet 的 shell 中 -curl http://localhost:3000 -curl http://localhost:8000/health -ss -ltnp | grep -E ':3000|:8000' -``` - -如果这里能看到 `0.0.0.0:3000` 和 `0.0.0.0:8000`，但局域网 IP 访问失败，请在管理员 PowerShell 中配置： - -```powershell -netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=3000 connectaddress=127.0.0.1 connectport=3000 -netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=8000 connectaddress=127.0.0.1 connectport=8000 - -New-NetFirewallRule -DisplayName "WSL Planet 3000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 3000 -New-NetFirewallRule -DisplayName "WSL Planet 8000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 8000 -``` - -## Earth - -Earth 是公开的 3D 态势页面，入口： - -```text -http://localhost:3000/earth -``` - -它是独立前端，实际页面位于： - -- `frontend/public/earth/index.html` -- `frontend/public/earth/js/` -- `frontend/public/earth/css/` - -React 路由中的 `/earth` 只是用 iframe 承载它。 - -### 主要用途 - -Earth 用于在一个地球视图中观察： - -- BGP 事件、异常和观测态势 -- 卫星和轨迹 -- 海缆与登陆点 -- 算力中心 -- 国界线、经纬线、高清材质、云图、地形 -- 新闻直播和态势新闻 -- 搜索和聚焦对象详情 - -### 图层控制 - -右侧图层面板用于打开或关闭可视图层。 - -常见图层包括： - -- 经纬线 -- 国界线 -- 高清材质 -- 大气云图 -- 海缆 -- 算力中心 -- BGP 观测 -- 卫星 -- AIS 船只 -- 轨迹 -- 地形 - -部分图层存在依赖关系： - -- 地形依赖高清材质 -- 轨迹依赖卫星 -- 高清材质关闭时，地球会显示基座地图和边缘识别效果 - -### 图例 - -左下角图例会跟随当前聚焦或启用的图层切换。 - -当前已覆盖： - -- 海缆 -- 卫星 -- 国界线 -- 算力中心 -- BGP -- AIS 船只 - -AIS 船只图例按船型显示颜色： - -- 货轮 -- 油轮 -- 客船 -- 渔船 -- 军舰 -- 停泊/低速 -- 其他船只 - -船只图例中的三角形对应地图上的航行船只标记，圆点对应停泊或低速状态。 - -### 搜索 - -Earth 搜索支持查找当前地球对象，例如： - -- 海缆 -- 登陆点 -- 卫星 -- 算力中心 -- BGP 事件 -- BGP 观测站 - -搜索结果可以用于快速定位对象，并打开对应详情。 - -### 位置候选采集 - -算力中心和 BGP 观测站详情卡支持自动采集坐标候选。点击对象后，使用详情卡中的 `自动采集坐标候选` 或 `重新自动采集坐标` 按钮，后端会从源坐标、开放组织注册 API 和在线地理编码中整理候选位置；这些常规来源没有候选时，会使用当前默认 AI Provider 做一次 LLM factcheck 兜底。BGP 观测站的已存储位置只用于补齐查询上下文，不会作为候选直接返回。 - -候选可以直接在 Earth 上预览。算力中心候选点击 `保存` 后会写入 `compute_center_locations` 维表，并立即刷新图层。算力中心图层左上角的通知气泡显示无法渲染的待定位数量；点击后可查看列表，单条采集候选，或用 `一键采用` 从上到下保存最高置信候选。没有可用候选的记录会留在列表中，不会被国家中心点或硬编码 hint 伪造位置。详细流程见 [Earth 位置候选采集使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-user.md)。 - -### 设置 - -设置面板包含： - -- 旋转模式 / 巡航模式 / 动捕模式 -- 巡航模块：BGP、新闻、算力中心、船只、海缆、卫星 -- 视图设置：卫星显示风格、日夜模式、面板显示开关 -- 动捕调试模式、动捕输入源、只显示骨骼 -- 地球默认大小 -- 地形透明度 -- 重置设置 - -这些设置会保存在浏览器本地存储中。换浏览器或清理站点数据后会恢复默认值。 - -### 视角控制 - -Earth 支持鼠标、触控板和触屏操作。 - -常用控制方式： - -| 操作 | 作用 | -| --- | --- | -| 鼠标左键拖动 | 旋转地球 | -| 手指单指拖动 | 在触屏设备上旋转地球 | -| 鼠标滚轮 | 放大或缩小视角 | -| 双指捏合 | 在触屏设备上放大或缩小视角 | -| 缩放按钮 | 按固定步长调整缩放 | -| 点击缩放百分比 | 重置到默认缩放 | - -缩放时，顶部胶囊会短暂显示当前缩放比例，例如 `缩放 180%`。这个提示只表示当前视角缩放，不代表数据加载进度；如果页面正在加载数据，加载提示优先显示，缩放提示不会打断加载状态。 - -拖动灵敏度会根据当前缩放自动调整。默认视角附近保持常规旋转速度；放大后拖动会逐步变细，适合检查某个区域、船只、卫星或 BGP 事件；缩小后拖动会略快，方便快速浏览全球态势。 - -### 动作捕捉控制 - -Earth 预留了动作捕捉控制入口，面向大屏和未来 3D 展示。实时链路有两种输入源：默认的 `浏览器摄像头` 会直接用网页 `getUserMedia` 在本机浏览器识别；高级的 `Motion Agent` 会走 `摄像头/RTSP/HTTP -> 本地 Agent -> 本地 WebSocket -> Earth 页面`。两种模式都不会把摄像头帧或实时手势发到云端，也不会复用新闻/RSS 聚合接口。 - -默认不自动启用。打开设置里的 `动捕调试模式`，或用 `?motion=1` 打开 Earth 动捕连接后，系统会启动当前选择的输入源。输入源默认是 `浏览器摄像头`，无需安装应用，但页面必须运行在 HTTPS 或 localhost，且用户需要允许浏览器摄像头权限。需要双摄、USB index、手机/网络摄像头流或客户端/边缘设备时，可在设置中切到 `Motion Agent`。默认 Agent 地址是 `ws://127.0.0.1:8765/ws/gestures`，也可用 URL 参数 `motionAgent` 覆盖。 - -URL 参数也可以直接指定输入源：`?motion=1&motionProvider=browser` 使用浏览器摄像头；`?motion=1&motionProvider=agent` 使用 Motion Agent；传入 `motionAgent=ws://...` 时会自动选择 Motion Agent。 - -当前手势语义： - -| 手势事件 | 作用 | -| --- | --- | -| `rotate_left` | 地球向左旋转 | -| `rotate_right` | 地球向右旋转 | -| `rotate_up` | 地球向上旋转 | -| `rotate_down` | 地球向下旋转 | -| `zoom_in` | 放大视角 | -| `zoom_out` | 缩小视角 | -| `focus_prev` / `focus_next` | 在当前动捕图层内切换可交互目标 | -| `layer_prev` / `layer_next` | 切换动捕候选图层，并巡航到新图层最近目标 | -| `confirm` | 确认当前已选目标；当前浏览器识别暂未启用双手上举确认 | - -设置面板中的 `动捕调试模式` 会打开调试面板。浏览器摄像头输入源会在面板内显示本机实时预览，并在其上绘制关节点和连线；`Motion Agent` 输入源只发送归一化骨架事件，不发送原始视频帧。面板里的 `只显示骨骼` 会隐藏视频预览、只保留深色背景和骨架；`停止匹配动作` 会暂停手势触发，但摄像头预览和骨架绘制仍可继续用于调试。未匹配动作时骨架为红色，匹配后变绿并显示当前动作名称。该入口和 `动捕输入源` 控件都已预留 Gatekeeper 权限标记，后续可接入鉴权控制。 - -### 巡航模式 - -巡航模式会让 Earth 自动轮播聚焦目标。 - -当前巡航模块包括： - -- BGP -- 新闻 -- 算力中心 -- 船只 -- 海缆 -- 卫星 - -适合演示、监控大屏或无人值守展示。 - -### 移动端 - -Earth 有移动端抽屉布局。小屏下： - -- 图层控制进入移动抽屉 -- 搜索、设置、详情会使用移动端面板 -- 主要交互仍围绕地球对象点击、搜索和图层开关 - -### 常见问题 - -#### Earth 打不开 - -先检查前端是否在线： - -```bash -./planet.sh health -./planet.sh log -f -``` - -如果前端端口不是 `3000`，使用启动时输出的实际端口。 - -#### 图层没有数据 - -检查后端和数据源： - -```bash -./planet.sh health -./planet.sh log -b -``` - -然后进入控制台查看： - -- `/datasources` -- `/data` -- `/bgp` - -#### 卫星、BGP 或海缆加载慢 - -这些图层可能依赖后端接口、外部数据源或首次加载任务。先等待启动任务完成，再查看日志和控制台数据源状态。 - -## 控制台 - -控制台入口： - -```text -http://localhost:3000/admin -``` - -控制台需要登录。首次使用先创建用户： - -```bash -./planet.sh createuser -``` - -### 页面结构 - -控制台使用 React + Ant Design，左侧菜单按工作域组织。 - -常见入口： +控制台 `http://localhost:3000/admin` 使用 React + Ant Design，左侧菜单按工作域组织。 | 页面 | 路由 | 用途 | | --- | --- | --- | | 仪表盘 | `/admin` | 系统概览 | -| Earth | `/earth` | 打开公开 Earth 页面 | -| 数据源 | `/datasources` | 查看数据源和触发采集 | -| 采集数据 | `/data` | 查看采集后的数据 | -| BGP 观测 | `/bgp` | 查看 BGP 专题数据 | +| Earth | `/earth` | 跳到公开 Earth 页面 | +| 数据源 | `/datasources` | 数据源目录、触发采集 | +| 采集数据 | `/data` | 已落库的数据 | +| BGP 观测 | `/bgp` | BGP 专题观测 | | 系统告警 | `/alerts/system` | 系统级告警 | | BGP 告警 | `/alerts/bgp` | BGP 相关告警 | | 态势告警 | `/alerts/situational` | 态势研判告警 | -| AI | `/ai` | 模型供应商、WebSearch 等工具和测试台 | -| 系统日志 | `/logs` | 查看系统日志，通常仅 super admin 可见 | -| 用户管理 | `/users` | 管理用户 | -| 系统配置 | `/settings` | 系统配置和电视直播源等设置 | +| AI | `/ai` | 模型供应商、工具、测试台 | +| 系统日志 | `/logs` | 通常仅 super admin 可见 | +| 用户管理 | `/users` | 创建/删除/改角色/调权限组 | +| 系统配置 | `/settings` | 系统、SMTP、TV、采集器设置 | -### 数据源 +权限不足时菜单项会自动隐藏。如果发现某个菜单看不到，先确认自己的角色和 Gatekeeper 权限组。 -`/datasources` 用于查看采集来源和触发采集。当前页面是“数据源目录”，会把内置数据源和自定义数据源放在同一张列表里展示。 +## 配置数据采集器 -常见操作： +`/settings?tab=collector_credentials` 是"采集器设置"页。这里统一维护所有采集器的连接配置，不仅是凭证。 -- 查看数据源状态 -- 触发采集 -- 查看最近采集任务 -- 打开详情抽屉查看 endpoint、请求头、基础配置和是否为内置数据源 - -如果 Earth 上某类对象缺失，通常先到这里确认数据源是否可用。 - -数据源列表中的名称点击后只打开信息抽屉，不再承担编辑入口。接口地址、凭证、请求头和自定义数据源配置统一到 `/settings` 的“采集器设置”里维护。 - -当有采集任务正在运行时，总体进度下方会出现 `采集中 N` 标签。这个标签和其他状态标签放在同一排，但带有可点击样式；点击后会弹出当前采集中任务列表，显示每个任务的阶段、进度和处理数量。 - -### 采集数据 - -`/data` 用于查看采集后的数据表。 - -适合排查： - -- 数据是否已经进入系统 -- 数据更新时间是否符合预期 -- 某个数据源是否产出了有效记录 - -### BGP 观测 - -`/bgp` 是 BGP 专题页面。 - -它和 Earth 的 BGP 图层互补： - -- Earth 强调空间态势和可视聚焦 -- 控制台 BGP 页面强调列表、状态、详情和研判 - -### 告警 - -告警入口包括： - -- `/alerts/system` -- `/alerts/bgp` -- `/alerts/situational` - -用于查看系统、网络和态势相关告警。 - -### 系统配置 - -`/settings` 用于管理系统级配置。 - -当前常见用途包括： - -- 系统设置 -- 电视直播源配置 -- 采集器设置 - -### AI - -`/ai` 用于管理 AI 运行链路，已经从系统配置中独立出来。旧链接 `/playground` 会跳转到 `/ai?tab=playground`。 - -当前包含： - -- `模型供应商`：默认 LLM provider、模型、Base URL、API Key、本地 `aiprovider` 代理和连接测试 -- `工具`：WebSearch provider、搜索 API Key、Base URL、最大结果数、超时和高级 provider 参数 -- `测试台`：AI Provider 状态、预设请求和真实分析链路调试 - -旧链接 `/settings?tab=ai` 会跳转到 `/ai?tab=providers`。 - -具体可用配置取决于当前登录用户权限。 - -#### 采集器设置 - -`/settings?tab=collector_credentials` 当前显示为“采集器设置”。这里统一维护所有采集器的连接配置，而不是只维护凭证。 - -使用方式： +操作步骤： 1. 在下拉框选择采集器。 2. 查看状态标签： @@ -545,167 +94,242 @@ http://localhost:3000/admin - 所属模块 - `启用` / `禁用` - `未检查` / `可用` / `不可用` -3. 点击下拉框右侧的插头图标执行健康检查。 -4. 如果检查通过，状态会变为 `可用`。 -5. 修改 endpoint、请求头、超时或重试次数后保存。 +3. 点击下拉框右侧的插头图标执行健康检查。检查通过状态变为 `可用`。 +4. 修改 endpoint、请求头、超时、重试次数等，然后保存。 -对于免费且不需要凭证的采集器，连接检查会直接请求对应 endpoint。对于需要凭证的采集器，连接检查会走对应凭证链路；如果凭证或 endpoint 相比上次验证成功时发生变化，需要重新点击连接。 +对于免费且不需要凭证的采集器，连接检查直接打 endpoint；对于需要凭证的采集器走对应凭证链路。如果凭证或 endpoint 相比上次验证成功时发生变化，需要重新点击连接。 -系统判断“已连接”的条件是： +系统判断"已连接"的条件： -- 当前配置已经成功采集过数据；或 -- 当前配置已经点击过连接按钮并验证成功。 +- 当前配置已经成功采集过数据，或 +- 当前配置已经点击过连接按钮并验证成功 -#### BarentsWatch AIS 凭证 +### BarentsWatch AIS 凭证 -`BarentsWatch AIS` 是需要凭证的内置采集器。选择该采集器后，凭证区域会显示在基础配置上方。 - -配置项： +`BarentsWatch AIS` 是需要凭证的内置采集器。选择该采集器后，凭证区域显示在基础配置上方： - `Client ID` - `Client Secret` - `Endpoint` -如果已经配置过 secret，输入框会显示脱敏预览。保存时如果保持这个脱敏预览不变，系统会保留原 secret；只有输入新的 secret 才会替换。 - -BarentsWatch AIS 支持从以下位置读取凭证： - -1. 控制台采集器设置中保存的凭证。 -2. 后端环境变量： - - `BARENTSWATCH_CLIENT_ID` - - `BARENTSWATCH_CLIENT_SECRET` - - 兼容历史拼写：`BARRENTSWATCH_CLIENT_ID`、`BARRENTSWATCH_CLIENT_SECRET` -3. `~/.zshrc` 中的同名 `export`。 +如果已配置过 secret，输入框会显示脱敏预览。保存时保持脱敏预览不变就会保留原 secret；只有输入新的 secret 才会替换。 如果连接失败，页面会弹出凭证获取教程。教程支持： -- 查看默认教程。 -- 点击“教程不好用”让 AI Provider 根据默认 prompt 重新生成教程。 -- 点击“重置”恢复默认教程。 +- 查看默认教程 +- 点击"教程不好用"让 AI Provider 用默认 prompt 重新生成 +- 点击"重置"恢复默认教程 -默认教程以 BarentsWatch 官方 tutorial 为准，并提醒 Live AIS 应选择 `AIS - API`，不是普通 `BarentsWatch - API`。 +默认教程以 BarentsWatch 官方 tutorial 为准，并提醒 Live AIS 应选择 `AIS - API`。 -### 系统日志 +### AISStream 实时船舶 -`/logs` 用于查看系统日志。若菜单中不可见，通常是当前用户角色没有权限。 +`AISStream 实时船舶` 是全球 AIS WebSocket 采集器。连接测试通过只说明 API Key 和 endpoint 格式可用；真正的全球船只数据来自后台 `aisstream_vessels` collector 长连接运行并写入 `ais_raw_observations`。 -排查问题时常用组合： +操作步骤： -```bash -./planet.sh health -./planet.sh log -``` +1. `/settings?tab=collector_credentials` 选择 `AISStream 实时船舶 : aisstream_vessels` +2. 在 `AISStream 凭证` 填入 API Key +3. Endpoint 保持默认 `wss://stream.aisstream.io/v0/stream` +4. 点击插头图标进行连接测试，确认显示 `可用` +5. 保存采集器设置 +6. 到采集调度入口运行 `aisstream_vessels` collector +7. 在 `AISStream 运行状态` 中观察： + - `streaming` / `connected` 表示正在接收实时流 + - `本轮消息数` 应持续增长 + - 如果显示 `disconnected` 且错误为 `ConnectionResetError`，需要重新触发或等待重连 -再进入 `/logs` 查看更结构化的运行信息。 +## 配置 AI 凭证 -## Docs +`/ai?tab=providers` 是 AI 模型管理入口。包含两个核心子 tab： -文档站入口： +- `模型供应商`：默认 LLM provider、模型、Base URL、API Key、本地 `aiprovider` 代理和连接测试 +- `工具`：通过下拉菜单选择具体工具，当前支持 WebSearch 和 OCR -```text -http://localhost:3000/docs -``` +### 模型供应商 -Docs 正文由后端 API 按权限读取，不再把全部 Markdown 直接打进前端构建产物。当前文档源文件仍位于： +provider 和模型既可选预设也可直接输入自定义 id/name。常用字段： -```text -docs/technical/zh/*.md -docs/technical/en/*.md -``` +- Provider：例如 `minimax`、`openai`、`anthropic`、`ollama` +- 协议适配：`OpenAI Chat Completions` / `Anthropic Messages` / `Ollama Generate` +- Base URL：模型 API 地址 +- 默认模型：例如 `gpt-5.1`、`MiniMax-M2.7` +- API Key：填入后保存；保存的 Key 在 UI 中显示为脱敏预览 +- Max Tokens、Anthropic Version：可保持默认 +- Timeout / Retry：超时和重试次数 -未登录访客默认只能看到 `public` 文档，例如首页、快速开始和使用手册。登录用户如果被分配 Gatekeeper 权限组，可以看到更多技术文档： +Base URL 输入框尾端的插头图标会触发连接测试。测试通过会显示当前模型返回的简短回复。 -- `docs_user`：用户操作类文档。 -- `docs_developer`：Earth、前端、后端、采集器和 AI Provider 等开发文档。 -- `docs_admin`：服务控制、运维、环境变量和敏感操作文档。 +### 工具 -`admin` 默认拥有管理文档权限，`super_admin` 拥有全部 Docs 权限。Gatekeeper 权限组在控制台“用户管理”中配置。 +- **WebSearch**：provider、API Key、Base URL、最大结果数、超时、高级 provider 参数。未启用时除"启用"开关外其它配置项和连接测试都会置灰 +- **OCR**：provider、Base URL、API Key、模型/engine、识别语言、超时、最大文件大小、输出格式 -Docs 支持： +旧链接 `/settings?tab=ai` 会跳到 `/ai?tab=providers`。 -- 分类导航 -- Markdown 渲染 -- 表格和代码块 -- 文档内目录 -- 对当前可见文档搜索 -- technical 文档之间的内部链接跳转 +## 系统设置 -如果新增 technical 文档，应同步检查： +`/settings` 用于管理系统级配置，常用子 tab： -- 是否有清晰的一级标题 -- 是否需要加入后端 Docs metadata 的人工分类和排序 -- 应归入 `public`、`docs_user`、`docs_developer` 还是 `docs_admin` +- **系统显示**：系统名称、刷新间隔、数据保留天数、最大并发任务 +- **通知策略**：告警邮件开关、收件邮箱、严重/警告/每日摘要通知 +- **安全策略**：会话超时、最大登录尝试、密码策略 +- **SMTP 邮件**：注册和找回密码所需的发件配置（仅 `admin` / `super_admin` 可见） +- **电视直播**：电视直播源管理 +- **AI / WebSearch / OCR**：见上节 -## 开发命令约定 +### SMTP 邮件设置 -前端命令必须使用 Bun： +公开注册和验证码功能依赖这一项。`admin` 或 `super_admin` 用户在 `/settings` 进入 `SMTP 邮件` 子 tab，填： -```bash -cd frontend -bun install -bun run dev -bun run build -``` +- SMTP 主机、端口 +- 账号、密码 +- 发件地址（必填）、发件人名称 +- STARTTLS（端口 587 常用） 或 隐式 TLS（端口 465） +- 超时秒数 -不要使用 `npm run ...`。项目在 WSL / Windows 混合环境中优先依赖 Bun，避免 Node/npm 路径差异带来的兼容问题。 +填完保存，再点"发送测试邮件"按钮，输入收件人地址试发一封。测试通过后即可让普通用户走 `/register` 自助注册。 -验证前端构建： +如果保留密码字段中的脱敏预览不动，保存时不会覆盖原密码；要换密码就输入新值。 -```bash -source ~/.zshrc && bun run build -``` +## 用户管理（管理员） -## 故障排查顺序 +`/users` 仅 `super_admin` 可创建/删除用户。该页支持： -遇到问题时，建议按这个顺序排查： +- 查看用户列表（用户名、邮箱、角色、是否激活、邮箱是否已验证） +- 创建用户：与公开注册等价，但跳过邮箱验证（管理员认账） +- 修改角色：`viewer` / `operator` / `admin` / `super_admin` +- 调整 Gatekeeper 权限组：`docs_user` / `docs_developer` / `docs_admin`，影响 Docs 站可见文档范围 +- 禁用 / 启用账号 -1. 看服务状态： +要让普通用户能看开发或运维文档，进 `/users` 给他加 `docs_developer` 或 `docs_admin`。 -```bash -./planet.sh health -``` +## 数据探索 -2. 看最近日志： +- `/datasources`：数据源目录。可以按产品域、层级、启用状态、最近执行状态、是否已有采集数据和关键词筛选；勾选多行后可批量采集选中项，未勾选时“采集当前筛选”只触发当前筛选范围。点击名称打开信息抽屉查看 endpoint、请求头、基础配置和是否内置；接口、凭证、请求头的编辑统一在 `/settings` 的"采集器设置"。总体进度下方的 `采集中 N` 标签可点击，展开当前采集任务列表 +- `/data`：采集后数据表，适合排查"数据是否已经进入系统"、"更新时间是否符合预期"、"某个数据源是否产出有效记录" +- `/bgp`：BGP 专题页面，列表 + 详情 + 研判，与 Earth 的 BGP 图层互补 +- `/alerts/system`、`/alerts/bgp`、`/alerts/situational`：系统、BGP、态势告警 -```bash -./planet.sh log -``` +## AI 测试台 -3. 按模块查看日志： +`/ai?tab=playground` 用于真实分析链路调试。可以： -```bash -./planet.sh log -f -./planet.sh log -b -./planet.sh log -a -``` +- 选择当前 provider +- 用预设请求或自定义 prompt 触发分析 +- 观察 AI Provider 状态和返回内容 -4. 只重启有问题的模块： +旧链接 `/playground` 会跳到这里。 -```bash -./planet.sh restart -f -./planet.sh restart -b -./planet.sh restart -a -``` +## Earth 公开页面 -5. 如果数据库或缓存异常，再重启数据库： +Earth `http://localhost:3000/earth` 是公开 3D 态势页面，不需要登录。React 路由中的 `/earth` 用 iframe 承载独立前端（位于 `frontend/public/earth/`）。 -```bash -./planet.sh restart -d -``` +### 主要用途 -6. 仍无法恢复时，执行全量重启： +在一个地球视图中观察：BGP 事件与观测态势、卫星和轨迹、海缆与登陆点、算力中心、国界线/经纬线/高清材质/云图/地形、新闻直播和态势新闻、搜索和聚焦对象详情。 -```bash -./planet.sh restart -``` +### 图层控制 + +右侧图层面板用于打开或关闭图层。常见图层：经纬线、国界线、高清材质、大气云图、海缆、算力中心、BGP 观测、卫星、AIS 船只、轨迹、地形。 + +依赖关系： + +- 地形依赖高清材质 +- 轨迹依赖卫星 +- 高清材质关闭时，地球显示基座地图和边缘识别效果 + +### 图例 + +左下角图例会跟随当前聚焦或启用的图层切换。已覆盖：海缆、卫星、国界线、算力中心、BGP、AIS 船只。 + +AIS 船只图例按船型显示颜色：货轮、油轮、客船、渔船、军舰、停泊/低速、其他船只。三角形对应航行船只标记，圆点对应停泊或低速状态。 + +### 搜索 + +支持查找海缆、登陆点、卫星、算力中心、BGP 事件、BGP 观测站。结果可快速定位并打开详情。 + +### 位置候选采集 + +算力中心和 BGP 观测站详情卡支持自动采集坐标候选。点击对象后用"自动采集坐标候选"或"重新自动采集坐标"按钮，后端会从源坐标、开放组织注册 API 和在线地理编码中整理候选；常规来源没有候选时使用当前默认 AI Provider 做 LLM factcheck 兜底。BGP 观测站的已存储位置只用于补齐查询上下文，不会作为候选直接返回。 + +候选可以直接在 Earth 预览。算力中心候选点击"保存"后写入 `compute_center_locations` 维表并刷新图层。算力中心图层左上角的通知气泡显示无法渲染的待定位数量；点击查看列表，单条采集候选，或用"一键采用"从上到下保存最高置信候选。没有可用候选的记录会留在列表中，不会被国家中心点或硬编码 hint 伪造位置。详细流程见 [Earth 位置候选采集使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-user.md)。 + +### 设置 + +设置面板包含：旋转模式 / 巡航模式 / 动捕模式、巡航模块（BGP/新闻/算力中心/船只/海缆/卫星）、视图设置（卫星显示风格、日夜模式、面板显示开关）、动捕调试模式 / 输入源 / 只显示骨骼、地球默认大小、地形透明度、重置设置。 + +这些设置保存在浏览器本地存储，换浏览器或清理站点数据后会恢复默认值。 + +### 视角控制 + +| 操作 | 作用 | +| --- | --- | +| 鼠标左键拖动 | 旋转地球 | +| 手指单指拖动 | 触屏旋转地球 | +| 鼠标滚轮 | 放大或缩小 | +| 双指捏合 | 触屏放大或缩小 | +| 缩放按钮 | 固定步长调整缩放 | +| 点击缩放百分比 | 重置到默认缩放 | + +缩放时顶部胶囊会短暂显示当前缩放比例。这个提示不代表数据加载进度；如果页面正在加载数据，加载提示优先显示。 + +拖动灵敏度会根据当前缩放自动调整：默认视角附近保持常规旋转速度；放大后拖动会逐步变细，缩小后拖动会略快。 + +### 动作捕捉控制 + +Earth 预留了动作捕捉控制入口。实时链路两种输入源： + +- **浏览器摄像头**（默认）：直接用网页 `getUserMedia` 在本机浏览器识别；无需安装应用，但页面必须运行在 HTTPS 或 localhost，且需允许浏览器摄像头权限 +- **Motion Agent**：摄像头/RTSP/HTTP → 本地 Agent → 本地 WebSocket → Earth 页面；用于双摄、USB index、手机/网络摄像头流 + +打开方式：设置中开启"动捕调试模式"，或加 URL 参数 `?motion=1` 打开 Earth 动捕连接。Motion Agent 默认地址 `ws://127.0.0.1:8765/ws/gestures`，可用 `motionAgent` URL 参数覆盖。也可以直接 `?motion=1&motionProvider=browser` 或 `?motion=1&motionProvider=agent`。 + +两种模式都不会把摄像头帧或实时手势发到云端，也不会复用新闻/RSS 聚合接口。 + +手势语义： + +| 手势事件 | 作用 | +| --- | --- | +| `rotate_left/right/up/down` | 地球向对应方向旋转 | +| `zoom_in/out` | 放大或缩小视角 | +| `focus_prev/next` | 在当前动捕图层内切换可交互目标 | +| `layer_prev/next` | 切换动捕候选图层并巡航到新图层最近目标 | +| `confirm` | 确认当前已选目标 | + +调试面板中：浏览器摄像头输入会显示本机实时预览并绘制关节点和连线；Motion Agent 只发归一化骨架事件，不发原始帧。"只显示骨骼"会隐藏视频预览只保留骨架；"停止匹配动作"会暂停手势触发但保留预览和骨架。未匹配时骨架红色，匹配后变绿并显示动作名称。 + +### 巡航模式 + +巡航模式让 Earth 自动轮播聚焦目标。当前巡航模块：BGP、新闻、算力中心、船只、海缆、卫星。适合演示、监控大屏或无人值守。 + +### 移动端 + +移动端抽屉布局：图层控制进入移动抽屉，搜索、设置、详情使用移动端面板。主要交互仍围绕地球对象点击、搜索和图层开关。 + +### 常见问题 + +- **Earth 打不开**：先确认前端服务是否在线；如果端口不是 `3000`，使用启动输出的实际端口 +- **图层没有数据**：进 `/datasources` 看数据源状态、是否已采集和最近执行结果，再到 `/data` 或 `/bgp` 看是否有记录 +- **卫星 / BGP / 海缆加载慢**：这些图层依赖后端接口和外部数据源，首次加载需要等启动任务完成 + +## Docs 文档站 + +文档站 `http://localhost:3000/docs` 由后端按权限读取，不再把全部 Markdown 直接打进前端构建产物。 + +未登录访客默认只能看到 `public` 文档：首页、快速开始、使用手册、常见问题、Earth 位置候选采集使用手册。登录用户被分配 Gatekeeper 权限组后可以看到更多技术文档： + +- `docs_user`：用户操作类文档 +- `docs_developer`：Earth、前端、后端、采集器和 AI Provider 等开发文档 +- `docs_admin`：服务控制、运维、环境变量和敏感操作文档（包括运维手册） + +`admin` 默认拥有 `docs_admin`，`super_admin` 拥有全部 Docs 权限。Gatekeeper 权限组在"用户管理"中配置。 + +Docs 支持分类导航、Markdown 渲染、表格和代码块、文档内目录、对当前可见文档搜索、technical 文档间内部链接跳转。 ## 相关文档 - [快速开始](/home/ray/dev/linkong/planet/docs/technical/zh/quickstart.md) -- [控制台前端结构](/home/ray/dev/linkong/planet/docs/technical/zh/frontend-admin-frontend-context.md) -- [Earth 前端结构](/home/ray/dev/linkong/planet/docs/technical/zh/earth-frontend-context.md) -- [Earth 图层样式属性索引](/home/ray/dev/linkong/planet/docs/technical/zh/earth-layer-style-reference.md) +- [常见问题](/home/ray/dev/linkong/planet/docs/technical/zh/faq.md) - [Earth 位置候选采集使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-user.md) -- [系统服务控制](/home/ray/dev/linkong/planet/docs/technical/zh/backend-system-service-control.md) -- [数据采集系统](/home/ray/dev/linkong/planet/docs/technical/zh/backend-collectors.md) -- [数据源、采集器设置与连接验证](/home/ray/dev/linkong/planet/docs/technical/zh/datasource-collector-settings-connectivity.md) +- [Planet 运维手册](/home/ray/dev/linkong/planet/docs/technical/zh/ops-runbook.md) diff --git a/docs/technical/zh/ops-runbook.md b/docs/technical/zh/ops-runbook.md new file mode 100644 index 00000000..9e75cc4d --- /dev/null +++ b/docs/technical/zh/ops-runbook.md @@ -0,0 +1,244 @@ +# Planet 运维手册 + +这份手册面向部署、值班和二次开发的运维人员。客户面向的 UI 使用流程见 [Planet 使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/manual.md)，本手册只覆盖 shell、Docker、日志、环境变量和故障排查。 + +## 首次启动 + +```bash +./planet.sh start +``` + +默认行为： + +- 启动 PostgreSQL 和 Redis +- 启动 AI Provider +- 启动后端 API +- 启动前端 Vite dev server +- 输出 Earth、控制台、Playground 和后端 API 文档入口 + +首次启动会自动写入两个默认账号（见 `backend/app/db/session.py` 的 `DEFAULT_LOGIN_USERS`）： + +| 用户名 | 密码 | 角色 | +| --- | --- | --- | +| `admin` | `admin123` | `super_admin` | +| `linkong` | `12345678` | `super_admin` | + +两个默认账号 `email_verified` 为 `true`，可直接登录控制台。任何在该列表之外的账号都必须走公开注册 + 邮箱验证流程（见使用手册），或用 `./planet.sh createuser` 命令式创建。 + +可指定端口： + +```bash +./planet.sh start -b 8001 -f 3001 -a 8101 +``` + +| 参数 | 含义 | +| --- | --- | +| `-b ` | 后端端口 | +| `-f ` | 前端端口 | +| `-a ` | AI Provider 端口 | +| `--allow-lan` | 允许局域网访问 | +| `--verbose` | 显示更多命令输出 | + +## 启停与按模块重启 + +停止全部： + +```bash +./planet.sh stop +``` + +会停止后端、AI Provider、前端、PostgreSQL、Redis。 + +按模块重启： + +```bash +./planet.sh restart # 全量 +./planet.sh restart -b # 后端 +./planet.sh restart -f # 前端 +./planet.sh restart -a # AI Provider +./planet.sh restart -d # 数据库 +``` + +按模块重启适合日常开发，能避免无关服务被打断。 + +## 健康检查 + +```bash +./planet.sh health +``` + +会检查： + +- `planet_*` 容器状态 +- 后端 `/health` +- AI Provider `/health` +- 前端页面可达性 + +如果某项显示 offline，优先看对应日志。 + +## 日志 + +最近日志： + +```bash +./planet.sh log +``` + +持续跟随： + +```bash +./planet.sh log -f # 前端: /tmp/planet_frontend.log +./planet.sh log -b # 后端: /tmp/planet_backend.log +./planet.sh log -a # AI Provider: planet_aiprovider 容器日志 +``` + +## 命令式创建用户 + +```bash +./planet.sh createuser +``` + +交互式提示用户名、密码、角色，直接落库并标记 `email_verified = TRUE`。 + +适用场景： + +- SMTP 还没配置好，但需要先发账号给一名管理员 +- 想批量预置内部测试账号 +- 公开注册流程因任何原因不可用，需要应急兜底 + +正式用户开通推荐走控制台 `/settings -> SMTP 邮件` 配好发件后，让用户在 `/register` 自助注册。 + +## 局域网 / WSL 访问 + +```bash +./planet.sh start --allow-lan +``` + +适用于： + +- WSL 中启动，Windows 浏览器访问 +- 手机或平板演示 Earth +- 局域网其他机器访问同一开发实例 + +`--allow-lan` 只负责让前端和后端监听 `0.0.0.0`。WSL 中运行时，Windows 本机一般可以通过 `localhost` 访问，但局域网其他机器访问 `http://:3000` 还需要 Windows 端口转发和防火墙放行。 + +建议按顺序排查： + +```bash +# 在运行 Planet 的 shell 中 +curl http://localhost:3000 +curl http://localhost:8000/health +ss -ltnp | grep -E ':3000|:8000' +``` + +如果看到 `0.0.0.0:3000` 和 `0.0.0.0:8000`，但局域网 IP 仍访问失败，在管理员 PowerShell 中配置： + +```powershell +netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=3000 connectaddress=127.0.0.1 connectport=3000 +netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=8000 connectaddress=127.0.0.1 connectport=8000 + +New-NetFirewallRule -DisplayName "WSL Planet 3000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 3000 +New-NetFirewallRule -DisplayName "WSL Planet 8000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 8000 +``` + +## AI Provider 环境变量与构建 + +AI Provider 运行期配置可以放在两处： + +| 位置 | 适合内容 | 说明 | +| --- | --- | --- | +| `aiprovider/.env` | 团队约定的本地默认配置 | Docker Compose 作为 `env_file` 读取 | +| `~/.zshrc` | 个人 provider、模型、密钥、代理 | `planet.sh` 启动时读取常见 `AI_*`、`SERVICE_*`、`PYTHON_IMAGE`、`UV_IMAGE` | + +推荐写法： + +```bash +export AI_PROVIDER=minimax +export AI_PROVIDER_API=anthropic-messages +export AI_BASE_URL=https://api.example.com/anthropic +export AI_API_KEY=sk-change-me +export AI_MODEL=MiniMax-M2.7 +export AI_PROVIDER_SERVICE_TOKEN=change_me +``` + +`planet.sh` 默认只静态解析 `~/.zshrc` 中简单的 `export KEY=value` 或 `KEY=value` 行。需要复杂 shell 展开时显式启用： + +```bash +PLANET_LOAD_ZSHRC_ENV=source ./planet.sh start -a +``` + +完全忽略 `~/.zshrc`： + +```bash +PLANET_LOAD_ZSHRC_ENV=0 ./planet.sh start -a +``` + +AI Provider 镜像只在代码、Dockerfile、Compose 配置或相关 Python 依赖变化时重建。修改密钥或 Base URL 后只需重启容器： + +```bash +./planet.sh restart -a +``` + +构建较慢时按层排查： + +| 现象 | 常见原因 | 处理方式 | +| --- | --- | --- | +| `transferring context` 很大 | build context 含前端资源等无关文件 | `.dockerignore` 只发送必需文件 | +| `uv sync` 下载较慢 | 首次构建或缓存为空 | 等待首次完成，后续复用 BuildKit 缓存 | +| 改密钥后仍是旧配置 | 容器未重启 | `./planet.sh restart -a` | + +## SMTP 邮件（公开注册依赖） + +公开注册和邮箱验证依赖 SMTP。管理员在控制台 `/settings -> SMTP 邮件` 子 tab 中填写主机、端口、账号、密码、发件地址、TLS 模式，然后用"发送测试邮件"按钮验证。配置同时写入 `system_settings.smtp` 表行。 + +未配置时 `POST /api/v1/auth/register` 返回 `503 EMAIL_PROVIDER_NOT_CONFIGURED`，前端的注册流程会给出明确提示。运维兜底方式是 `./planet.sh createuser`。 + +OTP 一次性验证码走 Redis，key 格式 `otp:{purpose}:{email}`，TTL 600 秒。错误尝试 5 次后该 key 失效；重发冷却 60 秒，由 `otp_rate:{purpose}:{email}` 控制。 + +## 故障排查顺序 + +```bash +./planet.sh health # 1. 看服务状态 +./planet.sh log # 2. 看最近日志 +./planet.sh log -f # 3. 按模块查看 +./planet.sh log -b +./planet.sh log -a +./planet.sh restart -f # 4. 只重启有问题的模块 +./planet.sh restart -b +./planet.sh restart -a +./planet.sh restart -d # 5. 数据库 / 缓存异常时 +./planet.sh restart # 6. 仍无法恢复时全量重启 +``` + +## 开发命令约定 + +前端必须使用 Bun： + +```bash +cd frontend +bun install +bun run dev +bun run build +``` + +不要使用 `npm run ...`。项目在 WSL / Windows 混合环境优先依赖 Bun，避免 Node/npm 路径差异。 + +验证前端构建： + +```bash +source ~/.zshrc && bun run build +``` + +后端依赖通过 uv 管理： + +```bash +uv sync +uv run pytest backend/tests/test_otp_service.py +``` + +## 相关文档 + +- [planet.sh 启动机制](/home/ray/dev/linkong/planet/docs/technical/zh/ops-planet-sh-startup.md) +- [系统服务控制](/home/ray/dev/linkong/planet/docs/technical/zh/backend-system-service-control.md) +- [Docker + Compose + Buildx 升级](/home/ray/dev/linkong/planet/docs/technical/zh/ops-docker-compose-buildx-upgrade.md) +- [数据采集系统](/home/ray/dev/linkong/planet/docs/technical/zh/backend-collectors.md) diff --git a/docs/technical/zh/quickstart.md b/docs/technical/zh/quickstart.md index 0541b09f..4187a47d 100644 --- a/docs/technical/zh/quickstart.md +++ b/docs/technical/zh/quickstart.md @@ -1,236 +1,66 @@ # 快速开始 -这份快速开始面向第一次启动 Planet 的开发者或演示操作者。目标是用最短路径把服务跑起来，并知道应该打开哪些入口。 +这份快速开始面向 Planet 的最终用户：你拿到了管理员给的访问地址，要从打开浏览器到第一次完成配置之间的最短路径。所有操作都在浏览器里完成。 -如果遇到端口占用、Windows / WSL 局域网访问、`uv` / `bun`、摄像头或 Docs 权限问题，先看 [常见问题](/home/ray/dev/linkong/planet/docs/technical/zh/faq.md)。 +如果你是负责部署或运维的同事，请改读 [Planet 运维手册](/home/ray/dev/linkong/planet/docs/technical/zh/ops-runbook.md)。 -## 前置条件 +## 1. 打开访问地址 -推荐在 WSL / Linux shell 中运行。 +打开管理员给的 URL，例如 `http://planet.example.com`。本地演示一般是 `http://localhost:3000`。 -需要具备： +入口分两类： -- Docker / Docker Compose 可用 -- 当前 shell 能访问 `uv` 和 `bun` -- 仓库已 clone 到本机 +- 公开页：`/earth`（3D 态势）、`/docs`（公共文档） +- 登录后：`/admin`（控制台）、`/ai`（AI）、`/settings`（系统配置） -如果是新机器，优先执行仓库自带初始化脚本： +## 2. 注册账号 -```bash -./scripts/bootstrap-dev.sh -``` +1. 打开 `/login`，点击表单下方"注册账户"。 +2. 在 `/register` 填用户名、邮箱、密码（至少 8 位）。 +3. 提交后，到邮箱查收 6 位验证码（10 分钟内有效）。 +4. 在验证页输入验证码，点击"验证并登录"，系统会自动跳到控制台。 -这个脚本会检查并同步常用依赖，并在缺少时生成： +收不到邮件时： -- `backend/.env` -- `aiprovider/.env` -- `frontend/.env.local` +- 看一下垃圾邮件或企业邮件网关 +- 验证页右下角的"重新发送验证码"会显示倒计时，结束后可以重发 +- 提示"邮件服务尚未配置"时联系管理员开 SMTP -AI Provider 的个人配置也可以放在 `~/.zshrc`。`planet.sh` 会读取简单的 `export AI_...=...` 或 `AI_...=...` 行，并在启动 AI Provider 时传给容器。修改模型、密钥或 Base URL 后，通常只需要重启 AI Provider： +默认角色是 `viewer`，可登录但只能看公共内容。要看采集器、用户管理、系统设置，请联系管理员升级角色或加 Gatekeeper 权限组。 -```bash -./planet.sh restart -a -``` +## 3. 第一次登录 -AISStream、BarentsWatch 等采集器凭证也可以先写在 `~/.zshrc` 里供连接验证读取，例如： +进入 `/admin` 仪表盘后，建议按这个顺序熟悉控制台： -```bash -export AISSTREAM_API_KEY="..." -export BARENTSWATCH_CLIENT_ID="..." -export BARENTSWATCH_CLIENT_SECRET="..." -``` +1. `/settings?tab=collector_credentials`：选一个采集器，点插头图标做连接测试。免费 collector（开源 BGP 等）通常直接可用；像 `AISStream`、`BarentsWatch` 这类需要凭证的，需要先填 API Key/Client Secret +2. `/ai?tab=providers`：填一个 LLM provider（例如 `minimax` / `openai`）、模型名、Base URL、API Key，点 Base URL 末端的插头测试连接。WebSearch / OCR 工具可选 +3. `/datasources` 或 `/data`：看采集器是否已经产出数据 +4. `/alerts/system`：看系统告警是否正常 +5. `/users`（仅 `super_admin`）：根据需要给同事开账号或调权限组 -正式采集更推荐在控制台 `设置 -> 采集器设置` 保存凭证，尤其是 AISStream 这类长连接 WebSocket collector。这样连接验证、后端采集任务和 Earth 实时船只聚合会使用同一份配置。 +## 4. 打开 Earth -## 1. 启动服务 +访问 `/earth`，公开页面，不需要登录。 -在仓库根目录执行： +进入后建议确认： -```bash -./planet.sh start -``` - -启动完成后，常用入口是： - -| 入口 | 默认地址 | 用途 | -| --- | --- | --- | -| Earth | `http://localhost:3000/earth` | 公开 3D Earth 可视化页面 | -| 控制台 | `http://localhost:3000/admin` | 登录后的管理后台 | -| 文档站 | `http://localhost:3000/docs` | 使用手册公开；开发/运维文档按 Gatekeeper 权限组开放 | -| AI | `http://localhost:3000/ai` | 登录后的模型供应商、工具和测试台入口 | -| 后端 API 文档 | `http://localhost:8000/docs` | FastAPI / OpenAPI 接口文档 | - -如果默认端口被占用，可以指定端口： - -```bash -./planet.sh start -f 3001 -b 8001 -a 8101 -``` - -后端 `8000` 被 Windows listener 或旧 portproxy 占用时，排查顺序见 [常见问题](/home/ray/dev/linkong/planet/docs/technical/zh/faq.md)。 - -## 2. 创建登录用户 - -控制台需要登录。首次使用可以执行： - -```bash -./planet.sh createuser -``` - -按提示输入用户名、密码和角色。 - -如果需要阅读开发或运维文档，用 `super_admin` 登录控制台后，在“用户管理”里给目标用户分配 Gatekeeper 权限组：`docs_developer` 用于开发文档，`docs_admin` 用于服务控制和运维文档。 - -## 3. 打开 Earth - -访问： - -```text -http://localhost:3000/earth -``` - -Earth 是公开页面，不需要登录。 - -进入后可以先确认： - -- 地球正常显示 -- 右侧图层控制可打开/关闭图层 +- 地球正常显示，右侧图层面板可以打开/关闭 - 搜索可以查找海缆、卫星、算力中心、BGP 事件 -- 算力中心和 BGP 观测站详情卡可以自动采集并预览坐标候选；常规来源无候选时会用当前默认 AI Provider 做 LLM factcheck 兜底；算力中心待定位气泡可以打开列表并保存候选 -- 鼠标拖动、滚轮缩放和缩放百分比提示正常工作 -- 设置面板可以切换旋转 / 巡航 / 动捕模式、日夜模式、卫星显示风格；动捕调试模式下浏览器摄像头可显示本机预览和骨架 +- 算力中心和 BGP 观测站详情卡可以自动采集坐标候选，并能在 Earth 上预览 +- 鼠标拖动、滚轮缩放、缩放百分比提示工作正常 +- 设置面板的旋转 / 巡航 / 动捕模式可以切换 -## 4. 打开控制台 +## 5. 找回密码 -访问： +忘记密码时打开 `/forgot-password`，输入邮箱并接收验证码，再输入验证码 + 新密码即可。系统对未注册的邮箱也会返回同样的提示（防止账号枚举）。 -```text -http://localhost:3000/admin -``` +## 6. 看文档 -控制台用于数据源、采集数据、专题观测、告警、系统日志和配置管理。 - -首次排查建议查看： - -- `/datasources`：数据源目录和采集触发；接口、请求头和凭证配置在 `/settings` 的“采集器设置” -- `/data`：已采集数据 -- `/bgp`：BGP 专题观测 -- `/ai`：AI，管理模型供应商、WebSearch 等工具和测试台 -- `/alerts/system`：系统告警 -- `/settings`：系统配置 - -## 5. 查看运行状态 - -```bash -./planet.sh health -``` - -这个命令会显示容器状态，并检查： - -- 后端 -- AI Provider -- 前端 - -## 6. 查看日志 - -最近日志： - -```bash -./planet.sh log -``` - -持续查看某个服务： - -```bash -./planet.sh log -f -./planet.sh log -b -./planet.sh log -a -``` - -含义： - -- `-f`：前端日志 -- `-b`：后端日志 -- `-a`：AI Provider 日志 - -## 7. 常用重启 - -只重启前端： - -```bash -./planet.sh restart -f -``` - -只重启后端： - -```bash -./planet.sh restart -b -``` - -只重启 AI Provider： - -```bash -./planet.sh restart -a -``` - -只重启数据库： - -```bash -./planet.sh restart -d -``` - -全量重启： - -```bash -./planet.sh restart -``` - -## 8. 局域网访问 - -如果希望 Windows 浏览器、手机或同一局域网的其他设备访问： - -```bash -./planet.sh start --allow-lan -``` - -这会让前端和后端监听局域网可访问地址。 - -注意：`--allow-lan` 只负责让 Planet 服务监听 `0.0.0.0`，不等于自动把 WSL 服务暴露到 Windows 局域网 IP。常见情况是： - -- WSL 内 `localhost:3000` / `localhost:8000` 能访问 -- Windows 本机 `localhost:3000` / `localhost:8000` 能访问 -- 但手机或其他电脑访问 `http://:3000` 失败 - -这通常说明 Windows 端还缺少端口转发或防火墙放行。 - -如果访问失败，先在运行 Planet 的 shell 中检查： - -```bash -curl http://localhost:3000 -curl http://localhost:8000/health -ss -ltnp | grep -E ':3000|:8000' -``` - -如果确认 WSL 中已监听 `0.0.0.0:3000` 和 `0.0.0.0:8000`，但局域网 IP 仍不能访问，请在管理员 PowerShell 中配置 Windows 端转发和防火墙： - -```powershell -netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=3000 connectaddress=127.0.0.1 connectport=3000 -netsh interface portproxy add v4tov4 listenaddress=0.0.0.0 listenport=8000 connectaddress=127.0.0.1 connectport=8000 - -New-NetFirewallRule -DisplayName "WSL Planet 3000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 3000 -New-NetFirewallRule -DisplayName "WSL Planet 8000" -Direction Inbound -Action Allow -Protocol TCP -LocalPort 8000 -``` - -## 9. 停止服务 - -```bash -./planet.sh stop -``` - -停止后会关闭前端、后端、AI Provider、PostgreSQL 和 Redis。 +`/docs` 是文档站。未登录可看：本快速开始、使用手册、常见问题。登录后被分配 `docs_user` / `docs_developer` / `docs_admin` 权限组的用户可以看更多技术文档。 ## 下一步 -- 完整操作说明见 [Planet 使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/manual.md) -- 控制台结构见 [控制台前端结构](/home/ray/dev/linkong/planet/docs/technical/zh/frontend-admin-frontend-context.md) -- Earth 结构见 [Earth 前端结构](/home/ray/dev/linkong/planet/docs/technical/zh/earth-frontend-context.md) -- 后端采集器见 [数据采集系统](/home/ray/dev/linkong/planet/docs/technical/zh/backend-collectors.md) +- 完整 UI 操作说明：[Planet 使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/manual.md) +- 排障与配置疑问：[常见问题](/home/ray/dev/linkong/planet/docs/technical/zh/faq.md) +- Earth 坐标候选采集详细流程：[Earth 位置候选采集使用手册](/home/ray/dev/linkong/planet/docs/technical/zh/location-pipeline-user.md) +- 部署 / 运维相关命令：[Planet 运维手册](/home/ray/dev/linkong/planet/docs/technical/zh/ops-runbook.md) diff --git a/docs/version-history.md b/docs/version-history.md index 90609d3a..f8f8a49a 100644 --- a/docs/version-history.md +++ b/docs/version-history.md @@ -16,12 +16,13 @@ ## Current Version - `main` 当前主线历史推导到：`0.16.5` -- `dev` 当前开发分支历史推导到：`0.51.1` +- `dev` 当前开发分支历史推导到：`0.52.0` ## Timeline | Version | Type | Branch | Commit | Summary | | --- | --- | --- | --- | --- | +| `0.52.0` | feature | `dev` | `pending` | 新增邮箱验证码账号自助链路、AISStream 船只实时采集与受控 snapshot/WS 展示、数据产品统计接口、受控图层接口和数据源批量运维 | | `0.51.1` | bugfix | `dev` | `pending` | 修复 Earth 静态资源模块相对路径与 Material Symbols 本地字体加载，避免部署路径变化或外部字体不可用时图标/边界资源失效 | | `0.51.0` | feature | `dev` | `pending` | 新增 AI Settings 控制台与 ai_tools 工具层；重写算力中心候选预览/保存交互（呼吸圈 + 即时图标）；动作捕捉 zoom 改为 mirror-safe trend + pose hold 双通道，支持持续触发 | | `0.50.0` | feature | `dev` | `pending` | 新增 Earth 动捕双通道控制、Motion Agent、Presentation 持久展示、AI Provider 多 provider 设置、位置候选 LLM 兜底与 FAQ | diff --git a/frontend/package.json b/frontend/package.json index 417af8a6..0831d18a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "planet-frontend", - "version": "0.51.1", + "version": "0.52.0", "private": true, "packageManager": "bun@1", "dependencies": { diff --git a/frontend/public/earth/js/constants.js b/frontend/public/earth/js/constants.js index b6e06248..5779831e 100644 --- a/frontend/public/earth/js/constants.js +++ b/frontend/public/earth/js/constants.js @@ -206,7 +206,7 @@ export const PATHS = { cablesApi: '/api/v1/visualization/geo/cables', landingPointsApi: '/api/v1/visualization/geo/landing-points', computeCentersApi: '/api/v1/visualization/geo/compute-centers', - vesselsApi: '/api/v1/visualization/geo/vessels', + vesselsApi: '/api/v1/vessels/snapshot', vesselTrackApi: (mmsi) => `/api/v1/visualization/vessels/${encodeURIComponent(mmsi)}/track`, bgpApi: '/api/v1/visualization/geo/bgp-anomalies', bgpIncidentsApi: '/api/v1/visualization/geo/bgp-incidents', diff --git a/frontend/public/earth/js/vessels.js b/frontend/public/earth/js/vessels.js index ed5b8a6f..ae832f2b 100644 --- a/frontend/public/earth/js/vessels.js +++ b/frontend/public/earth/js/vessels.js @@ -258,6 +258,11 @@ const vesselIconLayer = createInteractableLayer({ }), }); +const DEFAULT_VESSEL_VIEWPORT = { + bbox: [-180, -90, 180, 90], + zoom: 2, +}; + export function getVesselMarkers() { return vesselIconLayer.getMarkers(); } @@ -313,6 +318,14 @@ export function clearVesselData(earth) { export async function loadVessels(_scene, earth, options = {}) { const params = new URLSearchParams(); const requestedLimit = Number(options.limit ?? VESSEL_CONFIG.maxRenderedMarkers); + const bbox = Array.isArray(options.bbox) && options.bbox.length === 4 + ? options.bbox + : DEFAULT_VESSEL_VIEWPORT.bbox; + const zoom = Number.isFinite(Number(options.zoom)) + ? Number(options.zoom) + : DEFAULT_VESSEL_VIEWPORT.zoom; + params.set("bbox", bbox.join(",")); + params.set("zoom", String(zoom)); if (Number.isFinite(requestedLimit) && requestedLimit > 0) { params.set("limit", String(requestedLimit)); } @@ -352,7 +365,15 @@ export function startVesselRealtime(earth, { onUpdate } = {}) { connected: true, }; onUpdate?.({ totalCount: getVesselCount(), stream: getVesselRealtimeStats() }); - socket.send(JSON.stringify({ type: "subscribe", data: { channels: ["vessels"] } })); + socket.send(JSON.stringify({ + type: "subscribe", + data: { + channel: "vessels", + bbox: DEFAULT_VESSEL_VIEWPORT.bbox, + zoom: DEFAULT_VESSEL_VIEWPORT.zoom, + limit: VESSEL_CONFIG.maxRenderedMarkers, + }, + })); }; socket.onmessage = (event) => { let message; diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 18c129c7..12239f44 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,11 +1,14 @@ import { Suspense, lazy } from 'react' import { Spin } from 'antd' -import { Routes, Route, Navigate } from 'react-router-dom' +import { Routes, Route, Navigate, useLocation } from 'react-router-dom' import { useAuthStore } from './stores/auth' import Login from './pages/Login/Login' +const Register = lazy(() => import('./pages/Register/Register')) +const VerifyEmail = lazy(() => import('./pages/VerifyEmail/VerifyEmail')) +const ForgotPassword = lazy(() => import('./pages/ForgotPassword/ForgotPassword')) const SystemAlerts = lazy(() => import('./pages/Alerts/SystemAlerts')) const BGPAlerts = lazy(() => import('./pages/Alerts/BGPAlerts')) const SituationalAlerts = lazy(() => import('./pages/Alerts/SituationalAlerts')) @@ -25,7 +28,8 @@ const EARTH_ROUTE = '/earth' const DOCS_ROUTE = '/docs' const DOCS_ROUTE_PATTERN = '/docs/:slug' const DOCS_ROUTE_PREFIX = `${DOCS_ROUTE}/` -const PUBLIC_EXACT_ROUTES = new Set([ROOT_ROUTE, EARTH_ROUTE, DOCS_ROUTE]) +const AUTH_ROUTES = new Set(['/login', '/register', '/verify-email', '/forgot-password']) +const PUBLIC_EXACT_ROUTES = new Set([ROOT_ROUTE, EARTH_ROUTE, DOCS_ROUTE, ...AUTH_ROUTES]) function isPublicPath(pathname: string) { return PUBLIC_EXACT_ROUTES.has(pathname) || pathname.startsWith(DOCS_ROUTE_PREFIX) @@ -33,7 +37,8 @@ function isPublicPath(pathname: string) { function App() { const { token } = useAuthStore() - const isPublicRoute = isPublicPath(window.location.pathname) + const { pathname } = useLocation() + const isPublicRoute = isPublicPath(pathname) if (!token && !isPublicRoute) { return @@ -48,6 +53,10 @@ function App() { )} > + } /> + } /> + } /> + } /> } /> } /> } /> @@ -71,4 +80,4 @@ function App() { ) } -export default App +export default App diff --git a/frontend/src/components/ConnectionTestInput/ConnectionTestInput.tsx b/frontend/src/components/ConnectionTestInput/ConnectionTestInput.tsx new file mode 100644 index 00000000..dd436444 --- /dev/null +++ b/frontend/src/components/ConnectionTestInput/ConnectionTestInput.tsx @@ -0,0 +1,107 @@ +import type { ReactNode } from 'react' +import { Button, Input, Tooltip, type InputProps } from 'antd' + +export function PlugConnectIcon() { + return ( + + ) +} + +interface ConnectionTestInputProps extends Omit { + testTitle?: ReactNode + testing?: boolean + testDisabled?: boolean + onTest?: () => void + extraSuffix?: ReactNode +} + +export default function ConnectionTestInput({ + testTitle = '测试连接', + testing = false, + testDisabled = false, + disabled, + onTest, + extraSuffix, + ...inputProps +}: ConnectionTestInputProps) { + const actionDisabled = disabled || testDisabled + + return ( + + + - - - - - ({ - value: preset.provider, - label: preset.label, - }))} - onChange={(value) => applyWebSearchProviderSelection(value)} + value={selectedToolKey} + options={TOOL_OPTIONS} + onChange={setSelectedToolKey} /> -
- - - - - - -
- - - - - {selectedWebSearchSecret?.configured ? '已配置' : '未配置'} - - 保存时只会更新当前 WebSearch Provider 的 key。 - - - : } - onClick={() => { void handleWebSearchApiKeyVisibleChange(!webSearchApiKeyRevealed) }} - aria-label={webSearchApiKeyRevealed ? '隐藏 WebSearch API key' : '显示 WebSearch API key'} - /> - )} + {selectedToolKey === 'web_search' ? ( + <> + + + + + + + { void testWebSearchConnection() }} + /> - - + + + + + {selectedWebSearchSecret?.configured ? '已配置' : '未配置'} + + 保存时只会更新当前 WebSearch Provider 的 key。 + + + + +
+ + + } placeholder="注册时使用的邮箱" size="large" autoComplete="email" /> + + + + +
+ navigate('/login')}>返回登录 +
+ + ) : ( +
+ + 已将验证码发送至 {email}，10 分钟内有效。 + + + } placeholder="6 位验证码" size="large" maxLength={6} /> + + + } placeholder="新密码 (至少 8 位)" size="large" autoComplete="new-password" /> + + + + +
+ setStep('request')}>修改邮箱 + +
+
+ )} +
+ + ) +} + +export default ForgotPassword diff --git a/frontend/src/pages/Login/Login.tsx b/frontend/src/pages/Login/Login.tsx index 74f07bda..f1844009 100644 --- a/frontend/src/pages/Login/Login.tsx +++ b/frontend/src/pages/Login/Login.tsx @@ -1,9 +1,16 @@ import { useState } from 'react' -import { Input, Button, Form, message } from 'antd' -import { UserOutlined, LockOutlined } from '@ant-design/icons' +import { Button, Form, Input, Typography, message } from 'antd' +import { LockOutlined, UserOutlined } from '@ant-design/icons' import { useNavigate } from 'react-router-dom' import { useAuthStore } from '../../stores/auth' +interface LoginError { + response?: { + data?: { detail?: string | { code?: string; email?: string; message?: string } } + status?: number + } +} + function Login() { const [loading, setLoading] = useState(false) const navigate = useNavigate() @@ -16,8 +23,16 @@ function Login() { message.success('登录成功') navigate('/admin', { replace: true }) } catch (error: unknown) { - const err = error as { response?: { data?: { detail?: string } } } - message.error(err.response?.data?.detail || '登录失败') + const err = error as LoginError + const detail = err.response?.data?.detail + if (detail && typeof detail === 'object' && detail.code === 'EMAIL_NOT_VERIFIED') { + message.warning('邮箱未验证，请先完成邮箱验证') + const email = detail.email ? `?email=${encodeURIComponent(detail.email)}` : '' + navigate(`/verify-email${email}`) + return + } + const fallback = typeof detail === 'string' ? detail : detail?.message + message.error(fallback || '登录失败') } finally { setLoading(false) } @@ -36,6 +51,7 @@ function Login() { prefix={} placeholder="用户名" size="large" + autoComplete="username" /> } placeholder="密码" size="large" + autoComplete="current-password" /> @@ -59,6 +76,10 @@ function Login() { 登录 +
+ navigate('/register')}>注册账户 + navigate('/forgot-password')}>忘记密码？ +
diff --git a/frontend/src/pages/Playground/Playground.tsx b/frontend/src/pages/Playground/Playground.tsx index 5483a054..f7b0443d 100644 --- a/frontend/src/pages/Playground/Playground.tsx +++ b/frontend/src/pages/Playground/Playground.tsx @@ -225,6 +225,7 @@ export function PlaygroundWorkspace({ embedded = false }: { embedded?: boolean } const [helpExpanded, setHelpExpanded] = useState(true) const [settingsOpen, setSettingsOpen] = useState(false) const [servicePanelOpen, setServicePanelOpen] = useState(false) + const [helpPanelOpen, setHelpPanelOpen] = useState(false) const [selectedPresetKey, setSelectedPresetKey] = useState(PLAYGROUND_PRESETS[0].key) const [title, setTitle] = useState(PLAYGROUND_PRESETS[0].values.title) const [objective, setObjective] = useState(PLAYGROUND_PRESETS[0].values.objective) @@ -582,14 +583,16 @@ export function PlaygroundWorkspace({ embedded = false }: { embedded?: boolean } className="playground-card playground-card--provider" title="Provider 状态" extra={( - + +
+ navigate('/login')}>已有账号，去登录 +
+ + ) : ( +
+ + 已将 6 位验证码发送至 {email}，10 分钟内有效。 + + + } + placeholder="6 位验证码" + size="large" + maxLength={6} + inputMode="numeric" + /> + + + + +
+ setStep('register')}>修改邮箱 + +
+
+ )} + + + ) +} + +export default Register diff --git a/frontend/src/pages/Settings/Settings.tsx b/frontend/src/pages/Settings/Settings.tsx index 40e487f8..e1cc465b 100644 --- a/frontend/src/pages/Settings/Settings.tsx +++ b/frontend/src/pages/Settings/Settings.tsx @@ -18,6 +18,7 @@ import { } from '@ant-design/icons' import { Alert, + AutoComplete, Button, Card, Checkbox, @@ -37,11 +38,13 @@ import { } from 'antd' import axios from 'axios' import AppLayout from '../../components/AppLayout/AppLayout' +import ConnectionTestInput, { PlugConnectIcon } from '../../components/ConnectionTestInput/ConnectionTestInput' import Scrollbar from '../../components/Scrollbar/Scrollbar' import TableScrollRegion from '../../components/Scrollbar/TableScrollRegion' import MarkdownRenderer from '../../components/MarkdownRenderer/MarkdownRenderer' import { formatDateTimeZhCN } from '../../utils/datetime' import { useNavigate, useSearchParams } from 'react-router-dom' +import SmtpPanel from './SmtpPanel' const { Title, Text } = Typography const ANTHROPIC_MESSAGES_MAX_TOKENS = 1200 @@ -318,62 +321,6 @@ const formatLagSeconds = (value: number | null | undefined) => { return `${Math.round(value / 3600)} 小时` } -function PlugConnectIcon() { - return ( - - ) -} - function SettingsPanel({ loading, children, @@ -441,6 +388,7 @@ function Settings() { const [tvEditForm] = Form.useForm() const selectedAiProvider = Form.useWatch(['ai_provider', 'provider'], integrationForm) const selectedWebSearchProvider = Form.useWatch(['web_search', 'provider'], integrationForm) + const webSearchEnabled = Form.useWatch(['web_search', 'enabled'], integrationForm) const selectedAiProviderSecret = selectedAiProvider ? integrations?.ai_provider.providers?.[selectedAiProvider]?.api_key || integrations?.ai_provider.api_key : integrations?.ai_provider.api_key @@ -1907,6 +1855,11 @@ function Settings() { ), }, + { + key: 'smtp', + label: 'SMTP 邮件', + children: , + }, { key: 'tv', label: '电视直播', @@ -2036,42 +1989,43 @@ function Settings() { > LLM Provider}> - - - - - - - - - + + { void testAiProviderConnection() }} + extraSuffix={( + + + { void testWebSearchConnection() }} + /> @@ -2208,15 +2163,19 @@ function Settings() { : } - onClick={() => { void handleWebSearchApiKeyVisibleChange(!webSearchApiKeyRevealed) }} - aria-label={webSearchApiKeyRevealed ? '隐藏 WebSearch API key' : '显示 WebSearch API key'} - /> + + + + + + setTestOpen(false)} + onOk={onTest} + okText="发送" + okButtonProps={{ loading: testing }} + cancelText="取消" + > + + 会用当前表单中的 SMTP 配置（未保存的修改也会生效）发送一封测试邮件。 + + setTestTo(event.target.value)} + autoFocus + /> + + + ) +} diff --git a/frontend/src/pages/VerifyEmail/VerifyEmail.tsx b/frontend/src/pages/VerifyEmail/VerifyEmail.tsx new file mode 100644 index 00000000..845f1999 --- /dev/null +++ b/frontend/src/pages/VerifyEmail/VerifyEmail.tsx @@ -0,0 +1,121 @@ +import { useEffect, useState } from 'react' +import { Button, Form, Input, message, Typography } from 'antd' +import { MailOutlined, SafetyCertificateOutlined } from '@ant-design/icons' +import axios from 'axios' +import { useNavigate, useSearchParams } from 'react-router-dom' +import { useAuthStore } from '../../stores/auth' + +const API_URL = (import.meta as any).env?.VITE_API_URL || '/api/v1' + +interface ErrorBody { + response?: { + data?: { detail?: string | { code?: string; message?: string; retry_after_seconds?: number } } + } +} + +function extractDetail(error: unknown): string { + const err = error as ErrorBody + const detail = err.response?.data?.detail + if (typeof detail === 'string') return detail + if (detail && typeof detail === 'object') return detail.message || detail.code || '操作失败' + return '操作失败' +} + +function VerifyEmail() { + const navigate = useNavigate() + const [search] = useSearchParams() + const [email, setEmail] = useState(search.get('email') || '') + const [loading, setLoading] = useState(false) + const [resending, setResending] = useState(false) + const [cooldown, setCooldown] = useState(0) + + useEffect(() => { + if (cooldown <= 0) return + const timer = setTimeout(() => setCooldown((v) => v - 1), 1000) + return () => clearTimeout(timer) + }, [cooldown]) + + const onVerify = async (values: { code: string }) => { + setLoading(true) + try { + const response = await axios.post(`${API_URL}/auth/verify-email`, { email, code: values.code }) + const { access_token, user } = response.data + useAuthStore.setState({ token: access_token, user }) + axios.defaults.headers.common.Authorization = `Bearer ${access_token}` + message.success('邮箱验证成功，正在登录…') + navigate('/admin', { replace: true }) + } catch (error) { + message.error(extractDetail(error)) + } finally { + setLoading(false) + } + } + + const onResend = async () => { + if (!email || cooldown > 0) return + setResending(true) + try { + await axios.post(`${API_URL}/auth/resend-code`, { email, purpose: 'register' }) + setCooldown(60) + message.success('验证码已重发') + } catch (error) { + const err = error as ErrorBody + const detail = err.response?.data?.detail + if (detail && typeof detail === 'object' && detail.retry_after_seconds) { + setCooldown(detail.retry_after_seconds) + } + message.error(extractDetail(error)) + } finally { + setResending(false) + } + } + + return ( +
+
+

验证邮箱

+
+ + } + size="large" + value={email} + onChange={(event) => setEmail(event.target.value)} + placeholder="注册时使用的邮箱" + autoComplete="email" + /> + + + } + placeholder="6 位验证码" + size="large" + maxLength={6} + inputMode="numeric" + /> + + + + +
+ navigate('/login')}>返回登录 + +
+
+
+
+ ) +} + +export default VerifyEmail diff --git a/pyproject.toml b/pyproject.toml index e1079e66..fdf2d673 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "planet" -version = "0.51.1" +version = "0.52.0" description = "智能星球计划 - 态势感知系统" requires-python = ">=3.14" dependencies = [ @@ -21,6 +21,7 @@ dependencies = [ "aiofiles>=23.2.1", "python-dotenv>=1.0.0", "email-validator>=2.1.0", + "aiosmtplib>=3.0.0", "apscheduler>=3.10.4", "networkx>=3.0", "mediapipe>=0.10.35", diff --git a/rules.md b/rules.md index 2b10ae72..4b064dab 100644 --- a/rules.md +++ b/rules.md @@ -255,6 +255,7 @@ Changing layout, visual hierarchy, controls, interaction states, responsive beha - Use icons in buttons for common tools/actions when an established icon exists. - Keep icon-only buttons accessible with `aria-label` and `title`. - Use segmented controls for modes, switches/checkboxes for binary settings, sliders/inputs for numeric values, menus/selects for option sets, and tabs for views. +- Connection-test actions for endpoint/Base URL inputs must use the shared `ConnectionTestInput` pattern: a single plug/connector icon at the input suffix, no adjacent text button; disabled integrations should grey out the field and its test action. - Do not put cards inside cards. - Do not use visible in-app text to explain obvious UI features or styling. - Text must fit within its parent on mobile and desktop. diff --git a/uv.lock b/uv.lock index 47bd5465..502cc3af 100644 --- a/uv.lock +++ b/uv.lock @@ -20,6 +20,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/8a/340a1555ae33d7354dbca4faa54948d76d89a27ceef032c8c3bc661d003e/aiofiles-25.1.0-py3-none-any.whl", hash = "sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695", size = 14668, upload-time = "2025-10-09T20:51:03.174Z" }, ] +[[package]] +name = "aiosmtplib" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/ad/240a7ce4e50713b111dff8b781a898d8d4770e5d6ad4899103f84c86005c/aiosmtplib-5.1.0.tar.gz", hash = "sha256:2504a23b2b63c9de6bc4ea719559a38996dba68f73f6af4eb97be20ee4c5e6c4", size = 66176, upload-time = "2026-01-25T01:51:11.408Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/82/70f2c452acd7ed18c558c8ace9a8cf4fdcc70eae9a41749b5bdc53eb6f45/aiosmtplib-5.1.0-py3-none-any.whl", hash = "sha256:368029440645b486b69db7029208a7a78c6691b90d24a5332ddba35d9109d55b", size = 27778, upload-time = "2026-01-25T01:51:10.026Z" }, +] + [[package]] name = "annotated-doc" version = "0.0.4" @@ -748,10 +757,11 @@ wheels = [ [[package]] name = "planet" -version = "0.51.1" +version = "0.52.0" source = { virtual = "." } dependencies = [ { name = "aiofiles" }, + { name = "aiosmtplib" }, { name = "apscheduler" }, { name = "asyncpg" }, { name = "bcrypt" }, @@ -785,6 +795,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "aiofiles", specifier = ">=23.2.1" }, + { name = "aiosmtplib", specifier = ">=3.0.0" }, { name = "apscheduler", specifier = ">=3.10.4" }, { name = "asyncpg", specifier = ">=0.29.0" }, { name = "bcrypt", specifier = ">=4.0.0" }, diff --git a/websocket_protocol.md b/websocket_protocol.md index 60faf14e..5adb7833 100644 --- a/websocket_protocol.md +++ b/websocket_protocol.md @@ -422,25 +422,72 @@ Same as `data_frame` with `update_type: "full"` ## Subscription Management +The current backend accepts `type: "subscribe"` / `type: "unsubscribe"` messages. Older examples that use `type: "subscription"` describe the same intent but should not be used for new clients. + ### Subscribe ```json { - "type": "subscription", + "type": "subscribe", "timestamp": "2024-01-20T10:30:00.000Z", "data": { - "action": "subscribe", "channels": ["gpu_clusters", "alerts"] } } ``` +### Vessel Viewport Subscribe + +The `vessels` channel is viewport-scoped. Clients must provide the current bbox, zoom, and limit. The server stores only this lightweight subscription filter and sends updates that match the subscribed bbox. + +```json +{ + "type": "subscribe", + "timestamp": "2026-05-12T10:30:00.000Z", + "data": { + "channel": "vessels", + "bbox": [120.8, 30.7, 122.1, 31.8], + "zoom": 12, + "limit": 1000 + } +} +``` + +`limit` is capped at 5000 per subscription, and each WebSocket data frame is capped to 1000 vessel updates. Collector updates are throttled to one flush per second; within a flush window, only the latest update per MMSI is retained. + +Vessel data frames use the normal `data_frame` envelope: + +```json +{ + "type": "data_frame", + "channel": "vessels", + "timestamp": "2026-05-12T10:30:01.000Z", + "payload": { + "action": "upsert", + "vessels": [ + { + "mmsi": 257123000, + "lat": 59.91, + "lon": 10.73, + "sog": 12.4, + "cog": 214, + "received_at": "2026-05-12T10:30:00Z" + } + ], + "subscription": { + "bbox": [120.8, 30.7, 122.1, 31.8], + "zoom": 12, + "limit": 1000 + } + } +} +``` + ### Unsubscribe ```json { - "type": "subscription", + "type": "unsubscribe", "timestamp": "2024-01-20T10:30:00.000Z", "data": { - "action": "unsubscribe", "channels": ["alerts"] } }