release: bump version to 0.70.0
Some checks failed
ci / backend (push) Has been cancelled
ci / frontend (push) Has been cancelled
release / images (push) Has been cancelled
ci / delivery (push) Has been cancelled

This commit is contained in:
linkong
2026-06-04 17:16:23 +08:00
parent acbbfdf9e2
commit 8c204717cd
78 changed files with 1762 additions and 703 deletions

View File

@@ -4,6 +4,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import text
from app.core.config import settings
from app.core.enums import OtpPurpose, UserRole
from app.core.logging import get_logger
from app.core.security import (
create_access_token,
@@ -170,7 +171,7 @@ async def get_me(current_user: User = Depends(get_current_user)):
}
async def _send_code_or_raise(db: AsyncSession, email: str, code: str, purpose: str) -> None:
async def _send_code_or_raise(db: AsyncSession, email: str, code: str, purpose: OtpPurpose) -> None:
try:
await send_verification_email(db, to=email, code=code, purpose=purpose)
except EmailNotConfiguredError as exc:
@@ -207,7 +208,7 @@ async def register(payload: UserRegister, db: AsyncSession = Depends(get_db)):
username=payload.username,
email=payload.email,
password_hash=get_password_hash(payload.password),
role="viewer",
role=UserRole.VIEWER.value,
is_active=True,
email_verified=False,
)
@@ -215,13 +216,13 @@ async def register(payload: UserRegister, db: AsyncSession = Depends(get_db)):
await db.commit()
try:
code = otp.issue_code(payload.email, "register")
code = otp.issue_code(payload.email, OtpPurpose.REGISTER)
except otp.OtpResendRateLimited as exc:
raise HTTPException(
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
detail={"code": exc.code, "retry_after_seconds": exc.retry_after_seconds},
) from exc
await _send_code_or_raise(db, payload.email, code, "register")
await _send_code_or_raise(db, payload.email, code, OtpPurpose.REGISTER)
return {"status": "pending_verification", "email": payload.email}
@@ -266,7 +267,7 @@ async def resend_code(payload: ResendCodeRequest, db: AsyncSession = Depends(get
if user is None:
# Avoid email enumeration; pretend success.
return {"status": "ok"}
if payload.purpose == "register" and user.email_verified:
if payload.purpose is OtpPurpose.REGISTER and user.email_verified:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail={"code": "ALREADY_VERIFIED"},
@@ -289,12 +290,17 @@ async def forgot_password(payload: ForgotPasswordRequest, db: AsyncSession = Dep
# Don't leak whether an email is registered.
return {"status": "ok"}
try:
code = otp.issue_code(payload.email, "reset_password")
code = otp.issue_code(payload.email, OtpPurpose.RESET_PASSWORD)
except otp.OtpResendRateLimited:
# Silently accept; the user can retry after the cooldown.
return {"status": "ok"}
try:
await send_verification_email(db, to=payload.email, code=code, purpose="reset_password")
await send_verification_email(
db,
to=payload.email,
code=code,
purpose=OtpPurpose.RESET_PASSWORD,
)
except EmailNotConfiguredError as exc:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
@@ -304,7 +310,11 @@ async def forgot_password(payload: ForgotPasswordRequest, db: AsyncSession = Dep
logger.warning_event(
"SMTP send failed",
event="auth.email.send_failed",
context={"email": payload.email, "purpose": "reset_password", "error": str(exc)},
context={
"email": payload.email,
"purpose": OtpPurpose.RESET_PASSWORD.value,
"error": str(exc),
},
)
return {"status": "ok"}
@@ -318,7 +328,7 @@ async def reset_password(payload: ResetPasswordRequest, db: AsyncSession = Depen
detail={"code": "OTP_INVALID"},
)
try:
otp.verify_code(payload.email, "reset_password", payload.code)
otp.verify_code(payload.email, OtpPurpose.RESET_PASSWORD, payload.code)
except otp.OtpExpired as exc:
raise HTTPException(
status_code=status.HTTP_410_GONE,

View File

@@ -13,6 +13,7 @@ import httpx
from app.core.target_schema_registry import get_target_schema, list_target_schemas
from app.core.datasource_defaults import DEFAULT_DATASOURCES
from app.core.enums import AuthType, MappingValidationStatus, UserRole
from app.db.session import get_db
from app.models.user import User
from app.models.datasource_config import DataSourceConfig
@@ -34,7 +35,6 @@ from app.services.datasource_mapping import (
)
from app.services.custom_datasource_runtime import (
CustomDatasourceRuntimeError,
fetch_rest_payload,
get_custom_stream_status,
run_mapped_rest_config,
run_mapped_websocket_config,
@@ -42,8 +42,6 @@ from app.services.custom_datasource_runtime import (
stop_custom_stream,
test_websocket_config,
)
DATASOURCE_MAPPING_PROMPT_KEY = "datasource.mapping"
from app.services.datasource_connectivity import (
_resolve_aisstream_api_key,
_resolve_spacetrack_credentials_with_override,
@@ -57,7 +55,8 @@ from app.services.persistent_logs import record_audit_log
router = APIRouter()
SECRET_REVEAL_ROLES = {"admin", "super_admin"}
DATASOURCE_MAPPING_PROMPT_KEY = "datasource.mapping"
SECRET_REVEAL_ROLES = {UserRole.ADMIN.value, UserRole.SUPER_ADMIN.value}
def _user_role_value(user: User) -> str:
@@ -124,7 +123,7 @@ class DataSourceConfigCreate(BaseModel):
description: Optional[str] = None
source_type: str = Field(..., description="rest, websocket, http, api, database")
endpoint: str = Field(..., max_length=500)
auth_type: str = Field(default="none", description="none, bearer, api_key, basic")
auth_type: AuthType = Field(default=AuthType.NONE, description="none, bearer, api_key, basic")
auth_config: dict = Field(default={})
headers: dict = Field(default={})
config: dict = Field(default={"timeout": 30, "retry": 3})
@@ -135,7 +134,7 @@ class DataSourceConfigUpdate(BaseModel):
description: Optional[str] = None
source_type: Optional[str] = None
endpoint: Optional[str] = Field(None, max_length=500)
auth_type: Optional[str] = None
auth_type: Optional[AuthType] = None
auth_config: Optional[dict] = None
headers: Optional[dict] = None
config: Optional[dict] = None
@@ -210,7 +209,7 @@ class MappingTemplateCreate(BaseModel):
mapping_json: dict
sample_payload: Any | None = None
sample_payload_hash: Optional[str] = None
validation_status: str = Field(default="draft", pattern="^(draft|valid|invalid)$")
validation_status: MappingValidationStatus = MappingValidationStatus.DRAFT
is_active: bool = False
@@ -219,7 +218,7 @@ class MappingTemplateUpdate(BaseModel):
mapping_json: Optional[dict] = None
sample_payload: Any | None = None
sample_payload_hash: Optional[str] = None
validation_status: Optional[str] = Field(default=None, pattern="^(draft|valid|invalid)$")
validation_status: Optional[MappingValidationStatus] = None
is_active: Optional[bool] = None
@@ -918,6 +917,7 @@ async def get_datasource_target_schemas(
@router.post("/mappings/propose")
async def propose_datasource_mapping(
payload: MappingProposeRequest,
db: AsyncSession = Depends(get_db),
current_user: User = Depends(get_current_user),
ai_client: AIProviderClient = Depends(get_ai_provider_client),
):

View File

@@ -7,6 +7,7 @@ from sqlalchemy import func, or_, select, text
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.logging import get_logger
from app.core.enums import JobStatus, SnapshotStatus
from app.core.time import to_iso8601_utc
from app.core.security import get_current_user
from app.core.data_sources import get_data_sources_config
@@ -581,7 +582,7 @@ async def get_running_task(db: AsyncSession, datasource_id: int) -> Optional[Col
result = await db.execute(
select(CollectionTask)
.where(CollectionTask.datasource_id == datasource_id)
.where(CollectionTask.status == "running")
.where(CollectionTask.status == JobStatus.RUNNING.value)
.order_by(CollectionTask.started_at.desc())
.limit(1)
)
@@ -609,8 +610,8 @@ async def get_running_task(db: AsyncSession, datasource_id: int) -> Optional[Col
f"Marked failed automatically after stale running timeout "
f"({STALE_RUNNING_TASK_TIMEOUT_MINUTES}m)"
)
task.status = "failed"
task.phase = "failed"
task.status = JobStatus.FAILED.value
task.phase = JobStatus.FAILED.value
task.completed_at = now
task.error_message = f"{existing_error}\n{stale_reason}".strip() if existing_error else stale_reason
await db.commit()
@@ -647,7 +648,7 @@ async def rollback_orphaned_running_task(
)
if snapshot is not None:
snapshot.status = "cancelled"
snapshot.status = SnapshotStatus.CANCELLED.value
snapshot.is_current = False
snapshot.completed_at = datetime.now(timezone.utc)
summary = dict(snapshot.summary or {})
@@ -670,13 +671,13 @@ async def rollback_orphaned_running_task(
{"snapshot_id": snapshot.parent_snapshot_id},
)
running_task.status = "cancelled"
running_task.phase = "cancelled"
running_task.status = JobStatus.CANCELLED.value
running_task.phase = JobStatus.CANCELLED.value
running_task.completed_at = datetime.now(timezone.utc)
existing_error = (running_task.error_message or "").strip()
cancel_reason = "Cancelled after backend restart because the running task handle was lost; incomplete writes rolled back"
running_task.error_message = f"{existing_error}\n{cancel_reason}".strip() if existing_error else cancel_reason
datasource.last_status = "cancelled"
datasource.last_status = JobStatus.CANCELLED.value
datasource.last_run_at = datetime.now(timezone.utc)
await db.commit()
@@ -711,7 +712,7 @@ async def fail_and_rollback_stale_running_task(
)
if snapshot is not None:
snapshot.status = "failed"
snapshot.status = SnapshotStatus.FAILED.value
snapshot.is_current = False
snapshot.completed_at = datetime.now(timezone.utc)
summary = dict(snapshot.summary or {})
@@ -739,11 +740,11 @@ async def fail_and_rollback_stale_running_task(
f"Marked failed automatically after stale running timeout "
f"({STALE_RUNNING_TASK_TIMEOUT_MINUTES}m); incomplete writes rolled back"
)
running_task.status = "failed"
running_task.phase = "failed"
running_task.status = JobStatus.FAILED.value
running_task.phase = JobStatus.FAILED.value
running_task.completed_at = datetime.now(timezone.utc)
running_task.error_message = f"{existing_error}\n{stale_reason}".strip() if existing_error else stale_reason
datasource.last_status = "failed"
datasource.last_status = JobStatus.FAILED.value
datasource.last_run_at = datetime.now(timezone.utc)
await db.commit()

View File

@@ -105,7 +105,7 @@ def _parse_layer_bbox(bbox: str) -> tuple[float, float, float, float]:
@router.get("/vessels/snapshot")
async def get_vessel_layer_snapshot(
bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"),
zoom: int = Query(..., ge=1, le=20),
zoom: float = Query(..., ge=1, le=20),
limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1),
vessel_type: Optional[str] = Query(None, alias="type"),
since_minutes: int = Query(60, ge=1, le=1440),

View File

@@ -12,6 +12,7 @@ from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.logging import get_logger
from app.core.enums import ProviderApi, TVSourceType, UserRole
from app.core.security import get_current_user
from app.core.time import to_iso8601_utc
from app.core.config import settings as app_settings
@@ -73,7 +74,7 @@ router = APIRouter()
logger = get_logger(__name__, service="api")
AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS = 5
AI_CONNECTION_TEST_PROMPT_KEY = "ai.connection_test"
SECRET_REVEAL_ROLES = {"admin", "super_admin"}
SECRET_REVEAL_ROLES = {UserRole.ADMIN.value, UserRole.SUPER_ADMIN.value}
DEFAULT_SETTINGS = {
"system": {
@@ -236,7 +237,7 @@ class TVStreamSourceUpdate(BaseModel):
provider: str = Field(default="Unknown", max_length=100)
region: str = Field(default="Global", max_length=100)
language: str = Field(default="und", max_length=32)
source_type: str = Field(default="iframe", pattern="^(iframe|hls|video|external|youtube)$")
source_type: TVSourceType = TVSourceType.IFRAME
embed_url: str = ""
stream_url: str = ""
homepage_url: str = ""
@@ -279,7 +280,7 @@ class AIProviderIntegrationUpdate(BaseModel):
service_token: Optional[str] = None
default_provider: Optional[str] = None
provider: str = Field(default="minimax", max_length=80)
provider_api: str = Field(default="anthropic-messages", max_length=80)
provider_api: ProviderApi = ProviderApi.ANTHROPIC_MESSAGES
base_url: str = Field(default="", max_length=500)
model: str = Field(default="", max_length=200)
api_key: Optional[str] = None
@@ -423,7 +424,7 @@ def _get_provider_preset(provider: str) -> dict:
except ValueError:
return {
"provider": provider,
"provider_api": "openai-completions",
"provider_api": ProviderApi.OPENAI_COMPLETIONS.value,
"base_url": "",
"model": "",
"models": [],
@@ -487,12 +488,12 @@ def _provider_defaults(provider: str) -> dict:
preset = _get_provider_preset(provider)
return {
"provider": provider,
"provider_api": preset.get("provider_api") or "openai-completions",
"provider_api": preset.get("provider_api") or ProviderApi.OPENAI_COMPLETIONS.value,
"base_url": preset.get("base_url") or "",
"model": preset.get("model") or "",
"api_key": "",
"max_tokens": (
1200 if preset.get("provider_api") == "anthropic-messages" else 4096
1200 if preset.get("provider_api") == ProviderApi.ANTHROPIC_MESSAGES.value else 4096
),
"anthropic_version": "2023-06-01",
"model_provider_apis": preset.get("model_provider_apis") or {},
@@ -665,7 +666,7 @@ def _runtime_config_from_ai_payload(ai_payload: dict) -> dict:
),
"llm_config": {
"provider": default_provider,
"provider_api": provider_config.get("provider_api") or "anthropic-messages",
"provider_api": provider_config.get("provider_api") or ProviderApi.ANTHROPIC_MESSAGES.value,
"base_url": provider_config.get("base_url") or "",
"model": provider_config.get("model") or "",
"api_key": api_key,
@@ -781,7 +782,10 @@ def _contains_model(model_ids: list[str], model: str) -> bool:
async def _check_ai_provider_lightweight(llm_config: dict, timeout_seconds: int) -> dict:
provider = _normalize_provider_id(llm_config.get("provider") or "")
configured_api = str(llm_config.get("provider_api") or "").strip() or "openai-completions"
configured_api = (
str(llm_config.get("provider_api") or "").strip()
or ProviderApi.OPENAI_COMPLETIONS.value
)
model = str(llm_config.get("model") or "").strip()
base_url = str(llm_config.get("base_url") or "").strip().rstrip("/")
api_key = str(llm_config.get("api_key") or "").strip()
@@ -802,7 +806,7 @@ async def _check_ai_provider_lightweight(llm_config: dict, timeout_seconds: int)
"message": "当前 provider/base_url/model 未完整配置。",
"mode": "lightweight_config",
}
if provider_api != "ollama-generate" and not api_key:
if provider_api != ProviderApi.OLLAMA_GENERATE.value and not api_key:
return {
"success": False,
"connected": False,
@@ -813,13 +817,13 @@ async def _check_ai_provider_lightweight(llm_config: dict, timeout_seconds: int)
if provider == "opencode-go":
url = _join_provider_url(base_url, "/models")
headers = {"Authorization": f"Bearer {api_key}"}
elif provider_api == "ollama-generate":
elif provider_api == ProviderApi.OLLAMA_GENERATE.value:
url = _join_provider_url(base_url, "/api/tags")
headers: dict[str, str] = {}
elif provider_api == "openai-completions":
elif provider_api == ProviderApi.OPENAI_COMPLETIONS.value:
url = _join_provider_url(base_url, "/models")
headers = {"Authorization": f"Bearer {api_key}"}
elif provider_api == "anthropic-messages":
elif provider_api == ProviderApi.ANTHROPIC_MESSAGES.value:
url = _join_provider_url(base_url, "/models")
headers = {
"x-api-key": api_key,
@@ -1165,7 +1169,7 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
api_key, api_key_source = _resolve_provider_api_key(provider_id, provider_config)
providers_payload[provider_id] = {
"provider": provider_id,
"provider_api": provider_config.get("provider_api") or "openai-completions",
"provider_api": provider_config.get("provider_api") or ProviderApi.OPENAI_COMPLETIONS.value,
"base_url": provider_config.get("base_url") or "",
"model": provider_config.get("model") or "",
"api_key": _mask_secret(api_key, api_key_source),
@@ -1215,7 +1219,7 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
"service_token": _mask_secret(*_resolve_service_token(normalized_ai)),
"default_provider": default_provider,
"provider": default_provider,
"provider_api": display_llm_config.get("provider_api") or "anthropic-messages",
"provider_api": display_llm_config.get("provider_api") or ProviderApi.ANTHROPIC_MESSAGES.value,
"base_url": display_llm_config.get("base_url") or "https://api.minimaxi.com/anthropic",
"model": display_llm_config.get("model") or "MiniMax-M2.7",
"api_key": display_llm_config.get("api_key") or _mask_secret(None),
@@ -1458,7 +1462,7 @@ async def update_smtp_settings(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
if current_user.role not in ("admin", "super_admin"):
if current_user.role not in (UserRole.ADMIN.value, UserRole.SUPER_ADMIN.value):
raise HTTPException(status_code=403, detail="Only administrators can change SMTP settings")
current = await get_setting_payload(db, "smtp")
merged = _build_smtp_payload(current, payload)
@@ -1472,7 +1476,7 @@ async def test_smtp_settings(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
if current_user.role not in ("admin", "super_admin"):
if current_user.role not in (UserRole.ADMIN.value, UserRole.SUPER_ADMIN.value):
raise HTTPException(status_code=403, detail="Only administrators can test SMTP settings")
from app.services.email import EmailError, send_email

View File

@@ -5,6 +5,7 @@ from fastapi import APIRouter, Depends, HTTPException, status
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import text
from app.core.enums import UserRole
from app.core.security import get_current_user, get_password_hash
from app.db.session import get_db
from app.models.user import User
@@ -13,6 +14,8 @@ from app.schemas.user import UserCreate, UserUpdate
router = APIRouter()
VALID_GATEKEEPER_GROUPS = {"docs_user", "docs_developer", "docs_admin"}
ADMIN_ROLES = [UserRole.SUPER_ADMIN.value, UserRole.ADMIN.value]
SUPER_ADMIN_ROLES = [UserRole.SUPER_ADMIN.value]
def check_permission(current_user: User, required_roles: List[str]) -> bool:
@@ -32,7 +35,7 @@ async def list_users(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
if not check_permission(current_user, ["super_admin", "admin"]):
if not check_permission(current_user, ADMIN_ROLES):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Insufficient permissions",
@@ -91,7 +94,7 @@ async def get_user(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
if not check_permission(current_user, ["super_admin", "admin"]) and current_user.id != user_id:
if not check_permission(current_user, ADMIN_ROLES) and current_user.id != user_id:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Insufficient permissions",
@@ -128,7 +131,7 @@ async def create_user(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
if not check_permission(current_user, ["super_admin"]):
if not check_permission(current_user, SUPER_ADMIN_ROLES):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only super_admin can create users",
@@ -196,18 +199,18 @@ async def update_user(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
if not check_permission(current_user, ["super_admin", "admin"]) and current_user.id != user_id:
if not check_permission(current_user, ADMIN_ROLES) and current_user.id != user_id:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Insufficient permissions",
)
if not check_permission(current_user, ["super_admin"]) and user_data.role is not None:
if not check_permission(current_user, SUPER_ADMIN_ROLES) and user_data.role is not None:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only super_admin can change user role",
)
if not check_permission(current_user, ["super_admin"]) and user_data.gatekeeper_groups is not None:
if not check_permission(current_user, SUPER_ADMIN_ROLES) and user_data.gatekeeper_groups is not None:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only super_admin can change Gatekeeper groups",
@@ -260,7 +263,7 @@ async def delete_user(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
if not check_permission(current_user, ["super_admin"]):
if not check_permission(current_user, SUPER_ADMIN_ROLES):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="Only super_admin can delete users",

View File

@@ -1,4 +1,4 @@
"""Bounded vessel snapshot APIs for viewport-first consumers."""
"""Bounded vessel snapshot APIs backed by the latest vessel state table."""
from typing import Optional
@@ -14,8 +14,8 @@ router = APIRouter()
@router.get("/snapshot")
async def get_vessel_snapshot(
bbox: Optional[str] = Query(None, description="Viewport bbox as lon_min,lat_min,lon_max,lat_max"),
zoom: int = Query(..., ge=1, le=20, description="Current map zoom level"),
bbox: Optional[str] = Query(None, description="Snapshot bbox as lon_min,lat_min,lon_max,lat_max"),
zoom: float = Query(..., ge=1, le=20, description="Current map zoom level"),
type: Optional[str] = Query(
None,
description="Comma-separated vessel types: cargo,tanker,passenger,fishing,military,other",

View File

@@ -18,6 +18,7 @@ from sqlalchemy import select, func
from typing import List, Dict, Any, Optional
from app.core.collected_data_fields import get_record_field
from app.core.enums import BGPStatus
from app.core.satellite_tle import build_tle_lines_from_elements
from app.core.time import to_iso8601_utc
from app.db.session import get_db
@@ -25,7 +26,7 @@ from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_incident import BGPIncident
from app.models.bgp_observation import BGPObservation
from app.models.collected_data import CollectedData
from app.models.vessel import AISSourceHealth, VesselPosition, VesselStatic
from app.models.vessel import AISSourceHealth, VesselCurrentState, VesselPosition, VesselStatic
from app.services.bgp_collectors import build_bgp_collector_coverage
from app.services.cable_graph import build_graph_from_data, CableGraph, haversine_distance
from app.services.compute_center_locations import (
@@ -47,11 +48,10 @@ from app.services.location.llm_fallback import (
from app.services.persistent_logs import record_system_log
from app.services.vessel_ais_aggregation import (
build_field_conflict_candidates,
count_unique_raw_vessel_mmsi,
get_aggregated_vessel,
get_aggregated_vessel_track,
get_aggregated_vessels,
get_aggregated_vessels_snapshot,
get_current_vessels_snapshot,
get_vessel_conflict_records,
get_vessel_raw_observations,
MAX_SNAPSHOT_LIMIT,
@@ -75,7 +75,6 @@ TERRAIN_TILE_BATCH_MAX_ITEMS = 128
TERRAIN_TILE_BATCH_CONCURRENCY = 16
_terrain_tile_cache: OrderedDict[tuple[int, int, int], tuple[bytes, str, dict[str, str]]] = OrderedDict()
VESSEL_NAME_FALLBACK_PATTERN = re.compile(r"^mmsi\s*\d+$", re.IGNORECASE)
VESSEL_SNAPSHOT_LEGACY_FALLBACK_ENABLED = True
SECONDS_PER_MINUTE = 60
BYTES_PER_MIB = 1024 * 1024
CABLE_CACHE_FRESH_SECONDS = 6 * 60 * SECONDS_PER_MINUTE
@@ -839,9 +838,14 @@ def convert_aggregated_vessels_to_geojson(vessels: List[dict[str, Any]]) -> Dict
continue
source_summary = {}
for source, summary in (vessel.get("source_summary") or {}).items():
latest_observed_at = summary.get("latest_observed_at")
source_summary[source] = {
**summary,
"latest_observed_at": to_iso8601_utc(summary.get("latest_observed_at")),
"latest_observed_at": (
to_iso8601_utc(latest_observed_at)
if isinstance(latest_observed_at, datetime)
else latest_observed_at
),
}
props = {
"mmsi": vessel["mmsi"],
@@ -1075,7 +1079,7 @@ async def build_vessel_snapshot_response(
db: AsyncSession,
*,
bbox: tuple[float, float, float, float] | None,
zoom: int | None,
zoom: float | None,
type_filter: str | None,
limit: int | None,
since_minutes: int = 60,
@@ -2340,58 +2344,35 @@ async def _load_raw_vessel_snapshot_features(
observed_since: datetime,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
if bbox is None:
aggregated_vessels = await get_aggregated_vessels(
db,
limit=limit,
observed_since=observed_since,
)
else:
aggregated_vessels = await get_aggregated_vessels_snapshot(
db,
bbox=bbox,
limit=limit,
observed_since=observed_since,
)
raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels)
raw_features = raw_geojson.get("features", [])
features = raw_features
legacy_features: list[dict[str, Any]] = []
legacy_fallback_used = False
if not raw_features and VESSEL_SNAPSHOT_LEGACY_FALLBACK_ENABLED:
legacy_features = await _load_legacy_vessel_snapshot_features(
db,
bbox=bbox,
limit=limit,
)
features, _merge_diagnostics = _merge_vessel_features(raw_features, legacy_features)
legacy_fallback_used = bool(legacy_features)
return [], {
"source": "vessel_current_state",
"current_state_count": 0,
"final_unique_mmsi": 0,
}
current_vessels = await get_current_vessels_snapshot(
db,
bbox=bbox,
limit=limit,
observed_since=observed_since,
)
features = convert_aggregated_vessels_to_geojson(current_vessels).get("features", [])
unique_mmsi = len(
{
key
for key in (_feature_mmsi_key(feature) for feature in features)
if key is not None
}
)
return features, {
"raw_feature_count": len(raw_features),
"raw_unique_mmsi": len(
{
key
for key in (_feature_mmsi_key(feature) for feature in raw_features)
if key is not None
}
),
"legacy_feature_count": len(legacy_features),
"legacy_backfilled_mmsi": len(
{
key
for key in (_feature_mmsi_key(feature) for feature in legacy_features)
if key is not None
}
),
"legacy_fallback_enabled": VESSEL_SNAPSHOT_LEGACY_FALLBACK_ENABLED,
"legacy_fallback_used": legacy_fallback_used,
"final_unique_mmsi": len(
{
key
for key in (_feature_mmsi_key(feature) for feature in features)
if key is not None
}
),
"source": "vessel_current_state",
"current_state_count": len(features),
"final_unique_mmsi": unique_mmsi,
"raw_feature_count": 0,
"raw_unique_mmsi": 0,
"legacy_feature_count": 0,
"legacy_backfilled_mmsi": 0,
"legacy_fallback_enabled": False,
"legacy_fallback_used": False,
}
@router.get("/vessels/custom-supplements")
@@ -2759,10 +2740,10 @@ async def _build_visualization_geo_summary(db: AsyncSession) -> dict[str, Any]:
compute_center_count = supercomputer_count + gpu_cluster_count
active_incident_result = await db.execute(
select(func.count(BGPIncident.id)).where(BGPIncident.status == "active"),
select(func.count(BGPIncident.id)).where(BGPIncident.status == BGPStatus.ACTIVE.value),
)
active_anomaly_result = await db.execute(
select(func.count(BGPAnomaly.id)).where(BGPAnomaly.status == "active"),
select(func.count(BGPAnomaly.id)).where(BGPAnomaly.status == BGPStatus.ACTIVE.value),
)
active_incident_count = int(active_incident_result.scalar() or 0)
active_anomaly_count = int(active_anomaly_result.scalar() or 0)
@@ -2783,21 +2764,14 @@ async def _build_visualization_geo_summary(db: AsyncSession) -> dict[str, Any]:
)
else:
bgp_collector_count = int(bgp_collector_scalar or 0)
raw_unique_window_hours = 24
raw_unique_mmsi = await count_unique_raw_vessel_mmsi(
db,
observed_since=datetime.now(UTC) - timedelta(hours=raw_unique_window_hours),
vessel_current_window_minutes = 60
vessel_current_result = await db.execute(
select(func.count(VesselCurrentState.mmsi)).where(
VesselCurrentState.observed_at
>= datetime.now(UTC) - timedelta(minutes=vessel_current_window_minutes)
)
)
legacy_unique_result = await db.execute(
select(func.count(func.distinct(VesselPosition.mmsi)))
)
legacy_unique_mmsi = int(legacy_unique_result.scalar() or 0)
legacy_fallback_active = (
VESSEL_SNAPSHOT_LEGACY_FALLBACK_ENABLED
and raw_unique_mmsi == 0
and legacy_unique_mmsi > 0
)
vessel_count = legacy_unique_mmsi if legacy_fallback_active else raw_unique_mmsi
vessel_count = int(vessel_current_result.scalar() or 0)
aisstream_health = await db.get(AISSourceHealth, "aisstream_vessels")
return {
@@ -2808,11 +2782,10 @@ async def _build_visualization_geo_summary(db: AsyncSession) -> dict[str, Any]:
"satellite_count": satellite_count,
"compute_center_count": compute_center_count,
"vessel_count": vessel_count,
"vessel_count_source": "legacy_fallback" if legacy_fallback_active else "raw_recent",
"vessel_legacy_fallback_enabled": VESSEL_SNAPSHOT_LEGACY_FALLBACK_ENABLED,
"vessel_raw_unique_mmsi": raw_unique_mmsi,
"vessel_raw_unique_window_hours": raw_unique_window_hours,
"vessel_legacy_unique_mmsi": legacy_unique_mmsi,
"vessel_count_source": "vessel_current_state",
"vessel_current_window_minutes": vessel_current_window_minutes,
"vessel_raw_unique_mmsi": 0,
"vessel_legacy_unique_mmsi": 0,
"aisstream_connection_state": aisstream_health.connection_state if aisstream_health else None,
"aisstream_last_seen_at": to_iso8601_utc(aisstream_health.last_seen_at) if aisstream_health else None,
"aisstream_message_rate": aisstream_health.message_rate if aisstream_health else None,

View File

@@ -9,6 +9,7 @@ from jose import jwt, JWTError
from sqlalchemy import text
from app.core.config import settings
from app.core.enums import UserRole
from app.core.logging import get_logger
from app.core.time import to_iso8601_utc
from app.core.websocket.manager import manager
@@ -96,7 +97,7 @@ async def websocket_endpoint(
"earth_news",
EARTH_UPDATES_CHANNEL,
]
if user_role == "super_admin":
if user_role == UserRole.SUPER_ADMIN.value:
supported_channels = [*supported_channels, LOG_TAIL_CHANNEL]
await manager.connect(websocket, user_id)
@@ -138,7 +139,7 @@ async def websocket_endpoint(
if channel and channel not in channels:
channels = [*channels, channel]
if LOG_TAIL_CHANNEL in channels:
if user_role != "super_admin":
if user_role != UserRole.SUPER_ADMIN.value:
await websocket.send_json(
{
"type": "subscription_error",