linkong/planet
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

release: bump version to 0.52.0

Browse Source
This commit is contained in:
linkong
2026-05-12 17:15:02 +08:00
parent b15d097b9c
commit b87cb310fd
70 changed files with 5589 additions and 2187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
backend/app/api/main.py
View File

@@ -8,12 +8,14 @@ from app.api.v1 import (
docs,
tasks,
dashboard,
websocket,
alerts,
settings,
collected_data,
data_products,
layers,
visualization,
vessel_aggregation,
vessels,
bgp,
news,
system_control,
@@ -36,12 +38,15 @@ api_router.include_router(dashboard.router, prefix="/dashboard", tags=["dashboar
api_router.include_router(alerts.router, prefix="/alerts", tags=["alerts"])
api_router.include_router(settings.router, prefix="/settings", tags=["settings"])
api_router.include_router(system_control.router, prefix="/system", tags=["system"])
api_router.include_router(data_products.router, prefix="/data-products", tags=["data-products"])
api_router.include_router(layers.router, prefix="/layers", tags=["layers"])
api_router.include_router(visualization.router, prefix="/visualization", tags=["visualization"])
api_router.include_router(
vessel_aggregation.router,
prefix="/vessel-aggregation",
tags=["vessel-aggregation"],
)
api_router.include_router(vessels.router, prefix="/vessels", tags=["vessels"])
api_router.include_router(bgp.router, prefix="/bgp", tags=["bgp"])
api_router.include_router(tv.router, prefix="/tv", tags=["tv"])
api_router.include_router(news.router, prefix="/news", tags=["news"])

269
backend/app/api/v1/auth.py
View File

@@ -1,26 +1,85 @@
from datetime import timedelta
from fastapi import APIRouter, Depends, HTTPException, status
from fastapi.security import OAuth2PasswordRequestForm
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy import text
from app.core.config import settings
from app.core.logging import get_logger
from app.core.security import (
create_access_token,
create_refresh_token,
blacklist_token,
get_current_user,
get_password_hash,
verify_password,
)
from app.db.session import get_db
from app.models.user import User
from app.schemas.token import Token
from app.schemas.user import UserCreate, UserResponse
from app.schemas.user import (
ForgotPasswordRequest,
ResendCodeRequest,
ResetPasswordRequest,
UserRegister,
UserResponse,
VerifyEmailRequest,
)
from app.services import otp
from app.services.email import (
EmailError,
EmailNotConfiguredError,
send_verification_email,
)
logger = get_logger(__name__)
router = APIRouter()
def _token_response(user: User) -> dict:
access_token = create_access_token(data={"sub": user.id})
refresh = create_refresh_token(data={"sub": user.id})
expires_in = (
settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60
if settings.ACCESS_TOKEN_EXPIRE_MINUTES > 0
else None
)
return {
"access_token": access_token,
"token_type": "bearer",
"expires_in": expires_in,
"refresh_token": refresh,
"user": {
"id": user.id,
"username": user.username,
"role": user.role,
"gatekeeper_groups": user.gatekeeper_groups or [],
},
}
async def _load_user_by_email(db: AsyncSession, email: str) -> User | None:
result = await db.execute(
text(
"SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups, email_verified "
"FROM users WHERE email = :email"
),
{"email": email},
)
row = result.fetchone()
if row is None:
return None
user = User()
user.id = row[0]
user.username = row[1]
user.email = row[2]
user.password_hash = row[3]
user.role = row[4]
user.is_active = row[5]
user.gatekeeper_groups = row[6] or []
user.email_verified = bool(row[7])
return user
@router.post("/login", response_model=Token)
async def login(
form_data: OAuth2PasswordRequestForm = Depends(),
@@ -28,7 +87,8 @@ async def login(
):
result = await db.execute(
text(
"SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups FROM users WHERE username = :username"
"SELECT id, username, email, password_hash, role, is_active, gatekeeper_groups, email_verified "
"FROM users WHERE username = :username"
),
{"username": form_data.username},
)
@@ -47,6 +107,7 @@ async def login(
user.role = row[4]
user.is_active = row[5]
user.gatekeeper_groups = row[6] or []
user.email_verified = bool(row[7])
if not verify_password(form_data.password, user.password_hash):
raise HTTPException(
@@ -58,25 +119,13 @@ async def login(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="User is inactive",
)
if not user.email_verified:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail={"code": "EMAIL_NOT_VERIFIED", "email": user.email},
)
access_token = create_access_token(data={"sub": user.id})
refresh_token = create_refresh_token(data={"sub": user.id})
expires_in = None
if settings.ACCESS_TOKEN_EXPIRE_MINUTES > 0:
expires_in = settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60
return {
"access_token": access_token,
"token_type": "bearer",
"expires_in": expires_in,
"user": {
"id": user.id,
"username": user.username,
"role": user.role,
"gatekeeper_groups": user.gatekeeper_groups or [],
},
}
return _token_response(user)
@router.post("/refresh", response_model=Token)
@@ -116,5 +165,179 @@ async def get_me(current_user: User = Depends(get_current_user)):
"role": current_user.role,
"gatekeeper_groups": current_user.gatekeeper_groups or [],
"is_active": current_user.is_active,
"email_verified": getattr(current_user, "email_verified", True),
"created_at": current_user.created_at,
}
async def _send_code_or_raise(db: AsyncSession, email: str, code: str, purpose: str) -> None:
try:
await send_verification_email(db, to=email, code=code, purpose=purpose)
except EmailNotConfiguredError as exc:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail={"code": exc.code, "message": str(exc)},
) from exc
except EmailError as exc:
logger.warning_event(
"SMTP send failed",
event="auth.email.send_failed",
context={"email": email, "purpose": purpose, "error": str(exc)},
)
raise HTTPException(
status_code=status.HTTP_502_BAD_GATEWAY,
detail={"code": exc.code, "message": str(exc)},
) from exc
@router.post("/register", status_code=status.HTTP_201_CREATED)
async def register(payload: UserRegister, db: AsyncSession = Depends(get_db)):
existing = await db.execute(
text("SELECT id, email_verified FROM users WHERE username = :u OR email = :e"),
{"u": payload.username, "e": payload.email},
)
row = existing.fetchone()
if row is not None:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail={"code": "USER_ALREADY_EXISTS", "message": "Username or email already in use"},
)
user = User(
username=payload.username,
email=payload.email,
password_hash=get_password_hash(payload.password),
role="viewer",
is_active=True,
email_verified=False,
)
db.add(user)
await db.commit()
try:
code = otp.issue_code(payload.email, "register")
except otp.OtpResendRateLimited as exc:
raise HTTPException(
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
detail={"code": exc.code, "retry_after_seconds": exc.retry_after_seconds},
) from exc
await _send_code_or_raise(db, payload.email, code, "register")
return {"status": "pending_verification", "email": payload.email}
@router.post("/verify-email", response_model=Token)
async def verify_email(payload: VerifyEmailRequest, db: AsyncSession = Depends(get_db)):
user = await _load_user_by_email(db, payload.email)
if user is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail={"code": "USER_NOT_FOUND"},
)
try:
otp.verify_code(payload.email, "register", payload.code)
except otp.OtpExpired as exc:
raise HTTPException(
status_code=status.HTTP_410_GONE,
detail={"code": exc.code, "message": str(exc)},
) from exc
except otp.OtpAttemptsExceeded as exc:
raise HTTPException(
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
detail={"code": exc.code, "message": str(exc)},
) from exc
except otp.OtpInvalid as exc:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail={"code": exc.code, "message": str(exc)},
) from exc
await db.execute(
text("UPDATE users SET email_verified = TRUE WHERE id = :id"),
{"id": user.id},
)
await db.commit()
user.email_verified = True
return _token_response(user)
@router.post("/resend-code")
async def resend_code(payload: ResendCodeRequest, db: AsyncSession = Depends(get_db)):
user = await _load_user_by_email(db, payload.email)
if user is None:
# Avoid email enumeration; pretend success.
return {"status": "ok"}
if payload.purpose == "register" and user.email_verified:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail={"code": "ALREADY_VERIFIED"},
)
try:
code = otp.issue_code(payload.email, payload.purpose)
except otp.OtpResendRateLimited as exc:
raise HTTPException(
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
detail={"code": exc.code, "retry_after_seconds": exc.retry_after_seconds},
) from exc
await _send_code_or_raise(db, payload.email, code, payload.purpose)
return {"status": "ok"}
@router.post("/forgot-password")
async def forgot_password(payload: ForgotPasswordRequest, db: AsyncSession = Depends(get_db)):
user = await _load_user_by_email(db, payload.email)
if user is None:
# Don't leak whether an email is registered.
return {"status": "ok"}
try:
code = otp.issue_code(payload.email, "reset_password")
except otp.OtpResendRateLimited:
# Silently accept; the user can retry after the cooldown.
return {"status": "ok"}
try:
await send_verification_email(db, to=payload.email, code=code, purpose="reset_password")
except EmailNotConfiguredError as exc:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail={"code": exc.code, "message": str(exc)},
) from exc
except EmailError as exc:
logger.warning_event(
"SMTP send failed",
event="auth.email.send_failed",
context={"email": payload.email, "purpose": "reset_password", "error": str(exc)},
)
return {"status": "ok"}
@router.post("/reset-password")
async def reset_password(payload: ResetPasswordRequest, db: AsyncSession = Depends(get_db)):
user = await _load_user_by_email(db, payload.email)
if user is None:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail={"code": "OTP_INVALID"},
)
try:
otp.verify_code(payload.email, "reset_password", payload.code)
except otp.OtpExpired as exc:
raise HTTPException(
status_code=status.HTTP_410_GONE,
detail={"code": exc.code, "message": str(exc)},
) from exc
except otp.OtpAttemptsExceeded as exc:
raise HTTPException(
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
detail={"code": exc.code, "message": str(exc)},
) from exc
except otp.OtpInvalid as exc:
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail={"code": exc.code, "message": str(exc)},
) from exc
await db.execute(
text("UPDATE users SET password_hash = :p, email_verified = TRUE WHERE id = :id"),
{"p": get_password_hash(payload.new_password), "id": user.id},
)
await db.commit()
return {"status": "ok"}

98
backend/app/api/v1/data_products.py Normal file
View File

@@ -0,0 +1,98 @@
from datetime import UTC, datetime
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.v1.visualization import get_visualization_geo_summary
from app.core.time import to_iso8601_utc
from app.db.session import get_db
router = APIRouter()
PRODUCT_DEFINITIONS: dict[str, dict] = {
"vessels": {
"name": "船只",
"sources": ["aisstream_vessels", "barentswatch_vessels"],
"primary_stat_key": "vessel_count",
"stat_keys": ["vessel_count", "vessel_raw_unique_mmsi", "vessel_legacy_unique_mmsi"],
},
"cables": {
"name": "海底光缆",
"sources": [
"arcgis_cables",
"arcgis_landing_points",
"arcgis_cable_landing_relation",
"telegeography_cables",
"telegeography_landing",
"telegeography_systems",
"fao_landing_points",
],
"primary_stat_key": "cable_count",
"stat_keys": ["cable_count", "landing_point_count"],
},
"satellites": {
"name": "卫星",
"sources": ["celestrak_tle", "spacetrack_tle"],
"primary_stat_key": "satellite_count",
"stat_keys": ["satellite_count"],
},
"bgp": {
"name": "BGP",
"sources": [
"ris_live_bgp",
"bgpstream_bgp",
"iptoasn_prefix_geo",
"opengeofeed_prefix_geo",
"nro_delegated_prefix_geo",
],
"primary_stat_key": "bgp_event_count",
"stat_keys": ["bgp_event_count", "bgp_incident_count", "bgp_anomaly_count", "bgp_collector_count"],
},
"compute": {
"name": "算力",
"sources": ["top500", "epoch_ai_gpu"],
"primary_stat_key": "compute_center_count",
"stat_keys": ["compute_center_count", "supercomputer_count", "gpu_cluster_count"],
},
}
def _build_product_status(product_id: str, summary: dict) -> dict:
definition = PRODUCT_DEFINITIONS[product_id]
stats = summary.get("stats", {})
product_stats = {key: stats.get(key, 0) for key in definition["stat_keys"]}
total_count = int(product_stats.get(definition["primary_stat_key"]) or 0)
return {
"product_id": product_id,
"name": definition["name"],
"sources": definition["sources"],
"generated_at": summary.get("generated_at") or to_iso8601_utc(datetime.now(UTC)),
"total_count": total_count,
"stats": product_stats,
"build_state": "ready",
"stats_scope": "global",
"stats_freshness": "cached_or_indexed",
}
@router.get("")
async def list_data_products(db: AsyncSession = Depends(get_db)):
summary = await get_visualization_geo_summary(db)
return {
"generated_at": summary.get("generated_at"),
"data": [
_build_product_status(product_id, summary)
for product_id in PRODUCT_DEFINITIONS
],
}
@router.get("/{product_id}/status")
async def get_data_product_status(
product_id: str,
db: AsyncSession = Depends(get_db),
):
if product_id not in PRODUCT_DEFINITIONS:
raise HTTPException(status_code=404, detail="Unknown data product")
summary = await get_visualization_geo_summary(db)
return _build_product_status(product_id, summary)

403
backend/app/api/v1/datasources.py
View File

@@ -3,7 +3,8 @@ from datetime import datetime, timedelta, timezone
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import func, select, text
from pydantic import BaseModel, Field
from sqlalchemy import func, or_, select, text
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.time import to_iso8601_utc
@@ -27,6 +28,29 @@ from app.services.scheduler import (
router = APIRouter()
STALE_RUNNING_TASK_TIMEOUT_MINUTES = 90
PRODUCT_SOURCE_KEYWORDS: tuple[tuple[str, tuple[str, ...]], ...] = (
("vessels", ("vessel", "ais")),
("cables", ("cable", "landing", "telegeography", "arcgis", "fao")),
("satellites", ("tle", "satellite", "spacetrack", "celestrak")),
("bgp", ("bgp", "asn", "prefix_geo", "opengeofeed", "nro")),
("compute", ("top500", "gpu", "supercomputer", "compute")),
("ai", ("huggingface", "epoch_ai")),
("media", ("news", "tv", "live_stream")),
)
class DatasourceBatchTriggerRequest(BaseModel):
source_ids: list[int] = Field(default_factory=list)
force: bool = False
module: Optional[str] = None
product: Optional[str] = None
is_active: Optional[bool] = None
priority: Optional[str] = None
run_status: Optional[str] = None
collected: Optional[bool] = None
credential_status: Optional[str] = None
q: Optional[str] = None
def format_frequency_label(minutes: int) -> str:
if minutes % 1440 == 0:
@@ -47,6 +71,20 @@ def datasource_metadata(source: str) -> dict:
}
def datasource_product_key(datasource: DataSource) -> str:
haystack = " ".join(
[
datasource.source or "",
datasource.name or "",
datasource.collector_class or "",
]
).lower()
for product, keywords in PRODUCT_SOURCE_KEYWORDS:
if any(keyword in haystack for keyword in keywords):
return product
return "other"
def is_due_for_collection(datasource: DataSource, now: datetime) -> bool:
if datasource.last_run_at is None:
return True
@@ -110,6 +148,22 @@ async def _load_latest_task_ids(
return {datasource_id: task_id for datasource_id, task_id in result.all()}
async def _load_collected_record_counts(
db: AsyncSession,
sources: list[str],
) -> dict[str, int]:
if not sources:
return {}
result = await db.execute(
select(CollectedData.source, func.count(CollectedData.id))
.where(CollectedData.source.in_(sources))
.where(CollectedData.is_current.is_(True))
.group_by(CollectedData.source)
)
return {source: int(count or 0) for source, count in result.all()}
async def _load_datasource_endpoint_overrides(
db: AsyncSession,
sources: list[str],
@@ -161,6 +215,192 @@ async def _load_datasource_list_context(
return running_tasks, endpoint_overrides
def _apply_datasource_query_filters(
query,
*,
module: Optional[str] = None,
is_active: Optional[bool] = None,
priority: Optional[str] = None,
run_status: Optional[str] = None,
q: Optional[str] = None,
) -> object:
if module:
query = query.where(DataSource.module == module)
if is_active is not None:
query = query.where(DataSource.is_active == is_active)
if priority:
query = query.where(DataSource.priority == priority)
if run_status and run_status not in {"running", "collected", "uncollected"}:
if run_status == "not_run":
query = query.where(DataSource.last_status.is_(None))
else:
query = query.where(DataSource.last_status == run_status)
if q:
like_value = f"%{q.strip()}%"
query = query.where(
or_(
DataSource.name.ilike(like_value),
DataSource.source.ilike(like_value),
DataSource.collector_class.ilike(like_value),
)
)
return query
def _filter_datasources_in_memory(
datasources: list[DataSource],
*,
running_tasks: dict[int, CollectionTask],
record_counts: dict[str, int],
product: Optional[str] = None,
run_status: Optional[str] = None,
collected: Optional[bool] = None,
credential_status: Optional[str] = None,
) -> list[DataSource]:
filtered: list[DataSource] = []
for datasource in datasources:
record_count = record_counts.get(datasource.source, 0)
if product and datasource_product_key(datasource) != product:
continue
if collected is not None and (record_count > 0) != collected:
continue
if credential_status:
metadata = datasource_metadata(datasource.source)
if metadata["credential_status"] != credential_status:
continue
if run_status == "running" and datasource.id not in running_tasks:
continue
if run_status == "collected" and record_count <= 0:
continue
if run_status == "uncollected" and record_count > 0:
continue
filtered.append(datasource)
return filtered
async def _trigger_datasource_batch(
db: AsyncSession,
datasources: list[DataSource],
*,
force: bool,
) -> dict:
if not datasources:
return {
"status": "noop",
"message": "No matching data sources to trigger",
"force": force,
"triggered": [],
"skipped": [],
"failed": [],
}
previous_task_ids: dict[int, Optional[int]] = {}
triggered_sources: list[dict] = []
skipped_sources: list[dict] = []
failed_sources: list[dict] = []
now = datetime.now(timezone.utc)
running_tasks = await _load_latest_running_tasks(
db,
[datasource.id for datasource in datasources],
)
for datasource in datasources:
if not datasource.is_active:
skipped_sources.append(
{
"id": datasource.id,
"source": datasource.source,
"name": datasource.name,
"reason": "disabled",
}
)
continue
running_task = running_tasks.get(datasource.id)
if running_task is not None:
if not force:
skipped_sources.append(
{
"id": datasource.id,
"source": datasource.source,
"name": datasource.name,
"reason": "already_running",
"task_id": running_task.id,
}
)
continue
cancelled = await cancel_running_collector_now(datasource.source)
if not cancelled:
await rollback_orphaned_running_task(db, datasource, running_task)
if not force and not is_due_for_collection(datasource, now):
skipped_sources.append(
{
"id": datasource.id,
"source": datasource.source,
"name": datasource.name,
"reason": "within_frequency_window",
"last_run_at": to_iso8601_utc(datasource.last_run_at),
"next_run_at": to_iso8601_utc(
datasource.last_run_at + timedelta(minutes=datasource.frequency_minutes)
),
}
)
continue
previous_task_ids[datasource.id] = None
success = run_collector_now(datasource.source)
if not success:
failed_sources.append(
{
"id": datasource.id,
"source": datasource.source,
"name": datasource.name,
"reason": "trigger_failed",
}
)
continue
triggered_sources.append(
{
"id": datasource.id,
"source": datasource.source,
"name": datasource.name,
"task_id": None,
}
)
latest_task_ids = await _load_latest_task_ids(
db,
[datasource.id for datasource in datasources],
)
for datasource_id in previous_task_ids:
previous_task_ids[datasource_id] = latest_task_ids.get(datasource_id)
for _ in range(20):
await asyncio.sleep(0.1)
pending = [item for item in triggered_sources if item["task_id"] is None]
if not pending:
break
latest_task_ids = await _load_latest_task_ids(
db,
[item["id"] for item in pending],
)
for item in pending:
task_id = latest_task_ids.get(item["id"])
if task_id is not None and task_id != previous_task_ids.get(item["id"]):
item["task_id"] = task_id
return {
"status": "triggered" if triggered_sources else "partial",
"message": f"Triggered {len(triggered_sources)} data sources",
"force": force,
"triggered": triggered_sources,
"skipped": skipped_sources,
"failed": failed_sources,
}
async def get_datasource_record(db: AsyncSession, source_id: str) -> Optional[DataSource]:
datasource = None
try:
@@ -355,16 +595,23 @@ async def list_datasources(
module: Optional[str] = None,
is_active: Optional[bool] = None,
priority: Optional[str] = None,
product: Optional[str] = None,
run_status: Optional[str] = None,
collected: Optional[bool] = None,
credential_status: Optional[str] = None,
q: Optional[str] = None,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
query = select(DataSource).order_by(DataSource.module, DataSource.id)
if module:
query = query.where(DataSource.module == module)
if is_active is not None:
query = query.where(DataSource.is_active == is_active)
if priority:
query = query.where(DataSource.priority == priority)
query = _apply_datasource_query_filters(
query,
module=module,
is_active=is_active,
priority=priority,
run_status=run_status,
q=q,
)
result = await db.execute(query)
datasources = result.scalars().all()
@@ -372,11 +619,22 @@ async def list_datasources(
collector_list = []
config = get_data_sources_config()
running_tasks, endpoint_overrides = await _load_datasource_list_context(db, datasources)
record_counts = await _load_collected_record_counts(db, [datasource.source for datasource in datasources])
datasources = _filter_datasources_in_memory(
datasources,
running_tasks=running_tasks,
record_counts=record_counts,
product=product,
run_status=run_status,
collected=collected,
credential_status=credential_status,
)
for datasource in datasources:
running_task = running_tasks.get(datasource.id)
endpoint = endpoint_overrides.get(datasource.source) or config.get_yaml_url(datasource.source)
last_run_at = datasource.last_run_at
last_status = datasource.last_status
collected_records = record_counts.get(datasource.source, 0)
collector_list.append(
{
@@ -384,6 +642,7 @@ async def list_datasources(
"source": datasource.source,
"name": datasource.name,
**datasource_metadata(datasource.source),
"product": datasource_product_key(datasource),
"module": datasource.module,
"priority": datasource.priority,
"frequency": format_frequency_label(datasource.frequency_minutes),
@@ -405,6 +664,8 @@ async def list_datasources(
"phase_unit": running_task.phase_unit if running_task else None,
"records_processed": running_task.records_processed if running_task else None,
"total_records": running_task.total_records if running_task else None,
"collected_records": collected_records,
"has_collected_data": collected_records > 0,
}
)
@@ -419,110 +680,46 @@ async def trigger_all_datasources(
):
result = await db.execute(
select(DataSource)
.where(DataSource.is_active == True)
.where(DataSource.is_active.is_(True))
.order_by(DataSource.module, DataSource.id)
)
datasources = result.scalars().all()
return await _trigger_datasource_batch(db, datasources, force=force)
if not datasources:
return {
"status": "noop",
"message": "No active data sources to trigger",
"triggered": [],
"skipped": [],
"failed": [],
}
previous_task_ids: dict[int, Optional[int]] = {}
triggered_sources: list[dict] = []
skipped_sources: list[dict] = []
failed_sources: list[dict] = []
now = datetime.now(timezone.utc)
running_tasks = await _load_latest_running_tasks(
db,
[datasource.id for datasource in datasources],
)
for datasource in datasources:
running_task = running_tasks.get(datasource.id)
if running_task is not None:
skipped_sources.append(
{
"id": datasource.id,
"source": datasource.source,
"name": datasource.name,
"reason": "already_running",
"task_id": running_task.id,
}
)
continue
if not force and not is_due_for_collection(datasource, now):
skipped_sources.append(
{
"id": datasource.id,
"source": datasource.source,
"name": datasource.name,
"reason": "within_frequency_window",
"last_run_at": to_iso8601_utc(datasource.last_run_at),
"next_run_at": to_iso8601_utc(
datasource.last_run_at + timedelta(minutes=datasource.frequency_minutes)
),
}
)
continue
previous_task_ids[datasource.id] = None
success = run_collector_now(datasource.source)
if not success:
failed_sources.append(
{
"id": datasource.id,
"source": datasource.source,
"name": datasource.name,
"reason": "trigger_failed",
}
)
continue
triggered_sources.append(
{
"id": datasource.id,
"source": datasource.source,
"name": datasource.name,
"task_id": None,
}
@router.post("/trigger-batch")
async def trigger_datasource_batch(
payload: DatasourceBatchTriggerRequest,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
query = select(DataSource).order_by(DataSource.module, DataSource.id)
if payload.source_ids:
query = query.where(DataSource.id.in_(payload.source_ids))
else:
query = _apply_datasource_query_filters(
query,
module=payload.module,
is_active=payload.is_active,
priority=payload.priority,
run_status=payload.run_status,
q=payload.q,
)
latest_task_ids = await _load_latest_task_ids(
db,
[datasource.id for datasource in datasources],
result = await db.execute(query)
datasources = result.scalars().all()
running_tasks, _ = await _load_datasource_list_context(db, datasources)
record_counts = await _load_collected_record_counts(db, [datasource.source for datasource in datasources])
datasources = _filter_datasources_in_memory(
datasources,
running_tasks=running_tasks,
record_counts=record_counts,
product=None if payload.source_ids else payload.product,
run_status=None if payload.source_ids else payload.run_status,
collected=None if payload.source_ids else payload.collected,
credential_status=None if payload.source_ids else payload.credential_status,
)
for datasource_id in previous_task_ids:
previous_task_ids[datasource_id] = latest_task_ids.get(datasource_id)
for _ in range(20):
await asyncio.sleep(0.1)
pending = [item for item in triggered_sources if item["task_id"] is None]
if not pending:
break
latest_task_ids = await _load_latest_task_ids(
db,
[item["id"] for item in pending],
)
for item in pending:
task_id = latest_task_ids.get(item["id"])
if task_id is not None and task_id != previous_task_ids.get(item["id"]):
item["task_id"] = task_id
return {
"status": "triggered" if triggered_sources else "partial",
"message": f"Triggered {len(triggered_sources)} data sources",
"force": force,
"triggered": triggered_sources,
"skipped": skipped_sources,
"failed": failed_sources,
}
return await _trigger_datasource_batch(db, datasources, force=payload.force)
@router.get("/{source_id}")

229
backend/app/api/v1/layers.py Normal file
View File

@@ -0,0 +1,229 @@
from typing import Any, Optional
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.v1.visualization import (
_parse_bbox,
get_bgp_anomalies_geojson,
get_bgp_collectors_geojson,
get_bgp_incidents_geojson,
get_cables_geojson,
get_landing_points_geojson,
get_satellites_geojson,
)
from app.api.v1.vessels import build_vessel_snapshot_response
from app.db.session import get_db
router = APIRouter()
DEFAULT_LAYER_LIMIT = 1000
MAX_LAYER_LIMIT = 5000
LOW_ZOOM_FEATURE_LIMIT = 500
def _clamp_limit(limit: int, zoom: int) -> tuple[int, bool]:
clamped = min(max(limit, 1), MAX_LAYER_LIMIT)
if zoom <= 3:
return min(clamped, LOW_ZOOM_FEATURE_LIMIT), clamped != limit or clamped > LOW_ZOOM_FEATURE_LIMIT
return clamped, clamped != limit
def _coordinate_in_bbox(coord: Any, bbox: tuple[float, float, float, float]) -> bool:
if not isinstance(coord, (list, tuple)) or len(coord) < 2:
return False
try:
lon = float(coord[0])
lat = float(coord[1])
except (TypeError, ValueError):
return False
lon_min, lat_min, lon_max, lat_max = bbox
return lon_min <= lon <= lon_max and lat_min <= lat <= lat_max
def _geometry_intersects_bbox(geometry: dict, bbox: tuple[float, float, float, float]) -> bool:
coordinates = geometry.get("coordinates")
geometry_type = geometry.get("type")
if geometry_type == "Point":
return _coordinate_in_bbox(coordinates, bbox)
if geometry_type in {"LineString", "MultiPoint"}:
return any(_coordinate_in_bbox(coord, bbox) for coord in coordinates or [])
if geometry_type in {"Polygon", "MultiLineString"}:
return any(
_coordinate_in_bbox(coord, bbox)
for line in coordinates or []
for coord in line
)
if geometry_type == "MultiPolygon":
return any(
_coordinate_in_bbox(coord, bbox)
for polygon in coordinates or []
for line in polygon
for coord in line
)
return False
def _guard_geojson_layer(
geojson: dict,
*,
bbox: tuple[float, float, float, float],
zoom: int,
limit: int,
) -> dict:
bounded_limit, limit_clamped = _clamp_limit(limit, zoom)
features = [
feature
for feature in geojson.get("features", [])
if _geometry_intersects_bbox(feature.get("geometry") or {}, bbox)
]
visible_count = len(features)
returned_features = features[:bounded_limit]
return {
**geojson,
"features": returned_features,
"visible_count": visible_count,
"returned_count": len(returned_features),
"diagnostics": {
"bbox_limited": True,
"limit": bounded_limit,
"limit_clamped": limit_clamped,
"truncated": visible_count > len(returned_features),
"degraded": zoom <= 3 or visible_count > len(returned_features),
"stats_scope": "viewport",
},
}
def _parse_layer_bbox(bbox: str) -> tuple[float, float, float, float]:
parsed = _parse_bbox(bbox)
if parsed is None:
raise HTTPException(status_code=400, detail="bbox is required")
return parsed
@router.get("/vessels/snapshot")
async def get_vessel_layer_snapshot(
bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"),
zoom: int = Query(..., ge=1, le=20),
limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1),
vessel_type: Optional[str] = Query(None, alias="type"),
since_minutes: int = Query(60, ge=1, le=1440),
db: AsyncSession = Depends(get_db),
):
parsed_bbox = _parse_layer_bbox(bbox)
return await build_vessel_snapshot_response(
db,
bbox=parsed_bbox,
zoom=zoom,
limit=limit,
vessel_type=vessel_type,
since_minutes=since_minutes,
)
@router.get("/cables")
async def get_cable_layer(
bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"),
zoom: int = Query(..., ge=1, le=20),
limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1),
db: AsyncSession = Depends(get_db),
):
return _guard_geojson_layer(
await get_cables_geojson(db),
bbox=_parse_layer_bbox(bbox),
zoom=zoom,
limit=limit,
)
@router.get("/landing-points")
async def get_landing_point_layer(
bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"),
zoom: int = Query(..., ge=1, le=20),
limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1),
db: AsyncSession = Depends(get_db),
):
return _guard_geojson_layer(
await get_landing_points_geojson(db),
bbox=_parse_layer_bbox(bbox),
zoom=zoom,
limit=limit,
)
@router.get("/satellites")
async def get_satellite_layer(
bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"),
zoom: int = Query(..., ge=1, le=20),
limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1),
db: AsyncSession = Depends(get_db),
):
bounded_limit, _ = _clamp_limit(limit, zoom)
return _guard_geojson_layer(
await get_satellites_geojson(limit=bounded_limit, db=db),
bbox=_parse_layer_bbox(bbox),
zoom=zoom,
limit=limit,
)
@router.get("/bgp/anomalies")
async def get_bgp_anomaly_layer(
bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"),
zoom: int = Query(..., ge=1, le=20),
limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1),
severity: Optional[str] = Query(None),
status: Optional[str] = Query("active"),
db: AsyncSession = Depends(get_db),
):
bounded_limit, _ = _clamp_limit(limit, zoom)
return _guard_geojson_layer(
await get_bgp_anomalies_geojson(
severity=severity,
status=status,
limit=bounded_limit,
db=db,
),
bbox=_parse_layer_bbox(bbox),
zoom=zoom,
limit=limit,
)
@router.get("/bgp/incidents")
async def get_bgp_incident_layer(
bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"),
zoom: int = Query(..., ge=1, le=20),
limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1),
severity: Optional[str] = Query(None),
status: Optional[str] = Query("active"),
db: AsyncSession = Depends(get_db),
):
bounded_limit, _ = _clamp_limit(limit, zoom)
return _guard_geojson_layer(
await get_bgp_incidents_geojson(
severity=severity,
status=status,
limit=min(bounded_limit, 500),
db=db,
),
bbox=_parse_layer_bbox(bbox),
zoom=zoom,
limit=limit,
)
@router.get("/bgp/collectors")
async def get_bgp_collector_layer(
bbox: str = Query(..., description="lon_min,lat_min,lon_max,lat_max"),
zoom: int = Query(..., ge=1, le=20),
limit: int = Query(DEFAULT_LAYER_LIMIT, ge=1),
db: AsyncSession = Depends(get_db),
):
return _guard_geojson_layer(
await get_bgp_collectors_geojson(db),
bbox=_parse_layer_bbox(bbox),
zoom=zoom,
limit=limit,
)

232
backend/app/api/v1/settings.py
View File

@@ -81,6 +81,17 @@ DEFAULT_SETTINGS = {
"password_policy": "medium",
},
"tv": DEFAULT_TV_SETTINGS,
"smtp": {
"host": "",
"port": 587,
"username": "",
"password": "",
"from_address": "",
"from_name": "Planet",
"use_tls": False,
"use_starttls": True,
"timeout_seconds": 20,
},
"external_integrations": {
"ai_provider": {
"service_url": "",
@@ -95,6 +106,17 @@ DEFAULT_SETTINGS = {
"default_provider": "tavily",
"providers": {},
},
"ocr": {
"enabled": False,
"provider": "paddleocr",
"base_url": "",
"api_key": "",
"model": "",
"languages": ["zh", "en"],
"timeout_seconds": 30,
"max_file_size_mb": 20,
"output_format": "markdown",
},
},
}
@@ -153,6 +175,24 @@ class TVSettingsUpdate(BaseModel):
sources: list[TVStreamSourceUpdate] = Field(default_factory=list)
class SMTPSettingsUpdate(BaseModel):
host: str = Field(default="", max_length=255)
port: int = Field(default=587, ge=1, le=65535)
username: str = Field(default="", max_length=255)
password: Optional[str] = None
clear_password: bool = False
from_address: str = Field(default="", max_length=255)
from_name: str = Field(default="Planet", max_length=120)
use_tls: bool = False
use_starttls: bool = True
timeout_seconds: int = Field(default=20, ge=3, le=300)
class SMTPTestRequest(BaseModel):
to: EmailStr
settings: Optional[SMTPSettingsUpdate] = None
class AIProviderIntegrationUpdate(BaseModel):
service_url: str = ""
service_token: Optional[str] = None
@@ -198,10 +238,23 @@ class WebSearchIntegrationUpdate(BaseModel):
scrape_formats: list[str] = Field(default_factory=lambda: ["markdown"])
class OCRIntegrationUpdate(BaseModel):
enabled: bool = False
provider: str = Field(default="paddleocr", max_length=80)
base_url: str = Field(default="", max_length=500)
api_key: Optional[str] = None
model: str = Field(default="", max_length=200)
languages: list[str] = Field(default_factory=lambda: ["zh", "en"])
timeout_seconds: int = Field(default=30, ge=3, le=300)
max_file_size_mb: int = Field(default=20, ge=1, le=200)
output_format: str = Field(default="markdown", pattern="^(markdown|text|json)$")
class ExternalIntegrationsUpdate(BaseModel):
ai_provider: AIProviderIntegrationUpdate
barentswatch: BarentsWatchIntegrationUpdate
web_search: WebSearchIntegrationUpdate | None = None
ocr: OCRIntegrationUpdate | None = None
def merge_with_defaults(category: str, payload: Optional[dict]) -> dict:
@@ -649,6 +702,59 @@ def _runtime_config_from_web_search_payload(web_search_payload: dict) -> WebSear
)
def _normalize_ocr_payload(ocr_payload: dict | None) -> dict:
raw = dict(ocr_payload or {})
languages = raw.get("languages")
if not isinstance(languages, list) or not languages:
languages = ["zh", "en"]
return {
"enabled": bool(raw.get("enabled", False)),
"provider": str(raw.get("provider") or "paddleocr").strip().lower() or "paddleocr",
"base_url": str(raw.get("base_url") or "").strip(),
"api_key": str(raw.get("api_key") or "").strip(),
"model": str(raw.get("model") or "").strip(),
"languages": [str(item).strip() for item in languages if str(item).strip()],
"timeout_seconds": int(raw.get("timeout_seconds") or 30),
"max_file_size_mb": int(raw.get("max_file_size_mb") or 20),
"output_format": str(raw.get("output_format") or "markdown").strip() or "markdown",
}
def _resolve_ocr_api_key(ocr_config: dict) -> tuple[str, str]:
saved_key = ocr_config.get("api_key") or ""
if saved_key:
return str(saved_key), "runtime"
return _resolve_web_search_env_secret("OCR_API_KEY")
def _build_ocr_payload(
current_payload: dict,
update: OCRIntegrationUpdate | None,
) -> dict:
current_ocr = _normalize_ocr_payload(current_payload.get("ocr") or {})
if update is None:
return current_ocr
current_key, current_key_source = _resolve_ocr_api_key(current_ocr)
current_key_preview = _mask_secret(current_key, current_key_source)["preview"]
ocr_payload = {
"enabled": update.enabled,
"provider": update.provider.strip().lower() or current_ocr.get("provider") or "paddleocr",
"base_url": update.base_url.strip(),
"model": update.model.strip(),
"languages": [item.strip() for item in update.languages if item.strip()] or ["zh", "en"],
"timeout_seconds": update.timeout_seconds,
"max_file_size_mb": update.max_file_size_mb,
"output_format": update.output_format.strip() or "markdown",
}
if not _is_secret_placeholder(update.api_key, current_key_preview):
ocr_payload["api_key"] = str(update.api_key).strip()
elif current_ocr.get("api_key"):
ocr_payload["api_key"] = current_ocr.get("api_key") or ""
else:
ocr_payload["api_key"] = ""
return ocr_payload
async def get_runtime_web_search_config(db: AsyncSession) -> WebSearchConfig:
runtime_record = await get_setting_record(db, "external_integrations")
payload = merge_with_defaults(
@@ -684,6 +790,7 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
)
normalized_ai = _normalize_ai_provider_payload(raw_payload.get("ai_provider") or {})
normalized_web_search = _normalize_web_search_payload(raw_payload.get("web_search") or {})
normalized_ocr = _normalize_ocr_payload(raw_payload.get("ocr") or {})
default_provider = normalized_ai["default_provider"]
providers_payload: dict[str, dict] = {}
for provider in sorted({
@@ -731,6 +838,7 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
web_search_providers_payload.get(normalized_web_search["default_provider"])
or _web_search_provider_defaults(normalized_web_search["default_provider"])
)
ocr_api_key, ocr_api_key_source = _resolve_ocr_api_key(normalized_ocr)
barentswatch_record = await get_barentswatch_config_record(db)
barentswatch_auth = barentswatch_record.auth_config if barentswatch_record else {}
barentswatch_auth = barentswatch_auth or {}
@@ -782,6 +890,18 @@ async def serialize_external_integrations(db: AsyncSession) -> dict:
"scrape_formats": display_web_search_config.get("scrape_formats") or ["markdown"],
"source": "runtime" if runtime_setting else "env",
},
"ocr": {
"enabled": normalized_ocr["enabled"],
"provider": normalized_ocr["provider"],
"base_url": normalized_ocr["base_url"],
"api_key": _mask_secret(ocr_api_key, ocr_api_key_source),
"model": normalized_ocr["model"],
"languages": normalized_ocr["languages"],
"timeout_seconds": normalized_ocr["timeout_seconds"],
"max_file_size_mb": normalized_ocr["max_file_size_mb"],
"output_format": normalized_ocr["output_format"],
"source": "runtime" if normalized_ocr.get("api_key") else (ocr_api_key_source or "default"),
},
}
@@ -792,11 +912,12 @@ async def save_external_integrations_payload(
current_payload = await get_setting_payload(db, "external_integrations")
ai_payload = _build_ai_provider_payload(current_payload, update.ai_provider)
web_search_payload = _build_web_search_payload(current_payload, update.web_search)
ocr_payload = _build_ocr_payload(current_payload, update.ocr)
await save_setting_payload(
db,
"external_integrations",
{"ai_provider": ai_payload, "web_search": web_search_payload},
{"ai_provider": ai_payload, "web_search": web_search_payload, "ocr": ocr_payload},
)
default_endpoint = get_data_sources_config().get_yaml_url("barentswatch_vessels")
@@ -919,6 +1040,97 @@ async def update_security_settings(
return {"status": "updated", "security": payload}
def _serialize_smtp_payload(payload: dict) -> dict:
password = str(payload.get("password") or "")
return {
"host": payload.get("host") or "",
"port": int(payload.get("port") or 587),
"username": payload.get("username") or "",
"password": _mask_secret(password, "runtime" if password else ""),
"from_address": payload.get("from_address") or "",
"from_name": payload.get("from_name") or "Planet",
"use_tls": bool(payload.get("use_tls", False)),
"use_starttls": bool(payload.get("use_starttls", True)),
"timeout_seconds": int(payload.get("timeout_seconds") or 20),
"configured": bool(payload.get("host") and payload.get("from_address")),
}
def _build_smtp_payload(current_payload: dict, update: SMTPSettingsUpdate) -> dict:
current_password = str(current_payload.get("password") or "")
current_preview = _mask_secret(current_password, "runtime" if current_password else "")["preview"]
if update.clear_password:
password = ""
elif _is_secret_placeholder(update.password, current_preview):
password = current_password
else:
password = str(update.password).strip()
return {
"host": update.host.strip(),
"port": update.port,
"username": update.username.strip(),
"password": password,
"from_address": update.from_address.strip(),
"from_name": update.from_name.strip() or "Planet",
"use_tls": update.use_tls,
"use_starttls": update.use_starttls,
"timeout_seconds": update.timeout_seconds,
}
@router.get("/smtp")
async def get_smtp_settings(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
return {"smtp": _serialize_smtp_payload(await get_setting_payload(db, "smtp"))}
@router.put("/smtp")
async def update_smtp_settings(
payload: SMTPSettingsUpdate,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
if current_user.role not in ("admin", "super_admin"):
raise HTTPException(status_code=403, detail="Only administrators can change SMTP settings")
current = await get_setting_payload(db, "smtp")
merged = _build_smtp_payload(current, payload)
saved = await save_setting_payload(db, "smtp", merged)
return {"status": "updated", "smtp": _serialize_smtp_payload(saved)}
@router.post("/smtp/test")
async def test_smtp_settings(
payload: SMTPTestRequest,
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
if current_user.role not in ("admin", "super_admin"):
raise HTTPException(status_code=403, detail="Only administrators can test SMTP settings")
from app.services.email import EmailError, send_email
current = await get_setting_payload(db, "smtp")
config = _build_smtp_payload(current, payload.settings) if payload.settings else current
if not config.get("host") or not config.get("from_address"):
raise HTTPException(
status_code=400,
detail="Host and from_address are required to send a test email",
)
try:
await send_email(
db,
to=payload.to,
subject="Planet SMTP test",
text_body="This is a test email from Planet SMTP settings.",
html_body="<p>This is a test email from Planet SMTP settings.</p>",
config=config,
)
except EmailError as exc:
return {"success": False, "message": str(exc), "code": exc.code}
return {"success": True, "message": "Test email sent"}
@router.get("/tv")
async def get_tv_settings(
current_user: User = Depends(get_current_user),
@@ -1033,10 +1245,11 @@ async def connect_ai_provider_integration(
)
)
current_web_search = _normalize_web_search_payload(current_payload.get("web_search") or {})
current_ocr = _normalize_ocr_payload(current_payload.get("ocr") or {})
await save_setting_payload(
db,
"external_integrations",
{"ai_provider": draft_ai_payload, "web_search": current_web_search},
{"ai_provider": draft_ai_payload, "web_search": current_web_search, "ocr": current_ocr},
)
return {
"success": True,
@@ -1110,6 +1323,21 @@ async def reveal_web_search_secrets(
}
@router.get("/integrations/ocr/secrets")
async def reveal_ocr_secrets(
current_user: User = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
current_payload = await get_setting_payload(db, "external_integrations")
ocr_payload = _normalize_ocr_payload(current_payload.get("ocr") or {})
api_key, api_key_source = _resolve_ocr_api_key(ocr_payload)
return {
"provider": ocr_payload["provider"],
"api_key": api_key,
"api_key_source": api_key_source,
}
@router.post("/integrations/web-search/connect")
async def connect_web_search_integration(
payload: WebSearchIntegrationUpdate,

39
backend/app/api/v1/vessels.py Normal file
View File

@@ -0,0 +1,39 @@
"""Bounded vessel snapshot APIs for viewport-first consumers."""
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.v1.visualization import _parse_bbox, build_vessel_snapshot_response
from app.db.session import get_db
from app.services.vessel_ais_aggregation import MAX_SNAPSHOT_LIMIT
router = APIRouter()
@router.get("/snapshot")
async def get_vessel_snapshot(
bbox: Optional[str] = Query(None, description="Viewport bbox as lon_min,lat_min,lon_max,lat_max"),
zoom: int = Query(..., ge=1, le=20, description="Current map zoom level"),
type: Optional[str] = Query(
None,
description="Comma-separated vessel types: cargo,tanker,passenger,fishing,military,other",
),
limit: int = Query(1000, ge=1, le=MAX_SNAPSHOT_LIMIT),
since_minutes: int = Query(60, ge=1, le=1440),
db: AsyncSession = Depends(get_db),
):
if not bbox:
raise HTTPException(status_code=400, detail="bbox is required")
parsed_bbox = _parse_bbox(bbox)
if parsed_bbox is None:
raise HTTPException(status_code=400, detail="bbox is required")
return await build_vessel_snapshot_response(
db,
bbox=parsed_bbox,
zoom=zoom,
type_filter=type,
limit=limit,
since_minutes=since_minutes,
)

148
backend/app/api/v1/visualization.py
View File

@@ -51,8 +51,10 @@ from app.services.vessel_ais_aggregation import (
get_aggregated_vessel,
get_aggregated_vessel_track,
get_aggregated_vessels,
get_aggregated_vessels_snapshot,
get_vessel_conflict_records,
get_vessel_raw_observations,
MAX_SNAPSHOT_LIMIT,
)
from app.core.logging import get_logger
@@ -960,6 +962,52 @@ def _build_vessel_stats(features: List[dict[str, Any]]) -> dict[str, Any]:
}
def _safe_vessel_limit(value: int | None, *, default: int = 1000) -> int:
if value is None or value <= 0:
return default
return min(value, MAX_SNAPSHOT_LIMIT)
async def build_vessel_snapshot_response(
db: AsyncSession,
*,
bbox: tuple[float, float, float, float] | None,
zoom: int | None,
type_filter: str | None,
limit: int | None,
since_minutes: int = 60,
) -> dict[str, Any]:
requested_types = _requested_vessel_types(type_filter)
safe_limit = _safe_vessel_limit(limit)
safe_since_minutes = min(max(int(since_minutes or 60), 1), 1440)
observed_since = datetime.now(UTC) - timedelta(minutes=safe_since_minutes)
features, diagnostics = await _load_raw_vessel_snapshot_features(
db,
bbox=bbox,
limit=safe_limit,
observed_since=observed_since,
)
features = _filter_vessel_features(
features,
bbox=bbox,
requested_types=requested_types,
)[:safe_limit]
return {
"type": "FeatureCollection",
"features": features,
"count": len(features),
"stats": _build_vessel_stats(features),
"diagnostics": {
**diagnostics,
"filtered_count": len(features),
"bbox_applied": bbox is not None,
"zoom": zoom,
"limit": safe_limit,
"since_minutes": safe_since_minutes,
},
}
def convert_bgp_anomalies_to_geojson(
records: List[BGPAnomaly],
geography_hints: Optional[Dict[str, Dict[str, Any]]] = None,
@@ -2042,66 +2090,58 @@ async def get_vessels_geojson(
),
db: AsyncSession = Depends(get_db),
):
"""Return latest vessel positions as GeoJSON points."""
parsed_bbox = _parse_bbox(bbox)
requested_types = _requested_vessel_types(type)
merged_features, diagnostics = await _load_merged_vessel_features(db)
features = _filter_vessel_features(
merged_features,
bbox=parsed_bbox,
requested_types=requested_types,
"""Legacy vessel endpoint removed in favor of /api/v1/vessels/snapshot."""
raise HTTPException(
status_code=410,
detail=(
"Legacy vessel GeoJSON endpoint has been removed. "
"Use /api/v1/vessels/snapshot with bbox, zoom, and limit."
),
)
if limit and limit > 0:
features = features[:limit]
return {
"type": "FeatureCollection",
"features": features,
"count": len(features),
"stats": _build_vessel_stats(features),
"diagnostics": {
**diagnostics,
"filtered_count": len(features),
},
}
async def _load_merged_vessel_features(db: AsyncSession) -> tuple[list[dict[str, Any]], dict[str, Any]]:
aggregated_vessels = await get_aggregated_vessels(db)
async def _load_raw_vessel_snapshot_features(
db: AsyncSession,
*,
bbox: tuple[float, float, float, float] | None,
limit: int,
observed_since: datetime,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
if bbox is None:
aggregated_vessels = await get_aggregated_vessels(
db,
limit=limit,
observed_since=observed_since,
)
else:
aggregated_vessels = await get_aggregated_vessels_snapshot(
db,
bbox=bbox,
limit=limit,
observed_since=observed_since,
)
raw_geojson = convert_aggregated_vessels_to_geojson(aggregated_vessels)
latest_times = (
select(
VesselPosition.mmsi.label("mmsi"),
func.max(VesselPosition.received_at).label("received_at"),
)
.group_by(VesselPosition.mmsi)
.subquery()
)
stmt = (
select(VesselPosition, VesselStatic)
.join(
latest_times,
(VesselPosition.mmsi == latest_times.c.mmsi)
& (VesselPosition.received_at == latest_times.c.received_at),
)
.outerjoin(VesselStatic, VesselStatic.mmsi == VesselPosition.mmsi)
.order_by(VesselPosition.received_at.desc())
)
result = await db.execute(stmt)
rows = list(result.all())
legacy_geojson = convert_vessels_to_geojson(rows)
merged_features, diagnostics = _merge_vessel_features(
raw_geojson.get("features", []),
legacy_geojson.get("features", []),
)
return merged_features, {
**diagnostics,
"raw_feature_count": len(raw_geojson.get("features", [])),
"legacy_feature_count": len(legacy_geojson.get("features", [])),
features = raw_geojson.get("features", [])
return features, {
"raw_feature_count": len(features),
"raw_unique_mmsi": len(
{
key
for key in (_feature_mmsi_key(feature) for feature in features)
if key is not None
}
),
"legacy_feature_count": 0,
"legacy_backfilled_mmsi": 0,
"final_unique_mmsi": len(
{
key
for key in (_feature_mmsi_key(feature) for feature in features)
if key is not None
}
),
}
@router.get("/vessels/custom-supplements")
async def get_vessel_custom_supplements(db: AsyncSession = Depends(get_db)):
"""Group custom vessel_ais sources by their declared merge target for diagnostics."""

35
backend/app/api/v1/websocket.py
View File

@@ -1,7 +1,6 @@
"""WebSocket API endpoints"""
import asyncio
import json
from datetime import UTC, datetime
from typing import Optional
@@ -95,14 +94,44 @@ async def websocket_endpoint(
}
)
elif data.get("type") == "subscribe":
channels = data.get("data", {}).get("channels", [])
payload_data = data.get("data", {})
if not isinstance(payload_data, dict):
payload_data = {}
channels = payload_data.get("channels", [])
if isinstance(channels, str):
channels = [channels]
elif not isinstance(channels, list):
channels = []
channel = payload_data.get("channel")
if channel and channel not in channels:
channels = [*channels, channel]
if is_anonymous:
channels = [channel for channel in channels if channel in supported_channels]
vessel_subscription = None
if "vessels" in channels and "bbox" in payload_data:
try:
vessel_subscription = manager.subscribe_vessels(websocket, payload_data)
except ValueError as exc:
await websocket.send_json(
{
"type": "subscription_error",
"data": {"channel": "vessels", "detail": str(exc)},
}
)
continue
channels = [channel for channel in channels if channel != "vessels"]
manager.subscribe(websocket, channels)
await websocket.send_json(
{
"type": "subscription_confirmed",
"data": {"action": "subscribe", "channels": channels},
"data": {
"action": "subscribe",
"channels": [
*channels,
*(["vessels"] if vessel_subscription else []),
],
"vessels": vessel_subscription,
},
}
)
elif data.get("type") == "unsubscribe":

61
backend/app/core/websocket/broadcaster.py
View File

@@ -2,7 +2,7 @@
import asyncio
from datetime import UTC, datetime
from typing import Dict, Any, Optional
from typing import Dict, Any
from app.core.time import to_iso8601_utc
from app.core.websocket.manager import manager
@@ -15,6 +15,8 @@ class DataBroadcaster:
def __init__(self):
self.running = False
self.tasks: Dict[str, asyncio.Task] = {}
self._pending_vessel_updates: Dict[str, Dict[str, Any]] = {}
self._vessel_flush_interval = 1.0
async def get_dashboard_stats(self) -> Dict[str, Any]:
"""Get dashboard statistics"""
@@ -68,6 +70,9 @@ class DataBroadcaster:
async def broadcast_custom(self, channel: str, data: Dict[str, Any]):
"""Broadcast custom data to a specific channel"""
if channel == "vessels":
self.enqueue_vessel_update(data)
return
await manager.broadcast(
{
"type": "data_frame",
@@ -78,6 +83,58 @@ class DataBroadcaster:
channel=channel,
)
def enqueue_vessel_update(self, data: Dict[str, Any]):
vessels = data.get("vessels") if isinstance(data, dict) else None
if not isinstance(vessels, list):
return
source = data.get("source")
action = data.get("action") or "upsert"
created = data.get("created")
for vessel in vessels:
if not isinstance(vessel, dict):
continue
mmsi = vessel.get("mmsi")
if mmsi in (None, ""):
continue
self._pending_vessel_updates[str(mmsi)] = {
**vessel,
"_source": source,
"_action": action,
"_created": created,
}
async def flush_vessel_updates(self):
if not self._pending_vessel_updates:
return
pending = self._pending_vessel_updates
self._pending_vessel_updates = {}
vessels = []
for item in pending.values():
vessel = dict(item)
source = vessel.pop("_source", None)
action = vessel.pop("_action", "upsert")
created = vessel.pop("_created", None)
vessel["source"] = source
vessel["action"] = action
vessel["created"] = created
vessels.append(vessel)
await manager.broadcast_vessels(
{
"action": "upsert",
"source": "mixed",
"created": None,
"vessels": vessels,
}
)
async def broadcast_vessels_periodically(self):
while self.running:
try:
await self.flush_vessel_updates()
except Exception:
pass
await asyncio.sleep(self._vessel_flush_interval)
async def broadcast_datasource_task_update(self, data: Dict[str, Any]):
"""Broadcast datasource task progress updates to connected clients."""
await manager.broadcast(
@@ -95,6 +152,7 @@ class DataBroadcaster:
if not self.running:
self.running = True
self.tasks["dashboard"] = asyncio.create_task(self.broadcast_stats(5))
self.tasks["vessels"] = asyncio.create_task(self.broadcast_vessels_periodically())
def stop(self):
"""Stop all broadcasters"""
@@ -102,6 +160,7 @@ class DataBroadcaster:
for task in self.tasks.values():
task.cancel()
self.tasks.clear()
self._pending_vessel_updates.clear()
broadcaster = DataBroadcaster()

105
backend/app/core/websocket/manager.py
View File

@@ -1,11 +1,16 @@
"""WebSocket Connection Manager"""
from typing import Dict, Set, Optional
from datetime import UTC, datetime
from typing import Any, Dict, Set, Optional
from fastapi import WebSocket
import redis.asyncio as redis
from app.core.config import settings
MAX_VESSEL_SUBSCRIPTION_LIMIT = 5000
MAX_VESSEL_WS_MESSAGE_ITEMS = 1000
MAX_VESSEL_BBOX_AREA = 2500.0
class ConnectionManager:
"""Manages WebSocket connections"""
@@ -14,6 +19,7 @@ class ConnectionManager:
self.active_connections: Dict[str, Set[WebSocket]] = {} # user_id -> connections
self.channel_subscriptions: Dict[str, Set[WebSocket]] = {}
self.websocket_channels: Dict[WebSocket, Set[str]] = {}
self.vessel_subscriptions: Dict[WebSocket, dict[str, Any]] = {}
self.redis_client: Optional[redis.Redis] = None
async def connect(self, websocket: WebSocket, user_id: str):
@@ -72,6 +78,50 @@ class ConnectionManager:
channels = list(self.websocket_channels.get(websocket, set()))
if channels:
self.unsubscribe(websocket, channels)
self.vessel_subscriptions.pop(websocket, None)
def subscribe_vessels(self, websocket: WebSocket, config: dict[str, Any]) -> dict[str, Any]:
subscription = self._normalize_vessel_subscription(config)
self.channel_subscriptions.setdefault("vessels", set()).add(websocket)
self.websocket_channels.setdefault(websocket, set()).add("vessels")
self.vessel_subscriptions[websocket] = subscription
return subscription
def _normalize_vessel_subscription(self, config: dict[str, Any]) -> dict[str, Any]:
bbox = config.get("bbox")
if not isinstance(bbox, (list, tuple)) or len(bbox) != 4:
raise ValueError("vessels subscription requires bbox=[lon_min,lat_min,lon_max,lat_max]")
try:
lon_min, lat_min, lon_max, lat_max = [float(value) for value in bbox]
except (TypeError, ValueError) as exc:
raise ValueError("bbox values must be numbers") from exc
if lat_min > lat_max:
lat_min, lat_max = lat_max, lat_min
if lon_min > lon_max:
lon_min, lon_max = lon_max, lon_min
if not (-180 <= lon_min <= 180 and -180 <= lon_max <= 180):
raise ValueError("bbox longitude values must be between -180 and 180")
if not (-90 <= lat_min <= 90 and -90 <= lat_max <= 90):
raise ValueError("bbox latitude values must be between -90 and 90")
if (lon_max - lon_min) * (lat_max - lat_min) > MAX_VESSEL_BBOX_AREA:
raise ValueError("bbox is too large; zoom in or request a smaller viewport")
zoom = int(config.get("zoom") or 1)
if zoom < 1 or zoom > 20:
raise ValueError("zoom must be between 1 and 20")
limit = min(max(int(config.get("limit") or 1000), 1), MAX_VESSEL_SUBSCRIPTION_LIMIT)
vessel_types = {
str(item).strip().lower()
for item in str(config.get("type") or "").split(",")
if str(item).strip()
}
return {
"bbox": (lon_min, lat_min, lon_max, lat_max),
"zoom": zoom,
"limit": limit,
"type": vessel_types,
"last_sent_at": None,
}
async def send_personal_message(self, message: dict, user_id: str):
if user_id in self.active_connections:
@@ -92,6 +142,58 @@ class ConnectionManager:
except Exception:
self.unsubscribe_all(connection)
async def broadcast_vessels(self, data: dict[str, Any]):
vessels = data.get("vessels") if isinstance(data, dict) else None
if not isinstance(vessels, list) or not vessels:
return
for connection, subscription in list(self.vessel_subscriptions.items()):
matched = [
vessel
for vessel in vessels
if self._vessel_matches_subscription(vessel, subscription)
][: min(subscription["limit"], MAX_VESSEL_WS_MESSAGE_ITEMS)]
if not matched:
continue
subscription["last_sent_at"] = datetime.now(UTC)
message = {
"type": "data_frame",
"channel": "vessels",
"timestamp": subscription["last_sent_at"].isoformat(),
"payload": {
**data,
"vessels": matched,
"subscription": {
"bbox": list(subscription["bbox"]),
"zoom": subscription["zoom"],
"limit": subscription["limit"],
},
},
}
try:
await connection.send_json(message)
except Exception:
self.unsubscribe_all(connection)
def _vessel_matches_subscription(
self,
vessel: dict[str, Any],
subscription: dict[str, Any],
) -> bool:
try:
lon = float(vessel.get("lon"))
lat = float(vessel.get("lat"))
except (TypeError, ValueError):
return False
lon_min, lat_min, lon_max, lat_max = subscription["bbox"]
if not (lon_min <= lon <= lon_max and lat_min <= lat <= lat_max):
return False
requested_types = subscription.get("type") or set()
if not requested_types:
return True
type_name = str(vessel.get("vessel_type_name") or "").lower()
return any(requested_type in type_name for requested_type in requested_types)
async def close_all(self):
for user_id in self.active_connections:
for connection in self.active_connections[user_id]:
@@ -99,6 +201,7 @@ class ConnectionManager:
self.active_connections.clear()
self.channel_subscriptions.clear()
self.websocket_channels.clear()
self.vessel_subscriptions.clear()
manager = ConnectionManager()

40
backend/app/db/session.py
View File

@@ -107,6 +107,7 @@ async def ensure_default_admin_user(session: AsyncSession):
password_hash=get_password_hash(default_user["password"]),
role=default_user["role"],
is_active=True,
email_verified=True,
)
)
await session.commit()
@@ -148,14 +149,31 @@ async def init_db():
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
users_email_verified_existed = (
await conn.execute(
text(
"""
SELECT 1
FROM information_schema.columns
WHERE table_name = 'users' AND column_name = 'email_verified'
"""
)
)
).fetchone() is not None
await conn.execute(
text(
"""
ALTER TABLE users
ADD COLUMN IF NOT EXISTS gatekeeper_groups JSONB DEFAULT '[]'::jsonb
ADD COLUMN IF NOT EXISTS gatekeeper_groups JSONB DEFAULT '[]'::jsonb,
ADD COLUMN IF NOT EXISTS email_verified BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN IF NOT EXISTS pending_email VARCHAR(255)
"""
)
)
if not users_email_verified_existed:
await conn.execute(
text("UPDATE users SET email_verified = TRUE WHERE email_verified = FALSE")
)
await conn.execute(
text(
"""
@@ -216,6 +234,26 @@ async def init_db():
"""
)
)
await conn.execute(
text(
"""
CREATE INDEX IF NOT EXISTS idx_ais_raw_schema_observed_desc
ON ais_raw_observations (target_schema, observed_at DESC)
"""
)
)
await conn.execute(
text(
"""
CREATE INDEX IF NOT EXISTS idx_ais_raw_payload_lon_lat
ON ais_raw_observations (
((normalized_payload->>'lon')::double precision),
((normalized_payload->>'lat')::double precision)
)
WHERE target_schema = 'vessel_ais'
"""
)
)
await conn.execute(
text(
"""

2
backend/app/models/user.py
View File

@@ -14,6 +14,8 @@ class User(Base):
role = Column(String(20), default="viewer")
gatekeeper_groups = Column(JSON, default=list)
is_active = Column(Boolean, default=True)
email_verified = Column(Boolean, default=False, nullable=False)
pending_email = Column(String(255), nullable=True)
last_login_at = Column(DateTime(timezone=True))
created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(

27
backend/app/schemas/user.py
View File

@@ -39,7 +39,34 @@ class UserResponse(UserBase):
role: str
gatekeeper_groups: list[str] = Field(default_factory=list)
is_active: bool
email_verified: bool = False
created_at: datetime
class Config:
from_attributes = True
class UserRegister(BaseModel):
username: str = Field(..., min_length=3, max_length=50)
email: EmailStr
password: str = Field(..., min_length=8, max_length=128)
class VerifyEmailRequest(BaseModel):
email: EmailStr
code: str = Field(..., min_length=6, max_length=6)
class ResendCodeRequest(BaseModel):
email: EmailStr
purpose: str = Field(default="register", pattern="^(register|verify_email|reset_password)$")
class ForgotPasswordRequest(BaseModel):
email: EmailStr
class ResetPasswordRequest(BaseModel):
email: EmailStr
code: str = Field(..., min_length=6, max_length=6)
new_password: str = Field(..., min_length=8, max_length=128)

2
backend/app/services/collectors/aisstream.py
View File

@@ -138,7 +138,7 @@ class AISStreamCollector(BaseCollector):
try:
import websockets
except ImportError as exc:
except ImportError:
return {"status": "failed", "error": "Python package 'websockets' is required for AISStream"}
start_time = datetime.now(UTC)

1
backend/app/services/docs_gatekeeper.py
View File

@@ -53,6 +53,7 @@ DOCS_METADATA: tuple[DocsMetadata, ...] = (
DocsMetadata("backend-datasources-api-performance.md", "backend-datasources-api-performance", "docs_developer", "Backend", 33, "数据源 API 性能", "Datasource API Performance"),
DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 34, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"),
DocsMetadata("agents-aiprovider.md", "agents-aiprovider", "docs_developer", "Agents", 40, "AI Provider 指南", "AI Provider Guide"),
DocsMetadata("ops-runbook.md", "ops-runbook", "docs_admin", "Ops", 49, "Planet 运维手册", "Planet Ops Runbook"),
DocsMetadata("ops-docker-compose-buildx-upgrade.md", "ops-docker-compose-buildx-upgrade", "docs_admin", "Ops", 50, "Docker + Compose + Buildx 升级", "Docker + Compose + Buildx Upgrade"),
DocsMetadata("ops-planet-sh-startup.md", "ops-planet-sh-startup", "docs_admin", "Ops", 51, "planet.sh 启动机制", "planet.sh Startup"),
)

123
backend/app/services/email.py Normal file
View File

@@ -0,0 +1,123 @@
"""SMTP-backed email sender.
Generic primitive used by registration/verification today, reusable for alert
digests and other notifications later. Configuration lives in the `smtp` row of
`system_settings` and is loaded once per send (small surface, no caching layer
yet to keep behavior obvious after settings changes).
"""
from __future__ import annotations
from email.message import EmailMessage
from typing import Literal, Optional
import aiosmtplib
from sqlalchemy.ext.asyncio import AsyncSession
OtpPurpose = Literal["register", "verify_email", "reset_password"]
class EmailError(Exception):
code: str = "EMAIL_ERROR"
class EmailNotConfiguredError(EmailError):
code = "EMAIL_PROVIDER_NOT_CONFIGURED"
class EmailSendError(EmailError):
code = "EMAIL_SEND_FAILED"
async def _load_smtp_config(db: AsyncSession) -> dict:
from app.api.v1.settings import get_setting_payload # local import avoids cycle
payload = await get_setting_payload(db, "smtp")
if not payload.get("host") or not payload.get("from_address"):
raise EmailNotConfiguredError("SMTP host/from_address not set")
return payload
async def send_email(
db: AsyncSession,
*,
to: str,
subject: str,
text_body: str,
html_body: Optional[str] = None,
config: Optional[dict] = None,
) -> None:
cfg = config or await _load_smtp_config(db)
message = EmailMessage()
from_name = (cfg.get("from_name") or "").strip()
from_address = cfg["from_address"]
message["From"] = f"{from_name} <{from_address}>" if from_name else from_address
message["To"] = to
message["Subject"] = subject
message.set_content(text_body)
if html_body:
message.add_alternative(html_body, subtype="html")
use_tls = bool(cfg.get("use_tls", True))
use_starttls = bool(cfg.get("use_starttls", False))
port = int(cfg.get("port") or (465 if use_tls else 587))
try:
await aiosmtplib.send(
message,
hostname=cfg["host"],
port=port,
username=cfg.get("username") or None,
password=cfg.get("password") or None,
use_tls=use_tls and not use_starttls,
start_tls=use_starttls,
timeout=int(cfg.get("timeout_seconds") or 20),
)
except aiosmtplib.SMTPException as exc:
raise EmailSendError(str(exc)) from exc
except OSError as exc:
raise EmailSendError(str(exc)) from exc
_SUBJECTS: dict[OtpPurpose, str] = {
"register": "Confirm your Planet account",
"verify_email": "Verify your Planet email",
"reset_password": "Reset your Planet password",
}
_HEADLINES: dict[OtpPurpose, str] = {
"register": "Welcome to Planet — confirm your email to activate your account.",
"verify_email": "Confirm your new email address to keep your Planet account active.",
"reset_password": "Use this code to set a new password for your Planet account.",
}
async def send_verification_email(
db: AsyncSession,
*,
to: str,
code: str,
purpose: OtpPurpose,
config: Optional[dict] = None,
) -> None:
subject = _SUBJECTS[purpose]
headline = _HEADLINES[purpose]
text_body = (
f"{headline}\n\n"
f"Your verification code: {code}\n"
"This code expires in 10 minutes. If you did not request it, ignore this email.\n"
)
html_body = (
f"<p>{headline}</p>"
f"<p style=\"font-size:24px;letter-spacing:4px;font-family:monospace\"><b>{code}</b></p>"
"<p>This code expires in 10 minutes. If you did not request it, ignore this email.</p>"
)
await send_email(
db,
to=to,
subject=subject,
text_body=text_body,
html_body=html_body,
config=config,
)

100
backend/app/services/otp.py Normal file
View File

@@ -0,0 +1,100 @@
"""One-time verification codes backed by Redis.
Reusable primitive for register/verify-email/reset-password (and any future 2FA or
phone-number verification). Codes are bcrypt-hashed before storage so a Redis dump
does not leak active codes.
"""
from __future__ import annotations
import json
import secrets
from typing import Literal
import bcrypt
from app.core.security import redis_client
OtpPurpose = Literal["register", "verify_email", "reset_password"]
CODE_TTL_SECONDS = 600 # 10 minutes
RESEND_COOLDOWN_SECONDS = 60
MAX_ATTEMPTS = 5
CODE_LENGTH = 6
class OtpError(Exception):
code: str = "OTP_ERROR"
class OtpResendRateLimited(OtpError):
code = "OTP_RESEND_RATE_LIMITED"
def __init__(self, retry_after_seconds: int) -> None:
super().__init__(f"Resend allowed in {retry_after_seconds}s")
self.retry_after_seconds = retry_after_seconds
class OtpInvalid(OtpError):
code = "OTP_INVALID"
class OtpExpired(OtpError):
code = "OTP_EXPIRED"
class OtpAttemptsExceeded(OtpError):
code = "OTP_ATTEMPTS_EXCEEDED"
def _code_key(email: str, purpose: OtpPurpose) -> str:
return f"otp:{purpose}:{email.lower()}"
def _rate_key(email: str, purpose: OtpPurpose) -> str:
return f"otp_rate:{purpose}:{email.lower()}"
def _generate_code() -> str:
# secrets.randbelow gives uniform 0..10**CODE_LENGTH-1 without modulo bias
return f"{secrets.randbelow(10 ** CODE_LENGTH):0{CODE_LENGTH}d}"
def check_resend_allowed(email: str, purpose: OtpPurpose) -> None:
ttl = redis_client.ttl(_rate_key(email, purpose))
if ttl and ttl > 0:
raise OtpResendRateLimited(ttl)
def issue_code(email: str, purpose: OtpPurpose) -> str:
"""Generate a new code, persist its hash, and start the resend cooldown.
Caller is responsible for delivering the returned plaintext (e.g. via email).
Any pre-existing code for the same (purpose, email) is overwritten.
"""
check_resend_allowed(email, purpose)
code = _generate_code()
hashed = bcrypt.hashpw(code.encode(), bcrypt.gensalt()).decode()
payload = json.dumps({"hash": hashed, "attempts": 0})
redis_client.set(_code_key(email, purpose), payload, ex=CODE_TTL_SECONDS)
redis_client.set(_rate_key(email, purpose), "1", ex=RESEND_COOLDOWN_SECONDS)
return code
def verify_code(email: str, purpose: OtpPurpose, code: str) -> None:
"""Validate and consume a code. Raises subclasses of OtpError on failure."""
key = _code_key(email, purpose)
raw = redis_client.get(key)
if raw is None:
raise OtpExpired("Code expired or never issued")
record = json.loads(raw)
attempts = int(record.get("attempts", 0))
if attempts >= MAX_ATTEMPTS:
redis_client.delete(key)
raise OtpAttemptsExceeded("Too many invalid attempts")
if not bcrypt.checkpw(code.encode(), record["hash"].encode()):
record["attempts"] = attempts + 1
ttl = redis_client.ttl(key)
redis_client.set(key, json.dumps(record), ex=max(ttl, 1))
raise OtpInvalid("Incorrect code")
redis_client.delete(key)

45
backend/app/services/vessel_ais_aggregation.py
View File

@@ -6,6 +6,7 @@ import json
from typing import Any, Iterable
from sqlalchemy import select
from sqlalchemy import Float
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth
@@ -17,6 +18,10 @@ from app.services.vessel_types import normalize_vessel_type_name
VESSEL_AIS_SCHEMA = "vessel_ais"
DEFAULT_AGGREGATION_WINDOW_HOURS = 24
DEFAULT_SNAPSHOT_WINDOW_MINUTES = 60
MAX_SNAPSHOT_LIMIT = 5000
MAX_SNAPSHOT_CANDIDATE_MULTIPLIER = 20
MAX_SNAPSHOT_CANDIDATE_OBSERVATIONS = 100_000
BARENTSWATCH_DELIVERY_MODE = "polling"
BARENTSWATCH_TRANSPORT = "http"
AISSTREAM_DELIVERY_MODE = "realtime_stream"
@@ -564,6 +569,46 @@ async def get_aggregated_vessels(
return vessels
async def get_aggregated_vessels_snapshot(
db: AsyncSession,
*,
bbox: tuple[float, float, float, float],
limit: int = 1000,
observed_since: datetime | None = None,
) -> list[dict[str, Any]]:
"""Return a bounded viewport snapshot without loading the global AIS window."""
observed_since = observed_since or (
datetime.now(UTC) - timedelta(minutes=DEFAULT_SNAPSHOT_WINDOW_MINUTES)
)
safe_limit = min(max(int(limit or 1000), 1), MAX_SNAPSHOT_LIMIT)
candidate_limit = min(
max(safe_limit * MAX_SNAPSHOT_CANDIDATE_MULTIPLIER, safe_limit),
MAX_SNAPSHOT_CANDIDATE_OBSERVATIONS,
)
lon_min, lat_min, lon_max, lat_max = bbox
payload_lon = AISRawObservation.normalized_payload["lon"].as_string().cast(Float)
payload_lat = AISRawObservation.normalized_payload["lat"].as_string().cast(Float)
stmt = (
select(AISRawObservation)
.where(AISRawObservation.target_schema == VESSEL_AIS_SCHEMA)
.where(AISRawObservation.observed_at >= observed_since)
.where(payload_lon >= lon_min)
.where(payload_lon <= lon_max)
.where(payload_lat >= lat_min)
.where(payload_lat <= lat_max)
.order_by(AISRawObservation.observed_at.desc(), AISRawObservation.id.desc())
.limit(candidate_limit)
)
result = await db.execute(stmt)
if not hasattr(result, "scalars"):
return []
vessels = await aggregate_vessel_observations(db, result.scalars().all())
return vessels[:safe_limit]
async def get_aggregated_vessel(db: AsyncSession, mmsi: int) -> dict[str, Any] | None:
observations = await get_vessel_raw_observations(db, mmsi, limit=1000)
vessels = await aggregate_vessel_observations(db, observations)