release: bump version to 0.74.4

backend/app/api/v1/settings.py

@@ -75,6 +75,7 @@ logger = get_logger(__name__, service="api")
 AI_PROVIDER_QUICK_CONNECT_TIMEOUT_SECONDS = 5
 AI_CONNECTION_TEST_PROMPT_KEY = "ai.connection_test"
 SECRET_REVEAL_ROLES = {UserRole.ADMIN.value, UserRole.SUPER_ADMIN.value}
+LLM_PROVIDER_PRESET_CATEGORY_PREFIX = "llm_provider_preset:"
 
 DEFAULT_SETTINGS = {
     "system": {
@@ -2044,8 +2045,18 @@ async def reset_provider_credential_guide(
 @router.get("/integrations/ai-provider/presets")
 async def get_ai_provider_presets(
     current_user: User = Depends(get_current_user),
+    db: AsyncSession = Depends(get_db),
 ):
-    return {"data": list_fallback_llm_provider_presets()}
+    presets = list_fallback_llm_provider_presets()
+    saved = await get_setting_payloads(
+        db, [f"{LLM_PROVIDER_PRESET_CATEGORY_PREFIX}{preset['provider']}" for preset in presets]
+    )
+    return {
+        "data": [
+            {**preset, **saved[f"{LLM_PROVIDER_PRESET_CATEGORY_PREFIX}{preset['provider']}"]}
+            for preset in presets
+        ]
+    }
 
 
 @router.post("/integrations/ai-provider/presets/{provider}/refresh")
@@ -2056,19 +2067,39 @@ async def refresh_ai_provider_preset(
 ):
     try:
         provider_id = _normalize_provider_id(provider)
+        get_fallback_llm_provider_preset(provider_id)
+    except ValueError as exc:
+        raise HTTPException(status_code=404, detail=str(exc)) from exc
+
+    try:
         api_key = None
         if provider_id == "opencode-go":
             current_payload = await get_setting_payload(db, "external_integrations")
             ai_payload = _normalize_ai_provider_payload(current_payload.get("ai_provider") or {})
-            provider_config = ai_payload["providers"].get(provider_id) or _provider_defaults(provider_id)
+            provider_config = ai_payload["providers"].get(provider_id) or _provider_defaults(
+                provider_id
+            )
             api_key, _api_key_source = _resolve_provider_api_key(provider_id, provider_config)
-        return {"data": await refresh_llm_provider_preset(provider_id, api_key=api_key)}
-    except ValueError as exc:
-        raise HTTPException(status_code=404, detail=str(exc)) from exc
+        refreshed = await refresh_llm_provider_preset(provider_id, api_key=api_key)
     except Exception as exc:
-        fallback = get_fallback_llm_provider_preset(provider)
-        fallback["refresh_error"] = str(exc)
-        return {"data": fallback}
+        logger.warning(
+            "LLM provider catalog refresh failed",
+            extra={
+                "event": "settings.ai_provider.catalog.failed",
+                "context": {
+                    "provider": provider_id,
+                    "error_type": type(exc).__name__,
+                },
+            },
+        )
+        raise HTTPException(
+            status_code=502,
+            detail="模型列表刷新失败，已保留上次模型列表。",
+        ) from exc
+
+    refreshed["refreshed_at"] = to_iso8601_utc(datetime.now(UTC))
+    await save_setting_payload(db, f"{LLM_PROVIDER_PRESET_CATEGORY_PREFIX}{provider_id}", refreshed)
+    return {"data": refreshed}
 
 
 @router.put("/integrations")

backend/app/api/v1/tv.py

@@ -7,6 +7,7 @@ from fastapi.responses import Response
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.db.session import get_db
+from app.services.tv_catalog import get_tv_catalog_page
 from app.services.tv_streams import get_public_tv_payload, is_allowed_tv_proxy_url
 
 router = APIRouter()
@@ -34,9 +35,13 @@ def _should_strip_hls_metadata_line(line: str) -> bool:
 
 @router.get("/streams")
 async def list_public_tv_streams(
+    offset: int = Query(0, ge=0),
+    limit: int = Query(50, ge=1, le=100),
+    q: str = Query("", max_length=200),
+    selected_id: str | None = Query(None, max_length=200),
     db: AsyncSession = Depends(get_db),
 ):
-    return await get_public_tv_payload(db)
+    return await get_tv_catalog_page(db, offset=offset, limit=limit, q=q, selected_id=selected_id)
 
 
 @router.get("/proxy")

backend/app/api/v1/visualization.py

@@ -1121,6 +1121,7 @@ async def build_vessel_snapshot_response(
     safe_limit = _safe_vessel_limit(limit)
     safe_since_minutes = min(max(int(since_minutes or 60), 1), 1440)
     observed_since = datetime.now(UTC) - timedelta(minutes=safe_since_minutes)
+    snapshot_started_at = datetime.now(UTC)
     features, diagnostics = await _load_raw_vessel_snapshot_features(
         db,
         bbox=bbox,
@@ -1137,6 +1138,7 @@ async def build_vessel_snapshot_response(
         "features": features,
         "count": len(features),
         "stats": _build_vessel_stats(features),
+        "generated_at": to_iso8601_utc(snapshot_started_at),
         "diagnostics": {
             **diagnostics,
             "filtered_count": len(features),

backend/app/api/v1/websocket.py

@@ -161,7 +161,7 @@ async def websocket_endpoint(
                     if is_anonymous:
                         channels = [channel for channel in channels if channel in supported_channels]
                     vessel_subscription = None
-                    if "vessels" in channels and "bbox" in payload_data:
+                    if "vessels" in channels:
                         try:
                             vessel_subscription = manager.subscribe_vessels(websocket, payload_data)
                         except ValueError as exc:

backend/app/core/websocket/broadcaster.py

@@ -4,11 +4,14 @@ import asyncio
 from datetime import UTC, datetime
 from typing import Dict, Any
 
+from app.core.logging import get_logger
 from app.core.time import to_iso8601_utc
 from app.core.websocket.manager import manager
 
 
 EARTH_UPDATES_CHANNEL = "earth_updates"
+VESSEL_STATE_QUERY_BATCH_SIZE = 1000
+logger = get_logger(__name__, service="websocket")
 
 
 class DataBroadcaster:
@@ -114,16 +117,22 @@ class DataBroadcaster:
             return
         pending = self._pending_vessel_updates
         self._pending_vessel_updates = {}
-        vessels = []
-        for item in pending.values():
-            vessel = dict(item)
-            source = vessel.pop("_source", None)
-            action = vessel.pop("_action", "upsert")
-            created = vessel.pop("_created", None)
-            vessel["source"] = source
-            vessel["action"] = action
-            vessel["created"] = created
-            vessels.append(vessel)
+        try:
+            vessels = await self._load_current_vessel_updates(list(pending))
+        except Exception:
+            # Preserve newer updates that arrived during the failed database read.
+            self._pending_vessel_updates = {**pending, **self._pending_vessel_updates}
+            raise
+        if not vessels:
+            return
+        vessels = [
+            {
+                **vessel,
+                "action": vessel.get("action", "upsert"),
+                "created": pending.get(str(vessel["mmsi"]), {}).get("_created"),
+            }
+            for vessel in vessels
+        ]
         await manager.broadcast_vessels(
             {
                 "action": "upsert",
@@ -133,12 +142,31 @@ class DataBroadcaster:
             }
         )
 
+    async def _load_current_vessel_updates(self, keys: list[str]) -> list[Dict[str, Any]]:
+        from app.db.session import async_session_factory
+        from app.services.vessel_ais_aggregation import get_current_vessels_by_mmsi
+
+        mmsis = [int(key) for key in keys if key.isdigit()]
+        vessels = []
+        async with async_session_factory() as db:
+            for offset in range(0, len(mmsis), VESSEL_STATE_QUERY_BATCH_SIZE):
+                vessels.extend(await get_current_vessels_by_mmsi(
+                    db, mmsis[offset:offset + VESSEL_STATE_QUERY_BATCH_SIZE]
+                ))
+        present = {int(vessel["mmsi"]) for vessel in vessels}
+        vessels.extend({"mmsi": mmsi, "action": "remove"} for mmsi in mmsis if mmsi not in present)
+        return vessels
+
     async def broadcast_vessels_periodically(self):
         while self.running:
             try:
                 await self.flush_vessel_updates()
-            except Exception:
-                pass
+            except Exception as exc:
+                logger.exception_event(
+                    "Failed to flush vessel updates",
+                    event="vessels.broadcast.failed",
+                    context={"error": str(exc)},
+                )
             await asyncio.sleep(self._vessel_flush_interval)
 
     async def broadcast_datasource_task_update(self, data: Dict[str, Any]):

backend/app/core/websocket/manager.py

@@ -63,6 +63,8 @@ class ConnectionManager:
 
     def unsubscribe(self, websocket: WebSocket, channels: list[str]):
         for channel in {str(channel).strip() for channel in channels if str(channel).strip()}:
+            if channel == "vessels":
+                self.vessel_subscriptions.pop(websocket, None)
             subscribers = self.channel_subscriptions.get(channel)
             if subscribers is not None:
                 subscribers.discard(websocket)
@@ -88,7 +90,8 @@ class ConnectionManager:
         return subscription
 
     def _normalize_vessel_subscription(self, config: dict[str, Any]) -> dict[str, Any]:
-        bbox = config.get("bbox")
+        global_scope = config.get("scope") == "global"
+        bbox = [-180, -90, 180, 90] if global_scope else config.get("bbox")
         if not isinstance(bbox, (list, tuple)) or len(bbox) != 4:
             raise ValueError("vessels subscription requires bbox=[lon_min,lat_min,lon_max,lat_max]")
         try:
@@ -103,7 +106,7 @@ class ConnectionManager:
             raise ValueError("bbox longitude values must be between -180 and 180")
         if not (-90 <= lat_min <= 90 and -90 <= lat_max <= 90):
             raise ValueError("bbox latitude values must be between -90 and 90")
-        if (lon_max - lon_min) * (lat_max - lat_min) > MAX_VESSEL_BBOX_AREA:
+        if not global_scope and (lon_max - lon_min) * (lat_max - lat_min) > MAX_VESSEL_BBOX_AREA:
             raise ValueError("bbox is too large; zoom in or request a smaller viewport")
 
         zoom = int(config.get("zoom") or 1)
@@ -116,10 +119,11 @@ class ConnectionManager:
             if str(item).strip()
         }
         return {
+            "scope": "global" if global_scope else "viewport",
             "bbox": (lon_min, lat_min, lon_max, lat_max),
             "zoom": zoom,
             "limit": limit,
-            "type": vessel_types,
+            "type": sorted(vessel_types),
             "last_sent_at": None,
         }
 
@@ -152,7 +156,9 @@ class ConnectionManager:
                 vessel
                 for vessel in vessels
                 if self._vessel_matches_subscription(vessel, subscription)
-            ][: min(subscription["limit"], MAX_VESSEL_WS_MESSAGE_ITEMS)]
+            ]
+            if subscription.get("scope") != "global":
+                matched = matched[:subscription["limit"]]
             if not matched:
                 continue
             subscription["last_sent_at"] = datetime.now(UTC)
@@ -162,16 +168,24 @@ class ConnectionManager:
                 "timestamp": subscription["last_sent_at"].isoformat(),
                 "payload": {
                     **data,
-                    "vessels": matched,
+                    "vessels": [],
                     "subscription": {
                         "bbox": list(subscription["bbox"]),
                         "zoom": subscription["zoom"],
                         "limit": subscription["limit"],
+                        "scope": subscription.get("scope", "viewport"),
                     },
                 },
             }
             try:
-                await connection.send_json(message)
+                for offset in range(0, len(matched), MAX_VESSEL_WS_MESSAGE_ITEMS):
+                    await connection.send_json({
+                        **message,
+                        "payload": {
+                            **message["payload"],
+                            "vessels": matched[offset:offset + MAX_VESSEL_WS_MESSAGE_ITEMS],
+                        },
+                    })
             except Exception:
                 self.unsubscribe_all(connection)
 
@@ -180,6 +194,8 @@ class ConnectionManager:
         vessel: dict[str, Any],
         subscription: dict[str, Any],
     ) -> bool:
+        if vessel.get("action") == "remove" and subscription.get("scope") == "global":
+            return True
         try:
             lon = float(vessel.get("lon"))
             lat = float(vessel.get("lat"))

backend/app/services/collectors/news_live_streams.py

@@ -35,7 +35,7 @@ class NewsLiveStreamsCollector(BaseCollector):
     DEFAULT_IPTV_ORG_LOGOS_URL = "https://iptv-org.github.io/api/logos.json"
     DEFAULT_IPTV_ORG_NEWS_CATEGORIES = ("news", "business", "weather")
     DEFAULT_IPTV_ORG_EXCLUDE_CATEGORIES = ("music", "sports", "kids", "entertainment")
-    DEFAULT_IPTV_ORG_MAX_SOURCES = 120
+    DEFAULT_IPTV_ORG_MAX_SOURCES = 0  # Zero keeps the complete matching channel catalog.
 
     async def fetch(self) -> list[dict[str, Any]]:
         request_url = (self._resolved_url or "").strip()
@@ -445,7 +445,7 @@ class NewsLiveStreamsCollector(BaseCollector):
                     "reference_date": datetime.now(UTC).isoformat(),
                 }
             )
-            if len(normalized) >= max_sources:
+            if max_sources > 0 and len(normalized) >= max_sources:
                 break
 
         return normalized

backend/app/services/earth_db_change_listener.py

@@ -56,6 +56,8 @@ def build_earth_update_from_db_payload(payload: dict[str, Any]) -> dict[str, Any
     source_has_adapter = bool(get_earth_update_layers_for_source(source))
     effective_source = source if source_has_adapter else (table_name if table_name else source)
     operation = payload.get("operation")
+    if "vessels" in layers and operation in {"DELETE", "TRUNCATE"}:
+        refresh_strategy = "reload"
     update: dict[str, Any] = {
         "event": "earth.layer.changed",
         "action": "database_changed",
@@ -113,6 +115,8 @@ class PendingEarthDbChange:
         operation = payload.get("operation")
         if operation:
             self.operations.add(str(operation))
+            if "vessels" in self.layers and operation in {"DELETE", "TRUNCATE"}:
+                self.refresh_strategy = "reload"
         entity_keys = payload.get("entity_keys")
         if not isinstance(entity_keys, list):
             entity_key = payload.get("entity_key")
@@ -176,6 +180,17 @@ class EarthDbChangeDispatcher:
         if not update:
             return False
 
+        if payload.get("table") == "vessel_current_state" and payload.get("operation") == "DELETE":
+            from app.core.websocket.broadcaster import broadcaster
+
+            keys = payload.get("entity_keys")
+            if not isinstance(keys, list):
+                keys = [payload.get("entity_key")]
+            broadcaster.enqueue_vessel_update({
+                "action": "remove",
+                "vessels": [{"mmsi": key} for key in keys if key is not None],
+            })
+
         source = update["source"]
         pending = self._pending.get(source)
         if pending is None:

backend/app/services/earth_layer_adapters.py

@@ -25,6 +25,7 @@ EARTH_LAYER_ADAPTERS: tuple[EarthLayerAdapter, ...] = (
         layers=("vessels",),
         cache_patterns=("vessels*", "summary*"),
         derived_models=("ais_raw_observations", "ais_conflict_records", "ais_source_health"),
+        refresh_strategy="delta",
     ),
     EarthLayerAdapter(
         sources=frozenset(

backend/app/services/llm_provider_catalog.py

@@ -9,6 +9,11 @@ import httpx
 MODELS_DEV_URL = "https://models.dev/api.json"
 OPENCODE_GO_MODELS_URL = "https://opencode.ai/zen/go/v1/models"
 
+
+class LLMProviderCatalogError(RuntimeError):
+    """The upstream catalog cannot supply a usable model list."""
+
+
 OPENCODE_GO_MODEL_PROVIDER_APIS = {
     "minimax-m2.7": "anthropic-messages",
     "minimax-m2.5": "anthropic-messages",
@@ -171,9 +176,9 @@ async def refresh_llm_provider_preset(provider: str, api_key: str | None = None)
             str(item.get("id"))
             for item in data
             if isinstance(item, dict) and item.get("id")
-        ][:120]
+        ]
         if not model_ids:
-            model_ids = fallback["models"]
+            raise LLMProviderCatalogError("The provider returned an empty model catalog")
         return {
             **fallback,
             "model": fallback["model"] if fallback["model"] in model_ids else model_ids[0],
@@ -184,7 +189,7 @@ async def refresh_llm_provider_preset(provider: str, api_key: str | None = None)
 
     models_dev_key = MODELS_DEV_PROVIDER_KEYS.get(fallback["provider"])
     if not models_dev_key:
-        return fallback
+        raise LLMProviderCatalogError("Live catalog refresh is unavailable for this provider")
 
     async with httpx.AsyncClient(timeout=15.0, follow_redirects=True) as client:
         response = await client.get(
@@ -194,12 +199,23 @@ async def refresh_llm_provider_preset(provider: str, api_key: str | None = None)
         response.raise_for_status()
         catalog = response.json()
 
-    upstream = catalog.get(models_dev_key)
+    upstream = catalog.get(models_dev_key) if isinstance(catalog, dict) else None
     if not isinstance(upstream, dict):
-        return fallback
+        raise LLMProviderCatalogError("The provider is missing from the model catalog")
 
     upstream_models = upstream.get("models") if isinstance(upstream.get("models"), dict) else {}
-    model_ids = list(upstream_models.keys())[:80]
+    # Catalog insertion order is not release order; old entries can appear first.
+    model_ids = sorted(
+        (
+            model_id
+            for model_id, model in upstream_models.items()
+            if model_id and isinstance(model, dict)
+        ),
+        key=lambda model_id: (str(upstream_models[model_id].get("release_date") or ""), model_id),
+        reverse=True,
+    )
+    if not model_ids:
+        raise LLMProviderCatalogError("The provider returned an empty model catalog")
     base_url = upstream.get("api") or fallback["base_url"]
     if fallback["provider"] == "deepseek" and base_url == "https://api.deepseek.com":
         base_url = "https://api.deepseek.com/v1"
@@ -208,8 +224,8 @@ async def refresh_llm_provider_preset(provider: str, api_key: str | None = None)
         **fallback,
         "label": upstream.get("name") or fallback["label"],
         "base_url": base_url,
-        "model": model_ids[0] if model_ids else fallback["model"],
-        "models": model_ids or fallback["models"],
+        "model": model_ids[0],
+        "models": model_ids,
         "api_key_env": (upstream.get("env") or [fallback["api_key_env"]])[0],
         "source": MODELS_DEV_URL,
     }

backend/app/services/tv_catalog.py

@@ -0,0 +1,148 @@
+"""Search and paginate the public live TV catalog at the database boundary."""
+
+from typing import Any
+
+from sqlalchemy import Select, func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.time import to_iso8601_utc
+from app.models.collected_data import CollectedData
+from app.services.tv_streams import (
+    TV_LIVE_SOURCE_COLLECTOR,
+    TV_LIVE_SOURCE_DATA_TYPE,
+    _build_collected_tv_source,
+    build_public_tv_payload,
+    get_tv_settings_payload,
+)
+
+
+def _source_key():
+    return func.coalesce(
+        func.nullif(CollectedData.extra_data["id"].as_string(), ""),
+        func.nullif(CollectedData.source_id, ""),
+        CollectedData.entity_key,
+    )
+
+
+def _collected_catalog_query(configured_ids: list[str]) -> Select[tuple[CollectedData]]:
+    metadata = CollectedData.extra_data
+    source_id = _source_key()
+    enabled = func.lower(func.trim(func.coalesce(metadata["is_enabled"].as_string(), "true")))
+    ranked = (
+        select(
+            CollectedData.id,
+            func.row_number()
+            .over(
+                partition_by=source_id,
+                order_by=CollectedData.id.desc(),
+            )
+            .label("source_rank"),
+        )
+        .where(
+            CollectedData.source == TV_LIVE_SOURCE_COLLECTOR,
+            CollectedData.data_type == TV_LIVE_SOURCE_DATA_TYPE,
+            CollectedData.is_current.is_(True),
+            CollectedData.is_valid == 1,
+            CollectedData.deleted_at.is_(None),
+            enabled.notin_(("false", "0", "no", "off")),
+            source_id.notin_(configured_ids),
+        )
+        .subquery()
+    )
+    return (
+        select(CollectedData)
+        .join(ranked, ranked.c.id == CollectedData.id)
+        .where(ranked.c.source_rank == 1)
+    )
+
+
+def _filter_catalog_query(
+    query: Select[tuple[CollectedData]], terms: list[str]
+) -> Select[tuple[CollectedData]]:
+    metadata = CollectedData.extra_data
+    searchable = func.lower(
+        func.concat_ws(
+            " ",
+            CollectedData.name,
+            CollectedData.title,
+            CollectedData.source_id,
+            metadata["name"].as_string(),
+            metadata["provider"].as_string(),
+            metadata["region"].as_string(),
+            metadata["country"].as_string(),
+            metadata["language"].as_string(),
+        )
+    )
+    for term in terms:
+        query = query.where(searchable.contains(term, autoescape=True))
+    return query
+
+
+def _matches_source(source: dict[str, Any], terms: list[str]) -> bool:
+    searchable = " ".join(
+        str(source.get(key) or "") for key in ("id", "name", "provider", "region", "language")
+    ).lower()
+    return all(term in searchable for term in terms)
+
+
+async def get_tv_catalog_page(
+    db: AsyncSession,
+    *,
+    offset: int = 0,
+    limit: int = 50,
+    q: str = "",
+    selected_id: str | None = None,
+) -> dict[str, Any]:
+    settings = await get_tv_settings_payload(db)
+    payload = build_public_tv_payload(settings, [])
+    configured = payload["sources"]
+    query = _collected_catalog_query([source["id"] for source in settings["sources"]])
+    if selected_id:
+        selected = next((source for source in configured if source["id"] == selected_id), None)
+        if selected is None:
+            record = await db.scalar(query.where(_source_key() == selected_id).limit(1))
+            selected = _build_collected_tv_source(record, 0) if record else None
+        if selected:
+            payload["selected_source"] = selected
+    summary = (
+        await db.execute(
+            select(func.count(), func.max(CollectedData.collected_at))
+            .select_from(CollectedData)
+            .where(CollectedData.id.in_(query.with_only_columns(CollectedData.id)))
+        )
+    ).one()
+    total_collected, latest_update = summary
+    terms = q.lower().split()
+    matched_configured = [source for source in configured if _matches_source(source, terms)]
+    filtered_query = _filter_catalog_query(query, terms)
+    matched_collected = (
+        await db.scalar(select(func.count()).select_from(filtered_query.subquery()))
+        if terms
+        else total_collected
+    )
+    sources = matched_configured[offset : offset + limit]
+    remaining = limit - len(sources)
+    if remaining:
+        rows = await db.scalars(
+            filtered_query.order_by(func.lower(CollectedData.name), CollectedData.id)
+            .offset(max(0, offset - len(matched_configured)))
+            .limit(remaining)
+        )
+        sources.extend(
+            _build_collected_tv_source(record, index) for index, record in enumerate(rows)
+        )
+    total = len(matched_configured) + matched_collected
+    next_offset = offset + len(sources)
+    return {
+        **payload,
+        "sources": sources,
+        "source_count": len(configured) + total_collected,
+        "latest_updated_at": (
+            to_iso8601_utc(latest_update) if latest_update else payload["latest_updated_at"]
+        ),
+        "total": total,
+        "offset": offset,
+        "limit": limit,
+        "has_more": next_offset < total,
+        "next_offset": next_offset if next_offset < total else None,
+    }

backend/app/services/tv_streams.py

@@ -11,7 +11,7 @@ from app.core.time import to_iso8601_utc
 from app.models.collected_data import CollectedData
 from app.models.system_setting import SystemSetting
 
-DEFAULT_TV_SOURCE_ID = "cgtn-en"
+DEFAULT_TV_SOURCE_ID = "aljazeera-mubasher"
 TV_SETTINGS_CATEGORY = "tv"
 TV_LIVE_SOURCE_COLLECTOR = "news_live_streams"
 TV_LIVE_SOURCE_DATA_TYPE = "news_live_stream"
@@ -300,8 +300,12 @@ def normalize_tv_settings(payload: dict[str, Any] | None) -> dict[str, Any]:
     ]
 
     if not any(source["id"] == DEFAULT_TV_SOURCE_ID for source in normalized_sources):
+        default_source = next(
+            source for source in DEFAULT_TV_SETTINGS["sources"]
+            if source["id"] == DEFAULT_TV_SOURCE_ID
+        )
         normalized_sources.append(
-            normalize_tv_source(DEFAULT_TV_SETTINGS["sources"][0], index=len(normalized_sources))
+            normalize_tv_source(default_source, index=len(normalized_sources))
         )
 
     default_source_exists = any(

backend/app/services/vessel_ais_aggregation.py

@@ -614,6 +614,18 @@ async def get_current_vessels_snapshot(
     return [item.to_dict() for item in result.scalars().all()]
 
 
+async def get_current_vessels_by_mmsi(
+    db: AsyncSession, mmsis: list[int]
+) -> list[dict[str, Any]]:
+    """Read canonical render state for the vessels changed by a stream flush."""
+    if not mmsis:
+        return []
+    result = await db.execute(
+        select(VesselCurrentState).where(VesselCurrentState.mmsi.in_(mmsis))
+    )
+    return [_jsonable(item.to_dict()) for item in result.scalars().all()]
+
+
 async def aggregate_vessel_observations(
     db: AsyncSession,
     observations: Iterable[AISRawObservation],

backend/tests/test_earth_db_change_listener.py

@@ -63,6 +63,21 @@ def test_build_earth_update_maps_derived_tables_to_layers():
     assert vessel_update is not None
     assert vessel_update["source"] == "vessel_position"
     assert vessel_update["layers"] == ["vessels"]
+    assert vessel_update["refresh_strategy"] == "reload"
+
+
+def test_vessel_stream_changes_do_not_request_full_layer_rebuilds():
+    from app.services.earth_db_change_listener import PendingEarthDbChange
+
+    for source in ("aisstream_vessels", "barentswatch_vessels"):
+        update = build_earth_update_from_db_payload({
+            "source": source, "table": "vessel_current_state", "operation": "UPDATE",
+        })
+        assert update["refresh_strategy"] == "delta"
+        pending = PendingEarthDbChange(source=source, layers=["vessels"], refresh_strategy="delta")
+        pending.add({"operation": "UPDATE"})
+        pending.add({"operation": "DELETE"})
+        assert pending.refresh_strategy == "reload"
 
 
 def test_build_earth_update_maps_interactable_delete_to_delta():

backend/tests/test_llm_provider_catalog.py

@@ -0,0 +1,64 @@
+import httpx
+import pytest
+
+from app.services import llm_provider_catalog as catalog
+
+
+def mock_catalog(monkeypatch, payload):
+    client_type = httpx.AsyncClient
+    transport = httpx.MockTransport(lambda request: httpx.Response(200, json=payload))
+    monkeypatch.setattr(
+        catalog.httpx, "AsyncClient", lambda **kwargs: client_type(transport=transport, **kwargs)
+    )
+
+
+@pytest.mark.asyncio
+async def test_refresh_orders_by_release_date_before_choosing_default(monkeypatch):
+    mock_catalog(
+        monkeypatch,
+        {
+            "minimax": {
+                "models": {
+                    "MiniMax-M2": {"release_date": "2025-10-27"},
+                    "MiniMax-M3": {"release_date": "2026-06-01"},
+                    "MiniMax-M2.7": {"release_date": "2026-03-18"},
+                    "undated-model": {},
+                }
+            }
+        },
+    )
+
+    refreshed = await catalog.refresh_llm_provider_preset("minimax")
+
+    assert refreshed["model"] == "MiniMax-M3"
+    assert refreshed["models"] == ["MiniMax-M3", "MiniMax-M2.7", "MiniMax-M2", "undated-model"]
+    assert refreshed["source"] == catalog.MODELS_DEV_URL
+
+
+@pytest.mark.asyncio
+async def test_refresh_does_not_truncate_new_models(monkeypatch):
+    models = {f"older-{index}": {"release_date": "2025-01-01"} for index in range(85)}
+    models["latest"] = {"release_date": "2026-06-01"}
+    mock_catalog(monkeypatch, {"minimax": {"models": models}})
+
+    refreshed = await catalog.refresh_llm_provider_preset("minimax")
+
+    assert refreshed["model"] == "latest"
+    assert len(refreshed["models"]) == 86
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("payload", [{}, {"minimax": {"models": {}}}, {"minimax": []}])
+async def test_invalid_catalog_fails_instead_of_claiming_fallback_is_fresh(monkeypatch, payload):
+    mock_catalog(monkeypatch, payload)
+
+    with pytest.raises(catalog.LLMProviderCatalogError):
+        await catalog.refresh_llm_provider_preset("minimax")
+
+
+@pytest.mark.asyncio
+async def test_empty_opencode_catalog_is_a_refresh_failure(monkeypatch):
+    mock_catalog(monkeypatch, {"data": []})
+
+    with pytest.raises(catalog.LLMProviderCatalogError):
+        await catalog.refresh_llm_provider_preset("opencode-go", api_key="test-key")

backend/tests/test_settings_ai_provider.py

@@ -1,5 +1,7 @@
+from copy import deepcopy
 from types import SimpleNamespace
 
+import httpx
 import pytest
 
 from app.api.v1 import settings as settings_api
@@ -24,6 +26,71 @@ from app.api.v1.settings import (
 from app.services.llm_provider_catalog import get_fallback_llm_provider_preset
 
 
+@pytest.mark.asyncio
+async def test_refreshed_presets_survive_listing_without_changing_runtime_settings(monkeypatch):
+    stored = {
+        "external_integrations": {
+            "ai_provider": {
+                "default_provider": "minimax",
+                "service_token": "internal-token",
+                "providers": {"minimax": {"model": "custom-model", "api_key": "provider-key"}},
+            }
+        }
+    }
+    runtime_before = deepcopy(stored["external_integrations"])
+
+    async def fake_save(_db, category, payload):
+        stored[category] = deepcopy(payload)
+        return stored[category]
+
+    async def fake_get_many(_db, categories):
+        return {category: deepcopy(stored.get(category, {})) for category in categories}
+
+    async def fake_refresh(provider, api_key=None):
+        return {
+            **get_fallback_llm_provider_preset(provider),
+            "model": "new-model",
+            "models": ["new-model", "older-model"],
+            "source": "https://models.dev/api.json",
+        }
+
+    monkeypatch.setattr(settings_api, "save_setting_payload", fake_save)
+    monkeypatch.setattr(settings_api, "get_setting_payloads", fake_get_many)
+    monkeypatch.setattr(settings_api, "refresh_llm_provider_preset", fake_refresh)
+    user = SimpleNamespace(id=1, role="admin")
+
+    await settings_api.refresh_ai_provider_preset("minimax", user, object())
+    await settings_api.refresh_ai_provider_preset("openai", user, object())
+    listed = await settings_api.get_ai_provider_presets(user, object())
+    presets = {preset["provider"]: preset for preset in listed["data"]}
+
+    assert presets["minimax"]["models"] == ["new-model", "older-model"]
+    assert presets["openai"]["model"] == "new-model"
+    assert presets["anthropic"]["source"] == "fallback"
+    assert presets["minimax"]["refreshed_at"]
+    assert stored["external_integrations"] == runtime_before
+    assert "provider-key" not in str(listed)
+    assert "internal-token" not in str(listed)
+
+
+@pytest.mark.asyncio
+async def test_failed_preset_refresh_is_an_error_and_does_not_save(monkeypatch):
+    async def fail_refresh(*args, **kwargs):
+        raise httpx.ConnectError("upstream error with secret-value")
+
+    async def fail_save(*args, **kwargs):
+        pytest.fail("failed refresh must preserve the last saved preset")
+
+    monkeypatch.setattr(settings_api, "refresh_llm_provider_preset", fail_refresh)
+    monkeypatch.setattr(settings_api, "save_setting_payload", fail_save)
+
+    with pytest.raises(settings_api.HTTPException) as error:
+        await settings_api.refresh_ai_provider_preset("minimax", SimpleNamespace(id=1), object())
+
+    assert error.value.status_code == 502
+    assert "secret-value" not in error.value.detail
+
+
 @pytest.fixture(autouse=True)
 def isolated_ai_provider_env_file(monkeypatch, tmp_path):
     env_file = tmp_path / ".env"

backend/tests/test_tv_catalog.py

@@ -0,0 +1,160 @@
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock
+
+import pytest
+from sqlalchemy import create_engine, event
+from sqlalchemy.orm import Session
+
+from app.db.session import Base
+from app.models.collected_data import CollectedData
+from app.models.data_snapshot import DataSnapshot
+from app.models.system_setting import SystemSetting
+from app.models.task import CollectionTask
+from app.services.collectors.news_live_streams import NewsLiveStreamsCollector
+from app.services.tv_catalog import get_tv_catalog_page
+from app.services.tv_streams import normalize_tv_settings
+
+
+class CatalogSession:
+    """Execute the real catalog queries against an isolated SQLite database."""
+
+    def __init__(self, session):
+        self.session = session
+
+    async def execute(self, query):
+        return self.session.execute(query)
+
+    async def scalar(self, query):
+        return self.session.scalar(query)
+
+    async def scalars(self, query):
+        return self.session.scalars(query)
+
+
+@pytest.fixture
+def catalog_db():
+    engine = create_engine("sqlite:///:memory:")
+
+    @event.listens_for(engine, "connect")
+    def register_functions(connection, _record):
+        connection.create_function(
+            "concat_ws",
+            -1,
+            lambda sep, *args: sep.join(str(arg) for arg in args if arg is not None),
+        )
+
+    Base.metadata.create_all(
+        engine,
+        tables=[
+            CollectionTask.__table__,
+            DataSnapshot.__table__,
+            CollectedData.__table__,
+            SystemSetting.__table__,
+        ],
+    )
+    with Session(engine) as session:
+        for index in range(135):
+            session.add(
+                CollectedData(
+                    source="news_live_streams",
+                    source_id=f"channel-{index:03}",
+                    data_type="news_live_stream",
+                    name=f"Channel {index:03}",
+                    collected_at=datetime(2026, 9, 13, tzinfo=UTC),
+                    is_current=True,
+                    is_valid=1,
+                    extra_data={
+                        "stream_url": "https://example.invalid/live.m3u8",
+                        "region": "Canada",
+                    },
+                )
+            )
+        session.flush()
+        yield CatalogSession(session)
+    engine.dispose()
+
+
+@pytest.mark.asyncio
+async def test_pages_include_entire_catalog_without_overlap(catalog_db):
+    first = await get_tv_catalog_page(catalog_db, limit=50)
+    second = await get_tv_catalog_page(catalog_db, offset=first["next_offset"], limit=50)
+    third = await get_tv_catalog_page(catalog_db, offset=second["next_offset"], limit=50)
+    ids = [source["id"] for page in (first, second, third) for source in page["sources"]]
+    assert [len(page["sources"]) for page in (first, second, third)] == [50, 50, 45]
+    assert len(set(ids)) == first["source_count"] == 145
+    assert ids[-1] == "channel-134"
+    assert third["next_offset"] is None and not third["has_more"]
+    beyond = await get_tv_catalog_page(catalog_db, offset=200)
+    assert beyond["sources"] == [] and not beyond["has_more"]
+
+
+@pytest.mark.asyncio
+async def test_search_finds_later_pages_and_treats_wildcards_literally(catalog_db):
+    payload = await get_tv_catalog_page(catalog_db, q="CANADA 134")
+    assert [source["id"] for source in payload["sources"]] == ["channel-134"]
+    assert payload["total"] == 1 and payload["source_count"] == 145
+    assert (await get_tv_catalog_page(catalog_db, q="%"))["total"] == 0
+
+
+@pytest.mark.asyncio
+async def test_selection_survives_refresh_when_outside_first_page(catalog_db):
+    payload = await get_tv_catalog_page(catalog_db, selected_id="channel-134")
+    assert payload["selected_source"]["id"] == "channel-134"
+    assert "channel-134" not in [source["id"] for source in payload["sources"]]
+    assert payload["default_source_id"] == "aljazeera-mubasher"
+    default = (await get_tv_catalog_page(catalog_db, selected_id="removed"))["selected_source"]
+    assert default["id"] == "aljazeera-mubasher" and default["source_type"] == "hls"
+
+
+@pytest.mark.asyncio
+async def test_catalog_hides_inactive_records_and_deduplicates_ids(catalog_db):
+    for name, values in [
+        ("Disabled", {"extra_data": {"is_enabled": False}}),
+        ("Historical", {"is_current": False}),
+        ("Invalid", {"is_valid": 0}),
+        ("Deleted", {"deleted_at": datetime.now(UTC)}),
+        ("Replacement", {"source_id": "channel-134"}),
+    ]:
+        record = dict(
+            source="news_live_streams",
+            source_id=name,
+            name=name,
+            data_type="news_live_stream",
+            is_current=True,
+            is_valid=1,
+        )
+        catalog_db.session.add(CollectedData(**{**record, **values}))
+    catalog_db.session.flush()
+    payload = await get_tv_catalog_page(catalog_db, q="replacement")
+    assert payload["source_count"] == 145
+    assert [source["id"] for source in payload["sources"]] == ["channel-134"]
+
+
+def test_missing_builtin_default_adds_aljazeera_without_losing_custom_source():
+    settings = normalize_tv_settings({"sources": [{"id": "custom", "name": "Custom"}]})
+    assert settings["default_source_id"] == "aljazeera-mubasher"
+    assert {source["id"] for source in settings["sources"]} == {"custom", "aljazeera-mubasher"}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "config, expected", [({}, 135), ({"max_sources": 0}, 135), ({"max_sources": 7}, 7)]
+)
+async def test_collector_keeps_all_matching_channels_unless_explicitly_limited(
+    monkeypatch, config, expected
+):
+    collector = NewsLiveStreamsCollector()
+    channels = [
+        {"id": f"channel-{i}", "name": f"Channel {i}", "categories": ["news"]} for i in range(135)
+    ]
+    channels.append({"id": "sport", "name": "Sports", "categories": ["sports"]})
+    streams = [
+        {"channel": channel["id"], "url": "https://example.invalid/live.m3u8"}
+        for channel in channels
+    ]
+    monkeypatch.setattr(
+        collector, "_gather_iptv_org_payloads", AsyncMock(return_value=(channels, streams, []))
+    )
+    records = await collector._fetch_iptv_org("https://example.invalid/channels.json", config)
+    assert len(records) == expected
+    assert all(record["source_id"] != "sport" for record in records)

backend/tests/test_websocket_manager.py

@@ -1,5 +1,9 @@
-import pytest
 import importlib
+import json
+
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
 
 from app.core.websocket.manager import ConnectionManager
 from app.core.websocket.broadcaster import DataBroadcaster
@@ -113,6 +117,11 @@ async def test_vessel_broadcaster_keeps_latest_update_per_mmsi(monkeypatch):
     broadcaster_module = importlib.import_module("app.core.websocket.broadcaster")
     monkeypatch.setattr(broadcaster_module.manager, "broadcast_vessels", fake_broadcast_vessels)
     broadcaster = DataBroadcaster()
+    async def load_current(keys):
+        assert keys == ["1"]
+        return [{"mmsi": 1, "lat": 60.0, "lon": 11.0, "source": "barentswatch_vessels"}]
+
+    monkeypatch.setattr(broadcaster, "_load_current_vessel_updates", load_current)
     broadcaster.enqueue_vessel_update(
         {
             "source": "aisstream_vessels",
@@ -129,10 +138,67 @@ async def test_vessel_broadcaster_keeps_latest_update_per_mmsi(monkeypatch):
     assert sent[0]["vessels"] == [
         {
             "mmsi": 1,
-            "lat": 59.1,
-            "lon": 10.1,
-            "source": "aisstream_vessels",
+            "lat": 60.0,
+            "lon": 11.0,
+            "source": "barentswatch_vessels",
             "action": "upsert",
             "created": None,
         }
     ]
+
+
+@pytest.mark.asyncio
+async def test_global_vessel_subscription_delivers_every_item_in_bounded_frames():
+    manager = ConnectionManager()
+    socket = FakeWebSocket()
+    config = manager.subscribe_vessels(socket, {"scope": "global", "zoom": 4})
+    json.dumps(config)
+    vessels = [{"mmsi": index, "lat": 60, "lon": 10} for index in range(2501)]
+    await manager.broadcast_vessels({"vessels": vessels})
+
+    assert [len(frame["payload"]["vessels"]) for frame in socket.sent] == [1000, 1000, 501]
+    assert [item for frame in socket.sent for item in frame["payload"]["vessels"]] == vessels
+    manager.unsubscribe(socket, ["vessels"])
+    await manager.broadcast_vessels({"vessels": vessels})
+    assert len(socket.sent) == 3
+
+
+@pytest.mark.asyncio
+async def test_global_vessel_removal_does_not_require_coordinates():
+    manager = ConnectionManager()
+    socket = FakeWebSocket()
+    manager.subscribe_vessels(socket, {"scope": "global", "zoom": 4})
+    await manager.broadcast_vessels({"vessels": [{"mmsi": 123, "action": "remove"}]})
+    assert socket.sent[0]["payload"]["vessels"] == [{"mmsi": 123, "action": "remove"}]
+
+
+def test_anonymous_earth_can_confirm_global_vessel_subscription(monkeypatch):
+    websocket_module = importlib.import_module("app.api.v1.websocket")
+    monkeypatch.setattr(websocket_module, "manager", ConnectionManager())
+    app = FastAPI()
+    app.include_router(websocket_module.router)
+    with TestClient(app) as client, client.websocket_connect("/ws") as socket:
+        assert socket.receive_json()["type"] == "connection_established"
+        socket.send_json({
+            "type": "subscribe",
+            "data": {"channels": ["earth_updates", "vessels"], "scope": "global", "zoom": 4},
+        })
+        response = socket.receive_json()
+        assert response["type"] == "subscription_confirmed"
+        assert response["data"]["vessels"]["scope"] == "global"
+        assert response["data"]["vessels"]["type"] == []
+
+
+@pytest.mark.asyncio
+async def test_vessel_flush_retries_without_overwriting_newer_queued_updates(monkeypatch):
+    broadcaster = DataBroadcaster()
+    broadcaster.enqueue_vessel_update({"vessels": [{"mmsi": 1, "lat": 59, "lon": 10}]})
+
+    async def fail_read(_keys):
+        broadcaster.enqueue_vessel_update({"vessels": [{"mmsi": 1, "lat": 61, "lon": 12}]})
+        raise RuntimeError("database unavailable")
+
+    monkeypatch.setattr(broadcaster, "_load_current_vessel_updates", fail_read)
+    with pytest.raises(RuntimeError, match="database unavailable"):
+        await broadcaster.flush_vessel_updates()
+    assert broadcaster._pending_vessel_updates["1"]["lat"] == 61