release: bump version to 0.70.0
Some checks failed
ci / backend (push) Has been cancelled
ci / frontend (push) Has been cancelled
release / images (push) Has been cancelled
ci / delivery (push) Has been cancelled

This commit is contained in:
linkong
2026-06-04 17:16:23 +08:00
parent acbbfdf9e2
commit 8c204717cd
78 changed files with 1762 additions and 703 deletions

View File

@@ -6,6 +6,7 @@ from collections import Counter, defaultdict
from datetime import UTC, datetime
from typing import Any
from app.core.enums import BGPStatus
from app.models.bgp_anomaly import BGPAnomaly
@@ -127,7 +128,7 @@ def detect_origin_change_anomalies(
source=source,
anomaly_type=anomaly_type,
severity=severity,
status="active",
status=BGPStatus.ACTIVE.value,
entity_key=f"{anomaly_type}:{prefix}:{new_origin}",
prefix=prefix,
origin_asn=sorted(historic)[0] if historic else None,
@@ -197,7 +198,7 @@ def detect_more_specific_burst_anomalies(
source=source,
anomaly_type="more_specific_burst",
severity="high",
status="active",
status=BGPStatus.ACTIVE.value,
entity_key=f"more_specific_burst:{root_prefix}:{len(unique_prefixes)}:{len(related_collectors)}",
prefix=sample.get("prefix"),
origin_asn=sample.get("origin_asn"),
@@ -267,7 +268,7 @@ def detect_mass_withdrawal_anomalies(
source=source,
anomaly_type="mass_withdrawal",
severity=severity,
status="active",
status=BGPStatus.ACTIVE.value,
entity_key=f"mass_withdrawal:{prefix}:{origin_asn}:{len(related_collectors)}:{count}",
prefix=prefix,
origin_asn=origin_asn,
@@ -354,7 +355,7 @@ def detect_route_leak_anomalies(
source=source,
anomaly_type="route_leak_candidate",
severity="high" if max_path_length >= dominant_length + 3 else "medium",
status="active",
status=BGPStatus.ACTIVE.value,
entity_key=f"route_leak_candidate:{prefix}:{max_path_length}:{len(related_collectors)}",
prefix=prefix,
origin_asn=sample_metadata.get("origin_asn"),
@@ -435,7 +436,7 @@ def detect_path_flap_anomalies(
source=source,
anomaly_type="path_flap",
severity=severity,
status="active",
status=BGPStatus.ACTIVE.value,
entity_key=f"path_flap:{prefix}:{transitions}:{len(distinct_paths)}",
prefix=prefix,
origin_asn=sample_metadata.get("origin_asn"),

View File

@@ -9,6 +9,7 @@ from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.collected_data_fields import get_record_field
from app.core.enums import BGPStatus
from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_incident import BGPIncident
from app.models.collected_data import CollectedData
@@ -290,7 +291,7 @@ async def create_bgp_incidents_for_anomalies(
existing.title = title
existing.summary = summary
existing.severity = severity
existing.status = "active"
existing.status = BGPStatus.ACTIVE.value
existing.confidence = confidence
existing.started_at = primary.started_at or existing.started_at or datetime.now(UTC)
existing.ended_at = None
@@ -313,7 +314,7 @@ async def create_bgp_incidents_for_anomalies(
title=title,
summary=summary,
severity=severity,
status="active",
status=BGPStatus.ACTIVE.value,
confidence=confidence,
started_at=primary.started_at or datetime.now(UTC),
affected_prefixes=prefixes,

View File

@@ -12,11 +12,11 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.core.collected_data_fields import build_dynamic_metadata, get_record_field
from app.core.countries import normalize_country
from app.core.enums import JobStatus, SnapshotStatus
from app.core.logging import get_logger
from app.core.time import to_iso8601_utc
from app.core.websocket.broadcaster import broadcaster
from app.services.business_logs import emit_business_log, exception_context
from app.services.earth_layer_adapters import get_earth_update_layers_for_source
logger = get_logger(__name__, service="collector")
@@ -238,7 +238,7 @@ class BaseCollector(ABC):
snapshot = await db.get(DataSnapshot, snapshot_id)
if snapshot:
parent_snapshot_id = snapshot.parent_snapshot_id
snapshot.status = "cancelled"
snapshot.status = SnapshotStatus.CANCELLED.value
snapshot.is_current = False
snapshot.completed_at = datetime.now(UTC)
summary = dict(snapshot.summary or {})
@@ -308,7 +308,7 @@ class BaseCollector(ABC):
task.datasource_id = datasource_id
task.source = task.source or self.name
task.task_type = task.task_type or "collect"
task.status = "running"
task.status = JobStatus.RUNNING.value
task.phase = "queued"
task.started_at = task.started_at or start_time
task.completed_at = None
@@ -393,7 +393,7 @@ class BaseCollector(ABC):
},
)
task.status = "success"
task.status = JobStatus.SUCCESS.value
task.phase = "completed"
task.phase_progress = 100.0
task.phase_message = "采集完成"
@@ -428,7 +428,7 @@ class BaseCollector(ABC):
}
except asyncio.CancelledError:
await db.rollback()
task.status = "cancelled"
task.status = JobStatus.CANCELLED.value
task.phase = "cancelled"
task.phase_message = "采集已取消"
task.error_message = "Collection cancelled by operator and rolled back"
@@ -456,7 +456,7 @@ class BaseCollector(ABC):
raise
except Exception as e:
await db.rollback()
task.status = "failed"
task.status = JobStatus.FAILED.value
task.phase = "failed"
task.phase_message = str(e)
task.error_message = str(e)
@@ -464,7 +464,7 @@ class BaseCollector(ABC):
if snapshot_id is not None:
snapshot = await db.get(DataSnapshot, snapshot_id)
if snapshot:
snapshot.status = "failed"
snapshot.status = SnapshotStatus.FAILED.value
snapshot.completed_at = datetime.now(UTC)
snapshot.summary = {"error": str(e)}
await db.commit()
@@ -509,7 +509,7 @@ class BaseCollector(ABC):
if snapshot:
snapshot.record_count = 0
snapshot.summary = {"created": 0, "updated": 0, "unchanged": 0}
snapshot.status = "success"
snapshot.status = SnapshotStatus.SUCCESS.value
snapshot.completed_at = datetime.now(UTC)
await db.commit()
return 0
@@ -642,7 +642,7 @@ class BaseCollector(ABC):
snapshot = await db.get(DataSnapshot, snapshot_id)
if snapshot:
snapshot.record_count = records_added
snapshot.status = "success"
snapshot.status = SnapshotStatus.SUCCESS.value
snapshot.completed_at = datetime.now(UTC)
snapshot.summary = {
"created": created_count,

View File

@@ -6,6 +6,7 @@ from typing import Any
import httpx
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.enums import SnapshotStatus
from app.core.time import to_iso8601_utc
from app.core.websocket.broadcaster import broadcaster
from app.services.barentswatch import (
@@ -125,7 +126,7 @@ class VesselAISCollector(BaseCollector):
snapshot = await db.get(DataSnapshot, snapshot_id)
if snapshot:
snapshot.record_count = records_added
snapshot.status = "success"
snapshot.status = SnapshotStatus.SUCCESS.value
snapshot.completed_at = now
snapshot.summary = {
"created": records_added,

View File

@@ -12,6 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.core.cache import cache
from app.core.config import settings
from app.core.enums import JobStatus, JobType, RollbackPolicy
from app.core.logging import get_logger
from app.core.time import to_iso8601_utc
from app.core.websocket.broadcaster import broadcaster
@@ -36,17 +37,17 @@ from app.services.scheduler import sync_datasource_job
logger = get_logger(__name__)
JOB_TYPE_COLLECT = "collect"
JOB_TYPE_CLEAR_DATA = "clear_data"
JOB_TYPE_CLEAR_CACHE = "clear_cache"
JOB_TYPE_EARTH_REFRESH = "earth_refresh"
JOB_TYPE_COLLECT = JobType.COLLECT.value
JOB_TYPE_CLEAR_DATA = JobType.CLEAR_DATA.value
JOB_TYPE_CLEAR_CACHE = JobType.CLEAR_CACHE.value
JOB_TYPE_EARTH_REFRESH = JobType.EARTH_REFRESH.value
JOB_STATUS_QUEUED = "queued"
JOB_STATUS_RUNNING = "running"
JOB_STATUS_CANCELLING = "cancelling"
JOB_STATUS_SUCCESS = "success"
JOB_STATUS_FAILED = "failed"
JOB_STATUS_CANCELLED = "cancelled"
JOB_STATUS_QUEUED = JobStatus.QUEUED.value
JOB_STATUS_RUNNING = JobStatus.RUNNING.value
JOB_STATUS_CANCELLING = JobStatus.CANCELLING.value
JOB_STATUS_SUCCESS = JobStatus.SUCCESS.value
JOB_STATUS_FAILED = JobStatus.FAILED.value
JOB_STATUS_CANCELLED = JobStatus.CANCELLED.value
ACTIVE_JOB_STATUSES = (JOB_STATUS_QUEUED, JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING)
TERMINAL_JOB_STATUSES = (JOB_STATUS_SUCCESS, JOB_STATUS_FAILED, JOB_STATUS_CANCELLED)
@@ -80,7 +81,7 @@ async def enqueue_datasource_job(
task_type: str,
*,
payload: dict[str, Any] | None = None,
rollback_policy: str = "keep_committed_batches",
rollback_policy: str = RollbackPolicy.KEEP_COMMITTED_BATCHES.value,
dedupe_key: str | None = None,
) -> CollectionTask:
if dedupe_key:

View File

@@ -13,6 +13,7 @@ from sqlalchemy import func, select
from app.core.data_sources import get_data_sources_config
from app.core.datasource_defaults import DEFAULT_DATASOURCES
from app.core.enums import JobStatus
from app.models.collected_data import CollectedData
from app.models.datasource import DataSource
from app.models.datasource_config import DataSourceConfig
@@ -398,7 +399,7 @@ async def has_collected_data(db, source: str) -> bool:
datasource_result = await db.execute(select(DataSource).where(DataSource.source == source))
datasource = datasource_result.scalar_one_or_none()
return bool(datasource and datasource.last_status == "success")
return bool(datasource and datasource.last_status == JobStatus.SUCCESS.value)
async def get_builtin_connection_status(

View File

@@ -6,6 +6,7 @@ from dataclasses import dataclass
from pathlib import Path
from typing import Literal
from app.core.enums import UserRole
from app.models.user import User
DocsAccess = Literal["public", "docs_user", "docs_developer", "docs_admin"]
@@ -43,7 +44,9 @@ DOCS_METADATA: tuple[DocsMetadata, ...] = (
DocsMetadata("earth-satellite-footprint-policy.md", "earth-satellite-footprint-policy", "docs_developer", "Earth", 13, "智能星球卫星覆盖策略", "Intelligent Planet Satellite Footprint Policy"),
DocsMetadata("earth-bgp-context.md", "earth-bgp-context", "docs_developer", "Earth", 14, "BGP 态势上下文", "BGP Context"),
DocsMetadata("earth-interactable-usage.md", "earth-interactable-usage", "docs_developer", "Earth", 16, "智能星球可交互图标接入", "Intelligent Planet Interactable Usage"),
DocsMetadata("earth-interactable-clustering.md", "earth-interactable-clustering", "docs_developer", "Earth", 17, "智能星球可交互图标聚类策略", "Intelligent Planet Interactable Clustering"),
DocsMetadata("earth-toolbar-overlay-coordination.md", "earth-toolbar-overlay-coordination", "docs_developer", "Earth", 17, "智能星球工具栏与浮层协同", "Intelligent Planet Toolbar and Overlay Coordination"),
DocsMetadata("earth-news-sources.md", "earth-news-sources", "docs_developer", "Earth", 19, "智能星球新闻源配置", "Intelligent Planet News Source Configuration"),
DocsMetadata("frontend-admin-frontend-context.md", "frontend-admin-frontend-context", "docs_developer", "Frontend", 20, "控制台前端结构", "Admin Frontend Context"),
DocsMetadata("frontend-layout-guidelines.md", "frontend-layout-guidelines", "docs_developer", "Frontend", 21, "前端布局指南", "Frontend Layout Guidelines"),
DocsMetadata("tactile-ui-components.md", "tactile-ui-components", "docs_developer", "Frontend", 24, "Tactile UI 组件库", "Tactile UI Components"),
@@ -52,7 +55,8 @@ DOCS_METADATA: tuple[DocsMetadata, ...] = (
DocsMetadata("datasource-collector-settings-connectivity.md", "datasource-collector-settings-connectivity", "docs_developer", "Backend", 32, "数据源、采集器设置与连接验证", "Datasource Collector Settings and Connectivity"),
DocsMetadata("backend-datasources-api-performance.md", "backend-datasources-api-performance", "docs_developer", "Backend", 33, "数据源 API 性能", "Datasource API Performance"),
DocsMetadata("data-job-earth-sync-architecture.md", "data-job-earth-sync-architecture", "docs_developer", "Backend", 34, "数据作业与 Outbox 技术架构", "Data Jobs and Outbox Architecture"),
DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 35, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"),
DocsMetadata("backend-enum-contracts.md", "backend-enum-contracts", "docs_developer", "Backend", 35, "后端枚举与字符串兼容契约", "Backend Enum and String Compatibility Contract"),
DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 36, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"),
DocsMetadata("earth-news-live-streams-collector-format.md", "earth-news-live-streams-collector-format", "docs_developer", "Backend", 36, "新闻直播采集格式", "News Live Streams Collector Format"),
DocsMetadata("docs-gatekeeper-development.md", "docs-gatekeeper-development", "docs_developer", "Backend", 37, "Docs Gatekeeper 开发说明", "Docs Gatekeeper Development Guide"),
DocsMetadata("agents-aiprovider.md", "agents-aiprovider", "docs_developer", "Agents", 40, "AI Provider 指南", "AI Provider Guide"),
@@ -69,9 +73,9 @@ def get_user_gatekeeper_groups(user: User | None) -> set[str]:
return set()
role = user.role.value if hasattr(user.role, "value") else str(user.role or "")
if role == "super_admin":
if role == UserRole.SUPER_ADMIN.value:
return {"docs_user", "docs_developer", "docs_admin"}
if role == "admin":
if role == UserRole.ADMIN.value:
return {"docs_user", "docs_developer", "docs_admin"}
groups = set()

View File

@@ -20,8 +20,8 @@ class EarthLayerAdapter:
EARTH_LAYER_ADAPTERS: tuple[EarthLayerAdapter, ...] = (
EarthLayerAdapter(
sources=frozenset({"barentswatch_vessels", "aisstream_vessels", "vessel_static", "vessel_position", "ais_raw_observations", "ais_source_health"}),
tables=frozenset({"vessel_static", "vessel_position", "ais_raw_observations", "ais_source_health"}),
sources=frozenset({"barentswatch_vessels", "aisstream_vessels", "vessel_static", "vessel_position", "vessel_current_state", "ais_raw_observations", "ais_source_health"}),
tables=frozenset({"vessel_static", "vessel_position", "vessel_current_state", "ais_raw_observations", "ais_source_health"}),
layers=("vessels",),
cache_patterns=("vessels*", "summary*"),
derived_models=("ais_raw_observations", "ais_conflict_records", "ais_source_health"),

View File

@@ -20,10 +20,27 @@ from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.countries import COUNTRY_VARIANTS_MAP, get_country_centroid, normalize_country
from app.core.enums import (
BreakingLevel,
BreakingScope,
BreakingSource,
NewsEnrichmentStatus,
NewsImportanceLevel,
NewsMarketImpact,
NewsSourceType,
NewsTaggingSource,
)
from app.models.system_setting import SystemSetting
from app.ai_tasks.prompts import EffectiveAIPrompt, get_effective_prompt
from app.schemas.ai import SituationalAnalysisRequest
from app.services.ai_client import AIProviderClient
from app.services.earth_news_classification import (
apply_news_classification as _apply_news_classification,
breaking_sort_rank as _breaking_sort_rank,
highest_breaking_level as _highest_breaking_level,
normalize_breaking_level as _normalize_breaking_level_enum,
normalize_breaking_scope as _normalize_breaking_scope_enum,
)
from app.services.location.resolvers.nominatim import build_default_nominatim_geocoder
@@ -69,7 +86,7 @@ class NewsFeedEndpoint:
id: str
name: str
url: str
type: str = "rss"
type: str = NewsSourceType.RSS.value
region: str = ""
enabled: bool = True
default_category: str = "other"
@@ -85,7 +102,7 @@ class NewsFeedSource:
feed_url: str
homepage_url: str
feed_directory_url: str = ""
source_type: str = "rss"
source_type: str = NewsSourceType.RSS.value
feed_urls: tuple[str, ...] = ()
feeds: tuple[NewsFeedEndpoint, ...] = ()
priority: int = 100
@@ -120,7 +137,7 @@ class ParsedNewsItem:
published_at: datetime | None
content_language: str = "en"
localizations: dict[str, dict[str, str]] = field(default_factory=dict)
enrichment_status: str = "pending"
enrichment_status: str = NewsEnrichmentStatus.PENDING.value
enrichment_error: str | None = None
enriched_at: datetime | None = None
target_location: NewsTargetLocation | None = None
@@ -132,16 +149,22 @@ class ParsedNewsItem:
location_patch: dict[str, Any] | None = None
source_tags: list[str] = field(default_factory=list)
feed_id: str = ""
feed_type: str = "rss"
feed_type: str = NewsSourceType.RSS.value
feed_default_category: str = "other"
category: str = "other"
item_tags: list[str] = field(default_factory=list)
tagging_source: str = "rules"
tagging_source: str = NewsTaggingSource.RULES.value
tagging_confidence: float = 0.0
importance_score: int = 0
importance_level: str = "low"
importance_level: str = NewsImportanceLevel.LOW.value
importance_reasons: list[str] = field(default_factory=list)
market_impact: str = "none"
market_impact: str = NewsMarketImpact.NONE.value
breaking_level: str = BreakingLevel.NONE.value
breaking_scope: str = BreakingScope.REGIONAL.value
breaking_reasons: list[str] = field(default_factory=list)
breaking_source: str = BreakingSource.RULES.value
breaking_confidence: float = 0.0
breaking_expires_at: datetime | None = None
@dataclass
@@ -1863,34 +1886,12 @@ def _parse_feed_entries(
return items
def _contains_keyword(text: str, keyword: str) -> bool:
keyword_text = str(keyword or "").strip().lower()
if not keyword_text:
return False
if re.search(r"[\u4e00-\u9fff]", keyword_text):
return keyword_text in text
return re.search(rf"(?<![a-z0-9]){re.escape(keyword_text)}(?![a-z0-9])", text) is not None
def _normalize_breaking_level(value: str | None) -> str:
return _normalize_breaking_level_enum(value).value
def _score_category(text: str, title_text: str, category: dict[str, Any]) -> int:
score = 0
keywords = category.get("keywords") if isinstance(category.get("keywords"), list) else []
for keyword in keywords:
if _contains_keyword(title_text, keyword):
score += 3
elif _contains_keyword(text, keyword):
score += 1
return score
def _importance_level(score: int) -> str:
if score >= 80:
return "critical"
if score >= 60:
return "high"
if score >= 35:
return "medium"
return "low"
def _normalize_breaking_scope(value: str | None) -> str:
return _normalize_breaking_scope_enum(value).value
def apply_news_classification(
@@ -1901,74 +1902,7 @@ def apply_news_classification(
config_payload: dict[str, Any] | None = None,
) -> ParsedNewsItem:
config = normalize_earth_news_sources_payload(config_payload)
title_text = item.title.lower()
combined_text = f"{item.title} {item.summary} {item.source} {item.feed_name}".lower()
feed_default_category = (feed.default_category if feed else item.feed_default_category) or source.default_category or "other"
best_key = feed_default_category
best_score = 0
second_score = 0
for category in config["categories"]:
if not isinstance(category, dict) or category.get("enabled") is False:
continue
score = _score_category(combined_text, title_text, category)
if score > best_score:
second_score = best_score
best_score = score
best_key = str(category.get("key") or "other")
elif score > second_score:
second_score = score
item_tags: list[str] = []
for rule in config["item_tag_rules"]:
if not isinstance(rule, dict):
continue
keywords = rule.get("keywords") if isinstance(rule.get("keywords"), list) else []
if any(_contains_keyword(combined_text, keyword) for keyword in keywords):
tag_key = str(rule.get("key") or "").strip()
if tag_key and tag_key not in item_tags:
item_tags.append(tag_key)
if best_score < 3 and rule.get("category"):
best_key = str(rule["category"])
best_score = 3
confidence = round(best_score / (best_score + second_score + 1), 2) if best_score else 0.35
if best_score < 3 and feed_default_category:
best_key = feed_default_category
confidence = 0.45
importance_score = max(0, min(100, 18 + source.importance_weight + best_score * 6))
reasons: list[str] = []
source_tag_set = set(source.source_tags)
if "official_data" in source_tag_set:
importance_score += 20
reasons.append("官方数据源")
if "press_release" in source_tag_set:
importance_score = max(0, importance_score - 12)
reasons.append("企业公告基础权重较低")
ecommerce_terms = ("网上零售额", "电商物流指数", "gmv", "订单量", "物流指数", "履约", "直播电商", "跨境电商")
if any(_contains_keyword(combined_text, term) for term in ecommerce_terms):
importance_score += 25
reasons.append("命中电商数据指标")
major_platforms = ("amazon", "shopify", "walmart", "alibaba", "jd.com", "pinduoduo", "tiktok shop", "shein", "阿里", "京东", "拼多多", "抖音")
if any(_contains_keyword(combined_text, term) for term in major_platforms):
importance_score += 15
reasons.append("涉及大型平台")
if any(term in combined_text for term in ("同比", "环比", "%", "billion", "million", "增长", "下降")):
importance_score += 10
reasons.append("包含量化指标")
importance_score = max(0, min(100, importance_score))
item.category = best_key or "other"
item.item_tags = item_tags
item.tagging_source = "rules"
item.tagging_confidence = confidence
item.importance_score = importance_score
item.importance_level = _importance_level(importance_score)
item.importance_reasons = reasons or ["按来源权重和分类规则计算"]
item.market_impact = "global" if "global" in source_tag_set else "national" if {"china", "us"} & source_tag_set else "sector"
item.source_tags = list(source.source_tags)
return item
return _apply_news_classification(item, source, feed=feed, config=config)
def _serialize_sources(sources: list[NewsFeedSource]) -> list[dict[str, Any]]:
@@ -2020,6 +1954,10 @@ def _serialize_enriched_at(value: datetime | None) -> str | None:
return value.isoformat().replace("+00:00", "Z") if value else None
def _serialize_breaking_expires_at(value: datetime | None) -> str | None:
return value.isoformat().replace("+00:00", "Z") if value else None
def _content_patch(item: ParsedNewsItem) -> dict[str, Any]:
return {
"content_language": item.content_language,
@@ -2047,6 +1985,12 @@ def _news_meta_patch(item: ParsedNewsItem) -> dict[str, Any]:
"importance_level": item.importance_level,
"importance_reasons": list(item.importance_reasons or []),
"market_impact": item.market_impact,
"breaking_level": _normalize_breaking_level(item.breaking_level),
"breaking_scope": _normalize_breaking_scope(item.breaking_scope),
"breaking_reasons": list(item.breaking_reasons or []),
"breaking_source": item.breaking_source,
"breaking_confidence": item.breaking_confidence,
"breaking_expires_at": _serialize_breaking_expires_at(item.breaking_expires_at),
}
@@ -2142,6 +2086,12 @@ def build_target_location_job_payload(item: ParsedNewsItem) -> dict[str, Any]:
"importance_level": item.importance_level,
"importance_reasons": list(item.importance_reasons or []),
"market_impact": item.market_impact,
"breaking_level": _normalize_breaking_level(item.breaking_level),
"breaking_scope": _normalize_breaking_scope(item.breaking_scope),
"breaking_reasons": list(item.breaking_reasons or []),
"breaking_source": item.breaking_source,
"breaking_confidence": item.breaking_confidence,
"breaking_expires_at": _serialize_breaking_expires_at(item.breaking_expires_at),
}
@@ -2173,6 +2123,12 @@ def parsed_news_item_from_job_payload(payload: dict[str, Any]) -> ParsedNewsItem
importance_level=str(payload.get("importance_level") or "low"),
importance_reasons=list(payload.get("importance_reasons") or []),
market_impact=str(payload.get("market_impact") or "none"),
breaking_level=_normalize_breaking_level(str(payload.get("breaking_level") or "none")),
breaking_scope=_normalize_breaking_scope(str(payload.get("breaking_scope") or "regional")),
breaking_reasons=list(payload.get("breaking_reasons") or []),
breaking_source=str(payload.get("breaking_source") or "rules"),
breaking_confidence=float(payload.get("breaking_confidence") or 0),
breaking_expires_at=_parse_datetime(_coerce_str(payload.get("breaking_expires_at"))),
)
@@ -2218,6 +2174,12 @@ def _serialize_item(item: ParsedNewsItem, *, active_region: str, locale: str = D
"importance_level": item.importance_level,
"importance_reasons": list(item.importance_reasons or []),
"market_impact": item.market_impact,
"breaking_level": _normalize_breaking_level(item.breaking_level),
"breaking_scope": _normalize_breaking_scope(item.breaking_scope),
"breaking_reasons": list(item.breaking_reasons or []),
"breaking_source": item.breaking_source,
"breaking_confidence": item.breaking_confidence,
"breaking_expires_at": _serialize_breaking_expires_at(item.breaking_expires_at),
}
@@ -2239,6 +2201,9 @@ def _build_payload(
) -> dict[str, Any]:
profile = get_region_profile(active_region)
timestamp = generated_at or datetime.now(UTC)
visible_items = list(items)
cruise_visible_items = list(cruise_items if cruise_items is not None else items)
highest_breaking_level = _highest_breaking_level(visible_items + cruise_visible_items)
return {
"generated_at": timestamp.isoformat().replace("+00:00", "Z"),
"focus": {
@@ -2256,11 +2221,13 @@ def _build_payload(
"sources": sorted(source_ids or []),
"limit": limit,
"locale": locale,
"has_breaking": highest_breaking_level != "none",
"highest_breaking_level": highest_breaking_level,
},
"items": [_serialize_item(item, active_region=active_region, locale=locale) for item in items],
"items": [_serialize_item(item, active_region=active_region, locale=locale) for item in visible_items],
"cruise_items": [
_serialize_item(item, active_region=active_region, locale=locale)
for item in (cruise_items if cruise_items is not None else items)
for item in cruise_visible_items
],
"errors": errors,
"stale": stale,
@@ -2282,7 +2249,11 @@ def _rank_and_trim_items(
return sorted(
deduped.values(),
key=lambda item: (
False if active_region == "global" else item.feed_region != active_region,
-_breaking_sort_rank(item),
False
if active_region == "global"
or (_breaking_sort_rank(item) > 0 and _normalize_breaking_scope(item.breaking_scope) == "global")
else item.feed_region != active_region,
item.published_at is None,
-(item.published_at.timestamp() if item.published_at else 0),
item.feed_name,

View File

@@ -0,0 +1,258 @@
"""Classification, importance, and breaking-news policy for Earth news."""
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
import re
from typing import Any, Protocol
from app.core.enums import (
BreakingLevel,
BreakingScope,
BreakingSource,
NewsImportanceLevel,
NewsMarketImpact,
NewsTaggingSource,
parse_enum,
)
class NewsItemLike(Protocol):
title: str
summary: str
source: str
feed_name: str
published_at: datetime | None
feed_default_category: str
category: str
item_tags: list[str]
tagging_source: str
tagging_confidence: float
importance_score: int
importance_level: str
importance_reasons: list[str]
market_impact: str
source_tags: list[str]
breaking_level: str
breaking_scope: str
breaking_reasons: list[str]
breaking_source: str
breaking_confidence: float
breaking_expires_at: datetime | None
class NewsSourceLike(Protocol):
default_category: str
importance_weight: int
source_tags: tuple[str, ...]
class NewsFeedLike(Protocol):
default_category: str
@dataclass(frozen=True)
class BreakingRule:
level: BreakingLevel
scope: BreakingScope
reason: str
keywords: tuple[str, ...]
IMPORTANCE_THRESHOLDS: tuple[tuple[int, NewsImportanceLevel], ...] = (
(80, NewsImportanceLevel.CRITICAL),
(60, NewsImportanceLevel.HIGH),
(35, NewsImportanceLevel.MEDIUM),
(0, NewsImportanceLevel.LOW),
)
BREAKING_LEVEL_RANK: dict[BreakingLevel, int] = {
BreakingLevel.NONE: 0,
BreakingLevel.WATCH: 1,
BreakingLevel.BREAKING: 2,
BreakingLevel.CRITICAL: 3,
}
BREAKING_TTL: dict[BreakingLevel, timedelta] = {
BreakingLevel.WATCH: timedelta(hours=6),
BreakingLevel.BREAKING: timedelta(hours=12),
BreakingLevel.CRITICAL: timedelta(hours=24),
}
BREAKING_RULES: tuple[BreakingRule, ...] = (
BreakingRule(BreakingLevel.CRITICAL, BreakingScope.GLOBAL, "核事故或核风险", ("nuclear accident", "nuclear emergency", "radiation leak", "核事故", "核泄漏", "辐射泄漏")),
BreakingRule(BreakingLevel.CRITICAL, BreakingScope.REGIONAL, "重大军事冲突升级", ("airstrike", "missile strike", "invasion", "martial law", "空袭", "导弹袭击", "入侵", "戒严")),
BreakingRule(BreakingLevel.BREAKING, BreakingScope.REGIONAL, "战争或安全事件", ("war escalates", "terror attack", "coup", "hostage", "战争升级", "恐袭", "政变", "人质")),
BreakingRule(BreakingLevel.BREAKING, BreakingScope.REGIONAL, "重大灾害应急", ("major earthquake", "tsunami", "volcanic eruption", "state of emergency", "强震", "海啸", "火山喷发", "紧急状态")),
BreakingRule(BreakingLevel.BREAKING, BreakingScope.GLOBAL, "金融市场异常", ("market halt", "trading halt", "flash crash", "bank run", "金融熔断", "交易暂停", "银行挤兑")),
BreakingRule(BreakingLevel.WATCH, BreakingScope.GLOBAL, "大规模网络安全事件", ("massive cyberattack", "ransomware attack", "data breach", "大规模网络攻击", "勒索软件", "数据泄露")),
BreakingRule(BreakingLevel.WATCH, BreakingScope.REGIONAL, "航天或卫星事故", ("rocket explosion", "satellite collision", "space station emergency", "火箭爆炸", "卫星碰撞", "空间站事故")),
)
def contains_keyword(text: str, keyword: str) -> bool:
keyword_text = str(keyword or "").strip().lower()
if not keyword_text:
return False
if re.search(r"[\u4e00-\u9fff]", keyword_text):
return keyword_text in text
return re.search(rf"(?<![a-z0-9]){re.escape(keyword_text)}(?![a-z0-9])", text) is not None
def score_category(text: str, title_text: str, category: dict[str, Any]) -> int:
score = 0
keywords = category.get("keywords") if isinstance(category.get("keywords"), list) else []
for keyword in keywords:
if contains_keyword(title_text, keyword):
score += 3
elif contains_keyword(text, keyword):
score += 1
return score
def importance_level(score: int) -> NewsImportanceLevel:
normalized_score = max(0, min(100, int(score)))
for threshold, level in IMPORTANCE_THRESHOLDS:
if normalized_score >= threshold:
return level
return NewsImportanceLevel.LOW
def normalize_breaking_level(value: object) -> BreakingLevel:
return parse_enum(BreakingLevel, value, BreakingLevel.NONE)
def normalize_breaking_scope(value: object) -> BreakingScope:
return parse_enum(BreakingScope, value, BreakingScope.REGIONAL)
def breaking_expires_at(level: object, published_at: datetime | None) -> datetime | None:
normalized = normalize_breaking_level(level)
if normalized is BreakingLevel.NONE:
return None
base = published_at or datetime.now(UTC)
base = base.replace(tzinfo=UTC) if base.tzinfo is None else base.astimezone(UTC)
return base + BREAKING_TTL[normalized]
def is_breaking_active(item: NewsItemLike, *, now: datetime | None = None) -> bool:
if normalize_breaking_level(item.breaking_level) is BreakingLevel.NONE:
return False
expires_at = item.breaking_expires_at
if expires_at is None:
return True
expires_at = expires_at.replace(tzinfo=UTC) if expires_at.tzinfo is None else expires_at.astimezone(UTC)
return expires_at > (now or datetime.now(UTC))
def breaking_sort_rank(item: NewsItemLike) -> int:
if not is_breaking_active(item):
return 0
return BREAKING_LEVEL_RANK[normalize_breaking_level(item.breaking_level)]
def highest_breaking_level(items: list[NewsItemLike]) -> BreakingLevel:
active = [normalize_breaking_level(item.breaking_level) for item in items if is_breaking_active(item)]
return max(active, key=BREAKING_LEVEL_RANK.get) if active else BreakingLevel.NONE
def apply_breaking_rules(item: NewsItemLike) -> None:
combined_text = f"{item.title} {item.summary} {item.source} {item.feed_name}".lower()
best_level = BreakingLevel.NONE
best_scope = BreakingScope.REGIONAL
reasons: list[str] = []
confidence = 0.0
for rule in BREAKING_RULES:
if not any(contains_keyword(combined_text, keyword) for keyword in rule.keywords):
continue
if BREAKING_LEVEL_RANK[rule.level] > BREAKING_LEVEL_RANK[best_level]:
best_level = rule.level
best_scope = rule.scope
if rule.reason not in reasons:
reasons.append(rule.reason)
confidence = max(confidence, 0.72 if rule.level is BreakingLevel.CRITICAL else 0.64 if rule.level is BreakingLevel.BREAKING else 0.52)
item.breaking_level = best_level.value
item.breaking_scope = (best_scope if best_level is not BreakingLevel.NONE else BreakingScope.REGIONAL).value
item.breaking_reasons = reasons
item.breaking_source = BreakingSource.RULES.value
item.breaking_confidence = round(confidence, 2)
item.breaking_expires_at = breaking_expires_at(best_level, item.published_at)
def apply_news_classification(
item: NewsItemLike,
source: NewsSourceLike,
*,
feed: NewsFeedLike | None,
config: dict[str, Any],
) -> NewsItemLike:
title_text = item.title.lower()
combined_text = f"{item.title} {item.summary} {item.source} {item.feed_name}".lower()
feed_default_category = (feed.default_category if feed else item.feed_default_category) or source.default_category or "other"
best_key = feed_default_category
best_score = second_score = 0
for category in config["categories"]:
if not isinstance(category, dict) or category.get("enabled") is False:
continue
score = score_category(combined_text, title_text, category)
if score > best_score:
second_score, best_score = best_score, score
best_key = str(category.get("key") or "other")
elif score > second_score:
second_score = score
item_tags: list[str] = []
for rule in config["item_tag_rules"]:
if not isinstance(rule, dict):
continue
keywords = rule.get("keywords") if isinstance(rule.get("keywords"), list) else []
if any(contains_keyword(combined_text, keyword) for keyword in keywords):
tag_key = str(rule.get("key") or "").strip()
if tag_key and tag_key not in item_tags:
item_tags.append(tag_key)
if best_score < 3 and rule.get("category"):
best_key, best_score = str(rule["category"]), 3
confidence = round(best_score / (best_score + second_score + 1), 2) if best_score else 0.35
if best_score < 3 and feed_default_category:
best_key, confidence = feed_default_category, 0.45
score = max(0, min(100, 18 + source.importance_weight + best_score * 6))
reasons: list[str] = []
source_tags = set(source.source_tags)
if "official_data" in source_tags:
score += 20
reasons.append("官方数据源")
if "press_release" in source_tags:
score = max(0, score - 12)
reasons.append("企业公告基础权重较低")
if any(contains_keyword(combined_text, term) for term in ("网上零售额", "电商物流指数", "gmv", "订单量", "物流指数", "履约", "直播电商", "跨境电商")):
score += 25
reasons.append("命中电商数据指标")
if any(contains_keyword(combined_text, term) for term in ("amazon", "shopify", "walmart", "alibaba", "jd.com", "pinduoduo", "tiktok shop", "shein", "阿里", "京东", "拼多多", "抖音")):
score += 15
reasons.append("涉及大型平台")
if any(term in combined_text for term in ("同比", "环比", "%", "billion", "million", "增长", "下降")):
score += 10
reasons.append("包含量化指标")
score = max(0, min(100, score))
item.category = best_key or "other"
item.item_tags = item_tags
item.tagging_source = NewsTaggingSource.RULES.value
item.tagging_confidence = confidence
item.importance_score = score
item.importance_level = importance_level(score).value
item.importance_reasons = reasons or ["按来源权重和分类规则计算"]
item.market_impact = (
NewsMarketImpact.GLOBAL.value
if "global" in source_tags
else NewsMarketImpact.NATIONAL.value
if {"china", "us"} & source_tags
else NewsMarketImpact.SECTOR.value
)
item.source_tags = list(source.source_tags)
apply_breaking_rules(item)
return item

View File

@@ -3,7 +3,7 @@ from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import func, select
from sqlalchemy import func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.earth_news import EarthNewsItem
@@ -13,6 +13,11 @@ from app.services.earth_news import (
build_anchor_location_patch,
_news_meta_patch,
)
from app.services.earth_news_classification import (
breaking_sort_rank,
normalize_breaking_level,
normalize_breaking_scope,
)
def _coerce_datetime(value: datetime | None) -> datetime | None:
@@ -23,6 +28,17 @@ def _coerce_datetime(value: datetime | None) -> datetime | None:
return value.astimezone(UTC)
def _coerce_meta_datetime(value: Any) -> datetime | None:
if isinstance(value, datetime):
return _coerce_datetime(value)
if not isinstance(value, str) or not value.strip():
return None
try:
return _coerce_datetime(datetime.fromisoformat(value.replace("Z", "+00:00")))
except ValueError:
return None
def _location_patch_from_record(record: EarthNewsItem) -> dict[str, Any]:
return {
"latitude": record.latitude,
@@ -64,10 +80,32 @@ def record_to_parsed_news_item(record: EarthNewsItem) -> ParsedNewsItem:
importance_level=str(news_meta.get("importance_level") or "low"),
importance_reasons=list(news_meta.get("importance_reasons") or []),
market_impact=str(news_meta.get("market_impact") or "none"),
breaking_level=normalize_breaking_level(news_meta.get("breaking_level")).value,
breaking_scope=normalize_breaking_scope(news_meta.get("breaking_scope")).value,
breaking_reasons=list(news_meta.get("breaking_reasons") or []),
breaking_source=str(news_meta.get("breaking_source") or "rules"),
breaking_confidence=float(news_meta.get("breaking_confidence") or 0),
breaking_expires_at=_coerce_meta_datetime(news_meta.get("breaking_expires_at")),
)
return apply_enrichment_patch_to_item(item, _location_patch_from_record(record))
def _sort_parsed_news_items(items: list[ParsedNewsItem], *, active_region: str) -> list[ParsedNewsItem]:
return sorted(
items,
key=lambda item: (
-breaking_sort_rank(item),
False
if active_region == "global"
or (breaking_sort_rank(item) > 0 and normalize_breaking_scope(item.breaking_scope).value == "global")
else item.feed_region != active_region,
item.published_at is None,
-(item.published_at.timestamp() if item.published_at else 0),
item.feed_name,
),
)
def _query_sort_key(active_region: str):
if active_region == "global":
return (
@@ -139,14 +177,20 @@ async def list_earth_news_items(
categories: set[str] | None = None,
source_ids: set[str] | None = None,
) -> list[ParsedNewsItem]:
query_limit = limit if source_ids else min(max(limit * 4, limit), 100)
query_limit = limit if source_ids else min(max(limit * 8, limit), 200)
query = (
select(EarthNewsItem)
.order_by(*_query_sort_key(active_region))
.limit(query_limit)
)
if active_region != "global":
query = query.where(EarthNewsItem.region.in_({"global", active_region}))
news_meta = EarthNewsItem.location_meta.op("->")("news_meta")
query = query.where(
or_(
EarthNewsItem.region.in_({"global", active_region}),
news_meta.op("->>")("breaking_scope") == "global",
)
)
category_clause = _category_filter_clause(categories)
if category_clause is not None:
query = query.where(category_clause)
@@ -155,11 +199,11 @@ async def list_earth_news_items(
query = query.where(source_clause)
result = await db.execute(query)
records = list(result.scalars().all())
if not source_ids:
records = _diversify_records_by_source(records, limit=limit)
else:
records = records[:limit]
return [record_to_parsed_news_item(record) for record in records]
items = _sort_parsed_news_items(
[record_to_parsed_news_item(record) for record in records],
active_region=active_region,
)
return items[:limit]
async def list_earth_news_cruise_items(
@@ -177,7 +221,7 @@ async def list_earth_news_cruise_items(
EarthNewsItem.last_seen_at.desc(),
EarthNewsItem.feed_name.asc(),
)
.limit(limit)
.limit(min(max(limit * 4, limit), 200))
)
category_clause = _category_filter_clause(categories)
if category_clause is not None:
@@ -186,7 +230,11 @@ async def list_earth_news_cruise_items(
if source_clause is not None:
query = query.where(source_clause)
result = await db.execute(query)
return [record_to_parsed_news_item(record) for record in result.scalars().all()]
items = _sort_parsed_news_items(
[record_to_parsed_news_item(record) for record in result.scalars().all()],
active_region="global",
)
return items[:limit]
async def get_earth_news_freshness(

View File

@@ -9,12 +9,12 @@ yet to keep behavior obvious after settings changes).
from __future__ import annotations
from email.message import EmailMessage
from typing import Literal, Optional
from typing import Optional
import aiosmtplib
from sqlalchemy.ext.asyncio import AsyncSession
OtpPurpose = Literal["register", "verify_email", "reset_password"]
from app.core.enums import OtpPurpose
class EmailError(Exception):
@@ -81,15 +81,15 @@ async def send_email(
_SUBJECTS: dict[OtpPurpose, str] = {
"register": "Confirm your Planet account",
"verify_email": "Verify your Planet email",
"reset_password": "Reset your Planet password",
OtpPurpose.REGISTER: "Confirm your Planet account",
OtpPurpose.VERIFY_EMAIL: "Verify your Planet email",
OtpPurpose.RESET_PASSWORD: "Reset your Planet password",
}
_HEADLINES: dict[OtpPurpose, str] = {
"register": "Welcome to Planet — confirm your email to activate your account.",
"verify_email": "Confirm your new email address to keep your Planet account active.",
"reset_password": "Use this code to set a new password for your Planet account.",
OtpPurpose.REGISTER: "Welcome to Planet — confirm your email to activate your account.",
OtpPurpose.VERIFY_EMAIL: "Confirm your new email address to keep your Planet account active.",
OtpPurpose.RESET_PASSWORD: "Use this code to set a new password for your Planet account.",
}

View File

@@ -9,14 +9,12 @@ from __future__ import annotations
import json
import secrets
from typing import Literal
import bcrypt
from app.core.enums import OtpPurpose
from app.core.security import redis_client
OtpPurpose = Literal["register", "verify_email", "reset_password"]
CODE_TTL_SECONDS = 600 # 10 minutes
RESEND_COOLDOWN_SECONDS = 60
MAX_ATTEMPTS = 5

View File

@@ -7,9 +7,14 @@ from time import perf_counter
from uuid import uuid4
from fastapi import HTTPException, status
from sqlalchemy import func, select, update
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.enums import (
PlaygroundMessageKind,
PlaygroundMessageRole,
PlaygroundMessageStatus,
)
from app.core.logging import get_logger
from app.db.session import async_session_factory
from app.models.playground_message import PlaygroundMessage
@@ -21,7 +26,6 @@ from app.schemas.ai import (
PlaygroundMessageRecord,
PlaygroundMessageResendRequest,
PlaygroundMessageStopRequest,
PlaygroundSessionResponse,
PlaygroundSessionState,
PlaygroundSessionUpsertRequest,
PlaygroundThreadResponse,
@@ -37,6 +41,13 @@ STREAM_CHUNK_SIZE = 24
STREAM_INTERVAL_SECONDS = 0.08
THINKING_PREVIEW_SECONDS = 2.6
ORPHANED_RUN_MESSAGE = "后台生成任务已中断,请点击上一条用户消息的重试按钮重新生成。"
ACTIVE_MESSAGE_STATUSES = frozenset(
{
PlaygroundMessageStatus.PENDING.value,
PlaygroundMessageStatus.THINKING.value,
PlaygroundMessageStatus.ANSWERING.value,
}
)
class _ActiveRun:
@@ -93,7 +104,11 @@ async def _require_visible_message(
result = await db.execute(select(PlaygroundMessage).where(*conditions))
message = result.scalar_one_or_none()
if message is None:
detail = "User message not found" if role == "user" else "Playground message not found"
detail = (
"User message not found"
if role == PlaygroundMessageRole.USER.value
else "Playground message not found"
)
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=detail)
return message
@@ -108,7 +123,7 @@ def _message_to_record(message: PlaygroundMessage, parent_public_id: str | None
content=message.content or "",
thinking_content=message.thinking_content or "",
meta=list(message.meta or []),
markdown=message.role != "system",
markdown=message.role != PlaygroundMessageRole.SYSTEM.value,
provider=message.provider,
model=message.model,
request_id=message.request_id,
@@ -197,11 +212,11 @@ async def _reconcile_orphaned_active_messages(
) -> list[PlaygroundMessage]:
changed = False
for item in messages:
if item.status not in {"pending", "thinking", "answering"}:
if item.status not in ACTIVE_MESSAGE_STATUSES:
continue
if item.public_id in _ACTIVE_RUNS:
continue
item.status = "error"
item.status = PlaygroundMessageStatus.ERROR.value
item.content = item.content or ORPHANED_RUN_MESSAGE
orphan_meta = "错误: 后台任务已中断"
if orphan_meta not in (item.meta or []):
@@ -330,9 +345,9 @@ async def create_turn(
public_id=uuid4().hex,
session_id=session.id,
user_id=user_id,
role="user",
kind="message",
status="done",
role=PlaygroundMessageRole.USER.value,
kind=PlaygroundMessageKind.MESSAGE.value,
status=PlaygroundMessageStatus.DONE.value,
title=payload.selected_preset_key,
content=payload.input,
meta=[payload.title],
@@ -343,9 +358,9 @@ async def create_turn(
session_id=session.id,
user_id=user_id,
parent_message_id=None,
role="assistant",
kind="thinking",
status="pending",
role=PlaygroundMessageRole.ASSISTANT.value,
kind=PlaygroundMessageKind.THINKING.value,
status=PlaygroundMessageStatus.PENDING.value,
title="AI 回应",
content="",
thinking_content="",
@@ -397,9 +412,9 @@ async def _create_assistant_retry_turn(
session_id=session.id,
user_id=user_id,
parent_message_id=user_message.id,
role="assistant",
kind="thinking",
status="pending",
role=PlaygroundMessageRole.ASSISTANT.value,
kind=PlaygroundMessageKind.THINKING.value,
status=PlaygroundMessageStatus.PENDING.value,
title="AI 回应",
content="",
thinking_content="",
@@ -439,7 +454,7 @@ async def stop_message(
session = await _require_session(db, user_id=user_id, session_key=payload.session_key)
message = await _require_visible_message(db, user_id=user_id, public_id=payload.message_id)
if message.status not in {"pending", "thinking", "answering"}:
if message.status not in ACTIVE_MESSAGE_STATUSES:
return await _build_action_response(db, session=session)
active_run = _ACTIVE_RUNS.get(message.public_id)
@@ -447,7 +462,7 @@ async def stop_message(
active_run.stop_requested.set()
active_run.task.cancel()
message.status = "stopped"
message.status = PlaygroundMessageStatus.STOPPED.value
if "已手动停止生成" not in (message.meta or []):
message.meta = [*(message.meta or []), "已手动停止生成"]
await db.flush()
@@ -469,7 +484,7 @@ async def resend_turn(
db,
user_id=user_id,
public_id=payload.user_message_id,
role="user",
role=PlaygroundMessageRole.USER.value,
)
later_messages = await db.execute(
@@ -481,7 +496,7 @@ async def resend_turn(
)
for item in later_messages.scalars().all():
item.is_visible = False
if item.status in {"pending", "thinking", "answering"}:
if item.status in ACTIVE_MESSAGE_STATUSES:
active_run = _ACTIVE_RUNS.get(item.public_id)
if active_run is not None:
active_run.stop_requested.set()
@@ -520,7 +535,7 @@ async def edit_user_message(
db,
user_id=user_id,
public_id=payload.user_message_id,
role="user",
role=PlaygroundMessageRole.USER.value,
)
user_message.content = payload.content.strip()
@@ -566,12 +581,12 @@ def _build_conversation_history(messages: Sequence[PlaygroundMessage], current_u
for item in messages:
if item.id >= current_user_message_id:
break
if item.role == "system":
if item.role == PlaygroundMessageRole.SYSTEM.value:
continue
history.append(
{
"role": item.role,
"kind": item.kind or "message",
"kind": item.kind or PlaygroundMessageKind.MESSAGE.value,
"title": item.title,
"content": item.content or "",
}
@@ -668,7 +683,11 @@ async def _run_assistant_message(
assistant_message = await _mark_message_state(
db,
message_id=assistant_message_id,
status="thinking" if analysis.thinking_blocks else "answering",
status=(
PlaygroundMessageStatus.THINKING.value
if analysis.thinking_blocks
else PlaygroundMessageStatus.ANSWERING.value
),
title=f"{analysis.provider} / {analysis.model}",
provider=analysis.provider,
model=analysis.model,
@@ -706,7 +725,7 @@ async def _run_assistant_message(
await _mark_message_state(
db,
message_id=assistant_message_id,
status="answering",
status=PlaygroundMessageStatus.ANSWERING.value,
content=content[:cursor],
)
await db.commit()
@@ -717,7 +736,7 @@ async def _run_assistant_message(
assistant_message = await _mark_message_state(
db,
message_id=assistant_message_id,
status="done",
status=PlaygroundMessageStatus.DONE.value,
content=content,
meta=[
f"Request ID: {request_id}",
@@ -762,8 +781,8 @@ async def _run_assistant_message(
async with async_session_factory() as db:
result = await db.execute(select(PlaygroundMessage).where(PlaygroundMessage.id == assistant_message_id))
message = result.scalar_one_or_none()
if message is not None and message.status in {"pending", "thinking", "answering"}:
message.status = "stopped"
if message is not None and message.status in ACTIVE_MESSAGE_STATUSES:
message.status = PlaygroundMessageStatus.STOPPED.value
if "已手动停止生成" not in (message.meta or []):
message.meta = [*(message.meta or []), "已手动停止生成"]
await db.flush()
@@ -795,7 +814,7 @@ async def _run_assistant_message(
result = await db.execute(select(PlaygroundMessage).where(PlaygroundMessage.id == assistant_message_id))
message = result.scalar_one_or_none()
if message is not None:
message.status = "error"
message.status = PlaygroundMessageStatus.ERROR.value
message.content = message.content or f"分析失败:{error_message}"
message.meta = [
*(message.meta or []),

View File

@@ -8,6 +8,7 @@ from apscheduler.schedulers.asyncio import AsyncIOScheduler
from apscheduler.triggers.interval import IntervalTrigger
from sqlalchemy import select
from app.core.enums import JobStatus
from app.core.logging import get_logger
from app.db.session import async_session_factory
from app.core.time import to_iso8601_utc
@@ -140,7 +141,7 @@ async def run_collector_task(collector_name: str):
select(CollectionTask)
.where(
CollectionTask.datasource_id == datasource.id,
CollectionTask.status == "running",
CollectionTask.status == JobStatus.RUNNING.value,
)
.order_by(CollectionTask.started_at.desc(), CollectionTask.id.desc())
.limit(1)
@@ -184,7 +185,7 @@ async def run_collector_task(collector_name: str):
f"Marked failed automatically after stale running timeout "
f"({RUNNING_TASK_GUARD_TIMEOUT_MINUTES}m) in scheduler guard"
)
existing_running.status = "failed"
existing_running.status = JobStatus.FAILED.value
existing_running.phase = "failed"
existing_running.completed_at = now
existing_running.error_message = (
@@ -243,7 +244,7 @@ async def run_collector_task(collector_name: str):
return
datasource.last_run_at = datetime.now(UTC)
datasource.last_status = task_result.get("status")
if datasource.last_status == "success":
if datasource.last_status == JobStatus.SUCCESS.value:
effective_candidate = await get_builtin_effective_candidate(db, datasource_source)
checksum, _credential_context = await build_builtin_connectivity_checksum(
datasource_source,
@@ -284,7 +285,7 @@ async def run_collector_task(collector_name: str):
await db.rollback()
datasource = await db.get(DataSource, datasource_id)
datasource.last_run_at = datetime.now(UTC)
datasource.last_status = "cancelled"
datasource.last_status = JobStatus.CANCELLED.value
await db.commit()
logger.warning_event(
"Collector cancelled by operator",
@@ -306,7 +307,7 @@ async def run_collector_task(collector_name: str):
await db.rollback()
datasource = await db.get(DataSource, datasource_id)
datasource.last_run_at = datetime.now(UTC)
datasource.last_status = "failed"
datasource.last_status = JobStatus.FAILED.value
await db.commit()
logger.exception_event(
"Collector failed",
@@ -335,7 +336,7 @@ async def cleanup_stale_running_tasks(max_age_hours: int = 2) -> int:
async with async_session_factory() as db:
result = await db.execute(
select(CollectionTask).where(
CollectionTask.status == "running",
CollectionTask.status == JobStatus.RUNNING.value,
CollectionTask.started_at.is_not(None),
CollectionTask.started_at < cutoff,
)
@@ -343,7 +344,7 @@ async def cleanup_stale_running_tasks(max_age_hours: int = 2) -> int:
stale_tasks = result.scalars().all()
for task in stale_tasks:
task.status = "failed"
task.status = JobStatus.FAILED.value
task.phase = "failed"
task.completed_at = datetime.now(UTC)
existing_error = (task.error_message or "").strip()

View File

@@ -6,6 +6,7 @@ from typing import Any
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.enums import BGPStatus
from app.models.alert import Alert, AlertSeverity, AlertStatus
from app.models.bgp_anomaly import BGPAnomaly
from app.models.bgp_incident import BGPIncident
@@ -49,11 +50,11 @@ async def build_situational_alert_brief_request(
total_incidents_result = await db.execute(select(func.count(BGPIncident.id)))
active_incidents_result = await db.execute(
select(func.count(BGPIncident.id)).where(BGPIncident.status == "active")
select(func.count(BGPIncident.id)).where(BGPIncident.status == BGPStatus.ACTIVE.value)
)
bgp_severity_result = await db.execute(
select(BGPIncident.severity, func.count(BGPIncident.id))
.where(BGPIncident.status == "active")
.where(BGPIncident.status == BGPStatus.ACTIVE.value)
.group_by(BGPIncident.severity)
)
bgp_region_counter: Counter[str] = Counter()
@@ -65,11 +66,11 @@ async def build_situational_alert_brief_request(
total_anomalies_result = await db.execute(select(func.count(BGPAnomaly.id)))
active_anomalies_result = await db.execute(
select(func.count(BGPAnomaly.id)).where(BGPAnomaly.status == "active")
select(func.count(BGPAnomaly.id)).where(BGPAnomaly.status == BGPStatus.ACTIVE.value)
)
anomaly_type_result = await db.execute(
select(BGPAnomaly.anomaly_type, func.count(BGPAnomaly.id))
.where(BGPAnomaly.status == "active")
.where(BGPAnomaly.status == BGPStatus.ACTIVE.value)
.group_by(BGPAnomaly.anomaly_type)
.order_by(func.count(BGPAnomaly.id).desc())
.limit(6)

View File

@@ -7,6 +7,7 @@ from pathlib import Path
from typing import Any
from app.core.config import ROOT_DIR
from app.core.enums import UserRole
from app.core.security import redis_client
SYSTEM_TASK_TTL_SECONDS = 24 * 60 * 60
@@ -47,7 +48,7 @@ def normalize_user_role(role: Any) -> str:
def require_super_admin(user_role: Any) -> bool:
return normalize_user_role(user_role) == "super_admin"
return normalize_user_role(user_role) == UserRole.SUPER_ADMIN.value
def build_task_id(prefix: str = "restart") -> str:

View File

@@ -13,6 +13,7 @@ from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from app.core.enums import LogLevel
from app.core.security import redis_client
from app.models.system_log import AuditLog, ObservabilityEvent, ObservabilityEventGroup, SystemLog
from sqlalchemy import select
@@ -24,11 +25,11 @@ BUFFER_LOG_LIMIT = 1000
BUFFER_LOG_TTL_SECONDS = 7 * 24 * 60 * 60
LOG_BUFFER_KEY_PREFIX = "planet:system_logs"
LOG_LEVEL_ERROR = "error"
LOG_LEVEL_WARNING = "warning"
LOG_LEVEL_INFO = "info"
LOG_LEVEL_DEBUG = "debug"
LOG_LEVEL_ALL = "all"
LOG_LEVEL_ERROR = LogLevel.ERROR.value
LOG_LEVEL_WARNING = LogLevel.WARNING.value
LOG_LEVEL_INFO = LogLevel.INFO.value
LOG_LEVEL_DEBUG = LogLevel.DEBUG.value
LOG_LEVEL_ALL = LogLevel.ALL.value
SUPPORTED_LOG_LEVELS = {
LOG_LEVEL_ALL,

View File

@@ -9,7 +9,12 @@ from sqlalchemy import select
from sqlalchemy import Float
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth
from app.models.vessel import (
AISConflictRecord,
AISRawObservation,
AISSourceHealth,
VesselCurrentState,
)
from app.services.vessel_aggregation_strategy import (
DEFAULT_STRATEGY,
load_strategy,
@@ -44,6 +49,17 @@ CONFLICT_FIELDS = (
"width",
"draught",
)
CURRENT_STATE_STATIC_FIELDS = (
"name",
"callsign",
"vessel_type",
"vessel_type_name",
"flag",
"length",
"width",
"draught",
"imo",
)
def _json_default(value: Any) -> Any:
@@ -489,9 +505,115 @@ async def record_vessel_ais_observation(
quality_flags=quality_flags or [],
)
db.add(observation)
await upsert_vessel_current_state(
db,
source=source,
normalized_payload=normalized_json,
observed_at=observed_at,
quality_flags=quality_flags or [],
)
return observation
async def upsert_vessel_current_state(
db: AsyncSession,
*,
source: str,
normalized_payload: dict[str, Any],
observed_at: datetime,
quality_flags: list[str] | None = None,
) -> VesselCurrentState | None:
"""Keep one latest renderable row per MMSI while preserving useful static fields."""
if not _has_valid_position(normalized_payload):
return None
mmsi = int(normalized_payload["mmsi"])
current = await db.get(VesselCurrentState, mmsi)
if current is not None and current.observed_at is not None:
current_observed_at = _coerce_datetime(current.observed_at)
if current_observed_at is not None and observed_at < current_observed_at:
return current
if current is None:
current = VesselCurrentState(mmsi=mmsi)
db.add(current)
current.lat = float(normalized_payload["lat"])
current.lon = float(normalized_payload["lon"])
current.source = source
current.observed_at = observed_at
current.updated_at = datetime.now(UTC)
updated_fields: set[str] = {"lat", "lon"}
for field in DYNAMIC_FIELDS:
if field in {"lat", "lon"}:
continue
value = _payload_value(normalized_payload, field)
if value is not None:
setattr(current, field, value)
updated_fields.add(field)
field_sources = dict(current.field_sources or {})
for field in CURRENT_STATE_STATIC_FIELDS:
value = _payload_value(normalized_payload, field)
if value is None:
continue
existing_source = field_sources.get(field)
existing_value = getattr(current, field, None)
if (
existing_value in (None, "")
or _strategy_source_rank(source, DEFAULT_STRATEGY)
>= _strategy_source_rank(str(existing_source or ""), DEFAULT_STRATEGY)
):
setattr(current, field, value)
updated_fields.add(field)
current.vessel_type_name = current.vessel_type_name or normalize_vessel_type_name(
current.vessel_type
)
selected_reasons = dict(current.selected_reasons or {})
for field in updated_fields:
field_sources[field] = source
selected_reasons[field] = (
"newest_observation" if field in DYNAMIC_FIELDS else "source_priority"
)
current.field_sources = field_sources
current.selected_reasons = selected_reasons
current.source_summary = {
**dict(current.source_summary or {}),
source: {
"latest_observed_at": observed_at.isoformat(),
},
}
current.quality_flags = sorted(set((current.quality_flags or []) + (quality_flags or [])))
return current
async def get_current_vessels_snapshot(
db: AsyncSession,
*,
bbox: tuple[float, float, float, float],
limit: int = 1000,
observed_since: datetime,
) -> list[dict[str, Any]]:
"""Read the bounded latest-state table used by Earth rendering."""
safe_limit = min(max(int(limit or 1000), 1), MAX_SNAPSHOT_LIMIT)
lon_min, lat_min, lon_max, lat_max = bbox
stmt = (
select(VesselCurrentState)
.where(VesselCurrentState.observed_at >= observed_since)
.where(VesselCurrentState.lon >= lon_min)
.where(VesselCurrentState.lon <= lon_max)
.where(VesselCurrentState.lat >= lat_min)
.where(VesselCurrentState.lat <= lat_max)
.order_by(VesselCurrentState.observed_at.desc(), VesselCurrentState.mmsi.asc())
.limit(safe_limit)
)
result = await db.execute(stmt)
if not hasattr(result, "scalars"):
return []
return [item.to_dict() for item in result.scalars().all()]
async def aggregate_vessel_observations(
db: AsyncSession,
observations: Iterable[AISRawObservation],