release: bump version to 0.70.0
This commit is contained in:
@@ -6,6 +6,7 @@ from collections import Counter, defaultdict
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from app.core.enums import BGPStatus
|
||||
from app.models.bgp_anomaly import BGPAnomaly
|
||||
|
||||
|
||||
@@ -127,7 +128,7 @@ def detect_origin_change_anomalies(
|
||||
source=source,
|
||||
anomaly_type=anomaly_type,
|
||||
severity=severity,
|
||||
status="active",
|
||||
status=BGPStatus.ACTIVE.value,
|
||||
entity_key=f"{anomaly_type}:{prefix}:{new_origin}",
|
||||
prefix=prefix,
|
||||
origin_asn=sorted(historic)[0] if historic else None,
|
||||
@@ -197,7 +198,7 @@ def detect_more_specific_burst_anomalies(
|
||||
source=source,
|
||||
anomaly_type="more_specific_burst",
|
||||
severity="high",
|
||||
status="active",
|
||||
status=BGPStatus.ACTIVE.value,
|
||||
entity_key=f"more_specific_burst:{root_prefix}:{len(unique_prefixes)}:{len(related_collectors)}",
|
||||
prefix=sample.get("prefix"),
|
||||
origin_asn=sample.get("origin_asn"),
|
||||
@@ -267,7 +268,7 @@ def detect_mass_withdrawal_anomalies(
|
||||
source=source,
|
||||
anomaly_type="mass_withdrawal",
|
||||
severity=severity,
|
||||
status="active",
|
||||
status=BGPStatus.ACTIVE.value,
|
||||
entity_key=f"mass_withdrawal:{prefix}:{origin_asn}:{len(related_collectors)}:{count}",
|
||||
prefix=prefix,
|
||||
origin_asn=origin_asn,
|
||||
@@ -354,7 +355,7 @@ def detect_route_leak_anomalies(
|
||||
source=source,
|
||||
anomaly_type="route_leak_candidate",
|
||||
severity="high" if max_path_length >= dominant_length + 3 else "medium",
|
||||
status="active",
|
||||
status=BGPStatus.ACTIVE.value,
|
||||
entity_key=f"route_leak_candidate:{prefix}:{max_path_length}:{len(related_collectors)}",
|
||||
prefix=prefix,
|
||||
origin_asn=sample_metadata.get("origin_asn"),
|
||||
@@ -435,7 +436,7 @@ def detect_path_flap_anomalies(
|
||||
source=source,
|
||||
anomaly_type="path_flap",
|
||||
severity=severity,
|
||||
status="active",
|
||||
status=BGPStatus.ACTIVE.value,
|
||||
entity_key=f"path_flap:{prefix}:{transitions}:{len(distinct_paths)}",
|
||||
prefix=prefix,
|
||||
origin_asn=sample_metadata.get("origin_asn"),
|
||||
|
||||
@@ -9,6 +9,7 @@ from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.collected_data_fields import get_record_field
|
||||
from app.core.enums import BGPStatus
|
||||
from app.models.bgp_anomaly import BGPAnomaly
|
||||
from app.models.bgp_incident import BGPIncident
|
||||
from app.models.collected_data import CollectedData
|
||||
@@ -290,7 +291,7 @@ async def create_bgp_incidents_for_anomalies(
|
||||
existing.title = title
|
||||
existing.summary = summary
|
||||
existing.severity = severity
|
||||
existing.status = "active"
|
||||
existing.status = BGPStatus.ACTIVE.value
|
||||
existing.confidence = confidence
|
||||
existing.started_at = primary.started_at or existing.started_at or datetime.now(UTC)
|
||||
existing.ended_at = None
|
||||
@@ -313,7 +314,7 @@ async def create_bgp_incidents_for_anomalies(
|
||||
title=title,
|
||||
summary=summary,
|
||||
severity=severity,
|
||||
status="active",
|
||||
status=BGPStatus.ACTIVE.value,
|
||||
confidence=confidence,
|
||||
started_at=primary.started_at or datetime.now(UTC),
|
||||
affected_prefixes=prefixes,
|
||||
|
||||
@@ -12,11 +12,11 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.collected_data_fields import build_dynamic_metadata, get_record_field
|
||||
from app.core.countries import normalize_country
|
||||
from app.core.enums import JobStatus, SnapshotStatus
|
||||
from app.core.logging import get_logger
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
from app.services.business_logs import emit_business_log, exception_context
|
||||
from app.services.earth_layer_adapters import get_earth_update_layers_for_source
|
||||
|
||||
|
||||
logger = get_logger(__name__, service="collector")
|
||||
@@ -238,7 +238,7 @@ class BaseCollector(ABC):
|
||||
snapshot = await db.get(DataSnapshot, snapshot_id)
|
||||
if snapshot:
|
||||
parent_snapshot_id = snapshot.parent_snapshot_id
|
||||
snapshot.status = "cancelled"
|
||||
snapshot.status = SnapshotStatus.CANCELLED.value
|
||||
snapshot.is_current = False
|
||||
snapshot.completed_at = datetime.now(UTC)
|
||||
summary = dict(snapshot.summary or {})
|
||||
@@ -308,7 +308,7 @@ class BaseCollector(ABC):
|
||||
task.datasource_id = datasource_id
|
||||
task.source = task.source or self.name
|
||||
task.task_type = task.task_type or "collect"
|
||||
task.status = "running"
|
||||
task.status = JobStatus.RUNNING.value
|
||||
task.phase = "queued"
|
||||
task.started_at = task.started_at or start_time
|
||||
task.completed_at = None
|
||||
@@ -393,7 +393,7 @@ class BaseCollector(ABC):
|
||||
},
|
||||
)
|
||||
|
||||
task.status = "success"
|
||||
task.status = JobStatus.SUCCESS.value
|
||||
task.phase = "completed"
|
||||
task.phase_progress = 100.0
|
||||
task.phase_message = "采集完成"
|
||||
@@ -428,7 +428,7 @@ class BaseCollector(ABC):
|
||||
}
|
||||
except asyncio.CancelledError:
|
||||
await db.rollback()
|
||||
task.status = "cancelled"
|
||||
task.status = JobStatus.CANCELLED.value
|
||||
task.phase = "cancelled"
|
||||
task.phase_message = "采集已取消"
|
||||
task.error_message = "Collection cancelled by operator and rolled back"
|
||||
@@ -456,7 +456,7 @@ class BaseCollector(ABC):
|
||||
raise
|
||||
except Exception as e:
|
||||
await db.rollback()
|
||||
task.status = "failed"
|
||||
task.status = JobStatus.FAILED.value
|
||||
task.phase = "failed"
|
||||
task.phase_message = str(e)
|
||||
task.error_message = str(e)
|
||||
@@ -464,7 +464,7 @@ class BaseCollector(ABC):
|
||||
if snapshot_id is not None:
|
||||
snapshot = await db.get(DataSnapshot, snapshot_id)
|
||||
if snapshot:
|
||||
snapshot.status = "failed"
|
||||
snapshot.status = SnapshotStatus.FAILED.value
|
||||
snapshot.completed_at = datetime.now(UTC)
|
||||
snapshot.summary = {"error": str(e)}
|
||||
await db.commit()
|
||||
@@ -509,7 +509,7 @@ class BaseCollector(ABC):
|
||||
if snapshot:
|
||||
snapshot.record_count = 0
|
||||
snapshot.summary = {"created": 0, "updated": 0, "unchanged": 0}
|
||||
snapshot.status = "success"
|
||||
snapshot.status = SnapshotStatus.SUCCESS.value
|
||||
snapshot.completed_at = datetime.now(UTC)
|
||||
await db.commit()
|
||||
return 0
|
||||
@@ -642,7 +642,7 @@ class BaseCollector(ABC):
|
||||
snapshot = await db.get(DataSnapshot, snapshot_id)
|
||||
if snapshot:
|
||||
snapshot.record_count = records_added
|
||||
snapshot.status = "success"
|
||||
snapshot.status = SnapshotStatus.SUCCESS.value
|
||||
snapshot.completed_at = datetime.now(UTC)
|
||||
snapshot.summary = {
|
||||
"created": created_count,
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Any
|
||||
import httpx
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.enums import SnapshotStatus
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
from app.services.barentswatch import (
|
||||
@@ -125,7 +126,7 @@ class VesselAISCollector(BaseCollector):
|
||||
snapshot = await db.get(DataSnapshot, snapshot_id)
|
||||
if snapshot:
|
||||
snapshot.record_count = records_added
|
||||
snapshot.status = "success"
|
||||
snapshot.status = SnapshotStatus.SUCCESS.value
|
||||
snapshot.completed_at = now
|
||||
snapshot.summary = {
|
||||
"created": records_added,
|
||||
|
||||
@@ -12,6 +12,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.cache import cache
|
||||
from app.core.config import settings
|
||||
from app.core.enums import JobStatus, JobType, RollbackPolicy
|
||||
from app.core.logging import get_logger
|
||||
from app.core.time import to_iso8601_utc
|
||||
from app.core.websocket.broadcaster import broadcaster
|
||||
@@ -36,17 +37,17 @@ from app.services.scheduler import sync_datasource_job
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
JOB_TYPE_COLLECT = "collect"
|
||||
JOB_TYPE_CLEAR_DATA = "clear_data"
|
||||
JOB_TYPE_CLEAR_CACHE = "clear_cache"
|
||||
JOB_TYPE_EARTH_REFRESH = "earth_refresh"
|
||||
JOB_TYPE_COLLECT = JobType.COLLECT.value
|
||||
JOB_TYPE_CLEAR_DATA = JobType.CLEAR_DATA.value
|
||||
JOB_TYPE_CLEAR_CACHE = JobType.CLEAR_CACHE.value
|
||||
JOB_TYPE_EARTH_REFRESH = JobType.EARTH_REFRESH.value
|
||||
|
||||
JOB_STATUS_QUEUED = "queued"
|
||||
JOB_STATUS_RUNNING = "running"
|
||||
JOB_STATUS_CANCELLING = "cancelling"
|
||||
JOB_STATUS_SUCCESS = "success"
|
||||
JOB_STATUS_FAILED = "failed"
|
||||
JOB_STATUS_CANCELLED = "cancelled"
|
||||
JOB_STATUS_QUEUED = JobStatus.QUEUED.value
|
||||
JOB_STATUS_RUNNING = JobStatus.RUNNING.value
|
||||
JOB_STATUS_CANCELLING = JobStatus.CANCELLING.value
|
||||
JOB_STATUS_SUCCESS = JobStatus.SUCCESS.value
|
||||
JOB_STATUS_FAILED = JobStatus.FAILED.value
|
||||
JOB_STATUS_CANCELLED = JobStatus.CANCELLED.value
|
||||
|
||||
ACTIVE_JOB_STATUSES = (JOB_STATUS_QUEUED, JOB_STATUS_RUNNING, JOB_STATUS_CANCELLING)
|
||||
TERMINAL_JOB_STATUSES = (JOB_STATUS_SUCCESS, JOB_STATUS_FAILED, JOB_STATUS_CANCELLED)
|
||||
@@ -80,7 +81,7 @@ async def enqueue_datasource_job(
|
||||
task_type: str,
|
||||
*,
|
||||
payload: dict[str, Any] | None = None,
|
||||
rollback_policy: str = "keep_committed_batches",
|
||||
rollback_policy: str = RollbackPolicy.KEEP_COMMITTED_BATCHES.value,
|
||||
dedupe_key: str | None = None,
|
||||
) -> CollectionTask:
|
||||
if dedupe_key:
|
||||
|
||||
@@ -13,6 +13,7 @@ from sqlalchemy import func, select
|
||||
|
||||
from app.core.data_sources import get_data_sources_config
|
||||
from app.core.datasource_defaults import DEFAULT_DATASOURCES
|
||||
from app.core.enums import JobStatus
|
||||
from app.models.collected_data import CollectedData
|
||||
from app.models.datasource import DataSource
|
||||
from app.models.datasource_config import DataSourceConfig
|
||||
@@ -398,7 +399,7 @@ async def has_collected_data(db, source: str) -> bool:
|
||||
|
||||
datasource_result = await db.execute(select(DataSource).where(DataSource.source == source))
|
||||
datasource = datasource_result.scalar_one_or_none()
|
||||
return bool(datasource and datasource.last_status == "success")
|
||||
return bool(datasource and datasource.last_status == JobStatus.SUCCESS.value)
|
||||
|
||||
|
||||
async def get_builtin_connection_status(
|
||||
|
||||
@@ -6,6 +6,7 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from app.core.enums import UserRole
|
||||
from app.models.user import User
|
||||
|
||||
DocsAccess = Literal["public", "docs_user", "docs_developer", "docs_admin"]
|
||||
@@ -43,7 +44,9 @@ DOCS_METADATA: tuple[DocsMetadata, ...] = (
|
||||
DocsMetadata("earth-satellite-footprint-policy.md", "earth-satellite-footprint-policy", "docs_developer", "Earth", 13, "智能星球卫星覆盖策略", "Intelligent Planet Satellite Footprint Policy"),
|
||||
DocsMetadata("earth-bgp-context.md", "earth-bgp-context", "docs_developer", "Earth", 14, "BGP 态势上下文", "BGP Context"),
|
||||
DocsMetadata("earth-interactable-usage.md", "earth-interactable-usage", "docs_developer", "Earth", 16, "智能星球可交互图标接入", "Intelligent Planet Interactable Usage"),
|
||||
DocsMetadata("earth-interactable-clustering.md", "earth-interactable-clustering", "docs_developer", "Earth", 17, "智能星球可交互图标聚类策略", "Intelligent Planet Interactable Clustering"),
|
||||
DocsMetadata("earth-toolbar-overlay-coordination.md", "earth-toolbar-overlay-coordination", "docs_developer", "Earth", 17, "智能星球工具栏与浮层协同", "Intelligent Planet Toolbar and Overlay Coordination"),
|
||||
DocsMetadata("earth-news-sources.md", "earth-news-sources", "docs_developer", "Earth", 19, "智能星球新闻源配置", "Intelligent Planet News Source Configuration"),
|
||||
DocsMetadata("frontend-admin-frontend-context.md", "frontend-admin-frontend-context", "docs_developer", "Frontend", 20, "控制台前端结构", "Admin Frontend Context"),
|
||||
DocsMetadata("frontend-layout-guidelines.md", "frontend-layout-guidelines", "docs_developer", "Frontend", 21, "前端布局指南", "Frontend Layout Guidelines"),
|
||||
DocsMetadata("tactile-ui-components.md", "tactile-ui-components", "docs_developer", "Frontend", 24, "Tactile UI 组件库", "Tactile UI Components"),
|
||||
@@ -52,7 +55,8 @@ DOCS_METADATA: tuple[DocsMetadata, ...] = (
|
||||
DocsMetadata("datasource-collector-settings-connectivity.md", "datasource-collector-settings-connectivity", "docs_developer", "Backend", 32, "数据源、采集器设置与连接验证", "Datasource Collector Settings and Connectivity"),
|
||||
DocsMetadata("backend-datasources-api-performance.md", "backend-datasources-api-performance", "docs_developer", "Backend", 33, "数据源 API 性能", "Datasource API Performance"),
|
||||
DocsMetadata("data-job-earth-sync-architecture.md", "data-job-earth-sync-architecture", "docs_developer", "Backend", 34, "数据作业与 Outbox 技术架构", "Data Jobs and Outbox Architecture"),
|
||||
DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 35, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"),
|
||||
DocsMetadata("backend-enum-contracts.md", "backend-enum-contracts", "docs_developer", "Backend", 35, "后端枚举与字符串兼容契约", "Backend Enum and String Compatibility Contract"),
|
||||
DocsMetadata("location-pipeline-development.md", "location-pipeline-development", "docs_developer", "Backend", 36, "通用位置估算管线开发说明", "Shared Location Resolution Pipeline Development Guide"),
|
||||
DocsMetadata("earth-news-live-streams-collector-format.md", "earth-news-live-streams-collector-format", "docs_developer", "Backend", 36, "新闻直播采集格式", "News Live Streams Collector Format"),
|
||||
DocsMetadata("docs-gatekeeper-development.md", "docs-gatekeeper-development", "docs_developer", "Backend", 37, "Docs Gatekeeper 开发说明", "Docs Gatekeeper Development Guide"),
|
||||
DocsMetadata("agents-aiprovider.md", "agents-aiprovider", "docs_developer", "Agents", 40, "AI Provider 指南", "AI Provider Guide"),
|
||||
@@ -69,9 +73,9 @@ def get_user_gatekeeper_groups(user: User | None) -> set[str]:
|
||||
return set()
|
||||
|
||||
role = user.role.value if hasattr(user.role, "value") else str(user.role or "")
|
||||
if role == "super_admin":
|
||||
if role == UserRole.SUPER_ADMIN.value:
|
||||
return {"docs_user", "docs_developer", "docs_admin"}
|
||||
if role == "admin":
|
||||
if role == UserRole.ADMIN.value:
|
||||
return {"docs_user", "docs_developer", "docs_admin"}
|
||||
|
||||
groups = set()
|
||||
|
||||
@@ -20,8 +20,8 @@ class EarthLayerAdapter:
|
||||
|
||||
EARTH_LAYER_ADAPTERS: tuple[EarthLayerAdapter, ...] = (
|
||||
EarthLayerAdapter(
|
||||
sources=frozenset({"barentswatch_vessels", "aisstream_vessels", "vessel_static", "vessel_position", "ais_raw_observations", "ais_source_health"}),
|
||||
tables=frozenset({"vessel_static", "vessel_position", "ais_raw_observations", "ais_source_health"}),
|
||||
sources=frozenset({"barentswatch_vessels", "aisstream_vessels", "vessel_static", "vessel_position", "vessel_current_state", "ais_raw_observations", "ais_source_health"}),
|
||||
tables=frozenset({"vessel_static", "vessel_position", "vessel_current_state", "ais_raw_observations", "ais_source_health"}),
|
||||
layers=("vessels",),
|
||||
cache_patterns=("vessels*", "summary*"),
|
||||
derived_models=("ais_raw_observations", "ais_conflict_records", "ais_source_health"),
|
||||
|
||||
@@ -20,10 +20,27 @@ from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.countries import COUNTRY_VARIANTS_MAP, get_country_centroid, normalize_country
|
||||
from app.core.enums import (
|
||||
BreakingLevel,
|
||||
BreakingScope,
|
||||
BreakingSource,
|
||||
NewsEnrichmentStatus,
|
||||
NewsImportanceLevel,
|
||||
NewsMarketImpact,
|
||||
NewsSourceType,
|
||||
NewsTaggingSource,
|
||||
)
|
||||
from app.models.system_setting import SystemSetting
|
||||
from app.ai_tasks.prompts import EffectiveAIPrompt, get_effective_prompt
|
||||
from app.schemas.ai import SituationalAnalysisRequest
|
||||
from app.services.ai_client import AIProviderClient
|
||||
from app.services.earth_news_classification import (
|
||||
apply_news_classification as _apply_news_classification,
|
||||
breaking_sort_rank as _breaking_sort_rank,
|
||||
highest_breaking_level as _highest_breaking_level,
|
||||
normalize_breaking_level as _normalize_breaking_level_enum,
|
||||
normalize_breaking_scope as _normalize_breaking_scope_enum,
|
||||
)
|
||||
from app.services.location.resolvers.nominatim import build_default_nominatim_geocoder
|
||||
|
||||
|
||||
@@ -69,7 +86,7 @@ class NewsFeedEndpoint:
|
||||
id: str
|
||||
name: str
|
||||
url: str
|
||||
type: str = "rss"
|
||||
type: str = NewsSourceType.RSS.value
|
||||
region: str = ""
|
||||
enabled: bool = True
|
||||
default_category: str = "other"
|
||||
@@ -85,7 +102,7 @@ class NewsFeedSource:
|
||||
feed_url: str
|
||||
homepage_url: str
|
||||
feed_directory_url: str = ""
|
||||
source_type: str = "rss"
|
||||
source_type: str = NewsSourceType.RSS.value
|
||||
feed_urls: tuple[str, ...] = ()
|
||||
feeds: tuple[NewsFeedEndpoint, ...] = ()
|
||||
priority: int = 100
|
||||
@@ -120,7 +137,7 @@ class ParsedNewsItem:
|
||||
published_at: datetime | None
|
||||
content_language: str = "en"
|
||||
localizations: dict[str, dict[str, str]] = field(default_factory=dict)
|
||||
enrichment_status: str = "pending"
|
||||
enrichment_status: str = NewsEnrichmentStatus.PENDING.value
|
||||
enrichment_error: str | None = None
|
||||
enriched_at: datetime | None = None
|
||||
target_location: NewsTargetLocation | None = None
|
||||
@@ -132,16 +149,22 @@ class ParsedNewsItem:
|
||||
location_patch: dict[str, Any] | None = None
|
||||
source_tags: list[str] = field(default_factory=list)
|
||||
feed_id: str = ""
|
||||
feed_type: str = "rss"
|
||||
feed_type: str = NewsSourceType.RSS.value
|
||||
feed_default_category: str = "other"
|
||||
category: str = "other"
|
||||
item_tags: list[str] = field(default_factory=list)
|
||||
tagging_source: str = "rules"
|
||||
tagging_source: str = NewsTaggingSource.RULES.value
|
||||
tagging_confidence: float = 0.0
|
||||
importance_score: int = 0
|
||||
importance_level: str = "low"
|
||||
importance_level: str = NewsImportanceLevel.LOW.value
|
||||
importance_reasons: list[str] = field(default_factory=list)
|
||||
market_impact: str = "none"
|
||||
market_impact: str = NewsMarketImpact.NONE.value
|
||||
breaking_level: str = BreakingLevel.NONE.value
|
||||
breaking_scope: str = BreakingScope.REGIONAL.value
|
||||
breaking_reasons: list[str] = field(default_factory=list)
|
||||
breaking_source: str = BreakingSource.RULES.value
|
||||
breaking_confidence: float = 0.0
|
||||
breaking_expires_at: datetime | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -1863,34 +1886,12 @@ def _parse_feed_entries(
|
||||
return items
|
||||
|
||||
|
||||
def _contains_keyword(text: str, keyword: str) -> bool:
|
||||
keyword_text = str(keyword or "").strip().lower()
|
||||
if not keyword_text:
|
||||
return False
|
||||
if re.search(r"[\u4e00-\u9fff]", keyword_text):
|
||||
return keyword_text in text
|
||||
return re.search(rf"(?<![a-z0-9]){re.escape(keyword_text)}(?![a-z0-9])", text) is not None
|
||||
def _normalize_breaking_level(value: str | None) -> str:
|
||||
return _normalize_breaking_level_enum(value).value
|
||||
|
||||
|
||||
def _score_category(text: str, title_text: str, category: dict[str, Any]) -> int:
|
||||
score = 0
|
||||
keywords = category.get("keywords") if isinstance(category.get("keywords"), list) else []
|
||||
for keyword in keywords:
|
||||
if _contains_keyword(title_text, keyword):
|
||||
score += 3
|
||||
elif _contains_keyword(text, keyword):
|
||||
score += 1
|
||||
return score
|
||||
|
||||
|
||||
def _importance_level(score: int) -> str:
|
||||
if score >= 80:
|
||||
return "critical"
|
||||
if score >= 60:
|
||||
return "high"
|
||||
if score >= 35:
|
||||
return "medium"
|
||||
return "low"
|
||||
def _normalize_breaking_scope(value: str | None) -> str:
|
||||
return _normalize_breaking_scope_enum(value).value
|
||||
|
||||
|
||||
def apply_news_classification(
|
||||
@@ -1901,74 +1902,7 @@ def apply_news_classification(
|
||||
config_payload: dict[str, Any] | None = None,
|
||||
) -> ParsedNewsItem:
|
||||
config = normalize_earth_news_sources_payload(config_payload)
|
||||
title_text = item.title.lower()
|
||||
combined_text = f"{item.title} {item.summary} {item.source} {item.feed_name}".lower()
|
||||
|
||||
feed_default_category = (feed.default_category if feed else item.feed_default_category) or source.default_category or "other"
|
||||
best_key = feed_default_category
|
||||
best_score = 0
|
||||
second_score = 0
|
||||
for category in config["categories"]:
|
||||
if not isinstance(category, dict) or category.get("enabled") is False:
|
||||
continue
|
||||
score = _score_category(combined_text, title_text, category)
|
||||
if score > best_score:
|
||||
second_score = best_score
|
||||
best_score = score
|
||||
best_key = str(category.get("key") or "other")
|
||||
elif score > second_score:
|
||||
second_score = score
|
||||
|
||||
item_tags: list[str] = []
|
||||
for rule in config["item_tag_rules"]:
|
||||
if not isinstance(rule, dict):
|
||||
continue
|
||||
keywords = rule.get("keywords") if isinstance(rule.get("keywords"), list) else []
|
||||
if any(_contains_keyword(combined_text, keyword) for keyword in keywords):
|
||||
tag_key = str(rule.get("key") or "").strip()
|
||||
if tag_key and tag_key not in item_tags:
|
||||
item_tags.append(tag_key)
|
||||
if best_score < 3 and rule.get("category"):
|
||||
best_key = str(rule["category"])
|
||||
best_score = 3
|
||||
|
||||
confidence = round(best_score / (best_score + second_score + 1), 2) if best_score else 0.35
|
||||
if best_score < 3 and feed_default_category:
|
||||
best_key = feed_default_category
|
||||
confidence = 0.45
|
||||
|
||||
importance_score = max(0, min(100, 18 + source.importance_weight + best_score * 6))
|
||||
reasons: list[str] = []
|
||||
source_tag_set = set(source.source_tags)
|
||||
if "official_data" in source_tag_set:
|
||||
importance_score += 20
|
||||
reasons.append("官方数据源")
|
||||
if "press_release" in source_tag_set:
|
||||
importance_score = max(0, importance_score - 12)
|
||||
reasons.append("企业公告基础权重较低")
|
||||
ecommerce_terms = ("网上零售额", "电商物流指数", "gmv", "订单量", "物流指数", "履约", "直播电商", "跨境电商")
|
||||
if any(_contains_keyword(combined_text, term) for term in ecommerce_terms):
|
||||
importance_score += 25
|
||||
reasons.append("命中电商数据指标")
|
||||
major_platforms = ("amazon", "shopify", "walmart", "alibaba", "jd.com", "pinduoduo", "tiktok shop", "shein", "阿里", "京东", "拼多多", "抖音")
|
||||
if any(_contains_keyword(combined_text, term) for term in major_platforms):
|
||||
importance_score += 15
|
||||
reasons.append("涉及大型平台")
|
||||
if any(term in combined_text for term in ("同比", "环比", "%", "billion", "million", "增长", "下降")):
|
||||
importance_score += 10
|
||||
reasons.append("包含量化指标")
|
||||
|
||||
importance_score = max(0, min(100, importance_score))
|
||||
item.category = best_key or "other"
|
||||
item.item_tags = item_tags
|
||||
item.tagging_source = "rules"
|
||||
item.tagging_confidence = confidence
|
||||
item.importance_score = importance_score
|
||||
item.importance_level = _importance_level(importance_score)
|
||||
item.importance_reasons = reasons or ["按来源权重和分类规则计算"]
|
||||
item.market_impact = "global" if "global" in source_tag_set else "national" if {"china", "us"} & source_tag_set else "sector"
|
||||
item.source_tags = list(source.source_tags)
|
||||
return item
|
||||
return _apply_news_classification(item, source, feed=feed, config=config)
|
||||
|
||||
|
||||
def _serialize_sources(sources: list[NewsFeedSource]) -> list[dict[str, Any]]:
|
||||
@@ -2020,6 +1954,10 @@ def _serialize_enriched_at(value: datetime | None) -> str | None:
|
||||
return value.isoformat().replace("+00:00", "Z") if value else None
|
||||
|
||||
|
||||
def _serialize_breaking_expires_at(value: datetime | None) -> str | None:
|
||||
return value.isoformat().replace("+00:00", "Z") if value else None
|
||||
|
||||
|
||||
def _content_patch(item: ParsedNewsItem) -> dict[str, Any]:
|
||||
return {
|
||||
"content_language": item.content_language,
|
||||
@@ -2047,6 +1985,12 @@ def _news_meta_patch(item: ParsedNewsItem) -> dict[str, Any]:
|
||||
"importance_level": item.importance_level,
|
||||
"importance_reasons": list(item.importance_reasons or []),
|
||||
"market_impact": item.market_impact,
|
||||
"breaking_level": _normalize_breaking_level(item.breaking_level),
|
||||
"breaking_scope": _normalize_breaking_scope(item.breaking_scope),
|
||||
"breaking_reasons": list(item.breaking_reasons or []),
|
||||
"breaking_source": item.breaking_source,
|
||||
"breaking_confidence": item.breaking_confidence,
|
||||
"breaking_expires_at": _serialize_breaking_expires_at(item.breaking_expires_at),
|
||||
}
|
||||
|
||||
|
||||
@@ -2142,6 +2086,12 @@ def build_target_location_job_payload(item: ParsedNewsItem) -> dict[str, Any]:
|
||||
"importance_level": item.importance_level,
|
||||
"importance_reasons": list(item.importance_reasons or []),
|
||||
"market_impact": item.market_impact,
|
||||
"breaking_level": _normalize_breaking_level(item.breaking_level),
|
||||
"breaking_scope": _normalize_breaking_scope(item.breaking_scope),
|
||||
"breaking_reasons": list(item.breaking_reasons or []),
|
||||
"breaking_source": item.breaking_source,
|
||||
"breaking_confidence": item.breaking_confidence,
|
||||
"breaking_expires_at": _serialize_breaking_expires_at(item.breaking_expires_at),
|
||||
}
|
||||
|
||||
|
||||
@@ -2173,6 +2123,12 @@ def parsed_news_item_from_job_payload(payload: dict[str, Any]) -> ParsedNewsItem
|
||||
importance_level=str(payload.get("importance_level") or "low"),
|
||||
importance_reasons=list(payload.get("importance_reasons") or []),
|
||||
market_impact=str(payload.get("market_impact") or "none"),
|
||||
breaking_level=_normalize_breaking_level(str(payload.get("breaking_level") or "none")),
|
||||
breaking_scope=_normalize_breaking_scope(str(payload.get("breaking_scope") or "regional")),
|
||||
breaking_reasons=list(payload.get("breaking_reasons") or []),
|
||||
breaking_source=str(payload.get("breaking_source") or "rules"),
|
||||
breaking_confidence=float(payload.get("breaking_confidence") or 0),
|
||||
breaking_expires_at=_parse_datetime(_coerce_str(payload.get("breaking_expires_at"))),
|
||||
)
|
||||
|
||||
|
||||
@@ -2218,6 +2174,12 @@ def _serialize_item(item: ParsedNewsItem, *, active_region: str, locale: str = D
|
||||
"importance_level": item.importance_level,
|
||||
"importance_reasons": list(item.importance_reasons or []),
|
||||
"market_impact": item.market_impact,
|
||||
"breaking_level": _normalize_breaking_level(item.breaking_level),
|
||||
"breaking_scope": _normalize_breaking_scope(item.breaking_scope),
|
||||
"breaking_reasons": list(item.breaking_reasons or []),
|
||||
"breaking_source": item.breaking_source,
|
||||
"breaking_confidence": item.breaking_confidence,
|
||||
"breaking_expires_at": _serialize_breaking_expires_at(item.breaking_expires_at),
|
||||
}
|
||||
|
||||
|
||||
@@ -2239,6 +2201,9 @@ def _build_payload(
|
||||
) -> dict[str, Any]:
|
||||
profile = get_region_profile(active_region)
|
||||
timestamp = generated_at or datetime.now(UTC)
|
||||
visible_items = list(items)
|
||||
cruise_visible_items = list(cruise_items if cruise_items is not None else items)
|
||||
highest_breaking_level = _highest_breaking_level(visible_items + cruise_visible_items)
|
||||
return {
|
||||
"generated_at": timestamp.isoformat().replace("+00:00", "Z"),
|
||||
"focus": {
|
||||
@@ -2256,11 +2221,13 @@ def _build_payload(
|
||||
"sources": sorted(source_ids or []),
|
||||
"limit": limit,
|
||||
"locale": locale,
|
||||
"has_breaking": highest_breaking_level != "none",
|
||||
"highest_breaking_level": highest_breaking_level,
|
||||
},
|
||||
"items": [_serialize_item(item, active_region=active_region, locale=locale) for item in items],
|
||||
"items": [_serialize_item(item, active_region=active_region, locale=locale) for item in visible_items],
|
||||
"cruise_items": [
|
||||
_serialize_item(item, active_region=active_region, locale=locale)
|
||||
for item in (cruise_items if cruise_items is not None else items)
|
||||
for item in cruise_visible_items
|
||||
],
|
||||
"errors": errors,
|
||||
"stale": stale,
|
||||
@@ -2282,7 +2249,11 @@ def _rank_and_trim_items(
|
||||
return sorted(
|
||||
deduped.values(),
|
||||
key=lambda item: (
|
||||
False if active_region == "global" else item.feed_region != active_region,
|
||||
-_breaking_sort_rank(item),
|
||||
False
|
||||
if active_region == "global"
|
||||
or (_breaking_sort_rank(item) > 0 and _normalize_breaking_scope(item.breaking_scope) == "global")
|
||||
else item.feed_region != active_region,
|
||||
item.published_at is None,
|
||||
-(item.published_at.timestamp() if item.published_at else 0),
|
||||
item.feed_name,
|
||||
|
||||
258
backend/app/services/earth_news_classification.py
Normal file
258
backend/app/services/earth_news_classification.py
Normal file
@@ -0,0 +1,258 @@
|
||||
"""Classification, importance, and breaking-news policy for Earth news."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
import re
|
||||
from typing import Any, Protocol
|
||||
|
||||
from app.core.enums import (
|
||||
BreakingLevel,
|
||||
BreakingScope,
|
||||
BreakingSource,
|
||||
NewsImportanceLevel,
|
||||
NewsMarketImpact,
|
||||
NewsTaggingSource,
|
||||
parse_enum,
|
||||
)
|
||||
|
||||
|
||||
class NewsItemLike(Protocol):
|
||||
title: str
|
||||
summary: str
|
||||
source: str
|
||||
feed_name: str
|
||||
published_at: datetime | None
|
||||
feed_default_category: str
|
||||
category: str
|
||||
item_tags: list[str]
|
||||
tagging_source: str
|
||||
tagging_confidence: float
|
||||
importance_score: int
|
||||
importance_level: str
|
||||
importance_reasons: list[str]
|
||||
market_impact: str
|
||||
source_tags: list[str]
|
||||
breaking_level: str
|
||||
breaking_scope: str
|
||||
breaking_reasons: list[str]
|
||||
breaking_source: str
|
||||
breaking_confidence: float
|
||||
breaking_expires_at: datetime | None
|
||||
|
||||
|
||||
class NewsSourceLike(Protocol):
|
||||
default_category: str
|
||||
importance_weight: int
|
||||
source_tags: tuple[str, ...]
|
||||
|
||||
|
||||
class NewsFeedLike(Protocol):
|
||||
default_category: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BreakingRule:
|
||||
level: BreakingLevel
|
||||
scope: BreakingScope
|
||||
reason: str
|
||||
keywords: tuple[str, ...]
|
||||
|
||||
|
||||
IMPORTANCE_THRESHOLDS: tuple[tuple[int, NewsImportanceLevel], ...] = (
|
||||
(80, NewsImportanceLevel.CRITICAL),
|
||||
(60, NewsImportanceLevel.HIGH),
|
||||
(35, NewsImportanceLevel.MEDIUM),
|
||||
(0, NewsImportanceLevel.LOW),
|
||||
)
|
||||
|
||||
BREAKING_LEVEL_RANK: dict[BreakingLevel, int] = {
|
||||
BreakingLevel.NONE: 0,
|
||||
BreakingLevel.WATCH: 1,
|
||||
BreakingLevel.BREAKING: 2,
|
||||
BreakingLevel.CRITICAL: 3,
|
||||
}
|
||||
|
||||
BREAKING_TTL: dict[BreakingLevel, timedelta] = {
|
||||
BreakingLevel.WATCH: timedelta(hours=6),
|
||||
BreakingLevel.BREAKING: timedelta(hours=12),
|
||||
BreakingLevel.CRITICAL: timedelta(hours=24),
|
||||
}
|
||||
|
||||
BREAKING_RULES: tuple[BreakingRule, ...] = (
|
||||
BreakingRule(BreakingLevel.CRITICAL, BreakingScope.GLOBAL, "核事故或核风险", ("nuclear accident", "nuclear emergency", "radiation leak", "核事故", "核泄漏", "辐射泄漏")),
|
||||
BreakingRule(BreakingLevel.CRITICAL, BreakingScope.REGIONAL, "重大军事冲突升级", ("airstrike", "missile strike", "invasion", "martial law", "空袭", "导弹袭击", "入侵", "戒严")),
|
||||
BreakingRule(BreakingLevel.BREAKING, BreakingScope.REGIONAL, "战争或安全事件", ("war escalates", "terror attack", "coup", "hostage", "战争升级", "恐袭", "政变", "人质")),
|
||||
BreakingRule(BreakingLevel.BREAKING, BreakingScope.REGIONAL, "重大灾害应急", ("major earthquake", "tsunami", "volcanic eruption", "state of emergency", "强震", "海啸", "火山喷发", "紧急状态")),
|
||||
BreakingRule(BreakingLevel.BREAKING, BreakingScope.GLOBAL, "金融市场异常", ("market halt", "trading halt", "flash crash", "bank run", "金融熔断", "交易暂停", "银行挤兑")),
|
||||
BreakingRule(BreakingLevel.WATCH, BreakingScope.GLOBAL, "大规模网络安全事件", ("massive cyberattack", "ransomware attack", "data breach", "大规模网络攻击", "勒索软件", "数据泄露")),
|
||||
BreakingRule(BreakingLevel.WATCH, BreakingScope.REGIONAL, "航天或卫星事故", ("rocket explosion", "satellite collision", "space station emergency", "火箭爆炸", "卫星碰撞", "空间站事故")),
|
||||
)
|
||||
|
||||
|
||||
def contains_keyword(text: str, keyword: str) -> bool:
|
||||
keyword_text = str(keyword or "").strip().lower()
|
||||
if not keyword_text:
|
||||
return False
|
||||
if re.search(r"[\u4e00-\u9fff]", keyword_text):
|
||||
return keyword_text in text
|
||||
return re.search(rf"(?<![a-z0-9]){re.escape(keyword_text)}(?![a-z0-9])", text) is not None
|
||||
|
||||
|
||||
def score_category(text: str, title_text: str, category: dict[str, Any]) -> int:
|
||||
score = 0
|
||||
keywords = category.get("keywords") if isinstance(category.get("keywords"), list) else []
|
||||
for keyword in keywords:
|
||||
if contains_keyword(title_text, keyword):
|
||||
score += 3
|
||||
elif contains_keyword(text, keyword):
|
||||
score += 1
|
||||
return score
|
||||
|
||||
|
||||
def importance_level(score: int) -> NewsImportanceLevel:
|
||||
normalized_score = max(0, min(100, int(score)))
|
||||
for threshold, level in IMPORTANCE_THRESHOLDS:
|
||||
if normalized_score >= threshold:
|
||||
return level
|
||||
return NewsImportanceLevel.LOW
|
||||
|
||||
|
||||
def normalize_breaking_level(value: object) -> BreakingLevel:
|
||||
return parse_enum(BreakingLevel, value, BreakingLevel.NONE)
|
||||
|
||||
|
||||
def normalize_breaking_scope(value: object) -> BreakingScope:
|
||||
return parse_enum(BreakingScope, value, BreakingScope.REGIONAL)
|
||||
|
||||
|
||||
def breaking_expires_at(level: object, published_at: datetime | None) -> datetime | None:
|
||||
normalized = normalize_breaking_level(level)
|
||||
if normalized is BreakingLevel.NONE:
|
||||
return None
|
||||
base = published_at or datetime.now(UTC)
|
||||
base = base.replace(tzinfo=UTC) if base.tzinfo is None else base.astimezone(UTC)
|
||||
return base + BREAKING_TTL[normalized]
|
||||
|
||||
|
||||
def is_breaking_active(item: NewsItemLike, *, now: datetime | None = None) -> bool:
|
||||
if normalize_breaking_level(item.breaking_level) is BreakingLevel.NONE:
|
||||
return False
|
||||
expires_at = item.breaking_expires_at
|
||||
if expires_at is None:
|
||||
return True
|
||||
expires_at = expires_at.replace(tzinfo=UTC) if expires_at.tzinfo is None else expires_at.astimezone(UTC)
|
||||
return expires_at > (now or datetime.now(UTC))
|
||||
|
||||
|
||||
def breaking_sort_rank(item: NewsItemLike) -> int:
|
||||
if not is_breaking_active(item):
|
||||
return 0
|
||||
return BREAKING_LEVEL_RANK[normalize_breaking_level(item.breaking_level)]
|
||||
|
||||
|
||||
def highest_breaking_level(items: list[NewsItemLike]) -> BreakingLevel:
|
||||
active = [normalize_breaking_level(item.breaking_level) for item in items if is_breaking_active(item)]
|
||||
return max(active, key=BREAKING_LEVEL_RANK.get) if active else BreakingLevel.NONE
|
||||
|
||||
|
||||
def apply_breaking_rules(item: NewsItemLike) -> None:
|
||||
combined_text = f"{item.title} {item.summary} {item.source} {item.feed_name}".lower()
|
||||
best_level = BreakingLevel.NONE
|
||||
best_scope = BreakingScope.REGIONAL
|
||||
reasons: list[str] = []
|
||||
confidence = 0.0
|
||||
for rule in BREAKING_RULES:
|
||||
if not any(contains_keyword(combined_text, keyword) for keyword in rule.keywords):
|
||||
continue
|
||||
if BREAKING_LEVEL_RANK[rule.level] > BREAKING_LEVEL_RANK[best_level]:
|
||||
best_level = rule.level
|
||||
best_scope = rule.scope
|
||||
if rule.reason not in reasons:
|
||||
reasons.append(rule.reason)
|
||||
confidence = max(confidence, 0.72 if rule.level is BreakingLevel.CRITICAL else 0.64 if rule.level is BreakingLevel.BREAKING else 0.52)
|
||||
|
||||
item.breaking_level = best_level.value
|
||||
item.breaking_scope = (best_scope if best_level is not BreakingLevel.NONE else BreakingScope.REGIONAL).value
|
||||
item.breaking_reasons = reasons
|
||||
item.breaking_source = BreakingSource.RULES.value
|
||||
item.breaking_confidence = round(confidence, 2)
|
||||
item.breaking_expires_at = breaking_expires_at(best_level, item.published_at)
|
||||
|
||||
|
||||
def apply_news_classification(
|
||||
item: NewsItemLike,
|
||||
source: NewsSourceLike,
|
||||
*,
|
||||
feed: NewsFeedLike | None,
|
||||
config: dict[str, Any],
|
||||
) -> NewsItemLike:
|
||||
title_text = item.title.lower()
|
||||
combined_text = f"{item.title} {item.summary} {item.source} {item.feed_name}".lower()
|
||||
feed_default_category = (feed.default_category if feed else item.feed_default_category) or source.default_category or "other"
|
||||
best_key = feed_default_category
|
||||
best_score = second_score = 0
|
||||
for category in config["categories"]:
|
||||
if not isinstance(category, dict) or category.get("enabled") is False:
|
||||
continue
|
||||
score = score_category(combined_text, title_text, category)
|
||||
if score > best_score:
|
||||
second_score, best_score = best_score, score
|
||||
best_key = str(category.get("key") or "other")
|
||||
elif score > second_score:
|
||||
second_score = score
|
||||
|
||||
item_tags: list[str] = []
|
||||
for rule in config["item_tag_rules"]:
|
||||
if not isinstance(rule, dict):
|
||||
continue
|
||||
keywords = rule.get("keywords") if isinstance(rule.get("keywords"), list) else []
|
||||
if any(contains_keyword(combined_text, keyword) for keyword in keywords):
|
||||
tag_key = str(rule.get("key") or "").strip()
|
||||
if tag_key and tag_key not in item_tags:
|
||||
item_tags.append(tag_key)
|
||||
if best_score < 3 and rule.get("category"):
|
||||
best_key, best_score = str(rule["category"]), 3
|
||||
|
||||
confidence = round(best_score / (best_score + second_score + 1), 2) if best_score else 0.35
|
||||
if best_score < 3 and feed_default_category:
|
||||
best_key, confidence = feed_default_category, 0.45
|
||||
|
||||
score = max(0, min(100, 18 + source.importance_weight + best_score * 6))
|
||||
reasons: list[str] = []
|
||||
source_tags = set(source.source_tags)
|
||||
if "official_data" in source_tags:
|
||||
score += 20
|
||||
reasons.append("官方数据源")
|
||||
if "press_release" in source_tags:
|
||||
score = max(0, score - 12)
|
||||
reasons.append("企业公告基础权重较低")
|
||||
if any(contains_keyword(combined_text, term) for term in ("网上零售额", "电商物流指数", "gmv", "订单量", "物流指数", "履约", "直播电商", "跨境电商")):
|
||||
score += 25
|
||||
reasons.append("命中电商数据指标")
|
||||
if any(contains_keyword(combined_text, term) for term in ("amazon", "shopify", "walmart", "alibaba", "jd.com", "pinduoduo", "tiktok shop", "shein", "阿里", "京东", "拼多多", "抖音")):
|
||||
score += 15
|
||||
reasons.append("涉及大型平台")
|
||||
if any(term in combined_text for term in ("同比", "环比", "%", "billion", "million", "增长", "下降")):
|
||||
score += 10
|
||||
reasons.append("包含量化指标")
|
||||
|
||||
score = max(0, min(100, score))
|
||||
item.category = best_key or "other"
|
||||
item.item_tags = item_tags
|
||||
item.tagging_source = NewsTaggingSource.RULES.value
|
||||
item.tagging_confidence = confidence
|
||||
item.importance_score = score
|
||||
item.importance_level = importance_level(score).value
|
||||
item.importance_reasons = reasons or ["按来源权重和分类规则计算"]
|
||||
item.market_impact = (
|
||||
NewsMarketImpact.GLOBAL.value
|
||||
if "global" in source_tags
|
||||
else NewsMarketImpact.NATIONAL.value
|
||||
if {"china", "us"} & source_tags
|
||||
else NewsMarketImpact.SECTOR.value
|
||||
)
|
||||
item.source_tags = list(source.source_tags)
|
||||
apply_breaking_rules(item)
|
||||
return item
|
||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.earth_news import EarthNewsItem
|
||||
@@ -13,6 +13,11 @@ from app.services.earth_news import (
|
||||
build_anchor_location_patch,
|
||||
_news_meta_patch,
|
||||
)
|
||||
from app.services.earth_news_classification import (
|
||||
breaking_sort_rank,
|
||||
normalize_breaking_level,
|
||||
normalize_breaking_scope,
|
||||
)
|
||||
|
||||
|
||||
def _coerce_datetime(value: datetime | None) -> datetime | None:
|
||||
@@ -23,6 +28,17 @@ def _coerce_datetime(value: datetime | None) -> datetime | None:
|
||||
return value.astimezone(UTC)
|
||||
|
||||
|
||||
def _coerce_meta_datetime(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return _coerce_datetime(value)
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return None
|
||||
try:
|
||||
return _coerce_datetime(datetime.fromisoformat(value.replace("Z", "+00:00")))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _location_patch_from_record(record: EarthNewsItem) -> dict[str, Any]:
|
||||
return {
|
||||
"latitude": record.latitude,
|
||||
@@ -64,10 +80,32 @@ def record_to_parsed_news_item(record: EarthNewsItem) -> ParsedNewsItem:
|
||||
importance_level=str(news_meta.get("importance_level") or "low"),
|
||||
importance_reasons=list(news_meta.get("importance_reasons") or []),
|
||||
market_impact=str(news_meta.get("market_impact") or "none"),
|
||||
breaking_level=normalize_breaking_level(news_meta.get("breaking_level")).value,
|
||||
breaking_scope=normalize_breaking_scope(news_meta.get("breaking_scope")).value,
|
||||
breaking_reasons=list(news_meta.get("breaking_reasons") or []),
|
||||
breaking_source=str(news_meta.get("breaking_source") or "rules"),
|
||||
breaking_confidence=float(news_meta.get("breaking_confidence") or 0),
|
||||
breaking_expires_at=_coerce_meta_datetime(news_meta.get("breaking_expires_at")),
|
||||
)
|
||||
return apply_enrichment_patch_to_item(item, _location_patch_from_record(record))
|
||||
|
||||
|
||||
def _sort_parsed_news_items(items: list[ParsedNewsItem], *, active_region: str) -> list[ParsedNewsItem]:
|
||||
return sorted(
|
||||
items,
|
||||
key=lambda item: (
|
||||
-breaking_sort_rank(item),
|
||||
False
|
||||
if active_region == "global"
|
||||
or (breaking_sort_rank(item) > 0 and normalize_breaking_scope(item.breaking_scope).value == "global")
|
||||
else item.feed_region != active_region,
|
||||
item.published_at is None,
|
||||
-(item.published_at.timestamp() if item.published_at else 0),
|
||||
item.feed_name,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _query_sort_key(active_region: str):
|
||||
if active_region == "global":
|
||||
return (
|
||||
@@ -139,14 +177,20 @@ async def list_earth_news_items(
|
||||
categories: set[str] | None = None,
|
||||
source_ids: set[str] | None = None,
|
||||
) -> list[ParsedNewsItem]:
|
||||
query_limit = limit if source_ids else min(max(limit * 4, limit), 100)
|
||||
query_limit = limit if source_ids else min(max(limit * 8, limit), 200)
|
||||
query = (
|
||||
select(EarthNewsItem)
|
||||
.order_by(*_query_sort_key(active_region))
|
||||
.limit(query_limit)
|
||||
)
|
||||
if active_region != "global":
|
||||
query = query.where(EarthNewsItem.region.in_({"global", active_region}))
|
||||
news_meta = EarthNewsItem.location_meta.op("->")("news_meta")
|
||||
query = query.where(
|
||||
or_(
|
||||
EarthNewsItem.region.in_({"global", active_region}),
|
||||
news_meta.op("->>")("breaking_scope") == "global",
|
||||
)
|
||||
)
|
||||
category_clause = _category_filter_clause(categories)
|
||||
if category_clause is not None:
|
||||
query = query.where(category_clause)
|
||||
@@ -155,11 +199,11 @@ async def list_earth_news_items(
|
||||
query = query.where(source_clause)
|
||||
result = await db.execute(query)
|
||||
records = list(result.scalars().all())
|
||||
if not source_ids:
|
||||
records = _diversify_records_by_source(records, limit=limit)
|
||||
else:
|
||||
records = records[:limit]
|
||||
return [record_to_parsed_news_item(record) for record in records]
|
||||
items = _sort_parsed_news_items(
|
||||
[record_to_parsed_news_item(record) for record in records],
|
||||
active_region=active_region,
|
||||
)
|
||||
return items[:limit]
|
||||
|
||||
|
||||
async def list_earth_news_cruise_items(
|
||||
@@ -177,7 +221,7 @@ async def list_earth_news_cruise_items(
|
||||
EarthNewsItem.last_seen_at.desc(),
|
||||
EarthNewsItem.feed_name.asc(),
|
||||
)
|
||||
.limit(limit)
|
||||
.limit(min(max(limit * 4, limit), 200))
|
||||
)
|
||||
category_clause = _category_filter_clause(categories)
|
||||
if category_clause is not None:
|
||||
@@ -186,7 +230,11 @@ async def list_earth_news_cruise_items(
|
||||
if source_clause is not None:
|
||||
query = query.where(source_clause)
|
||||
result = await db.execute(query)
|
||||
return [record_to_parsed_news_item(record) for record in result.scalars().all()]
|
||||
items = _sort_parsed_news_items(
|
||||
[record_to_parsed_news_item(record) for record in result.scalars().all()],
|
||||
active_region="global",
|
||||
)
|
||||
return items[:limit]
|
||||
|
||||
|
||||
async def get_earth_news_freshness(
|
||||
|
||||
@@ -9,12 +9,12 @@ yet to keep behavior obvious after settings changes).
|
||||
from __future__ import annotations
|
||||
|
||||
from email.message import EmailMessage
|
||||
from typing import Literal, Optional
|
||||
from typing import Optional
|
||||
|
||||
import aiosmtplib
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
OtpPurpose = Literal["register", "verify_email", "reset_password"]
|
||||
from app.core.enums import OtpPurpose
|
||||
|
||||
|
||||
class EmailError(Exception):
|
||||
@@ -81,15 +81,15 @@ async def send_email(
|
||||
|
||||
|
||||
_SUBJECTS: dict[OtpPurpose, str] = {
|
||||
"register": "Confirm your Planet account",
|
||||
"verify_email": "Verify your Planet email",
|
||||
"reset_password": "Reset your Planet password",
|
||||
OtpPurpose.REGISTER: "Confirm your Planet account",
|
||||
OtpPurpose.VERIFY_EMAIL: "Verify your Planet email",
|
||||
OtpPurpose.RESET_PASSWORD: "Reset your Planet password",
|
||||
}
|
||||
|
||||
_HEADLINES: dict[OtpPurpose, str] = {
|
||||
"register": "Welcome to Planet — confirm your email to activate your account.",
|
||||
"verify_email": "Confirm your new email address to keep your Planet account active.",
|
||||
"reset_password": "Use this code to set a new password for your Planet account.",
|
||||
OtpPurpose.REGISTER: "Welcome to Planet — confirm your email to activate your account.",
|
||||
OtpPurpose.VERIFY_EMAIL: "Confirm your new email address to keep your Planet account active.",
|
||||
OtpPurpose.RESET_PASSWORD: "Use this code to set a new password for your Planet account.",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -9,14 +9,12 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import secrets
|
||||
from typing import Literal
|
||||
|
||||
import bcrypt
|
||||
|
||||
from app.core.enums import OtpPurpose
|
||||
from app.core.security import redis_client
|
||||
|
||||
OtpPurpose = Literal["register", "verify_email", "reset_password"]
|
||||
|
||||
CODE_TTL_SECONDS = 600 # 10 minutes
|
||||
RESEND_COOLDOWN_SECONDS = 60
|
||||
MAX_ATTEMPTS = 5
|
||||
|
||||
@@ -7,9 +7,14 @@ from time import perf_counter
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.enums import (
|
||||
PlaygroundMessageKind,
|
||||
PlaygroundMessageRole,
|
||||
PlaygroundMessageStatus,
|
||||
)
|
||||
from app.core.logging import get_logger
|
||||
from app.db.session import async_session_factory
|
||||
from app.models.playground_message import PlaygroundMessage
|
||||
@@ -21,7 +26,6 @@ from app.schemas.ai import (
|
||||
PlaygroundMessageRecord,
|
||||
PlaygroundMessageResendRequest,
|
||||
PlaygroundMessageStopRequest,
|
||||
PlaygroundSessionResponse,
|
||||
PlaygroundSessionState,
|
||||
PlaygroundSessionUpsertRequest,
|
||||
PlaygroundThreadResponse,
|
||||
@@ -37,6 +41,13 @@ STREAM_CHUNK_SIZE = 24
|
||||
STREAM_INTERVAL_SECONDS = 0.08
|
||||
THINKING_PREVIEW_SECONDS = 2.6
|
||||
ORPHANED_RUN_MESSAGE = "后台生成任务已中断,请点击上一条用户消息的重试按钮重新生成。"
|
||||
ACTIVE_MESSAGE_STATUSES = frozenset(
|
||||
{
|
||||
PlaygroundMessageStatus.PENDING.value,
|
||||
PlaygroundMessageStatus.THINKING.value,
|
||||
PlaygroundMessageStatus.ANSWERING.value,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class _ActiveRun:
|
||||
@@ -93,7 +104,11 @@ async def _require_visible_message(
|
||||
result = await db.execute(select(PlaygroundMessage).where(*conditions))
|
||||
message = result.scalar_one_or_none()
|
||||
if message is None:
|
||||
detail = "User message not found" if role == "user" else "Playground message not found"
|
||||
detail = (
|
||||
"User message not found"
|
||||
if role == PlaygroundMessageRole.USER.value
|
||||
else "Playground message not found"
|
||||
)
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=detail)
|
||||
return message
|
||||
|
||||
@@ -108,7 +123,7 @@ def _message_to_record(message: PlaygroundMessage, parent_public_id: str | None
|
||||
content=message.content or "",
|
||||
thinking_content=message.thinking_content or "",
|
||||
meta=list(message.meta or []),
|
||||
markdown=message.role != "system",
|
||||
markdown=message.role != PlaygroundMessageRole.SYSTEM.value,
|
||||
provider=message.provider,
|
||||
model=message.model,
|
||||
request_id=message.request_id,
|
||||
@@ -197,11 +212,11 @@ async def _reconcile_orphaned_active_messages(
|
||||
) -> list[PlaygroundMessage]:
|
||||
changed = False
|
||||
for item in messages:
|
||||
if item.status not in {"pending", "thinking", "answering"}:
|
||||
if item.status not in ACTIVE_MESSAGE_STATUSES:
|
||||
continue
|
||||
if item.public_id in _ACTIVE_RUNS:
|
||||
continue
|
||||
item.status = "error"
|
||||
item.status = PlaygroundMessageStatus.ERROR.value
|
||||
item.content = item.content or ORPHANED_RUN_MESSAGE
|
||||
orphan_meta = "错误: 后台任务已中断"
|
||||
if orphan_meta not in (item.meta or []):
|
||||
@@ -330,9 +345,9 @@ async def create_turn(
|
||||
public_id=uuid4().hex,
|
||||
session_id=session.id,
|
||||
user_id=user_id,
|
||||
role="user",
|
||||
kind="message",
|
||||
status="done",
|
||||
role=PlaygroundMessageRole.USER.value,
|
||||
kind=PlaygroundMessageKind.MESSAGE.value,
|
||||
status=PlaygroundMessageStatus.DONE.value,
|
||||
title=payload.selected_preset_key,
|
||||
content=payload.input,
|
||||
meta=[payload.title],
|
||||
@@ -343,9 +358,9 @@ async def create_turn(
|
||||
session_id=session.id,
|
||||
user_id=user_id,
|
||||
parent_message_id=None,
|
||||
role="assistant",
|
||||
kind="thinking",
|
||||
status="pending",
|
||||
role=PlaygroundMessageRole.ASSISTANT.value,
|
||||
kind=PlaygroundMessageKind.THINKING.value,
|
||||
status=PlaygroundMessageStatus.PENDING.value,
|
||||
title="AI 回应",
|
||||
content="",
|
||||
thinking_content="",
|
||||
@@ -397,9 +412,9 @@ async def _create_assistant_retry_turn(
|
||||
session_id=session.id,
|
||||
user_id=user_id,
|
||||
parent_message_id=user_message.id,
|
||||
role="assistant",
|
||||
kind="thinking",
|
||||
status="pending",
|
||||
role=PlaygroundMessageRole.ASSISTANT.value,
|
||||
kind=PlaygroundMessageKind.THINKING.value,
|
||||
status=PlaygroundMessageStatus.PENDING.value,
|
||||
title="AI 回应",
|
||||
content="",
|
||||
thinking_content="",
|
||||
@@ -439,7 +454,7 @@ async def stop_message(
|
||||
session = await _require_session(db, user_id=user_id, session_key=payload.session_key)
|
||||
message = await _require_visible_message(db, user_id=user_id, public_id=payload.message_id)
|
||||
|
||||
if message.status not in {"pending", "thinking", "answering"}:
|
||||
if message.status not in ACTIVE_MESSAGE_STATUSES:
|
||||
return await _build_action_response(db, session=session)
|
||||
|
||||
active_run = _ACTIVE_RUNS.get(message.public_id)
|
||||
@@ -447,7 +462,7 @@ async def stop_message(
|
||||
active_run.stop_requested.set()
|
||||
active_run.task.cancel()
|
||||
|
||||
message.status = "stopped"
|
||||
message.status = PlaygroundMessageStatus.STOPPED.value
|
||||
if "已手动停止生成" not in (message.meta or []):
|
||||
message.meta = [*(message.meta or []), "已手动停止生成"]
|
||||
await db.flush()
|
||||
@@ -469,7 +484,7 @@ async def resend_turn(
|
||||
db,
|
||||
user_id=user_id,
|
||||
public_id=payload.user_message_id,
|
||||
role="user",
|
||||
role=PlaygroundMessageRole.USER.value,
|
||||
)
|
||||
|
||||
later_messages = await db.execute(
|
||||
@@ -481,7 +496,7 @@ async def resend_turn(
|
||||
)
|
||||
for item in later_messages.scalars().all():
|
||||
item.is_visible = False
|
||||
if item.status in {"pending", "thinking", "answering"}:
|
||||
if item.status in ACTIVE_MESSAGE_STATUSES:
|
||||
active_run = _ACTIVE_RUNS.get(item.public_id)
|
||||
if active_run is not None:
|
||||
active_run.stop_requested.set()
|
||||
@@ -520,7 +535,7 @@ async def edit_user_message(
|
||||
db,
|
||||
user_id=user_id,
|
||||
public_id=payload.user_message_id,
|
||||
role="user",
|
||||
role=PlaygroundMessageRole.USER.value,
|
||||
)
|
||||
|
||||
user_message.content = payload.content.strip()
|
||||
@@ -566,12 +581,12 @@ def _build_conversation_history(messages: Sequence[PlaygroundMessage], current_u
|
||||
for item in messages:
|
||||
if item.id >= current_user_message_id:
|
||||
break
|
||||
if item.role == "system":
|
||||
if item.role == PlaygroundMessageRole.SYSTEM.value:
|
||||
continue
|
||||
history.append(
|
||||
{
|
||||
"role": item.role,
|
||||
"kind": item.kind or "message",
|
||||
"kind": item.kind or PlaygroundMessageKind.MESSAGE.value,
|
||||
"title": item.title,
|
||||
"content": item.content or "",
|
||||
}
|
||||
@@ -668,7 +683,11 @@ async def _run_assistant_message(
|
||||
assistant_message = await _mark_message_state(
|
||||
db,
|
||||
message_id=assistant_message_id,
|
||||
status="thinking" if analysis.thinking_blocks else "answering",
|
||||
status=(
|
||||
PlaygroundMessageStatus.THINKING.value
|
||||
if analysis.thinking_blocks
|
||||
else PlaygroundMessageStatus.ANSWERING.value
|
||||
),
|
||||
title=f"{analysis.provider} / {analysis.model}",
|
||||
provider=analysis.provider,
|
||||
model=analysis.model,
|
||||
@@ -706,7 +725,7 @@ async def _run_assistant_message(
|
||||
await _mark_message_state(
|
||||
db,
|
||||
message_id=assistant_message_id,
|
||||
status="answering",
|
||||
status=PlaygroundMessageStatus.ANSWERING.value,
|
||||
content=content[:cursor],
|
||||
)
|
||||
await db.commit()
|
||||
@@ -717,7 +736,7 @@ async def _run_assistant_message(
|
||||
assistant_message = await _mark_message_state(
|
||||
db,
|
||||
message_id=assistant_message_id,
|
||||
status="done",
|
||||
status=PlaygroundMessageStatus.DONE.value,
|
||||
content=content,
|
||||
meta=[
|
||||
f"Request ID: {request_id}",
|
||||
@@ -762,8 +781,8 @@ async def _run_assistant_message(
|
||||
async with async_session_factory() as db:
|
||||
result = await db.execute(select(PlaygroundMessage).where(PlaygroundMessage.id == assistant_message_id))
|
||||
message = result.scalar_one_or_none()
|
||||
if message is not None and message.status in {"pending", "thinking", "answering"}:
|
||||
message.status = "stopped"
|
||||
if message is not None and message.status in ACTIVE_MESSAGE_STATUSES:
|
||||
message.status = PlaygroundMessageStatus.STOPPED.value
|
||||
if "已手动停止生成" not in (message.meta or []):
|
||||
message.meta = [*(message.meta or []), "已手动停止生成"]
|
||||
await db.flush()
|
||||
@@ -795,7 +814,7 @@ async def _run_assistant_message(
|
||||
result = await db.execute(select(PlaygroundMessage).where(PlaygroundMessage.id == assistant_message_id))
|
||||
message = result.scalar_one_or_none()
|
||||
if message is not None:
|
||||
message.status = "error"
|
||||
message.status = PlaygroundMessageStatus.ERROR.value
|
||||
message.content = message.content or f"分析失败:{error_message}"
|
||||
message.meta = [
|
||||
*(message.meta or []),
|
||||
|
||||
@@ -8,6 +8,7 @@ from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.enums import JobStatus
|
||||
from app.core.logging import get_logger
|
||||
from app.db.session import async_session_factory
|
||||
from app.core.time import to_iso8601_utc
|
||||
@@ -140,7 +141,7 @@ async def run_collector_task(collector_name: str):
|
||||
select(CollectionTask)
|
||||
.where(
|
||||
CollectionTask.datasource_id == datasource.id,
|
||||
CollectionTask.status == "running",
|
||||
CollectionTask.status == JobStatus.RUNNING.value,
|
||||
)
|
||||
.order_by(CollectionTask.started_at.desc(), CollectionTask.id.desc())
|
||||
.limit(1)
|
||||
@@ -184,7 +185,7 @@ async def run_collector_task(collector_name: str):
|
||||
f"Marked failed automatically after stale running timeout "
|
||||
f"({RUNNING_TASK_GUARD_TIMEOUT_MINUTES}m) in scheduler guard"
|
||||
)
|
||||
existing_running.status = "failed"
|
||||
existing_running.status = JobStatus.FAILED.value
|
||||
existing_running.phase = "failed"
|
||||
existing_running.completed_at = now
|
||||
existing_running.error_message = (
|
||||
@@ -243,7 +244,7 @@ async def run_collector_task(collector_name: str):
|
||||
return
|
||||
datasource.last_run_at = datetime.now(UTC)
|
||||
datasource.last_status = task_result.get("status")
|
||||
if datasource.last_status == "success":
|
||||
if datasource.last_status == JobStatus.SUCCESS.value:
|
||||
effective_candidate = await get_builtin_effective_candidate(db, datasource_source)
|
||||
checksum, _credential_context = await build_builtin_connectivity_checksum(
|
||||
datasource_source,
|
||||
@@ -284,7 +285,7 @@ async def run_collector_task(collector_name: str):
|
||||
await db.rollback()
|
||||
datasource = await db.get(DataSource, datasource_id)
|
||||
datasource.last_run_at = datetime.now(UTC)
|
||||
datasource.last_status = "cancelled"
|
||||
datasource.last_status = JobStatus.CANCELLED.value
|
||||
await db.commit()
|
||||
logger.warning_event(
|
||||
"Collector cancelled by operator",
|
||||
@@ -306,7 +307,7 @@ async def run_collector_task(collector_name: str):
|
||||
await db.rollback()
|
||||
datasource = await db.get(DataSource, datasource_id)
|
||||
datasource.last_run_at = datetime.now(UTC)
|
||||
datasource.last_status = "failed"
|
||||
datasource.last_status = JobStatus.FAILED.value
|
||||
await db.commit()
|
||||
logger.exception_event(
|
||||
"Collector failed",
|
||||
@@ -335,7 +336,7 @@ async def cleanup_stale_running_tasks(max_age_hours: int = 2) -> int:
|
||||
async with async_session_factory() as db:
|
||||
result = await db.execute(
|
||||
select(CollectionTask).where(
|
||||
CollectionTask.status == "running",
|
||||
CollectionTask.status == JobStatus.RUNNING.value,
|
||||
CollectionTask.started_at.is_not(None),
|
||||
CollectionTask.started_at < cutoff,
|
||||
)
|
||||
@@ -343,7 +344,7 @@ async def cleanup_stale_running_tasks(max_age_hours: int = 2) -> int:
|
||||
stale_tasks = result.scalars().all()
|
||||
|
||||
for task in stale_tasks:
|
||||
task.status = "failed"
|
||||
task.status = JobStatus.FAILED.value
|
||||
task.phase = "failed"
|
||||
task.completed_at = datetime.now(UTC)
|
||||
existing_error = (task.error_message or "").strip()
|
||||
|
||||
@@ -6,6 +6,7 @@ from typing import Any
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.enums import BGPStatus
|
||||
from app.models.alert import Alert, AlertSeverity, AlertStatus
|
||||
from app.models.bgp_anomaly import BGPAnomaly
|
||||
from app.models.bgp_incident import BGPIncident
|
||||
@@ -49,11 +50,11 @@ async def build_situational_alert_brief_request(
|
||||
|
||||
total_incidents_result = await db.execute(select(func.count(BGPIncident.id)))
|
||||
active_incidents_result = await db.execute(
|
||||
select(func.count(BGPIncident.id)).where(BGPIncident.status == "active")
|
||||
select(func.count(BGPIncident.id)).where(BGPIncident.status == BGPStatus.ACTIVE.value)
|
||||
)
|
||||
bgp_severity_result = await db.execute(
|
||||
select(BGPIncident.severity, func.count(BGPIncident.id))
|
||||
.where(BGPIncident.status == "active")
|
||||
.where(BGPIncident.status == BGPStatus.ACTIVE.value)
|
||||
.group_by(BGPIncident.severity)
|
||||
)
|
||||
bgp_region_counter: Counter[str] = Counter()
|
||||
@@ -65,11 +66,11 @@ async def build_situational_alert_brief_request(
|
||||
|
||||
total_anomalies_result = await db.execute(select(func.count(BGPAnomaly.id)))
|
||||
active_anomalies_result = await db.execute(
|
||||
select(func.count(BGPAnomaly.id)).where(BGPAnomaly.status == "active")
|
||||
select(func.count(BGPAnomaly.id)).where(BGPAnomaly.status == BGPStatus.ACTIVE.value)
|
||||
)
|
||||
anomaly_type_result = await db.execute(
|
||||
select(BGPAnomaly.anomaly_type, func.count(BGPAnomaly.id))
|
||||
.where(BGPAnomaly.status == "active")
|
||||
.where(BGPAnomaly.status == BGPStatus.ACTIVE.value)
|
||||
.group_by(BGPAnomaly.anomaly_type)
|
||||
.order_by(func.count(BGPAnomaly.id).desc())
|
||||
.limit(6)
|
||||
|
||||
@@ -7,6 +7,7 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.core.config import ROOT_DIR
|
||||
from app.core.enums import UserRole
|
||||
from app.core.security import redis_client
|
||||
|
||||
SYSTEM_TASK_TTL_SECONDS = 24 * 60 * 60
|
||||
@@ -47,7 +48,7 @@ def normalize_user_role(role: Any) -> str:
|
||||
|
||||
|
||||
def require_super_admin(user_role: Any) -> bool:
|
||||
return normalize_user_role(user_role) == "super_admin"
|
||||
return normalize_user_role(user_role) == UserRole.SUPER_ADMIN.value
|
||||
|
||||
|
||||
def build_task_id(prefix: str = "restart") -> str:
|
||||
|
||||
@@ -13,6 +13,7 @@ from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.core.enums import LogLevel
|
||||
from app.core.security import redis_client
|
||||
from app.models.system_log import AuditLog, ObservabilityEvent, ObservabilityEventGroup, SystemLog
|
||||
from sqlalchemy import select
|
||||
@@ -24,11 +25,11 @@ BUFFER_LOG_LIMIT = 1000
|
||||
BUFFER_LOG_TTL_SECONDS = 7 * 24 * 60 * 60
|
||||
LOG_BUFFER_KEY_PREFIX = "planet:system_logs"
|
||||
|
||||
LOG_LEVEL_ERROR = "error"
|
||||
LOG_LEVEL_WARNING = "warning"
|
||||
LOG_LEVEL_INFO = "info"
|
||||
LOG_LEVEL_DEBUG = "debug"
|
||||
LOG_LEVEL_ALL = "all"
|
||||
LOG_LEVEL_ERROR = LogLevel.ERROR.value
|
||||
LOG_LEVEL_WARNING = LogLevel.WARNING.value
|
||||
LOG_LEVEL_INFO = LogLevel.INFO.value
|
||||
LOG_LEVEL_DEBUG = LogLevel.DEBUG.value
|
||||
LOG_LEVEL_ALL = LogLevel.ALL.value
|
||||
|
||||
SUPPORTED_LOG_LEVELS = {
|
||||
LOG_LEVEL_ALL,
|
||||
|
||||
@@ -9,7 +9,12 @@ from sqlalchemy import select
|
||||
from sqlalchemy import Float
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.vessel import AISConflictRecord, AISRawObservation, AISSourceHealth
|
||||
from app.models.vessel import (
|
||||
AISConflictRecord,
|
||||
AISRawObservation,
|
||||
AISSourceHealth,
|
||||
VesselCurrentState,
|
||||
)
|
||||
from app.services.vessel_aggregation_strategy import (
|
||||
DEFAULT_STRATEGY,
|
||||
load_strategy,
|
||||
@@ -44,6 +49,17 @@ CONFLICT_FIELDS = (
|
||||
"width",
|
||||
"draught",
|
||||
)
|
||||
CURRENT_STATE_STATIC_FIELDS = (
|
||||
"name",
|
||||
"callsign",
|
||||
"vessel_type",
|
||||
"vessel_type_name",
|
||||
"flag",
|
||||
"length",
|
||||
"width",
|
||||
"draught",
|
||||
"imo",
|
||||
)
|
||||
|
||||
|
||||
def _json_default(value: Any) -> Any:
|
||||
@@ -489,9 +505,115 @@ async def record_vessel_ais_observation(
|
||||
quality_flags=quality_flags or [],
|
||||
)
|
||||
db.add(observation)
|
||||
await upsert_vessel_current_state(
|
||||
db,
|
||||
source=source,
|
||||
normalized_payload=normalized_json,
|
||||
observed_at=observed_at,
|
||||
quality_flags=quality_flags or [],
|
||||
)
|
||||
return observation
|
||||
|
||||
|
||||
async def upsert_vessel_current_state(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
source: str,
|
||||
normalized_payload: dict[str, Any],
|
||||
observed_at: datetime,
|
||||
quality_flags: list[str] | None = None,
|
||||
) -> VesselCurrentState | None:
|
||||
"""Keep one latest renderable row per MMSI while preserving useful static fields."""
|
||||
|
||||
if not _has_valid_position(normalized_payload):
|
||||
return None
|
||||
mmsi = int(normalized_payload["mmsi"])
|
||||
current = await db.get(VesselCurrentState, mmsi)
|
||||
if current is not None and current.observed_at is not None:
|
||||
current_observed_at = _coerce_datetime(current.observed_at)
|
||||
if current_observed_at is not None and observed_at < current_observed_at:
|
||||
return current
|
||||
|
||||
if current is None:
|
||||
current = VesselCurrentState(mmsi=mmsi)
|
||||
db.add(current)
|
||||
|
||||
current.lat = float(normalized_payload["lat"])
|
||||
current.lon = float(normalized_payload["lon"])
|
||||
current.source = source
|
||||
current.observed_at = observed_at
|
||||
current.updated_at = datetime.now(UTC)
|
||||
updated_fields: set[str] = {"lat", "lon"}
|
||||
for field in DYNAMIC_FIELDS:
|
||||
if field in {"lat", "lon"}:
|
||||
continue
|
||||
value = _payload_value(normalized_payload, field)
|
||||
if value is not None:
|
||||
setattr(current, field, value)
|
||||
updated_fields.add(field)
|
||||
field_sources = dict(current.field_sources or {})
|
||||
for field in CURRENT_STATE_STATIC_FIELDS:
|
||||
value = _payload_value(normalized_payload, field)
|
||||
if value is None:
|
||||
continue
|
||||
existing_source = field_sources.get(field)
|
||||
existing_value = getattr(current, field, None)
|
||||
if (
|
||||
existing_value in (None, "")
|
||||
or _strategy_source_rank(source, DEFAULT_STRATEGY)
|
||||
>= _strategy_source_rank(str(existing_source or ""), DEFAULT_STRATEGY)
|
||||
):
|
||||
setattr(current, field, value)
|
||||
updated_fields.add(field)
|
||||
|
||||
current.vessel_type_name = current.vessel_type_name or normalize_vessel_type_name(
|
||||
current.vessel_type
|
||||
)
|
||||
selected_reasons = dict(current.selected_reasons or {})
|
||||
for field in updated_fields:
|
||||
field_sources[field] = source
|
||||
selected_reasons[field] = (
|
||||
"newest_observation" if field in DYNAMIC_FIELDS else "source_priority"
|
||||
)
|
||||
current.field_sources = field_sources
|
||||
current.selected_reasons = selected_reasons
|
||||
current.source_summary = {
|
||||
**dict(current.source_summary or {}),
|
||||
source: {
|
||||
"latest_observed_at": observed_at.isoformat(),
|
||||
},
|
||||
}
|
||||
current.quality_flags = sorted(set((current.quality_flags or []) + (quality_flags or [])))
|
||||
return current
|
||||
|
||||
|
||||
async def get_current_vessels_snapshot(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
bbox: tuple[float, float, float, float],
|
||||
limit: int = 1000,
|
||||
observed_since: datetime,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Read the bounded latest-state table used by Earth rendering."""
|
||||
|
||||
safe_limit = min(max(int(limit or 1000), 1), MAX_SNAPSHOT_LIMIT)
|
||||
lon_min, lat_min, lon_max, lat_max = bbox
|
||||
stmt = (
|
||||
select(VesselCurrentState)
|
||||
.where(VesselCurrentState.observed_at >= observed_since)
|
||||
.where(VesselCurrentState.lon >= lon_min)
|
||||
.where(VesselCurrentState.lon <= lon_max)
|
||||
.where(VesselCurrentState.lat >= lat_min)
|
||||
.where(VesselCurrentState.lat <= lat_max)
|
||||
.order_by(VesselCurrentState.observed_at.desc(), VesselCurrentState.mmsi.asc())
|
||||
.limit(safe_limit)
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
if not hasattr(result, "scalars"):
|
||||
return []
|
||||
return [item.to_dict() for item in result.scalars().all()]
|
||||
|
||||
|
||||
async def aggregate_vessel_observations(
|
||||
db: AsyncSession,
|
||||
observations: Iterable[AISRawObservation],
|
||||
|
||||
Reference in New Issue
Block a user