259 lines
10 KiB
Python
259 lines
10 KiB
Python
"""Classification, importance, and breaking-news policy for Earth news."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime, timedelta
|
|
import re
|
|
from typing import Any, Protocol
|
|
|
|
from app.core.enums import (
|
|
BreakingLevel,
|
|
BreakingScope,
|
|
BreakingSource,
|
|
NewsImportanceLevel,
|
|
NewsMarketImpact,
|
|
NewsTaggingSource,
|
|
parse_enum,
|
|
)
|
|
|
|
|
|
class NewsItemLike(Protocol):
|
|
title: str
|
|
summary: str
|
|
source: str
|
|
feed_name: str
|
|
published_at: datetime | None
|
|
feed_default_category: str
|
|
category: str
|
|
item_tags: list[str]
|
|
tagging_source: str
|
|
tagging_confidence: float
|
|
importance_score: int
|
|
importance_level: str
|
|
importance_reasons: list[str]
|
|
market_impact: str
|
|
source_tags: list[str]
|
|
breaking_level: str
|
|
breaking_scope: str
|
|
breaking_reasons: list[str]
|
|
breaking_source: str
|
|
breaking_confidence: float
|
|
breaking_expires_at: datetime | None
|
|
|
|
|
|
class NewsSourceLike(Protocol):
|
|
default_category: str
|
|
importance_weight: int
|
|
source_tags: tuple[str, ...]
|
|
|
|
|
|
class NewsFeedLike(Protocol):
|
|
default_category: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class BreakingRule:
|
|
level: BreakingLevel
|
|
scope: BreakingScope
|
|
reason: str
|
|
keywords: tuple[str, ...]
|
|
|
|
|
|
IMPORTANCE_THRESHOLDS: tuple[tuple[int, NewsImportanceLevel], ...] = (
|
|
(80, NewsImportanceLevel.CRITICAL),
|
|
(60, NewsImportanceLevel.HIGH),
|
|
(35, NewsImportanceLevel.MEDIUM),
|
|
(0, NewsImportanceLevel.LOW),
|
|
)
|
|
|
|
BREAKING_LEVEL_RANK: dict[BreakingLevel, int] = {
|
|
BreakingLevel.NONE: 0,
|
|
BreakingLevel.WATCH: 1,
|
|
BreakingLevel.BREAKING: 2,
|
|
BreakingLevel.CRITICAL: 3,
|
|
}
|
|
|
|
BREAKING_TTL: dict[BreakingLevel, timedelta] = {
|
|
BreakingLevel.WATCH: timedelta(hours=6),
|
|
BreakingLevel.BREAKING: timedelta(hours=12),
|
|
BreakingLevel.CRITICAL: timedelta(hours=24),
|
|
}
|
|
|
|
BREAKING_RULES: tuple[BreakingRule, ...] = (
|
|
BreakingRule(BreakingLevel.CRITICAL, BreakingScope.GLOBAL, "核事故或核风险", ("nuclear accident", "nuclear emergency", "radiation leak", "核事故", "核泄漏", "辐射泄漏")),
|
|
BreakingRule(BreakingLevel.CRITICAL, BreakingScope.REGIONAL, "重大军事冲突升级", ("airstrike", "missile strike", "invasion", "martial law", "空袭", "导弹袭击", "入侵", "戒严")),
|
|
BreakingRule(BreakingLevel.BREAKING, BreakingScope.REGIONAL, "战争或安全事件", ("war escalates", "terror attack", "coup", "hostage", "战争升级", "恐袭", "政变", "人质")),
|
|
BreakingRule(BreakingLevel.BREAKING, BreakingScope.REGIONAL, "重大灾害应急", ("major earthquake", "tsunami", "volcanic eruption", "state of emergency", "强震", "海啸", "火山喷发", "紧急状态")),
|
|
BreakingRule(BreakingLevel.BREAKING, BreakingScope.GLOBAL, "金融市场异常", ("market halt", "trading halt", "flash crash", "bank run", "金融熔断", "交易暂停", "银行挤兑")),
|
|
BreakingRule(BreakingLevel.WATCH, BreakingScope.GLOBAL, "大规模网络安全事件", ("massive cyberattack", "ransomware attack", "data breach", "大规模网络攻击", "勒索软件", "数据泄露")),
|
|
BreakingRule(BreakingLevel.WATCH, BreakingScope.REGIONAL, "航天或卫星事故", ("rocket explosion", "satellite collision", "space station emergency", "火箭爆炸", "卫星碰撞", "空间站事故")),
|
|
)
|
|
|
|
|
|
def contains_keyword(text: str, keyword: str) -> bool:
|
|
keyword_text = str(keyword or "").strip().lower()
|
|
if not keyword_text:
|
|
return False
|
|
if re.search(r"[\u4e00-\u9fff]", keyword_text):
|
|
return keyword_text in text
|
|
return re.search(rf"(?<![a-z0-9]){re.escape(keyword_text)}(?![a-z0-9])", text) is not None
|
|
|
|
|
|
def score_category(text: str, title_text: str, category: dict[str, Any]) -> int:
|
|
score = 0
|
|
keywords = category.get("keywords") if isinstance(category.get("keywords"), list) else []
|
|
for keyword in keywords:
|
|
if contains_keyword(title_text, keyword):
|
|
score += 3
|
|
elif contains_keyword(text, keyword):
|
|
score += 1
|
|
return score
|
|
|
|
|
|
def importance_level(score: int) -> NewsImportanceLevel:
|
|
normalized_score = max(0, min(100, int(score)))
|
|
for threshold, level in IMPORTANCE_THRESHOLDS:
|
|
if normalized_score >= threshold:
|
|
return level
|
|
return NewsImportanceLevel.LOW
|
|
|
|
|
|
def normalize_breaking_level(value: object) -> BreakingLevel:
|
|
return parse_enum(BreakingLevel, value, BreakingLevel.NONE)
|
|
|
|
|
|
def normalize_breaking_scope(value: object) -> BreakingScope:
|
|
return parse_enum(BreakingScope, value, BreakingScope.REGIONAL)
|
|
|
|
|
|
def breaking_expires_at(level: object, published_at: datetime | None) -> datetime | None:
|
|
normalized = normalize_breaking_level(level)
|
|
if normalized is BreakingLevel.NONE:
|
|
return None
|
|
base = published_at or datetime.now(UTC)
|
|
base = base.replace(tzinfo=UTC) if base.tzinfo is None else base.astimezone(UTC)
|
|
return base + BREAKING_TTL[normalized]
|
|
|
|
|
|
def is_breaking_active(item: NewsItemLike, *, now: datetime | None = None) -> bool:
|
|
if normalize_breaking_level(item.breaking_level) is BreakingLevel.NONE:
|
|
return False
|
|
expires_at = item.breaking_expires_at
|
|
if expires_at is None:
|
|
return True
|
|
expires_at = expires_at.replace(tzinfo=UTC) if expires_at.tzinfo is None else expires_at.astimezone(UTC)
|
|
return expires_at > (now or datetime.now(UTC))
|
|
|
|
|
|
def breaking_sort_rank(item: NewsItemLike) -> int:
|
|
if not is_breaking_active(item):
|
|
return 0
|
|
return BREAKING_LEVEL_RANK[normalize_breaking_level(item.breaking_level)]
|
|
|
|
|
|
def highest_breaking_level(items: list[NewsItemLike]) -> BreakingLevel:
|
|
active = [normalize_breaking_level(item.breaking_level) for item in items if is_breaking_active(item)]
|
|
return max(active, key=BREAKING_LEVEL_RANK.get) if active else BreakingLevel.NONE
|
|
|
|
|
|
def apply_breaking_rules(item: NewsItemLike) -> None:
|
|
combined_text = f"{item.title} {item.summary} {item.source} {item.feed_name}".lower()
|
|
best_level = BreakingLevel.NONE
|
|
best_scope = BreakingScope.REGIONAL
|
|
reasons: list[str] = []
|
|
confidence = 0.0
|
|
for rule in BREAKING_RULES:
|
|
if not any(contains_keyword(combined_text, keyword) for keyword in rule.keywords):
|
|
continue
|
|
if BREAKING_LEVEL_RANK[rule.level] > BREAKING_LEVEL_RANK[best_level]:
|
|
best_level = rule.level
|
|
best_scope = rule.scope
|
|
if rule.reason not in reasons:
|
|
reasons.append(rule.reason)
|
|
confidence = max(confidence, 0.72 if rule.level is BreakingLevel.CRITICAL else 0.64 if rule.level is BreakingLevel.BREAKING else 0.52)
|
|
|
|
item.breaking_level = best_level.value
|
|
item.breaking_scope = (best_scope if best_level is not BreakingLevel.NONE else BreakingScope.REGIONAL).value
|
|
item.breaking_reasons = reasons
|
|
item.breaking_source = BreakingSource.RULES.value
|
|
item.breaking_confidence = round(confidence, 2)
|
|
item.breaking_expires_at = breaking_expires_at(best_level, item.published_at)
|
|
|
|
|
|
def apply_news_classification(
|
|
item: NewsItemLike,
|
|
source: NewsSourceLike,
|
|
*,
|
|
feed: NewsFeedLike | None,
|
|
config: dict[str, Any],
|
|
) -> NewsItemLike:
|
|
title_text = item.title.lower()
|
|
combined_text = f"{item.title} {item.summary} {item.source} {item.feed_name}".lower()
|
|
feed_default_category = (feed.default_category if feed else item.feed_default_category) or source.default_category or "other"
|
|
best_key = feed_default_category
|
|
best_score = second_score = 0
|
|
for category in config["categories"]:
|
|
if not isinstance(category, dict) or category.get("enabled") is False:
|
|
continue
|
|
score = score_category(combined_text, title_text, category)
|
|
if score > best_score:
|
|
second_score, best_score = best_score, score
|
|
best_key = str(category.get("key") or "other")
|
|
elif score > second_score:
|
|
second_score = score
|
|
|
|
item_tags: list[str] = []
|
|
for rule in config["item_tag_rules"]:
|
|
if not isinstance(rule, dict):
|
|
continue
|
|
keywords = rule.get("keywords") if isinstance(rule.get("keywords"), list) else []
|
|
if any(contains_keyword(combined_text, keyword) for keyword in keywords):
|
|
tag_key = str(rule.get("key") or "").strip()
|
|
if tag_key and tag_key not in item_tags:
|
|
item_tags.append(tag_key)
|
|
if best_score < 3 and rule.get("category"):
|
|
best_key, best_score = str(rule["category"]), 3
|
|
|
|
confidence = round(best_score / (best_score + second_score + 1), 2) if best_score else 0.35
|
|
if best_score < 3 and feed_default_category:
|
|
best_key, confidence = feed_default_category, 0.45
|
|
|
|
score = max(0, min(100, 18 + source.importance_weight + best_score * 6))
|
|
reasons: list[str] = []
|
|
source_tags = set(source.source_tags)
|
|
if "official_data" in source_tags:
|
|
score += 20
|
|
reasons.append("官方数据源")
|
|
if "press_release" in source_tags:
|
|
score = max(0, score - 12)
|
|
reasons.append("企业公告基础权重较低")
|
|
if any(contains_keyword(combined_text, term) for term in ("网上零售额", "电商物流指数", "gmv", "订单量", "物流指数", "履约", "直播电商", "跨境电商")):
|
|
score += 25
|
|
reasons.append("命中电商数据指标")
|
|
if any(contains_keyword(combined_text, term) for term in ("amazon", "shopify", "walmart", "alibaba", "jd.com", "pinduoduo", "tiktok shop", "shein", "阿里", "京东", "拼多多", "抖音")):
|
|
score += 15
|
|
reasons.append("涉及大型平台")
|
|
if any(term in combined_text for term in ("同比", "环比", "%", "billion", "million", "增长", "下降")):
|
|
score += 10
|
|
reasons.append("包含量化指标")
|
|
|
|
score = max(0, min(100, score))
|
|
item.category = best_key or "other"
|
|
item.item_tags = item_tags
|
|
item.tagging_source = NewsTaggingSource.RULES.value
|
|
item.tagging_confidence = confidence
|
|
item.importance_score = score
|
|
item.importance_level = importance_level(score).value
|
|
item.importance_reasons = reasons or ["按来源权重和分类规则计算"]
|
|
item.market_impact = (
|
|
NewsMarketImpact.GLOBAL.value
|
|
if "global" in source_tags
|
|
else NewsMarketImpact.NATIONAL.value
|
|
if {"china", "us"} & source_tags
|
|
else NewsMarketImpact.SECTOR.value
|
|
)
|
|
item.source_tags = list(source.source_tags)
|
|
apply_breaking_rules(item)
|
|
return item
|