from __future__ import annotations from dataclasses import dataclass from datetime import UTC, datetime import hashlib import html import json import re from typing import Any from bs4 import BeautifulSoup from sqlalchemy import delete, func, select from sqlalchemy.ext.asyncio import AsyncSession from app.core.enums import NewsEnrichmentStatus, NewsSourceType, NewsTaggingSource from app.core.websocket.broadcaster import broadcaster from app.models.earth_news import EarthNewsItem from app.models.system_setting import SystemSetting from app.services.earth_news import ( ALLOWED_NEWS_CATEGORY_KEYS, DEFAULT_NEWS_LOCALE, REGION_ANCHORS, NewsFeedEndpoint, NewsFeedSource, NewsTargetLocation, ParsedNewsItem, apply_news_classification, build_anchor_location_patch, build_target_location_job_payload, build_target_location_patch, _serialize_item, ) from app.services.earth_news_queue import enqueue_target_location_job from app.services.earth_news_store import record_to_parsed_news_item MANUAL_NEWS_SOURCE_ID = "manual" MANUAL_NEWS_SOURCE_LABEL = "手动添加" MANUAL_NEWS_MAX_IMPORT_ITEMS = 500 MANUAL_NEWS_MAX_TITLE_LENGTH = 500 MANUAL_NEWS_MAX_SUMMARY_LENGTH = 1200 MANUAL_NEWS_MAX_CONTENT_LENGTH = 12000 EARTH_NEWS_MANUAL_GROUPS_CATEGORY = "earth_news_manual_groups" DEFAULT_MANUAL_NEWS_GROUP_ID = "manual-default" DEFAULT_MANUAL_NEWS_GROUP_NAME = "新建新闻组" @dataclass(frozen=True) class ManualNewsWriteResult: item: EarthNewsItem created: bool queued: bool @dataclass(frozen=True) class ManualNewsGroup: id: str name: str sort_order: int = 0 def _clean_text(value: object, *, max_length: int) -> str: raw = "" if value is None else str(value) text = BeautifulSoup(html.unescape(raw), "html.parser").get_text(" ", strip=True) text = re.sub(r"\s+", " ", text).strip() if len(text) > max_length: return text[: max_length - 1].rstrip() + "…" return text def _parse_datetime(value: object) -> datetime | None: if value is None or str(value).strip() == "": return None if isinstance(value, datetime): parsed = value else: try: parsed = datetime.fromisoformat(str(value).strip().replace("Z", "+00:00")) except ValueError as exc: raise ValueError("published_at 必须是 ISO8601 时间。") from exc if parsed.tzinfo is None: return parsed.replace(tzinfo=UTC) return parsed.astimezone(UTC) def _detect_language(*parts: str) -> str: text = " ".join(part for part in parts if part) cjk_count = len(re.findall(r"[\u4e00-\u9fff]", text)) latin_count = len(re.findall(r"[A-Za-z]", text)) return "zh-CN" if cjk_count >= max(4, latin_count // 3) else "en-US" def _manual_item_id(*, title: str, published_at: datetime | None, url: str, source: str) -> str: published = published_at.isoformat() if published_at else "" basis = "\n".join([title.strip().lower(), published, url.strip().lower(), source.strip().lower()]) return f"manual:{hashlib.sha1(basis.encode('utf-8')).hexdigest()[:16]}" def _manual_group_id(name: str) -> str: basis = f"{name.strip().lower()}\n{datetime.now(UTC).isoformat()}" return f"manual-group:{hashlib.sha1(basis.encode('utf-8')).hexdigest()[:10]}" def _news_meta(record: EarthNewsItem) -> dict[str, Any]: location_meta = record.location_meta if isinstance(record.location_meta, dict) else {} news_meta = location_meta.get("news_meta") return dict(news_meta) if isinstance(news_meta, dict) else {} def _record_source_type(record: EarthNewsItem) -> str: return str(_news_meta(record).get("feed_type") or _news_meta(record).get("source_type") or "rss") def _record_manual_group_id(record: EarthNewsItem) -> str: return str(_news_meta(record).get("manual_group_id") or DEFAULT_MANUAL_NEWS_GROUP_ID) def _rss_group_id(record: EarthNewsItem) -> str: basis = "\n".join( [ _record_source_type(record), str(record.feed_name or ""), str(record.source or ""), ] ) return f"rss:{hashlib.sha1(basis.encode('utf-8')).hexdigest()[:12]}" def _default_manual_group() -> dict[str, Any]: return { "id": DEFAULT_MANUAL_NEWS_GROUP_ID, "name": DEFAULT_MANUAL_NEWS_GROUP_NAME, "sort_order": 0, } def _normalize_manual_groups_payload(payload: Any) -> list[dict[str, Any]]: raw_groups = payload.get("groups") if isinstance(payload, dict) else None normalized: list[dict[str, Any]] = [] seen: set[str] = set() for index, item in enumerate(raw_groups if isinstance(raw_groups, list) else []): if not isinstance(item, dict): continue group_id = str(item.get("id") or "").strip() name = _clean_text(item.get("name"), max_length=120) if not group_id or not name or group_id in seen: continue normalized.append( { "id": group_id, "name": name, "sort_order": int(item.get("sort_order") or index), } ) seen.add(group_id) if DEFAULT_MANUAL_NEWS_GROUP_ID not in seen: normalized.insert(0, _default_manual_group()) return sorted(normalized, key=lambda item: (int(item.get("sort_order") or 0), str(item.get("name") or ""))) async def _get_manual_groups_record(db: AsyncSession) -> SystemSetting | None: result = await db.execute( select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_MANUAL_GROUPS_CATEGORY) ) return result.scalar_one_or_none() async def get_manual_news_groups(db: AsyncSession) -> list[dict[str, Any]]: record = await _get_manual_groups_record(db) return _normalize_manual_groups_payload(record.payload if record else None) async def _save_manual_news_groups(db: AsyncSession, groups: list[dict[str, Any]]) -> list[dict[str, Any]]: normalized = _normalize_manual_groups_payload({"groups": groups}) record = await _get_manual_groups_record(db) payload = {"groups": normalized} if record is None: db.add(SystemSetting(category=EARTH_NEWS_MANUAL_GROUPS_CATEGORY, payload=payload)) else: record.payload = payload await db.flush() return normalized async def resolve_manual_news_group(db: AsyncSession, group_id: str | None) -> ManualNewsGroup: normalized_id = str(group_id or DEFAULT_MANUAL_NEWS_GROUP_ID).strip() or DEFAULT_MANUAL_NEWS_GROUP_ID groups = await get_manual_news_groups(db) match = next((item for item in groups if item.get("id") == normalized_id), None) if match is None and normalized_id != DEFAULT_MANUAL_NEWS_GROUP_ID: raise ValueError(f"手动新闻组不存在：{normalized_id}") match = match or _default_manual_group() return ManualNewsGroup( id=str(match["id"]), name=str(match["name"]), sort_order=int(match.get("sort_order") or 0), ) async def create_manual_news_group(db: AsyncSession, name: str) -> dict[str, Any]: group_name = _clean_text(name, max_length=120) if not group_name: raise ValueError("新闻组名称不能为空。") groups = await get_manual_news_groups(db) group = {"id": _manual_group_id(group_name), "name": group_name, "sort_order": len(groups)} groups.append(group) await _save_manual_news_groups(db, groups) return group async def rename_manual_news_group(db: AsyncSession, group_id: str, name: str) -> dict[str, Any]: group_name = _clean_text(name, max_length=120) if not group_name: raise ValueError("新闻组名称不能为空。") groups = await get_manual_news_groups(db) match = next((item for item in groups if item.get("id") == group_id), None) if match is None: raise ValueError(f"手动新闻组不存在：{group_id}") match["name"] = group_name await _save_manual_news_groups(db, groups) result = await db.execute( select(EarthNewsItem).where( EarthNewsItem.location_meta.op("->")("news_meta").op("->>")("feed_type") == NewsSourceType.MANUAL.value ) ) for record in result.scalars().all(): if _record_manual_group_id(record) != group_id: continue location_meta = dict(record.location_meta or {}) news_meta = dict(location_meta.get("news_meta") or {}) news_meta["manual_group_name"] = group_name location_meta["news_meta"] = news_meta record.location_meta = location_meta await db.flush() return match def _normalize_region(value: object) -> str: region = str(value or "global").strip().lower() or "global" if region not in REGION_ANCHORS: raise ValueError(f"region 不支持：{region}") return region def _normalize_tags(value: object) -> list[str]: if value is None: return [] if isinstance(value, str): parts = re.split(r"[,，\n]", value) elif isinstance(value, list): parts = [str(item) for item in value] else: raise ValueError("tags 必须是字符串数组或逗号分隔字符串。") return [item.strip() for item in parts if item.strip()][:20] def _normalize_location(value: object) -> NewsTargetLocation | None: if value in (None, ""): return None if not isinstance(value, dict): raise ValueError("location 必须是对象。") lat = value.get("latitude") lon = value.get("longitude") if lat in (None, "") and lon in (None, ""): return None try: latitude = float(lat) longitude = float(lon) except (TypeError, ValueError) as exc: raise ValueError("location.latitude / longitude 必须是数字。") from exc if not -90 <= latitude <= 90 or not -180 <= longitude <= 180: raise ValueError("location 经纬度超出范围。") label = _clean_text(value.get("label"), max_length=255) if not label: label = f"{latitude:.4f}, {longitude:.4f}" return NewsTargetLocation( latitude=latitude, longitude=longitude, label=label, source="manual_location", confidence=1.0, country=_clean_text(value.get("country"), max_length=100) or None, city=_clean_text(value.get("city"), max_length=100) or None, ) def _manual_source(source_name: str, *, region: str) -> NewsFeedSource: return NewsFeedSource( id=MANUAL_NEWS_SOURCE_ID, name=source_name or MANUAL_NEWS_SOURCE_LABEL, region=region, feed_url="", homepage_url="", source_type=NewsSourceType.MANUAL.value, default_category="other", source_tags=("manual",), ) def _manual_feed(category: str) -> NewsFeedEndpoint: return NewsFeedEndpoint( id=MANUAL_NEWS_SOURCE_ID, name=MANUAL_NEWS_SOURCE_LABEL, url="", type=NewsSourceType.MANUAL.value, default_category=category or "other", tags=("manual",), priority=1, ) def parsed_manual_news_item( payload: dict[str, Any], *, item_id_override: str | None = None, ) -> tuple[ParsedNewsItem, NewsTargetLocation | None, str]: title = _clean_text(payload.get("title"), max_length=MANUAL_NEWS_MAX_TITLE_LENGTH) if not title: raise ValueError("title 不能为空。") content = _clean_text(payload.get("content"), max_length=MANUAL_NEWS_MAX_CONTENT_LENGTH) summary = _clean_text(payload.get("summary"), max_length=MANUAL_NEWS_MAX_SUMMARY_LENGTH) if not summary: summary = _clean_text(content, max_length=240) if content else title source = _clean_text(payload.get("source"), max_length=255) or MANUAL_NEWS_SOURCE_LABEL region = _normalize_region(payload.get("region")) published_at = _parse_datetime(payload.get("published_at")) or datetime.now(UTC) url = str(payload.get("url") or "").strip() category = str(payload.get("category") or "other").strip().lower() or "other" if category not in ALLOWED_NEWS_CATEGORY_KEYS: raise ValueError(f"category 不支持：{category}") tags = _normalize_tags(payload.get("tags")) target = _normalize_location(payload.get("location")) language = str(payload.get("content_language") or "").strip() or _detect_language(title, summary, content) localizations = { language: { "title": title, "summary": summary, } } item = ParsedNewsItem( id=item_id_override or _manual_item_id(title=title, published_at=published_at, url=url, source=source), title=title, summary=summary, url=url, source=source, feed_name=MANUAL_NEWS_SOURCE_LABEL, feed_region=region, homepage_url=str(payload.get("homepage_url") or ""), published_at=published_at, content_language=language, localizations=localizations, enrichment_status=NewsEnrichmentStatus.PENDING.value, source_tags=["manual"], feed_id=MANUAL_NEWS_SOURCE_ID, feed_type=NewsSourceType.MANUAL.value, feed_default_category=category, category=category, item_tags=tags, tagging_source=NewsTaggingSource.MANUAL.value if payload.get("category") else NewsTaggingSource.RULES.value, tagging_confidence=0.9 if payload.get("category") else 0.0, ) source_config = _manual_source(source, region=region) feed = _manual_feed(category) apply_news_classification(item, source_config, feed=feed) if payload.get("category"): item.category = category item.tagging_source = NewsTaggingSource.MANUAL.value item.tagging_confidence = 0.9 if tags: item.item_tags = sorted(set([*item.item_tags, *tags])) return item, target, content def _manual_editable(record: EarthNewsItem) -> bool: if record.id.startswith("manual:"): return True news_meta = (record.location_meta or {}).get("news_meta") if isinstance(record.location_meta, dict) else None return isinstance(news_meta, dict) and news_meta.get("feed_type") == NewsSourceType.MANUAL.value async def _broadcast_news_reload() -> None: await broadcaster.broadcast_earth_update( { "action": "database_changed", "source": "earth_news_items", "layers": ["news"], "refresh_strategy": "reload", } ) async def upsert_manual_news_item( db: AsyncSession, payload: dict[str, Any], *, item_id_override: str | None = None, group_id: str | None = None, ) -> ManualNewsWriteResult: item, target, content = parsed_manual_news_item(payload, item_id_override=item_id_override) group = await resolve_manual_news_group(db, group_id or payload.get("group_id")) existing = await db.get(EarthNewsItem, item.id) created = existing is None patch = build_target_location_patch(item, target) if target else build_anchor_location_patch(item) patch_meta = dict(patch.get("location_meta") or {}) patch_news_meta = dict(patch_meta.get("news_meta") or {}) patch_news_meta["feed_type"] = NewsSourceType.MANUAL.value patch_news_meta["source_type"] = NewsSourceType.MANUAL.value patch_news_meta["manual_group_id"] = group.id patch_news_meta["manual_group_name"] = group.name patch_meta["news_meta"] = patch_news_meta patch["location_meta"] = patch_meta now = datetime.now(UTC) record = existing or EarthNewsItem( id=item.id, title=item.title, summary=item.summary, content_language=item.content_language, localizations=dict(item.localizations or {}), url=item.url, source=item.source, feed_name=item.feed_name, region=item.feed_region, homepage_url=item.homepage_url, published_at=item.published_at, latitude=patch["latitude"], longitude=patch["longitude"], location_label=patch["location_label"], location_source=patch["location_source"], verified=patch["verified"], location_meta=patch["location_meta"], first_seen_at=now, last_seen_at=now, resolved_at=now if patch["verified"] else None, enrichment_status=item.enrichment_status, ) if existing is None: db.add(record) else: if not _manual_editable(record): raise PermissionError("RSS 新闻不允许通过手动新闻接口编辑。") record.title = item.title record.summary = item.summary record.content_language = item.content_language record.localizations = dict(item.localizations or {}) record.url = item.url record.source = item.source record.feed_name = item.feed_name record.region = item.feed_region record.homepage_url = item.homepage_url record.published_at = item.published_at record.last_seen_at = now if target is None and record.location_source == "manual_location": merged_meta = dict(record.location_meta or {}) patch_meta = patch.get("location_meta") if isinstance(patch, dict) else None patch_news_meta = patch_meta.get("news_meta") if isinstance(patch_meta, dict) else None if isinstance(patch_news_meta, dict): merged_meta["news_meta"] = patch_news_meta record.location_meta = merged_meta else: record.location_meta = patch["location_meta"] if target: record.latitude = patch["latitude"] record.longitude = patch["longitude"] record.location_label = patch["location_label"] record.location_source = patch["location_source"] record.verified = patch["verified"] record.resolved_at = now elif record.location_source != "manual_location": record.latitude = patch["latitude"] record.longitude = patch["longitude"] record.location_label = patch["location_label"] record.location_source = patch["location_source"] record.verified = patch["verified"] record.resolved_at = None record.enrichment_status = NewsEnrichmentStatus.PENDING.value record.enrichment_error = None record.enriched_at = None if content: meta = dict(record.location_meta or {}) meta["manual_content"] = content record.location_meta = meta await db.flush() queued = await enqueue_target_location_job(build_target_location_job_payload(item), force=True) if queued: record.enrichment_status = NewsEnrichmentStatus.QUEUED.value await db.flush() return ManualNewsWriteResult(item=record, created=created, queued=queued) async def import_manual_news_items( db: AsyncSession, payload: list[Any], *, group_id: str | None = None, ) -> dict[str, Any]: if len(payload) > MANUAL_NEWS_MAX_IMPORT_ITEMS: raise ValueError(f"单次最多导入 {MANUAL_NEWS_MAX_IMPORT_ITEMS} 条。") created = 0 updated = 0 queued = 0 errors: list[dict[str, Any]] = [] for index, raw_item in enumerate(payload): if not isinstance(raw_item, dict): errors.append({"index": index, "error": "条目必须是 JSON 对象。"}) continue try: result = await upsert_manual_news_item(db, raw_item, group_id=group_id) created += 1 if result.created else 0 updated += 0 if result.created else 1 queued += 1 if result.queued else 0 except Exception as exc: errors.append({"index": index, "error": str(exc)}) if errors and created == 0 and updated == 0: raise ValueError("导入失败，未写入任何新闻。") return {"created": created, "updated": updated, "queued": queued, "failed": len(errors), "errors": errors} async def parse_manual_news_import_upload(raw_bytes: bytes) -> list[Any]: try: payload = json.loads(raw_bytes.decode("utf-8-sig")) except UnicodeDecodeError as exc: raise ValueError("JSON 文件必须使用 UTF-8 编码。") from exc except json.JSONDecodeError as exc: raise ValueError(f"JSON 解析失败：第 {exc.lineno} 行第 {exc.colno} 列。") from exc if not isinstance(payload, list): raise ValueError("JSON 顶层必须是数组。") return payload def serialize_news_record(record: EarthNewsItem, *, locale: str = DEFAULT_NEWS_LOCALE) -> dict[str, Any]: item = record_to_parsed_news_item(record) payload = _serialize_item(item, active_region=item.feed_region, locale=locale) news_meta = _news_meta(record) payload["editable"] = _manual_editable(record) payload["source_type"] = payload.get("feed_type") payload["status"] = record.enrichment_status payload["translated"] = bool((record.localizations or {}).get("zh-CN") and (record.localizations or {}).get("en-US")) payload["manual_content"] = (record.location_meta or {}).get("manual_content") if isinstance(record.location_meta, dict) else None payload["manual_group_id"] = news_meta.get("manual_group_id") payload["manual_group_name"] = news_meta.get("manual_group_name") return payload def _record_matches_group(record: EarthNewsItem, group_id: str) -> bool: source_type = _record_source_type(record) if source_type == NewsSourceType.MANUAL.value: return _record_manual_group_id(record) == group_id return _rss_group_id(record) == group_id async def list_news_records( db: AsyncSession, *, page: int, page_size: int, source_type: str | None = None, region: str | None = None, category: str | None = None, status_filter: str | None = None, group_id: str | None = None, ) -> dict[str, Any]: page = max(page, 1) page_size = min(max(page_size, 1), 100) query = select(EarthNewsItem) count_query = select(func.count(EarthNewsItem.id)) filters = [] if region and region != "all": filters.append(EarthNewsItem.region == region) if status_filter and status_filter != "all": filters.append(EarthNewsItem.enrichment_status == status_filter) if source_type and source_type != "all": filters.append(EarthNewsItem.location_meta.op("->")("news_meta").op("->>")("feed_type") == source_type) if category and category != "all": filters.append(EarthNewsItem.location_meta.op("->")("news_meta").op("->>")("category") == category) for clause in filters: query = query.where(clause) count_query = count_query.where(clause) ordered_query = query.order_by(EarthNewsItem.published_at.desc().nullslast(), EarthNewsItem.last_seen_at.desc()) if group_id: result = await db.execute(ordered_query) all_records = [record for record in result.scalars().all() if _record_matches_group(record, group_id)] total = len(all_records) records = all_records[(page - 1) * page_size : page * page_size] else: total_result = await db.execute(count_query) result = await db.execute( ordered_query.offset((page - 1) * page_size).limit(page_size) ) records = list(result.scalars().all()) total = int(total_result.scalar() or 0) return { "items": [serialize_news_record(record) for record in records], "page": page, "page_size": page_size, "total": total, } async def list_news_groups(db: AsyncSession, *, locale: str = DEFAULT_NEWS_LOCALE) -> dict[str, Any]: manual_groups = await get_manual_news_groups(db) manual_by_id: dict[str, dict[str, Any]] = { str(group["id"]): { "id": str(group["id"]), "name": str(group["name"]), "group_type": "manual", "source_type": NewsSourceType.MANUAL.value, "editable": True, "sort_order": int(group.get("sort_order") or 0), "count": 0, "items": [], } for group in manual_groups } rss_by_id: dict[str, dict[str, Any]] = {} result = await db.execute( select(EarthNewsItem).order_by(EarthNewsItem.published_at.desc().nullslast(), EarthNewsItem.last_seen_at.desc()) ) for record in result.scalars().all(): serialized = serialize_news_record(record, locale=locale) source_type = _record_source_type(record) if source_type == NewsSourceType.MANUAL.value: group_id = _record_manual_group_id(record) group = manual_by_id.setdefault( group_id, { "id": group_id, "name": str(_news_meta(record).get("manual_group_name") or DEFAULT_MANUAL_NEWS_GROUP_NAME), "group_type": "manual", "source_type": NewsSourceType.MANUAL.value, "editable": True, "sort_order": len(manual_by_id), "count": 0, "items": [], }, ) else: group_id = _rss_group_id(record) group = rss_by_id.setdefault( group_id, { "id": group_id, "name": record.feed_name or record.source or "RSS 新闻", "group_type": "rss", "source_type": source_type, "editable": False, "region": record.region, "source": record.source, "feed_name": record.feed_name, "count": 0, "items": [], }, ) group["count"] = int(group.get("count") or 0) + 1 group.setdefault("items", []).append(serialized) manual_items = sorted(manual_by_id.values(), key=lambda item: (int(item.get("sort_order") or 0), str(item.get("name") or ""))) rss_items = sorted(rss_by_id.values(), key=lambda item: str(item.get("name") or "")) return {"groups": [*manual_items, *rss_items], "manual_groups": manual_items, "rss_groups": rss_items} async def get_news_record_or_404(db: AsyncSession, item_id: str) -> EarthNewsItem | None: return await db.get(EarthNewsItem, item_id) async def delete_manual_news_item(db: AsyncSession, item_id: str) -> bool: record = await db.get(EarthNewsItem, item_id) if record is None: return False if not _manual_editable(record): raise PermissionError("RSS 新闻不允许通过手动新闻接口删除。") await db.execute(delete(EarthNewsItem).where(EarthNewsItem.id == item_id)) await db.flush() return True async def reprocess_manual_news_item(db: AsyncSession, item_id: str) -> bool: record = await db.get(EarthNewsItem, item_id) if record is None: return False if not _manual_editable(record): raise PermissionError("RSS 新闻不允许通过手动新闻接口重新处理。") item = record_to_parsed_news_item(record) queued = await enqueue_target_location_job(build_target_location_job_payload(item), force=True) if queued: record.enrichment_status = NewsEnrichmentStatus.QUEUED.value record.enrichment_error = None await db.flush() return queued async def broadcast_manual_news_changed() -> None: await _broadcast_news_reload()