694 lines
27 KiB
Python
694 lines
27 KiB
Python
from __future__ import annotations
|
||
|
||
from dataclasses import dataclass
|
||
from datetime import UTC, datetime
|
||
import hashlib
|
||
import html
|
||
import json
|
||
import re
|
||
from typing import Any
|
||
|
||
from bs4 import BeautifulSoup
|
||
from sqlalchemy import delete, func, select
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.core.enums import NewsEnrichmentStatus, NewsSourceType, NewsTaggingSource
|
||
from app.core.websocket.broadcaster import broadcaster
|
||
from app.models.earth_news import EarthNewsItem
|
||
from app.models.system_setting import SystemSetting
|
||
from app.services.earth_news import (
|
||
ALLOWED_NEWS_CATEGORY_KEYS,
|
||
DEFAULT_NEWS_LOCALE,
|
||
REGION_ANCHORS,
|
||
NewsFeedEndpoint,
|
||
NewsFeedSource,
|
||
NewsTargetLocation,
|
||
ParsedNewsItem,
|
||
apply_news_classification,
|
||
build_anchor_location_patch,
|
||
build_target_location_job_payload,
|
||
build_target_location_patch,
|
||
_serialize_item,
|
||
)
|
||
from app.services.earth_news_queue import enqueue_target_location_job
|
||
from app.services.earth_news_store import record_to_parsed_news_item
|
||
|
||
|
||
MANUAL_NEWS_SOURCE_ID = "manual"
|
||
MANUAL_NEWS_SOURCE_LABEL = "手动添加"
|
||
MANUAL_NEWS_MAX_IMPORT_ITEMS = 500
|
||
MANUAL_NEWS_MAX_TITLE_LENGTH = 500
|
||
MANUAL_NEWS_MAX_SUMMARY_LENGTH = 1200
|
||
MANUAL_NEWS_MAX_CONTENT_LENGTH = 12000
|
||
EARTH_NEWS_MANUAL_GROUPS_CATEGORY = "earth_news_manual_groups"
|
||
DEFAULT_MANUAL_NEWS_GROUP_ID = "manual-default"
|
||
DEFAULT_MANUAL_NEWS_GROUP_NAME = "新建新闻组"
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class ManualNewsWriteResult:
|
||
item: EarthNewsItem
|
||
created: bool
|
||
queued: bool
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class ManualNewsGroup:
|
||
id: str
|
||
name: str
|
||
sort_order: int = 0
|
||
|
||
|
||
def _clean_text(value: object, *, max_length: int) -> str:
|
||
raw = "" if value is None else str(value)
|
||
text = BeautifulSoup(html.unescape(raw), "html.parser").get_text(" ", strip=True)
|
||
text = re.sub(r"\s+", " ", text).strip()
|
||
if len(text) > max_length:
|
||
return text[: max_length - 1].rstrip() + "…"
|
||
return text
|
||
|
||
|
||
def _parse_datetime(value: object) -> datetime | None:
|
||
if value is None or str(value).strip() == "":
|
||
return None
|
||
if isinstance(value, datetime):
|
||
parsed = value
|
||
else:
|
||
try:
|
||
parsed = datetime.fromisoformat(str(value).strip().replace("Z", "+00:00"))
|
||
except ValueError as exc:
|
||
raise ValueError("published_at 必须是 ISO8601 时间。") from exc
|
||
if parsed.tzinfo is None:
|
||
return parsed.replace(tzinfo=UTC)
|
||
return parsed.astimezone(UTC)
|
||
|
||
|
||
def _detect_language(*parts: str) -> str:
|
||
text = " ".join(part for part in parts if part)
|
||
cjk_count = len(re.findall(r"[\u4e00-\u9fff]", text))
|
||
latin_count = len(re.findall(r"[A-Za-z]", text))
|
||
return "zh-CN" if cjk_count >= max(4, latin_count // 3) else "en-US"
|
||
|
||
|
||
def _manual_item_id(*, title: str, published_at: datetime | None, url: str, source: str) -> str:
|
||
published = published_at.isoformat() if published_at else ""
|
||
basis = "\n".join([title.strip().lower(), published, url.strip().lower(), source.strip().lower()])
|
||
return f"manual:{hashlib.sha1(basis.encode('utf-8')).hexdigest()[:16]}"
|
||
|
||
|
||
def _manual_group_id(name: str) -> str:
|
||
basis = f"{name.strip().lower()}\n{datetime.now(UTC).isoformat()}"
|
||
return f"manual-group:{hashlib.sha1(basis.encode('utf-8')).hexdigest()[:10]}"
|
||
|
||
|
||
def _news_meta(record: EarthNewsItem) -> dict[str, Any]:
|
||
location_meta = record.location_meta if isinstance(record.location_meta, dict) else {}
|
||
news_meta = location_meta.get("news_meta")
|
||
return dict(news_meta) if isinstance(news_meta, dict) else {}
|
||
|
||
|
||
def _record_source_type(record: EarthNewsItem) -> str:
|
||
return str(_news_meta(record).get("feed_type") or _news_meta(record).get("source_type") or "rss")
|
||
|
||
|
||
def _record_manual_group_id(record: EarthNewsItem) -> str:
|
||
return str(_news_meta(record).get("manual_group_id") or DEFAULT_MANUAL_NEWS_GROUP_ID)
|
||
|
||
|
||
def _rss_group_id(record: EarthNewsItem) -> str:
|
||
basis = "\n".join(
|
||
[
|
||
_record_source_type(record),
|
||
str(record.feed_name or ""),
|
||
str(record.source or ""),
|
||
]
|
||
)
|
||
return f"rss:{hashlib.sha1(basis.encode('utf-8')).hexdigest()[:12]}"
|
||
|
||
|
||
def _default_manual_group() -> dict[str, Any]:
|
||
return {
|
||
"id": DEFAULT_MANUAL_NEWS_GROUP_ID,
|
||
"name": DEFAULT_MANUAL_NEWS_GROUP_NAME,
|
||
"sort_order": 0,
|
||
}
|
||
|
||
|
||
def _normalize_manual_groups_payload(payload: Any) -> list[dict[str, Any]]:
|
||
raw_groups = payload.get("groups") if isinstance(payload, dict) else None
|
||
normalized: list[dict[str, Any]] = []
|
||
seen: set[str] = set()
|
||
for index, item in enumerate(raw_groups if isinstance(raw_groups, list) else []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
group_id = str(item.get("id") or "").strip()
|
||
name = _clean_text(item.get("name"), max_length=120)
|
||
if not group_id or not name or group_id in seen:
|
||
continue
|
||
normalized.append(
|
||
{
|
||
"id": group_id,
|
||
"name": name,
|
||
"sort_order": int(item.get("sort_order") or index),
|
||
}
|
||
)
|
||
seen.add(group_id)
|
||
if DEFAULT_MANUAL_NEWS_GROUP_ID not in seen:
|
||
normalized.insert(0, _default_manual_group())
|
||
return sorted(normalized, key=lambda item: (int(item.get("sort_order") or 0), str(item.get("name") or "")))
|
||
|
||
|
||
async def _get_manual_groups_record(db: AsyncSession) -> SystemSetting | None:
|
||
result = await db.execute(
|
||
select(SystemSetting).where(SystemSetting.category == EARTH_NEWS_MANUAL_GROUPS_CATEGORY)
|
||
)
|
||
return result.scalar_one_or_none()
|
||
|
||
|
||
async def get_manual_news_groups(db: AsyncSession) -> list[dict[str, Any]]:
|
||
record = await _get_manual_groups_record(db)
|
||
return _normalize_manual_groups_payload(record.payload if record else None)
|
||
|
||
|
||
async def _save_manual_news_groups(db: AsyncSession, groups: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
normalized = _normalize_manual_groups_payload({"groups": groups})
|
||
record = await _get_manual_groups_record(db)
|
||
payload = {"groups": normalized}
|
||
if record is None:
|
||
db.add(SystemSetting(category=EARTH_NEWS_MANUAL_GROUPS_CATEGORY, payload=payload))
|
||
else:
|
||
record.payload = payload
|
||
await db.flush()
|
||
return normalized
|
||
|
||
|
||
async def resolve_manual_news_group(db: AsyncSession, group_id: str | None) -> ManualNewsGroup:
|
||
normalized_id = str(group_id or DEFAULT_MANUAL_NEWS_GROUP_ID).strip() or DEFAULT_MANUAL_NEWS_GROUP_ID
|
||
groups = await get_manual_news_groups(db)
|
||
match = next((item for item in groups if item.get("id") == normalized_id), None)
|
||
if match is None and normalized_id != DEFAULT_MANUAL_NEWS_GROUP_ID:
|
||
raise ValueError(f"手动新闻组不存在：{normalized_id}")
|
||
match = match or _default_manual_group()
|
||
return ManualNewsGroup(
|
||
id=str(match["id"]),
|
||
name=str(match["name"]),
|
||
sort_order=int(match.get("sort_order") or 0),
|
||
)
|
||
|
||
|
||
async def create_manual_news_group(db: AsyncSession, name: str) -> dict[str, Any]:
|
||
group_name = _clean_text(name, max_length=120)
|
||
if not group_name:
|
||
raise ValueError("新闻组名称不能为空。")
|
||
groups = await get_manual_news_groups(db)
|
||
group = {"id": _manual_group_id(group_name), "name": group_name, "sort_order": len(groups)}
|
||
groups.append(group)
|
||
await _save_manual_news_groups(db, groups)
|
||
return group
|
||
|
||
|
||
async def rename_manual_news_group(db: AsyncSession, group_id: str, name: str) -> dict[str, Any]:
|
||
group_name = _clean_text(name, max_length=120)
|
||
if not group_name:
|
||
raise ValueError("新闻组名称不能为空。")
|
||
groups = await get_manual_news_groups(db)
|
||
match = next((item for item in groups if item.get("id") == group_id), None)
|
||
if match is None:
|
||
raise ValueError(f"手动新闻组不存在：{group_id}")
|
||
match["name"] = group_name
|
||
await _save_manual_news_groups(db, groups)
|
||
|
||
result = await db.execute(
|
||
select(EarthNewsItem).where(
|
||
EarthNewsItem.location_meta.op("->")("news_meta").op("->>")("feed_type") == NewsSourceType.MANUAL.value
|
||
)
|
||
)
|
||
for record in result.scalars().all():
|
||
if _record_manual_group_id(record) != group_id:
|
||
continue
|
||
location_meta = dict(record.location_meta or {})
|
||
news_meta = dict(location_meta.get("news_meta") or {})
|
||
news_meta["manual_group_name"] = group_name
|
||
location_meta["news_meta"] = news_meta
|
||
record.location_meta = location_meta
|
||
await db.flush()
|
||
return match
|
||
|
||
|
||
def _normalize_region(value: object) -> str:
|
||
region = str(value or "global").strip().lower() or "global"
|
||
if region not in REGION_ANCHORS:
|
||
raise ValueError(f"region 不支持：{region}")
|
||
return region
|
||
|
||
|
||
def _normalize_tags(value: object) -> list[str]:
|
||
if value is None:
|
||
return []
|
||
if isinstance(value, str):
|
||
parts = re.split(r"[,，\n]", value)
|
||
elif isinstance(value, list):
|
||
parts = [str(item) for item in value]
|
||
else:
|
||
raise ValueError("tags 必须是字符串数组或逗号分隔字符串。")
|
||
return [item.strip() for item in parts if item.strip()][:20]
|
||
|
||
|
||
def _normalize_location(value: object) -> NewsTargetLocation | None:
|
||
if value in (None, ""):
|
||
return None
|
||
if not isinstance(value, dict):
|
||
raise ValueError("location 必须是对象。")
|
||
lat = value.get("latitude")
|
||
lon = value.get("longitude")
|
||
if lat in (None, "") and lon in (None, ""):
|
||
return None
|
||
try:
|
||
latitude = float(lat)
|
||
longitude = float(lon)
|
||
except (TypeError, ValueError) as exc:
|
||
raise ValueError("location.latitude / longitude 必须是数字。") from exc
|
||
if not -90 <= latitude <= 90 or not -180 <= longitude <= 180:
|
||
raise ValueError("location 经纬度超出范围。")
|
||
label = _clean_text(value.get("label"), max_length=255)
|
||
if not label:
|
||
label = f"{latitude:.4f}, {longitude:.4f}"
|
||
return NewsTargetLocation(
|
||
latitude=latitude,
|
||
longitude=longitude,
|
||
label=label,
|
||
source="manual_location",
|
||
confidence=1.0,
|
||
country=_clean_text(value.get("country"), max_length=100) or None,
|
||
city=_clean_text(value.get("city"), max_length=100) or None,
|
||
)
|
||
|
||
|
||
def _manual_source(source_name: str, *, region: str) -> NewsFeedSource:
|
||
return NewsFeedSource(
|
||
id=MANUAL_NEWS_SOURCE_ID,
|
||
name=source_name or MANUAL_NEWS_SOURCE_LABEL,
|
||
region=region,
|
||
feed_url="",
|
||
homepage_url="",
|
||
source_type=NewsSourceType.MANUAL.value,
|
||
default_category="other",
|
||
source_tags=("manual",),
|
||
)
|
||
|
||
|
||
def _manual_feed(category: str) -> NewsFeedEndpoint:
|
||
return NewsFeedEndpoint(
|
||
id=MANUAL_NEWS_SOURCE_ID,
|
||
name=MANUAL_NEWS_SOURCE_LABEL,
|
||
url="",
|
||
type=NewsSourceType.MANUAL.value,
|
||
default_category=category or "other",
|
||
tags=("manual",),
|
||
priority=1,
|
||
)
|
||
|
||
|
||
def parsed_manual_news_item(
|
||
payload: dict[str, Any],
|
||
*,
|
||
item_id_override: str | None = None,
|
||
) -> tuple[ParsedNewsItem, NewsTargetLocation | None, str]:
|
||
title = _clean_text(payload.get("title"), max_length=MANUAL_NEWS_MAX_TITLE_LENGTH)
|
||
if not title:
|
||
raise ValueError("title 不能为空。")
|
||
content = _clean_text(payload.get("content"), max_length=MANUAL_NEWS_MAX_CONTENT_LENGTH)
|
||
summary = _clean_text(payload.get("summary"), max_length=MANUAL_NEWS_MAX_SUMMARY_LENGTH)
|
||
if not summary:
|
||
summary = _clean_text(content, max_length=240) if content else title
|
||
source = _clean_text(payload.get("source"), max_length=255) or MANUAL_NEWS_SOURCE_LABEL
|
||
region = _normalize_region(payload.get("region"))
|
||
published_at = _parse_datetime(payload.get("published_at")) or datetime.now(UTC)
|
||
url = str(payload.get("url") or "").strip()
|
||
category = str(payload.get("category") or "other").strip().lower() or "other"
|
||
if category not in ALLOWED_NEWS_CATEGORY_KEYS:
|
||
raise ValueError(f"category 不支持：{category}")
|
||
tags = _normalize_tags(payload.get("tags"))
|
||
target = _normalize_location(payload.get("location"))
|
||
language = str(payload.get("content_language") or "").strip() or _detect_language(title, summary, content)
|
||
localizations = {
|
||
language: {
|
||
"title": title,
|
||
"summary": summary,
|
||
}
|
||
}
|
||
item = ParsedNewsItem(
|
||
id=item_id_override
|
||
or _manual_item_id(title=title, published_at=published_at, url=url, source=source),
|
||
title=title,
|
||
summary=summary,
|
||
url=url,
|
||
source=source,
|
||
feed_name=MANUAL_NEWS_SOURCE_LABEL,
|
||
feed_region=region,
|
||
homepage_url=str(payload.get("homepage_url") or ""),
|
||
published_at=published_at,
|
||
content_language=language,
|
||
localizations=localizations,
|
||
enrichment_status=NewsEnrichmentStatus.PENDING.value,
|
||
source_tags=["manual"],
|
||
feed_id=MANUAL_NEWS_SOURCE_ID,
|
||
feed_type=NewsSourceType.MANUAL.value,
|
||
feed_default_category=category,
|
||
category=category,
|
||
item_tags=tags,
|
||
tagging_source=NewsTaggingSource.MANUAL.value if payload.get("category") else NewsTaggingSource.RULES.value,
|
||
tagging_confidence=0.9 if payload.get("category") else 0.0,
|
||
)
|
||
source_config = _manual_source(source, region=region)
|
||
feed = _manual_feed(category)
|
||
apply_news_classification(item, source_config, feed=feed)
|
||
if payload.get("category"):
|
||
item.category = category
|
||
item.tagging_source = NewsTaggingSource.MANUAL.value
|
||
item.tagging_confidence = 0.9
|
||
if tags:
|
||
item.item_tags = sorted(set([*item.item_tags, *tags]))
|
||
return item, target, content
|
||
|
||
|
||
def _manual_editable(record: EarthNewsItem) -> bool:
|
||
if record.id.startswith("manual:"):
|
||
return True
|
||
news_meta = (record.location_meta or {}).get("news_meta") if isinstance(record.location_meta, dict) else None
|
||
return isinstance(news_meta, dict) and news_meta.get("feed_type") == NewsSourceType.MANUAL.value
|
||
|
||
|
||
async def _broadcast_news_reload() -> None:
|
||
await broadcaster.broadcast_earth_update(
|
||
{
|
||
"action": "database_changed",
|
||
"source": "earth_news_items",
|
||
"layers": ["news"],
|
||
"refresh_strategy": "reload",
|
||
}
|
||
)
|
||
|
||
|
||
async def upsert_manual_news_item(
|
||
db: AsyncSession,
|
||
payload: dict[str, Any],
|
||
*,
|
||
item_id_override: str | None = None,
|
||
group_id: str | None = None,
|
||
) -> ManualNewsWriteResult:
|
||
item, target, content = parsed_manual_news_item(payload, item_id_override=item_id_override)
|
||
group = await resolve_manual_news_group(db, group_id or payload.get("group_id"))
|
||
existing = await db.get(EarthNewsItem, item.id)
|
||
created = existing is None
|
||
patch = build_target_location_patch(item, target) if target else build_anchor_location_patch(item)
|
||
patch_meta = dict(patch.get("location_meta") or {})
|
||
patch_news_meta = dict(patch_meta.get("news_meta") or {})
|
||
patch_news_meta["feed_type"] = NewsSourceType.MANUAL.value
|
||
patch_news_meta["source_type"] = NewsSourceType.MANUAL.value
|
||
patch_news_meta["manual_group_id"] = group.id
|
||
patch_news_meta["manual_group_name"] = group.name
|
||
patch_meta["news_meta"] = patch_news_meta
|
||
patch["location_meta"] = patch_meta
|
||
now = datetime.now(UTC)
|
||
record = existing or EarthNewsItem(
|
||
id=item.id,
|
||
title=item.title,
|
||
summary=item.summary,
|
||
content_language=item.content_language,
|
||
localizations=dict(item.localizations or {}),
|
||
url=item.url,
|
||
source=item.source,
|
||
feed_name=item.feed_name,
|
||
region=item.feed_region,
|
||
homepage_url=item.homepage_url,
|
||
published_at=item.published_at,
|
||
latitude=patch["latitude"],
|
||
longitude=patch["longitude"],
|
||
location_label=patch["location_label"],
|
||
location_source=patch["location_source"],
|
||
verified=patch["verified"],
|
||
location_meta=patch["location_meta"],
|
||
first_seen_at=now,
|
||
last_seen_at=now,
|
||
resolved_at=now if patch["verified"] else None,
|
||
enrichment_status=item.enrichment_status,
|
||
)
|
||
if existing is None:
|
||
db.add(record)
|
||
else:
|
||
if not _manual_editable(record):
|
||
raise PermissionError("RSS 新闻不允许通过手动新闻接口编辑。")
|
||
record.title = item.title
|
||
record.summary = item.summary
|
||
record.content_language = item.content_language
|
||
record.localizations = dict(item.localizations or {})
|
||
record.url = item.url
|
||
record.source = item.source
|
||
record.feed_name = item.feed_name
|
||
record.region = item.feed_region
|
||
record.homepage_url = item.homepage_url
|
||
record.published_at = item.published_at
|
||
record.last_seen_at = now
|
||
if target is None and record.location_source == "manual_location":
|
||
merged_meta = dict(record.location_meta or {})
|
||
patch_meta = patch.get("location_meta") if isinstance(patch, dict) else None
|
||
patch_news_meta = patch_meta.get("news_meta") if isinstance(patch_meta, dict) else None
|
||
if isinstance(patch_news_meta, dict):
|
||
merged_meta["news_meta"] = patch_news_meta
|
||
record.location_meta = merged_meta
|
||
else:
|
||
record.location_meta = patch["location_meta"]
|
||
if target:
|
||
record.latitude = patch["latitude"]
|
||
record.longitude = patch["longitude"]
|
||
record.location_label = patch["location_label"]
|
||
record.location_source = patch["location_source"]
|
||
record.verified = patch["verified"]
|
||
record.resolved_at = now
|
||
elif record.location_source != "manual_location":
|
||
record.latitude = patch["latitude"]
|
||
record.longitude = patch["longitude"]
|
||
record.location_label = patch["location_label"]
|
||
record.location_source = patch["location_source"]
|
||
record.verified = patch["verified"]
|
||
record.resolved_at = None
|
||
record.enrichment_status = NewsEnrichmentStatus.PENDING.value
|
||
record.enrichment_error = None
|
||
record.enriched_at = None
|
||
if content:
|
||
meta = dict(record.location_meta or {})
|
||
meta["manual_content"] = content
|
||
record.location_meta = meta
|
||
await db.flush()
|
||
|
||
queued = await enqueue_target_location_job(build_target_location_job_payload(item), force=True)
|
||
if queued:
|
||
record.enrichment_status = NewsEnrichmentStatus.QUEUED.value
|
||
await db.flush()
|
||
return ManualNewsWriteResult(item=record, created=created, queued=queued)
|
||
|
||
|
||
async def import_manual_news_items(
|
||
db: AsyncSession,
|
||
payload: list[Any],
|
||
*,
|
||
group_id: str | None = None,
|
||
) -> dict[str, Any]:
|
||
if len(payload) > MANUAL_NEWS_MAX_IMPORT_ITEMS:
|
||
raise ValueError(f"单次最多导入 {MANUAL_NEWS_MAX_IMPORT_ITEMS} 条。")
|
||
created = 0
|
||
updated = 0
|
||
queued = 0
|
||
errors: list[dict[str, Any]] = []
|
||
for index, raw_item in enumerate(payload):
|
||
if not isinstance(raw_item, dict):
|
||
errors.append({"index": index, "error": "条目必须是 JSON 对象。"})
|
||
continue
|
||
try:
|
||
result = await upsert_manual_news_item(db, raw_item, group_id=group_id)
|
||
created += 1 if result.created else 0
|
||
updated += 0 if result.created else 1
|
||
queued += 1 if result.queued else 0
|
||
except Exception as exc:
|
||
errors.append({"index": index, "error": str(exc)})
|
||
if errors and created == 0 and updated == 0:
|
||
raise ValueError("导入失败，未写入任何新闻。")
|
||
return {"created": created, "updated": updated, "queued": queued, "failed": len(errors), "errors": errors}
|
||
|
||
|
||
async def parse_manual_news_import_upload(raw_bytes: bytes) -> list[Any]:
|
||
try:
|
||
payload = json.loads(raw_bytes.decode("utf-8-sig"))
|
||
except UnicodeDecodeError as exc:
|
||
raise ValueError("JSON 文件必须使用 UTF-8 编码。") from exc
|
||
except json.JSONDecodeError as exc:
|
||
raise ValueError(f"JSON 解析失败：第 {exc.lineno} 行第 {exc.colno} 列。") from exc
|
||
if not isinstance(payload, list):
|
||
raise ValueError("JSON 顶层必须是数组。")
|
||
return payload
|
||
|
||
|
||
def serialize_news_record(record: EarthNewsItem, *, locale: str = DEFAULT_NEWS_LOCALE) -> dict[str, Any]:
|
||
item = record_to_parsed_news_item(record)
|
||
payload = _serialize_item(item, active_region=item.feed_region, locale=locale)
|
||
news_meta = _news_meta(record)
|
||
payload["editable"] = _manual_editable(record)
|
||
payload["source_type"] = payload.get("feed_type")
|
||
payload["status"] = record.enrichment_status
|
||
payload["translated"] = bool((record.localizations or {}).get("zh-CN") and (record.localizations or {}).get("en-US"))
|
||
payload["manual_content"] = (record.location_meta or {}).get("manual_content") if isinstance(record.location_meta, dict) else None
|
||
payload["manual_group_id"] = news_meta.get("manual_group_id")
|
||
payload["manual_group_name"] = news_meta.get("manual_group_name")
|
||
return payload
|
||
|
||
|
||
def _record_matches_group(record: EarthNewsItem, group_id: str) -> bool:
|
||
source_type = _record_source_type(record)
|
||
if source_type == NewsSourceType.MANUAL.value:
|
||
return _record_manual_group_id(record) == group_id
|
||
return _rss_group_id(record) == group_id
|
||
|
||
|
||
async def list_news_records(
|
||
db: AsyncSession,
|
||
*,
|
||
page: int,
|
||
page_size: int,
|
||
source_type: str | None = None,
|
||
region: str | None = None,
|
||
category: str | None = None,
|
||
status_filter: str | None = None,
|
||
group_id: str | None = None,
|
||
) -> dict[str, Any]:
|
||
page = max(page, 1)
|
||
page_size = min(max(page_size, 1), 100)
|
||
query = select(EarthNewsItem)
|
||
count_query = select(func.count(EarthNewsItem.id))
|
||
filters = []
|
||
if region and region != "all":
|
||
filters.append(EarthNewsItem.region == region)
|
||
if status_filter and status_filter != "all":
|
||
filters.append(EarthNewsItem.enrichment_status == status_filter)
|
||
if source_type and source_type != "all":
|
||
filters.append(EarthNewsItem.location_meta.op("->")("news_meta").op("->>")("feed_type") == source_type)
|
||
if category and category != "all":
|
||
filters.append(EarthNewsItem.location_meta.op("->")("news_meta").op("->>")("category") == category)
|
||
for clause in filters:
|
||
query = query.where(clause)
|
||
count_query = count_query.where(clause)
|
||
ordered_query = query.order_by(EarthNewsItem.published_at.desc().nullslast(), EarthNewsItem.last_seen_at.desc())
|
||
if group_id:
|
||
result = await db.execute(ordered_query)
|
||
all_records = [record for record in result.scalars().all() if _record_matches_group(record, group_id)]
|
||
total = len(all_records)
|
||
records = all_records[(page - 1) * page_size : page * page_size]
|
||
else:
|
||
total_result = await db.execute(count_query)
|
||
result = await db.execute(
|
||
ordered_query.offset((page - 1) * page_size).limit(page_size)
|
||
)
|
||
records = list(result.scalars().all())
|
||
total = int(total_result.scalar() or 0)
|
||
return {
|
||
"items": [serialize_news_record(record) for record in records],
|
||
"page": page,
|
||
"page_size": page_size,
|
||
"total": total,
|
||
}
|
||
|
||
|
||
async def list_news_groups(db: AsyncSession, *, locale: str = DEFAULT_NEWS_LOCALE) -> dict[str, Any]:
|
||
manual_groups = await get_manual_news_groups(db)
|
||
manual_by_id: dict[str, dict[str, Any]] = {
|
||
str(group["id"]): {
|
||
"id": str(group["id"]),
|
||
"name": str(group["name"]),
|
||
"group_type": "manual",
|
||
"source_type": NewsSourceType.MANUAL.value,
|
||
"editable": True,
|
||
"sort_order": int(group.get("sort_order") or 0),
|
||
"count": 0,
|
||
"items": [],
|
||
}
|
||
for group in manual_groups
|
||
}
|
||
rss_by_id: dict[str, dict[str, Any]] = {}
|
||
result = await db.execute(
|
||
select(EarthNewsItem).order_by(EarthNewsItem.published_at.desc().nullslast(), EarthNewsItem.last_seen_at.desc())
|
||
)
|
||
for record in result.scalars().all():
|
||
serialized = serialize_news_record(record, locale=locale)
|
||
source_type = _record_source_type(record)
|
||
if source_type == NewsSourceType.MANUAL.value:
|
||
group_id = _record_manual_group_id(record)
|
||
group = manual_by_id.setdefault(
|
||
group_id,
|
||
{
|
||
"id": group_id,
|
||
"name": str(_news_meta(record).get("manual_group_name") or DEFAULT_MANUAL_NEWS_GROUP_NAME),
|
||
"group_type": "manual",
|
||
"source_type": NewsSourceType.MANUAL.value,
|
||
"editable": True,
|
||
"sort_order": len(manual_by_id),
|
||
"count": 0,
|
||
"items": [],
|
||
},
|
||
)
|
||
else:
|
||
group_id = _rss_group_id(record)
|
||
group = rss_by_id.setdefault(
|
||
group_id,
|
||
{
|
||
"id": group_id,
|
||
"name": record.feed_name or record.source or "RSS 新闻",
|
||
"group_type": "rss",
|
||
"source_type": source_type,
|
||
"editable": False,
|
||
"region": record.region,
|
||
"source": record.source,
|
||
"feed_name": record.feed_name,
|
||
"count": 0,
|
||
"items": [],
|
||
},
|
||
)
|
||
group["count"] = int(group.get("count") or 0) + 1
|
||
group.setdefault("items", []).append(serialized)
|
||
manual_items = sorted(manual_by_id.values(), key=lambda item: (int(item.get("sort_order") or 0), str(item.get("name") or "")))
|
||
rss_items = sorted(rss_by_id.values(), key=lambda item: str(item.get("name") or ""))
|
||
return {"groups": [*manual_items, *rss_items], "manual_groups": manual_items, "rss_groups": rss_items}
|
||
|
||
|
||
async def get_news_record_or_404(db: AsyncSession, item_id: str) -> EarthNewsItem | None:
|
||
return await db.get(EarthNewsItem, item_id)
|
||
|
||
|
||
async def delete_manual_news_item(db: AsyncSession, item_id: str) -> bool:
|
||
record = await db.get(EarthNewsItem, item_id)
|
||
if record is None:
|
||
return False
|
||
if not _manual_editable(record):
|
||
raise PermissionError("RSS 新闻不允许通过手动新闻接口删除。")
|
||
await db.execute(delete(EarthNewsItem).where(EarthNewsItem.id == item_id))
|
||
await db.flush()
|
||
return True
|
||
|
||
|
||
async def reprocess_manual_news_item(db: AsyncSession, item_id: str) -> bool:
|
||
record = await db.get(EarthNewsItem, item_id)
|
||
if record is None:
|
||
return False
|
||
if not _manual_editable(record):
|
||
raise PermissionError("RSS 新闻不允许通过手动新闻接口重新处理。")
|
||
item = record_to_parsed_news_item(record)
|
||
queued = await enqueue_target_location_job(build_target_location_job_payload(item), force=True)
|
||
if queued:
|
||
record.enrichment_status = NewsEnrichmentStatus.QUEUED.value
|
||
record.enrichment_error = None
|
||
await db.flush()
|
||
return queued
|
||
|
||
|
||
async def broadcast_manual_news_changed() -> None:
|
||
await _broadcast_news_reload()
|