571 lines
23 KiB
Python
571 lines
23 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import base64
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
from urllib.parse import urlparse
|
|
|
|
import httpx
|
|
from sqlalchemy import select
|
|
|
|
from app.core.data_sources import get_data_sources_config
|
|
from app.models.datasource_config import DataSourceConfig
|
|
from app.services.collectors.base import BaseCollector
|
|
|
|
|
|
class NewsLiveStreamsCollector(BaseCollector):
|
|
"""Collect normalized news live-stream sources from a JSON endpoint."""
|
|
|
|
name = "news_live_streams"
|
|
priority = "P2"
|
|
module = "L4"
|
|
frequency_hours = 12
|
|
data_type = "news_live_stream"
|
|
fail_on_empty = False
|
|
|
|
DEFAULT_TIMEOUT = 45.0
|
|
DEFAULT_HEADERS = {
|
|
"User-Agent": "Planet-Intelligence-System/1.0 (Python/collector)",
|
|
"Accept": "application/json",
|
|
}
|
|
RESPONSE_CANDIDATE_KEYS = ("sources", "streams", "channels", "items", "results", "data")
|
|
DEFAULT_ADAPTER = "iptv_org"
|
|
DEFAULT_IPTV_ORG_STREAMS_URL = "https://iptv-org.github.io/api/streams.json"
|
|
DEFAULT_IPTV_ORG_LOGOS_URL = "https://iptv-org.github.io/api/logos.json"
|
|
DEFAULT_IPTV_ORG_NEWS_CATEGORIES = ("news", "business", "weather")
|
|
DEFAULT_IPTV_ORG_EXCLUDE_CATEGORIES = ("music", "sports", "kids", "entertainment")
|
|
DEFAULT_IPTV_ORG_MAX_SOURCES = 120
|
|
|
|
async def fetch(self) -> list[dict[str, Any]]:
|
|
request_url = (self._resolved_url or "").strip()
|
|
if not request_url:
|
|
return []
|
|
|
|
datasource_config = await self._load_datasource_config()
|
|
effective_config = self._get_effective_config(datasource_config)
|
|
adapter = str(effective_config.get("adapter") or "").strip().lower()
|
|
if adapter == "iptv_org":
|
|
return await self._fetch_iptv_org(request_url, effective_config)
|
|
|
|
request_headers = self._build_request_headers(datasource_config)
|
|
request_config = self._get_request_config(datasource_config)
|
|
request_params = self._build_request_params(datasource_config)
|
|
request_json = self._build_request_json_body(datasource_config)
|
|
request_data = self._build_request_form_body(datasource_config)
|
|
timeout = self._get_timeout(datasource_config)
|
|
|
|
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
|
response = await client.request(
|
|
request_config["method"],
|
|
request_url,
|
|
headers=request_headers,
|
|
params=request_params or None,
|
|
json=request_json,
|
|
data=request_data,
|
|
)
|
|
response.raise_for_status()
|
|
return self.parse_response(
|
|
response.json(),
|
|
response_path=request_config["response_path"],
|
|
)
|
|
|
|
async def _load_datasource_config(self) -> DataSourceConfig | None:
|
|
if not self._db_session:
|
|
return None
|
|
|
|
result = await self._db_session.execute(
|
|
select(DataSourceConfig)
|
|
.where(DataSourceConfig.name == self.name)
|
|
.where(DataSourceConfig.is_active.is_(True))
|
|
.limit(1)
|
|
)
|
|
return result.scalar_one_or_none()
|
|
|
|
def _get_effective_config(self, datasource_config: DataSourceConfig | None) -> dict[str, Any]:
|
|
payload = dict(datasource_config.config or {}) if datasource_config else {}
|
|
if payload:
|
|
return payload
|
|
|
|
yaml_config = get_data_sources_config()
|
|
return {
|
|
"adapter": self.DEFAULT_ADAPTER,
|
|
"streams_url": yaml_config.get_yaml_value("news_live_streams.streams_url")
|
|
or self.DEFAULT_IPTV_ORG_STREAMS_URL,
|
|
"logos_url": yaml_config.get_yaml_value("news_live_streams.logos_url")
|
|
or self.DEFAULT_IPTV_ORG_LOGOS_URL,
|
|
"news_categories": list(self.DEFAULT_IPTV_ORG_NEWS_CATEGORIES),
|
|
"exclude_categories": list(self.DEFAULT_IPTV_ORG_EXCLUDE_CATEGORIES),
|
|
"max_sources": self.DEFAULT_IPTV_ORG_MAX_SOURCES,
|
|
}
|
|
|
|
def _get_request_config(self, datasource_config: DataSourceConfig | None) -> dict[str, Any]:
|
|
payload = self._get_effective_config(datasource_config)
|
|
raw_method = payload.get("method") or payload.get("request_method") or "GET"
|
|
method = str(raw_method).strip().upper() or "GET"
|
|
if method not in {"GET", "POST"}:
|
|
method = "GET"
|
|
|
|
response_path = payload.get("response_path") or payload.get("payload_path") or payload.get("items_path")
|
|
if isinstance(response_path, str):
|
|
response_path = response_path.strip()
|
|
else:
|
|
response_path = None
|
|
|
|
return {
|
|
"method": method,
|
|
"response_path": response_path or None,
|
|
}
|
|
|
|
def _get_timeout(self, datasource_config: DataSourceConfig | None) -> float:
|
|
payload = self._get_effective_config(datasource_config)
|
|
try:
|
|
return float(payload.get("timeout", self.DEFAULT_TIMEOUT))
|
|
except (TypeError, ValueError):
|
|
return self.DEFAULT_TIMEOUT
|
|
|
|
def _build_request_headers(self, datasource_config: DataSourceConfig | None) -> dict[str, str]:
|
|
headers = dict(self.DEFAULT_HEADERS)
|
|
if datasource_config:
|
|
headers.update(self._normalize_headers(datasource_config.headers))
|
|
headers.update(self._build_auth_headers(datasource_config))
|
|
return headers
|
|
|
|
def _build_request_params(self, datasource_config: DataSourceConfig | None) -> dict[str, Any]:
|
|
params: dict[str, Any] = {}
|
|
if not datasource_config:
|
|
return params
|
|
|
|
payload = datasource_config.config or {}
|
|
candidate = payload.get("params") or payload.get("query_params")
|
|
if isinstance(candidate, dict):
|
|
params.update(candidate)
|
|
|
|
if datasource_config.auth_type == "api_key":
|
|
auth_config = datasource_config.auth_config or {}
|
|
if str(auth_config.get("in") or auth_config.get("location") or "header").lower() == "query":
|
|
api_key = auth_config.get("api_key")
|
|
key_name = auth_config.get("key_name") or auth_config.get("param_name") or "api_key"
|
|
if api_key and key_name:
|
|
params[str(key_name)] = api_key
|
|
|
|
return params
|
|
|
|
def _build_request_json_body(self, datasource_config: DataSourceConfig | None) -> Any:
|
|
if not datasource_config:
|
|
return None
|
|
|
|
payload = datasource_config.config or {}
|
|
body = payload.get("json_body")
|
|
if body is None and str(payload.get("body_type") or "").lower() in {"json", ""}:
|
|
candidate = payload.get("body")
|
|
if isinstance(candidate, (dict, list)):
|
|
body = candidate
|
|
return body
|
|
|
|
def _build_request_form_body(self, datasource_config: DataSourceConfig | None) -> Any:
|
|
if not datasource_config:
|
|
return None
|
|
|
|
payload = datasource_config.config or {}
|
|
form_body = payload.get("form_body")
|
|
if form_body is not None:
|
|
return form_body
|
|
|
|
if str(payload.get("body_type") or "").lower() == "form":
|
|
candidate = payload.get("body")
|
|
if isinstance(candidate, dict):
|
|
return candidate
|
|
return None
|
|
|
|
def _normalize_headers(self, headers: Any) -> dict[str, str]:
|
|
if not isinstance(headers, dict):
|
|
return {}
|
|
normalized: dict[str, str] = {}
|
|
for key, value in headers.items():
|
|
header_name = str(key).strip()
|
|
if not header_name or value is None:
|
|
continue
|
|
normalized[header_name] = str(value)
|
|
return normalized
|
|
|
|
def _build_auth_headers(self, datasource_config: DataSourceConfig | None) -> dict[str, str]:
|
|
if not datasource_config:
|
|
return {}
|
|
|
|
auth_type = str(datasource_config.auth_type or "none").lower()
|
|
auth_config = datasource_config.auth_config or {}
|
|
if auth_type == "bearer" and auth_config.get("token"):
|
|
return {"Authorization": f"Bearer {auth_config['token']}"}
|
|
|
|
if auth_type == "api_key" and auth_config.get("api_key"):
|
|
location = str(auth_config.get("in") or auth_config.get("location") or "header").lower()
|
|
if location == "query":
|
|
return {}
|
|
key_name = auth_config.get("key_name") or "X-API-Key"
|
|
return {str(key_name): str(auth_config["api_key"])}
|
|
|
|
if auth_type == "basic":
|
|
username = str(auth_config.get("username") or "")
|
|
password = str(auth_config.get("password") or "")
|
|
encoded = base64.b64encode(f"{username}:{password}".encode()).decode()
|
|
return {"Authorization": f"Basic {encoded}"}
|
|
|
|
return {}
|
|
|
|
def _extract_candidates(self, response: Any, response_path: str | None) -> list[Any]:
|
|
if response_path:
|
|
extracted = self._extract_from_path(response, response_path)
|
|
if isinstance(extracted, list):
|
|
return extracted
|
|
if isinstance(extracted, dict):
|
|
for key in self.RESPONSE_CANDIDATE_KEYS:
|
|
nested = extracted.get(key)
|
|
if isinstance(nested, list):
|
|
return nested
|
|
return [extracted]
|
|
|
|
if isinstance(response, dict):
|
|
for key in self.RESPONSE_CANDIDATE_KEYS:
|
|
nested = response.get(key)
|
|
if isinstance(nested, list):
|
|
return nested
|
|
return []
|
|
|
|
if isinstance(response, list):
|
|
return response
|
|
return []
|
|
|
|
def _extract_from_path(self, payload: Any, path: str) -> Any:
|
|
current = payload
|
|
for segment in (part.strip() for part in path.split(".") if part.strip()):
|
|
if isinstance(current, dict):
|
|
current = current.get(segment)
|
|
continue
|
|
if isinstance(current, list):
|
|
try:
|
|
current = current[int(segment)]
|
|
except (TypeError, ValueError, IndexError):
|
|
return None
|
|
continue
|
|
return None
|
|
return current
|
|
|
|
def _infer_source_type(self, item: dict[str, Any]) -> str:
|
|
explicit = str(item.get("source_type") or item.get("type") or "").strip().lower()
|
|
if explicit in {"iframe", "hls", "video", "external", "youtube"}:
|
|
return explicit
|
|
|
|
youtube_video_id = self._clean_text(
|
|
item.get("youtube_video_id")
|
|
or item.get("video_id")
|
|
or item.get("youtubeVideoId")
|
|
)
|
|
youtube_channel = self._clean_text(item.get("youtube_channel") or item.get("channel_handle"))
|
|
embed_url = self._clean_url(item.get("embed_url") or item.get("embed") or item.get("page_url"))
|
|
stream_url = self._clean_url(item.get("stream_url") or item.get("stream") or item.get("playback_url") or item.get("hls_url"))
|
|
homepage_url = self._clean_url(item.get("homepage_url") or item.get("source_url") or item.get("website"))
|
|
|
|
if youtube_video_id or youtube_channel:
|
|
return "youtube"
|
|
if stream_url.endswith(".m3u8"):
|
|
return "hls"
|
|
if stream_url:
|
|
return "video"
|
|
if embed_url:
|
|
parsed = urlparse(embed_url)
|
|
if "youtube.com" in (parsed.netloc or "") or "youtu.be" in (parsed.netloc or ""):
|
|
return "youtube"
|
|
return "iframe"
|
|
if homepage_url:
|
|
return "external"
|
|
return "iframe"
|
|
|
|
def _parse_enabled(self, item: dict[str, Any]) -> bool:
|
|
if "is_enabled" in item:
|
|
return self._to_bool(item.get("is_enabled"), default=True)
|
|
if "enabled" in item:
|
|
return self._to_bool(item.get("enabled"), default=True)
|
|
if "active" in item:
|
|
return self._to_bool(item.get("active"), default=True)
|
|
if "status" in item:
|
|
status = str(item.get("status") or "").strip().lower()
|
|
if status in {"disabled", "inactive", "offline"}:
|
|
return False
|
|
if status in {"enabled", "active", "online", "live"}:
|
|
return True
|
|
return True
|
|
|
|
def _to_bool(self, value: Any, *, default: bool) -> bool:
|
|
if isinstance(value, bool):
|
|
return value
|
|
if value in (None, ""):
|
|
return default
|
|
if isinstance(value, str):
|
|
lowered = value.strip().lower()
|
|
if lowered in {"1", "true", "yes", "on", "enabled", "active", "online", "live"}:
|
|
return True
|
|
if lowered in {"0", "false", "no", "off", "disabled", "inactive", "offline"}:
|
|
return False
|
|
return bool(value)
|
|
|
|
def _clean_text(self, value: Any) -> str:
|
|
if value is None:
|
|
return ""
|
|
return str(value).strip()
|
|
|
|
def _clean_url(self, value: Any) -> str:
|
|
text = self._clean_text(value)
|
|
if not text:
|
|
return ""
|
|
parsed = urlparse(text)
|
|
if parsed.scheme and parsed.scheme not in {"http", "https"}:
|
|
return ""
|
|
if parsed.scheme and not parsed.netloc:
|
|
return ""
|
|
return text
|
|
|
|
async def _fetch_iptv_org(self, channels_url: str, collector_config: dict[str, Any]) -> list[dict[str, Any]]:
|
|
streams_url = self._clean_url(collector_config.get("streams_url")) or self.DEFAULT_IPTV_ORG_STREAMS_URL
|
|
logos_url = self._clean_url(collector_config.get("logos_url")) or self.DEFAULT_IPTV_ORG_LOGOS_URL
|
|
news_categories = {
|
|
self._clean_text(value).lower()
|
|
for value in (collector_config.get("news_categories") or self.DEFAULT_IPTV_ORG_NEWS_CATEGORIES)
|
|
if self._clean_text(value)
|
|
}
|
|
exclude_categories = {
|
|
self._clean_text(value).lower()
|
|
for value in (collector_config.get("exclude_categories") or self.DEFAULT_IPTV_ORG_EXCLUDE_CATEGORIES)
|
|
if self._clean_text(value)
|
|
}
|
|
try:
|
|
max_sources = int(collector_config.get("max_sources", self.DEFAULT_IPTV_ORG_MAX_SOURCES))
|
|
except (TypeError, ValueError):
|
|
max_sources = self.DEFAULT_IPTV_ORG_MAX_SOURCES
|
|
|
|
timeout = self.DEFAULT_TIMEOUT
|
|
try:
|
|
timeout = float(collector_config.get("timeout", self.DEFAULT_TIMEOUT))
|
|
except (TypeError, ValueError):
|
|
timeout = self.DEFAULT_TIMEOUT
|
|
|
|
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
|
channels_payload, streams_payload, logos_payload = await self._gather_iptv_org_payloads(
|
|
client,
|
|
channels_url,
|
|
streams_url,
|
|
logos_url,
|
|
)
|
|
|
|
channels = channels_payload if isinstance(channels_payload, list) else []
|
|
streams = streams_payload if isinstance(streams_payload, list) else []
|
|
logos = logos_payload if isinstance(logos_payload, list) else []
|
|
|
|
logo_by_channel = {
|
|
self._clean_text(item.get("channel")): self._clean_url(item.get("url"))
|
|
for item in logos
|
|
if isinstance(item, dict) and self._clean_text(item.get("channel")) and self._clean_url(item.get("url"))
|
|
}
|
|
|
|
streams_by_channel: dict[str, list[dict[str, Any]]] = {}
|
|
for stream in streams:
|
|
if not isinstance(stream, dict):
|
|
continue
|
|
channel_id = self._clean_text(stream.get("channel"))
|
|
if not channel_id:
|
|
continue
|
|
streams_by_channel.setdefault(channel_id, []).append(stream)
|
|
|
|
normalized: list[dict[str, Any]] = []
|
|
for channel in channels:
|
|
if not isinstance(channel, dict):
|
|
continue
|
|
|
|
categories = [
|
|
self._clean_text(value).lower()
|
|
for value in (channel.get("categories") or [])
|
|
if self._clean_text(value)
|
|
]
|
|
if news_categories and not any(category in news_categories for category in categories):
|
|
continue
|
|
if exclude_categories and any(category in exclude_categories for category in categories):
|
|
continue
|
|
if channel.get("is_nsfw") is True:
|
|
continue
|
|
if channel.get("closed"):
|
|
continue
|
|
|
|
channel_id = self._clean_text(channel.get("id"))
|
|
if not channel_id:
|
|
continue
|
|
|
|
stream = self._pick_iptv_org_stream(streams_by_channel.get(channel_id) or [])
|
|
if not stream:
|
|
continue
|
|
|
|
stream_url = self._clean_url(stream.get("url"))
|
|
if not stream_url:
|
|
continue
|
|
|
|
name = self._clean_text(channel.get("name")) or channel_id
|
|
notes_parts = [
|
|
f"Imported from IPTV-org catalog ({channel_id})",
|
|
f"Categories: {', '.join(categories)}" if categories else "",
|
|
f"Quality: {self._clean_text(stream.get('quality'))}" if self._clean_text(stream.get("quality")) else "",
|
|
]
|
|
metadata = {
|
|
"provider": self._clean_text(channel.get("network")) or "IPTV-org",
|
|
"region": self._clean_text(channel.get("country")) or "Global",
|
|
"language": "und",
|
|
"source_type": "hls" if stream_url.endswith(".m3u8") else "video",
|
|
"embed_url": "",
|
|
"stream_url": stream_url,
|
|
"homepage_url": self._clean_url(channel.get("website")),
|
|
"poster_url": logo_by_channel.get(channel_id, ""),
|
|
"youtube_video_id": "",
|
|
"youtube_channel": "",
|
|
"sort_order": 400 + len(normalized),
|
|
"notes": "; ".join(part for part in notes_parts if part),
|
|
"is_enabled": True,
|
|
"collector_adapter": "iptv_org",
|
|
"channel_id": channel_id,
|
|
"categories": categories,
|
|
"quality": self._clean_text(stream.get("quality")),
|
|
"stream_label": self._clean_text(stream.get("label") or stream.get("title")),
|
|
"stream_referrer": self._clean_text(stream.get("referrer")),
|
|
"stream_user_agent": self._clean_text(stream.get("user_agent")),
|
|
}
|
|
|
|
normalized.append(
|
|
{
|
|
"source_id": channel_id,
|
|
"name": name,
|
|
"description": metadata["notes"],
|
|
"metadata": metadata,
|
|
"reference_date": datetime.now(UTC).isoformat(),
|
|
}
|
|
)
|
|
if len(normalized) >= max_sources:
|
|
break
|
|
|
|
return normalized
|
|
|
|
async def _gather_iptv_org_payloads(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
channels_url: str,
|
|
streams_url: str,
|
|
logos_url: str,
|
|
) -> tuple[Any, Any, Any]:
|
|
headers = dict(self.DEFAULT_HEADERS)
|
|
channels_payload, streams_payload, logos_payload = await asyncio.gather(
|
|
client.get(channels_url, headers=headers),
|
|
client.get(streams_url, headers=headers),
|
|
client.get(logos_url, headers=headers),
|
|
)
|
|
channels_payload.raise_for_status()
|
|
streams_payload.raise_for_status()
|
|
logos_payload.raise_for_status()
|
|
return channels_payload.json(), streams_payload.json(), logos_payload.json()
|
|
|
|
def _pick_iptv_org_stream(self, streams: list[dict[str, Any]]) -> dict[str, Any] | None:
|
|
if not streams:
|
|
return None
|
|
|
|
def score(stream: dict[str, Any]) -> tuple[int, int]:
|
|
url = self._clean_url(stream.get("url"))
|
|
quality = self._clean_text(stream.get("quality")).lower()
|
|
quality_score = 0
|
|
if quality.endswith("p"):
|
|
try:
|
|
quality_score = int(quality[:-1])
|
|
except ValueError:
|
|
quality_score = 0
|
|
stream_score = 1000 if url.endswith(".m3u8") else 0
|
|
return stream_score, quality_score
|
|
|
|
sorted_streams = sorted(streams, key=score, reverse=True)
|
|
return sorted_streams[0]
|
|
|
|
def parse_response(self, response: Any, *, response_path: str | None = None) -> list[dict[str, Any]]:
|
|
candidates = self._extract_candidates(response, response_path)
|
|
|
|
normalized: list[dict[str, Any]] = []
|
|
for index, item in enumerate(candidates):
|
|
if not isinstance(item, dict):
|
|
continue
|
|
|
|
stream_id = (
|
|
item.get("id")
|
|
or item.get("source_id")
|
|
or item.get("slug")
|
|
or item.get("channel_id")
|
|
or item.get("code")
|
|
or f"news-live-{index + 1}"
|
|
)
|
|
name = self._clean_text(
|
|
item.get("name")
|
|
or item.get("title")
|
|
or item.get("channel")
|
|
or item.get("display_name")
|
|
or f"News Live {index + 1}"
|
|
)
|
|
if not name:
|
|
continue
|
|
|
|
source_type = self._infer_source_type(item)
|
|
stream_url = self._clean_url(
|
|
item.get("stream_url")
|
|
or item.get("stream")
|
|
or item.get("playback_url")
|
|
or item.get("hls_url")
|
|
or item.get("m3u8_url")
|
|
)
|
|
embed_url = self._clean_url(
|
|
item.get("embed_url")
|
|
or item.get("embed")
|
|
or item.get("page_url")
|
|
or (item.get("url") if source_type == "iframe" else "")
|
|
)
|
|
homepage_url = self._clean_url(
|
|
item.get("homepage_url")
|
|
or item.get("source_url")
|
|
or item.get("website")
|
|
or item.get("url")
|
|
)
|
|
metadata = {
|
|
"provider": self._clean_text(item.get("provider") or item.get("publisher") or item.get("network")) or "Collector",
|
|
"region": self._clean_text(item.get("region") or item.get("country") or item.get("market")) or "Global",
|
|
"language": self._clean_text(item.get("language") or item.get("lang") or item.get("locale")) or "und",
|
|
"source_type": source_type,
|
|
"embed_url": embed_url,
|
|
"stream_url": stream_url,
|
|
"homepage_url": homepage_url,
|
|
"poster_url": self._clean_url(item.get("poster_url") or item.get("thumbnail_url") or item.get("logo_url")),
|
|
"youtube_video_id": self._clean_text(
|
|
item.get("youtube_video_id")
|
|
or item.get("video_id")
|
|
or item.get("youtubeVideoId")
|
|
),
|
|
"youtube_channel": self._clean_text(
|
|
item.get("youtube_channel")
|
|
or item.get("channel_handle")
|
|
or item.get("youtubeChannel")
|
|
),
|
|
"sort_order": item.get("sort_order", 200 + index),
|
|
"notes": self._clean_text(item.get("notes") or item.get("description") or item.get("summary")),
|
|
"is_enabled": self._parse_enabled(item),
|
|
}
|
|
|
|
normalized.append(
|
|
{
|
|
"source_id": str(stream_id),
|
|
"name": name,
|
|
"description": metadata["notes"],
|
|
"metadata": metadata,
|
|
"reference_date": item.get("reference_date") or datetime.now(UTC).isoformat(),
|
|
}
|
|
)
|
|
|
|
return normalized
|