import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { Button, Card, Checkbox, Col, Drawer, Alert, Form, Input, Modal, Progress, Row, Space, Table, Tag, Tooltip, Typography, message, } from 'antd' import { CopyOutlined, InfoCircleOutlined, PauseCircleOutlined, PlayCircleOutlined, SyncOutlined, } from '@ant-design/icons' import axios, { type AxiosResponse } from 'axios' import { useNavigate } from 'react-router-dom' import AppLayout from '../../components/AppLayout/AppLayout' import ScrollbarOverlay from '../../components/Scrollbar/ScrollbarOverlay' import { formatDateTimeZhCN } from '../../utils/datetime' import { formatPhaseMetric, getPhaseDisplay, getPhaseSummary } from '../../utils/phaseProgress' const { Text } = Typography const COLLECTION_REFRESH_DELAY_MS = 800 interface BuiltInDataSource { id: number source: string name: string display_name?: string module: string priority: string frequency: string endpoint?: string is_active: boolean collector_class: string last_run: string | null last_run_at?: string | null last_status?: string | null is_running: boolean task_id: number | null progress: number | null phase?: string | null phase_progress?: number | null phase_message?: string | null phase_current?: number | null phase_total?: number | null phase_unit?: string | null records_processed: number | null total_records: number | null is_free?: boolean requires_credentials?: boolean credential_provider?: string | null credential_status?: string } interface CustomDataSourceOverride { id: number name: string description: string | null source_type: string endpoint: string auth_type: string is_active: boolean created_at: string updated_at: string | null } interface EditableDataSourceConfig { id: number name: string description: string | null source_type: string endpoint: string auth_type: string auth_config: Record headers: Record config: Record is_active?: boolean } interface UnifiedDataSource { key: string id: number name: string display_name: string source: string module?: string priority?: string frequency?: string endpoint?: string is_active: boolean collector_class?: string source_type?: string auth_type?: string last_run_at?: string | null last_status?: string | null is_running?: boolean progress?: number | null phase?: string | null phase_progress?: number | null phase_message?: string | null phase_current?: number | null phase_total?: number | null phase_unit?: string | null task_id?: number | null created_at?: string updated_at?: string | null description?: string | null headers?: Record config?: Record is_free?: boolean requires_credentials?: boolean credential_provider?: string | null credential_status?: string } interface ViewDataSource extends UnifiedDataSource { headers: Record config: Record } type TriggerDatasourceConflict = { reason?: string message?: string progress?: number | null phase?: string | null phase_progress?: number | null phase_message?: string | null phase_current?: number | null phase_total?: number | null phase_unit?: string | null records_processed?: number | null total_records?: number | null } type TriggerDatasourceResult = | { ok: true; response: AxiosResponse } | { ok: false; status: number; detail?: string | TriggerDatasourceConflict } type DatasourceTaskStatus = { is_running: boolean task_id?: number | null progress?: number | null phase?: string | null phase_progress?: number | null phase_message?: string | null phase_current?: number | null phase_total?: number | null phase_unit?: string | null records_processed?: number | null total_records?: number | null status?: string | null } function normalizeBuiltin(source: BuiltInDataSource): UnifiedDataSource { return { key: `builtin:${source.id}`, id: source.id, name: source.name, display_name: source.display_name || source.name, source: source.source, module: source.module, priority: source.priority, frequency: source.frequency, endpoint: source.endpoint, is_active: source.is_active, collector_class: source.collector_class, last_run_at: source.last_run_at || source.last_run, last_status: source.last_status, is_running: source.is_running, progress: source.progress, phase: source.phase, phase_progress: source.phase_progress, phase_message: source.phase_message, phase_current: source.phase_current, phase_total: source.phase_total, phase_unit: source.phase_unit, task_id: source.task_id, is_free: source.is_free, requires_credentials: source.requires_credentials, credential_provider: source.credential_provider, credential_status: source.credential_status, headers: {}, config: {}, } } function DataSources() { const [messageApi, contextHolder] = message.useMessage() const navigate = useNavigate() const [modal, modalContextHolder] = Modal.useModal() const [builtInSources, setBuiltInSources] = useState([]) const [customOverrides, setCustomOverrides] = useState([]) const [loading, setLoading] = useState(false) const [triggerAllLoading, setTriggerAllLoading] = useState(false) const [forceTriggerAll, setForceTriggerAll] = useState(false) const [viewDrawerVisible, setViewDrawerVisible] = useState(false) const [runningTasksVisible, setRunningTasksVisible] = useState(false) const [viewingSource, setViewingSource] = useState(null) const [recordCount, setRecordCount] = useState(null) const [tableHeight, setTableHeight] = useState(360) const tableRegionRef = useRef(null) const allSources = useMemo(() => builtInSources.map(normalizeBuiltin), [builtInSources]) const activeBuiltInCount = builtInSources.filter((source) => source.is_active).length const runningBuiltInSources = builtInSources.filter((source) => source.is_running) const runningBuiltInCount = runningBuiltInSources.length const aggregateProgress = runningBuiltInCount > 0 ? Math.round( builtInSources .filter((source) => source.is_running) .reduce((sum, source) => sum + (source.progress || 0), 0) / runningBuiltInCount, ) : 0 const fetchData = useCallback(async () => { setLoading(true) try { const [builtinRes, customRes] = await Promise.all([ axios.get('/api/v1/datasources'), axios.get('/api/v1/datasources/configs'), ]) setBuiltInSources(builtinRes.data.data || []) setCustomOverrides(customRes.data.data || []) } catch (error) { console.error('Failed to fetch data:', error) messageApi.error('获取数据源列表失败') } finally { setLoading(false) } }, [messageApi]) useEffect(() => { void fetchData() }, [fetchData]) useEffect(() => { const updateHeight = () => { setTableHeight(Math.max(260, (tableRegionRef.current?.offsetHeight || 0) - 56)) } updateHeight() if (typeof ResizeObserver === 'undefined') return undefined const observer = new ResizeObserver(updateHeight) if (tableRegionRef.current) observer.observe(tableRegionRef.current) return () => observer.disconnect() }, [allSources.length]) const fetchDatasourceTaskStatus = async (id: number) => { const res = await axios.get(`/api/v1/datasources/${id}/task-status`) return res.data } const triggerDatasource = async (id: number, options?: { force?: boolean }) => { const res = await axios.post(`/api/v1/datasources/${id}/trigger`, null, { params: { force: options?.force ?? false }, validateStatus: (status) => status < 500, }) if (res.status >= 400) { return { ok: false, status: res.status, detail: res.data?.detail } satisfies TriggerDatasourceResult } if (res.data.task_id) { void fetchData() } else { window.setTimeout(fetchData, COLLECTION_REFRESH_DELAY_MS) } return { ok: true, response: res } satisfies TriggerDatasourceResult } const confirmForceTrigger = (id: number, taskInfo?: TriggerDatasourceConflict) => { modal.confirm({ title: '当前任务未完成', content: (

{taskInfo?.message || '当前采集任务仍在运行,重新触发会丢失本次未完成进度。'}

当前阶段: {getPhaseDisplay(taskInfo || {})}

当前进度: {typeof taskInfo?.progress === 'number' ? `${Math.round(taskInfo.progress)}%` : '未知'}

确认后会强制取消当前采集,并重新开始采集。

), okText: '强制重新采集', cancelText: '取消', okButtonProps: { danger: true }, onOk: async () => { const result = await triggerDatasource(id, { force: true }) if (result.ok) { messageApi.success('已强制重新触发') return } const detail = result.detail messageApi.error(typeof detail === 'string' ? detail : detail?.message || '强制重新采集失败') }, }) } const triggerDatasourceWithPrecheck = async (id: number) => { const taskStatus = await fetchDatasourceTaskStatus(id) if (taskStatus.is_running) { confirmForceTrigger(id, taskStatus) return } const result = await triggerDatasource(id) if (result.ok) { messageApi.success('任务已触发') return } const detail = result.detail if (result.status === 409 && typeof detail === 'object' && detail?.reason === 'running_task_in_progress') { confirmForceTrigger(id, detail) return } messageApi.error(typeof detail === 'string' ? detail : detail?.message || '触发失败') } const handleTriggerAll = async () => { try { setTriggerAllLoading(true) const res = await axios.post('/api/v1/datasources/trigger-all', null, { params: { force: forceTriggerAll }, }) const triggered = res.data.triggered || [] const skipped = res.data.skipped || [] const failed = res.data.failed || [] messageApi.success([ `已触发 ${triggered.length} 个`, skipped.length ? `跳过 ${skipped.length} 个` : null, failed.length ? `失败 ${failed.length} 个` : null, ].filter(Boolean).join(',')) void fetchData() } catch (error: unknown) { const err = error as { response?: { data?: { detail?: string } } } messageApi.error(err.response?.data?.detail || '全触发失败') } finally { setTriggerAllLoading(false) } } const handleToggle = async (id: number, current: boolean) => { const endpoint = current ? 'disable' : 'enable' try { await axios.post(`/api/v1/datasources/${id}/${endpoint}`) messageApi.success(`${current ? '已禁用' : '已启用'}`) void fetchData() } catch (error: unknown) { const err = error as { response?: { data?: { detail?: string } } } messageApi.error(err.response?.data?.detail || '操作失败') } } const handleViewSource = async (source: UnifiedDataSource) => { try { { const override = customOverrides.find((item) => item.name === source.source) const [detailRes, statsRes, overrideDetail] = await Promise.all([ axios.get(`/api/v1/datasources/${source.id}`), axios.get(`/api/v1/datasources/${source.id}/stats`), override ? axios.get(`/api/v1/datasources/configs/${override.id}`).then((res) => res.data) : Promise.resolve(null), ]) const data = detailRes.data setViewingSource({ ...source, name: data.name, display_name: data.display_name || source.display_name, endpoint: overrideDetail?.endpoint || data.endpoint || source.endpoint || '', auth_type: overrideDetail?.auth_type || 'none', headers: overrideDetail?.headers || {}, config: overrideDetail?.config || {}, collector_class: data.collector_class, module: data.module, priority: data.priority, frequency: data.frequency, is_free: data.is_free, requires_credentials: data.requires_credentials, credential_provider: data.credential_provider, credential_status: data.credential_status, }) setRecordCount(statsRes.data.total_records || 0) } setViewDrawerVisible(true) } catch (error) { console.error(error) messageApi.error('获取数据源信息失败') } } const handleCopyLink = async (value: string, successText: string) => { try { await navigator.clipboard.writeText(value) messageApi.success(successText) } catch { messageApi.error('复制失败,请手动复制') } } const columns = [ { title: '名称', dataIndex: 'display_name', key: 'name', width: 260, ellipsis: true, render: (_: string, record: UnifiedDataSource) => ( {record.source} ), }, { title: '类型', key: 'kind', width: 100, render: () => 内置, }, { title: '层级/类型', key: 'module', width: 120, render: (_: unknown, record: UnifiedDataSource) => {record.module}, }, { title: '频率', dataIndex: 'frequency', key: 'frequency', width: 90, render: (value: string | undefined) => value || '-', }, { title: '最近采集', dataIndex: 'last_run_at', key: 'last_run_at', width: 180, render: (value: string | null | undefined) => formatDateTimeZhCN(value) || '-', }, { title: '状态', key: 'status', width: 180, render: (_: unknown, record: UnifiedDataSource) => { if (record.is_running) { return ( {getPhaseSummary(record)} ) } if (!record.last_status) return 未执行 return {record.last_status} }, }, { title: '操作', key: 'action', fixed: 'right' as const, width: 190, render: (_: unknown, record: UnifiedDataSource) => ( ), }, ] return ( {contextHolder} {modalContextHolder}

数据源

采集实时进度
总体进度 {aggregateProgress}%
0 ? 'active' : 'normal'} showInfo={false} strokeColor="#1677ff" />
全部 {allSources.length}
内置 {builtInSources.length}
已启用内置 {activeBuiltInCount}
{runningBuiltInCount > 0 ? ( ) : null}
setForceTriggerAll(event.target.checked)}> 强制全部采集
setRunningTasksVisible(false)} footer={} width={680} > {runningBuiltInSources.length ? runningBuiltInSources.map((source) => (
{source.display_name || source.name} {source.source}{source.task_id ? ` · #${source.task_id}` : ''} {getPhaseSummary(source)}
{source.phase_message ? ( {source.phase_message} ) : null} {source.phase_unit === 'bytes' ? `下载 ${formatPhaseMetric(source) || '准备中'}` : `已处理 ${source.records_processed ?? 0}${source.total_records ? ` / ${source.total_records}` : ''}`}
)) : ( 当前没有采集中任务。 )}
{ setViewDrawerVisible(false) setViewingSource(null) setRecordCount(null) }} footer={
} > {viewingSource && (
内置数据源 {viewingSource.is_active ? '启用' : '禁用'}
名称
标识
模块
优先级
频率
数据量
采集器
{viewingSource.requires_credentials ? ( navigate(`/settings?tab=collector_credentials&collector=${encodeURIComponent(viewingSource.source)}`)} > 去配置 ) : undefined} /> ) : null}