Files
planet/frontend/src/pages/DataSources/DataSources.tsx
2026-05-07 18:06:06 +08:00

713 lines
26 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
import {
Button,
Card,
Checkbox,
Col,
Drawer,
Alert,
Form,
Input,
Modal,
Progress,
Row,
Space,
Table,
Tag,
Tooltip,
Typography,
message,
} from 'antd'
import {
CopyOutlined,
InfoCircleOutlined,
PauseCircleOutlined,
PlayCircleOutlined,
SyncOutlined,
} from '@ant-design/icons'
import axios, { type AxiosResponse } from 'axios'
import { useNavigate } from 'react-router-dom'
import AppLayout from '../../components/AppLayout/AppLayout'
import ScrollbarOverlay from '../../components/Scrollbar/ScrollbarOverlay'
import { formatDateTimeZhCN } from '../../utils/datetime'
import { formatPhaseMetric, getPhaseDisplay, getPhaseSummary } from '../../utils/phaseProgress'
const { Text } = Typography
const COLLECTION_REFRESH_DELAY_MS = 800
interface BuiltInDataSource {
id: number
source: string
name: string
display_name?: string
module: string
priority: string
frequency: string
endpoint?: string
is_active: boolean
collector_class: string
last_run: string | null
last_run_at?: string | null
last_status?: string | null
is_running: boolean
task_id: number | null
progress: number | null
phase?: string | null
phase_progress?: number | null
phase_message?: string | null
phase_current?: number | null
phase_total?: number | null
phase_unit?: string | null
records_processed: number | null
total_records: number | null
is_free?: boolean
requires_credentials?: boolean
credential_provider?: string | null
credential_status?: string
}
interface CustomDataSourceOverride {
id: number
name: string
description: string | null
source_type: string
endpoint: string
auth_type: string
is_active: boolean
created_at: string
updated_at: string | null
}
interface EditableDataSourceConfig {
id: number
name: string
description: string | null
source_type: string
endpoint: string
auth_type: string
auth_config: Record<string, any>
headers: Record<string, string>
config: Record<string, any>
is_active?: boolean
}
interface UnifiedDataSource {
key: string
id: number
name: string
display_name: string
source: string
module?: string
priority?: string
frequency?: string
endpoint?: string
is_active: boolean
collector_class?: string
source_type?: string
auth_type?: string
last_run_at?: string | null
last_status?: string | null
is_running?: boolean
progress?: number | null
phase?: string | null
phase_progress?: number | null
phase_message?: string | null
phase_current?: number | null
phase_total?: number | null
phase_unit?: string | null
task_id?: number | null
created_at?: string
updated_at?: string | null
description?: string | null
headers?: Record<string, string>
config?: Record<string, any>
is_free?: boolean
requires_credentials?: boolean
credential_provider?: string | null
credential_status?: string
}
interface ViewDataSource extends UnifiedDataSource {
headers: Record<string, string>
config: Record<string, any>
}
type TriggerDatasourceConflict = {
reason?: string
message?: string
progress?: number | null
phase?: string | null
phase_progress?: number | null
phase_message?: string | null
phase_current?: number | null
phase_total?: number | null
phase_unit?: string | null
records_processed?: number | null
total_records?: number | null
}
type TriggerDatasourceResult =
| { ok: true; response: AxiosResponse<any> }
| { ok: false; status: number; detail?: string | TriggerDatasourceConflict }
type DatasourceTaskStatus = {
is_running: boolean
task_id?: number | null
progress?: number | null
phase?: string | null
phase_progress?: number | null
phase_message?: string | null
phase_current?: number | null
phase_total?: number | null
phase_unit?: string | null
records_processed?: number | null
total_records?: number | null
status?: string | null
}
function normalizeBuiltin(source: BuiltInDataSource): UnifiedDataSource {
return {
key: `builtin:${source.id}`,
id: source.id,
name: source.name,
display_name: source.display_name || source.name,
source: source.source,
module: source.module,
priority: source.priority,
frequency: source.frequency,
endpoint: source.endpoint,
is_active: source.is_active,
collector_class: source.collector_class,
last_run_at: source.last_run_at || source.last_run,
last_status: source.last_status,
is_running: source.is_running,
progress: source.progress,
phase: source.phase,
phase_progress: source.phase_progress,
phase_message: source.phase_message,
phase_current: source.phase_current,
phase_total: source.phase_total,
phase_unit: source.phase_unit,
task_id: source.task_id,
is_free: source.is_free,
requires_credentials: source.requires_credentials,
credential_provider: source.credential_provider,
credential_status: source.credential_status,
headers: {},
config: {},
}
}
function DataSources() {
const [messageApi, contextHolder] = message.useMessage()
const navigate = useNavigate()
const [modal, modalContextHolder] = Modal.useModal()
const [builtInSources, setBuiltInSources] = useState<BuiltInDataSource[]>([])
const [customOverrides, setCustomOverrides] = useState<CustomDataSourceOverride[]>([])
const [loading, setLoading] = useState(false)
const [triggerAllLoading, setTriggerAllLoading] = useState(false)
const [forceTriggerAll, setForceTriggerAll] = useState(false)
const [viewDrawerVisible, setViewDrawerVisible] = useState(false)
const [runningTasksVisible, setRunningTasksVisible] = useState(false)
const [viewingSource, setViewingSource] = useState<ViewDataSource | null>(null)
const [recordCount, setRecordCount] = useState<number | null>(null)
const [tableHeight, setTableHeight] = useState(360)
const tableRegionRef = useRef<HTMLDivElement | null>(null)
const allSources = useMemo(() => builtInSources.map(normalizeBuiltin), [builtInSources])
const activeBuiltInCount = builtInSources.filter((source) => source.is_active).length
const runningBuiltInSources = builtInSources.filter((source) => source.is_running)
const runningBuiltInCount = runningBuiltInSources.length
const aggregateProgress = runningBuiltInCount > 0
? Math.round(
builtInSources
.filter((source) => source.is_running)
.reduce((sum, source) => sum + (source.progress || 0), 0) / runningBuiltInCount,
)
: 0
const fetchData = useCallback(async () => {
setLoading(true)
try {
const [builtinRes, customRes] = await Promise.all([
axios.get('/api/v1/datasources'),
axios.get('/api/v1/datasources/configs'),
])
setBuiltInSources(builtinRes.data.data || [])
setCustomOverrides(customRes.data.data || [])
} catch (error) {
console.error('Failed to fetch data:', error)
messageApi.error('获取数据源列表失败')
} finally {
setLoading(false)
}
}, [messageApi])
useEffect(() => {
void fetchData()
}, [fetchData])
useEffect(() => {
const updateHeight = () => {
setTableHeight(Math.max(260, (tableRegionRef.current?.offsetHeight || 0) - 56))
}
updateHeight()
if (typeof ResizeObserver === 'undefined') return undefined
const observer = new ResizeObserver(updateHeight)
if (tableRegionRef.current) observer.observe(tableRegionRef.current)
return () => observer.disconnect()
}, [allSources.length])
const fetchDatasourceTaskStatus = async (id: number) => {
const res = await axios.get<DatasourceTaskStatus>(`/api/v1/datasources/${id}/task-status`)
return res.data
}
const triggerDatasource = async (id: number, options?: { force?: boolean }) => {
const res = await axios.post(`/api/v1/datasources/${id}/trigger`, null, {
params: { force: options?.force ?? false },
validateStatus: (status) => status < 500,
})
if (res.status >= 400) {
return { ok: false, status: res.status, detail: res.data?.detail } satisfies TriggerDatasourceResult
}
if (res.data.task_id) {
void fetchData()
} else {
window.setTimeout(fetchData, COLLECTION_REFRESH_DELAY_MS)
}
return { ok: true, response: res } satisfies TriggerDatasourceResult
}
const confirmForceTrigger = (id: number, taskInfo?: TriggerDatasourceConflict) => {
modal.confirm({
title: '当前任务未完成',
content: (
<div>
<p>{taskInfo?.message || '当前采集任务仍在运行,重新触发会丢失本次未完成进度。'}</p>
<p>: {getPhaseDisplay(taskInfo || {})}</p>
<p>: {typeof taskInfo?.progress === 'number' ? `${Math.round(taskInfo.progress)}%` : '未知'}</p>
<p></p>
</div>
),
okText: '强制重新采集',
cancelText: '取消',
okButtonProps: { danger: true },
onOk: async () => {
const result = await triggerDatasource(id, { force: true })
if (result.ok) {
messageApi.success('已强制重新触发')
return
}
const detail = result.detail
messageApi.error(typeof detail === 'string' ? detail : detail?.message || '强制重新采集失败')
},
})
}
const triggerDatasourceWithPrecheck = async (id: number) => {
const taskStatus = await fetchDatasourceTaskStatus(id)
if (taskStatus.is_running) {
confirmForceTrigger(id, taskStatus)
return
}
const result = await triggerDatasource(id)
if (result.ok) {
messageApi.success('任务已触发')
return
}
const detail = result.detail
if (result.status === 409 && typeof detail === 'object' && detail?.reason === 'running_task_in_progress') {
confirmForceTrigger(id, detail)
return
}
messageApi.error(typeof detail === 'string' ? detail : detail?.message || '触发失败')
}
const handleTriggerAll = async () => {
try {
setTriggerAllLoading(true)
const res = await axios.post('/api/v1/datasources/trigger-all', null, {
params: { force: forceTriggerAll },
})
const triggered = res.data.triggered || []
const skipped = res.data.skipped || []
const failed = res.data.failed || []
messageApi.success([
`已触发 ${triggered.length}`,
skipped.length ? `跳过 ${skipped.length}` : null,
failed.length ? `失败 ${failed.length}` : null,
].filter(Boolean).join(''))
void fetchData()
} catch (error: unknown) {
const err = error as { response?: { data?: { detail?: string } } }
messageApi.error(err.response?.data?.detail || '全触发失败')
} finally {
setTriggerAllLoading(false)
}
}
const handleToggle = async (id: number, current: boolean) => {
const endpoint = current ? 'disable' : 'enable'
try {
await axios.post(`/api/v1/datasources/${id}/${endpoint}`)
messageApi.success(`${current ? '已禁用' : '已启用'}`)
void fetchData()
} catch (error: unknown) {
const err = error as { response?: { data?: { detail?: string } } }
messageApi.error(err.response?.data?.detail || '操作失败')
}
}
const handleViewSource = async (source: UnifiedDataSource) => {
try {
{
const override = customOverrides.find((item) => item.name === source.source)
const [detailRes, statsRes, overrideDetail] = await Promise.all([
axios.get(`/api/v1/datasources/${source.id}`),
axios.get(`/api/v1/datasources/${source.id}/stats`),
override ? axios.get<EditableDataSourceConfig>(`/api/v1/datasources/configs/${override.id}`).then((res) => res.data) : Promise.resolve(null),
])
const data = detailRes.data
setViewingSource({
...source,
name: data.name,
display_name: data.display_name || source.display_name,
endpoint: overrideDetail?.endpoint || data.endpoint || source.endpoint || '',
auth_type: overrideDetail?.auth_type || 'none',
headers: overrideDetail?.headers || {},
config: overrideDetail?.config || {},
collector_class: data.collector_class,
module: data.module,
priority: data.priority,
frequency: data.frequency,
is_free: data.is_free,
requires_credentials: data.requires_credentials,
credential_provider: data.credential_provider,
credential_status: data.credential_status,
})
setRecordCount(statsRes.data.total_records || 0)
}
setViewDrawerVisible(true)
} catch (error) {
console.error(error)
messageApi.error('获取数据源信息失败')
}
}
const handleCopyLink = async (value: string, successText: string) => {
try {
await navigator.clipboard.writeText(value)
messageApi.success(successText)
} catch {
messageApi.error('复制失败,请手动复制')
}
}
const columns = [
{
title: '名称',
dataIndex: 'display_name',
key: 'name',
width: 260,
ellipsis: true,
render: (_: string, record: UnifiedDataSource) => (
<Space direction="vertical" size={0}>
<Button type="link" style={{ padding: 0, height: 22 }} onClick={() => { void handleViewSource(record) }}>
{record.display_name}
</Button>
<Text type="secondary" style={{ fontSize: 12 }}>{record.source}</Text>
</Space>
),
},
{
title: '类型',
key: 'kind',
width: 100,
render: () => <Tag color="blue"></Tag>,
},
{
title: '层级/类型',
key: 'module',
width: 120,
render: (_: unknown, record: UnifiedDataSource) => <Tag>{record.module}</Tag>,
},
{
title: '频率',
dataIndex: 'frequency',
key: 'frequency',
width: 90,
render: (value: string | undefined) => value || '-',
},
{
title: '最近采集',
dataIndex: 'last_run_at',
key: 'last_run_at',
width: 180,
render: (value: string | null | undefined) => formatDateTimeZhCN(value) || '-',
},
{
title: '状态',
key: 'status',
width: 180,
render: (_: unknown, record: UnifiedDataSource) => {
if (record.is_running) {
return (
<Tooltip title={getPhaseDisplay(record)}>
<Tag color="processing">{getPhaseSummary(record)}</Tag>
</Tooltip>
)
}
if (!record.last_status) return <Tag></Tag>
return <Tag color={record.last_status === 'success' ? 'success' : record.last_status === 'failed' ? 'error' : 'default'}>{record.last_status}</Tag>
},
},
{
title: '操作',
key: 'action',
fixed: 'right' as const,
width: 190,
render: (_: unknown, record: UnifiedDataSource) => (
<Space size={4}>
<Button type="link" size="small" icon={<SyncOutlined />} disabled={!record.is_active} onClick={() => { void triggerDatasourceWithPrecheck(record.id) }}>
</Button>
<Button
type="link"
size="small"
icon={record.is_active ? <PauseCircleOutlined /> : <PlayCircleOutlined />}
danger={record.is_active}
style={record.is_active ? undefined : { color: '#52c41a' }}
onClick={() => { void handleToggle(record.id, record.is_active) }}
>
{record.is_active ? '禁用' : '启用'}
</Button>
</Space>
),
},
]
return (
<AppLayout>
{contextHolder}
{modalContextHolder}
<div className="page-shell">
<div className="page-shell__header">
<h2 style={{ margin: 0 }}></h2>
</div>
<div className="page-shell__body data-source-builtin-tab">
<div className="data-source-bulk-toolbar">
<div className="data-source-bulk-toolbar__meta">
<div className="data-source-bulk-toolbar__title"></div>
<div className="data-source-bulk-toolbar__progress">
<div className="data-source-bulk-toolbar__progress-copy">
<span></span>
<strong>{aggregateProgress}%</strong>
</div>
<Progress percent={aggregateProgress} size="small" status={runningBuiltInCount > 0 ? 'active' : 'normal'} showInfo={false} strokeColor="#1677ff" />
</div>
<div className="data-source-bulk-toolbar__stats">
<div className="data-source-bulk-toolbar__stat-pill">
<span className="data-source-bulk-toolbar__stat-label"></span>
<strong>{allSources.length}</strong>
</div>
<div className="data-source-bulk-toolbar__stat-pill">
<span className="data-source-bulk-toolbar__stat-label"></span>
<strong>{builtInSources.length}</strong>
</div>
<div className="data-source-bulk-toolbar__stat-pill">
<span className="data-source-bulk-toolbar__stat-label"></span>
<strong>{activeBuiltInCount}</strong>
</div>
{runningBuiltInCount > 0 ? (
<Tooltip title="查看采集中任务">
<button
type="button"
className="data-source-bulk-toolbar__running-pill"
onClick={() => setRunningTasksVisible(true)}
>
<span className="data-source-bulk-toolbar__running-dot" />
<span className="data-source-bulk-toolbar__stat-label"></span>
<strong>{runningBuiltInCount}</strong>
<span className="data-source-bulk-toolbar__running-arrow"></span>
</button>
</Tooltip>
) : null}
</div>
</div>
<Space size={12} align="center">
<Tooltip title="这里用于查看数据源状态和触发采集;接口、凭证和自定义源配置请到设置中心维护。">
<InfoCircleOutlined style={{ fontSize: 16, color: '#8c8c8c', cursor: 'default' }} />
</Tooltip>
<Checkbox checked={forceTriggerAll} onChange={(event) => setForceTriggerAll(event.target.checked)}>
</Checkbox>
<Button type="primary" size="middle" icon={<SyncOutlined />} loading={triggerAllLoading} onClick={handleTriggerAll}>
</Button>
</Space>
</div>
<div ref={tableRegionRef} className="table-scroll-region data-source-table-region">
<Table
columns={columns}
dataSource={allSources}
rowKey="key"
loading={loading}
pagination={false}
scroll={{ x: 1100, y: tableHeight }}
tableLayout="fixed"
size="small"
/>
<ScrollbarOverlay containerRef={tableRegionRef} targetSelector=".ant-table-body" />
</div>
</div>
</div>
<Modal
title="采集中任务"
open={runningTasksVisible}
onCancel={() => setRunningTasksVisible(false)}
footer={<Button onClick={() => setRunningTasksVisible(false)}></Button>}
width={680}
>
<Space direction="vertical" size={12} style={{ width: '100%' }}>
{runningBuiltInSources.length ? runningBuiltInSources.map((source) => (
<Card key={source.id} size="small">
<Space direction="vertical" size={8} style={{ width: '100%' }}>
<div style={{ display: 'flex', justifyContent: 'space-between', gap: 12 }}>
<Space direction="vertical" size={0}>
<Text strong>{source.display_name || source.name}</Text>
<Text type="secondary" style={{ fontSize: 12 }}>
{source.source}{source.task_id ? ` · #${source.task_id}` : ''}
</Text>
</Space>
<Tooltip title={getPhaseDisplay(source)}>
<Tag color="processing">{getPhaseSummary(source)}</Tag>
</Tooltip>
</div>
{source.phase_message ? (
<Text type="secondary" style={{ fontSize: 12 }}>{source.phase_message}</Text>
) : null}
<Progress
percent={Math.round(source.progress || 0)}
size="small"
status="active"
strokeColor="#1677ff"
/>
<Text type="secondary" style={{ fontSize: 12 }}>
{source.phase_unit === 'bytes'
? `下载 ${formatPhaseMetric(source) || '准备中'}`
: `已处理 ${source.records_processed ?? 0}${source.total_records ? ` / ${source.total_records}` : ''}`}
</Text>
</Space>
</Card>
)) : (
<Text type="secondary"></Text>
)}
</Space>
</Modal>
<Drawer
title="查看数据源"
width={600}
open={viewDrawerVisible}
onClose={() => {
setViewDrawerVisible(false)
setViewingSource(null)
setRecordCount(null)
}}
footer={<div style={{ textAlign: 'right' }}><Button onClick={() => setViewDrawerVisible(false)}></Button></div>}
>
{viewingSource && (
<Space direction="vertical" size={16} style={{ width: '100%' }}>
<Card size="small" bordered={false} style={{ background: '#fafafa' }}>
<Row gutter={[12, 12]}>
<Col span={24}>
<Space>
<Tag color="blue"></Tag>
<Tag color={viewingSource.is_active ? 'green' : 'default'}>{viewingSource.is_active ? '启用' : '禁用'}</Tag>
</Space>
</Col>
<Col span={24}>
<div style={{ marginBottom: 4, color: '#8c8c8c', fontSize: 12 }}></div>
<Input value={viewingSource.display_name} disabled />
</Col>
<Col span={24}>
<div style={{ marginBottom: 4, color: '#8c8c8c', fontSize: 12 }}></div>
<Input value={viewingSource.source} disabled />
</Col>
<Col span={12}>
<div style={{ marginBottom: 4, color: '#8c8c8c', fontSize: 12 }}></div>
<Input value={viewingSource.module || '-'} disabled />
</Col>
<Col span={12}>
<div style={{ marginBottom: 4, color: '#8c8c8c', fontSize: 12 }}></div>
<Input value={viewingSource.priority || '-'} disabled />
</Col>
<Col span={12}>
<div style={{ marginBottom: 4, color: '#8c8c8c', fontSize: 12 }}></div>
<Input value={viewingSource.frequency || '-'} disabled />
</Col>
<Col span={12}>
<div style={{ marginBottom: 4, color: '#8c8c8c', fontSize: 12 }}></div>
<Input value={recordCount === null ? '-' : `${recordCount}`} disabled />
</Col>
<Col span={24}>
<div style={{ marginBottom: 4, color: '#8c8c8c', fontSize: 12 }}></div>
<Input value={viewingSource.collector_class || '-'} disabled />
</Col>
</Row>
</Card>
{viewingSource.requires_credentials ? (
<Alert
type={viewingSource.credential_status === 'supported' ? 'info' : 'warning'}
showIcon
message="需要采集器凭证"
description={viewingSource.credential_status === 'supported' ? '请在设置中心的采集器设置中维护该采集器凭证。' : '该采集器需要凭证,配置入口待接入。'}
action={viewingSource.credential_status === 'supported' ? (
<Button
size="small"
type="primary"
onClick={() => navigate(`/settings?tab=collector_credentials&collector=${encodeURIComponent(viewingSource.source)}`)}
>
</Button>
) : undefined}
/>
) : null}
<Form layout="vertical">
<Form.Item label="采集源 API 链接">
<Space.Compact style={{ width: '100%' }}>
<Input value={viewingSource.endpoint || '-'} readOnly />
<Tooltip title={viewingSource.endpoint ? '复制采集源 API 链接' : '当前没有可复制的采集源 API 链接'}>
<Button disabled={!viewingSource.endpoint} icon={<CopyOutlined />} onClick={() => viewingSource.endpoint && handleCopyLink(viewingSource.endpoint, '采集源 API 链接已复制')} />
</Tooltip>
</Space.Compact>
</Form.Item>
<Form.Item label="请求头">
<pre style={{ background: '#f5f5f5', padding: 12, borderRadius: 4, overflow: 'auto', margin: 0 }}>
{JSON.stringify(viewingSource.headers || {}, null, 2)}
</pre>
</Form.Item>
<Form.Item label="运行参数">
<pre style={{ background: '#f5f5f5', padding: 12, borderRadius: 4, overflow: 'auto', margin: 0 }}>
{JSON.stringify(viewingSource.config || {}, null, 2)}
</pre>
</Form.Item>
</Form>
</Space>
)}
</Drawer>
</AppLayout>
)
}
export default DataSources