Files
planet/frontend/src/pages/DataSources/DataSources.tsx

857 lines
29 KiB
TypeScript

import { useEffect, useRef, useState } from 'react'
import {
Table, Tag, Space, message, Button, Form, Input, Select,
Drawer, Tabs, Empty, Tooltip, Popconfirm, Collapse, InputNumber
} from 'antd'
import {
PlayCircleOutlined, PauseCircleOutlined, PlusOutlined,
EditOutlined, DeleteOutlined, ApiOutlined,
CheckCircleOutlined, CloseCircleOutlined, ExperimentOutlined,
SyncOutlined, ClearOutlined
} from '@ant-design/icons'
import axios from 'axios'
import AppLayout from '../../components/AppLayout/AppLayout'
interface BuiltInDataSource {
id: number
name: string
module: string
priority: string
frequency: string
is_active: boolean
collector_class: string
last_run: string | null
is_running: boolean
task_id: number | null
progress: number | null
records_processed: number | null
total_records: number | null
}
interface CustomDataSource {
id: number
name: string
description: string | null
source_type: string
endpoint: string
auth_type: string
is_active: boolean
created_at: string
updated_at: string | null
}
interface ViewDataSource {
id: number
name: string
description: string | null
source_type: string
endpoint: string
auth_type: string
headers: Record<string, string>
config: Record<string, any>
collector_class: string
module: string
priority: string
frequency: string
}
function DataSources() {
const [activeTab, setActiveTab] = useState('builtin')
const [builtInSources, setBuiltInSources] = useState<BuiltInDataSource[]>([])
const [customSources, setCustomSources] = useState<CustomDataSource[]>([])
const [loading, setLoading] = useState(false)
const [drawerVisible, setDrawerVisible] = useState(false)
const [viewDrawerVisible, setViewDrawerVisible] = useState(false)
const [editingConfig, setEditingConfig] = useState<CustomDataSource | null>(null)
const [viewingSource, setViewingSource] = useState<ViewDataSource | null>(null)
const [recordCount, setRecordCount] = useState<number>(0)
const [testing, setTesting] = useState(false)
const [testResult, setTestResult] = useState<any>(null)
const builtinTableRegionRef = useRef<HTMLDivElement | null>(null)
const customTableRegionRef = useRef<HTMLDivElement | null>(null)
const [builtinTableHeight, setBuiltinTableHeight] = useState(360)
const [customTableHeight, setCustomTableHeight] = useState(360)
const [form] = Form.useForm()
const fetchData = async () => {
setLoading(true)
try {
const [builtinRes, customRes] = await Promise.all([
axios.get('/api/v1/datasources'),
axios.get('/api/v1/datasources/configs')
])
setBuiltInSources(builtinRes.data.data || [])
setCustomSources(customRes.data.data || [])
} catch (error) {
console.error('Failed to fetch data:', error)
} finally {
setLoading(false)
}
}
const [taskProgress, setTaskProgress] = useState<Record<number, { progress: number; is_running: boolean }>>({})
useEffect(() => {
fetchData()
}, [])
useEffect(() => {
const updateHeights = () => {
const builtinRegionHeight = builtinTableRegionRef.current?.offsetHeight || 0
const customRegionHeight = customTableRegionRef.current?.offsetHeight || 0
setBuiltinTableHeight(Math.max(220, builtinRegionHeight - 56))
setCustomTableHeight(Math.max(220, customRegionHeight - 56))
}
updateHeights()
if (typeof ResizeObserver === 'undefined') {
return undefined
}
const observer = new ResizeObserver(updateHeights)
if (builtinTableRegionRef.current) observer.observe(builtinTableRegionRef.current)
if (customTableRegionRef.current) observer.observe(customTableRegionRef.current)
return () => observer.disconnect()
}, [activeTab, builtInSources.length, customSources.length])
useEffect(() => {
const runningSources = builtInSources.filter(s => s.is_running)
if (runningSources.length === 0) return
const interval = setInterval(async () => {
const progressMap: Record<number, { progress: number; is_running: boolean }> = {}
await Promise.all(
runningSources.map(async (source) => {
try {
const res = await axios.get(`/api/v1/datasources/${source.id}/task-status`)
progressMap[source.id] = {
progress: res.data.progress || 0,
is_running: res.data.is_running
}
} catch {
progressMap[source.id] = { progress: 0, is_running: false }
}
})
)
setTaskProgress(prev => ({ ...prev, ...progressMap }))
}, 2000)
return () => clearInterval(interval)
}, [builtInSources.map(s => s.id).join(',')])
const handleTrigger = async (id: number) => {
try {
await axios.post(`/api/v1/datasources/${id}/trigger`)
message.success('任务已触发')
// Trigger polling immediately
setTaskProgress(prev => ({ ...prev, [id]: { progress: 0, is_running: true } }))
// Also refresh data
fetchData()
// Also fetch the running task status
pollTaskStatus(id)
} catch (error: unknown) {
const err = error as { response?: { data?: { detail?: string } } }
message.error(err.response?.data?.detail || '触发失败')
}
}
const pollTaskStatus = async (sourceId: number) => {
const poll = async () => {
try {
const res = await axios.get(`/api/v1/datasources/${sourceId}/task-status`)
const data = res.data
setTaskProgress(prev => ({ ...prev, [sourceId]: {
progress: data.progress || 0,
is_running: data.is_running
} }))
// Keep polling while running
if (data.is_running) {
setTimeout(poll, 2000)
} else {
// Task completed - refresh data and clear this source from progress
setTimeout(() => {
setTaskProgress(prev => {
const newState = { ...prev }
delete newState[sourceId]
return newState
})
}, 1000)
fetchData()
}
} catch {
// Stop polling on error
}
}
poll()
}
const handleToggle = async (id: number, current: boolean) => {
const endpoint = current ? 'disable' : 'enable'
try {
await axios.post(`/api/v1/datasources/${id}/${endpoint}`)
message.success(`${current ? '已禁用' : '已启用'}`)
fetchData()
} catch (error: unknown) {
const err = error as { response?: { data?: { detail?: string } } }
message.error(err.response?.data?.detail || '操作失败')
}
}
const handleClearDataFromDrawer = async () => {
if (!viewingSource) return
try {
const res = await axios.delete(`/api/v1/datasources/${viewingSource.id}/data`)
message.success(res.data.message || '数据已删除')
setViewDrawerVisible(false)
fetchData()
} catch (error: unknown) {
const err = error as { response?: { data?: { detail?: string } } }
message.error(err.response?.data?.detail || '删除数据失败')
}
}
const handleViewSource = async (source: BuiltInDataSource) => {
try {
const [res, statsRes] = await Promise.all([
axios.get(`/api/v1/datasources/${source.id}`),
axios.get(`/api/v1/datasources/${source.id}/stats`)
])
const data = res.data
setViewingSource({
id: data.id,
name: data.name,
description: null,
source_type: data.collector_class,
endpoint: '',
auth_type: 'none',
headers: {},
config: {},
collector_class: data.collector_class,
module: data.module,
priority: data.priority,
frequency: data.frequency,
})
setRecordCount(statsRes.data.total_records || 0)
setViewDrawerVisible(true)
} catch (error) {
message.error('获取数据源信息失败')
}
}
const handleUpdateSource = async () => {
if (!viewingSource) return
try {
await axios.post(`/api/v1/datasources/${viewingSource.id}/trigger`)
message.success('已触发更新')
setViewDrawerVisible(false)
} catch (error: unknown) {
const err = error as { response?: { data?: { detail?: string } } }
message.error(err.response?.data?.detail || '更新失败')
}
}
const handleTest = async () => {
try {
const values = await form.validateFields()
setTesting(true)
setTestResult(null)
const res = await axios.post('/api/v1/datasources/configs/test', values)
setTestResult(res.data)
if (res.data.success) {
message.success('连接测试成功')
} else {
message.error('连接测试失败')
}
} catch (error: unknown) {
const err = error as { response?: { data?: { detail?: string; message?: string } } }
message.error(err.response?.data?.message || err.response?.data?.detail || '测试失败')
} finally {
setTesting(false)
}
}
const handleSave = async () => {
try {
const values = await form.validateFields()
if (editingConfig) {
await axios.put(`/api/v1/datasources/configs/${editingConfig.id}`, values)
message.success('配置已更新')
} else {
await axios.post('/api/v1/datasources/configs', values)
message.success('配置已创建')
}
setDrawerVisible(false)
form.resetFields()
setEditingConfig(null)
setTestResult(null)
fetchData()
} catch (error: unknown) {
const err = error as { response?: { data?: { detail?: string; message?: string } } }
message.error(err.response?.data?.message || err.response?.data?.detail || '保存失败')
}
}
const handleDelete = async (id: number) => {
try {
await axios.delete(`/api/v1/datasources/configs/${id}`)
message.success('配置已删除')
fetchData()
} catch (error: unknown) {
const err = error as { response?: { data?: { detail?: string } } }
message.error(err.response?.data?.detail || '删除失败')
}
}
const handleToggleCustom = async (id: number, current: boolean) => {
try {
await axios.put(`/api/v1/datasources/configs/${id}`, { is_active: !current })
message.success(`${current ? '已禁用' : '已启用'}`)
fetchData()
} catch (error: unknown) {
const err = error as { response?: { data?: { detail?: string } } }
message.error(err.response?.data?.detail || '操作失败')
}
}
const openDrawer = (config?: CustomDataSource) => {
setEditingConfig(config || null)
if (config) {
form.setFieldsValue({
...config,
auth_config: {},
})
} else {
form.resetFields()
form.setFieldsValue({
source_type: 'http',
auth_type: 'none',
config: { timeout: 30, retry: 3 },
headers: {},
})
}
setDrawerVisible(true)
setTestResult(null)
}
const builtinColumns = [
{ title: 'ID', dataIndex: 'id', key: 'id', width: 60, fixed: 'left' as const },
{
title: '名称',
dataIndex: 'name',
key: 'name',
width: 180,
ellipsis: true,
render: (name: string, record: BuiltInDataSource) => (
<Button type="link" onClick={() => handleViewSource(record)}>
{name}
</Button>
),
},
{ title: '模块', dataIndex: 'module', key: 'module', width: 80 },
{
title: '优先级',
dataIndex: 'priority',
key: 'priority',
width: 80,
render: (p: string) => <Tag color={p === 'P0' ? 'red' : 'orange'}>{p}</Tag>,
},
{ title: '频率', dataIndex: 'frequency', key: 'frequency', width: 80 },
{
title: '最近采集',
dataIndex: 'last_run',
key: 'last_run',
width: 140,
render: (lastRun: string | null) => lastRun || '-',
},
{
title: '状态',
dataIndex: 'is_active',
key: 'is_active',
width: 100,
render: (_: unknown, record: BuiltInDataSource) => {
const progress = taskProgress[record.id]
if (progress?.is_running || record.is_running) {
const pct = progress?.progress ?? record.progress ?? 0
return (
<Tag color="blue">
{Math.round(pct)}%
</Tag>
)
}
return <Tag color={record.is_active ? 'green' : 'red'}>{record.is_active ? '运行中' : '已暂停'}</Tag>
},
},
{
title: '操作',
key: 'action',
width: 200,
fixed: 'right' as const,
render: (_: unknown, record: BuiltInDataSource) => (
<Space size="small">
<Button
type="link"
size="small"
icon={<SyncOutlined />}
onClick={() => handleTrigger(record.id)}
>
</Button>
<Button
type="link"
size="small"
icon={record.is_active ? <PauseCircleOutlined /> : <PlayCircleOutlined />}
onClick={() => handleToggle(record.id, record.is_active)}
>
{record.is_active ? '禁用' : '启用'}
</Button>
</Space>
),
},
]
const customColumns = [
{ title: 'ID', dataIndex: 'id', key: 'id', width: 60, fixed: 'left' as const },
{ title: '名称', dataIndex: 'name', key: 'name', width: 150, ellipsis: true },
{ title: '类型', dataIndex: 'source_type', key: 'source_type', width: 100 },
{
title: '状态',
dataIndex: 'is_active',
key: 'is_active',
width: 80,
render: (active: boolean) => (
<Tag color={active ? 'green' : 'red'}>{active ? '启用' : '禁用'}</Tag>
),
},
{ title: '创建时间', dataIndex: 'created_at', key: 'created_at', width: 160 },
{
title: '操作',
key: 'action',
width: 150,
fixed: 'right' as const,
render: (_: unknown, record: CustomDataSource) => (
<Space size="small">
<Tooltip title="编辑">
<Button type="link" size="small" icon={<EditOutlined />} onClick={() => openDrawer(record)} />
</Tooltip>
<Tooltip title={record.is_active ? '禁用' : '启用'}>
<Button
type="link"
size="small"
icon={record.is_active ? <PauseCircleOutlined /> : <PlayCircleOutlined />}
onClick={() => handleToggleCustom(record.id, record.is_active)}
/>
</Tooltip>
<Popconfirm
title="确定删除此配置?"
onConfirm={() => handleDelete(record.id)}
>
<Tooltip title="删除">
<Button type="link" size="small" danger icon={<DeleteOutlined />} />
</Tooltip>
</Popconfirm>
</Space>
),
},
]
const tabItems = [
{
key: 'builtin',
label: '内置数据源',
children: (
<div className="page-shell__body">
<div ref={builtinTableRegionRef} className="table-scroll-region data-source-table-region">
<Table
columns={builtinColumns}
dataSource={builtInSources}
rowKey="id"
loading={loading}
pagination={false}
scroll={{ x: 800, y: builtinTableHeight }}
tableLayout="fixed"
size="small"
virtual
/>
</div>
</div>
),
},
{
key: 'custom',
label: (
<span>
<ApiOutlined />
</span>
),
children: (
<div className="page-shell__body data-source-custom-tab">
<div className="data-source-custom-toolbar">
<Button type="primary" icon={<PlusOutlined />} onClick={() => openDrawer()}>
</Button>
</div>
{customSources.length === 0 ? (
<div className="data-source-empty-state">
<Empty description="暂无自定义数据源" />
</div>
) : (
<div ref={customTableRegionRef} className="table-scroll-region data-source-table-region">
<Table
columns={customColumns}
dataSource={customSources}
rowKey="id"
loading={loading}
pagination={false}
scroll={{ x: 600, y: customTableHeight }}
tableLayout="fixed"
size="small"
virtual
/>
</div>
)}
</div>
),
},
]
return (
<AppLayout>
<div className="page-shell">
<div className="page-shell__header">
<h2 style={{ margin: 0 }}></h2>
</div>
<div className="page-shell__body">
<div className="data-source-tabs-shell">
<Tabs className="data-source-tabs" activeKey={activeTab} onChange={setActiveTab} items={tabItems} />
</div>
</div>
</div>
<Drawer
title={editingConfig ? '编辑数据源' : '添加数据源'}
width={600}
open={drawerVisible}
onClose={() => {
setDrawerVisible(false)
form.resetFields()
setEditingConfig(null)
setTestResult(null)
}}
footer={
<div style={{ display: 'flex', justifyContent: 'space-between' }}>
<Button
icon={<ExperimentOutlined />}
loading={testing}
onClick={handleTest}
>
</Button>
<Space>
<Button onClick={() => setDrawerVisible(false)}></Button>
<Button type="primary" onClick={handleSave}>
</Button>
</Space>
</div>
}
>
<Form form={form} layout="vertical">
<Form.Item
name="name"
label="名称"
rules={[{ required: true, message: '请输入名称' }]}
>
<Input placeholder="My API Data Source" />
</Form.Item>
<Form.Item name="description" label="描述">
<Input.TextArea rows={2} placeholder="数据源描述" />
</Form.Item>
<Form.Item
name="source_type"
label="数据源类型"
rules={[{ required: true, message: '请选择类型' }]}
>
<Select>
<Select.Option value="http">HTTP API</Select.Option>
<Select.Option value="api">REST API</Select.Option>
<Select.Option value="database"></Select.Option>
</Select>
</Form.Item>
<Form.Item
name="endpoint"
label="接口地址"
rules={[{ required: true, message: '请输入接口地址' }]}
>
<Input placeholder="https://api.example.com/data" />
</Form.Item>
<Collapse
items={[
{
key: 'auth',
label: '认证配置',
children: (
<>
<Form.Item name="auth_type" label="认证方式">
<Select>
<Select.Option value="none"></Select.Option>
<Select.Option value="bearer">Bearer Token</Select.Option>
<Select.Option value="api_key">API Key</Select.Option>
<Select.Option value="basic">Basic Auth</Select.Option>
</Select>
</Form.Item>
<div>
<Form.Item noStyle shouldUpdate={(_, { auth_type }) => auth_type === 'bearer'}>
{({ getFieldValue }) => {
if (getFieldValue('auth_type') === 'bearer') {
return (
<Form.Item name={['auth_config', 'token']} label="Token">
<Input.Password placeholder="Bearer Token" />
</Form.Item>
)
}
return null
}}
</Form.Item>
<Form.Item noStyle shouldUpdate={(_, { auth_type }) => auth_type === 'api_key'}>
{({ getFieldValue }) => {
if (getFieldValue('auth_type') === 'api_key') {
return (
<>
<Form.Item name={['auth_config', 'key_name']} label="Header名称" initialValue="X-API-Key">
<Input placeholder="X-API-Key" />
</Form.Item>
<Form.Item name={['auth_config', 'api_key']} label="API Key">
<Input.Password placeholder="API Key" />
</Form.Item>
</>
)
}
return null
}}
</Form.Item>
<Form.Item noStyle shouldUpdate={(_, { auth_type }) => auth_type === 'basic'}>
{({ getFieldValue }) => {
if (getFieldValue('auth_type') === 'basic') {
return (
<>
<Form.Item name={['auth_config', 'username']} label="用户名">
<Input placeholder="Username" />
</Form.Item>
<Form.Item name={['auth_config', 'password']} label="密码">
<Input.Password placeholder="Password" />
</Form.Item>
</>
)
}
return null
}}
</Form.Item>
</div>
</>
),
},
]}
/>
<Collapse
items={[
{
key: 'headers',
label: '请求头',
children: (
<Form.List name="headers">
{(fields, { add, remove }) => (
<>
{fields.map(({ key, name, ...restField }) => (
<Space key={key} style={{ display: 'flex', marginBottom: 8 }} align="baseline">
<Form.Item {...restField} name={[name, 'key']} rules={[{ required: true, message: 'Header键' }]}>
<Input placeholder="Content-Type" />
</Form.Item>
<Form.Item {...restField} name={[name, 'value']} rules={[{ required: true, message: 'Header值' }]}>
<Input placeholder="application/json" />
</Form.Item>
<Button type="link" danger onClick={() => remove(name)}></Button>
</Space>
))}
<Button type="dashed" onClick={() => add()} block>
</Button>
</>
)}
</Form.List>
),
},
]}
/>
<Collapse
items={[
{
key: 'config',
label: '高级配置',
children: (
<>
<Form.Item name={['config', 'timeout']} label="超时时间(秒)">
<InputNumber min={1} max={300} style={{ width: '100%' }} />
</Form.Item>
<Form.Item name={['config', 'retry']} label="重试次数">
<InputNumber min={0} max={10} style={{ width: '100%' }} />
</Form.Item>
</>
),
},
]}
/>
{testResult && (
<div style={{
marginTop: 16,
padding: 16,
background: testResult.success ? '#f6ffed' : '#fff2f0',
border: `1px solid ${testResult.success ? '#b7eb8f' : '#ffa39e'}`,
borderRadius: 4,
}}>
<div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 8 }}>
{testResult.success ? (
<CheckCircleOutlined style={{ color: '#52c41a', fontSize: 18 }} />
) : (
<CloseCircleOutlined style={{ color: '#ff4d4f', fontSize: 18 }} />
)}
<span style={{ fontWeight: 'bold' }}>
{testResult.success ? '连接成功' : '连接失败'}
</span>
</div>
{testResult.status_code && <div>: {testResult.status_code}</div>}
{testResult.response_time_ms && <div>: {testResult.response_time_ms.toFixed(0)}ms</div>}
{testResult.error && <div style={{ color: '#ff4d4f' }}>: {testResult.error}</div>}
{testResult.data_preview && (
<div style={{ marginTop: 8, fontSize: 12, color: '#666' }}>
: {testResult.data_preview}
</div>
)}
</div>
)}
</Form>
</Drawer>
<Drawer
title="查看数据源"
width={600}
open={viewDrawerVisible}
onClose={() => {
setViewDrawerVisible(false)
setViewingSource(null)
}}
footer={
<div style={{ display: 'flex', justifyContent: 'space-between' }}>
<Popconfirm
title={`确定删除"${viewingSource?.name}"的所有数据?`}
onConfirm={handleClearDataFromDrawer}
okText="确定"
cancelText="取消"
>
<Button danger icon={<ClearOutlined />}>
</Button>
</Popconfirm>
<Space>
<Button
icon={<ExperimentOutlined />}
loading={testing}
onClick={handleTest}
>
</Button>
<Button onClick={() => setViewDrawerVisible(false)}></Button>
<Button
type="primary"
icon={<SyncOutlined />}
onClick={handleUpdateSource}
>
</Button>
</Space>
</div>
}
>
{viewingSource && (
<Form layout="vertical">
<Form.Item label="名称">
<Input value={viewingSource.name} disabled />
</Form.Item>
<Form.Item label="数据量">
<Input value={`${recordCount}`} disabled />
</Form.Item>
<Form.Item label="采集器">
<Input value={viewingSource.collector_class} disabled />
</Form.Item>
<Form.Item label="模块">
<Input value={viewingSource.module} disabled />
</Form.Item>
<Form.Item label="优先级">
<Input value={viewingSource.priority} disabled />
</Form.Item>
<Form.Item label="频率">
<Input value={viewingSource.frequency} disabled />
</Form.Item>
<Collapse
items={[
{
key: 'auth',
label: '认证配置',
children: (
<Form.Item label="认证方式">
<Input value={viewingSource.auth_type || 'none'} disabled />
</Form.Item>
),
},
{
key: 'headers',
label: '请求头',
children: viewingSource.headers && Object.keys(viewingSource.headers).length > 0 ? (
<pre style={{ background: '#f5f5f5', padding: 12, borderRadius: 4, overflow: 'auto' }}>
{JSON.stringify(viewingSource.headers, null, 2)}
</pre>
) : (
<div style={{ color: '#999' }}></div>
),
},
{
key: 'config',
label: '高级配置',
children: viewingSource.config && Object.keys(viewingSource.config).length > 0 ? (
<pre style={{ background: '#f5f5f5', padding: 12, borderRadius: 4, overflow: 'auto' }}>
{JSON.stringify(viewingSource.config, null, 2)}
</pre>
) : (
<div style={{ color: '#999' }}></div>
),
},
]}
/>
</Form>
)}
</Drawer>
</AppLayout>
)
}
export default DataSources