Refine data management and collection workflows
This commit is contained in:
linkong
@@ -7,7 +7,7 @@ import {
|
||||
PlayCircleOutlined, PauseCircleOutlined, PlusOutlined,
|
||||
EditOutlined, DeleteOutlined, ApiOutlined,
|
||||
CheckCircleOutlined, CloseCircleOutlined, ExperimentOutlined,
|
||||
SyncOutlined, ClearOutlined
|
||||
SyncOutlined, ClearOutlined, CopyOutlined
|
||||
} from '@ant-design/icons'
|
||||
import axios from 'axios'
|
||||
import AppLayout from '../../components/AppLayout/AppLayout'
|
||||
@@ -18,16 +18,28 @@ interface BuiltInDataSource {
|
||||
module: string
|
||||
priority: string
|
||||
frequency: string
|
||||
endpoint?: string
|
||||
is_active: boolean
|
||||
collector_class: string
|
||||
last_run: string | null
|
||||
is_running: boolean
|
||||
task_id: number | null
|
||||
progress: number | null
|
||||
phase?: string | null
|
||||
records_processed: number | null
|
||||
total_records: number | null
|
||||
}
|
||||
|
||||
interface TaskTrackerState {
|
||||
task_id: number | null
|
||||
is_running: boolean
|
||||
progress: number
|
||||
phase: string | null
|
||||
status?: string | null
|
||||
records_processed?: number | null
|
||||
total_records?: number | null
|
||||
}
|
||||
|
||||
interface CustomDataSource {
|
||||
id: number
|
||||
name: string
|
||||
@@ -89,7 +101,7 @@ function DataSources() {
|
||||
}
|
||||
}
|
||||
|
||||
const [taskProgress, setTaskProgress] = useState<Record<number, { progress: number; is_running: boolean }>>({})
|
||||
const [taskProgress, setTaskProgress] = useState<Record<number, TaskTrackerState>>({})
|
||||
|
||||
useEffect(() => {
|
||||
fetchData()
|
||||
@@ -118,80 +130,85 @@ function DataSources() {
|
||||
}, [activeTab, builtInSources.length, customSources.length])
|
||||
|
||||
useEffect(() => {
|
||||
const runningSources = builtInSources.filter(s => s.is_running)
|
||||
if (runningSources.length === 0) return
|
||||
const trackedSources = builtInSources.filter((source) => {
|
||||
const trackedTask = taskProgress[source.id]
|
||||
return Boolean((trackedTask?.task_id ?? source.task_id) && (trackedTask?.is_running ?? source.is_running))
|
||||
})
|
||||
|
||||
if (trackedSources.length === 0) return
|
||||
|
||||
const interval = setInterval(async () => {
|
||||
const progressMap: Record<number, { progress: number; is_running: boolean }> = {}
|
||||
|
||||
const updates: Record<number, TaskTrackerState> = {}
|
||||
|
||||
await Promise.all(
|
||||
runningSources.map(async (source) => {
|
||||
trackedSources.map(async (source) => {
|
||||
const trackedTaskId = taskProgress[source.id]?.task_id ?? source.task_id
|
||||
if (!trackedTaskId) return
|
||||
|
||||
try {
|
||||
const res = await axios.get(`/api/v1/datasources/${source.id}/task-status`)
|
||||
progressMap[source.id] = {
|
||||
const res = await axios.get(`/api/v1/datasources/${source.id}/task-status`, {
|
||||
params: { task_id: trackedTaskId },
|
||||
})
|
||||
updates[source.id] = {
|
||||
task_id: res.data.task_id ?? trackedTaskId,
|
||||
progress: res.data.progress || 0,
|
||||
is_running: res.data.is_running
|
||||
is_running: !!res.data.is_running,
|
||||
phase: res.data.phase || null,
|
||||
status: res.data.status || null,
|
||||
records_processed: res.data.records_processed,
|
||||
total_records: res.data.total_records,
|
||||
}
|
||||
} catch {
|
||||
progressMap[source.id] = { progress: 0, is_running: false }
|
||||
updates[source.id] = {
|
||||
task_id: trackedTaskId,
|
||||
progress: 0,
|
||||
is_running: false,
|
||||
phase: 'failed',
|
||||
status: 'failed',
|
||||
}
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
setTaskProgress(prev => ({ ...prev, ...progressMap }))
|
||||
|
||||
setTaskProgress((prev) => {
|
||||
const next = { ...prev, ...updates }
|
||||
for (const [sourceId, state] of Object.entries(updates)) {
|
||||
if (!state.is_running && state.status !== 'running') {
|
||||
delete next[Number(sourceId)]
|
||||
}
|
||||
}
|
||||
return next
|
||||
})
|
||||
|
||||
if (Object.values(updates).some((state) => !state.is_running)) {
|
||||
fetchData()
|
||||
}
|
||||
}, 2000)
|
||||
|
||||
return () => clearInterval(interval)
|
||||
}, [builtInSources.map(s => s.id).join(',')])
|
||||
}, [builtInSources, taskProgress])
|
||||
|
||||
const handleTrigger = async (id: number) => {
|
||||
try {
|
||||
await axios.post(`/api/v1/datasources/${id}/trigger`)
|
||||
const res = await axios.post(`/api/v1/datasources/${id}/trigger`)
|
||||
message.success('任务已触发')
|
||||
// Trigger polling immediately
|
||||
setTaskProgress(prev => ({ ...prev, [id]: { progress: 0, is_running: true } }))
|
||||
// Also refresh data
|
||||
setTaskProgress(prev => ({
|
||||
...prev,
|
||||
[id]: {
|
||||
task_id: res.data.task_id ?? null,
|
||||
progress: 0,
|
||||
is_running: true,
|
||||
phase: 'queued',
|
||||
status: 'running',
|
||||
},
|
||||
}))
|
||||
fetchData()
|
||||
// Also fetch the running task status
|
||||
pollTaskStatus(id)
|
||||
} catch (error: unknown) {
|
||||
const err = error as { response?: { data?: { detail?: string } } }
|
||||
message.error(err.response?.data?.detail || '触发失败')
|
||||
}
|
||||
}
|
||||
|
||||
const pollTaskStatus = async (sourceId: number) => {
|
||||
const poll = async () => {
|
||||
try {
|
||||
const res = await axios.get(`/api/v1/datasources/${sourceId}/task-status`)
|
||||
const data = res.data
|
||||
|
||||
setTaskProgress(prev => ({ ...prev, [sourceId]: {
|
||||
progress: data.progress || 0,
|
||||
is_running: data.is_running
|
||||
} }))
|
||||
|
||||
// Keep polling while running
|
||||
if (data.is_running) {
|
||||
setTimeout(poll, 2000)
|
||||
} else {
|
||||
// Task completed - refresh data and clear this source from progress
|
||||
setTimeout(() => {
|
||||
setTaskProgress(prev => {
|
||||
const newState = { ...prev }
|
||||
delete newState[sourceId]
|
||||
return newState
|
||||
})
|
||||
}, 1000)
|
||||
fetchData()
|
||||
}
|
||||
} catch {
|
||||
// Stop polling on error
|
||||
}
|
||||
}
|
||||
poll()
|
||||
}
|
||||
|
||||
const handleToggle = async (id: number, current: boolean) => {
|
||||
const endpoint = current ? 'disable' : 'enable'
|
||||
try {
|
||||
@@ -229,7 +246,7 @@ function DataSources() {
|
||||
name: data.name,
|
||||
description: null,
|
||||
source_type: data.collector_class,
|
||||
endpoint: '',
|
||||
endpoint: data.endpoint || '',
|
||||
auth_type: 'none',
|
||||
headers: {},
|
||||
config: {},
|
||||
@@ -340,6 +357,27 @@ function DataSources() {
|
||||
setTestResult(null)
|
||||
}
|
||||
|
||||
const handleCopyLink = async (value: string, successText: string) => {
|
||||
try {
|
||||
if (navigator.clipboard?.writeText) {
|
||||
await navigator.clipboard.writeText(value)
|
||||
} else {
|
||||
const textArea = document.createElement('textarea')
|
||||
textArea.value = value
|
||||
textArea.style.position = 'fixed'
|
||||
textArea.style.opacity = '0'
|
||||
document.body.appendChild(textArea)
|
||||
textArea.focus()
|
||||
textArea.select()
|
||||
document.execCommand('copy')
|
||||
document.body.removeChild(textArea)
|
||||
}
|
||||
message.success(successText)
|
||||
} catch {
|
||||
message.error('复制失败，请手动复制')
|
||||
}
|
||||
}
|
||||
|
||||
const builtinColumns = [
|
||||
{ title: 'ID', dataIndex: 'id', key: 'id', width: 60, fixed: 'left' as const },
|
||||
{
|
||||
@@ -374,15 +412,31 @@ function DataSources() {
|
||||
title: '状态',
|
||||
dataIndex: 'is_active',
|
||||
key: 'is_active',
|
||||
width: 100,
|
||||
width: 180,
|
||||
render: (_: unknown, record: BuiltInDataSource) => {
|
||||
const progress = taskProgress[record.id]
|
||||
if (progress?.is_running || record.is_running) {
|
||||
const pct = progress?.progress ?? record.progress ?? 0
|
||||
const taskState = taskProgress[record.id]
|
||||
const isTaskRunning = taskState?.is_running || record.is_running
|
||||
|
||||
const phaseLabelMap: Record<string, string> = {
|
||||
queued: '排队中',
|
||||
fetching: '抓取中',
|
||||
transforming: '处理中',
|
||||
saving: '保存中',
|
||||
completed: '已完成',
|
||||
failed: '失败',
|
||||
}
|
||||
|
||||
if (isTaskRunning) {
|
||||
const pct = taskState?.progress ?? record.progress ?? 0
|
||||
const phase = taskState?.phase || record.phase || 'queued'
|
||||
return (
|
||||
<Tag color="blue">
|
||||
采集中 {Math.round(pct)}%
|
||||
</Tag>
|
||||
<Space size={6} wrap>
|
||||
<Tag color={record.is_active ? 'green' : 'red'}>{record.is_active ? '运行中' : '已暂停'}</Tag>
|
||||
<Tag color="processing">
|
||||
{phaseLabelMap[phase] || phase}
|
||||
{pct > 0 ? ` ${Math.round(pct)}%` : ''}
|
||||
</Tag>
|
||||
</Space>
|
||||
)
|
||||
}
|
||||
return <Tag color={record.is_active ? 'green' : 'red'}>{record.is_active ? '运行中' : '已暂停'}</Tag>
|
||||
@@ -420,6 +474,22 @@ function DataSources() {
|
||||
{ title: 'ID', dataIndex: 'id', key: 'id', width: 60, fixed: 'left' as const },
|
||||
{ title: '名称', dataIndex: 'name', key: 'name', width: 150, ellipsis: true },
|
||||
{ title: '类型', dataIndex: 'source_type', key: 'source_type', width: 100 },
|
||||
{
|
||||
title: 'API链接',
|
||||
dataIndex: 'endpoint',
|
||||
key: 'endpoint',
|
||||
width: 280,
|
||||
ellipsis: true,
|
||||
render: (endpoint: string) => (
|
||||
endpoint ? (
|
||||
<Tooltip title={endpoint}>
|
||||
<a href={endpoint} target="_blank" rel="noreferrer">
|
||||
{endpoint}
|
||||
</a>
|
||||
</Tooltip>
|
||||
) : '-'
|
||||
),
|
||||
},
|
||||
{
|
||||
title: '状态',
|
||||
dataIndex: 'is_active',
|
||||
@@ -477,7 +547,6 @@ function DataSources() {
|
||||
scroll={{ x: 800, y: builtinTableHeight }}
|
||||
tableLayout="fixed"
|
||||
size="small"
|
||||
virtual
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
@@ -509,10 +578,9 @@ function DataSources() {
|
||||
rowKey="id"
|
||||
loading={loading}
|
||||
pagination={false}
|
||||
scroll={{ x: 600, y: customTableHeight }}
|
||||
scroll={{ x: 900, y: customTableHeight }}
|
||||
tableLayout="fixed"
|
||||
size="small"
|
||||
virtual
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
@@ -811,6 +879,19 @@ function DataSources() {
|
||||
<Input value={viewingSource.frequency} disabled />
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item label="采集源 API 链接">
|
||||
<Space.Compact style={{ width: '100%' }}>
|
||||
<Input value={viewingSource.endpoint || '-'} readOnly />
|
||||
<Tooltip title={viewingSource.endpoint ? '复制采集源 API 链接' : '当前没有可复制的采集源 API 链接'}>
|
||||
<Button
|
||||
disabled={!viewingSource.endpoint}
|
||||
icon={<CopyOutlined />}
|
||||
onClick={() => viewingSource.endpoint && handleCopyLink(viewingSource.endpoint, '采集源 API 链接已复制')}
|
||||
/>
|
||||
</Tooltip>
|
||||
</Space.Compact>
|
||||
</Form.Item>
|
||||
|
||||
<Collapse
|
||||
items={[
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user