feat(backend): Add cable graph service and data collectors

## Changelog ### New Features #### Cable Graph Service - Add cable_graph.py for finding shortest path between landing points - Implement haversine distance calculation for great circle distances - Support for dateline crossing (longitude normalization) - NetworkX-based graph for optimal path finding #### Data Collectors - Add ArcGISCableCollector for fetching submarine cable data from ArcGIS GeoJSON API - Add FAOLandingPointCollector for fetching landing point data from FAO CSV API ### Backend Changes #### API Updates - auth.py: Update authentication logic - datasources.py: Add datasource endpoints and management - visualization.py: Add visualization API endpoints - config.py: Update configuration settings - security.py: Improve security settings #### Models & Schemas - task.py: Update task model with new fields - token.py: Update token schema #### Services - collectors/base.py: Improve base collector with better error handling - collectors/__init__.py: Register new collectors - scheduler.py: Update scheduler logic - tasks/scheduler.py: Add task scheduling ### Frontend Changes - AppLayout.tsx: Improve layout component - index.css: Add global styles - DataSources.tsx: Enhance data sources management page - vite.config.ts: Add Vite configuration for earth module
2026-03-11 16:38:49 +08:00
parent 6cb4398f3a
commit aaae6a53c3
18 changed files with 990 additions and 146 deletions
--- a/frontend/src/components/AppLayout/AppLayout.tsx
+++ b/frontend/src/components/AppLayout/AppLayout.tsx
@@ -1,4 +1,4 @@
-import { ReactNode } from 'react'
+import { ReactNode, useState } from 'react'
 import { Layout, Menu, Typography, Button } from 'antd'
 import {
  DashboardOutlined,
@@ -7,6 +7,7 @@ import {
  SettingOutlined,
  BarChartOutlined,
  MenuUnfoldOutlined,
+  MenuFoldOutlined,
 } from '@ant-design/icons'
 import { Link, useLocation } from 'react-router-dom'
 import { useAuthStore } from '../../stores/auth'
@@ -21,6 +22,7 @@ interface AppLayoutProps {
 function AppLayout({ children }: AppLayoutProps) {
  const location = useLocation()
  const { user, logout } = useAuthStore()
+  const [collapsed, setCollapsed] = useState(false)

  const menuItems = [
    { key: '/', icon: <DashboardOutlined />, label: <Link to="/">仪表盘</Link> },
@@ -34,30 +36,18 @@ function AppLayout({ children }: AppLayoutProps) {
    <Layout className="dashboard-layout">
      <Sider 
        width={240} 
+        collapsedWidth={80}
        collapsible 
-        collapsed={false} 
-        onCollapse={(collapsed) => {
-          const sider = document.querySelector('.dashboard-sider') as HTMLElement
-          if (sider) {
-            sider.style.width = collapsed ? '80px' : '240px'
-            sider.style.minWidth = collapsed ? '80px' : '240px'
-            sider.style.maxWidth = collapsed ? '80px' : '240px'
-          }
-        }}
+        collapsed={collapsed} 
+        onCollapse={setCollapsed}
        className="dashboard-sider"
-        trigger={null}
-        breakpoint="lg"
-        onBreakpoint={(broken) => {
-          const sider = document.querySelector('.dashboard-sider') as HTMLElement
-          if (sider) {
-            sider.style.width = broken ? '80px' : '240px'
-            sider.style.minWidth = broken ? '80px' : '240px'
-            sider.style.maxWidth = broken ? '80px' : '240px'
-          }
-        }}
      >
        <div style={{ height: 64, display: 'flex', alignItems: 'center', justifyContent: 'center' }}>
-          <Text strong style={{ color: 'white', fontSize: 18 }}>智能星球</Text>
+          {collapsed ? (
+            <Text strong style={{ color: 'white', fontSize: 20 }}>🌏</Text>
+          ) : (
+            <Text strong style={{ color: 'white', fontSize: 18 }}>智能星球</Text>
+          )}
        </div>
        <Menu 
          theme="dark" 
@@ -70,17 +60,8 @@ function AppLayout({ children }: AppLayoutProps) {
        <Header className="dashboard-header" style={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between', padding: '0 16px' }}>
          <Button 
            type="text" 
-            icon={<MenuUnfoldOutlined />} 
-            onClick={() => {
-              const sider = document.querySelector('.ant-layout-sider') as HTMLElement
-              if (sider) {
-                const currentWidth = sider.style.width || '240px'
-                const isCollapsed = currentWidth === '80px'
-                sider.style.width = isCollapsed ? '240px' : '80px'
-                sider.style.minWidth = isCollapsed ? '240px' : '80px'
-                sider.style.maxWidth = isCollapsed ? '240px' : '80px'
-              }
-            }}
+            icon={collapsed ? <MenuUnfoldOutlined /> : <MenuFoldOutlined />} 
+            onClick={() => setCollapsed(!collapsed)}
            style={{ fontSize: 16 }}
          />
          <Text strong>欢迎, {user?.username}</Text>
--- a/frontend/src/index.css
+++ b/frontend/src/index.css
@@ -32,6 +32,10 @@ body {
  background: #001529 !important;
 }

+.ant-layout-sider-trigger {
+  display: none !important;
+}
+
 .dashboard-header {
  background: white;
  padding: 0 24px;
--- a/frontend/src/pages/DataSources/DataSources.tsx
+++ b/frontend/src/pages/DataSources/DataSources.tsx
@@ -7,7 +7,7 @@ import {
  PlayCircleOutlined, PauseCircleOutlined, PlusOutlined,
  EditOutlined, DeleteOutlined, ApiOutlined,
  CheckCircleOutlined, CloseCircleOutlined, ExperimentOutlined,
-  SyncOutlined
+  SyncOutlined, ClearOutlined
 } from '@ant-design/icons'
 import axios from 'axios'
 import AppLayout from '../../components/AppLayout/AppLayout'
@@ -20,6 +20,12 @@ interface BuiltInDataSource {
  frequency: string
  is_active: boolean
  collector_class: string
+  last_run: string | null
+  is_running: boolean
+  task_id: number | null
+  progress: number | null
+  records_processed: number | null
+  total_records: number | null
 }

 interface CustomDataSource {
@@ -58,6 +64,7 @@ function DataSources() {
  const [viewDrawerVisible, setViewDrawerVisible] = useState(false)
  const [editingConfig, setEditingConfig] = useState<CustomDataSource | null>(null)
  const [viewingSource, setViewingSource] = useState<ViewDataSource | null>(null)
+  const [recordCount, setRecordCount] = useState<number>(0)
  const [testing, setTesting] = useState(false)
  const [testResult, setTestResult] = useState<any>(null)
  const [form] = Form.useForm()
@@ -78,20 +85,87 @@ function DataSources() {
    }
  }

+  const [taskProgress, setTaskProgress] = useState<Record<number, { progress: number; is_running: boolean }>>({})
+
  useEffect(() => {
    fetchData()
  }, [])

+  useEffect(() => {
+    const runningSources = builtInSources.filter(s => s.is_running)
+    if (runningSources.length === 0) return
+
+    const interval = setInterval(async () => {
+      const progressMap: Record<number, { progress: number; is_running: boolean }> = {}
+      
+      await Promise.all(
+        runningSources.map(async (source) => {
+          try {
+            const res = await axios.get(`/api/v1/datasources/${source.id}/task-status`)
+            progressMap[source.id] = {
+              progress: res.data.progress || 0,
+              is_running: res.data.is_running
+            }
+          } catch {
+            progressMap[source.id] = { progress: 0, is_running: false }
+          }
+        })
+      )
+      
+      setTaskProgress(prev => ({ ...prev, ...progressMap }))
+    }, 2000)
+
+    return () => clearInterval(interval)
+  }, [builtInSources.map(s => s.id).join(',')])
+
  const handleTrigger = async (id: number) => {
    try {
      await axios.post(`/api/v1/datasources/${id}/trigger`)
      message.success('任务已触发')
+      // Trigger polling immediately
+      setTaskProgress(prev => ({ ...prev, [id]: { progress: 0, is_running: true } }))
+      // Also refresh data
+      fetchData()
+      // Also fetch the running task status
+      pollTaskStatus(id)
    } catch (error: unknown) {
      const err = error as { response?: { data?: { detail?: string } } }
      message.error(err.response?.data?.detail || '触发失败')
    }
  }

+  const pollTaskStatus = async (sourceId: number) => {
+    const poll = async () => {
+      try {
+        const res = await axios.get(`/api/v1/datasources/${sourceId}/task-status`)
+        const data = res.data
+        
+        setTaskProgress(prev => ({ ...prev, [sourceId]: { 
+          progress: data.progress || 0, 
+          is_running: data.is_running 
+        } }))
+        
+        // Keep polling while running
+        if (data.is_running) {
+          setTimeout(poll, 2000)
+        } else {
+          // Task completed - refresh data and clear this source from progress
+          setTimeout(() => {
+            setTaskProgress(prev => {
+              const newState = { ...prev }
+              delete newState[sourceId]
+              return newState
+            })
+          }, 1000)
+          fetchData()
+        }
+      } catch {
+        // Stop polling on error
+      }
+    }
+    poll()
+  }
+
  const handleToggle = async (id: number, current: boolean) => {
    const endpoint = current ? 'disable' : 'enable'
    try {
@@ -104,9 +178,25 @@ function DataSources() {
    }
  }

+  const handleClearDataFromDrawer = async () => {
+    if (!viewingSource) return
+    try {
+      const res = await axios.delete(`/api/v1/datasources/${viewingSource.id}/data`)
+      message.success(res.data.message || '数据已删除')
+      setViewDrawerVisible(false)
+      fetchData()
+    } catch (error: unknown) {
+      const err = error as { response?: { data?: { detail?: string } } }
+      message.error(err.response?.data?.detail || '删除数据失败')
+    }
+  }
+
  const handleViewSource = async (source: BuiltInDataSource) => {
    try {
-      const res = await axios.get(`/api/v1/datasources/${source.id}`)
+      const [res, statsRes] = await Promise.all([
+        axios.get(`/api/v1/datasources/${source.id}`),
+        axios.get(`/api/v1/datasources/${source.id}/stats`)
+      ])
      const data = res.data
      setViewingSource({
        id: data.id,
@@ -122,6 +212,7 @@ function DataSources() {
        priority: data.priority,
        frequency: data.frequency,
      })
+      setRecordCount(statsRes.data.total_records || 0)
      setViewDrawerVisible(true)
    } catch (error) {
      message.error('获取数据源信息失败')
@@ -224,40 +315,63 @@ function DataSources() {
  }

  const builtinColumns = [
-    { title: 'ID', dataIndex: 'id', key: 'id', width: 60 },
+    { title: 'ID', dataIndex: 'id', key: 'id', width: 60, fixed: 'left' as const },
    {
      title: '名称',
      dataIndex: 'name',
      key: 'name',
+      width: 180,
+      ellipsis: true,
      render: (name: string, record: BuiltInDataSource) => (
        <Button type="link" onClick={() => handleViewSource(record)}>
          {name}
        </Button>
      ),
    },
-    { title: '模块', dataIndex: 'module', key: 'module' },
+    { title: '模块', dataIndex: 'module', key: 'module', width: 80 },
    {
      title: '优先级',
      dataIndex: 'priority',
      key: 'priority',
+      width: 80,
      render: (p: string) => <Tag color={p === 'P0' ? 'red' : 'orange'}>{p}</Tag>,
    },
-    { title: '频率', dataIndex: 'frequency', key: 'frequency' },
+    { title: '频率', dataIndex: 'frequency', key: 'frequency', width: 80 },
+    {
+      title: '最近采集',
+      dataIndex: 'last_run',
+      key: 'last_run',
+      width: 140,
+      render: (lastRun: string | null) => lastRun || '-',
+    },
    {
      title: '状态',
      dataIndex: 'is_active',
      key: 'is_active',
-      render: (active: boolean) => (
-        <Tag color={active ? 'green' : 'red'}>{active ? '运行中' : '已暂停'}</Tag>
-      ),
+      width: 100,
+      render: (_: unknown, record: BuiltInDataSource) => {
+        const progress = taskProgress[record.id]
+        if (progress?.is_running || record.is_running) {
+          const pct = progress?.progress ?? record.progress ?? 0
+          return (
+            <Tag color="blue">
+              采集中 {Math.round(pct)}%
+            </Tag>
+          )
+        }
+        return <Tag color={record.is_active ? 'green' : 'red'}>{record.is_active ? '运行中' : '已暂停'}</Tag>
+      },
    },
    {
      title: '操作',
      key: 'action',
+      width: 200,
+      fixed: 'right' as const,
      render: (_: unknown, record: BuiltInDataSource) => (
-        <Space>
+        <Space size="small">
          <Button
            type="link"
+            size="small"
            icon={<SyncOutlined />}
            onClick={() => handleTrigger(record.id)}
          >
@@ -265,6 +379,7 @@ function DataSources() {
          </Button>
          <Button
            type="link"
+            size="small"
            icon={record.is_active ? <PauseCircleOutlined /> : <PlayCircleOutlined />}
            onClick={() => handleToggle(record.id, record.is_active)}
          >
@@ -276,29 +391,33 @@ function DataSources() {
  ]

  const customColumns = [
-    { title: 'ID', dataIndex: 'id', key: 'id', width: 60 },
-    { title: '名称', dataIndex: 'name', key: 'name' },
-    { title: '类型', dataIndex: 'source_type', key: 'source_type' },
+    { title: 'ID', dataIndex: 'id', key: 'id', width: 60, fixed: 'left' as const },
+    { title: '名称', dataIndex: 'name', key: 'name', width: 150, ellipsis: true },
+    { title: '类型', dataIndex: 'source_type', key: 'source_type', width: 100 },
    {
      title: '状态',
      dataIndex: 'is_active',
      key: 'is_active',
+      width: 80,
      render: (active: boolean) => (
        <Tag color={active ? 'green' : 'red'}>{active ? '启用' : '禁用'}</Tag>
      ),
    },
-    { title: '创建时间', dataIndex: 'created_at', key: 'created_at' },
+    { title: '创建时间', dataIndex: 'created_at', key: 'created_at', width: 160 },
    {
      title: '操作',
      key: 'action',
+      width: 150,
+      fixed: 'right' as const,
      render: (_: unknown, record: CustomDataSource) => (
-        <Space>
+        <Space size="small">
          <Tooltip title="编辑">
-            <Button type="link" icon={<EditOutlined />} onClick={() => openDrawer(record)} />
+            <Button type="link" size="small" icon={<EditOutlined />} onClick={() => openDrawer(record)} />
          </Tooltip>
          <Tooltip title={record.is_active ? '禁用' : '启用'}>
            <Button
              type="link"
+              size="small"
              icon={record.is_active ? <PauseCircleOutlined /> : <PlayCircleOutlined />}
              onClick={() => handleToggleCustom(record.id, record.is_active)}
            />
@@ -308,7 +427,7 @@ function DataSources() {
            onConfirm={() => handleDelete(record.id)}
          >
            <Tooltip title="删除">
-              <Button type="link" danger icon={<DeleteOutlined />} />
+              <Button type="link" size="small" danger icon={<DeleteOutlined />} />
            </Tooltip>
          </Popconfirm>
        </Space>
@@ -327,8 +446,9 @@ function DataSources() {
          rowKey="id"
          loading={loading}
          pagination={false}
-          scroll={{ x: 'max-content' }}
+          scroll={{ x: 800, y: 'auto' }}
          tableLayout="fixed"
+          size="small"
        />
      ),
    },
@@ -355,8 +475,9 @@ function DataSources() {
              rowKey="id"
              loading={loading}
              pagination={false}
-              scroll={{ x: 'max-content' }}
+              scroll={{ x: 600, y: 'auto' }}
              tableLayout="fixed"
+              size="small"
            />
          )}
        </>
@@ -590,14 +711,24 @@ function DataSources() {
        }}
        footer={
          <div style={{ display: 'flex', justifyContent: 'space-between' }}>
-            <Button
-              icon={<ExperimentOutlined />}
-              loading={testing}
-              onClick={handleTest}
+            <Popconfirm
+              title={`确定删除"${viewingSource?.name}"的所有数据?`}
+              onConfirm={handleClearDataFromDrawer}
+              okText="确定"
+              cancelText="取消"
            >
-              测试连接
-            </Button>
+              <Button danger icon={<ClearOutlined />}>
+                删除数据
+              </Button>
+            </Popconfirm>
            <Space>
+              <Button
+                icon={<ExperimentOutlined />}
+                loading={testing}
+                onClick={handleTest}
+              >
+                测试连接
+              </Button>
              <Button onClick={() => setViewDrawerVisible(false)}>关闭</Button>
              <Button
                type="primary"
@@ -616,6 +747,10 @@ function DataSources() {
              <Input value={viewingSource.name} disabled />
            </Form.Item>

+            <Form.Item label="数据量">
+              <Input value={`${recordCount} 条`} disabled />
+            </Form.Item>
+
            <Form.Item label="采集器">
              <Input value={viewingSource.collector_class} disabled />
            </Form.Item>
--- a/frontend/vite.config.ts
+++ b/frontend/vite.config.ts
@@ -1,5 +1,6 @@
 import { defineConfig } from 'vite'
 import react from '@vitejs/plugin-react'
+import path from 'path'

 export default defineConfig({
  plugins: [react()],
@@ -18,5 +19,14 @@ export default defineConfig({
        secure: false,
      },
    },
+    fs: {
+      allow: ['..'],
+    },
+  },
+  publicDir: 'public',
+  resolve: {
+    alias: {
+      '@': path.resolve(__dirname, './src'),
+    },
  },
 })