fix: add controlled service restart console

This commit is contained in:
linkong
2026-03-31 17:56:31 +08:00
parent e384318b50
commit e903723877
14 changed files with 1350 additions and 13 deletions

View File

@@ -1,15 +1,17 @@
import { useEffect, useState } from 'react'
import { Card, Row, Col, Statistic, Typography, Button, Tag, Spin, Space } from 'antd'
import { Card, Row, Col, Statistic, Typography, Button, Tag, Spin, Space, Modal, Alert, Select } from 'antd'
import {
DatabaseOutlined,
BarChartOutlined,
AlertOutlined,
GlobalOutlined,
PoweroffOutlined,
WifiOutlined,
DisconnectOutlined,
ReloadOutlined,
} from '@ant-design/icons'
import { Link } from 'react-router-dom'
import axios from 'axios'
import { useAuthStore } from '../../stores/auth'
import AppLayout from '../../components/AppLayout/AppLayout'
import { formatDateTimeZhCN } from '../../utils/datetime'
@@ -29,12 +31,91 @@ interface Stats {
}
}
interface RestartTask {
task_id: string
action: string
status: string
stage: string
message: string
created_at: string
updated_at: string
}
interface RestartTaskLogs {
task_id: string
lines: string[]
}
type RestartAction = 'restart-backend' | 'restart-database' | 'restart-system'
type RestartStage = 'confirming' | 'waiting_for_shutdown' | 'waiting_for_recovery' | 'recovered' | 'failed' | 'timeout'
const RESTART_ACTION_OPTIONS: Array<{ value: RestartAction; label: string; description: string; command: string }> = [
{
value: 'restart-backend',
label: '重启服务器',
description: '只重启后端服务,页面通常会短暂失联后自动恢复。',
command: './planet.sh restart -b',
},
{
value: 'restart-database',
label: '重启数据库',
description: '重启 PostgreSQL 和 Redis 容器,前端页面保持在线。',
command: './planet.sh restart -d',
},
{
value: 'restart-system',
label: '完全重启',
description: '重启前后端和相关服务,页面会短暂不可用,恢复后自动刷新。',
command: './planet.sh restart',
},
]
const RESTART_GUIDE_LINES: Record<RestartAction, string[]> = {
'restart-backend': [
'[ctl] preparing backend restart task',
'[ctl] handing restart to detached runner',
'[ctl] waiting for backend health recovery',
],
'restart-database': [
'[ctl] preparing database restart task',
'[ctl] restarting PostgreSQL and Redis containers',
'[ctl] waiting for containers to settle',
],
'restart-system': [
'[ctl] preparing full system restart',
'[ctl] notifying operator that frontend may disconnect',
'[ctl] stopping frontend and backend services',
'[ctl] restarting platform services',
'[ctl] polling for frontend re-entry window',
],
}
function getRestartConfirmMessage(action: RestartAction): string {
if (action === 'restart-database') {
return '将重启 PostgreSQL 和 Redis页面通常保持在线但相关请求可能短暂波动。'
}
if (action === 'restart-system') {
return '将完全重启前后端和相关服务,页面会短暂不可用,恢复后会自动刷新。'
}
return '将重启后端服务,页面会短暂不可用。'
}
function Dashboard() {
const { token, clearAuth } = useAuthStore()
const { token, clearAuth, user } = useAuthStore()
const [stats, setStats] = useState<Stats | null>(null)
const [loading, setLoading] = useState(true)
const [wsConnected, setWsConnected] = useState(false)
const [error, setError] = useState<string | null>(null)
const [restartModalOpen, setRestartModalOpen] = useState(false)
const [restartSubmitting, setRestartSubmitting] = useState(false)
const [restartAction, setRestartAction] = useState<RestartAction>('restart-backend')
const [restartTaskId, setRestartTaskId] = useState<string | null>(null)
const [restartMessage, setRestartMessage] = useState(getRestartConfirmMessage('restart-backend'))
const [restartStage, setRestartStage] = useState<RestartStage>('confirming')
const [restartLogs, setRestartLogs] = useState<string[]>([])
const [restartStartedAt, setRestartStartedAt] = useState<number | null>(null)
const isSuperAdmin = user?.role === 'super_admin'
const selectedRestartAction = RESTART_ACTION_OPTIONS.find((item) => item.value === restartAction) ?? RESTART_ACTION_OPTIONS[0]
useEffect(() => {
if (!token) return
@@ -115,6 +196,177 @@ function Dashboard() {
window.location.reload()
}
const openRestartModal = () => {
setRestartAction('restart-backend')
setRestartTaskId(null)
setRestartLogs([])
setRestartSubmitting(false)
setRestartStartedAt(null)
setRestartStage('confirming')
setRestartMessage(getRestartConfirmMessage('restart-backend'))
setRestartModalOpen(true)
}
const closeRestartModal = () => {
if (restartSubmitting || restartStage === 'waiting_for_shutdown' || restartStage === 'waiting_for_recovery') {
return
}
setRestartModalOpen(false)
}
const handleRestartAction = async () => {
setRestartSubmitting(true)
setRestartLogs(RESTART_GUIDE_LINES[restartAction].slice(0, restartAction === 'restart-system' ? 3 : 1))
try {
const res = await axios.post<RestartTask>('/api/v1/system/restart-tasks', { action: restartAction })
setRestartTaskId(res.data.task_id)
setRestartStartedAt(Date.now())
setRestartStage('waiting_for_shutdown')
setRestartMessage(
restartAction === 'restart-system'
? '已发送完全重启指令,页面可能暂时失联,恢复后会自动刷新。'
: '已发送重启指令,正在等待服务进入重启流程。'
)
setRestartLogs((current) => [...current, `任务已创建: ${res.data.task_id}`])
} catch (restartError: unknown) {
const err = restartError as { response?: { data?: { detail?: string } } }
setRestartStage('failed')
setRestartMessage(err.response?.data?.detail || '提交重启任务失败')
setRestartLogs((current) => [...current, '提交重启任务失败'])
} finally {
setRestartSubmitting(false)
}
}
useEffect(() => {
if (!restartModalOpen || restartStage !== 'confirming') return
setRestartMessage(getRestartConfirmMessage(restartAction))
}, [restartAction, restartModalOpen, restartStage])
useEffect(() => {
if (!restartModalOpen || !restartTaskId || restartStartedAt === null) return
let cancelled = false
let sawUnhealthy = false
let healthyStreak = 0
let frontendHealthyStreak = 0
let pollTimer: number | null = null
const appendLog = (line: string) => {
setRestartLogs((current) => (current[current.length - 1] === line ? current : [...current, line].slice(-8)))
}
const poll = async () => {
if (cancelled) return
const elapsed = Date.now() - restartStartedAt
if (elapsed > 90_000) {
setRestartStage('timeout')
setRestartMessage('恢复超时,请手动检查后端服务状态。')
appendLog('恢复超时,请手动检查服务状态')
return
}
try {
const taskRes = await axios.get<RestartTask>(`/api/v1/system/restart-tasks/${restartTaskId}`, { timeout: 1500 })
const task = taskRes.data
if (!cancelled && task?.message) {
setRestartMessage(task.message)
}
if (!cancelled && restartAction !== 'restart-system') {
const logsRes = await axios.get<RestartTaskLogs>(`/api/v1/system/restart-tasks/${restartTaskId}/logs`, { timeout: 1500 })
if (logsRes.data.lines.length > 0) {
setRestartLogs(logsRes.data.lines.slice(-8))
}
}
if (!cancelled && typeof task?.stage === 'string') {
if (task.stage === 'healthy') {
setRestartStage('recovered')
setRestartMessage('服务已恢复,正在刷新页面。')
appendLog('后端已恢复,正在刷新页面')
window.setTimeout(() => window.location.reload(), 600)
return
}
if (task.status === 'failed' || task.status === 'timeout') {
setRestartStage(task.status === 'timeout' ? 'timeout' : 'failed')
setRestartMessage(task.message || '重启任务失败')
appendLog(task.message || '重启任务失败')
return
}
}
} catch {
// Backend may be temporarily down during restart; handled by health polling below.
}
if (restartAction === 'restart-system') {
try {
const rootRes = await fetch(`/?restart_probe=${Date.now()}`, { cache: 'no-store' })
if (rootRes.ok) {
frontendHealthyStreak += 1
if (sawUnhealthy && frontendHealthyStreak >= 2) {
setRestartStage('recovered')
setRestartMessage('系统已恢复,正在刷新页面。')
appendLog('[ctl] frontend entrypoint reachable again')
window.setTimeout(() => window.location.reload(), 600)
return
}
} else {
sawUnhealthy = true
frontendHealthyStreak = 0
setRestartStage('waiting_for_recovery')
setRestartMessage('系统正在完全重启,正在等待前端恢复访问。')
appendLog('[ctl] frontend is temporarily unavailable')
}
} catch {
sawUnhealthy = true
frontendHealthyStreak = 0
setRestartStage('waiting_for_recovery')
setRestartMessage('系统正在完全重启,正在等待前端恢复访问。')
appendLog('[ctl] frontend is temporarily unavailable')
}
pollTimer = window.setTimeout(poll, 1500)
return
}
try {
const healthRes = await fetch('/health', { cache: 'no-store' })
if (healthRes.ok) {
healthyStreak += 1
if (sawUnhealthy && healthyStreak >= 2) {
setRestartStage('recovered')
setRestartMessage('服务已恢复,正在刷新页面。')
appendLog('健康检查已恢复,正在刷新页面')
window.setTimeout(() => window.location.reload(), 600)
return
}
} else {
sawUnhealthy = true
healthyStreak = 0
setRestartStage('waiting_for_recovery')
setRestartMessage('后端已停止响应,正在等待服务恢复。')
appendLog('检测到后端已停止响应')
}
} catch {
sawUnhealthy = true
healthyStreak = 0
setRestartStage('waiting_for_recovery')
setRestartMessage('后端已停止响应,正在等待服务恢复。')
appendLog('检测到后端已停止响应')
}
pollTimer = window.setTimeout(poll, 1500)
}
pollTimer = window.setTimeout(poll, 1200)
return () => {
cancelled = true
if (pollTimer !== null) {
window.clearTimeout(pollTimer)
}
}
}, [restartAction, restartModalOpen, restartStartedAt, restartTaskId])
if (loading && !stats) {
return (
<div style={{ display: 'flex', justifyContent: 'center', alignItems: 'center', height: '100vh' }}>
@@ -137,7 +389,17 @@ function Dashboard() {
) : (
<Tag className="dashboard-status-tag" icon={<DisconnectOutlined />} color="default">线</Tag>
)}
<Button className="dashboard-refresh-button" icon={<ReloadOutlined />} onClick={handleRetry}></Button>
{isSuperAdmin ? (
<Button
className="dashboard-action-button dashboard-restart-button"
icon={<PoweroffOutlined />}
danger
onClick={openRestartModal}
>
</Button>
) : null}
<Button className="dashboard-action-button dashboard-refresh-button" icon={<ReloadOutlined />} onClick={handleRetry}></Button>
</Space>
</div>
@@ -209,6 +471,84 @@ function Dashboard() {
{wsConnected && <Tag className="dashboard-status-tag" color="green" style={{ marginLeft: 8 }}></Tag>}
</div>
)}
<Modal
title="重启服务"
open={restartModalOpen}
onCancel={closeRestartModal}
maskClosable={false}
closable={!restartSubmitting && restartStage !== 'waiting_for_shutdown' && restartStage !== 'waiting_for_recovery'}
footer={[
<Button
key="close"
onClick={closeRestartModal}
disabled={restartSubmitting || restartStage === 'waiting_for_shutdown' || restartStage === 'waiting_for_recovery'}
>
{restartStage === 'confirming' ? '取消' : '关闭'}
</Button>,
<Button
key="submit"
type="primary"
danger
loading={restartSubmitting}
disabled={restartSubmitting || restartStage !== 'confirming'}
onClick={handleRestartAction}
>
</Button>,
]}
>
<div className="dashboard-restart-modal">
<div className="dashboard-restart-toolbar">
<div className="dashboard-restart-toolbar__field">
<Text className="dashboard-restart-section__label"></Text>
<Select
value={restartAction}
onChange={(value) => setRestartAction(value)}
disabled={restartSubmitting || restartStage !== 'confirming'}
options={RESTART_ACTION_OPTIONS.map((item) => ({
value: item.value,
label: item.label,
}))}
/>
</div>
<div className="dashboard-restart-toolbar__meta">
<div className="dashboard-restart-toolbar__item">
<Text type="secondary"></Text>
<Text code>{selectedRestartAction.command}</Text>
</div>
<div className="dashboard-restart-toolbar__item">
<Text type="secondary"> ID</Text>
<Text>{restartTaskId || '等待创建'}</Text>
</div>
</div>
</div>
<div className="dashboard-restart-section">
<Text className="dashboard-restart-section__label"></Text>
<Alert
type={
restartStage === 'failed' || restartStage === 'timeout'
? 'error'
: restartStage === 'recovered'
? 'success'
: 'info'
}
message={restartMessage}
showIcon
/>
</div>
<div className="dashboard-restart-section">
<Text className="dashboard-restart-section__label"></Text>
<div className="dashboard-restart-log">
{restartLogs.length > 0 ? restartLogs.map((line, index) => (
<div key={`${line}-${index}`}>{line}</div>
)) : <div></div>}
</div>
</div>
</div>
</Modal>
</div>
</AppLayout>
)