功能:完善云管助手与GPU故障恢复核对

This commit is contained in:
songyiming committed 2026-11-13 19:08:40 +08:00
1 parent b6026f9759
commit 9f7b01e4db
42 files changed
+3576 -120

No files matched your search

+12 -2
View File
@@ -43,7 +43,7 @@ func (s *Server) settings(c *gin.Context) {
if pve != nil {
provider = "proxmox"
}
c.JSON(200, gin.H{"mode": s.Config.Mode, "computeProvider": provider, "pveConfigured": pve != nil, "terminalTransport": "https-poll", "capabilities": []string{"diagnostics-center", "admin-mfa", "administrator-sessions", "task-history", "audit-query-export", "terminal-metric-alerts", "terminal-system-metrics", "gpu-inventory", "desktop-batches", "desktop-snapshots", "pve-connection", "desktop-hardware", "login-rate-limit", "csv-imports", "session-history", "monitor-alerts", "administrators", "password", "auth", "resources", "desktops", "templates", "pools", "clone", "desktop-connection", "terminals", "terminal-lifecycle", "terminal-policies", "users", "batch-bindings", "bindings", "tasks", "audit"}})
c.JSON(200, gin.H{"mode": s.Config.Mode, "computeProvider": provider, "pveConfigured": pve != nil, "terminalTransport": "https-poll", "capabilities": []string{"common-assistant", "diagnostics-center", "admin-mfa", "administrator-sessions", "task-history", "audit-query-export", "terminal-metric-alerts", "terminal-system-metrics", "gpu-inventory", "gpu-health-agent", "desktop-batches", "desktop-snapshots", "pve-connection", "desktop-hardware", "login-rate-limit", "csv-imports", "session-history", "monitor-alerts", "administrators", "password", "auth", "resources", "desktops", "templates", "pools", "clone", "desktop-connection", "terminals", "terminal-lifecycle", "terminal-policies", "users", "batch-bindings", "bindings", "tasks", "audit"}})
}
func (s *Server) resources(c *gin.Context) {
items := []model.Resource{}
@@ -514,6 +514,7 @@ func (s *Server) RunWorker(ctx context.Context) {
workerCtx, stop := context.WithCancel(ctx)
provisionDone := make(chan struct{}, 1)
provisionBusy := false
var nextGPUCheck time.Time
defer func() {
stop()
if provisionBusy {
@@ -532,7 +533,8 @@ func (s *Server) RunWorker(ctx context.Context) {
if err = conn.PingContext(ctx); err != nil {
return
}
// Unknown outcomes are never automatically retried after a worker restart.
// Unknown commands are never replayed. Known GPU tasks are observed
// separately below so a transient PVE read failure can recover.
var expired []model.Task
if s.DB.SelectContext(ctx, &expired, "SELECT * FROM tasks WHERE status IN ('queued','running') AND ((kind NOT IN ('provision','hardware','snapshot','batch-desktop','gpu-prepare') AND created_at<now()-interval '2 minutes') OR (kind='gpu-prepare' AND created_at<now()-interval '5 minutes') OR (kind='batch-desktop' AND ((status='queued' AND created_at<now()-interval '2 hours') OR (status='running' AND updated_at<now()-interval '2 minutes'))) OR (kind IN ('provision','hardware','snapshot') AND updated_at<now()-interval '35 minutes'))") == nil {
for _, t := range expired {
@@ -563,6 +565,14 @@ func (s *Server) RunWorker(ctx context.Context) {
_ = s.complete(ctx, t, "timeout", "结果未确认,请核对设备或 PVE 实际状态;不会自动重放")
}
}
if !provisionBusy && !time.Now().Before(nextGPUCheck) {
nextGPUCheck = time.Now().Add(15 * time.Second)
if recovery, readErr := s.nextGPURecovery(ctx); readErr == nil {
provisionBusy = true
go func(task model.Task) { s.recoverGPUPrepare(workerCtx, task); provisionDone <- struct{}{} }(recovery)
continue
}
}
var t model.Task
err := s.DB.GetContext(ctx, &t, `UPDATE tasks SET status='running',updated_at=now(),message='正在执行,等待最终回执' WHERE id=(SELECT id FROM tasks WHERE ((status='queued' AND kind!='terminal') OR (kind='provision' AND status='running' AND provider_ref!='') OR (kind IN ('hardware','snapshot','batch-desktop') AND status='running')) AND ($1 OR kind NOT IN ('provision','hardware','snapshot','batch-desktop','gpu-prepare')) ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING *`, !provisionBusy)
if err != nil {