功能：完善云管助手与GPU故障恢复核对

platform/internal/server/gpu_recovery.go

@@ -0,0 +1,145 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"vdi-cloud/platform/internal/model"
+	"vdi-cloud/platform/internal/store"
+)
+
+func (s *Server) knownGPURecovery(state gpuPrepareState) bool {
+	return state.UPID != "" && state.Node == s.Config.ExclusiveGPUNode &&
+		fmt.Sprint(state.Members) == fmt.Sprint(s.Config.ExclusiveGPUVMIDs) &&
+		(state.Phase == "start-waiting" || state.Phase == "shutdown-waiting" || state.Phase == "service-waiting")
+}
+
+func (s *Server) gpuRecoveryPending(task model.Task) bool {
+	var state gpuPrepareState
+	return task.Status == "timeout" && time.Since(task.CreatedAt) < 5*time.Minute &&
+		json.Unmarshal([]byte(task.ProviderRef), &state) == nil && s.knownGPURecovery(state)
+}
+
+// Recovery observes the original UPID and every member under the same group
+// lock as preparation. It never sends a power command, even after an API
+// restart or a lost submission response.
+func (s *Server) recoverGPUPrepare(parent context.Context, task model.Task) {
+	ctx, cancel := context.WithTimeout(parent, 60*time.Second)
+	defer cancel()
+	if s.RefreshCompute(ctx) != nil {
+		return
+	}
+	pve := s.currentPVE()
+	if pve == nil {
+		return
+	}
+	tx, err := s.DB.BeginTxx(ctx, nil)
+	if err != nil {
+		return
+	}
+	defer tx.Rollback()
+	locked, err := tryGPULock(ctx, tx)
+	if err != nil || !locked {
+		return
+	}
+	if err = tx.GetContext(ctx, &task, "SELECT * FROM tasks WHERE id=$1 AND kind='gpu-prepare' AND status IN ('running','timeout') FOR UPDATE", task.ID); err != nil {
+		return
+	}
+	finish := func(status, message string, power map[string]string) {
+		for id, value := range power {
+			if _, err = tx.ExecContext(ctx, "UPDATE desktops SET power_state=$1,service_state=CASE WHEN $2=$3 AND $4='success' THEN 'ready' ELSE 'unknown' END,updated_at=now() WHERE id=$2", value, id, task.TargetID, status); err != nil {
+				return
+			}
+		}
+		if _, err = tx.ExecContext(ctx, "UPDATE tasks SET status=$1,message=$2,updated_at=now() WHERE id=$3", status, message, task.ID); err != nil {
+			return
+		}
+		if status != "timeout" {
+			if _, err = tx.ExecContext(ctx, "INSERT INTO audit_logs VALUES($1,'system','auto-reconcile-gpu-switch',$2,$3,$4,now())", store.ID("audit_"), task.TargetID, status, task.ID); err != nil {
+				return
+			}
+		}
+		_ = tx.Commit()
+	}
+	var state gpuPrepareState
+	if json.Unmarshal([]byte(task.ProviderRef), &state) == nil && state.Phase == "queued" && state.UPID == "" {
+		finish("failed", "服务重启前尚未提交电源操作，可重新选择桌面", nil)
+		return
+	}
+	if json.Unmarshal([]byte(task.ProviderRef), &state) != nil || !s.knownGPURecovery(state) {
+		finish("timeout", "未记录可核对的原任务或独占组配置已变化，请管理员核查；不会重复发送电源命令", nil)
+		return
+	}
+	original, err := pve.Task(ctx, state.Node, state.UPID)
+	if err != nil || original.Status != "stopped" || original.ExitStatus == "" {
+		finish("timeout", "平台正在自动核对原 PVE 任务，暂不重复切换；计算节点恢复后将继续检查", nil)
+		return
+	}
+	members, err := s.gpuMembers(ctx, tx)
+	if err != nil {
+		finish("timeout", "独占显卡组成员不完整，请管理员核查", nil)
+		return
+	}
+	powers := map[string]string{}
+	running, matches, submittedMember := 0, true, false
+	var target model.Desktop
+	for _, d := range members {
+		power, readErr := pve.Power(ctx, d.Node, d.VMID)
+		if readErr != nil || (power != "running" && power != "stopped") {
+			finish("timeout", "平台正在自动核对虚拟机实际电源状态，暂不重复切换", nil)
+			return
+		}
+		powers[d.ID] = power
+		if d.VMID == state.VMID {
+			submittedMember = true
+		}
+		if power == "running" {
+			running++
+		}
+		if d.ID == task.TargetID {
+			target = d
+			matches = matches && power == "running"
+		} else {
+			matches = matches && power == "stopped"
+		}
+	}
+	if running > 1 || target.ID == "" || !submittedMember {
+		finish("timeout", "实际电源或显卡归属存在冲突，请管理员核查；平台不会自动关机", powers)
+		return
+	}
+	if original.ExitStatus != "OK" || !matches {
+		finish("failed", "已自动核对原任务和电源状态，本次准备未完成；可重新选择桌面", powers)
+		return
+	}
+	var allowed bool
+	err = tx.GetContext(ctx, &allowed, `SELECT EXISTS(SELECT 1 FROM bindings b JOIN users u ON u.id=b.user_id JOIN terminals t ON t.id=u.terminal_id WHERE b.desktop_id=$1 AND t.id=$2 AND u.enabled AND t.enabled)`, target.ID, state.Terminal)
+	if err != nil {
+		return
+	}
+	if !allowed || target.SetupRequired || target.Endpoint == "" {
+		finish("failed", "原任务已核对，但桌面授权或连接配置已变化，请检查配置后重新选择", powers)
+		return
+	}
+	if !sunshineReady(ctx, target.Endpoint) {
+		if time.Since(task.CreatedAt) < 5*time.Minute {
+			finish("timeout", "原任务已确认成功，正在等待桌面连接服务恢复", powers)
+		} else {
+			finish("failed", "原任务及显卡归属已自动确认，但 Sunshine 仍未响应，请检查桌面服务", powers)
+		}
+		return
+	}
+	finish("success", "已自动核对原任务、独占显卡归属和连接服务，桌面可连接", powers)
+}
+
+// Only called by the worker while no GPU/provisioning operation is running.
+// A restart resumes observations, never the command sequence.
+func (s *Server) nextGPURecovery(ctx context.Context) (model.Task, error) {
+	var task model.Task
+	err := s.DB.GetContext(ctx, &task, `SELECT * FROM tasks WHERE kind='gpu-prepare'
+AND ((status='running')
+ OR (status='timeout' AND provider_ref::jsonb->>'upid' IS NOT NULL AND provider_ref::jsonb->>'upid' != '' AND created_at>now()-interval '24 hours'))
+ORDER BY updated_at,id LIMIT 1`)
+	return task, err
+}