package server import ( "context" "database/sql" "encoding/json" "encoding/xml" "errors" "fmt" "io" "log" "net" "net/http" "strconv" "strings" "time" "github.com/gin-gonic/gin" "github.com/jmoiron/sqlx" "vdi-cloud/platform/internal/model" "vdi-cloud/platform/internal/store" "vdi-cloud/platform/pkg/proxmox" ) // This optional, single-host group is provisioned by the administrator. Device // requests may select a bound desktop; they cannot choose VMIDs or PCI devices. const exclusiveGPULock int64 = 7651092039 type gpuPrepareState struct { Phase string `json:"phase"` Node string `json:"node"` Members []int `json:"members"` Terminal string `json:"terminal"` VMID int `json:"vmid,omitempty"` UPID string `json:"upid,omitempty"` } type gpuExclusiveMember struct { DesktopID string `json:"desktopId"` Name string `json:"name"` VMID int `json:"vmid"` PowerState string `json:"powerState"` PowerObserved bool `json:"powerObserved"` ActiveSessions int `json:"activeSessions"` ActiveTasks int `json:"activeTasks"` Conflicts []string `json:"conflicts"` Eligible bool `json:"eligible"` } type gpuExclusiveStatus struct { Enabled bool `json:"enabled"` Node string `json:"node"` VMIDs []int `json:"vmids"` ObservedAt time.Time `json:"observedAt"` LockAvailable bool `json:"lockAvailable"` GroupComplete bool `json:"groupComplete"` Members []gpuExclusiveMember `json:"members"` Conflicts []string `json:"conflicts"` AllocationReady bool `json:"allocationReady"` ReleaseReady bool `json:"releaseReady"` ReleasePolicy string `json:"releasePolicy"` ActiveTasks []model.Task `json:"activeTasks"` } // exclusiveGPUStatus is a read-only preflight view. It never changes VM power // or task state; it is used by the advanced console before allocation/release. func (s *Server) exclusiveGPUStatus(c *gin.Context) { node := strings.TrimSpace(c.Query("node")) status := gpuExclusiveStatus{ObservedAt: time.Now().UTC(), Node: s.Config.ExclusiveGPUNode, VMIDs: append([]int{}, s.Config.ExclusiveGPUVMIDs...), Members: []gpuExclusiveMember{}, Conflicts: []string{}, ActiveTasks: []model.Task{}, ReleasePolicy: "仅在无活动会话、无未完成任务且锁可用时可释放;不会自动关机或重放操作"} if node != "" && (len(node) > 100 || strings.ContainsAny(node, "/\\?#")) { s.fail(c, 400, "invalid_node", "节点名称无效") return } if status.Node == "" || len(status.VMIDs) == 0 { c.JSON(200, status) return } if node != "" && node != status.Node { status.Conflicts = []string{"当前节点未配置独占显卡组"} c.JSON(200, status) return } status.Enabled = true tx, err := s.DB.BeginTxx(c.Request.Context(), &sql.TxOptions{ReadOnly: true}) if err != nil { s.dbError(c, err) return } defer tx.Rollback() locked, err := tryGPULock(c.Request.Context(), tx) if err != nil { s.dbError(c, err) return } status.LockAvailable = locked if !locked { status.Conflicts = append(status.Conflicts, "独占显卡锁当前被占用") } var members []model.Desktop members, err = s.gpuMembers(c.Request.Context(), tx) if err != nil { status.Conflicts = append(status.Conflicts, "独占显卡组成员不完整") } else { status.GroupComplete = true } var tasks []model.Task if err = tx.SelectContext(c.Request.Context(), &tasks, `SELECT * FROM tasks WHERE kind='gpu-prepare' AND status IN ('queued','running','timeout') ORDER BY created_at DESC`); err == nil { status.ActiveTasks = tasks if len(tasks) > 0 { status.Conflicts = append(status.Conflicts, "存在未完成或结果未核对的显卡切换任务") } } pve := s.currentPVE() for _, d := range members { m := gpuExclusiveMember{DesktopID: d.ID, Name: d.Name, VMID: d.VMID, PowerState: d.PowerState, Conflicts: []string{}} var n int if err = tx.GetContext(c.Request.Context(), &n, `SELECT count(*) FROM terminals WHERE session_desktop_id=$1 AND last_seen>now()-interval '90 seconds'`, d.ID); err == nil { m.ActiveSessions = n } if err = tx.GetContext(c.Request.Context(), &n, `SELECT count(*) FROM tasks WHERE target_id=$1 AND status IN ('queued','running','timeout') AND kind!='terminal'`, d.ID); err == nil { m.ActiveTasks = n } if m.ActiveSessions > 0 { m.Conflicts = append(m.Conflicts, "存在活跃终端会话") } if m.ActiveTasks > 0 { m.Conflicts = append(m.Conflicts, "存在未完成管理任务") } if pve != nil { power, powerErr := pve.Power(c.Request.Context(), d.Node, d.VMID) if powerErr == nil { m.PowerState, m.PowerObserved = power, true } else { m.PowerState = "unknown" m.Conflicts = append(m.Conflicts, "无法读取 PVE 实际电源状态") } } else { m.PowerState = "unknown" m.Conflicts = append(m.Conflicts, "尚未配置 PVE") } if m.PowerObserved && m.PowerState != "stopped" { m.Conflicts = append(m.Conflicts, "虚拟机当前仍在运行") } m.Eligible = len(m.Conflicts) == 0 && m.PowerState == "stopped" status.Members = append(status.Members, m) } if !status.GroupComplete { status.Conflicts = append(status.Conflicts, "数据库中的独占显卡组成员数量与配置不一致") } allEligible := len(status.Members) == len(status.VMIDs) for _, member := range status.Members { if !member.Eligible { allEligible = false break } } status.AllocationReady = status.Enabled && status.GroupComplete && status.LockAvailable && len(status.ActiveTasks) == 0 && len(status.Conflicts) == 0 && allEligible status.ReleaseReady = status.Enabled && status.GroupComplete && status.LockAvailable && len(status.ActiveTasks) == 0 && allEligible c.JSON(200, status) } func (s *Server) exclusiveGPU(d model.Desktop) bool { if d.Provider != "proxmox" || d.Node != s.Config.ExclusiveGPUNode { return false } for _, id := range s.Config.ExclusiveGPUVMIDs { if d.VMID == id { return true } } return false } // Maintenance submissions take the same group lock as terminal preparations. // Call inside their existing transaction; the durable reservation also covers // the interval between enqueueing a preparation and claiming it in the worker. func (s *Server) exclusiveGPUMaintenance(ctx context.Context, db sqlx.ExtContext, d model.Desktop) error { if !s.exclusiveGPU(d) { return nil } locked, err := tryGPULock(ctx, db) if err != nil { return err } var busy bool if locked { err = sqlx.GetContext(ctx, db, &busy, `SELECT EXISTS(SELECT 1 FROM tasks WHERE kind='gpu-prepare' AND status IN ('queued','running','timeout'))`) } if err != nil { return err } if !locked || busy { return errors.New("显卡正在切换或结果未确认,请先核对切换任务") } return nil } func (s *Server) gpuMembers(ctx context.Context, db sqlx.ExtContext) ([]model.Desktop, error) { var members []model.Desktop err := sqlx.SelectContext(ctx, db, &members, `SELECT * FROM desktops WHERE provider='proxmox' AND node=$1 AND vmid=ANY($2) ORDER BY vmid`, s.Config.ExclusiveGPUNode, s.Config.ExclusiveGPUVMIDs) if err == nil && len(members) != len(s.Config.ExclusiveGPUVMIDs) { err = errors.New("exclusive GPU group is incomplete") } return members, err } func tryGPULock(ctx context.Context, db sqlx.ExtContext) (bool, error) { var locked bool err := sqlx.GetContext(ctx, db, &locked, "SELECT pg_try_advisory_xact_lock($1)", exclusiveGPULock) return locked, err } func (s *Server) gpuAllowsConnection(c *gin.Context, d model.Desktop) bool { if !s.exclusiveGPU(d) { return true } locked, err := tryGPULock(c.Request.Context(), deviceTx(c)) var busy bool if err == nil && locked { err = deviceTx(c).GetContext(c.Request.Context(), &busy, `SELECT EXISTS(SELECT 1 FROM tasks WHERE kind='gpu-prepare' AND status IN ('queued','running','timeout'))`) } if err != nil { s.dbError(c, err) return false } if !locked || busy { s.fail(c, 409, "gpu_switch_busy", "显卡正在切换或上次切换尚未确认,请稍后重试或联系管理员") return false } pve := s.currentPVE() if pve == nil { s.fail(c, 503, "pve_unavailable", "计算平台不可用") return false } for _, id := range s.Config.ExclusiveGPUVMIDs { power, e := pve.Power(c.Request.Context(), d.Node, id) expected := "stopped" if id == d.VMID { expected = "running" } if e != nil || power != expected { s.fail(c, 409, "gpu_owner_not_ready", "桌面电源或显卡归属尚未就绪,请先准备桌面") return false } } return true } func gpuGroupBusy(ctx context.Context, db sqlx.ExtContext, members []model.Desktop, taskID string) (bool, error) { ids := make([]string, 0, len(members)) for _, d := range members { ids = append(ids, d.ID) } var busy bool err := sqlx.GetContext(ctx, db, &busy, `SELECT EXISTS(SELECT 1 FROM terminals WHERE session_desktop_id=ANY($1) AND last_seen>now()-interval '90 seconds') OR EXISTS(SELECT 1 FROM tasks WHERE target_id=ANY($1) AND id!=$2 AND kind!='terminal' AND status IN ('queued','running','timeout'))`, ids, taskID) return busy, err } func (s *Server) prepareGPUDesktop(c *gin.Context) { var in struct { DesktopID string `json:"desktopId" binding:"required,max=100"` } if !s.decode(c, &in) { return } tx := deviceTx(c) ctx := c.Request.Context() terminal := c.GetString("terminalId") var d model.Desktop err := tx.GetContext(ctx, &d, `SELECT d.* FROM desktops d JOIN bindings b ON b.desktop_id=d.id JOIN users u ON u.id=b.user_id JOIN terminals t ON t.id=u.terminal_id WHERE d.id=$1 AND t.id=$2 AND u.enabled AND t.enabled`, in.DesktopID, terminal) if err == sql.ErrNoRows { s.fail(c, 403, "desktop_forbidden", "该终端没有桌面使用权限") return } if err != nil { s.dbError(c, err) return } if !s.exclusiveGPU(d) || d.SetupRequired || d.Endpoint == "" { s.fail(c, 409, "desktop_not_preparable", "该桌面未启用显卡切换或连接配置尚未完成") return } locked, err := tryGPULock(ctx, tx) if err != nil { s.dbError(c, err) return } if !locked { s.fail(c, 409, "gpu_switch_busy", "显卡正在切换,请稍后重试") return } var previous model.Task err = tx.GetContext(ctx, &previous, `SELECT * FROM tasks WHERE kind='gpu-prepare' AND status IN ('queued','running','timeout') ORDER BY created_at DESC LIMIT 1`) if err == nil { if previous.Actor == "device:"+terminal && previous.TargetID == d.ID && previous.Status != "timeout" { s.deviceJSON(c, 202, gin.H{"taskId": previous.ID, "status": previous.Status, "message": previous.Message}) return } s.fail(c, 409, "gpu_switch_busy", "已有显卡切换任务或未确认结果,请稍后重试或联系管理员") return } if err != sql.ErrNoRows { s.dbError(c, err) return } members, err := s.gpuMembers(ctx, tx) if err != nil { s.dbError(c, err) return } for _, member := range members { if !s.hardwareAllowsConnection(c, member.ID) { return } } busy, err := gpuGroupBusy(ctx, tx, members, "") if err != nil { s.dbError(c, err) return } if busy { s.fail(c, 409, "gpu_group_in_use", "请先退出正在使用的云桌面;其他终端使用中或管理任务未结束时无法切换") return } state := gpuPrepareState{Phase: "queued", Node: d.Node, Members: s.Config.ExclusiveGPUVMIDs, Terminal: terminal} encoded, _ := json.Marshal(state) id := store.ID("task_") _, err = tx.ExecContext(ctx, `INSERT INTO tasks(id,kind,action,target_id,target_name,actor,status,message,provider_ref,idempotency_key) VALUES($1,'gpu-prepare','prepare-desktop',$2,$3,$4,'queued','正在准备桌面,等待显卡切换',$5,$1)`, id, d.ID, d.Name, "device:"+terminal, string(encoded)) if err != nil { s.dbError(c, err) return } s.deviceJSON(c, 202, gin.H{"taskId": id, "status": "queued", "message": "正在准备桌面,首次启动可能需要两分钟"}) } func (s *Server) gpuPrepareStatus(c *gin.Context) { var task model.Task err := deviceTx(c).GetContext(c.Request.Context(), &task, `SELECT * FROM tasks WHERE id=$1 AND kind='gpu-prepare' AND actor=$2`, c.Param("id"), "device:"+c.GetString("terminalId")) if err == sql.ErrNoRows { s.fail(c, 404, "task_not_found", "切换任务不存在") return } if err != nil { s.dbError(c, err) return } s.deviceJSON(c, 200, gin.H{"taskId": task.ID, "status": task.Status, "message": task.Message}) } func (s *Server) saveGPUStage(ctx context.Context, t model.Task, state gpuPrepareState, message string) error { data, _ := json.Marshal(state) result, err := s.DB.ExecContext(ctx, `UPDATE tasks SET provider_ref=$1,message=$2,updated_at=now() WHERE id=$3 AND status='running'`, string(data), message, t.ID) if err == nil { n, _ := result.RowsAffected() if n != 1 { err = errors.New("GPU task reservation lost") } } return err } func sunshineReady(ctx context.Context, endpoint string) bool { host := endpoint if parsed, _, err := net.SplitHostPort(endpoint); err == nil { host = parsed } // Platform endpoints are hostnames/IPs, never arbitrary URLs or paths. if strings.ContainsAny(host, "/\\@?#") || host == "" { return false } req, err := http.NewRequestWithContext(ctx, "GET", "http://"+net.JoinHostPort(host, "47989")+"/serverinfo", nil) if err != nil { return false } client := http.Client{Timeout: 2 * time.Second, Transport: &http.Transport{Proxy: nil}, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} defer client.CloseIdleConnections() response, err := client.Do(req) if err != nil { return false } defer response.Body.Close() var info struct { XMLName xml.Name `xml:"root"` Status string `xml:"status_code,attr"` Hostname string `xml:"hostname"` } return response.StatusCode == 200 && xml.NewDecoder(io.LimitReader(response.Body, 65536)).Decode(&info) == nil && info.Status == "200" && info.Hostname != "" } func (s *Server) executeGPUPrepare(parent context.Context, t model.Task) { ctx, cancel := context.WithTimeout(parent, 180*time.Second) defer cancel() state := gpuPrepareState{} err := json.Unmarshal([]byte(t.ProviderRef), &state) uncertain := false if err == nil && state.Phase != "queued" && state.Phase != "" { uncertain = true } if err == nil && (state.Phase != "queued" || state.Node != s.Config.ExclusiveGPUNode || fmt.Sprint(state.Members) != fmt.Sprint(s.Config.ExclusiveGPUVMIDs)) { err = errors.New("GPU group changed or task already submitted") } var tx *sqlx.Tx if err == nil { tx, err = s.DB.BeginTxx(ctx, nil) } if tx != nil { defer tx.Rollback() } // Hold this lock on a dedicated transaction throughout the switch. Tickets // fail promptly instead of waiting behind a VM boot; no root credential leaves // the platform. An API restart never automatically replays an uncertain POST. if err == nil { var locked bool locked, err = tryGPULock(ctx, tx) if err == nil && !locked { err = errors.New("GPU group locked") } } var members []model.Desktop if err == nil { members, err = s.gpuMembers(ctx, tx) } var target model.Desktop if err == nil { for _, d := range members { if d.ID == t.TargetID { target = d } } var allowed bool err = tx.GetContext(ctx, &allowed, `SELECT EXISTS(SELECT 1 FROM bindings b JOIN users u ON u.id=b.user_id JOIN terminals term ON term.id=u.terminal_id WHERE b.desktop_id=$1 AND u.terminal_id=$2 AND u.enabled AND term.enabled)`, t.TargetID, state.Terminal) if err == nil && (!allowed || target.ID == "" || target.SetupRequired || target.Endpoint == "") { err = errors.New("desktop authorization/configuration changed") } } if err == nil { var busy bool busy, err = gpuGroupBusy(ctx, tx, members, t.ID) if err == nil && busy { err = errors.New("GPU group is in use") } } if err == nil { err = s.RefreshCompute(ctx) } pve := s.currentPVE() if err == nil && pve == nil { err = errors.New("PVE unavailable") } action := func(d model.Desktop, operation string) error { state.Phase = operation + "-submitting" state.VMID = d.VMID state.UPID = "" if err := s.saveGPUStage(ctx, t, state, "正在"+map[string]string{"start": "启动所选桌面", "shutdown": "正常关闭上一台桌面"}[operation]); err != nil { return err } uncertain = true upid, err := pve.Action(ctx, d.Node, d.VMID, operation) if err != nil { var rejected *proxmox.HTTPError if errors.As(err, &rejected) && rejected.Status >= 400 && rejected.Status < 500 { uncertain = false } return err } state.Phase = operation + "-waiting" state.UPID = upid if err = s.saveGPUStage(ctx, t, state, "正在等待虚拟机电源操作完成"); err != nil { return err } if err = pve.Wait(ctx, d.Node, upid, time.Second); err != nil { return err } power, err := pve.Power(ctx, d.Node, d.VMID) expected := "running" if operation == "shutdown" { expected = "stopped" } if err == nil && power != expected { err = errors.New("VM power transition not confirmed") } if err == nil { uncertain = false } return err } if err == nil { for _, d := range members { if d.ID == target.ID { continue } power, e := pve.Power(ctx, d.Node, d.VMID) if e != nil { err = e break } if power == "running" { err = action(d, "shutdown") } else if power != "stopped" { err = errors.New("unexpected VM power state") } if err != nil { break } } } if err == nil { power, e := pve.Power(ctx, target.Node, target.VMID) err = e if err == nil && power == "stopped" { err = action(target, "start") } else if err == nil && power != "running" { err = errors.New("unexpected target state") } } if err == nil { state.Phase = "service-waiting" err = s.saveGPUStage(ctx, t, state, "桌面已开机,正在等待连接服务") for err == nil && !sunshineReady(ctx, target.Endpoint) { select { case <-ctx.Done(): err = ctx.Err() case <-time.After(time.Second): } } } if err == nil { // Recheck all owners immediately before releasing the reservation. for _, d := range members { power, e := pve.Power(ctx, d.Node, d.VMID) expected := "stopped" if d.ID == target.ID { expected = "running" } if e != nil || power != expected { err = errors.New("final GPU ownership check failed") break } if _, e = s.DB.ExecContext(ctx, `UPDATE desktops SET power_state=$1,service_state='unknown',updated_at=now() WHERE id=$2`, power, d.ID); e != nil { err = e break } } } status, message := "success", "桌面已启动,连接服务已响应" if err != nil { log.Printf("GPU prepare task=%s phase=%s error=%v", t.ID, state.Phase, err) status = "failed" message = "桌面准备失败,请检查桌面授权、占用状态及 PVE" if uncertain { status = "timeout" message = "电源操作或服务状态尚未确认,已阻止再次切换;请管理员核对 PVE 原任务" } } finalCtx, finish := context.WithTimeout(context.Background(), 5*time.Second) defer finish() if err == nil { state.Phase = "completed" _ = s.saveGPUStage(finalCtx, t, state, message) } _ = s.complete(finalCtx, t, status, message) } // Reconcile only a known original PVE task. An unknown submission requires // manual operator investigation; this endpoint never resubmits a power action. func (s *Server) reconcileGPUPrepare(c *gin.Context) { var task model.Task if err := s.DB.GetContext(c.Request.Context(), &task, `SELECT * FROM tasks WHERE id=$1 AND kind='gpu-prepare' AND status='timeout'`, c.Param("id")); err != nil { s.fail(c, 409, "not_uncertain", "该任务无需核对或不存在") return } var state gpuPrepareState if json.Unmarshal([]byte(task.ProviderRef), &state) != nil || state.UPID == "" { s.fail(c, 409, "unknown_submission", "未记录原 PVE 任务编号,需要在 PVE 手工核对,不能自动重发") return } tx, err := s.DB.BeginTxx(c.Request.Context(), nil) if err != nil { s.dbError(c, err) return } defer tx.Rollback() locked, err := tryGPULock(c.Request.Context(), tx) if err != nil || !locked { s.fail(c, 409, "gpu_switch_busy", "显卡正在切换") return } pve := s.currentPVE() if pve == nil { s.fail(c, 503, "pve_unavailable", "计算平台不可用") return } original, err := pve.Task(c.Request.Context(), state.Node, state.UPID) if err != nil || original.Status != "stopped" { s.fail(c, 409, "pve_task_pending", "原 PVE 任务未结束或无法核对") return } for _, id := range state.Members { power, e := pve.Power(c.Request.Context(), state.Node, id) if e != nil || (power != "running" && power != "stopped") { s.fail(c, 409, "power_unknown", "虚拟机实际电源状态未确认") return } } _, err = tx.ExecContext(c.Request.Context(), `UPDATE tasks SET status='failed',message='管理员已核对原 PVE 任务及实际电源状态,可重新准备桌面',updated_at=now() WHERE id=$1 AND status='timeout'`, task.ID) if err == nil { _, err = tx.ExecContext(c.Request.Context(), `INSERT INTO audit_logs VALUES($1,$2,'reconcile-gpu-switch',$3,'success',$4,now())`, store.ID("audit_"), c.MustGet("admin").(model.Administrator).Username, task.TargetName, task.ID) } if err == nil { err = tx.Commit() } if err != nil { s.dbError(c, err) return } c.JSON(200, gin.H{"reconciled": true, "vmid": strconv.Itoa(state.VMID)}) }