238 lines
7.8 KiB
Go
238 lines
7.8 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"github.com/gin-gonic/gin"
|
|
"github.com/jmoiron/sqlx"
|
|
"time"
|
|
"vdi-cloud/platform/internal/model"
|
|
"vdi-cloud/platform/pkg/proxmox"
|
|
)
|
|
|
|
type powerBatchJob struct {
|
|
Stage string `db:"stage"`
|
|
Fingerprint string `db:"fingerprint"`
|
|
}
|
|
|
|
func (s *Server) claimBatchPower(ctx context.Context, t model.Task) (bool, error) {
|
|
tx, err := s.DB.BeginTxx(ctx, nil)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
defer tx.Rollback()
|
|
var id string
|
|
if err = tx.GetContext(ctx, &id, "SELECT id FROM desktops WHERE id=$1 FOR UPDATE", t.TargetID); err != nil {
|
|
return false, err
|
|
}
|
|
r, err := tx.ExecContext(ctx, `UPDATE power_batch_items SET stage='submitting' WHERE task_id=$1 AND stage='prepared' AND EXISTS(SELECT 1 FROM tasks WHERE id=$1 AND kind='batch-desktop' AND status IN ('queued','running'))`, t.ID)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
n, _ := r.RowsAffected()
|
|
if n != 1 {
|
|
return false, nil
|
|
}
|
|
err = tx.Commit()
|
|
return err == nil, err
|
|
}
|
|
func (s *Server) confirmBatchPower(ctx context.Context, db sqlx.ExtContext, t model.Task) error {
|
|
var d model.Desktop
|
|
if err := sqlx.GetContext(ctx, db, &d, "SELECT * FROM desktops WHERE id=$1", t.TargetID); err != nil {
|
|
return err
|
|
}
|
|
power := d.PowerState
|
|
if d.Provider == "proxmox" {
|
|
pve := s.currentPVE()
|
|
if pve == nil || t.ProviderRef == "" {
|
|
return errors.New("missing provider task")
|
|
}
|
|
task, err := pve.Task(ctx, d.Node, t.ProviderRef)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if task.Status != "stopped" || task.ExitStatus != "OK" {
|
|
return errors.New("original power task not successful")
|
|
}
|
|
power, err = pve.Power(ctx, d.Node, d.VMID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
} else if d.Provider != "demo" || s.Config.Mode != "demo" {
|
|
return errors.New("unsupported provider")
|
|
}
|
|
expected := "running"
|
|
if t.Action == "shutdown" {
|
|
expected = "stopped"
|
|
}
|
|
if power != expected {
|
|
return errors.New("actual power differs")
|
|
}
|
|
return nil
|
|
}
|
|
func (s *Server) executeBatchPower(parent context.Context, t model.Task) {
|
|
ctx, cancel := context.WithTimeout(parent, 90*time.Second)
|
|
defer cancel()
|
|
var job powerBatchJob
|
|
err := s.DB.GetContext(ctx, &job, "SELECT stage,fingerprint FROM power_batch_items WHERE task_id=$1", t.ID)
|
|
submitted := job.Stage == "submitting" || job.Stage == "uncertain"
|
|
if err == nil {
|
|
err = s.RefreshCompute(ctx)
|
|
}
|
|
var d model.Desktop
|
|
if err == nil && !submitted {
|
|
var item powerBatchCheckItem
|
|
d, item, err = s.powerBatchPreflight(ctx, s.DB, t.TargetID, t.Action, t.ID)
|
|
if err == nil && (!item.Available || powerFingerprint(d, item.PowerState) != job.Fingerprint) {
|
|
err = errors.New("power preflight changed")
|
|
}
|
|
if err == nil {
|
|
var claimed bool
|
|
claimed, err = s.claimBatchPower(ctx, t)
|
|
if err == nil && !claimed {
|
|
return
|
|
}
|
|
if err == nil {
|
|
submitted = true
|
|
if d.Provider == "demo" {
|
|
power := "running"
|
|
if t.Action == "shutdown" {
|
|
power = "stopped"
|
|
}
|
|
_, err = s.DB.ExecContext(ctx, "UPDATE desktops SET power_state=$1,service_state='unknown',updated_at=now() WHERE id=$2", power, d.ID)
|
|
} else {
|
|
t.ProviderRef, err = s.currentPVE().Action(ctx, d.Node, d.VMID, t.Action)
|
|
var rejected *proxmox.HTTPError
|
|
if errors.As(err, &rejected) && rejected.Status >= 400 && rejected.Status < 500 {
|
|
submitted = false
|
|
}
|
|
if err == nil {
|
|
_, err = s.DB.ExecContext(ctx, "UPDATE tasks SET provider_ref=$1,updated_at=now() WHERE id=$2", t.ProviderRef, t.ID)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if err == nil {
|
|
err = s.DB.GetContext(ctx, &d, "SELECT * FROM desktops WHERE id=$1", t.TargetID)
|
|
if err == nil && d.Provider == "proxmox" {
|
|
if t.ProviderRef == "" {
|
|
err = errors.New("missing original UPID")
|
|
} else {
|
|
err = s.currentPVE().Wait(ctx, d.Node, t.ProviderRef, time.Second)
|
|
}
|
|
}
|
|
}
|
|
if err == nil {
|
|
err = s.confirmBatchPower(ctx, s.DB, t)
|
|
}
|
|
status, stage, message := "success", "completed", "电源任务和实际状态已确认；连接服务需另行确认"
|
|
if err != nil {
|
|
status, stage, message = "failed", "failed", "执行前状态发生变化或 PVE 拒绝请求，未发送有效修改"
|
|
if submitted {
|
|
status, stage, message = "timeout", "uncertain", "批量电源结果未确认，已阻止后续修改和新连接；只核对原任务，不会自动重发"
|
|
}
|
|
}
|
|
s.finishBatchPower(t, status, stage, message)
|
|
}
|
|
func (s *Server) finishBatchPower(t model.Task, status, stage, message string) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
tx, err := s.DB.BeginTxx(ctx, nil)
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer tx.Rollback()
|
|
var id, current, jobStage string
|
|
if err = tx.GetContext(ctx, &id, "SELECT id FROM desktops WHERE id=$1 FOR UPDATE", t.TargetID); err != nil {
|
|
return
|
|
}
|
|
if err = tx.GetContext(ctx, ¤t, "SELECT status FROM tasks WHERE id=$1 FOR UPDATE", t.ID); err != nil || current != "queued" && current != "running" {
|
|
return
|
|
}
|
|
if err = tx.GetContext(ctx, &jobStage, "SELECT stage FROM power_batch_items WHERE task_id=$1 FOR UPDATE", t.ID); err != nil {
|
|
return
|
|
}
|
|
if stage == "expired" {
|
|
status, stage, message = "failed", "failed", "批量电源任务提交前过期，未发送修改"
|
|
if jobStage != "prepared" {
|
|
status, stage, message = "timeout", "uncertain", "原批量电源任务结果未确认，不会自动重发"
|
|
}
|
|
}
|
|
_, err = tx.ExecContext(ctx, "UPDATE power_batch_items SET stage=$1 WHERE task_id=$2", stage, t.ID)
|
|
if err == nil && status == "success" {
|
|
power := "running"
|
|
if t.Action == "shutdown" {
|
|
power = "stopped"
|
|
}
|
|
_, err = tx.ExecContext(ctx, "UPDATE desktops SET power_state=$1,service_state='unknown',updated_at=now() WHERE id=$2", power, t.TargetID)
|
|
}
|
|
if err == nil {
|
|
err = s.completeTx(ctx, tx, t, status, message)
|
|
}
|
|
if err == nil {
|
|
_ = tx.Commit()
|
|
}
|
|
}
|
|
func (s *Server) reconcileBatchPower(c *gin.Context) {
|
|
tx, actor := s.accountTransaction(c, false)
|
|
if tx == nil {
|
|
return
|
|
}
|
|
defer tx.Rollback()
|
|
if actor.Role != "super_admin" && actor.Role != "operator" {
|
|
s.fail(c, 403, "forbidden", "当前账号不能核对电源任务")
|
|
return
|
|
}
|
|
ctx := c.Request.Context()
|
|
var t model.Task
|
|
if err := tx.GetContext(ctx, &t, "SELECT * FROM tasks WHERE id=$1 AND kind='batch-desktop'", c.Param("id")); err != nil {
|
|
s.dbError(c, err)
|
|
return
|
|
}
|
|
var id string
|
|
if err := tx.GetContext(ctx, &id, "SELECT id FROM desktops WHERE id=$1 FOR UPDATE", t.TargetID); err != nil {
|
|
s.dbError(c, err)
|
|
return
|
|
}
|
|
if err := tx.GetContext(ctx, &t, "SELECT * FROM tasks WHERE id=$1 FOR UPDATE", t.ID); err != nil {
|
|
s.dbError(c, err)
|
|
return
|
|
}
|
|
var stage string
|
|
if err := tx.GetContext(ctx, &stage, "SELECT stage FROM power_batch_items WHERE task_id=$1 FOR UPDATE", t.ID); err != nil {
|
|
s.dbError(c, err)
|
|
return
|
|
}
|
|
if t.Status != "timeout" || stage != "uncertain" {
|
|
s.fail(c, 409, "reconcile_unavailable", "仅可核对结果未确认的批量电源任务")
|
|
return
|
|
}
|
|
if err := s.confirmBatchPower(ctx, tx, t); err != nil {
|
|
s.fail(c, 409, "power_unconfirmed", "原 PVE 任务和实际电源状态尚未确认；未重新发送修改")
|
|
return
|
|
}
|
|
_, err := tx.ExecContext(ctx, "UPDATE power_batch_items SET stage='completed' WHERE task_id=$1", t.ID)
|
|
if err == nil {
|
|
_, err = tx.ExecContext(ctx, "UPDATE tasks SET status='success',message='只读核对原任务和实际电源状态通过，未重新发送修改',updated_at=now() WHERE id=$1", t.ID)
|
|
}
|
|
if err == nil {
|
|
power := "running"
|
|
if t.Action == "shutdown" {
|
|
power = "stopped"
|
|
}
|
|
_, err = tx.ExecContext(ctx, "UPDATE desktops SET power_state=$1,service_state='unknown',updated_at=now() WHERE id=$2", power, t.TargetID)
|
|
}
|
|
if err == nil {
|
|
err = accountAudit(ctx, tx, actor.Username, "reconcile_batch_power", t.ID, c.GetString("requestId"))
|
|
}
|
|
if err == nil {
|
|
err = tx.Commit()
|
|
}
|
|
if err != nil {
|
|
s.dbError(c, err)
|
|
return
|
|
}
|
|
c.JSON(200, gin.H{"accepted": true})
|
|
}
|