功能:完善云管助手与GPU故障恢复核对
This commit is contained in:
1 parent
b6026f9759
commit
9f7b01e4db
42 files changed
+3576
-120
No files matched your search
@@ -43,7 +43,7 @@ func (s *Server) settings(c *gin.Context) {
|
||||
if pve != nil {
|
||||
provider = "proxmox"
|
||||
}
|
||||
c.JSON(200, gin.H{"mode": s.Config.Mode, "computeProvider": provider, "pveConfigured": pve != nil, "terminalTransport": "https-poll", "capabilities": []string{"diagnostics-center", "admin-mfa", "administrator-sessions", "task-history", "audit-query-export", "terminal-metric-alerts", "terminal-system-metrics", "gpu-inventory", "desktop-batches", "desktop-snapshots", "pve-connection", "desktop-hardware", "login-rate-limit", "csv-imports", "session-history", "monitor-alerts", "administrators", "password", "auth", "resources", "desktops", "templates", "pools", "clone", "desktop-connection", "terminals", "terminal-lifecycle", "terminal-policies", "users", "batch-bindings", "bindings", "tasks", "audit"}})
|
||||
c.JSON(200, gin.H{"mode": s.Config.Mode, "computeProvider": provider, "pveConfigured": pve != nil, "terminalTransport": "https-poll", "capabilities": []string{"common-assistant", "diagnostics-center", "admin-mfa", "administrator-sessions", "task-history", "audit-query-export", "terminal-metric-alerts", "terminal-system-metrics", "gpu-inventory", "gpu-health-agent", "desktop-batches", "desktop-snapshots", "pve-connection", "desktop-hardware", "login-rate-limit", "csv-imports", "session-history", "monitor-alerts", "administrators", "password", "auth", "resources", "desktops", "templates", "pools", "clone", "desktop-connection", "terminals", "terminal-lifecycle", "terminal-policies", "users", "batch-bindings", "bindings", "tasks", "audit"}})
|
||||
}
|
||||
func (s *Server) resources(c *gin.Context) {
|
||||
items := []model.Resource{}
|
||||
@@ -514,6 +514,7 @@ func (s *Server) RunWorker(ctx context.Context) {
|
||||
workerCtx, stop := context.WithCancel(ctx)
|
||||
provisionDone := make(chan struct{}, 1)
|
||||
provisionBusy := false
|
||||
var nextGPUCheck time.Time
|
||||
defer func() {
|
||||
stop()
|
||||
if provisionBusy {
|
||||
@@ -532,7 +533,8 @@ func (s *Server) RunWorker(ctx context.Context) {
|
||||
if err = conn.PingContext(ctx); err != nil {
|
||||
return
|
||||
}
|
||||
// Unknown outcomes are never automatically retried after a worker restart.
|
||||
// Unknown commands are never replayed. Known GPU tasks are observed
|
||||
// separately below so a transient PVE read failure can recover.
|
||||
var expired []model.Task
|
||||
if s.DB.SelectContext(ctx, &expired, "SELECT * FROM tasks WHERE status IN ('queued','running') AND ((kind NOT IN ('provision','hardware','snapshot','batch-desktop','gpu-prepare') AND created_at<now()-interval '2 minutes') OR (kind='gpu-prepare' AND created_at<now()-interval '5 minutes') OR (kind='batch-desktop' AND ((status='queued' AND created_at<now()-interval '2 hours') OR (status='running' AND updated_at<now()-interval '2 minutes'))) OR (kind IN ('provision','hardware','snapshot') AND updated_at<now()-interval '35 minutes'))") == nil {
|
||||
for _, t := range expired {
|
||||
@@ -563,6 +565,14 @@ func (s *Server) RunWorker(ctx context.Context) {
|
||||
_ = s.complete(ctx, t, "timeout", "结果未确认,请核对设备或 PVE 实际状态;不会自动重放")
|
||||
}
|
||||
}
|
||||
if !provisionBusy && !time.Now().Before(nextGPUCheck) {
|
||||
nextGPUCheck = time.Now().Add(15 * time.Second)
|
||||
if recovery, readErr := s.nextGPURecovery(ctx); readErr == nil {
|
||||
provisionBusy = true
|
||||
go func(task model.Task) { s.recoverGPUPrepare(workerCtx, task); provisionDone <- struct{}{} }(recovery)
|
||||
continue
|
||||
}
|
||||
}
|
||||
var t model.Task
|
||||
err := s.DB.GetContext(ctx, &t, `UPDATE tasks SET status='running',updated_at=now(),message='正在执行,等待最终回执' WHERE id=(SELECT id FROM tasks WHERE ((status='queued' AND kind!='terminal') OR (kind='provision' AND status='running' AND provider_ref!='') OR (kind IN ('hardware','snapshot','batch-desktop') AND status='running')) AND ($1 OR kind NOT IN ('provision','hardware','snapshot','batch-desktop','gpu-prepare')) ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING *`, !provisionBusy)
|
||||
if err != nil {
|
||||
|
||||
Reference in new issue
Block a user