功能：补充固定场景助手与GPU健康诊断能力
This commit is contained in:
songyiming committed
1 parent
9f05aa20ea
commit
b6026f9759
29 files changed
+1333
-22
No files matched your search
@@ -53,6 +53,59 @@ type diagnosticReport struct {
|
||||
Source string `json:"source"`
|
||||
Notes []string `json:"notes"`
|
||||
}
|
||||
type terminalLogArchive struct {
|
||||
ID string `json:"id" db:"id"`
|
||||
TaskID string `json:"taskId" db:"task_id"`
|
||||
TerminalID string `json:"terminalId" db:"terminal_id"`
|
||||
TerminalName string `json:"terminalName" db:"terminal_name"`
|
||||
CapturedAt time.Time `json:"capturedAt" db:"captured_at"`
|
||||
BootID string `json:"bootId" db:"boot_id"`
|
||||
AgentVersion string `json:"agentVersion" db:"agent_version"`
|
||||
KioskState string `json:"kioskState" db:"kiosk_state"`
|
||||
LogText string `json:"logText" db:"log_text"`
|
||||
}
|
||||
type terminalLogArchivePage struct {
|
||||
Items []terminalLogArchive `json:"items"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"pageSize"`
|
||||
Total int `json:"total"`
|
||||
ObservedAt time.Time `json:"observedAt"`
|
||||
RetentionDays int `json:"retentionDays"`
|
||||
}
|
||||
|
||||
func (s *Server) terminalLogArchives(c *gin.Context) {
|
||||
terminalID, keyword := strings.TrimSpace(c.Query("terminalId")), strings.TrimSpace(c.Query("keyword"))
|
||||
if len(terminalID) > 100 || len(keyword) > 200 || strings.IndexByte(terminalID, 0) >= 0 || strings.IndexByte(keyword, 0) >= 0 {
|
||||
s.fail(c, 400, "invalid_request", "日志筛选条件无效")
|
||||
return
|
||||
}
|
||||
from, to, ok := timeRange(c, 30)
|
||||
if !ok {
|
||||
s.fail(c, 400, "invalid_range", "请选择有效时间范围，最长 30 天，时间需包含时区")
|
||||
return
|
||||
}
|
||||
page, size := pageParams(c)
|
||||
args := []any{from, to, terminalID, keyword}
|
||||
var total int
|
||||
if err := s.DB.GetContext(c.Request.Context(), &total, `SELECT count(*) FROM terminal_log_archives WHERE captured_at>=$1 AND captured_at<=$2 AND ($3='' OR terminal_id=$3) AND ($4='' OR log_text ILIKE '%'||$4||'%')`, args...); err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
items := []terminalLogArchive{}
|
||||
args = append(args, size, (page-1)*size)
|
||||
if err := s.DB.SelectContext(c.Request.Context(), &items, `SELECT a.id,a.task_id,a.terminal_id,t.name terminal_name,a.captured_at,a.boot_id,a.agent_version,a.kiosk_state,a.log_text FROM terminal_log_archives a JOIN terminals t ON t.id=a.terminal_id WHERE a.captured_at>=$1 AND a.captured_at<=$2 AND ($3='' OR a.terminal_id=$3) AND ($4='' OR a.log_text ILIKE '%'||$4||'%') ORDER BY a.captured_at DESC,a.id DESC LIMIT $5 OFFSET $6`, args...); err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
for i := range items {
|
||||
items[i].BootID = redactDiagnostic(items[i].BootID)
|
||||
items[i].AgentVersion = redactDiagnostic(items[i].AgentVersion)
|
||||
items[i].KioskState = redactDiagnostic(items[i].KioskState)
|
||||
items[i].TerminalName = redactDiagnostic(items[i].TerminalName)
|
||||
items[i].LogText = redactDiagnostic(items[i].LogText)
|
||||
}
|
||||
c.JSON(200, terminalLogArchivePage{Items: items, Page: page, PageSize: size, Total: total, ObservedAt: time.Now().UTC(), RetentionDays: 30})
|
||||
}
|
||||
|
||||
// Correlation uses stable IDs, never a display name. Desktop logs are not
|
||||
// inferred from a device's current binding: that device may serve other VMs.
|
||||
|
||||
@@ -35,6 +35,129 @@ type gpuPrepareState struct {
|
||||
UPID string `json:"upid,omitempty"`
|
||||
}
|
||||
|
||||
type gpuExclusiveMember struct {
|
||||
DesktopID string `json:"desktopId"`
|
||||
Name string `json:"name"`
|
||||
VMID int `json:"vmid"`
|
||||
PowerState string `json:"powerState"`
|
||||
PowerObserved bool `json:"powerObserved"`
|
||||
ActiveSessions int `json:"activeSessions"`
|
||||
ActiveTasks int `json:"activeTasks"`
|
||||
Conflicts []string `json:"conflicts"`
|
||||
Eligible bool `json:"eligible"`
|
||||
}
|
||||
type gpuExclusiveStatus struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Node string `json:"node"`
|
||||
VMIDs []int `json:"vmids"`
|
||||
ObservedAt time.Time `json:"observedAt"`
|
||||
LockAvailable bool `json:"lockAvailable"`
|
||||
GroupComplete bool `json:"groupComplete"`
|
||||
Members []gpuExclusiveMember `json:"members"`
|
||||
Conflicts []string `json:"conflicts"`
|
||||
AllocationReady bool `json:"allocationReady"`
|
||||
ReleaseReady bool `json:"releaseReady"`
|
||||
ReleasePolicy string `json:"releasePolicy"`
|
||||
ActiveTasks []model.Task `json:"activeTasks"`
|
||||
}
|
||||
|
||||
// exclusiveGPUStatus is a read-only preflight view. It never changes VM power
|
||||
// or task state; it is used by the advanced console before allocation/release.
|
||||
func (s *Server) exclusiveGPUStatus(c *gin.Context) {
|
||||
node := strings.TrimSpace(c.Query("node"))
|
||||
status := gpuExclusiveStatus{ObservedAt: time.Now().UTC(), Node: s.Config.ExclusiveGPUNode, VMIDs: append([]int{}, s.Config.ExclusiveGPUVMIDs...), Members: []gpuExclusiveMember{}, Conflicts: []string{}, ActiveTasks: []model.Task{}, ReleasePolicy: "仅在无活动会话、无未完成任务且锁可用时可释放；不会自动关机或重放操作"}
|
||||
if node != "" && (len(node) > 100 || strings.ContainsAny(node, "/\\?#")) {
|
||||
s.fail(c, 400, "invalid_node", "节点名称无效")
|
||||
return
|
||||
}
|
||||
if status.Node == "" || len(status.VMIDs) == 0 {
|
||||
c.JSON(200, status)
|
||||
return
|
||||
}
|
||||
if node != "" && node != status.Node {
|
||||
status.Conflicts = []string{"当前节点未配置独占显卡组"}
|
||||
c.JSON(200, status)
|
||||
return
|
||||
}
|
||||
status.Enabled = true
|
||||
tx, err := s.DB.BeginTxx(c.Request.Context(), &sql.TxOptions{ReadOnly: true})
|
||||
if err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
defer tx.Rollback()
|
||||
locked, err := tryGPULock(c.Request.Context(), tx)
|
||||
if err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
status.LockAvailable = locked
|
||||
if !locked {
|
||||
status.Conflicts = append(status.Conflicts, "独占显卡锁当前被占用")
|
||||
}
|
||||
var members []model.Desktop
|
||||
members, err = s.gpuMembers(c.Request.Context(), tx)
|
||||
if err != nil {
|
||||
status.Conflicts = append(status.Conflicts, "独占显卡组成员不完整")
|
||||
} else {
|
||||
status.GroupComplete = true
|
||||
}
|
||||
var tasks []model.Task
|
||||
if err = tx.SelectContext(c.Request.Context(), &tasks, `SELECT * FROM tasks WHERE kind='gpu-prepare' AND status IN ('queued','running','timeout') ORDER BY created_at DESC`); err == nil {
|
||||
status.ActiveTasks = tasks
|
||||
if len(tasks) > 0 {
|
||||
status.Conflicts = append(status.Conflicts, "存在未完成或结果未核对的显卡切换任务")
|
||||
}
|
||||
}
|
||||
pve := s.currentPVE()
|
||||
for _, d := range members {
|
||||
m := gpuExclusiveMember{DesktopID: d.ID, Name: d.Name, VMID: d.VMID, PowerState: d.PowerState, Conflicts: []string{}}
|
||||
var n int
|
||||
if err = tx.GetContext(c.Request.Context(), &n, `SELECT count(*) FROM terminals WHERE session_desktop_id=$1 AND last_seen>now()-interval '90 seconds'`, d.ID); err == nil {
|
||||
m.ActiveSessions = n
|
||||
}
|
||||
if err = tx.GetContext(c.Request.Context(), &n, `SELECT count(*) FROM tasks WHERE target_id=$1 AND status IN ('queued','running','timeout') AND kind!='terminal'`, d.ID); err == nil {
|
||||
m.ActiveTasks = n
|
||||
}
|
||||
if m.ActiveSessions > 0 {
|
||||
m.Conflicts = append(m.Conflicts, "存在活跃终端会话")
|
||||
}
|
||||
if m.ActiveTasks > 0 {
|
||||
m.Conflicts = append(m.Conflicts, "存在未完成管理任务")
|
||||
}
|
||||
if pve != nil {
|
||||
power, powerErr := pve.Power(c.Request.Context(), d.Node, d.VMID)
|
||||
if powerErr == nil {
|
||||
m.PowerState, m.PowerObserved = power, true
|
||||
} else {
|
||||
m.PowerState = "unknown"
|
||||
m.Conflicts = append(m.Conflicts, "无法读取 PVE 实际电源状态")
|
||||
}
|
||||
} else {
|
||||
m.PowerState = "unknown"
|
||||
m.Conflicts = append(m.Conflicts, "尚未配置 PVE")
|
||||
}
|
||||
if m.PowerObserved && m.PowerState != "stopped" {
|
||||
m.Conflicts = append(m.Conflicts, "虚拟机当前仍在运行")
|
||||
}
|
||||
m.Eligible = len(m.Conflicts) == 0 && m.PowerState == "stopped"
|
||||
status.Members = append(status.Members, m)
|
||||
}
|
||||
if !status.GroupComplete {
|
||||
status.Conflicts = append(status.Conflicts, "数据库中的独占显卡组成员数量与配置不一致")
|
||||
}
|
||||
allEligible := len(status.Members) == len(status.VMIDs)
|
||||
for _, member := range status.Members {
|
||||
if !member.Eligible {
|
||||
allEligible = false
|
||||
break
|
||||
}
|
||||
}
|
||||
status.AllocationReady = status.Enabled && status.GroupComplete && status.LockAvailable && len(status.ActiveTasks) == 0 && len(status.Conflicts) == 0 && allEligible
|
||||
status.ReleaseReady = status.Enabled && status.GroupComplete && status.LockAvailable && len(status.ActiveTasks) == 0 && allEligible
|
||||
c.JSON(200, status)
|
||||
}
|
||||
|
||||
func (s *Server) exclusiveGPU(d model.Desktop) bool {
|
||||
if d.Provider != "proxmox" || d.Node != s.Config.ExclusiveGPUNode {
|
||||
return false
|
||||
|
||||
@@ -119,6 +119,23 @@ func TestExclusiveGPUAuthorizationAndOccupiedGroup(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExclusiveGPUStatusPreflightIsReadOnly(t *testing.T) {
|
||||
f, p := exclusiveFixture(t, "normal")
|
||||
admin := f.login("admin")
|
||||
code, out := f.request("GET", "/resources/gpu-exclusive?node=gpu-node", admin, nil, "")
|
||||
expect(t, 200, code, out)
|
||||
if out["enabled"] != true || out["groupComplete"] != true || out["allocationReady"] != false || out["releaseReady"] != false {
|
||||
t.Fatalf("unexpected preflight: %#v", out)
|
||||
}
|
||||
if len(p.actions) != 0 {
|
||||
t.Fatalf("preflight changed power: %#v", p.actions)
|
||||
}
|
||||
code, _ = f.request("POST", "/resources/gpu-exclusive", admin, nil, "")
|
||||
if code != 404 {
|
||||
t.Fatalf("exclusive GPU status unexpectedly exposes write route: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExclusiveGPUReservationBlocksAdministrativePower(t *testing.T) {
|
||||
f, p := exclusiveFixture(t, "normal")
|
||||
prepareExclusive(t, f)
|
||||
|
||||
@@ -37,8 +37,20 @@ type gpuItem struct {
|
||||
Class string `json:"class"`
|
||||
IOMMUGroup *int `json:"iommuGroup"`
|
||||
State string `json:"state"`
|
||||
Health gpuHealth `json:"health"`
|
||||
References []gpuReference `json:"references"`
|
||||
}
|
||||
type gpuHealth struct {
|
||||
Status string `json:"status"`
|
||||
Source string `json:"source"`
|
||||
ObservedAt time.Time `json:"observedAt"`
|
||||
TemperatureC *float64 `json:"temperatureC"`
|
||||
UtilizationPercent *float64 `json:"utilizationPercent"`
|
||||
MemoryUsedBytes *int64 `json:"memoryUsedBytes"`
|
||||
MemoryTotalBytes *int64 `json:"memoryTotalBytes"`
|
||||
Driver string `json:"driver"`
|
||||
DriverBinding string `json:"driverBinding"`
|
||||
}
|
||||
type gpuIssue struct {
|
||||
Code string `json:"code"`
|
||||
VMID int `json:"vmid"`
|
||||
@@ -92,6 +104,7 @@ func (s *Server) gpuInventory(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
items := []gpuItem{}
|
||||
observedAt := time.Now().UTC()
|
||||
for _, d := range devices {
|
||||
if !d.IsGPU() {
|
||||
continue
|
||||
@@ -105,7 +118,11 @@ func (s *Server) gpuInventory(c *gin.Context) {
|
||||
if group != nil && *group < 0 {
|
||||
group = nil
|
||||
}
|
||||
items = append(items, gpuItem{ID: node + ":" + address, Address: address, Name: d.DeviceName, Vendor: d.VendorName, VendorID: d.Vendor, DeviceID: d.Device, Class: d.Class, IOMMUGroup: group, References: []gpuReference{}})
|
||||
driverBinding := "unknown"
|
||||
if strings.TrimSpace(d.Driver) != "" {
|
||||
driverBinding = "bound"
|
||||
}
|
||||
items = append(items, gpuItem{ID: node + ":" + address, Address: address, Name: d.DeviceName, Vendor: d.VendorName, VendorID: d.Vendor, DeviceID: d.Device, Class: d.Class, IOMMUGroup: group, Health: gpuHealth{Status: "not_collected", Source: "pve-pci", ObservedAt: observedAt, Driver: d.Driver, DriverBinding: driverBinding}, References: []gpuReference{}})
|
||||
}
|
||||
fullVisibility, err := pve.HasPermission(ctx, "/", "VM.Audit")
|
||||
if err != nil || !fullVisibility {
|
||||
@@ -227,7 +244,7 @@ func (s *Server) gpuInventory(c *gin.Context) {
|
||||
}
|
||||
}
|
||||
sort.Slice(items, func(i, j int) bool { return items[i].Address < items[j].Address })
|
||||
c.JSON(200, gin.H{"node": node, "observedAt": time.Now().UTC(), "complete": len(issues) == 0, "visibleVmCount": visibleCount, "scannedVmCount": scannedCount, "items": items, "issues": issues})
|
||||
c.JSON(200, gin.H{"node": node, "observedAt": observedAt, "complete": len(issues) == 0, "visibleVmCount": visibleCount, "scannedVmCount": scannedCount, "items": items, "issues": issues, "health": gin.H{"status": "partial", "source": "pve-pci", "temperature": "not_collected", "utilization": "not_collected", "memory": "not_collected", "driverBinding": "reported_when_pve_exposes_driver"}})
|
||||
}
|
||||
|
||||
func rawOptionText(value json.RawMessage) string {
|
||||
|
||||
@@ -299,7 +299,7 @@ func (s *Server) observe(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
// Bounded retention. Current incidents and active session records are preserved.
|
||||
for _, query := range []string{"DELETE FROM login_limits WHERE window_started<now()-interval '1 day'", "DELETE FROM resource_samples WHERE sampled_at<now()-interval '30 days'", "DELETE FROM terminal_metric_samples WHERE sampled_at<now()-interval '30 days'", "DELETE FROM session_history WHERE ended_at<now()-interval '90 days'", "DELETE FROM alerts WHERE resolved_at<now()-interval '90 days'"} {
|
||||
for _, query := range []string{"DELETE FROM login_limits WHERE window_started<now()-interval '1 day'", "DELETE FROM resource_samples WHERE sampled_at<now()-interval '30 days'", "DELETE FROM terminal_metric_samples WHERE sampled_at<now()-interval '30 days'", "DELETE FROM terminal_log_archives WHERE captured_at<now()-interval '30 days'", "DELETE FROM session_history WHERE ended_at<now()-interval '90 days'", "DELETE FROM alerts WHERE resolved_at<now()-interval '90 days'"} {
|
||||
if _, err = tx.ExecContext(ctx, query); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -110,6 +110,7 @@ func New(cfg config.Config, db *sqlx.DB, pve *proxmox.Client) *Server {
|
||||
a.POST("/desktops/:id/hardware", s.allow("super_admin"), s.queueHardware)
|
||||
a.GET("/resources", s.resources)
|
||||
a.GET("/resources/gpus", s.gpuInventory)
|
||||
a.GET("/resources/gpu-exclusive", s.exclusiveGPUStatus)
|
||||
a.GET("/resources/vms", s.discoverVMs)
|
||||
a.GET("/resources/templates", s.discoverTemplates)
|
||||
a.GET("/resources/storage", s.storageList)
|
||||
@@ -160,6 +161,7 @@ func New(cfg config.Config, db *sqlx.DB, pve *proxmox.Client) *Server {
|
||||
a.GET("/tasks/:id/diagnostics", s.allow("super_admin", "operator"), s.taskDiagnostics)
|
||||
a.GET("/diagnostics", s.allow("super_admin", "operator"), s.diagnostics)
|
||||
a.GET("/diagnostics/export", s.allow("super_admin", "operator"), s.exportDiagnostics)
|
||||
a.GET("/terminal-logs", s.allow("super_admin", "operator"), s.terminalLogArchives)
|
||||
a.GET("/audit-logs", s.audits)
|
||||
a.GET("/audit-logs/export", s.exportAudits)
|
||||
r.NoRoute(func(c *gin.Context) { s.fail(c, 404, "not_found", "接口不存在") })
|
||||
|
||||
@@ -253,6 +253,11 @@ func TestDeviceIdentityBindingTicketAndAcknowledgement(t *testing.T) {
|
||||
if out["logTail"] != "private diagnostic" {
|
||||
t.Fatal("diagnostic not retained")
|
||||
}
|
||||
status, out = f.request("GET", "/terminal-logs?terminalId="+id+"&keyword=private", admin, nil, "")
|
||||
expect(t, 200, status, out)
|
||||
if out["total"] != float64(1) || len(out["items"].([]any)) != 1 || out["items"].([]any)[0].(map[string]any)["logText"] != "private diagnostic" {
|
||||
t.Fatalf("terminal log archive not searchable: %#v", out)
|
||||
}
|
||||
_ = otherID
|
||||
}
|
||||
|
||||
|
||||
@@ -233,6 +233,15 @@ func (s *Server) deviceAck(c *gin.Context) {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
// Keep a dedicated, indexed archive for long-lived terminal log search.
|
||||
// The task row remains the audit record; this copy is bounded by the
|
||||
// retention cleanup in observability.go.
|
||||
logText := redactDiagnostic(in.Diagnostics.LogTail)
|
||||
meta, _ := json.Marshal(gin.H{"agentVersion": redactDiagnostic(in.Diagnostics.AgentVersion), "kioskState": redactDiagnostic(in.Diagnostics.KioskState), "uptimeSeconds": in.Diagnostics.UptimeSeconds})
|
||||
if _, err = deviceTx(c).ExecContext(c.Request.Context(), `INSERT INTO terminal_log_archives(id,task_id,terminal_id,captured_at,boot_id,agent_version,kiosk_state,log_text,metadata) VALUES($1,$2,$3,now(),$4,$5,$6,$7,$8) ON CONFLICT(task_id) DO UPDATE SET log_text=EXCLUDED.log_text,metadata=EXCLUDED.metadata`, store.ID("log_"), task.ID, c.GetString("terminalId"), redactDiagnostic(in.Diagnostics.BootID), redactDiagnostic(in.Diagnostics.AgentVersion), redactDiagnostic(in.Diagnostics.KioskState), logText, string(meta)); err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
// Agent text can contain sensitive device details. Keep the public audit message fixed.
|
||||
if err = s.completeTx(c.Request.Context(), deviceTx(c), task, in.Status, "已收到终端执行回执"); err != nil {
|
||||
|
||||
@@ -73,6 +73,15 @@ CREATE TABLE IF NOT EXISTS task_diagnostics (
|
||||
task_id text PRIMARY KEY REFERENCES tasks(id) ON DELETE CASCADE,
|
||||
content jsonb NOT NULL, created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS terminal_log_archives (
|
||||
id text PRIMARY KEY, task_id text NOT NULL UNIQUE REFERENCES tasks(id) ON DELETE CASCADE,
|
||||
terminal_id text NOT NULL REFERENCES terminals(id) ON DELETE CASCADE,
|
||||
captured_at timestamptz NOT NULL DEFAULT now(), boot_id text NOT NULL DEFAULT '',
|
||||
agent_version text NOT NULL DEFAULT '', kiosk_state text NOT NULL DEFAULT '', log_text text NOT NULL DEFAULT '',
|
||||
metadata jsonb NOT NULL DEFAULT '{}'::jsonb
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS terminal_log_archives_terminal_time ON terminal_log_archives(terminal_id,captured_at DESC,id);
|
||||
CREATE INDEX IF NOT EXISTS terminal_log_archives_time ON terminal_log_archives(captured_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS image_templates (
|
||||
id text PRIMARY KEY, name text NOT NULL, version text NOT NULL, os text NOT NULL CHECK(os IN ('Windows','Ubuntu','麒麟')),
|
||||
node text NOT NULL, vmid integer NOT NULL CHECK(vmid>=100), digest text NOT NULL,
|
||||
|
||||
@@ -18,6 +18,7 @@ type PCIDevice struct {
|
||||
Device string `json:"device"`
|
||||
DeviceName string `json:"device_name"`
|
||||
IOMMUGroup *int `json:"iommugroup"`
|
||||
Driver string `json:"driver"`
|
||||
}
|
||||
|
||||
type PCIMapping struct {
|
||||
|
||||
Reference in new issue
Block a user