songyiming/vdi-cloud
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

功能：完善通用云管工作台、审计与诊断能力

Browse Source
This commit is contained in:
songyiming committed 2026-11-11 18:46:16 +08:00
1 parent 5ac692724d
commit 9f05aa20ea
225 files changed
+14266 -141
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

platform/internal/server/admin.go
+28 -22
View File
@@ -3,6 +3,7 @@ package server
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"github.com/gin-gonic/gin"
@@ -42,7 +43,7 @@ func (s *Server) settings(c *gin.Context) {
if pve != nil {
provider = "proxmox"
}
c.JSON(200, gin.H{"mode": s.Config.Mode, "computeProvider": provider, "pveConfigured": pve != nil, "terminalTransport": "https-poll", "capabilities": []string{"terminal-system-metrics", "gpu-inventory", "desktop-batches", "desktop-snapshots", "pve-connection", "desktop-hardware", "login-rate-limit", "csv-imports", "session-history", "monitor-alerts", "administrators", "password", "auth", "resources", "desktops", "templates", "pools", "clone", "desktop-connection", "terminals", "terminal-lifecycle", "terminal-policies", "users", "batch-bindings", "bindings", "tasks", "audit"}})
c.JSON(200, gin.H{"mode": s.Config.Mode, "computeProvider": provider, "pveConfigured": pve != nil, "terminalTransport": "https-poll", "capabilities": []string{"diagnostics-center", "admin-mfa", "administrator-sessions", "task-history", "audit-query-export", "terminal-metric-alerts", "terminal-system-metrics", "gpu-inventory", "desktop-batches", "desktop-snapshots", "pve-connection", "desktop-hardware", "login-rate-limit", "csv-imports", "session-history", "monitor-alerts", "administrators", "password", "auth", "resources", "desktops", "templates", "pools", "clone", "desktop-connection", "terminals", "terminal-lifecycle", "terminal-policies", "users", "batch-bindings", "bindings", "tasks", "audit"}})
}
func (s *Server) resources(c *gin.Context) {
items := []model.Resource{}
@@ -195,14 +196,6 @@ func (s *Server) bindings(c *gin.Context) {
}
c.JSON(200, user)
}
func (s *Server) tasks(c *gin.Context) {
items := []model.Task{}
if err := s.DB.SelectContext(c.Request.Context(), &items, "SELECT * FROM tasks ORDER BY created_at DESC LIMIT 1000"); err != nil {
s.dbError(c, err)
return
}
list(c, items)
}
func (s *Server) task(c *gin.Context) {
var t model.Task
if err := s.DB.GetContext(c.Request.Context(), &t, "SELECT * FROM tasks WHERE id=$1", c.Param("id")); err != nil {
@@ -211,14 +204,6 @@ func (s *Server) task(c *gin.Context) {
}
c.JSON(200, t)
}
func (s *Server) audits(c *gin.Context) {
items := []model.Audit{}
if err := s.DB.SelectContext(c.Request.Context(), &items, "SELECT * FROM audit_logs ORDER BY created_at DESC LIMIT 1000"); err != nil {
s.dbError(c, err)
return
}
list(c, items)
}
func (s *Server) discoverVMs(c *gin.Context) {
pve := s.currentPVE()
if pve == nil {
@@ -322,6 +307,10 @@ func (s *Server) desktopAction(c *gin.Context) {
s.fail(c, 503, "pve_unconfigured", "Proxmox 尚未配置")
return
}
if s.exclusiveGPU(d) && in.Action != "shutdown" {
s.fail(c, 409, "exclusive_gpu_power", "此桌面共用显卡，请先退出当前云桌面，再从终端选择页启动")
return
}
unresolved, err := maintenanceBlocked(c.Request.Context(), s.DB, d.ID, "", false)
if err != nil {
s.dbError(c, err)
@@ -376,11 +365,15 @@ func (s *Server) submitTask(c *gin.Context, kind, action, targetID, targetName s
}
defer tx.Rollback()
if kind == "desktop" {
var id string
if err = tx.GetContext(ctx, &id, "SELECT id FROM desktops WHERE id=$1 FOR UPDATE", targetID); err != nil {
var desktop model.Desktop
if err = tx.GetContext(ctx, &desktop, "SELECT * FROM desktops WHERE id=$1 FOR UPDATE", targetID); err != nil {
s.dbError(c, err)
return
}
if err = s.exclusiveGPUMaintenance(ctx, tx, desktop); err != nil {
s.fail(c, 409, "gpu_switch_busy", err.Error())
return
}
unresolved, err := maintenanceBlocked(ctx, tx, targetID, "", false)
if err != nil {
s.dbError(c, err)
@@ -541,8 +534,17 @@ func (s *Server) RunWorker(ctx context.Context) {
}
// Unknown outcomes are never automatically retried after a worker restart.
var expired []model.Task
if s.DB.SelectContext(ctx, &expired, "SELECT * FROM tasks WHERE status IN ('queued','running') AND ((kind NOT IN ('provision','hardware','snapshot','batch-desktop') AND created_at<now()-interval '2 minutes') OR (kind='batch-desktop' AND ((status='queued' AND created_at<now()-interval '2 hours') OR (status='running' AND updated_at<now()-interval '2 minutes'))) OR (kind IN ('provision','hardware','snapshot') AND updated_at<now()-interval '35 minutes'))") == nil {
if s.DB.SelectContext(ctx, &expired, "SELECT * FROM tasks WHERE status IN ('queued','running') AND ((kind NOT IN ('provision','hardware','snapshot','batch-desktop','gpu-prepare') AND created_at<now()-interval '2 minutes') OR (kind='gpu-prepare' AND created_at<now()-interval '5 minutes') OR (kind='batch-desktop' AND ((status='queued' AND created_at<now()-interval '2 hours') OR (status='running' AND updated_at<now()-interval '2 minutes'))) OR (kind IN ('provision','hardware','snapshot') AND updated_at<now()-interval '35 minutes'))") == nil {
for _, t := range expired {
if t.Kind == "gpu-prepare" {
var state gpuPrepareState
status := "timeout"
if json.Unmarshal([]byte(t.ProviderRef), &state) == nil && state.Phase == "queued" {
status = "failed"
}
_ = s.complete(ctx, t, status, "桌面准备超时，请核对任务和实际电源状态；不会自动重放")
continue
}
if t.Kind == "batch-desktop" {
s.finishBatchPower(t, "", "expired", "")
continue
@@ -562,11 +564,11 @@ func (s *Server) RunWorker(ctx context.Context) {
}
}
var t model.Task
err := s.DB.GetContext(ctx, &t, `UPDATE tasks SET status='running',updated_at=now(),message='正在执行，等待最终回执' WHERE id=(SELECT id FROM tasks WHERE ((status='queued' AND kind!='terminal') OR (kind='provision' AND status='running' AND provider_ref!='') OR (kind IN ('hardware','snapshot','batch-desktop') AND status='running')) AND ($1 OR kind NOT IN ('provision','hardware','snapshot','batch-desktop')) ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING *`, !provisionBusy)
err := s.DB.GetContext(ctx, &t, `UPDATE tasks SET status='running',updated_at=now(),message='正在执行，等待最终回执' WHERE id=(SELECT id FROM tasks WHERE ((status='queued' AND kind!='terminal') OR (kind='provision' AND status='running' AND provider_ref!='') OR (kind IN ('hardware','snapshot','batch-desktop') AND status='running')) AND ($1 OR kind NOT IN ('provision','hardware','snapshot','batch-desktop','gpu-prepare')) ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 1) RETURNING *`, !provisionBusy)
if err != nil {
continue
}
if t.Kind == "provision" || t.Kind == "hardware" || t.Kind == "snapshot" || t.Kind == "batch-desktop" {
if t.Kind == "provision" || t.Kind == "hardware" || t.Kind == "snapshot" || t.Kind == "batch-desktop" || t.Kind == "gpu-prepare" {
provisionBusy = true
go func(task model.Task) { s.execute(workerCtx, task); provisionDone <- struct{}{} }(t)
} else {
@@ -594,6 +596,10 @@ func (s *Server) RunSync(ctx context.Context) {
}
}
func (s *Server) execute(parent context.Context, t model.Task) {
if t.Kind == "gpu-prepare" {
s.executeGPUPrepare(parent, t)
return
}
if t.Kind == "batch-desktop" {
s.executeBatchPower(parent, t)
return
platform/internal/server/admin_sessions.go
+125
View File
@@ -0,0 +1,125 @@
package server
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"strings"
"time"
"unicode"
"github.com/gin-gonic/gin"
)
type administratorSession struct {
ID string `json:"-" db:"id"`
Key string `json:"key" db:"-"`
CreatedAt *time.Time `json:"createdAt" db:"created_at"`
ExpiresAt time.Time `json:"expiresAt" db:"expires_at"`
ClientIP string `json:"clientIp" db:"client_ip"`
UserAgent string `json:"userAgent" db:"user_agent"`
Current bool `json:"current" db:"-"`
}
// A management handle must never disclose the JWT's session identifier.
func (s *Server) administratorSessionKey(id string) string {
mac := hmac.New(sha256.New, []byte(s.Config.JWTSecret))
mac.Write([]byte("vdi-admin-session-management\x00" + id))
return hex.EncodeToString(mac.Sum(nil))
}
func sessionUserAgent(raw string) string {
runes := []rune(strings.Map(func(r rune) rune {
if unicode.IsControl(r) {
return ' '
}
return r
}, raw))
if len(runes) > 500 {
runes = runes[:500]
}
return string(runes)
}
func (s *Server) administratorSessions(c *gin.Context) {
tx, _ := s.accountTransaction(c, true)
if tx == nil {
return
}
defer tx.Rollback()
var username string
if err := tx.GetContext(c.Request.Context(), &username, "SELECT username FROM administrators WHERE id=$1", c.Param("id")); err != nil {
s.dbError(c, err)
return
}
items := []administratorSession{}
err := tx.SelectContext(c.Request.Context(), &items, `SELECT x.id,x.expires_at,d.created_at,COALESCE(d.client_ip,'') client_ip,COALESCE(d.user_agent,'') user_agent
FROM admin_sessions x LEFT JOIN admin_session_details d ON d.session_id=x.id
WHERE x.admin_id=$1 AND x.expires_at>now() ORDER BY x.expires_at DESC,x.id DESC`, c.Param("id"))
if err != nil {
s.dbError(c, err)
return
}
for i := range items {
items[i].Key = s.administratorSessionKey(items[i].ID)
items[i].Current = items[i].ID == c.GetString("sessionId")
}
list(c, items)
}
func (s *Server) revokeAdministratorSession(c *gin.Context) {
key := c.Param("sessionKey")
if len(key) != 64 {
s.fail(c, 404, "not_found", "登录会话不存在或已失效")
return
}
tx, actor := s.accountTransaction(c, true)
if tx == nil {
return
}
defer tx.Rollback()
ctx := c.Request.Context()
var username string
if err := tx.GetContext(ctx, &username, "SELECT username FROM administrators WHERE id=$1 FOR UPDATE", c.Param("id")); err != nil {
s.dbError(c, err)
return
}
var ids []string
if err := tx.SelectContext(ctx, &ids, "SELECT id FROM admin_sessions WHERE admin_id=$1 AND expires_at>now()", c.Param("id")); err != nil {
s.dbError(c, err)
return
}
for _, id := range ids {
if !hmac.Equal([]byte(s.administratorSessionKey(id)), []byte(key)) {
continue
}
if id == c.GetString("sessionId") {
s.fail(c, 409, "current_session_protected", "当前登录请使用退出登录入口")
return
}
result, err := tx.ExecContext(ctx, "DELETE FROM admin_sessions WHERE id=$1 AND admin_id=$2 AND expires_at>now()", id, c.Param("id"))
if err != nil {
s.dbError(c, err)
return
}
count, err := result.RowsAffected()
if err != nil {
s.dbError(c, err)
return
}
if count == 0 {
break
}
err = accountAudit(ctx, tx, actor.Username, "revoke_administrator_session", username+":"+key, c.GetString("requestId"))
if err == nil {
err = tx.Commit()
}
if err != nil {
s.dbError(c, err)
return
}
c.Status(204)
return
}
s.fail(c, 404, "not_found", "登录会话不存在或已失效")
}
platform/internal/server/admin_sessions_test.go
+194
View File
@@ -0,0 +1,194 @@
package server
import (
"encoding/json"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"github.com/golang-jwt/jwt/v5"
)
func rawAdminSession(t *testing.T, f *fixture, token string) string {
t.Helper()
c := &claims{}
_, err := jwt.ParseWithClaims(token, c, func(*jwt.Token) (any, error) { return []byte(f.server.Config.JWTSecret), nil })
if err != nil {
t.Fatal(err)
}
return c.ID
}
func TestAdminSessionsSelectiveRevocationAndAudit(t *testing.T) {
f := newFixtureWorker(t, false)
owner := f.login("admin")
target := createTestAccount(t, f, owner, "session-reader", "viewer")
first := accountLogin(t, f, "session-reader", testAccountPassword)
second := accountLogin(t, f, "session-reader", testAccountPassword)
raw := rawAdminSession(t, f, first)
key := f.server.administratorSessionKey(raw)
code, out := f.request("GET", "/administrators/"+target+"/sessions", owner, nil, "")
expect(t, 200, code, out)
if out["total"] != float64(2) {
t.Fatal(out)
}
encoded, _ := json.Marshal(out)
if strings.Contains(string(encoded), raw) || strings.Contains(string(encoded), first) {
t.Fatal("credential material disclosed")
}
code, out = f.request("DELETE", "/administrators/"+target+"/sessions/"+key, owner, nil, "")
expect(t, 204, code, out)
code, out = f.request("GET", "/auth/me", first, nil, "")
expect(t, 401, code, out)
code, out = f.request("GET", "/auth/me", second, nil, "")
expect(t, 200, code, out)
code, out = f.request("DELETE", "/administrators/"+target+"/sessions/"+key, owner, nil, "")
expect(t, 404, code, out)
var count int
f.db.Get(&count, "SELECT count(*) FROM admin_session_details WHERE session_id=$1", raw)
if count != 0 {
t.Fatal("metadata not cascaded")
}
var audit string
if err := f.db.Get(&audit, "SELECT target FROM audit_logs WHERE action='revoke_administrator_session'"); err != nil {
t.Fatal(err)
}
if audit != "session-reader:"+key || strings.Contains(audit, raw) {
t.Fatal("wrong audit handle")
}
}
func TestAdminSessionsPermissionsMetadataAndLegacy(t *testing.T) {
f := newFixtureWorker(t, false)
owner := f.login("admin")
var target string
f.db.Get(&target, "SELECT id FROM administrators WHERE username='admin'")
raw := rawAdminSession(t, f, owner)
key := f.server.administratorSessionKey(raw)
path := "/administrators/" + target + "/sessions"
for _, user := range []string{"", "viewer", "operator"} {
token := ""
want := 401
if user != "" {
token = f.login(user)
want = 403
}
for _, req := range []struct{ method, path string }{{"GET", path}, {"DELETE", path + "/" + key}} {
code, out := f.request(req.method, req.path, token, nil, "")
expect(t, want, code, out)
}
}
code, out := f.request("GET", path, owner, nil, "")
expect(t, 200, code, out)
row := out["items"].([]any)[0].(map[string]any)
if row["current"] != true || row["createdAt"] == nil {
t.Fatal(row)
}
code, out = f.request("DELETE", path+"/"+key, owner, nil, "")
expect(t, 409, code, out)
code, out = f.request("DELETE", path+"/"+raw, owner, nil, "")
expect(t, 404, code, out)
code, out = f.request("GET", "/administrators/missing/sessions", owner, nil, "")
expect(t, 404, code, out)
// Pre-upgrade sessions remain valid, with unknown source/time rather than invented metadata.
f.db.MustExec("DELETE FROM admin_session_details WHERE session_id=$1", raw)
code, out = f.request("GET", path, owner, nil, "")
expect(t, 200, code, out)
row = out["items"].([]any)[0].(map[string]any)
if row["createdAt"] != nil || row["clientIp"] != "" || row["userAgent"] != "" {
t.Fatal(row)
}
f.db.MustExec("INSERT INTO admin_sessions VALUES('expired-session',$1,now()-interval '1 second')", target)
code, out = f.request("GET", path, owner, nil, "")
expect(t, 200, code, out)
if out["total"] != float64(1) {
t.Fatal(out)
}
code, out = f.request("DELETE", path+"/"+f.server.administratorSessionKey("expired-session"), owner, nil, "")
expect(t, 404, code, out)
request := httptest.NewRequest("POST", "/api/v1/auth/login", strings.NewReader(`{"username":"viewer","password":"Demo@2026"}`))
request.RemoteAddr = "192.0.2.8:42000"
request.Header.Set("Content-Type", "application/json")
request.Header.Set("X-Forwarded-For", "198.51.100.99")
request.Header.Set("User-Agent", "Client\n"+strings.Repeat("界", 600))
response := httptest.NewRecorder()
f.server.Router.ServeHTTP(response, request)
if response.Code != 200 {
t.Fatal(response.Body.String())
}
var source struct {
IP string `db:"client_ip"`
Agent string `db:"user_agent"`
}
if err := f.db.Get(&source, "SELECT client_ip,user_agent FROM admin_session_details d JOIN admin_sessions x ON x.id=d.session_id JOIN administrators a ON a.id=x.admin_id WHERE a.username='viewer' ORDER BY d.created_at DESC LIMIT 1"); err != nil {
t.Fatal(err)
}
if source.IP != "192.0.2.8" || len([]rune(source.Agent)) != 500 || strings.Contains(source.Agent, "\n") {
t.Fatal("invalid login source")
}
}
func TestAdminSessionsConcurrentRevocationAndActorRecheck(t *testing.T) {
f := newFixtureWorker(t, false)
owner := f.login("admin")
target := createTestAccount(t, f, owner, "concurrent-sessions", "viewer")
token := accountLogin(t, f, "concurrent-sessions", testAccountPassword)
path := "/administrators/" + target + "/sessions/" + f.server.administratorSessionKey(rawAdminSession(t, f, token))
codes := make(chan int, 2)
var wg sync.WaitGroup
for i := 0; i < 2; i++ {
wg.Add(1)
go func() { defer wg.Done(); code, _ := f.request("DELETE", path, owner, nil, ""); codes <- code }()
}
wg.Wait()
close(codes)
sum := 0
for code := range codes {
if code != 204 && code != 404 {
t.Fatal(code)
}
sum += code
}
if sum != 608 {
t.Fatal("duplicate successful revocation", sum)
}
var count int
f.db.Get(&count, "SELECT count(*) FROM audit_logs WHERE action='revoke_administrator_session'")
if count != 1 {
t.Fatal(count)
}
// Middleware authenticated before revocation must not authorize a write after waiting on the account lock.
tx := f.db.MustBegin()
defer tx.Rollback()
tx.MustExec("SELECT pg_advisory_xact_lock(7651092028)")
done := make(chan int, 1)
go func() { code, _ := f.request("DELETE", path, owner, nil, ""); done <- code }()
deadline := time.Now().Add(3 * time.Second)
for {
var waiting int
if err := f.db.Get(&waiting, "SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='advisory' AND pid<>pg_backend_pid()"); err != nil {
t.Fatal(err)
}
if waiting > 0 {
break
}
if time.Now().After(deadline) {
t.Fatal("request did not reach account transaction")
}
time.Sleep(10 * time.Millisecond)
}
tx.MustExec("DELETE FROM admin_sessions WHERE id=$1", rawAdminSession(t, f, owner))
if err := tx.Commit(); err != nil {
t.Fatal(err)
}
select {
case code := <-done:
if code != 401 {
t.Fatal(code)
}
case <-time.After(3 * time.Second):
t.Fatal("blocked request")
}
}
platform/internal/server/audits.go
+150
View File
@@ -0,0 +1,150 @@
package server
import (
"bytes"
"database/sql"
"encoding/csv"
"fmt"
"strings"
"time"
"unicode"
"unicode/utf8"
"github.com/gin-gonic/gin"
"vdi-cloud/platform/internal/model"
"vdi-cloud/platform/internal/store"
)
const auditExportLimit = 10000
// Filters are shared by the paginated view and export. No implicit history cap.
func (s *Server) auditFilter(c *gin.Context) (string, []any, bool) {
where, args := " FROM audit_logs WHERE true", []any{}
add := func(expr string, value any) {
args = append(args, value)
where += fmt.Sprintf(" AND "+expr, len(args))
}
var from, to time.Time
for _, key := range []string{"from", "to"} {
if raw := c.Query(key); raw != "" {
parsed, err := time.Parse(time.RFC3339Nano, raw)
if err != nil {
s.fail(c, 400, "invalid_range", "时间格式无效，请使用含时区的时间")
return "", nil, false
}
if key == "from" {
from = parsed
add("created_at >= $%d", parsed)
} else {
to = parsed
add("created_at <= $%d", parsed)
}
}
}
if !from.IsZero() && !to.IsZero() && !to.After(from) {
s.fail(c, 400, "invalid_range", "结束时间必须晚于开始时间")
return "", nil, false
}
for _, key := range []string{"keyword", "actor", "action", "result"} {
value := c.Query(key)
if utf8.RuneCountInString(value) > 200 || strings.IndexByte(value, 0) >= 0 {
s.fail(c, 400, "invalid_request", "筛选条件最长 200 字，且不能包含空字符")
return "", nil, false
}
if value == "" {
continue
}
if key == "action" || key == "result" {
add(key+" = $%d", value)
continue
}
// Search terms are literal, including SQL LIKE wildcards supplied by users.
pattern := "%" + strings.NewReplacer(`\`, `\\`, "%", `\%`, "_", `\_`).Replace(value) + "%"
if key == "actor" {
add("actor ILIKE $%d", pattern)
} else {
args = append(args, pattern)
p := fmt.Sprintf("$%d", len(args))
where += " AND (id ILIKE " + p + " OR actor ILIKE " + p + " OR action ILIKE " + p + " OR target ILIKE " + p + " OR result ILIKE " + p + " OR request_id ILIKE " + p + ")"
}
}
return where, args, true
}
func (s *Server) audits(c *gin.Context) {
where, args, ok := s.auditFilter(c)
if !ok {
return
}
page, size := pageParams(c)
tx, err := s.DB.BeginTxx(c.Request.Context(), &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
if err != nil {
s.dbError(c, err)
return
}
defer tx.Rollback()
var total int
if err = tx.GetContext(c.Request.Context(), &total, "SELECT count(*)"+where, args...); err != nil {
s.dbError(c, err)
return
}
items := []model.Audit{}
query := "SELECT *" + where + fmt.Sprintf(" ORDER BY created_at DESC,id DESC LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2)
args = append(args, size, (page-1)*size)
if err = tx.SelectContext(c.Request.Context(), &items, query, args...); err != nil {
s.dbError(c, err)
return
}
c.JSON(200, gin.H{"items": items, "total": total, "page": page, "pageSize": size, "requestId": c.GetString("requestId")})
}
func safeCSVCell(value string) string {
trimmed := strings.TrimLeftFunc(value, func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) || r == '\ufeff' })
if len(trimmed) > 0 && strings.ContainsRune("=+-@", rune(trimmed[0])) {
return "'" + value
}
return value
}
func (s *Server) exportAudits(c *gin.Context) {
where, args, ok := s.auditFilter(c)
if !ok {
return
}
// An extra row detects oversized exports without silently truncating evidence.
items := []model.Audit{}
if err := s.DB.SelectContext(c.Request.Context(), &items, "SELECT *"+where+" ORDER BY created_at DESC,id DESC LIMIT 10001", args...); err != nil {
s.dbError(c, err)
return
}
if len(items) > auditExportLimit {
s.fail(c, 413, "export_too_large", "匹配记录超过 10000 条，请缩小时间范围或增加筛选条件后导出")
return
}
var content bytes.Buffer
content.WriteString("\ufeff")
w := csv.NewWriter(&content)
w.UseCRLF = true
_ = w.Write([]string{"时间（UTC）", "操作者", "动作", "对象", "结果", "请求标识", "审计标识"})
for _, row := range items {
cells := []string{row.CreatedAt.UTC().Format(time.RFC3339Nano), row.Actor, row.Action, row.Target, row.Result, row.RequestID, row.ID}
for i := range cells {
cells[i] = safeCSVCell(cells[i])
}
_ = w.Write(cells)
}
w.Flush()
if err := w.Error(); err != nil {
s.fail(c, 500, "export_failed", "导出生成失败，请重试")
return
}
// Record preparation, without storing credentials or the user's search terms.
if _, err := s.DB.ExecContext(c.Request.Context(), "INSERT INTO audit_logs(id,actor,action,target,result,request_id) VALUES($1,$2,'export_audit_logs',$3,'success',$4)", store.ID("audit_"), s.admin(c).Username, fmt.Sprintf("rows=%d", len(items)), c.GetString("requestId")); err != nil {
s.dbError(c, err)
return
}
c.Header("Content-Disposition", `attachment; filename="audit-logs-`+time.Now().UTC().Format("20060102-150405")+`.csv"`)
c.Header("X-Export-Count", fmt.Sprint(len(items)))
c.Header("X-Content-Type-Options", "nosniff")
c.Data(200, "text/csv; charset=utf-8", content.Bytes())
}
platform/internal/server/audits_test.go
+133
View File
@@ -0,0 +1,133 @@
package server
import (
"encoding/csv"
"fmt"
"net/http/httptest"
"net/url"
"strings"
"testing"
)
func TestAuditHistoryFiltersAndDatabasePagination(t *testing.T) {
f := newFixtureWorker(t, false)
token := f.login("viewer")
_, err := f.db.Exec(`INSERT INTO audit_logs SELECT 'history_'||lpad(i::text,5,'0'),'worker','save_alert_rule','node_'||i,'success','request_'||i,'2025-01-01T00:00:00Z'::timestamptz FROM generate_series(1,1050) i`)
if err != nil {
t.Fatal(err)
}
if _, err = f.db.Exec(`INSERT INTO audit_logs VALUES('needle','Root_100%','save_alert_rule','研发,"一组"','denied','req_trace','2025-01-02T00:00:00Z')`); err != nil {
t.Fatal(err)
}
code, out := f.request("GET", "/audit-logs?action=save_alert_rule&page=11&pageSize=100", token, nil, "")
expect(t, 200, code, out)
items := out["items"].([]any)
if out["total"] != float64(1051) || len(items) != 51 || items[50].(map[string]any)["id"] != "history_00001" {
t.Fatal(out)
}
q := url.Values{"keyword": {"req_trace"}, "actor": {"_100%"}, "action": {"save_alert_rule"}, "result": {"denied"}, "from": {"2025-01-01T08:00:00+08:00"}, "to": {"2025-01-02T08:00:00+08:00"}}
code, out = f.request("GET", "/audit-logs?"+q.Encode(), token, nil, "")
expect(t, 200, code, out)
if out["total"] != float64(1) || out["items"].([]any)[0].(map[string]any)["id"] != "needle" {
t.Fatal(out)
}
for _, term := range []string{"_100%", "研发,", "' OR true --"} {
code, out = f.request("GET", "/audit-logs?keyword="+url.QueryEscape(term), token, nil, "")
expect(t, 200, code, out)
want := float64(0)
if term == "_100%" || term == "研发," {
want = 1
}
if out["total"] != want {
t.Fatal(term, out)
}
}
code, out = f.request("GET", "/audit-logs?page=99999999&pageSize=999", token, nil, "")
expect(t, 200, code, out)
if out["page"] != float64(1000000) || out["pageSize"] != float64(100) || len(out["items"].([]any)) != 0 {
t.Fatal(out)
}
for _, query := range []string{"from=broken", "to=broken", "from=2025-01-02T00:00:00Z&to=2025-01-01T00:00:00Z", "keyword=" + strings.Repeat("x", 201), "actor=%00"} {
for _, path := range []string{"/audit-logs?", "/audit-logs/export?"} {
code, out = f.request("GET", path+query, token, nil, "")
expect(t, 400, code, out)
}
}
}
func auditDownload(f *fixture, token, query string) *httptest.ResponseRecorder {
r := httptest.NewRequest("GET", "/api/v1/audit-logs/export"+query, nil)
if token != "" {
r.Header.Set("Authorization", "Bearer "+token)
}
w := httptest.NewRecorder()
f.server.Router.ServeHTTP(w, r)
return w
}
func TestAuditCSVExportPermissionsCompletenessAndSafety(t *testing.T) {
f := newFixtureWorker(t, false)
if _, err := f.db.Exec(`INSERT INTO audit_logs VALUES('csv-special','=malicious','custom_action',$1,'success','req-csv','2025-01-01T00:00:00Z')`, "研发,\"一组\"\r\n第二行"); err != nil {
t.Fatal(err)
}
w := auditDownload(f, "", "")
if w.Code != 401 {
t.Fatal(w.Code)
}
for _, role := range []string{"admin", "operator", "viewer"} {
token := f.login(role)
w = auditDownload(f, token, "?action=custom_action&page=999&pageSize=1")
if w.Code != 200 || w.Header().Get("X-Export-Count") != "1" || !strings.HasPrefix(w.Header().Get("Content-Type"), "text/csv") || !strings.Contains(w.Header().Get("Content-Disposition"), ".csv") {
t.Fatal(w.Code, w.Body.String())
}
body := w.Body.String()
if !strings.HasPrefix(body, "\ufeff") {
t.Fatal("missing Excel UTF-8 BOM")
}
rows, err := csv.NewReader(strings.NewReader(strings.TrimPrefix(body, "\ufeff"))).ReadAll()
if err != nil || len(rows) != 2 || len(rows[0]) != 7 || rows[1][1] != "'=malicious" || rows[1][3] != "研发,\"一组\"\n第二行" || rows[1][6] != "csv-special" {
t.Fatal(rows, err)
}
var logged int
if err := f.db.Get(&logged, "SELECT count(*) FROM audit_logs WHERE action='export_audit_logs' AND actor=$1 AND target='rows=1'", role); err != nil || logged != 1 {
t.Fatal(logged, err)
}
}
for _, cell := range []string{"=1+2", "+cmd", "-1", "@SUM(A1)", "\t=HYPERLINK(\"x\")", "\r\n @cmd", "\ufeff=cmd"} {
if safeCSVCell(cell) != "'"+cell {
t.Fatalf("unsafe: %q", cell)
}
}
for _, cell := range []string{"研发", "", "request-id", "success", "2025-01-01"} {
if safeCSVCell(cell) != cell {
t.Fatal(cell)
}
}
}
func TestAuditExportLimitAndEmptyResults(t *testing.T) {
f := newFixtureWorker(t, false)
token := f.login("admin")
if _, err := f.db.Exec(`INSERT INTO audit_logs SELECT 'bulk_'||i,'worker','bulk','target','success','request','2025-01-01T00:00:00Z'::timestamptz FROM generate_series(1,10001) i`); err != nil {
t.Fatal(err)
}
w := auditDownload(f, token, "?action=bulk")
if w.Code != 413 || strings.Contains(w.Header().Get("Content-Type"), "csv") {
t.Fatal(w.Code, w.Body.String())
}
if _, err := f.db.Exec("DELETE FROM audit_logs WHERE id='bulk_10001'"); err != nil {
t.Fatal(err)
}
w = auditDownload(f, token, "?action=bulk&pageSize=1")
if w.Code != 200 || w.Header().Get("X-Export-Count") != fmt.Sprint(auditExportLimit) {
t.Fatal(w.Code)
}
rows, err := csv.NewReader(strings.NewReader(strings.TrimPrefix(w.Body.String(), "\ufeff"))).ReadAll()
if err != nil || len(rows) != auditExportLimit+1 {
t.Fatal(len(rows), err)
}
w = auditDownload(f, token, "?action=nonexistent")
if w.Code != 200 || w.Header().Get("X-Export-Count") != "0" {
t.Fatal(w.Code)
}
}
platform/internal/server/diagnostics.go
+253
View File
@@ -0,0 +1,253 @@
package server
import (
"archive/zip"
"bytes"
"database/sql"
"encoding/json"
"fmt"
"net/http"
"regexp"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/jmoiron/sqlx"
"vdi-cloud/platform/internal/store"
)
const diagnosticExportLimit = 2000
type diagnosticSubject struct {
Kind string `json:"kind" db:"kind"`
ID string `json:"id" db:"id"`
Name string `json:"name" db:"name"`
State string `json:"state" db:"state"`
Node string `json:"node" db:"node"`
VMID int `json:"vmid" db:"vmid"`
AgentVersion string `json:"agentVersion" db:"agent_version"`
LastSeen *time.Time `json:"lastSeen" db:"last_seen"`
}
type diagnosticEvent struct {
ID string `json:"id" db:"id"`
Source string `json:"source" db:"source"`
Time time.Time `json:"time" db:"time"`
Action string `json:"action" db:"action"`
Status string `json:"status" db:"status"`
TargetID string `json:"targetId" db:"target_id"`
TargetName string `json:"targetName" db:"target_name"`
Actor string `json:"actor" db:"actor"`
Message string `json:"message" db:"message"`
Details string `json:"details" db:"details"`
}
type diagnosticReport struct {
Subject diagnosticSubject `json:"subject"`
From time.Time `json:"from"`
To time.Time `json:"to"`
ObservedAt time.Time `json:"observedAt"`
Items []diagnosticEvent `json:"items"`
Counts map[string]int `json:"counts"`
Total int `json:"total"`
Page int `json:"page"`
PageSize int `json:"pageSize"`
Source string `json:"source"`
Notes []string `json:"notes"`
}
// Correlation uses stable IDs, never a display name. Desktop logs are not
// inferred from a device's current binding: that device may serve other VMs.
const diagnosticEventsSQL = `WITH scoped_tasks AS (
SELECT * FROM tasks WHERE
($4='terminal' AND ((kind='terminal' AND target_id=$1) OR (kind='gpu-prepare' AND actor='device:'||$1)))
OR ($4='desktop' AND target_id=$1 AND kind!='terminal')
), scoped_sessions AS (
SELECT * FROM session_history WHERE ($4='terminal' AND terminal_id=$1) OR ($4='desktop' AND desktop_id=$1)
), scoped_alerts AS (
SELECT * FROM alerts WHERE ($4='terminal' AND target_id=$1 AND metric LIKE 'terminal_%')
OR ($4='desktop' AND (target_id=$1 OR (target_id=$5 AND metric LIKE 'node_%')))
), events AS (
SELECT 'task:'||id id,'tasks' source,created_at time,action,status,target_id,target_name,actor,
'任务提交；状态为查询时当前状态' message,jsonb_build_object('taskId',id,'kind',kind)::text details FROM scoped_tasks
UNION ALL
SELECT 'task-update:'||id,'tasks',updated_at,action,status,target_id,target_name,actor,message,
jsonb_build_object('taskId',id,'kind',kind)::text FROM scoped_tasks WHERE updated_at>created_at
UNION ALL
SELECT 'audit:'||a.id,'audits',a.created_at,a.action,a.result,a.target,a.target,a.actor,'配置或操作审计',
jsonb_build_object('auditId',a.id,'requestId',a.request_id)::text FROM audit_logs a
WHERE a.target=$1 OR a.target IN (SELECT id FROM scoped_tasks) OR a.request_id IN (SELECT id FROM scoped_tasks)
UNION ALL
SELECT 'session-start:'||id,'sessions',reserved_at,'session-reserved','connecting',desktop_id,desktop_name,username,
terminal_name||' → '||desktop_name,jsonb_build_object('historyId',id,'terminalId',terminal_id,'terminalName',terminal_name,'desktopId',desktop_id,'source',source)::text FROM scoped_sessions
UNION ALL
SELECT 'session-connected:'||id,'sessions',connected_at,'session-connected','connected',desktop_id,desktop_name,username,
terminal_name||' → '||desktop_name,jsonb_build_object('historyId',id,'terminalId',terminal_id,'terminalName',terminal_name,'source',source)::text FROM scoped_sessions WHERE connected_at IS NOT NULL
UNION ALL
SELECT 'session-ended:'||id,'sessions',ended_at,'session-ended',state,desktop_id,desktop_name,username,end_reason,
jsonb_build_object('historyId',id,'terminalId',terminal_id,'terminalName',terminal_name)::text FROM scoped_sessions WHERE ended_at IS NOT NULL
UNION ALL
SELECT 'alert-open:'||id,'alerts',opened_at,'alert-opened','firing',target_id,target_name,'',rule_name,
jsonb_build_object('alertId',id,'metric',metric,'value',value,'threshold',threshold)::text FROM scoped_alerts
UNION ALL
SELECT 'alert-ack:'||id,'alerts',acknowledged_at,'acknowledge_alert','acknowledged',target_id,target_name,acknowledged_by,rule_name,
jsonb_build_object('alertId',id)::text FROM scoped_alerts WHERE acknowledged_at IS NOT NULL
UNION ALL
SELECT 'alert-resolved:'||id,'alerts',resolved_at,'alert-resolved','resolved',target_id,target_name,'',rule_name,
jsonb_build_object('alertId',id,'reason',resolve_reason)::text FROM scoped_alerts WHERE resolved_at IS NOT NULL
UNION ALL
SELECT 'log:'||t.id,'logs',d.created_at,'collect-logs','success',t.target_id,t.target_name,t.actor,'终端已回传的日志片段',
jsonb_build_object('taskId',t.id,'agentVersion',left(d.content->>'agentVersion',80),'bootId',left(d.content->>'bootId',80),
'uptimeSeconds',d.content->'uptimeSeconds','kioskState',left(d.content->>'kioskState',80),'logTail',left(d.content->>'logTail',16384))::text
FROM task_diagnostics d JOIN scoped_tasks t ON t.id=d.task_id WHERE $4='terminal' AND t.kind='terminal' AND t.action='collect-logs' AND t.status='success'
), filtered AS (SELECT * FROM events WHERE time>=$2 AND time<=$3)
`
// Redact common credential formats in free text before it reaches either the
// browser or the archive. No provider payload, ticket or activation data is read.
var diagnosticRedactions = []*regexp.Regexp{
regexp.MustCompile(`(?is)-----BEGIN [^-]*PRIVATE KEY-----.*?-----END [^-]*PRIVATE KEY-----`),
regexp.MustCompile(`(?i)\b(?:Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+`),
regexp.MustCompile(`(?i)\bPVEAPIToken=[^\s"'<>]+`),
regexp.MustCompile(`(?i)\b(?:device|activate|ticket)_[a-f0-9]{16,}\b`),
regexp.MustCompile(`\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b`),
regexp.MustCompile(`(?i)(?:"|')?(?:password|passwd|token|access[_-]?token|refresh[_-]?token|activation[_-]?secret|secret|api[_-]?key|authorization|cookie|ticket)(?:"|')?\s*[:=]\s*(?:"[^"\r\n]*"|'[^'\r\n]*'|[^\s&,;\r\n]+)`),
regexp.MustCompile(`(?i)\b[a-z][a-z0-9+.-]*://[^\s/@]+:[^\s/@]+@`),
}
func redactDiagnostic(value string) string {
for _, pattern := range diagnosticRedactions {
value = pattern.ReplaceAllString(value, "[REDACTED]")
}
return value
}
func sanitizeDiagnosticEvent(e *diagnosticEvent) {
e.TargetName = redactDiagnostic(e.TargetName)
e.Actor = redactDiagnostic(e.Actor)
e.Message = redactDiagnostic(e.Message)
var fields map[string]any
if json.Unmarshal([]byte(e.Details), &fields) != nil {
e.Details = "{}"
return
}
for key, value := range fields {
if text, ok := value.(string); ok {
fields[key] = redactDiagnostic(text)
}
}
data, _ := json.Marshal(fields)
e.Details = string(data)
}
func (s *Server) diagnosticReport(c *gin.Context, export bool) (*diagnosticReport, bool) {
kind, id, source := c.Query("targetType"), c.Query("targetId"), c.Query("source")
if (kind != "terminal" && kind != "desktop") || id == "" || len(id) > 100 || strings.IndexByte(id, 0) >= 0 || (source != "" && source != "tasks" && source != "sessions" && source != "alerts" && source != "audits" && source != "logs") {
s.fail(c, 400, "invalid_request", "请选择设备或桌面，以及有效的记录来源")
return nil, false
}
from, to, ok := timeRange(c, 7)
if !ok {
s.fail(c, 400, "invalid_range", "请选择有效时间范围，最长 7 天，时间需包含时区")
return nil, false
}
tx, err := s.DB.BeginTxx(c.Request.Context(), &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
if err != nil {
s.dbError(c, err)
return nil, false
}
defer tx.Rollback()
report := &diagnosticReport{From: from, To: to, ObservedAt: time.Now().UTC(), Items: []diagnosticEvent{}, Counts: map[string]int{"tasks": 0, "audits": 0, "sessions": 0, "alerts": 0, "logs": 0}, Source: source,
Notes: []string{"按唯一标识关联；按名称记录且缺少稳定关联标识的旧审计不会推测关联。", "当前对象快照独立于历史时间范围；任务提交条目的状态为查询时当前状态，观察到的会话起点不代表真实首次连接时间。", "桌面视角包含所属节点告警；终端日志只在设备视角展示，不根据当前绑定推断历史日志属于哪台桌面。", "日志为设备已回传片段，不是持续实时日志；未收集、未上报和超出留存期均可能没有记录。", "已遮盖常见凭据格式；诊断包仍含设备标识、人员名称及运行信息，请按实际需要共享。"}}
if err = loadDiagnosticSubject(c, tx, kind, id, &report.Subject); err != nil {
s.dbError(c, err)
return nil, false
}
args := []any{id, from, to, kind, report.Subject.Node}
var counts []struct {
Source string `db:"source"`
Total int `db:"total"`
}
if err = tx.SelectContext(c.Request.Context(), &counts, diagnosticEventsSQL+"SELECT source,count(*) total FROM filtered GROUP BY source", args...); err != nil {
s.dbError(c, err)
return nil, false
}
for _, count := range counts {
report.Counts[count.Source] = count.Total
if source == "" || source == count.Source {
report.Total += count.Total
}
}
if export && report.Total > diagnosticExportLimit {
s.fail(c, 413, "export_too_large", "诊断事件超过 2000 条，请缩小时间范围或选择记录来源后导出")
return nil, false
}
report.Page, report.PageSize = pageParams(c)
if export {
report.Page = 1
report.PageSize = diagnosticExportLimit
}
args = append(args, source, report.PageSize, (report.Page-1)*report.PageSize)
if err = tx.SelectContext(c.Request.Context(), &report.Items, diagnosticEventsSQL+"SELECT * FROM filtered WHERE ($6='' OR source=$6) ORDER BY time DESC,source,id LIMIT $7 OFFSET $8", args...); err != nil {
s.dbError(c, err)
return nil, false
}
for i := range report.Items {
sanitizeDiagnosticEvent(&report.Items[i])
}
report.Subject.Name = redactDiagnostic(report.Subject.Name)
report.Subject.AgentVersion = redactDiagnostic(report.Subject.AgentVersion)
return report, true
}
func loadDiagnosticSubject(c *gin.Context, tx *sqlx.Tx, kind, id string, subject *diagnosticSubject) error {
if kind == "terminal" {
return tx.GetContext(c.Request.Context(), subject, `SELECT 'terminal' kind,id,name,CASE WHEN NOT enabled THEN 'disabled' WHEN status='online' AND last_seen<now()-interval '90 seconds' THEN 'offline' ELSE status END state,'' node,0 vmid,agent_version,last_seen FROM terminals WHERE id=$1`, id)
}
return tx.GetContext(c.Request.Context(), subject, `SELECT 'desktop' kind,id,name,power_state state,node,vmid,'' agent_version,NULL::timestamptz last_seen FROM desktops WHERE id=$1`, id)
}
func (s *Server) diagnostics(c *gin.Context) {
report, ok := s.diagnosticReport(c, false)
if !ok {
return
}
c.Header("Cache-Control", "no-store")
c.JSON(http.StatusOK, report)
}
func (s *Server) exportDiagnostics(c *gin.Context) {
report, ok := s.diagnosticReport(c, true)
if !ok {
return
}
content, err := json.MarshalIndent(report, "", " ")
if err != nil {
s.fail(c, 500, "export_failed", "诊断包生成失败")
return
}
var output bytes.Buffer
archive := zip.NewWriter(&output)
for _, entry := range []struct {
name string
data []byte
}{
{"diagnostics.json", content},
{"README.txt", []byte("VDI Cloud 诊断包\n包含所选唯一对象、时间范围和记录来源的全部匹配事件，最多 2000 条。\n当前对象信息为生成时快照。事件按 UTC 排序；任务当前结果不代表历史状态变更日志。\n日志为设备已回传片段；没有日志不代表设备没有故障。桌面视角不推断终端日志归属。\n遮盖常见凭据格式不保证消除所有个人信息或自由文本中的敏感信息。\n生成时不主动读取平台密钥、设备令牌、激活密钥或 PVE 原始任务载荷。自由日志中的自定义敏感格式仍需人工检查。\n")},
} {
writer, e := archive.Create(entry.name)
if e == nil {
_, e = writer.Write(entry.data)
}
if e != nil {
s.fail(c, 500, "export_failed", "诊断包生成失败")
return
}
}
if err = archive.Close(); err != nil {
s.fail(c, 500, "export_failed", "诊断包生成失败")
return
}
if _, err = s.DB.ExecContext(c.Request.Context(), `INSERT INTO audit_logs(id,actor,action,target,result,request_id) VALUES($1,$2,'export_diagnostics',$3,$4,$5)`, store.ID("audit_"), s.admin(c).Username, report.Subject.ID, fmt.Sprintf("success;events=%d", report.Total), c.GetString("requestId")); err != nil {
s.dbError(c, err)
return
}
c.Header("Cache-Control", "no-store")
c.Header("X-Content-Type-Options", "nosniff")
c.Header("Content-Disposition", `attachment; filename="vdi-diagnostics-`+time.Now().UTC().Format("20060102-150405")+`.zip"`)
c.Data(200, "application/zip", output.Bytes())
}
platform/internal/server/diagnostics_test.go
+219
View File
@@ -0,0 +1,219 @@
package server
import (
"archive/zip"
"bytes"
"encoding/json"
"io"
"net/http/httptest"
"strings"
"testing"
)
const diagnosticTestQuery = "?targetType=terminal&targetId=term_demo_001&from=2025-01-01T00:00:00Z&to=2025-01-02T00:00:00Z"
func diagnosticFixture(t *testing.T) *fixture {
f := newFixtureWorker(t, false)
statements := []string{
`UPDATE terminals SET name='同名盒子' WHERE id IN ('term_demo_001','term_demo_002')`,
`INSERT INTO tasks(id,kind,action,target_id,target_name,actor,status,message,provider_ref,idempotency_key,created_at,updated_at) VALUES
('diag-task','terminal','collect-logs','term_demo_001','同名盒子','admin','success','password=task-secret','provider-secret','diag-task','2025-01-01T01:00:00Z','2025-01-01T02:00:00Z'),
('unrelated-task','terminal','reboot','term_demo_002','同名盒子','admin','failed','unrelated-private','','unrelated','2025-01-01T01:00:00Z','2025-01-01T01:00:00Z'),
('gpu-task','gpu-prepare','prepare-desktop','desk_demo_windows','设计专属桌面','device:term_demo_001','timeout','gpu unknown','','gpu-task','2025-01-01T03:00:00Z','2025-01-01T03:00:00Z'),
('desktop-task','desktop','start','desk_demo_windows','设计专属桌面','admin','success','desktop-result','','desktop-task','2025-01-01T04:00:00Z','2025-01-01T04:00:00Z')`,
`INSERT INTO audit_logs VALUES
('direct-audit','admin','update_terminal','term_demo_001','success','trace','2025-01-01T03:00:00Z'),
('task-audit','admin','read_diagnostics','diag-task','success','trace2','2025-01-01T03:00:00Z'),
('name-only-audit','admin','reboot','同名盒子','success','trace3','2025-01-01T03:00:00Z'),
('unrelated-audit','admin','update_terminal','term_demo_002','success','trace4','2025-01-01T03:00:00Z')`,
`INSERT INTO session_history(id,session_id,terminal_id,terminal_name,desktop_id,desktop_name,state,source,reserved_at,connected_at,ended_at,end_reason) VALUES
('old-session','session-old','term_demo_001','同名盒子','desk_demo_windows','设计专属桌面','ended','ticket','2024-12-20T00:00:00Z','2024-12-20T00:01:00Z','2025-01-01T04:00:00Z','client-ended'),
('other-session','session-other','term_demo_002','同名盒子','desk_demo_ubuntu','研发办公桌面','ended','ticket','2025-01-01T01:00:00Z',NULL,'2025-01-01T02:00:00Z','unrelated-private')`,
`INSERT INTO alert_rules(id,name,metric,threshold,hold_seconds) VALUES('diag-rule','设备离线','terminal_offline',120,0)`,
`INSERT INTO alerts(id,rule_id,rule_name,metric,target_id,target_name,value,threshold,status,started_at,opened_at) VALUES
('term-alert','diag-rule','设备离线','terminal_offline','term_demo_001','同名盒子',121,120,'firing','2025-01-01T00:00:00Z','2025-01-01T00:00:00Z'),
('node-alert','diag-rule','节点异常','node_unavailable','demo-node-a','节点 A',121,120,'firing','2025-01-01T00:00:00Z','2025-01-01T00:00:00Z')`,
`INSERT INTO task_diagnostics(task_id,content,created_at) VALUES('diag-task','{"agentVersion":"qt-test","bootId":"test-boot","kioskState":"DESKTOP_LIST","uptimeSeconds":500,"logTail":"ordinary error\nAuthorization: Bearer log-secret\npassword=private-value\nhttps://user:url-secret@example.test/path\n<script>alert(1)</script>","unexpectedPassword":"unlisted-secret"}','2025-01-01T05:00:00Z')`,
}
for _, statement := range statements {
if _, err := f.db.Exec(statement); err != nil {
t.Fatal(err)
}
}
return f
}
func diagnosticDownload(f *fixture, token, query string) *httptest.ResponseRecorder {
r := httptest.NewRequest("GET", "/api/v1/diagnostics/export"+query, nil)
if token != "" {
r.Header.Set("Authorization", "Bearer "+token)
}
w := httptest.NewRecorder()
f.server.Router.ServeHTTP(w, r)
return w
}
func readDiagnosticArchive(t *testing.T, body []byte) map[string][]byte {
t.Helper()
archive, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
if err != nil {
t.Fatal(err)
}
files := map[string][]byte{}
for _, file := range archive.File {
r, e := file.Open()
if e != nil {
t.Fatal(e)
}
data, e := io.ReadAll(r)
r.Close()
if e != nil {
t.Fatal(e)
}
files[file.Name] = data
}
return files
}
func TestDiagnosticsStableCorrelationAndLifecycleTime(t *testing.T) {
f := diagnosticFixture(t)
token := f.login("operator")
code, out := f.request("GET", "/diagnostics"+diagnosticTestQuery, token, nil, "")
expect(t, 200, code, out)
data, _ := json.Marshal(out)
text := string(data)
for _, want := range []string{"task:diag-task", "task:gpu-task", "session-ended:old-session", "audit:task-audit", "alert-open:term-alert", "log:diag-task"} {
if !strings.Contains(text, want) {
t.Fatal("missing correlated event", want)
}
}
for _, private := range []string{"unrelated-task", "unrelated-audit", "name-only-audit", "other-session", "provider-secret", "task-secret", "log-secret", "private-value", "url-secret", "unlisted-secret"} {
if strings.Contains(text, private) {
t.Fatal("incorrectly included content", private)
}
}
if out["counts"].(map[string]any)["logs"] != float64(1) {
t.Fatal(out)
}
// Paging is performed after all sources are merged, using a stable ordering.
code, page := f.request("GET", "/diagnostics"+diagnosticTestQuery+"&pageSize=2&page=2", token, nil, "")
expect(t, 200, code, page)
items := page["items"].([]any)
all := out["items"].([]any)
if len(items) != 2 || items[0].(map[string]any)["id"] != all[2].(map[string]any)["id"] || page["total"] != out["total"] {
t.Fatal(page)
}
var tasks int
f.db.Get(&tasks, "SELECT count(*) FROM tasks")
if tasks != 4 {
t.Fatal("read created tasks", tasks)
}
}
func TestDiagnosticsDesktopScopeAndSourceFilters(t *testing.T) {
f := diagnosticFixture(t)
token := f.login("admin")
query := strings.Replace(diagnosticTestQuery, "targetType=terminal&targetId=term_demo_001", "targetType=desktop&targetId=desk_demo_windows", 1)
code, out := f.request("GET", "/diagnostics"+query, token, nil, "")
expect(t, 200, code, out)
data, _ := json.Marshal(out)
text := string(data)
if !strings.Contains(text, "alert-open:node-alert") || !strings.Contains(text, "session-ended:old-session") || strings.Contains(text, "log:diag-task") || strings.Contains(text, "other-session") {
t.Fatal(text)
}
code, out = f.request("GET", "/diagnostics"+diagnosticTestQuery+"&source=logs", token, nil, "")
expect(t, 200, code, out)
if out["total"] != float64(1) || out["items"].([]any)[0].(map[string]any)["source"] != "logs" {
t.Fatal(out)
}
}
func TestDiagnosticsPermissionsAndValidation(t *testing.T) {
f := newFixtureWorker(t, false)
for _, role := range []string{"", "viewer", "operator", "admin"} {
token := ""
if role != "" {
token = f.login(role)
}
want := 200
if role == "" {
want = 401
}
if role == "viewer" {
want = 403
}
code, out := f.request("GET", "/diagnostics"+diagnosticTestQuery, token, nil, "")
expect(t, want, code, out)
w := diagnosticDownload(f, token, diagnosticTestQuery)
if w.Code != want {
t.Fatal(role, w.Code)
}
}
token := f.login("admin")
for _, query := range []string{"", "?targetType=all&targetId=x", diagnosticTestQuery + "&source=shell", "?targetType=terminal&targetId=term_demo_001&from=2025-01-01T00:00:00&to=2025-01-02T00:00:00Z", "?targetType=terminal&targetId=term_demo_001&from=2025-01-01T00:00:00Z&to=2025-02-01T00:00:00Z"} {
code, out := f.request("GET", "/diagnostics"+query, token, nil, "")
expect(t, 400, code, out)
}
code, out := f.request("GET", strings.Replace("/diagnostics"+diagnosticTestQuery, "term_demo_001", "missing", 1), token, nil, "")
expect(t, 404, code, out)
}
func TestDiagnosticsZIPCompletenessAndRedaction(t *testing.T) {
f := diagnosticFixture(t)
token := f.login("operator")
w := diagnosticDownload(f, token, diagnosticTestQuery+"&source=logs&page=999&pageSize=1")
if w.Code != 200 || w.Header().Get("Content-Type") != "application/zip" || w.Header().Get("Cache-Control") != "no-store" {
t.Fatal(w.Code, w.Body.String())
}
files := readDiagnosticArchive(t, w.Body.Bytes())
if len(files) != 2 || len(files["README.txt"]) == 0 {
t.Fatal("invalid archive entries")
}
var report diagnosticReport
if err := json.Unmarshal(files["diagnostics.json"], &report); err != nil {
t.Fatal(err)
}
if report.Total != 1 || len(report.Items) != 1 || report.Page != 1 || report.Items[0].Source != "logs" {
t.Fatal(report)
}
text := string(files["diagnostics.json"])
for _, secret := range []string{"provider-secret", "log-secret", "private-value", "url-secret", "unlisted-secret"} {
if strings.Contains(text, secret) {
t.Fatal("unredacted secret", secret)
}
}
if !strings.Contains(text, "ordinary error") || !strings.Contains(text, "REDACTED") {
t.Fatal(text)
}
var exports int
f.db.Get(&exports, "SELECT count(*) FROM audit_logs WHERE action='export_diagnostics' AND target='term_demo_001' AND actor='operator'")
if exports != 1 {
t.Fatal("missing export audit")
}
}
func TestDiagnosticsExportLimitAndEmptyArchive(t *testing.T) {
f := newFixtureWorker(t, false)
token := f.login("admin")
if _, err := f.db.Exec(`INSERT INTO audit_logs SELECT 'diag-bulk-'||i,'admin','update_terminal','term_demo_001','success','req','2025-01-01T01:00:00Z'::timestamptz FROM generate_series(1,2001) i`); err != nil {
t.Fatal(err)
}
w := diagnosticDownload(f, token, diagnosticTestQuery+"&source=audits")
if w.Code != 413 || strings.Contains(w.Header().Get("Content-Type"), "zip") {
t.Fatal(w.Code)
}
w = diagnosticDownload(f, token, diagnosticTestQuery+"&source=logs")
if w.Code != 200 {
t.Fatal(w.Code, w.Body.String())
}
files := readDiagnosticArchive(t, w.Body.Bytes())
var report diagnosticReport
json.Unmarshal(files["diagnostics.json"], &report)
if report.Total != 0 || len(report.Items) != 0 {
t.Fatal(report)
}
}
func TestDiagnosticRedactionFormats(t *testing.T) {
for _, value := range []string{`password="two word secret"`, `{"accessToken":"private-value"}`, "Authorization: Bearer private-value", "PVEAPIToken=admin@pve!id=private-value", "device_0123456789abcdef0123456789abcdef", "https://user:private-value@example.test/path", "-----BEGIN OPENSSH PRIVATE KEY-----\nprivate-value\n-----END OPENSSH PRIVATE KEY-----"} {
if result := redactDiagnostic(value); strings.Contains(result, "private-value") || strings.Contains(result, "two word secret") || strings.Contains(result, "0123456789abcdef") {
t.Fatalf("unredacted: %s", result)
}
}
if redactDiagnostic("network error: offline") != "network error: offline" {
t.Fatal("ordinary diagnostic lost")
}
}
platform/internal/server/exclusive_gpu.go
+493
View File
@@ -0,0 +1,493 @@
package server
import (
"context"
"database/sql"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"io"
"log"
"net"
"net/http"
"strconv"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/jmoiron/sqlx"
"vdi-cloud/platform/internal/model"
"vdi-cloud/platform/internal/store"
"vdi-cloud/platform/pkg/proxmox"
)
// This optional, single-host group is provisioned by the administrator. Device
// requests may select a bound desktop; they cannot choose VMIDs or PCI devices.
const exclusiveGPULock int64 = 7651092039
type gpuPrepareState struct {
Phase string `json:"phase"`
Node string `json:"node"`
Members []int `json:"members"`
Terminal string `json:"terminal"`
VMID int `json:"vmid,omitempty"`
UPID string `json:"upid,omitempty"`
}
func (s *Server) exclusiveGPU(d model.Desktop) bool {
if d.Provider != "proxmox" || d.Node != s.Config.ExclusiveGPUNode {
return false
}
for _, id := range s.Config.ExclusiveGPUVMIDs {
if d.VMID == id {
return true
}
}
return false
}
// Maintenance submissions take the same group lock as terminal preparations.
// Call inside their existing transaction; the durable reservation also covers
// the interval between enqueueing a preparation and claiming it in the worker.
func (s *Server) exclusiveGPUMaintenance(ctx context.Context, db sqlx.ExtContext, d model.Desktop) error {
if !s.exclusiveGPU(d) {
return nil
}
locked, err := tryGPULock(ctx, db)
if err != nil {
return err
}
var busy bool
if locked {
err = sqlx.GetContext(ctx, db, &busy, `SELECT EXISTS(SELECT 1 FROM tasks WHERE kind='gpu-prepare' AND status IN ('queued','running','timeout'))`)
}
if err != nil {
return err
}
if !locked || busy {
return errors.New("显卡正在切换或结果未确认，请先核对切换任务")
}
return nil
}
func (s *Server) gpuMembers(ctx context.Context, db sqlx.ExtContext) ([]model.Desktop, error) {
var members []model.Desktop
err := sqlx.SelectContext(ctx, db, &members, `SELECT * FROM desktops WHERE provider='proxmox' AND node=$1 AND vmid=ANY($2) ORDER BY vmid`, s.Config.ExclusiveGPUNode, s.Config.ExclusiveGPUVMIDs)
if err == nil && len(members) != len(s.Config.ExclusiveGPUVMIDs) {
err = errors.New("exclusive GPU group is incomplete")
}
return members, err
}
func tryGPULock(ctx context.Context, db sqlx.ExtContext) (bool, error) {
var locked bool
err := sqlx.GetContext(ctx, db, &locked, "SELECT pg_try_advisory_xact_lock($1)", exclusiveGPULock)
return locked, err
}
func (s *Server) gpuAllowsConnection(c *gin.Context, d model.Desktop) bool {
if !s.exclusiveGPU(d) {
return true
}
locked, err := tryGPULock(c.Request.Context(), deviceTx(c))
var busy bool
if err == nil && locked {
err = deviceTx(c).GetContext(c.Request.Context(), &busy, `SELECT EXISTS(SELECT 1 FROM tasks WHERE kind='gpu-prepare' AND status IN ('queued','running','timeout'))`)
}
if err != nil {
s.dbError(c, err)
return false
}
if !locked || busy {
s.fail(c, 409, "gpu_switch_busy", "显卡正在切换或上次切换尚未确认，请稍后重试或联系管理员")
return false
}
pve := s.currentPVE()
if pve == nil {
s.fail(c, 503, "pve_unavailable", "计算平台不可用")
return false
}
for _, id := range s.Config.ExclusiveGPUVMIDs {
power, e := pve.Power(c.Request.Context(), d.Node, id)
expected := "stopped"
if id == d.VMID {
expected = "running"
}
if e != nil || power != expected {
s.fail(c, 409, "gpu_owner_not_ready", "桌面电源或显卡归属尚未就绪，请先准备桌面")
return false
}
}
return true
}
func gpuGroupBusy(ctx context.Context, db sqlx.ExtContext, members []model.Desktop, taskID string) (bool, error) {
ids := make([]string, 0, len(members))
for _, d := range members {
ids = append(ids, d.ID)
}
var busy bool
err := sqlx.GetContext(ctx, db, &busy, `SELECT EXISTS(SELECT 1 FROM terminals WHERE session_desktop_id=ANY($1) AND last_seen>now()-interval '90 seconds') OR EXISTS(SELECT 1 FROM tasks WHERE target_id=ANY($1) AND id!=$2 AND kind!='terminal' AND status IN ('queued','running','timeout'))`, ids, taskID)
return busy, err
}
func (s *Server) prepareGPUDesktop(c *gin.Context) {
var in struct {
DesktopID string `json:"desktopId" binding:"required,max=100"`
}
if !s.decode(c, &in) {
return
}
tx := deviceTx(c)
ctx := c.Request.Context()
terminal := c.GetString("terminalId")
var d model.Desktop
err := tx.GetContext(ctx, &d, `SELECT d.* FROM desktops d JOIN bindings b ON b.desktop_id=d.id JOIN users u ON u.id=b.user_id JOIN terminals t ON t.id=u.terminal_id WHERE d.id=$1 AND t.id=$2 AND u.enabled AND t.enabled`, in.DesktopID, terminal)
if err == sql.ErrNoRows {
s.fail(c, 403, "desktop_forbidden", "该终端没有桌面使用权限")
return
}
if err != nil {
s.dbError(c, err)
return
}
if !s.exclusiveGPU(d) || d.SetupRequired || d.Endpoint == "" {
s.fail(c, 409, "desktop_not_preparable", "该桌面未启用显卡切换或连接配置尚未完成")
return
}
locked, err := tryGPULock(ctx, tx)
if err != nil {
s.dbError(c, err)
return
}
if !locked {
s.fail(c, 409, "gpu_switch_busy", "显卡正在切换，请稍后重试")
return
}
var previous model.Task
err = tx.GetContext(ctx, &previous, `SELECT * FROM tasks WHERE kind='gpu-prepare' AND status IN ('queued','running','timeout') ORDER BY created_at DESC LIMIT 1`)
if err == nil {
if previous.Actor == "device:"+terminal && previous.TargetID == d.ID && previous.Status != "timeout" {
s.deviceJSON(c, 202, gin.H{"taskId": previous.ID, "status": previous.Status, "message": previous.Message})
return
}
s.fail(c, 409, "gpu_switch_busy", "已有显卡切换任务或未确认结果，请稍后重试或联系管理员")
return
}
if err != sql.ErrNoRows {
s.dbError(c, err)
return
}
members, err := s.gpuMembers(ctx, tx)
if err != nil {
s.dbError(c, err)
return
}
for _, member := range members {
if !s.hardwareAllowsConnection(c, member.ID) {
return
}
}
busy, err := gpuGroupBusy(ctx, tx, members, "")
if err != nil {
s.dbError(c, err)
return
}
if busy {
s.fail(c, 409, "gpu_group_in_use", "请先退出正在使用的云桌面；其他终端使用中或管理任务未结束时无法切换")
return
}
state := gpuPrepareState{Phase: "queued", Node: d.Node, Members: s.Config.ExclusiveGPUVMIDs, Terminal: terminal}
encoded, _ := json.Marshal(state)
id := store.ID("task_")
_, err = tx.ExecContext(ctx, `INSERT INTO tasks(id,kind,action,target_id,target_name,actor,status,message,provider_ref,idempotency_key) VALUES($1,'gpu-prepare','prepare-desktop',$2,$3,$4,'queued','正在准备桌面，等待显卡切换',$5,$1)`, id, d.ID, d.Name, "device:"+terminal, string(encoded))
if err != nil {
s.dbError(c, err)
return
}
s.deviceJSON(c, 202, gin.H{"taskId": id, "status": "queued", "message": "正在准备桌面，首次启动可能需要两分钟"})
}
func (s *Server) gpuPrepareStatus(c *gin.Context) {
var task model.Task
err := deviceTx(c).GetContext(c.Request.Context(), &task, `SELECT * FROM tasks WHERE id=$1 AND kind='gpu-prepare' AND actor=$2`, c.Param("id"), "device:"+c.GetString("terminalId"))
if err == sql.ErrNoRows {
s.fail(c, 404, "task_not_found", "切换任务不存在")
return
}
if err != nil {
s.dbError(c, err)
return
}
s.deviceJSON(c, 200, gin.H{"taskId": task.ID, "status": task.Status, "message": task.Message})
}
func (s *Server) saveGPUStage(ctx context.Context, t model.Task, state gpuPrepareState, message string) error {
data, _ := json.Marshal(state)
result, err := s.DB.ExecContext(ctx, `UPDATE tasks SET provider_ref=$1,message=$2,updated_at=now() WHERE id=$3 AND status='running'`, string(data), message, t.ID)
if err == nil {
n, _ := result.RowsAffected()
if n != 1 {
err = errors.New("GPU task reservation lost")
}
}
return err
}
func sunshineReady(ctx context.Context, endpoint string) bool {
host := endpoint
if parsed, _, err := net.SplitHostPort(endpoint); err == nil {
host = parsed
}
// Platform endpoints are hostnames/IPs, never arbitrary URLs or paths.
if strings.ContainsAny(host, "/\\@?#") || host == "" {
return false
}
req, err := http.NewRequestWithContext(ctx, "GET", "http://"+net.JoinHostPort(host, "47989")+"/serverinfo", nil)
if err != nil {
return false
}
client := http.Client{Timeout: 2 * time.Second, Transport: &http.Transport{Proxy: nil}, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
defer client.CloseIdleConnections()
response, err := client.Do(req)
if err != nil {
return false
}
defer response.Body.Close()
var info struct {
XMLName xml.Name `xml:"root"`
Status string `xml:"status_code,attr"`
Hostname string `xml:"hostname"`
}
return response.StatusCode == 200 && xml.NewDecoder(io.LimitReader(response.Body, 65536)).Decode(&info) == nil && info.Status == "200" && info.Hostname != ""
}
func (s *Server) executeGPUPrepare(parent context.Context, t model.Task) {
ctx, cancel := context.WithTimeout(parent, 180*time.Second)
defer cancel()
state := gpuPrepareState{}
err := json.Unmarshal([]byte(t.ProviderRef), &state)
uncertain := false
if err == nil && state.Phase != "queued" && state.Phase != "" {
uncertain = true
}
if err == nil && (state.Phase != "queued" || state.Node != s.Config.ExclusiveGPUNode || fmt.Sprint(state.Members) != fmt.Sprint(s.Config.ExclusiveGPUVMIDs)) {
err = errors.New("GPU group changed or task already submitted")
}
var tx *sqlx.Tx
if err == nil {
tx, err = s.DB.BeginTxx(ctx, nil)
}
if tx != nil {
defer tx.Rollback()
}
// Hold this lock on a dedicated transaction throughout the switch. Tickets
// fail promptly instead of waiting behind a VM boot; no root credential leaves
// the platform. An API restart never automatically replays an uncertain POST.
if err == nil {
var locked bool
locked, err = tryGPULock(ctx, tx)
if err == nil && !locked {
err = errors.New("GPU group locked")
}
}
var members []model.Desktop
if err == nil {
members, err = s.gpuMembers(ctx, tx)
}
var target model.Desktop
if err == nil {
for _, d := range members {
if d.ID == t.TargetID {
target = d
}
}
var allowed bool
err = tx.GetContext(ctx, &allowed, `SELECT EXISTS(SELECT 1 FROM bindings b JOIN users u ON u.id=b.user_id JOIN terminals term ON term.id=u.terminal_id WHERE b.desktop_id=$1 AND u.terminal_id=$2 AND u.enabled AND term.enabled)`, t.TargetID, state.Terminal)
if err == nil && (!allowed || target.ID == "" || target.SetupRequired || target.Endpoint == "") {
err = errors.New("desktop authorization/configuration changed")
}
}
if err == nil {
var busy bool
busy, err = gpuGroupBusy(ctx, tx, members, t.ID)
if err == nil && busy {
err = errors.New("GPU group is in use")
}
}
if err == nil {
err = s.RefreshCompute(ctx)
}
pve := s.currentPVE()
if err == nil && pve == nil {
err = errors.New("PVE unavailable")
}
action := func(d model.Desktop, operation string) error {
state.Phase = operation + "-submitting"
state.VMID = d.VMID
state.UPID = ""
if err := s.saveGPUStage(ctx, t, state, "正在"+map[string]string{"start": "启动所选桌面", "shutdown": "正常关闭上一台桌面"}[operation]); err != nil {
return err
}
uncertain = true
upid, err := pve.Action(ctx, d.Node, d.VMID, operation)
if err != nil {
var rejected *proxmox.HTTPError
if errors.As(err, &rejected) && rejected.Status >= 400 && rejected.Status < 500 {
uncertain = false
}
return err
}
state.Phase = operation + "-waiting"
state.UPID = upid
if err = s.saveGPUStage(ctx, t, state, "正在等待虚拟机电源操作完成"); err != nil {
return err
}
if err = pve.Wait(ctx, d.Node, upid, time.Second); err != nil {
return err
}
power, err := pve.Power(ctx, d.Node, d.VMID)
expected := "running"
if operation == "shutdown" {
expected = "stopped"
}
if err == nil && power != expected {
err = errors.New("VM power transition not confirmed")
}
if err == nil {
uncertain = false
}
return err
}
if err == nil {
for _, d := range members {
if d.ID == target.ID {
continue
}
power, e := pve.Power(ctx, d.Node, d.VMID)
if e != nil {
err = e
break
}
if power == "running" {
err = action(d, "shutdown")
} else if power != "stopped" {
err = errors.New("unexpected VM power state")
}
if err != nil {
break
}
}
}
if err == nil {
power, e := pve.Power(ctx, target.Node, target.VMID)
err = e
if err == nil && power == "stopped" {
err = action(target, "start")
} else if err == nil && power != "running" {
err = errors.New("unexpected target state")
}
}
if err == nil {
state.Phase = "service-waiting"
err = s.saveGPUStage(ctx, t, state, "桌面已开机，正在等待连接服务")
for err == nil && !sunshineReady(ctx, target.Endpoint) {
select {
case <-ctx.Done():
err = ctx.Err()
case <-time.After(time.Second):
}
}
}
if err == nil {
// Recheck all owners immediately before releasing the reservation.
for _, d := range members {
power, e := pve.Power(ctx, d.Node, d.VMID)
expected := "stopped"
if d.ID == target.ID {
expected = "running"
}
if e != nil || power != expected {
err = errors.New("final GPU ownership check failed")
break
}
if _, e = s.DB.ExecContext(ctx, `UPDATE desktops SET power_state=$1,service_state='unknown',updated_at=now() WHERE id=$2`, power, d.ID); e != nil {
err = e
break
}
}
}
status, message := "success", "桌面已启动，连接服务已响应"
if err != nil {
log.Printf("GPU prepare task=%s phase=%s error=%v", t.ID, state.Phase, err)
status = "failed"
message = "桌面准备失败，请检查桌面授权、占用状态及 PVE"
if uncertain {
status = "timeout"
message = "电源操作或服务状态尚未确认，已阻止再次切换；请管理员核对 PVE 原任务"
}
}
finalCtx, finish := context.WithTimeout(context.Background(), 5*time.Second)
defer finish()
if err == nil {
state.Phase = "completed"
_ = s.saveGPUStage(finalCtx, t, state, message)
}
_ = s.complete(finalCtx, t, status, message)
}
// Reconcile only a known original PVE task. An unknown submission requires
// manual operator investigation; this endpoint never resubmits a power action.
func (s *Server) reconcileGPUPrepare(c *gin.Context) {
var task model.Task
if err := s.DB.GetContext(c.Request.Context(), &task, `SELECT * FROM tasks WHERE id=$1 AND kind='gpu-prepare' AND status='timeout'`, c.Param("id")); err != nil {
s.fail(c, 409, "not_uncertain", "该任务无需核对或不存在")
return
}
var state gpuPrepareState
if json.Unmarshal([]byte(task.ProviderRef), &state) != nil || state.UPID == "" {
s.fail(c, 409, "unknown_submission", "未记录原 PVE 任务编号，需要在 PVE 手工核对，不能自动重发")
return
}
tx, err := s.DB.BeginTxx(c.Request.Context(), nil)
if err != nil {
s.dbError(c, err)
return
}
defer tx.Rollback()
locked, err := tryGPULock(c.Request.Context(), tx)
if err != nil || !locked {
s.fail(c, 409, "gpu_switch_busy", "显卡正在切换")
return
}
pve := s.currentPVE()
if pve == nil {
s.fail(c, 503, "pve_unavailable", "计算平台不可用")
return
}
original, err := pve.Task(c.Request.Context(), state.Node, state.UPID)
if err != nil || original.Status != "stopped" {
s.fail(c, 409, "pve_task_pending", "原 PVE 任务未结束或无法核对")
return
}
for _, id := range state.Members {
power, e := pve.Power(c.Request.Context(), state.Node, id)
if e != nil || (power != "running" && power != "stopped") {
s.fail(c, 409, "power_unknown", "虚拟机实际电源状态未确认")
return
}
}
_, err = tx.ExecContext(c.Request.Context(), `UPDATE tasks SET status='failed',message='管理员已核对原 PVE 任务及实际电源状态，可重新准备桌面',updated_at=now() WHERE id=$1 AND status='timeout'`, task.ID)
if err == nil {
_, err = tx.ExecContext(c.Request.Context(), `INSERT INTO audit_logs VALUES($1,$2,'reconcile-gpu-switch',$3,'success',$4,now())`, store.ID("audit_"), c.MustGet("admin").(model.Administrator).Username, task.TargetName, task.ID)
}
if err == nil {
err = tx.Commit()
}
if err != nil {
s.dbError(c, err)
return
}
c.JSON(200, gin.H{"reconciled": true, "vmid": strconv.Itoa(state.VMID)})
}
platform/internal/server/exclusive_gpu_test.go
+211
View File
@@ -0,0 +1,211 @@
package server
import (
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"vdi-cloud/platform/internal/model"
"vdi-cloud/platform/internal/store"
"vdi-cloud/platform/pkg/proxmox"
)
type exclusiveTestPVE struct {
mu sync.Mutex
power map[int]string
actions []string
mode string
confirmed bool
}
func exclusiveFixture(t *testing.T, mode string) (*fixture, *exclusiveTestPVE) {
t.Helper()
f := newFixtureWorker(t, false)
f.server.Config.ExclusiveGPUNode = "gpu-node"
f.server.Config.ExclusiveGPUVMIDs = []int{100, 101, 102}
for i, id := range []string{"desk_demo_windows", "desk_demo_kylin", "desk_demo_ubuntu"} {
if _, err := f.db.Exec(`UPDATE desktops SET provider='proxmox',node='gpu-node',vmid=$1,endpoint='127.0.0.1',setup_required=false,power_state='stopped' WHERE id=$2`, 100+i, id); err != nil {
t.Fatal(err)
}
}
if _, err := f.db.Exec(`INSERT INTO terminals(id,sn,name,bootstrap_hash,token_hash,last_seen) VALUES('gpu-box','GPU-BOX','GPU test','',$1,now());`, store.Hash("gpu-device")); err != nil {
t.Fatal(err)
}
if _, err := f.db.Exec(`UPDATE users SET terminal_id='gpu-box' WHERE id='user_demo_wang'; INSERT INTO bindings(user_id,desktop_id) SELECT 'user_demo_wang',id FROM desktops ON CONFLICT DO NOTHING`); err != nil {
t.Fatal(err)
}
p := &exclusiveTestPVE{power: map[int]string{100: "running", 101: "stopped", 102: "stopped"}, mode: mode}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
p.mu.Lock()
defer p.mu.Unlock()
path := strings.TrimPrefix(r.URL.Path, "/api2/json")
var data any
var id int
switch {
case strings.Contains(path, "/tasks/"):
if p.mode == "poll-failure" {
w.WriteHeader(503)
return
}
for _, action := range p.actions {
if action == "100:shutdown" {
p.power[100] = "stopped"
p.confirmed = true
}
if action == "102:start" {
p.power[102] = "running"
}
}
data = map[string]string{"status": "stopped", "exitstatus": "OK"}
case strings.HasSuffix(path, "/status/current"):
fmt.Sscanf(path, "/nodes/gpu-node/qemu/%d/status/current", &id)
data = map[string]string{"status": p.power[id]}
case r.Method == "POST" && strings.Contains(path, "/status/"):
fmt.Sscanf(path, "/nodes/gpu-node/qemu/%d/status/", &id)
action := path[strings.LastIndex(path, "/")+1:]
if action == "start" && (!p.confirmed || p.power[100] != "stopped") {
t.Error("start before confirmed shutdown")
}
p.actions = append(p.actions, fmt.Sprintf("%d:%s", id, action))
if p.mode == "ambiguous" {
w.WriteHeader(500)
return
}
if p.mode == "rejected" {
w.WriteHeader(403)
return
}
data = "UPID:gpu-test"
default:
t.Error("unexpected GPU switch request", r.Method, path)
http.NotFound(w, r)
return
}
json.NewEncoder(w).Encode(map[string]any{"data": data})
}))
t.Cleanup(srv.Close)
f.server.PVE = &proxmox.Client{BaseURL: srv.URL, HTTP: srv.Client()}
return f, p
}
func prepareExclusive(t *testing.T, f *fixture) model.Task {
t.Helper()
code, out := f.request("POST", "/terminal/desktop-prepare", "gpu-device", map[string]any{"desktopId": "desk_demo_ubuntu"}, "")
expect(t, 202, code, out)
var task model.Task
if err := f.db.Get(&task, `UPDATE tasks SET status='running' WHERE id=$1 RETURNING *`, out["taskId"]); err != nil {
t.Fatal(err)
}
return task
}
func TestExclusiveGPUAuthorizationAndOccupiedGroup(t *testing.T) {
f, p := exclusiveFixture(t, "normal")
code, out := f.request("POST", "/terminal/desktop-prepare", "gpu-device", map[string]any{"desktopId": "unbound"}, "")
expect(t, 403, code, out)
if _, err := f.db.Exec(`UPDATE terminals SET session_desktop_id='desk_demo_windows' WHERE id='gpu-box'`); err != nil {
t.Fatal(err)
}
code, out = f.request("POST", "/terminal/desktop-prepare", "gpu-device", map[string]any{"desktopId": "desk_demo_ubuntu"}, "")
expect(t, 409, code, out)
if len(p.actions) != 0 {
t.Fatal("occupied group changed power", p.actions)
}
}
func TestExclusiveGPUReservationBlocksAdministrativePower(t *testing.T) {
f, p := exclusiveFixture(t, "normal")
prepareExclusive(t, f)
admin := f.login("admin")
for _, action := range []string{"start", "shutdown", "reboot"} {
code, out := f.request("POST", "/desktops/desk_demo_windows/actions", admin, map[string]any{"action": action}, "gpu-admin-"+action)
expect(t, 409, code, out)
}
_, check, err := f.server.powerBatchPreflight(context.Background(), f.db, "desk_demo_windows", "start", "")
if err != nil || check.Code != "exclusive_gpu_power" || len(p.actions) != 0 {
t.Fatal(check, err, p.actions)
}
}
func TestExclusiveGPUShutdownThenStartAndStaleCacheTicket(t *testing.T) {
f, p := exclusiveFixture(t, "normal")
listener, err := net.Listen("tcp", "127.0.0.1:47989")
if err != nil {
t.Fatal(err)
}
ready := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, `<root status_code="200"><hostname>test-sunshine</hostname></root>`)
})}
go ready.Serve(listener)
t.Cleanup(func() { ready.Close() })
task := prepareExclusive(t, f)
f.server.executeGPUPrepare(context.Background(), task)
var status string
f.db.Get(&status, "SELECT status FROM tasks WHERE id=$1", task.ID)
if status != "success" {
t.Fatal("prepare failed", status)
}
if fmt.Sprint(p.actions) != "[100:shutdown 102:start]" {
t.Fatal(p.actions)
}
// Sync cache may lag behind real PVE state after an immediate start.
f.db.Exec(`UPDATE desktops SET power_state='stopped' WHERE id='desk_demo_ubuntu'`)
code, out := f.request("POST", "/terminal/connect-ticket", "gpu-device", map[string]any{"desktopId": "desk_demo_ubuntu"}, "")
expect(t, 201, code, out)
code, out = f.request("POST", "/terminal/connect-ticket/consume", "gpu-device", map[string]any{"ticket": out["ticket"]}, "")
expect(t, 200, code, out)
}
func TestExclusiveGPUUnknownSubmissionNeverReplayed(t *testing.T) {
for _, mode := range []string{"ambiguous", "poll-failure", "rejected"} {
t.Run(mode, func(t *testing.T) {
f, p := exclusiveFixture(t, mode)
task := prepareExclusive(t, f)
f.server.executeGPUPrepare(context.Background(), task)
var result model.Task
f.db.Get(&result, "SELECT * FROM tasks WHERE id=$1", task.ID)
want := "timeout"
if mode == "rejected" {
want = "failed"
}
if result.Status != want || len(p.actions) != 1 {
t.Fatal(result.Status, p.actions)
}
if mode == "rejected" {
return
}
f.server.executeGPUPrepare(context.Background(), result)
if len(p.actions) != 1 {
t.Fatal("replayed uncertain power action", p.actions)
}
code, out := f.request("POST", "/terminal/desktop-prepare", "gpu-device", map[string]any{"desktopId": "desk_demo_ubuntu"}, "")
expect(t, 409, code, out)
code, out = f.request("POST", "/tasks/"+task.ID+"/gpu-reconcile", f.login("admin"), nil, "")
if mode == "ambiguous" {
expect(t, 409, code, out)
} else {
expect(t, 409, code, out) /* original task cannot be read */
}
if len(p.actions) != 1 {
t.Fatal("reconciliation changed power")
}
if mode == "poll-failure" {
p.mu.Lock()
p.mode = "normal"
p.mu.Unlock()
code, out = f.request("POST", "/tasks/"+task.ID+"/gpu-reconcile", f.login("admin"), nil, "")
expect(t, 200, code, out)
f.db.Get(&result, "SELECT * FROM tasks WHERE id=$1", task.ID)
if result.Status != "failed" || len(p.actions) != 1 {
t.Fatal("reconciliation must release reservation without replay", result.Status, p.actions)
}
}
})
}
}
platform/internal/server/hardware.go
+3
View File
@@ -62,6 +62,9 @@ func (s *Server) hardwarePreflight(ctx context.Context, id string, in hardwareIn
if err != nil {
return d, h, config, errors.New("无法读取 PVE 规格")
}
if err = s.exclusiveGPUMaintenance(ctx, db, d); err != nil {
return d, h, config, err
}
reject := func(message string) (model.Desktop, proxmox.VMHardware, map[string]any, error) {
return d, h, config, errors.New(message)
}
platform/internal/server/login_limits.go
+13 -2
View File
@@ -14,6 +14,9 @@ import (
// Count attempts before password hashing. Shared storage keeps the budget across
// API replicas and restarts; neither passwords nor cleartext account/IP keys persist.
func (s *Server) allowLoginAttempt(c *gin.Context, username string) bool {
return s.allowAuthAttempt(c, username, "login")
}
func (s *Server) allowAuthAttempt(c *gin.Context, username, purpose string) bool {
tx, err := s.DB.BeginTxx(c.Request.Context(), nil)
if err != nil {
s.dbError(c, err)
@@ -26,7 +29,11 @@ func (s *Server) allowLoginAttempt(c *gin.Context, username string) bool {
max int
}{{"account:" + strings.ToLower(strings.TrimSpace(username)), 10}, {"ip:" + c.ClientIP(), 30}} {
mac := hmac.New(sha256.New, []byte(s.Config.JWTSecret))
mac.Write([]byte("login-limit:" + limit.key))
prefix := "login-limit:"
if purpose != "login" {
prefix = purpose + "-limit:"
}
mac.Write([]byte(prefix + limit.key))
key := hex.EncodeToString(mac.Sum(nil))
var row struct {
Attempts int `db:"attempts"`
@@ -52,7 +59,11 @@ func (s *Server) allowLoginAttempt(c *gin.Context, username string) bool {
}
if retry > 0 {
c.Header("Retry-After", strconv.Itoa(retry))
s.fail(c, 429, "login_rate_limited", "登录尝试过于频繁，请在 "+strconv.Itoa(retry)+" 秒后重试")
code, label := "login_rate_limited", "登录"
if purpose != "login" {
code, label = "authentication_rate_limited", "验证"
}
s.fail(c, 429, code, label+"尝试过于频繁，请在 "+strconv.Itoa(retry)+" 秒后重试")
return false
}
return true
platform/internal/server/mfa.go
+367
View File
@@ -0,0 +1,367 @@
package server
import (
"context"
"crypto/aes"
"crypto/cipher"
"crypto/hmac"
"crypto/rand"
"crypto/sha1"
"crypto/sha256"
"database/sql"
"encoding/base32"
"encoding/base64"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"net/url"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/jmoiron/sqlx"
"golang.org/x/crypto/bcrypt"
"vdi-cloud/platform/internal/store"
)
type mfaRecord struct {
AdminID string `db:"admin_id"`
Secret string `db:"encrypted_secret"`
LastCounter int64 `db:"last_counter"`
Pending string `db:"pending_secret"`
PendingSession string `db:"pending_session"`
PendingExpires *time.Time `db:"pending_expires"`
EnabledAt *time.Time `db:"enabled_at"`
}
func (s *Server) cryptMFA(adminID, value string, seal bool) (string, error) {
key := sha256.Sum256([]byte("vdi-admin-mfa-v1:" + s.Config.JWTSecret))
block, _ := aes.NewCipher(key[:])
gcm, _ := cipher.NewGCM(block)
aad := []byte("admin_mfa:v1:" + adminID)
if seal {
nonce := make([]byte, gcm.NonceSize())
if _, err := rand.Read(nonce); err != nil {
return "", err
}
return base64.StdEncoding.EncodeToString(gcm.Seal(nonce, nonce, []byte(value), aad)), nil
}
data, err := base64.StdEncoding.DecodeString(value)
if err != nil || len(data) < gcm.NonceSize() {
return "", errors.New("invalid encrypted MFA secret")
}
plain, err := gcm.Open(nil, data[:gcm.NonceSize()], data[gcm.NonceSize():], aad)
return string(plain), err
}
// RFC 6238 / RFC 4226, SHA-1 with dynamic truncation. Production uses 6 digits and 30 seconds.
func totpCode(secret string, counter int64, digits int) (string, error) {
key, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(secret)
if err != nil || len(key) < 20 || counter < 0 || (digits != 6 && digits != 8) {
return "", errors.New("invalid TOTP parameters")
}
var input [8]byte
binary.BigEndian.PutUint64(input[:], uint64(counter))
mac := hmac.New(sha1.New, key)
mac.Write(input[:])
digest := mac.Sum(nil)
offset := int(digest[len(digest)-1] & 15)
value := binary.BigEndian.Uint32(digest[offset:offset+4]) & 0x7fffffff
mod := uint32(1000000)
if digits == 8 {
mod = 100000000
}
return fmt.Sprintf("%0*d", digits, value%mod), nil
}
func matchTOTPCounter(secret, code string, now time.Time, last int64) (int64, bool) {
if len(code) != 6 {
return 0, false
}
for _, r := range code {
if r < '0' || r > '9' {
return 0, false
}
}
for delta := int64(1); delta >= -1; delta-- {
counter := now.Unix()/30 + delta
if counter <= last {
continue
}
value, err := totpCode(secret, counter, 6)
if err == nil && hmac.Equal([]byte(value), []byte(code)) {
return counter, true
}
}
return 0, false
}
func recoveryHash(code string) string {
code = strings.ToLower(strings.NewReplacer("-", "", " ", "").Replace(strings.TrimSpace(code)))
if len(code) != 32 {
return ""
}
if _, err := hex.DecodeString(code); err != nil {
return ""
}
return store.Hash("vdi-mfa-recovery-v1:" + code)
}
func newRecoveryCodes(ctx context.Context, tx *sqlx.Tx, id string) ([]string, error) {
if _, err := tx.ExecContext(ctx, "DELETE FROM admin_mfa_recovery WHERE admin_id=$1", id); err != nil {
return nil, err
}
codes := make([]string, 0, 8)
for i := 0; i < 8; i++ {
var raw [16]byte
if _, err := rand.Read(raw[:]); err != nil {
return nil, err
}
value := strings.ToUpper(hex.EncodeToString(raw[:]))
code := value[:8] + "-" + value[8:16] + "-" + value[16:24] + "-" + value[24:]
if _, err := tx.ExecContext(ctx, "INSERT INTO admin_mfa_recovery(admin_id,code_hash) VALUES($1,$2)", id, recoveryHash(code)); err != nil {
return nil, err
}
codes = append(codes, code)
}
return codes, nil
}
// Caller locks the administrator first, serializing login, account changes and MFA writes.
func (s *Server) consumeMFA(ctx context.Context, tx *sqlx.Tx, row mfaRecord, code string) (bool, error) {
if hash := recoveryHash(code); hash != "" {
result, err := tx.ExecContext(ctx, "DELETE FROM admin_mfa_recovery WHERE admin_id=$1 AND code_hash=$2", row.AdminID, hash)
if err != nil {
return false, err
}
count, err := result.RowsAffected()
return count == 1, err
}
secret, err := s.cryptMFA(row.AdminID, row.Secret, false)
if err != nil {
return false, err
}
counter, ok := matchTOTPCounter(secret, code, time.Now(), row.LastCounter)
if !ok {
return false, nil
}
_, err = tx.ExecContext(ctx, "UPDATE admin_mfa SET last_counter=$1 WHERE admin_id=$2", counter, row.AdminID)
return err == nil, err
}
func (s *Server) mfaStatus(c *gin.Context) {
var status struct {
Enabled bool `json:"enabled" db:"enabled"`
RecoveryRemaining int `json:"recoveryRemaining" db:"recovery_remaining"`
}
err := s.DB.GetContext(c.Request.Context(), &status, `SELECT EXISTS(SELECT 1 FROM admin_mfa WHERE admin_id=$1 AND encrypted_secret!='') enabled,
(SELECT count(*) FROM admin_mfa_recovery WHERE admin_id=$1) recovery_remaining`, s.admin(c).ID)
if err != nil {
s.dbError(c, err)
return
}
c.JSON(200, status)
}
func (s *Server) beginMFA(c *gin.Context) {
var in struct {
Password string `json:"password" binding:"required,max=200"`
}
if !s.decodeConfiguration(c, &in) || !s.allowAuthAttempt(c, s.admin(c).Username, "mfa") {
return
}
tx, actor := s.accountTransaction(c, false)
if tx == nil {
return
}
defer tx.Rollback()
if bcrypt.CompareHashAndPassword([]byte(actor.PasswordHash), []byte(in.Password)) != nil {
s.fail(c, 400, "current_password_invalid", "当前密码不正确")
return
}
var enabled bool
if err := tx.GetContext(c.Request.Context(), &enabled, "SELECT EXISTS(SELECT 1 FROM admin_mfa WHERE admin_id=$1 AND encrypted_secret!='')", actor.ID); err != nil {
s.dbError(c, err)
return
}
if enabled {
s.fail(c, 409, "mfa_enabled", "已开启双重验证，请先关闭后重新绑定")
return
}
var raw [20]byte
if _, err := rand.Read(raw[:]); err != nil {
s.dbError(c, err)
return
}
secret := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(raw[:])
encrypted, err := s.cryptMFA(actor.ID, secret, true)
expires := time.Now().Add(10 * time.Minute)
if err == nil {
_, err = tx.ExecContext(c.Request.Context(), `INSERT INTO admin_mfa(admin_id,pending_secret,pending_session,pending_expires) VALUES($1,$2,$3,$4)
ON CONFLICT(admin_id) DO UPDATE SET pending_secret=$2,pending_session=$3,pending_expires=$4`, actor.ID, encrypted, c.GetString("sessionId"), expires)
}
if err == nil {
err = tx.Commit()
}
if err != nil {
s.dbError(c, err)
return
}
uri := url.URL{Scheme: "otpauth", Host: "totp", Path: "/VDI Cloud:" + actor.Username}
uri.RawQuery = url.Values{"secret": {secret}, "issuer": {"VDI Cloud"}, "algorithm": {"SHA1"}, "digits": {"6"}, "period": {"30"}}.Encode()
c.JSON(200, gin.H{"secret": secret, "provisioningUri": uri.String(), "expiresAt": expires})
}
func (s *Server) confirmMFA(c *gin.Context) {
var in struct {
Code string `json:"code" binding:"required,len=6"`
}
if !s.decodeConfiguration(c, &in) || !s.allowAuthAttempt(c, s.admin(c).Username, "mfa") {
return
}
tx, actor := s.accountTransaction(c, false)
if tx == nil {
return
}
defer tx.Rollback()
ctx := c.Request.Context()
var row mfaRecord
err := tx.GetContext(ctx, &row, "SELECT * FROM admin_mfa WHERE admin_id=$1", actor.ID)
if errors.Is(err, sql.ErrNoRows) {
s.fail(c, 409, "mfa_setup_expired", "请重新开始绑定")
return
}
if err != nil {
s.dbError(c, err)
return
}
if row.Secret != "" {
s.fail(c, 409, "mfa_enabled", "已开启双重验证")
return
}
if row.Pending == "" || row.PendingExpires == nil || !row.PendingExpires.After(time.Now()) || row.PendingSession != c.GetString("sessionId") {
s.fail(c, 409, "mfa_setup_expired", "绑定已过期或来自其他登录，请重新开始")
return
}
secret, err := s.cryptMFA(actor.ID, row.Pending, false)
if err != nil {
s.dbError(c, err)
return
}
counter, ok := matchTOTPCounter(secret, in.Code, time.Now(), -1)
if !ok {
s.fail(c, 400, "mfa_invalid", "验证码错误或已过期，请核对设备时间")
return
}
_, err = tx.ExecContext(ctx, "UPDATE admin_mfa SET encrypted_secret=pending_secret,last_counter=$1,enabled_at=now(),pending_secret='',pending_session='',pending_expires=NULL WHERE admin_id=$2", counter, actor.ID)
var codes []string
if err == nil {
codes, err = newRecoveryCodes(ctx, tx, actor.ID)
}
if err == nil {
_, err = tx.ExecContext(ctx, "DELETE FROM admin_sessions WHERE admin_id=$1 AND id!=$2", actor.ID, c.GetString("sessionId"))
}
if err == nil {
err = accountAudit(ctx, tx, actor.Username, "enable_mfa", actor.Username, c.GetString("requestId"))
}
if err == nil {
err = tx.Commit()
}
if err != nil {
s.dbError(c, err)
return
}
c.JSON(200, gin.H{"recoveryCodes": codes})
}
func (s *Server) changeMFA(c *gin.Context, disable bool) {
var in struct {
Password string `json:"password" binding:"required,max=200"`
Code string `json:"code" binding:"required,max=80"`
}
if !s.decodeConfiguration(c, &in) || !s.allowAuthAttempt(c, s.admin(c).Username, "mfa") {
return
}
tx, actor := s.accountTransaction(c, false)
if tx == nil {
return
}
defer tx.Rollback()
ctx := c.Request.Context()
if bcrypt.CompareHashAndPassword([]byte(actor.PasswordHash), []byte(in.Password)) != nil {
s.fail(c, 400, "current_password_invalid", "当前密码不正确")
return
}
var row mfaRecord
err := tx.GetContext(ctx, &row, "SELECT * FROM admin_mfa WHERE admin_id=$1 AND encrypted_secret!=''", actor.ID)
if errors.Is(err, sql.ErrNoRows) {
s.fail(c, 409, "mfa_disabled", "尚未开启双重验证")
return
}
if err != nil {
s.dbError(c, err)
return
}
ok, err := s.consumeMFA(ctx, tx, row, in.Code)
if err != nil {
s.dbError(c, err)
return
}
if !ok {
s.fail(c, 400, "mfa_invalid", "验证码错误、已使用或已过期")
return
}
action := "rotate_mfa_recovery"
var codes []string
if disable {
action = "disable_mfa"
_, err = tx.ExecContext(ctx, "DELETE FROM admin_mfa WHERE admin_id=$1", actor.ID)
} else {
codes, err = newRecoveryCodes(ctx, tx, actor.ID)
}
if err == nil {
_, err = tx.ExecContext(ctx, "DELETE FROM admin_sessions WHERE admin_id=$1 AND id!=$2", actor.ID, c.GetString("sessionId"))
}
if err == nil {
err = accountAudit(ctx, tx, actor.Username, action, actor.Username, c.GetString("requestId"))
}
if err == nil {
err = tx.Commit()
}
if err != nil {
s.dbError(c, err)
return
}
if disable {
c.Status(204)
} else {
c.JSON(200, gin.H{"recoveryCodes": codes})
}
}
func (s *Server) loginMFA(c *gin.Context, tx *sqlx.Tx, adminID, code string) bool {
var row mfaRecord
err := tx.GetContext(c.Request.Context(), &row, "SELECT * FROM admin_mfa WHERE admin_id=$1 AND encrypted_secret!=''", adminID)
if errors.Is(err, sql.ErrNoRows) {
return true
}
if err != nil {
s.dbError(c, err)
return false
}
if code == "" {
s.fail(c, 401, "mfa_required", "请输入身份验证器动态码或一次性恢复码")
return false
}
ok, err := s.consumeMFA(c.Request.Context(), tx, row, code)
if err != nil {
s.dbError(c, err)
return false
}
if !ok {
s.fail(c, 401, "mfa_invalid", "验证码错误、已使用或已过期")
return false
}
return true
}
platform/internal/server/mfa_test.go
+245
View File
@@ -0,0 +1,245 @@
package server
import (
"context"
"encoding/base32"
"encoding/json"
"os"
"strings"
"sync"
"testing"
"time"
"vdi-cloud/platform/internal/store"
)
func TestTOTPRFC6238VectorsAndReplay(t *testing.T) {
secret := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString([]byte("12345678901234567890"))
for _, tc := range []struct {
unix int64
want string
}{{59, "94287082"}, {1111111109, "07081804"}, {1111111111, "14050471"}, {1234567890, "89005924"}, {2000000000, "69279037"}, {20000000000, "65353130"}} {
got, err := totpCode(secret, tc.unix/30, 8)
if err != nil || got != tc.want {
t.Fatal(tc.unix, got, err)
}
}
now := time.Unix(1234567890, 0)
for delta := int64(-2); delta <= 2; delta++ {
code, _ := totpCode(secret, now.Unix()/30+delta, 6)
counter, ok := matchTOTPCounter(secret, code, now, -1)
if ok != (delta >= -1 && delta <= 1) {
t.Fatal("wrong time window", delta)
}
if ok {
if _, replay := matchTOTPCounter(secret, code, now, counter); replay {
t.Fatal("replay accepted")
}
}
}
}
func setupTestMFA(t *testing.T, f *fixture, token, password string) (string, []string) {
t.Helper()
code, out := f.request("POST", "/auth/mfa/setup", token, map[string]any{"password": password}, "")
expect(t, 200, code, out)
secret := out["secret"].(string)
if !strings.HasPrefix(out["provisioningUri"].(string), "otpauth://totp/") {
t.Fatal("invalid provisioning URI")
}
value, _ := totpCode(secret, time.Now().Unix()/30, 6)
code, out = f.request("POST", "/auth/mfa/confirm", token, map[string]any{"code": value}, "")
expect(t, 200, code, out)
items := out["recoveryCodes"].([]any)
if len(items) != 8 {
t.Fatal("missing recovery codes")
}
codes := make([]string, len(items))
for i, v := range items {
codes[i] = v.(string)
}
return secret, codes
}
func TestMFALifecycleLoginRecoveryRotationAndDisable(t *testing.T) {
f := newFixtureWorker(t, false)
owner := f.login("admin")
id := createTestAccount(t, f, owner, "mfa-user", "viewer")
user := accountLogin(t, f, "mfa-user", testAccountPassword)
old := accountLogin(t, f, "mfa-user", testAccountPassword)
secret, codes := setupTestMFA(t, f, user, testAccountPassword)
code, out := f.request("GET", "/auth/me", old, nil, "")
expect(t, 401, code, out)
code, out = f.request("GET", "/auth/mfa", user, nil, "")
expect(t, 200, code, out)
if out["enabled"] != true || out["recoveryRemaining"] != float64(8) {
t.Fatal(out)
}
encoded, _ := json.Marshal(out)
if strings.Contains(string(encoded), secret) {
t.Fatal("status leaks secret")
}
var record mfaRecord
if err := f.db.Get(&record, "SELECT * FROM admin_mfa WHERE admin_id=$1", id); err != nil {
t.Fatal(err)
}
if strings.Contains(record.Secret, secret) || record.Pending != "" {
t.Fatal("secret not encrypted or pending not cleared")
}
if _, err := f.server.cryptMFA("wrong-admin", record.Secret, false); err == nil {
t.Fatal("secret not bound to account")
}
code, out = f.request("POST", "/auth/login", "", map[string]any{"username": "mfa-user", "password": testAccountPassword}, "")
expect(t, 401, code, out)
if out["code"] != "mfa_required" || out["accessToken"] != nil {
t.Fatal(out)
}
// Failed password must not reveal the MFA requirement or consume a recovery code.
code, out = f.request("POST", "/auth/login", "", map[string]any{"username": "mfa-user", "password": "incorrect", "code": codes[0]}, "")
expect(t, 401, code, out)
if out["code"] != "invalid_credentials" {
t.Fatal(out)
}
code, out = f.request("POST", "/auth/login", "", map[string]any{"username": "mfa-user", "password": testAccountPassword, "code": strings.ToLower(codes[0])}, "")
expect(t, 200, code, out)
newToken := out["accessToken"].(string)
code, out = f.request("POST", "/auth/login", "", map[string]any{"username": "mfa-user", "password": testAccountPassword, "code": codes[0]}, "")
expect(t, 401, code, out)
// Rotate consumes one code, invalidates all old recovery codes and other sessions.
code, out = f.request("POST", "/auth/mfa/recovery", newToken, map[string]any{"password": testAccountPassword, "code": codes[1]}, "")
expect(t, 200, code, out)
newCodes := out["recoveryCodes"].([]any)
code, out = f.request("GET", "/auth/me", user, nil, "")
expect(t, 401, code, out)
code, out = f.request("DELETE", "/auth/mfa", newToken, map[string]any{"password": testAccountPassword, "code": codes[2]}, "")
expect(t, 400, code, out)
code, out = f.request("DELETE", "/auth/mfa", newToken, map[string]any{"password": testAccountPassword, "code": newCodes[0]}, "")
expect(t, 204, code, out)
code, out = f.request("GET", "/auth/mfa", newToken, nil, "")
expect(t, 200, code, out)
if out["enabled"] != false || out["recoveryRemaining"] != float64(0) {
t.Fatal(out)
}
_ = accountLogin(t, f, "mfa-user", testAccountPassword)
var actions []string
f.db.Select(&actions, "SELECT action FROM audit_logs WHERE actor='mfa-user' AND action IN ('enable_mfa','disable_mfa','rotate_mfa_recovery') ORDER BY created_at,id")
if len(actions) != 3 {
t.Fatal("MFA mutations not audited", actions)
}
}
func TestMFASetupExpirySessionBindingValidationAndLimits(t *testing.T) {
f := newFixtureWorker(t, false)
user := f.login("viewer")
other := f.login("viewer")
for _, path := range []string{"/auth/mfa", "/auth/mfa/setup", "/auth/mfa/confirm", "/auth/mfa/recovery"} {
method := "POST"
if path == "/auth/mfa" {
method = "GET"
}
code, out := f.request(method, path, "", map[string]any{}, "")
expect(t, 401, code, out)
}
code, out := f.request("POST", "/auth/mfa/setup", user, map[string]any{"password": "incorrect"}, "")
expect(t, 400, code, out)
code, out = f.request("POST", "/auth/mfa/setup", user, map[string]any{"password": "Demo@2026", "adminId": "other"}, "")
expect(t, 400, code, out)
code, out = f.request("POST", "/auth/mfa/setup", user, map[string]any{"password": "Demo@2026"}, "")
expect(t, 200, code, out)
secret := out["secret"].(string)
value, _ := totpCode(secret, time.Now().Unix()/30, 6)
code, out = f.request("POST", "/auth/mfa/confirm", other, map[string]any{"code": value}, "")
expect(t, 409, code, out)
f.db.MustExec("UPDATE admin_mfa SET pending_expires=now()-interval '1 second'")
code, out = f.request("POST", "/auth/mfa/confirm", user, map[string]any{"code": value}, "")
expect(t, 409, code, out)
code, out = f.request("GET", "/auth/mfa", user, nil, "")
expect(t, 200, code, out)
if out["enabled"] != false {
t.Fatal(out)
}
f.db.MustExec("TRUNCATE login_limits")
for i := 0; i < 11; i++ {
code, out = f.request("POST", "/auth/mfa/setup", user, map[string]any{"password": "incorrect"}, "")
want := 400
if i == 10 {
want = 429
}
expect(t, want, code, out)
}
// MFA attempt budget does not block ordinary login for an unbound account.
_ = f.login("viewer")
}
func TestMFAConcurrentCodeConsumptionAndTOTPLogin(t *testing.T) {
f := newFixtureWorker(t, false)
user := f.login("operator")
secret, codes := setupTestMFA(t, f, user, "Demo@2026")
var id string
f.db.Get(&id, "SELECT id FROM administrators WHERE username='operator'")
// Clear only the test row's counter to exercise a fresh current-time code without a 30-second sleep.
f.db.MustExec("UPDATE admin_mfa SET last_counter=-1 WHERE admin_id=$1", id)
value, _ := totpCode(secret, time.Now().Unix()/30, 6)
var wg sync.WaitGroup
results := make(chan int, 2)
for i := 0; i < 2; i++ {
wg.Add(1)
go func() {
defer wg.Done()
code, _ := f.request("POST", "/auth/login", "", map[string]any{"username": "operator", "password": "Demo@2026", "code": value}, "")
results <- code
}()
}
wg.Wait()
close(results)
sum := 0
for code := range results {
sum += code
}
if sum != 601 {
t.Fatal("TOTP replay/concurrency", sum)
}
results = make(chan int, 2)
for i := 0; i < 2; i++ {
wg.Add(1)
go func() {
defer wg.Done()
code, _ := f.request("POST", "/auth/login", "", map[string]any{"username": "operator", "password": "Demo@2026", "code": codes[0]}, "")
results <- code
}()
}
wg.Wait()
close(results)
sum = 0
for code := range results {
sum += code
}
if sum != 601 {
t.Fatal("recovery code replay/concurrency", sum)
}
}
func TestMFASuperAdminRestartAndPasswordChangePreserveSecondFactor(t *testing.T) {
f := newFixtureWorker(t, false)
owner := f.login("admin")
_, codes := setupTestMFA(t, f, owner, "Demo@2026")
db, err := store.Open(context.Background(), os.Getenv("VDI_TEST_DATABASE_URL"))
if err != nil {
t.Fatal(err)
}
db.Close()
code, out := f.request("GET", "/auth/mfa", owner, nil, "")
expect(t, 200, code, out)
if out["enabled"] != true || out["recoveryRemaining"] != float64(8) {
t.Fatal("restart changed MFA", out)
}
code, out = f.request("POST", "/auth/password", owner, map[string]any{"currentPassword": "Demo@2026", "newPassword": "Changed-mfa-2026"}, "")
expect(t, 204, code, out)
code, out = f.request("POST", "/auth/login", "", map[string]any{"username": "admin", "password": "Changed-mfa-2026"}, "")
expect(t, 401, code, out)
if out["code"] != "mfa_required" {
t.Fatal("password change bypassed MFA", out)
}
code, out = f.request("POST", "/auth/login", "", map[string]any{"username": "admin", "password": "Changed-mfa-2026", "code": codes[0]}, "")
expect(t, 200, code, out)
}
platform/internal/server/observability.go
+39 -14
View File
@@ -13,12 +13,13 @@ import (
)
type alertRule struct {
ID string `json:"id" db:"id"`
Name string `json:"name" db:"name" binding:"required,max=80"`
Metric string `json:"metric" db:"metric" binding:"required,oneof=node_cpu node_memory node_storage node_unavailable terminal_offline"`
Threshold float64 `json:"threshold" db:"threshold" binding:"required"`
HoldSeconds int `json:"holdSeconds" db:"hold_seconds" binding:"min=0,max=3600"`
Enabled bool `json:"enabled" db:"enabled"`
ID string `json:"id" db:"id"`
Name string `json:"name" db:"name" binding:"required,max=80"`
Metric string `json:"metric" db:"metric" binding:"required,oneof=node_cpu node_memory node_storage node_unavailable terminal_offline terminal_cpu terminal_memory terminal_temperature"`
Threshold float64 `json:"threshold" db:"threshold" binding:"required"`
HoldSeconds int `json:"holdSeconds" db:"hold_seconds" binding:"min=0,max=3600"`
Enabled bool `json:"enabled" db:"enabled"`
UpdatedAt time.Time `json:"-" db:"updated_at"`
}
type alertRecord struct {
ID string `json:"id" db:"id"`
@@ -36,6 +37,7 @@ type alertRecord struct {
ResolveReason string `json:"resolveReason" db:"resolve_reason"`
AcknowledgedAt *time.Time `json:"acknowledgedAt" db:"acknowledged_at"`
AcknowledgedBy string `json:"acknowledgedBy" db:"acknowledged_by"`
DataFresh *bool `json:"dataFresh" db:"data_fresh"`
}
type resourcePoint struct {
Time time.Time `json:"time" db:"time"`
@@ -92,8 +94,12 @@ func (s *Server) saveAlertRule(c *gin.Context) {
}
in.Name = strings.TrimSpace(in.Name)
offline := in.Metric == "terminal_offline" || in.Metric == "node_unavailable"
if in.Name == "" || math.IsNaN(in.Threshold) || math.IsInf(in.Threshold, 0) || (!offline && (in.Threshold < 1 || in.Threshold > 100)) || (offline && (in.Threshold < 90 || in.Threshold > 86400 || in.HoldSeconds != 0)) {
s.fail(c, 400, "invalid_request", "水位阈值为 1～100%；离线阈值为 90～86400 秒且不额外设置持续时间")
maximum := 100.0
if in.Metric == "terminal_temperature" {
maximum = 150
}
if in.Name == "" || math.IsNaN(in.Threshold) || math.IsInf(in.Threshold, 0) || (!offline && (in.Threshold < 1 || in.Threshold > maximum)) || (offline && (in.Threshold < 90 || in.Threshold > 86400 || in.HoldSeconds != 0)) {
s.fail(c, 400, "invalid_request", "使用率阈值为 1～100%，SoC 温度为 1～150°C；离线阈值为 90～86400 秒且不额外设置持续时间")
return
}
ctx := c.Request.Context()
@@ -113,7 +119,7 @@ func (s *Server) saveAlertRule(c *gin.Context) {
_, err = tx.ExecContext(ctx, "INSERT INTO alert_rules(id,name,metric,threshold,hold_seconds,enabled) VALUES($1,$2,$3,$4,$5,$6)", in.ID, in.Name, in.Metric, in.Threshold, in.HoldSeconds, in.Enabled)
} else {
var id string
err = tx.GetContext(ctx, &id, "UPDATE alert_rules SET name=$1,metric=$2,threshold=$3,hold_seconds=$4,enabled=$5 WHERE id=$6 RETURNING id", in.Name, in.Metric, in.Threshold, in.HoldSeconds, in.Enabled, in.ID)
err = tx.GetContext(ctx, &id, "UPDATE alert_rules SET name=$1,metric=$2,threshold=$3,hold_seconds=$4,enabled=$5,updated_at=now() WHERE id=$6 RETURNING id", in.Name, in.Metric, in.Threshold, in.HoldSeconds, in.Enabled, in.ID)
if err == nil {
_, err = tx.ExecContext(ctx, "UPDATE alerts SET status='resolved',resolved_at=now(),resolve_reason='rule_changed' WHERE rule_id=$1 AND status='firing'", in.ID)
}
@@ -153,7 +159,7 @@ func (s *Server) alerts(c *gin.Context) {
}
items := []alertRecord{}
args = append(args, size, (page-1)*size)
if err := s.DB.SelectContext(c.Request.Context(), &items, "SELECT *"+filter+" ORDER BY opened_at DESC,id LIMIT $3 OFFSET $4", args...); err != nil {
if err := s.DB.SelectContext(c.Request.Context(), &items, "SELECT *,"+terminalAlertFreshSQL+" AS data_fresh"+filter+" ORDER BY opened_at DESC,id LIMIT $3 OFFSET $4", args...); err != nil {
s.dbError(c, err)
return
}
@@ -193,10 +199,11 @@ func (s *Server) acknowledgeAlert(c *gin.Context) {
}
type observation struct {
ID string `db:"id"`
Name string `db:"name"`
Value *float64 `db:"value"`
Unavailable bool `db:"unavailable"`
ID string `db:"id"`
Name string `db:"name"`
Value *float64 `db:"value"`
Unavailable bool `db:"unavailable"`
SampledAt *time.Time `db:"sampled_at"`
}
// One transaction serializes rule edits, expiry and condition evaluation across replicas.
@@ -229,6 +236,10 @@ func (s *Server) observe(ctx context.Context) error {
query = `SELECT r.id,r.name,CASE WHEN r.status='online' AND r.updated_at>now()-interval '90 seconds' THEN s.` + column + `*100 ELSE NULL END value,false unavailable
FROM resources r LEFT JOIN LATERAL(SELECT ` + column + ` FROM resource_samples WHERE resource_id=r.id AND sampled_at>now()-interval '90 seconds' ORDER BY sampled_at DESC LIMIT 1) s ON true WHERE r.provider='proxmox'`
}
terminalColumn := terminalAlertColumn(rule.Metric)
if terminalColumn != "" {
query = terminalAlertQuery(terminalColumn)
}
if err = tx.SelectContext(ctx, &targets, query); err != nil {
return err
}
@@ -236,6 +247,14 @@ func (s *Server) observe(ctx context.Context) error {
for _, target := range targets {
ids = append(ids, target.ID)
if target.Value == nil {
if terminalColumn != "" {
// Unknown data interrupts the pending duration, while an open
// incident remains open until a fresh normal sample arrives.
if _, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1 AND target_id=$2", rule.ID, target.ID); err != nil {
return err
}
continue
}
if _, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1 AND target_id=$2 AND NOT EXISTS(SELECT 1 FROM alerts WHERE rule_id=$1 AND target_id=$2 AND status='firing')", rule.ID, target.ID); err != nil {
return err
}
@@ -250,6 +269,12 @@ func (s *Server) observe(ctx context.Context) error {
}
continue
}
if terminalColumn != "" {
if err = evaluateTerminalHigh(ctx, tx, rule, target, terminalColumn); err != nil {
return err
}
continue
}
if _, err = tx.ExecContext(ctx, "INSERT INTO alert_states(rule_id,target_id) VALUES($1,$2) ON CONFLICT DO NOTHING", rule.ID, target.ID); err != nil {
return err
}
platform/internal/server/power_batches.go
+6
View File
@@ -74,6 +74,12 @@ func (s *Server) powerBatchPreflight(ctx context.Context, db sqlx.ExtContext, id
item.Message = message
return d, item, nil
}
if s.exclusiveGPU(d) && action != "shutdown" {
return reject("exclusive_gpu_power", "此桌面共用显卡，请从终端选择页启动")
}
if err = s.exclusiveGPUMaintenance(ctx, db, d); err != nil {
return reject("gpu_switch_busy", err.Error())
}
blocked, err := maintenanceBlocked(ctx, db, id, own, true)
if err != nil {
return d, item, err
platform/internal/server/server.go
+23 -1
View File
@@ -69,14 +69,23 @@ func New(cfg config.Config, db *sqlx.DB, pve *proxmox.Client) *Server {
t.POST("/connect-ticket", s.createTicket)
t.POST("/connect-ticket/consume", s.consumeTicket)
t.POST("/session", s.deviceSession)
t.POST("/desktop-prepare", s.prepareGPUDesktop)
t.GET("/desktop-prepare/:id", s.gpuPrepareStatus)
a := r.Group("/api/v1", s.adminAuth, s.refreshComputeRequest)
a.GET("/auth/me", func(c *gin.Context) { c.JSON(200, c.MustGet("admin")) })
a.POST("/auth/logout", s.logout)
a.POST("/auth/password", s.changeOwnPassword)
a.GET("/auth/mfa", s.mfaStatus)
a.POST("/auth/mfa/setup", s.beginMFA)
a.POST("/auth/mfa/confirm", s.confirmMFA)
a.POST("/auth/mfa/recovery", func(c *gin.Context) { s.changeMFA(c, false) })
a.DELETE("/auth/mfa", func(c *gin.Context) { s.changeMFA(c, true) })
a.GET("/administrators", s.allow("super_admin"), s.administratorList)
a.POST("/administrators", s.allow("super_admin"), s.createAdministrator)
a.PUT("/administrators/:id", s.allow("super_admin"), s.updateAdministrator)
a.POST("/administrators/:id/password", s.allow("super_admin"), s.resetAdministratorPassword)
a.GET("/administrators/:id/sessions", s.allow("super_admin"), s.administratorSessions)
a.DELETE("/administrators/:id/sessions/:sessionKey", s.allow("super_admin"), s.revokeAdministratorSession)
a.GET("/overview", s.overview)
a.GET("/sessions/history", s.sessionHistory)
a.GET("/monitor/series", s.resourceSeries)
@@ -96,6 +105,7 @@ func New(cfg config.Config, db *sqlx.DB, pve *proxmox.Client) *Server {
a.POST("/desktops/:id/snapshots/check", s.allow("super_admin"), s.checkSnapshot)
a.POST("/desktops/:id/snapshots/actions", s.allow("super_admin"), s.queueSnapshot)
a.POST("/tasks/:id/snapshot-reconcile", s.allow("super_admin"), s.reconcileSnapshot)
a.POST("/tasks/:id/gpu-reconcile", s.allow("super_admin", "operator"), s.reconcileGPUPrepare)
a.POST("/desktops/:id/hardware/check", s.allow("super_admin"), s.checkHardware)
a.POST("/desktops/:id/hardware", s.allow("super_admin"), s.queueHardware)
a.GET("/resources", s.resources)
@@ -143,11 +153,15 @@ func New(cfg config.Config, db *sqlx.DB, pve *proxmox.Client) *Server {
a.POST("/users/batch-bindings", s.allow("super_admin"), s.batchBindings)
a.PUT("/users/:id/bindings", s.allow("super_admin"), s.bindings)
a.GET("/tasks", s.tasks)
a.GET("/tasks/export", s.exportTasks)
a.GET("/tasks/:id", s.task)
a.POST("/tasks/:id/hardware-reconcile", s.allow("super_admin"), s.reconcileHardware)
a.POST("/tasks/:id/reconcile", s.allow("super_admin"), s.reconcileProvision)
a.GET("/tasks/:id/diagnostics", s.allow("super_admin", "operator"), s.taskDiagnostics)
a.GET("/diagnostics", s.allow("super_admin", "operator"), s.diagnostics)
a.GET("/diagnostics/export", s.allow("super_admin", "operator"), s.exportDiagnostics)
a.GET("/audit-logs", s.audits)
a.GET("/audit-logs/export", s.exportAudits)
r.NoRoute(func(c *gin.Context) { s.fail(c, 404, "not_found", "接口不存在") })
return s
}
@@ -216,6 +230,7 @@ func (s *Server) login(c *gin.Context) {
var input struct {
Username string `json:"username" binding:"required,max=80"`
Password string `json:"password" binding:"required,max=200"`
Code string `json:"code" binding:"max=80"`
}
if !s.decode(c, &input) {
return
@@ -231,11 +246,14 @@ func (s *Server) login(c *gin.Context) {
}
defer tx.Rollback()
// Serialize session creation against account disable, role changes and password reset.
err = tx.GetContext(c.Request.Context(), &a, "SELECT * FROM administrators WHERE username=$1 AND enabled FOR SHARE", input.Username)
err = tx.GetContext(c.Request.Context(), &a, "SELECT * FROM administrators WHERE username=$1 AND enabled FOR UPDATE", input.Username)
if err != nil || bcrypt.CompareHashAndPassword([]byte(a.PasswordHash), []byte(input.Password)) != nil {
s.fail(c, 401, "invalid_credentials", "用户名或密码错误")
return
}
if !s.loginMFA(c, tx, a.ID, input.Code) {
return
}
now := time.Now()
id := store.ID("sess_")
expires := now.Add(time.Hour)
@@ -248,6 +266,10 @@ func (s *Server) login(c *gin.Context) {
s.dbError(c, err)
return
}
if _, err = tx.ExecContext(c.Request.Context(), "INSERT INTO admin_session_details(session_id,client_ip,user_agent) VALUES($1,$2,$3)", id, c.ClientIP(), sessionUserAgent(c.GetHeader("User-Agent"))); err != nil {
s.dbError(c, err)
return
}
if err = tx.Commit(); err != nil {
s.dbError(c, err)
return
platform/internal/server/snapshots.go
+3
View File
@@ -117,6 +117,9 @@ func (s *Server) snapshotPreflight(ctx context.Context, db sqlx.ExtContext, id s
if err != nil {
return d, "", errors.New("无法读取 PVE 当前配置")
}
if err = s.exclusiveGPUMaintenance(ctx, db, d); err != nil {
return d, "", err
}
blockers, _ := snapshotBaseBlockers(h, config)
if len(blockers) > 0 {
return d, "", errors.New(strings.Join(blockers, "；"))
platform/internal/server/task_history.go
+153
View File
@@ -0,0 +1,153 @@
package server
import (
"bytes"
"database/sql"
"encoding/csv"
"fmt"
"strings"
"time"
"unicode/utf8"
"github.com/gin-gonic/gin"
"vdi-cloud/platform/internal/model"
"vdi-cloud/platform/internal/store"
)
// The same filter serves SQL pagination, pending-operation polling and CSV.
func (s *Server) taskFilter(c *gin.Context) (string, []any, bool) {
where, args := " FROM tasks WHERE true", []any{}
add := func(expr string, value any) {
args = append(args, value)
where += fmt.Sprintf(" AND "+expr, len(args))
}
var from, to time.Time
for _, key := range []string{"from", "to"} {
if raw := c.Query(key); raw != "" {
v, err := time.Parse(time.RFC3339Nano, raw)
if err != nil {
s.fail(c, 400, "invalid_range", "时间格式无效，请使用含时区的时间")
return "", nil, false
}
if key == "from" {
from = v
add("created_at >= $%d", v)
} else {
to = v
add("created_at <= $%d", v)
}
}
}
if !from.IsZero() && !to.IsZero() && !to.After(from) {
s.fail(c, 400, "invalid_range", "结束时间必须晚于开始时间")
return "", nil, false
}
if active := c.Query("active"); active != "" {
if active != "true" && active != "false" {
s.fail(c, 400, "invalid_request", "active 只能为 true 或 false")
return "", nil, false
}
if active == "true" {
where += " AND status IN ('queued','running','timeout')"
}
}
for _, key := range []string{"keyword", "actor", "action", "kind", "status", "targetId"} {
value := c.Query(key)
if utf8.RuneCountInString(value) > 200 || strings.IndexByte(value, 0) >= 0 {
s.fail(c, 400, "invalid_request", "筛选条件最长 200 字，且不能包含空字符")
return "", nil, false
}
if value == "" {
continue
}
if key == "status" && value != "queued" && value != "running" && value != "success" && value != "failed" && value != "timeout" {
s.fail(c, 400, "invalid_request", "任务状态无效")
return "", nil, false
}
if key != "keyword" && key != "actor" {
column := key
if key == "targetId" {
column = "target_id"
}
add(column+" = $%d", value)
continue
}
pattern := "%" + strings.NewReplacer(`\`, `\\`, "%", `\%`, "_", `\_`).Replace(value) + "%"
if key == "actor" {
add("actor ILIKE $%d", pattern)
} else {
args = append(args, pattern)
p := fmt.Sprintf("$%d", len(args))
where += " AND (id ILIKE " + p + " OR target_id ILIKE " + p + " OR target_name ILIKE " + p + " OR actor ILIKE " + p + " OR action ILIKE " + p + " OR message ILIKE " + p + ")"
}
}
return where, args, true
}
func (s *Server) tasks(c *gin.Context) {
where, args, ok := s.taskFilter(c)
if !ok {
return
}
page, size := pageParams(c)
tx, err := s.DB.BeginTxx(c.Request.Context(), &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
if err != nil {
s.dbError(c, err)
return
}
defer tx.Rollback()
var total int
if err = tx.GetContext(c.Request.Context(), &total, "SELECT count(*)"+where, args...); err != nil {
s.dbError(c, err)
return
}
items := []model.Task{}
query := "SELECT *" + where + fmt.Sprintf(" ORDER BY created_at DESC,id DESC LIMIT $%d OFFSET $%d", len(args)+1, len(args)+2)
args = append(args, size, (page-1)*size)
if err = tx.SelectContext(c.Request.Context(), &items, query, args...); err != nil {
s.dbError(c, err)
return
}
c.JSON(200, gin.H{"items": items, "total": total, "page": page, "pageSize": size, "requestId": c.GetString("requestId")})
}
func (s *Server) exportTasks(c *gin.Context) {
where, args, ok := s.taskFilter(c)
if !ok {
return
}
items := []model.Task{}
if err := s.DB.SelectContext(c.Request.Context(), &items, "SELECT *"+where+" ORDER BY created_at DESC,id DESC LIMIT 10001", args...); err != nil {
s.dbError(c, err)
return
}
if len(items) > 10000 {
s.fail(c, 413, "export_too_large", "匹配任务超过 10000 条，请缩小时间范围或增加筛选条件后导出")
return
}
var content bytes.Buffer
content.WriteString("\ufeff")
w := csv.NewWriter(&content)
w.UseCRLF = true
_ = w.Write([]string{"提交时间（UTC）", "更新时间（UTC）", "任务标识", "任务类型", "动作", "操作者", "目标标识", "目标名称", "状态", "执行结果"})
for _, t := range items {
cells := []string{t.CreatedAt.UTC().Format(time.RFC3339Nano), t.UpdatedAt.UTC().Format(time.RFC3339Nano), t.ID, t.Kind, t.Action, t.Actor, t.TargetID, t.TargetName, t.Status, t.Message}
for i := range cells {
cells[i] = safeCSVCell(cells[i])
}
_ = w.Write(cells)
}
w.Flush()
if err := w.Error(); err != nil {
s.fail(c, 500, "export_failed", "导出生成失败，请重试")
return
}
if _, err := s.DB.ExecContext(c.Request.Context(), "INSERT INTO audit_logs(id,actor,action,target,result,request_id) VALUES($1,$2,'export_tasks',$3,'success',$4)", store.ID("audit_"), s.admin(c).Username, fmt.Sprintf("rows=%d", len(items)), c.GetString("requestId")); err != nil {
s.dbError(c, err)
return
}
c.Header("Content-Disposition", `attachment; filename="tasks-`+time.Now().UTC().Format("20060102-150405")+`.csv"`)
c.Header("X-Export-Count", fmt.Sprint(len(items)))
c.Header("X-Content-Type-Options", "nosniff")
c.Data(200, "text/csv; charset=utf-8", content.Bytes())
}
platform/internal/server/task_history_test.go
+129
View File
@@ -0,0 +1,129 @@
package server
import (
"encoding/csv"
"net/http/httptest"
"net/url"
"strings"
"testing"
)
func TestTaskHistoryPaginationFiltersAndOldPending(t *testing.T) {
f := newFixtureWorker(t, false)
token := f.login("viewer")
_, err := f.db.Exec(`INSERT INTO tasks(id,kind,action,target_id,target_name,actor,status,message,idempotency_key,created_at,updated_at)
SELECT 'history_'||lpad(i::text,5,'0'),'desktop','start','target_'||i,'测试桌面_'||i,'worker','success','完成','history_'||i,'2025-01-02T00:00:00Z'::timestamptz,'2025-01-02T00:00:01Z'::timestamptz FROM generate_series(1,1050) i;
INSERT INTO tasks(id,kind,action,target_id,target_name,actor,status,message,idempotency_key,created_at) VALUES
('old-pending','snapshot','snapshot-create','pending-target','旧快照','Root_100%','timeout','结果未确认','old-pending','2024-01-01T00:00:00Z'),
('old-running','terminal','collect-logs','old-terminal','诊断设备','worker','running','正在执行','old-running','2024-01-01T00:00:00Z')`)
if err != nil {
t.Fatal(err)
}
code, out := f.request("GET", "/tasks?page=11&pageSize=100", token, nil, "")
expect(t, 200, code, out)
items := out["items"].([]any)
if out["total"] != float64(1052) || len(items) != 52 || items[len(items)-1].(map[string]any)["id"] != "old-pending" {
t.Fatal(out)
}
code, out = f.request("GET", "/tasks?active=true", token, nil, "")
expect(t, 200, code, out)
if out["total"] != float64(2) {
t.Fatal(out)
}
q := url.Values{"actor": {"_100%"}, "status": {"timeout"}, "action": {"snapshot-create"}, "kind": {"snapshot"}, "targetId": {"pending-target"}, "keyword": {"结果未确认"}, "from": {"2024-01-01T08:00:00+08:00"}, "to": {"2024-01-02T08:00:00+08:00"}}
code, out = f.request("GET", "/tasks?"+q.Encode(), token, nil, "")
expect(t, 200, code, out)
if out["total"] != float64(1) || out["items"].([]any)[0].(map[string]any)["id"] != "old-pending" {
t.Fatal(out)
}
for _, term := range []string{"桌面_1", "%", "' OR true --"} {
code, out = f.request("GET", "/tasks?keyword="+url.QueryEscape(term), token, nil, "")
expect(t, 200, code, out)
want := float64(0)
if term == "桌面_1" {
want = 162
}
if term == "%" {
want = 1
}
if out["total"] != want {
t.Fatal(term, out["total"])
}
}
for _, query := range []string{"from=broken", "from=2025-01-02T00:00:00Z&to=2025-01-01T00:00:00Z", "status=invalid", "active=maybe", "actor=%00", "keyword=" + strings.Repeat("x", 201)} {
for _, path := range []string{"/tasks?", "/tasks/export?"} {
code, out = f.request("GET", path+query, token, nil, "")
expect(t, 400, code, out)
}
}
code, out = f.request("GET", "/tasks?page=999999999&pageSize=999", token, nil, "")
expect(t, 200, code, out)
if out["page"] != float64(1000000) || out["pageSize"] != float64(100) || len(out["items"].([]any)) != 0 {
t.Fatal(out)
}
}
func downloadTaskCSV(f *fixture, token, query string) *httptest.ResponseRecorder {
r := httptest.NewRequest("GET", "/api/v1/tasks/export"+query, nil)
if token != "" {
r.Header.Set("Authorization", "Bearer "+token)
}
w := httptest.NewRecorder()
f.server.Router.ServeHTTP(w, r)
return w
}
func TestTaskHistoryExportRolesAndPrivateFields(t *testing.T) {
f := newFixtureWorker(t, false)
_, err := f.db.Exec(`INSERT INTO tasks(id,kind,action,target_id,target_name,actor,status,message,idempotency_key,provider_ref) VALUES ('csv-task','desktop','start','csv-target',$1,'worker','failed',$2,'private-idempotency','private-provider-payload')`, "研发,\"一组\"\n第二行", "\t=HYPERLINK(\"x\")")
if err != nil {
t.Fatal(err)
}
if w := downloadTaskCSV(f, "", ""); w.Code != 401 {
t.Fatal(w.Code)
}
for _, role := range []string{"admin", "operator", "viewer"} {
token := f.login(role)
w := downloadTaskCSV(f, token, "?keyword=csv-task&page=100&pageSize=1")
if w.Code != 200 || w.Header().Get("X-Export-Count") != "1" {
t.Fatal(w.Code, w.Body.String())
}
body := w.Body.String()
if !strings.HasPrefix(body, "\ufeff") || strings.Contains(body, "private-") {
t.Fatal("CSV exposed internal fields")
}
rows, err := csv.NewReader(strings.NewReader(strings.TrimPrefix(body, "\ufeff"))).ReadAll()
if err != nil || len(rows) != 2 || len(rows[0]) != 10 || rows[1][2] != "csv-task" || rows[1][7] != "研发,\"一组\"\n第二行" || rows[1][9] != "'\t=HYPERLINK(\"x\")" {
t.Fatal(rows, err)
}
var count int
err = f.db.Get(&count, "SELECT count(*) FROM audit_logs WHERE action='export_tasks' AND actor=$1 AND target='rows=1'", role)
if err != nil || count != 1 {
t.Fatal(count, err)
}
}
}
func TestTaskHistoryExportCompleteAndLimit(t *testing.T) {
f := newFixtureWorker(t, false)
token := f.login("admin")
_, err := f.db.Exec(`INSERT INTO tasks(id,kind,action,target_id,target_name,actor,status,idempotency_key) SELECT 'bulk_'||i,'desktop','start','target_'||i,'桌面','worker','success','bulk_'||i FROM generate_series(1,10001) i`)
if err != nil {
t.Fatal(err)
}
if w := downloadTaskCSV(f, token, ""); w.Code != 413 {
t.Fatal(w.Code)
}
if _, err = f.db.Exec("DELETE FROM tasks WHERE id='bulk_10001'"); err != nil {
t.Fatal(err)
}
w := downloadTaskCSV(f, token, "?pageSize=1")
rows, err := csv.NewReader(strings.NewReader(strings.TrimPrefix(w.Body.String(), "\ufeff"))).ReadAll()
if w.Code != 200 || w.Header().Get("X-Export-Count") != "10000" || err != nil || len(rows) != 10001 {
t.Fatal(w.Code, len(rows), err)
}
w = downloadTaskCSV(f, token, "?keyword=nonexistent")
if w.Code != 200 || w.Header().Get("X-Export-Count") != "0" {
t.Fatal(w.Code)
}
}
platform/internal/server/terminal.go
+18 -4
View File
@@ -73,7 +73,7 @@ func (s *Server) deviceAuth(c *gin.Context) {
return
}
defer tx.Rollback()
if strings.HasSuffix(c.FullPath(), "/connect-ticket") || strings.HasSuffix(c.FullPath(), "/connect-ticket/consume") {
if strings.HasSuffix(c.FullPath(), "/connect-ticket") || strings.HasSuffix(c.FullPath(), "/connect-ticket/consume") || strings.HasSuffix(c.FullPath(), "/desktop-prepare") {
if !s.lockAuthorization(c, tx) {
return
}
@@ -174,7 +174,15 @@ func (s *Server) deviceDesktops(c *gin.Context) {
return
}
if s.commitDevice(c) {
list(c, items)
type item struct {
model.Desktop
PowerOnConnect bool `json:"powerOnConnect"`
}
result := make([]item, 0, len(items))
for _, d := range items {
result = append(result, item{d, s.exclusiveGPU(d) && !d.SetupRequired && d.Endpoint != ""})
}
list(c, result)
}
}
func (s *Server) deviceCommands(c *gin.Context) {
@@ -258,7 +266,10 @@ func (s *Server) createTicket(c *gin.Context) {
if !s.hardwareAllowsConnection(c, d.ID) {
return
}
if d.SetupRequired || d.PowerState != "running" || d.Endpoint == "" {
if !s.gpuAllowsConnection(c, d) {
return
}
if d.SetupRequired || (d.PowerState != "running" && !s.exclusiveGPU(d)) || d.Endpoint == "" {
s.fail(c, 409, "desktop_not_ready", "桌面未开机或尚未配置连接地址")
return
}
@@ -309,7 +320,10 @@ func (s *Server) consumeTicket(c *gin.Context) {
if !s.hardwareAllowsConnection(c, d.ID) {
return
}
if d.SetupRequired || d.PowerState != "running" || d.Endpoint == "" {
if !s.gpuAllowsConnection(c, d) {
return
}
if d.SetupRequired || (d.PowerState != "running" && !s.exclusiveGPU(d)) || d.Endpoint == "" {
s.fail(c, 409, "desktop_not_ready", "桌面当前不可连接")
return
}
platform/internal/server/terminal_alerts.go
+52
View File
@@ -0,0 +1,52 @@
package server
import (
"context"
"github.com/jmoiron/sqlx"
"vdi-cloud/platform/internal/store"
)
// Column names are exclusively from this allowlist, never request text.
func terminalAlertColumn(metric string) string {
return map[string]string{"terminal_cpu": "cpu_percent", "terminal_memory": "memory_percent", "terminal_temperature": "soc_temperature_c"}[metric]
}
func terminalAlertQuery(column string) string {
return `SELECT t.id,t.name,false unavailable,m.sampled_at,
CASE WHEN m.supported AND t.last_seen>now()-interval '90 seconds' AND m.sampled_at>now()-interval '90 seconds'
AND m.sampled_at<=now() THEN m.` + column + ` ELSE NULL END value
FROM terminals t LEFT JOIN terminal_metrics_current m ON m.terminal_id=t.id
WHERE t.enabled AND t.token_hash IS NOT NULL AND t.status!='unregistered'`
}
// Non-terminal metrics keep a null freshness flag for backward compatibility.
// A value becomes unknown on missing/old samples, unsupported agents or revocation.
const terminalAlertFreshSQL = `CASE WHEN metric IN ('terminal_cpu','terminal_memory','terminal_temperature') THEN EXISTS(
SELECT 1 FROM terminals t JOIN terminal_metrics_current m ON m.terminal_id=t.id
WHERE t.id=alerts.target_id AND t.enabled AND t.token_hash IS NOT NULL AND t.status!='unregistered'
AND t.last_seen>now()-interval '90 seconds' AND m.supported AND m.sampled_at>now()-interval '90 seconds' AND m.sampled_at<=now()
AND CASE alerts.metric WHEN 'terminal_cpu' THEN m.cpu_percent WHEN 'terminal_memory' THEN m.memory_percent
WHEN 'terminal_temperature' THEN m.soc_temperature_c END IS NOT NULL) ELSE NULL END`
func evaluateTerminalHigh(ctx context.Context, tx *sqlx.Tx, rule alertRule, target observation, column string) error {
// Count source time, not repeated evaluator ticks. A source gap over 90s or
// an intervening null/normal history sample starts a new continuous segment.
// The history range is at most 90 seconds in the non-gap case and uses its PK.
_, err := tx.ExecContext(ctx, `INSERT INTO alert_states(rule_id,target_id,first_seen,last_sample_at) VALUES($1,$2,GREATEST($3::timestamptz,$5::timestamptz),$3)
ON CONFLICT(rule_id,target_id) DO UPDATE SET first_seen=CASE
WHEN alert_states.last_sample_at IS NULL OR EXCLUDED.last_sample_at-alert_states.last_sample_at>interval '90 seconds'
OR EXISTS(SELECT 1 FROM terminal_metric_samples s WHERE s.terminal_id=$2
AND s.sampled_at>GREATEST(alert_states.last_sample_at,EXCLUDED.last_sample_at-interval '90 seconds') AND s.sampled_at<=EXCLUDED.last_sample_at
AND (s.`+column+` IS NULL OR s.`+column+`<$4)) THEN EXCLUDED.first_seen ELSE alert_states.first_seen END,
last_sample_at=EXCLUDED.last_sample_at
WHERE alert_states.last_sample_at IS NULL OR EXCLUDED.last_sample_at>alert_states.last_sample_at`, rule.ID, target.ID, target.SampledAt, rule.Threshold, rule.UpdatedAt)
if err != nil {
return err
}
_, err = tx.ExecContext(ctx, `INSERT INTO alerts(id,rule_id,rule_name,metric,target_id,target_name,value,threshold,started_at)
SELECT $1,$2,$3,$4,$5,$6,$7,$8,first_seen FROM alert_states WHERE rule_id=$2 AND target_id=$5
AND ($9=0 OR last_sample_at-first_seen>=make_interval(secs=>$9) OR EXISTS(SELECT 1 FROM alerts WHERE rule_id=$2 AND target_id=$5 AND status='firing'))
ON CONFLICT(rule_id,target_id) WHERE status='firing' DO UPDATE SET value=EXCLUDED.value`,
store.ID("alert_"), rule.ID, rule.Name, rule.Metric, target.ID, target.Name, *target.Value, rule.Threshold, rule.HoldSeconds)
return err
}
platform/internal/server/terminal_alerts_test.go
+266
View File
@@ -0,0 +1,266 @@
package server
import (
"os"
"testing"
"vdi-cloud/platform/internal/store"
)
func TestTerminalAlertSchemaUpgradeIsRepeatable(t *testing.T) {
f, _, _ := terminalAlertFixture(t)
if _, err := f.db.Exec(`INSERT INTO alert_rules(id,name,metric,threshold,hold_seconds,enabled) VALUES('upgrade-old','原有停用规则','node_cpu',87,45,false);
INSERT INTO alerts(id,rule_id,rule_name,metric,target_id,target_name,value,threshold,started_at) VALUES('upgrade-event','upgrade-old','原有停用规则','node_cpu','old-node','原有节点',95,87,now());
INSERT INTO alert_states(rule_id,target_id) VALUES('upgrade-old','old-node')`); err != nil {
t.Fatal(err)
}
if _, err := f.db.Exec(`ALTER TABLE alert_rules DROP CONSTRAINT alert_rules_metric_check;
ALTER TABLE alert_rules ADD CONSTRAINT alert_rules_metric_check CHECK(metric IN ('node_cpu','node_memory','node_storage','node_unavailable','terminal_offline'));
ALTER TABLE alert_states DROP COLUMN last_sample_at;
ALTER TABLE alert_rules DROP COLUMN updated_at`); err != nil {
t.Fatal(err)
}
for i := 0; i < 2; i++ {
db, err := store.Open(t.Context(), os.Getenv("VDI_TEST_DATABASE_URL"))
if err != nil {
t.Fatal(err)
}
db.Close()
}
var count int
if err := f.db.Get(&count, "SELECT count(*) FROM alert_rules WHERE id='upgrade-old' AND NOT enabled AND threshold=87 AND hold_seconds=45"); err != nil || count != 1 {
t.Fatal("old rules changed", count, err)
}
if err := f.db.Get(&count, "SELECT count(*) FROM alerts WHERE id='upgrade-event' AND status='firing' AND value=95"); err != nil || count != 1 {
t.Fatal("old event changed", count, err)
}
if err := f.db.Get(&count, "SELECT count(*) FROM alert_states WHERE rule_id='upgrade-old' AND target_id='old-node' AND last_sample_at IS NULL"); err != nil || count != 1 {
t.Fatal("old pending state changed", count, err)
}
if _, err := f.db.Exec("INSERT INTO alert_rules(id,name,metric,threshold) VALUES('upgrade-temp','升级温度规则','terminal_temperature',80)"); err != nil {
t.Fatal(err)
}
}
func terminalAlertFixture(t *testing.T) (*fixture, string, func(int, any, any, any)) {
t.Helper()
f := newFixtureWorker(t, false)
exec := func(query string, args ...any) {
t.Helper()
if _, err := f.db.Exec(query, args...); err != nil {
t.Fatal(err)
}
}
exec("UPDATE alert_rules SET enabled=false")
exec(`INSERT INTO terminals(id,sn,name,bootstrap_hash,token_hash,last_seen,status)
VALUES('alert-box','ALERT-BOX','真实指标测试终端','',$1,now(),'online')`, store.Hash("alert-device"))
exec(`INSERT INTO terminal_metrics_current(terminal_id,sampled_at,supported) VALUES('alert-box',now(),true)`)
report := func(age int, cpu, memory, temp any) {
t.Helper()
exec("UPDATE terminals SET last_seen=now() WHERE id='alert-box'")
exec(`UPDATE terminal_metrics_current SET supported=true,sampled_at=now()-make_interval(secs=>$1),cpu_percent=$2,memory_percent=$3,soc_temperature_c=$4 WHERE terminal_id='alert-box'`, age, cpu, memory, temp)
exec(`INSERT INTO terminal_metric_samples(terminal_id,sampled_at,cpu_percent,memory_percent,soc_temperature_c)
SELECT terminal_id,sampled_at,cpu_percent,memory_percent,soc_temperature_c FROM terminal_metrics_current WHERE terminal_id='alert-box' ON CONFLICT DO NOTHING`)
}
return f, f.login("admin"), report
}
func TestTerminalMetricAlertsSourceTimeMissingAndRecovery(t *testing.T) {
f, admin, report := terminalAlertFixture(t)
evaluate := func() {
t.Helper()
if err := f.server.observe(t.Context()); err != nil {
t.Fatal(err)
}
}
count := func(query string) int {
t.Helper()
var n int
if err := f.db.Get(&n, query); err != nil {
t.Fatal(err)
}
return n
}
for _, metric := range []string{"terminal_cpu", "terminal_memory", "terminal_temperature"} {
threshold := 90
if metric == "terminal_temperature" {
threshold = 80
}
code, out := f.request("POST", "/monitor/rules", admin, map[string]any{"name": metric, "metric": metric, "threshold": threshold, "holdSeconds": 60, "enabled": true}, "")
expect(t, 200, code, out)
}
// Simulate rules that were already enabled before the first source sample.
f.db.Exec("UPDATE alert_rules SET updated_at=now()-interval '70 seconds' WHERE enabled")
report(65, 95, 96, 100)
evaluate()
evaluate()
if count("SELECT count(*) FROM alerts") != 0 {
t.Fatal("repeated evaluation advanced source duration")
}
report(0, 96, 97, 101)
evaluate()
evaluate()
if count("SELECT count(*) FROM alerts WHERE status='firing'") != 3 {
t.Fatal("missing or duplicate terminal events")
}
viewer := f.login("viewer")
code, out := f.request("GET", "/monitor/alerts", viewer, nil, "")
expect(t, 200, code, out)
for _, item := range out["items"].([]any) {
if item.(map[string]any)["dataFresh"] != true {
t.Fatal(out)
}
}
// A missing sensor cannot resolve an open event or be represented as zero.
report(0, 96, nil, 101)
evaluate()
code, out = f.request("GET", "/monitor/alerts", viewer, nil, "")
expect(t, 200, code, out)
for _, item := range out["items"].([]any) {
row := item.(map[string]any)
if row["metric"] == "terminal_memory" && row["dataFresh"] != false {
t.Fatal(row)
}
}
if count("SELECT count(*) FROM alerts WHERE status='firing'") != 3 {
t.Fatal("null sensor claimed recovery")
}
f.db.Exec("UPDATE terminal_metrics_current SET sampled_at=now()-interval '91 seconds'")
evaluate()
if count("SELECT count(*) FROM alert_states WHERE target_id='alert-box'") != 0 {
t.Fatal("stale pending duration retained")
}
code, out = f.request("GET", "/monitor/alerts", viewer, nil, "")
expect(t, 200, code, out)
for _, item := range out["items"].([]any) {
if item.(map[string]any)["dataFresh"] != false {
t.Fatal(out)
}
}
report(0, 96, 97, 101)
evaluate()
if count("SELECT count(*) FROM alerts") != 3 {
t.Fatal("data resume duplicated an open event")
}
f.db.Exec("UPDATE terminal_metrics_current SET supported=false")
evaluate()
if count("SELECT count(*) FROM alerts WHERE status='firing'") != 3 {
t.Fatal("old agent claimed recovery")
}
report(0, 10, 30, 40)
evaluate()
if count("SELECT count(*) FROM alerts WHERE status='resolved' AND resolve_reason='condition_cleared'") != 3 {
t.Fatal("fresh normal metrics did not resolve")
}
// Zero-duration rules can trigger from the first actual high sample.
for _, item := range []string{"terminal_cpu", "terminal_memory", "terminal_temperature"} {
f.db.Exec("UPDATE alert_rules SET hold_seconds=0 WHERE metric=$1", item)
}
report(0, 95, 96, 100)
evaluate()
f.db.Exec("UPDATE terminals SET enabled=false WHERE id='alert-box'")
evaluate()
if count("SELECT count(*) FROM alerts WHERE status='resolved' AND resolve_reason='target_unavailable'") != 3 {
t.Fatal("revocation did not retire alerts")
}
}
func TestTerminalMetricAlertContinuityAcrossMissedEvaluations(t *testing.T) {
f, admin, report := terminalAlertFixture(t)
code, out := f.request("POST", "/monitor/rules", admin, map[string]any{"name": "连续 CPU", "metric": "terminal_cpu", "threshold": 90, "holdSeconds": 60, "enabled": true}, "")
expect(t, 200, code, out)
f.db.Exec("UPDATE alert_rules SET updated_at=now()-interval '80 seconds' WHERE enabled")
evaluate := func() {
t.Helper()
if err := f.server.observe(t.Context()); err != nil {
t.Fatal(err)
}
}
noAlert := func() {
t.Helper()
var n int
f.db.Get(&n, "SELECT count(*) FROM alerts")
if n != 0 {
t.Fatal("non-continuous samples triggered an alert")
}
}
report(75, 95, nil, nil)
evaluate()
if _, err := f.db.Exec(`INSERT INTO terminal_metric_samples(terminal_id,sampled_at,cpu_percent) VALUES('alert-box',now()-interval '40 seconds',20)`); err != nil {
t.Fatal(err)
}
report(0, 95, nil, nil)
evaluate()
noAlert() // the observer missed a normal sample
f.db.Exec("UPDATE alert_states SET first_seen=now()-interval '120 seconds',last_sample_at=now()-interval '100 seconds'")
report(0, 95, nil, nil)
evaluate()
noAlert() // monitoring gap restarts the duration
report(0, nil, nil, nil)
evaluate()
var n int
f.db.Get(&n, "SELECT count(*) FROM alert_states WHERE target_id='alert-box'")
if n != 0 {
t.Fatal("null sample did not reset pending state")
}
report(0, 95, nil, nil)
evaluate()
noAlert()
// Missing history samples also interrupt a pending high segment.
f.db.Exec("UPDATE alert_states SET first_seen=now()-interval '80 seconds',last_sample_at=now()-interval '80 seconds'")
f.db.Exec("INSERT INTO terminal_metric_samples(terminal_id,sampled_at) VALUES('alert-box',now()-interval '30 seconds')")
report(0, 95, nil, nil)
evaluate()
noAlert()
}
func TestTerminalAlertRuleBoundsAndPermissions(t *testing.T) {
f, admin, _ := terminalAlertFixture(t)
for _, test := range []struct {
metric string
threshold float64
want int
}{{"terminal_temperature", 150, 200}, {"terminal_temperature", 150.1, 400}, {"terminal_temperature", 0, 400}, {"terminal_cpu", 101, 400}, {"terminal_memory", 100, 200}, {"unexpected", 20, 400}} {
code, out := f.request("POST", "/monitor/rules", admin, map[string]any{"name": "阈值", "metric": test.metric, "threshold": test.threshold, "holdSeconds": 0, "enabled": false}, "")
expect(t, test.want, code, out)
}
for _, role := range []string{"viewer", "operator"} {
code, out := f.request("POST", "/monitor/rules", f.login(role), map[string]any{"name": "无权修改", "metric": "terminal_temperature", "threshold": 80, "holdSeconds": 60, "enabled": true}, "")
expect(t, 403, code, out)
}
}
func TestNewTerminalRuleDoesNotCountPreEnableSamples(t *testing.T) {
f, admin, report := terminalAlertFixture(t)
body := map[string]any{"name": "新规则", "metric": "terminal_cpu", "threshold": 90, "holdSeconds": 60, "enabled": true}
code, out := f.request("POST", "/monitor/rules", admin, body, "")
expect(t, 200, code, out)
id := out["id"].(string)
report(65, 95, nil, nil)
if err := f.server.observe(t.Context()); err != nil {
t.Fatal(err)
}
var clamped bool
if err := f.db.Get(&clamped, "SELECT s.first_seen>=r.updated_at AND s.first_seen>s.last_sample_at FROM alert_states s JOIN alert_rules r ON r.id=s.rule_id WHERE r.id=$1", id); err != nil || !clamped {
t.Fatal("pre-enable sample was counted", err)
}
report(0, 95, nil, nil)
if err := f.server.observe(t.Context()); err != nil {
t.Fatal(err)
}
var count int
f.db.Get(&count, "SELECT count(*) FROM alerts WHERE rule_id=$1", id)
if count != 0 {
t.Fatal("rule fired before its own hold interval")
}
// A zero hold still triggers immediately from a valid current reading.
body["holdSeconds"] = 0
code, out = f.request("PUT", "/monitor/rules/"+id, admin, body, "")
expect(t, 200, code, out)
if err := f.server.observe(t.Context()); err != nil {
t.Fatal(err)
}
f.db.Get(&count, "SELECT count(*) FROM alerts WHERE rule_id=$1 AND status='firing'", id)
if count != 1 {
t.Fatal("zero hold did not trigger")
}
}