package server import ( "context" "log" "math" "strings" "time" "github.com/gin-gonic/gin" "github.com/jmoiron/sqlx" "vdi-cloud/platform/internal/store" ) type alertRule struct { ID string `json:"id" db:"id"` Name string `json:"name" db:"name" binding:"required,max=80"` Metric string `json:"metric" db:"metric" binding:"required,oneof=node_cpu node_memory node_storage node_unavailable terminal_offline"` Threshold float64 `json:"threshold" db:"threshold" binding:"required"` HoldSeconds int `json:"holdSeconds" db:"hold_seconds" binding:"min=0,max=3600"` Enabled bool `json:"enabled" db:"enabled"` } type alertRecord struct { ID string `json:"id" db:"id"` RuleID string `json:"ruleId" db:"rule_id"` RuleName string `json:"ruleName" db:"rule_name"` Metric string `json:"metric" db:"metric"` TargetID string `json:"targetId" db:"target_id"` TargetName string `json:"targetName" db:"target_name"` Value float64 `json:"value" db:"value"` Threshold float64 `json:"threshold" db:"threshold"` Status string `json:"status" db:"status"` StartedAt time.Time `json:"startedAt" db:"started_at"` OpenedAt time.Time `json:"openedAt" db:"opened_at"` ResolvedAt *time.Time `json:"resolvedAt" db:"resolved_at"` ResolveReason string `json:"resolveReason" db:"resolve_reason"` AcknowledgedAt *time.Time `json:"acknowledgedAt" db:"acknowledged_at"` AcknowledgedBy string `json:"acknowledgedBy" db:"acknowledged_by"` } type resourcePoint struct { Time time.Time `json:"time" db:"time"` CPU *float64 `json:"cpu" db:"cpu"` Memory *float64 `json:"memory" db:"memory"` Storage *float64 `json:"storage" db:"storage"` Samples int `json:"samples" db:"samples"` } func (s *Server) resourceSeries(c *gin.Context) { from, to, ok := timeRange(c, 7) if !ok { s.fail(c, 400, "invalid_range", "请选择有效时间范围,最长 7 天") return } id := c.Query("resourceId") if id == "" || len(id) > 100 { s.fail(c, 400, "invalid_request", "请选择节点") return } bucket := 60 if to.Sub(from) > 6*time.Hour { bucket = 300 } if to.Sub(from) > 24*time.Hour { bucket = 1800 } items := []resourcePoint{} err := s.DB.SelectContext(c.Request.Context(), &items, `SELECT date_bin(make_interval(secs=>$4),sampled_at,'2000-01-01'::timestamptz) time, avg(cpu)*100 cpu,avg(memory)*100 memory,avg(storage)*100 storage,count(*) samples FROM resource_samples WHERE resource_id=$1 AND sampled_at>=$2 AND sampled_at<=$3 GROUP BY 1 ORDER BY 1`, id, from, to, bucket) if err != nil { s.dbError(c, err) return } c.JSON(200, gin.H{"items": items, "bucketSeconds": bucket, "retentionDays": 30}) } func (s *Server) alertRules(c *gin.Context) { items := []alertRule{} if err := s.DB.SelectContext(c.Request.Context(), &items, "SELECT * FROM alert_rules ORDER BY name,id"); err != nil { s.dbError(c, err) return } list(c, items) } func observabilityLock(ctx context.Context, tx *sqlx.Tx) error { _, err := tx.ExecContext(ctx, "SELECT pg_advisory_xact_lock(76511102026)") return err } func (s *Server) saveAlertRule(c *gin.Context) { var in alertRule if !s.decode(c, &in) { return } in.Name = strings.TrimSpace(in.Name) offline := in.Metric == "terminal_offline" || in.Metric == "node_unavailable" if in.Name == "" || math.IsNaN(in.Threshold) || math.IsInf(in.Threshold, 0) || (!offline && (in.Threshold < 1 || in.Threshold > 100)) || (offline && (in.Threshold < 90 || in.Threshold > 86400 || in.HoldSeconds != 0)) { s.fail(c, 400, "invalid_request", "水位阈值为 1~100%;离线阈值为 90~86400 秒且不额外设置持续时间") return } ctx := c.Request.Context() tx, err := s.DB.BeginTxx(ctx, nil) if err != nil { s.dbError(c, err) return } defer tx.Rollback() if err = observabilityLock(ctx, tx); err != nil { s.dbError(c, err) return } in.ID = c.Param("id") if in.ID == "" { in.ID = store.ID("rule_") _, err = tx.ExecContext(ctx, "INSERT INTO alert_rules(id,name,metric,threshold,hold_seconds,enabled) VALUES($1,$2,$3,$4,$5,$6)", in.ID, in.Name, in.Metric, in.Threshold, in.HoldSeconds, in.Enabled) } else { var id string err = tx.GetContext(ctx, &id, "UPDATE alert_rules SET name=$1,metric=$2,threshold=$3,hold_seconds=$4,enabled=$5 WHERE id=$6 RETURNING id", in.Name, in.Metric, in.Threshold, in.HoldSeconds, in.Enabled, in.ID) if err == nil { _, err = tx.ExecContext(ctx, "UPDATE alerts SET status='resolved',resolved_at=now(),resolve_reason='rule_changed' WHERE rule_id=$1 AND status='firing'", in.ID) } if err == nil { _, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1", in.ID) } } if err == nil { err = accountAudit(ctx, tx, s.admin(c).Username, "save_alert_rule", in.ID, c.GetString("requestId")) } if err == nil { err = tx.Commit() } if err != nil { s.dbError(c, err) return } c.JSON(200, in) } func (s *Server) alerts(c *gin.Context) { status := c.Query("status") if status != "" && status != "firing" && status != "resolved" { s.fail(c, 400, "invalid_request", "告警状态无效") return } page, size := pageParams(c) filter := " FROM alerts WHERE ($1='' OR status=$1) AND (target_name ILIKE $2 OR rule_name ILIKE $2)" args := []any{status, "%" + c.Query("keyword") + "%"} var total, firing, unacknowledged int if err := s.DB.GetContext(c.Request.Context(), &total, "SELECT count(*)"+filter, args...); err != nil { s.dbError(c, err) return } if err := s.DB.QueryRowContext(c.Request.Context(), "SELECT count(*),count(*) FILTER(WHERE acknowledged_at IS NULL) FROM alerts WHERE status='firing'").Scan(&firing, &unacknowledged); err != nil { s.dbError(c, err) return } items := []alertRecord{} args = append(args, size, (page-1)*size) if err := s.DB.SelectContext(c.Request.Context(), &items, "SELECT *"+filter+" ORDER BY opened_at DESC,id LIMIT $3 OFFSET $4", args...); err != nil { s.dbError(c, err) return } c.JSON(200, gin.H{"items": items, "total": total, "page": page, "pageSize": size, "firingCount": firing, "unacknowledgedCount": unacknowledged, "requestId": c.GetString("requestId")}) } func (s *Server) acknowledgeAlert(c *gin.Context) { ctx := c.Request.Context() tx, err := s.DB.BeginTxx(ctx, nil) if err != nil { s.dbError(c, err) return } defer tx.Rollback() var row alertRecord if err = tx.GetContext(ctx, &row, "SELECT * FROM alerts WHERE id=$1 FOR UPDATE", c.Param("id")); err != nil { s.dbError(c, err) return } if row.Status != "firing" { s.fail(c, 409, "alert_resolved", "告警已经恢复") return } if row.AcknowledgedAt == nil { _, err = tx.ExecContext(ctx, "UPDATE alerts SET acknowledged_at=now(),acknowledged_by=$1 WHERE id=$2", s.admin(c).Username, row.ID) if err == nil { err = accountAudit(ctx, tx, s.admin(c).Username, "acknowledge_alert", row.ID, c.GetString("requestId")) } } if err == nil { err = tx.Commit() } if err != nil { s.dbError(c, err) return } c.JSON(200, gin.H{"accepted": true}) } type observation struct { ID string `db:"id"` Name string `db:"name"` Value *float64 `db:"value"` Unavailable bool `db:"unavailable"` } // One transaction serializes rule edits, expiry and condition evaluation across replicas. func (s *Server) observe(ctx context.Context) error { tx, err := s.DB.BeginTxx(ctx, nil) if err != nil { return err } defer tx.Rollback() var acquired bool if err = tx.GetContext(ctx, &acquired, "SELECT pg_try_advisory_xact_lock(76511102026)"); err != nil || !acquired { return err } if _, err = tx.ExecContext(ctx, `UPDATE session_history SET state='interrupted',ended_at=last_reported_at+interval '90 seconds',end_reason='heartbeat_timeout' WHERE ended_at IS NULL AND last_reported_atnow()-interval '90 seconds' THEN s.` + column + `*100 ELSE NULL END value,false unavailable FROM resources r LEFT JOIN LATERAL(SELECT ` + column + ` FROM resource_samples WHERE resource_id=r.id AND sampled_at>now()-interval '90 seconds' ORDER BY sampled_at DESC LIMIT 1) s ON true WHERE r.provider='proxmox'` } if err = tx.SelectContext(ctx, &targets, query); err != nil { return err } ids := []string{} for _, target := range targets { ids = append(ids, target.ID) if target.Value == nil { if _, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1 AND target_id=$2 AND NOT EXISTS(SELECT 1 FROM alerts WHERE rule_id=$1 AND target_id=$2 AND status='firing')", rule.ID, target.ID); err != nil { return err } continue } if *target.Value < rule.Threshold && !target.Unavailable { if _, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1 AND target_id=$2", rule.ID, target.ID); err != nil { return err } if _, err = tx.ExecContext(ctx, "UPDATE alerts SET status='resolved',resolved_at=now(),resolve_reason='condition_cleared' WHERE rule_id=$1 AND target_id=$2 AND status='firing'", rule.ID, target.ID); err != nil { return err } continue } if _, err = tx.ExecContext(ctx, "INSERT INTO alert_states(rule_id,target_id) VALUES($1,$2) ON CONFLICT DO NOTHING", rule.ID, target.ID); err != nil { return err } hold := rule.HoldSeconds if rule.Metric == "node_unavailable" && target.Unavailable { hold = int(math.Ceil(rule.Threshold)) if err = tx.GetContext(ctx, target.Value, "SELECT extract(epoch FROM(now()-first_seen))::double precision FROM alert_states WHERE rule_id=$1 AND target_id=$2", rule.ID, target.ID); err != nil { return err } } if _, err = tx.ExecContext(ctx, `INSERT INTO alerts(id,rule_id,rule_name,metric,target_id,target_name,value,threshold,started_at) SELECT $1,$2,$3,$4,$5,$6,$7,$8,first_seen FROM alert_states WHERE rule_id=$2 AND target_id=$5 AND first_seen<=now()-make_interval(secs=>$9) ON CONFLICT(rule_id,target_id) WHERE status='firing' DO UPDATE SET value=EXCLUDED.value`, store.ID("alert_"), rule.ID, rule.Name, rule.Metric, target.ID, target.Name, *target.Value, rule.Threshold, hold); err != nil { return err } } if _, err = tx.ExecContext(ctx, "UPDATE alerts SET status='resolved',resolved_at=now(),resolve_reason='target_unavailable' WHERE rule_id=$1 AND NOT(target_id=ANY($2::text[])) AND status='firing'", rule.ID, ids); err != nil { return err } if _, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1 AND NOT(target_id=ANY($2::text[]))", rule.ID, ids); err != nil { return err } } // Bounded retention. Current incidents and active session records are preserved. for _, query := range []string{"DELETE FROM login_limits WHERE window_started