功能：扩展云管资源运维、账号授权及监控管理
This commit is contained in:
songyiming committed
1 parent
eef485306f
commit
6134d49c16
87 files changed
+29498
-126
No files matched your search
@@ -0,0 +1,299 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"math"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"vdi-cloud/platform/internal/store"
|
||||
)
|
||||
|
||||
type alertRule struct {
|
||||
ID string `json:"id" db:"id"`
|
||||
Name string `json:"name" db:"name" binding:"required,max=80"`
|
||||
Metric string `json:"metric" db:"metric" binding:"required,oneof=node_cpu node_memory node_storage node_unavailable terminal_offline"`
|
||||
Threshold float64 `json:"threshold" db:"threshold" binding:"required"`
|
||||
HoldSeconds int `json:"holdSeconds" db:"hold_seconds" binding:"min=0,max=3600"`
|
||||
Enabled bool `json:"enabled" db:"enabled"`
|
||||
}
|
||||
type alertRecord struct {
|
||||
ID string `json:"id" db:"id"`
|
||||
RuleID string `json:"ruleId" db:"rule_id"`
|
||||
RuleName string `json:"ruleName" db:"rule_name"`
|
||||
Metric string `json:"metric" db:"metric"`
|
||||
TargetID string `json:"targetId" db:"target_id"`
|
||||
TargetName string `json:"targetName" db:"target_name"`
|
||||
Value float64 `json:"value" db:"value"`
|
||||
Threshold float64 `json:"threshold" db:"threshold"`
|
||||
Status string `json:"status" db:"status"`
|
||||
StartedAt time.Time `json:"startedAt" db:"started_at"`
|
||||
OpenedAt time.Time `json:"openedAt" db:"opened_at"`
|
||||
ResolvedAt *time.Time `json:"resolvedAt" db:"resolved_at"`
|
||||
ResolveReason string `json:"resolveReason" db:"resolve_reason"`
|
||||
AcknowledgedAt *time.Time `json:"acknowledgedAt" db:"acknowledged_at"`
|
||||
AcknowledgedBy string `json:"acknowledgedBy" db:"acknowledged_by"`
|
||||
}
|
||||
type resourcePoint struct {
|
||||
Time time.Time `json:"time" db:"time"`
|
||||
CPU *float64 `json:"cpu" db:"cpu"`
|
||||
Memory *float64 `json:"memory" db:"memory"`
|
||||
Storage *float64 `json:"storage" db:"storage"`
|
||||
Samples int `json:"samples" db:"samples"`
|
||||
}
|
||||
|
||||
func (s *Server) resourceSeries(c *gin.Context) {
|
||||
from, to, ok := timeRange(c, 7)
|
||||
if !ok {
|
||||
s.fail(c, 400, "invalid_range", "请选择有效时间范围，最长 7 天")
|
||||
return
|
||||
}
|
||||
id := c.Query("resourceId")
|
||||
if id == "" || len(id) > 100 {
|
||||
s.fail(c, 400, "invalid_request", "请选择节点")
|
||||
return
|
||||
}
|
||||
bucket := 60
|
||||
if to.Sub(from) > 6*time.Hour {
|
||||
bucket = 300
|
||||
}
|
||||
if to.Sub(from) > 24*time.Hour {
|
||||
bucket = 1800
|
||||
}
|
||||
items := []resourcePoint{}
|
||||
err := s.DB.SelectContext(c.Request.Context(), &items, `SELECT date_bin(make_interval(secs=>$4),sampled_at,'2000-01-01'::timestamptz) time,
|
||||
avg(cpu)*100 cpu,avg(memory)*100 memory,avg(storage)*100 storage,count(*) samples FROM resource_samples
|
||||
WHERE resource_id=$1 AND sampled_at>=$2 AND sampled_at<=$3 GROUP BY 1 ORDER BY 1`, id, from, to, bucket)
|
||||
if err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"items": items, "bucketSeconds": bucket, "retentionDays": 30})
|
||||
}
|
||||
func (s *Server) alertRules(c *gin.Context) {
|
||||
items := []alertRule{}
|
||||
if err := s.DB.SelectContext(c.Request.Context(), &items, "SELECT * FROM alert_rules ORDER BY name,id"); err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
list(c, items)
|
||||
}
|
||||
func observabilityLock(ctx context.Context, tx *sqlx.Tx) error {
|
||||
_, err := tx.ExecContext(ctx, "SELECT pg_advisory_xact_lock(76511102026)")
|
||||
return err
|
||||
}
|
||||
func (s *Server) saveAlertRule(c *gin.Context) {
|
||||
var in alertRule
|
||||
if !s.decode(c, &in) {
|
||||
return
|
||||
}
|
||||
in.Name = strings.TrimSpace(in.Name)
|
||||
offline := in.Metric == "terminal_offline" || in.Metric == "node_unavailable"
|
||||
if in.Name == "" || math.IsNaN(in.Threshold) || math.IsInf(in.Threshold, 0) || (!offline && (in.Threshold < 1 || in.Threshold > 100)) || (offline && (in.Threshold < 90 || in.Threshold > 86400 || in.HoldSeconds != 0)) {
|
||||
s.fail(c, 400, "invalid_request", "水位阈值为 1～100%；离线阈值为 90～86400 秒且不额外设置持续时间")
|
||||
return
|
||||
}
|
||||
ctx := c.Request.Context()
|
||||
tx, err := s.DB.BeginTxx(ctx, nil)
|
||||
if err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if err = observabilityLock(ctx, tx); err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
in.ID = c.Param("id")
|
||||
if in.ID == "" {
|
||||
in.ID = store.ID("rule_")
|
||||
_, err = tx.ExecContext(ctx, "INSERT INTO alert_rules(id,name,metric,threshold,hold_seconds,enabled) VALUES($1,$2,$3,$4,$5,$6)", in.ID, in.Name, in.Metric, in.Threshold, in.HoldSeconds, in.Enabled)
|
||||
} else {
|
||||
var id string
|
||||
err = tx.GetContext(ctx, &id, "UPDATE alert_rules SET name=$1,metric=$2,threshold=$3,hold_seconds=$4,enabled=$5 WHERE id=$6 RETURNING id", in.Name, in.Metric, in.Threshold, in.HoldSeconds, in.Enabled, in.ID)
|
||||
if err == nil {
|
||||
_, err = tx.ExecContext(ctx, "UPDATE alerts SET status='resolved',resolved_at=now(),resolve_reason='rule_changed' WHERE rule_id=$1 AND status='firing'", in.ID)
|
||||
}
|
||||
if err == nil {
|
||||
_, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1", in.ID)
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
err = accountAudit(ctx, tx, s.admin(c).Username, "save_alert_rule", in.ID, c.GetString("requestId"))
|
||||
}
|
||||
if err == nil {
|
||||
err = tx.Commit()
|
||||
}
|
||||
if err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, in)
|
||||
}
|
||||
func (s *Server) alerts(c *gin.Context) {
|
||||
status := c.Query("status")
|
||||
if status != "" && status != "firing" && status != "resolved" {
|
||||
s.fail(c, 400, "invalid_request", "告警状态无效")
|
||||
return
|
||||
}
|
||||
page, size := pageParams(c)
|
||||
filter := " FROM alerts WHERE ($1='' OR status=$1) AND (target_name ILIKE $2 OR rule_name ILIKE $2)"
|
||||
args := []any{status, "%" + c.Query("keyword") + "%"}
|
||||
var total, firing, unacknowledged int
|
||||
if err := s.DB.GetContext(c.Request.Context(), &total, "SELECT count(*)"+filter, args...); err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
if err := s.DB.QueryRowContext(c.Request.Context(), "SELECT count(*),count(*) FILTER(WHERE acknowledged_at IS NULL) FROM alerts WHERE status='firing'").Scan(&firing, &unacknowledged); err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
items := []alertRecord{}
|
||||
args = append(args, size, (page-1)*size)
|
||||
if err := s.DB.SelectContext(c.Request.Context(), &items, "SELECT *"+filter+" ORDER BY opened_at DESC,id LIMIT $3 OFFSET $4", args...); err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"items": items, "total": total, "page": page, "pageSize": size, "firingCount": firing, "unacknowledgedCount": unacknowledged, "requestId": c.GetString("requestId")})
|
||||
}
|
||||
func (s *Server) acknowledgeAlert(c *gin.Context) {
|
||||
ctx := c.Request.Context()
|
||||
tx, err := s.DB.BeginTxx(ctx, nil)
|
||||
if err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var row alertRecord
|
||||
if err = tx.GetContext(ctx, &row, "SELECT * FROM alerts WHERE id=$1 FOR UPDATE", c.Param("id")); err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
if row.Status != "firing" {
|
||||
s.fail(c, 409, "alert_resolved", "告警已经恢复")
|
||||
return
|
||||
}
|
||||
if row.AcknowledgedAt == nil {
|
||||
_, err = tx.ExecContext(ctx, "UPDATE alerts SET acknowledged_at=now(),acknowledged_by=$1 WHERE id=$2", s.admin(c).Username, row.ID)
|
||||
if err == nil {
|
||||
err = accountAudit(ctx, tx, s.admin(c).Username, "acknowledge_alert", row.ID, c.GetString("requestId"))
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
err = tx.Commit()
|
||||
}
|
||||
if err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"accepted": true})
|
||||
}
|
||||
|
||||
type observation struct {
|
||||
ID string `db:"id"`
|
||||
Name string `db:"name"`
|
||||
Value *float64 `db:"value"`
|
||||
Unavailable bool `db:"unavailable"`
|
||||
}
|
||||
|
||||
// One transaction serializes rule edits, expiry and condition evaluation across replicas.
|
||||
func (s *Server) observe(ctx context.Context) error {
|
||||
tx, err := s.DB.BeginTxx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var acquired bool
|
||||
if err = tx.GetContext(ctx, &acquired, "SELECT pg_try_advisory_xact_lock(76511102026)"); err != nil || !acquired {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `UPDATE session_history SET state='interrupted',ended_at=last_reported_at+interval '90 seconds',end_reason='heartbeat_timeout'
|
||||
WHERE ended_at IS NULL AND last_reported_at<now()-interval '90 seconds'`); err != nil {
|
||||
return err
|
||||
}
|
||||
rules := []alertRule{}
|
||||
if err = tx.SelectContext(ctx, &rules, "SELECT * FROM alert_rules WHERE enabled ORDER BY id"); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, rule := range rules {
|
||||
targets := []observation{}
|
||||
query := `SELECT id,name,extract(epoch FROM(now()-last_seen))::double precision value,false unavailable FROM terminals WHERE enabled AND token_hash IS NOT NULL AND last_seen IS NOT NULL AND status!='unregistered'`
|
||||
if rule.Metric == "node_unavailable" {
|
||||
query = `SELECT id,name,extract(epoch FROM(now()-updated_at))::double precision value,(status!='online') unavailable FROM resources WHERE provider='proxmox'`
|
||||
}
|
||||
column := map[string]string{"node_cpu": "cpu", "node_memory": "memory", "node_storage": "storage"}[rule.Metric]
|
||||
if column != "" {
|
||||
query = `SELECT r.id,r.name,CASE WHEN r.status='online' AND r.updated_at>now()-interval '90 seconds' THEN s.` + column + `*100 ELSE NULL END value,false unavailable
|
||||
FROM resources r LEFT JOIN LATERAL(SELECT ` + column + ` FROM resource_samples WHERE resource_id=r.id AND sampled_at>now()-interval '90 seconds' ORDER BY sampled_at DESC LIMIT 1) s ON true WHERE r.provider='proxmox'`
|
||||
}
|
||||
if err = tx.SelectContext(ctx, &targets, query); err != nil {
|
||||
return err
|
||||
}
|
||||
ids := []string{}
|
||||
for _, target := range targets {
|
||||
ids = append(ids, target.ID)
|
||||
if target.Value == nil {
|
||||
if _, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1 AND target_id=$2 AND NOT EXISTS(SELECT 1 FROM alerts WHERE rule_id=$1 AND target_id=$2 AND status='firing')", rule.ID, target.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if *target.Value < rule.Threshold && !target.Unavailable {
|
||||
if _, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1 AND target_id=$2", rule.ID, target.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, "UPDATE alerts SET status='resolved',resolved_at=now(),resolve_reason='condition_cleared' WHERE rule_id=$1 AND target_id=$2 AND status='firing'", rule.ID, target.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, "INSERT INTO alert_states(rule_id,target_id) VALUES($1,$2) ON CONFLICT DO NOTHING", rule.ID, target.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
hold := rule.HoldSeconds
|
||||
if rule.Metric == "node_unavailable" && target.Unavailable {
|
||||
hold = int(math.Ceil(rule.Threshold))
|
||||
if err = tx.GetContext(ctx, target.Value, "SELECT extract(epoch FROM(now()-first_seen))::double precision FROM alert_states WHERE rule_id=$1 AND target_id=$2", rule.ID, target.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, `INSERT INTO alerts(id,rule_id,rule_name,metric,target_id,target_name,value,threshold,started_at)
|
||||
SELECT $1,$2,$3,$4,$5,$6,$7,$8,first_seen FROM alert_states WHERE rule_id=$2 AND target_id=$5 AND first_seen<=now()-make_interval(secs=>$9)
|
||||
ON CONFLICT(rule_id,target_id) WHERE status='firing' DO UPDATE SET value=EXCLUDED.value`, store.ID("alert_"), rule.ID, rule.Name, rule.Metric, target.ID, target.Name, *target.Value, rule.Threshold, hold); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, "UPDATE alerts SET status='resolved',resolved_at=now(),resolve_reason='target_unavailable' WHERE rule_id=$1 AND NOT(target_id=ANY($2::text[])) AND status='firing'", rule.ID, ids); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1 AND NOT(target_id=ANY($2::text[]))", rule.ID, ids); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Bounded retention. Current incidents and active session records are preserved.
|
||||
for _, query := range []string{"DELETE FROM login_limits WHERE window_started<now()-interval '1 day'", "DELETE FROM resource_samples WHERE sampled_at<now()-interval '30 days'", "DELETE FROM terminal_metric_samples WHERE sampled_at<now()-interval '30 days'", "DELETE FROM session_history WHERE ended_at<now()-interval '90 days'", "DELETE FROM alerts WHERE resolved_at<now()-interval '90 days'"} {
|
||||
if _, err = tx.ExecContext(ctx, query); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
func (s *Server) RunObservability(ctx context.Context) {
|
||||
ticker := time.NewTicker(15 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
readCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
if err := s.observe(readCtx); err != nil {
|
||||
log.Printf("observability evaluation failed: %v", err)
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user