功能：完善通用云管工作台、审计与诊断能力
This commit is contained in:
songyiming committed
1 parent
5ac692724d
commit
9f05aa20ea
225 files changed
+14266
-141
No files matched your search
@@ -13,12 +13,13 @@ import (
|
||||
)
|
||||
|
||||
type alertRule struct {
|
||||
ID string `json:"id" db:"id"`
|
||||
Name string `json:"name" db:"name" binding:"required,max=80"`
|
||||
Metric string `json:"metric" db:"metric" binding:"required,oneof=node_cpu node_memory node_storage node_unavailable terminal_offline"`
|
||||
Threshold float64 `json:"threshold" db:"threshold" binding:"required"`
|
||||
HoldSeconds int `json:"holdSeconds" db:"hold_seconds" binding:"min=0,max=3600"`
|
||||
Enabled bool `json:"enabled" db:"enabled"`
|
||||
ID string `json:"id" db:"id"`
|
||||
Name string `json:"name" db:"name" binding:"required,max=80"`
|
||||
Metric string `json:"metric" db:"metric" binding:"required,oneof=node_cpu node_memory node_storage node_unavailable terminal_offline terminal_cpu terminal_memory terminal_temperature"`
|
||||
Threshold float64 `json:"threshold" db:"threshold" binding:"required"`
|
||||
HoldSeconds int `json:"holdSeconds" db:"hold_seconds" binding:"min=0,max=3600"`
|
||||
Enabled bool `json:"enabled" db:"enabled"`
|
||||
UpdatedAt time.Time `json:"-" db:"updated_at"`
|
||||
}
|
||||
type alertRecord struct {
|
||||
ID string `json:"id" db:"id"`
|
||||
@@ -36,6 +37,7 @@ type alertRecord struct {
|
||||
ResolveReason string `json:"resolveReason" db:"resolve_reason"`
|
||||
AcknowledgedAt *time.Time `json:"acknowledgedAt" db:"acknowledged_at"`
|
||||
AcknowledgedBy string `json:"acknowledgedBy" db:"acknowledged_by"`
|
||||
DataFresh *bool `json:"dataFresh" db:"data_fresh"`
|
||||
}
|
||||
type resourcePoint struct {
|
||||
Time time.Time `json:"time" db:"time"`
|
||||
@@ -92,8 +94,12 @@ func (s *Server) saveAlertRule(c *gin.Context) {
|
||||
}
|
||||
in.Name = strings.TrimSpace(in.Name)
|
||||
offline := in.Metric == "terminal_offline" || in.Metric == "node_unavailable"
|
||||
if in.Name == "" || math.IsNaN(in.Threshold) || math.IsInf(in.Threshold, 0) || (!offline && (in.Threshold < 1 || in.Threshold > 100)) || (offline && (in.Threshold < 90 || in.Threshold > 86400 || in.HoldSeconds != 0)) {
|
||||
s.fail(c, 400, "invalid_request", "水位阈值为 1～100%；离线阈值为 90～86400 秒且不额外设置持续时间")
|
||||
maximum := 100.0
|
||||
if in.Metric == "terminal_temperature" {
|
||||
maximum = 150
|
||||
}
|
||||
if in.Name == "" || math.IsNaN(in.Threshold) || math.IsInf(in.Threshold, 0) || (!offline && (in.Threshold < 1 || in.Threshold > maximum)) || (offline && (in.Threshold < 90 || in.Threshold > 86400 || in.HoldSeconds != 0)) {
|
||||
s.fail(c, 400, "invalid_request", "使用率阈值为 1～100%，SoC 温度为 1～150°C；离线阈值为 90～86400 秒且不额外设置持续时间")
|
||||
return
|
||||
}
|
||||
ctx := c.Request.Context()
|
||||
@@ -113,7 +119,7 @@ func (s *Server) saveAlertRule(c *gin.Context) {
|
||||
_, err = tx.ExecContext(ctx, "INSERT INTO alert_rules(id,name,metric,threshold,hold_seconds,enabled) VALUES($1,$2,$3,$4,$5,$6)", in.ID, in.Name, in.Metric, in.Threshold, in.HoldSeconds, in.Enabled)
|
||||
} else {
|
||||
var id string
|
||||
err = tx.GetContext(ctx, &id, "UPDATE alert_rules SET name=$1,metric=$2,threshold=$3,hold_seconds=$4,enabled=$5 WHERE id=$6 RETURNING id", in.Name, in.Metric, in.Threshold, in.HoldSeconds, in.Enabled, in.ID)
|
||||
err = tx.GetContext(ctx, &id, "UPDATE alert_rules SET name=$1,metric=$2,threshold=$3,hold_seconds=$4,enabled=$5,updated_at=now() WHERE id=$6 RETURNING id", in.Name, in.Metric, in.Threshold, in.HoldSeconds, in.Enabled, in.ID)
|
||||
if err == nil {
|
||||
_, err = tx.ExecContext(ctx, "UPDATE alerts SET status='resolved',resolved_at=now(),resolve_reason='rule_changed' WHERE rule_id=$1 AND status='firing'", in.ID)
|
||||
}
|
||||
@@ -153,7 +159,7 @@ func (s *Server) alerts(c *gin.Context) {
|
||||
}
|
||||
items := []alertRecord{}
|
||||
args = append(args, size, (page-1)*size)
|
||||
if err := s.DB.SelectContext(c.Request.Context(), &items, "SELECT *"+filter+" ORDER BY opened_at DESC,id LIMIT $3 OFFSET $4", args...); err != nil {
|
||||
if err := s.DB.SelectContext(c.Request.Context(), &items, "SELECT *,"+terminalAlertFreshSQL+" AS data_fresh"+filter+" ORDER BY opened_at DESC,id LIMIT $3 OFFSET $4", args...); err != nil {
|
||||
s.dbError(c, err)
|
||||
return
|
||||
}
|
||||
@@ -193,10 +199,11 @@ func (s *Server) acknowledgeAlert(c *gin.Context) {
|
||||
}
|
||||
|
||||
type observation struct {
|
||||
ID string `db:"id"`
|
||||
Name string `db:"name"`
|
||||
Value *float64 `db:"value"`
|
||||
Unavailable bool `db:"unavailable"`
|
||||
ID string `db:"id"`
|
||||
Name string `db:"name"`
|
||||
Value *float64 `db:"value"`
|
||||
Unavailable bool `db:"unavailable"`
|
||||
SampledAt *time.Time `db:"sampled_at"`
|
||||
}
|
||||
|
||||
// One transaction serializes rule edits, expiry and condition evaluation across replicas.
|
||||
@@ -229,6 +236,10 @@ func (s *Server) observe(ctx context.Context) error {
|
||||
query = `SELECT r.id,r.name,CASE WHEN r.status='online' AND r.updated_at>now()-interval '90 seconds' THEN s.` + column + `*100 ELSE NULL END value,false unavailable
|
||||
FROM resources r LEFT JOIN LATERAL(SELECT ` + column + ` FROM resource_samples WHERE resource_id=r.id AND sampled_at>now()-interval '90 seconds' ORDER BY sampled_at DESC LIMIT 1) s ON true WHERE r.provider='proxmox'`
|
||||
}
|
||||
terminalColumn := terminalAlertColumn(rule.Metric)
|
||||
if terminalColumn != "" {
|
||||
query = terminalAlertQuery(terminalColumn)
|
||||
}
|
||||
if err = tx.SelectContext(ctx, &targets, query); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -236,6 +247,14 @@ func (s *Server) observe(ctx context.Context) error {
|
||||
for _, target := range targets {
|
||||
ids = append(ids, target.ID)
|
||||
if target.Value == nil {
|
||||
if terminalColumn != "" {
|
||||
// Unknown data interrupts the pending duration, while an open
|
||||
// incident remains open until a fresh normal sample arrives.
|
||||
if _, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1 AND target_id=$2", rule.ID, target.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, "DELETE FROM alert_states WHERE rule_id=$1 AND target_id=$2 AND NOT EXISTS(SELECT 1 FROM alerts WHERE rule_id=$1 AND target_id=$2 AND status='firing')", rule.ID, target.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -250,6 +269,12 @@ func (s *Server) observe(ctx context.Context) error {
|
||||
}
|
||||
continue
|
||||
}
|
||||
if terminalColumn != "" {
|
||||
if err = evaluateTerminalHigh(ctx, tx, rule, target, terminalColumn); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, err = tx.ExecContext(ctx, "INSERT INTO alert_states(rule_id,target_id) VALUES($1,$2) ON CONFLICT DO NOTHING", rule.ID, target.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user