267 lines
11 KiB
Go
267 lines
11 KiB
Go
package server
|
|
|
|
import (
|
|
"os"
|
|
"testing"
|
|
"vdi-cloud/platform/internal/store"
|
|
)
|
|
|
|
func TestTerminalAlertSchemaUpgradeIsRepeatable(t *testing.T) {
|
|
f, _, _ := terminalAlertFixture(t)
|
|
if _, err := f.db.Exec(`INSERT INTO alert_rules(id,name,metric,threshold,hold_seconds,enabled) VALUES('upgrade-old','原有停用规则','node_cpu',87,45,false);
|
|
INSERT INTO alerts(id,rule_id,rule_name,metric,target_id,target_name,value,threshold,started_at) VALUES('upgrade-event','upgrade-old','原有停用规则','node_cpu','old-node','原有节点',95,87,now());
|
|
INSERT INTO alert_states(rule_id,target_id) VALUES('upgrade-old','old-node')`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := f.db.Exec(`ALTER TABLE alert_rules DROP CONSTRAINT alert_rules_metric_check;
|
|
ALTER TABLE alert_rules ADD CONSTRAINT alert_rules_metric_check CHECK(metric IN ('node_cpu','node_memory','node_storage','node_unavailable','terminal_offline'));
|
|
ALTER TABLE alert_states DROP COLUMN last_sample_at;
|
|
ALTER TABLE alert_rules DROP COLUMN updated_at`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 2; i++ {
|
|
db, err := store.Open(t.Context(), os.Getenv("VDI_TEST_DATABASE_URL"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
db.Close()
|
|
}
|
|
var count int
|
|
if err := f.db.Get(&count, "SELECT count(*) FROM alert_rules WHERE id='upgrade-old' AND NOT enabled AND threshold=87 AND hold_seconds=45"); err != nil || count != 1 {
|
|
t.Fatal("old rules changed", count, err)
|
|
}
|
|
if err := f.db.Get(&count, "SELECT count(*) FROM alerts WHERE id='upgrade-event' AND status='firing' AND value=95"); err != nil || count != 1 {
|
|
t.Fatal("old event changed", count, err)
|
|
}
|
|
if err := f.db.Get(&count, "SELECT count(*) FROM alert_states WHERE rule_id='upgrade-old' AND target_id='old-node' AND last_sample_at IS NULL"); err != nil || count != 1 {
|
|
t.Fatal("old pending state changed", count, err)
|
|
}
|
|
if _, err := f.db.Exec("INSERT INTO alert_rules(id,name,metric,threshold) VALUES('upgrade-temp','升级温度规则','terminal_temperature',80)"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func terminalAlertFixture(t *testing.T) (*fixture, string, func(int, any, any, any)) {
|
|
t.Helper()
|
|
f := newFixtureWorker(t, false)
|
|
exec := func(query string, args ...any) {
|
|
t.Helper()
|
|
if _, err := f.db.Exec(query, args...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
exec("UPDATE alert_rules SET enabled=false")
|
|
exec(`INSERT INTO terminals(id,sn,name,bootstrap_hash,token_hash,last_seen,status)
|
|
VALUES('alert-box','ALERT-BOX','真实指标测试终端','',$1,now(),'online')`, store.Hash("alert-device"))
|
|
exec(`INSERT INTO terminal_metrics_current(terminal_id,sampled_at,supported) VALUES('alert-box',now(),true)`)
|
|
report := func(age int, cpu, memory, temp any) {
|
|
t.Helper()
|
|
exec("UPDATE terminals SET last_seen=now() WHERE id='alert-box'")
|
|
exec(`UPDATE terminal_metrics_current SET supported=true,sampled_at=now()-make_interval(secs=>$1),cpu_percent=$2,memory_percent=$3,soc_temperature_c=$4 WHERE terminal_id='alert-box'`, age, cpu, memory, temp)
|
|
exec(`INSERT INTO terminal_metric_samples(terminal_id,sampled_at,cpu_percent,memory_percent,soc_temperature_c)
|
|
SELECT terminal_id,sampled_at,cpu_percent,memory_percent,soc_temperature_c FROM terminal_metrics_current WHERE terminal_id='alert-box' ON CONFLICT DO NOTHING`)
|
|
}
|
|
return f, f.login("admin"), report
|
|
}
|
|
|
|
func TestTerminalMetricAlertsSourceTimeMissingAndRecovery(t *testing.T) {
|
|
f, admin, report := terminalAlertFixture(t)
|
|
evaluate := func() {
|
|
t.Helper()
|
|
if err := f.server.observe(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
count := func(query string) int {
|
|
t.Helper()
|
|
var n int
|
|
if err := f.db.Get(&n, query); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return n
|
|
}
|
|
for _, metric := range []string{"terminal_cpu", "terminal_memory", "terminal_temperature"} {
|
|
threshold := 90
|
|
if metric == "terminal_temperature" {
|
|
threshold = 80
|
|
}
|
|
code, out := f.request("POST", "/monitor/rules", admin, map[string]any{"name": metric, "metric": metric, "threshold": threshold, "holdSeconds": 60, "enabled": true}, "")
|
|
expect(t, 200, code, out)
|
|
}
|
|
// Simulate rules that were already enabled before the first source sample.
|
|
f.db.Exec("UPDATE alert_rules SET updated_at=now()-interval '70 seconds' WHERE enabled")
|
|
report(65, 95, 96, 100)
|
|
evaluate()
|
|
evaluate()
|
|
if count("SELECT count(*) FROM alerts") != 0 {
|
|
t.Fatal("repeated evaluation advanced source duration")
|
|
}
|
|
report(0, 96, 97, 101)
|
|
evaluate()
|
|
evaluate()
|
|
if count("SELECT count(*) FROM alerts WHERE status='firing'") != 3 {
|
|
t.Fatal("missing or duplicate terminal events")
|
|
}
|
|
viewer := f.login("viewer")
|
|
code, out := f.request("GET", "/monitor/alerts", viewer, nil, "")
|
|
expect(t, 200, code, out)
|
|
for _, item := range out["items"].([]any) {
|
|
if item.(map[string]any)["dataFresh"] != true {
|
|
t.Fatal(out)
|
|
}
|
|
}
|
|
// A missing sensor cannot resolve an open event or be represented as zero.
|
|
report(0, 96, nil, 101)
|
|
evaluate()
|
|
code, out = f.request("GET", "/monitor/alerts", viewer, nil, "")
|
|
expect(t, 200, code, out)
|
|
for _, item := range out["items"].([]any) {
|
|
row := item.(map[string]any)
|
|
if row["metric"] == "terminal_memory" && row["dataFresh"] != false {
|
|
t.Fatal(row)
|
|
}
|
|
}
|
|
if count("SELECT count(*) FROM alerts WHERE status='firing'") != 3 {
|
|
t.Fatal("null sensor claimed recovery")
|
|
}
|
|
f.db.Exec("UPDATE terminal_metrics_current SET sampled_at=now()-interval '91 seconds'")
|
|
evaluate()
|
|
if count("SELECT count(*) FROM alert_states WHERE target_id='alert-box'") != 0 {
|
|
t.Fatal("stale pending duration retained")
|
|
}
|
|
code, out = f.request("GET", "/monitor/alerts", viewer, nil, "")
|
|
expect(t, 200, code, out)
|
|
for _, item := range out["items"].([]any) {
|
|
if item.(map[string]any)["dataFresh"] != false {
|
|
t.Fatal(out)
|
|
}
|
|
}
|
|
report(0, 96, 97, 101)
|
|
evaluate()
|
|
if count("SELECT count(*) FROM alerts") != 3 {
|
|
t.Fatal("data resume duplicated an open event")
|
|
}
|
|
f.db.Exec("UPDATE terminal_metrics_current SET supported=false")
|
|
evaluate()
|
|
if count("SELECT count(*) FROM alerts WHERE status='firing'") != 3 {
|
|
t.Fatal("old agent claimed recovery")
|
|
}
|
|
report(0, 10, 30, 40)
|
|
evaluate()
|
|
if count("SELECT count(*) FROM alerts WHERE status='resolved' AND resolve_reason='condition_cleared'") != 3 {
|
|
t.Fatal("fresh normal metrics did not resolve")
|
|
}
|
|
// Zero-duration rules can trigger from the first actual high sample.
|
|
for _, item := range []string{"terminal_cpu", "terminal_memory", "terminal_temperature"} {
|
|
f.db.Exec("UPDATE alert_rules SET hold_seconds=0 WHERE metric=$1", item)
|
|
}
|
|
report(0, 95, 96, 100)
|
|
evaluate()
|
|
f.db.Exec("UPDATE terminals SET enabled=false WHERE id='alert-box'")
|
|
evaluate()
|
|
if count("SELECT count(*) FROM alerts WHERE status='resolved' AND resolve_reason='target_unavailable'") != 3 {
|
|
t.Fatal("revocation did not retire alerts")
|
|
}
|
|
}
|
|
|
|
func TestTerminalMetricAlertContinuityAcrossMissedEvaluations(t *testing.T) {
|
|
f, admin, report := terminalAlertFixture(t)
|
|
code, out := f.request("POST", "/monitor/rules", admin, map[string]any{"name": "连续 CPU", "metric": "terminal_cpu", "threshold": 90, "holdSeconds": 60, "enabled": true}, "")
|
|
expect(t, 200, code, out)
|
|
f.db.Exec("UPDATE alert_rules SET updated_at=now()-interval '80 seconds' WHERE enabled")
|
|
evaluate := func() {
|
|
t.Helper()
|
|
if err := f.server.observe(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
noAlert := func() {
|
|
t.Helper()
|
|
var n int
|
|
f.db.Get(&n, "SELECT count(*) FROM alerts")
|
|
if n != 0 {
|
|
t.Fatal("non-continuous samples triggered an alert")
|
|
}
|
|
}
|
|
report(75, 95, nil, nil)
|
|
evaluate()
|
|
if _, err := f.db.Exec(`INSERT INTO terminal_metric_samples(terminal_id,sampled_at,cpu_percent) VALUES('alert-box',now()-interval '40 seconds',20)`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
report(0, 95, nil, nil)
|
|
evaluate()
|
|
noAlert() // the observer missed a normal sample
|
|
f.db.Exec("UPDATE alert_states SET first_seen=now()-interval '120 seconds',last_sample_at=now()-interval '100 seconds'")
|
|
report(0, 95, nil, nil)
|
|
evaluate()
|
|
noAlert() // monitoring gap restarts the duration
|
|
report(0, nil, nil, nil)
|
|
evaluate()
|
|
var n int
|
|
f.db.Get(&n, "SELECT count(*) FROM alert_states WHERE target_id='alert-box'")
|
|
if n != 0 {
|
|
t.Fatal("null sample did not reset pending state")
|
|
}
|
|
report(0, 95, nil, nil)
|
|
evaluate()
|
|
noAlert()
|
|
// Missing history samples also interrupt a pending high segment.
|
|
f.db.Exec("UPDATE alert_states SET first_seen=now()-interval '80 seconds',last_sample_at=now()-interval '80 seconds'")
|
|
f.db.Exec("INSERT INTO terminal_metric_samples(terminal_id,sampled_at) VALUES('alert-box',now()-interval '30 seconds')")
|
|
report(0, 95, nil, nil)
|
|
evaluate()
|
|
noAlert()
|
|
}
|
|
|
|
func TestTerminalAlertRuleBoundsAndPermissions(t *testing.T) {
|
|
f, admin, _ := terminalAlertFixture(t)
|
|
for _, test := range []struct {
|
|
metric string
|
|
threshold float64
|
|
want int
|
|
}{{"terminal_temperature", 150, 200}, {"terminal_temperature", 150.1, 400}, {"terminal_temperature", 0, 400}, {"terminal_cpu", 101, 400}, {"terminal_memory", 100, 200}, {"unexpected", 20, 400}} {
|
|
code, out := f.request("POST", "/monitor/rules", admin, map[string]any{"name": "阈值", "metric": test.metric, "threshold": test.threshold, "holdSeconds": 0, "enabled": false}, "")
|
|
expect(t, test.want, code, out)
|
|
}
|
|
for _, role := range []string{"viewer", "operator"} {
|
|
code, out := f.request("POST", "/monitor/rules", f.login(role), map[string]any{"name": "无权修改", "metric": "terminal_temperature", "threshold": 80, "holdSeconds": 60, "enabled": true}, "")
|
|
expect(t, 403, code, out)
|
|
}
|
|
}
|
|
|
|
func TestNewTerminalRuleDoesNotCountPreEnableSamples(t *testing.T) {
|
|
f, admin, report := terminalAlertFixture(t)
|
|
body := map[string]any{"name": "新规则", "metric": "terminal_cpu", "threshold": 90, "holdSeconds": 60, "enabled": true}
|
|
code, out := f.request("POST", "/monitor/rules", admin, body, "")
|
|
expect(t, 200, code, out)
|
|
id := out["id"].(string)
|
|
report(65, 95, nil, nil)
|
|
if err := f.server.observe(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var clamped bool
|
|
if err := f.db.Get(&clamped, "SELECT s.first_seen>=r.updated_at AND s.first_seen>s.last_sample_at FROM alert_states s JOIN alert_rules r ON r.id=s.rule_id WHERE r.id=$1", id); err != nil || !clamped {
|
|
t.Fatal("pre-enable sample was counted", err)
|
|
}
|
|
report(0, 95, nil, nil)
|
|
if err := f.server.observe(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var count int
|
|
f.db.Get(&count, "SELECT count(*) FROM alerts WHERE rule_id=$1", id)
|
|
if count != 0 {
|
|
t.Fatal("rule fired before its own hold interval")
|
|
}
|
|
// A zero hold still triggers immediately from a valid current reading.
|
|
body["holdSeconds"] = 0
|
|
code, out = f.request("PUT", "/monitor/rules/"+id, admin, body, "")
|
|
expect(t, 200, code, out)
|
|
if err := f.server.observe(t.Context()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.db.Get(&count, "SELECT count(*) FROM alerts WHERE rule_id=$1 AND status='firing'", id)
|
|
if count != 1 {
|
|
t.Fatal("zero hold did not trigger")
|
|
}
|
|
}
|