功能:完善云管助手与GPU故障恢复核对
This commit is contained in:
1 parent
b6026f9759
commit
9f7b01e4db
42 files changed
+3576
-120
No files matched your search
@@ -2,6 +2,7 @@ package proxmox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -58,6 +59,70 @@ func TestTLSAndTaskLifecycle(t *testing.T) {
|
||||
t.Fatal("restore must never map to hard reset")
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservationRetriesDoNotReplayPowerCommands(t *testing.T) {
|
||||
for _, endpoint := range []string{"task", "power"} {
|
||||
t.Run(endpoint, func(t *testing.T) {
|
||||
reads, writes := 0, 0
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == "POST" {
|
||||
writes++
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
reads++
|
||||
if reads < 3 {
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
fmtBody := `{"data":{"status":"running"}}`
|
||||
if endpoint == "task" {
|
||||
fmtBody = `{"data":{"status":"stopped","exitstatus":"OK"}}`
|
||||
}
|
||||
w.Write([]byte(fmtBody))
|
||||
}))
|
||||
defer ts.Close()
|
||||
c := &Client{BaseURL: ts.URL, HTTP: ts.Client()}
|
||||
var err error
|
||||
if endpoint == "task" {
|
||||
err = c.Wait(context.Background(), "node", "UPID:one", time.Millisecond)
|
||||
} else {
|
||||
_, err = c.Power(context.Background(), "node", 100)
|
||||
}
|
||||
if err != nil || reads != 3 {
|
||||
t.Fatal("observation did not recover", reads, err)
|
||||
}
|
||||
if _, err = c.Action(context.Background(), "node", 100, "start"); err == nil || writes != 1 {
|
||||
t.Fatal("command response loss must not replay POST", writes, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservationRetriesAreBoundedAndRespectCancellation(t *testing.T) {
|
||||
for _, status := range []int{http.StatusForbidden, http.StatusServiceUnavailable} {
|
||||
calls := 0
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls++
|
||||
w.WriteHeader(status)
|
||||
}))
|
||||
c := &Client{BaseURL: ts.URL, HTTP: ts.Client()}
|
||||
_, err := c.Power(context.Background(), "node", 100)
|
||||
want := 1
|
||||
if status == http.StatusServiceUnavailable {
|
||||
want = 3
|
||||
}
|
||||
if err == nil || calls != want {
|
||||
t.Fatal("incorrect retry limit", calls, err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err = c.Power(ctx, "node", 100); !errors.Is(err, context.Canceled) || calls != want {
|
||||
t.Fatal("cancellation ignored", calls, err)
|
||||
}
|
||||
ts.Close()
|
||||
}
|
||||
}
|
||||
func TestFailedPVEJobIsNotSuccess(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(`{"data":{"status":"stopped","exitstatus":"ERROR: disk full"}}`))
|
||||
|
||||
Reference in new issue
Block a user