package laya import ( "context" "encoding/json" "net/http" "net/http/httptest" "strings" "testing" "time" ) // fake serves the Laya API with a fixed answer per action label, and records // the last state and questions it was asked. type fake struct { choice string prob map[string]float64 lastState string lastQs map[string]json.RawMessage status int batchSizes int } func (f *fake) server(t *testing.T) *httptest.Server { t.Helper() mux := http.NewServeMux() mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { writeJSON(w, Health{Status: "ok", Model: "test-model"}) }) mux.HandleFunc("/v1/predict", func(w http.ResponseWriter, r *http.Request) { var req struct { State json.RawMessage `json:"state"` Questions map[string]json.RawMessage `json:"questions"` } if err := json.NewDecoder(r.Body).Decode(&req); err != nil { t.Errorf("decode request: %v", err) } f.lastState = string(req.State) f.lastQs = req.Questions f.respond(w, map[string]Answer{ "action": {Type: KindChoice, Choice: f.choice, Probabilities: f.prob}, "severity": {Type: KindChoice, Choice: "high", Probabilities: map[string]float64{"high": 0.8}}, "urgency": {Type: KindScore, Score: 3}, }) }) mux.HandleFunc("/v1/predict/batch", func(w http.ResponseWriter, r *http.Request) { var req struct { States []json.RawMessage `json:"states"` } if err := json.NewDecoder(r.Body).Decode(&req); err != nil { t.Errorf("decode batch request: %v", err) } f.batchSizes = len(req.States) results := make([]Response, 0, len(req.States)) for range req.States { results = append(results, Response{Model: "test-model", Answers: map[string]Answer{ "action": {Type: KindChoice, Choice: f.choice, Probabilities: f.prob}, }}) } writeJSON(w, map[string]any{"results": results}) }) mux.HandleFunc("/boom", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusBadGateway) _, _ = w.Write([]byte("upstream exploded")) }) srv := httptest.NewServer(mux) t.Cleanup(srv.Close) return srv } func (f *fake) respond(w http.ResponseWriter, answers map[string]Answer) { if f.status != 0 { w.WriteHeader(f.status) return } writeJSON(w, Response{Model: "test-model", Answers: answers, Usage: Usage{InputTokens: 128}}) } func writeJSON(w http.ResponseWriter, v any) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(v) } func TestPredictSendsTypedQuestionsAndReadsAnswers(t *testing.T) { f := &fake{choice: "restart", prob: map[string]float64{"restart": 0.9, "notify": 0.1}} srv := f.server(t) c := NewClient(srv.URL) resp, err := c.Predict(context.Background(), "checkout-api is 500ing", Questions()) if err != nil { t.Fatalf("Predict: %v", err) } if got := resp.Answers["action"].Choice; got != "restart" { t.Errorf("choice = %q, want restart", got) } if got := resp.Answers["urgency"].Score; got != 3 { t.Errorf("urgency score = %v, want 3", got) } if got := resp.Usage.InputTokens; got != 128 { t.Errorf("input tokens = %d, want 128", got) } // The choice question must carry a criteria object and the score question an // array, from the same "criteria" key. if q := string(f.lastQs["action"]); !strings.Contains(q, `"criteria":{"escalate"`) { t.Errorf("action question = %s, want a criteria object", q) } if q := string(f.lastQs["urgency"]); !strings.Contains(q, `"criteria":[`) { t.Errorf("urgency question = %s, want a criteria array", q) } if f.lastState != `"checkout-api is 500ing"` { t.Errorf("state = %s, want the JSON-encoded state", f.lastState) } } func TestPredictBatchKeepsOrder(t *testing.T) { srv := (&fake{choice: "notify", prob: map[string]float64{"notify": 1}}).server(t) got, err := NewClient(srv.URL).PredictBatch(context.Background(), []any{"a", "b", "c"}, Questions()) if err != nil { t.Fatalf("PredictBatch: %v", err) } if len(got) != 3 { t.Fatalf("got %d results, want 3", len(got)) } for i, r := range got { if r.Answers["action"].Choice != "notify" { t.Errorf("result %d choice = %q", i, r.Answers["action"].Choice) } } } func TestPredictErrorsAreSafeToLog(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusServiceUnavailable) })) defer srv.Close() _, err := NewClient(srv.URL).Predict(context.Background(), "x", Questions()) if err == nil { t.Fatal("expected an error") } if !strings.Contains(err.Error(), "503") { t.Errorf("error = %v, want the status code", err) } } func TestScrubRemovesCredentialsFromURL(t *testing.T) { got := scrub(`Post "https://user:hunter2@laya.k6n.net/v1/predict": timeout`) if strings.Contains(got, "hunter2") { t.Errorf("scrub(%q) leaked the password: %s", "user:hunter2@", got) } if !strings.Contains(got, "laya.k6n.net") { t.Errorf("scrub(%q) dropped the host: %s", "user:hunter2@", got) } } func TestHealth(t *testing.T) { srv := (&fake{}).server(t) h, err := NewClient(srv.URL).Health(context.Background()) if err != nil { t.Fatalf("Health: %v", err) } if h.Model != "test-model" || h.Status != "ok" { t.Errorf("health = %+v, want status ok from test-model", h) } } // --- state rendering and gates --- func nowFixture() time.Time { return time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) } func healthyPod() Pod { return Pod{Namespace: "shop", Deployment: "checkout-api", Name: "checkout-api-7d9", Found: true, Ready: true, Phase: "Running"} } func engineFor(t *testing.T, policy Policy, choice string, prob float64) (*Engine, *fake) { t.Helper() f := &fake{choice: choice, prob: map[string]float64{choice: prob}} srv := f.server(t) policy.Now = nowFixture return NewEngine(NewClient(srv.URL), policy), f } func TestRenderStatesPodEvents(t *testing.T) { exit := int32(137) in := Input{ AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 412, Window: 5 * time.Minute, BaselineMean: 3, NewErrors: true, Clients: 42, Patterns: []Pattern{{What: "HTTP 500 | /api/cart | panic", Category: "service-defect", Status: 500, Count: 300, Baseline: 2, New: true, Clients: 42}}, Pods: []Pod{{ Namespace: "shop", Deployment: "checkout-api", Name: "checkout-api-7d9", Found: true, Phase: "Running", Restarts: 3, PodRestarts: 9, Reason: "OOMKilled", ExitCode: &exit, LastTerminatedAt: nowFixture().Add(-30 * time.Second), LastEvents: []string{"panic: runtime error: invalid memory address"}, }}, Deploys: []Deploy{{Namespace: "shop", Workload: "checkout-api", Age: 4 * time.Minute, Ready: 1, Desired: 3}}, } state := in.Render(nowFixture()) for _, want := range []string{ "shop/checkout-api pod checkout-api-7d9", "OOMKilled", "exit code 137", "its container restarted 30s ago and is still coming up", "9 restarts across the Deployment", "panic: runtime error: invalid memory address", "rollout in progress started 4m0s ago", "137x normal", } { if !strings.Contains(state, want) { t.Errorf("state is missing %q:\n%s", want, state) } } } func TestDecideExecutesRestartOnlyWhenPodsSupportIt(t *testing.T) { e, f := engineFor(t, Policy{Actions: []Action{ActionRestart, ActionIgnore, ActionNotify, ActionEscalate}, MinProbability: 0.5}, "restart", 0.82) v := e.Decide(context.Background(), Input{ AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 100, Window: 5 * time.Minute, BaselineMean: 2, Clients: 30, Patterns: []Pattern{{What: "HTTP 500 | /api/cart", Category: "service-defect", Count: 90, Baseline: 1, New: true, Clients: 30}}, Pods: []Pod{{Namespace: "shop", Deployment: "checkout-api", Name: "p1", Found: true, Ready: false, Phase: "Running"}}, }) if !v.Executable() || v.Action != ActionRestart { t.Fatalf("action = %q executable=%v gates=%v, want an executed restart", v.Action, v.Executable(), v.Gates) } if len(v.Targets) != 1 || v.Targets[0] != "shop/checkout-api" { t.Errorf("targets = %v, want shop/checkout-api", v.Targets) } if !strings.Contains(f.lastState, "pod p1") { t.Errorf("model was not given the pod state:\n%s", f.lastState) } } func TestDecideRefusesRestartDuringCrashLoop(t *testing.T) { e, _ := engineFor(t, Policy{Actions: []Action{ActionRestart}, MinProbability: 0.5}, "restart", 0.95) v := e.Decide(context.Background(), Input{ AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 300, Window: 5 * time.Minute, BaselineMean: 2, Patterns: []Pattern{{What: "HTTP 500 | panic", Category: "service-defect", Count: 300, New: true}}, Pods: []Pod{{Namespace: "shop", Deployment: "checkout-api", Name: "p1", Found: true, Phase: "Running", Restarts: 5, LastTerminatedAt: nowFixture().Add(-20 * time.Second)}}, }) if v.Action != ActionNotify { t.Errorf("action = %q, want notify: a crash loop needs a new release, not a restart", v.Action) } if len(v.Gates) == 0 || !strings.Contains(strings.Join(v.Gates, " "), "crash-looping") { t.Errorf("gates = %v, want the crash-loop reason", v.Gates) } } // A short 503 burst while every pod is being replaced is the rollout's own, so // a restart must never be fired into it. func TestDecideRefusesRestartDuringRollout(t *testing.T) { e, _ := engineFor(t, Policy{Actions: []Action{ActionRestart, ActionIgnore}, MinProbability: 0.5}, "restart", 0.9) v := e.Decide(context.Background(), Input{ AlertID: "a1", Service: "checkout-api", Priority: "medium", ErrorCount: 40, Window: 5 * time.Minute, BaselineMean: 2, Patterns: []Pattern{{What: "HTTP 503 | /api/cart", Category: "capacity-saturation", Status: 503, Count: 40}}, Pods: []Pod{{Namespace: "shop", Deployment: "checkout-api", Name: "p1", Found: true, Phase: "Pending", LastTerminatedAt: nowFixture().Add(-40 * time.Second)}}, Deploys: []Deploy{{Namespace: "shop", Workload: "checkout-api", Age: 2 * time.Minute, Ready: 1, Desired: 3}}, }) if v.Action == ActionRestart { t.Fatalf("action = restart during a rollout, want it withheld (gates=%v)", v.Gates) } if !strings.Contains(strings.Join(v.Gates, " "), "replaced by a rollout") { t.Errorf("gates = %v, want the rollout reason", v.Gates) } } func TestDecideRefusesIgnoreOfANewError(t *testing.T) { e, _ := engineFor(t, Policy{Actions: []Action{ActionIgnore}, MinProbability: 0.5}, "ignore", 0.9) v := e.Decide(context.Background(), Input{ AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 200, Window: 5 * time.Minute, BaselineMean: 2, NewErrors: true, Patterns: []Pattern{{What: "HTTP 500 | /api/cart", Count: 200, New: true}}, Pods: []Pod{healthyPod()}, }) if v.Action != ActionNotify { t.Errorf("action = %q, want notify: a new error was never confirmed harmless", v.Action) } } func TestDecideAllowsIgnoreOfKnownNoise(t *testing.T) { e, _ := engineFor(t, Policy{Actions: []Action{ActionIgnore}, MinProbability: 0.5}, "ignore", 0.88) v := e.Decide(context.Background(), Input{ AlertID: "a1", Service: "blog", Priority: "low", ErrorCount: 1800, Window: 5 * time.Minute, BaselineMean: 20, KnownNoise: true, Patterns: []Pattern{{What: "HTTP 404 | /old-post", Status: 404, Count: 1800}}, }) if v.Action != ActionIgnore || !v.Executable() { t.Errorf("action = %q executable = %v, want an executable ignore", v.Action, v.Executable()) } } func TestDecideWithholdsActionsOutsideTheAllowlist(t *testing.T) { e, _ := engineFor(t, Policy{Actions: []Action{ActionNotify}, MinProbability: 0.5}, "ignore", 0.99) v := e.Decide(context.Background(), Input{ AlertID: "a1", Service: "blog", Priority: "low", ErrorCount: 100, Window: 5 * time.Minute, BaselineMean: 20, Patterns: []Pattern{{What: "HTTP 404", Status: 404, Count: 100}}, Pods: []Pod{healthyPod()}, }) if v.Action != ActionNotify || v.Requested != ActionIgnore || !v.Changed() { t.Errorf("action = %q requested = %q changed = %v, want ignore downgraded to notify", v.Action, v.Requested, v.Changed()) } } func TestDecideDowngradesALowConfidenceChoice(t *testing.T) { e, _ := engineFor(t, Policy{Actions: []Action{ActionRestart, ActionIgnore}, MinProbability: 0.6}, "restart", 0.41) v := e.Decide(context.Background(), Input{ AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 90, Window: 5 * time.Minute, BaselineMean: 2, Patterns: []Pattern{{What: "HTTP 500", Count: 90, New: true}}, Pods: []Pod{{Namespace: "shop", Deployment: "checkout-api", Found: true, Ready: false, Phase: "Running"}}, }) if v.Action != ActionNotify { t.Errorf("action = %q, want notify: 41%% is a guess", v.Action) } } func TestRestartBudgetIsEnforced(t *testing.T) { e, _ := engineFor(t, Policy{Actions: []Action{ActionRestart}, MinProbability: 0.5, MaxRestartsPerHour: 1}, "restart", 0.9) in := Input{ AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 90, Window: 5 * time.Minute, BaselineMean: 2, Patterns: []Pattern{{What: "HTTP 500", Count: 90, New: true}}, Pods: []Pod{{Namespace: "shop", Deployment: "checkout-api", Found: true, Ready: false, Phase: "Running"}}, } if v := e.Decide(context.Background(), in); v.Action != ActionRestart { t.Fatalf("first decision = %q, want restart", v.Action) } second := e.Decide(context.Background(), in) if second.Action != ActionNotify { t.Errorf("second decision = %q, want notify: the hourly restart budget is spent", second.Action) } } func TestDecideDegradesToNotifyWhenTheModelIsDown(t *testing.T) { f := &fake{status: http.StatusServiceUnavailable} srv := f.server(t) e := NewEngine(NewClient(srv.URL), Policy{Actions: []Action{ActionRestart}, Now: nowFixture}) v := e.Decide(context.Background(), Input{AlertID: "a1", Service: "checkout-api", ErrorCount: 90, Window: 5 * time.Minute}) if v.Action != ActionNotify || v.Executable() || v.Error == "" { t.Errorf("action = %q executable = %v error = %q, want a plain notification", v.Action, v.Executable(), v.Error) } } func TestDecideRejectsAnUnknownAction(t *testing.T) { f := &fake{choice: "delete-the-cluster", prob: map[string]float64{"delete-the-cluster": 1}} srv := f.server(t) e := NewEngine(NewClient(srv.URL), Policy{Actions: []Action{ActionRestart}, Now: nowFixture}) v := e.Decide(context.Background(), Input{AlertID: "a1", Service: "x", ErrorCount: 1, Window: time.Minute}) if v.Error == "" || v.Executable() { t.Errorf("error = %q executable = %v, want an unknown action to be rejected", v.Error, v.Executable()) } }