developer
8d71ef06c6
Adds internal/laya, a client and safety-gated decision engine for the Laya decision API (https://laya.k6n.net). The API's OpenAPI document leaves every response schema empty, so the wire format was pinned by probing the live service: questions are typed (choice/score/noul) and carry "criteria" as an object for a choice and an array for a score. An alert is rendered into a compact factual state and asked four typed questions: which action (restart/ignore/notify/escalate), severity and how urgent. The state always includes the pods' last events - restart count, readiness, last termination reason and exit code, log tail, and any overlapping rollout - because a 503 burst from a pod being replaced is otherwise indistinguishable from an outage. The model proposes, the policy disposes. Every gate records why it changed an answer: an executable-action allowlist, a probability and lead-over-runner-up threshold, no restart while pods are crash-looping (the kubelet is already doing it) or while a rollout is replacing every pod (the 503s are the rollout's own), never ignoring a new error or a high-priority spike without a human's mute, an hourly restart budget per deployment, and a plain notification whenever the model is unreachable. The margin, not the raw probability, is the useful signal: label probabilities are diffuse and a correct answer rarely exceeds 0.5. topology.PodLog now carries the pod's lifecycle facts alongside the log tail. PodsNotReady/CrashLooping helpers build on them. Verified end to end against the live API: a crash loop is withheld as "already crash-looping", a rollout's 503 burst is held back at a 2% margin, and crawler 404s are ignored at 13%. LAYA_URL=... go test ./internal/laya -run TestLiveModel replays it.
345 lines
14 KiB
Go
345 lines
14 KiB
Go
package laya
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// fake serves the Laya API with a fixed answer per action label, and records
|
|
// the last state and questions it was asked.
|
|
type fake struct {
|
|
choice string
|
|
prob map[string]float64
|
|
lastState string
|
|
lastQs map[string]json.RawMessage
|
|
status int
|
|
batchSizes int
|
|
}
|
|
|
|
func (f *fake) server(t *testing.T) *httptest.Server {
|
|
t.Helper()
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
|
|
writeJSON(w, Health{Status: "ok", Model: "test-model"})
|
|
})
|
|
mux.HandleFunc("/v1/predict", func(w http.ResponseWriter, r *http.Request) {
|
|
var req struct {
|
|
State json.RawMessage `json:"state"`
|
|
Questions map[string]json.RawMessage `json:"questions"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
t.Errorf("decode request: %v", err)
|
|
}
|
|
f.lastState = string(req.State)
|
|
f.lastQs = req.Questions
|
|
f.respond(w, map[string]Answer{
|
|
"action": {Type: KindChoice, Choice: f.choice, Probabilities: f.prob},
|
|
"severity": {Type: KindChoice, Choice: "high", Probabilities: map[string]float64{"high": 0.8}},
|
|
"urgency": {Type: KindScore, Score: 3},
|
|
})
|
|
})
|
|
mux.HandleFunc("/v1/predict/batch", func(w http.ResponseWriter, r *http.Request) {
|
|
var req struct {
|
|
States []json.RawMessage `json:"states"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
t.Errorf("decode batch request: %v", err)
|
|
}
|
|
f.batchSizes = len(req.States)
|
|
results := make([]Response, 0, len(req.States))
|
|
for range req.States {
|
|
results = append(results, Response{Model: "test-model", Answers: map[string]Answer{
|
|
"action": {Type: KindChoice, Choice: f.choice, Probabilities: f.prob},
|
|
}})
|
|
}
|
|
writeJSON(w, map[string]any{"results": results})
|
|
})
|
|
mux.HandleFunc("/boom", func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusBadGateway)
|
|
_, _ = w.Write([]byte("upstream exploded"))
|
|
})
|
|
srv := httptest.NewServer(mux)
|
|
t.Cleanup(srv.Close)
|
|
return srv
|
|
}
|
|
|
|
func (f *fake) respond(w http.ResponseWriter, answers map[string]Answer) {
|
|
if f.status != 0 {
|
|
w.WriteHeader(f.status)
|
|
return
|
|
}
|
|
writeJSON(w, Response{Model: "test-model", Answers: answers, Usage: Usage{InputTokens: 128}})
|
|
}
|
|
|
|
func writeJSON(w http.ResponseWriter, v any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_ = json.NewEncoder(w).Encode(v)
|
|
}
|
|
|
|
func TestPredictSendsTypedQuestionsAndReadsAnswers(t *testing.T) {
|
|
f := &fake{choice: "restart", prob: map[string]float64{"restart": 0.9, "notify": 0.1}}
|
|
srv := f.server(t)
|
|
c := NewClient(srv.URL)
|
|
|
|
resp, err := c.Predict(context.Background(), "checkout-api is 500ing", Questions())
|
|
if err != nil {
|
|
t.Fatalf("Predict: %v", err)
|
|
}
|
|
if got := resp.Answers["action"].Choice; got != "restart" {
|
|
t.Errorf("choice = %q, want restart", got)
|
|
}
|
|
if got := resp.Answers["urgency"].Score; got != 3 {
|
|
t.Errorf("urgency score = %v, want 3", got)
|
|
}
|
|
if got := resp.Usage.InputTokens; got != 128 {
|
|
t.Errorf("input tokens = %d, want 128", got)
|
|
}
|
|
// The choice question must carry a criteria object and the score question an
|
|
// array, from the same "criteria" key.
|
|
if q := string(f.lastQs["action"]); !strings.Contains(q, `"criteria":{"escalate"`) {
|
|
t.Errorf("action question = %s, want a criteria object", q)
|
|
}
|
|
if q := string(f.lastQs["urgency"]); !strings.Contains(q, `"criteria":[`) {
|
|
t.Errorf("urgency question = %s, want a criteria array", q)
|
|
}
|
|
if f.lastState != `"checkout-api is 500ing"` {
|
|
t.Errorf("state = %s, want the JSON-encoded state", f.lastState)
|
|
}
|
|
}
|
|
|
|
func TestPredictBatchKeepsOrder(t *testing.T) {
|
|
srv := (&fake{choice: "notify", prob: map[string]float64{"notify": 1}}).server(t)
|
|
got, err := NewClient(srv.URL).PredictBatch(context.Background(), []any{"a", "b", "c"}, Questions())
|
|
if err != nil {
|
|
t.Fatalf("PredictBatch: %v", err)
|
|
}
|
|
if len(got) != 3 {
|
|
t.Fatalf("got %d results, want 3", len(got))
|
|
}
|
|
for i, r := range got {
|
|
if r.Answers["action"].Choice != "notify" {
|
|
t.Errorf("result %d choice = %q", i, r.Answers["action"].Choice)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPredictErrorsAreSafeToLog(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusServiceUnavailable)
|
|
}))
|
|
defer srv.Close()
|
|
_, err := NewClient(srv.URL).Predict(context.Background(), "x", Questions())
|
|
if err == nil {
|
|
t.Fatal("expected an error")
|
|
}
|
|
if !strings.Contains(err.Error(), "503") {
|
|
t.Errorf("error = %v, want the status code", err)
|
|
}
|
|
}
|
|
|
|
func TestScrubRemovesCredentialsFromURL(t *testing.T) {
|
|
got := scrub(`Post "https://user:hunter2@laya.k6n.net/v1/predict": timeout`)
|
|
if strings.Contains(got, "hunter2") {
|
|
t.Errorf("scrub(%q) leaked the password: %s", "user:hunter2@", got)
|
|
}
|
|
if !strings.Contains(got, "laya.k6n.net") {
|
|
t.Errorf("scrub(%q) dropped the host: %s", "user:hunter2@", got)
|
|
}
|
|
}
|
|
|
|
func TestHealth(t *testing.T) {
|
|
srv := (&fake{}).server(t)
|
|
h, err := NewClient(srv.URL).Health(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("Health: %v", err)
|
|
}
|
|
if h.Model != "test-model" || h.Status != "ok" {
|
|
t.Errorf("health = %+v, want status ok from test-model", h)
|
|
}
|
|
}
|
|
|
|
// --- state rendering and gates ---
|
|
|
|
func nowFixture() time.Time { return time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) }
|
|
|
|
func healthyPod() Pod {
|
|
return Pod{Namespace: "shop", Deployment: "checkout-api", Name: "checkout-api-7d9", Found: true, Ready: true, Phase: "Running"}
|
|
}
|
|
|
|
func engineFor(t *testing.T, policy Policy, choice string, prob float64) (*Engine, *fake) {
|
|
t.Helper()
|
|
f := &fake{choice: choice, prob: map[string]float64{choice: prob}}
|
|
srv := f.server(t)
|
|
policy.Now = nowFixture
|
|
return NewEngine(NewClient(srv.URL), policy), f
|
|
}
|
|
|
|
func TestRenderStatesPodEvents(t *testing.T) {
|
|
exit := int32(137)
|
|
in := Input{
|
|
AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 412, Window: 5 * time.Minute,
|
|
BaselineMean: 3, NewErrors: true, Clients: 42,
|
|
Patterns: []Pattern{{What: "HTTP 500 | /api/cart | panic", Category: "service-defect", Status: 500, Count: 300, Baseline: 2, New: true, Clients: 42}},
|
|
Pods: []Pod{{
|
|
Namespace: "shop", Deployment: "checkout-api", Name: "checkout-api-7d9", Found: true, Phase: "Running",
|
|
Restarts: 3, PodRestarts: 9, Reason: "OOMKilled", ExitCode: &exit, LastTerminatedAt: nowFixture().Add(-30 * time.Second),
|
|
LastEvents: []string{"panic: runtime error: invalid memory address"},
|
|
}},
|
|
Deploys: []Deploy{{Namespace: "shop", Workload: "checkout-api", Age: 4 * time.Minute, Ready: 1, Desired: 3}},
|
|
}
|
|
state := in.Render(nowFixture())
|
|
for _, want := range []string{
|
|
"shop/checkout-api pod checkout-api-7d9",
|
|
"OOMKilled",
|
|
"exit code 137",
|
|
"its container restarted 30s ago and is still coming up",
|
|
"9 restarts across the Deployment",
|
|
"panic: runtime error: invalid memory address",
|
|
"rollout in progress started 4m0s ago",
|
|
"137x normal",
|
|
} {
|
|
if !strings.Contains(state, want) {
|
|
t.Errorf("state is missing %q:\n%s", want, state)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDecideExecutesRestartOnlyWhenPodsSupportIt(t *testing.T) {
|
|
e, f := engineFor(t, Policy{Actions: []Action{ActionRestart, ActionIgnore, ActionNotify, ActionEscalate}, MinProbability: 0.5}, "restart", 0.82)
|
|
v := e.Decide(context.Background(), Input{
|
|
AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 100, Window: 5 * time.Minute, BaselineMean: 2, Clients: 30,
|
|
Patterns: []Pattern{{What: "HTTP 500 | /api/cart", Category: "service-defect", Count: 90, Baseline: 1, New: true, Clients: 30}},
|
|
Pods: []Pod{{Namespace: "shop", Deployment: "checkout-api", Name: "p1", Found: true, Ready: false, Phase: "Running"}},
|
|
})
|
|
if !v.Executable() || v.Action != ActionRestart {
|
|
t.Fatalf("action = %q executable=%v gates=%v, want an executed restart", v.Action, v.Executable(), v.Gates)
|
|
}
|
|
if len(v.Targets) != 1 || v.Targets[0] != "shop/checkout-api" {
|
|
t.Errorf("targets = %v, want shop/checkout-api", v.Targets)
|
|
}
|
|
if !strings.Contains(f.lastState, "pod p1") {
|
|
t.Errorf("model was not given the pod state:\n%s", f.lastState)
|
|
}
|
|
}
|
|
|
|
func TestDecideRefusesRestartDuringCrashLoop(t *testing.T) {
|
|
e, _ := engineFor(t, Policy{Actions: []Action{ActionRestart}, MinProbability: 0.5}, "restart", 0.95)
|
|
v := e.Decide(context.Background(), Input{
|
|
AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 300, Window: 5 * time.Minute, BaselineMean: 2,
|
|
Patterns: []Pattern{{What: "HTTP 500 | panic", Category: "service-defect", Count: 300, New: true}},
|
|
Pods: []Pod{{Namespace: "shop", Deployment: "checkout-api", Name: "p1", Found: true, Phase: "Running", Restarts: 5, LastTerminatedAt: nowFixture().Add(-20 * time.Second)}},
|
|
})
|
|
if v.Action != ActionNotify {
|
|
t.Errorf("action = %q, want notify: a crash loop needs a new release, not a restart", v.Action)
|
|
}
|
|
if len(v.Gates) == 0 || !strings.Contains(strings.Join(v.Gates, " "), "crash-looping") {
|
|
t.Errorf("gates = %v, want the crash-loop reason", v.Gates)
|
|
}
|
|
}
|
|
|
|
// A short 503 burst while every pod is being replaced is the rollout's own, so
|
|
// a restart must never be fired into it.
|
|
func TestDecideRefusesRestartDuringRollout(t *testing.T) {
|
|
e, _ := engineFor(t, Policy{Actions: []Action{ActionRestart, ActionIgnore}, MinProbability: 0.5}, "restart", 0.9)
|
|
v := e.Decide(context.Background(), Input{
|
|
AlertID: "a1", Service: "checkout-api", Priority: "medium", ErrorCount: 40, Window: 5 * time.Minute, BaselineMean: 2,
|
|
Patterns: []Pattern{{What: "HTTP 503 | /api/cart", Category: "capacity-saturation", Status: 503, Count: 40}},
|
|
Pods: []Pod{{Namespace: "shop", Deployment: "checkout-api", Name: "p1", Found: true, Phase: "Pending", LastTerminatedAt: nowFixture().Add(-40 * time.Second)}},
|
|
Deploys: []Deploy{{Namespace: "shop", Workload: "checkout-api", Age: 2 * time.Minute, Ready: 1, Desired: 3}},
|
|
})
|
|
if v.Action == ActionRestart {
|
|
t.Fatalf("action = restart during a rollout, want it withheld (gates=%v)", v.Gates)
|
|
}
|
|
if !strings.Contains(strings.Join(v.Gates, " "), "replaced by a rollout") {
|
|
t.Errorf("gates = %v, want the rollout reason", v.Gates)
|
|
}
|
|
}
|
|
|
|
func TestDecideRefusesIgnoreOfANewError(t *testing.T) {
|
|
e, _ := engineFor(t, Policy{Actions: []Action{ActionIgnore}, MinProbability: 0.5}, "ignore", 0.9)
|
|
v := e.Decide(context.Background(), Input{
|
|
AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 200, Window: 5 * time.Minute, BaselineMean: 2, NewErrors: true,
|
|
Patterns: []Pattern{{What: "HTTP 500 | /api/cart", Count: 200, New: true}},
|
|
Pods: []Pod{healthyPod()},
|
|
})
|
|
if v.Action != ActionNotify {
|
|
t.Errorf("action = %q, want notify: a new error was never confirmed harmless", v.Action)
|
|
}
|
|
}
|
|
|
|
func TestDecideAllowsIgnoreOfKnownNoise(t *testing.T) {
|
|
e, _ := engineFor(t, Policy{Actions: []Action{ActionIgnore}, MinProbability: 0.5}, "ignore", 0.88)
|
|
v := e.Decide(context.Background(), Input{
|
|
AlertID: "a1", Service: "blog", Priority: "low", ErrorCount: 1800, Window: 5 * time.Minute, BaselineMean: 20, KnownNoise: true,
|
|
Patterns: []Pattern{{What: "HTTP 404 | /old-post", Status: 404, Count: 1800}},
|
|
})
|
|
if v.Action != ActionIgnore || !v.Executable() {
|
|
t.Errorf("action = %q executable = %v, want an executable ignore", v.Action, v.Executable())
|
|
}
|
|
}
|
|
|
|
func TestDecideWithholdsActionsOutsideTheAllowlist(t *testing.T) {
|
|
e, _ := engineFor(t, Policy{Actions: []Action{ActionNotify}, MinProbability: 0.5}, "ignore", 0.99)
|
|
v := e.Decide(context.Background(), Input{
|
|
AlertID: "a1", Service: "blog", Priority: "low", ErrorCount: 100, Window: 5 * time.Minute, BaselineMean: 20,
|
|
Patterns: []Pattern{{What: "HTTP 404", Status: 404, Count: 100}},
|
|
Pods: []Pod{healthyPod()},
|
|
})
|
|
if v.Action != ActionNotify || v.Requested != ActionIgnore || !v.Changed() {
|
|
t.Errorf("action = %q requested = %q changed = %v, want ignore downgraded to notify", v.Action, v.Requested, v.Changed())
|
|
}
|
|
}
|
|
|
|
func TestDecideDowngradesALowConfidenceChoice(t *testing.T) {
|
|
e, _ := engineFor(t, Policy{Actions: []Action{ActionRestart, ActionIgnore}, MinProbability: 0.6}, "restart", 0.41)
|
|
v := e.Decide(context.Background(), Input{
|
|
AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 90, Window: 5 * time.Minute, BaselineMean: 2,
|
|
Patterns: []Pattern{{What: "HTTP 500", Count: 90, New: true}},
|
|
Pods: []Pod{{Namespace: "shop", Deployment: "checkout-api", Found: true, Ready: false, Phase: "Running"}},
|
|
})
|
|
if v.Action != ActionNotify {
|
|
t.Errorf("action = %q, want notify: 41%% is a guess", v.Action)
|
|
}
|
|
}
|
|
|
|
func TestRestartBudgetIsEnforced(t *testing.T) {
|
|
e, _ := engineFor(t, Policy{Actions: []Action{ActionRestart}, MinProbability: 0.5, MaxRestartsPerHour: 1}, "restart", 0.9)
|
|
in := Input{
|
|
AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 90, Window: 5 * time.Minute, BaselineMean: 2,
|
|
Patterns: []Pattern{{What: "HTTP 500", Count: 90, New: true}},
|
|
Pods: []Pod{{Namespace: "shop", Deployment: "checkout-api", Found: true, Ready: false, Phase: "Running"}},
|
|
}
|
|
if v := e.Decide(context.Background(), in); v.Action != ActionRestart {
|
|
t.Fatalf("first decision = %q, want restart", v.Action)
|
|
}
|
|
second := e.Decide(context.Background(), in)
|
|
if second.Action != ActionNotify {
|
|
t.Errorf("second decision = %q, want notify: the hourly restart budget is spent", second.Action)
|
|
}
|
|
}
|
|
|
|
func TestDecideDegradesToNotifyWhenTheModelIsDown(t *testing.T) {
|
|
f := &fake{status: http.StatusServiceUnavailable}
|
|
srv := f.server(t)
|
|
e := NewEngine(NewClient(srv.URL), Policy{Actions: []Action{ActionRestart}, Now: nowFixture})
|
|
v := e.Decide(context.Background(), Input{AlertID: "a1", Service: "checkout-api", ErrorCount: 90, Window: 5 * time.Minute})
|
|
if v.Action != ActionNotify || v.Executable() || v.Error == "" {
|
|
t.Errorf("action = %q executable = %v error = %q, want a plain notification", v.Action, v.Executable(), v.Error)
|
|
}
|
|
}
|
|
|
|
func TestDecideRejectsAnUnknownAction(t *testing.T) {
|
|
f := &fake{choice: "delete-the-cluster", prob: map[string]float64{"delete-the-cluster": 1}}
|
|
srv := f.server(t)
|
|
e := NewEngine(NewClient(srv.URL), Policy{Actions: []Action{ActionRestart}, Now: nowFixture})
|
|
v := e.Decide(context.Background(), Input{AlertID: "a1", Service: "x", ErrorCount: 1, Window: time.Minute})
|
|
if v.Error == "" || v.Executable() {
|
|
t.Errorf("error = %q executable = %v, want an unknown action to be rejected", v.Error, v.Executable())
|
|
}
|
|
}
|