go-otel/internal/laya/laya_test.go

package laya

import (
	"context"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"
)

// fake serves the Laya API with a fixed answer per action label, and records
// the last state and questions it was asked.
type fake struct {
	choice     string
	prob       map[string]float64
	lastState  string
	lastQs     map[string]json.RawMessage
	status     int
	batchSizes int
}

func (f *fake) server(t *testing.T) *httptest.Server {
	t.Helper()
	mux := http.NewServeMux()
	mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
		writeJSON(w, Health{Status: "ok", Model: "test-model"})
	})
	mux.HandleFunc("/v1/predict", func(w http.ResponseWriter, r *http.Request) {
		var req struct {
			State     json.RawMessage            `json:"state"`
			Questions map[string]json.RawMessage `json:"questions"`
		}
		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
			t.Errorf("decode request: %v", err)
		}
		f.lastState = string(req.State)
		f.lastQs = req.Questions
		f.respond(w, map[string]Answer{
			"action":   {Type: KindChoice, Choice: f.choice, Probabilities: f.prob},
			"severity": {Type: KindChoice, Choice: "high", Probabilities: map[string]float64{"high": 0.8}},
			"urgency":  {Type: KindScore, Score: 3},
		})
	})
	mux.HandleFunc("/v1/predict/batch", func(w http.ResponseWriter, r *http.Request) {
		var req struct {
			States []json.RawMessage `json:"states"`
		}
		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
			t.Errorf("decode batch request: %v", err)
		}
		f.batchSizes = len(req.States)
		results := make([]Response, 0, len(req.States))
		for range req.States {
			results = append(results, Response{Model: "test-model", Answers: map[string]Answer{
				"action": {Type: KindChoice, Choice: f.choice, Probabilities: f.prob},
			}})
		}
		writeJSON(w, map[string]any{"results": results})
	})
	mux.HandleFunc("/boom", func(w http.ResponseWriter, _ *http.Request) {
		w.WriteHeader(http.StatusBadGateway)
		_, _ = w.Write([]byte("upstream exploded"))
	})
	srv := httptest.NewServer(mux)
	t.Cleanup(srv.Close)
	return srv
}

func (f *fake) respond(w http.ResponseWriter, answers map[string]Answer) {
	if f.status != 0 {
		w.WriteHeader(f.status)
		return
	}
	writeJSON(w, Response{Model: "test-model", Answers: answers, Usage: Usage{InputTokens: 128}})
}

func writeJSON(w http.ResponseWriter, v any) {
	w.Header().Set("Content-Type", "application/json")
	_ = json.NewEncoder(w).Encode(v)
}

func TestPredictSendsTypedQuestionsAndReadsAnswers(t *testing.T) {
	f := &fake{choice: "restart", prob: map[string]float64{"restart": 0.9, "notify": 0.1}}
	srv := f.server(t)
	c := NewClient(srv.URL)

	resp, err := c.Predict(context.Background(), "checkout-api is 500ing", Questions())
	if err != nil {
		t.Fatalf("Predict: %v", err)
	}
	if got := resp.Answers["action"].Choice; got != "restart" {
		t.Errorf("choice = %q, want restart", got)
	}
	if got := resp.Answers["urgency"].Score; got != 3 {
		t.Errorf("urgency score = %v, want 3", got)
	}
	if got := resp.Usage.InputTokens; got != 128 {
		t.Errorf("input tokens = %d, want 128", got)
	}
	// The choice question must carry a criteria object and the score question an
	// array, from the same "criteria" key.
	if q := string(f.lastQs["action"]); !strings.Contains(q, `"criteria":{"escalate"`) {
		t.Errorf("action question = %s, want a criteria object", q)
	}
	if q := string(f.lastQs["urgency"]); !strings.Contains(q, `"criteria":[`) {
		t.Errorf("urgency question = %s, want a criteria array", q)
	}
	if f.lastState != `"checkout-api is 500ing"` {
		t.Errorf("state = %s, want the JSON-encoded state", f.lastState)
	}
}

func TestPredictBatchKeepsOrder(t *testing.T) {
	srv := (&fake{choice: "notify", prob: map[string]float64{"notify": 1}}).server(t)
	got, err := NewClient(srv.URL).PredictBatch(context.Background(), []any{"a", "b", "c"}, Questions())
	if err != nil {
		t.Fatalf("PredictBatch: %v", err)
	}
	if len(got) != 3 {
		t.Fatalf("got %d results, want 3", len(got))
	}
	for i, r := range got {
		if r.Answers["action"].Choice != "notify" {
			t.Errorf("result %d choice = %q", i, r.Answers["action"].Choice)
		}
	}
}

func TestPredictErrorsAreSafeToLog(t *testing.T) {
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		w.WriteHeader(http.StatusServiceUnavailable)
	}))
	defer srv.Close()
	_, err := NewClient(srv.URL).Predict(context.Background(), "x", Questions())
	if err == nil {
		t.Fatal("expected an error")
	}
	if !strings.Contains(err.Error(), "503") {
		t.Errorf("error = %v, want the status code", err)
	}
}

func TestScrubRemovesCredentialsFromURL(t *testing.T) {
	got := scrub(`Post "https://user:hunter2@laya.k6n.net/v1/predict": timeout`)
	if strings.Contains(got, "hunter2") {
		t.Errorf("scrub(%q) leaked the password: %s", "user:hunter2@", got)
	}
	if !strings.Contains(got, "laya.k6n.net") {
		t.Errorf("scrub(%q) dropped the host: %s", "user:hunter2@", got)
	}
}

func TestHealth(t *testing.T) {
	srv := (&fake{}).server(t)
	h, err := NewClient(srv.URL).Health(context.Background())
	if err != nil {
		t.Fatalf("Health: %v", err)
	}
	if h.Model != "test-model" || h.Status != "ok" {
		t.Errorf("health = %+v, want status ok from test-model", h)
	}
}

// --- state rendering and gates ---

func nowFixture() time.Time { return time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) }

func healthyPod() Pod {
	return Pod{Namespace: "shop", Deployment: "checkout-api", Name: "checkout-api-7d9", Found: true, Ready: true, Phase: "Running"}
}

func engineFor(t *testing.T, policy Policy, choice string, prob float64) (*Engine, *fake) {
	t.Helper()
	f := &fake{choice: choice, prob: map[string]float64{choice: prob}}
	srv := f.server(t)
	policy.Now = nowFixture
	return NewEngine(NewClient(srv.URL), policy), f
}

func TestRenderStatesPodEvents(t *testing.T) {
	exit := int32(137)
	in := Input{
		AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 412, Window: 5 * time.Minute,
		BaselineMean: 3, NewErrors: true, Clients: 42,
		Patterns: []Pattern{{What: "HTTP 500 | /api/cart | panic", Category: "service-defect", Status: 500, Count: 300, Baseline: 2, New: true, Clients: 42}},
		Pods: []Pod{{
			Namespace: "shop", Deployment: "checkout-api", Name: "checkout-api-7d9", Found: true, Phase: "Running",
			Restarts: 3, PodRestarts: 9, Reason: "OOMKilled", ExitCode: &exit, LastTerminatedAt: nowFixture().Add(-30 * time.Second),
			LastEvents: []string{"panic: runtime error: invalid memory address"},
		}},
		Deploys: []Deploy{{Namespace: "shop", Workload: "checkout-api", Age: 4 * time.Minute, Ready: 1, Desired: 3}},
	}
	state := in.Render(nowFixture())
	for _, want := range []string{
		"shop/checkout-api pod checkout-api-7d9",
		"OOMKilled",
		"exit code 137",
		"its container restarted 30s ago and is still coming up",
		"9 restarts across the Deployment",
		"panic: runtime error: invalid memory address",
		"rollout in progress started 4m0s ago",
		"137x normal",
	} {
		if !strings.Contains(state, want) {
			t.Errorf("state is missing %q:\n%s", want, state)
		}
	}
}

func TestDecideExecutesRestartOnlyWhenPodsSupportIt(t *testing.T) {
	e, f := engineFor(t, Policy{Actions: []Action{ActionRestart, ActionIgnore, ActionNotify, ActionEscalate}, MinProbability: 0.5}, "restart", 0.82)
	v := e.Decide(context.Background(), Input{
		AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 100, Window: 5 * time.Minute, BaselineMean: 2, Clients: 30,
		Patterns: []Pattern{{What: "HTTP 500 | /api/cart", Category: "service-defect", Count: 90, Baseline: 1, New: true, Clients: 30}},
		Pods:     []Pod{{Namespace: "shop", Deployment: "checkout-api", Name: "p1", Found: true, Ready: false, Phase: "Running"}},
	})
	if !v.Executable() || v.Action != ActionRestart {
		t.Fatalf("action = %q executable=%v gates=%v, want an executed restart", v.Action, v.Executable(), v.Gates)
	}
	if len(v.Targets) != 1 || v.Targets[0] != "shop/checkout-api" {
		t.Errorf("targets = %v, want shop/checkout-api", v.Targets)
	}
	if !strings.Contains(f.lastState, "pod p1") {
		t.Errorf("model was not given the pod state:\n%s", f.lastState)
	}
}

func TestDecideRefusesRestartDuringCrashLoop(t *testing.T) {
	e, _ := engineFor(t, Policy{Actions: []Action{ActionRestart}, MinProbability: 0.5}, "restart", 0.95)
	v := e.Decide(context.Background(), Input{
		AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 300, Window: 5 * time.Minute, BaselineMean: 2,
		Patterns: []Pattern{{What: "HTTP 500 | panic", Category: "service-defect", Count: 300, New: true}},
		Pods:     []Pod{{Namespace: "shop", Deployment: "checkout-api", Name: "p1", Found: true, Phase: "Running", Restarts: 5, LastTerminatedAt: nowFixture().Add(-20 * time.Second)}},
	})
	if v.Action != ActionNotify {
		t.Errorf("action = %q, want notify: a crash loop needs a new release, not a restart", v.Action)
	}
	if len(v.Gates) == 0 || !strings.Contains(strings.Join(v.Gates, " "), "crash-looping") {
		t.Errorf("gates = %v, want the crash-loop reason", v.Gates)
	}
}

// A short 503 burst while every pod is being replaced is the rollout's own, so
// a restart must never be fired into it.
func TestDecideRefusesRestartDuringRollout(t *testing.T) {
	e, _ := engineFor(t, Policy{Actions: []Action{ActionRestart, ActionIgnore}, MinProbability: 0.5}, "restart", 0.9)
	v := e.Decide(context.Background(), Input{
		AlertID: "a1", Service: "checkout-api", Priority: "medium", ErrorCount: 40, Window: 5 * time.Minute, BaselineMean: 2,
		Patterns: []Pattern{{What: "HTTP 503 | /api/cart", Category: "capacity-saturation", Status: 503, Count: 40}},
		Pods:     []Pod{{Namespace: "shop", Deployment: "checkout-api", Name: "p1", Found: true, Phase: "Pending", LastTerminatedAt: nowFixture().Add(-40 * time.Second)}},
		Deploys:  []Deploy{{Namespace: "shop", Workload: "checkout-api", Age: 2 * time.Minute, Ready: 1, Desired: 3}},
	})
	if v.Action == ActionRestart {
		t.Fatalf("action = restart during a rollout, want it withheld (gates=%v)", v.Gates)
	}
	if !strings.Contains(strings.Join(v.Gates, " "), "replaced by a rollout") {
		t.Errorf("gates = %v, want the rollout reason", v.Gates)
	}
}

func TestDecideRefusesIgnoreOfANewError(t *testing.T) {
	e, _ := engineFor(t, Policy{Actions: []Action{ActionIgnore}, MinProbability: 0.5}, "ignore", 0.9)
	v := e.Decide(context.Background(), Input{
		AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 200, Window: 5 * time.Minute, BaselineMean: 2, NewErrors: true,
		Patterns: []Pattern{{What: "HTTP 500 | /api/cart", Count: 200, New: true}},
		Pods:     []Pod{healthyPod()},
	})
	if v.Action != ActionNotify {
		t.Errorf("action = %q, want notify: a new error was never confirmed harmless", v.Action)
	}
}

func TestDecideAllowsIgnoreOfKnownNoise(t *testing.T) {
	e, _ := engineFor(t, Policy{Actions: []Action{ActionIgnore}, MinProbability: 0.5}, "ignore", 0.88)
	v := e.Decide(context.Background(), Input{
		AlertID: "a1", Service: "blog", Priority: "low", ErrorCount: 1800, Window: 5 * time.Minute, BaselineMean: 20, KnownNoise: true,
		Patterns: []Pattern{{What: "HTTP 404 | /old-post", Status: 404, Count: 1800}},
	})
	if v.Action != ActionIgnore || !v.Executable() {
		t.Errorf("action = %q executable = %v, want an executable ignore", v.Action, v.Executable())
	}
}

func TestDecideWithholdsActionsOutsideTheAllowlist(t *testing.T) {
	e, _ := engineFor(t, Policy{Actions: []Action{ActionNotify}, MinProbability: 0.5}, "ignore", 0.99)
	v := e.Decide(context.Background(), Input{
		AlertID: "a1", Service: "blog", Priority: "low", ErrorCount: 100, Window: 5 * time.Minute, BaselineMean: 20,
		Patterns: []Pattern{{What: "HTTP 404", Status: 404, Count: 100}},
		Pods:     []Pod{healthyPod()},
	})
	if v.Action != ActionNotify || v.Requested != ActionIgnore || !v.Changed() {
		t.Errorf("action = %q requested = %q changed = %v, want ignore downgraded to notify", v.Action, v.Requested, v.Changed())
	}
}

func TestDecideDowngradesALowConfidenceChoice(t *testing.T) {
	e, _ := engineFor(t, Policy{Actions: []Action{ActionRestart, ActionIgnore}, MinProbability: 0.6}, "restart", 0.41)
	v := e.Decide(context.Background(), Input{
		AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 90, Window: 5 * time.Minute, BaselineMean: 2,
		Patterns: []Pattern{{What: "HTTP 500", Count: 90, New: true}},
		Pods:     []Pod{{Namespace: "shop", Deployment: "checkout-api", Found: true, Ready: false, Phase: "Running"}},
	})
	if v.Action != ActionNotify {
		t.Errorf("action = %q, want notify: 41%% is a guess", v.Action)
	}
}

func TestRestartBudgetIsEnforced(t *testing.T) {
	e, _ := engineFor(t, Policy{Actions: []Action{ActionRestart}, MinProbability: 0.5, MaxRestartsPerHour: 1}, "restart", 0.9)
	in := Input{
		AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 90, Window: 5 * time.Minute, BaselineMean: 2,
		Patterns: []Pattern{{What: "HTTP 500", Count: 90, New: true}},
		Pods:     []Pod{{Namespace: "shop", Deployment: "checkout-api", Found: true, Ready: false, Phase: "Running"}},
	}
	if v := e.Decide(context.Background(), in); v.Action != ActionRestart {
		t.Fatalf("first decision = %q, want restart", v.Action)
	}
	second := e.Decide(context.Background(), in)
	if second.Action != ActionNotify {
		t.Errorf("second decision = %q, want notify: the hourly restart budget is spent", second.Action)
	}
}

func TestDecideDegradesToNotifyWhenTheModelIsDown(t *testing.T) {
	f := &fake{status: http.StatusServiceUnavailable}
	srv := f.server(t)
	e := NewEngine(NewClient(srv.URL), Policy{Actions: []Action{ActionRestart}, Now: nowFixture})
	v := e.Decide(context.Background(), Input{AlertID: "a1", Service: "checkout-api", ErrorCount: 90, Window: 5 * time.Minute})
	if v.Action != ActionNotify || v.Executable() || v.Error == "" {
		t.Errorf("action = %q executable = %v error = %q, want a plain notification", v.Action, v.Executable(), v.Error)
	}
}

func TestDecideRejectsAnUnknownAction(t *testing.T) {
	f := &fake{choice: "delete-the-cluster", prob: map[string]float64{"delete-the-cluster": 1}}
	srv := f.server(t)
	e := NewEngine(NewClient(srv.URL), Policy{Actions: []Action{ActionRestart}, Now: nowFixture})
	v := e.Decide(context.Background(), Input{AlertID: "a1", Service: "x", ErrorCount: 1, Window: time.Minute})
	if v.Error == "" || v.Executable() {
		t.Errorf("error = %q executable = %v, want an unknown action to be rejected", v.Error, v.Executable())
	}
}