124 lines
4.5 KiB
Go
124 lines
4.5 KiB
Go
package alerts
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.tornberg.me/go-otel/internal/features"
|
|
)
|
|
|
|
// layaReply is the shape laya.k6n.net/predict returned when probed.
|
|
const layaReply = `{"model":"laya-rl-agent","answers":{
|
|
"remediation_action":{"type":"choice","choice":"check_capacity","probabilities":{"ignore":0.1,"restart_deployment":0.12,"check_capacity":0.62,"fix_dependency":0.1,"investigate_code":0.06},"confidence":0.3,"answer_confidence":0.62},
|
|
"requires_human_attention":{"type":"noul","noul":0.7123,"confidence":0.7123,"answer_confidence":0.7123},
|
|
"urgency":{"type":"score","score":1.03,"legend":{"0":"cosmetic","1":"degraded performance","2":"service outage"},"probabilities":{"0":0.11,"1":0.75,"2":0.14},"confidence":0.32,"answer_confidence":0.75}},
|
|
"usage":{"input_tokens":141,"output_tokens":0}}`
|
|
|
|
func layaServer(t *testing.T, status int, body string, got *map[string]any) *httptest.Server {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path != "/predict" {
|
|
t.Errorf("path %q", r.URL.Path)
|
|
}
|
|
if got != nil {
|
|
_ = json.NewDecoder(r.Body).Decode(got)
|
|
}
|
|
w.WriteHeader(status)
|
|
_, _ = w.Write([]byte(body))
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
return srv
|
|
}
|
|
|
|
func timeoutAlert() Alert {
|
|
now := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
|
|
return NewAlert(features.Spike{Service: "codesearch", WindowStart: now.Add(-5 * time.Minute), WindowEnd: now, ErrorCount: 15, BaselineMean: 0, Excess: 15,
|
|
Patterns: []features.PatternDelta{{Signature: "t", Count: 15, Excess: 15, Novel: true,
|
|
Example: features.Event{Service: "codesearch", Kind: "span", SpanName: "upload", Message: "context deadline exceeded", IsTimeout: true}}}}, now)
|
|
}
|
|
|
|
func TestLayaClassifySendsAlertTextAndParsesVerdict(t *testing.T) {
|
|
var req map[string]any
|
|
srv := layaServer(t, 200, layaReply, &req)
|
|
a := timeoutAlert()
|
|
v, err := (&Laya{URL: srv.URL + "/"}).Classify(context.Background(), a)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if req["state"] != a.Text("") {
|
|
t.Fatalf("state is not the alert text:\n%v", req["state"])
|
|
}
|
|
qs := req["questions"].(map[string]any)
|
|
for _, k := range []string{"remediation_action", "requires_human_attention", "urgency"} {
|
|
if qs[k] == nil {
|
|
t.Fatalf("question %s missing", k)
|
|
}
|
|
}
|
|
if v.Action != "check_capacity" || v.ActionConfidence != 0.62 || v.Human != 0.7123 || v.Urgency != "degraded performance" || v.UrgencyConf != 0.75 {
|
|
t.Fatalf("verdict %+v", v)
|
|
}
|
|
if got := v.Summary(); got != "check_capacity 62% · human 71% · degraded performance 75%" {
|
|
t.Fatalf("summary %q", got)
|
|
}
|
|
if got := v.RunnersUp(); !strings.HasPrefix(got, "restart_deployment 12%, ") {
|
|
t.Fatalf("runners-up %q", got)
|
|
}
|
|
}
|
|
|
|
func TestLayaErrors(t *testing.T) {
|
|
for name, tc := range map[string]struct {
|
|
status int
|
|
body string
|
|
}{
|
|
"http": {502, "bad gateway"},
|
|
"json": {200, "not json"},
|
|
"missing": {200, `{"answers":{"urgency":{}}}`},
|
|
} {
|
|
srv := layaServer(t, tc.status, tc.body, nil)
|
|
if _, err := (&Laya{URL: srv.URL}).Classify(context.Background(), timeoutAlert()); err == nil {
|
|
t.Errorf("%s: want error", name)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestLayaVerdictInNotifications(t *testing.T) {
|
|
a := timeoutAlert()
|
|
a.Laya = &LayaVerdict{Action: "restart_deployment", ActionConfidence: 0.8, ActionProbs: map[string]float64{"restart_deployment": 0.8, "ignore": 0.2}, Human: 0.4, Urgency: "service outage", UrgencyConf: 0.6}
|
|
h := a.HTML("https://otel.example", telegramMax)
|
|
if !strings.Contains(h, "🤖 <b>laya</b> <i>(shadow, no action taken)</i>: <code>restart_deployment</code> 80% · human 40% · service outage 60%") ||
|
|
!strings.Contains(h, "other: ignore 20%") {
|
|
t.Fatalf("html:\n%s", h)
|
|
}
|
|
if txt := a.Text(""); !strings.Contains(txt, "laya (shadow, no action taken): restart_deployment 80%") {
|
|
t.Fatalf("text:\n%s", txt)
|
|
}
|
|
a.Laya = nil
|
|
if strings.Contains(a.HTML("", telegramMax), "laya") {
|
|
t.Fatal("no verdict must render no laya line")
|
|
}
|
|
}
|
|
|
|
// A laya outage must never cost a page; the alert just goes out without a verdict.
|
|
func TestRunnerAttachesVerdictAndSurvivesLayaFailure(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
status int
|
|
want bool
|
|
}{{200, true}, {500, false}} {
|
|
srv := layaServer(t, tc.status, layaReply, nil)
|
|
f := &fake{}
|
|
r := newRunner(nil, f)
|
|
r.laya = &Laya{URL: srv.URL}
|
|
r.deliver(context.Background(), timeoutAlert())
|
|
if len(f.sent) != 1 {
|
|
t.Fatalf("status %d: sent %d", tc.status, len(f.sent))
|
|
}
|
|
if got := f.sent[0].Laya != nil; got != tc.want {
|
|
t.Fatalf("status %d: verdict attached=%v", tc.status, got)
|
|
}
|
|
}
|
|
}
|