eval

internal/alerts/laya.go

@@ -0,0 +1,180 @@
+package alerts
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Laya asks the laya model (POST {URL}/predict) what it would do about an
+// alert. It runs in shadow mode: the verdict is recorded on the alert and
+// shown in notifications, but nothing is ever acted on, muted or dropped
+// because of it. A failed call leaves the alert exactly as it was.
+type Laya struct {
+	URL    string // base URL, e.g. https://laya.k6n.net
+	Client *http.Client
+}
+
+// LayaActions are the remediations laya may choose between. They are
+// suggestions only; none is wired to an automated action yet.
+var LayaActions = map[string]string{
+	"ignore":             "transient blip or noise, nobody needs to look at it",
+	"restart_deployment": "the service is stuck, leaking or crashed; restarting its pods is likely to clear it",
+	"check_capacity":     "requests time out or resources are exhausted; check load, connection pools, CPU/memory limits and scaling",
+	"fix_dependency":     "a database, upstream service or network path this service depends on is failing",
+	"investigate_code":   "a bug or bad release in this service; needs a code fix or a rollback",
+}
+
+var layaUrgency = []string{"cosmetic", "degraded performance", "service outage"}
+
+// LayaVerdict is laya's answer, reduced to what a person needs to judge it.
+type LayaVerdict struct {
+	Model            string             `json:"model,omitempty"`
+	Action           string             `json:"action"`
+	ActionConfidence float64            `json:"actionConfidence"` // probability of the chosen action
+	ActionProbs      map[string]float64 `json:"actionProbs,omitempty"`
+	Human            float64            `json:"human"` // probability a person must intervene
+	Urgency          string             `json:"urgency"`
+	UrgencyScore     float64            `json:"urgencyScore"` // expected value on 0 (cosmetic) .. 2 (outage)
+	UrgencyConf      float64            `json:"urgencyConfidence"`
+}
+
+func layaRequest(state string) map[string]any {
+	return map[string]any{
+		"state": state,
+		"questions": map[string]any{
+			"remediation_action": map[string]any{
+				"type":         "choice",
+				"instructions": "Which operational response fits this production error alert best?",
+				"criteria":     LayaActions,
+			},
+			"requires_human_attention": map[string]any{
+				"type":         "noul",
+				"instructions": "Does this alert need a human engineer to intervene, or will it resolve or matter little on its own?",
+				"criteria": map[string]string{
+					"true":  "errors are new or rising well above normal and users or requests are failing",
+					"false": "a transient blip, client noise or bot traffic",
+				},
+			},
+			"urgency": map[string]any{
+				"type":         "score",
+				"instructions": "Rate the operational severity of this alert.",
+				"criteria":     layaUrgency,
+			},
+		},
+	}
+}
+
+type layaAnswer struct {
+	Choice           string             `json:"choice"`
+	Probabilities    map[string]float64 `json:"probabilities"`
+	AnswerConfidence float64            `json:"answer_confidence"`
+	Noul             float64            `json:"noul"`
+	Score            float64            `json:"score"`
+	Legend           map[string]string  `json:"legend"`
+}
+
+type layaResponse struct {
+	Model   string                `json:"model"`
+	Answers map[string]layaAnswer `json:"answers"`
+}
+
+// Classify asks laya about the alert. The state is the alert's plain-text
+// message, so the model sees exactly the facts a person would, nothing more.
+func (l *Laya) Classify(ctx context.Context, a Alert) (*LayaVerdict, error) {
+	a.Laya = nil
+	return l.classifyState(ctx, a.Text(""))
+}
+
+func (l *Laya) classifyState(ctx context.Context, state string) (*LayaVerdict, error) {
+	raw, err := json.Marshal(layaRequest(state))
+	if err != nil {
+		return nil, err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(l.URL, "/")+"/predict", bytes.NewReader(raw))
+	if err != nil {
+		return nil, errors.New("laya: invalid URL")
+	}
+	req.Header.Set("Content-Type", "application/json")
+	client := l.Client
+	if client == nil {
+		client = &http.Client{Timeout: 10 * time.Second}
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("laya: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode/100 != 2 {
+		detail, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
+		return nil, fmt.Errorf("laya: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(detail)))
+	}
+	var lr layaResponse
+	if err := json.NewDecoder(resp.Body).Decode(&lr); err != nil {
+		return nil, fmt.Errorf("laya: invalid response: %w", err)
+	}
+	return lr.verdict()
+}
+
+func (lr layaResponse) verdict() (*LayaVerdict, error) {
+	act, ok1 := lr.Answers["remediation_action"]
+	hum, ok2 := lr.Answers["requires_human_attention"]
+	urg, ok3 := lr.Answers["urgency"]
+	if !ok1 || !ok2 || !ok3 || act.Choice == "" {
+		return nil, errors.New("laya: response is missing answers")
+	}
+	v := &LayaVerdict{Model: lr.Model, Action: act.Choice, ActionConfidence: act.Probabilities[act.Choice], ActionProbs: act.Probabilities,
+		Human: hum.Noul, UrgencyScore: urg.Score}
+	// The urgency label is the most probable bucket, not the rounded score:
+	// a 50/50 cosmetic/outage split must not read as "degraded".
+	best := -1.0
+	for k, p := range urg.Probabilities {
+		if p > best || (p == best && k < v.Urgency) {
+			best, v.Urgency = p, k
+		}
+	}
+	v.UrgencyConf = best
+	if label, ok := urg.Legend[v.Urgency]; ok {
+		v.Urgency = label
+	} else if i, err := strconv.Atoi(v.Urgency); err == nil && i >= 0 && i < len(layaUrgency) {
+		v.Urgency = layaUrgency[i]
+	}
+	return v, nil
+}
+
+// Summary is a one-line rendering, e.g.
+// "check_capacity 62% · human 71% · degraded performance 75%".
+func (v LayaVerdict) Summary() string {
+	return fmt.Sprintf("%s %s · human %s · %s %s", v.Action, pct(v.ActionConfidence), pct(v.Human), v.Urgency, pct(v.UrgencyConf))
+}
+
+// RunnersUp lists the other actions by probability, for judging how close
+// the call was.
+func (v LayaVerdict) RunnersUp() string {
+	type kv struct {
+		k string
+		p float64
+	}
+	var rest []kv
+	for k, p := range v.ActionProbs {
+		if k != v.Action {
+			rest = append(rest, kv{k, p})
+		}
+	}
+	sort.Slice(rest, func(i, j int) bool { return rest[i].p > rest[j].p })
+	parts := make([]string, 0, len(rest))
+	for _, r := range rest {
+		parts = append(parts, r.k+" "+pct(r.p))
+	}
+	return strings.Join(parts, ", ")
+}
+
+func pct(p float64) string { return strconv.Itoa(int(p*100+0.5)) + "%" }

internal/alerts/laya_test.go

@@ -0,0 +1,123 @@
+package alerts
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/features"
+)
+
+// layaReply is the shape laya.k6n.net/predict returned when probed.
+const layaReply = `{"model":"laya-rl-agent","answers":{
+ "remediation_action":{"type":"choice","choice":"check_capacity","probabilities":{"ignore":0.1,"restart_deployment":0.12,"check_capacity":0.62,"fix_dependency":0.1,"investigate_code":0.06},"confidence":0.3,"answer_confidence":0.62},
+ "requires_human_attention":{"type":"noul","noul":0.7123,"confidence":0.7123,"answer_confidence":0.7123},
+ "urgency":{"type":"score","score":1.03,"legend":{"0":"cosmetic","1":"degraded performance","2":"service outage"},"probabilities":{"0":0.11,"1":0.75,"2":0.14},"confidence":0.32,"answer_confidence":0.75}},
+ "usage":{"input_tokens":141,"output_tokens":0}}`
+
+func layaServer(t *testing.T, status int, body string, got *map[string]any) *httptest.Server {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/predict" {
+			t.Errorf("path %q", r.URL.Path)
+		}
+		if got != nil {
+			_ = json.NewDecoder(r.Body).Decode(got)
+		}
+		w.WriteHeader(status)
+		_, _ = w.Write([]byte(body))
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func timeoutAlert() Alert {
+	now := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	return NewAlert(features.Spike{Service: "codesearch", WindowStart: now.Add(-5 * time.Minute), WindowEnd: now, ErrorCount: 15, BaselineMean: 0, Excess: 15,
+		Patterns: []features.PatternDelta{{Signature: "t", Count: 15, Excess: 15, Novel: true,
+			Example: features.Event{Service: "codesearch", Kind: "span", SpanName: "upload", Message: "context deadline exceeded", IsTimeout: true}}}}, now)
+}
+
+func TestLayaClassifySendsAlertTextAndParsesVerdict(t *testing.T) {
+	var req map[string]any
+	srv := layaServer(t, 200, layaReply, &req)
+	a := timeoutAlert()
+	v, err := (&Laya{URL: srv.URL + "/"}).Classify(context.Background(), a)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req["state"] != a.Text("") {
+		t.Fatalf("state is not the alert text:\n%v", req["state"])
+	}
+	qs := req["questions"].(map[string]any)
+	for _, k := range []string{"remediation_action", "requires_human_attention", "urgency"} {
+		if qs[k] == nil {
+			t.Fatalf("question %s missing", k)
+		}
+	}
+	if v.Action != "check_capacity" || v.ActionConfidence != 0.62 || v.Human != 0.7123 || v.Urgency != "degraded performance" || v.UrgencyConf != 0.75 {
+		t.Fatalf("verdict %+v", v)
+	}
+	if got := v.Summary(); got != "check_capacity 62% · human 71% · degraded performance 75%" {
+		t.Fatalf("summary %q", got)
+	}
+	if got := v.RunnersUp(); !strings.HasPrefix(got, "restart_deployment 12%, ") {
+		t.Fatalf("runners-up %q", got)
+	}
+}
+
+func TestLayaErrors(t *testing.T) {
+	for name, tc := range map[string]struct {
+		status int
+		body   string
+	}{
+		"http":    {502, "bad gateway"},
+		"json":    {200, "not json"},
+		"missing": {200, `{"answers":{"urgency":{}}}`},
+	} {
+		srv := layaServer(t, tc.status, tc.body, nil)
+		if _, err := (&Laya{URL: srv.URL}).Classify(context.Background(), timeoutAlert()); err == nil {
+			t.Errorf("%s: want error", name)
+		}
+	}
+}
+
+func TestLayaVerdictInNotifications(t *testing.T) {
+	a := timeoutAlert()
+	a.Laya = &LayaVerdict{Action: "restart_deployment", ActionConfidence: 0.8, ActionProbs: map[string]float64{"restart_deployment": 0.8, "ignore": 0.2}, Human: 0.4, Urgency: "service outage", UrgencyConf: 0.6}
+	h := a.HTML("https://otel.example", telegramMax)
+	if !strings.Contains(h, "🤖 <b>laya</b> <i>(shadow, no action taken)</i>: <code>restart_deployment</code> 80% · human 40% · service outage 60%") ||
+		!strings.Contains(h, "other: ignore 20%") {
+		t.Fatalf("html:\n%s", h)
+	}
+	if txt := a.Text(""); !strings.Contains(txt, "laya (shadow, no action taken): restart_deployment 80%") {
+		t.Fatalf("text:\n%s", txt)
+	}
+	a.Laya = nil
+	if strings.Contains(a.HTML("", telegramMax), "laya") {
+		t.Fatal("no verdict must render no laya line")
+	}
+}
+
+// A laya outage must never cost a page; the alert just goes out without a verdict.
+func TestRunnerAttachesVerdictAndSurvivesLayaFailure(t *testing.T) {
+	for _, tc := range []struct {
+		status int
+		want   bool
+	}{{200, true}, {500, false}} {
+		srv := layaServer(t, tc.status, layaReply, nil)
+		f := &fake{}
+		r := newRunner(nil, f)
+		r.laya = &Laya{URL: srv.URL}
+		r.deliver(context.Background(), timeoutAlert())
+		if len(f.sent) != 1 {
+			t.Fatalf("status %d: sent %d", tc.status, len(f.sent))
+		}
+		if got := f.sent[0].Laya != nil; got != tc.want {
+			t.Fatalf("status %d: verdict attached=%v", tc.status, got)
+		}
+	}
+}

internal/alerts/notify.go

@@ -29,6 +29,9 @@ type Alert struct {
 	Spike    features.Spike  `json:"spike"`
 	Patterns []PatternReport `json:"patterns"`
 	Test     bool            `json:"test,omitempty"`
+	// Laya is the laya model's shadow verdict, when it was asked. Advisory
+	// only: nothing is acted on or suppressed because of it.
+	Laya *LayaVerdict `json:"laya,omitempty"`
 }
 
 // PatternReport is a spike pattern with its triage verdict, ready to show.
@@ -214,6 +217,9 @@ func (a Alert) Text(baseURL string) string {
 	window := a.Spike.WindowEnd.Sub(a.Spike.WindowStart).Round(time.Minute)
 	fmt.Fprintf(&b, "%s: %s is failing (%d errors in the last %s, normally about %.0f)\n",
 		prefix, a.Spike.Service, a.Spike.ErrorCount, window, a.Spike.BaselineMean)
+	if a.Laya != nil {
+		fmt.Fprintf(&b, "laya (shadow, no action taken): %s\n", a.Laya.Summary())
+	}
 	base := strings.TrimRight(baseURL, "/")
 	for i, p := range a.Patterns {
 		fmt.Fprintf(&b, "\n%d. %s\n", i+1, p.What)
@@ -282,6 +288,13 @@ func (a Alert) HTML(baseURL string, max int) string {
 	if base != "" {
 		foot = fmt.Sprintf("\n<a href=\"%s\">All issues</a>", esc(base+"/issues"))
 	}
+	if a.Laya != nil {
+		head += fmt.Sprintf("🤖 <b>laya</b> <i>(shadow, no action taken)</i>: <code>%s</code> %s · human %s · %s %s\n",
+			esc(a.Laya.Action), pct(a.Laya.ActionConfidence), pct(a.Laya.Human), esc(a.Laya.Urgency), pct(a.Laya.UrgencyConf))
+		if r := a.Laya.RunnersUp(); r != "" {
+			head += "<i>   other: " + esc(r) + "</i>\n"
+		}
+	}
 	var b strings.Builder
 	b.WriteString(head)
 	for i, p := range a.Patterns {

internal/alerts/runner.go

@@ -34,6 +34,9 @@ type Config struct {
 	Muted                     func(issueID string) bool
 	WebhookURL, WebhookSecret string
 	AODURL                    string
+	// LayaURL, when set, asks laya (POST {LayaURL}/predict) for a shadow
+	// verdict on every alert and shows it in the notification.
+	LayaURL string
 }
 
 func ConfigFromEnv() Config {
@@ -54,6 +57,7 @@ func ConfigFromEnv() Config {
 		WebhookURL:       os.Getenv("OTEL_ALERT_WEBHOOK_URL"),
 		WebhookSecret:    os.Getenv("OTEL_ALERT_WEBHOOK_SECRET"),
 		AODURL:           os.Getenv("OTEL_ALERT_AOD_URL"),
+		LayaURL:          os.Getenv("OTEL_ALERT_LAYA_URL"),
 	}
 }
 
@@ -106,6 +110,7 @@ type Runner struct {
 	dataSince func(ctx context.Context) time.Time
 	notifiers []Notifier
 	resolver  Resolver // optional: ingress -> service -> deployment for edge errors
+	laya      *Laya    // optional: shadow verdict attached to each alert
 	sup       *features.Suppressor
 
 	mu        sync.Mutex
@@ -137,6 +142,10 @@ func Start(ctx context.Context, cfg Config, conn driver.Conn, database string, r
 		},
 		dataSince: func(ctx context.Context) time.Time { return DataSince(ctx, conn, database) },
 	}
+	if cfg.LayaURL != "" {
+		r.laya = &Laya{URL: cfg.LayaURL}
+		log.Printf("alerts: laya shadow verdicts from %s", cfg.LayaURL)
+	}
 	names := make([]string, 0, len(notifiers))
 	for _, n := range notifiers {
 		names = append(names, n.Name())
@@ -239,6 +248,7 @@ func (r *Runner) deliver(ctx context.Context, a Alert) {
 	if pr, ok := r.resolver.(PodResolver); ok {
 		a.EnrichPods(ctx, pr)
 	}
+	r.askLaya(ctx, &a)
 	r.mu.Lock()
 	r.recent = append(r.recent, a)
 	if len(r.recent) > maxRecent {
@@ -261,6 +271,23 @@ func (r *Runner) deliver(ctx context.Context, a Alert) {
 	}
 }
 
+// askLaya attaches laya's shadow verdict. It is bounded so a slow model
+// delays a page by seconds at most, and a failure only costs the verdict.
+func (r *Runner) askLaya(ctx context.Context, a *Alert) {
+	if r.laya == nil {
+		return
+	}
+	lctx, cancel := context.WithTimeout(ctx, 8*time.Second)
+	defer cancel()
+	v, err := r.laya.Classify(lctx, *a)
+	if err != nil {
+		log.Printf("alerts: laya verdict for %s failed: %v", a.ID, err)
+		return
+	}
+	a.Laya = v
+	log.Printf("alerts: laya[shadow] %s: %s", a.ID, v.Summary())
+}
+
 // Status is the JSON served by GET /api/alerts.
 type Status struct {
 	Enabled   bool      `json:"enabled"`
@@ -305,6 +332,7 @@ func (r *Runner) HandleTest(w http.ResponseWriter, req *http.Request) {
 		Example: features.Event{Service: "alert-test", Kind: "span", StatusCode: 500, Route: "/api/example", Message: "this is a test alert"},
 	}}}, now)
 	a.Test = true
+	r.askLaya(req.Context(), &a)
 	results := map[string]string{}
 	for _, n := range r.notifiers {
 		if err := n.Send(req.Context(), a); err != nil {