go-otel/internal/alerts/laya.go

package alerts

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"time"
)

// Laya asks the laya model (POST {URL}/predict) what it would do about an
// alert. It runs in shadow mode: the verdict is recorded on the alert and
// shown in notifications, but nothing is ever acted on, muted or dropped
// because of it. A failed call leaves the alert exactly as it was.
type Laya struct {
	URL    string // base URL, e.g. https://laya.k6n.net
	Client *http.Client
}

// LayaActions are the remediations laya may choose between. They are
// suggestions only; none is wired to an automated action yet.
var LayaActions = map[string]string{
	"ignore":             "transient blip or noise, nobody needs to look at it",
	"restart_deployment": "the service is stuck, leaking or crashed; restarting its pods is likely to clear it",
	"check_capacity":     "requests time out or resources are exhausted; check load, connection pools, CPU/memory limits and scaling",
	"fix_dependency":     "a database, upstream service or network path this service depends on is failing",
	"investigate_code":   "a bug or bad release in this service; needs a code fix or a rollback",
}

var layaUrgency = []string{"cosmetic", "degraded performance", "service outage"}

// LayaVerdict is laya's answer, reduced to what a person needs to judge it.
type LayaVerdict struct {
	Model            string             `json:"model,omitempty"`
	Action           string             `json:"action"`
	ActionConfidence float64            `json:"actionConfidence"` // probability of the chosen action
	ActionProbs      map[string]float64 `json:"actionProbs,omitempty"`
	Human            float64            `json:"human"` // probability a person must intervene
	Urgency          string             `json:"urgency"`
	UrgencyScore     float64            `json:"urgencyScore"` // expected value on 0 (cosmetic) .. 2 (outage)
	UrgencyConf      float64            `json:"urgencyConfidence"`
}

func layaRequest(state string) map[string]any {
	return map[string]any{
		"state": state,
		"questions": map[string]any{
			"remediation_action": map[string]any{
				"type":         "choice",
				"instructions": "Which operational response fits this production error alert best?",
				"criteria":     LayaActions,
			},
			"requires_human_attention": map[string]any{
				"type":         "noul",
				"instructions": "Does this alert need a human engineer to intervene, or will it resolve or matter little on its own?",
				"criteria": map[string]string{
					"true":  "errors are new or rising well above normal and users or requests are failing",
					"false": "a transient blip, client noise or bot traffic",
				},
			},
			"urgency": map[string]any{
				"type":         "score",
				"instructions": "Rate the operational severity of this alert.",
				"criteria":     layaUrgency,
			},
		},
	}
}

type layaAnswer struct {
	Choice           string             `json:"choice"`
	Probabilities    map[string]float64 `json:"probabilities"`
	AnswerConfidence float64            `json:"answer_confidence"`
	Noul             float64            `json:"noul"`
	Score            float64            `json:"score"`
	Legend           map[string]string  `json:"legend"`
}

type layaResponse struct {
	Model   string                `json:"model"`
	Answers map[string]layaAnswer `json:"answers"`
}

// Classify asks laya about the alert. The state is the alert's plain-text
// message, so the model sees exactly the facts a person would, nothing more.
func (l *Laya) Classify(ctx context.Context, a Alert) (*LayaVerdict, error) {
	a.Laya = nil
	return l.classifyState(ctx, a.Text(""))
}

func (l *Laya) classifyState(ctx context.Context, state string) (*LayaVerdict, error) {
	raw, err := json.Marshal(layaRequest(state))
	if err != nil {
		return nil, err
	}
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(l.URL, "/")+"/predict", bytes.NewReader(raw))
	if err != nil {
		return nil, errors.New("laya: invalid URL")
	}
	req.Header.Set("Content-Type", "application/json")
	client := l.Client
	if client == nil {
		client = &http.Client{Timeout: 10 * time.Second}
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("laya: %w", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode/100 != 2 {
		detail, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
		return nil, fmt.Errorf("laya: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(detail)))
	}
	var lr layaResponse
	if err := json.NewDecoder(resp.Body).Decode(&lr); err != nil {
		return nil, fmt.Errorf("laya: invalid response: %w", err)
	}
	return lr.verdict()
}

func (lr layaResponse) verdict() (*LayaVerdict, error) {
	act, ok1 := lr.Answers["remediation_action"]
	hum, ok2 := lr.Answers["requires_human_attention"]
	urg, ok3 := lr.Answers["urgency"]
	if !ok1 || !ok2 || !ok3 || act.Choice == "" {
		return nil, errors.New("laya: response is missing answers")
	}
	v := &LayaVerdict{Model: lr.Model, Action: act.Choice, ActionConfidence: act.Probabilities[act.Choice], ActionProbs: act.Probabilities,
		Human: hum.Noul, UrgencyScore: urg.Score}
	// The urgency label is the most probable bucket, not the rounded score:
	// a 50/50 cosmetic/outage split must not read as "degraded".
	best := -1.0
	for k, p := range urg.Probabilities {
		if p > best || (p == best && k < v.Urgency) {
			best, v.Urgency = p, k
		}
	}
	v.UrgencyConf = best
	if label, ok := urg.Legend[v.Urgency]; ok {
		v.Urgency = label
	} else if i, err := strconv.Atoi(v.Urgency); err == nil && i >= 0 && i < len(layaUrgency) {
		v.Urgency = layaUrgency[i]
	}
	return v, nil
}

// Summary is a one-line rendering, e.g.
// "check_capacity 62% · human 71% · degraded performance 75%".
func (v LayaVerdict) Summary() string {
	return fmt.Sprintf("%s %s · human %s · %s %s", v.Action, pct(v.ActionConfidence), pct(v.Human), v.Urgency, pct(v.UrgencyConf))
}

// RunnersUp lists the other actions by probability, for judging how close
// the call was.
func (v LayaVerdict) RunnersUp() string {
	type kv struct {
		k string
		p float64
	}
	var rest []kv
	for k, p := range v.ActionProbs {
		if k != v.Action {
			rest = append(rest, kv{k, p})
		}
	}
	sort.Slice(rest, func(i, j int) bool { return rest[i].p > rest[j].p })
	parts := make([]string, 0, len(rest))
	for _, r := range rest {
		parts = append(parts, r.k+" "+pct(r.p))
	}
	return strings.Join(parts, ", ")
}

func pct(p float64) string { return strconv.Itoa(int(p*100+0.5)) + "%" }