eval

cmd/laya-eval/main.go

@@ -0,0 +1,144 @@
+// laya-eval runs alert scenarios through laya and compares its shadow verdict
+// with what a person judged the right response to be, so the model can be
+// assessed before it is ever allowed to act.
+//
+//	go run ./cmd/laya-eval                        # built-in scenarios
+//	go run ./cmd/laya-eval -file alerts.json      # a GET /api/alerts dump (no expectations)
+//	go run ./cmd/laya-eval -send                  # also deliver each as a TEST alert
+//
+// -send uses the channels configured by OTEL_ALERT_* (see alerts.ConfigFromEnv).
+package main
+
+import (
+	"context"
+	_ "embed"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"text/tabwriter"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/alerts"
+)
+
+//go:embed scenarios.json
+var builtin []byte
+
+// Scenario is one alert and the response a person would pick for it.
+type Scenario struct {
+	Name   string       `json:"name"`
+	Expect *Expectation `json:"expect,omitempty"`
+	Alert  alerts.Alert `json:"alert"`
+}
+
+type Expectation struct {
+	Action string `json:"action"`
+	Human  bool   `json:"human"`
+	Why    string `json:"why"`
+}
+
+func main() {
+	url := flag.String("url", envOr("OTEL_ALERT_LAYA_URL", "https://laya.k6n.net"), "laya base URL")
+	file := flag.String("file", "", "scenarios JSON, or a GET /api/alerts response (default: built-in scenarios)")
+	send := flag.Bool("send", false, "deliver each scenario, with its verdict, as a TEST alert to the configured channels")
+	flag.Parse()
+
+	raw := builtin
+	if *file != "" {
+		var err error
+		if raw, err = os.ReadFile(*file); err != nil {
+			fail(err)
+		}
+	}
+	scenarios, err := parse(raw)
+	if err != nil {
+		fail(err)
+	}
+	var notifiers []alerts.Notifier
+	if *send {
+		cfg := alerts.ConfigFromEnv()
+		if notifiers = cfg.Notifiers(); len(notifiers) == 0 {
+			fail(fmt.Errorf("-send: no channel configured (OTEL_ALERT_TELEGRAM_TOKEN + OTEL_ALERT_TELEGRAM_CHAT_ID, OTEL_ALERT_WEBHOOK_URL or OTEL_ALERT_AOD_URL)"))
+		}
+	}
+
+	laya := &alerts.Laya{URL: *url}
+	tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(tw, "SCENARIO\tLAYA ACTION\tHUMAN\tURGENCY\tEXPECTED\tMATCH")
+	judged, actionOK, humanOK := 0, 0, 0
+	for _, s := range scenarios {
+		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+		v, err := laya.Classify(ctx, s.Alert)
+		cancel()
+		if err != nil {
+			fmt.Fprintf(tw, "%s\terror: %v\t\t\t\t\n", s.Name, err)
+			continue
+		}
+		exp, match := "-", "-"
+		if s.Expect != nil {
+			judged++
+			a, h := v.Action == s.Expect.Action, (v.Human >= 0.5) == s.Expect.Human
+			if a {
+				actionOK++
+			}
+			if h {
+				humanOK++
+			}
+			exp = fmt.Sprintf("%s, human=%v", s.Expect.Action, s.Expect.Human)
+			match = fmt.Sprintf("action %s, human %s", mark(a), mark(h))
+		}
+		fmt.Fprintf(tw, "%s\t%s %.0f%%\t%.0f%%\t%s %.0f%%\t%s\t%s\n", s.Name, v.Action, v.ActionConfidence*100, v.Human*100, v.Urgency, v.UrgencyConf*100, exp, match)
+
+		if *send {
+			a := s.Alert
+			a.Test, a.Laya = true, v
+			for _, n := range notifiers {
+				sctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+				if err := n.Send(sctx, a); err != nil {
+					fmt.Fprintf(os.Stderr, "%s: %s: %v\n", s.Name, n.Name(), err)
+				}
+				cancel()
+			}
+		}
+	}
+	tw.Flush()
+	if judged > 0 {
+		fmt.Printf("\nagreement over %d judged scenarios: action %d/%d, needs-human %d/%d\n", judged, actionOK, judged, humanOK, judged)
+	}
+}
+
+// parse accepts a scenario list or a raw GET /api/alerts response.
+func parse(raw []byte) ([]Scenario, error) {
+	var list []Scenario
+	if err := json.Unmarshal(raw, &list); err == nil {
+		return list, nil
+	}
+	var st alerts.Status
+	if err := json.Unmarshal(raw, &st); err != nil {
+		return nil, fmt.Errorf("neither a scenario list nor an /api/alerts response: %w", err)
+	}
+	for _, a := range st.Alerts {
+		list = append(list, Scenario{Name: a.ID + " " + a.Spike.Service, Alert: a})
+	}
+	return list, nil
+}
+
+func mark(ok bool) string {
+	if ok {
+		return "ok"
+	}
+	return "MISS"
+}
+
+func envOr(k, def string) string {
+	if v := os.Getenv(k); v != "" {
+		return v
+	}
+	return def
+}
+
+func fail(err error) {
+	fmt.Fprintln(os.Stderr, err)
+	os.Exit(1)
+}