145 lines
4.0 KiB
Go
145 lines
4.0 KiB
Go
// laya-eval runs alert scenarios through laya and compares its shadow verdict
|
|
// with what a person judged the right response to be, so the model can be
|
|
// assessed before it is ever allowed to act.
|
|
//
|
|
// go run ./cmd/laya-eval # built-in scenarios
|
|
// go run ./cmd/laya-eval -file alerts.json # a GET /api/alerts dump (no expectations)
|
|
// go run ./cmd/laya-eval -send # also deliver each as a TEST alert
|
|
//
|
|
// -send uses the channels configured by OTEL_ALERT_* (see alerts.ConfigFromEnv).
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
_ "embed"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"os"
|
|
"text/tabwriter"
|
|
"time"
|
|
|
|
"git.tornberg.me/go-otel/internal/alerts"
|
|
)
|
|
|
|
//go:embed scenarios.json
|
|
var builtin []byte
|
|
|
|
// Scenario is one alert and the response a person would pick for it.
|
|
type Scenario struct {
|
|
Name string `json:"name"`
|
|
Expect *Expectation `json:"expect,omitempty"`
|
|
Alert alerts.Alert `json:"alert"`
|
|
}
|
|
|
|
type Expectation struct {
|
|
Action string `json:"action"`
|
|
Human bool `json:"human"`
|
|
Why string `json:"why"`
|
|
}
|
|
|
|
func main() {
|
|
url := flag.String("url", envOr("OTEL_ALERT_LAYA_URL", "https://laya.k6n.net"), "laya base URL")
|
|
file := flag.String("file", "", "scenarios JSON, or a GET /api/alerts response (default: built-in scenarios)")
|
|
send := flag.Bool("send", false, "deliver each scenario, with its verdict, as a TEST alert to the configured channels")
|
|
flag.Parse()
|
|
|
|
raw := builtin
|
|
if *file != "" {
|
|
var err error
|
|
if raw, err = os.ReadFile(*file); err != nil {
|
|
fail(err)
|
|
}
|
|
}
|
|
scenarios, err := parse(raw)
|
|
if err != nil {
|
|
fail(err)
|
|
}
|
|
var notifiers []alerts.Notifier
|
|
if *send {
|
|
cfg := alerts.ConfigFromEnv()
|
|
if notifiers = cfg.Notifiers(); len(notifiers) == 0 {
|
|
fail(fmt.Errorf("-send: no channel configured (OTEL_ALERT_TELEGRAM_TOKEN + OTEL_ALERT_TELEGRAM_CHAT_ID, OTEL_ALERT_WEBHOOK_URL or OTEL_ALERT_AOD_URL)"))
|
|
}
|
|
}
|
|
|
|
laya := &alerts.Laya{URL: *url}
|
|
tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
|
|
fmt.Fprintln(tw, "SCENARIO\tLAYA ACTION\tHUMAN\tURGENCY\tEXPECTED\tMATCH")
|
|
judged, actionOK, humanOK := 0, 0, 0
|
|
for _, s := range scenarios {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
|
v, err := laya.Classify(ctx, s.Alert)
|
|
cancel()
|
|
if err != nil {
|
|
fmt.Fprintf(tw, "%s\terror: %v\t\t\t\t\n", s.Name, err)
|
|
continue
|
|
}
|
|
exp, match := "-", "-"
|
|
if s.Expect != nil {
|
|
judged++
|
|
a, h := v.Action == s.Expect.Action, (v.Human >= 0.5) == s.Expect.Human
|
|
if a {
|
|
actionOK++
|
|
}
|
|
if h {
|
|
humanOK++
|
|
}
|
|
exp = fmt.Sprintf("%s, human=%v", s.Expect.Action, s.Expect.Human)
|
|
match = fmt.Sprintf("action %s, human %s", mark(a), mark(h))
|
|
}
|
|
fmt.Fprintf(tw, "%s\t%s %.0f%%\t%.0f%%\t%s %.0f%%\t%s\t%s\n", s.Name, v.Action, v.ActionConfidence*100, v.Human*100, v.Urgency, v.UrgencyConf*100, exp, match)
|
|
|
|
if *send {
|
|
a := s.Alert
|
|
a.Test, a.Laya = true, v
|
|
for _, n := range notifiers {
|
|
sctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
|
if err := n.Send(sctx, a); err != nil {
|
|
fmt.Fprintf(os.Stderr, "%s: %s: %v\n", s.Name, n.Name(), err)
|
|
}
|
|
cancel()
|
|
}
|
|
}
|
|
}
|
|
tw.Flush()
|
|
if judged > 0 {
|
|
fmt.Printf("\nagreement over %d judged scenarios: action %d/%d, needs-human %d/%d\n", judged, actionOK, judged, humanOK, judged)
|
|
}
|
|
}
|
|
|
|
// parse accepts a scenario list or a raw GET /api/alerts response.
|
|
func parse(raw []byte) ([]Scenario, error) {
|
|
var list []Scenario
|
|
if err := json.Unmarshal(raw, &list); err == nil {
|
|
return list, nil
|
|
}
|
|
var st alerts.Status
|
|
if err := json.Unmarshal(raw, &st); err != nil {
|
|
return nil, fmt.Errorf("neither a scenario list nor an /api/alerts response: %w", err)
|
|
}
|
|
for _, a := range st.Alerts {
|
|
list = append(list, Scenario{Name: a.ID + " " + a.Spike.Service, Alert: a})
|
|
}
|
|
return list, nil
|
|
}
|
|
|
|
func mark(ok bool) string {
|
|
if ok {
|
|
return "ok"
|
|
}
|
|
return "MISS"
|
|
}
|
|
|
|
func envOr(k, def string) string {
|
|
if v := os.Getenv(k); v != "" {
|
|
return v
|
|
}
|
|
return def
|
|
}
|
|
|
|
func fail(err error) {
|
|
fmt.Fprintln(os.Stderr, err)
|
|
os.Exit(1)
|
|
}
|