all the changes

cmd/triage-eval/main.go

@@ -0,0 +1,98 @@
+// triage-eval applies the rules classifier to a JSONL file of patterns. Rows
+// with an "expected" category are scored; the rest are written out with the
+// prediction so they can be reviewed and labelled by hand.
+//
+//	triage-eval -input holdout.jsonl            # score labelled rows
+//	triage-eval -input raw.jsonl -label-out holdout.jsonl   # prepare for labelling
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"sort"
+
+	"git.tornberg.me/go-otel/internal/triage"
+)
+
+type row struct {
+	Service        string `json:"service"`
+	Kind           string `json:"kind"`
+	StatusCode     int    `json:"status_code"`
+	Route          string `json:"route"`
+	Title          string `json:"title"`
+	Message        string `json:"message"`
+	IsTimeout      uint8  `json:"is_timeout"`
+	IsCancellation uint8  `json:"is_cancellation"`
+	IsRetry        uint8  `json:"is_retry"`
+	Count          uint64 `json:"n"`
+	Expected       string `json:"expected"`
+	Predicted      string `json:"predicted"`
+	Rule           string `json:"rule"`
+}
+
+func main() {
+	input := flag.String("input", "", "JSONL input")
+	labelOut := flag.String("label-out", "", "write rows with predictions here for hand labelling")
+	flag.Parse()
+	f, err := os.Open(*input)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+	var rows []row
+	sc := bufio.NewScanner(f)
+	sc.Buffer(make([]byte, 1<<20), 1<<20)
+	for sc.Scan() {
+		var r row
+		if err := json.Unmarshal(sc.Bytes(), &r); err != nil {
+			log.Fatalf("bad row: %v", err)
+		}
+		res := triage.Classify(triage.Input{Service: r.Service, Kind: r.Kind, Status: r.StatusCode, Route: r.Route, Title: r.Title, Message: r.Message,
+			IsTimeout: r.IsTimeout == 1, IsCancellation: r.IsCancellation == 1, IsRetry: r.IsRetry == 1})
+		r.Predicted, r.Rule = string(res.Category), res.Rule
+		rows = append(rows, r)
+	}
+	if *labelOut != "" {
+		out, err := os.Create(*labelOut)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer out.Close()
+		enc := json.NewEncoder(out)
+		for _, r := range rows {
+			_ = enc.Encode(r)
+		}
+		log.Printf("wrote %d rows to %s; fill in \"expected\" then re-run without -label-out", len(rows), *labelOut)
+		return
+	}
+	scored, correct := 0, 0
+	confusion := map[[2]string]int{}
+	for _, r := range rows {
+		if r.Expected == "" {
+			continue
+		}
+		scored++
+		if r.Expected == r.Predicted {
+			correct++
+		} else {
+			confusion[[2]string{r.Expected, r.Predicted}]++
+			fmt.Printf("miss  expected=%-24s got=%-24s rule=%-18s %s %d %s %.80s\n", r.Expected, r.Predicted, r.Rule, r.Service, r.StatusCode, r.Route, r.Message)
+		}
+	}
+	if scored == 0 {
+		log.Fatal("no labelled rows (set \"expected\")")
+	}
+	fmt.Printf("\naccuracy %d/%d = %.0f%%\n", correct, scored, 100*float64(correct)/float64(scored))
+	keys := make([][2]string, 0, len(confusion))
+	for k := range confusion {
+		keys = append(keys, k)
+	}
+	sort.Slice(keys, func(i, j int) bool { return confusion[keys[i]] > confusion[keys[j]] })
+	for _, k := range keys {
+		fmt.Printf("  %-24s -> %-24s x%d\n", k[0], k[1], confusion[k])
+	}
+}