99 lines
2.9 KiB
Go
99 lines
2.9 KiB
Go
// triage-eval applies the rules classifier to a JSONL file of patterns. Rows
|
|
// with an "expected" category are scored; the rest are written out with the
|
|
// prediction so they can be reviewed and labelled by hand.
|
|
//
|
|
// triage-eval -input holdout.jsonl # score labelled rows
|
|
// triage-eval -input raw.jsonl -label-out holdout.jsonl # prepare for labelling
|
|
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"sort"
|
|
|
|
"git.tornberg.me/go-otel/internal/triage"
|
|
)
|
|
|
|
type row struct {
|
|
Service string `json:"service"`
|
|
Kind string `json:"kind"`
|
|
StatusCode int `json:"status_code"`
|
|
Route string `json:"route"`
|
|
Title string `json:"title"`
|
|
Message string `json:"message"`
|
|
IsTimeout uint8 `json:"is_timeout"`
|
|
IsCancellation uint8 `json:"is_cancellation"`
|
|
IsRetry uint8 `json:"is_retry"`
|
|
Count uint64 `json:"n"`
|
|
Expected string `json:"expected"`
|
|
Predicted string `json:"predicted"`
|
|
Rule string `json:"rule"`
|
|
}
|
|
|
|
func main() {
|
|
input := flag.String("input", "", "JSONL input")
|
|
labelOut := flag.String("label-out", "", "write rows with predictions here for hand labelling")
|
|
flag.Parse()
|
|
f, err := os.Open(*input)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
defer f.Close()
|
|
var rows []row
|
|
sc := bufio.NewScanner(f)
|
|
sc.Buffer(make([]byte, 1<<20), 1<<20)
|
|
for sc.Scan() {
|
|
var r row
|
|
if err := json.Unmarshal(sc.Bytes(), &r); err != nil {
|
|
log.Fatalf("bad row: %v", err)
|
|
}
|
|
res := triage.Classify(triage.Input{Service: r.Service, Kind: r.Kind, Status: r.StatusCode, Route: r.Route, Title: r.Title, Message: r.Message,
|
|
IsTimeout: r.IsTimeout == 1, IsCancellation: r.IsCancellation == 1, IsRetry: r.IsRetry == 1})
|
|
r.Predicted, r.Rule = string(res.Category), res.Rule
|
|
rows = append(rows, r)
|
|
}
|
|
if *labelOut != "" {
|
|
out, err := os.Create(*labelOut)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
defer out.Close()
|
|
enc := json.NewEncoder(out)
|
|
for _, r := range rows {
|
|
_ = enc.Encode(r)
|
|
}
|
|
log.Printf("wrote %d rows to %s; fill in \"expected\" then re-run without -label-out", len(rows), *labelOut)
|
|
return
|
|
}
|
|
scored, correct := 0, 0
|
|
confusion := map[[2]string]int{}
|
|
for _, r := range rows {
|
|
if r.Expected == "" {
|
|
continue
|
|
}
|
|
scored++
|
|
if r.Expected == r.Predicted {
|
|
correct++
|
|
} else {
|
|
confusion[[2]string{r.Expected, r.Predicted}]++
|
|
fmt.Printf("miss expected=%-24s got=%-24s rule=%-18s %s %d %s %.80s\n", r.Expected, r.Predicted, r.Rule, r.Service, r.StatusCode, r.Route, r.Message)
|
|
}
|
|
}
|
|
if scored == 0 {
|
|
log.Fatal("no labelled rows (set \"expected\")")
|
|
}
|
|
fmt.Printf("\naccuracy %d/%d = %.0f%%\n", correct, scored, 100*float64(correct)/float64(scored))
|
|
keys := make([][2]string, 0, len(confusion))
|
|
for k := range confusion {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Slice(keys, func(i, j int) bool { return confusion[keys[i]] > confusion[keys[j]] })
|
|
for _, k := range keys {
|
|
fmt.Printf(" %-24s -> %-24s x%d\n", k[0], k[1], confusion[k])
|
|
}
|
|
}
|