all the changes

cmd/spike-alerts/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/ClickHouse/clickhouse-go/v2"
 
 	"git.tornberg.me/go-otel/internal/features"
+	"git.tornberg.me/go-otel/internal/triage"
 )
 
 func main() {
@@ -33,7 +34,8 @@ func main() {
 	minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert")
 	multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline")
 	cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles")
-	needle := flag.String("needle", os.Getenv("OTEL_NEEDLE_ADDR"), "Needle sidecar URL; empty skips classification")
+	needle := flag.String("needle", "", "Needle sidecar URL (dormant: it is not accurate enough to gate alerts); empty skips it")
+	showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)")
 	asJSON := flag.Bool("json", false, "emit JSON lines instead of text")
 	flag.Parse()
 	if *since == "" || *until == "" {
@@ -55,9 +57,10 @@ func main() {
 	cache := map[string]classification{}
 	first := start.Add(time.Duration(*baselineWindows) * *window)
 	suppress := features.NewSuppressor(*cooldown)
+	muted := map[triage.Category]int{}
 	alerts := 0
 	for at := first.Add(*window); !at.After(end); at = at.Add(*step) {
-		for _, spike := range suppress.Filter(features.DetectSpikes(events, at, cfg)) {
+		for _, spike := range suppress.Filter(triageFilter(features.DetectSpikes(events, at, cfg), *showMuted, muted)) {
 			alerts++
 			var cls []classification
 			for _, p := range spike.Patterns {
@@ -79,6 +82,9 @@ func main() {
 			printSpike(spike, cls)
 		}
 	}
+	for c, n := range muted {
+		log.Printf("muted %d pattern-window(s) as %s", n, c)
+	}
 	log.Printf("%d alert(s) between %s and %s", alerts, first.Format(time.RFC3339), end.Format(time.RFC3339))
 }
 
@@ -89,6 +95,35 @@ type classification struct {
 	Err        string  `json:"error,omitempty"`
 }
 
+func triageOf(e features.Event) triage.Result {
+	return triage.Classify(triage.Input{Service: e.Service, Kind: e.Kind, Status: e.StatusCode, Route: e.Route, Title: e.SpanName, Message: e.Message,
+		IsTimeout: e.IsTimeout, IsCancellation: e.IsCancellation, IsRetry: e.IsRetry})
+}
+
+// triageFilter keeps only actionable patterns (unless showMuted), recomputing
+// each spike's excess so a probe storm cannot page anyone by itself.
+func triageFilter(spikes []features.Spike, showMuted bool, muted map[triage.Category]int) []features.Spike {
+	var out []features.Spike
+	for _, sp := range spikes {
+		var kept []features.PatternDelta
+		sp.Excess = 0
+		for _, p := range sp.Patterns {
+			c := triageOf(p.Example).Category
+			if !showMuted && !c.Actionable() {
+				muted[c]++
+				continue
+			}
+			kept = append(kept, p)
+			sp.Excess += p.Excess
+		}
+		if len(kept) > 0 {
+			sp.Patterns = kept
+			out = append(out, sp)
+		}
+	}
+	return out
+}
+
 func classify(base, text string) classification {
 	body, _ := json.Marshal(map[string]string{"text": text})
 	resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body))
@@ -127,9 +162,9 @@ func printSpike(s features.Spike, cls []classification) {
 		if p.Novel {
 			tag = "NEW"
 		}
-		what := "-"
+		what := string(triageOf(p.Example).Category)
 		if cls[i].Action != "" {
-			what = fmt.Sprintf("%s (%.2f)", cls[i].Action, cls[i].Confidence)
+			what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence)
 		}
 		fmt.Printf("  %-6s x%-4d base %-6.1f %-24s %s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example))
 	}
@@ -144,7 +179,7 @@ func load(addr, database, user, password string, start, end time.Time) []feature
 	}
 	defer conn.Close()
 	// Errors only, and only columns present in every deployed schema version.
-	rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT timestamp, service, release, kind, source, status_code, span_name, route, exception_type, message_normalized, pattern_signature
+	rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT timestamp, service, release, kind, source, status_code, span_name, route, exception_type, message_normalized, pattern_signature, is_timeout, is_cancellation, is_retry
 		FROM %s.normalized_events FINAL
 		WHERE is_error = 1 AND timestamp >= toDateTime64('%s', 9, 'UTC') AND timestamp < toDateTime64('%s', 9, 'UTC')
 		ORDER BY timestamp LIMIT 2000000`, database, chTime(start), chTime(end)))
@@ -156,10 +191,12 @@ func load(addr, database, user, password string, start, end time.Time) []feature
 	for rows.Next() {
 		e := features.Event{IsError: true}
 		var status uint16
-		if err := rows.Scan(&e.Timestamp, &e.Service, &e.Release, &e.Kind, &e.Source, &status, &e.SpanName, &e.Route, &e.ExceptionType, &e.Message, &e.PatternSignature); err != nil {
+		var timeout, cancel, retry uint8
+		if err := rows.Scan(&e.Timestamp, &e.Service, &e.Release, &e.Kind, &e.Source, &status, &e.SpanName, &e.Route, &e.ExceptionType, &e.Message, &e.PatternSignature, &timeout, &cancel, &retry); err != nil {
 			log.Fatal(err)
 		}
 		e.StatusCode = int(status)
+		e.IsTimeout, e.IsCancellation, e.IsRetry = timeout == 1, cancel == 1, retry == 1
 		out = append(out, e)
 	}
 	if err := rows.Err(); err != nil {