// spike-alerts replays the deterministic spike rule over a bounded window of // normalized_events and, optionally, asks the Needle sidecar what kind of // failure each top pattern is. It only reads ClickHouse. package main import ( "bytes" "context" "encoding/json" "flag" "fmt" "log" "net/http" "os" "strings" "time" "github.com/ClickHouse/clickhouse-go/v2" "git.tornberg.me/go-otel/internal/features" "git.tornberg.me/go-otel/internal/triage" ) func main() { addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port") database := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "ClickHouse database") user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "ClickHouse user") password := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "ClickHouse password") since := flag.String("since", "", "inclusive UTC RFC3339 start of the range to replay") until := flag.String("until", "", "exclusive UTC RFC3339 end of the range") window := flag.Duration("window", 5*time.Minute, "observation window") step := flag.Duration("step", 5*time.Minute, "replay step") baselineWindows := flag.Int("baseline-windows", 36, "preceding windows used as baseline") minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert") multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline") cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles") needle := flag.String("needle", "", "Needle sidecar URL (dormant: it is not accurate enough to gate alerts); empty skips it") showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)") asJSON := flag.Bool("json", false, "emit JSON lines instead of text") flag.Parse() if *since == "" || *until == "" { log.Fatal("-since and -until are required") } start, err := time.Parse(time.RFC3339, *since) if err != nil { log.Fatal(err) } end, err := time.Parse(time.RFC3339, *until) if err != nil || !start.Before(end) { log.Fatal("invalid range") } cfg := features.SpikeConfig{Window: *window, BaselineWindows: *baselineWindows, Multiple: *multiple, MinExcess: *minExcess} events := load(*addr, *database, *user, *password, start, end) cfg.DataSince = dataSince(*addr, *database, *user, *password) log.Printf("loaded %d error events", len(events)) cache := map[string]classification{} first := start.Add(time.Duration(*baselineWindows) * *window) suppress := features.NewSuppressor(*cooldown) muted := map[triage.Category]int{} alerts := 0 for at := first.Add(*window); !at.After(end); at = at.Add(*step) { for _, spike := range suppress.Filter(triageFilter(features.DetectSpikes(events, at, cfg), *showMuted, muted)) { alerts++ var cls []classification for _, p := range spike.Patterns { c := classification{} if *needle != "" { if hit, ok := cache[p.Signature]; ok { c = hit } else { c = classify(*needle, needleText(p.Example)) cache[p.Signature] = c } } cls = append(cls, c) } if *asJSON { _ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike, "classifications": cls}) continue } printSpike(spike, cls) } } for c, n := range muted { log.Printf("muted %d pattern-window(s) as %s", n, c) } log.Printf("%d alert(s) between %s and %s", alerts, first.Format(time.RFC3339), end.Format(time.RFC3339)) } type classification struct { Action string `json:"action"` Confidence float64 `json:"confidence"` Cached bool `json:"cached"` Err string `json:"error,omitempty"` } func triageOf(e features.Event) triage.Result { return triage.Classify(triage.Input{Service: e.Service, Kind: e.Kind, Status: e.StatusCode, Route: e.Route, Title: e.SpanName, Message: e.Message, IsTimeout: e.IsTimeout, IsCancellation: e.IsCancellation, IsRetry: e.IsRetry}) } // triageFilter keeps only actionable patterns (unless showMuted), recomputing // each spike's excess so a probe storm cannot page anyone by itself. func triageFilter(spikes []features.Spike, showMuted bool, muted map[triage.Category]int) []features.Spike { var out []features.Spike for _, sp := range spikes { var kept []features.PatternDelta sp.Excess = 0 for _, p := range sp.Patterns { c := triageOf(p.Example).Category if !showMuted && !c.Actionable() { muted[c]++ continue } kept = append(kept, p) sp.Excess += p.Excess } if len(kept) > 0 { sp.Patterns = kept out = append(out, sp) } } return out } func classify(base, text string) classification { body, _ := json.Marshal(map[string]string{"text": text}) resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body)) if err != nil { return classification{Action: "?", Err: err.Error()} } defer resp.Body.Close() var out classification if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { return classification{Action: "?", Err: err.Error()} } return out } func needleText(e features.Event) string { parts := []string{"Service: " + e.Service, "Kind: " + e.Kind} if e.SpanName != "" { parts = append(parts, "Title: "+e.SpanName) } if e.Message != "" { parts = append(parts, "Message: "+e.Message) } if e.StatusCode > 0 { parts = append(parts, fmt.Sprintf("Status: %d", e.StatusCode)) } if e.Route != "" { parts = append(parts, "Route: "+e.Route) } return strings.Join(parts, " | ") } func printSpike(s features.Spike, cls []classification) { fmt.Printf("\n%s %s errors=%d (baseline %.1f/window, excess %.0f)\n", s.WindowEnd.Format("01-02 15:04"), s.Service, s.ErrorCount, s.BaselineMean, s.Excess) for i, p := range s.Patterns { tag := "rising" if p.Novel { tag = "NEW" } what := string(triageOf(p.Example).Category) if cls[i].Action != "" { what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence) } fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example)) } } func load(addr, database, user, password string, start, end time.Time) []features.Event { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{addr}, Auth: clickhouse.Auth{Database: database, Username: user, Password: password}}) if err != nil { log.Fatal(err) } defer conn.Close() // Errors only, and only columns present in every deployed schema version. rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT timestamp, service, release, kind, source, status_code, span_name, route, exception_type, message_normalized, pattern_signature, is_timeout, is_cancellation, is_retry FROM %s.normalized_events FINAL WHERE is_error = 1 AND timestamp >= toDateTime64('%s', 9, 'UTC') AND timestamp < toDateTime64('%s', 9, 'UTC') ORDER BY timestamp LIMIT 2000000`, database, chTime(start), chTime(end))) if err != nil { log.Fatal(err) } defer rows.Close() var out []features.Event for rows.Next() { e := features.Event{IsError: true} var status uint16 var timeout, cancel, retry uint8 if err := rows.Scan(&e.Timestamp, &e.Service, &e.Release, &e.Kind, &e.Source, &status, &e.SpanName, &e.Route, &e.ExceptionType, &e.Message, &e.PatternSignature, &timeout, &cancel, &retry); err != nil { log.Fatal(err) } e.StatusCode = int(status) e.IsTimeout, e.IsCancellation, e.IsRetry = timeout == 1, cancel == 1, retry == 1 out = append(out, e) } if err := rows.Err(); err != nil { log.Fatal(err) } return out } func chTime(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05.000000000") } func envOr(k, f string) string { if v := os.Getenv(k); v != "" { return v } return f } // dataSince returns the earliest normalized row, used as the warm-up guard. func dataSince(addr, database, user, password string) time.Time { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{addr}, Auth: clickhouse.Auth{Database: database, Username: user, Password: password}}) if err != nil { log.Fatal(err) } defer conn.Close() var t time.Time if err := conn.QueryRow(ctx, fmt.Sprintf(`SELECT min(timestamp) FROM %s.normalized_events`, database)).Scan(&t); err != nil { log.Printf("warm-up guard disabled: %v", err) return time.Time{} } if t.Year() < 2000 { // empty table return time.Time{} } return t }