// spike-alerts replays the deterministic spike rule over a bounded window of // normalized_events. It only reads ClickHouse. package main import ( "context" "encoding/json" "flag" "fmt" "log" "os" "strings" "time" "github.com/ClickHouse/clickhouse-go/v2" "git.tornberg.me/go-otel/internal/alerts" "git.tornberg.me/go-otel/internal/features" "git.tornberg.me/go-otel/internal/triage" "git.tornberg.me/go-otel/internal/uaclass" ) func main() { addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port") database := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "ClickHouse database") user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "ClickHouse user") password := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "ClickHouse password") since := flag.String("since", "", "inclusive UTC RFC3339 start of the range to replay") until := flag.String("until", "", "exclusive UTC RFC3339 end of the range") window := flag.Duration("window", 5*time.Minute, "observation window") step := flag.Duration("step", 5*time.Minute, "replay step") baselineWindows := flag.Int("baseline-windows", 36, "preceding windows used as baseline") minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert") multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline") cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles") showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)") asJSON := flag.Bool("json", false, "emit JSON lines instead of text") flag.Parse() if *since == "" || *until == "" { log.Fatal("-since and -until are required") } start, err := time.Parse(time.RFC3339, *since) if err != nil { log.Fatal(err) } end, err := time.Parse(time.RFC3339, *until) if err != nil || !start.Before(end) { log.Fatal("invalid range") } cfg := features.SpikeConfig{Window: *window, BaselineWindows: *baselineWindows, Multiple: *multiple, MinExcess: *minExcess} conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{*addr}, Auth: clickhouse.Auth{Database: *database, Username: *user, Password: *password}}) if err != nil { log.Fatal(err) } defer conn.Close() ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() events, err := alerts.LoadErrorEvents(ctx, conn, *database, start, end) if err != nil { log.Fatal(err) } cfg.DataSince = alerts.DataSince(ctx, conn, *database) features.MarkBadActors(events, features.BadActorConfig{}) badActors := map[string]int{} for _, e := range events { if e.BadActor != "" { badActors[e.BadActor]++ } } log.Printf("bad-actor events (ignored for alerts, kept as data): %v of %d", badActors, len(events)) log.Printf("loaded %d error events", len(events)) first := start.Add(time.Duration(*baselineWindows) * *window) suppress := features.NewSuppressor(*cooldown) muted := map[triage.Category]int{} fired := 0 for at := first.Add(*window); !at.After(end); at = at.Add(*step) { detected := alerts.DetectAlerts(events, at, cfg) // exactly what the server alerts on if *showMuted { detected = alerts.FilterActionable(features.DetectSpikes(events, at, cfg), true, muted) } for _, spike := range suppress.Filter(detected) { fired++ if *asJSON { _ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike}) continue } printSpike(spike) } } for c, n := range muted { log.Printf("muted %d pattern-window(s) as %s", n, c) } log.Printf("%d alert(s) between %s and %s", fired, first.Format(time.RFC3339), end.Format(time.RFC3339)) } func eventText(e features.Event) string { parts := []string{"Service: " + e.Service, "Kind: " + e.Kind} if e.SpanName != "" { parts = append(parts, "Title: "+e.SpanName) } if e.Message != "" { parts = append(parts, "Message: "+e.Message) } if e.StatusCode > 0 { parts = append(parts, fmt.Sprintf("Status: %d", e.StatusCode)) } if e.Route != "" { parts = append(parts, "Route: "+e.Route) } return strings.Join(parts, " | ") } func printSpike(s features.Spike) { fmt.Printf("\n%s %s errors=%d (baseline %.1f/window, excess %.0f)\n", s.WindowEnd.Format("01-02 15:04"), s.Service, s.ErrorCount, s.BaselineMean, s.Excess) for _, p := range s.Patterns { tag := "rising" if p.Novel { tag = "NEW" } what := string(alerts.TriageOf(p.Example).Category) clients := "" if p.Clients > 0 { clients = fmt.Sprintf(" [%d client%s]", p.Clients, map[bool]string{true: "", false: "s"}[p.Clients == 1]) } if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) { clients += " [" + p.Example.UAClass + "]" } fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, eventText(p.Example), clients) } } func chTime(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05.000000000") } func envOr(k, f string) string { if v := os.Getenv(k); v != "" { return v } return f }