more ml

cmd/spike-alerts/main.go

@@ -0,0 +1,177 @@
+// spike-alerts replays the deterministic spike rule over a bounded window of
+// normalized_events and, optionally, asks the Needle sidecar what kind of
+// failure each top pattern is. It only reads ClickHouse.
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+
+	"git.tornberg.me/go-otel/internal/features"
+)
+
+func main() {
+	addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port")
+	database := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "ClickHouse database")
+	user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "ClickHouse user")
+	password := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "ClickHouse password")
+	since := flag.String("since", "", "inclusive UTC RFC3339 start of the range to replay")
+	until := flag.String("until", "", "exclusive UTC RFC3339 end of the range")
+	window := flag.Duration("window", 5*time.Minute, "observation window")
+	step := flag.Duration("step", 5*time.Minute, "replay step")
+	baselineWindows := flag.Int("baseline-windows", 36, "preceding windows used as baseline")
+	minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert")
+	multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline")
+	cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles")
+	needle := flag.String("needle", os.Getenv("OTEL_NEEDLE_ADDR"), "Needle sidecar URL; empty skips classification")
+	asJSON := flag.Bool("json", false, "emit JSON lines instead of text")
+	flag.Parse()
+	if *since == "" || *until == "" {
+		log.Fatal("-since and -until are required")
+	}
+	start, err := time.Parse(time.RFC3339, *since)
+	if err != nil {
+		log.Fatal(err)
+	}
+	end, err := time.Parse(time.RFC3339, *until)
+	if err != nil || !start.Before(end) {
+		log.Fatal("invalid range")
+	}
+	cfg := features.SpikeConfig{Window: *window, BaselineWindows: *baselineWindows, Multiple: *multiple, MinExcess: *minExcess}
+
+	events := load(*addr, *database, *user, *password, start, end)
+	log.Printf("loaded %d error events", len(events))
+
+	cache := map[string]classification{}
+	first := start.Add(time.Duration(*baselineWindows) * *window)
+	suppress := features.NewSuppressor(*cooldown)
+	alerts := 0
+	for at := first.Add(*window); !at.After(end); at = at.Add(*step) {
+		for _, spike := range suppress.Filter(features.DetectSpikes(events, at, cfg)) {
+			alerts++
+			var cls []classification
+			for _, p := range spike.Patterns {
+				c := classification{}
+				if *needle != "" {
+					if hit, ok := cache[p.Signature]; ok {
+						c = hit
+					} else {
+						c = classify(*needle, needleText(p.Example))
+						cache[p.Signature] = c
+					}
+				}
+				cls = append(cls, c)
+			}
+			if *asJSON {
+				_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike, "classifications": cls})
+				continue
+			}
+			printSpike(spike, cls)
+		}
+	}
+	log.Printf("%d alert(s) between %s and %s", alerts, first.Format(time.RFC3339), end.Format(time.RFC3339))
+}
+
+type classification struct {
+	Action     string  `json:"action"`
+	Confidence float64 `json:"confidence"`
+	Cached     bool    `json:"cached"`
+	Err        string  `json:"error,omitempty"`
+}
+
+func classify(base, text string) classification {
+	body, _ := json.Marshal(map[string]string{"text": text})
+	resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body))
+	if err != nil {
+		return classification{Action: "?", Err: err.Error()}
+	}
+	defer resp.Body.Close()
+	var out classification
+	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
+		return classification{Action: "?", Err: err.Error()}
+	}
+	return out
+}
+
+func needleText(e features.Event) string {
+	parts := []string{"Service: " + e.Service, "Kind: " + e.Kind}
+	if e.SpanName != "" {
+		parts = append(parts, "Title: "+e.SpanName)
+	}
+	if e.Message != "" {
+		parts = append(parts, "Message: "+e.Message)
+	}
+	if e.StatusCode > 0 {
+		parts = append(parts, fmt.Sprintf("Status: %d", e.StatusCode))
+	}
+	if e.Route != "" {
+		parts = append(parts, "Route: "+e.Route)
+	}
+	return strings.Join(parts, " | ")
+}
+
+func printSpike(s features.Spike, cls []classification) {
+	fmt.Printf("\n%s  %s  errors=%d (baseline %.1f/window, excess %.0f)\n", s.WindowEnd.Format("01-02 15:04"), s.Service, s.ErrorCount, s.BaselineMean, s.Excess)
+	for i, p := range s.Patterns {
+		tag := "rising"
+		if p.Novel {
+			tag = "NEW"
+		}
+		what := "-"
+		if cls[i].Action != "" {
+			what = fmt.Sprintf("%s (%.2f)", cls[i].Action, cls[i].Confidence)
+		}
+		fmt.Printf("  %-6s x%-4d base %-6.1f %-24s %s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example))
+	}
+}
+
+func load(addr, database, user, password string, start, end time.Time) []features.Event {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+	conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{addr}, Auth: clickhouse.Auth{Database: database, Username: user, Password: password}})
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer conn.Close()
+	// Errors only, and only columns present in every deployed schema version.
+	rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT timestamp, service, release, kind, source, status_code, span_name, route, exception_type, message_normalized, pattern_signature
+		FROM %s.normalized_events FINAL
+		WHERE is_error = 1 AND timestamp >= toDateTime64('%s', 9, 'UTC') AND timestamp < toDateTime64('%s', 9, 'UTC')
+		ORDER BY timestamp LIMIT 2000000`, database, chTime(start), chTime(end)))
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+	var out []features.Event
+	for rows.Next() {
+		e := features.Event{IsError: true}
+		var status uint16
+		if err := rows.Scan(&e.Timestamp, &e.Service, &e.Release, &e.Kind, &e.Source, &status, &e.SpanName, &e.Route, &e.ExceptionType, &e.Message, &e.PatternSignature); err != nil {
+			log.Fatal(err)
+		}
+		e.StatusCode = int(status)
+		out = append(out, e)
+	}
+	if err := rows.Err(); err != nil {
+		log.Fatal(err)
+	}
+	return out
+}
+
+func chTime(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05.000000000") }
+func envOr(k, f string) string {
+	if v := os.Getenv(k); v != "" {
+		return v
+	}
+	return f
+}