go-otel/cmd/spike-alerts/main.go

// spike-alerts replays the deterministic spike rule over a bounded window of
// normalized_events and, optionally, asks the Needle sidecar what kind of
// failure each top pattern is. It only reads ClickHouse.
package main

import (
	"bytes"
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/ClickHouse/clickhouse-go/v2"

	"git.tornberg.me/go-otel/internal/alerts"
	"git.tornberg.me/go-otel/internal/features"
	"git.tornberg.me/go-otel/internal/triage"
	"git.tornberg.me/go-otel/internal/uaclass"
)

func main() {
	addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port")
	database := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "ClickHouse database")
	user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "ClickHouse user")
	password := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "ClickHouse password")
	since := flag.String("since", "", "inclusive UTC RFC3339 start of the range to replay")
	until := flag.String("until", "", "exclusive UTC RFC3339 end of the range")
	window := flag.Duration("window", 5*time.Minute, "observation window")
	step := flag.Duration("step", 5*time.Minute, "replay step")
	baselineWindows := flag.Int("baseline-windows", 36, "preceding windows used as baseline")
	minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert")
	multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline")
	cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles")
	needle := flag.String("needle", "", "Needle sidecar URL (dormant: it is not accurate enough to gate alerts); empty skips it")
	showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)")
	asJSON := flag.Bool("json", false, "emit JSON lines instead of text")
	flag.Parse()
	if *since == "" || *until == "" {
		log.Fatal("-since and -until are required")
	}
	start, err := time.Parse(time.RFC3339, *since)
	if err != nil {
		log.Fatal(err)
	}
	end, err := time.Parse(time.RFC3339, *until)
	if err != nil || !start.Before(end) {
		log.Fatal("invalid range")
	}
	cfg := features.SpikeConfig{Window: *window, BaselineWindows: *baselineWindows, Multiple: *multiple, MinExcess: *minExcess}

	conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{*addr}, Auth: clickhouse.Auth{Database: *database, Username: *user, Password: *password}})
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
	defer cancel()
	events, err := alerts.LoadErrorEvents(ctx, conn, *database, start, end)
	if err != nil {
		log.Fatal(err)
	}
	cfg.DataSince = alerts.DataSince(ctx, conn, *database)
	log.Printf("loaded %d error events", len(events))

	cache := map[string]classification{}
	first := start.Add(time.Duration(*baselineWindows) * *window)
	suppress := features.NewSuppressor(*cooldown)
	muted := map[triage.Category]int{}
	fired := 0
	for at := first.Add(*window); !at.After(end); at = at.Add(*step) {
		for _, spike := range suppress.Filter(alerts.FilterActionable(features.DetectSpikes(events, at, cfg), *showMuted, muted)) {
			fired++
			var cls []classification
			for _, p := range spike.Patterns {
				c := classification{}
				if *needle != "" {
					if hit, ok := cache[p.Signature]; ok {
						c = hit
					} else {
						c = classify(*needle, needleText(p.Example))
						cache[p.Signature] = c
					}
				}
				cls = append(cls, c)
			}
			if *asJSON {
				_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike, "classifications": cls})
				continue
			}
			printSpike(spike, cls)
		}
	}
	for c, n := range muted {
		log.Printf("muted %d pattern-window(s) as %s", n, c)
	}
	log.Printf("%d alert(s) between %s and %s", fired, first.Format(time.RFC3339), end.Format(time.RFC3339))
}

type classification struct {
	Action     string  `json:"action"`
	Confidence float64 `json:"confidence"`
	Cached     bool    `json:"cached"`
	Err        string  `json:"error,omitempty"`
}

func classify(base, text string) classification {
	body, _ := json.Marshal(map[string]string{"text": text})
	resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body))
	if err != nil {
		return classification{Action: "?", Err: err.Error()}
	}
	defer resp.Body.Close()
	var out classification
	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
		return classification{Action: "?", Err: err.Error()}
	}
	return out
}

func needleText(e features.Event) string {
	parts := []string{"Service: " + e.Service, "Kind: " + e.Kind}
	if e.SpanName != "" {
		parts = append(parts, "Title: "+e.SpanName)
	}
	if e.Message != "" {
		parts = append(parts, "Message: "+e.Message)
	}
	if e.StatusCode > 0 {
		parts = append(parts, fmt.Sprintf("Status: %d", e.StatusCode))
	}
	if e.Route != "" {
		parts = append(parts, "Route: "+e.Route)
	}
	return strings.Join(parts, " | ")
}

func printSpike(s features.Spike, cls []classification) {
	fmt.Printf("\n%s  %s  errors=%d (baseline %.1f/window, excess %.0f)\n", s.WindowEnd.Format("01-02 15:04"), s.Service, s.ErrorCount, s.BaselineMean, s.Excess)
	for i, p := range s.Patterns {
		tag := "rising"
		if p.Novel {
			tag = "NEW"
		}
		what := string(alerts.TriageOf(p.Example).Category)
		if cls[i].Action != "" {
			what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence)
		}
		clients := ""
		if p.Clients > 0 {
			clients = fmt.Sprintf(" [%d client%s]", p.Clients, map[bool]string{true: "", false: "s"}[p.Clients == 1])
		}
		if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) {
			clients += " [" + p.Example.UAClass + "]"
		}
		fmt.Printf("  %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example), clients)
	}
}

func chTime(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05.000000000") }
func envOr(k, f string) string {
	if v := os.Getenv(k); v != "" {
		return v
	}
	return f
}