183 lines
6.4 KiB
Go
183 lines
6.4 KiB
Go
// spike-alerts replays the deterministic spike rule over a bounded window of
|
|
// normalized_events and, optionally, asks the Needle sidecar what kind of
|
|
// failure each top pattern is. It only reads ClickHouse.
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/ClickHouse/clickhouse-go/v2"
|
|
|
|
"git.tornberg.me/go-otel/internal/alerts"
|
|
"git.tornberg.me/go-otel/internal/features"
|
|
"git.tornberg.me/go-otel/internal/triage"
|
|
"git.tornberg.me/go-otel/internal/uaclass"
|
|
)
|
|
|
|
func main() {
|
|
addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port")
|
|
database := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "ClickHouse database")
|
|
user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "ClickHouse user")
|
|
password := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "ClickHouse password")
|
|
since := flag.String("since", "", "inclusive UTC RFC3339 start of the range to replay")
|
|
until := flag.String("until", "", "exclusive UTC RFC3339 end of the range")
|
|
window := flag.Duration("window", 5*time.Minute, "observation window")
|
|
step := flag.Duration("step", 5*time.Minute, "replay step")
|
|
baselineWindows := flag.Int("baseline-windows", 36, "preceding windows used as baseline")
|
|
minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert")
|
|
multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline")
|
|
cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles")
|
|
needle := flag.String("needle", "", "Needle sidecar URL (dormant: it is not accurate enough to gate alerts); empty skips it")
|
|
showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)")
|
|
asJSON := flag.Bool("json", false, "emit JSON lines instead of text")
|
|
flag.Parse()
|
|
if *since == "" || *until == "" {
|
|
log.Fatal("-since and -until are required")
|
|
}
|
|
start, err := time.Parse(time.RFC3339, *since)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
end, err := time.Parse(time.RFC3339, *until)
|
|
if err != nil || !start.Before(end) {
|
|
log.Fatal("invalid range")
|
|
}
|
|
cfg := features.SpikeConfig{Window: *window, BaselineWindows: *baselineWindows, Multiple: *multiple, MinExcess: *minExcess}
|
|
|
|
conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{*addr}, Auth: clickhouse.Auth{Database: *database, Username: *user, Password: *password}})
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
defer conn.Close()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
|
defer cancel()
|
|
events, err := alerts.LoadErrorEvents(ctx, conn, *database, start, end)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
cfg.DataSince = alerts.DataSince(ctx, conn, *database)
|
|
features.MarkBadActors(events, features.BadActorConfig{})
|
|
badActors := map[string]int{}
|
|
for _, e := range events {
|
|
if e.BadActor != "" {
|
|
badActors[e.BadActor]++
|
|
}
|
|
}
|
|
log.Printf("bad-actor events (ignored for alerts, kept as data): %v of %d", badActors, len(events))
|
|
log.Printf("loaded %d error events", len(events))
|
|
|
|
cache := map[string]classification{}
|
|
first := start.Add(time.Duration(*baselineWindows) * *window)
|
|
suppress := features.NewSuppressor(*cooldown)
|
|
muted := map[triage.Category]int{}
|
|
fired := 0
|
|
for at := first.Add(*window); !at.After(end); at = at.Add(*step) {
|
|
detected := alerts.DetectAlerts(events, at, cfg) // exactly what the server alerts on
|
|
if *showMuted {
|
|
detected = alerts.FilterActionable(features.DetectSpikes(events, at, cfg), true, muted)
|
|
}
|
|
for _, spike := range suppress.Filter(detected) {
|
|
fired++
|
|
var cls []classification
|
|
for _, p := range spike.Patterns {
|
|
c := classification{}
|
|
if *needle != "" {
|
|
if hit, ok := cache[p.Signature]; ok {
|
|
c = hit
|
|
} else {
|
|
c = classify(*needle, needleText(p.Example))
|
|
cache[p.Signature] = c
|
|
}
|
|
}
|
|
cls = append(cls, c)
|
|
}
|
|
if *asJSON {
|
|
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike, "classifications": cls})
|
|
continue
|
|
}
|
|
printSpike(spike, cls)
|
|
}
|
|
}
|
|
for c, n := range muted {
|
|
log.Printf("muted %d pattern-window(s) as %s", n, c)
|
|
}
|
|
log.Printf("%d alert(s) between %s and %s", fired, first.Format(time.RFC3339), end.Format(time.RFC3339))
|
|
}
|
|
|
|
type classification struct {
|
|
Action string `json:"action"`
|
|
Confidence float64 `json:"confidence"`
|
|
Cached bool `json:"cached"`
|
|
Err string `json:"error,omitempty"`
|
|
}
|
|
|
|
func classify(base, text string) classification {
|
|
body, _ := json.Marshal(map[string]string{"text": text})
|
|
resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body))
|
|
if err != nil {
|
|
return classification{Action: "?", Err: err.Error()}
|
|
}
|
|
defer resp.Body.Close()
|
|
var out classification
|
|
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
|
return classification{Action: "?", Err: err.Error()}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func needleText(e features.Event) string {
|
|
parts := []string{"Service: " + e.Service, "Kind: " + e.Kind}
|
|
if e.SpanName != "" {
|
|
parts = append(parts, "Title: "+e.SpanName)
|
|
}
|
|
if e.Message != "" {
|
|
parts = append(parts, "Message: "+e.Message)
|
|
}
|
|
if e.StatusCode > 0 {
|
|
parts = append(parts, fmt.Sprintf("Status: %d", e.StatusCode))
|
|
}
|
|
if e.Route != "" {
|
|
parts = append(parts, "Route: "+e.Route)
|
|
}
|
|
return strings.Join(parts, " | ")
|
|
}
|
|
|
|
func printSpike(s features.Spike, cls []classification) {
|
|
fmt.Printf("\n%s %s errors=%d (baseline %.1f/window, excess %.0f)\n", s.WindowEnd.Format("01-02 15:04"), s.Service, s.ErrorCount, s.BaselineMean, s.Excess)
|
|
for i, p := range s.Patterns {
|
|
tag := "rising"
|
|
if p.Novel {
|
|
tag = "NEW"
|
|
}
|
|
what := string(alerts.TriageOf(p.Example).Category)
|
|
if cls[i].Action != "" {
|
|
what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence)
|
|
}
|
|
clients := ""
|
|
if p.Clients > 0 {
|
|
clients = fmt.Sprintf(" [%d client%s]", p.Clients, map[bool]string{true: "", false: "s"}[p.Clients == 1])
|
|
}
|
|
if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) {
|
|
clients += " [" + p.Example.UAClass + "]"
|
|
}
|
|
fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example), clients)
|
|
}
|
|
}
|
|
|
|
func chTime(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05.000000000") }
|
|
func envOr(k, f string) string {
|
|
if v := os.Getenv(k); v != "" {
|
|
return v
|
|
}
|
|
return f
|
|
}
|