215 lines
7.7 KiB
Go
215 lines
7.7 KiB
Go
// spike-alerts replays the deterministic spike rule over a bounded window of
|
|
// normalized_events and, optionally, asks the Needle sidecar what kind of
|
|
// failure each top pattern is. It only reads ClickHouse.
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/ClickHouse/clickhouse-go/v2"
|
|
|
|
"git.tornberg.me/go-otel/internal/features"
|
|
"git.tornberg.me/go-otel/internal/triage"
|
|
)
|
|
|
|
func main() {
|
|
addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port")
|
|
database := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "ClickHouse database")
|
|
user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "ClickHouse user")
|
|
password := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "ClickHouse password")
|
|
since := flag.String("since", "", "inclusive UTC RFC3339 start of the range to replay")
|
|
until := flag.String("until", "", "exclusive UTC RFC3339 end of the range")
|
|
window := flag.Duration("window", 5*time.Minute, "observation window")
|
|
step := flag.Duration("step", 5*time.Minute, "replay step")
|
|
baselineWindows := flag.Int("baseline-windows", 36, "preceding windows used as baseline")
|
|
minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert")
|
|
multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline")
|
|
cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles")
|
|
needle := flag.String("needle", "", "Needle sidecar URL (dormant: it is not accurate enough to gate alerts); empty skips it")
|
|
showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)")
|
|
asJSON := flag.Bool("json", false, "emit JSON lines instead of text")
|
|
flag.Parse()
|
|
if *since == "" || *until == "" {
|
|
log.Fatal("-since and -until are required")
|
|
}
|
|
start, err := time.Parse(time.RFC3339, *since)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
end, err := time.Parse(time.RFC3339, *until)
|
|
if err != nil || !start.Before(end) {
|
|
log.Fatal("invalid range")
|
|
}
|
|
cfg := features.SpikeConfig{Window: *window, BaselineWindows: *baselineWindows, Multiple: *multiple, MinExcess: *minExcess}
|
|
|
|
events := load(*addr, *database, *user, *password, start, end)
|
|
log.Printf("loaded %d error events", len(events))
|
|
|
|
cache := map[string]classification{}
|
|
first := start.Add(time.Duration(*baselineWindows) * *window)
|
|
suppress := features.NewSuppressor(*cooldown)
|
|
muted := map[triage.Category]int{}
|
|
alerts := 0
|
|
for at := first.Add(*window); !at.After(end); at = at.Add(*step) {
|
|
for _, spike := range suppress.Filter(triageFilter(features.DetectSpikes(events, at, cfg), *showMuted, muted)) {
|
|
alerts++
|
|
var cls []classification
|
|
for _, p := range spike.Patterns {
|
|
c := classification{}
|
|
if *needle != "" {
|
|
if hit, ok := cache[p.Signature]; ok {
|
|
c = hit
|
|
} else {
|
|
c = classify(*needle, needleText(p.Example))
|
|
cache[p.Signature] = c
|
|
}
|
|
}
|
|
cls = append(cls, c)
|
|
}
|
|
if *asJSON {
|
|
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike, "classifications": cls})
|
|
continue
|
|
}
|
|
printSpike(spike, cls)
|
|
}
|
|
}
|
|
for c, n := range muted {
|
|
log.Printf("muted %d pattern-window(s) as %s", n, c)
|
|
}
|
|
log.Printf("%d alert(s) between %s and %s", alerts, first.Format(time.RFC3339), end.Format(time.RFC3339))
|
|
}
|
|
|
|
type classification struct {
|
|
Action string `json:"action"`
|
|
Confidence float64 `json:"confidence"`
|
|
Cached bool `json:"cached"`
|
|
Err string `json:"error,omitempty"`
|
|
}
|
|
|
|
func triageOf(e features.Event) triage.Result {
|
|
return triage.Classify(triage.Input{Service: e.Service, Kind: e.Kind, Status: e.StatusCode, Route: e.Route, Title: e.SpanName, Message: e.Message,
|
|
IsTimeout: e.IsTimeout, IsCancellation: e.IsCancellation, IsRetry: e.IsRetry})
|
|
}
|
|
|
|
// triageFilter keeps only actionable patterns (unless showMuted), recomputing
|
|
// each spike's excess so a probe storm cannot page anyone by itself.
|
|
func triageFilter(spikes []features.Spike, showMuted bool, muted map[triage.Category]int) []features.Spike {
|
|
var out []features.Spike
|
|
for _, sp := range spikes {
|
|
var kept []features.PatternDelta
|
|
sp.Excess = 0
|
|
for _, p := range sp.Patterns {
|
|
c := triageOf(p.Example).Category
|
|
if !showMuted && !c.Actionable() {
|
|
muted[c]++
|
|
continue
|
|
}
|
|
kept = append(kept, p)
|
|
sp.Excess += p.Excess
|
|
}
|
|
if len(kept) > 0 {
|
|
sp.Patterns = kept
|
|
out = append(out, sp)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func classify(base, text string) classification {
|
|
body, _ := json.Marshal(map[string]string{"text": text})
|
|
resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body))
|
|
if err != nil {
|
|
return classification{Action: "?", Err: err.Error()}
|
|
}
|
|
defer resp.Body.Close()
|
|
var out classification
|
|
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
|
return classification{Action: "?", Err: err.Error()}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func needleText(e features.Event) string {
|
|
parts := []string{"Service: " + e.Service, "Kind: " + e.Kind}
|
|
if e.SpanName != "" {
|
|
parts = append(parts, "Title: "+e.SpanName)
|
|
}
|
|
if e.Message != "" {
|
|
parts = append(parts, "Message: "+e.Message)
|
|
}
|
|
if e.StatusCode > 0 {
|
|
parts = append(parts, fmt.Sprintf("Status: %d", e.StatusCode))
|
|
}
|
|
if e.Route != "" {
|
|
parts = append(parts, "Route: "+e.Route)
|
|
}
|
|
return strings.Join(parts, " | ")
|
|
}
|
|
|
|
func printSpike(s features.Spike, cls []classification) {
|
|
fmt.Printf("\n%s %s errors=%d (baseline %.1f/window, excess %.0f)\n", s.WindowEnd.Format("01-02 15:04"), s.Service, s.ErrorCount, s.BaselineMean, s.Excess)
|
|
for i, p := range s.Patterns {
|
|
tag := "rising"
|
|
if p.Novel {
|
|
tag = "NEW"
|
|
}
|
|
what := string(triageOf(p.Example).Category)
|
|
if cls[i].Action != "" {
|
|
what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence)
|
|
}
|
|
fmt.Printf(" %-6s x%-4d base %-6.1f %-24s %s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example))
|
|
}
|
|
}
|
|
|
|
func load(addr, database, user, password string, start, end time.Time) []features.Event {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
|
defer cancel()
|
|
conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{addr}, Auth: clickhouse.Auth{Database: database, Username: user, Password: password}})
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
defer conn.Close()
|
|
// Errors only, and only columns present in every deployed schema version.
|
|
rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT timestamp, service, release, kind, source, status_code, span_name, route, exception_type, message_normalized, pattern_signature, is_timeout, is_cancellation, is_retry
|
|
FROM %s.normalized_events FINAL
|
|
WHERE is_error = 1 AND timestamp >= toDateTime64('%s', 9, 'UTC') AND timestamp < toDateTime64('%s', 9, 'UTC')
|
|
ORDER BY timestamp LIMIT 2000000`, database, chTime(start), chTime(end)))
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
defer rows.Close()
|
|
var out []features.Event
|
|
for rows.Next() {
|
|
e := features.Event{IsError: true}
|
|
var status uint16
|
|
var timeout, cancel, retry uint8
|
|
if err := rows.Scan(&e.Timestamp, &e.Service, &e.Release, &e.Kind, &e.Source, &status, &e.SpanName, &e.Route, &e.ExceptionType, &e.Message, &e.PatternSignature, &timeout, &cancel, &retry); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
e.StatusCode = int(status)
|
|
e.IsTimeout, e.IsCancellation, e.IsRetry = timeout == 1, cancel == 1, retry == 1
|
|
out = append(out, e)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func chTime(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05.000000000") }
|
|
func envOr(k, f string) string {
|
|
if v := os.Getenv(k); v != "" {
|
|
return v
|
|
}
|
|
return f
|
|
}
|