278 lines
8.1 KiB
Go
278 lines
8.1 KiB
Go
package features
|
|
|
|
import (
|
|
"sort"
|
|
"time"
|
|
)
|
|
|
|
// SpikeConfig controls the deterministic "is this service misbehaving right
|
|
// now" rule. It works on excess errors: a pattern that fires at its usual
|
|
// rate (a client polling a 401 endpoint every few seconds) contributes zero,
|
|
// so only new or rising patterns can raise an alert.
|
|
type SpikeConfig struct {
|
|
Window time.Duration // current observation window
|
|
BaselineWindows int // preceding windows used as the baseline
|
|
Multiple float64 // a pattern must be novel or >= Multiple * its own baseline mean
|
|
MinExcess int // and excess errors must be >= MinExcess
|
|
TopPatterns int
|
|
// DataSince is when data collection began (earliest row of any kind). When
|
|
// set, no alert fires until WarmupWindows windows of baseline exist,
|
|
// otherwise the first window after data begins reports everything as NEW.
|
|
// Zero disables the guard.
|
|
DataSince time.Time
|
|
WarmupWindows int
|
|
}
|
|
|
|
func DefaultSpikeConfig() SpikeConfig {
|
|
return SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 36, Multiple: 3, MinExcess: 5, TopPatterns: 5}
|
|
}
|
|
|
|
// PatternDelta is one error pattern's contribution to a spike.
|
|
type PatternDelta struct {
|
|
Signature string `json:"signature"`
|
|
Count int `json:"count"`
|
|
BaselineMean float64 `json:"baselineMean"`
|
|
Excess float64 `json:"excess"`
|
|
Novel bool `json:"novel"` // never seen in the baseline
|
|
Clients int `json:"clients,omitempty"` // distinct resolved client IPs in the current window
|
|
Services []string `json:"services,omitempty"` // services with an error in the same requests, sorted
|
|
Example Event `json:"example"`
|
|
}
|
|
|
|
type Spike struct {
|
|
Service string `json:"service"`
|
|
WindowStart time.Time `json:"windowStart"`
|
|
WindowEnd time.Time `json:"windowEnd"`
|
|
ErrorCount int `json:"errorCount"`
|
|
BaselineMean float64 `json:"baselineMean"`
|
|
Excess float64 `json:"excess"`
|
|
Patterns []PatternDelta `json:"patterns"`
|
|
}
|
|
|
|
// DetectSpikes evaluates the window ending at `at` against the preceding
|
|
// BaselineWindows windows of the same length. Baseline data never includes
|
|
// the current window. Only error events are inspected.
|
|
func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike {
|
|
d := DefaultSpikeConfig()
|
|
if cfg.Window <= 0 {
|
|
cfg.Window = d.Window
|
|
}
|
|
if cfg.BaselineWindows <= 0 {
|
|
cfg.BaselineWindows = d.BaselineWindows
|
|
}
|
|
if cfg.Multiple <= 0 {
|
|
cfg.Multiple = d.Multiple
|
|
}
|
|
if cfg.MinExcess <= 0 {
|
|
cfg.MinExcess = d.MinExcess
|
|
}
|
|
if cfg.TopPatterns <= 0 {
|
|
cfg.TopPatterns = d.TopPatterns
|
|
}
|
|
at = at.UTC()
|
|
start := at.Add(-cfg.Window)
|
|
baseStart := start.Add(-time.Duration(cfg.BaselineWindows) * cfg.Window)
|
|
if !cfg.DataSince.IsZero() {
|
|
warm := cfg.WarmupWindows
|
|
if warm <= 0 {
|
|
warm = min(cfg.BaselineWindows, 6)
|
|
}
|
|
if start.Sub(cfg.DataSince.UTC()) < time.Duration(warm)*cfg.Window {
|
|
return nil
|
|
}
|
|
}
|
|
|
|
type agg struct {
|
|
cur int
|
|
base []int // per baseline window
|
|
example Event
|
|
clients map[string]struct{}
|
|
services map[string]struct{}
|
|
}
|
|
type svc struct {
|
|
cur int
|
|
base []int // per baseline window
|
|
patterns map[string]*agg
|
|
}
|
|
services := map[string]*svc{}
|
|
for _, e := range events {
|
|
// Bad actors (scanners, crawlers getting 4xx) are noise for alerting.
|
|
// A duplicate is a later hop of a request whose outermost error is
|
|
// already counted; counting it again multiplies every failure by the
|
|
// number of services it crossed.
|
|
if !e.IsError || e.IsDuplicate || e.BadActor != "" {
|
|
continue
|
|
}
|
|
ts := e.Timestamp.UTC()
|
|
inCur := !ts.Before(start) && ts.Before(at)
|
|
inBase := !ts.Before(baseStart) && ts.Before(start)
|
|
if !inCur && !inBase {
|
|
continue
|
|
}
|
|
name := e.Service
|
|
if name == "" {
|
|
name = "unknown"
|
|
}
|
|
s := services[name]
|
|
if s == nil {
|
|
s = &svc{base: make([]int, cfg.BaselineWindows), patterns: map[string]*agg{}}
|
|
services[name] = s
|
|
}
|
|
key := e.PatternSignature
|
|
if key == "" {
|
|
key = issueKey(e)
|
|
}
|
|
p := s.patterns[key]
|
|
if p == nil {
|
|
p = &agg{base: make([]int, cfg.BaselineWindows)}
|
|
s.patterns[key] = p
|
|
}
|
|
if inCur {
|
|
s.cur++
|
|
p.cur++
|
|
p.example = e
|
|
if p.services == nil {
|
|
p.services = map[string]struct{}{}
|
|
}
|
|
p.services[name] = struct{}{}
|
|
for _, sv := range e.TraceServices {
|
|
p.services[sv] = struct{}{}
|
|
}
|
|
if e.ClientIP != "" {
|
|
if p.clients == nil {
|
|
p.clients = map[string]struct{}{}
|
|
}
|
|
p.clients[e.ClientIP] = struct{}{}
|
|
}
|
|
} else {
|
|
idx := int(ts.Sub(baseStart) / cfg.Window)
|
|
if idx >= cfg.BaselineWindows {
|
|
idx = cfg.BaselineWindows - 1
|
|
}
|
|
s.base[idx]++
|
|
p.base[idx]++
|
|
if p.example.PatternSignature == "" && p.cur == 0 {
|
|
p.example = e
|
|
}
|
|
}
|
|
}
|
|
|
|
var spikes []Spike
|
|
for name, s := range services {
|
|
baseMean := trimmedMean(s.base, baselineTrim)
|
|
if s.cur == 0 {
|
|
continue
|
|
}
|
|
spike := Spike{Service: name, WindowStart: start, WindowEnd: at, ErrorCount: s.cur, BaselineMean: baseMean}
|
|
for sig, p := range s.patterns {
|
|
mean := trimmedMean(p.base, baselineTrim)
|
|
seen := sum(p.base) > 0
|
|
excess := float64(p.cur) - mean
|
|
// A pattern only counts when it is novel or well above its own
|
|
// baseline; the service-wide total is masked by chronic noise.
|
|
if p.cur == 0 || excess <= 0 || (seen && float64(p.cur) < cfg.Multiple*mean) {
|
|
continue
|
|
}
|
|
spike.Excess += excess
|
|
spike.Patterns = append(spike.Patterns, PatternDelta{
|
|
Signature: sig, Count: p.cur, BaselineMean: mean, Excess: excess, Novel: !seen, Example: p.example, Clients: len(p.clients), Services: sortedKeys(p.services),
|
|
})
|
|
}
|
|
if spike.Excess < float64(cfg.MinExcess) {
|
|
continue
|
|
}
|
|
sort.Slice(spike.Patterns, func(i, j int) bool {
|
|
if spike.Patterns[i].Excess != spike.Patterns[j].Excess {
|
|
return spike.Patterns[i].Excess > spike.Patterns[j].Excess
|
|
}
|
|
return spike.Patterns[i].Signature < spike.Patterns[j].Signature
|
|
})
|
|
if len(spike.Patterns) > cfg.TopPatterns {
|
|
spike.Patterns = spike.Patterns[:cfg.TopPatterns]
|
|
}
|
|
spikes = append(spikes, spike)
|
|
}
|
|
sort.Slice(spikes, func(i, j int) bool { return spikes[i].Excess > spikes[j].Excess })
|
|
return spikes
|
|
}
|
|
|
|
// baselineTrim is the fraction of the busiest baseline windows discarded
|
|
// before averaging, so one earlier storm cannot inflate the baseline and hide
|
|
// the next one.
|
|
const baselineTrim = 0.2
|
|
|
|
func sum(v []int) (t int) {
|
|
for _, x := range v {
|
|
t += x
|
|
}
|
|
return t
|
|
}
|
|
|
|
func trimmedMean(counts []int, trim float64) float64 {
|
|
if len(counts) == 0 {
|
|
return 0
|
|
}
|
|
sorted := append([]int(nil), counts...)
|
|
sort.Ints(sorted)
|
|
keep := len(sorted) - int(float64(len(sorted))*trim)
|
|
if keep < 1 {
|
|
keep = 1
|
|
}
|
|
return float64(sum(sorted[:keep])) / float64(keep)
|
|
}
|
|
|
|
// Suppressor stops the same incident from alerting in every consecutive
|
|
// window. A pattern re-alerts once its cooldown has elapsed, or sooner when
|
|
// its count at least doubles (an escalation).
|
|
type Suppressor struct {
|
|
Cooldown time.Duration
|
|
last map[string]suppressed
|
|
}
|
|
|
|
type suppressed struct {
|
|
at time.Time
|
|
count int
|
|
}
|
|
|
|
func NewSuppressor(cooldown time.Duration) *Suppressor {
|
|
if cooldown <= 0 {
|
|
cooldown = 30 * time.Minute
|
|
}
|
|
return &Suppressor{Cooldown: cooldown, last: map[string]suppressed{}}
|
|
}
|
|
|
|
// Filter returns spikes reduced to the patterns that are new or escalated;
|
|
// spikes with no remaining patterns are dropped.
|
|
func (s *Suppressor) Filter(spikes []Spike) []Spike {
|
|
var out []Spike
|
|
for _, spike := range spikes {
|
|
var kept []PatternDelta
|
|
var excess float64
|
|
for _, p := range spike.Patterns {
|
|
key := spike.Service + "\x00" + p.Signature
|
|
prev, ok := s.last[key]
|
|
if ok && spike.WindowEnd.Sub(prev.at) < s.Cooldown && p.Count < 2*prev.count {
|
|
continue
|
|
}
|
|
s.last[key] = suppressed{at: spike.WindowEnd, count: p.Count}
|
|
kept = append(kept, p)
|
|
excess += p.Excess
|
|
}
|
|
if len(kept) == 0 {
|
|
continue
|
|
}
|
|
spike.Patterns, spike.Excess = kept, excess
|
|
out = append(out, spike)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func sortedKeys(m map[string]struct{}) []string {
|
|
out := make([]string, 0, len(m))
|
|
for k := range m {
|
|
out = append(out, k)
|
|
}
|
|
sort.Strings(out)
|
|
return out
|
|
}
|