package features import ( "sort" "time" ) // SpikeConfig controls the deterministic "is this service misbehaving right // now" rule. It works on excess errors: a pattern that fires at its usual // rate (a client polling a 401 endpoint every few seconds) contributes zero, // so only new or rising patterns can raise an alert. type SpikeConfig struct { Window time.Duration // current observation window BaselineWindows int // preceding windows used as the baseline Multiple float64 // a pattern must be novel or >= Multiple * its own baseline mean MinExcess int // and excess errors must be >= MinExcess TopPatterns int // DataSince is when data collection began (earliest row of any kind). When // set, no alert fires until WarmupWindows windows of baseline exist, // otherwise the first window after data begins reports everything as NEW. // Zero disables the guard. DataSince time.Time WarmupWindows int } func DefaultSpikeConfig() SpikeConfig { return SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 36, Multiple: 3, MinExcess: 5, TopPatterns: 5} } // PatternDelta is one error pattern's contribution to a spike. type PatternDelta struct { Signature string `json:"signature"` Count int `json:"count"` BaselineMean float64 `json:"baselineMean"` Excess float64 `json:"excess"` Novel bool `json:"novel"` // never seen in the baseline Example Event `json:"example"` } type Spike struct { Service string `json:"service"` WindowStart time.Time `json:"windowStart"` WindowEnd time.Time `json:"windowEnd"` ErrorCount int `json:"errorCount"` BaselineMean float64 `json:"baselineMean"` Excess float64 `json:"excess"` Patterns []PatternDelta `json:"patterns"` } // DetectSpikes evaluates the window ending at `at` against the preceding // BaselineWindows windows of the same length. Baseline data never includes // the current window. Only error events are inspected. func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike { d := DefaultSpikeConfig() if cfg.Window <= 0 { cfg.Window = d.Window } if cfg.BaselineWindows <= 0 { cfg.BaselineWindows = d.BaselineWindows } if cfg.Multiple <= 0 { cfg.Multiple = d.Multiple } if cfg.MinExcess <= 0 { cfg.MinExcess = d.MinExcess } if cfg.TopPatterns <= 0 { cfg.TopPatterns = d.TopPatterns } at = at.UTC() start := at.Add(-cfg.Window) baseStart := start.Add(-time.Duration(cfg.BaselineWindows) * cfg.Window) if !cfg.DataSince.IsZero() { warm := cfg.WarmupWindows if warm <= 0 { warm = min(cfg.BaselineWindows, 6) } if start.Sub(cfg.DataSince.UTC()) < time.Duration(warm)*cfg.Window { return nil } } type agg struct { cur int base []int // per baseline window example Event } type svc struct { cur int base []int // per baseline window patterns map[string]*agg } services := map[string]*svc{} for _, e := range events { if !e.IsError { continue } ts := e.Timestamp.UTC() inCur := !ts.Before(start) && ts.Before(at) inBase := !ts.Before(baseStart) && ts.Before(start) if !inCur && !inBase { continue } name := e.Service if name == "" { name = "unknown" } s := services[name] if s == nil { s = &svc{base: make([]int, cfg.BaselineWindows), patterns: map[string]*agg{}} services[name] = s } key := e.PatternSignature if key == "" { key = issueKey(e) } p := s.patterns[key] if p == nil { p = &agg{base: make([]int, cfg.BaselineWindows)} s.patterns[key] = p } if inCur { s.cur++ p.cur++ p.example = e } else { idx := int(ts.Sub(baseStart) / cfg.Window) if idx >= cfg.BaselineWindows { idx = cfg.BaselineWindows - 1 } s.base[idx]++ p.base[idx]++ if p.example.PatternSignature == "" && p.cur == 0 { p.example = e } } } var spikes []Spike for name, s := range services { baseMean := trimmedMean(s.base, baselineTrim) if s.cur == 0 { continue } spike := Spike{Service: name, WindowStart: start, WindowEnd: at, ErrorCount: s.cur, BaselineMean: baseMean} for sig, p := range s.patterns { mean := trimmedMean(p.base, baselineTrim) seen := sum(p.base) > 0 excess := float64(p.cur) - mean // A pattern only counts when it is novel or well above its own // baseline; the service-wide total is masked by chronic noise. if p.cur == 0 || excess <= 0 || (seen && float64(p.cur) < cfg.Multiple*mean) { continue } spike.Excess += excess spike.Patterns = append(spike.Patterns, PatternDelta{ Signature: sig, Count: p.cur, BaselineMean: mean, Excess: excess, Novel: !seen, Example: p.example, }) } if spike.Excess < float64(cfg.MinExcess) { continue } sort.Slice(spike.Patterns, func(i, j int) bool { if spike.Patterns[i].Excess != spike.Patterns[j].Excess { return spike.Patterns[i].Excess > spike.Patterns[j].Excess } return spike.Patterns[i].Signature < spike.Patterns[j].Signature }) if len(spike.Patterns) > cfg.TopPatterns { spike.Patterns = spike.Patterns[:cfg.TopPatterns] } spikes = append(spikes, spike) } sort.Slice(spikes, func(i, j int) bool { return spikes[i].Excess > spikes[j].Excess }) return spikes } // baselineTrim is the fraction of the busiest baseline windows discarded // before averaging, so one earlier storm cannot inflate the baseline and hide // the next one. const baselineTrim = 0.2 func sum(v []int) (t int) { for _, x := range v { t += x } return t } func trimmedMean(counts []int, trim float64) float64 { if len(counts) == 0 { return 0 } sorted := append([]int(nil), counts...) sort.Ints(sorted) keep := len(sorted) - int(float64(len(sorted))*trim) if keep < 1 { keep = 1 } return float64(sum(sorted[:keep])) / float64(keep) } // Suppressor stops the same incident from alerting in every consecutive // window. A pattern re-alerts once its cooldown has elapsed, or sooner when // its count at least doubles (an escalation). type Suppressor struct { Cooldown time.Duration last map[string]suppressed } type suppressed struct { at time.Time count int } func NewSuppressor(cooldown time.Duration) *Suppressor { if cooldown <= 0 { cooldown = 30 * time.Minute } return &Suppressor{Cooldown: cooldown, last: map[string]suppressed{}} } // Filter returns spikes reduced to the patterns that are new or escalated; // spikes with no remaining patterns are dropped. func (s *Suppressor) Filter(spikes []Spike) []Spike { var out []Spike for _, spike := range spikes { var kept []PatternDelta var excess float64 for _, p := range spike.Patterns { key := spike.Service + "\x00" + p.Signature prev, ok := s.last[key] if ok && spike.WindowEnd.Sub(prev.at) < s.Cooldown && p.Count < 2*prev.count { continue } s.last[key] = suppressed{at: spike.WindowEnd, count: p.Count} kept = append(kept, p) excess += p.Excess } if len(kept) == 0 { continue } spike.Patterns, spike.Excess = kept, excess out = append(out, spike) } return out }