package features import ( "sort" "time" ) // SpikeConfig controls the deterministic "is this service misbehaving right // now" rule. It works on excess errors: a pattern that fires at its usual // rate (a client polling a 401 endpoint every few seconds) contributes zero, // so only new or rising patterns can raise an alert. type SpikeConfig struct { Window time.Duration // current observation window BaselineWindows int // preceding windows used as the baseline Multiple float64 // a pattern must be novel or >= Multiple * its own baseline mean MinExcess int // and excess errors must be >= MinExcess TopPatterns int } func DefaultSpikeConfig() SpikeConfig { return SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 36, Multiple: 3, MinExcess: 5, TopPatterns: 5} } // PatternDelta is one error pattern's contribution to a spike. type PatternDelta struct { Signature string `json:"signature"` Count int `json:"count"` BaselineMean float64 `json:"baselineMean"` Excess float64 `json:"excess"` Novel bool `json:"novel"` // never seen in the baseline Example Event `json:"example"` } type Spike struct { Service string `json:"service"` WindowStart time.Time `json:"windowStart"` WindowEnd time.Time `json:"windowEnd"` ErrorCount int `json:"errorCount"` BaselineMean float64 `json:"baselineMean"` Excess float64 `json:"excess"` Patterns []PatternDelta `json:"patterns"` } // DetectSpikes evaluates the window ending at `at` against the preceding // BaselineWindows windows of the same length. Baseline data never includes // the current window. Only error events are inspected. func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike { d := DefaultSpikeConfig() if cfg.Window <= 0 { cfg.Window = d.Window } if cfg.BaselineWindows <= 0 { cfg.BaselineWindows = d.BaselineWindows } if cfg.Multiple <= 0 { cfg.Multiple = d.Multiple } if cfg.MinExcess <= 0 { cfg.MinExcess = d.MinExcess } if cfg.TopPatterns <= 0 { cfg.TopPatterns = d.TopPatterns } at = at.UTC() start := at.Add(-cfg.Window) baseStart := start.Add(-time.Duration(cfg.BaselineWindows) * cfg.Window) type agg struct { cur, base int example Event } type svc struct { cur, base int patterns map[string]*agg } services := map[string]*svc{} for _, e := range events { if !e.IsError { continue } ts := e.Timestamp.UTC() inCur := !ts.Before(start) && ts.Before(at) inBase := !ts.Before(baseStart) && ts.Before(start) if !inCur && !inBase { continue } name := e.Service if name == "" { name = "unknown" } s := services[name] if s == nil { s = &svc{patterns: map[string]*agg{}} services[name] = s } key := e.PatternSignature if key == "" { key = issueKey(e) } p := s.patterns[key] if p == nil { p = &agg{} s.patterns[key] = p } if inCur { s.cur++ p.cur++ p.example = e } else { s.base++ p.base++ if p.example.PatternSignature == "" && p.cur == 0 { p.example = e } } } var spikes []Spike n := float64(cfg.BaselineWindows) for name, s := range services { baseMean := float64(s.base) / n if s.cur == 0 { continue } spike := Spike{Service: name, WindowStart: start, WindowEnd: at, ErrorCount: s.cur, BaselineMean: baseMean} for sig, p := range s.patterns { mean := float64(p.base) / n excess := float64(p.cur) - mean // A pattern only counts when it is novel or well above its own // baseline; the service-wide total is masked by chronic noise. if p.cur == 0 || excess <= 0 || (p.base > 0 && float64(p.cur) < cfg.Multiple*mean) { continue } spike.Excess += excess spike.Patterns = append(spike.Patterns, PatternDelta{ Signature: sig, Count: p.cur, BaselineMean: mean, Excess: excess, Novel: p.base == 0, Example: p.example, }) } if spike.Excess < float64(cfg.MinExcess) { continue } sort.Slice(spike.Patterns, func(i, j int) bool { if spike.Patterns[i].Excess != spike.Patterns[j].Excess { return spike.Patterns[i].Excess > spike.Patterns[j].Excess } return spike.Patterns[i].Signature < spike.Patterns[j].Signature }) if len(spike.Patterns) > cfg.TopPatterns { spike.Patterns = spike.Patterns[:cfg.TopPatterns] } spikes = append(spikes, spike) } sort.Slice(spikes, func(i, j int) bool { return spikes[i].Excess > spikes[j].Excess }) return spikes }