more ml

internal/features/spike.go

@@ -66,12 +66,14 @@ func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike {
 	baseStart := start.Add(-time.Duration(cfg.BaselineWindows) * cfg.Window)
 
 	type agg struct {
-		cur, base int
-		example   Event
+		cur     int
+		base    []int // per baseline window
+		example Event
 	}
 	type svc struct {
-		cur, base int
-		patterns  map[string]*agg
+		cur      int
+		base     []int // per baseline window
+		patterns map[string]*agg
 	}
 	services := map[string]*svc{}
 	for _, e := range events {
@@ -90,7 +92,7 @@ func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike {
 		}
 		s := services[name]
 		if s == nil {
-			s = &svc{patterns: map[string]*agg{}}
+			s = &svc{base: make([]int, cfg.BaselineWindows), patterns: map[string]*agg{}}
 			services[name] = s
 		}
 		key := e.PatternSignature
@@ -99,7 +101,7 @@ func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike {
 		}
 		p := s.patterns[key]
 		if p == nil {
-			p = &agg{}
+			p = &agg{base: make([]int, cfg.BaselineWindows)}
 			s.patterns[key] = p
 		}
 		if inCur {
@@ -107,8 +109,12 @@ func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike {
 			p.cur++
 			p.example = e
 		} else {
-			s.base++
-			p.base++
+			idx := int(ts.Sub(baseStart) / cfg.Window)
+			if idx >= cfg.BaselineWindows {
+				idx = cfg.BaselineWindows - 1
+			}
+			s.base[idx]++
+			p.base[idx]++
 			if p.example.PatternSignature == "" && p.cur == 0 {
 				p.example = e
 			}
@@ -116,24 +122,24 @@ func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike {
 	}
 
 	var spikes []Spike
-	n := float64(cfg.BaselineWindows)
 	for name, s := range services {
-		baseMean := float64(s.base) / n
+		baseMean := trimmedMean(s.base, baselineTrim)
 		if s.cur == 0 {
 			continue
 		}
 		spike := Spike{Service: name, WindowStart: start, WindowEnd: at, ErrorCount: s.cur, BaselineMean: baseMean}
 		for sig, p := range s.patterns {
-			mean := float64(p.base) / n
+			mean := trimmedMean(p.base, baselineTrim)
+			seen := sum(p.base) > 0
 			excess := float64(p.cur) - mean
 			// A pattern only counts when it is novel or well above its own
 			// baseline; the service-wide total is masked by chronic noise.
-			if p.cur == 0 || excess <= 0 || (p.base > 0 && float64(p.cur) < cfg.Multiple*mean) {
+			if p.cur == 0 || excess <= 0 || (seen && float64(p.cur) < cfg.Multiple*mean) {
 				continue
 			}
 			spike.Excess += excess
 			spike.Patterns = append(spike.Patterns, PatternDelta{
-				Signature: sig, Count: p.cur, BaselineMean: mean, Excess: excess, Novel: p.base == 0, Example: p.example,
+				Signature: sig, Count: p.cur, BaselineMean: mean, Excess: excess, Novel: !seen, Example: p.example,
 			})
 		}
 		if spike.Excess < float64(cfg.MinExcess) {
@@ -153,3 +159,74 @@ func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike {
 	sort.Slice(spikes, func(i, j int) bool { return spikes[i].Excess > spikes[j].Excess })
 	return spikes
 }
+
+// baselineTrim is the fraction of the busiest baseline windows discarded
+// before averaging, so one earlier storm cannot inflate the baseline and hide
+// the next one.
+const baselineTrim = 0.2
+
+func sum(v []int) (t int) {
+	for _, x := range v {
+		t += x
+	}
+	return t
+}
+
+func trimmedMean(counts []int, trim float64) float64 {
+	if len(counts) == 0 {
+		return 0
+	}
+	sorted := append([]int(nil), counts...)
+	sort.Ints(sorted)
+	keep := len(sorted) - int(float64(len(sorted))*trim)
+	if keep < 1 {
+		keep = 1
+	}
+	return float64(sum(sorted[:keep])) / float64(keep)
+}
+
+// Suppressor stops the same incident from alerting in every consecutive
+// window. A pattern re-alerts once its cooldown has elapsed, or sooner when
+// its count at least doubles (an escalation).
+type Suppressor struct {
+	Cooldown time.Duration
+	last     map[string]suppressed
+}
+
+type suppressed struct {
+	at    time.Time
+	count int
+}
+
+func NewSuppressor(cooldown time.Duration) *Suppressor {
+	if cooldown <= 0 {
+		cooldown = 30 * time.Minute
+	}
+	return &Suppressor{Cooldown: cooldown, last: map[string]suppressed{}}
+}
+
+// Filter returns spikes reduced to the patterns that are new or escalated;
+// spikes with no remaining patterns are dropped.
+func (s *Suppressor) Filter(spikes []Spike) []Spike {
+	var out []Spike
+	for _, spike := range spikes {
+		var kept []PatternDelta
+		var excess float64
+		for _, p := range spike.Patterns {
+			key := spike.Service + "\x00" + p.Signature
+			prev, ok := s.last[key]
+			if ok && spike.WindowEnd.Sub(prev.at) < s.Cooldown && p.Count < 2*prev.count {
+				continue
+			}
+			s.last[key] = suppressed{at: spike.WindowEnd, count: p.Count}
+			kept = append(kept, p)
+			excess += p.Excess
+		}
+		if len(kept) == 0 {
+			continue
+		}
+		spike.Patterns, spike.Excess = kept, excess
+		out = append(out, spike)
+	}
+	return out
+}

internal/features/spike_test.go

@@ -39,3 +39,41 @@ func TestDetectSpikesRequiresMinimumExcess(t *testing.T) {
 		t.Fatalf("3 errors should stay under the floor: %+v", got)
 	}
 }
+
+func TestBaselineTrimIgnoresEarlierStorm(t *testing.T) {
+	at := time.Date(2026, 9, 19, 12, 0, 0, 0, time.UTC)
+	cfg := SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 10}
+	var events []Event
+	add := func(sig string, n int, when time.Time) {
+		for i := 0; i < n; i++ {
+			events = append(events, Event{Timestamp: when.Add(time.Duration(i) * time.Millisecond), Service: "api", IsError: true, PatternSignature: sig})
+		}
+	}
+	// one earlier storm in 1 of 10 baseline windows, then a smaller repeat now
+	add("storm", 1000, at.Add(-6*cfg.Window))
+	add("storm", 60, at.Add(-cfg.Window))
+	got := DetectSpikes(events, at, cfg)
+	if len(got) != 1 || got[0].Patterns[0].BaselineMean != 0 {
+		t.Fatalf("storm baseline should be trimmed to 0, got %+v", got)
+	}
+}
+
+func TestSuppressorCooldownAndEscalation(t *testing.T) {
+	at := time.Date(2026, 9, 19, 12, 0, 0, 0, time.UTC)
+	spike := func(at time.Time, count int) []Spike {
+		return []Spike{{Service: "api", WindowEnd: at, Patterns: []PatternDelta{{Signature: "x", Count: count, Excess: float64(count)}}}}
+	}
+	s := NewSuppressor(30 * time.Minute)
+	if len(s.Filter(spike(at, 20))) != 1 {
+		t.Fatal("first alert should pass")
+	}
+	if len(s.Filter(spike(at.Add(5*time.Minute), 22))) != 0 {
+		t.Fatal("repeat inside cooldown should be suppressed")
+	}
+	if len(s.Filter(spike(at.Add(10*time.Minute), 45))) != 1 {
+		t.Fatal("2x escalation should re-alert")
+	}
+	if len(s.Filter(spike(at.Add(50*time.Minute), 46))) != 1 {
+		t.Fatal("alert after cooldown should pass")
+	}
+}