more

internal/features/features.go

@@ -28,6 +28,9 @@ type Event struct {
 	IsCancellation   bool
 	Message          string
 	SpanName         string
+	Route            string
+	StatusCode       int
+	ExceptionType    string
 }
 
 // Config controls feature and label windows. A zero Config receives the

internal/features/spike.go

@@ -0,0 +1,155 @@
+package features
+
+import (
+	"sort"
+	"time"
+)
+
+// SpikeConfig controls the deterministic "is this service misbehaving right
+// now" rule. It works on excess errors: a pattern that fires at its usual
+// rate (a client polling a 401 endpoint every few seconds) contributes zero,
+// so only new or rising patterns can raise an alert.
+type SpikeConfig struct {
+	Window          time.Duration // current observation window
+	BaselineWindows int           // preceding windows used as the baseline
+	Multiple        float64       // a pattern must be novel or >= Multiple * its own baseline mean
+	MinExcess       int           // and excess errors must be >= MinExcess
+	TopPatterns     int
+}
+
+func DefaultSpikeConfig() SpikeConfig {
+	return SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 36, Multiple: 3, MinExcess: 5, TopPatterns: 5}
+}
+
+// PatternDelta is one error pattern's contribution to a spike.
+type PatternDelta struct {
+	Signature    string  `json:"signature"`
+	Count        int     `json:"count"`
+	BaselineMean float64 `json:"baselineMean"`
+	Excess       float64 `json:"excess"`
+	Novel        bool    `json:"novel"` // never seen in the baseline
+	Example      Event   `json:"example"`
+}
+
+type Spike struct {
+	Service      string         `json:"service"`
+	WindowStart  time.Time      `json:"windowStart"`
+	WindowEnd    time.Time      `json:"windowEnd"`
+	ErrorCount   int            `json:"errorCount"`
+	BaselineMean float64        `json:"baselineMean"`
+	Excess       float64        `json:"excess"`
+	Patterns     []PatternDelta `json:"patterns"`
+}
+
+// DetectSpikes evaluates the window ending at `at` against the preceding
+// BaselineWindows windows of the same length. Baseline data never includes
+// the current window. Only error events are inspected.
+func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike {
+	d := DefaultSpikeConfig()
+	if cfg.Window <= 0 {
+		cfg.Window = d.Window
+	}
+	if cfg.BaselineWindows <= 0 {
+		cfg.BaselineWindows = d.BaselineWindows
+	}
+	if cfg.Multiple <= 0 {
+		cfg.Multiple = d.Multiple
+	}
+	if cfg.MinExcess <= 0 {
+		cfg.MinExcess = d.MinExcess
+	}
+	if cfg.TopPatterns <= 0 {
+		cfg.TopPatterns = d.TopPatterns
+	}
+	at = at.UTC()
+	start := at.Add(-cfg.Window)
+	baseStart := start.Add(-time.Duration(cfg.BaselineWindows) * cfg.Window)
+
+	type agg struct {
+		cur, base int
+		example   Event
+	}
+	type svc struct {
+		cur, base int
+		patterns  map[string]*agg
+	}
+	services := map[string]*svc{}
+	for _, e := range events {
+		if !e.IsError {
+			continue
+		}
+		ts := e.Timestamp.UTC()
+		inCur := !ts.Before(start) && ts.Before(at)
+		inBase := !ts.Before(baseStart) && ts.Before(start)
+		if !inCur && !inBase {
+			continue
+		}
+		name := e.Service
+		if name == "" {
+			name = "unknown"
+		}
+		s := services[name]
+		if s == nil {
+			s = &svc{patterns: map[string]*agg{}}
+			services[name] = s
+		}
+		key := e.PatternSignature
+		if key == "" {
+			key = issueKey(e)
+		}
+		p := s.patterns[key]
+		if p == nil {
+			p = &agg{}
+			s.patterns[key] = p
+		}
+		if inCur {
+			s.cur++
+			p.cur++
+			p.example = e
+		} else {
+			s.base++
+			p.base++
+			if p.example.PatternSignature == "" && p.cur == 0 {
+				p.example = e
+			}
+		}
+	}
+
+	var spikes []Spike
+	n := float64(cfg.BaselineWindows)
+	for name, s := range services {
+		baseMean := float64(s.base) / n
+		if s.cur == 0 {
+			continue
+		}
+		spike := Spike{Service: name, WindowStart: start, WindowEnd: at, ErrorCount: s.cur, BaselineMean: baseMean}
+		for sig, p := range s.patterns {
+			mean := float64(p.base) / n
+			excess := float64(p.cur) - mean
+			// A pattern only counts when it is novel or well above its own
+			// baseline; the service-wide total is masked by chronic noise.
+			if p.cur == 0 || excess <= 0 || (p.base > 0 && float64(p.cur) < cfg.Multiple*mean) {
+				continue
+			}
+			spike.Excess += excess
+			spike.Patterns = append(spike.Patterns, PatternDelta{
+				Signature: sig, Count: p.cur, BaselineMean: mean, Excess: excess, Novel: p.base == 0, Example: p.example,
+			})
+		}
+		if spike.Excess < float64(cfg.MinExcess) {
+			continue
+		}
+		sort.Slice(spike.Patterns, func(i, j int) bool {
+			if spike.Patterns[i].Excess != spike.Patterns[j].Excess {
+				return spike.Patterns[i].Excess > spike.Patterns[j].Excess
+			}
+			return spike.Patterns[i].Signature < spike.Patterns[j].Signature
+		})
+		if len(spike.Patterns) > cfg.TopPatterns {
+			spike.Patterns = spike.Patterns[:cfg.TopPatterns]
+		}
+		spikes = append(spikes, spike)
+	}
+	sort.Slice(spikes, func(i, j int) bool { return spikes[i].Excess > spikes[j].Excess })
+	return spikes
+}

internal/features/spike_test.go

@@ -0,0 +1,41 @@
+package features
+
+import (
+	"testing"
+	"time"
+)
+
+func TestDetectSpikesIgnoresChronicPatterns(t *testing.T) {
+	at := time.Date(2026, 9, 19, 12, 0, 0, 0, time.UTC)
+	cfg := SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12}
+	var events []Event
+	add := func(sig string, n int, when time.Time) {
+		for i := 0; i < n; i++ {
+			events = append(events, Event{Timestamp: when.Add(time.Duration(i) * time.Second), Service: "api", IsError: true, PatternSignature: sig})
+		}
+	}
+	// chronic poller: 50 errors in every window, including the current one
+	for w := 0; w <= 12; w++ {
+		add("poll-401", 50, at.Add(-time.Duration(w+1)*cfg.Window))
+	}
+	if got := DetectSpikes(events, at, cfg); len(got) != 0 {
+		t.Fatalf("chronic pattern alone alerted: %+v", got)
+	}
+	// a novel failure appears in the current window
+	add("db-timeout", 20, at.Add(-cfg.Window))
+	got := DetectSpikes(events, at, cfg)
+	if len(got) != 1 || got[0].Patterns[0].Signature != "db-timeout" || !got[0].Patterns[0].Novel {
+		t.Fatalf("expected db-timeout spike, got %+v", got)
+	}
+}
+
+func TestDetectSpikesRequiresMinimumExcess(t *testing.T) {
+	at := time.Date(2026, 9, 19, 12, 0, 0, 0, time.UTC)
+	events := []Event{}
+	for i := 0; i < 3; i++ {
+		events = append(events, Event{Timestamp: at.Add(-time.Minute), Service: "api", IsError: true, PatternSignature: "x"})
+	}
+	if got := DetectSpikes(events, at, SpikeConfig{}); len(got) != 0 {
+		t.Fatalf("3 errors should stay under the floor: %+v", got)
+	}
+}