go-otel/internal/features/spike.go

package features

import (
	"sort"
	"time"
)

// SpikeConfig controls the deterministic "is this service misbehaving right
// now" rule. It works on excess errors: a pattern that fires at its usual
// rate (a client polling a 401 endpoint every few seconds) contributes zero,
// so only new or rising patterns can raise an alert.
type SpikeConfig struct {
	Window          time.Duration // current observation window
	BaselineWindows int           // preceding windows used as the baseline
	Multiple        float64       // a pattern must be novel or >= Multiple * its own baseline mean
	MinExcess       int           // and excess errors must be >= MinExcess
	TopPatterns     int
}

func DefaultSpikeConfig() SpikeConfig {
	return SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 36, Multiple: 3, MinExcess: 5, TopPatterns: 5}
}

// PatternDelta is one error pattern's contribution to a spike.
type PatternDelta struct {
	Signature    string  `json:"signature"`
	Count        int     `json:"count"`
	BaselineMean float64 `json:"baselineMean"`
	Excess       float64 `json:"excess"`
	Novel        bool    `json:"novel"` // never seen in the baseline
	Example      Event   `json:"example"`
}

type Spike struct {
	Service      string         `json:"service"`
	WindowStart  time.Time      `json:"windowStart"`
	WindowEnd    time.Time      `json:"windowEnd"`
	ErrorCount   int            `json:"errorCount"`
	BaselineMean float64        `json:"baselineMean"`
	Excess       float64        `json:"excess"`
	Patterns     []PatternDelta `json:"patterns"`
}

// DetectSpikes evaluates the window ending at `at` against the preceding
// BaselineWindows windows of the same length. Baseline data never includes
// the current window. Only error events are inspected.
func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike {
	d := DefaultSpikeConfig()
	if cfg.Window <= 0 {
		cfg.Window = d.Window
	}
	if cfg.BaselineWindows <= 0 {
		cfg.BaselineWindows = d.BaselineWindows
	}
	if cfg.Multiple <= 0 {
		cfg.Multiple = d.Multiple
	}
	if cfg.MinExcess <= 0 {
		cfg.MinExcess = d.MinExcess
	}
	if cfg.TopPatterns <= 0 {
		cfg.TopPatterns = d.TopPatterns
	}
	at = at.UTC()
	start := at.Add(-cfg.Window)
	baseStart := start.Add(-time.Duration(cfg.BaselineWindows) * cfg.Window)

	type agg struct {
		cur, base int
		example   Event
	}
	type svc struct {
		cur, base int
		patterns  map[string]*agg
	}
	services := map[string]*svc{}
	for _, e := range events {
		if !e.IsError {
			continue
		}
		ts := e.Timestamp.UTC()
		inCur := !ts.Before(start) && ts.Before(at)
		inBase := !ts.Before(baseStart) && ts.Before(start)
		if !inCur && !inBase {
			continue
		}
		name := e.Service
		if name == "" {
			name = "unknown"
		}
		s := services[name]
		if s == nil {
			s = &svc{patterns: map[string]*agg{}}
			services[name] = s
		}
		key := e.PatternSignature
		if key == "" {
			key = issueKey(e)
		}
		p := s.patterns[key]
		if p == nil {
			p = &agg{}
			s.patterns[key] = p
		}
		if inCur {
			s.cur++
			p.cur++
			p.example = e
		} else {
			s.base++
			p.base++
			if p.example.PatternSignature == "" && p.cur == 0 {
				p.example = e
			}
		}
	}

	var spikes []Spike
	n := float64(cfg.BaselineWindows)
	for name, s := range services {
		baseMean := float64(s.base) / n
		if s.cur == 0 {
			continue
		}
		spike := Spike{Service: name, WindowStart: start, WindowEnd: at, ErrorCount: s.cur, BaselineMean: baseMean}
		for sig, p := range s.patterns {
			mean := float64(p.base) / n
			excess := float64(p.cur) - mean
			// A pattern only counts when it is novel or well above its own
			// baseline; the service-wide total is masked by chronic noise.
			if p.cur == 0 || excess <= 0 || (p.base > 0 && float64(p.cur) < cfg.Multiple*mean) {
				continue
			}
			spike.Excess += excess
			spike.Patterns = append(spike.Patterns, PatternDelta{
				Signature: sig, Count: p.cur, BaselineMean: mean, Excess: excess, Novel: p.base == 0, Example: p.example,
			})
		}
		if spike.Excess < float64(cfg.MinExcess) {
			continue
		}
		sort.Slice(spike.Patterns, func(i, j int) bool {
			if spike.Patterns[i].Excess != spike.Patterns[j].Excess {
				return spike.Patterns[i].Excess > spike.Patterns[j].Excess
			}
			return spike.Patterns[i].Signature < spike.Patterns[j].Signature
		})
		if len(spike.Patterns) > cfg.TopPatterns {
			spike.Patterns = spike.Patterns[:cfg.TopPatterns]
		}
		spikes = append(spikes, spike)
	}
	sort.Slice(spikes, func(i, j int) bool { return spikes[i].Excess > spikes[j].Excess })
	return spikes
}