go-otel/internal/alerts/filter.go

package alerts

import (
	"git.tornberg.me/go-otel/internal/features"
	"git.tornberg.me/go-otel/internal/triage"
)

// TriageOf classifies a pattern's example event with the deterministic rules.
func TriageOf(e features.Event) triage.Result {
	return triage.Classify(triage.Input{Service: e.Service, Kind: e.Kind, Status: e.StatusCode, Route: e.Route, Title: e.SpanName, Message: e.Message,
		IsTimeout: e.IsTimeout, IsCancellation: e.IsCancellation, IsRetry: e.IsRetry})
}

// edgeClientError reports a 4xx seen at the nginx edge: registry auth
// challenges, /oauth2/auth subrequests, 405s from scanners. These are the
// caller's problem and never page, even next to a real 5xx.
func edgeClientError(e features.Event) bool {
	return e.Service == "nginx" && e.StatusCode >= 400 && e.StatusCode < 500
}

// FilterActionable keeps only patterns whose category should page a human
// (unless keepMuted), recomputing each spike's excess so a probe storm cannot
// alert by itself. muted, when non-nil, counts what was dropped per category.
func FilterActionable(spikes []features.Spike, keepMuted bool, muted map[triage.Category]int) []features.Spike {
	var out []features.Spike
	for _, sp := range spikes {
		var kept []features.PatternDelta
		sp.Excess = 0
		for _, p := range sp.Patterns {
			c := TriageOf(p.Example).Category
			if !keepMuted && (!c.Actionable() || edgeClientError(p.Example)) {
				if muted != nil {
					muted[c]++
				}
				continue
			}
			kept = append(kept, p)
			sp.Excess += p.Excess
		}
		if len(kept) > 0 {
			sp.Patterns = kept
			out = append(out, sp)
		}
	}
	return out
}

// DropMuted removes patterns whose issue the user ignored, recomputing each
// spike's excess and dropping spikes left with nothing to report. A nil
// muted keeps everything.
func DropMuted(spikes []features.Spike, muted func(issueID string) bool) []features.Spike {
	if muted == nil {
		return spikes
	}
	var out []features.Spike
	for _, sp := range spikes {
		var kept []features.PatternDelta
		sp.Excess = 0
		for _, p := range sp.Patterns {
			if id := p.Example.IssueFingerprint; id != "" && muted(id) {
				continue
			}
			kept = append(kept, p)
			sp.Excess += p.Excess
		}
		if len(kept) > 0 {
			sp.Patterns = kept
			out = append(out, sp)
		}
	}
	return out
}

// Priority ranks how urgently a pattern needs a human. Client errors (4xx)
// are the caller's problem more often than ours, so they rank below server
// failures and count for less when deciding whether to page.
type Priority string

const (
	PriorityHigh   Priority = "high"
	PriorityMedium Priority = "medium"
	PriorityLow    Priority = "low"
)

// lowPriorityWeight is how much a low-priority pattern's excess counts toward
// the paging threshold: five 4xx errors weigh as one 5xx.
const lowPriorityWeight = 0.2

// PriorityOf ranks an example event: 4xx is low; server failures (5xx, or a
// service defect, saturation or unreachable dependency) are high.
func PriorityOf(e features.Event, c triage.Category) Priority {
	switch {
	case e.StatusCode >= 400 && e.StatusCode < 500:
		return PriorityLow
	case e.StatusCode >= 500, c == triage.ServiceDefect, c == triage.CapacitySaturation, c == triage.DependencyUnreachable:
		return PriorityHigh
	}
	return PriorityMedium
}

// PagingExcess is a spike's excess with low-priority patterns discounted, so a
// pile of 4xx cannot page on its own but still tips a real 5xx spike.
func PagingExcess(sp features.Spike) float64 {
	var total float64
	for _, p := range sp.Patterns {
		if PriorityOf(p.Example, TriageOf(p.Example).Category) == PriorityLow {
			total += p.Excess * lowPriorityWeight
		} else {
			total += p.Excess
		}
	}
	return total
}