107 lines
3.2 KiB
Go
107 lines
3.2 KiB
Go
package alerts
|
|
|
|
import (
|
|
"git.tornberg.me/go-otel/internal/features"
|
|
"git.tornberg.me/go-otel/internal/triage"
|
|
)
|
|
|
|
// TriageOf classifies a pattern's example event with the deterministic rules.
|
|
func TriageOf(e features.Event) triage.Result {
|
|
return triage.Classify(triage.Input{Service: e.Service, Kind: e.Kind, Status: e.StatusCode, Route: e.Route, Title: e.SpanName, Message: e.Message,
|
|
IsTimeout: e.IsTimeout, IsCancellation: e.IsCancellation, IsRetry: e.IsRetry})
|
|
}
|
|
|
|
// FilterActionable keeps only patterns whose category should page a human
|
|
// (unless keepMuted), recomputing each spike's excess so a probe storm cannot
|
|
// alert by itself. muted, when non-nil, counts what was dropped per category.
|
|
func FilterActionable(spikes []features.Spike, keepMuted bool, muted map[triage.Category]int) []features.Spike {
|
|
var out []features.Spike
|
|
for _, sp := range spikes {
|
|
var kept []features.PatternDelta
|
|
sp.Excess = 0
|
|
for _, p := range sp.Patterns {
|
|
c := TriageOf(p.Example).Category
|
|
if !keepMuted && !c.Actionable() {
|
|
if muted != nil {
|
|
muted[c]++
|
|
}
|
|
continue
|
|
}
|
|
kept = append(kept, p)
|
|
sp.Excess += p.Excess
|
|
}
|
|
if len(kept) > 0 {
|
|
sp.Patterns = kept
|
|
out = append(out, sp)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// DropMuted removes patterns whose issue the user ignored, recomputing each
|
|
// spike's excess and dropping spikes left with nothing to report. A nil
|
|
// muted keeps everything.
|
|
func DropMuted(spikes []features.Spike, muted func(issueID string) bool) []features.Spike {
|
|
if muted == nil {
|
|
return spikes
|
|
}
|
|
var out []features.Spike
|
|
for _, sp := range spikes {
|
|
var kept []features.PatternDelta
|
|
sp.Excess = 0
|
|
for _, p := range sp.Patterns {
|
|
if id := p.Example.IssueFingerprint; id != "" && muted(id) {
|
|
continue
|
|
}
|
|
kept = append(kept, p)
|
|
sp.Excess += p.Excess
|
|
}
|
|
if len(kept) > 0 {
|
|
sp.Patterns = kept
|
|
out = append(out, sp)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Priority ranks how urgently a pattern needs a human. Client errors (4xx)
|
|
// are the caller's problem more often than ours, so they rank below server
|
|
// failures and count for less when deciding whether to page.
|
|
type Priority string
|
|
|
|
const (
|
|
PriorityHigh Priority = "high"
|
|
PriorityMedium Priority = "medium"
|
|
PriorityLow Priority = "low"
|
|
)
|
|
|
|
// lowPriorityWeight is how much a low-priority pattern's excess counts toward
|
|
// the paging threshold: five 4xx errors weigh as one 5xx.
|
|
const lowPriorityWeight = 0.2
|
|
|
|
// PriorityOf ranks an example event: 4xx is low; server failures (5xx, or a
|
|
// service defect, saturation or unreachable dependency) are high.
|
|
func PriorityOf(e features.Event, c triage.Category) Priority {
|
|
switch {
|
|
case e.StatusCode >= 400 && e.StatusCode < 500:
|
|
return PriorityLow
|
|
case e.StatusCode >= 500, c == triage.ServiceDefect, c == triage.CapacitySaturation, c == triage.DependencyUnreachable:
|
|
return PriorityHigh
|
|
}
|
|
return PriorityMedium
|
|
}
|
|
|
|
// PagingExcess is a spike's excess with low-priority patterns discounted, so a
|
|
// pile of 4xx cannot page on its own but still tips a real 5xx spike.
|
|
func PagingExcess(sp features.Spike) float64 {
|
|
var total float64
|
|
for _, p := range sp.Patterns {
|
|
if PriorityOf(p.Example, TriageOf(p.Example).Category) == PriorityLow {
|
|
total += p.Excess * lowPriorityWeight
|
|
} else {
|
|
total += p.Excess
|
|
}
|
|
}
|
|
return total
|
|
}
|