package alerts import ( "git.tornberg.me/go-otel/internal/features" "git.tornberg.me/go-otel/internal/triage" ) // TriageOf classifies a pattern's example event with the deterministic rules. func TriageOf(e features.Event) triage.Result { return triage.Classify(triage.Input{Service: e.Service, Kind: e.Kind, Status: e.StatusCode, Route: e.Route, Title: e.SpanName, Message: e.Message, IsTimeout: e.IsTimeout, IsCancellation: e.IsCancellation, IsRetry: e.IsRetry}) } // edgeClientError reports a 4xx seen at the nginx edge: registry auth // challenges, /oauth2/auth subrequests, 405s from scanners. These are the // caller's problem and never page, even next to a real 5xx. func edgeClientError(e features.Event) bool { return e.Service == "nginx" && e.StatusCode >= 400 && e.StatusCode < 500 } // FilterActionable keeps only patterns whose category should page a human // (unless keepMuted), recomputing each spike's excess so a probe storm cannot // alert by itself. muted, when non-nil, counts what was dropped per category. func FilterActionable(spikes []features.Spike, keepMuted bool, muted map[triage.Category]int) []features.Spike { var out []features.Spike for _, sp := range spikes { var kept []features.PatternDelta sp.Excess = 0 for _, p := range sp.Patterns { c := TriageOf(p.Example).Category if !keepMuted && (!c.Actionable() || edgeClientError(p.Example)) { if muted != nil { muted[c]++ } continue } kept = append(kept, p) sp.Excess += p.Excess } if len(kept) > 0 { sp.Patterns = kept out = append(out, sp) } } return out } // DropMuted removes patterns whose issue the user ignored, recomputing each // spike's excess and dropping spikes left with nothing to report. A nil // muted keeps everything. func DropMuted(spikes []features.Spike, muted func(issueID string) bool) []features.Spike { if muted == nil { return spikes } var out []features.Spike for _, sp := range spikes { var kept []features.PatternDelta sp.Excess = 0 for _, p := range sp.Patterns { if id := p.Example.IssueFingerprint; id != "" && muted(id) { continue } kept = append(kept, p) sp.Excess += p.Excess } if len(kept) > 0 { sp.Patterns = kept out = append(out, sp) } } return out } // Priority ranks how urgently a pattern needs a human. Client errors (4xx) // are the caller's problem more often than ours, so they rank below server // failures and count for less when deciding whether to page. type Priority string const ( PriorityHigh Priority = "high" PriorityMedium Priority = "medium" PriorityLow Priority = "low" ) // lowPriorityWeight is how much a low-priority pattern's excess counts toward // the paging threshold: five 4xx errors weigh as one 5xx. const lowPriorityWeight = 0.2 // PriorityOf ranks an example event: 4xx is low; server failures (5xx, or a // service defect, saturation or unreachable dependency) are high. func PriorityOf(e features.Event, c triage.Category) Priority { switch { case e.StatusCode >= 400 && e.StatusCode < 500: return PriorityLow case e.StatusCode >= 500, c == triage.ServiceDefect, c == triage.CapacitySaturation, c == triage.DependencyUnreachable: return PriorityHigh } return PriorityMedium } // PagingExcess is a spike's excess with low-priority patterns discounted, so a // pile of 4xx cannot page on its own but still tips a real 5xx spike. func PagingExcess(sp features.Spike) float64 { var total float64 for _, p := range sp.Patterns { if PriorityOf(p.Example, TriageOf(p.Example).Category) == PriorityLow { total += p.Excess * lowPriorityWeight } else { total += p.Excess } } return total }