package alerts import ( "git.tornberg.me/go-otel/internal/features" "git.tornberg.me/go-otel/internal/triage" ) // TriageOf classifies a pattern's example event with the deterministic rules. func TriageOf(e features.Event) triage.Result { return triage.Classify(triage.Input{Service: e.Service, Kind: e.Kind, Status: e.StatusCode, Route: e.Route, Title: e.SpanName, Message: e.Message, IsTimeout: e.IsTimeout, IsCancellation: e.IsCancellation, IsRetry: e.IsRetry}) } // FilterActionable keeps only patterns whose category should page a human // (unless keepMuted), recomputing each spike's excess so a probe storm cannot // alert by itself. muted, when non-nil, counts what was dropped per category. func FilterActionable(spikes []features.Spike, keepMuted bool, muted map[triage.Category]int) []features.Spike { var out []features.Spike for _, sp := range spikes { var kept []features.PatternDelta sp.Excess = 0 for _, p := range sp.Patterns { c := TriageOf(p.Example).Category if !keepMuted && !c.Actionable() { if muted != nil { muted[c]++ } continue } kept = append(kept, p) sp.Excess += p.Excess } if len(kept) > 0 { sp.Patterns = kept out = append(out, sp) } } return out } // Priority ranks how urgently a pattern needs a human. Client errors (4xx) // are the caller's problem more often than ours, so they rank below server // failures and count for less when deciding whether to page. type Priority string const ( PriorityHigh Priority = "high" PriorityMedium Priority = "medium" PriorityLow Priority = "low" ) // lowPriorityWeight is how much a low-priority pattern's excess counts toward // the paging threshold: five 4xx errors weigh as one 5xx. const lowPriorityWeight = 0.2 // PriorityOf ranks an example event: 4xx is low; server failures (5xx, or a // service defect, saturation or unreachable dependency) are high. func PriorityOf(e features.Event, c triage.Category) Priority { switch { case e.StatusCode >= 400 && e.StatusCode < 500: return PriorityLow case e.StatusCode >= 500, c == triage.ServiceDefect, c == triage.CapacitySaturation, c == triage.DependencyUnreachable: return PriorityHigh } return PriorityMedium } // PagingExcess is a spike's excess with low-priority patterns discounted, so a // pile of 4xx cannot page on its own but still tips a real 5xx spike. func PagingExcess(sp features.Spike) float64 { var total float64 for _, p := range sp.Patterns { if PriorityOf(p.Example, TriageOf(p.Example).Category) == PriorityLow { total += p.Excess * lowPriorityWeight } else { total += p.Excess } } return total }