fixes

internal/alerts/runner.go

@@ -98,6 +98,7 @@ type Runner struct {
 	load      Loader
 	dataSince func(ctx context.Context) time.Time
 	notifiers []Notifier
+	resolver  Resolver // optional: ingress -> service -> deployment for edge errors
 	sup       *features.Suppressor
 
 	mu        sync.Mutex
@@ -113,7 +114,7 @@ const maxRecent = 50
 
 // Start launches the runner against ClickHouse and returns it, or nil when
 // alerting is not configured.
-func Start(ctx context.Context, cfg Config, conn driver.Conn, database string) *Runner {
+func Start(ctx context.Context, cfg Config, conn driver.Conn, database string, resolver Resolver) *Runner {
 	notifiers := cfg.Notifiers()
 	if len(notifiers) == 0 && !cfg.DryRun {
 		log.Printf("alerts: disabled (set OTEL_ALERT_TELEGRAM_TOKEN + OTEL_ALERT_TELEGRAM_CHAT_ID and/or OTEL_ALERT_WEBHOOK_URL)")
@@ -122,6 +123,7 @@ func Start(ctx context.Context, cfg Config, conn driver.Conn, database string) *
 	r := &Runner{
 		cfg:       cfg,
 		notifiers: notifiers,
+		resolver:  resolver,
 		sup:       features.NewSuppressor(cfg.Cooldown),
 		load: func(ctx context.Context, start, end time.Time) ([]features.Event, error) {
 			return LoadErrorEvents(ctx, conn, database, start, end)
@@ -205,10 +207,11 @@ func DetectAlerts(events []features.Event, at time.Time, cfg features.SpikeConfi
 		minExcess = features.DefaultSpikeConfig().MinExcess
 	}
 	var out []features.Spike
-	// The excess threshold is applied after triage: a service whose total
-	// excess is mostly probes must not page on the little that is left.
+	// The excess threshold is applied after triage, with 4xx discounted: a
+	// service whose excess is mostly probes or client errors must not page on
+	// the little that is left.
 	for _, sp := range FilterActionable(features.DetectSpikes(events, at, cfg), false, nil) {
-		if sp.Excess >= float64(minExcess) {
+		if PagingExcess(sp) >= float64(minExcess) {
 			out = append(out, sp)
 		}
 	}
@@ -216,6 +219,7 @@ func DetectAlerts(events []features.Event, at time.Time, cfg features.SpikeConfi
 }
 
 func (r *Runner) deliver(ctx context.Context, a Alert) {
+	a.Enrich(r.resolver)
 	r.mu.Lock()
 	r.recent = append(r.recent, a)
 	if len(r.recent) > maxRecent {