fix

internal/alerts/alerts_test.go

@@ -128,3 +128,38 @@ func TestTelegramTruncates(t *testing.T) {
 		t.Fatalf("message too long: %d", n)
 	}
 }
+
+func TestScannerClientIsMutedEntirely(t *testing.T) {
+	at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	cur := at.Add(-2 * time.Minute)
+	scanner := func(route string, n int) []features.Event {
+		ev := burst("p-"+route, n, cur, 503, route)
+		for i := range ev {
+			ev[i].ClientIP = "198.51.100.9"
+		}
+		return ev
+	}
+	var events []features.Event
+	for _, r := range []string{"/.env", "/.cursorrules", "/.codex/auth.json"} {
+		events = append(events, scanner(r, 4)...)
+	}
+	events = append(events, scanner("/", 12)...) // the scanner's innocent-looking root request
+	if got := DetectAlerts(events, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5}); len(got) != 0 {
+		t.Fatalf("scanner traffic alerted: %+v", got)
+	}
+	// a different client with the same 503 on "/" still alerts
+	other := burst("real-503", 12, cur, 503, "/")
+	if got := DetectAlerts(other, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5}); len(got) != 1 {
+		t.Fatalf("real failure was muted: %+v", got)
+	}
+}
+
+func TestMinExcessAppliesAfterTriage(t *testing.T) {
+	at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	cur := at.Add(-2 * time.Minute)
+	// 200 probe errors + 6 real ones: total excess is large, actionable excess is 6 < 10
+	events := append(burst("probe", 200, cur, 404, "/.env"), burst("real", 6, cur, 503, "/api")...)
+	if got := DetectAlerts(events, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 10}); len(got) != 0 {
+		t.Fatalf("paged on excess that was mostly probes: %+v", got)
+	}
+}

internal/alerts/runner.go

@@ -198,7 +198,21 @@ func (r *Runner) Tick(ctx context.Context, now time.Time) {
 // DetectAlerts is the detection half of a tick: spikes reduced to actionable
 // patterns. Exported so the CLI and the runner cannot drift apart.
 func DetectAlerts(events []features.Event, at time.Time, cfg features.SpikeConfig) []features.Spike {
-	return FilterActionable(features.DetectSpikes(events, at, cfg), false, nil)
+	// Scanners, route enumerators and crawlers getting 4xx are noise for paging.
+	features.MarkBadActors(events, features.BadActorConfig{})
+	minExcess := cfg.MinExcess
+	if minExcess <= 0 {
+		minExcess = features.DefaultSpikeConfig().MinExcess
+	}
+	var out []features.Spike
+	// The excess threshold is applied after triage: a service whose total
+	// excess is mostly probes must not page on the little that is left.
+	for _, sp := range FilterActionable(features.DetectSpikes(events, at, cfg), false, nil) {
+		if sp.Excess >= float64(minExcess) {
+			out = append(out, sp)
+		}
+	}
+	return out
 }
 
 func (r *Runner) deliver(ctx context.Context, a Alert) {

internal/features/badactor.go

@@ -0,0 +1,71 @@
+package features
+
+import (
+	"git.tornberg.me/go-otel/internal/triage"
+	"git.tornberg.me/go-otel/internal/uaclass"
+)
+
+// Reasons an event is attributed to a bad actor. The empty string means it is
+// not. These are noise for alerting and for the model's label, not verdicts:
+// the rows are kept, so crawler traffic stays available as training data.
+const (
+	BadActorProbe      = "probe-scanner" // client hit several probe paths
+	BadActorEnumerator = "enumerator"    // client walked many routes that do not exist
+	BadActorBot        = "bot-4xx"       // self-declared crawler/agent/scanner getting a client error
+)
+
+// BadActorConfig sets the per-client thresholds, evaluated over the events
+// passed to MarkBadActors (a detector range or an export).
+type BadActorConfig struct {
+	ProbePaths int // distinct probe paths that make a client a scanner
+	EnumPaths  int // distinct not-found routes that make a client an enumerator
+}
+
+func DefaultBadActorConfig() BadActorConfig { return BadActorConfig{ProbePaths: 3, EnumPaths: 15} }
+
+// MarkBadActors sets Event.BadActor in place.
+//
+// Client rules need a resolved ClientIP (see internal/tracectx) and apply to
+// every event of the client, so a scanner's innocent-looking 503 on "/" is
+// covered too. The bot rule needs only the User-Agent class, and covers 4xx
+// only: a crawler that provokes a 5xx has found a real defect.
+func MarkBadActors(events []Event, cfg BadActorConfig) {
+	d := DefaultBadActorConfig()
+	if cfg.ProbePaths <= 0 {
+		cfg.ProbePaths = d.ProbePaths
+	}
+	if cfg.EnumPaths <= 0 {
+		cfg.EnumPaths = d.EnumPaths
+	}
+	probe := map[string]map[string]struct{}{}
+	missing := map[string]map[string]struct{}{}
+	add := func(m map[string]map[string]struct{}, ip, route string) {
+		if m[ip] == nil {
+			m[ip] = map[string]struct{}{}
+		}
+		m[ip][route] = struct{}{}
+	}
+	for _, e := range events {
+		if e.ClientIP == "" || e.IsDuplicate {
+			continue
+		}
+		if triage.IsProbePath(e.Route) {
+			add(probe, e.ClientIP, e.Route)
+		}
+		switch e.StatusCode {
+		case 400, 404, 405, 410:
+			add(missing, e.ClientIP, e.Route)
+		}
+	}
+	for i := range events {
+		e := &events[i]
+		switch {
+		case e.ClientIP != "" && len(probe[e.ClientIP]) >= cfg.ProbePaths:
+			e.BadActor = BadActorProbe
+		case e.ClientIP != "" && len(missing[e.ClientIP]) >= cfg.EnumPaths:
+			e.BadActor = BadActorEnumerator
+		case uaclass.IsCrawler(e.UAClass) && e.StatusCode >= 400 && e.StatusCode < 500:
+			e.BadActor = BadActorBot
+		}
+	}
+}

internal/features/features.go

@@ -41,6 +41,7 @@ type Event struct {
 	IsDuplicate bool   // repeats an ancestor span's failure; not a new request
 	IsRootCause bool   // error span with no erroring child
 	HasUpstream bool   // an ingress span that had a child in another service
+	BadActor    string // reason this event is attributed to a bad actor (see MarkBadActors); empty if not
 }
 
 // Config controls feature and label windows. A zero Config receives the
@@ -116,13 +117,24 @@ type Row struct {
 	TopClientShare  float64 `json:"topClientShare"`
 	BotErrorShare   float64 `json:"botErrorShare"`
 
+	// Clean errors are logical requests (no duplicate hops) not attributed to a
+	// bad actor. Label targets the clean future error rate, so scanner and
+	// crawler noise does not teach the model to alert; RawLabel is the old
+	// definition on raw error counts, kept for comparison.
+	BadActorErrors int `json:"badActorErrors"`
+	CleanErrors    int `json:"cleanErrors"`
+
 	BaselineErrorRate float64 `json:"baselineErrorRate"`
+	BaselineCleanRate float64 `json:"baselineCleanRate"`
 	BaselineSamples   int     `json:"baselineSamples"`
 	FutureEventCount  int     `json:"futureEventCount"`
 	FutureErrorCount  int     `json:"futureErrorCount"`
 	FutureErrorRate   float64 `json:"futureErrorRate"`
+	FutureCleanErrors int     `json:"futureCleanErrors"`
+	FutureCleanRate   float64 `json:"futureCleanRate"`
 	LabelKnown        bool    `json:"labelKnown"`
 	Label             bool    `json:"label"`
+	RawLabel          bool    `json:"rawLabel"`
 }
 
 // Build aggregates every service/release over rolling observation windows.
@@ -181,16 +193,18 @@ func Build(input []Event, cfg Config) []Row {
 					row.NewIssueCount++
 				}
 			}
-			futureTotal, futureErrors := count(events, start.Add(cfg.Window), start.Add(cfg.Window).Add(cfg.Future), row.Service)
-			baselineRate, samples := baseline(events, start, cfg.Window, cfg.BaselineDays, row.Service)
-			row.FutureEventCount, row.FutureErrorCount = futureTotal, futureErrors
+			futureTotal, futureErrors, futureClean := count(events, start.Add(cfg.Window), start.Add(cfg.Window).Add(cfg.Future), row.Service)
+			baselineRate, baselineClean, samples := baseline(events, start, cfg.Window, cfg.BaselineDays, row.Service)
+			row.FutureEventCount, row.FutureErrorCount, row.FutureCleanErrors = futureTotal, futureErrors, futureClean
 			row.FutureErrorRate = rate(futureErrors, futureTotal)
-			row.BaselineErrorRate, row.BaselineSamples = baselineRate, samples
+			row.FutureCleanRate = rate(futureClean, futureTotal)
+			row.BaselineErrorRate, row.BaselineCleanRate, row.BaselineSamples = baselineRate, baselineClean, samples
 			// A target is only known once the input has reached its exclusive
 			// end. This deliberately withholds the tail of an export instead of
 			// treating an incomplete future interval as a negative label.
 			row.LabelKnown = samples >= cfg.MinBaselineSamples && !start.Add(cfg.Window).Add(cfg.Future).After(observedUntil)
-			row.Label = row.LabelKnown && futureErrors >= cfg.MinFutureErrors && row.FutureErrorRate >= cfg.SpikeMultiple*baselineRate
+			row.RawLabel = row.LabelKnown && futureErrors >= cfg.MinFutureErrors && row.FutureErrorRate >= cfg.SpikeMultiple*baselineRate
+			row.Label = row.LabelKnown && futureClean >= cfg.MinFutureErrors && row.FutureCleanRate >= cfg.SpikeMultiple*baselineClean
 			rows = append(rows, row)
 		}
 	}
@@ -250,7 +264,12 @@ func (a *accumulator) add(e Event) {
 	a.value.ErrorCount++
 	if e.IsDuplicate {
 		a.value.DuplicateErrors++
-	} else if e.ClientIP != "" {
+	} else if e.BadActor != "" {
+		a.value.BadActorErrors++
+	} else {
+		a.value.CleanErrors++
+	}
+	if !e.IsDuplicate && e.ClientIP != "" {
 		a.clients[e.ClientIP]++
 	}
 	if e.IsRootCause {
@@ -310,35 +329,41 @@ func (a *accumulator) row() Row {
 	return a.value
 }
 
-func count(events []Event, start, end time.Time, service string) (total, errors int) {
+// count returns events, raw errors and clean errors (logical requests not
+// attributed to a bad actor) for a service in [start, end).
+func count(events []Event, start, end time.Time, service string) (total, errors, clean int) {
 	for _, e := range events {
 		if e.Service == service && !e.Timestamp.Before(start) && e.Timestamp.Before(end) {
 			total++
 			if e.IsError {
 				errors++
+				if !e.IsDuplicate && e.BadActor == "" {
+					clean++
+				}
 			}
 		}
 	}
-	return total, errors
+	return total, errors, clean
 }
 
 // baseline samples the same UTC time window on the preceding calendar days.
 // All intervals end before the current observation start, so no future data
 // leaks into a training label.
-func baseline(events []Event, start time.Time, window time.Duration, days int, service string) (float64, int) {
-	var total, errors, samples int
+func baseline(events []Event, start time.Time, window time.Duration, days int, service string) (raw, clean float64, samples int) {
+	var total, errors, cleanErrors int
 	for day := 1; day <= days; day++ {
 		from := start.AddDate(0, 0, -day)
 		to := from.Add(window)
-		countTotal, countErrors := count(events, from, to, service)
+		countTotal, countErrors, countClean := count(events, from, to, service)
 		if countTotal == 0 {
 			continue
 		}
 		total += countTotal
 		errors += countErrors
+		cleanErrors += countClean
 		samples++
 	}
-	return rate(errors, total), samples
+	return rate(errors, total), rate(cleanErrors, total), samples
 }
 
 func percentile(values []float64, p float64) float64 {

internal/features/features_test.go

@@ -75,3 +75,35 @@ func TestBuildLogicalRequestFeatures(t *testing.T) {
 		t.Fatalf("bot share wrong: %v", r.BotErrorShare)
 	}
 }
+
+func TestBuildLabelIgnoresBadActorNoise(t *testing.T) {
+	base := time.Date(2026, 9, 15, 12, 0, 0, 0, time.UTC)
+	var events []Event
+	mk := func(at time.Time, err bool, bad string) Event {
+		return Event{Timestamp: at, Service: "api", Kind: "span", IsError: err, BadActor: bad, PatternSignature: "p"}
+	}
+	// seven days of a quiet baseline, then a future window of pure bot errors
+	for d := 0; d <= 7; d++ {
+		day := base.AddDate(0, 0, d)
+		for i := 0; i < 50; i++ {
+			events = append(events, mk(day.Add(time.Duration(i)*time.Second), i == 0, ""))
+		}
+	}
+	target := base.AddDate(0, 0, 7)
+	for i := 0; i < 30; i++ {
+		events = append(events, mk(target.Add(6*time.Minute+time.Duration(i)*time.Second), true, BadActorBot))
+	}
+	rows := Build(events, Config{ObservedUntil: target.Add(time.Hour)})
+	var found bool
+	for _, r := range rows {
+		if r.WindowStart.Equal(target) && r.Service == "api" {
+			found = true
+			if !r.LabelKnown || !r.RawLabel || r.Label {
+				t.Fatalf("raw label should fire on bot errors and clean label must not: %+v", r)
+			}
+		}
+	}
+	if !found {
+		t.Fatal("target row missing")
+	}
+}

internal/features/spike.go

@@ -94,10 +94,11 @@ func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike {
 	}
 	services := map[string]*svc{}
 	for _, e := range events {
+		// Bad actors (scanners, crawlers getting 4xx) are noise for alerting.
 		// A duplicate is a later hop of a request whose outermost error is
 		// already counted; counting it again multiplies every failure by the
 		// number of services it crossed.
-		if !e.IsError || e.IsDuplicate {
+		if !e.IsError || e.IsDuplicate || e.BadActor != "" {
 			continue
 		}
 		ts := e.Timestamp.UTC()

internal/features/spike_test.go

@@ -121,3 +121,37 @@ func TestDetectSpikesSkipsDuplicateHopsAndCountsClients(t *testing.T) {
 		t.Fatalf("clients = %d, want 2", got[0].Patterns[0].Clients)
 	}
 }
+
+func TestMarkBadActors(t *testing.T) {
+	at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	ev := func(ip, route string, status int, class string) Event {
+		return Event{Timestamp: at, Service: "nginx", IsError: true, ClientIP: ip, Route: route, StatusCode: status, UAClass: class}
+	}
+	var events []Event
+	for _, r := range []string{"/.env", "/.cursorrules", "/.codex/auth.json"} { // probe scanner
+		events = append(events, ev("198.51.100.1", r, 404, "browser"))
+	}
+	events = append(events, ev("198.51.100.1", "/", 503, "browser")) // its innocent request
+	for i := 0; i < 15; i++ {                                        // enumerator of ordinary-looking paths
+		events = append(events, ev("198.51.100.2", "/api/x"+string(rune('a'+i)), 404, "browser"))
+	}
+	events = append(events,
+		ev("203.0.113.5", "/missing", 404, "ai_crawler"),    // declared crawler 4xx: bad actor
+		ev("203.0.113.5", "/api/orders", 500, "ai_crawler"), // ...but its 5xx is a real defect
+		ev("203.0.113.6", "/api/orders", 500, "browser"),
+		ev("203.0.113.7", "/missing", 404, "library"), // libraries are not crawlers
+	)
+	MarkBadActors(events, BadActorConfig{})
+	got := map[string]int{}
+	for _, e := range events {
+		got[e.BadActor]++
+	}
+	if got[BadActorProbe] != 4 || got[BadActorEnumerator] != 15 || got[BadActorBot] != 1 || got[""] != 3 {
+		t.Fatalf("marking wrong: %+v", got)
+	}
+	for _, e := range events {
+		if e.ClientIP == "203.0.113.5" && e.StatusCode == 500 && e.BadActor != "" {
+			t.Fatal("a crawler-provoked 5xx must stay visible")
+		}
+	}
+}

internal/issues/route_test.go

@@ -0,0 +1,18 @@
+package issues
+
+import "testing"
+
+func TestNormalizeRoutePrefixedIDs(t *testing.T) {
+	cases := map[string]string{
+		"/api/terminals/term_ab315d5ce24c4662b34a8ad998be79cf/ws": "/api/terminals/{id}/ws",
+		"/api/sessions/sess-9f2c41aa77/events":                    "/api/sessions/{id}/events",
+		"/api/skills":                                             "/api/skills",
+		"/api/v2/settings":                                        "/api/v2/settings",
+		"/api/user_settings":                                      "/api/user_settings",
+	}
+	for in, want := range cases {
+		if got := NormalizeRoute(in); got != want {
+			t.Errorf("NormalizeRoute(%q) = %q, want %q", in, got, want)
+		}
+	}
+}

internal/issues/tracker.go

@@ -346,7 +346,7 @@ func NormalizeRoute(route string) string {
 		if part == "" {
 			continue
 		}
-		if uuidRe.MatchString(part) || hexIDRe.MatchString(part) || numRe.MatchString(part) {
+		if uuidRe.MatchString(part) || hexIDRe.MatchString(part) || numRe.MatchString(part) || prefixedIDRe.MatchString(part) {
 			parts[i] = "{id}"
 		}
 	}
@@ -623,6 +623,9 @@ var (
 	hexIDRe      = regexp.MustCompile(`(?i)\b[0-9a-f]{8,}\b`)
 	httpStatusRe = regexp.MustCompile(`(?i)\bhttp\s+([1-5]\d{2})\b`)
 	numRe        = regexp.MustCompile(`\b\d+\b`)
+	// A route segment that is a short prefix plus a long hex id (term_ab31…,
+	// sess-9f2c…): \b does not fire between "_" and a hex digit, so hexIDRe misses it.
+	prefixedIDRe = regexp.MustCompile(`(?i)^[a-z]{1,12}[_-][0-9a-f]{8,}$`)
 	spaceRe      = regexp.MustCompile(`\s+`)
 )
 

internal/triage/triage.go

@@ -47,7 +47,8 @@ var (
 	// control, CMS/admin paths, path traversal. Deliberately path-only.
 	probeRoute = regexp.MustCompile(`(?i)(\.env\b|\.git\b|\.htpasswd|\.htaccess|\.npmrc|\.claude|\.docker|\.kube|\.aws|\.ssh|` +
 		`wp-|/wp/|/wp$|wordpress|phpmyadmin|phpinfo|cgi-bin|xmlrpc|userfiles|@fs/|etc/passwd|proc/self|\.\./|%2e%2e|` +
-		`\.php\b|_debugbar|debug/pprof|server-info|actuator|__env|\.bash_|\.profile$|graphql/console|` +
+		`\.php\b|\.codex|\.cursor|\.windsurf|\.aider|\.clinerules|\.roo(/|code|$)|\.continue/|\.gemini|\.idea/|\.vscode/|copilot-instructions|claude\.md|agents\.md|mcp\.json|` +
+		`_debugbar|debug/pprof|server-info|actuator|__env|\.bash_|\.profile$|graphql/console|` +
 		`credentials|secrets?\b|firebase|serviceaccount|/config(/|\.)|settings\.(js|json|py)$|bootstrap\.properties|` +
 		`\.(ya?ml|toml|properties|sql|bak|old|swp|zip|tar|tgz|gz|ini|conf|cfg|key|pem)$)`)
 	bugText  = regexp.MustCompile(`(?i)panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault`)

internal/triage/triage_test.go

@@ -41,3 +41,16 @@ func TestProbeBeatsLifecycleFlags(t *testing.T) {
 		t.Fatalf("got %s", got.Category)
 	}
 }
+
+func TestAIToolConfigPathsAreProbes(t *testing.T) {
+	for _, p := range []string{"/.cursorrules", "/.codex/auth.json", "/backup/.claude/credentials.json", "/.windsurfrules", "/.vscode/launch.json", "/.idea/WebServers.xml", "/x/.github/copilot-instructions.md", "/CLAUDE.md", "/mcp.json", "/.clinerules"} {
+		if !IsProbePath(p) {
+			t.Errorf("%s should be a probe", p)
+		}
+	}
+	for _, p := range []string{"/mcp", "/api/agents", "/api/sessions/{id}", "/api/continue"} {
+		if IsProbePath(p) {
+			t.Errorf("%s must not be a probe", p)
+		}
+	}
+}

internal/uaclass/uaclass.go

@@ -269,3 +269,14 @@ func IsBot(class string) bool {
 	}
 	return false
 }
+
+// IsCrawler reports self-declared crawlers, agents and scanners: the classes
+// whose 4xx noise is not user impact. Monitors and plain HTTP libraries are
+// excluded because their failures can be real.
+func IsCrawler(class string) bool {
+	switch class {
+	case AICrawler, AIAgent, SearchCrawler, SEOCrawler, ScanTool, OtherBot:
+		return true
+	}
+	return false
+}