fix

internal/features/badactor.go

@@ -0,0 +1,71 @@
+package features
+
+import (
+	"git.tornberg.me/go-otel/internal/triage"
+	"git.tornberg.me/go-otel/internal/uaclass"
+)
+
+// Reasons an event is attributed to a bad actor. The empty string means it is
+// not. These are noise for alerting and for the model's label, not verdicts:
+// the rows are kept, so crawler traffic stays available as training data.
+const (
+	BadActorProbe      = "probe-scanner" // client hit several probe paths
+	BadActorEnumerator = "enumerator"    // client walked many routes that do not exist
+	BadActorBot        = "bot-4xx"       // self-declared crawler/agent/scanner getting a client error
+)
+
+// BadActorConfig sets the per-client thresholds, evaluated over the events
+// passed to MarkBadActors (a detector range or an export).
+type BadActorConfig struct {
+	ProbePaths int // distinct probe paths that make a client a scanner
+	EnumPaths  int // distinct not-found routes that make a client an enumerator
+}
+
+func DefaultBadActorConfig() BadActorConfig { return BadActorConfig{ProbePaths: 3, EnumPaths: 15} }
+
+// MarkBadActors sets Event.BadActor in place.
+//
+// Client rules need a resolved ClientIP (see internal/tracectx) and apply to
+// every event of the client, so a scanner's innocent-looking 503 on "/" is
+// covered too. The bot rule needs only the User-Agent class, and covers 4xx
+// only: a crawler that provokes a 5xx has found a real defect.
+func MarkBadActors(events []Event, cfg BadActorConfig) {
+	d := DefaultBadActorConfig()
+	if cfg.ProbePaths <= 0 {
+		cfg.ProbePaths = d.ProbePaths
+	}
+	if cfg.EnumPaths <= 0 {
+		cfg.EnumPaths = d.EnumPaths
+	}
+	probe := map[string]map[string]struct{}{}
+	missing := map[string]map[string]struct{}{}
+	add := func(m map[string]map[string]struct{}, ip, route string) {
+		if m[ip] == nil {
+			m[ip] = map[string]struct{}{}
+		}
+		m[ip][route] = struct{}{}
+	}
+	for _, e := range events {
+		if e.ClientIP == "" || e.IsDuplicate {
+			continue
+		}
+		if triage.IsProbePath(e.Route) {
+			add(probe, e.ClientIP, e.Route)
+		}
+		switch e.StatusCode {
+		case 400, 404, 405, 410:
+			add(missing, e.ClientIP, e.Route)
+		}
+	}
+	for i := range events {
+		e := &events[i]
+		switch {
+		case e.ClientIP != "" && len(probe[e.ClientIP]) >= cfg.ProbePaths:
+			e.BadActor = BadActorProbe
+		case e.ClientIP != "" && len(missing[e.ClientIP]) >= cfg.EnumPaths:
+			e.BadActor = BadActorEnumerator
+		case uaclass.IsCrawler(e.UAClass) && e.StatusCode >= 400 && e.StatusCode < 500:
+			e.BadActor = BadActorBot
+		}
+	}
+}

internal/features/features.go

@@ -41,6 +41,7 @@ type Event struct {
 	IsDuplicate bool   // repeats an ancestor span's failure; not a new request
 	IsRootCause bool   // error span with no erroring child
 	HasUpstream bool   // an ingress span that had a child in another service
+	BadActor    string // reason this event is attributed to a bad actor (see MarkBadActors); empty if not
 }
 
 // Config controls feature and label windows. A zero Config receives the
@@ -116,13 +117,24 @@ type Row struct {
 	TopClientShare  float64 `json:"topClientShare"`
 	BotErrorShare   float64 `json:"botErrorShare"`
 
+	// Clean errors are logical requests (no duplicate hops) not attributed to a
+	// bad actor. Label targets the clean future error rate, so scanner and
+	// crawler noise does not teach the model to alert; RawLabel is the old
+	// definition on raw error counts, kept for comparison.
+	BadActorErrors int `json:"badActorErrors"`
+	CleanErrors    int `json:"cleanErrors"`
+
 	BaselineErrorRate float64 `json:"baselineErrorRate"`
+	BaselineCleanRate float64 `json:"baselineCleanRate"`
 	BaselineSamples   int     `json:"baselineSamples"`
 	FutureEventCount  int     `json:"futureEventCount"`
 	FutureErrorCount  int     `json:"futureErrorCount"`
 	FutureErrorRate   float64 `json:"futureErrorRate"`
+	FutureCleanErrors int     `json:"futureCleanErrors"`
+	FutureCleanRate   float64 `json:"futureCleanRate"`
 	LabelKnown        bool    `json:"labelKnown"`
 	Label             bool    `json:"label"`
+	RawLabel          bool    `json:"rawLabel"`
 }
 
 // Build aggregates every service/release over rolling observation windows.
@@ -181,16 +193,18 @@ func Build(input []Event, cfg Config) []Row {
 					row.NewIssueCount++
 				}
 			}
-			futureTotal, futureErrors := count(events, start.Add(cfg.Window), start.Add(cfg.Window).Add(cfg.Future), row.Service)
-			baselineRate, samples := baseline(events, start, cfg.Window, cfg.BaselineDays, row.Service)
-			row.FutureEventCount, row.FutureErrorCount = futureTotal, futureErrors
+			futureTotal, futureErrors, futureClean := count(events, start.Add(cfg.Window), start.Add(cfg.Window).Add(cfg.Future), row.Service)
+			baselineRate, baselineClean, samples := baseline(events, start, cfg.Window, cfg.BaselineDays, row.Service)
+			row.FutureEventCount, row.FutureErrorCount, row.FutureCleanErrors = futureTotal, futureErrors, futureClean
 			row.FutureErrorRate = rate(futureErrors, futureTotal)
-			row.BaselineErrorRate, row.BaselineSamples = baselineRate, samples
+			row.FutureCleanRate = rate(futureClean, futureTotal)
+			row.BaselineErrorRate, row.BaselineCleanRate, row.BaselineSamples = baselineRate, baselineClean, samples
 			// A target is only known once the input has reached its exclusive
 			// end. This deliberately withholds the tail of an export instead of
 			// treating an incomplete future interval as a negative label.
 			row.LabelKnown = samples >= cfg.MinBaselineSamples && !start.Add(cfg.Window).Add(cfg.Future).After(observedUntil)
-			row.Label = row.LabelKnown && futureErrors >= cfg.MinFutureErrors && row.FutureErrorRate >= cfg.SpikeMultiple*baselineRate
+			row.RawLabel = row.LabelKnown && futureErrors >= cfg.MinFutureErrors && row.FutureErrorRate >= cfg.SpikeMultiple*baselineRate
+			row.Label = row.LabelKnown && futureClean >= cfg.MinFutureErrors && row.FutureCleanRate >= cfg.SpikeMultiple*baselineClean
 			rows = append(rows, row)
 		}
 	}
@@ -250,7 +264,12 @@ func (a *accumulator) add(e Event) {
 	a.value.ErrorCount++
 	if e.IsDuplicate {
 		a.value.DuplicateErrors++
-	} else if e.ClientIP != "" {
+	} else if e.BadActor != "" {
+		a.value.BadActorErrors++
+	} else {
+		a.value.CleanErrors++
+	}
+	if !e.IsDuplicate && e.ClientIP != "" {
 		a.clients[e.ClientIP]++
 	}
 	if e.IsRootCause {
@@ -310,35 +329,41 @@ func (a *accumulator) row() Row {
 	return a.value
 }
 
-func count(events []Event, start, end time.Time, service string) (total, errors int) {
+// count returns events, raw errors and clean errors (logical requests not
+// attributed to a bad actor) for a service in [start, end).
+func count(events []Event, start, end time.Time, service string) (total, errors, clean int) {
 	for _, e := range events {
 		if e.Service == service && !e.Timestamp.Before(start) && e.Timestamp.Before(end) {
 			total++
 			if e.IsError {
 				errors++
+				if !e.IsDuplicate && e.BadActor == "" {
+					clean++
+				}
 			}
 		}
 	}
-	return total, errors
+	return total, errors, clean
 }
 
 // baseline samples the same UTC time window on the preceding calendar days.
 // All intervals end before the current observation start, so no future data
 // leaks into a training label.
-func baseline(events []Event, start time.Time, window time.Duration, days int, service string) (float64, int) {
-	var total, errors, samples int
+func baseline(events []Event, start time.Time, window time.Duration, days int, service string) (raw, clean float64, samples int) {
+	var total, errors, cleanErrors int
 	for day := 1; day <= days; day++ {
 		from := start.AddDate(0, 0, -day)
 		to := from.Add(window)
-		countTotal, countErrors := count(events, from, to, service)
+		countTotal, countErrors, countClean := count(events, from, to, service)
 		if countTotal == 0 {
 			continue
 		}
 		total += countTotal
 		errors += countErrors
+		cleanErrors += countClean
 		samples++
 	}
-	return rate(errors, total), samples
+	return rate(errors, total), rate(cleanErrors, total), samples
 }
 
 func percentile(values []float64, p float64) float64 {

internal/features/features_test.go

@@ -75,3 +75,35 @@ func TestBuildLogicalRequestFeatures(t *testing.T) {
 		t.Fatalf("bot share wrong: %v", r.BotErrorShare)
 	}
 }
+
+func TestBuildLabelIgnoresBadActorNoise(t *testing.T) {
+	base := time.Date(2026, 9, 15, 12, 0, 0, 0, time.UTC)
+	var events []Event
+	mk := func(at time.Time, err bool, bad string) Event {
+		return Event{Timestamp: at, Service: "api", Kind: "span", IsError: err, BadActor: bad, PatternSignature: "p"}
+	}
+	// seven days of a quiet baseline, then a future window of pure bot errors
+	for d := 0; d <= 7; d++ {
+		day := base.AddDate(0, 0, d)
+		for i := 0; i < 50; i++ {
+			events = append(events, mk(day.Add(time.Duration(i)*time.Second), i == 0, ""))
+		}
+	}
+	target := base.AddDate(0, 0, 7)
+	for i := 0; i < 30; i++ {
+		events = append(events, mk(target.Add(6*time.Minute+time.Duration(i)*time.Second), true, BadActorBot))
+	}
+	rows := Build(events, Config{ObservedUntil: target.Add(time.Hour)})
+	var found bool
+	for _, r := range rows {
+		if r.WindowStart.Equal(target) && r.Service == "api" {
+			found = true
+			if !r.LabelKnown || !r.RawLabel || r.Label {
+				t.Fatalf("raw label should fire on bot errors and clean label must not: %+v", r)
+			}
+		}
+	}
+	if !found {
+		t.Fatal("target row missing")
+	}
+}

internal/features/spike.go

@@ -94,10 +94,11 @@ func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike {
 	}
 	services := map[string]*svc{}
 	for _, e := range events {
+		// Bad actors (scanners, crawlers getting 4xx) are noise for alerting.
 		// A duplicate is a later hop of a request whose outermost error is
 		// already counted; counting it again multiplies every failure by the
 		// number of services it crossed.
-		if !e.IsError || e.IsDuplicate {
+		if !e.IsError || e.IsDuplicate || e.BadActor != "" {
 			continue
 		}
 		ts := e.Timestamp.UTC()

internal/features/spike_test.go

@@ -121,3 +121,37 @@ func TestDetectSpikesSkipsDuplicateHopsAndCountsClients(t *testing.T) {
 		t.Fatalf("clients = %d, want 2", got[0].Patterns[0].Clients)
 	}
 }
+
+func TestMarkBadActors(t *testing.T) {
+	at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	ev := func(ip, route string, status int, class string) Event {
+		return Event{Timestamp: at, Service: "nginx", IsError: true, ClientIP: ip, Route: route, StatusCode: status, UAClass: class}
+	}
+	var events []Event
+	for _, r := range []string{"/.env", "/.cursorrules", "/.codex/auth.json"} { // probe scanner
+		events = append(events, ev("198.51.100.1", r, 404, "browser"))
+	}
+	events = append(events, ev("198.51.100.1", "/", 503, "browser")) // its innocent request
+	for i := 0; i < 15; i++ {                                        // enumerator of ordinary-looking paths
+		events = append(events, ev("198.51.100.2", "/api/x"+string(rune('a'+i)), 404, "browser"))
+	}
+	events = append(events,
+		ev("203.0.113.5", "/missing", 404, "ai_crawler"),    // declared crawler 4xx: bad actor
+		ev("203.0.113.5", "/api/orders", 500, "ai_crawler"), // ...but its 5xx is a real defect
+		ev("203.0.113.6", "/api/orders", 500, "browser"),
+		ev("203.0.113.7", "/missing", 404, "library"), // libraries are not crawlers
+	)
+	MarkBadActors(events, BadActorConfig{})
+	got := map[string]int{}
+	for _, e := range events {
+		got[e.BadActor]++
+	}
+	if got[BadActorProbe] != 4 || got[BadActorEnumerator] != 15 || got[BadActorBot] != 1 || got[""] != 3 {
+		t.Fatalf("marking wrong: %+v", got)
+	}
+	for _, e := range events {
+		if e.ClientIP == "203.0.113.5" && e.StatusCode == 500 && e.BadActor != "" {
+			t.Fatal("a crawler-provoked 5xx must stay visible")
+		}
+	}
+}