fix

cmd/spike-alerts/main.go

@@ -65,6 +65,14 @@ func main() {
 		log.Fatal(err)
 	}
 	cfg.DataSince = alerts.DataSince(ctx, conn, *database)
+	features.MarkBadActors(events, features.BadActorConfig{})
+	badActors := map[string]int{}
+	for _, e := range events {
+		if e.BadActor != "" {
+			badActors[e.BadActor]++
+		}
+	}
+	log.Printf("bad-actor events (ignored for alerts, kept as data): %v of %d", badActors, len(events))
 	log.Printf("loaded %d error events", len(events))
 
 	cache := map[string]classification{}
@@ -73,7 +81,11 @@ func main() {
 	muted := map[triage.Category]int{}
 	fired := 0
 	for at := first.Add(*window); !at.After(end); at = at.Add(*step) {
-		for _, spike := range suppress.Filter(alerts.FilterActionable(features.DetectSpikes(events, at, cfg), *showMuted, muted)) {
+		detected := alerts.DetectAlerts(events, at, cfg) // exactly what the server alerts on
+		if *showMuted {
+			detected = alerts.FilterActionable(features.DetectSpikes(events, at, cfg), true, muted)
+		}
+		for _, spike := range suppress.Filter(detected) {
 			fired++
 			var cls []classification
 			for _, p := range spike.Patterns {

cmd/window-features/main.go

@@ -107,5 +107,6 @@ func readEvents(path string) []features.Event {
 	if err := scanner.Err(); err != nil {
 		log.Fatal(err)
 	}
+	features.MarkBadActors(result, features.BadActorConfig{})
 	return result
 }

docs/STATUS.md

@@ -143,6 +143,22 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue`
     orchestrator pattern signature); `HasUpstream` is loaded but not yet a triage
     rule; trace propagation is missing in cart/checkout/profile/scheduling (roots only).
 
+- Alerts (`internal/alerts`, env `OTEL_ALERT_*`, secret `otel-alerts`): the server
+  runs the spike detector every minute and posts to Telegram (and/or a generic
+  webhook). First tick after a start only primes the suppressor. `GET /api/alerts`,
+  `POST /api/alerts/test`, `go run ./cmd/alert-test`. Recent alerts are in memory only.
+- Bad actors (`features.MarkBadActors`): an event is marked, never dropped, when its
+  client hit >=3 probe paths (`probe-scanner`), walked >=15 not-found routes
+  (`enumerator`), or is a declared crawler/agent/scanner getting a 4xx (`bot-4xx`;
+  their 5xx stay visible). Marked events are ignored by `DetectSpikes`/alerts and
+  counted in `Row.BadActorErrors`. **`Row.Label` now targets the clean future error
+  rate** (logical requests, no bad actors); `RawLabel` is the old raw-count label. AI
+  crawler rows stay in the data on purpose (see memory: flag, not block).
+  Client rules need a resolved client IP, so they only cover data since the trace
+  columns shipped; the bot rule works on any row with a UA class.
+- Route ids: `term_<hex>` style segments normalize to `{id}` (per-session websocket
+  routes were creating a new pattern each session).
+
 ## Commands
 
 ```bash

internal/alerts/alerts_test.go

@@ -128,3 +128,38 @@ func TestTelegramTruncates(t *testing.T) {
 		t.Fatalf("message too long: %d", n)
 	}
 }
+
+func TestScannerClientIsMutedEntirely(t *testing.T) {
+	at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	cur := at.Add(-2 * time.Minute)
+	scanner := func(route string, n int) []features.Event {
+		ev := burst("p-"+route, n, cur, 503, route)
+		for i := range ev {
+			ev[i].ClientIP = "198.51.100.9"
+		}
+		return ev
+	}
+	var events []features.Event
+	for _, r := range []string{"/.env", "/.cursorrules", "/.codex/auth.json"} {
+		events = append(events, scanner(r, 4)...)
+	}
+	events = append(events, scanner("/", 12)...) // the scanner's innocent-looking root request
+	if got := DetectAlerts(events, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5}); len(got) != 0 {
+		t.Fatalf("scanner traffic alerted: %+v", got)
+	}
+	// a different client with the same 503 on "/" still alerts
+	other := burst("real-503", 12, cur, 503, "/")
+	if got := DetectAlerts(other, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5}); len(got) != 1 {
+		t.Fatalf("real failure was muted: %+v", got)
+	}
+}
+
+func TestMinExcessAppliesAfterTriage(t *testing.T) {
+	at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	cur := at.Add(-2 * time.Minute)
+	// 200 probe errors + 6 real ones: total excess is large, actionable excess is 6 < 10
+	events := append(burst("probe", 200, cur, 404, "/.env"), burst("real", 6, cur, 503, "/api")...)
+	if got := DetectAlerts(events, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 10}); len(got) != 0 {
+		t.Fatalf("paged on excess that was mostly probes: %+v", got)
+	}
+}

internal/alerts/runner.go

@@ -198,7 +198,21 @@ func (r *Runner) Tick(ctx context.Context, now time.Time) {
 // DetectAlerts is the detection half of a tick: spikes reduced to actionable
 // patterns. Exported so the CLI and the runner cannot drift apart.
 func DetectAlerts(events []features.Event, at time.Time, cfg features.SpikeConfig) []features.Spike {
-	return FilterActionable(features.DetectSpikes(events, at, cfg), false, nil)
+	// Scanners, route enumerators and crawlers getting 4xx are noise for paging.
+	features.MarkBadActors(events, features.BadActorConfig{})
+	minExcess := cfg.MinExcess
+	if minExcess <= 0 {
+		minExcess = features.DefaultSpikeConfig().MinExcess
+	}
+	var out []features.Spike
+	// The excess threshold is applied after triage: a service whose total
+	// excess is mostly probes must not page on the little that is left.
+	for _, sp := range FilterActionable(features.DetectSpikes(events, at, cfg), false, nil) {
+		if sp.Excess >= float64(minExcess) {
+			out = append(out, sp)
+		}
+	}
+	return out
 }
 
 func (r *Runner) deliver(ctx context.Context, a Alert) {

internal/features/badactor.go

@@ -0,0 +1,71 @@
+package features
+
+import (
+	"git.tornberg.me/go-otel/internal/triage"
+	"git.tornberg.me/go-otel/internal/uaclass"
+)
+
+// Reasons an event is attributed to a bad actor. The empty string means it is
+// not. These are noise for alerting and for the model's label, not verdicts:
+// the rows are kept, so crawler traffic stays available as training data.
+const (
+	BadActorProbe      = "probe-scanner" // client hit several probe paths
+	BadActorEnumerator = "enumerator"    // client walked many routes that do not exist
+	BadActorBot        = "bot-4xx"       // self-declared crawler/agent/scanner getting a client error
+)
+
+// BadActorConfig sets the per-client thresholds, evaluated over the events
+// passed to MarkBadActors (a detector range or an export).
+type BadActorConfig struct {
+	ProbePaths int // distinct probe paths that make a client a scanner
+	EnumPaths  int // distinct not-found routes that make a client an enumerator
+}
+
+func DefaultBadActorConfig() BadActorConfig { return BadActorConfig{ProbePaths: 3, EnumPaths: 15} }
+
+// MarkBadActors sets Event.BadActor in place.
+//
+// Client rules need a resolved ClientIP (see internal/tracectx) and apply to
+// every event of the client, so a scanner's innocent-looking 503 on "/" is
+// covered too. The bot rule needs only the User-Agent class, and covers 4xx
+// only: a crawler that provokes a 5xx has found a real defect.
+func MarkBadActors(events []Event, cfg BadActorConfig) {
+	d := DefaultBadActorConfig()
+	if cfg.ProbePaths <= 0 {
+		cfg.ProbePaths = d.ProbePaths
+	}
+	if cfg.EnumPaths <= 0 {
+		cfg.EnumPaths = d.EnumPaths
+	}
+	probe := map[string]map[string]struct{}{}
+	missing := map[string]map[string]struct{}{}
+	add := func(m map[string]map[string]struct{}, ip, route string) {
+		if m[ip] == nil {
+			m[ip] = map[string]struct{}{}
+		}
+		m[ip][route] = struct{}{}
+	}
+	for _, e := range events {
+		if e.ClientIP == "" || e.IsDuplicate {
+			continue
+		}
+		if triage.IsProbePath(e.Route) {
+			add(probe, e.ClientIP, e.Route)
+		}
+		switch e.StatusCode {
+		case 400, 404, 405, 410:
+			add(missing, e.ClientIP, e.Route)
+		}
+	}
+	for i := range events {
+		e := &events[i]
+		switch {
+		case e.ClientIP != "" && len(probe[e.ClientIP]) >= cfg.ProbePaths:
+			e.BadActor = BadActorProbe
+		case e.ClientIP != "" && len(missing[e.ClientIP]) >= cfg.EnumPaths:
+			e.BadActor = BadActorEnumerator
+		case uaclass.IsCrawler(e.UAClass) && e.StatusCode >= 400 && e.StatusCode < 500:
+			e.BadActor = BadActorBot
+		}
+	}
+}

internal/features/features.go

@@ -41,6 +41,7 @@ type Event struct {
 	IsDuplicate bool   // repeats an ancestor span's failure; not a new request
 	IsRootCause bool   // error span with no erroring child
 	HasUpstream bool   // an ingress span that had a child in another service
+	BadActor    string // reason this event is attributed to a bad actor (see MarkBadActors); empty if not
 }
 
 // Config controls feature and label windows. A zero Config receives the
@@ -116,13 +117,24 @@ type Row struct {
 	TopClientShare  float64 `json:"topClientShare"`
 	BotErrorShare   float64 `json:"botErrorShare"`
 
+	// Clean errors are logical requests (no duplicate hops) not attributed to a
+	// bad actor. Label targets the clean future error rate, so scanner and
+	// crawler noise does not teach the model to alert; RawLabel is the old
+	// definition on raw error counts, kept for comparison.
+	BadActorErrors int `json:"badActorErrors"`
+	CleanErrors    int `json:"cleanErrors"`
+
 	BaselineErrorRate float64 `json:"baselineErrorRate"`
+	BaselineCleanRate float64 `json:"baselineCleanRate"`
 	BaselineSamples   int     `json:"baselineSamples"`
 	FutureEventCount  int     `json:"futureEventCount"`
 	FutureErrorCount  int     `json:"futureErrorCount"`
 	FutureErrorRate   float64 `json:"futureErrorRate"`
+	FutureCleanErrors int     `json:"futureCleanErrors"`
+	FutureCleanRate   float64 `json:"futureCleanRate"`
 	LabelKnown        bool    `json:"labelKnown"`
 	Label             bool    `json:"label"`
+	RawLabel          bool    `json:"rawLabel"`
 }
 
 // Build aggregates every service/release over rolling observation windows.
@@ -181,16 +193,18 @@ func Build(input []Event, cfg Config) []Row {
 					row.NewIssueCount++
 				}
 			}
-			futureTotal, futureErrors := count(events, start.Add(cfg.Window), start.Add(cfg.Window).Add(cfg.Future), row.Service)
+			futureTotal, futureErrors, futureClean := count(events, start.Add(cfg.Window), start.Add(cfg.Window).Add(cfg.Future), row.Service)
-			baselineRate, samples := baseline(events, start, cfg.Window, cfg.BaselineDays, row.Service)
+			baselineRate, baselineClean, samples := baseline(events, start, cfg.Window, cfg.BaselineDays, row.Service)
-			row.FutureEventCount, row.FutureErrorCount = futureTotal, futureErrors
+			row.FutureEventCount, row.FutureErrorCount, row.FutureCleanErrors = futureTotal, futureErrors, futureClean
 			row.FutureErrorRate = rate(futureErrors, futureTotal)
-			row.BaselineErrorRate, row.BaselineSamples = baselineRate, samples
+			row.FutureCleanRate = rate(futureClean, futureTotal)
+			row.BaselineErrorRate, row.BaselineCleanRate, row.BaselineSamples = baselineRate, baselineClean, samples
 			// A target is only known once the input has reached its exclusive
 			// end. This deliberately withholds the tail of an export instead of
 			// treating an incomplete future interval as a negative label.
 			row.LabelKnown = samples >= cfg.MinBaselineSamples && !start.Add(cfg.Window).Add(cfg.Future).After(observedUntil)
-			row.Label = row.LabelKnown && futureErrors >= cfg.MinFutureErrors && row.FutureErrorRate >= cfg.SpikeMultiple*baselineRate
+			row.RawLabel = row.LabelKnown && futureErrors >= cfg.MinFutureErrors && row.FutureErrorRate >= cfg.SpikeMultiple*baselineRate
+			row.Label = row.LabelKnown && futureClean >= cfg.MinFutureErrors && row.FutureCleanRate >= cfg.SpikeMultiple*baselineClean
 			rows = append(rows, row)
 		}
 	}
@@ -250,7 +264,12 @@ func (a *accumulator) add(e Event) {
 	a.value.ErrorCount++
 	if e.IsDuplicate {
 		a.value.DuplicateErrors++
-	} else if e.ClientIP != "" {
+	} else if e.BadActor != "" {
+		a.value.BadActorErrors++
+	} else {
+		a.value.CleanErrors++
+	}
+	if !e.IsDuplicate && e.ClientIP != "" {
 		a.clients[e.ClientIP]++
 	}
 	if e.IsRootCause {
@@ -310,35 +329,41 @@ func (a *accumulator) row() Row {
 	return a.value
 }
 
-func count(events []Event, start, end time.Time, service string) (total, errors int) {
+// count returns events, raw errors and clean errors (logical requests not
+// attributed to a bad actor) for a service in [start, end).
+func count(events []Event, start, end time.Time, service string) (total, errors, clean int) {
 	for _, e := range events {
 		if e.Service == service && !e.Timestamp.Before(start) && e.Timestamp.Before(end) {
 			total++
 			if e.IsError {
 				errors++
+				if !e.IsDuplicate && e.BadActor == "" {
+					clean++
+				}
 			}
 		}
 	}
-	return total, errors
+	return total, errors, clean
 }
 
 // baseline samples the same UTC time window on the preceding calendar days.
 // All intervals end before the current observation start, so no future data
 // leaks into a training label.
-func baseline(events []Event, start time.Time, window time.Duration, days int, service string) (float64, int) {
+func baseline(events []Event, start time.Time, window time.Duration, days int, service string) (raw, clean float64, samples int) {
-	var total, errors, samples int
+	var total, errors, cleanErrors int
 	for day := 1; day <= days; day++ {
 		from := start.AddDate(0, 0, -day)
 		to := from.Add(window)
-		countTotal, countErrors := count(events, from, to, service)
+		countTotal, countErrors, countClean := count(events, from, to, service)
 		if countTotal == 0 {
 			continue
 		}
 		total += countTotal
 		errors += countErrors
+		cleanErrors += countClean
 		samples++
 	}
-	return rate(errors, total), samples
+	return rate(errors, total), rate(cleanErrors, total), samples
 }
 
 func percentile(values []float64, p float64) float64 {

internal/features/features_test.go

@@ -75,3 +75,35 @@ func TestBuildLogicalRequestFeatures(t *testing.T) {
 		t.Fatalf("bot share wrong: %v", r.BotErrorShare)
 	}
 }
+
+func TestBuildLabelIgnoresBadActorNoise(t *testing.T) {
+	base := time.Date(2026, 9, 15, 12, 0, 0, 0, time.UTC)
+	var events []Event
+	mk := func(at time.Time, err bool, bad string) Event {
+		return Event{Timestamp: at, Service: "api", Kind: "span", IsError: err, BadActor: bad, PatternSignature: "p"}
+	}
+	// seven days of a quiet baseline, then a future window of pure bot errors
+	for d := 0; d <= 7; d++ {
+		day := base.AddDate(0, 0, d)
+		for i := 0; i < 50; i++ {
+			events = append(events, mk(day.Add(time.Duration(i)*time.Second), i == 0, ""))
+		}
+	}
+	target := base.AddDate(0, 0, 7)
+	for i := 0; i < 30; i++ {
+		events = append(events, mk(target.Add(6*time.Minute+time.Duration(i)*time.Second), true, BadActorBot))
+	}
+	rows := Build(events, Config{ObservedUntil: target.Add(time.Hour)})
+	var found bool
+	for _, r := range rows {
+		if r.WindowStart.Equal(target) && r.Service == "api" {
+			found = true
+			if !r.LabelKnown || !r.RawLabel || r.Label {
+				t.Fatalf("raw label should fire on bot errors and clean label must not: %+v", r)
+			}
+		}
+	}
+	if !found {
+		t.Fatal("target row missing")
+	}
+}

internal/features/spike.go

@@ -94,10 +94,11 @@ func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike {
 	}
 	services := map[string]*svc{}
 	for _, e := range events {
+		// Bad actors (scanners, crawlers getting 4xx) are noise for alerting.
 		// A duplicate is a later hop of a request whose outermost error is
 		// already counted; counting it again multiplies every failure by the
 		// number of services it crossed.
-		if !e.IsError || e.IsDuplicate {
+		if !e.IsError || e.IsDuplicate || e.BadActor != "" {
 			continue
 		}
 		ts := e.Timestamp.UTC()

internal/features/spike_test.go

@@ -121,3 +121,37 @@ func TestDetectSpikesSkipsDuplicateHopsAndCountsClients(t *testing.T) {
 		t.Fatalf("clients = %d, want 2", got[0].Patterns[0].Clients)
 	}
 }
+
+func TestMarkBadActors(t *testing.T) {
+	at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	ev := func(ip, route string, status int, class string) Event {
+		return Event{Timestamp: at, Service: "nginx", IsError: true, ClientIP: ip, Route: route, StatusCode: status, UAClass: class}
+	}
+	var events []Event
+	for _, r := range []string{"/.env", "/.cursorrules", "/.codex/auth.json"} { // probe scanner
+		events = append(events, ev("198.51.100.1", r, 404, "browser"))
+	}
+	events = append(events, ev("198.51.100.1", "/", 503, "browser")) // its innocent request
+	for i := 0; i < 15; i++ {                                        // enumerator of ordinary-looking paths
+		events = append(events, ev("198.51.100.2", "/api/x"+string(rune('a'+i)), 404, "browser"))
+	}
+	events = append(events,
+		ev("203.0.113.5", "/missing", 404, "ai_crawler"),    // declared crawler 4xx: bad actor
+		ev("203.0.113.5", "/api/orders", 500, "ai_crawler"), // ...but its 5xx is a real defect
+		ev("203.0.113.6", "/api/orders", 500, "browser"),
+		ev("203.0.113.7", "/missing", 404, "library"), // libraries are not crawlers
+	)
+	MarkBadActors(events, BadActorConfig{})
+	got := map[string]int{}
+	for _, e := range events {
+		got[e.BadActor]++
+	}
+	if got[BadActorProbe] != 4 || got[BadActorEnumerator] != 15 || got[BadActorBot] != 1 || got[""] != 3 {
+		t.Fatalf("marking wrong: %+v", got)
+	}
+	for _, e := range events {
+		if e.ClientIP == "203.0.113.5" && e.StatusCode == 500 && e.BadActor != "" {
+			t.Fatal("a crawler-provoked 5xx must stay visible")
+		}
+	}
+}

internal/issues/route_test.go

@@ -0,0 +1,18 @@
+package issues
+
+import "testing"
+
+func TestNormalizeRoutePrefixedIDs(t *testing.T) {
+	cases := map[string]string{
+		"/api/terminals/term_ab315d5ce24c4662b34a8ad998be79cf/ws": "/api/terminals/{id}/ws",
+		"/api/sessions/sess-9f2c41aa77/events":                    "/api/sessions/{id}/events",
+		"/api/skills":                                             "/api/skills",
+		"/api/v2/settings":                                        "/api/v2/settings",
+		"/api/user_settings":                                      "/api/user_settings",
+	}
+	for in, want := range cases {
+		if got := NormalizeRoute(in); got != want {
+			t.Errorf("NormalizeRoute(%q) = %q, want %q", in, got, want)
+		}
+	}
+}

internal/issues/tracker.go

@@ -346,7 +346,7 @@ func NormalizeRoute(route string) string {
 		if part == "" {
 			continue
 		}
-		if uuidRe.MatchString(part) || hexIDRe.MatchString(part) || numRe.MatchString(part) {
+		if uuidRe.MatchString(part) || hexIDRe.MatchString(part) || numRe.MatchString(part) || prefixedIDRe.MatchString(part) {
 			parts[i] = "{id}"
 		}
 	}
@@ -623,6 +623,9 @@ var (
 	hexIDRe      = regexp.MustCompile(`(?i)\b[0-9a-f]{8,}\b`)
 	httpStatusRe = regexp.MustCompile(`(?i)\bhttp\s+([1-5]\d{2})\b`)
 	numRe        = regexp.MustCompile(`\b\d+\b`)
+	// A route segment that is a short prefix plus a long hex id (term_ab31…,
+	// sess-9f2c…): \b does not fire between "_" and a hex digit, so hexIDRe misses it.
+	prefixedIDRe = regexp.MustCompile(`(?i)^[a-z]{1,12}[_-][0-9a-f]{8,}$`)
 	spaceRe      = regexp.MustCompile(`\s+`)
 )
 

internal/triage/triage.go

@@ -47,7 +47,8 @@ var (
 	// control, CMS/admin paths, path traversal. Deliberately path-only.
 	probeRoute = regexp.MustCompile(`(?i)(\.env\b|\.git\b|\.htpasswd|\.htaccess|\.npmrc|\.claude|\.docker|\.kube|\.aws|\.ssh|` +
 		`wp-|/wp/|/wp$|wordpress|phpmyadmin|phpinfo|cgi-bin|xmlrpc|userfiles|@fs/|etc/passwd|proc/self|\.\./|%2e%2e|` +
-		`\.php\b|_debugbar|debug/pprof|server-info|actuator|__env|\.bash_|\.profile$|graphql/console|` +
+		`\.php\b|\.codex|\.cursor|\.windsurf|\.aider|\.clinerules|\.roo(/|code|$)|\.continue/|\.gemini|\.idea/|\.vscode/|copilot-instructions|claude\.md|agents\.md|mcp\.json|` +
+		`_debugbar|debug/pprof|server-info|actuator|__env|\.bash_|\.profile$|graphql/console|` +
 		`credentials|secrets?\b|firebase|serviceaccount|/config(/|\.)|settings\.(js|json|py)$|bootstrap\.properties|` +
 		`\.(ya?ml|toml|properties|sql|bak|old|swp|zip|tar|tgz|gz|ini|conf|cfg|key|pem)$)`)
 	bugText  = regexp.MustCompile(`(?i)panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault`)

internal/triage/triage_test.go

@@ -41,3 +41,16 @@ func TestProbeBeatsLifecycleFlags(t *testing.T) {
 		t.Fatalf("got %s", got.Category)
 	}
 }
+
+func TestAIToolConfigPathsAreProbes(t *testing.T) {
+	for _, p := range []string{"/.cursorrules", "/.codex/auth.json", "/backup/.claude/credentials.json", "/.windsurfrules", "/.vscode/launch.json", "/.idea/WebServers.xml", "/x/.github/copilot-instructions.md", "/CLAUDE.md", "/mcp.json", "/.clinerules"} {
+		if !IsProbePath(p) {
+			t.Errorf("%s should be a probe", p)
+		}
+	}
+	for _, p := range []string{"/mcp", "/api/agents", "/api/sessions/{id}", "/api/continue"} {
+		if IsProbePath(p) {
+			t.Errorf("%s must not be a probe", p)
+		}
+	}
+}

internal/uaclass/uaclass.go

@@ -269,3 +269,14 @@ func IsBot(class string) bool {
 	}
 	return false
 }
+
+// IsCrawler reports self-declared crawlers, agents and scanners: the classes
+// whose 4xx noise is not user impact. Monitors and plain HTTP libraries are
+// excluded because their failures can be real.
+func IsCrawler(class string) bool {
+	switch class {
+	case AICrawler, AIAgent, SearchCrawler, SEOCrawler, ScanTool, OtherBot:
+		return true
+	}
+	return false
+}

otel-ui/index.html

@@ -4,7 +4,7 @@
     <meta charset="UTF-8" />
     <link rel="icon" type="image/svg+xml" href="/vite.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>HyperScope OTel — Distributed Telemetry</title>
+    <title>OhhTel — Distributed Telemetry</title>
     <link rel="preconnect" href="https://fonts.googleapis.com" />
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
     <!-- Design-spec typefaces: Bricolage Grotesque for headings and

otel-ui/src/components/IssuesVisualizer.tsx

@@ -1,4 +1,4 @@
-import { useMemo, useState, type ReactNode } from "react"
+import { useEffect, useMemo, useState, type ReactNode } from "react"
 import {
   Activity,
   AlertOctagon,
@@ -56,6 +56,7 @@ const without = (set: Set<string>, value: string) => {
 }
 
 type KindFilter = "all" | "span_error" | "log_error" | "frontend_error"
+type GroupBy = "none" | "service"
 
 interface HoneypotIP {
   ip: string
@@ -304,11 +305,13 @@ const IssueCard = ({
   onStatusChange,
   statusBusy,
   onFocus,
+  detectedIPByAddress,
 }: {
   issue: Issue
   onStatusChange: (issue: Issue, status: string) => void
   statusBusy: boolean
   onFocus: (issue: Issue, traceId?: string) => void
+  detectedIPByAddress: Map<string, DetectedIP>
 }) => {
   const [expanded, setExpanded] = useState(false)
   const [needleResult, setNeedleResult] = useState<NeedleResult | null>(null)
@@ -337,6 +340,10 @@ const IssueCard = ({
   const hasMappedStacktrace = issue.samples.some((sample) => Boolean(sample.resolvedStacktrace))
   const contextSample = issue.samples.find((s) => s.httpMethod || s.httpRoute || s.httpStatus)
   const correlate = strongestCorrelate(issue)
+  const clientLabels = [...new Set(issue.samples.map((sample) => sample.uaClass).filter((value): value is string => Boolean(value)))]
+    .filter((value) => value !== "browser" && value !== "unknown" && value !== "none")
+    .map(uaClassLabel)
+  const badActorSources = [...new Set(issue.samples.flatMap((sample) => sample.sourceIp ? detectedIPByAddress.get(sample.sourceIp)?.sources ?? [] : []))]
 
   const viewTrace = (traceId: string) => onFocus(issue, traceId)
 
@@ -368,6 +375,18 @@ const IssueCard = ({
               <meta.Icon className="h-3 w-3 shrink-0" aria-hidden="true" />
               {meta.label}
             </span>
+            {clientLabels.slice(0, 2).map((label) => (
+              <span key={label} className="inline-flex h-[23px] shrink-0 items-center gap-1.5 whitespace-nowrap rounded-[6px] border border-ds-accent/30 bg-ds-accent-bg px-2 text-[10.5px] font-bold uppercase tracking-wider text-ds-accent">
+                <Bot className="h-3 w-3" aria-hidden="true" />
+                {label}
+              </span>
+            ))}
+            {badActorSources.length > 0 && (
+              <span className="inline-flex h-[23px] shrink-0 items-center gap-1.5 whitespace-nowrap rounded-[6px] border border-ds-warn/30 bg-ds-warn-bg px-2 text-[10.5px] font-bold uppercase tracking-wider text-ds-warn" title={`Detected by ${badActorSources.join(" and ")}`}>
+                <ShieldBan className="h-3 w-3" aria-hidden="true" />
+                Bad actor IP
+              </span>
+            )}
             <CopyRef id={issue.id} path={`/issues/${encodeURIComponent(issue.id)}`} />
             <span className="text-[12.5px] text-ds-text-3">
               Detected {formatRelativeTime(issue.firstSeen / 1e6)} · last seen{" "}
@@ -464,6 +483,9 @@ const IssueCard = ({
             {(contextSample?.sourceIp || latestSample?.sourceIp) && (
               <IpButton ip={(contextSample?.sourceIp || latestSample?.sourceIp)!} />
             )}
+            {badActorSources.length > 0 && (
+              <span className="font-[family-name:var(--font-data)] text-ds-warn">detected_ip={badActorSources.join("+")}</span>
+            )}
             {(contextSample?.release || latestSample?.release) && (
               <span>release: {contextSample?.release || latestSample?.release}</span>
             )}
@@ -688,6 +710,7 @@ export const IssuesVisualizer = ({
   const focusIssue = (issue: Issue, traceId?: string) => go(traceId ? [issue.id, "trace", traceId] : [issue.id])
   const [statusFilter, setStatusFilter] = useState<StatusFilter>("all")
   const [kindFilter, setKindFilter] = useState<KindFilter>("all")
+  const [groupBy, setGroupBy] = useState<GroupBy>("none")
   const [needleModalOpen, setNeedleModalOpen] = useState(false)
   const [detectorTest, setDetectorTest] = useState<DetectorTestResult | null>(null)
   const [detectorTestLoading, setDetectorTestLoading] = useState(false)
@@ -720,6 +743,12 @@ export const IssuesVisualizer = ({
     }
   }
 
+  useEffect(() => {
+    void loadSpamIPs()
+    // The detector set is loaded once when the Issues view opens.
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [])
+
   const detectedIPs = useMemo(() => {
     const byIP = new Map<string, DetectedIP>()
     for (const entry of spamIPs) byIP.set(entry.ip, { ip: entry.ip, sources: ["honeypot"], count: entry.count })
@@ -731,6 +760,7 @@ export const IssuesVisualizer = ({
     return Array.from(byIP.values())
   }, [spamIPs, blockerStatus])
   const spamIPSet = useMemo(() => new Set(detectedIPs.map((entry) => entry.ip)), [detectedIPs])
+  const detectedIPByAddress = useMemo(() => new Map(detectedIPs.map((entry) => [entry.ip, entry])), [detectedIPs])
 
   const crawlersHidden = CRAWLER_CLASSES.every((c) => clientExclude.has(c))
   const toggleCrawlers = () => {
@@ -800,6 +830,18 @@ export const IssuesVisualizer = ({
     }
   }, [filteredIssues, statusFilter])
 
+  const groupedPrimaryIssues = useMemo(() => {
+    if (groupBy === "none") return [["", primaryIssues] as const]
+    const groups = new Map<string, Issue[]>()
+    for (const issue of primaryIssues) {
+      const name = issue.service || "unknown"
+      const group = groups.get(name) ?? []
+      group.push(issue)
+      groups.set(name, group)
+    }
+    return [...groups.entries()].sort((a, b) => b[1].reduce((n, issue) => n + issue.count, 0) - a[1].reduce((n, issue) => n + issue.count, 0))
+  }, [groupBy, primaryIssues])
+
   const [showSecondary, setShowSecondary] = useState(false)
 
   const runDetectorTest = async () => {
@@ -912,6 +954,7 @@ export const IssuesVisualizer = ({
     resetFilters()
     setSearchQuery("")
     setStatusFilter("all")
+    setGroupBy("none")
     setExcludeSpamIPs(false)
     setClientInclude(new Set())
     setClientExclude(new Set())
@@ -991,6 +1034,21 @@ export const IssuesVisualizer = ({
                 </button>
               ))}
             </div>
+            <div className="flex items-center gap-1 rounded-[9px] border border-ds-line bg-ds-surface p-0.5" aria-label="Group issues">
+              {(["none", "service"] as GroupBy[]).map((value) => (
+                <button
+                  key={value}
+                  type="button"
+                  onClick={() => setGroupBy(value)}
+                  aria-pressed={groupBy === value}
+                  className={`rounded-[6px] px-2.5 py-1 text-[11.5px] font-semibold transition-colors ${
+                    groupBy === value ? "bg-ds-fill text-ds-text" : "text-ds-text-3 hover:text-ds-text-2"
+                  }`}
+                >
+                  {value === "none" ? "Flat" : "By service"}
+                </button>
+              ))}
+            </div>
             <button
               type="button"
               onClick={() => {
@@ -1150,9 +1208,19 @@ export const IssuesVisualizer = ({
               </p>
             </div>
           ) : (
-            <div className="flex flex-col gap-3">
+            <div className="flex flex-col gap-5">
-              {primaryIssues.map((issue) => (
+              {groupedPrimaryIssues.map(([service, group]) => (
-                <IssueCard key={issue.id} issue={issue} onStatusChange={onStatusChange} statusBusy={statusBusy} onFocus={focusIssue} />
+                <section key={service || "all"} className="flex flex-col gap-3">
+                  {groupBy === "service" && (
+                    <div className="flex items-baseline gap-2 px-1">
+                      <h2 className="font-[family-name:var(--font-display)] text-[15px] font-bold text-ds-text">{service}</h2>
+                      <span className="font-[family-name:var(--font-data)] text-[11px] text-ds-text-3">{group.length} issue{group.length === 1 ? "" : "s"} · {group.reduce((n, issue) => n + issue.count, 0)} occurrences</span>
+                    </div>
+                  )}
+                  {group.map((issue) => (
+                    <IssueCard key={issue.id} issue={issue} onStatusChange={onStatusChange} statusBusy={statusBusy} onFocus={focusIssue} detectedIPByAddress={detectedIPByAddress} />
+                  ))}
+                </section>
               ))}
             </div>
           )}
@@ -1174,7 +1242,7 @@ export const IssuesVisualizer = ({
               {showSecondary && (
                 <div className="flex flex-col gap-3">
                   {secondaryIssues.map((issue) => (
-                    <IssueCard key={issue.id} issue={issue} onStatusChange={onStatusChange} statusBusy={statusBusy} onFocus={focusIssue} />
+                    <IssueCard key={issue.id} issue={issue} onStatusChange={onStatusChange} statusBusy={statusBusy} onFocus={focusIssue} detectedIPByAddress={detectedIPByAddress} />
                   ))}
                 </div>
               )}

otel-ui/src/components/SearchAndWatcher.tsx

@@ -1,4 +1,4 @@
-import React, { useState, useRef } from "react";
+import React, { useState, useRef, useEffect } from "react";
 import {
   Search,
   Radar,
@@ -49,14 +49,13 @@ export const SearchAndWatcher: React.FC = () => {
     setTimeout(() => setCopiedId(null), 1500);
   };
 
-  const handleRunSearch = async (e?: React.FormEvent) => {
+  const runSearch = async (query = searchQuery) => {
-    if (e) e.preventDefault();
     setSearchLoading(true);
     setSearchError(null);
 
     try {
       const params = new URLSearchParams();
-      if (searchQuery.trim()) params.set("q", searchQuery.trim());
+      if (query.trim()) params.set("q", query.trim());
       if (searchType) params.set("type", searchType);
       params.set("limit", String(searchLimit));
 
@@ -83,6 +82,20 @@ export const SearchAndWatcher: React.FC = () => {
     }
   };
 
+  const handleRunSearch = async (e?: React.FormEvent) => {
+    e?.preventDefault();
+    await runSearch();
+  };
+
+  useEffect(() => {
+    const query = new URLSearchParams(window.location.search).get("q");
+    if (!query) return;
+    setSearchQuery(query);
+    void runSearch(query);
+    // The query is intentionally read once when arriving from a release link.
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
   const startWatcher = async () => {
     if (isWatching) return;
     setIsWatching(true);

otel-ui/src/components/TrendsVisualizer.tsx

@@ -11,7 +11,7 @@ import {
   XAxis,
   YAxis,
 } from "recharts"
-import { AlertOctagon, GitCompareArrows, RefreshCw, Server } from "lucide-react"
+import { AlertOctagon, ExternalLink, GitCompareArrows, Info, RefreshCw, Search, Server } from "lucide-react"
 import { useStats } from "../hooks/useStats"
 import type { StatsPoint } from "../lib/types"
 
@@ -20,9 +20,6 @@ const compactFormatter = new Intl.NumberFormat("en-US", {
   maximumFractionDigits: 1,
 })
 
-// Stable palette for release lines; cycles when there are many releases.
-const RELEASE_COLORS = ["var(--ds-accent)", "var(--ds-accent)", "var(--ds-ok)", "var(--ds-warn)", "var(--ds-accent)", "var(--ds-critical)"]
-
 const label = (bucket: string, granularity: "hour" | "day") => {
   const d = new Date(bucket.replace(" ", "T") + "Z")
   if (Number.isNaN(d.getTime())) return bucket
@@ -59,6 +56,27 @@ const ChartTooltip = ({ active, payload, granularity }: any) => {
   )
 }
 
+const ReleaseTooltip = ({ active, payload, label: bucket }: any) => {
+  if (!active || !payload?.length) return null
+  const d = new Date(String(bucket).replace(" ", "T") + "Z")
+  return (
+    <div className="rounded-xl border border-ds-line bg-ds-bg/95 p-3 text-xs">
+      <div className="mb-1 font-mono text-[10px] text-ds-text-2">
+        {d.toLocaleString([], { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit", hour12: false })}
+      </div>
+      {payload.map((entry: any) => (
+        <div key={entry.name} className="flex items-center justify-between gap-4">
+          <span className="flex items-center gap-1.5 text-ds-text-2">
+            <span className="h-1.5 w-1.5 rounded-full bg-ds-accent" />
+            {entry.name}
+          </span>
+          <span className="font-mono font-bold text-ds-text">{Number(entry.value).toFixed(2)}%</span>
+        </div>
+      ))}
+    </div>
+  )
+}
+
 const rangeOptions = [24, 72, 168, 720] // 1d, 3d, 7d, 30d
 
 export const TrendsVisualizer = () => {
@@ -81,12 +99,12 @@ export const TrendsVisualizer = () => {
 
   const releases = useMemo(() => Object.keys(releaseSeries).sort(), [releaseSeries])
   const releaseData = useMemo(() => {
-    // Recharts wants rows, not per-series arrays: bucket -> {release: errors}.
+    // Recharts wants rows, not per-series arrays: bucket -> {release: rate}.
     const byBucket: Record<string, Record<string, number | string>> = {}
     releases.forEach((rel) => {
       releaseSeries[rel].forEach((p) => {
         byBucket[p.bucket] = byBucket[p.bucket] || { bucket: p.bucket }
-        byBucket[p.bucket][rel] = p.errors
+        byBucket[p.bucket][rel] = p.total > 0 ? (p.errors / p.total) * 100 : 0
       })
     })
     return Object.values(byBucket).sort((a, b) =>
@@ -94,6 +112,13 @@ export const TrendsVisualizer = () => {
     )
   }, [releaseSeries, releases])
 
+  const releaseSummary = useMemo(() => releases.map((release) => {
+    const points = releaseSeries[release]
+    const total = points.reduce((sum, point) => sum + point.total, 0)
+    const errors = points.reduce((sum, point) => sum + point.errors, 0)
+    return { release, total, errors, rate: total > 0 ? (errors / total) * 100 : 0 }
+  }), [releaseSeries, releases])
+
   const totals = useMemo(() => {
     let total = 0
     let errors = 0
@@ -196,16 +221,6 @@ export const TrendsVisualizer = () => {
           <div className="h-64 w-full">
             <ResponsiveContainer width="100%" height="100%">
               <AreaChart data={errorData} margin={{ top: 5, right: 10, left: 0, bottom: 0 }}>
-                <defs>
-                  <linearGradient id="trendFill" x1="0" y1="0" x2="0" y2="100%">
-                    <stop offset="5%" stopColor="var(--ds-accent)" stopOpacity={0.35} />
-                    <stop offset="95%" stopColor="var(--ds-accent)" stopOpacity={0} />
-                  </linearGradient>
-                  <linearGradient id="trendErrFill" x1="0" y1="0" x2="0" y2="100%">
-                    <stop offset="5%" stopColor="var(--ds-critical)" stopOpacity={0.45} />
-                    <stop offset="95%" stopColor="var(--ds-critical)" stopOpacity={0} />
-                  </linearGradient>
-                </defs>
                 <CartesianGrid strokeDasharray="3 3" vertical={false} stroke="var(--ds-line-strong)" />
                 <XAxis
                   dataKey="bucket"
@@ -234,7 +249,7 @@ export const TrendsVisualizer = () => {
                   name="Total"
                   stroke="var(--ds-accent)"
                   strokeWidth={2}
-                  fill="url(#trendFill)"
+                  fill="var(--ds-accent-bg)"
                   isAnimationActive={false}
                 />
                 <Area
@@ -243,7 +258,7 @@ export const TrendsVisualizer = () => {
                   name="Errors"
                   stroke="var(--ds-critical)"
                   strokeWidth={2}
-                  fill="url(#trendErrFill)"
+                  fill="var(--ds-critical-bg)"
                   isAnimationActive={false}
                 />
               </AreaChart>
@@ -254,15 +269,21 @@ export const TrendsVisualizer = () => {
 
       {/* Release comparison */}
       <section className="rounded-[14px] border border-ds-line bg-ds-surface-2 p-4">
-        <header className="mb-3 flex items-center justify-between">
+        <header className="mb-3 flex flex-wrap items-center justify-between gap-2">
           <h3 className="flex items-center gap-2 text-xs font-bold uppercase tracking-[0.2em] text-ds-text-2">
             <GitCompareArrows className="h-3.5 w-3.5 text-ds-accent" />
-            Errors by release
+            Release error rate
           </h3>
           <span className="text-[10px] uppercase tracking-wider text-ds-text-3">
-            {releases.length} release{releases.length === 1 ? "" : "s"} · overlay
+            {releases.length} release{releases.length === 1 ? "" : "s"} · percent of entries
           </span>
         </header>
+        <div className="mb-4 flex items-start gap-2 border border-ds-line bg-ds-fill p-3 text-[12px] leading-relaxed text-ds-text-2">
+          <Info className="mt-0.5 h-3.5 w-3.5 shrink-0 text-ds-accent" aria-hidden="true" />
+          <span>
+            Each point is <span className="font-[family-name:var(--font-data)] text-ds-text">errors ÷ all entries</span> carrying that release in the bucket. This is a rate, so releases with different traffic volumes can be compared. The table below keeps the underlying counts visible.
+          </span>
+        </div>
         {releaseData.length === 0 || releases.length === 0 ? (
           <p className="py-12 text-center text-sm text-ds-text-3">
             No release-tagged telemetry in this range. Have services send{" "}
@@ -289,9 +310,10 @@ export const TrendsVisualizer = () => {
                   tickLine={false}
                   axisLine={false}
                   width={36}
-                  tickFormatter={(v) => compactFormatter.format(v)}
+                  tickFormatter={(v) => `${v}%`}
+                  domain={[0, "auto"]}
                 />
-                <Tooltip content={<ChartTooltip granularity="hour" />} />
+                <Tooltip content={<ReleaseTooltip />} />
                 <Legend
                   wrapperStyle={{ fontSize: 10, textTransform: "uppercase", letterSpacing: "0.1em" }}
                 />
@@ -300,7 +322,8 @@ export const TrendsVisualizer = () => {
                     key={rel}
                     type="monotone"
                     dataKey={rel}
-                    stroke={RELEASE_COLORS[i % RELEASE_COLORS.length]}
+                    stroke="var(--ds-accent)"
+                    strokeDasharray={i % 3 === 1 ? "6 3" : i % 3 === 2 ? "2 2" : undefined}
                     strokeWidth={2}
                     dot={false}
                     isAnimationActive={false}
@@ -310,6 +333,45 @@ export const TrendsVisualizer = () => {
             </ResponsiveContainer>
           </div>
         )}
+        {releaseSummary.length > 0 && (
+          <div className="mt-4 overflow-x-auto border-t border-ds-line pt-3">
+            <table className="w-full min-w-[560px] text-left text-[12px]">
+              <caption className="mb-2 text-left text-[10px] uppercase tracking-[0.15em] text-ds-text-3">
+                Range totals · {service || "all services"}
+              </caption>
+              <thead className="text-[10px] uppercase tracking-wider text-ds-text-3">
+                <tr>
+                  <th className="pb-2 font-medium">Release</th>
+                  <th className="pb-2 text-right font-medium">Entries</th>
+                  <th className="pb-2 text-right font-medium">Errors</th>
+                  <th className="pb-2 text-right font-medium">Error rate</th>
+                  <th className="pb-2 text-right font-medium">Inspect</th>
+                </tr>
+              </thead>
+              <tbody>
+                {releaseSummary.map(({ release, total, errors, rate }) => (
+                  <tr key={release} className="border-t border-ds-line text-ds-text-2">
+                    <td className="py-2 font-[family-name:var(--font-data)] text-ds-text">{release}</td>
+                    <td className="py-2 text-right font-[family-name:var(--font-data)]">{compactFormatter.format(total)}</td>
+                    <td className="py-2 text-right font-[family-name:var(--font-data)]">{compactFormatter.format(errors)}</td>
+                    <td className="py-2 text-right font-[family-name:var(--font-data)] text-ds-critical">{rate.toFixed(2)}%</td>
+                    <td className="py-2 text-right">
+                      <a
+                        href={`/search?q=${encodeURIComponent(release)}`}
+                        className="inline-flex items-center gap-1 text-ds-accent hover:underline"
+                        title={`Search telemetry containing ${release}`}
+                      >
+                        <Search className="h-3.5 w-3.5" aria-hidden="true" />
+                        Search
+                        <ExternalLink className="h-3 w-3" aria-hidden="true" />
+                      </a>
+                    </td>
+                  </tr>
+                ))}
+              </tbody>
+            </table>
+          </div>
+        )}
       </section>
 
       <p className="flex items-center justify-center gap-1.5 pb-2 text-center text-[10px] text-ds-text-3">