all the changes

.gitignore

@@ -6,3 +6,6 @@ otel-agent.log
 /otel-ui/node_modules/
 /otel-ui/dist/
 /otel-ui/dist-ssr/
+
+# Local credentials and scratch
+.env

AGENTS.md

@@ -149,11 +149,12 @@ Lessons from getting here:
   `index.html`. Rebuild (`npm --prefix otel-ui run build`) before concluding a
   screen isn't wired.
 - Still to polish (functional, but not yet re-laid-out to the screen
-  pattern): `TraceVisualization`, `MetricsVisualizer`, `StructuredLogTerminal`,
+  pattern): `TraceVisualization`, `StructuredLogTerminal`,
   `SearchAndWatcher`, `TrendsVisualizer`, `SetupPanel`. They use `ds-*`
   colors via a mechanical token swap but keep their old layout/type scale, use
   `onClick` on non-buttons in places, and `CommandAndQueryBar` /
-  `WaveformHistogram` still sit above every tab except Issues and RUM sessions (`isTriageOverview` in `App.tsx`). Restructure
+  `WaveformHistogram` still sit above every tab except Issues, logs, RUM
+  sessions, rollouts and metrics (`isTriageOverview` in `App.tsx`). Restructure
   them onto `FilterBar` + the card/aside pattern when touching them.
 
 ### Finding old-pattern usages

cmd/bad-actors/main.go

@@ -0,0 +1,184 @@
+// bad-actors ranks client IPs by scanner behaviour in raw nginx/ingress spans
+// and reports which would be safe to block. It only reads ClickHouse; nothing
+// is blocked. Review its output before feeding any blocker.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+
+	"git.tornberg.me/go-otel/internal/honeypot"
+	"git.tornberg.me/go-otel/internal/triage"
+)
+
+type otlp struct {
+	ResourceSpans []struct {
+		ScopeSpans []struct {
+			Spans []struct {
+				Attributes []struct {
+					Key   string                     `json:"key"`
+					Value map[string]json.RawMessage `json:"value"`
+				} `json:"attributes"`
+			} `json:"spans"`
+		} `json:"scopeSpans"`
+	} `json:"resourceSpans"`
+}
+
+type actor struct {
+	IP        string
+	Total     int
+	Probes    int
+	Paths     map[string]struct{}
+	Statuses  map[string]int
+	UserAgent string
+}
+
+func main() {
+	addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port")
+	db := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "database")
+	user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "user")
+	pass := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "password")
+	since := flag.String("since", "", "inclusive UTC RFC3339")
+	until := flag.String("until", "", "exclusive UTC RFC3339")
+	service := flag.String("service", "nginx", "ingress service name")
+	minProbes := flag.Int("min-probes", 10, "minimum probe requests to be blockable")
+	minPaths := flag.Int("min-paths", 3, "minimum distinct probe paths")
+	minShare := flag.Float64("min-share", 0.8, "minimum share of the IP's requests that are probes")
+	broadPaths := flag.Int("broad-paths", 50, "distinct probe paths that mark a scanner regardless of share")
+	broadProbes := flag.Int("broad-probes", 100, "probe requests required with -broad-paths")
+	top := flag.Int("top", 30, "rows to print")
+	flag.Parse()
+	start, err1 := time.Parse(time.RFC3339, *since)
+	end, err2 := time.Parse(time.RFC3339, *until)
+	if err1 != nil || err2 != nil || !start.Before(end) {
+		log.Fatal("-since/-until must be valid RFC3339 and ordered")
+	}
+	conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{*addr}, Auth: clickhouse.Auth{Database: *db, Username: *user, Password: *pass}})
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer conn.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	defer cancel()
+	rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT data FROM %s.entries WHERE type='traces' AND service=? AND timestamp >= toDateTime64('%s',9,'UTC') AND timestamp < toDateTime64('%s',9,'UTC')`,
+		*db, start.UTC().Format("2006-01-02 15:04:05.000000000"), end.UTC().Format("2006-01-02 15:04:05.000000000")), *service)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+
+	actors := map[string]*actor{}
+	spans := 0
+	for rows.Next() {
+		var data string
+		if err := rows.Scan(&data); err != nil {
+			log.Fatal(err)
+		}
+		var d otlp
+		if json.Unmarshal([]byte(data), &d) != nil {
+			continue
+		}
+		for _, rs := range d.ResourceSpans {
+			for _, ss := range rs.ScopeSpans {
+				for _, sp := range ss.Spans {
+					at := map[string]string{}
+					for _, a := range sp.Attributes {
+						for _, raw := range a.Value {
+							var v any
+							_ = json.Unmarshal(raw, &v)
+							at[a.Key] = fmt.Sprint(v)
+						}
+					}
+					ip := strings.TrimSpace(strings.SplitN(first(at, "http.request.header.x-real-ip", "http.request.header.x-forwarded-for", "client.address", "net.peer.ip"), ",", 2)[0])
+					if ip == "" {
+						continue
+					}
+					spans++
+					path := strings.SplitN(first(at, "http.target", "url.path", "http.route"), "?", 2)[0]
+					a := actors[ip]
+					if a == nil {
+						a = &actor{IP: ip, Paths: map[string]struct{}{}, Statuses: map[string]int{}}
+						actors[ip] = a
+					}
+					a.Total++
+					if a.UserAgent == "" {
+						a.UserAgent = first(at, "http.user_agent")
+					}
+					if triage.IsProbePath(path) {
+						a.Probes++
+						a.Paths[path] = struct{}{}
+						a.Statuses[first(at, "http.status_code")]++
+					}
+				}
+			}
+		}
+	}
+	if err := rows.Err(); err != nil {
+		log.Fatal(err)
+	}
+
+	var list []*actor
+	totalProbes, blockableProbes, blockable, skippedInternal, skippedInternalProbes := 0, 0, 0, 0, 0
+	for _, a := range actors {
+		totalProbes += a.Probes
+		if a.Probes == 0 {
+			continue
+		}
+		if internal(a.IP) {
+			skippedInternal++
+			skippedInternalProbes += a.Probes
+			a.UserAgent = "[INTERNAL] " + a.UserAgent
+		} else if (a.Probes >= *minProbes && len(a.Paths) >= *minPaths && float64(a.Probes)/float64(a.Total) >= *minShare) ||
+			(a.Probes >= *broadProbes && len(a.Paths) >= *broadPaths) {
+			blockable++
+			blockableProbes += a.Probes
+			a.UserAgent = "[BLOCK] " + a.UserAgent
+		}
+		list = append(list, a)
+	}
+	sort.Slice(list, func(i, j int) bool { return list[i].Probes > list[j].Probes })
+	fmt.Printf("%d spans with an IP, %d distinct IPs, %d probe requests\n", spans, len(actors), totalProbes)
+	fmt.Printf("blockable (public; >=%d probes/>=%d paths/>=%.0f%% probes, or >=%d probes over >=%d paths): %d IPs covering %d probes (%.0f%%)\n", *minProbes, *minPaths, *minShare*100, *broadProbes, *broadPaths, blockable, blockableProbes, pct(blockableProbes, totalProbes))
+	fmt.Printf("internal/loopback (never blocked): %d IPs, %d probes (%.0f%%) - real client is hidden behind a proxy hop\n\n", skippedInternal, skippedInternalProbes, pct(skippedInternalProbes, totalProbes))
+	fmt.Printf("%-40s %6s %6s %6s  %s\n", "ip", "probes", "total", "paths", "verdict / user-agent / statuses")
+	for i, a := range list {
+		if i >= *top {
+			break
+		}
+		fmt.Printf("%-40s %6d %6d %6d  %.60s %v\n", a.IP, a.Probes, a.Total, len(a.Paths), a.UserAgent, a.Statuses)
+	}
+}
+
+// internal reports addresses that must never be blocked; it is the same
+// definition the tracker and blocker use.
+func internal(s string) bool { return honeypot.Untrackable(s) }
+
+func first(m map[string]string, keys ...string) string {
+	for _, k := range keys {
+		if v := m[k]; v != "" && v != "<nil>" {
+			return v
+		}
+	}
+	return ""
+}
+func pct(a, b int) float64 {
+	if b == 0 {
+		return 0
+	}
+	return 100 * float64(a) / float64(b)
+}
+func envOr(k, f string) string {
+	if v := os.Getenv(k); v != "" {
+		return v
+	}
+	return f
+}

cmd/needle-sidecar/eval/golden.jsonl

@@ -1,40 +1,40 @@
-{"id": "ng-404-root", "src": "clickhouse", "expected": "log_404", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /"}
+{"id": "ng-404-root", "src": "clickhouse", "expected": "stale_reference", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /"}
-{"id": "ng-401-repos", "src": "clickhouse", "expected": "auth_failure", "note": "polled ~700x/h by an unauthenticated client", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/repos/available"}
+{"id": "ng-401-repos", "src": "clickhouse", "expected": "client_credential_issue", "note": "polled ~700x/h by an unauthenticated client", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/repos/available"}
-{"id": "ng-401-ctrl", "src": "clickhouse", "expected": "auth_failure", "note": "", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/controller/instances"}
+{"id": "ng-401-ctrl", "src": "clickhouse", "expected": "client_credential_issue", "note": "", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/controller/instances"}
-{"id": "orch-401-repos", "src": "clickhouse", "expected": "auth_failure", "note": "same request as ng-401-repos, second span", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/repos/available"}
+{"id": "orch-401-repos", "src": "clickhouse", "expected": "client_credential_issue", "note": "same request as ng-401-repos, second span", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/repos/available"}
-{"id": "orch-401-chat", "src": "clickhouse", "expected": "auth_failure", "note": "", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/sessions/{id}/collab/chat"}
+{"id": "orch-401-chat", "src": "clickhouse", "expected": "client_credential_issue", "note": "", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/sessions/{id}/collab/chat"}
-{"id": "ng-401-manifest", "src": "clickhouse", "expected": "auth_failure", "note": "registry pull without token", "text": "Service: nginx | Kind: span | Status: 401 | Route: /v2/eventsourced-ai-worker/manifests/latest"}
+{"id": "ng-401-manifest", "src": "clickhouse", "expected": "client_credential_issue", "note": "registry pull without token", "text": "Service: nginx | Kind: span | Status: 401 | Route: /v2/eventsourced-ai-worker/manifests/latest"}
-{"id": "ng-503-store", "src": "clickhouse", "expected": "escalate_outage", "note": "upstream 503, real user-facing route", "text": "Service: nginx | Kind: span | Status: 503 | Route: /store/regions"}
+{"id": "ng-503-store", "src": "clickhouse", "expected": "dependency_unreachable", "note": "nginx 503 burst: upstream down is the likeliest cause", "text": "Service: nginx | Kind: span | Status: 503 | Route: /store/regions"}
-{"id": "ng-500-root", "src": "clickhouse", "expected": "escalate_outage", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /"}
+{"id": "ng-500-root", "src": "clickhouse", "expected": "service_defect", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /"}
-{"id": "ng-500-wp", "src": "clickhouse", "expected": "security_alert", "note": "WordPress path that does not exist here", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp"}
+{"id": "ng-500-wp", "src": "clickhouse", "expected": "scanner_probe", "note": "WordPress path that does not exist here", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp"}
-{"id": "ng-500-blog", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /blog"}
+{"id": "ng-500-blog", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /blog"}
-{"id": "ng-500-wpjson", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp-json/batch/v1"}
+{"id": "ng-500-wpjson", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp-json/batch/v1"}
-{"id": "ng-500-wordpress", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wordpress"}
+{"id": "ng-500-wordpress", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wordpress"}
-{"id": "ng-404-env", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env"}
+{"id": "ng-404-env", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env"}
-{"id": "ng-404-envlocal", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env.local"}
+{"id": "ng-404-envlocal", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env.local"}
-{"id": "ng-404-userfiles", "src": "clickhouse", "expected": "security_alert", "note": "ambiguous: probe or stale link", "text": "Service: nginx | Kind: span | Status: 404 | Route: /userfiles"}
+{"id": "ng-404-userfiles", "src": "clickhouse", "expected": "scanner_probe", "note": "ambiguous: probe or stale link", "text": "Service: nginx | Kind: span | Status: 404 | Route: /userfiles"}
-{"id": "ng-404-commit", "src": "clickhouse", "expected": "log_404", "note": "git UI link to missing commit", "text": "Service: nginx | Kind: span | Status: 404 | Route: /mats/go-cart-actor/commit/{id}"}
+{"id": "ng-404-commit", "src": "clickhouse", "expected": "stale_reference", "note": "git UI link to missing commit", "text": "Service: nginx | Kind: span | Status: 404 | Route: /mats/go-cart-actor/commit/{id}"}
-{"id": "ng-404-vite", "src": "clickhouse", "expected": "log_404", "note": "dev tooling request", "text": "Service: nginx | Kind: span | Status: 404 | Route: /__vite_rsc_findSourceMapURL"}
+{"id": "ng-404-vite", "src": "clickhouse", "expected": "stale_reference", "note": "dev tooling request", "text": "Service: nginx | Kind: span | Status: 404 | Route: /__vite_rsc_findSourceMapURL"}
-{"id": "ng-404-blob", "src": "clickhouse", "expected": "log_404", "note": "registry blob existence check", "text": "Service: nginx | Kind: span | Status: 404 | Route: /v2/eventsourced-ai/blobs/{id}"}
+{"id": "ng-404-blob", "src": "clickhouse", "expected": "stale_reference", "note": "registry blob existence check", "text": "Service: nginx | Kind: span | Status: 404 | Route: /v2/eventsourced-ai/blobs/{id}"}
-{"id": "ng-499-repos", "src": "clickhouse", "expected": "none", "note": "client closed request", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/repos/available"}
+{"id": "ng-499-repos", "src": "clickhouse", "expected": "expected_lifecycle", "note": "client closed request", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/repos/available"}
-{"id": "ng-499-ctrl", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/controller/instances"}
+{"id": "ng-499-ctrl", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/controller/instances"}
-{"id": "ng-308-blog", "src": "clickhouse", "expected": "none", "note": "redirect", "text": "Service: nginx | Kind: span | Status: 308 | Route: /blog/"}
+{"id": "ng-308-blog", "src": "clickhouse", "expected": "expected_lifecycle", "note": "redirect", "text": "Service: nginx | Kind: span | Status: 308 | Route: /blog/"}
-{"id": "ng-200-speed", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/network-speed"}
+{"id": "ng-200-speed", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/network-speed"}
-{"id": "ng-200-matrix", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /_matrix/federation/v1/send/{id}"}
+{"id": "ng-200-matrix", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /_matrix/federation/v1/send/{id}"}
-{"id": "ng-200-overview", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/overview"}
+{"id": "ng-200-overview", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/overview"}
-{"id": "sched-readyz", "src": "clickhouse", "expected": "none", "note": "health check", "text": "Service: scheduling | Kind: span | Title: GET /readyz | Message: get /readyz | Status: 200 | Route: /readyz"}
+{"id": "sched-readyz", "src": "clickhouse", "expected": "expected_lifecycle", "note": "health check", "text": "Service: scheduling | Kind: span | Title: GET /readyz | Message: get /readyz | Status: 200 | Route: /readyz"}
-{"id": "cart-ping", "src": "clickhouse", "expected": "none", "note": "gRPC keepalive", "text": "Service: cart | Kind: span | Title: /control_plane_messages.ControlPlane/Ping | Message: /control_plane_messages.controlplane/ping"}
+{"id": "cart-ping", "src": "clickhouse", "expected": "expected_lifecycle", "note": "gRPC keepalive", "text": "Service: cart | Kind: span | Title: /control_plane_messages.ControlPlane/Ping | Message: /control_plane_messages.controlplane/ping"}
-{"id": "rag-request", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: rust-rag | Kind: span | Title: request | Message: request"}
+{"id": "rag-request", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: rust-rag | Kind: span | Title: request | Message: request"}
-{"id": "orch-session-load", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: eventsourced-orchestrator | Kind: log | Message: session snapshot/replay loaded"}
+{"id": "orch-session-load", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: eventsourced-orchestrator | Kind: log | Message: session snapshot/replay loaded"}
-{"id": "worker-dialerr", "src": "clickhouse", "expected": "none", "note": "dependency down; arguably escalate_outage - decide", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:<n>/...: connection refused"}
+{"id": "worker-dialerr", "src": "clickhouse", "expected": "dependency_unreachable", "note": "", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:<n>/...: connection refused"}
-{"id": "worker-eof", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *errors.errorstring: eof"}
+{"id": "worker-eof", "src": "clickhouse", "expected": "dependency_unreachable", "note": "", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *errors.errorstring: eof"}
-{"id": "ledger-skip", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: ledger | Kind: log | Message: offsite replicate: content unchanged since last push - skipping"}
+{"id": "ledger-skip", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: ledger | Kind: log | Message: offsite replicate: content unchanged since last push - skipping"}
-{"id": "web-cls", "src": "clickhouse", "expected": "record_vital", "note": "real frontend log; is this a vital or a console line?", "text": "Service: eventsourced-ai-web | Kind: log | Message: cls <num>"}
+{"id": "web-cls", "src": "clickhouse", "expected": "expected_lifecycle", "note": "real frontend log; is this a vital or a console line?", "text": "Service: eventsourced-ai-web | Kind: log | Message: cls <num>"}
-{"id": "syn-js-1", "src": "synthetic", "expected": "client_js_crash", "note": "", "text": "Service: web-frontend | Kind: log | Title: TypeError: Cannot read properties of undefined | Message: Uncaught TypeError: Cannot read properties of undefined (reading 'map') | Route: /checkout"}
+{"id": "syn-js-1", "src": "synthetic", "expected": "service_defect", "note": "", "text": "Service: web-frontend | Kind: log | Title: TypeError: Cannot read properties of undefined | Message: Uncaught TypeError: Cannot read properties of undefined (reading 'map') | Route: /checkout"}
-{"id": "syn-js-2", "src": "synthetic", "expected": "client_js_crash", "note": "", "text": "Service: shop-ui | Kind: log | Title: ChunkLoadError | Message: Loading chunk 42 failed | Route: /products/<id>"}
+{"id": "syn-js-2", "src": "synthetic", "expected": "service_defect", "note": "", "text": "Service: shop-ui | Kind: log | Title: ChunkLoadError | Message: Loading chunk 42 failed | Route: /products/<id>"}
-{"id": "syn-vital-1", "src": "synthetic", "expected": "record_vital", "note": "", "text": "Service: web-frontend | Kind: metric | Title: web_vital TTFB | Message: TTFB 812ms"}
+{"id": "syn-vital-1", "src": "synthetic", "expected": "expected_lifecycle", "note": "", "text": "Service: web-frontend | Kind: metric | Title: web_vital TTFB | Message: TTFB 812ms"}
-{"id": "syn-db-1", "src": "synthetic", "expected": "database_error", "note": "", "text": "Service: checkout | Kind: log | Title: database timeout after <n>ms | Message: context deadline exceeded querying orders"}
+{"id": "syn-db-1", "src": "synthetic", "expected": "capacity_saturation", "note": "", "text": "Service: checkout | Kind: log | Title: database timeout after <n>ms | Message: context deadline exceeded querying orders"}
-{"id": "syn-db-2", "src": "synthetic", "expected": "database_error", "note": "", "text": "Service: inventory | Kind: log | Title: deadlock detected | Message: pq: deadlock detected while updating stock"}
+{"id": "syn-db-2", "src": "synthetic", "expected": "service_defect", "note": "", "text": "Service: inventory | Kind: log | Title: deadlock detected | Message: pq: deadlock detected while updating stock"}
-{"id": "syn-db-3", "src": "synthetic", "expected": "database_error", "note": "", "text": "Service: orders | Kind: log | Title: connection pool exhausted | Message: too many connections in pool"}
+{"id": "syn-db-3", "src": "synthetic", "expected": "capacity_saturation", "note": "", "text": "Service: orders | Kind: log | Title: connection pool exhausted | Message: too many connections in pool"}
-{"id": "syn-oom", "src": "synthetic", "expected": "none", "note": "", "text": "Service: kubelet-proxy | Kind: log | Title: OOMKilled | Message: container exceeded memory limit and was restarted"}
+{"id": "syn-oom", "src": "synthetic", "expected": "capacity_saturation", "note": "", "text": "Service: kubelet-proxy | Kind: log | Title: OOMKilled | Message: container exceeded memory limit and was restarted"}
-{"id": "syn-retry", "src": "synthetic", "expected": "none", "note": "", "text": "Service: worker | Kind: span | Title: retry attempt 2 | Message: retrying after transient error"}
+{"id": "syn-retry", "src": "synthetic", "expected": "expected_lifecycle", "note": "", "text": "Service: worker | Kind: span | Title: retry attempt 2 | Message: retrying after transient error"}

cmd/needle-sidecar/eval/holdout.jsonl

@@ -0,0 +1,60 @@
+{"service":"nginx","kind":"span","status_code":404,"route":"/ses.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/api/v1/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/api/w/admins/jobs_u/get_log_file/../../../../proc/self/environ","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"eventsourced-ai-web","kind":"log","status_code":0,"route":"","title":"","message":"cls \u003cnum\u003e","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":16,"expected":"","predicted":"expected_lifecycle","rule":"lifecycle-text"}
+{"service":"nginx","kind":"span","status_code":500,"route":"/wordpress","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":30,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/private/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/settings.js","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":200,"route":"/","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":17,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"}
+{"service":"eventsourced-orchestrator","kind":"span","status_code":200,"route":"/api/memories","title":"GET /api/memories","message":"get /api/memories","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":5,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"}
+{"service":"cart","kind":"log","status_code":0,"route":"","title":"","message":"handleownershipchange from \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e: accepted_remote=\u003cnum\u003e kept_local=\u003cnum\u003e legacy_no_stamp=\u003cnum\u003e","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"expected_lifecycle","rule":"no-status-signal"}
+{"service":"eventsourced-orchestrator","kind":"span","status_code":0,"route":"","title":"streamproc.replay","message":"streamproc.replay","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"","predicted":"expected_lifecycle","rule":"lifecycle-text"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/.htpasswd","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":401,"route":"/v2/eventsourced-ai-worker/manifests/6d2e5b3-sandbox","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":16,"expected":"","predicted":"client_credential_issue","rule":"status-401"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/api/config","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/ssl/server.key","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/_debugbar/open","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":500,"route":"/register","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"","predicted":"service_defect","rule":"status-5xx"}
+{"service":"nginx","kind":"span","status_code":500,"route":"/wp/wordpress/wp-json/batch/v1","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/old/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/mcp","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":401,"route":"/mcp","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"client_credential_issue","rule":"status-401"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/graphql/console","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/debug/pprof","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/panel/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":200,"route":"/api/actions/runner.v1.RunnerService/FetchTask","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":729,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/server-info","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"eventsourced-orchestrator","kind":"span","status_code":401,"route":"/api/overview","title":"GET /","message":"get /","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"","predicted":"client_credential_issue","rule":"status-401"}
+{"service":"nginx","kind":"span","status_code":101,"route":"/api/sessions/{id}/collab/chat-prompt/ws","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"needs_review","rule":"no-rule"}
+{"service":"nginx","kind":"span","status_code":401,"route":"/v2/eventsourced-ai/blobs/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"","predicted":"client_credential_issue","rule":"status-401"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/v2/eventsourced-ai/blobs/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":17,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/api/v2/settings","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/.well-known/jwks.json","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"eventsourced-ai-web","kind":"log","status_code":0,"route":"","title":"","message":"ttfb \u003cnum\u003e","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":5,"expected":"","predicted":"expected_lifecycle","rule":"lifecycle-text"}
+{"service":"nginx","kind":"span","status_code":200,"route":"/api/stream","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":9,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"}
+{"service":"eventsourced-orchestrator","kind":"span","status_code":200,"route":"/api/sessions/{id}/replay","title":"GET /api/sessions/{id}/replay","message":"get /api/sessions/{id}/replay","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"","predicted":"expected_lifecycle","rule":"lifecycle-text"}
+{"service":"nginx","kind":"span","status_code":201,"route":"/v2/ecommerce/auth/blobs/uploads/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"needs_review","rule":"no-rule"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/cms/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/__env.js","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":504,"route":"/api/stream","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"","predicted":"dependency_unreachable","rule":"status-5xx-upstream"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/api/v2/config","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/actuator/threaddump","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/scripts/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/.streamlit/secrets.toml","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/web/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"cart","kind":"log","status_code":0,"route":"","title":"","message":"\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e taking ownership of \u003cnum\u003e ids","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"expected_lifecycle","rule":"no-status-signal"}
+{"service":"nginx","kind":"span","status_code":200,"route":"/api/schemas/harness_evidence","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/agent/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/settings.py","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/conf/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/dev/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"eventsourced-orchestrator","kind":"span","status_code":302,"route":"/auth/login","title":"GET /auth/login","message":"get /auth/login","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/favicon.ico","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":200,"route":"/api/dev-machines","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":120,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"}
+{"service":"nginx","kind":"span","status_code":308,"route":"/wp-json/batch/v1/","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/.bash_profile","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/@fs/home/ubuntu/.aws/credentials","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/serviceAccountKey.json","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/graphql","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/api","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":200,"route":"/v2/ecommerce/auth/manifests/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"}

cmd/needle-sidecar/eval/rules.py

@@ -0,0 +1,42 @@
+"""Deterministic baseline for the cause categories. Needle has to beat this."""
+import re
+
+PROBE = re.compile(r"(^|/)(\.env|\.git|wp-|wp/|wp$|wordpress|blog/wp|phpmyadmin|cgi-bin|\.aws|\.ssh|xmlrpc|userfiles|@fs/|etc/passwd)", re.I)
+DEP = re.compile(r"dial|connection refused|eof|no such host|upstream|unavailable|broken pipe|reset by peer", re.I)
+CAP = re.compile(r"timeout|deadline exceeded|pool|memory limit|oom|too many|queue full|exhaust", re.I)
+BUG = re.compile(r"panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault", re.I)
+LIFE = re.compile(r"healthz|readyz|/ping|retry|cancel|keepalive|vital|\bcls\b|ttfb|fcp|lcp|snapshot|replay|unchanged|session\.load", re.I)
+
+
+def fields(text):
+    d = {}
+    for part in text.split(" | "):
+        k, _, v = part.partition(": ")
+        d[k.lower()] = v
+    return d
+
+
+def classify(text):
+    f = fields(text)
+    status = int(f.get("status") or 0)
+    route = f.get("route", "")
+    blob = " ".join([f.get("title", ""), f.get("message", ""), route])
+    if PROBE.search(route):
+        return "scanner_probe"
+    if BUG.search(blob):
+        return "service_defect"
+    if CAP.search(blob):
+        return "capacity_saturation"
+    if DEP.search(blob):
+        return "dependency_unreachable"
+    if status == 401:
+        return "client_credential_issue"
+    if status in (502, 503, 504):
+        return "dependency_unreachable"
+    if status >= 500:
+        return "service_defect"
+    if status == 404:
+        return "stale_reference"
+    if LIFE.search(blob) or status in (200, 202, 204, 301, 302, 304, 308, 499) or (status == 0 and f.get("kind") in ("span", "log")):
+        return "expected_lifecycle"
+    return "needs_review"

cmd/needle-sidecar/eval/run_eval.py

@@ -4,8 +4,7 @@
 Usage: run_eval.py [--url http://127.0.0.1:5005] [--golden eval/golden.jsonl] [--json out.json]
 
 Each golden row: {"id", "expected", "text"}; `expected` is a sidecar action name
-or "none" when no tool should fit. The sidecar has no fallback tool, so "none"
+including "needs_review" for the fallback.
-rows measure how often it force-fits (see the confidence column).
 """
 import argparse, json, os, sys, time, urllib.request
 from collections import Counter, defaultdict
@@ -29,7 +28,7 @@ def main():
     ap.add_argument("--golden", default=os.path.join(HERE, "golden.jsonl"))
     ap.add_argument("--json", help="write per-row results here")
     ap.add_argument("--min-conf", type=float, default=0.0,
-                    help="predictions below this confidence count as 'none'")
+                    help="predictions below this confidence count as needs_review")
     a = ap.parse_args()
 
     rows = [json.loads(l) for l in open(a.golden) if l.strip()]
@@ -41,10 +40,10 @@ def main():
         except Exception as e:
             print(f"sidecar error on {row['id']}: {e}", file=sys.stderr)
             sys.exit(2)
-        pred = out.get("action", "unclassified")
+        pred = out.get("action", "needs_review")
         conf = out.get("confidence", 0.0)
-        if conf < a.min_conf or pred == "unclassified":
+        if conf < a.min_conf:
-            pred = "none"
+            pred = "needs_review"
         lat.append(out.get("latency_ms", 0.0))
         confusion[(row["expected"], pred)] += 1
         results.append({**row, "predicted": pred, "confidence": conf, "ok": pred == row["expected"]})

cmd/needle-sidecar/server.py

@@ -16,46 +16,49 @@ import needle
 PORT = int(os.environ.get("NEEDLE_PORT", 5005))
 HOST = os.environ.get("NEEDLE_HOST", "0.0.0.0")
 
-# --- Define Triage Tools for Needle 3 ---
+# --- Cause-level triage tools ---
+# These deliberately do not restate what the event already carries (status
+# code, is_error, severity). Each one is a judgement about *why* it happened
+# or *who should care*, which is what an alert reader needs.
 @needle.tool
-def report_client_javascript_crash(service: str, message: str, url: str):
+def report_scanner_probe(service: str, path: str, status_code: int):
-    """Report client-side JavaScript crash or error logged by browser RUM."""
+    """Automated scan or exploit crawler requesting sensitive or nonexistent paths such as .env, wp-admin, .git, phpmyadmin. Applies whatever status the probe received, including 500 or 502."""
-    return {"action": "client_js_crash", "service": service, "message": message, "url": url}
+    return {"action": "scanner_probe", "service": service, "path": path, "status_code": status_code}
 
 @needle.tool
-def alert_vulnerability_probe(scanner: str, target: str, status_code: int):
+def report_dependency_unreachable(service: str, dependency: str, error_message: str):
-    """Alert on malicious vulnerability scans or exploit crawlers."""
+    """A downstream or upstream service is down, refusing connections, closed the stream (EOF), or returned upstream unavailable."""
-    return {"action": "security_alert", "scanner": scanner, "target": target, "status_code": status_code}
+    return {"action": "dependency_unreachable", "service": service, "dependency": dependency, "error": error_message}
 
 @needle.tool
-def record_frontend_web_vital(service: str, metric_name: str, duration_ms: int):
+def report_client_credential_issue(service: str, route: str, reason: str):
-    """Record RUM performance metrics like TTFB or FCP."""
+    """A caller lacks a valid or unexpired token, for example an unauthenticated client polling an API or a registry pull without login. The caller's fault, not a fault in this service."""
-    return {"action": "record_vital", "service": service, "metric": metric_name, "ms": duration_ms}
+    return {"action": "client_credential_issue", "service": service, "route": route, "reason": reason}
 
 @needle.tool
-def log_missing_resource_404(path: str, host: str):
+def report_stale_reference(service: str, path: str):
-    """Log benign 404 Not Found requests for missing pages or git commits."""
+    """Harmless missing resource caused by an old link, removed asset, deleted commit, or dev tooling request."""
-    return {"action": "log_404", "path": path, "host": host}
+    return {"action": "stale_reference", "service": service, "path": path}
 
 @needle.tool
-def report_database_error(service: str, table: str, error_message: str):
+def report_service_defect(service: str, error_message: str):
-    """Report database query errors, deadlocks, or connection pool timeouts."""
+    """A bug in the service's own code: panic, nil dereference, unhandled exception, index out of range, deadlock, or a 500 from a real application route."""
-    return {"action": "database_error", "service": service, "table": table, "error": error_message}
+    return {"action": "service_defect", "service": service, "error": error_message}
 
 @needle.tool
-def report_auth_failure(service: str, user_or_token: str, reason: str):
+def report_capacity_saturation(service: str, resource: str, error_message: str):
-    """Report authentication/authorization failure or invalid credentials."""
+    """Resource exhaustion or overload: timeouts, connection pool exhausted, memory limit exceeded, OOM, queue full."""
-    return {"action": "auth_failure", "service": service, "user_or_token": user_or_token, "reason": reason}
+    return {"action": "capacity_saturation", "service": service, "resource": resource, "error": error_message}
 
 @needle.tool
-def escalate_backend_outage(service: str, error_message: str):
+def report_expected_lifecycle(service: str, what: str):
-    """Escalate severe backend 5xx crash or unhandled panic to on-call."""
+    """Normal behaviour that is not a problem: health and readiness checks, keepalive pings, retries, cancellations, client disconnects, redirects, successful requests, routine info logs, web vitals."""
-    return {"action": "escalate_outage", "service": service, "error": error_message}
+    return {"action": "expected_lifecycle", "service": service, "what": what}
 
 @needle.tool
-def report_unclassified(service: str, summary: str):
+def report_needs_review(service: str, summary: str):
-    """Fallback when the event matches none of the other categories: retries, cancellations, restarts, OOM, health checks, or anything unclear."""
+    """Fallback when none of the other categories clearly applies."""
-    return {"action": "unclassified", "service": service, "summary": summary}
+    return {"action": "needs_review", "service": service, "summary": summary}
 
 # --- Redaction: nothing PII-bearing should reach the model or the cache ---
 _REDACTIONS = [
@@ -81,14 +84,14 @@ _stats = {"hits": 0, "misses": 0}
 print(f"Loading Needle 3 model (Cactus-Compute/needle3)...")
 t0 = time.perf_counter()
 agent = needle.Needle(tools=[
-    report_client_javascript_crash,
+    report_scanner_probe,
-    alert_vulnerability_probe,
+    report_dependency_unreachable,
-    record_frontend_web_vital,
+    report_client_credential_issue,
-    log_missing_resource_404,
+    report_stale_reference,
-    report_database_error,
+    report_service_defect,
-    report_auth_failure,
+    report_capacity_saturation,
-    escalate_backend_outage,
+    report_expected_lifecycle,
-    report_unclassified
+    report_needs_review,
 ])
 print(f"Needle 3 loaded in {(time.perf_counter() - t0):.2f}s")
 
@@ -116,14 +119,8 @@ class NeedleHandler(BaseHTTPRequestHandler):
                 "model": "Cactus-Compute/needle3",
                 "cache": {**_stats, "size": len(_cache)},
                 "tools": [
-                    "client_js_crash",
+                    "scanner_probe", "dependency_unreachable", "client_credential_issue", "stale_reference",
-                    "security_alert",
+                    "service_defect", "capacity_saturation", "expected_lifecycle", "needs_review"
-                    "record_vital",
-                    "log_404",
-                    "database_error",
-                    "auth_failure",
-                    "escalate_outage",
-                    "unclassified"
                 ]
             }
             self.wfile.write(json.dumps(resp).encode("utf-8"))
@@ -193,7 +190,7 @@ class NeedleHandler(BaseHTTPRequestHandler):
 
             results = res.get("results", [])
             first_result = results[0] if results and isinstance(results[0], dict) else {}
-            action = first_result.get("action", "unclassified")
+            action = first_result.get("action", "needs_review")
 
             calls = res.get("function_calls", [])
             tool_name = calls[0].get("name") if calls else "triage_tool"

cmd/normalized-backfill/main.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"git.tornberg.me/go-otel/internal/normalization"
+	"git.tornberg.me/go-otel/internal/storage"
 	"github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 )
@@ -50,7 +51,7 @@ func main() {
 		log.Fatal(err)
 	}
 	if *write {
-		if err := createSchema(ctx, conn, *database); err != nil {
+		if err := storage.EnsureNormalizedSchema(ctx, conn, *database); err != nil {
 			log.Fatal(err)
 		}
 	}
@@ -82,7 +83,7 @@ func main() {
 	rawCount, eventCount := 0, 0
 	var batch driver.Batch
 	if *write {
-		batch, err = conn.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.normalized_events", *database))
+		batch, err = conn.PrepareBatch(ctx, storage.NormalizedInsertSQL(*database))
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -115,12 +116,12 @@ func main() {
 			}
 			counts[key].Count++
 			if *write {
-				if err := appendEvent(batch, event); err != nil {
+				if err := storage.AppendNormalizedEvent(batch, event); err != nil {
 					log.Fatal(err)
 				}
 				if eventCount%*batchSize == 0 {
 					flush()
-					batch, err = conn.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.normalized_events", *database))
+					batch, err = conn.PrepareBatch(ctx, storage.NormalizedInsertSQL(*database))
 					if err != nil {
 						log.Fatal(err)
 					}
@@ -146,70 +147,6 @@ func main() {
 	}
 }
 
-func createSchema(ctx context.Context, conn clickhouse.Conn, database string) error {
-	query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.normalized_events (
-		event_id String,
-		entry_type LowCardinality(String),
-		event_index UInt32,
-		ingest_timestamp DateTime64(9, 'UTC'),
-		timestamp DateTime64(9, 'UTC'),
-		service LowCardinality(String), release LowCardinality(String), environment LowCardinality(String),
-		kind LowCardinality(String), source LowCardinality(String), is_error UInt8,
-		status_class LowCardinality(String), status_code UInt16, duration_ms Float64,
-		route String, span_name String, metric_name String, metric_type LowCardinality(String), metric_unit String,
-		metric_value Float64, metric_count UInt64, metric_sum Float64, metric_min Float64, metric_max Float64,
-		exception_type String, message_normalized String, stack_signature String, trace_id String, issue_fingerprint String,
-		is_retry UInt8, is_timeout UInt8, is_cancellation UInt8,
-		exact_signature String, pattern_signature String, version UInt8 DEFAULT 1,
-		method String DEFAULT '', host String DEFAULT '', user_agent String DEFAULT '',
-		origin_host String DEFAULT '', referer_host String DEFAULT '', fetch_site String DEFAULT '', fetch_mode String DEFAULT ''
-	) ENGINE = ReplacingMergeTree(version)
-	PARTITION BY toYYYYMM(timestamp)
-	ORDER BY (service, kind, pattern_signature, timestamp, event_id)
-	TTL toDateTime(timestamp) + INTERVAL 180 DAY`, database)
-	if err := conn.Exec(ctx, query); err != nil {
-		return err
-	}
-	// Existing developer tables predate the explicit model flags. These
-	// additive migrations preserve historical rows with safe zero defaults.
-	for _, statement := range []string{
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS issue_fingerprint String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_retry UInt8 DEFAULT 0", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_timeout UInt8 DEFAULT 0", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_cancellation UInt8 DEFAULT 0", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS method String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS host String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS user_agent String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS origin_host String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS referer_host String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS fetch_site String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS fetch_mode String DEFAULT ''", database),
-	} {
-		if err := conn.Exec(ctx, statement); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func appendEvent(batch driver.Batch, event normalization.Event) error {
-	return batch.Append(
-		event.EventID, event.EntryType, event.EventIndex, time.Unix(0, event.IngestTimestamp), time.Unix(0, event.Timestamp),
-		event.Service, event.Release, event.Environment, event.Kind, event.Source, boolUint8(event.IsError),
-		event.StatusClass, event.StatusCode, event.DurationMS, event.Route, event.SpanName, event.MetricName, event.MetricType, event.MetricUnit,
-		event.MetricValue, event.MetricCount, event.MetricSum, event.MetricMin, event.MetricMax,
-		event.ExceptionType, event.MessageNormalized, event.StackSignature, event.TraceID, event.IssueFingerprint,
-		boolUint8(event.IsRetry), boolUint8(event.IsTimeout), boolUint8(event.IsCancellation), event.ExactSignature, event.PatternSignature,
-		uint8(1), event.Method, event.Host, event.UserAgent, event.OriginHost, event.RefererHost, event.FetchSite, event.FetchMode,
-	)
-}
-
-func boolUint8(value bool) uint8 {
-	if value {
-		return 1
-	}
-	return 0
-}
 func envOr(key, fallback string) string {
 	if value := os.Getenv(key); value != "" {
 		return value

cmd/server/main.go

@@ -11,6 +11,7 @@ import (
 
 	"git.tornberg.me/go-otel/internal/auth"
 	"git.tornberg.me/go-otel/internal/collector"
+	"git.tornberg.me/go-otel/internal/blocker"
 	"git.tornberg.me/go-otel/internal/honeypot"
 	kube "git.tornberg.me/go-otel/internal/kubernetes"
 	"git.tornberg.me/go-otel/internal/prometheus"
@@ -235,6 +236,12 @@ func main() {
 	wsServer := websocket.New(coll)
 	honeypotTracker := honeypot.New(honeypot.ConfigFromEnv())
 	coll.SetHoneypotRecorder(honeypotTracker)
+	// Optional ingress blocker: off unless OTEL_BLOCKER_ENABLED=true, and
+	// dry-run unless OTEL_BLOCKER_DRY_RUN=false.
+	ipBlocker, err := blocker.Start(context.Background(), blocker.ConfigFromEnv(), honeypotTracker)
+	if err != nil {
+		log.Printf("ip blocker disabled: %v", err)
+	}
 	// Update collector with wsServer
 	coll.SetBroadcaster(wsServer)
 	ingestKey := os.Getenv("OTEL_INGEST_KEY")
@@ -293,6 +300,9 @@ func main() {
 	uiMux.HandleFunc("POST /api/auth/token", oidcAuth.HandleTokenExchange)
 	uiMux.HandleFunc("GET /api/auth/config", oidcAuth.HandleConfig)
 	uiMux.Handle("/api/honeypot/ips", oidcAuth.RequireAuth(http.HandlerFunc(honeypotTracker.HandleIPsForUser)))
+	if ipBlocker != nil {
+		uiMux.Handle("GET /api/blocker/status", oidcAuth.RequireAuth(http.HandlerFunc(ipBlocker.HandleStatus)))
+	}
 	// Source-map uploads are intentionally public so the release build script
 	// can upload maps without an OIDC session or bearer token.
 	uiMux.HandleFunc("POST /api/sourcemaps", coll.HandleSourceMap)

cmd/spike-alerts/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/ClickHouse/clickhouse-go/v2"
 
 	"git.tornberg.me/go-otel/internal/features"
+	"git.tornberg.me/go-otel/internal/triage"
 )
 
 func main() {
@@ -33,7 +34,8 @@ func main() {
 	minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert")
 	multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline")
 	cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles")
-	needle := flag.String("needle", os.Getenv("OTEL_NEEDLE_ADDR"), "Needle sidecar URL; empty skips classification")
+	needle := flag.String("needle", "", "Needle sidecar URL (dormant: it is not accurate enough to gate alerts); empty skips it")
+	showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)")
 	asJSON := flag.Bool("json", false, "emit JSON lines instead of text")
 	flag.Parse()
 	if *since == "" || *until == "" {
@@ -55,9 +57,10 @@ func main() {
 	cache := map[string]classification{}
 	first := start.Add(time.Duration(*baselineWindows) * *window)
 	suppress := features.NewSuppressor(*cooldown)
+	muted := map[triage.Category]int{}
 	alerts := 0
 	for at := first.Add(*window); !at.After(end); at = at.Add(*step) {
-		for _, spike := range suppress.Filter(features.DetectSpikes(events, at, cfg)) {
+		for _, spike := range suppress.Filter(triageFilter(features.DetectSpikes(events, at, cfg), *showMuted, muted)) {
 			alerts++
 			var cls []classification
 			for _, p := range spike.Patterns {
@@ -79,6 +82,9 @@ func main() {
 			printSpike(spike, cls)
 		}
 	}
+	for c, n := range muted {
+		log.Printf("muted %d pattern-window(s) as %s", n, c)
+	}
 	log.Printf("%d alert(s) between %s and %s", alerts, first.Format(time.RFC3339), end.Format(time.RFC3339))
 }
 
@@ -89,6 +95,35 @@ type classification struct {
 	Err        string  `json:"error,omitempty"`
 }
 
+func triageOf(e features.Event) triage.Result {
+	return triage.Classify(triage.Input{Service: e.Service, Kind: e.Kind, Status: e.StatusCode, Route: e.Route, Title: e.SpanName, Message: e.Message,
+		IsTimeout: e.IsTimeout, IsCancellation: e.IsCancellation, IsRetry: e.IsRetry})
+}
+
+// triageFilter keeps only actionable patterns (unless showMuted), recomputing
+// each spike's excess so a probe storm cannot page anyone by itself.
+func triageFilter(spikes []features.Spike, showMuted bool, muted map[triage.Category]int) []features.Spike {
+	var out []features.Spike
+	for _, sp := range spikes {
+		var kept []features.PatternDelta
+		sp.Excess = 0
+		for _, p := range sp.Patterns {
+			c := triageOf(p.Example).Category
+			if !showMuted && !c.Actionable() {
+				muted[c]++
+				continue
+			}
+			kept = append(kept, p)
+			sp.Excess += p.Excess
+		}
+		if len(kept) > 0 {
+			sp.Patterns = kept
+			out = append(out, sp)
+		}
+	}
+	return out
+}
+
 func classify(base, text string) classification {
 	body, _ := json.Marshal(map[string]string{"text": text})
 	resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body))
@@ -127,9 +162,9 @@ func printSpike(s features.Spike, cls []classification) {
 		if p.Novel {
 			tag = "NEW"
 		}
-		what := "-"
+		what := string(triageOf(p.Example).Category)
 		if cls[i].Action != "" {
-			what = fmt.Sprintf("%s (%.2f)", cls[i].Action, cls[i].Confidence)
+			what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence)
 		}
 		fmt.Printf("  %-6s x%-4d base %-6.1f %-24s %s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example))
 	}
@@ -144,7 +179,7 @@ func load(addr, database, user, password string, start, end time.Time) []feature
 	}
 	defer conn.Close()
 	// Errors only, and only columns present in every deployed schema version.
-	rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT timestamp, service, release, kind, source, status_code, span_name, route, exception_type, message_normalized, pattern_signature
+	rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT timestamp, service, release, kind, source, status_code, span_name, route, exception_type, message_normalized, pattern_signature, is_timeout, is_cancellation, is_retry
 		FROM %s.normalized_events FINAL
 		WHERE is_error = 1 AND timestamp >= toDateTime64('%s', 9, 'UTC') AND timestamp < toDateTime64('%s', 9, 'UTC')
 		ORDER BY timestamp LIMIT 2000000`, database, chTime(start), chTime(end)))
@@ -156,10 +191,12 @@ func load(addr, database, user, password string, start, end time.Time) []feature
 	for rows.Next() {
 		e := features.Event{IsError: true}
 		var status uint16
-		if err := rows.Scan(&e.Timestamp, &e.Service, &e.Release, &e.Kind, &e.Source, &status, &e.SpanName, &e.Route, &e.ExceptionType, &e.Message, &e.PatternSignature); err != nil {
+		var timeout, cancel, retry uint8
+		if err := rows.Scan(&e.Timestamp, &e.Service, &e.Release, &e.Kind, &e.Source, &status, &e.SpanName, &e.Route, &e.ExceptionType, &e.Message, &e.PatternSignature, &timeout, &cancel, &retry); err != nil {
 			log.Fatal(err)
 		}
 		e.StatusCode = int(status)
+		e.IsTimeout, e.IsCancellation, e.IsRetry = timeout == 1, cancel == 1, retry == 1
 		out = append(out, e)
 	}
 	if err := rows.Err(); err != nil {

cmd/triage-eval/main.go

@@ -0,0 +1,98 @@
+// triage-eval applies the rules classifier to a JSONL file of patterns. Rows
+// with an "expected" category are scored; the rest are written out with the
+// prediction so they can be reviewed and labelled by hand.
+//
+//	triage-eval -input holdout.jsonl            # score labelled rows
+//	triage-eval -input raw.jsonl -label-out holdout.jsonl   # prepare for labelling
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"sort"
+
+	"git.tornberg.me/go-otel/internal/triage"
+)
+
+type row struct {
+	Service        string `json:"service"`
+	Kind           string `json:"kind"`
+	StatusCode     int    `json:"status_code"`
+	Route          string `json:"route"`
+	Title          string `json:"title"`
+	Message        string `json:"message"`
+	IsTimeout      uint8  `json:"is_timeout"`
+	IsCancellation uint8  `json:"is_cancellation"`
+	IsRetry        uint8  `json:"is_retry"`
+	Count          uint64 `json:"n"`
+	Expected       string `json:"expected"`
+	Predicted      string `json:"predicted"`
+	Rule           string `json:"rule"`
+}
+
+func main() {
+	input := flag.String("input", "", "JSONL input")
+	labelOut := flag.String("label-out", "", "write rows with predictions here for hand labelling")
+	flag.Parse()
+	f, err := os.Open(*input)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+	var rows []row
+	sc := bufio.NewScanner(f)
+	sc.Buffer(make([]byte, 1<<20), 1<<20)
+	for sc.Scan() {
+		var r row
+		if err := json.Unmarshal(sc.Bytes(), &r); err != nil {
+			log.Fatalf("bad row: %v", err)
+		}
+		res := triage.Classify(triage.Input{Service: r.Service, Kind: r.Kind, Status: r.StatusCode, Route: r.Route, Title: r.Title, Message: r.Message,
+			IsTimeout: r.IsTimeout == 1, IsCancellation: r.IsCancellation == 1, IsRetry: r.IsRetry == 1})
+		r.Predicted, r.Rule = string(res.Category), res.Rule
+		rows = append(rows, r)
+	}
+	if *labelOut != "" {
+		out, err := os.Create(*labelOut)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer out.Close()
+		enc := json.NewEncoder(out)
+		for _, r := range rows {
+			_ = enc.Encode(r)
+		}
+		log.Printf("wrote %d rows to %s; fill in \"expected\" then re-run without -label-out", len(rows), *labelOut)
+		return
+	}
+	scored, correct := 0, 0
+	confusion := map[[2]string]int{}
+	for _, r := range rows {
+		if r.Expected == "" {
+			continue
+		}
+		scored++
+		if r.Expected == r.Predicted {
+			correct++
+		} else {
+			confusion[[2]string{r.Expected, r.Predicted}]++
+			fmt.Printf("miss  expected=%-24s got=%-24s rule=%-18s %s %d %s %.80s\n", r.Expected, r.Predicted, r.Rule, r.Service, r.StatusCode, r.Route, r.Message)
+		}
+	}
+	if scored == 0 {
+		log.Fatal("no labelled rows (set \"expected\")")
+	}
+	fmt.Printf("\naccuracy %d/%d = %.0f%%\n", correct, scored, 100*float64(correct)/float64(scored))
+	keys := make([][2]string, 0, len(confusion))
+	for k := range confusion {
+		keys = append(keys, k)
+	}
+	sort.Slice(keys, func(i, j int) bool { return confusion[keys[i]] > confusion[keys[j]] })
+	for _, k := range keys {
+		fmt.Printf("  %-24s -> %-24s x%d\n", k[0], k[1], confusion[k])
+	}
+}

docs/ML.md

@@ -1,5 +1,8 @@
 # Error detection model
 
+> Latest status, decisions and next steps: [STATUS.md](STATUS.md). Where they
+> differ (ClickHouse is now a LoadBalancer; Needle is dormant), STATUS.md wins.
+
 ## Current handoff (2026-09-18)
 
 The first iteration is intentionally explainable and deterministic. Raw OTLP

docs/STATUS.md

@@ -0,0 +1,132 @@
+# Status: error triage, spike detection, IP blocking (2026-09-19)
+
+Pick-up notes. Nothing below is committed, deployed or applied to the cluster
+unless stated. Greenfield, single user: radical changes are fine.
+
+## Access
+
+- ClickHouse (otel instance) is now a **LoadBalancer**: `monitoring/clickhouse`
+  at `10.10.3.45` (native 9000, HTTP 8123). Set in `k8s/clickhouse.yaml` and
+  patched live. `docs/ML.md` still says "keep ClusterIP"; that is superseded.
+- Credentials are in `.env` (gitignored, mode 600). Load with
+  `set -a; . ./.env; set +a`. `clickhouse-client` is installed locally.
+- `.env` deliberately does not enable Needle; pass `-needle URL` explicitly.
+
+## What exists now
+
+| Piece | Where | State |
+| --- | --- | --- |
+| Normalized rows written on ingest | `internal/storage/normalized.go`, `Append` in `clickhouse.go` | Code done, tests pass. **Running collector is still the old build**, so nothing new is written until deployed. |
+| Live schema migration | `EnsureNormalizedSchema` (runs at collector start) | Applied once by hand via backfill; `issue_fingerprint` etc. now exist. |
+| 404/410 are not errors | `isErrorStatus` in `internal/normalization/normalizer.go` | Done. Rows stored before this still count 404s as errors. |
+| Backfill | `cmd/normalized-backfill` | Ran 2026-09-19 12:00 to ~21:00 UTC. Queries need `FINAL` (duplicates until merge). |
+| Spike rule | `internal/features/spike.go` | Excess-error rule, trimmed-mean baseline, `Suppressor` (cooldown, 2x escalation). Tested. |
+| Replay CLI | `cmd/spike-alerts` | Replays a range; mutes non-actionable categories; Needle optional and off. |
+| Cause categories | `internal/triage` | Deterministic rules, first match wins. Tested. |
+| Rules scoring | `cmd/triage-eval` | Scores a labelled JSONL; also prepares rows for labelling. |
+| Offender report | `cmd/bad-actors` | Read-only. Ranks IPs from raw nginx spans, marks `[BLOCK]` / `[INTERNAL]`. |
+| Tracker | `internal/honeypot` | Now needs distinct paths (default 3), never tracks internal/reserved IPs, uses `triage.IsProbePath`. |
+| Blocker | `internal/blocker` | Writes ingress-nginx `block-cidrs`. **Dry-run by default.** Tested with a fake client. Wired into `cmd/server`. |
+| RBAC + env | `k8s/deployment.yaml` | Role limited to the one ConfigMap; blocker env set to enabled + dry-run. Not applied. |
+
+Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue`,
+`stale_reference`, `service_defect`, `capacity_saturation`, `expected_lifecycle`,
+`needs_review`. They are cause-level verdicts, not restatements of status.
+
+## Findings worth remembering
+
+- **Needle scored 10% on the 40-row golden set** (about chance for 8 classes).
+  Reversing the tool order changed its answers wholesale (15%, 34/40 `service_defect`),
+  so it picks by position, not content. Confidence is uncorrelated with
+  correctness. Decision: **Needle stays dormant.** Intended later use: a gate that
+  decides whether a larger model should act on an issue (affects anyone? fixable by
+  a smart model?). Do not use it for categorization.
+- **Rules scored 39/40 on the golden set, but that is a fit** (written after seeing
+  the rows). A fresh 60-row holdout is in `cmd/needle-sidecar/eval/holdout.jsonl`
+  (from 2026-09-18, unseen); it still needs your `expected` labels. Known
+  probe-like paths that still land in `stale_reference`: `/ssl/server.key`,
+  `/_debugbar/open`, `/debug/pprof`, `/actuator/*`, `/graphql`, `/server-info`,
+  `/.bash_profile`, `/settings.py`, `/__env.js`.
+- **The top error sources were noise:** 404 probes and a 401 poller
+  (`/api/repos/available`, `/api/controller/instances`, about 700 per hour, counted
+  twice: nginx and the orchestrator).
+- **IP data:** 44 public IPs account for 63% of probe requests (12.5 h window).
+  **34% show a peer of `127.0.0.1`** because `net.peer.ip` is the TCP peer behind
+  the PROXY-protocol hop, so the tracker cannot see them. The ingress has
+  `enable-real-ip=true`, `forwarded-for-header=proxy_protocol`,
+  `proxy-real-ip-cidr=127.0.0.1/32`, so `block-cidrs` itself works on the real
+  client address.
+- **Internal addresses are a hard stop** in both tracker and blocker
+  (`honeypot.Untrackable`): loopback, RFC1918, ULA, link-local, multicast,
+  unspecified, IPv4-mapped, CGNAT `100.64/10` (Tailscale), `192.0.0.0/24`,
+  `198.18/15`, `240/4`. The allowlist only adds exclusions.
+
+## Before enabling enforcement
+
+1. Deploy the new build (dry-run on). Watch `GET /api/blocker/status` for a day;
+   compare against `go run ./cmd/bad-actors -since ... -until ...`.
+2. Put your own public IP(s) in `OTEL_BLOCKER_ALLOW`. A blocked IP gets 403, so
+   you would lock yourself out of the UI. Recovery: `kubectl edit cm
+   ingress-nginx-controller -n ingress-nginx` (remove entries; managed ones are
+   listed in the `otel-debug.k6n.net/blocked` annotation).
+3. Then set `OTEL_BLOCKER_DRY_RUN=false`.
+4. Do not `kubectl apply` the whole `k8s/deployment.yaml` blindly: the dry run
+   showed it would create a new `otel-sourcemaps` PVC not present in the cluster.
+
+## Next steps (suggested order)
+
+1. Deploy the collector so normalized rows stay current; verify with
+   `SELECT max(timestamp) FROM otel.normalized_events`.
+2. Label `holdout.jsonl` (fill `expected`), run
+   `go run ./cmd/triage-eval -input cmd/needle-sidecar/eval/holdout.jsonl`, fix
+   the rule families that miss. Then re-export a fresh holdout so scoring stays
+   honest.
+3. Warm-up guard in `DetectSpikes`: require baseline data before alerting (the
+   first window after data begins alerts on everything as NEW).
+4. Real client IP in nginx spans, to close the 127.0.0.1 gap (inspect the
+   ingress OTel config first; not looked at yet).
+5. Alert delivery: run `DetectSpikes` on a timer in the collector and post
+   somewhere (UI panel / ntfy / Slack). Category chips and `same / different /
+   noise` + "mute pattern" in the UI, storing labels.
+6. Reduce 401-poller noise: mute list or per-pattern rate limit;
+   `client_credential_issue` is the caller's fault, likely a lower severity.
+7. After 2+ weeks of data: time-of-day baseline, rollout proximity, then the
+   logistic regression / MLP from `docs/ML.md`; the rule detector is what they
+   must beat.
+
+## Commands
+
+```bash
+set -a; . ./.env; set +a
+
+# replay the spike rule over a range (Needle off)
+go run ./cmd/spike-alerts -since 2026-09-19T12:00:00Z -until 2026-09-19T20:30:00Z -baseline-windows 12
+# add -show-muted to see probes / stale / lifecycle patterns too
+
+# who would be blocked
+go run ./cmd/bad-actors -since 2026-09-19T08:00:00Z -until 2026-09-19T20:30:00Z
+
+# score the rules against labelled rows
+go run ./cmd/triage-eval -input cmd/needle-sidecar/eval/holdout.jsonl
+
+# Needle eval (needs the sidecar running; venv was in scratch and is gone)
+pip install cactus-needle   # python 3.12 recommended
+NEEDLE_PORT=5005 NEEDLE_HOST=127.0.0.1 python cmd/needle-sidecar/server.py
+python cmd/needle-sidecar/eval/run_eval.py
+```
+
+Backfill in small windows only (10 to 30 minutes over the LAN); one-hour windows
+through a `kubectl port-forward` hung and dropped.
+
+## Git state
+
+Everything is uncommitted. Files changed by this work: `internal/{storage,
+normalization,features,triage,honeypot,blocker}`, `cmd/{spike-alerts,
+triage-eval,bad-actors,normalized-backfill,server,needle-sidecar}`,
+`k8s/{clickhouse,deployment}.yaml`, `.gitignore`.
+
+**Not from this work; check before committing:** `AGENTS.md`,
+`internal/collector/{collector_test,issue_extract}.go`, `otel-ui/src/**`
+(including a deleted `GaugeMeter.tsx`). Also tracked `__pycache__/*.pyc` files
+under `cmd/needle-sidecar` show as modified; they should be untracked and
+gitignored.

internal/blocker/blocker.go

@@ -0,0 +1,383 @@
+// Package blocker turns the honeypot's promoted-IP feed into ingress-nginx
+// `block-cidrs` entries.
+//
+// Safety properties, all deliberate:
+//   - dry-run is the default; nothing is written unless DryRun is false;
+//   - it only manages entries it added itself (tracked in an annotation on the
+//     ConfigMap), so manually configured block-cidrs are preserved untouched;
+//   - loopback/private/link-local addresses and an operator allowlist are never
+//     blocked;
+//   - blocks expire, the set is capped, growth per tick is limited, and writes
+//     are rate-limited because every ConfigMap change reloads nginx.
+package blocker
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"net/netip"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+
+	"git.tornberg.me/go-otel/internal/honeypot"
+)
+
+const AnnotationManaged = "otel-debug.k6n.net/blocked"
+
+type Source interface {
+	PromotedIPs() []honeypot.IP
+}
+
+type Config struct {
+	Enabled       bool
+	DryRun        bool
+	Namespace     string
+	ConfigMap     string
+	Key           string
+	Interval      time.Duration // how often the feed is evaluated
+	MinWrite      time.Duration // minimum gap between ConfigMap writes
+	BlockTTL      time.Duration // how long a block lasts after last sighting
+	MinProbes     int           // per-window probe hits required to block
+	MinPaths      int           // distinct probe paths required to block
+	BroadProbes   int           // hits+paths that block regardless of MinProbes
+	BroadPaths    int
+	MaxEntries    int // cap on entries this blocker manages
+	MaxNewPerTick int // limit on newly added entries per evaluation
+	Allow         []netip.Prefix
+}
+
+func DefaultConfig() Config {
+	return Config{
+		DryRun: true, Namespace: "ingress-nginx", ConfigMap: "ingress-nginx-controller", Key: "block-cidrs",
+		Interval: time.Minute, MinWrite: 5 * time.Minute, BlockTTL: 24 * time.Hour,
+		MinProbes: 10, MinPaths: 3, BroadProbes: 100, BroadPaths: 50, MaxEntries: 500, MaxNewPerTick: 20,
+	}
+}
+
+// ConfigFromEnv reads OTEL_BLOCKER_*. Enabled defaults to false and DryRun to
+// true; both must be changed on purpose.
+func ConfigFromEnv() Config {
+	c := DefaultConfig()
+	c.Enabled = strings.EqualFold(os.Getenv("OTEL_BLOCKER_ENABLED"), "true")
+	if v := os.Getenv("OTEL_BLOCKER_DRY_RUN"); v != "" {
+		c.DryRun = !strings.EqualFold(v, "false")
+	}
+	str := func(env string, dst *string) {
+		if v := os.Getenv(env); v != "" {
+			*dst = v
+		}
+	}
+	num := func(env string, dst *int) {
+		if n, err := strconv.Atoi(os.Getenv(env)); err == nil && n > 0 {
+			*dst = n
+		}
+	}
+	dur := func(env string, dst *time.Duration) {
+		if d, err := time.ParseDuration(os.Getenv(env)); err == nil && d > 0 {
+			*dst = d
+		}
+	}
+	str("OTEL_BLOCKER_NAMESPACE", &c.Namespace)
+	str("OTEL_BLOCKER_CONFIGMAP", &c.ConfigMap)
+	str("OTEL_BLOCKER_KEY", &c.Key)
+	num("OTEL_BLOCKER_MIN_PROBES", &c.MinProbes)
+	num("OTEL_BLOCKER_MIN_PATHS", &c.MinPaths)
+	num("OTEL_BLOCKER_MAX_ENTRIES", &c.MaxEntries)
+	dur("OTEL_BLOCKER_INTERVAL", &c.Interval)
+	dur("OTEL_BLOCKER_MIN_WRITE", &c.MinWrite)
+	dur("OTEL_BLOCKER_TTL", &c.BlockTTL)
+	for _, item := range strings.Split(os.Getenv("OTEL_BLOCKER_ALLOW"), ",") {
+		item = strings.TrimSpace(item)
+		if item == "" {
+			continue
+		}
+		if p, err := netip.ParsePrefix(item); err == nil {
+			c.Allow = append(c.Allow, p)
+		} else if a, err := netip.ParseAddr(item); err == nil {
+			c.Allow = append(c.Allow, netip.PrefixFrom(a, a.BitLen()))
+		} else {
+			log.Printf("blocker: ignoring invalid OTEL_BLOCKER_ALLOW entry %q", item)
+		}
+	}
+	return c
+}
+
+// Start builds a client (in-cluster, or KUBECONFIG locally) and runs the
+// blocker in the background. It returns nil, nil when disabled.
+func Start(ctx context.Context, cfg Config, source Source) (*Blocker, error) {
+	if !cfg.Enabled {
+		return nil, nil
+	}
+	rc, err := rest.InClusterConfig()
+	if err != nil {
+		kubeconfig := os.Getenv("KUBECONFIG")
+		if kubeconfig == "" {
+			home, herr := os.UserHomeDir()
+			if herr != nil {
+				return nil, herr
+			}
+			kubeconfig = home + "/.kube/config"
+		}
+		if rc, err = clientcmd.BuildConfigFromFlags("", kubeconfig); err != nil {
+			return nil, fmt.Errorf("blocker kubernetes config: %w", err)
+		}
+	}
+	client, err := kubernetes.NewForConfig(rc)
+	if err != nil {
+		return nil, fmt.Errorf("blocker kubernetes client: %w", err)
+	}
+	b := New(cfg, client, source)
+	go b.Run(ctx)
+	return b, nil
+}
+
+type Blocker struct {
+	cfg    Config
+	client kubernetes.Interface
+	source Source
+	now    func() time.Time
+
+	mu        sync.Mutex
+	lastWrite time.Time
+	lastErr   string
+	last      Status
+}
+
+// Status is a snapshot for the UI and for tests.
+type Status struct {
+	Enabled    bool             `json:"enabled"`
+	DryRun     bool             `json:"dryRun"`
+	Evaluated  time.Time        `json:"evaluated"`
+	Candidates []string         `json:"candidates"`
+	Managed    map[string]int64 `json:"managed"` // ip -> unix expiry
+	Wrote      bool             `json:"wrote"`
+	LastWrite  time.Time        `json:"lastWrite"`
+	Error      string           `json:"error,omitempty"`
+}
+
+func New(cfg Config, client kubernetes.Interface, source Source) *Blocker {
+	d := DefaultConfig()
+	if cfg.Interval <= 0 {
+		cfg.Interval = d.Interval
+	}
+	if cfg.MinWrite <= 0 {
+		cfg.MinWrite = d.MinWrite
+	}
+	if cfg.BlockTTL <= 0 {
+		cfg.BlockTTL = d.BlockTTL
+	}
+	if cfg.MaxEntries <= 0 {
+		cfg.MaxEntries = d.MaxEntries
+	}
+	if cfg.MaxNewPerTick <= 0 {
+		cfg.MaxNewPerTick = d.MaxNewPerTick
+	}
+	if cfg.MinProbes <= 0 {
+		cfg.MinProbes = d.MinProbes
+	}
+	if cfg.MinPaths <= 0 {
+		cfg.MinPaths = d.MinPaths
+	}
+	if cfg.BroadProbes <= 0 {
+		cfg.BroadProbes = d.BroadProbes
+	}
+	if cfg.BroadPaths <= 0 {
+		cfg.BroadPaths = d.BroadPaths
+	}
+	return &Blocker{cfg: cfg, client: client, source: source, now: time.Now}
+}
+
+// Run evaluates the feed until ctx is cancelled.
+func (b *Blocker) Run(ctx context.Context) {
+	log.Printf("blocker: started (dryRun=%t, %s/%s key=%s)", b.cfg.DryRun, b.cfg.Namespace, b.cfg.ConfigMap, b.cfg.Key)
+	t := time.NewTicker(b.cfg.Interval)
+	defer t.Stop()
+	for {
+		if err := b.Tick(ctx); err != nil {
+			log.Printf("blocker: %v", err)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+		}
+	}
+}
+
+// Blockable applies the policy to one promoted IP.
+func (b *Blocker) Blockable(ip honeypot.IP) bool {
+	if honeypot.Untrackable(ip.Address) {
+		return false
+	}
+	addr, err := netip.ParseAddr(ip.Address)
+	if err != nil {
+		return false
+	}
+	for _, p := range b.cfg.Allow {
+		if p.Contains(addr) {
+			return false
+		}
+	}
+	return (ip.Count >= b.cfg.MinProbes && ip.Paths >= b.cfg.MinPaths) ||
+		(ip.Count >= b.cfg.BroadProbes && ip.Paths >= b.cfg.BroadPaths)
+}
+
+// Tick runs one evaluation. It is exported so tests and a manual trigger can
+// drive it without waiting on the timer.
+func (b *Blocker) Tick(ctx context.Context) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	now := b.now()
+	status := Status{Enabled: true, DryRun: b.cfg.DryRun, Evaluated: now, Managed: map[string]int64{}, LastWrite: b.lastWrite}
+	defer func() { b.last = status }()
+
+	cms := b.client.CoreV1().ConfigMaps(b.cfg.Namespace)
+	cm, err := cms.Get(ctx, b.cfg.ConfigMap, metav1.GetOptions{})
+	if err != nil {
+		b.lastErr = err.Error()
+		status.Error = b.lastErr
+		return fmt.Errorf("get configmap: %w", err)
+	}
+
+	previous := map[string]int64{}
+	if raw := cm.Annotations[AnnotationManaged]; raw != "" {
+		if err := json.Unmarshal([]byte(raw), &previous); err != nil {
+			previous = map[string]int64{} // unreadable state: treat as empty rather than guess
+		}
+	}
+	managed := map[string]int64{}
+	for ip, exp := range previous {
+		if exp > now.Unix() {
+			managed[ip] = exp
+		}
+	}
+
+	added := 0
+	for _, cand := range b.source.PromotedIPs() {
+		if !b.Blockable(cand) {
+			continue
+		}
+		status.Candidates = append(status.Candidates, cand.Address)
+		_, exists := managed[cand.Address]
+		if !exists {
+			if added >= b.cfg.MaxNewPerTick {
+				continue
+			}
+			added++
+		}
+		managed[cand.Address] = now.Add(b.cfg.BlockTTL).Unix()
+	}
+	sort.Strings(status.Candidates)
+	for len(managed) > b.cfg.MaxEntries { // drop the soonest-expiring first
+		var oldest string
+		for ip, exp := range managed {
+			if oldest == "" || exp < managed[oldest] || (exp == managed[oldest] && ip < oldest) {
+				oldest = ip
+			}
+		}
+		delete(managed, oldest)
+	}
+	status.Managed = managed
+
+	current := splitCIDRs(cm.Data[b.cfg.Key])
+	manual := make([]string, 0, len(current))
+	for _, entry := range current {
+		if _, wasOurs := previous[entry]; !wasOurs {
+			manual = append(manual, entry)
+		}
+	}
+	desired := append([]string(nil), manual...)
+	for ip := range managed {
+		desired = append(desired, ip)
+	}
+	desired = dedupSorted(desired)
+	changed := strings.Join(desired, ",") != strings.Join(dedupSorted(current), ",")
+
+	if !changed {
+		return nil
+	}
+	if b.cfg.DryRun {
+		log.Printf("blocker[dry-run]: would set %s to %d entries (%d managed): %v", b.cfg.Key, len(desired), len(managed), managedList(managed))
+		return nil
+	}
+	if !b.lastWrite.IsZero() && now.Sub(b.lastWrite) < b.cfg.MinWrite {
+		return nil // rate-limited; the change is picked up on a later tick
+	}
+	state, _ := json.Marshal(managed)
+	if cm.Annotations == nil {
+		cm.Annotations = map[string]string{}
+	}
+	if cm.Data == nil {
+		cm.Data = map[string]string{}
+	}
+	cm.Annotations[AnnotationManaged] = string(state)
+	cm.Data[b.cfg.Key] = strings.Join(desired, ",")
+	if len(desired) == 0 {
+		delete(cm.Data, b.cfg.Key)
+	}
+	if _, err := cms.Update(ctx, cm, metav1.UpdateOptions{}); err != nil {
+		b.lastErr = err.Error()
+		status.Error = b.lastErr
+		return fmt.Errorf("update configmap: %w", err)
+	}
+	b.lastWrite, b.lastErr = now, ""
+	status.Wrote, status.LastWrite = true, now
+	log.Printf("blocker: updated %s: %d entries (%d managed)", b.cfg.Key, len(desired), len(managed))
+	return nil
+}
+
+// HandleStatus serves the last evaluation. Register it behind authentication.
+func (b *Blocker) HandleStatus(w http.ResponseWriter, r *http.Request) {
+	b.mu.Lock()
+	s := b.last
+	b.mu.Unlock()
+	if s.Evaluated.IsZero() {
+		s = Status{Enabled: true, DryRun: b.cfg.DryRun, Managed: map[string]int64{}}
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Cache-Control", "no-store")
+	_ = json.NewEncoder(w).Encode(s)
+}
+
+func splitCIDRs(v string) []string {
+	var out []string
+	for _, p := range strings.Split(v, ",") {
+		if p = strings.TrimSpace(p); p != "" {
+			out = append(out, p)
+		}
+	}
+	return out
+}
+
+func dedupSorted(in []string) []string {
+	seen := map[string]struct{}{}
+	out := make([]string, 0, len(in))
+	for _, v := range in {
+		if _, ok := seen[v]; !ok {
+			seen[v] = struct{}{}
+			out = append(out, v)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+func managedList(m map[string]int64) []string {
+	out := make([]string, 0, len(m))
+	for ip := range m {
+		out = append(out, ip)
+	}
+	sort.Strings(out)
+	return out
+}

internal/blocker/blocker_test.go

@@ -0,0 +1,153 @@
+package blocker
+
+import (
+	"context"
+	"encoding/json"
+	"net/netip"
+	"strings"
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+
+	"git.tornberg.me/go-otel/internal/honeypot"
+)
+
+type feed []honeypot.IP
+
+func (f feed) PromotedIPs() []honeypot.IP { return f }
+
+func setup(t *testing.T, cfg Config, data map[string]string, src Source) (*Blocker, *fake.Clientset, *time.Time) {
+	t.Helper()
+	client := fake.NewSimpleClientset(&corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "ingress-nginx", Name: "ingress-nginx-controller"},
+		Data:       data,
+	})
+	b := New(cfg, client, src)
+	clock := time.Date(2026, 9, 19, 12, 0, 0, 0, time.UTC)
+	b.now = func() time.Time { return clock }
+	return b, client, &clock
+}
+
+func current(t *testing.T, c *fake.Clientset) *corev1.ConfigMap {
+	t.Helper()
+	cm, err := c.CoreV1().ConfigMaps("ingress-nginx").Get(context.Background(), "ingress-nginx-controller", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return cm
+}
+
+func scanner(ip string) honeypot.IP { return honeypot.IP{Address: ip, Count: 40, Paths: 12} }
+
+func liveConfig() Config {
+	c := DefaultConfig()
+	c.DryRun = false
+	return c
+}
+
+func TestDryRunWritesNothing(t *testing.T) {
+	b, client, _ := setup(t, DefaultConfig(), map[string]string{"use-gzip": "true"}, feed{scanner("203.0.113.5")})
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	cm := current(t, client)
+	if _, ok := cm.Data["block-cidrs"]; ok || cm.Annotations[AnnotationManaged] != "" {
+		t.Fatalf("dry run wrote to the configmap: %+v %+v", cm.Data, cm.Annotations)
+	}
+	if len(b.last.Candidates) != 1 {
+		t.Fatalf("dry run should still report candidates: %+v", b.last)
+	}
+}
+
+func TestBlocksScannerAndPreservesManualEntries(t *testing.T) {
+	b, client, _ := setup(t, liveConfig(), map[string]string{"block-cidrs": "198.51.100.0/24", "use-gzip": "true"}, feed{scanner("203.0.113.5")})
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	cm := current(t, client)
+	if got := cm.Data["block-cidrs"]; got != "198.51.100.0/24,203.0.113.5" {
+		t.Fatalf("block-cidrs = %q", got)
+	}
+	if cm.Data["use-gzip"] != "true" {
+		t.Fatal("unrelated keys must be untouched")
+	}
+	var state map[string]int64
+	if err := json.Unmarshal([]byte(cm.Annotations[AnnotationManaged]), &state); err != nil || len(state) != 1 {
+		t.Fatalf("annotation = %q (%v)", cm.Annotations[AnnotationManaged], err)
+	}
+}
+
+func TestNeverBlocksInternalOrAllowlisted(t *testing.T) {
+	cfg := liveConfig()
+	cfg.Allow = []netip.Prefix{netip.MustParsePrefix("203.0.113.0/24")}
+	b, client, _ := setup(t, cfg, nil, feed{scanner("127.0.0.1"), scanner("10.244.4.1"), scanner("192.168.1.9"), scanner("203.0.113.77")})
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if v := current(t, client).Data["block-cidrs"]; v != "" {
+		t.Fatalf("blocked %q", v)
+	}
+}
+
+func TestThresholdsKeepOutOccasionalVisitors(t *testing.T) {
+	b, client, _ := setup(t, liveConfig(), nil, feed{
+		{Address: "203.0.113.1", Count: 4, Paths: 4},    // too few hits
+		{Address: "203.0.113.2", Count: 500, Paths: 1},  // one path hammered
+		{Address: "203.0.113.3", Count: 200, Paths: 60}, // broad scanner
+	})
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if v := current(t, client).Data["block-cidrs"]; v != "203.0.113.3" {
+		t.Fatalf("block-cidrs = %q", v)
+	}
+}
+
+func TestExpiryRemovesOnlyManagedEntries(t *testing.T) {
+	src := feed{scanner("203.0.113.5")}
+	b, client, clock := setup(t, liveConfig(), map[string]string{"block-cidrs": "198.51.100.7"}, src)
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	b.source = feed{} // scanner goes quiet
+	*clock = clock.Add(25 * time.Hour)
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if v := current(t, client).Data["block-cidrs"]; v != "198.51.100.7" {
+		t.Fatalf("after expiry block-cidrs = %q, want only the manual entry", v)
+	}
+}
+
+func TestWritesAreRateLimitedAndNewPerTickCapped(t *testing.T) {
+	cfg := liveConfig()
+	cfg.MaxNewPerTick = 2
+	var many feed
+	for _, ip := range []string{"203.0.113.1", "203.0.113.2", "203.0.113.3", "203.0.113.4"} {
+		many = append(many, scanner(ip))
+	}
+	b, client, clock := setup(t, cfg, nil, many)
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if n := len(strings.Split(current(t, client).Data["block-cidrs"], ",")); n != 2 {
+		t.Fatalf("first tick added %d, want 2", n)
+	}
+	*clock = clock.Add(time.Minute) // inside MinWrite
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if n := len(strings.Split(current(t, client).Data["block-cidrs"], ",")); n != 2 {
+		t.Fatalf("write was not rate limited: %d entries", n)
+	}
+	*clock = clock.Add(10 * time.Minute)
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if n := len(strings.Split(current(t, client).Data["block-cidrs"], ",")); n != 4 {
+		t.Fatalf("after the interval got %d entries, want 4", n)
+	}
+}

internal/collector/collector_test.go

@@ -392,6 +392,47 @@ func TestHandleIssuesAndByID(t *testing.T) {
 	}
 }
 
+func TestIssueTrendsUseTelemetryTimestamps(t *testing.T) {
+	coll := New(noopBroadcaster{}, 10, nil)
+
+	base := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
+	ld := plog.NewLogs()
+	rl := ld.ResourceLogs().AppendEmpty()
+	rl.Resource().Attributes().PutStr("service.name", "api")
+	records := rl.ScopeLogs().AppendEmpty().LogRecords()
+	first := records.AppendEmpty()
+	first.SetTimestamp(pcommon.NewTimestampFromTime(base))
+	first.SetSeverityNumber(plog.SeverityNumberError)
+	first.Body().SetStr("first failure")
+	second := records.AppendEmpty()
+	second.SetTimestamp(pcommon.NewTimestampFromTime(base.Add(time.Hour)))
+	second.SetSeverityNumber(plog.SeverityNumberError)
+	second.Body().SetStr("second failure")
+
+	if err := coll.ConsumeLogs(context.Background(), ld); err != nil {
+		t.Fatalf("ConsumeLogs failed: %v", err)
+	}
+
+	list := coll.ListIssues(0)
+	if len(list) != 2 {
+		t.Fatalf("expected two issues, got %d", len(list))
+	}
+	for _, issue := range list {
+		if len(issue.Trend) != 1 {
+			t.Fatalf("expected one trend bucket for %q, got %+v", issue.Title, issue.Trend)
+		}
+	}
+	expected := map[string]string{
+		"first failure":  issues.HourBucket(base.UnixNano()),
+		"second failure": issues.HourBucket(base.Add(time.Hour).UnixNano()),
+	}
+	for _, issue := range list {
+		if _, ok := issue.Trend[expected[issue.Title]]; !ok {
+			t.Fatalf("telemetry timestamp was not used for %q: %+v", issue.Title, issue.Trend)
+		}
+	}
+}
+
 func TestHandleMLDetectorTest(t *testing.T) {
 	coll := New(nil, 10, nil)
 	req := httptest.NewRequest(http.MethodPost, "/api/ml/test-spike", nil)

internal/collector/issue_extract.go

@@ -37,7 +37,14 @@ func extractSpanErrors(tracker *issues.Tracker, td ptrace.Traces, timestamp int6
 				service := spanServiceName(rs.Resource(), span)
 				title := spanErrorTitle(span)
 				found = true
-				tracker.Record("span_error", service, title, spanSample(span, rs.Resource(), timestamp, title))
+				occurrenceTimestamp := span.StartTimestamp()
+				if occurrenceTimestamp == 0 {
+					occurrenceTimestamp = span.EndTimestamp()
+				}
+				if occurrenceTimestamp == 0 {
+					occurrenceTimestamp = pcommon.Timestamp(timestamp)
+				}
+				tracker.Record("span_error", service, title, spanSample(span, rs.Resource(), int64(occurrenceTimestamp), title))
 			}
 		}
 	}
@@ -102,7 +109,14 @@ func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) bo
 				title := logErrorTitle(record)
 				found = true
 				kind := "log_error"
-				sample := logSample(record, rl.Resource(), timestamp, title)
+				occurrenceTimestamp := record.Timestamp()
+				if occurrenceTimestamp == 0 {
+					occurrenceTimestamp = record.ObservedTimestamp()
+				}
+				if occurrenceTimestamp == 0 {
+					occurrenceTimestamp = pcommon.Timestamp(timestamp)
+				}
+				sample := logSample(record, rl.Resource(), int64(occurrenceTimestamp), title)
 				if attrString(record.Attributes(), "source") == "frontend" {
 					kind = "frontend_error"
 					sample.URL = attrString(record.Attributes(), "browser.url")

internal/honeypot/honeypot.go

@@ -8,11 +8,14 @@ import (
 	"encoding/json"
 	"net"
 	"net/http"
+	"net/netip"
 	"os"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
+
+	"git.tornberg.me/go-otel/internal/triage"
 )
 
 const (
@@ -20,6 +23,7 @@ const (
 	defaultWindow    = 15 * time.Minute
 	defaultTTL       = 1 * time.Hour
 	maxTracked       = 10000
+	maxPathsTracked  = 64
 )
 
 type Config struct {
@@ -28,9 +32,14 @@ type Config struct {
 	TTL       time.Duration
 	Delay     time.Duration
 	Token     string
+	// MinPaths is how many distinct suspicious paths an IP must request
+	// before it is promoted. A legitimate client repeatedly fetching one
+	// odd path (a manifest.json) is not a scanner. Zero means 1.
+	MinPaths int
 }
 
 type entry struct {
+	Paths         map[string]struct{}
 	Count         int
 	FirstSeen     time.Time
 	LastSeen      time.Time
@@ -41,6 +50,7 @@ type entry struct {
 type IP struct {
 	Address       string    `json:"ip"`
 	Count         int       `json:"count"`
+	Paths         int       `json:"paths"`
 	FirstSeen     time.Time `json:"first_seen"`
 	LastSeen      time.Time `json:"last_seen"`
 	PromotedUntil time.Time `json:"promoted_until"`
@@ -52,6 +62,7 @@ type Tracker struct {
 	ttl       time.Duration
 	delay     time.Duration
 	token     string
+	minPaths  int
 
 	mu      sync.Mutex
 	clients map[string]entry
@@ -73,11 +84,17 @@ func New(cfg Config) *Tracker {
 	if cfg.Delay > 10*time.Second {
 		cfg.Delay = 10 * time.Second
 	}
-	return &Tracker{threshold: cfg.Threshold, window: cfg.Window, ttl: cfg.TTL, delay: cfg.Delay, token: cfg.Token, clients: make(map[string]entry)}
+	if cfg.MinPaths < 1 {
+		cfg.MinPaths = 1
+	}
+	return &Tracker{threshold: cfg.Threshold, window: cfg.Window, ttl: cfg.TTL, delay: cfg.Delay, token: cfg.Token, minPaths: cfg.MinPaths, clients: make(map[string]entry)}
 }
 
 func ConfigFromEnv() Config {
-	cfg := Config{Threshold: defaultThreshold, Window: defaultWindow, TTL: defaultTTL, Delay: 0, Token: os.Getenv("OTEL_HONEYPOT_TOKEN")}
+	cfg := Config{Threshold: defaultThreshold, Window: defaultWindow, TTL: defaultTTL, Delay: 0, Token: os.Getenv("OTEL_HONEYPOT_TOKEN"), MinPaths: 3}
+	if n, err := strconv.Atoi(os.Getenv("OTEL_HONEYPOT_MIN_PATHS")); err == nil && n > 0 {
+		cfg.MinPaths = n
+	}
 	if n, err := strconv.Atoi(os.Getenv("OTEL_HONEYPOT_THRESHOLD")); err == nil && n > 0 {
 		cfg.Threshold = n
 	}
@@ -102,7 +119,7 @@ func (t *Tracker) Middleware(next http.Handler) http.Handler {
 			return
 		}
 		ip := clientIP(r)
-		promoted := t.record(ip)
+		promoted := t.record(ip, r.URL.Path)
 		if promoted && t.delay > 0 {
 			timer := time.NewTimer(t.delay)
 			select {
@@ -125,17 +142,55 @@ func (t *Tracker) Middleware(next http.Handler) http.Handler {
 	})
 }
 
-func (t *Tracker) record(ip string) bool {
+// Untrackable reports addresses that must never be tracked or blocked:
+// loopback, private, link-local, multicast, unspecified or unparsable. Behind
+// a proxy hop the observed peer is often 127.0.0.1 or a node address, and
+// promoting it would cut off the ingress itself.
+func Untrackable(ip string) bool {
+	addr, err := netip.ParseAddr(strings.TrimSpace(ip))
+	if err != nil {
+		return true
+	}
+	addr = addr.Unmap()
+	if addr.IsLoopback() || addr.IsPrivate() || addr.IsLinkLocalUnicast() || addr.IsUnspecified() || addr.IsMulticast() {
+		return true
+	}
+	for _, p := range reservedPrefixes {
+		if p.Contains(addr) {
+			return true
+		}
+	}
+	return false
+}
+
+// reservedPrefixes are ranges the standard library helpers do not cover but
+// that are never a remote scanner: carrier-grade NAT (also used by Tailscale
+// and mobile carriers), IETF protocol assignments, benchmarking, and the
+// reserved/broadcast block.
+var reservedPrefixes = []netip.Prefix{
+	netip.MustParsePrefix("100.64.0.0/10"),
+	netip.MustParsePrefix("192.0.0.0/24"),
+	netip.MustParsePrefix("198.18.0.0/15"),
+	netip.MustParsePrefix("240.0.0.0/4"),
+}
+
+func (t *Tracker) record(ip, path string) bool {
+	if Untrackable(ip) {
+		return false
+	}
 	now := time.Now()
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	state, ok := t.clients[ip]
 	if !ok || now.Sub(state.LastSeen) > t.window {
-		state = entry{FirstSeen: now}
+		state = entry{FirstSeen: now, Paths: map[string]struct{}{}}
 	}
 	state.Count++
 	state.LastSeen = now
-	if state.Count >= t.threshold {
+	if len(state.Paths) < maxPathsTracked {
+		state.Paths[strings.ToLower(path)] = struct{}{}
+	}
+	if state.Count >= t.threshold && len(state.Paths) >= t.minPaths {
 		state.PromotedUntil = now.Add(t.ttl)
 	}
 	t.clients[ip] = state
@@ -152,10 +207,10 @@ func (t *Tracker) record(ip string) bool {
 // promoted-IP feed.
 func (t *Tracker) RecordTelemetry(ip, path string) bool {
 	ip = strings.TrimSpace(strings.SplitN(ip, ",", 2)[0])
-	if ip == "" || !isSuspiciousPath(path) {
+	if ip == "" || !(isSuspiciousPath(path) || triage.IsProbePath(path)) {
 		return false
 	}
-	return t.record(ip)
+	return t.record(ip, path)
 }
 
 // PromotedIPs returns a snapshot suitable for an ingress synchronizer.
@@ -168,7 +223,7 @@ func (t *Tracker) PromotedIPs() []IP {
 		if state.PromotedUntil.IsZero() || !now.Before(state.PromotedUntil) {
 			continue
 		}
-		result = append(result, IP{Address: address, Count: state.Count, FirstSeen: state.FirstSeen, LastSeen: state.LastSeen, PromotedUntil: state.PromotedUntil})
+		result = append(result, IP{Address: address, Count: state.Count, Paths: len(state.Paths), FirstSeen: state.FirstSeen, LastSeen: state.LastSeen, PromotedUntil: state.PromotedUntil})
 	}
 	return result
 }

internal/honeypot/honeypot_test.go

@@ -10,8 +10,8 @@ import (
 
 func TestPromotesAfterThresholdAndExpires(t *testing.T) {
 	tracker := New(Config{Threshold: 2, Window: time.Minute, TTL: time.Millisecond, Token: "secret"})
-	for i := 0; i < 3; i++ {
+	for _, path := range []string{"/.env", "/.git/config", "/wp-login.php"} {
-		req := httptest.NewRequest(http.MethodGet, "https://example.test/.env", nil)
+		req := httptest.NewRequest(http.MethodGet, "https://example.test"+path, nil)
 		req.RemoteAddr = "192.0.2.10:1234"
 		tracker.Middleware(http.HandlerFunc(http.NotFound)).ServeHTTP(httptest.NewRecorder(), req)
 	}
@@ -56,3 +56,29 @@ func TestRecordTelemetryPromotesTheObservedSourceIP(t *testing.T) {
 		t.Fatalf("unexpected promoted IPs: %+v", ips)
 	}
 }
+
+func TestNeverTracksInternalAddresses(t *testing.T) {
+	tracker := New(Config{Threshold: 1, TTL: time.Minute})
+	for _, ip := range []string{"127.0.0.1", "::1", "10.244.4.1", "192.168.1.5", "169.254.1.1", "100.64.0.1", "100.127.255.254", "198.18.0.1", "240.0.0.1", "255.255.255.255", "::ffff:10.1.2.3", "not-an-ip", ""} {
+		if tracker.RecordTelemetry(ip, "/.env") {
+			t.Fatalf("%q must never be promoted", ip)
+		}
+	}
+	if got := len(tracker.PromotedIPs()); got != 0 {
+		t.Fatalf("promoted %d internal addresses", got)
+	}
+}
+
+func TestMinPathsRequiresBreadth(t *testing.T) {
+	tracker := New(Config{Threshold: 2, TTL: time.Minute, MinPaths: 3})
+	for i := 0; i < 5; i++ {
+		tracker.RecordTelemetry("203.0.113.9", "/manifest.json")
+	}
+	if len(tracker.PromotedIPs()) != 0 {
+		t.Fatal("one repeated path must not promote")
+	}
+	tracker.RecordTelemetry("203.0.113.9", "/.env")
+	if !tracker.RecordTelemetry("203.0.113.9", "/wp-login.php") {
+		t.Fatal("three distinct paths must promote")
+	}
+}

internal/storage/clickhouse.go

@@ -153,6 +153,9 @@ func NewClickHouseStore(ctx context.Context, cfg ClickHouseConfig) (*ClickHouseS
 // initSchema creates the raw entries table and the materialized hourly
 // rollup used by the stats endpoints. Idempotent.
 func (s *ClickHouseStore) initSchema(ctx context.Context) error {
+	if err := EnsureNormalizedSchema(ctx, s.conn, s.db); err != nil {
+		return fmt.Errorf("normalized schema: %w", err)
+	}
 	if err := s.conn.Exec(ctx, fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.entries (
 		timestamp DateTime64(9, 'UTC') NOT NULL,
 		type      LowCardinality(String) NOT NULL,
@@ -361,6 +364,7 @@ func (s *ClickHouseStore) Append(ctx context.Context, entry websocket.HistoryEnt
 	if err := s.appendSpanFacts(ctx, ts, facts); err != nil {
 		return fmt.Errorf("clickhouse span facts: %w", err)
 	}
+	s.appendNormalized(ctx, entry)
 	return nil
 }
 

internal/storage/normalized.go

@@ -0,0 +1,107 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+
+	"git.tornberg.me/go-otel/internal/normalization"
+	"git.tornberg.me/go-otel/internal/websocket"
+)
+
+// normalizedColumns is the explicit insert column list. Tables migrated with
+// ALTER ADD COLUMN have a different physical column order than a fresh
+// CREATE, so positional inserts are not safe.
+const normalizedColumns = `event_id, entry_type, event_index, ingest_timestamp, timestamp, service, release, environment,
+	kind, source, is_error, status_class, status_code, duration_ms, route, span_name, metric_name, metric_type, metric_unit,
+	metric_value, metric_count, metric_sum, metric_min, metric_max, exception_type, message_normalized, stack_signature,
+	trace_id, issue_fingerprint, is_retry, is_timeout, is_cancellation, exact_signature, pattern_signature, version,
+	method, host, user_agent, origin_host, referer_host, fetch_site, fetch_mode`
+
+// NormalizedInsertSQL is the INSERT prefix for a normalized_events batch.
+func NormalizedInsertSQL(db string) string {
+	return fmt.Sprintf("INSERT INTO %s.normalized_events (%s)", db, normalizedColumns)
+}
+
+// EnsureNormalizedSchema creates normalized_events and applies the additive
+// migrations. Idempotent.
+func EnsureNormalizedSchema(ctx context.Context, conn driver.Conn, db string) error {
+	query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.normalized_events (
+		event_id String,
+		entry_type LowCardinality(String),
+		event_index UInt32,
+		ingest_timestamp DateTime64(9, 'UTC'),
+		timestamp DateTime64(9, 'UTC'),
+		service LowCardinality(String), release LowCardinality(String), environment LowCardinality(String),
+		kind LowCardinality(String), source LowCardinality(String), is_error UInt8,
+		status_class LowCardinality(String), status_code UInt16, duration_ms Float64,
+		route String, span_name String, metric_name String, metric_type LowCardinality(String), metric_unit String,
+		metric_value Float64, metric_count UInt64, metric_sum Float64, metric_min Float64, metric_max Float64,
+		exception_type String, message_normalized String, stack_signature String, trace_id String, issue_fingerprint String,
+		is_retry UInt8, is_timeout UInt8, is_cancellation UInt8,
+		exact_signature String, pattern_signature String, version UInt8 DEFAULT 1,
+		method String DEFAULT '', host String DEFAULT '', user_agent String DEFAULT '',
+		origin_host String DEFAULT '', referer_host String DEFAULT '', fetch_site String DEFAULT '', fetch_mode String DEFAULT ''
+	) ENGINE = ReplacingMergeTree(version)
+	PARTITION BY toYYYYMM(timestamp)
+	ORDER BY (service, kind, pattern_signature, timestamp, event_id)
+	TTL toDateTime(timestamp) + INTERVAL 180 DAY`, db)
+	if err := conn.Exec(ctx, query); err != nil {
+		return err
+	}
+	// Developer tables predate these columns; historical rows get safe defaults.
+	for _, col := range []string{
+		"issue_fingerprint String DEFAULT ''", "is_retry UInt8 DEFAULT 0", "is_timeout UInt8 DEFAULT 0",
+		"is_cancellation UInt8 DEFAULT 0", "method String DEFAULT ''", "host String DEFAULT ''",
+		"user_agent String DEFAULT ''", "origin_host String DEFAULT ''", "referer_host String DEFAULT ''",
+		"fetch_site String DEFAULT ''", "fetch_mode String DEFAULT ''",
+	} {
+		if err := conn.Exec(ctx, fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS %s", db, col)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// AppendNormalizedEvent adds one event to a batch prepared with NormalizedInsertSQL.
+func AppendNormalizedEvent(batch driver.Batch, e normalization.Event) error {
+	return batch.Append(
+		e.EventID, e.EntryType, e.EventIndex, time.Unix(0, e.IngestTimestamp), time.Unix(0, e.Timestamp),
+		e.Service, e.Release, e.Environment, e.Kind, e.Source, boolToUint8(e.IsError),
+		e.StatusClass, uint16(e.StatusCode), e.DurationMS, e.Route, e.SpanName, e.MetricName, e.MetricType, e.MetricUnit,
+		e.MetricValue, e.MetricCount, e.MetricSum, e.MetricMin, e.MetricMax,
+		e.ExceptionType, e.MessageNormalized, e.StackSignature, e.TraceID, e.IssueFingerprint,
+		boolToUint8(e.IsRetry), boolToUint8(e.IsTimeout), boolToUint8(e.IsCancellation), e.ExactSignature, e.PatternSignature,
+		uint8(1), e.Method, e.Host, e.UserAgent, e.OriginHost, e.RefererHost, e.FetchSite, e.FetchMode,
+	)
+}
+
+// appendNormalized derives and stores normalized rows for one raw entry.
+// The raw entry is the source of truth, so a failure here is logged and never
+// fails ingest; the backfill tool can rebuild any gap.
+func (s *ClickHouseStore) appendNormalized(ctx context.Context, entry websocket.HistoryEntry) {
+	events, err := normalization.NormalizeEntry(entry.Type, entry.Timestamp, entry.Data)
+	if err != nil || len(events) == 0 {
+		return
+	}
+	ctx = clickhouse.Context(ctx, clickhouse.WithSettings(clickhouse.Settings{"async_insert": 1, "wait_for_async_insert": 0}))
+	batch, err := s.conn.PrepareBatch(ctx, NormalizedInsertSQL(s.db))
+	if err != nil {
+		log.Printf("normalized insert prepare: %v", err)
+		return
+	}
+	for _, e := range events {
+		if err := AppendNormalizedEvent(batch, e); err != nil {
+			_ = batch.Abort()
+			log.Printf("normalized insert append: %v", err)
+			return
+		}
+	}
+	if err := batch.Send(); err != nil {
+		log.Printf("normalized insert send: %v", err)
+	}
+}

internal/triage/triage.go

@@ -0,0 +1,107 @@
+// Package triage assigns a cause-level category to a normalized error
+// pattern. The categories are deliberately not restatements of status or
+// severity: they answer "why did this happen and who should care".
+//
+// Rules are deterministic and ordered; the first match wins. Structured flags
+// (timeout, cancellation, retry) are preferred over message text.
+package triage
+
+import (
+	"regexp"
+	"strings"
+)
+
+type Category string
+
+const (
+	ScannerProbe          Category = "scanner_probe"
+	DependencyUnreachable Category = "dependency_unreachable"
+	ClientCredentialIssue Category = "client_credential_issue"
+	StaleReference        Category = "stale_reference"
+	ServiceDefect         Category = "service_defect"
+	CapacitySaturation    Category = "capacity_saturation"
+	ExpectedLifecycle     Category = "expected_lifecycle"
+	NeedsReview           Category = "needs_review"
+)
+
+// Input is the non-PII slice of a normalized event the rules look at.
+type Input struct {
+	Service        string
+	Kind           string
+	Status         int
+	Route          string
+	Title          string
+	Message        string
+	IsTimeout      bool
+	IsCancellation bool
+	IsRetry        bool
+}
+
+type Result struct {
+	Category Category `json:"category"`
+	Rule     string   `json:"rule"` // which rule fired, for review and tuning
+}
+
+var (
+	// Probe families: secrets/dotfiles, config and backup file types, source
+	// control, CMS/admin paths, path traversal. Deliberately path-only.
+	probeRoute = regexp.MustCompile(`(?i)(\.env\b|\.git\b|\.htpasswd|\.htaccess|\.npmrc|\.claude|\.docker|\.kube|\.aws|\.ssh|` +
+		`wp-|/wp/|/wp$|wordpress|phpmyadmin|phpinfo|cgi-bin|xmlrpc|userfiles|@fs/|etc/passwd|proc/self|\.\./|%2e%2e|` +
+		`\.php\b|credentials|secrets?\b|firebase|serviceaccount|/config(/|\.)|settings\.(js|json)$|bootstrap\.properties|` +
+		`\.(ya?ml|toml|properties|sql|bak|old|swp|zip|tar|tgz|gz|ini|conf|cfg)$)`)
+	bugText  = regexp.MustCompile(`(?i)panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault`)
+	capText  = regexp.MustCompile(`(?i)timeout|deadline exceeded|\bpool\b|memory limit|\boom(killed)?\b|too many|queue full|exhaust`)
+	depText  = regexp.MustCompile(`(?i)dial|connection refused|\beof\b|no such host|upstream|unavailable|broken pipe|reset by peer`)
+	lifeText = regexp.MustCompile(`(?i)healthz|readyz|/ping|keepalive|vital|\bcls\b|ttfb|fcp|lcp|snapshot|replay|unchanged|session\.load`)
+)
+
+// IsProbePath reports whether a request path looks like an automated scan.
+// It is the single definition shared by triage and IP-blocking analysis.
+func IsProbePath(path string) bool { return probeRoute.MatchString(path) }
+
+// Classify returns the category for one pattern's example event.
+func Classify(in Input) Result {
+	blob := strings.Join([]string{in.Title, in.Message, in.Route}, " ")
+	switch {
+	case probeRoute.MatchString(in.Route):
+		return Result{ScannerProbe, "probe-route"}
+	case in.IsCancellation:
+		return Result{ExpectedLifecycle, "cancellation-flag"}
+	case in.IsRetry:
+		return Result{ExpectedLifecycle, "retry-flag"}
+	case bugText.MatchString(blob):
+		return Result{ServiceDefect, "bug-text"}
+	case in.IsTimeout:
+		return Result{CapacitySaturation, "timeout-flag"}
+	case capText.MatchString(blob):
+		return Result{CapacitySaturation, "capacity-text"}
+	case depText.MatchString(blob):
+		return Result{DependencyUnreachable, "dependency-text"}
+	case in.Status == 401:
+		return Result{ClientCredentialIssue, "status-401"}
+	case in.Status == 502 || in.Status == 503 || in.Status == 504:
+		return Result{DependencyUnreachable, "status-5xx-upstream"}
+	case in.Status >= 500:
+		return Result{ServiceDefect, "status-5xx"}
+	case in.Status == 404 || in.Status == 410:
+		return Result{StaleReference, "status-404"}
+	case lifeText.MatchString(blob):
+		return Result{ExpectedLifecycle, "lifecycle-text"}
+	case in.Status == 499 || in.Status == 200 || in.Status == 202 || in.Status == 204 || (in.Status >= 300 && in.Status < 400):
+		return Result{ExpectedLifecycle, "benign-status"}
+	case in.Status == 0 && (in.Kind == "span" || in.Kind == "log"):
+		return Result{ExpectedLifecycle, "no-status-signal"}
+	}
+	return Result{NeedsReview, "no-rule"}
+}
+
+// Actionable reports whether alerts in this category should page a human by
+// default. Probes are a security channel rather than an ops one, and the rest
+// are expected noise or harmless.
+func (c Category) Actionable() bool {
+	switch c {
+	case ScannerProbe, StaleReference, ExpectedLifecycle:
+		return false
+	}
+	return true
+}

internal/triage/triage_test.go

@@ -0,0 +1,43 @@
+package triage
+
+import "testing"
+
+func TestClassify(t *testing.T) {
+	cases := []struct {
+		name string
+		in   Input
+		want Category
+	}{
+		{"wp 500 is a probe not an outage", Input{Status: 500, Route: "/wp"}, ScannerProbe},
+		{"php 405 is a probe", Input{Status: 405, Route: "/index.php"}, ScannerProbe},
+		{"ses.env", Input{Status: 404, Route: "/ses.env"}, ScannerProbe},
+		{"yaml config 302", Input{Status: 302, Route: "/config/aws.yml"}, ScannerProbe},
+		{"traversal", Input{Status: 404, Route: "/api/w/admins/get_log_file/../../../../proc/self/environ"}, ScannerProbe},
+		{"credentials file", Input{Status: 404, Route: "/firebase-credentials.json"}, ScannerProbe},
+		{"matrix rooms is not OOM", Input{Status: 200, Route: "/_matrix/client/r0/rooms/x/messages"}, ExpectedLifecycle},
+		{"real api 404 stays stale", Input{Status: 404, Route: "/api/config"}, StaleReference},
+		{"env 404", Input{Status: 404, Route: "/.env"}, ScannerProbe},
+		{"git config 502", Input{Status: 502, Route: "/.git/config"}, ScannerProbe},
+		{"poller 401", Input{Status: 401, Route: "/api/repos/available"}, ClientCredentialIssue},
+		{"upstream 503", Input{Status: 503, Route: "/store/regions"}, DependencyUnreachable},
+		{"real 500", Input{Status: 500, Route: "/"}, ServiceDefect},
+		{"panic in message", Input{Message: "panic: nil pointer dereference"}, ServiceDefect},
+		{"timeout flag", Input{IsTimeout: true, Message: "query"}, CapacitySaturation},
+		{"dial error", Input{Message: "*websocket.dialerror: connection refused"}, DependencyUnreachable},
+		{"missing commit", Input{Status: 404, Route: "/mats/repo/commit/{id}"}, StaleReference},
+		{"cancellation", Input{IsCancellation: true, Message: "context canceled"}, ExpectedLifecycle},
+		{"health", Input{Kind: "span", Status: 200, Route: "/readyz"}, ExpectedLifecycle},
+		{"unknown", Input{Kind: "metric", Status: 418}, NeedsReview},
+	}
+	for _, c := range cases {
+		if got := Classify(c.in); got.Category != c.want {
+			t.Errorf("%s: got %s (%s), want %s", c.name, got.Category, got.Rule, c.want)
+		}
+	}
+}
+
+func TestProbeBeatsLifecycleFlags(t *testing.T) {
+	if got := Classify(Input{Route: "/.env", IsRetry: true}); got.Category != ScannerProbe {
+		t.Fatalf("got %s", got.Category)
+	}
+}

k8s/clickhouse.yaml

@@ -143,4 +143,6 @@ spec:
   - name: http
     port: 8123
     targetPort: 8123
-  type: ClusterIP
+  # LoadBalancer for local-LAN development access only (training exports,
+  # spike-alerts). The receiver itself stays behind its own Service.
+  type: LoadBalancer

k8s/deployment.yaml

@@ -93,6 +93,16 @@ spec:
           value: "30s"
         - name: OTEL_CLICKHOUSE_CONN_MAX_LIFETIME
           value: "10m"
+        # Ingress blocker: promotes scanner IPs into ingress-nginx block-cidrs.
+        # Dry-run first: it logs what it would block and exposes it at
+        # /api/blocker/status. Set OTEL_BLOCKER_DRY_RUN=false to enforce, and
+        # put your own public IP(s) in OTEL_BLOCKER_ALLOW before doing so.
+        - name: OTEL_BLOCKER_ENABLED
+          value: "true"
+        - name: OTEL_BLOCKER_DRY_RUN
+          value: "true"
+        - name: OTEL_BLOCKER_ALLOW
+          value: ""
         - name: OTEL_STORE_WORKERS
           value: "4"
         # Optional rollout observation. The watcher is read-only and uses the
@@ -229,3 +239,32 @@ subjects:
 - kind: ServiceAccount
   name: otel-debug-receiver
   namespace: monitoring
+
+---
+# The blocker may read and update exactly one ConfigMap: the ingress-nginx
+# controller's, where `block-cidrs` lives. Nothing else in that namespace.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: otel-debug-receiver-ip-blocker
+  namespace: ingress-nginx
+rules:
+- apiGroups: [""]
+  resources: ["configmaps"]
+  resourceNames: ["ingress-nginx-controller"]
+  verbs: ["get", "update"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: otel-debug-receiver-ip-blocker
+  namespace: ingress-nginx
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: otel-debug-receiver-ip-blocker
+subjects:
+- kind: ServiceAccount
+  name: otel-debug-receiver
+  namespace: monitoring

otel-ui/src/App.tsx

@@ -71,10 +71,10 @@ function App() {
     typeof window !== "undefined" &&
     (window.location.pathname === "/auth_callback" ||
       window.location.search.includes("code="));
-  // The Issues tab is the redesigned triage overview: it owns its own filter
+  // The redesigned screens (issues, logs, RUM sessions, rollouts, metrics) own
-  // bar and cards, so the legacy anomaly banner and volume waveform above it
+  // their own filter bar and cards, so the legacy anomaly banner and volume
-  // would just duplicate that UI in a different visual language.
+  // waveform above them would just duplicate that UI in a different visual language.
-  const isTriageOverview = activeTab === "tab0" || activeTab === "tab3" || activeTab === "tab7";
+  const isTriageOverview = activeTab === "tab0" || activeTab === "tab3" || activeTab === "tab7" || activeTab === "tab9" || activeTab === "tab2";
 
   if (isCallback) {
     return <AuthCallbackView />;

otel-ui/src/components/AnomalyRadarBar.tsx

@@ -81,7 +81,7 @@ export const AnomalyRadarBar: React.FC = () => {
         <button
           type="button"
           onClick={() => setActiveTab("tab0")}
-          className="inline-flex h-7 items-center gap-1 rounded-[8px] bg-ds-critical px-2.5 text-[12px] font-semibold text-white hover:opacity-90"
+          className="inline-flex h-7 items-center gap-1 rounded-[8px] bg-ds-critical px-2.5 text-[12px] font-semibold text-ds-on-accent hover:opacity-90"
         >
           Triage issues <ArrowRight className="h-3 w-3" aria-hidden="true" />
         </button>

otel-ui/src/components/GaugeMeter.tsx

@@ -1,372 +0,0 @@
-import { useMemo } from "react";
-import { ArrowDownRight, ArrowUpRight, Minus } from "lucide-react";
-import { getServicePalette } from "../lib/otel-utils";
-
-export interface GaugeMeterProps {
-  name: string;
-  service?: string;
-  value: number;
-  min?: number;
-  max?: number;
-  avg?: number;
-  unit?: string;
-  size?: "sm" | "md" | "lg";
-  thresholds?: {
-    warn: number;
-    danger: number;
-  };
-  subtitle?: string;
-  history?: number[];
-  showTrend?: boolean;
-  isSum?: boolean;
-  aggLabel?: string;
-  soloPod?: string;
-  podCount?: number;
-  podSummaries?: Array<{ pod: string; value: number }>;
-}
-
-const formatNumber = (val: number, maxDecimals = 2) => {
-  if (Math.abs(val) >= 1_000_000) {
-    return `${(val / 1_000_000).toFixed(1)}M`;
-  }
-  if (Math.abs(val) >= 1_000) {
-    return `${(val / 1_000).toFixed(1)}k`;
-  }
-  if (Number.isInteger(val)) {
-    return val.toString();
-  }
-  return val.toFixed(maxDecimals);
-};
-
-// SVG arc calculations
-const polarToCartesian = (centerX: number, centerY: number, radius: number, angleInDegrees: number) => {
-  const angleInRadians = ((angleInDegrees - 90) * Math.PI) / 180.0;
-  return {
-    x: centerX + radius * Math.cos(angleInRadians),
-    y: centerY + radius * Math.sin(angleInRadians),
-  };
-};
-
-const describeArc = (x: number, y: number, radius: number, startAngle: number, endAngle: number) => {
-  const start = polarToCartesian(x, y, radius, endAngle);
-  const end = polarToCartesian(x, y, radius, startAngle);
-  const largeArcFlag = endAngle - startAngle <= 180 ? "0" : "1";
-  return ["M", start.x, start.y, "A", radius, radius, 0, largeArcFlag, 0, end.x, end.y].join(" ");
-};
-
-export const GaugeMeter = ({
-  name,
-  service,
-  value,
-  min: customMin,
-  max: customMax,
-  avg,
-  unit = "",
-  size = "md",
-  thresholds,
-  subtitle,
-  history = [],
-  showTrend = true,
-  isSum = false,
-  aggLabel,
-  soloPod,
-  podCount,
-  podSummaries,
-}: GaugeMeterProps) => {
-  const servicePalette = service ? getServicePalette(service) : null;
-  // Determine min/max range
-  const { min, max } = useMemo(() => {
-    let computedMin = customMin !== undefined ? customMin : 0;
-    let computedMax = customMax !== undefined ? customMax : 100;
-
-    if (history.length > 0 && customMin === undefined && customMax === undefined) {
-      const histMin = Math.min(...history);
-      const histMax = Math.max(...history);
-      if (histMax <= 1 && histMin >= 0) {
-        // Percentage ratio 0..1
-        computedMin = 0;
-        computedMax = 1;
-      } else if (histMax <= 100 && histMin >= 0) {
-        // 0..100%
-        computedMin = 0;
-        computedMax = 100;
-      } else {
-        computedMin = Math.floor(Math.min(0, histMin));
-        computedMax = Math.ceil(histMax * 1.15) || 100;
-      }
-    }
-
-    if (computedMax <= computedMin) {
-      computedMax = computedMin + 1;
-    }
-
-    return { min: computedMin, max: computedMax };
-  }, [customMin, customMax, history]);
-
-  // Normalized percent 0..100
-  const percent = useMemo(() => {
-    const clamped = Math.max(min, Math.min(value, max));
-    return ((clamped - min) / (max - min)) * 100;
-  }, [value, min, max]);
-
-  // Color determination based on thresholds
-  const warnThreshold = thresholds?.warn ?? 70;
-  const dangerThreshold = thresholds?.danger ?? 90;
-
-  const statusColor = useMemo(() => {
-    if (percent >= dangerThreshold) {
-      return {
-        text: "text-ds-critical",
-        stroke: "var(--ds-critical)",
-        gradientStart: "var(--ds-critical)",
-        gradientEnd: "var(--ds-critical)",
-        bg: "bg-ds-critical-bg",
-        border: "border-ds-critical/30",
-        glow: "color-mix(in srgb, var(--ds-critical) 35%, transparent)",
-        label: "Critical",
-      };
-    }
-    if (percent >= warnThreshold) {
-      return {
-        text: "text-ds-warn",
-        stroke: "var(--ds-warn)",
-        gradientStart: "var(--ds-warn)",
-        gradientEnd: "var(--ds-warn)",
-        bg: "bg-ds-warn-bg",
-        border: "border-ds-warn/30",
-        glow: "color-mix(in srgb, var(--ds-warn) 35%, transparent)",
-        label: "Warning",
-      };
-    }
-    return {
-      text: "text-ds-accent",
-      stroke: "var(--ds-accent)",
-      gradientStart: "var(--ds-accent)",
-      gradientEnd: "var(--ds-accent)",
-      bg: "bg-ds-accent-bg",
-      border: "border-ds-accent/30",
-      glow: "color-mix(in srgb, var(--ds-accent) 35%, transparent)",
-      label: "Nominal",
-    };
-  }, [percent, warnThreshold, dangerThreshold]);
-
-  // Trend calculation
-  const trend = useMemo(() => {
-    if (!showTrend || history.length < 2) return null;
-    const prev = history[history.length - 2];
-    const diff = value - prev;
-    if (Math.abs(diff) < 0.0001) return { direction: "flat", text: "steady", color: "text-ds-text-3" };
-    if (diff > 0) return { direction: "up", text: `+${formatNumber(diff)}`, color: "text-ds-critical" };
-    return { direction: "down", text: `${formatNumber(diff)}`, color: "text-ds-ok" };
-  }, [value, history, showTrend]);
-
-  // Dimensions
-  const dim = size === "sm" ? 120 : size === "lg" ? 220 : 160;
-  const strokeWidth = size === "sm" ? 8 : size === "lg" ? 14 : 10;
-  const radius = (dim - strokeWidth * 2) / 2;
-  const centerX = dim / 2;
-  const centerY = dim / 2 + (size === "sm" ? 4 : 8);
-
-  // 240 degree arc from -120 to +120 degrees
-  const startAngle = -120;
-  const endAngle = 120;
-  const totalAngle = endAngle - startAngle;
-  const valueAngle = startAngle + (percent / 100) * totalAngle;
-
-  const bgArc = describeArc(centerX, centerY, radius, startAngle, endAngle);
-  const valueArc = describeArc(centerX, centerY, radius, startAngle, Math.min(valueAngle, endAngle));
-
-  // Ticks at 0, 50, 100
-  const tickAngles = [startAngle, startAngle + totalAngle * 0.5, endAngle];
-
-  const gaugeId = useMemo(
-    () => `${service ? service.replace(/[^a-zA-Z0-9]/g, "_") + "_" : ""}${name.replace(/[^a-zA-Z0-9]/g, "_")}`,
-    [service, name]
-  );
-
-  return (
-    <div className="flex flex-col items-center justify-between rounded-[14px] border border-ds-line bg-ds-surface-2 p-4 transition-all hover:border-ds-line-strong">
-      <div className="flex w-full items-start justify-between gap-2 border-b border-ds-line pb-2">
-        <div className="flex flex-col min-w-0 pr-1">
-          <div className="flex items-center gap-1 flex-wrap mb-1">
-            {service && (
-              <span
-                className={`inline-flex items-center gap-1 self-start rounded px-1.5 py-0.5 text-[9px] font-mono font-bold uppercase tracking-wider border ${servicePalette?.bg} ${servicePalette?.text} ${servicePalette?.border}`}
-                title={`Owning Service: ${service}`}
-              >
-                <span className="w-1.5 h-1.5 rounded-full bg-current opacity-80" />
-                <span className="truncate max-w-[120px]">{service}</span>
-              </span>
-            )}
-            {podCount !== undefined && podCount > 1 && (
-              <span
-                className="inline-flex items-center gap-1 rounded bg-ds-accent-bg px-1.5 py-0.5 text-[9px] font-mono font-bold text-ds-accent border border-ds-accent/30"
-                title={`Sum of values across ${podCount} pods`}
-              >
-                <span className="text-[11px] font-bold leading-none text-ds-accent">∑</span>
-                <span>{podCount} pods</span>
-              </span>
-            )}
-          </div>
-          <span className="truncate text-xs font-bold uppercase tracking-wider text-ds-text-2" title={name}>
-            {name}
-          </span>
-        </div>
-        <span className={`shrink-0 rounded-full px-2 py-0.5 text-[9px] font-black uppercase tracking-wider ${statusColor.bg} ${statusColor.text} ${statusColor.border} border`}>
-          {statusColor.label}
-        </span>
-      </div>
-
-      <div className="relative my-2 flex items-center justify-center">
-        <svg width={dim} height={dim * 0.85} viewBox={`0 0 ${dim} ${dim * 0.85}`} className="overflow-visible">
-          <defs>
-            <linearGradient id={`gauge-grad-${gaugeId}`} x1="0%" y1="0%" x2="100%" y2="0%">
-              <stop offset="0%" stopColor={statusColor.gradientStart} />
-              <stop offset="100%" stopColor={statusColor.gradientEnd} />
-            </linearGradient>
-            <filter id={`gauge-glow-${gaugeId}`} x="-20%" y="-20%" width="140%" height="140%">
-              <feDropShadow dx="0" dy="0" stdDeviation="3" floodColor={statusColor.glow} />
-            </filter>
-          </defs>
-
-          {/* Background Track */}
-          <path
-            d={bgArc}
-            fill="none"
-            stroke="var(--ds-line-strong)"
-            strokeWidth={strokeWidth}
-            strokeLinecap="round"
-          />
-
-          {/* Ticks */}
-          {tickAngles.map((angle, idx) => {
-            const inner = polarToCartesian(centerX, centerY, radius - strokeWidth * 0.8, angle);
-            const outer = polarToCartesian(centerX, centerY, radius + strokeWidth * 0.8, angle);
-            return (
-              <line
-                key={idx}
-                x1={inner.x}
-                y1={inner.y}
-                x2={outer.x}
-                y2={outer.y}
-                stroke="var(--ds-line-strong)"
-                strokeWidth={1.5}
-              />
-            );
-          })}
-
-          {/* Active Value Arc */}
-          {percent > 0 && (
-            <path
-              d={valueArc}
-              fill="none"
-              stroke={`url(#gauge-grad-${gaugeId})`}
-              strokeWidth={strokeWidth}
-              strokeLinecap="round"
-              filter={`url(#gauge-glow-${gaugeId})`}
-            />
-          )}
-
-          {/* Needle / Value dot indicator */}
-          {percent > 0 && (() => {
-            const tip = polarToCartesian(centerX, centerY, radius, valueAngle);
-            return (
-              <circle
-                cx={tip.x}
-                cy={tip.y}
-                r={strokeWidth * 0.6}
-                fill="var(--ds-text)"
-                stroke={statusColor.stroke}
-                strokeWidth={2}
-                className="transition-all duration-300"
-              />
-            );
-          })()}
-        </svg>
-
-        {/* Center Readout */}
-        <div className="absolute inset-0 flex flex-col items-center justify-center pt-3 text-center pointer-events-none">
-          {soloPod ? (
-            <span className="text-[8px] font-mono uppercase font-bold text-ds-warn tracking-wider truncate max-w-[100px]" title={`Solo: ${soloPod}`}>
-              {soloPod}
-            </span>
-          ) : aggLabel ? (
-            <span className="text-[8.5px] font-mono uppercase font-bold text-ds-accent tracking-wider">
-              {aggLabel}
-            </span>
-          ) : podCount !== undefined && podCount > 1 ? (
-            <span className="text-[8.5px] font-mono uppercase font-bold text-ds-accent tracking-wider">
-              {isSum ? "Sum (∑)" : "Avg (μ)"}
-            </span>
-          ) : null}
-          <div className="flex items-baseline gap-1">
-            <span className={`font-mono font-black tracking-tight ${size === "sm" ? "text-lg" : size === "lg" ? "text-3xl" : "text-2xl"} ${statusColor.text}`}>
-              {formatNumber(value)}
-            </span>
-            {unit && (
-              <span className="text-[10px] font-bold uppercase text-ds-text-2">
-                {unit}
-              </span>
-            )}
-          </div>
-          {podCount !== undefined && podCount > 1 ? (
-            <span
-              className="text-[9px] font-mono text-ds-text-2 truncate max-w-[100px]"
-              title={isSum ? `Average: ${formatNumber(value / podCount)} per pod` : `Fleet average across ${podCount} pods`}
-            >
-              {isSum ? `~${formatNumber(value / podCount)}/pod` : `${podCount} pods avg`}
-            </span>
-          ) : subtitle ? (
-            <span className="text-[10px] text-ds-text-3 truncate max-w-[90px]">
-              {subtitle}
-            </span>
-          ) : null}
-        </div>
-      </div>
-
-      {/* Footer range indicators & trend */}
-      <div className="flex w-full items-center justify-between border-t border-ds-line pt-2 text-[10px] text-ds-text-2">
-        <span className="font-mono">
-          <span className="text-ds-text-3">min</span> {formatNumber(min)}
-        </span>
-        {trend && (
-          <span className={`inline-flex items-center gap-0.5 font-mono text-[9px] font-bold ${trend.color}`}>
-            {trend.direction === "up" && <ArrowUpRight className="h-2.5 w-2.5" />}
-            {trend.direction === "down" && <ArrowDownRight className="h-2.5 w-2.5" />}
-            {trend.direction === "flat" && <Minus className="h-2.5 w-2.5" />}
-            {trend.text}
-          </span>
-        )}
-        {avg !== undefined && (
-          <span className="font-mono text-ds-text-2">
-            <span className="text-ds-text-3">avg</span> {formatNumber(avg)}
-          </span>
-        )}
-        <span className="font-mono">
-          <span className="text-ds-text-3">max</span> {formatNumber(max)}
-        </span>
-      </div>
-
-      {/* Pod Breakdown pills if available */}
-      {podSummaries && podSummaries.length > 1 && (
-        <div className="mt-2 flex w-full flex-wrap items-center gap-1 border-t border-ds-line pt-1.5">
-          {podSummaries.slice(0, 3).map((ps) => (
-            <div
-              key={ps.pod}
-              className="flex items-center gap-1 rounded bg-ds-surface-2 px-1.5 py-0.5 text-[8.5px] font-mono text-ds-text-2"
-              title={`${ps.pod}: ${formatNumber(ps.value)} ${unit || ""}`}
-            >
-              <span className="truncate max-w-[65px] text-ds-text-2">{ps.pod}</span>
-              <span className="font-bold text-ds-accent">{formatNumber(ps.value)}</span>
-            </div>
-          ))}
-          {podSummaries.length > 3 && (
-            <span className="text-[8px] font-mono text-ds-text-3 self-center">
-              +{podSummaries.length - 3}
-            </span>
-          )}
-        </div>
-      )}
-    </div>
-  );
-};

otel-ui/src/components/RolloutsVisualizer.tsx

@@ -1,48 +1,473 @@
-import { useState } from "react"
+import { useEffect, useMemo, useState } from "react"
-import { Activity, CheckCircle2, CloudCog, RefreshCw, XCircle } from "lucide-react"
+import { Activity, CheckCircle2, CloudCog, RefreshCw, Rocket, XCircle } from "lucide-react"
 import { useRollouts } from "../hooks/useRollouts"
-import { formatRelativeTime } from "../lib/timeUtils"
+import { useObservabilityData } from "../hooks/useObservabilityData"
-import type { RolloutImpact } from "../lib/types"
+import { formatDuration, formatRelativeTime } from "../lib/timeUtils"
+import type { Rollout, RolloutImpact } from "../lib/types"
+import { FilterBar, type FilterChip } from "./FilterBar"
+import { MasterDetail } from "./MasterDetail"
+import { Segmented } from "./common/Segmented"
+import { DetailField, SectionLabel } from "./common/DetailField"
+import { useSubRoute } from "../lib/routes"
 
-const phaseStyle: Record<string, { className: string; Icon: typeof Activity }> = {
+type PhaseFilter = "progressing" | "completed" | "failed"
-  completed: { className: "text-ds-ok bg-ds-ok/15 border-ds-ok/20", Icon: CheckCircle2 },
+type SortMode = "recent" | "active"
-  failed: { className: "text-ds-critical bg-ds-critical-bg border-ds-critical/20", Icon: XCircle },
+
-  progressing: { className: "text-ds-warn bg-ds-warn-bg border-ds-warn/20", Icon: Activity },
+const phaseMeta: Record<string, { Icon: typeof Activity; pill: string; dot: string; bar: string }> = {
+  completed: { Icon: CheckCircle2, pill: "text-ds-ok bg-ds-ok/15 border-ds-ok/20", dot: "bg-ds-ok", bar: "bg-ds-ok" },
+  failed: { Icon: XCircle, pill: "text-ds-critical bg-ds-critical-bg border-ds-critical/20", dot: "bg-ds-critical", bar: "bg-ds-critical" },
+  progressing: { Icon: Activity, pill: "text-ds-warn bg-ds-warn-bg border-ds-warn/20", dot: "bg-ds-warn", bar: "bg-ds-warn" },
 }
 
+const phaseOf = (rollout: Rollout) => phaseMeta[rollout.phase] ?? phaseMeta.progressing
+
+const PhasePill = ({ rollout }: { rollout: Rollout }) => {
+  const { Icon, pill } = phaseOf(rollout)
+  return (
+    <span
+      className={`inline-flex shrink-0 items-center gap-1 whitespace-nowrap rounded border px-2 py-1 text-[10px] font-bold uppercase ${pill}`}
+    >
+      <Icon className="h-3 w-3" aria-hidden="true" /> {rollout.phase}
+    </span>
+  )
+}
+
+/** Real replica progress: available of desired, tinted by the rollout's actual phase. */
+const ReplicaBar = ({ rollout }: { rollout: Rollout }) => {
+  const { bar } = phaseOf(rollout)
+  const ratio = rollout.desiredReplicas > 0 ? Math.min(rollout.availableReplicas / rollout.desiredReplicas, 1) : 0
+  return (
+    <span className="flex flex-1 items-center gap-2">
+      <span className="h-1 flex-1 overflow-hidden rounded-full bg-ds-fill">
+        <span className={`block h-full rounded-full ${bar}`} style={{ width: `${Math.round(ratio * 100)}%` }} />
+      </span>
+      <span className="shrink-0 font-[family-name:var(--font-data)] text-[11px] text-ds-text-3">
+        {rollout.availableReplicas}/{rollout.desiredReplicas}
+      </span>
+    </span>
+  )
+}
+
+const RolloutListItem = ({ rollout }: { rollout: Rollout }) => {
+  const { dot } = phaseOf(rollout)
+  return (
+    <div className="flex flex-col gap-1.5">
+      <div className="flex items-center gap-2">
+        <span className={`h-2 w-2 shrink-0 rounded-full ${dot}`} aria-hidden="true" />
+        <span className="truncate font-[family-name:var(--font-data)] text-[12.5px] font-medium text-ds-text">
+          {rollout.workload}
+        </span>
+        <span className="shrink-0 font-[family-name:var(--font-data)] text-[11px] text-ds-text-3">
+          {rollout.namespace}
+        </span>
+        <span className="flex-grow" />
+        <span className="shrink-0 font-[family-name:var(--font-data)] text-[11px] text-ds-text-3">
+          {formatRelativeTime(rollout.startedAt / 1e6)}
+        </span>
+      </div>
+      <div className="flex items-center gap-2">
+        <span className="shrink-0 rounded-[6px] border border-ds-line-strong px-1.5 py-0.5 font-[family-name:var(--font-data)] text-[10.5px] text-ds-text-2">
+          rev {rollout.revision || "?"}
+        </span>
+        <ReplicaBar rollout={rollout} />
+      </div>
+    </div>
+  )
+}
+
+const formatDateTime = (ms: number) =>
+  new Date(ms).toLocaleString([], { month: "short", day: "numeric", hour: "2-digit", minute: "2-digit" })
+
+/**
+ * Error rate around the rollout, from Prometheus. A nonzero 5xx rate is drawn
+ * in the critical tone; the hairline marks when this revision started rolling.
+ */
+const ImpactChart = ({ impact, rollout }: { impact: RolloutImpact; rollout: Rollout }) => {
+  const samples = useMemo(
+    () =>
+      impact.points
+        .map((point) => ({ t: point.timestamp, v: Number.parseFloat(point.value) }))
+        .filter((sample) => Number.isFinite(sample.v)),
+    [impact.points],
+  )
+  if (samples.length === 0) {
+    return <p className="text-[12px] text-ds-text-3">No matching Prometheus series for this workload.</p>
+  }
+  const max = Math.max(...samples.map((s) => s.v))
+  const first = samples[0].t
+  const span = Math.max(samples[samples.length - 1].t - first, 1)
+  const startPct = ((rollout.startedAt / 1e9 - first) / span) * 100
+  const formatRate = (v: number) => (v >= 100 ? String(Math.round(v)) : String(Number(v.toFixed(3))))
+  return (
+    <div className="flex flex-col gap-1.5">
+      <div className="flex items-baseline justify-between gap-2">
+        <span className="font-[family-name:var(--font-data)] text-[10.5px] text-ds-text-3">
+          5xx/s · peak {formatRate(max)}
+        </span>
+        <span className="font-[family-name:var(--font-data)] text-[10.5px] text-ds-text-3">
+          {formatDateTime(first * 1000)} → {formatDateTime(samples[samples.length - 1].t * 1000)}
+        </span>
+      </div>
+      <div className="relative flex h-16 items-end gap-px rounded-[8px] bg-ds-fill/50 px-1 py-1">
+        {startPct > 0 && startPct < 100 && (
+          <span
+            className="absolute inset-y-0 w-px bg-ds-text-3"
+            style={{ left: `${startPct}%` }}
+            title={`Rollout started ${formatDateTime(rollout.startedAt / 1e6)}`}
+          />
+        )}
+        {samples.map((sample) => (
+          <span
+            key={sample.t}
+            className={`min-w-[1px] flex-1 rounded-t-[1px] ${sample.v > 0 ? "bg-ds-critical/70" : "bg-ds-line-strong/60"}`}
+            style={{ height: max > 0 ? `${Math.max((sample.v / max) * 100, 2)}%` : "2%" }}
+            title={`${formatDateTime(sample.t * 1000)} — ${formatRate(sample.v)} 5xx/s`}
+          />
+        ))}
+      </div>
+    </div>
+  )
+}
+
+const RolloutDetail = ({
+  rollout,
+  impactState,
+  impactLoading,
+}: {
+  rollout: Rollout
+  impactState?: RolloutImpact
+  impactLoading: boolean
+}) => {
+  const short = rollout.generation > 0 ? ` · generation ${rollout.generation}` : ""
+  const replicas = [
+    { label: "Desired", value: rollout.desiredReplicas },
+    { label: "Updated", value: rollout.updatedReplicas },
+    { label: "Available", value: rollout.availableReplicas, warn: rollout.availableReplicas < rollout.desiredReplicas },
+    { label: "Ready", value: rollout.readyReplicas, warn: rollout.readyReplicas < rollout.desiredReplicas },
+  ]
+  return (
+    <div className="flex flex-col gap-4 rounded-[14px] border border-ds-line bg-ds-surface p-4">
+      <div className="flex items-center gap-3">
+        <PhasePill rollout={rollout} />
+        <div className="flex min-w-0 flex-1 flex-col">
+          <span className="truncate font-[family-name:var(--font-data)] text-[13px] font-semibold text-ds-text">
+            {rollout.namespace}/{rollout.workload}
+          </span>
+          <span className="font-[family-name:var(--font-data)] text-[11px] text-ds-text-3">
+            revision {rollout.revision || "unknown"}
+            {short}
+          </span>
+        </div>
+      </div>
+
+      <div className="grid grid-cols-2 gap-x-4 gap-y-3 border-y border-ds-fill py-3 sm:grid-cols-3">
+        <DetailField label="Service" value={rollout.service} />
+        <DetailField label="Release" value={rollout.release} />
+        <DetailField label="Last state change" value={formatRelativeTime(rollout.lastSeen / 1e6)} />
+        <DetailField label="Image" value={rollout.image} />
+        <DetailField label="Reason" value={rollout.reason} />
+        <DetailField label="Message" value={rollout.message} />
+      </div>
+
+      <div className="grid grid-cols-4 gap-2">
+        {replicas.map((stat) => (
+          <div key={stat.label} className="flex flex-col gap-1">
+            <span className="text-[10.5px] font-bold uppercase tracking-[0.09em] text-ds-text-3">{stat.label}</span>
+            <span
+              className={`font-[family-name:var(--font-display)] text-[17px] font-bold ${stat.warn ? "text-ds-warn" : "text-ds-text"}`}
+            >
+              {stat.value}
+            </span>
+          </div>
+        ))}
+      </div>
+
+      <div className="flex flex-col gap-2">
+        <SectionLabel>Timeline</SectionLabel>
+        <div className="flex items-baseline justify-between gap-2">
+          <span className="text-[12.5px] text-ds-text">Started</span>
+          <span className="text-right font-[family-name:var(--font-data)] text-[11.5px] text-ds-text-2">
+            {formatDateTime(rollout.startedAt / 1e6)} · {formatRelativeTime(rollout.startedAt / 1e6)}
+          </span>
+        </div>
+        <div className="flex items-baseline justify-between gap-2">
+          <span className="text-[12.5px] text-ds-text">Completed</span>
+          {rollout.completedAt ? (
+            <span className="text-right font-[family-name:var(--font-data)] text-[11.5px] text-ds-text-2">
+              {formatDateTime(rollout.completedAt / 1e6)} · took{" "}
+              {formatDuration(Math.max(rollout.completedAt - rollout.startedAt, 0))}
+            </span>
+          ) : (
+            <span className="font-[family-name:var(--font-data)] text-[11.5px] text-ds-text-3">not yet</span>
+          )}
+        </div>
+      </div>
+
+      <div className="flex flex-col gap-2">
+        <div className="flex items-center gap-2">
+          <SectionLabel>Prometheus error rate</SectionLabel>
+          {impactLoading && <RefreshCw className="h-3 w-3 animate-spin text-ds-text-3" aria-hidden="true" />}
+        </div>
+        {!impactState ? (
+          <p className="text-[12px] text-ds-text-3">Loading impact data…</p>
+        ) : !impactState.configured ? (
+          <p className="text-[12px] text-ds-text-3">Prometheus is not configured.</p>
+        ) : (
+          <ImpactChart impact={impactState} rollout={rollout} />
+        )}
+      </div>
+    </div>
+  )
+}
+
+const Tile = ({
+  label,
+  value,
+  detail,
+  tone,
+}: {
+  label: string
+  value: string
+  detail: string
+  tone?: "critical" | "warn" | "ok"
+}) => (
+  <div className="flex flex-col gap-1 rounded-[12px] border border-ds-line bg-ds-surface px-4 py-3">
+    <span className="text-[10.5px] font-bold uppercase tracking-[0.09em] text-ds-text-3">{label}</span>
+    <span
+      className={`font-[family-name:var(--font-display)] text-[22px] font-bold ${
+        tone === "critical" ? "text-ds-critical" : tone === "warn" ? "text-ds-warn" : tone === "ok" ? "text-ds-ok" : "text-ds-text"
+      }`}
+    >
+      {value}
+    </span>
+    <span className="text-[12px] text-ds-text-3">{detail}</span>
+  </div>
+)
+
+const phaseRank: Record<string, number> = { progressing: 0, failed: 1, completed: 2 }
+
 export function RolloutsVisualizer() {
   const { rollouts, loading, error, refresh, impact } = useRollouts()
-  const [selected, setSelected] = useState<string | null>(null)
+  const { searchQuery, setSearchQuery } = useObservabilityData()
-  const [impactData, setImpactData] = useState<Record<string, RolloutImpact>>({})
+  const { segments, go } = useSubRoute()
+  const [phaseFilter, setPhaseFilter] = useState<PhaseFilter | null>(null)
+  const [namespaceFilter, setNamespaceFilter] = useState<string | null>(null)
+  const [sortMode, setSortMode] = useState<SortMode>("recent")
+  const [impactById, setImpactById] = useState<Record<string, RolloutImpact>>({})
+  const [impactPending, setImpactPending] = useState<Record<string, boolean>>({})
+  const [impactFailed, setImpactFailed] = useState<Record<string, boolean>>({})
   const [impactError, setImpactError] = useState<string | null>(null)
 
-  const select = async (id: string) => {
+  const namespaces = useMemo(() => [...new Set(rollouts.map((r) => r.namespace))].sort(), [rollouts])
-    setSelected(selected === id ? null : id)
+
-    if (impactData[id]) return
+  const filtered = useMemo(() => {
-    try {
+    return rollouts.filter((r) => {
-      setImpactError(null)
+      if (phaseFilter && r.phase !== phaseFilter) return false
-      const result = await impact(id)
+      if (namespaceFilter && r.namespace !== namespaceFilter) return false
-      setImpactData((previous) => ({ ...previous, [id]: result }))
+      if (searchQuery) {
-    }
+        const haystack = [r.workload, r.namespace, r.service, r.release, r.image, r.revision]
-    catch (cause) { setImpactError(cause instanceof Error ? cause.message : String(cause)) }
+          .filter(Boolean)
+          .join(" ")
+          .toLowerCase()
+        if (!haystack.includes(searchQuery.toLowerCase())) return false
+      }
+      return true
+    })
+  }, [rollouts, phaseFilter, namespaceFilter, searchQuery])
+
+  const sorted = useMemo(() => {
+    const byStart = (a: Rollout, b: Rollout) => b.startedAt - a.startedAt
+    if (sortMode === "active") return [...filtered].sort((a, b) => (phaseRank[a.phase] ?? 3) - (phaseRank[b.phase] ?? 3) || byStart(a, b))
+    return [...filtered].sort(byStart)
+  }, [filtered, sortMode])
+
+  const selected = sorted.find((r) => r.id === segments[0]) ?? sorted[0] ?? null
+
+  // Fetch Prometheus impact for whatever the user selects; cached per rollout,
+  // failures recorded so a dead Prometheus isn't re-polled on every render.
+  const selectedId = selected?.id ?? null
+  useEffect(() => {
+    if (!selectedId || impactById[selectedId] || impactFailed[selectedId] || impactPending[selectedId]) return
+    setImpactPending((prev) => ({ ...prev, [selectedId]: true }))
+    impact(selectedId)
+      .then((result) => {
+        setImpactById((prev) => ({ ...prev, [selectedId]: result }))
+        setImpactError(null)
+      })
+      .catch((cause) => {
+        setImpactFailed((prev) => ({ ...prev, [selectedId]: true }))
+        setImpactError(cause instanceof Error ? cause.message : String(cause))
+      })
+      .finally(() => setImpactPending((prev) => ({ ...prev, [selectedId]: false })))
+  }, [selectedId, impactById, impactFailed, impactPending, impact])
+
+  const stats = useMemo(() => {
+    const dayAgoMs = Date.now() - 24 * 3600 * 1000
+    const progressing = rollouts.filter((r) => r.phase === "progressing").length
+    const failed = rollouts.filter((r) => r.phase === "failed").length
+    const recent = rollouts.filter((r) => r.startedAt / 1e6 >= dayAgoMs).length
+    const workloads = new Set(rollouts.map((r) => `${r.namespace}/${r.workload}`)).size
+    return { progressing, failed, recent, workloads }
+  }, [rollouts])
+
+  const activeFilterCount = (phaseFilter ? 1 : 0) + (namespaceFilter ? 1 : 0) + (searchQuery ? 1 : 0)
+
+  const chips: FilterChip[] = [
+    ...(phaseFilter
+      ? [
+          {
+            id: "phase",
+            filterKey: "phase",
+            value: phaseFilter,
+            tone: phaseFilter === "failed" ? ("critical" as const) : phaseFilter === "progressing" ? ("warn" as const) : ("ok" as const),
+            onRemove: () => setPhaseFilter(null),
+          },
+        ]
+      : []),
+    ...(namespaceFilter
+      ? [{ id: "namespace", filterKey: "namespace", value: namespaceFilter, tone: "neutral" as const, onRemove: () => setNamespaceFilter(null) }]
+      : []),
+    ...(searchQuery ? [{ id: "query", filterKey: "query", value: searchQuery, tone: "neutral" as const, onRemove: () => setSearchQuery("") }] : []),
+  ]
+
+  const addFilterOptions = [
+    ...(["progressing", "failed", "completed"] as PhaseFilter[])
+      .filter((phase) => phase !== phaseFilter)
+      .map((phase) => ({ value: `phase:${phase}`, label: `phase=${phase}` })),
+    ...namespaces.filter((ns) => ns !== namespaceFilter).map((ns) => ({ value: `ns:${ns}`, label: `namespace=${ns}` })),
+  ]
+
+  const onAddFilter = (value: string) => {
+    if (value.startsWith("phase:")) setPhaseFilter(value.slice(6) as PhaseFilter)
+    else if (value.startsWith("ns:")) setNamespaceFilter(value.slice(3))
   }
 
-  if (loading && rollouts.length === 0) return <div className="py-16 text-center text-sm text-ds-text-3">Loading cluster rollouts…</div>
+  const clearAll = () => {
-  return <div className="space-y-4">
+    setPhaseFilter(null)
-    <div className="flex flex-wrap items-center gap-3 border-b border-ds-line pb-3">
+    setNamespaceFilter(null)
-      <div><div className="flex items-center gap-2 text-[10px] font-bold uppercase tracking-[0.2em] text-ds-text-3"><CloudCog className="h-3 w-3 text-ds-accent" /> Cluster Rollouts</div><div className="mt-1 text-xs text-ds-text-3">Read-only Kubernetes deployment state with optional Prometheus impact data.</div></div>
+    setSearchQuery("")
-      <button onClick={() => void refresh()} className="ml-auto inline-flex items-center gap-2 rounded-lg border border-ds-line px-3 py-2 text-[10px] font-bold uppercase tracking-wider text-ds-text-2 hover:text-ds-text"><RefreshCw className="h-3 w-3" /> Refresh</button>
+  }
-    </div>
+
-    {error && <div className="rounded-lg border border-ds-warn/20 bg-ds-warn-bg p-3 text-xs text-ds-warn">Rollout watcher unavailable: {String(error)}</div>}
+  const matchText = `Matching ${filtered.length} of ${rollouts.length} rollouts · ${activeFilterCount} filter${activeFilterCount === 1 ? "" : "s"}`
-    {impactError && <div className="rounded-lg border border-ds-critical/20 bg-ds-critical-bg p-3 text-xs text-ds-critical">Impact query failed: {impactError}</div>}
+
-    {rollouts.length === 0 ? <div className="py-16 text-center text-sm text-ds-text-3">No rollout data. Enable the Kubernetes watcher and grant its read-only RBAC permissions.</div> : <div className="space-y-2">{rollouts.map((rollout) => {
+  if (error) {
-      const status = phaseStyle[rollout.phase] ?? phaseStyle.progressing
+    return (
-      const Icon = status.Icon
+      <div className="flex flex-col gap-5">
-      const data = impactData[rollout.id]
+        <FilterBar chips={[]} matchText={matchText} />
-      return <div key={rollout.id} className="rounded-xl border border-ds-line bg-ds-surface-2 p-4">
+        <div className="flex flex-col items-center justify-center gap-2 rounded-[14px] border border-dashed border-ds-line py-20 text-center">
-        <button onClick={() => void select(rollout.id)} className="w-full text-left"><div className="flex flex-wrap items-center gap-3"><span className={`inline-flex items-center gap-1 rounded border px-2 py-1 text-[10px] font-bold uppercase ${status.className}`}><Icon className="h-3 w-3" /> {rollout.phase}</span><span className="font-mono text-xs text-ds-text">{rollout.namespace}/{rollout.workload}</span><span className="text-xs text-ds-text-2">revision {rollout.revision || "unknown"}</span><span className="ml-auto text-[10px] text-ds-text-3">{formatRelativeTime(rollout.lastSeen / 1e6)}</span></div><div className="mt-3 grid gap-2 text-[11px] text-ds-text-2 sm:grid-cols-2 lg:grid-cols-4"><span>release: <b className="text-ds-text">{rollout.release || "unlabeled"}</b></span><span>image: <b className="font-mono text-ds-text">{rollout.image || "unknown"}</b></span><span>replicas: <b className="text-ds-text">{rollout.availableReplicas}/{rollout.desiredReplicas} ready</b></span><span>reason: <b className="text-ds-text">{rollout.reason || "unknown"}</b></span></div></button>
+          <CloudCog className="h-8 w-8 text-ds-text-3" aria-hidden="true" />
-        {selected === rollout.id && <div className="mt-4 border-t border-ds-line pt-3 text-xs text-ds-text-2"><div>Started {formatRelativeTime(rollout.startedAt / 1e6)}{rollout.completedAt ? ` · completed ${formatRelativeTime(rollout.completedAt / 1e6)}` : ""}</div><div className="mt-2 text-ds-text-3">{rollout.message || "No rollout message reported."}</div>{data && <div className="mt-3 rounded-lg bg-ds-surface-2 p-3"><div className="flex items-center gap-2 text-[10px] font-bold uppercase tracking-widest text-ds-text-3"><Activity className="h-3 w-3" /> Prometheus error-rate points</div>{!data.configured ? <div className="mt-2 text-ds-text-3">Prometheus is not configured.</div> : data.points.length === 0 ? <div className="mt-2 text-ds-text-3">No matching Prometheus series.</div> : <div className="mt-2 flex flex-wrap gap-2">{data.points.slice(-12).map((point) => <span key={`${point.timestamp}-${point.value}`} className="rounded bg-ds-accent-bg px-2 py-1 font-mono text-[10px] text-ds-accent">{new Date(point.timestamp * 1000).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" })}: {point.value}</span>)}</div>}</div>}</div>}
+          <p className="text-[14px] font-medium text-ds-text">Rollout watcher unavailable</p>
+          <p className="max-w-sm text-[12.5px] text-ds-text-3">{error.message}</p>
+        </div>
       </div>
-    })}</div>}
+    )
-  </div>
+  }
+
+  if (loading && rollouts.length === 0) {
+    return <div className="py-16 text-center text-sm text-ds-text-3">Loading cluster rollouts…</div>
+  }
+
+  if (rollouts.length === 0) {
+    return (
+      <div className="flex flex-col gap-5">
+        <FilterBar chips={[]} matchText={matchText} />
+        <div className="flex flex-col items-center justify-center gap-2 rounded-[14px] border border-dashed border-ds-line py-20 text-center">
+          <CloudCog className="h-8 w-8 text-ds-text-3" aria-hidden="true" />
+          <p className="text-[14px] font-medium text-ds-text">No rollout data</p>
+          <p className="max-w-sm text-[12.5px] text-ds-text-3">
+            Enable the Kubernetes watcher and grant its read-only RBAC permissions to see deployment rollouts here.
+          </p>
+        </div>
+      </div>
+    )
+  }
+
+  return (
+    <div className="flex flex-col gap-5">
+      <FilterBar
+        chips={chips}
+        addFilterOptions={addFilterOptions.length > 0 ? addFilterOptions : undefined}
+        onAddFilter={addFilterOptions.length > 0 ? onAddFilter : undefined}
+        matchText={matchText}
+        onClearAll={activeFilterCount > 0 ? clearAll : undefined}
+      />
+
+      <div className="grid grid-cols-2 gap-3 lg:grid-cols-4">
+        <Tile label="Tracked rollouts" value={String(rollouts.length)} detail={`across ${stats.workloads} workloads`} />
+        <Tile
+          label="In progress"
+          value={String(stats.progressing)}
+          detail={stats.progressing > 0 ? "rolling out right now" : "cluster is settled"}
+          tone={stats.progressing > 0 ? "warn" : "ok"}
+        />
+        <Tile
+          label="Failed"
+          value={String(stats.failed)}
+          detail={stats.failed > 0 ? "deadline exceeded or unavailable" : "no failing rollouts"}
+          tone={stats.failed > 0 ? "critical" : "ok"}
+        />
+        <Tile label="Last 24 hours" value={String(stats.recent)} detail="rollouts started in the last day" />
+      </div>
+
+      {sorted.length === 0 ? (
+        <div className="flex flex-col items-center justify-center gap-2 rounded-[14px] border border-ds-line bg-ds-surface py-16 text-center">
+          <Rocket className="h-6 w-6 text-ds-text-3" aria-hidden="true" />
+          <p className="text-[13px] text-ds-text-3">No rollouts match your filters.</p>
+        </div>
+      ) : (
+        <MasterDetail
+          items={sorted}
+          getKey={(r) => r.id}
+          selectedKey={selected?.id ?? null}
+          onSelect={(id) => go([id])}
+          label="Cluster rollouts"
+          storageKey="rollouts"
+          listSize={{ default: 400, min: 300, max: 620 }}
+          listHeader={
+            <div className="flex items-center justify-between gap-2">
+              <h2 className="text-[11px] font-bold uppercase tracking-[0.1em] text-ds-text-3">
+                Rollouts · {sortMode === "active" ? "active first" : "newest first"}
+              </h2>
+              <div className="flex items-center gap-2">
+                <Segmented
+                  label="Sort rollouts"
+                  value={sortMode}
+                  onChange={setSortMode}
+                  options={[
+                    { value: "recent", label: "Recent" },
+                    { value: "active", label: "Active" },
+                  ]}
+                />
+                <button
+                  type="button"
+                  onClick={() => void refresh()}
+                  aria-label="Refresh rollouts"
+                  title="Refresh rollouts"
+                  className="inline-flex h-7 w-7 items-center justify-center rounded-[7px] text-ds-text-3 hover:bg-ds-fill hover:text-ds-text-2"
+                >
+                  <RefreshCw className="h-3.5 w-3.5" aria-hidden="true" />
+                </button>
+              </div>
+            </div>
+          }
+          renderItem={(rollout) => <RolloutListItem rollout={rollout} />}
+          renderDetail={(rollout) => (
+            <RolloutDetail
+              rollout={rollout}
+              impactState={impactById[rollout.id]}
+              impactLoading={Boolean(impactPending[rollout.id])}
+            />
+          )}
+        />
+      )}
+
+      {impactError && (
+        <p className="rounded-lg border border-ds-warn/20 bg-ds-warn-bg p-3 text-xs text-ds-warn">
+          Impact query failed: {impactError}
+        </p>
+      )}
+    </div>
+  )
 }
+
+export default RolloutsVisualizer

otel-ui/src/hooks/useRollouts.ts

@@ -1,10 +1,29 @@
 import useSWR from "swr"
-import { authSWRFetcher, authFetch, tokens } from "../lib/auth"
+import { authFetch, tokens } from "../lib/auth"
 import type { Rollout, RolloutImpact } from "../lib/types"
 
+/**
+ * Like authSWRFetcher but keeps the API's JSON error body ("kubernetes
+ * integration is not configured") instead of reducing it to "HTTP 501".
+ */
+const rolloutFetcher = async (key: string): Promise<Rollout[]> => {
+  const response = await authFetch(key)
+  if (!response.ok) {
+    let message = `HTTP ${response.status}`
+    try {
+      const body = (await response.json()) as { error?: string }
+      if (body.error) message = body.error
+    } catch {
+      // not JSON - keep the status text
+    }
+    throw new Error(message)
+  }
+  return (await response.json()) as Rollout[]
+}
+
 export function useRollouts() {
   const key = tokens() ? "/api/rollouts" : null
-  const state = useSWR<Rollout[]>(key, authSWRFetcher, { refreshInterval: 30_000, revalidateOnFocus: false, shouldRetryOnError: false })
+  const state = useSWR<Rollout[]>(key, rolloutFetcher, { refreshInterval: 30_000, revalidateOnFocus: false, shouldRetryOnError: false })
   const impact = async (id: string) => {
     const response = await authFetch(`/api/rollouts/${encodeURIComponent(id)}/impact`)
     if (!response.ok) throw new Error(`HTTP ${response.status}`)