all the changes

cmd/bad-actors/main.go

@@ -0,0 +1,184 @@
+// bad-actors ranks client IPs by scanner behaviour in raw nginx/ingress spans
+// and reports which would be safe to block. It only reads ClickHouse; nothing
+// is blocked. Review its output before feeding any blocker.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+
+	"git.tornberg.me/go-otel/internal/honeypot"
+	"git.tornberg.me/go-otel/internal/triage"
+)
+
+type otlp struct {
+	ResourceSpans []struct {
+		ScopeSpans []struct {
+			Spans []struct {
+				Attributes []struct {
+					Key   string                     `json:"key"`
+					Value map[string]json.RawMessage `json:"value"`
+				} `json:"attributes"`
+			} `json:"spans"`
+		} `json:"scopeSpans"`
+	} `json:"resourceSpans"`
+}
+
+type actor struct {
+	IP        string
+	Total     int
+	Probes    int
+	Paths     map[string]struct{}
+	Statuses  map[string]int
+	UserAgent string
+}
+
+func main() {
+	addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port")
+	db := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "database")
+	user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "user")
+	pass := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "password")
+	since := flag.String("since", "", "inclusive UTC RFC3339")
+	until := flag.String("until", "", "exclusive UTC RFC3339")
+	service := flag.String("service", "nginx", "ingress service name")
+	minProbes := flag.Int("min-probes", 10, "minimum probe requests to be blockable")
+	minPaths := flag.Int("min-paths", 3, "minimum distinct probe paths")
+	minShare := flag.Float64("min-share", 0.8, "minimum share of the IP's requests that are probes")
+	broadPaths := flag.Int("broad-paths", 50, "distinct probe paths that mark a scanner regardless of share")
+	broadProbes := flag.Int("broad-probes", 100, "probe requests required with -broad-paths")
+	top := flag.Int("top", 30, "rows to print")
+	flag.Parse()
+	start, err1 := time.Parse(time.RFC3339, *since)
+	end, err2 := time.Parse(time.RFC3339, *until)
+	if err1 != nil || err2 != nil || !start.Before(end) {
+		log.Fatal("-since/-until must be valid RFC3339 and ordered")
+	}
+	conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{*addr}, Auth: clickhouse.Auth{Database: *db, Username: *user, Password: *pass}})
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer conn.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	defer cancel()
+	rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT data FROM %s.entries WHERE type='traces' AND service=? AND timestamp >= toDateTime64('%s',9,'UTC') AND timestamp < toDateTime64('%s',9,'UTC')`,
+		*db, start.UTC().Format("2006-01-02 15:04:05.000000000"), end.UTC().Format("2006-01-02 15:04:05.000000000")), *service)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+
+	actors := map[string]*actor{}
+	spans := 0
+	for rows.Next() {
+		var data string
+		if err := rows.Scan(&data); err != nil {
+			log.Fatal(err)
+		}
+		var d otlp
+		if json.Unmarshal([]byte(data), &d) != nil {
+			continue
+		}
+		for _, rs := range d.ResourceSpans {
+			for _, ss := range rs.ScopeSpans {
+				for _, sp := range ss.Spans {
+					at := map[string]string{}
+					for _, a := range sp.Attributes {
+						for _, raw := range a.Value {
+							var v any
+							_ = json.Unmarshal(raw, &v)
+							at[a.Key] = fmt.Sprint(v)
+						}
+					}
+					ip := strings.TrimSpace(strings.SplitN(first(at, "http.request.header.x-real-ip", "http.request.header.x-forwarded-for", "client.address", "net.peer.ip"), ",", 2)[0])
+					if ip == "" {
+						continue
+					}
+					spans++
+					path := strings.SplitN(first(at, "http.target", "url.path", "http.route"), "?", 2)[0]
+					a := actors[ip]
+					if a == nil {
+						a = &actor{IP: ip, Paths: map[string]struct{}{}, Statuses: map[string]int{}}
+						actors[ip] = a
+					}
+					a.Total++
+					if a.UserAgent == "" {
+						a.UserAgent = first(at, "http.user_agent")
+					}
+					if triage.IsProbePath(path) {
+						a.Probes++
+						a.Paths[path] = struct{}{}
+						a.Statuses[first(at, "http.status_code")]++
+					}
+				}
+			}
+		}
+	}
+	if err := rows.Err(); err != nil {
+		log.Fatal(err)
+	}
+
+	var list []*actor
+	totalProbes, blockableProbes, blockable, skippedInternal, skippedInternalProbes := 0, 0, 0, 0, 0
+	for _, a := range actors {
+		totalProbes += a.Probes
+		if a.Probes == 0 {
+			continue
+		}
+		if internal(a.IP) {
+			skippedInternal++
+			skippedInternalProbes += a.Probes
+			a.UserAgent = "[INTERNAL] " + a.UserAgent
+		} else if (a.Probes >= *minProbes && len(a.Paths) >= *minPaths && float64(a.Probes)/float64(a.Total) >= *minShare) ||
+			(a.Probes >= *broadProbes && len(a.Paths) >= *broadPaths) {
+			blockable++
+			blockableProbes += a.Probes
+			a.UserAgent = "[BLOCK] " + a.UserAgent
+		}
+		list = append(list, a)
+	}
+	sort.Slice(list, func(i, j int) bool { return list[i].Probes > list[j].Probes })
+	fmt.Printf("%d spans with an IP, %d distinct IPs, %d probe requests\n", spans, len(actors), totalProbes)
+	fmt.Printf("blockable (public; >=%d probes/>=%d paths/>=%.0f%% probes, or >=%d probes over >=%d paths): %d IPs covering %d probes (%.0f%%)\n", *minProbes, *minPaths, *minShare*100, *broadProbes, *broadPaths, blockable, blockableProbes, pct(blockableProbes, totalProbes))
+	fmt.Printf("internal/loopback (never blocked): %d IPs, %d probes (%.0f%%) - real client is hidden behind a proxy hop\n\n", skippedInternal, skippedInternalProbes, pct(skippedInternalProbes, totalProbes))
+	fmt.Printf("%-40s %6s %6s %6s  %s\n", "ip", "probes", "total", "paths", "verdict / user-agent / statuses")
+	for i, a := range list {
+		if i >= *top {
+			break
+		}
+		fmt.Printf("%-40s %6d %6d %6d  %.60s %v\n", a.IP, a.Probes, a.Total, len(a.Paths), a.UserAgent, a.Statuses)
+	}
+}
+
+// internal reports addresses that must never be blocked; it is the same
+// definition the tracker and blocker use.
+func internal(s string) bool { return honeypot.Untrackable(s) }
+
+func first(m map[string]string, keys ...string) string {
+	for _, k := range keys {
+		if v := m[k]; v != "" && v != "<nil>" {
+			return v
+		}
+	}
+	return ""
+}
+func pct(a, b int) float64 {
+	if b == 0 {
+		return 0
+	}
+	return 100 * float64(a) / float64(b)
+}
+func envOr(k, f string) string {
+	if v := os.Getenv(k); v != "" {
+		return v
+	}
+	return f
+}

cmd/needle-sidecar/eval/golden.jsonl

@@ -1,40 +1,40 @@
-{"id": "ng-404-root", "src": "clickhouse", "expected": "log_404", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /"}
-{"id": "ng-401-repos", "src": "clickhouse", "expected": "auth_failure", "note": "polled ~700x/h by an unauthenticated client", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/repos/available"}
-{"id": "ng-401-ctrl", "src": "clickhouse", "expected": "auth_failure", "note": "", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/controller/instances"}
-{"id": "orch-401-repos", "src": "clickhouse", "expected": "auth_failure", "note": "same request as ng-401-repos, second span", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/repos/available"}
-{"id": "orch-401-chat", "src": "clickhouse", "expected": "auth_failure", "note": "", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/sessions/{id}/collab/chat"}
-{"id": "ng-401-manifest", "src": "clickhouse", "expected": "auth_failure", "note": "registry pull without token", "text": "Service: nginx | Kind: span | Status: 401 | Route: /v2/eventsourced-ai-worker/manifests/latest"}
-{"id": "ng-503-store", "src": "clickhouse", "expected": "escalate_outage", "note": "upstream 503, real user-facing route", "text": "Service: nginx | Kind: span | Status: 503 | Route: /store/regions"}
-{"id": "ng-500-root", "src": "clickhouse", "expected": "escalate_outage", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /"}
-{"id": "ng-500-wp", "src": "clickhouse", "expected": "security_alert", "note": "WordPress path that does not exist here", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp"}
-{"id": "ng-500-blog", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /blog"}
-{"id": "ng-500-wpjson", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp-json/batch/v1"}
-{"id": "ng-500-wordpress", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wordpress"}
-{"id": "ng-404-env", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env"}
-{"id": "ng-404-envlocal", "src": "clickhouse", "expected": "security_alert", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env.local"}
-{"id": "ng-404-userfiles", "src": "clickhouse", "expected": "security_alert", "note": "ambiguous: probe or stale link", "text": "Service: nginx | Kind: span | Status: 404 | Route: /userfiles"}
-{"id": "ng-404-commit", "src": "clickhouse", "expected": "log_404", "note": "git UI link to missing commit", "text": "Service: nginx | Kind: span | Status: 404 | Route: /mats/go-cart-actor/commit/{id}"}
-{"id": "ng-404-vite", "src": "clickhouse", "expected": "log_404", "note": "dev tooling request", "text": "Service: nginx | Kind: span | Status: 404 | Route: /__vite_rsc_findSourceMapURL"}
-{"id": "ng-404-blob", "src": "clickhouse", "expected": "log_404", "note": "registry blob existence check", "text": "Service: nginx | Kind: span | Status: 404 | Route: /v2/eventsourced-ai/blobs/{id}"}
-{"id": "ng-499-repos", "src": "clickhouse", "expected": "none", "note": "client closed request", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/repos/available"}
-{"id": "ng-499-ctrl", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/controller/instances"}
-{"id": "ng-308-blog", "src": "clickhouse", "expected": "none", "note": "redirect", "text": "Service: nginx | Kind: span | Status: 308 | Route: /blog/"}
-{"id": "ng-200-speed", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/network-speed"}
-{"id": "ng-200-matrix", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /_matrix/federation/v1/send/{id}"}
-{"id": "ng-200-overview", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/overview"}
-{"id": "sched-readyz", "src": "clickhouse", "expected": "none", "note": "health check", "text": "Service: scheduling | Kind: span | Title: GET /readyz | Message: get /readyz | Status: 200 | Route: /readyz"}
-{"id": "cart-ping", "src": "clickhouse", "expected": "none", "note": "gRPC keepalive", "text": "Service: cart | Kind: span | Title: /control_plane_messages.ControlPlane/Ping | Message: /control_plane_messages.controlplane/ping"}
-{"id": "rag-request", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: rust-rag | Kind: span | Title: request | Message: request"}
-{"id": "orch-session-load", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: eventsourced-orchestrator | Kind: log | Message: session snapshot/replay loaded"}
-{"id": "worker-dialerr", "src": "clickhouse", "expected": "none", "note": "dependency down; arguably escalate_outage - decide", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:<n>/...: connection refused"}
-{"id": "worker-eof", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *errors.errorstring: eof"}
-{"id": "ledger-skip", "src": "clickhouse", "expected": "none", "note": "", "text": "Service: ledger | Kind: log | Message: offsite replicate: content unchanged since last push - skipping"}
-{"id": "web-cls", "src": "clickhouse", "expected": "record_vital", "note": "real frontend log; is this a vital or a console line?", "text": "Service: eventsourced-ai-web | Kind: log | Message: cls <num>"}
-{"id": "syn-js-1", "src": "synthetic", "expected": "client_js_crash", "note": "", "text": "Service: web-frontend | Kind: log | Title: TypeError: Cannot read properties of undefined | Message: Uncaught TypeError: Cannot read properties of undefined (reading 'map') | Route: /checkout"}
-{"id": "syn-js-2", "src": "synthetic", "expected": "client_js_crash", "note": "", "text": "Service: shop-ui | Kind: log | Title: ChunkLoadError | Message: Loading chunk 42 failed | Route: /products/<id>"}
-{"id": "syn-vital-1", "src": "synthetic", "expected": "record_vital", "note": "", "text": "Service: web-frontend | Kind: metric | Title: web_vital TTFB | Message: TTFB 812ms"}
-{"id": "syn-db-1", "src": "synthetic", "expected": "database_error", "note": "", "text": "Service: checkout | Kind: log | Title: database timeout after <n>ms | Message: context deadline exceeded querying orders"}
-{"id": "syn-db-2", "src": "synthetic", "expected": "database_error", "note": "", "text": "Service: inventory | Kind: log | Title: deadlock detected | Message: pq: deadlock detected while updating stock"}
-{"id": "syn-db-3", "src": "synthetic", "expected": "database_error", "note": "", "text": "Service: orders | Kind: log | Title: connection pool exhausted | Message: too many connections in pool"}
-{"id": "syn-oom", "src": "synthetic", "expected": "none", "note": "", "text": "Service: kubelet-proxy | Kind: log | Title: OOMKilled | Message: container exceeded memory limit and was restarted"}
-{"id": "syn-retry", "src": "synthetic", "expected": "none", "note": "", "text": "Service: worker | Kind: span | Title: retry attempt 2 | Message: retrying after transient error"}
+{"id": "ng-404-root", "src": "clickhouse", "expected": "stale_reference", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /"}
+{"id": "ng-401-repos", "src": "clickhouse", "expected": "client_credential_issue", "note": "polled ~700x/h by an unauthenticated client", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/repos/available"}
+{"id": "ng-401-ctrl", "src": "clickhouse", "expected": "client_credential_issue", "note": "", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/controller/instances"}
+{"id": "orch-401-repos", "src": "clickhouse", "expected": "client_credential_issue", "note": "same request as ng-401-repos, second span", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/repos/available"}
+{"id": "orch-401-chat", "src": "clickhouse", "expected": "client_credential_issue", "note": "", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/sessions/{id}/collab/chat"}
+{"id": "ng-401-manifest", "src": "clickhouse", "expected": "client_credential_issue", "note": "registry pull without token", "text": "Service: nginx | Kind: span | Status: 401 | Route: /v2/eventsourced-ai-worker/manifests/latest"}
+{"id": "ng-503-store", "src": "clickhouse", "expected": "dependency_unreachable", "note": "nginx 503 burst: upstream down is the likeliest cause", "text": "Service: nginx | Kind: span | Status: 503 | Route: /store/regions"}
+{"id": "ng-500-root", "src": "clickhouse", "expected": "service_defect", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /"}
+{"id": "ng-500-wp", "src": "clickhouse", "expected": "scanner_probe", "note": "WordPress path that does not exist here", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp"}
+{"id": "ng-500-blog", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /blog"}
+{"id": "ng-500-wpjson", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp-json/batch/v1"}
+{"id": "ng-500-wordpress", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wordpress"}
+{"id": "ng-404-env", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env"}
+{"id": "ng-404-envlocal", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env.local"}
+{"id": "ng-404-userfiles", "src": "clickhouse", "expected": "scanner_probe", "note": "ambiguous: probe or stale link", "text": "Service: nginx | Kind: span | Status: 404 | Route: /userfiles"}
+{"id": "ng-404-commit", "src": "clickhouse", "expected": "stale_reference", "note": "git UI link to missing commit", "text": "Service: nginx | Kind: span | Status: 404 | Route: /mats/go-cart-actor/commit/{id}"}
+{"id": "ng-404-vite", "src": "clickhouse", "expected": "stale_reference", "note": "dev tooling request", "text": "Service: nginx | Kind: span | Status: 404 | Route: /__vite_rsc_findSourceMapURL"}
+{"id": "ng-404-blob", "src": "clickhouse", "expected": "stale_reference", "note": "registry blob existence check", "text": "Service: nginx | Kind: span | Status: 404 | Route: /v2/eventsourced-ai/blobs/{id}"}
+{"id": "ng-499-repos", "src": "clickhouse", "expected": "expected_lifecycle", "note": "client closed request", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/repos/available"}
+{"id": "ng-499-ctrl", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/controller/instances"}
+{"id": "ng-308-blog", "src": "clickhouse", "expected": "expected_lifecycle", "note": "redirect", "text": "Service: nginx | Kind: span | Status: 308 | Route: /blog/"}
+{"id": "ng-200-speed", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/network-speed"}
+{"id": "ng-200-matrix", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /_matrix/federation/v1/send/{id}"}
+{"id": "ng-200-overview", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/overview"}
+{"id": "sched-readyz", "src": "clickhouse", "expected": "expected_lifecycle", "note": "health check", "text": "Service: scheduling | Kind: span | Title: GET /readyz | Message: get /readyz | Status: 200 | Route: /readyz"}
+{"id": "cart-ping", "src": "clickhouse", "expected": "expected_lifecycle", "note": "gRPC keepalive", "text": "Service: cart | Kind: span | Title: /control_plane_messages.ControlPlane/Ping | Message: /control_plane_messages.controlplane/ping"}
+{"id": "rag-request", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: rust-rag | Kind: span | Title: request | Message: request"}
+{"id": "orch-session-load", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: eventsourced-orchestrator | Kind: log | Message: session snapshot/replay loaded"}
+{"id": "worker-dialerr", "src": "clickhouse", "expected": "dependency_unreachable", "note": "", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:<n>/...: connection refused"}
+{"id": "worker-eof", "src": "clickhouse", "expected": "dependency_unreachable", "note": "", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *errors.errorstring: eof"}
+{"id": "ledger-skip", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: ledger | Kind: log | Message: offsite replicate: content unchanged since last push - skipping"}
+{"id": "web-cls", "src": "clickhouse", "expected": "expected_lifecycle", "note": "real frontend log; is this a vital or a console line?", "text": "Service: eventsourced-ai-web | Kind: log | Message: cls <num>"}
+{"id": "syn-js-1", "src": "synthetic", "expected": "service_defect", "note": "", "text": "Service: web-frontend | Kind: log | Title: TypeError: Cannot read properties of undefined | Message: Uncaught TypeError: Cannot read properties of undefined (reading 'map') | Route: /checkout"}
+{"id": "syn-js-2", "src": "synthetic", "expected": "service_defect", "note": "", "text": "Service: shop-ui | Kind: log | Title: ChunkLoadError | Message: Loading chunk 42 failed | Route: /products/<id>"}
+{"id": "syn-vital-1", "src": "synthetic", "expected": "expected_lifecycle", "note": "", "text": "Service: web-frontend | Kind: metric | Title: web_vital TTFB | Message: TTFB 812ms"}
+{"id": "syn-db-1", "src": "synthetic", "expected": "capacity_saturation", "note": "", "text": "Service: checkout | Kind: log | Title: database timeout after <n>ms | Message: context deadline exceeded querying orders"}
+{"id": "syn-db-2", "src": "synthetic", "expected": "service_defect", "note": "", "text": "Service: inventory | Kind: log | Title: deadlock detected | Message: pq: deadlock detected while updating stock"}
+{"id": "syn-db-3", "src": "synthetic", "expected": "capacity_saturation", "note": "", "text": "Service: orders | Kind: log | Title: connection pool exhausted | Message: too many connections in pool"}
+{"id": "syn-oom", "src": "synthetic", "expected": "capacity_saturation", "note": "", "text": "Service: kubelet-proxy | Kind: log | Title: OOMKilled | Message: container exceeded memory limit and was restarted"}
+{"id": "syn-retry", "src": "synthetic", "expected": "expected_lifecycle", "note": "", "text": "Service: worker | Kind: span | Title: retry attempt 2 | Message: retrying after transient error"}

cmd/needle-sidecar/eval/holdout.jsonl

@@ -0,0 +1,60 @@
+{"service":"nginx","kind":"span","status_code":404,"route":"/ses.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/api/v1/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/api/w/admins/jobs_u/get_log_file/../../../../proc/self/environ","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"eventsourced-ai-web","kind":"log","status_code":0,"route":"","title":"","message":"cls \u003cnum\u003e","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":16,"expected":"","predicted":"expected_lifecycle","rule":"lifecycle-text"}
+{"service":"nginx","kind":"span","status_code":500,"route":"/wordpress","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":30,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/private/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/settings.js","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":200,"route":"/","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":17,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"}
+{"service":"eventsourced-orchestrator","kind":"span","status_code":200,"route":"/api/memories","title":"GET /api/memories","message":"get /api/memories","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":5,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"}
+{"service":"cart","kind":"log","status_code":0,"route":"","title":"","message":"handleownershipchange from \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e: accepted_remote=\u003cnum\u003e kept_local=\u003cnum\u003e legacy_no_stamp=\u003cnum\u003e","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"expected_lifecycle","rule":"no-status-signal"}
+{"service":"eventsourced-orchestrator","kind":"span","status_code":0,"route":"","title":"streamproc.replay","message":"streamproc.replay","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"","predicted":"expected_lifecycle","rule":"lifecycle-text"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/.htpasswd","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":401,"route":"/v2/eventsourced-ai-worker/manifests/6d2e5b3-sandbox","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":16,"expected":"","predicted":"client_credential_issue","rule":"status-401"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/api/config","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/ssl/server.key","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/_debugbar/open","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":500,"route":"/register","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"","predicted":"service_defect","rule":"status-5xx"}
+{"service":"nginx","kind":"span","status_code":500,"route":"/wp/wordpress/wp-json/batch/v1","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/old/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/mcp","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":401,"route":"/mcp","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"client_credential_issue","rule":"status-401"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/graphql/console","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/debug/pprof","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/panel/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":200,"route":"/api/actions/runner.v1.RunnerService/FetchTask","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":729,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/server-info","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"eventsourced-orchestrator","kind":"span","status_code":401,"route":"/api/overview","title":"GET /","message":"get /","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"","predicted":"client_credential_issue","rule":"status-401"}
+{"service":"nginx","kind":"span","status_code":101,"route":"/api/sessions/{id}/collab/chat-prompt/ws","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"needs_review","rule":"no-rule"}
+{"service":"nginx","kind":"span","status_code":401,"route":"/v2/eventsourced-ai/blobs/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"","predicted":"client_credential_issue","rule":"status-401"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/v2/eventsourced-ai/blobs/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":17,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/api/v2/settings","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/.well-known/jwks.json","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"eventsourced-ai-web","kind":"log","status_code":0,"route":"","title":"","message":"ttfb \u003cnum\u003e","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":5,"expected":"","predicted":"expected_lifecycle","rule":"lifecycle-text"}
+{"service":"nginx","kind":"span","status_code":200,"route":"/api/stream","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":9,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"}
+{"service":"eventsourced-orchestrator","kind":"span","status_code":200,"route":"/api/sessions/{id}/replay","title":"GET /api/sessions/{id}/replay","message":"get /api/sessions/{id}/replay","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"","predicted":"expected_lifecycle","rule":"lifecycle-text"}
+{"service":"nginx","kind":"span","status_code":201,"route":"/v2/ecommerce/auth/blobs/uploads/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"needs_review","rule":"no-rule"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/cms/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/__env.js","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":504,"route":"/api/stream","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"","predicted":"dependency_unreachable","rule":"status-5xx-upstream"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/api/v2/config","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/actuator/threaddump","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/scripts/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/.streamlit/secrets.toml","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/web/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"cart","kind":"log","status_code":0,"route":"","title":"","message":"\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e taking ownership of \u003cnum\u003e ids","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"expected_lifecycle","rule":"no-status-signal"}
+{"service":"nginx","kind":"span","status_code":200,"route":"/api/schemas/harness_evidence","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/agent/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/settings.py","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/conf/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/dev/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"eventsourced-orchestrator","kind":"span","status_code":302,"route":"/auth/login","title":"GET /auth/login","message":"get /auth/login","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/favicon.ico","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":200,"route":"/api/dev-machines","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":120,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"}
+{"service":"nginx","kind":"span","status_code":308,"route":"/wp-json/batch/v1/","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/.bash_profile","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/@fs/home/ubuntu/.aws/credentials","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/serviceAccountKey.json","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/graphql","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":404,"route":"/api","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"}
+{"service":"nginx","kind":"span","status_code":200,"route":"/v2/ecommerce/auth/manifests/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"}

cmd/needle-sidecar/eval/rules.py

@@ -0,0 +1,42 @@
+"""Deterministic baseline for the cause categories. Needle has to beat this."""
+import re
+
+PROBE = re.compile(r"(^|/)(\.env|\.git|wp-|wp/|wp$|wordpress|blog/wp|phpmyadmin|cgi-bin|\.aws|\.ssh|xmlrpc|userfiles|@fs/|etc/passwd)", re.I)
+DEP = re.compile(r"dial|connection refused|eof|no such host|upstream|unavailable|broken pipe|reset by peer", re.I)
+CAP = re.compile(r"timeout|deadline exceeded|pool|memory limit|oom|too many|queue full|exhaust", re.I)
+BUG = re.compile(r"panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault", re.I)
+LIFE = re.compile(r"healthz|readyz|/ping|retry|cancel|keepalive|vital|\bcls\b|ttfb|fcp|lcp|snapshot|replay|unchanged|session\.load", re.I)
+
+
+def fields(text):
+    d = {}
+    for part in text.split(" | "):
+        k, _, v = part.partition(": ")
+        d[k.lower()] = v
+    return d
+
+
+def classify(text):
+    f = fields(text)
+    status = int(f.get("status") or 0)
+    route = f.get("route", "")
+    blob = " ".join([f.get("title", ""), f.get("message", ""), route])
+    if PROBE.search(route):
+        return "scanner_probe"
+    if BUG.search(blob):
+        return "service_defect"
+    if CAP.search(blob):
+        return "capacity_saturation"
+    if DEP.search(blob):
+        return "dependency_unreachable"
+    if status == 401:
+        return "client_credential_issue"
+    if status in (502, 503, 504):
+        return "dependency_unreachable"
+    if status >= 500:
+        return "service_defect"
+    if status == 404:
+        return "stale_reference"
+    if LIFE.search(blob) or status in (200, 202, 204, 301, 302, 304, 308, 499) or (status == 0 and f.get("kind") in ("span", "log")):
+        return "expected_lifecycle"
+    return "needs_review"

cmd/needle-sidecar/eval/run_eval.py

@@ -4,8 +4,7 @@
 Usage: run_eval.py [--url http://127.0.0.1:5005] [--golden eval/golden.jsonl] [--json out.json]
 
 Each golden row: {"id", "expected", "text"}; `expected` is a sidecar action name
-or "none" when no tool should fit. The sidecar has no fallback tool, so "none"
-rows measure how often it force-fits (see the confidence column).
+including "needs_review" for the fallback.
 """
 import argparse, json, os, sys, time, urllib.request
 from collections import Counter, defaultdict
@@ -29,7 +28,7 @@ def main():
     ap.add_argument("--golden", default=os.path.join(HERE, "golden.jsonl"))
     ap.add_argument("--json", help="write per-row results here")
     ap.add_argument("--min-conf", type=float, default=0.0,
-                    help="predictions below this confidence count as 'none'")
+                    help="predictions below this confidence count as needs_review")
     a = ap.parse_args()
 
     rows = [json.loads(l) for l in open(a.golden) if l.strip()]
@@ -41,10 +40,10 @@ def main():
         except Exception as e:
             print(f"sidecar error on {row['id']}: {e}", file=sys.stderr)
             sys.exit(2)
-        pred = out.get("action", "unclassified")
+        pred = out.get("action", "needs_review")
         conf = out.get("confidence", 0.0)
-        if conf < a.min_conf or pred == "unclassified":
-            pred = "none"
+        if conf < a.min_conf:
+            pred = "needs_review"
         lat.append(out.get("latency_ms", 0.0))
         confusion[(row["expected"], pred)] += 1
         results.append({**row, "predicted": pred, "confidence": conf, "ok": pred == row["expected"]})

cmd/needle-sidecar/server.py

@@ -16,46 +16,49 @@ import needle
 PORT = int(os.environ.get("NEEDLE_PORT", 5005))
 HOST = os.environ.get("NEEDLE_HOST", "0.0.0.0")
 
-# --- Define Triage Tools for Needle 3 ---
+# --- Cause-level triage tools ---
+# These deliberately do not restate what the event already carries (status
+# code, is_error, severity). Each one is a judgement about *why* it happened
+# or *who should care*, which is what an alert reader needs.
 @needle.tool
-def report_client_javascript_crash(service: str, message: str, url: str):
-    """Report client-side JavaScript crash or error logged by browser RUM."""
-    return {"action": "client_js_crash", "service": service, "message": message, "url": url}
+def report_scanner_probe(service: str, path: str, status_code: int):
+    """Automated scan or exploit crawler requesting sensitive or nonexistent paths such as .env, wp-admin, .git, phpmyadmin. Applies whatever status the probe received, including 500 or 502."""
+    return {"action": "scanner_probe", "service": service, "path": path, "status_code": status_code}
 
 @needle.tool
-def alert_vulnerability_probe(scanner: str, target: str, status_code: int):
-    """Alert on malicious vulnerability scans or exploit crawlers."""
-    return {"action": "security_alert", "scanner": scanner, "target": target, "status_code": status_code}
+def report_dependency_unreachable(service: str, dependency: str, error_message: str):
+    """A downstream or upstream service is down, refusing connections, closed the stream (EOF), or returned upstream unavailable."""
+    return {"action": "dependency_unreachable", "service": service, "dependency": dependency, "error": error_message}
 
 @needle.tool
-def record_frontend_web_vital(service: str, metric_name: str, duration_ms: int):
-    """Record RUM performance metrics like TTFB or FCP."""
-    return {"action": "record_vital", "service": service, "metric": metric_name, "ms": duration_ms}
+def report_client_credential_issue(service: str, route: str, reason: str):
+    """A caller lacks a valid or unexpired token, for example an unauthenticated client polling an API or a registry pull without login. The caller's fault, not a fault in this service."""
+    return {"action": "client_credential_issue", "service": service, "route": route, "reason": reason}
 
 @needle.tool
-def log_missing_resource_404(path: str, host: str):
-    """Log benign 404 Not Found requests for missing pages or git commits."""
-    return {"action": "log_404", "path": path, "host": host}
+def report_stale_reference(service: str, path: str):
+    """Harmless missing resource caused by an old link, removed asset, deleted commit, or dev tooling request."""
+    return {"action": "stale_reference", "service": service, "path": path}
 
 @needle.tool
-def report_database_error(service: str, table: str, error_message: str):
-    """Report database query errors, deadlocks, or connection pool timeouts."""
-    return {"action": "database_error", "service": service, "table": table, "error": error_message}
+def report_service_defect(service: str, error_message: str):
+    """A bug in the service's own code: panic, nil dereference, unhandled exception, index out of range, deadlock, or a 500 from a real application route."""
+    return {"action": "service_defect", "service": service, "error": error_message}
 
 @needle.tool
-def report_auth_failure(service: str, user_or_token: str, reason: str):
-    """Report authentication/authorization failure or invalid credentials."""
-    return {"action": "auth_failure", "service": service, "user_or_token": user_or_token, "reason": reason}
+def report_capacity_saturation(service: str, resource: str, error_message: str):
+    """Resource exhaustion or overload: timeouts, connection pool exhausted, memory limit exceeded, OOM, queue full."""
+    return {"action": "capacity_saturation", "service": service, "resource": resource, "error": error_message}
 
 @needle.tool
-def escalate_backend_outage(service: str, error_message: str):
-    """Escalate severe backend 5xx crash or unhandled panic to on-call."""
-    return {"action": "escalate_outage", "service": service, "error": error_message}
+def report_expected_lifecycle(service: str, what: str):
+    """Normal behaviour that is not a problem: health and readiness checks, keepalive pings, retries, cancellations, client disconnects, redirects, successful requests, routine info logs, web vitals."""
+    return {"action": "expected_lifecycle", "service": service, "what": what}
 
 @needle.tool
-def report_unclassified(service: str, summary: str):
-    """Fallback when the event matches none of the other categories: retries, cancellations, restarts, OOM, health checks, or anything unclear."""
-    return {"action": "unclassified", "service": service, "summary": summary}
+def report_needs_review(service: str, summary: str):
+    """Fallback when none of the other categories clearly applies."""
+    return {"action": "needs_review", "service": service, "summary": summary}
 
 # --- Redaction: nothing PII-bearing should reach the model or the cache ---
 _REDACTIONS = [
@@ -81,14 +84,14 @@ _stats = {"hits": 0, "misses": 0}
 print(f"Loading Needle 3 model (Cactus-Compute/needle3)...")
 t0 = time.perf_counter()
 agent = needle.Needle(tools=[
-    report_client_javascript_crash,
-    alert_vulnerability_probe,
-    record_frontend_web_vital,
-    log_missing_resource_404,
-    report_database_error,
-    report_auth_failure,
-    escalate_backend_outage,
-    report_unclassified
+    report_scanner_probe,
+    report_dependency_unreachable,
+    report_client_credential_issue,
+    report_stale_reference,
+    report_service_defect,
+    report_capacity_saturation,
+    report_expected_lifecycle,
+    report_needs_review,
 ])
 print(f"Needle 3 loaded in {(time.perf_counter() - t0):.2f}s")
 
@@ -116,14 +119,8 @@ class NeedleHandler(BaseHTTPRequestHandler):
                 "model": "Cactus-Compute/needle3",
                 "cache": {**_stats, "size": len(_cache)},
                 "tools": [
-                    "client_js_crash",
-                    "security_alert",
-                    "record_vital",
-                    "log_404",
-                    "database_error",
-                    "auth_failure",
-                    "escalate_outage",
-                    "unclassified"
+                    "scanner_probe", "dependency_unreachable", "client_credential_issue", "stale_reference",
+                    "service_defect", "capacity_saturation", "expected_lifecycle", "needs_review"
                 ]
             }
             self.wfile.write(json.dumps(resp).encode("utf-8"))
@@ -193,7 +190,7 @@ class NeedleHandler(BaseHTTPRequestHandler):
 
             results = res.get("results", [])
             first_result = results[0] if results and isinstance(results[0], dict) else {}
-            action = first_result.get("action", "unclassified")
+            action = first_result.get("action", "needs_review")
 
             calls = res.get("function_calls", [])
             tool_name = calls[0].get("name") if calls else "triage_tool"

cmd/normalized-backfill/main.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"git.tornberg.me/go-otel/internal/normalization"
+	"git.tornberg.me/go-otel/internal/storage"
 	"github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 )
@@ -50,7 +51,7 @@ func main() {
 		log.Fatal(err)
 	}
 	if *write {
-		if err := createSchema(ctx, conn, *database); err != nil {
+		if err := storage.EnsureNormalizedSchema(ctx, conn, *database); err != nil {
 			log.Fatal(err)
 		}
 	}
@@ -82,7 +83,7 @@ func main() {
 	rawCount, eventCount := 0, 0
 	var batch driver.Batch
 	if *write {
-		batch, err = conn.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.normalized_events", *database))
+		batch, err = conn.PrepareBatch(ctx, storage.NormalizedInsertSQL(*database))
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -115,12 +116,12 @@ func main() {
 			}
 			counts[key].Count++
 			if *write {
-				if err := appendEvent(batch, event); err != nil {
+				if err := storage.AppendNormalizedEvent(batch, event); err != nil {
 					log.Fatal(err)
 				}
 				if eventCount%*batchSize == 0 {
 					flush()
-					batch, err = conn.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.normalized_events", *database))
+					batch, err = conn.PrepareBatch(ctx, storage.NormalizedInsertSQL(*database))
 					if err != nil {
 						log.Fatal(err)
 					}
@@ -146,70 +147,6 @@ func main() {
 	}
 }
 
-func createSchema(ctx context.Context, conn clickhouse.Conn, database string) error {
-	query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.normalized_events (
-		event_id String,
-		entry_type LowCardinality(String),
-		event_index UInt32,
-		ingest_timestamp DateTime64(9, 'UTC'),
-		timestamp DateTime64(9, 'UTC'),
-		service LowCardinality(String), release LowCardinality(String), environment LowCardinality(String),
-		kind LowCardinality(String), source LowCardinality(String), is_error UInt8,
-		status_class LowCardinality(String), status_code UInt16, duration_ms Float64,
-		route String, span_name String, metric_name String, metric_type LowCardinality(String), metric_unit String,
-		metric_value Float64, metric_count UInt64, metric_sum Float64, metric_min Float64, metric_max Float64,
-		exception_type String, message_normalized String, stack_signature String, trace_id String, issue_fingerprint String,
-		is_retry UInt8, is_timeout UInt8, is_cancellation UInt8,
-		exact_signature String, pattern_signature String, version UInt8 DEFAULT 1,
-		method String DEFAULT '', host String DEFAULT '', user_agent String DEFAULT '',
-		origin_host String DEFAULT '', referer_host String DEFAULT '', fetch_site String DEFAULT '', fetch_mode String DEFAULT ''
-	) ENGINE = ReplacingMergeTree(version)
-	PARTITION BY toYYYYMM(timestamp)
-	ORDER BY (service, kind, pattern_signature, timestamp, event_id)
-	TTL toDateTime(timestamp) + INTERVAL 180 DAY`, database)
-	if err := conn.Exec(ctx, query); err != nil {
-		return err
-	}
-	// Existing developer tables predate the explicit model flags. These
-	// additive migrations preserve historical rows with safe zero defaults.
-	for _, statement := range []string{
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS issue_fingerprint String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_retry UInt8 DEFAULT 0", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_timeout UInt8 DEFAULT 0", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_cancellation UInt8 DEFAULT 0", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS method String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS host String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS user_agent String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS origin_host String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS referer_host String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS fetch_site String DEFAULT ''", database),
-		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS fetch_mode String DEFAULT ''", database),
-	} {
-		if err := conn.Exec(ctx, statement); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func appendEvent(batch driver.Batch, event normalization.Event) error {
-	return batch.Append(
-		event.EventID, event.EntryType, event.EventIndex, time.Unix(0, event.IngestTimestamp), time.Unix(0, event.Timestamp),
-		event.Service, event.Release, event.Environment, event.Kind, event.Source, boolUint8(event.IsError),
-		event.StatusClass, event.StatusCode, event.DurationMS, event.Route, event.SpanName, event.MetricName, event.MetricType, event.MetricUnit,
-		event.MetricValue, event.MetricCount, event.MetricSum, event.MetricMin, event.MetricMax,
-		event.ExceptionType, event.MessageNormalized, event.StackSignature, event.TraceID, event.IssueFingerprint,
-		boolUint8(event.IsRetry), boolUint8(event.IsTimeout), boolUint8(event.IsCancellation), event.ExactSignature, event.PatternSignature,
-		uint8(1), event.Method, event.Host, event.UserAgent, event.OriginHost, event.RefererHost, event.FetchSite, event.FetchMode,
-	)
-}
-
-func boolUint8(value bool) uint8 {
-	if value {
-		return 1
-	}
-	return 0
-}
 func envOr(key, fallback string) string {
 	if value := os.Getenv(key); value != "" {
 		return value

cmd/server/main.go

@@ -11,6 +11,7 @@ import (
 
 	"git.tornberg.me/go-otel/internal/auth"
 	"git.tornberg.me/go-otel/internal/collector"
+	"git.tornberg.me/go-otel/internal/blocker"
 	"git.tornberg.me/go-otel/internal/honeypot"
 	kube "git.tornberg.me/go-otel/internal/kubernetes"
 	"git.tornberg.me/go-otel/internal/prometheus"
@@ -235,6 +236,12 @@ func main() {
 	wsServer := websocket.New(coll)
 	honeypotTracker := honeypot.New(honeypot.ConfigFromEnv())
 	coll.SetHoneypotRecorder(honeypotTracker)
+	// Optional ingress blocker: off unless OTEL_BLOCKER_ENABLED=true, and
+	// dry-run unless OTEL_BLOCKER_DRY_RUN=false.
+	ipBlocker, err := blocker.Start(context.Background(), blocker.ConfigFromEnv(), honeypotTracker)
+	if err != nil {
+		log.Printf("ip blocker disabled: %v", err)
+	}
 	// Update collector with wsServer
 	coll.SetBroadcaster(wsServer)
 	ingestKey := os.Getenv("OTEL_INGEST_KEY")
@@ -293,6 +300,9 @@ func main() {
 	uiMux.HandleFunc("POST /api/auth/token", oidcAuth.HandleTokenExchange)
 	uiMux.HandleFunc("GET /api/auth/config", oidcAuth.HandleConfig)
 	uiMux.Handle("/api/honeypot/ips", oidcAuth.RequireAuth(http.HandlerFunc(honeypotTracker.HandleIPsForUser)))
+	if ipBlocker != nil {
+		uiMux.Handle("GET /api/blocker/status", oidcAuth.RequireAuth(http.HandlerFunc(ipBlocker.HandleStatus)))
+	}
 	// Source-map uploads are intentionally public so the release build script
 	// can upload maps without an OIDC session or bearer token.
 	uiMux.HandleFunc("POST /api/sourcemaps", coll.HandleSourceMap)

cmd/spike-alerts/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/ClickHouse/clickhouse-go/v2"
 
 	"git.tornberg.me/go-otel/internal/features"
+	"git.tornberg.me/go-otel/internal/triage"
 )
 
 func main() {
@@ -33,7 +34,8 @@ func main() {
 	minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert")
 	multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline")
 	cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles")
-	needle := flag.String("needle", os.Getenv("OTEL_NEEDLE_ADDR"), "Needle sidecar URL; empty skips classification")
+	needle := flag.String("needle", "", "Needle sidecar URL (dormant: it is not accurate enough to gate alerts); empty skips it")
+	showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)")
 	asJSON := flag.Bool("json", false, "emit JSON lines instead of text")
 	flag.Parse()
 	if *since == "" || *until == "" {
@@ -55,9 +57,10 @@ func main() {
 	cache := map[string]classification{}
 	first := start.Add(time.Duration(*baselineWindows) * *window)
 	suppress := features.NewSuppressor(*cooldown)
+	muted := map[triage.Category]int{}
 	alerts := 0
 	for at := first.Add(*window); !at.After(end); at = at.Add(*step) {
-		for _, spike := range suppress.Filter(features.DetectSpikes(events, at, cfg)) {
+		for _, spike := range suppress.Filter(triageFilter(features.DetectSpikes(events, at, cfg), *showMuted, muted)) {
 			alerts++
 			var cls []classification
 			for _, p := range spike.Patterns {
@@ -79,6 +82,9 @@ func main() {
 			printSpike(spike, cls)
 		}
 	}
+	for c, n := range muted {
+		log.Printf("muted %d pattern-window(s) as %s", n, c)
+	}
 	log.Printf("%d alert(s) between %s and %s", alerts, first.Format(time.RFC3339), end.Format(time.RFC3339))
 }
 
@@ -89,6 +95,35 @@ type classification struct {
 	Err        string  `json:"error,omitempty"`
 }
 
+func triageOf(e features.Event) triage.Result {
+	return triage.Classify(triage.Input{Service: e.Service, Kind: e.Kind, Status: e.StatusCode, Route: e.Route, Title: e.SpanName, Message: e.Message,
+		IsTimeout: e.IsTimeout, IsCancellation: e.IsCancellation, IsRetry: e.IsRetry})
+}
+
+// triageFilter keeps only actionable patterns (unless showMuted), recomputing
+// each spike's excess so a probe storm cannot page anyone by itself.
+func triageFilter(spikes []features.Spike, showMuted bool, muted map[triage.Category]int) []features.Spike {
+	var out []features.Spike
+	for _, sp := range spikes {
+		var kept []features.PatternDelta
+		sp.Excess = 0
+		for _, p := range sp.Patterns {
+			c := triageOf(p.Example).Category
+			if !showMuted && !c.Actionable() {
+				muted[c]++
+				continue
+			}
+			kept = append(kept, p)
+			sp.Excess += p.Excess
+		}
+		if len(kept) > 0 {
+			sp.Patterns = kept
+			out = append(out, sp)
+		}
+	}
+	return out
+}
+
 func classify(base, text string) classification {
 	body, _ := json.Marshal(map[string]string{"text": text})
 	resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body))
@@ -127,9 +162,9 @@ func printSpike(s features.Spike, cls []classification) {
 		if p.Novel {
 			tag = "NEW"
 		}
-		what := "-"
+		what := string(triageOf(p.Example).Category)
 		if cls[i].Action != "" {
-			what = fmt.Sprintf("%s (%.2f)", cls[i].Action, cls[i].Confidence)
+			what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence)
 		}
 		fmt.Printf("  %-6s x%-4d base %-6.1f %-24s %s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example))
 	}
@@ -144,7 +179,7 @@ func load(addr, database, user, password string, start, end time.Time) []feature
 	}
 	defer conn.Close()
 	// Errors only, and only columns present in every deployed schema version.
-	rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT timestamp, service, release, kind, source, status_code, span_name, route, exception_type, message_normalized, pattern_signature
+	rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT timestamp, service, release, kind, source, status_code, span_name, route, exception_type, message_normalized, pattern_signature, is_timeout, is_cancellation, is_retry
 		FROM %s.normalized_events FINAL
 		WHERE is_error = 1 AND timestamp >= toDateTime64('%s', 9, 'UTC') AND timestamp < toDateTime64('%s', 9, 'UTC')
 		ORDER BY timestamp LIMIT 2000000`, database, chTime(start), chTime(end)))
@@ -156,10 +191,12 @@ func load(addr, database, user, password string, start, end time.Time) []feature
 	for rows.Next() {
 		e := features.Event{IsError: true}
 		var status uint16
-		if err := rows.Scan(&e.Timestamp, &e.Service, &e.Release, &e.Kind, &e.Source, &status, &e.SpanName, &e.Route, &e.ExceptionType, &e.Message, &e.PatternSignature); err != nil {
+		var timeout, cancel, retry uint8
+		if err := rows.Scan(&e.Timestamp, &e.Service, &e.Release, &e.Kind, &e.Source, &status, &e.SpanName, &e.Route, &e.ExceptionType, &e.Message, &e.PatternSignature, &timeout, &cancel, &retry); err != nil {
 			log.Fatal(err)
 		}
 		e.StatusCode = int(status)
+		e.IsTimeout, e.IsCancellation, e.IsRetry = timeout == 1, cancel == 1, retry == 1
 		out = append(out, e)
 	}
 	if err := rows.Err(); err != nil {

cmd/triage-eval/main.go

@@ -0,0 +1,98 @@
+// triage-eval applies the rules classifier to a JSONL file of patterns. Rows
+// with an "expected" category are scored; the rest are written out with the
+// prediction so they can be reviewed and labelled by hand.
+//
+//	triage-eval -input holdout.jsonl            # score labelled rows
+//	triage-eval -input raw.jsonl -label-out holdout.jsonl   # prepare for labelling
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"sort"
+
+	"git.tornberg.me/go-otel/internal/triage"
+)
+
+type row struct {
+	Service        string `json:"service"`
+	Kind           string `json:"kind"`
+	StatusCode     int    `json:"status_code"`
+	Route          string `json:"route"`
+	Title          string `json:"title"`
+	Message        string `json:"message"`
+	IsTimeout      uint8  `json:"is_timeout"`
+	IsCancellation uint8  `json:"is_cancellation"`
+	IsRetry        uint8  `json:"is_retry"`
+	Count          uint64 `json:"n"`
+	Expected       string `json:"expected"`
+	Predicted      string `json:"predicted"`
+	Rule           string `json:"rule"`
+}
+
+func main() {
+	input := flag.String("input", "", "JSONL input")
+	labelOut := flag.String("label-out", "", "write rows with predictions here for hand labelling")
+	flag.Parse()
+	f, err := os.Open(*input)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+	var rows []row
+	sc := bufio.NewScanner(f)
+	sc.Buffer(make([]byte, 1<<20), 1<<20)
+	for sc.Scan() {
+		var r row
+		if err := json.Unmarshal(sc.Bytes(), &r); err != nil {
+			log.Fatalf("bad row: %v", err)
+		}
+		res := triage.Classify(triage.Input{Service: r.Service, Kind: r.Kind, Status: r.StatusCode, Route: r.Route, Title: r.Title, Message: r.Message,
+			IsTimeout: r.IsTimeout == 1, IsCancellation: r.IsCancellation == 1, IsRetry: r.IsRetry == 1})
+		r.Predicted, r.Rule = string(res.Category), res.Rule
+		rows = append(rows, r)
+	}
+	if *labelOut != "" {
+		out, err := os.Create(*labelOut)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer out.Close()
+		enc := json.NewEncoder(out)
+		for _, r := range rows {
+			_ = enc.Encode(r)
+		}
+		log.Printf("wrote %d rows to %s; fill in \"expected\" then re-run without -label-out", len(rows), *labelOut)
+		return
+	}
+	scored, correct := 0, 0
+	confusion := map[[2]string]int{}
+	for _, r := range rows {
+		if r.Expected == "" {
+			continue
+		}
+		scored++
+		if r.Expected == r.Predicted {
+			correct++
+		} else {
+			confusion[[2]string{r.Expected, r.Predicted}]++
+			fmt.Printf("miss  expected=%-24s got=%-24s rule=%-18s %s %d %s %.80s\n", r.Expected, r.Predicted, r.Rule, r.Service, r.StatusCode, r.Route, r.Message)
+		}
+	}
+	if scored == 0 {
+		log.Fatal("no labelled rows (set \"expected\")")
+	}
+	fmt.Printf("\naccuracy %d/%d = %.0f%%\n", correct, scored, 100*float64(correct)/float64(scored))
+	keys := make([][2]string, 0, len(confusion))
+	for k := range confusion {
+		keys = append(keys, k)
+	}
+	sort.Slice(keys, func(i, j int) bool { return confusion[keys[i]] > confusion[keys[j]] })
+	for _, k := range keys {
+		fmt.Printf("  %-24s -> %-24s x%d\n", k[0], k[1], confusion[k])
+	}
+}