Merge branch after pulling latest changes

Resolved merge conflict in IssuesVisualizer.tsx by accepting incoming changes which include Needle 3 triage feature and updated UI layout.

.gitignore

@@ -1,4 +1,4 @@
-./server
+/server
 .DS_Store
 otel-agent.log
 ./semgrep-results.json

build.sh

@@ -4,12 +4,18 @@ set -euo pipefail
 RELEASE="${RELEASE:-$(git rev-parse --short HEAD)}"
 PUBLIC_SOURCEMAPS="${PUBLIC_SOURCEMAPS:-false}"
 
+echo "Building Needle 3 sidecar image..."
+docker build -t matst80/needle-sidecar:latest cmd/needle-sidecar
+docker push matst80/needle-sidecar:latest
+
+echo "Building otel-debug receiver image..."
 docker build \
   --build-arg "APP_RELEASE=${RELEASE}" \
   --build-arg "PUBLIC_SOURCEMAPS=${PUBLIC_SOURCEMAPS}" \
   -t matst80/otel-debug:latest .
 
 docker push matst80/otel-debug:latest
+kubectl apply -f k8s/
 kubectl rollout restart deployment otel-debug-receiver -n monitoring
 kubectl rollout status deployment/otel-debug-receiver -n monitoring
 

cmd/issue-model/main.go

@@ -0,0 +1,240 @@
+// issue-model trains and queries an explainable prototype model for normalized
+// events. It uses deterministic pattern signatures as weak labels until human
+// labels are available.
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"math"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+type event struct {
+	EventID          string `json:"eventId"`
+	Service          string `json:"service"`
+	Kind             string `json:"kind"`
+	Source           string `json:"source"`
+	IsError          uint8  `json:"isError"`
+	StatusClass      string `json:"statusClass,omitempty"`
+	StatusCode       uint16 `json:"statusCode,omitempty"`
+	Route            string `json:"route,omitempty"`
+	SpanName         string `json:"spanName,omitempty"`
+	MetricName       string `json:"metricName,omitempty"`
+	MetricType       string `json:"metricType,omitempty"`
+	ExceptionType    string `json:"exceptionType,omitempty"`
+	Message          string `json:"messageNormalized,omitempty"`
+	StackSignature   string `json:"stackSignature,omitempty"`
+	PatternSignature string `json:"patternSignature"`
+}
+
+type prototype struct {
+	Pattern string             `json:"patternSignature"`
+	Count   int                `json:"count"`
+	Tokens  map[string]float64 `json:"tokens"`
+}
+
+type model struct {
+	Version     string      `json:"version"`
+	FeatureSpec string      `json:"featureSpec"`
+	Prototypes  []prototype `json:"prototypes"`
+}
+
+type prediction struct {
+	EventID       string   `json:"eventId"`
+	ActualPattern string   `json:"actualPattern,omitempty"`
+	Pattern       string   `json:"patternSignature"`
+	Score         float64  `json:"score"`
+	Matches       []string `json:"matches"`
+	Rank          int      `json:"rank"`
+	Correct       bool     `json:"correct,omitempty"`
+	Accepted      bool     `json:"accepted"`
+}
+
+func main() {
+	trainPath := flag.String("train", "", "JSONL training sample")
+	modelPath := flag.String("model", "/tmp/issue-model-v001.json", "model path")
+	predictPath := flag.String("predict", "", "JSONL events to score")
+	top := flag.Int("top", 3, "number of predictions per event")
+	minScore := flag.Float64("min-score", 0.75, "minimum top similarity to accept a match")
+	minMargin := flag.Float64("min-margin", 0.05, "minimum gap between first and second match")
+	flag.Parse()
+	if *trainPath == "" && *predictPath == "" {
+		log.Fatal("provide -train or -predict")
+	}
+	if *trainPath != "" {
+		trained := train(*trainPath)
+		writeJSON(*modelPath, trained)
+		fmt.Printf("trained prototypes=%d model=%s\n", len(trained.Prototypes), *modelPath)
+	}
+	if *predictPath != "" {
+		var trained model
+		readJSON(*modelPath, &trained)
+		predict(*predictPath, trained, *top, *minScore, *minMargin)
+	}
+}
+
+func train(path string) model {
+	groups := map[string]*prototype{}
+	forEachEvent(path, func(item event) {
+		if item.PatternSignature == "" {
+			return
+		}
+		p := groups[item.PatternSignature]
+		if p == nil {
+			p = &prototype{Pattern: item.PatternSignature, Tokens: map[string]float64{}}
+			groups[item.PatternSignature] = p
+		}
+		p.Count++
+		for token := range features(item) {
+			p.Tokens[token]++
+		}
+	})
+	result := model{Version: "v1", FeatureSpec: "categorical tokens excluding IDs, timestamps, trace IDs, IPs, and raw values"}
+	for _, p := range groups {
+		for token, value := range p.Tokens {
+			p.Tokens[token] = value / float64(p.Count)
+		}
+		result.Prototypes = append(result.Prototypes, *p)
+	}
+	sort.Slice(result.Prototypes, func(i, j int) bool { return result.Prototypes[i].Pattern < result.Prototypes[j].Pattern })
+	return result
+}
+
+func predict(path string, trained model, top int, minScore, minMargin float64) {
+	if top < 1 {
+		top = 1
+	}
+	if len(trained.Prototypes) == 0 {
+		log.Fatal("model contains no prototypes; train it with a non-empty normalized sample")
+	}
+	forEachEvent(path, func(item event) {
+		input := features(item)
+		results := make([]prediction, 0, len(trained.Prototypes))
+		for _, p := range trained.Prototypes {
+			score, matches := similarity(input, p.Tokens)
+			results = append(results, prediction{EventID: item.EventID, Pattern: p.Pattern, Score: score, Matches: matches})
+		}
+		sort.Slice(results, func(i, j int) bool {
+			if results[i].Score != results[j].Score {
+				return results[i].Score > results[j].Score
+			}
+			return results[i].Pattern < results[j].Pattern
+		})
+		if len(results) > top {
+			results = results[:top]
+		}
+		margin := results[0].Score
+		if len(results) > 1 {
+			margin -= results[1].Score
+		}
+		for i := range results {
+			results[i].ActualPattern = item.PatternSignature
+			results[i].Rank = i + 1
+			results[i].Correct = results[i].Pattern == item.PatternSignature
+			results[i].Accepted = results[0].Score >= minScore && margin >= minMargin
+		}
+		for _, result := range results {
+			writeLine(result)
+		}
+	})
+}
+
+func features(item event) map[string]struct{} {
+	result := map[string]struct{}{}
+	add := func(key, value string) {
+		if value != "" {
+			result[key+"="+value] = struct{}{}
+		}
+	}
+	add("service", item.Service)
+	add("kind", item.Kind)
+	add("source", item.Source)
+	add("status", item.StatusClass)
+	if item.StatusCode > 0 {
+		add("statusCode", strconv.Itoa(int(item.StatusCode)))
+	}
+	add("route", item.Route)
+	add("span", item.SpanName)
+	add("metric", item.MetricName)
+	add("metricType", item.MetricType)
+	add("exception", item.ExceptionType)
+	add("message", item.Message)
+	add("stack", item.StackSignature)
+	return result
+}
+
+func similarity(input map[string]struct{}, prototype map[string]float64) (float64, []string) {
+	if len(input) == 0 || len(prototype) == 0 {
+		return 0, nil
+	}
+	var dot, inputNorm, protoNorm float64
+	matches := []string{}
+	for token := range input {
+		inputNorm++
+		if weight, ok := prototype[token]; ok {
+			dot += weight
+			matches = append(matches, token)
+		}
+	}
+	for _, weight := range prototype {
+		protoNorm += weight * weight
+	}
+	if protoNorm == 0 {
+		return 0, matches
+	}
+	score := dot / math.Sqrt(inputNorm*protoNorm)
+	sort.Strings(matches)
+	return score, matches
+}
+
+func forEachEvent(path string, callback func(event)) {
+	file, err := os.Open(path)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer file.Close()
+	scanner := bufio.NewScanner(file)
+	scanner.Buffer(make([]byte, 64*1024), 4*1024*1024)
+	for scanner.Scan() {
+		var item event
+		if err := json.Unmarshal(scanner.Bytes(), &item); err != nil {
+			log.Fatal(err)
+		}
+		callback(item)
+	}
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+}
+func writeJSON(path string, value any) {
+	data, err := json.MarshalIndent(value, "", "  ")
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err := os.WriteFile(path, data, 0600); err != nil {
+		log.Fatal(err)
+	}
+}
+func readJSON(path string, value any) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err := json.Unmarshal(data, value); err != nil {
+		log.Fatal(err)
+	}
+}
+func writeLine(value any) {
+	data, err := json.Marshal(value)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(strings.TrimSpace(string(data)))
+}

cmd/needle-sidecar/Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.12-slim
+
+# Create non-root user with UID 1000 matching Kubernetes pod securityContext
+RUN groupadd -g 1000 appuser && useradd -u 1000 -g appuser -m -d /app appuser
+
+WORKDIR /app
+
+ENV HOME=/app
+ENV NEEDLE_PORT=5005
+ENV NEEDLE_HOST=0.0.0.0
+ENV NEEDLE_TELEMETRY=0
+
+# Install dependencies
+RUN pip install --no-cache-dir cactus-needle
+
+# Pre-cache model and shared library into /app/.cache as user 1000
+RUN python -c "import needle; needle.Needle()"
+
+COPY server.py /app/server.py
+
+# Ensure UID 1000 owns /app and all caches
+RUN chown -R appuser:appuser /app
+
+USER appuser
+
+EXPOSE 5005
+
+CMD ["python", "/app/server.py"]

cmd/needle-sidecar/server.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""
+Needle 3 AI Telemetry Triage Sidecar Service
+Accepts OTel incident summaries and uses Cactus-Compute/needle3 to classify and extract actionable insights.
+"""
+
+import os
+import sys
+import json
+import time
+from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
+import needle
+
+PORT = int(os.environ.get("NEEDLE_PORT", 5005))
+HOST = os.environ.get("NEEDLE_HOST", "0.0.0.0")
+
+# --- Define Triage Tools for Needle 3 ---
+@needle.tool
+def report_client_javascript_crash(service: str, message: str, url: str):
+    """Report client-side JavaScript crash or error logged by browser RUM."""
+    return {"action": "client_js_crash", "service": service, "message": message, "url": url}
+
+@needle.tool
+def alert_vulnerability_probe(scanner: str, target: str, status_code: int):
+    """Alert on malicious vulnerability scans or exploit crawlers."""
+    return {"action": "security_alert", "scanner": scanner, "target": target, "status_code": status_code}
+
+@needle.tool
+def record_frontend_web_vital(service: str, metric_name: str, duration_ms: int):
+    """Record RUM performance metrics like TTFB or FCP."""
+    return {"action": "record_vital", "service": service, "metric": metric_name, "ms": duration_ms}
+
+@needle.tool
+def log_missing_resource_404(path: str, host: str):
+    """Log benign 404 Not Found requests for missing pages or git commits."""
+    return {"action": "log_404", "path": path, "host": host}
+
+@needle.tool
+def report_database_error(service: str, table: str, error_message: str):
+    """Report database query errors, deadlocks, or connection pool timeouts."""
+    return {"action": "database_error", "service": service, "table": table, "error": error_message}
+
+@needle.tool
+def report_auth_failure(service: str, user_or_token: str, reason: str):
+    """Report authentication/authorization failure or invalid credentials."""
+    return {"action": "auth_failure", "service": service, "user_or_token": user_or_token, "reason": reason}
+
+@needle.tool
+def escalate_backend_outage(service: str, error_message: str):
+    """Escalate severe backend 5xx crash or unhandled panic to on-call."""
+    return {"action": "escalate_outage", "service": service, "error": error_message}
+
+# Initialize Needle Agent once
+print(f"Loading Needle 3 model (Cactus-Compute/needle3)...")
+t0 = time.perf_counter()
+agent = needle.Needle(tools=[
+    report_client_javascript_crash,
+    alert_vulnerability_probe,
+    record_frontend_web_vital,
+    log_missing_resource_404,
+    report_database_error,
+    report_auth_failure,
+    escalate_backend_outage
+])
+print(f"Needle 3 loaded in {(time.perf_counter() - t0):.2f}s")
+
+
+class NeedleHandler(BaseHTTPRequestHandler):
+    def _send_cors(self):
+        self.send_header("Access-Control-Allow-Origin", "*")
+        self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
+        self.send_header("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Otel-Agent-Token")
+
+    def do_OPTIONS(self):
+        self.send_response(204)
+        self._send_cors()
+        self.end_headers()
+
+    def do_GET(self):
+        if self.path == "/health" or self.path == "/api/needle/status":
+            self.send_response(200)
+            self._send_cors()
+            self.send_header("Content-Type", "application/json")
+            self.end_headers()
+            resp = {
+                "status": "ok",
+                "available": True,
+                "model": "Cactus-Compute/needle3",
+                "tools": [
+                    "client_js_crash",
+                    "security_alert",
+                    "record_vital",
+                    "log_404",
+                    "database_error",
+                    "auth_failure",
+                    "escalate_outage"
+                ]
+            }
+            self.wfile.write(json.dumps(resp).encode("utf-8"))
+        else:
+            self.send_response(404)
+            self._send_cors()
+            self.end_headers()
+
+    def do_POST(self):
+        if self.path in ("/classify", "/api/needle/classify"):
+            content_len = int(self.headers.get("Content-Length", 0))
+            body = self.rfile.read(content_len)
+            try:
+                data = json.loads(body) if body else {}
+            except Exception as e:
+                self.send_response(400)
+                self._send_cors()
+                self.send_header("Content-Type", "application/json")
+                self.end_headers()
+                self.wfile.write(json.dumps({"error": f"Invalid JSON: {e}"}).encode("utf-8"))
+                return
+
+            text = data.get("text", "").strip()
+            # If text is not provided, build from structured fields:
+            if not text:
+                parts = []
+                if data.get("service"):
+                    parts.append(f"Service: {data['service']}")
+                if data.get("kind"):
+                    parts.append(f"Kind: {data['kind']}")
+                if data.get("title"):
+                    parts.append(f"Title: {data['title']}")
+                if data.get("message"):
+                    parts.append(f"Message: {data['message']}")
+                if data.get("url"):
+                    parts.append(f"URL: {data['url']}")
+                if data.get("status_code"):
+                    parts.append(f"Status: {data['status_code']}")
+                text = " | ".join(parts)
+
+            if not text:
+                self.send_response(400)
+                self._send_cors()
+                self.send_header("Content-Type", "application/json")
+                self.end_headers()
+                self.wfile.write(json.dumps({"error": "Missing 'text' or telemetry fields"}).encode("utf-8"))
+                return
+
+            t_start = time.perf_counter()
+            agent.reset()
+            res = agent.run(text, strict=False)
+            latency_ms = (time.perf_counter() - t_start) * 1000
+
+            results = res.get("results", [])
+            first_result = results[0] if results and isinstance(results[0], dict) else {}
+            action = first_result.get("action", "unclassified")
+
+            calls = res.get("function_calls", [])
+            tool_name = calls[0].get("name") if calls else "triage_tool"
+
+            out = {
+                "success": True,
+                "input_text": text,
+                "action": action,
+                "tool": tool_name,
+                "arguments": first_result,
+                "confidence": float(res.get("confidence", 0.0)),
+                "reasoning": res.get("reasoning", ""),
+                "latency_ms": round(latency_ms, 2),
+                "model": "Cactus-Compute/needle3"
+            }
+
+            self.send_response(200)
+            self._send_cors()
+            self.send_header("Content-Type", "application/json")
+            self.end_headers()
+            self.wfile.write(json.dumps(out).encode("utf-8"))
+        else:
+            self.send_response(404)
+            self._send_cors()
+            self.end_headers()
+
+    def log_message(self, format, *args):
+        # Concise logging
+        print(f"[needle-sidecar] {args[0]} {args[1]} -> {args[2]}")
+
+
+def run():
+    server = ThreadingHTTPServer((HOST, PORT), NeedleHandler)
+    print(f"[needle-sidecar] Listening on http://{HOST}:{PORT}")
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        print("\nShutting down needle sidecar...")
+        server.server_close()
+
+
+if __name__ == "__main__":
+    run()

cmd/normalized-backfill/main.go

@@ -0,0 +1,227 @@
+// normalized-backfill previews or persists deterministic normalized events.
+// It is read-only unless -write is explicitly supplied.
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"sort"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/normalization"
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+)
+
+type patternCount struct {
+	Service string
+	Kind    string
+	Pattern string
+	Count   uint64
+}
+
+func main() {
+	addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port")
+	database := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "ClickHouse database")
+	user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "ClickHouse user")
+	password := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "ClickHouse password")
+	limit := flag.Int("limit", 0, "raw envelopes to process; 0 means all")
+	since := flag.String("since", "", "inclusive UTC window start, RFC3339")
+	until := flag.String("until", "", "exclusive UTC window end, RFC3339")
+	batchSize := flag.Int("batch-size", 1000, "normalized events per insert batch")
+	write := flag.Bool("write", false, "create the table and insert normalized events")
+	flag.Parse()
+	if *batchSize < 1 {
+		log.Fatal("-batch-size must be positive")
+	}
+	start, end := parseWindow(*since, *until)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
+	defer cancel()
+	conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{*addr}, Auth: clickhouse.Auth{Database: *database, Username: *user, Password: *password}})
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer conn.Close()
+	if err := conn.Ping(ctx); err != nil {
+		log.Fatal(err)
+	}
+	if *write {
+		if err := createSchema(ctx, conn, *database); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	whereSQL := ""
+	if !start.IsZero() {
+		whereSQL = fmt.Sprintf(" timestamp >= toDateTime64('%s', 9, 'UTC')", start.Format("2006-01-02 15:04:05.000000000"))
+	}
+	if !end.IsZero() {
+		if whereSQL != "" {
+			whereSQL += " AND"
+		}
+		whereSQL += fmt.Sprintf(" timestamp < toDateTime64('%s', 9, 'UTC')", end.Format("2006-01-02 15:04:05.000000000"))
+	}
+	if whereSQL != "" {
+		whereSQL = " WHERE" + whereSQL
+	}
+	limitSQL := ""
+	if *limit > 0 {
+		limitSQL = fmt.Sprintf(" LIMIT %d", *limit)
+	}
+	rows, err := conn.Query(ctx, fmt.Sprintf("SELECT timestamp, type, data FROM %s.entries%s%s", *database, whereSQL, limitSQL))
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+
+	counts := map[string]*patternCount{}
+	rawCount, eventCount := 0, 0
+	var batch driver.Batch
+	if *write {
+		batch, err = conn.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.normalized_events", *database))
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+	flush := func() {
+		if batch != nil {
+			if err := batch.Send(); err != nil {
+				log.Fatal(err)
+			}
+			batch = nil
+		}
+	}
+	for rows.Next() {
+		var timestamp time.Time
+		var entryType, data string
+		if err := rows.Scan(&timestamp, &entryType, &data); err != nil {
+			log.Fatal(err)
+		}
+		rawCount++
+		events, err := normalization.NormalizeEntry(entryType, timestamp.UnixNano(), []byte(data))
+		if err != nil {
+			log.Printf("skip %s: %v", entryType, err)
+			continue
+		}
+		for _, event := range events {
+			eventCount++
+			key := event.Service + "\x00" + event.Kind + "\x00" + event.PatternSignature
+			if counts[key] == nil {
+				counts[key] = &patternCount{Service: event.Service, Kind: event.Kind, Pattern: event.PatternSignature}
+			}
+			counts[key].Count++
+			if *write {
+				if err := appendEvent(batch, event); err != nil {
+					log.Fatal(err)
+				}
+				if eventCount%*batchSize == 0 {
+					flush()
+					batch, err = conn.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s.normalized_events", *database))
+					if err != nil {
+						log.Fatal(err)
+					}
+				}
+			}
+		}
+	}
+	if err := rows.Err(); err != nil {
+		log.Fatal(err)
+	}
+	flush()
+	top := make([]*patternCount, 0, len(counts))
+	for _, item := range counts {
+		top = append(top, item)
+	}
+	sort.Slice(top, func(i, j int) bool { return top[i].Count > top[j].Count })
+	if len(top) > 20 {
+		top = top[:20]
+	}
+	fmt.Printf("raw_envelopes=%d normalized_events=%d unique_patterns=%d write=%t\n", rawCount, eventCount, len(top), *write)
+	for _, item := range top {
+		fmt.Printf("pattern_count=%d service=%s kind=%s pattern=%s\n", item.Count, item.Service, item.Kind, item.Pattern)
+	}
+}
+
+func createSchema(ctx context.Context, conn clickhouse.Conn, database string) error {
+	query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.normalized_events (
+		event_id String,
+		entry_type LowCardinality(String),
+		event_index UInt32,
+		ingest_timestamp DateTime64(9, 'UTC'),
+		timestamp DateTime64(9, 'UTC'),
+		service LowCardinality(String), release LowCardinality(String), environment LowCardinality(String),
+		kind LowCardinality(String), source LowCardinality(String), is_error UInt8,
+		status_class LowCardinality(String), status_code UInt16, duration_ms Float64,
+		route String, span_name String, metric_name String, metric_type LowCardinality(String), metric_unit String,
+		metric_value Float64, metric_count UInt64, metric_sum Float64, metric_min Float64, metric_max Float64,
+		exception_type String, message_normalized String, stack_signature String, trace_id String, issue_fingerprint String,
+		is_retry UInt8, is_timeout UInt8, is_cancellation UInt8,
+		exact_signature String, pattern_signature String, version UInt8 DEFAULT 1
+	) ENGINE = ReplacingMergeTree(version)
+	PARTITION BY toYYYYMM(timestamp)
+	ORDER BY (service, kind, pattern_signature, timestamp, event_id)
+	TTL toDateTime(timestamp) + INTERVAL 180 DAY`, database)
+	if err := conn.Exec(ctx, query); err != nil {
+		return err
+	}
+	// Existing developer tables predate the explicit model flags. These
+	// additive migrations preserve historical rows with safe zero defaults.
+	for _, statement := range []string{
+		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS issue_fingerprint String DEFAULT ''", database),
+		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_retry UInt8 DEFAULT 0", database),
+		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_timeout UInt8 DEFAULT 0", database),
+		fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS is_cancellation UInt8 DEFAULT 0", database),
+	} {
+		if err := conn.Exec(ctx, statement); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func appendEvent(batch driver.Batch, event normalization.Event) error {
+	return batch.Append(
+		event.EventID, event.EntryType, event.EventIndex, time.Unix(0, event.IngestTimestamp), time.Unix(0, event.Timestamp),
+		event.Service, event.Release, event.Environment, event.Kind, event.Source, boolUint8(event.IsError),
+		event.StatusClass, event.StatusCode, event.DurationMS, event.Route, event.SpanName, event.MetricName, event.MetricType, event.MetricUnit,
+		event.MetricValue, event.MetricCount, event.MetricSum, event.MetricMin, event.MetricMax,
+		event.ExceptionType, event.MessageNormalized, event.StackSignature, event.TraceID, event.IssueFingerprint,
+		boolUint8(event.IsRetry), boolUint8(event.IsTimeout), boolUint8(event.IsCancellation), event.ExactSignature, event.PatternSignature,
+		uint8(1),
+	)
+}
+
+func boolUint8(value bool) uint8 {
+	if value {
+		return 1
+	}
+	return 0
+}
+func envOr(key, fallback string) string {
+	if value := os.Getenv(key); value != "" {
+		return value
+	}
+	return fallback
+}
+
+func parseWindow(since, until string) (time.Time, time.Time) {
+	parse := func(value, name string) time.Time {
+		if value == "" {
+			return time.Time{}
+		}
+		parsed, err := time.Parse(time.RFC3339Nano, value)
+		if err != nil {
+			log.Fatalf("invalid -%s %q: %v", name, value, err)
+		}
+		return parsed.UTC()
+	}
+	start, end := parse(since, "since"), parse(until, "until")
+	if !start.IsZero() && !end.IsZero() && !start.Before(end) {
+		log.Fatal("-since must be before -until")
+	}
+	return start, end
+}

cmd/normalized-preview/main.go

@@ -0,0 +1,199 @@
+// normalized-preview reads a small sample from ClickHouse and prints the
+// normalized rows without creating tables or writing any data.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/normalization"
+	"github.com/ClickHouse/clickhouse-go/v2"
+)
+
+func main() {
+	addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port")
+	database := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "ClickHouse database")
+	user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "ClickHouse user")
+	password := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "ClickHouse password")
+	limit := flag.Int("limit", 10, "raw envelopes to preview")
+	typeFilter := flag.String("type", "", "only preview one raw type: traces, logs, or metrics")
+	httpMode := flag.Bool("http", false, "use ClickHouse HTTP instead of native protocol")
+	summary := flag.Bool("summary", false, "aggregate by service, kind, and pattern instead of printing events")
+	flag.Parse()
+	if *httpMode {
+		previewHTTP(*addr, *database, *user, *password, *limit, *typeFilter, *summary)
+		return
+	}
+
+	conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{*addr}, Auth: clickhouse.Auth{Database: *database, Username: *user, Password: *password}})
+	if err != nil {
+		log.Fatalf("clickhouse open: %v", err)
+	}
+	defer conn.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	if err := conn.Ping(ctx); err != nil {
+		log.Fatalf("clickhouse ping: %v", err)
+	}
+	if *typeFilter != "" && *typeFilter != "traces" && *typeFilter != "logs" && *typeFilter != "metrics" {
+		log.Fatalf("invalid -type %q", *typeFilter)
+	}
+	whereSQL := ""
+	if *typeFilter != "" {
+		whereSQL = fmt.Sprintf(" WHERE type = '%s'", *typeFilter)
+	}
+	rows, err := conn.Query(ctx, fmt.Sprintf("SELECT timestamp, type, data FROM %s.entries%s ORDER BY timestamp DESC LIMIT ?", *database, whereSQL), *limit)
+	if err != nil {
+		log.Fatalf("query entries: %v", err)
+	}
+	defer rows.Close()
+	encoder := json.NewEncoder(os.Stdout)
+	groups := map[string]*group{}
+	count := 0
+	for rows.Next() {
+		var timestamp time.Time
+		var entryType, data string
+		if err := rows.Scan(&timestamp, &entryType, &data); err != nil {
+			log.Fatalf("scan entry: %v", err)
+		}
+		events, err := normalization.NormalizeEntry(entryType, timestamp.UnixNano(), []byte(data))
+		if err != nil {
+			log.Printf("skip %s at %s: %v", entryType, timestamp.Format(time.RFC3339), err)
+			continue
+		}
+		for _, event := range events {
+			if *summary {
+				addGroup(groups, event)
+			} else if err := encoder.Encode(event); err != nil {
+				log.Fatal(err)
+			}
+			count++
+		}
+	}
+	if err := rows.Err(); err != nil {
+		log.Fatal(err)
+	}
+	if *summary {
+		printGroups(encoder, groups)
+	}
+	log.Printf("previewed %d normalized events from %d raw envelopes", count, *limit)
+}
+
+func previewHTTP(addr, database, user, password string, limit int, typeFilter string, summary bool) {
+	limitSQL := ""
+	orderSQL := " ORDER BY timestamp DESC"
+	whereSQL := ""
+	if typeFilter != "" {
+		if typeFilter != "traces" && typeFilter != "logs" && typeFilter != "metrics" {
+			log.Fatalf("invalid -type %q", typeFilter)
+		}
+		whereSQL = fmt.Sprintf(" WHERE type = '%s'", typeFilter)
+	}
+	if limit > 0 {
+		limitSQL = fmt.Sprintf(" LIMIT %d", limit)
+	} else {
+		// A full dry run must stream the table. Sorting every raw payload first
+		// can consume more memory than the ClickHouse query limit allows.
+		orderSQL = ""
+	}
+	query := fmt.Sprintf("SELECT toUnixTimestamp64Nano(timestamp) AS timestamp, type, data FROM %s.entries%s%s%s FORMAT JSONEachRow", database, whereSQL, orderSQL, limitSQL)
+	req, err := http.NewRequest(http.MethodGet, "http://"+addr+"/?query="+url.QueryEscape(query), nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+	req.SetBasicAuth(user, password)
+	response, err := http.DefaultClient.Do(req)
+	if err != nil {
+		log.Fatalf("clickhouse HTTP request: %v", err)
+	}
+	defer response.Body.Close()
+	if response.StatusCode/100 != 2 {
+		body, _ := io.ReadAll(response.Body)
+		log.Fatalf("clickhouse HTTP %s: %s", response.Status, body)
+	}
+	decoder := json.NewDecoder(response.Body)
+	encoder := json.NewEncoder(os.Stdout)
+	groups := map[string]*group{}
+	rawCount, eventCount := 0, 0
+	for {
+		var row struct {
+			Timestamp json.RawMessage `json:"timestamp"`
+			Type      string          `json:"type"`
+			Data      string          `json:"data"`
+		}
+		if err := decoder.Decode(&row); err == io.EOF {
+			break
+		} else if err != nil {
+			log.Fatal(err)
+		}
+		rawCount++
+		timestamp := parseTimestamp(row.Timestamp)
+		events, err := normalization.NormalizeEntry(row.Type, timestamp, []byte(row.Data))
+		if err != nil {
+			log.Printf("skip %s: %v", row.Type, err)
+			continue
+		}
+		for _, event := range events {
+			if summary {
+				addGroup(groups, event)
+			} else if err := encoder.Encode(event); err != nil {
+				log.Fatal(err)
+			}
+			eventCount++
+		}
+	}
+	if summary {
+		printGroups(encoder, groups)
+	}
+	log.Printf("previewed %d normalized events from %d raw envelopes", eventCount, rawCount)
+}
+
+type group struct {
+	Service          string `json:"service"`
+	Kind             string `json:"kind"`
+	PatternSignature string `json:"patternSignature"`
+	Count            int    `json:"count"`
+}
+
+func addGroup(groups map[string]*group, event normalization.Event) {
+	key := event.Service + "\x00" + event.Kind + "\x00" + event.PatternSignature
+	if groups[key] == nil {
+		groups[key] = &group{Service: event.Service, Kind: event.Kind, PatternSignature: event.PatternSignature}
+	}
+	groups[key].Count++
+}
+
+func printGroups(encoder *json.Encoder, groups map[string]*group) {
+	for _, item := range groups {
+		if err := encoder.Encode(item); err != nil {
+			log.Fatal(err)
+		}
+	}
+}
+
+func parseTimestamp(raw json.RawMessage) int64 {
+	var text string
+	if json.Unmarshal(raw, &text) == nil {
+		value, _ := strconv.ParseInt(text, 10, 64)
+		return value
+	}
+	var value int64
+	_ = json.Unmarshal(raw, &value)
+	return value
+}
+
+func envOr(key, fallback string) string {
+	if value := os.Getenv(key); value != "" {
+		return value
+	}
+	return fallback
+}

cmd/server/main.go

@@ -293,6 +293,8 @@ func main() {
 	// Source-map uploads are intentionally public so the release build script
 	// can upload maps without an OIDC session or bearer token.
 	uiMux.HandleFunc("POST /api/sourcemaps", coll.HandleSourceMap)
+	uiMux.HandleFunc("GET /api/needle/status", coll.HandleNeedleStatus)
+	uiMux.HandleFunc("POST /api/needle/classify", coll.HandleNeedleClassify)
 
 	// Protected API & WebSocket routes
 	uiMux.Handle("/api/history", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleHistory)))
@@ -303,6 +305,7 @@ func main() {
 	uiMux.Handle("/api/history/wait", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleWait)))
 	uiMux.Handle("GET /api/traces/{traceId}", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleTraceByID)))
 	uiMux.Handle("/api/issues", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleIssues)))
+	uiMux.Handle("POST /api/ml/test-spike", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleMLDetectorTest)))
 	uiMux.Handle("GET /api/issues/{id}", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleIssueByID)))
 	uiMux.Handle("GET /api/issues/{id}/similar", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleIssueSimilar)))
 	uiMux.Handle("POST /api/issues/{id}/similar", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleSimilarityFeedback)))
@@ -357,9 +360,13 @@ func main() {
 		}
 	}()
 
+	httpPort := os.Getenv("PORT")
+	if httpPort == "" {
+		httpPort = "8080"
+	}
 	uiHandler := honeypotTracker.Middleware(uiMux)
-	if err := http.ListenAndServe(":8080", uiHandler); err != nil {
+	if err := http.ListenAndServe(":"+httpPort, uiHandler); err != nil {
-		log.Fatalf("failed to start HTTP server: %v", err)
+		log.Fatalf("failed to start HTTP server on :%s: %v", httpPort, err)
 	}
 }
 func spaHandler(root http.FileSystem) http.Handler {

cmd/training-sample/main.go

@@ -0,0 +1,130 @@
+// training-sample exports a bounded, time-windowed JSONL dataset from
+// normalized_events. It never writes to ClickHouse.
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+)
+
+type sample struct {
+	EventID          string    `json:"eventId"`
+	Timestamp        time.Time `json:"timestamp"`
+	Service          string    `json:"service"`
+	Release          string    `json:"release,omitempty"`
+	Kind             string    `json:"kind"`
+	Source           string    `json:"source"`
+	IsError          uint8     `json:"isError"`
+	StatusClass      string    `json:"statusClass,omitempty"`
+	StatusCode       uint16    `json:"statusCode,omitempty"`
+	DurationMS       float64   `json:"durationMs,omitempty"`
+	Route            string    `json:"route,omitempty"`
+	SpanName         string    `json:"spanName,omitempty"`
+	MetricName       string    `json:"metricName,omitempty"`
+	MetricType       string    `json:"metricType,omitempty"`
+	MetricValue      float64   `json:"metricValue,omitempty"`
+	MetricCount      uint64    `json:"metricCount,omitempty"`
+	MetricSum        float64   `json:"metricSum,omitempty"`
+	MetricMin        float64   `json:"metricMin,omitempty"`
+	MetricMax        float64   `json:"metricMax,omitempty"`
+	ExceptionType    string    `json:"exceptionType,omitempty"`
+	Message          string    `json:"messageNormalized,omitempty"`
+	StackSignature   string    `json:"stackSignature,omitempty"`
+	TraceID          string    `json:"traceId,omitempty"`
+	IssueFingerprint string    `json:"issueFingerprint,omitempty"`
+	IsRetry          uint8     `json:"isRetry"`
+	IsTimeout        uint8     `json:"isTimeout"`
+	IsCancellation   uint8     `json:"isCancellation"`
+	ExactSignature   string    `json:"exactSignature"`
+	PatternSignature string    `json:"patternSignature"`
+}
+
+func main() {
+	addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port")
+	database := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "ClickHouse database")
+	user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "ClickHouse user")
+	password := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "ClickHouse password")
+	since := flag.String("since", "", "inclusive UTC RFC3339 timestamp")
+	until := flag.String("until", "", "exclusive UTC RFC3339 timestamp")
+	maxPerPattern := flag.Int("max-per-pattern", 25, "maximum examples per pattern")
+	maxEvents := flag.Int("max-events", 10000, "maximum exported events")
+	output := flag.String("output", "", "write JSONL to this path instead of stdout")
+	flag.Parse()
+	if *since == "" || *until == "" || *maxPerPattern < 1 || *maxEvents < 1 {
+		log.Fatal("-since, -until, -max-per-pattern, and -max-events are required and must be valid")
+	}
+	start, err := time.Parse(time.RFC3339Nano, *since)
+	if err != nil {
+		log.Fatal(err)
+	}
+	end, err := time.Parse(time.RFC3339Nano, *until)
+	if err != nil || !start.Before(end) {
+		log.Fatal("invalid training window")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	defer cancel()
+	conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{*addr}, Auth: clickhouse.Auth{Database: *database, Username: *user, Password: *password}})
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer conn.Close()
+	if err := conn.Ping(ctx); err != nil {
+		log.Fatal(err)
+	}
+	writer := os.Stdout
+	if *output != "" {
+		file, err := os.Create(*output)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer file.Close()
+		writer = file
+	}
+	query := fmt.Sprintf(`SELECT event_id, timestamp, service, release, kind, source, is_error, status_class, status_code, duration_ms,
+		route, span_name, metric_name, metric_type, metric_value, metric_count, metric_sum, metric_min, metric_max,
+		exception_type, message_normalized, stack_signature, trace_id, issue_fingerprint, is_retry, is_timeout, is_cancellation, exact_signature, pattern_signature
+		FROM %s.normalized_events FINAL
+		WHERE timestamp >= toDateTime64('%s', 9, 'UTC') AND timestamp < toDateTime64('%s', 9, 'UTC')
+		ORDER BY pattern_signature, timestamp
+		LIMIT %d BY pattern_signature
+		LIMIT %d`, *database, clickhouseTime(start), clickhouseTime(end), *maxPerPattern, *maxEvents)
+	rows, err := conn.Query(ctx, query)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+	encoder := json.NewEncoder(writer)
+	count := 0
+	for rows.Next() {
+		var item sample
+		if err := rows.Scan(&item.EventID, &item.Timestamp, &item.Service, &item.Release, &item.Kind, &item.Source, &item.IsError, &item.StatusClass, &item.StatusCode, &item.DurationMS, &item.Route, &item.SpanName, &item.MetricName, &item.MetricType, &item.MetricValue, &item.MetricCount, &item.MetricSum, &item.MetricMin, &item.MetricMax, &item.ExceptionType, &item.Message, &item.StackSignature, &item.TraceID, &item.IssueFingerprint, &item.IsRetry, &item.IsTimeout, &item.IsCancellation, &item.ExactSignature, &item.PatternSignature); err != nil {
+			log.Fatal(err)
+		}
+		if err := encoder.Encode(item); err != nil {
+			log.Fatal(err)
+		}
+		count++
+	}
+	if err := rows.Err(); err != nil {
+		log.Fatal(err)
+	}
+	log.Printf("exported %d training events", count)
+}
+
+func clickhouseTime(value time.Time) string {
+	return value.UTC().Format("2006-01-02 15:04:05.000000000")
+}
+func envOr(key, fallback string) string {
+	if value := os.Getenv(key); value != "" {
+		return value
+	}
+	return fallback
+}

cmd/window-features/main.go

@@ -0,0 +1,104 @@
+// window-features exports fixed-width service/release feature rows and the
+// documented future error-spike labels from a normalized JSONL export.
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/features"
+)
+
+type event struct {
+	Timestamp        time.Time `json:"timestamp"`
+	Service          string    `json:"service"`
+	Release          string    `json:"release"`
+	Kind             string    `json:"kind"`
+	Source           string    `json:"source"`
+	IsError          uint8     `json:"isError"`
+	StatusClass      string    `json:"statusClass"`
+	DurationMS       float64   `json:"durationMs"`
+	TraceID          string    `json:"traceId"`
+	PatternSignature string    `json:"patternSignature"`
+	IssueFingerprint string    `json:"issueFingerprint"`
+	IsRetry          uint8     `json:"isRetry"`
+	IsTimeout        uint8     `json:"isTimeout"`
+	IsCancellation   uint8     `json:"isCancellation"`
+	Message          string    `json:"messageNormalized"`
+	SpanName         string    `json:"spanName"`
+}
+
+func main() {
+	input := flag.String("input", "", "normalized JSONL input")
+	window := flag.Duration("window", 5*time.Minute, "observation window")
+	step := flag.Duration("step", 5*time.Minute, "window start interval")
+	future := flag.Duration("future", 15*time.Minute, "future prediction interval")
+	until := flag.String("until", "", "exclusive source-export watermark, RFC3339; required to label a quiet tail safely")
+	baselineDays := flag.Int("baseline-days", 7, "same-time-of-day trailing baseline days")
+	minBaselineSamples := flag.Int("min-baseline-samples", 7, "minimum nonempty baseline windows required for a label")
+	spikeMultiple := flag.Float64("spike-multiple", 3, "future/baseline error-rate ratio for a positive label")
+	minFutureErrors := flag.Int("min-future-errors", 5, "minimum future errors for a positive label")
+	includeUnknown := flag.Bool("include-unknown-labels", false, "emit rows lacking a complete baseline (default omits them)")
+	flag.Parse()
+	if *input == "" || *window <= 0 || *step <= 0 || *future <= 0 || *baselineDays < 1 || *minBaselineSamples < 1 || *spikeMultiple <= 0 || *minFutureErrors < 1 {
+		log.Fatal("invalid feature configuration")
+	}
+	var observedUntil time.Time
+	if *until != "" {
+		parsed, err := time.Parse(time.RFC3339Nano, *until)
+		if err != nil {
+			log.Fatalf("invalid -until %q: %v", *until, err)
+		}
+		observedUntil = parsed.UTC()
+	}
+
+	items := readEvents(*input)
+	rows := features.Build(items, features.Config{
+		Window: *window, Step: *step, Future: *future, ObservedUntil: observedUntil, BaselineDays: *baselineDays,
+		MinBaselineSamples: *minBaselineSamples, SpikeMultiple: *spikeMultiple, MinFutureErrors: *minFutureErrors,
+	})
+	encoder := json.NewEncoder(os.Stdout)
+	count := 0
+	for _, row := range rows {
+		if !*includeUnknown && !row.LabelKnown {
+			continue
+		}
+		if err := encoder.Encode(row); err != nil {
+			log.Fatal(err)
+		}
+		count++
+	}
+	fmt.Fprintf(os.Stderr, "built %d feature rows from %d events\n", count, len(items))
+}
+
+func readEvents(path string) []features.Event {
+	file, err := os.Open(path)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer file.Close()
+	result := []features.Event{}
+	scanner := bufio.NewScanner(file)
+	scanner.Buffer(make([]byte, 64*1024), 4*1024*1024)
+	for scanner.Scan() {
+		var item event
+		if err := json.Unmarshal(scanner.Bytes(), &item); err != nil {
+			log.Fatal(err)
+		}
+		result = append(result, features.Event{
+			Timestamp: item.Timestamp, Service: item.Service, Release: item.Release, Kind: item.Kind, Source: item.Source,
+			IsError: item.IsError != 0, StatusClass: item.StatusClass, DurationMS: item.DurationMS, TraceID: item.TraceID,
+			PatternSignature: item.PatternSignature, IssueFingerprint: item.IssueFingerprint, IsRetry: item.IsRetry != 0,
+			IsTimeout: item.IsTimeout != 0, IsCancellation: item.IsCancellation != 0, Message: item.Message, SpanName: item.SpanName,
+		})
+	}
+	if err := scanner.Err(); err != nil {
+		log.Fatal(err)
+	}
+	return result
+}

docs/ML.md

@@ -1,5 +1,86 @@
 # Error detection model
 
+## Current handoff (2026-09-18)
+
+The first iteration is intentionally explainable and deterministic. Raw OTLP
+events remain the source of truth; normalized rows are stored in
+`otel.normalized_events` with deterministic `event_id`, `exact_signature`, and
+`pattern_signature` values.
+
+Implemented commands:
+
+```bash
+# Read-only normalization summary for a bounded raw window.
+go run ./cmd/normalized-backfill \
+  -since 2026-09-18T17:00:00Z \
+  -until 2026-09-18T18:00:00Z
+
+# Explicit normalized-table write; use one-hour windows.
+go run ./cmd/normalized-backfill \
+  -since 2026-09-18T17:00:00Z \
+  -until 2026-09-18T18:00:00Z -write
+
+# Export a capped training sample.
+go run ./cmd/training-sample \
+  -since 2026-09-18T17:00:00Z \
+  -until 2026-09-18T22:00:00Z \
+  -max-per-pattern 10 -max-events 1000 \
+  -output /tmp/training-v001.jsonl
+
+# Train and score the explainable prototype model.
+go run ./cmd/issue-model -train /tmp/training-v001.jsonl \
+  -model /tmp/issue-model-v001.json
+go run ./cmd/issue-model -model /tmp/issue-model-v001.json \
+  -predict /tmp/sample.jsonl -min-score 0.75 -min-margin 0.05
+
+# Inspect rolling features (the five-hour sample is too short to label).
+go run ./cmd/window-features -input /tmp/training-v001.jsonl \
+  -window 5m -step 5m -future 15m \
+  -until 2026-09-18T22:00:00Z -include-unknown-labels
+```
+
+`window-features` emits one fixed-width row per service/release/window. It
+includes counts, error rate, span duration percentiles, kind and HTTP-status
+counts, unique trace and issue counts, frontend/backend error counts, and
+retry/timeout/cancellation counts. Its label is generated from the following
+fifteen minutes and a same-time-of-day trailing baseline. Rows without seven
+available baseline days or a complete future interval are withheld by default;
+pass `-include-unknown-labels` only for inspection, never as training labels.
+Supply the source export's exclusive `-until` watermark so a quiet final
+interval can be labelled without relying on the timestamp of its last event.
+
+For fast candidate/persistence analysis, override the observation and step
+durations explicitly (for example `-window 15s -step 5s`). Those short rows
+are not substitutes for the documented five-minute spike-prediction label.
+
+The current prototype was trained on 17:00–22:00 UTC and tested on a separate
+05:00–06:00 UTC sample. It achieved 45% top-1 accuracy against deterministic
+weak labels. This is a pipeline check, not a production model evaluation.
+
+The default similarity gate is provisional: score `>= 0.75` and top/runner-up
+margin `>= 0.05`. Recalibrate it after human labels are available; select the
+threshold for the desired precision, not by optimizing accuracy.
+
+Use 15 seconds with a 5-second step as the fast window for the observed
+five-second polling pattern. Add 30-second confirmation and five-minute trend
+windows. A single event is a candidate; persistence across windows makes it
+an issue.
+
+## Next session
+
+1. Export a clean five-hour JSONL sample and review 20–50 predictions.
+2. Add `same`, `different`, and `unknown/noise` labels in the UI.
+3. Add persistence and rollout proximity to the window aggregates; count,
+   error count/rate, unique traces, issue counts, latency, sources and status
+   classes are already exported.
+4. Re-evaluate the threshold on reviewed labels.
+5. Keep the model in shadow mode until it beats the deterministic baseline.
+
+Do not run an unrestricted backfill from a workstation port-forward. Use
+one-hour windows or an authenticated in-cluster job. The normalized table has
+some smoke-test rows; always select training data by an explicit timestamp
+window.
+
 ## Current decision
 
 Keep the original OTLP payloads as the source of truth, but derive a stable
@@ -40,6 +121,11 @@ PII-bearing fields such as raw messages, URLs, user agents, source IPs, and
 stack traces must not be model features. They can remain in the raw history and
 issue views subject to the existing retention and access controls.
 
+The normalized row now carries the canonical `issue_fingerprint` for errors,
+plus structured `is_retry`, `is_timeout`, and `is_cancellation` flags. The
+flags are derived from OTLP attributes and protocol status—not message text—so
+the corresponding aggregate features do not depend on raw user content.
+
 ## First aggregate feature row
 
 For each service/release/window, derive:
@@ -58,6 +144,15 @@ The model input should be numeric and fixed-width. Categorical service and
 release values can initially be filtered per service; one global model can be
 considered after there is enough multi-service data.
 
+## UI verification
+
+The **Issues** view has a **Test spike detector** button. It calls the
+authenticated `POST /api/ml/test-spike` endpoint, which creates no telemetry
+and writes nothing to storage. It runs a deterministic scenario through the
+same feature and label builder: seven daily 10% baseline windows followed by a
+future interval containing five errors in ten events. A passing result shows
+the baseline, future counts/rate, and the positive spike label in the UI.
+
 ## Finding similar errors and spans
 
 Similarity should be a separate concern from the primary issue fingerprint.

index.html

@@ -141,9 +141,45 @@
           Clear Logs
         </button>
       </div>
+
+      <div class="data-section" style="border-left: 4px solid #10b981;">
+        <h2 style="border-bottom-color: #10b981; color: #065f46;">Needle 3 AI Telemetry Triage</h2>
+        <p style="font-size: 13px; color: #666; margin-top: 0;">Test on-device telemetry classification with Cactus-Compute/needle3:</p>
+        <div style="display: flex; gap: 8px; margin-bottom: 10px;">
+          <input id="needle-input" type="text" style="flex: 1; padding: 8px 12px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;" value="User reported client browser crash on service eventsourced-ai-web: Script error on url https://event-agent.k6n.net/#session=angry-koala-1" />
+          <button id="needle-btn" style="background: #10b981; color: white; border: none; padding: 8px 16px; border-radius: 4px; font-weight: bold; cursor: pointer;" onclick="classifyWithNeedle()">
+            Classify with Needle 3
+          </button>
+        </div>
+        <div id="needle-result" class="data-content" style="background: #ecfdf5; border: 1px solid #a7f3d0; min-height: 40px; color: #064e3b;">
+          Click "Classify with Needle 3" to test the sidecar...
+        </div>
+      </div>
     </div>
 
     <script>
+      async function classifyWithNeedle() {
+        const input = document.getElementById("needle-input").value;
+        const resultDiv = document.getElementById("needle-result");
+        const btn = document.getElementById("needle-btn");
+        if (!input.trim()) return;
+        btn.textContent = "Classifying...";
+        btn.disabled = true;
+        try {
+          const resp = await fetch("/api/needle/classify", {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ text: input }),
+          });
+          const data = await resp.json();
+          resultDiv.innerHTML = `<pre style="margin: 0; font-family: monospace;">${JSON.stringify(data, null, 2)}</pre>`;
+        } catch (err) {
+          resultDiv.textContent = "Error: " + err;
+        } finally {
+          btn.textContent = "Classify with Needle 3";
+          btn.disabled = false;
+        }
+      }
       let ws;
       let tracesCount = 0;
       let metricsCount = 0;

internal/collector/collector_test.go

@@ -392,6 +392,31 @@ func TestHandleIssuesAndByID(t *testing.T) {
 	}
 }
 
+func TestHandleMLDetectorTest(t *testing.T) {
+	coll := New(nil, 10, nil)
+	req := httptest.NewRequest(http.MethodPost, "/api/ml/test-spike", nil)
+	rec := httptest.NewRecorder()
+	coll.HandleMLDetectorTest(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("expected detector test success, got %d: %s", rec.Code, rec.Body.String())
+	}
+	var result struct {
+		Passed bool `json:"passed"`
+		Row    struct {
+			LabelKnown        bool    `json:"labelKnown"`
+			Label             bool    `json:"label"`
+			BaselineErrorRate float64 `json:"baselineErrorRate"`
+			FutureErrorCount  int     `json:"futureErrorCount"`
+		} `json:"row"`
+	}
+	if err := json.NewDecoder(rec.Body).Decode(&result); err != nil {
+		t.Fatal(err)
+	}
+	if !result.Passed || !result.Row.LabelKnown || !result.Row.Label || result.Row.BaselineErrorRate != .1 || result.Row.FutureErrorCount != 5 {
+		t.Fatalf("unexpected detector verification response: %+v", result)
+	}
+}
+
 func TestHandleIssuesFiltersAndPaginates(t *testing.T) {
 	coll := New(noopBroadcaster{}, 10, nil)
 	coll.issues.Record("span_error", "nginx", "GET /.env", issues.Sample{Timestamp: 300, SourceIP: "203.0.113.10", HTTPRoute: "/.env"})

internal/collector/handlers.go

@@ -10,10 +10,12 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"os"
 	"strconv"
 	"strings"
 	"time"
 
+	"git.tornberg.me/go-otel/internal/features"
 	"git.tornberg.me/go-otel/internal/issues"
 	"git.tornberg.me/go-otel/internal/storage"
 	"git.tornberg.me/go-otel/internal/websocket"
@@ -669,6 +671,41 @@ func (c *Collector) HandleHealth(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// HandleMLDetectorTest runs a deterministic, in-memory spike through the
+// production feature and label builder. It stores no telemetry and exists so
+// an operator can verify the detector wiring from the authenticated UI.
+func (c *Collector) HandleMLDetectorTest(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	const service = "ml-verification"
+	base := time.Date(2026, 1, 8, 12, 0, 0, 0, time.UTC)
+	events := make([]features.Event, 0, 100)
+	for day := 1; day <= 7; day++ {
+		for i := 0; i < 10; i++ {
+			events = append(events, features.Event{Timestamp: base.AddDate(0, 0, -day).Add(time.Duration(i) * time.Second), Service: service, Kind: "span", IsError: i == 0, DurationMS: float64(i + 1)})
+		}
+	}
+	for i := 0; i < 10; i++ {
+		events = append(events, features.Event{Timestamp: base.Add(time.Duration(i) * time.Second), Service: service, Release: "verification", Kind: "span", DurationMS: float64(i + 1)})
+		events = append(events, features.Event{Timestamp: base.Add(5 * time.Minute).Add(time.Duration(i) * time.Second), Service: service, Release: "verification", Kind: "log", IsError: i < 5, IssueFingerprint: "verification-spike"})
+	}
+	rows := features.Build(events, features.Config{ObservedUntil: base.Add(20 * time.Minute)})
+	for _, row := range rows {
+		if row.WindowStart.Equal(base) && row.Service == service && row.Release == "verification" {
+			w.Header().Set("Content-Type", "application/json")
+			_ = json.NewEncoder(w).Encode(map[string]interface{}{
+				"passed":   row.LabelKnown && row.Label,
+				"scenario": "five errors in ten future events against a seven-day 10% baseline",
+				"row":      row,
+			})
+			return
+		}
+	}
+	jsonError(w, http.StatusInternalServerError, "detector test did not generate its verification row")
+}
+
 // statsEnabled reports whether an aggregate-capable store is configured.
 func (c *Collector) statsEnabled() bool { return c.statsQuerier != nil }
 
@@ -767,3 +804,107 @@ func (c *Collector) HandleStatsServices(w http.ResponseWriter, r *http.Request)
 	w.Header().Set("Content-Type", "application/json")
 	_ = json.NewEncoder(w).Encode(map[string]interface{}{"services": services})
 }
+
+// needleAddr returns the configured or default address for the Needle 3 sidecar.
+func needleAddr() string {
+	addr := os.Getenv("OTEL_NEEDLE_ADDR")
+	if addr == "" {
+		addr = "http://127.0.0.1:5005"
+	}
+	return strings.TrimRight(addr, "/")
+}
+
+// HandleNeedleStatus checks if the Needle 3 sidecar is reachable and reports its health.
+func (c *Collector) HandleNeedleStatus(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	target := needleAddr() + "/health"
+	client := &http.Client{Timeout: 2 * time.Second}
+	resp, err := client.Get(target)
+	w.Header().Set("Content-Type", "application/json")
+	if err != nil {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		_ = json.NewEncoder(w).Encode(map[string]interface{}{
+			"available": false,
+			"error":     fmt.Sprintf("Needle sidecar unreachable at %s: %v", target, err),
+		})
+		return
+	}
+	defer resp.Body.Close()
+	w.WriteHeader(resp.StatusCode)
+	_, _ = io.Copy(w, resp.Body)
+}
+
+// HandleNeedleClassify proxies telemetry triage requests to the Needle 3 sidecar.
+func (c *Collector) HandleNeedleClassify(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	body, err := io.ReadAll(r.Body)
+	if err != nil {
+		http.Error(w, "failed to read request body", http.StatusBadRequest)
+		return
+	}
+
+	// If the payload specifies an issue_id and no text, look up the issue
+	var reqMap map[string]interface{}
+	if err := json.Unmarshal(body, &reqMap); err == nil {
+		if issueID, ok := reqMap["issue_id"].(string); ok && issueID != "" {
+			if issue, ok := c.GetIssue(issueID); ok {
+				var promptParts []string
+				promptParts = append(promptParts, fmt.Sprintf("Service: %s", issue.Service))
+				promptParts = append(promptParts, fmt.Sprintf("Title: %s", issue.Title))
+				promptParts = append(promptParts, fmt.Sprintf("Kind: %s", issue.Kind))
+				if len(issue.Samples) > 0 {
+					latest := issue.Samples[len(issue.Samples)-1]
+					if latest.Message != "" {
+						promptParts = append(promptParts, fmt.Sprintf("Message: %s", latest.Message))
+					}
+					if latest.URL != "" {
+						promptParts = append(promptParts, fmt.Sprintf("URL: %s", latest.URL))
+					}
+					if latest.HTTPStatus > 0 {
+						promptParts = append(promptParts, fmt.Sprintf("Status: %d", latest.HTTPStatus))
+					}
+					if latest.HTTPRoute != "" {
+						promptParts = append(promptParts, fmt.Sprintf("Route: %s", latest.HTTPRoute))
+					}
+				}
+				reqMap["text"] = strings.Join(promptParts, " | ")
+				reqMap["service"] = issue.Service
+				reqMap["title"] = issue.Title
+				body, _ = json.Marshal(reqMap)
+			}
+		}
+	}
+
+	target := needleAddr() + "/classify"
+	client := &http.Client{Timeout: 60 * time.Second}
+	proxyReq, err := http.NewRequestWithContext(r.Context(), http.MethodPost, target, strings.NewReader(string(body)))
+	if err != nil {
+		http.Error(w, "failed to create proxy request", http.StatusInternalServerError)
+		return
+	}
+	proxyReq.Header.Set("Content-Type", "application/json")
+
+	resp, err := client.Do(proxyReq)
+	if err != nil {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusServiceUnavailable)
+		_ = json.NewEncoder(w).Encode(map[string]interface{}{
+			"success": false,
+			"error":   fmt.Sprintf("Needle sidecar unreachable at %s: %v", target, err),
+		})
+		return
+	}
+	defer resp.Body.Close()
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(resp.StatusCode)
+	_, _ = io.Copy(w, resp.Body)
+}

internal/collector/issue_extract.go

@@ -87,7 +87,7 @@ func spanSample(span ptrace.Span, res pcommon.Resource, timestamp int64, title s
 		SpanID:      idString(span.SpanID()),
 		Message:     title,
 		HTTPMethod:  firstAttr(span.Attributes(), "http.request.method", "http.method"),
-		HTTPRoute:   firstAttr(span.Attributes(), "http.route", "http.target", "url.path"),
+		HTTPRoute:   httpRoute(span.Attributes()),
 		UserAgent:   firstAttr(span.Attributes(), "user_agent.original", "http.user_agent"),
 		SourceIP:    sourceIP(span.Attributes(), res.Attributes()),
 		Release:     firstAttr(span.Attributes(), "service.release"),
@@ -109,7 +109,7 @@ func logSample(record plog.LogRecord, res pcommon.Resource, timestamp int64, tit
 		Message:     title,
 		Stacktrace:  attrString(record.Attributes(), "exception.stacktrace"),
 		HTTPMethod:  firstAttr(record.Attributes(), "http.request.method", "http.method"),
-		HTTPRoute:   firstAttr(record.Attributes(), "http.route", "http.target", "url.path"),
+		HTTPRoute:   httpRoute(record.Attributes()),
 		UserAgent:   firstAttr(record.Attributes(), "user_agent.original", "http.user_agent", "browser.userAgent"),
 		SourceIP:    sourceIP(record.Attributes(), res.Attributes()),
 		Release:     firstAttr(record.Attributes(), "service.release"),
@@ -179,6 +179,18 @@ func firstAttr(attrs pcommon.Map, keys ...string) string {
 	return ""
 }
 
+// httpRoute prefers the concrete URL path when instrumentation reports "/"
+// as a generic router mount. This keeps API endpoints distinct in issue
+// fingerprints and avoids collapsing unrelated requests into one root route.
+func httpRoute(attrs pcommon.Map) string {
+	route := firstAttr(attrs, "http.route")
+	path := firstAttr(attrs, "url.path", "http.target")
+	if path != "" && (route == "" || route == "/" && path != "/") {
+		return path
+	}
+	return route
+}
+
 // getHTTPStatusCode extracts an HTTP status code from attributes, supporting
 // both new (http.response.status_code) and old (http.status_code) semconv keys,
 // stored as integer or string.

internal/features/features.go

@@ -0,0 +1,328 @@
+// Package features derives fixed-width, non-PII service window features from
+// normalized OTLP events. It deliberately has no ClickHouse dependency so the
+// exact same feature logic can be used for exports, offline evaluation and
+// eventual online scoring.
+package features
+
+import (
+	"sort"
+	"time"
+)
+
+// Event is the subset of a normalized event used by the detector. Text fields
+// are already normalized/redacted by the normalization package.
+type Event struct {
+	Timestamp        time.Time
+	Service          string
+	Release          string
+	Kind             string
+	Source           string
+	IsError          bool
+	StatusClass      string
+	DurationMS       float64
+	TraceID          string
+	PatternSignature string
+	IssueFingerprint string
+	IsRetry          bool
+	IsTimeout        bool
+	IsCancellation   bool
+	Message          string
+	SpanName         string
+}
+
+// Config controls feature and label windows. A zero Config receives the
+// documented five-minute observation / fifteen-minute prediction defaults.
+type Config struct {
+	Window time.Duration
+	Step   time.Duration
+	Future time.Duration
+	// ObservedUntil is the exclusive source-export watermark. Supplying it
+	// lets a quiet tail interval be labelled safely; when absent Build falls
+	// back to the latest event timestamp and is deliberately conservative.
+	ObservedUntil      time.Time
+	BaselineDays       int
+	MinBaselineSamples int
+	SpikeMultiple      float64
+	MinFutureErrors    int
+}
+
+// DefaultConfig is intentionally conservative: labels are withheld until all
+// seven same-time-of-day baseline observations are present.
+func DefaultConfig() Config {
+	return Config{
+		Window:             5 * time.Minute,
+		Step:               5 * time.Minute,
+		Future:             15 * time.Minute,
+		BaselineDays:       7,
+		MinBaselineSamples: 7,
+		SpikeMultiple:      3,
+		MinFutureErrors:    5,
+	}
+}
+
+// Row is a numeric, fixed-width model input. Label is only meaningful when
+// LabelKnown is true; this prevents short exports from silently producing
+// negative training examples without a seven-day history.
+type Row struct {
+	WindowStart time.Time `json:"windowStart"`
+	WindowEnd   time.Time `json:"windowEnd"`
+	Service     string    `json:"service"`
+	Release     string    `json:"release,omitempty"`
+
+	TotalEvents int     `json:"totalEvents"`
+	ErrorCount  int     `json:"errorCount"`
+	ErrorRate   float64 `json:"errorRate"`
+	P50Duration float64 `json:"p50DurationMs"`
+	P95Duration float64 `json:"p95DurationMs"`
+
+	SpanCount   int `json:"spanCount"`
+	LogCount    int `json:"logCount"`
+	MetricCount int `json:"metricCount"`
+	Status2xx   int `json:"status2xx"`
+	Status4xx   int `json:"status4xx"`
+	Status5xx   int `json:"status5xx"`
+
+	UniqueTraceCount  int `json:"uniqueTraceCount"`
+	UniqueIssueCount  int `json:"uniqueIssueCount"`
+	NewIssueCount     int `json:"newIssueCount"`
+	FrontendErrors    int `json:"frontendErrors"`
+	BackendErrors     int `json:"backendErrors"`
+	RetryCount        int `json:"retryCount"`
+	TimeoutCount      int `json:"timeoutCount"`
+	CancellationCount int `json:"cancellationCount"`
+
+	BaselineErrorRate float64 `json:"baselineErrorRate"`
+	BaselineSamples   int     `json:"baselineSamples"`
+	FutureEventCount  int     `json:"futureEventCount"`
+	FutureErrorCount  int     `json:"futureErrorCount"`
+	FutureErrorRate   float64 `json:"futureErrorRate"`
+	LabelKnown        bool    `json:"labelKnown"`
+	Label             bool    `json:"label"`
+}
+
+// Build aggregates every service/release over rolling observation windows.
+// The future target and baseline are service-wide as specified by the model
+// target; release is context for the input rather than a way to split alerts.
+func Build(input []Event, cfg Config) []Row {
+	cfg = normalizedConfig(cfg)
+	if len(input) == 0 {
+		return nil
+	}
+	events := append([]Event(nil), input...)
+	for i := range events {
+		events[i].Timestamp = events[i].Timestamp.UTC()
+		if events[i].Service == "" {
+			events[i].Service = "unknown"
+		}
+	}
+	sort.Slice(events, func(i, j int) bool { return events[i].Timestamp.Before(events[j].Timestamp) })
+	firstIssueSeen := make(map[string]time.Time)
+	for _, e := range events {
+		if !e.IsError || issueKey(e) == "" {
+			continue
+		}
+		key := e.Service + "\x00" + issueKey(e)
+		if seen, ok := firstIssueSeen[key]; !ok || e.Timestamp.Before(seen) {
+			firstIssueSeen[key] = e.Timestamp
+		}
+	}
+
+	first := events[0].Timestamp.Truncate(cfg.Step)
+	last := events[len(events)-1].Timestamp
+	observedUntil := cfg.ObservedUntil.UTC()
+	if observedUntil.IsZero() {
+		observedUntil = last
+	}
+	rows := make([]Row, 0)
+	for start := first; !start.After(last); start = start.Add(cfg.Step) {
+		end := start.Add(cfg.Window)
+		groups := map[string]*accumulator{}
+		for _, e := range events {
+			if e.Timestamp.Before(start) || !e.Timestamp.Before(end) {
+				continue
+			}
+			key := e.Service + "\x00" + e.Release
+			a := groups[key]
+			if a == nil {
+				a = newAccumulator(start, end, e.Service, e.Release)
+				groups[key] = a
+			}
+			a.add(e)
+		}
+		for _, a := range groups {
+			row := a.row()
+			for issue := range a.issues {
+				if seen := firstIssueSeen[row.Service+"\x00"+issue]; !seen.Before(start) && seen.Before(end) {
+					row.NewIssueCount++
+				}
+			}
+			futureTotal, futureErrors := count(events, start.Add(cfg.Window), start.Add(cfg.Window).Add(cfg.Future), row.Service)
+			baselineRate, samples := baseline(events, start, cfg.Window, cfg.BaselineDays, row.Service)
+			row.FutureEventCount, row.FutureErrorCount = futureTotal, futureErrors
+			row.FutureErrorRate = rate(futureErrors, futureTotal)
+			row.BaselineErrorRate, row.BaselineSamples = baselineRate, samples
+			// A target is only known once the input has reached its exclusive
+			// end. This deliberately withholds the tail of an export instead of
+			// treating an incomplete future interval as a negative label.
+			row.LabelKnown = samples >= cfg.MinBaselineSamples && !start.Add(cfg.Window).Add(cfg.Future).After(observedUntil)
+			row.Label = row.LabelKnown && futureErrors >= cfg.MinFutureErrors && row.FutureErrorRate >= cfg.SpikeMultiple*baselineRate
+			rows = append(rows, row)
+		}
+	}
+	sort.Slice(rows, func(i, j int) bool {
+		if !rows[i].WindowStart.Equal(rows[j].WindowStart) {
+			return rows[i].WindowStart.Before(rows[j].WindowStart)
+		}
+		if rows[i].Service != rows[j].Service {
+			return rows[i].Service < rows[j].Service
+		}
+		return rows[i].Release < rows[j].Release
+	})
+	return rows
+}
+
+type accumulator struct {
+	value     Row
+	durations []float64
+	traces    map[string]struct{}
+	issues    map[string]struct{}
+}
+
+func newAccumulator(start, end time.Time, service, release string) *accumulator {
+	return &accumulator{value: Row{WindowStart: start, WindowEnd: end, Service: service, Release: release}, traces: map[string]struct{}{}, issues: map[string]struct{}{}}
+}
+
+func (a *accumulator) add(e Event) {
+	a.value.TotalEvents++
+	switch e.Kind {
+	case "span":
+		a.value.SpanCount++
+	case "log":
+		a.value.LogCount++
+	case "metric":
+		a.value.MetricCount++
+	}
+	switch e.StatusClass {
+	case "2xx":
+		a.value.Status2xx++
+	case "4xx":
+		a.value.Status4xx++
+	case "5xx":
+		a.value.Status5xx++
+	}
+	if e.TraceID != "" {
+		a.traces[e.TraceID] = struct{}{}
+	}
+	if e.Kind == "span" && e.DurationMS >= 0 {
+		a.durations = append(a.durations, e.DurationMS)
+	}
+	if !e.IsError {
+		return
+	}
+	a.value.ErrorCount++
+	if e.Source == "frontend" {
+		a.value.FrontendErrors++
+	} else {
+		a.value.BackendErrors++
+	}
+	if issue := issueKey(e); issue != "" {
+		a.issues[issue] = struct{}{}
+	}
+	if e.IsRetry {
+		a.value.RetryCount++
+	}
+	if e.IsTimeout {
+		a.value.TimeoutCount++
+	}
+	if e.IsCancellation {
+		a.value.CancellationCount++
+	}
+}
+
+func issueKey(e Event) string {
+	if e.IssueFingerprint != "" {
+		return e.IssueFingerprint
+	}
+	return e.PatternSignature
+}
+
+func (a *accumulator) row() Row {
+	a.value.ErrorRate = rate(a.value.ErrorCount, a.value.TotalEvents)
+	a.value.UniqueTraceCount, a.value.UniqueIssueCount = len(a.traces), len(a.issues)
+	if len(a.durations) != 0 {
+		sort.Float64s(a.durations)
+		a.value.P50Duration = percentile(a.durations, .50)
+		a.value.P95Duration = percentile(a.durations, .95)
+	}
+	return a.value
+}
+
+func count(events []Event, start, end time.Time, service string) (total, errors int) {
+	for _, e := range events {
+		if e.Service == service && !e.Timestamp.Before(start) && e.Timestamp.Before(end) {
+			total++
+			if e.IsError {
+				errors++
+			}
+		}
+	}
+	return total, errors
+}
+
+// baseline samples the same UTC time window on the preceding calendar days.
+// All intervals end before the current observation start, so no future data
+// leaks into a training label.
+func baseline(events []Event, start time.Time, window time.Duration, days int, service string) (float64, int) {
+	var total, errors, samples int
+	for day := 1; day <= days; day++ {
+		from := start.AddDate(0, 0, -day)
+		to := from.Add(window)
+		countTotal, countErrors := count(events, from, to, service)
+		if countTotal == 0 {
+			continue
+		}
+		total += countTotal
+		errors += countErrors
+		samples++
+	}
+	return rate(errors, total), samples
+}
+
+func percentile(values []float64, p float64) float64 {
+	if len(values) == 0 {
+		return 0
+	}
+	index := int(p * float64(len(values)-1))
+	return values[index]
+}
+func rate(errors, total int) float64 {
+	if total == 0 {
+		return 0
+	}
+	return float64(errors) / float64(total)
+}
+func normalizedConfig(cfg Config) Config {
+	d := DefaultConfig()
+	if cfg.Window <= 0 {
+		cfg.Window = d.Window
+	}
+	if cfg.Step <= 0 {
+		cfg.Step = d.Step
+	}
+	if cfg.Future <= 0 {
+		cfg.Future = d.Future
+	}
+	if cfg.BaselineDays <= 0 {
+		cfg.BaselineDays = d.BaselineDays
+	}
+	if cfg.MinBaselineSamples <= 0 {
+		cfg.MinBaselineSamples = d.MinBaselineSamples
+	}
+	if cfg.SpikeMultiple <= 0 {
+		cfg.SpikeMultiple = d.SpikeMultiple
+	}
+	if cfg.MinFutureErrors <= 0 {
+		cfg.MinFutureErrors = d.MinFutureErrors
+	}
+	return cfg
+}

internal/features/features_test.go

@@ -0,0 +1,56 @@
+package features
+
+import (
+	"testing"
+	"time"
+)
+
+func TestBuildLabelsSpikeUsingOnlyEarlierSameTimeWindows(t *testing.T) {
+	base := time.Date(2026, 9, 15, 12, 0, 0, 0, time.UTC)
+	events := make([]Event, 0)
+	// The prior two daily windows have 10 events and one error each.
+	for day := 1; day <= 2; day++ {
+		for i := 0; i < 10; i++ {
+			events = append(events, Event{Timestamp: base.AddDate(0, 0, -day).Add(time.Duration(i) * time.Second), Service: "api", Kind: "span", IsError: i == 0, DurationMS: float64(i)})
+		}
+	}
+	// Current row is non-error telemetry. The following fifteen minutes have
+	// five errors out of ten events: 5x the 10% baseline and above the floor.
+	for i := 0; i < 10; i++ {
+		events = append(events, Event{Timestamp: base.Add(time.Duration(i) * time.Second), Service: "api", Release: "v2", Kind: "span", IsError: false, DurationMS: float64(i)})
+	}
+	for i := 0; i < 10; i++ {
+		events = append(events, Event{Timestamp: base.Add(5 * time.Minute).Add(time.Duration(i) * time.Second), Service: "api", Kind: "log", IsError: i < 5})
+	}
+	// The source export's explicit exclusive watermark proves that the rest of
+	// the future interval was observed even though it was quiet.
+	rows := Build(events, Config{Window: 5 * time.Minute, Step: 5 * time.Minute, Future: 15 * time.Minute, ObservedUntil: base.Add(20 * time.Minute), BaselineDays: 2, MinBaselineSamples: 2, SpikeMultiple: 3, MinFutureErrors: 5})
+	var row *Row
+	for i := range rows {
+		if rows[i].WindowStart.Equal(base) && rows[i].Service == "api" && rows[i].Release == "v2" {
+			row = &rows[i]
+			break
+		}
+	}
+	if row == nil {
+		t.Fatal("missing current api/v2 row")
+	}
+	if !row.LabelKnown || !row.Label {
+		t.Fatalf("expected a known positive label, got %+v", *row)
+	}
+	if row.BaselineErrorRate != .1 || row.FutureErrorRate != .5 || row.FutureErrorCount != 5 {
+		t.Fatalf("unexpected label inputs: %+v", *row)
+	}
+	if row.P50Duration != 4 || row.P95Duration != 8 || row.SpanCount != 10 {
+		t.Fatalf("unexpected fixed-width features: %+v", *row)
+	}
+}
+
+func TestBuildWithholdsLabelWithoutCompleteBaseline(t *testing.T) {
+	base := time.Date(2026, 9, 15, 12, 0, 0, 0, time.UTC)
+	events := []Event{{Timestamp: base, Service: "api", Kind: "log"}, {Timestamp: base.Add(20 * time.Minute), Service: "other", Kind: "log"}}
+	rows := Build(events, Config{Window: 5 * time.Minute, Step: 5 * time.Minute, Future: 15 * time.Minute, BaselineDays: 7, MinBaselineSamples: 7})
+	if len(rows) == 0 || rows[0].LabelKnown {
+		t.Fatalf("short history must not be labelled: %+v", rows)
+	}
+}

internal/issues/tracker.go

@@ -153,7 +153,7 @@ func NewTracker() *Tracker {
 // fingerprint.
 func (t *Tracker) Record(kind, service, title string, sample Sample) {
 	normalizeSample(&sample, title)
-	id := fingerprint(kind, service, NormalizeTitle(title))
+	id := Fingerprint(kind, service, NormalizeTitle(title))
 	bucket := HourBucket(sample.Timestamp)
 
 	t.mu.Lock()
@@ -174,7 +174,7 @@ func (t *Tracker) Record(kind, service, title string, sample Sample) {
 			NormalizedTitle: NormalizeTitle(title),
 		}
 		issue.PatternSignature = patternSignature(kind, service, title, sample)
-		issue.PatternID = fingerprint("pattern", issue.PatternSignature)
+		issue.PatternID = Fingerprint("pattern", issue.PatternSignature)
 		t.issues[id] = issue
 	}
 
@@ -653,7 +653,10 @@ func decodeStatus(encoded string) string {
 	return b.String()
 }
 
-func fingerprint(parts ...string) string {
+// Fingerprint creates the stable opaque identifier used for canonical issue
+// grouping. Normalization uses it for error rows so offline feature exports
+// can count the same issue identity as the live issue tracker.
+func Fingerprint(parts ...string) string {
 	h := fnv.New64a()
 	_, _ = h.Write([]byte(strings.Join(parts, "\x00")))
 	return fmt.Sprintf("%016x", h.Sum64())

internal/normalization/normalizer.go

@@ -0,0 +1,447 @@
+// Package normalization converts raw OTLP envelopes into stable, non-PII
+// event rows suitable for aggregation and later model training.
+package normalization
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"git.tornberg.me/go-otel/internal/issues"
+)
+
+const Version = "v1"
+
+type Event struct {
+	EventID           string  `json:"eventId"`
+	EntryType         string  `json:"entryType"`
+	EventIndex        uint32  `json:"eventIndex"`
+	IngestTimestamp   int64   `json:"ingestTimestamp"`
+	Timestamp         int64   `json:"timestamp"`
+	NormalizerVersion string  `json:"normalizerVersion"`
+	Service           string  `json:"service"`
+	Release           string  `json:"release,omitempty"`
+	Environment       string  `json:"environment,omitempty"`
+	Kind              string  `json:"kind"`
+	Source            string  `json:"source"`
+	IsError           bool    `json:"isError"`
+	StatusClass       string  `json:"statusClass,omitempty"`
+	StatusCode        int     `json:"statusCode,omitempty"`
+	DurationMS        float64 `json:"durationMs,omitempty"`
+	Route             string  `json:"route,omitempty"`
+	SpanName          string  `json:"spanName,omitempty"`
+	MetricName        string  `json:"metricName,omitempty"`
+	MetricType        string  `json:"metricType,omitempty"`
+	MetricUnit        string  `json:"metricUnit,omitempty"`
+	MetricValue       float64 `json:"metricValue,omitempty"`
+	MetricCount       uint64  `json:"metricCount,omitempty"`
+	MetricSum         float64 `json:"metricSum,omitempty"`
+	MetricMin         float64 `json:"metricMin,omitempty"`
+	MetricMax         float64 `json:"metricMax,omitempty"`
+	ExceptionType     string  `json:"exceptionType,omitempty"`
+	MessageNormalized string  `json:"messageNormalized,omitempty"`
+	StackSignature    string  `json:"stackSignature,omitempty"`
+	TraceID           string  `json:"traceId,omitempty"`
+	IssueFingerprint  string  `json:"issueFingerprint,omitempty"`
+	IsRetry           bool    `json:"isRetry"`
+	IsTimeout         bool    `json:"isTimeout"`
+	IsCancellation    bool    `json:"isCancellation"`
+	ExactSignature    string  `json:"exactSignature"`
+	PatternSignature  string  `json:"patternSignature"`
+}
+
+// NormalizeEntry expands one raw OTLP envelope into one row per span, log
+// record, or metric datapoint. It never mutates or stores the input payload.
+func NormalizeEntry(entryType string, ingestTimestamp int64, data []byte) ([]Event, error) {
+	var envelope map[string]json.RawMessage
+	if err := json.Unmarshal(data, &envelope); err != nil {
+		return nil, fmt.Errorf("decode OTLP JSON: %w", err)
+	}
+	var events []Event
+	switch entryType {
+	case "traces":
+		events = normalizeTraces(ingestTimestamp, envelope)
+	case "logs":
+		events = normalizeLogs(ingestTimestamp, envelope)
+	case "metrics":
+		events = normalizeMetrics(ingestTimestamp, envelope)
+	default:
+		return nil, fmt.Errorf("unsupported telemetry type %q", entryType)
+	}
+	for i := range events {
+		events[i].EntryType = entryType
+		events[i].EventIndex = uint32(i)
+		events[i].IngestTimestamp = ingestTimestamp
+		events[i].EventID = signature(entryType, strconv.FormatInt(ingestTimestamp, 10), strconv.Itoa(i), events[i].ExactSignature)
+	}
+	return events, nil
+}
+
+func normalizeTraces(ingest int64, envelope map[string]json.RawMessage) []Event {
+	var resources []struct {
+		Resource   object `json:"resource"`
+		ScopeSpans []struct {
+			Spans []span `json:"spans"`
+		} `json:"scopeSpans"`
+	}
+	_ = json.Unmarshal(envelope["resourceSpans"], &resources)
+	result := []Event{}
+	for _, resource := range resources {
+		for _, scope := range resource.ScopeSpans {
+			for _, item := range scope.Spans {
+				attrs := mergeAttrs(resource.Resource.Attributes, item.Attributes)
+				status := intValue(item.Status.Code)
+				statusCode := intAttr(attrs, "http.response.status_code", "http.status_code")
+				route := normalizeHTTPRoute(attrs)
+				message := item.Status.Message
+				exceptionType, exceptionMessage, stack := exception(item.Events)
+				if exceptionMessage != "" {
+					message = exceptionMessage
+				}
+				if message == "" {
+					message = item.Name
+				}
+				if exceptionType != "" && message != "" {
+					message = exceptionType + ": " + message
+				}
+				isError := status == 2 || statusCode >= 400 || intAttr(attrs, "rpc.grpc.status_code") > 0 || exceptionType != ""
+				e := baseEvent(ingest, attrs, "span", item.TraceID)
+				e.IsError, e.StatusCode, e.StatusClass = isError, statusCode, statusClass(statusCode)
+				e.DurationMS = durationMS(item.Start, item.End)
+				e.Route, e.SpanName, e.ExceptionType, e.MessageNormalized, e.StackSignature = route, item.Name, exceptionType, normalizeMessage(message), firstFrame(stack)
+				e.IsRetry, e.IsTimeout, e.IsCancellation = operationFlags(attrs, statusCode, intAttr(attrs, "rpc.grpc.status_code"))
+				result = append(result, finish(e))
+			}
+		}
+	}
+	return result
+}
+
+func normalizeLogs(ingest int64, envelope map[string]json.RawMessage) []Event {
+	var resources []struct {
+		Resource  object `json:"resource"`
+		ScopeLogs []struct {
+			LogRecords []logRecord `json:"logRecords"`
+		} `json:"scopeLogs"`
+	}
+	_ = json.Unmarshal(envelope["resourceLogs"], &resources)
+	result := []Event{}
+	for _, resource := range resources {
+		for _, scope := range resource.ScopeLogs {
+			for _, item := range scope.LogRecords {
+				attrs := mergeAttrs(resource.Resource.Attributes, item.Attributes)
+				message := bodyString(item.Body)
+				exceptionType := stringAttr(attrs, "exception.type")
+				stack := stringAttr(attrs, "exception.stacktrace")
+				statusCode := intAttr(attrs, "http.response.status_code", "http.status_code")
+				grpcCode := intAttr(attrs, "rpc.grpc.status_code")
+				e := baseEvent(timestampOr(ingest, item.Timestamp), attrs, "log", item.TraceID)
+				e.IsError = item.SeverityNumber >= 17 || strings.EqualFold(item.SeverityText, "ERROR") || strings.EqualFold(item.SeverityText, "FATAL") || exceptionType != "" || statusCode >= 400 || grpcCode > 0
+				e.StatusCode, e.StatusClass = statusCode, statusClass(statusCode)
+				e.Route, e.ExceptionType, e.MessageNormalized, e.StackSignature = normalizeHTTPRoute(attrs), exceptionType, normalizeMessage(message), firstFrame(stack)
+				e.IsRetry, e.IsTimeout, e.IsCancellation = operationFlags(attrs, statusCode, grpcCode)
+				result = append(result, finish(e))
+			}
+		}
+	}
+	return result
+}
+
+func normalizeMetrics(ingest int64, envelope map[string]json.RawMessage) []Event {
+	var resources []struct {
+		Resource     object `json:"resource"`
+		ScopeMetrics []struct {
+			Metrics []metric `json:"metrics"`
+		} `json:"scopeMetrics"`
+	}
+	_ = json.Unmarshal(envelope["resourceMetrics"], &resources)
+	result := []Event{}
+	for _, resource := range resources {
+		attrs := resource.Resource.Attributes
+		for _, scope := range resource.ScopeMetrics {
+			for _, item := range scope.Metrics {
+				for _, point := range item.Points() {
+					e := baseEvent(timestampOr(ingest, point.Timestamp), mergeAttrs(attrs, point.Attributes), "metric", "")
+					e.MetricName, e.MetricType, e.MetricUnit = item.Name, point.Type, item.Unit
+					e.MetricValue, e.MetricCount, e.MetricSum = point.Value, point.Count, point.Sum
+					e.MetricMin, e.MetricMax = point.Min, point.Max
+					result = append(result, finish(e))
+				}
+			}
+		}
+	}
+	return result
+}
+
+type object struct {
+	Attributes []attribute `json:"attributes"`
+}
+type attribute struct {
+	Key   string `json:"key"`
+	Value value  `json:"value"`
+}
+type value struct {
+	StringValue string      `json:"stringValue"`
+	IntValue    json.Number `json:"intValue"`
+	DoubleValue float64     `json:"doubleValue"`
+	BoolValue   bool        `json:"boolValue"`
+}
+type span struct {
+	TraceID    string      `json:"traceId"`
+	Name       string      `json:"name"`
+	Start      string      `json:"startTimeUnixNano"`
+	End        string      `json:"endTimeUnixNano"`
+	Attributes []attribute `json:"attributes"`
+	Events     []spanEvent `json:"events"`
+	Status     struct {
+		Code    json.Number `json:"code"`
+		Message string      `json:"message"`
+	} `json:"status"`
+}
+type spanEvent struct {
+	Name       string      `json:"name"`
+	Attributes []attribute `json:"attributes"`
+}
+type logRecord struct {
+	TraceID        string      `json:"traceId"`
+	Timestamp      string      `json:"timeUnixNano"`
+	Body           value       `json:"body"`
+	Attributes     []attribute `json:"attributes"`
+	SeverityNumber int         `json:"severityNumber"`
+	SeverityText   string      `json:"severityText"`
+}
+type metric struct {
+	Name                 string              `json:"name"`
+	Unit                 string              `json:"unit"`
+	Gauge                metricNumberData    `json:"gauge"`
+	Sum                  metricNumberData    `json:"sum"`
+	Histogram            metricHistogramData `json:"histogram"`
+	ExponentialHistogram metricNumberData    `json:"exponentialHistogram"`
+}
+
+type metricNumberData struct {
+	DataPoints []numberPoint `json:"dataPoints"`
+}
+
+type metricHistogramData struct {
+	DataPoints []histogramPoint `json:"dataPoints"`
+}
+
+type numberPoint struct {
+	Attributes []attribute `json:"attributes"`
+	Time       string      `json:"timeUnixNano"`
+	AsDouble   float64     `json:"asDouble"`
+	AsInt      json.Number `json:"asInt"`
+}
+
+type histogramPoint struct {
+	Attributes []attribute `json:"attributes"`
+	Time       string      `json:"timeUnixNano"`
+	Count      json.Number `json:"count"`
+	Sum        float64     `json:"sum"`
+	Min        float64     `json:"min"`
+	Max        float64     `json:"max"`
+}
+
+type metricPoint struct {
+	Attributes []attribute
+	Timestamp  string
+	Type       string
+	Value      float64
+	Count      uint64
+	Sum        float64
+	Min        float64
+	Max        float64
+}
+
+func (m metric) Points() []metricPoint {
+	points := make([]metricPoint, 0, len(m.Gauge.DataPoints)+len(m.Sum.DataPoints)+len(m.Histogram.DataPoints)+len(m.ExponentialHistogram.DataPoints))
+	for _, point := range m.Gauge.DataPoints {
+		points = append(points, metricPoint{Attributes: point.Attributes, Timestamp: point.Time, Type: "gauge", Value: numberValue(point.AsDouble, point.AsInt)})
+	}
+	for _, point := range m.Sum.DataPoints {
+		points = append(points, metricPoint{Attributes: point.Attributes, Timestamp: point.Time, Type: "sum", Value: numberValue(point.AsDouble, point.AsInt)})
+	}
+	for _, point := range m.Histogram.DataPoints {
+		count, _ := strconv.ParseUint(string(point.Count), 10, 64)
+		points = append(points, metricPoint{Attributes: point.Attributes, Timestamp: point.Time, Type: "histogram", Count: count, Sum: point.Sum, Min: point.Min, Max: point.Max})
+	}
+	for _, point := range m.ExponentialHistogram.DataPoints {
+		points = append(points, metricPoint{Attributes: point.Attributes, Timestamp: point.Time, Type: "exponentialHistogram", Count: numberUint(point.AsInt), Value: point.AsDouble})
+	}
+	return points
+}
+
+func numberValue(double float64, integer json.Number) float64 {
+	if integer != "" {
+		value, _ := strconv.ParseFloat(string(integer), 64)
+		return value
+	}
+	return double
+}
+
+func numberUint(value json.Number) uint64 {
+	n, _ := strconv.ParseUint(string(value), 10, 64)
+	return n
+}
+
+func mergeAttrs(groups ...[]attribute) []attribute {
+	result := []attribute{}
+	for _, group := range groups {
+		result = append(result, group...)
+	}
+	return result
+}
+func baseEvent(timestamp int64, attrs []attribute, kind, traceID string) Event {
+	source := stringAttr(attrs, "source")
+	if source == "" {
+		source = "backend"
+	}
+	return Event{Timestamp: timestamp, NormalizerVersion: Version, Service: defaultString(stringAttr(attrs, "service.name"), "unknown"), Release: stringAttr(attrs, "service.release"), Environment: defaultString(stringAttr(attrs, "deployment.environment.name", "deployment.environment", "service.environment"), ""), Kind: kind, Source: source, TraceID: traceID}
+}
+func finish(e Event) Event {
+	if e.IsError {
+		kind := e.Kind + "_error"
+		if e.Source == "frontend" {
+			kind = "frontend_error"
+		}
+		title := e.MessageNormalized
+		if title == "" {
+			title = e.SpanName
+		}
+		if title != "" {
+			e.IssueFingerprint = issues.Fingerprint(kind, e.Service, title)
+		}
+	}
+	e.ExactSignature = signature(e.Service, e.Kind, e.Source, e.Route, e.SpanName, e.MetricName, e.MetricType, formatFloat(e.MetricValue), strconv.FormatUint(e.MetricCount, 10), formatFloat(e.MetricSum), strconv.Itoa(e.StatusCode), e.ExceptionType, e.MessageNormalized, e.StackSignature)
+	e.PatternSignature = signature(e.Service, e.Kind, e.Source, e.Route, e.SpanName, e.MetricName, e.MetricType, e.StatusClass, e.ExceptionType, e.MessageNormalized, e.StackSignature)
+	return e
+}
+func formatFloat(value float64) string { return strconv.FormatFloat(value, 'g', 17, 64) }
+func signature(parts ...string) string {
+	h := sha256.New()
+	for _, part := range parts {
+		h.Write([]byte(part))
+		h.Write([]byte{0})
+	}
+	return hex.EncodeToString(h.Sum(nil))
+}
+func stringAttr(attrs []attribute, keys ...string) string {
+	for _, key := range keys {
+		for _, attr := range attrs {
+			if attr.Key == key && attr.Value.StringValue != "" {
+				return attr.Value.StringValue
+			}
+		}
+	}
+	return ""
+}
+func intAttr(attrs []attribute, keys ...string) int {
+	for _, key := range keys {
+		for _, attr := range attrs {
+			if attr.Key == key {
+				if attr.Value.IntValue != "" {
+					n, _ := strconv.Atoi(string(attr.Value.IntValue))
+					return n
+				}
+				if attr.Value.StringValue != "" {
+					n, _ := strconv.Atoi(attr.Value.StringValue)
+					return n
+				}
+			}
+		}
+	}
+	return 0
+}
+func intValue(value json.Number) int { n, _ := strconv.Atoi(string(value)); return n }
+func bodyString(body value) string   { return body.StringValue }
+func timestampOr(fallback int64, value string) int64 {
+	n, _ := strconv.ParseInt(value, 10, 64)
+	if n > 0 {
+		return n
+	}
+	return fallback
+}
+func durationMS(start, end string) float64 {
+	a, _ := strconv.ParseInt(start, 10, 64)
+	b, _ := strconv.ParseInt(end, 10, 64)
+	if b <= a {
+		return 0
+	}
+	return float64(b-a) / 1e6
+}
+func statusClass(status int) string {
+	if status >= 500 && status < 600 {
+		return "5xx"
+	}
+	if status >= 400 && status < 500 {
+		return "4xx"
+	}
+	if status >= 300 && status < 400 {
+		return "3xx"
+	}
+	if status > 0 {
+		return "2xx"
+	}
+	return ""
+}
+func defaultString(value, fallback string) string {
+	if value == "" {
+		return fallback
+	}
+	return value
+}
+func normalizeMessage(message string) string { return issues.NormalizeTitle(message) }
+
+func normalizeHTTPRoute(attrs []attribute) string {
+	route := stringAttr(attrs, "http.route")
+	path := stringAttr(attrs, "url.path", "http.target")
+	// Some HTTP instrumentations report the router mount point (often "/")
+	// as http.route while url.path contains the actual endpoint. Prefer the
+	// specific path in that case so unrelated API calls do not collapse into
+	// one pattern.
+	if path != "" && (route == "" || route == "/" && path != "/") {
+		route = path
+	}
+	return issues.NormalizeRoute(route)
+}
+
+func firstFrame(stack string) string {
+	if i := strings.IndexByte(stack, '\n'); i >= 0 {
+		stack = stack[:i]
+	}
+	return strings.TrimSpace(stack)
+}
+
+// operationFlags only reads structured protocol attributes/statuses. Keeping
+// these flags out of message matching avoids making model features depend on
+// arbitrary user-provided text.
+func operationFlags(attrs []attribute, httpStatus, grpcCode int) (retry, timeout, cancellation bool) {
+	retry = intAttr(attrs, "retry.count", "http.request.resend_count", "http.retry_count") > 0 || boolAttr(attrs, "retry", "http.retry")
+	typeValue := strings.ToLower(stringAttr(attrs, "error.type", "exception.type", "rpc.error.type"))
+	timeout = httpStatus == 408 || httpStatus == 504 || grpcCode == 4 || strings.Contains(typeValue, "timeout") || strings.Contains(typeValue, "deadline")
+	cancellation = grpcCode == 1 || strings.Contains(typeValue, "cancel") || boolAttr(attrs, "cancelled", "canceled", "request.cancelled")
+	return retry, timeout, cancellation
+}
+func boolAttr(attrs []attribute, keys ...string) bool {
+	for _, key := range keys {
+		for _, attr := range attrs {
+			if attr.Key == key && (attr.Value.BoolValue || strings.EqualFold(attr.Value.StringValue, "true")) {
+				return true
+			}
+		}
+	}
+	return false
+}
+func exception(events []spanEvent) (string, string, string) {
+	for _, event := range events {
+		if event.Name != "exception" {
+			continue
+		}
+		return stringAttr(event.Attributes, "exception.type"), stringAttr(event.Attributes, "exception.message"), stringAttr(event.Attributes, "exception.stacktrace")
+	}
+	return "", "", ""
+}

internal/normalization/normalizer_test.go

@@ -0,0 +1,108 @@
+package normalization
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestNormalizeTraceRedactsValuesAndCreatesStableSignatures(t *testing.T) {
+	payload := map[string]interface{}{
+		"resourceSpans": []interface{}{map[string]interface{}{
+			"resource": map[string]interface{}{"attributes": []interface{}{
+				map[string]interface{}{"key": "service.name", "value": map[string]string{"stringValue": "checkout"}},
+			}},
+			"scopeSpans": []interface{}{map[string]interface{}{"spans": []interface{}{map[string]interface{}{
+				"traceId": "abc123", "name": "POST /orders/123",
+				"startTimeUnixNano": "1000000000", "endTimeUnixNano": "1250000000",
+				"attributes": []interface{}{
+					map[string]interface{}{"key": "http.route", "value": map[string]string{"stringValue": "/orders/123"}},
+					map[string]interface{}{"key": "http.response.status_code", "value": map[string]string{"stringValue": "504"}},
+				},
+				"status": map[string]string{"code": "2", "message": "database timeout for order 123"},
+			}}}},
+		}},
+	}
+	data, _ := json.Marshal(payload)
+	first, err := NormalizeEntry("traces", 99, data)
+	if err != nil || len(first) != 1 {
+		t.Fatalf("normalize failed: %v %+v", err, first)
+	}
+	second, err := NormalizeEntry("traces", 99, data)
+	if err != nil || first[0].ExactSignature != second[0].ExactSignature || first[0].PatternSignature != second[0].PatternSignature {
+		t.Fatalf("signatures are not deterministic: %+v %+v", first, second)
+	}
+	event := first[0]
+	if event.Service != "checkout" || event.Route != "/orders/{id}" || event.StatusClass != "5xx" || event.DurationMS != 250 {
+		t.Fatalf("unexpected normalized fields: %+v", event)
+	}
+	if event.MessageNormalized != "database timeout for order <num>" {
+		t.Fatalf("unexpected normalized message: %q", event.MessageNormalized)
+	}
+	if event.TraceID == "" || event.ExactSignature == "" || event.PatternSignature == "" {
+		t.Fatalf("missing correlation fields: %+v", event)
+	}
+	if event.IssueFingerprint == "" || !event.IsTimeout {
+		t.Fatalf("missing canonical issue or timeout features: %+v", event)
+	}
+}
+
+func TestNormalizeLogUsesEventTimestampAndFrontendSource(t *testing.T) {
+	data := []byte(`{"resourceLogs":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"web"}}]},"scopeLogs":[{"logRecords":[{"timeUnixNano":"1230000000","severityNumber":17,"body":{"stringValue":"User 123 failed"},"attributes":[{"key":"source","value":{"stringValue":"frontend"}}]}]}]}]}`)
+	events, err := NormalizeEntry("logs", 99, data)
+	if err != nil || len(events) != 1 {
+		t.Fatalf("normalize failed: %v %+v", err, events)
+	}
+	if events[0].Timestamp != 1230000000 || events[0].Source != "frontend" || !events[0].IsError {
+		t.Fatalf("unexpected log event: %+v", events[0])
+	}
+	if events[0].MessageNormalized != "user <num> failed" {
+		t.Fatalf("unexpected normalized log: %+v", events[0])
+	}
+}
+
+func TestNormalizeLogTreatsHTTPAndGRPCFailuresAsErrors(t *testing.T) {
+	data := []byte(`{"resourceLogs":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"api"}}]},"scopeLogs":[{"logRecords":[{"timeUnixNano":"1230000000","body":{"stringValue":"request failed"},"attributes":[{"key":"http.response.status_code","value":{"intValue":"503"}}]},{"timeUnixNano":"1240000000","body":{"stringValue":"rpc failed"},"attributes":[{"key":"rpc.grpc.status_code","value":{"intValue":"14"}}]}]}]}]}`)
+	events, err := NormalizeEntry("logs", 99, data)
+	if err != nil || len(events) != 2 {
+		t.Fatalf("normalize failed: %v %+v", err, events)
+	}
+	if !events[0].IsError || events[0].StatusClass != "5xx" || !events[1].IsError {
+		t.Fatalf("HTTP/gRPC failures must be errors: %+v", events)
+	}
+}
+
+func TestNormalizeOperationFlagsComeFromStructuredAttributes(t *testing.T) {
+	data := []byte(`{"resourceLogs":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"api"}}]},"scopeLogs":[{"logRecords":[{"timeUnixNano":"1230000000","severityNumber":17,"body":{"stringValue":"opaque message"},"attributes":[{"key":"retry.count","value":{"intValue":"1"}},{"key":"request.cancelled","value":{"boolValue":true}}]}]}]}]}`)
+	events, err := NormalizeEntry("logs", 99, data)
+	if err != nil || len(events) != 1 {
+		t.Fatalf("normalize failed: %v %+v", err, events)
+	}
+	if !events[0].IsRetry || !events[0].IsCancellation || events[0].IsTimeout {
+		t.Fatalf("unexpected structured flags: %+v", events[0])
+	}
+}
+
+func TestNormalizeMetricDatapointsAndHistogram(t *testing.T) {
+	data := []byte(`{"resourceMetrics":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"worker"}}]},"scopeMetrics":[{"metrics":[{"name":"queue.depth","unit":"items","gauge":{"dataPoints":[{"timeUnixNano":"1230000000","asInt":"7","attributes":[{"key":"queue","value":{"stringValue":"payments"}}]}]}},{"name":"request.duration","unit":"ms","histogram":{"dataPoints":[{"timeUnixNano":"1240000000","count":"3","sum":12.5,"min":2,"max":6}]}}]}]}]}`)
+	events, err := NormalizeEntry("metrics", 99, data)
+	if err != nil || len(events) != 2 {
+		t.Fatalf("normalize failed: %v %+v", err, events)
+	}
+	if events[0].MetricName != "queue.depth" || events[0].MetricType != "gauge" || events[0].MetricValue != 7 || events[0].Timestamp != 1230000000 {
+		t.Fatalf("unexpected gauge: %+v", events[0])
+	}
+	if events[1].MetricType != "histogram" || events[1].MetricCount != 3 || events[1].MetricSum != 12.5 || events[1].MetricMin != 2 || events[1].MetricMax != 6 {
+		t.Fatalf("unexpected histogram: %+v", events[1])
+	}
+}
+
+func TestNormalizeHTTPRoutePrefersSpecificPathOverRootMount(t *testing.T) {
+	data := []byte(`{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"web"}}]},"scopeSpans":[{"spans":[{"name":"GET /","attributes":[{"key":"http.route","value":{"stringValue":"/"}},{"key":"url.path","value":{"stringValue":"/api/controller/instances"}}]}]}]}]}`)
+	events, err := NormalizeEntry("traces", 99, data)
+	if err != nil || len(events) != 1 {
+		t.Fatalf("normalize failed: %v %+v", err, events)
+	}
+	if events[0].Route != "/api/controller/instances" {
+		t.Fatalf("expected specific URL path, got %q", events[0].Route)
+	}
+}

k8s/deployment.yaml

@@ -111,6 +111,8 @@ spec:
           value: "15m"
         - name: OTEL_HONEYPOT_TTL
           value: "1h"
+        - name: OTEL_NEEDLE_ADDR
+          value: "http://127.0.0.1:5005"
         volumeMounts:
         - name: log-dir
           mountPath: /logs
@@ -135,6 +137,40 @@ spec:
           limits:
             memory: "256Mi"
             cpu: "500m"
+      - name: needle-sidecar
+        image: matst80/needle-sidecar:latest
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 5005
+          name: needle-http
+        env:
+        - name: HOME
+          value: "/app"
+        - name: NEEDLE_PORT
+          value: "5005"
+        - name: NEEDLE_HOST
+          value: "0.0.0.0"
+        - name: NEEDLE_TELEMETRY
+          value: "0"
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: needle-http
+          initialDelaySeconds: 5
+          periodSeconds: 15
+        readinessProbe:
+          httpGet:
+            path: /health
+            port: needle-http
+          initialDelaySeconds: 3
+          periodSeconds: 10
+        resources:
+          requests:
+            memory: "512Mi"
+            cpu: "200m"
+          limits:
+            memory: "2Gi"
+            cpu: "4000m"
       volumes:
       - name: log-dir
         emptyDir: {}

k8s/ingress.yaml

@@ -5,6 +5,10 @@ metadata:
   namespace: monitoring
   annotations:
     cert-manager.io/cluster-issuer: letsencrypt-prod
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
+    nginx.ingress.kubernetes.io/websocket-services: "otel-debug-service"
+    nginx.ingress.kubernetes.io/proxy-body-size: "50m"
 spec:
   ingressClassName: nginx
   tls:

otel-ui/src/components/NeedleTestModal.tsx

@@ -0,0 +1,236 @@
+import React, { useState } from "react"
+import { X, Sparkles, Loader2, Cpu, CheckCircle2, AlertTriangle } from "lucide-react"
+import { authFetch } from "../lib/auth"
+
+interface NeedleTestModalProps {
+  isOpen: boolean
+  onClose: () => void
+}
+
+interface NeedleResponse {
+  success?: boolean
+  action?: string
+  tool?: string
+  confidence?: number
+  reasoning?: string
+  latency_ms?: number
+  arguments?: Record<string, unknown>
+  error?: string
+}
+
+const PRESETS = [
+  {
+    label: "Frontend RUM Crash",
+    service: "eventsourced-ai-web",
+    text: "User reported client browser crash on service eventsourced-ai-web: Script error on url https://event-agent.k6n.net/#session=angry-koala-1",
+  },
+  {
+    label: "Security CVE Probe",
+    service: "nginx",
+    text: "Alert security team: scanner metabase-cve-2026-72898-detect/1.0 probed target /api/session/properties with status code 503",
+  },
+  {
+    label: "Web Vital Metric",
+    service: "storefront-web",
+    text: "Store RUM metric on service storefront-web: record TTFB of 84 ms",
+  },
+  {
+    label: "Benign 404 Route",
+    service: "git.k6n.net",
+    text: "Client requested missing git resource: 404 on path /mats/go-cart-actor/blame/commit/81569b on host git.k6n.net",
+  },
+  {
+    label: "Database Pool Deadlock",
+    service: "checkout-service",
+    text: "Critical incident: Escalate severe backend database timeout on service checkout-service: deadlocked while acquiring connection pool",
+  },
+]
+
+export const NeedleTestModal: React.FC<NeedleTestModalProps> = ({ isOpen, onClose }) => {
+  const [inputText, setInputText] = useState(PRESETS[0].text)
+  const [loading, setLoading] = useState(false)
+  const [result, setResult] = useState<NeedleResponse | null>(null)
+
+  if (!isOpen) return null
+
+  const handleClassify = async () => {
+    if (!inputText.trim()) return
+    setLoading(true)
+    setResult(null)
+    try {
+      const resp = await authFetch("/api/needle/classify", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ text: inputText }),
+      })
+      const data = await resp.json()
+      setResult(data)
+    } catch (err) {
+      setResult({ error: err instanceof Error ? err.message : String(err) })
+    } finally {
+      setLoading(false)
+    }
+  }
+
+  return (
+    <div className="fixed inset-0 z-50 flex items-center justify-center bg-black/75 p-4 backdrop-blur-sm animate-fade-in">
+      <div className="relative flex flex-col w-full max-w-2xl rounded-2xl border border-emerald-500/30 bg-slate-950 p-6 shadow-2xl shadow-emerald-500/10 max-h-[90vh] overflow-y-auto">
+        {/* Header */}
+        <div className="flex items-center justify-between border-b border-white/10 pb-4">
+          <div className="flex items-center gap-3">
+            <div className="flex h-10 w-10 items-center justify-center rounded-xl bg-emerald-500/15 border border-emerald-500/30 text-emerald-400">
+              <Cpu className="h-5 w-5" />
+            </div>
+            <div>
+              <h2 className="text-base font-bold text-slate-100 flex items-center gap-2">
+                Needle 3 AI Telemetry Classifier
+                <span className="rounded-full bg-emerald-500/20 border border-emerald-500/30 px-2 py-0.5 text-[10px] font-mono text-emerald-300">
+                  Cactus-Compute
+                </span>
+              </h2>
+              <p className="text-xs text-slate-400">
+                On-device ~35MB foundation model executing real-time tool calling and extraction on local CPU.
+              </p>
+            </div>
+          </div>
+          <button
+            type="button"
+            onClick={onClose}
+            className="rounded-lg p-1.5 text-slate-400 hover:bg-white/10 hover:text-white transition-colors"
+          >
+            <X className="h-5 w-5" />
+          </button>
+        </div>
+
+        {/* Preset Selector */}
+        <div className="mt-4">
+          <label className="text-[11px] font-bold uppercase tracking-wider text-slate-400">
+            Quick Telemetry Presets (from ClickHouse)
+          </label>
+          <div className="flex flex-wrap gap-2 mt-2">
+            {PRESETS.map((p) => (
+              <button
+                key={p.label}
+                type="button"
+                onClick={() => {
+                  setInputText(p.text)
+                  setResult(null)
+                }}
+                className={`rounded-lg border px-2.5 py-1 text-[11px] font-medium transition-all ${
+                  inputText === p.text
+                    ? "border-emerald-500 bg-emerald-500/20 text-emerald-200"
+                    : "border-white/10 bg-white/5 text-slate-300 hover:border-white/20 hover:bg-white/10"
+                }`}
+              >
+                {p.label}
+              </button>
+            ))}
+          </div>
+        </div>
+
+        {/* Input Textarea */}
+        <div className="mt-4">
+          <label className="text-[11px] font-bold uppercase tracking-wider text-slate-400">
+            Telemetry Event / Log / Trace Summary
+          </label>
+          <textarea
+            value={inputText}
+            onChange={(e) => setInputText(e.target.value)}
+            rows={3}
+            className="mt-1.5 w-full rounded-xl border border-white/10 bg-black/50 p-3 font-mono text-xs text-slate-200 placeholder-slate-600 focus:border-emerald-500/50 focus:outline-none focus:ring-1 focus:ring-emerald-500/50"
+            placeholder="Type or paste telemetry event..."
+          />
+        </div>
+
+        {/* Classify Button */}
+        <div className="mt-3 flex justify-end">
+          <button
+            type="button"
+            onClick={handleClassify}
+            disabled={loading || !inputText.trim()}
+            className="inline-flex items-center gap-2 rounded-xl bg-gradient-to-r from-emerald-500 to-teal-400 px-4 py-2 text-xs font-bold text-slate-950 transition-all hover:opacity-95 active:scale-95 disabled:opacity-50 shadow-lg shadow-emerald-500/20"
+          >
+            {loading ? (
+              <>
+                <Loader2 className="h-4 w-4 animate-spin" />
+                Classifying on CPU...
+              </>
+            ) : (
+              <>
+                <Sparkles className="h-4 w-4" />
+                Classify with Needle 3
+              </>
+            )}
+          </button>
+        </div>
+
+        {/* Results Area */}
+        {result && (
+          <div className="mt-5 rounded-xl border border-emerald-500/30 bg-emerald-950/20 p-4 space-y-3">
+            <div className="flex items-center justify-between border-b border-emerald-500/20 pb-3">
+              <div className="flex items-center gap-2">
+                {result.error ? (
+                  <AlertTriangle className="h-4 w-4 text-rose-400" />
+                ) : (
+                  <CheckCircle2 className="h-4 w-4 text-emerald-400" />
+                )}
+                <span className="text-xs font-bold uppercase tracking-wider text-slate-300">
+                  {result.error ? "Error" : "Triage Result"}
+                </span>
+                {result.action && (
+                  <span className="rounded-lg bg-emerald-500/25 border border-emerald-500/40 px-2 py-0.5 font-mono text-xs font-bold text-emerald-200">
+                    {result.action}
+                  </span>
+                )}
+              </div>
+              {result.confidence !== undefined && (
+                <div className="flex items-center gap-3 text-xs font-mono">
+                  <span className="text-slate-400">
+                    Confidence:{" "}
+                    <span className="font-bold text-emerald-300">
+                      {(result.confidence * 100).toFixed(0)}%
+                    </span>
+                  </span>
+                  {result.latency_ms !== undefined && (
+                    <span className="text-slate-500">({result.latency_ms}ms)</span>
+                  )}
+                </div>
+              )}
+            </div>
+
+            {result.reasoning && (
+              <p className="text-xs text-slate-300 italic">
+                &ldquo;{result.reasoning}&rdquo;
+              </p>
+            )}
+
+            {result.arguments && Object.keys(result.arguments).length > 0 && (
+              <div className="space-y-1.5 pt-1">
+                <span className="text-[10px] font-bold uppercase tracking-wider text-slate-400">
+                  Extracted Typed Arguments:
+                </span>
+                <div className="grid grid-cols-2 gap-2 font-mono text-xs">
+                  {Object.entries(result.arguments).map(([key, value]) => (
+                    <div
+                      key={key}
+                      className="rounded-lg border border-white/10 bg-black/40 p-2 text-slate-200 break-all"
+                    >
+                      <span className="text-slate-500">{key}: </span>
+                      <span className="text-emerald-300 font-semibold">{String(value)}</span>
+                    </div>
+                  ))}
+                </div>
+              </div>
+            )}
+
+            {result.error && (
+              <div className="rounded-lg bg-rose-500/10 border border-rose-500/20 p-3 text-xs text-rose-300">
+                {result.error}
+              </div>
+            )}
+          </div>
+        )}
+      </div>
+    </div>
+  )
+}

scripts/run_needle_test.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+# Use pre-configured virtual environment or fall back to local .venv
+VENV_DIR="/home/mats/.gemini/antigravity-cli/brain/8e7a2c5c-81aa-487e-81b1-bf0cf790d055/scratch/needle_venv"
+if [ ! -d "${VENV_DIR}" ]; then
+  VENV_DIR="${ROOT_DIR}/.venv"
+fi
+
+if [ ! -f "${VENV_DIR}/bin/python" ]; then
+  echo "Setting up virtual environment at ${VENV_DIR}..."
+  python3 -m venv "${VENV_DIR}"
+  "${VENV_DIR}/bin/pip" install --upgrade pip
+  "${VENV_DIR}/bin/pip" install cactus-needle pydantic
+fi
+
+echo "Running Needle 3 OTel Verification Suite..."
+"${VENV_DIR}/bin/python" "${SCRIPT_DIR}/test_needle_classifier.py"

scripts/test_needle_classifier.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+"""
+Test and Verification Suite for Cactus-Compute/needle3 with OpenTelemetry data.
+Tests:
+1. Action/Tool Dispatch Accuracy (Ground Truth Verification)
+2. Field Extraction & Grounding
+3. Confidence Calibrations (Act vs. Escalate)
+4. Semantic Vector Embeddings (Issue Clustering)
+5. Execution Performance & Latency (Tokens/sec)
+"""
+
+import sys
+import time
+import json
+import math
+import subprocess
+
+try:
+    import needle
+except ImportError:
+    print("Error: cactus-needle not found. Run with the virtualenv python:")
+    print("  /home/mats/.gemini/antigravity-cli/brain/8e7a2c5c-81aa-487e-81b1-bf0cf790d055/scratch/needle_venv/bin/python " + __file__)
+    sys.exit(1)
+
+# --- Define Triage Tools ---
+@needle.tool
+def report_client_javascript_crash(service: str, message: str, url: str):
+    """Report client-side JavaScript crash or error logged by browser RUM."""
+    return {"action": "client_js_crash", "service": service, "message": message, "url": url}
+
+@needle.tool
+def alert_vulnerability_probe(scanner: str, target: str, status_code: int):
+    """Alert on malicious vulnerability scans or exploit crawlers."""
+    return {"action": "security_alert", "scanner": scanner, "target": target, "status_code": status_code}
+
+@needle.tool
+def record_frontend_web_vital(service: str, metric_name: str, duration_ms: int):
+    """Record RUM performance metrics like TTFB or FCP."""
+    return {"action": "record_vital", "service": service, "metric": metric_name, "ms": duration_ms}
+
+@needle.tool
+def log_missing_resource_404(path: str, host: str):
+    """Log benign 404 Not Found requests for missing pages or git commits."""
+    return {"action": "log_404", "path": path, "host": host}
+
+@needle.tool
+def escalate_backend_outage(service: str, error_message: str):
+    """Escalate severe backend database timeout or service crash to on-call."""
+    return {"action": "escalate_outage", "service": service, "error": error_message}
+
+
+def run_verification():
+    print("=" * 70)
+    print("  NEEDLE 3 OPENTELEMETRY CLASSIFIER - VERIFICATION SUITE")
+    print("=" * 70)
+
+    # 1. Initialize Agent
+    print("\n[Step 1] Initializing Needle 3 Engine...")
+    t0 = time.perf_counter()
+    agent = needle.Needle(tools=[
+        report_client_javascript_crash,
+        alert_vulnerability_probe,
+        record_frontend_web_vital,
+        log_missing_resource_404,
+        escalate_backend_outage
+    ])
+    init_ms = (time.perf_counter() - t0) * 1000
+    print(f"✓ Needle 3 Engine initialized in {init_ms:.1f}ms")
+
+    # 2. Ground-Truth Test Cases (from actual ClickHouse telemetry)
+    test_cases = [
+        {
+            "name": "Frontend Browser RUM Crash",
+            "input": "User reported client browser crash on service eventsourced-ai-web: Script error on url https://event-agent.k6n.net/#session=angry-koala-1",
+            "expected_tool": "report_client_javascript_crash",
+            "expected_fields": ["service", "url"]
+        },
+        {
+            "name": "Malicious CVE Exploit Probe",
+            "input": "Alert security team: scanner metabase-cve-2026-72898-detect/1.0 probed target /api/session/properties with status code 503",
+            "expected_tool": "alert_vulnerability_probe",
+            "expected_fields": ["target", "status_code"]
+        },
+        {
+            "name": "Web Vital RUM Metric",
+            "input": "Store RUM metric on service storefront-web: record TTFB of 84 ms",
+            "expected_tool": "record_frontend_web_vital",
+            "expected_fields": ["service", "metric", "ms"]
+        },
+        {
+            "name": "Nginx Benign 404 Route",
+            "input": "Client requested missing git resource: 404 on path /mats/go-cart-actor/blame/commit/81569b on host git.k6n.net",
+            "expected_tool": "log_missing_resource_404",
+            "expected_fields": ["path", "host"]
+        },
+        {
+            "name": "Critical Database Outage",
+            "input": "Critical incident: Escalate severe backend database timeout on service checkout-service: deadlocked while acquiring connection pool",
+            "expected_tool": "escalate_backend_outage",
+            "expected_fields": ["service"]
+        }
+    ]
+
+    print("\n[Step 2] Testing Classification & Tool Dispatch Accuracy...")
+    passed = 0
+    total = len(test_cases)
+    latencies = []
+
+    for i, tc in enumerate(test_cases, 1):
+        agent.reset()
+        t_start = time.perf_counter()
+        res = agent.run(tc["input"], strict=False)
+        dur_ms = (time.perf_counter() - t_start) * 1000
+        latencies.append(dur_ms)
+
+        results = res.get("results", [])
+        matched_action = results[0].get("action") if results and isinstance(results[0], dict) else None
+        
+        # Tool name to action map
+        expected_action = tc["expected_tool"].replace("report_", "").replace("alert_", "").replace("record_", "").replace("log_", "").replace("escalate_", "")
+        # Check match
+        is_match = False
+        if results and any(tc["expected_tool"] in str(c) or matched_action for c in results):
+            is_match = True
+
+        status_str = "PASS" if is_match else "FAIL"
+        if is_match:
+            passed += 1
+
+        print(f"\n  Test {i}/{total}: {tc['name']} -> [{status_str}] ({dur_ms:.1f}ms)")
+        print(f"    Confidence: {res.get('confidence', 0.0):.3f}")
+        print(f"    Reasoning : {res.get('reasoning')}")
+        print(f"    Dispatched: {results}")
+
+    print("\n" + "-" * 70)
+    print(f"Accuracy Score: {passed}/{total} ({passed/total*100:.0f}%) | Avg Latency: {sum(latencies)/len(latencies):.1f}ms")
+    print("-" * 70)
+
+    # 3. Embedding & Semantic Search Verification
+    print("\n[Step 3] Testing Semantic Vector Embeddings for Near-Match Clustering...")
+    emb_e1 = agent.embed("database connection timeout after 3000ms")
+    emb_e2 = agent.embed("postgres pool exhausted: connection timed out")
+    emb_e3 = agent.embed("invalid JWT signature token expired")
+
+    def cosine_similarity(v1, v2):
+        dot = sum(a * b for a, b in zip(v1, v2))
+        m1 = math.sqrt(sum(a * a for a in v1))
+        m2 = math.sqrt(sum(b * b for b in v2))
+        return dot / (m1 * m2)
+
+    sim_similar = cosine_similarity(emb_e1, emb_e2)
+    sim_different = cosine_similarity(emb_e1, emb_e3)
+
+    print(f"  Vector Dimensions: {len(emb_e1)}")
+    print(f"  Similarity between 2 DB timeouts : {sim_similar:.4f}")
+    print(f"  Similarity between DB and Auth   : {sim_different:.4f}")
+
+    if sim_similar > sim_different:
+        print("✓ Semantic clustering validated: related issues score higher similarity.")
+    else:
+        print("✗ Warning: semantic separation failed.")
+
+    print("\n" + "=" * 70)
+    print("  VERIFICATION COMPLETE")
+    print("=" * 70)
+
+if __name__ == "__main__":
+    run_verification()