Merge origin/main into feature/issues-tui

Needle scored ~chance on the golden set and never gated alerts, so drop the
sidecar, proxy handlers, UI buttons, k8s container and build step. The labelled
eval sets move to cmd/triage-eval/data.

nginx 4xx (registry 401 challenges, /oauth2/auth, scanner 405s) is now dropped
in FilterActionable instead of only being discounted, so it cannot ride along
with a real 5xx. nginx 5xx still alerts.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

.gitea/workflows/build.yaml

@@ -1,6 +1,9 @@
 name: Build and Publish
 run-name: ${{ gitea.actor }} build 🚀
-on: [push]
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
 
 jobs:
   Metadata:
@@ -26,7 +29,7 @@ jobs:
           echo "version=$VERSION" >> $GITHUB_OUTPUT
           echo "build_date=$BUILD_DATE" >> $GITHUB_OUTPUT
 
-  BuildAndDeployAmd64:
+  BuildAndPushAmd64:
     needs: Metadata
     runs-on: amd64
     steps:
@@ -39,15 +42,32 @@ jobs:
             --build-arg GIT_COMMIT=${{ needs.Metadata.outputs.git_commit }} \
             --build-arg BUILD_DATE=${{ needs.Metadata.outputs.build_date }} \
             --progress=plain \
+            -t matst80/otel-debug:${{ needs.Metadata.outputs.version }} \
             -t matst80/otel-debug:latest \
             .
-      - name: Push amd64 images
+      - name: Push images
         run: |
+          docker push matst80/otel-debug:${{ needs.Metadata.outputs.version }}
           docker push matst80/otel-debug:latest
-      - name: Rollout amd64 otel deployment
+      - name: Reconcile after image is published
+        env:
+          GITOPS_WEBHOOK_SECRET: ${{ secrets.GITOPS_WEBHOOK_SECRET }}
         run: |
-          kubectl rollout restart deployment/otel-debug-receiver -n monitoring
-          kubectl rollout status deployment/otel-debug-receiver -n monitoring
+          if [ -z "$GITOPS_WEBHOOK_SECRET" ]; then
+            echo "GITOPS_WEBHOOK_SECRET must match the k8s-helper Gitea webhook secret" >&2
+            exit 1
+          fi
+          payload="$(mktemp)"
+          trap 'rm -f "$payload"' EXIT
+          printf '{"ref":"%s","after":"%s","repository":{"clone_url":"https://git.k6n.net/mats/go-otel.git","ssh_url":"git@git.k6n.net:mats/go-otel.git"}}' \
+            "$GITHUB_REF" "$GITHUB_SHA" > "$payload"
+          signature="$(openssl dgst -sha256 -hmac "$GITOPS_WEBHOOK_SECRET" "$payload" | awk '{print $NF}')"
+          curl --fail --silent --show-error \
+            -X POST \
+            -H "Content-Type: application/json" \
+            -H "X-Gitea-Signature: $signature" \
+            --data-binary "@$payload" \
+            https://ui.k6n.net/api/gitops/webhooks/gitea
       - name: Upload source maps
         env:
           RELEASE: ${{ needs.Metadata.outputs.version }}
@@ -69,31 +89,3 @@ jobs:
           done < <(find "$maps_dir" -type f -name '*.js.map' -print0)
           rm -rf "$maps_dir"
           [ "$found" = 1 ] || { echo "no source maps found in UI build" >&2; exit 1; }
-      # - name: Rollout amd64 deployment (pin to version)
-      #   run: |
-      #     kubectl set image deployment/cart-actor-x86 -n cart cart-actor-amd64=registry.knatofs.se/go-otel-debug-amd64:${{ needs.Metadata.outputs.version }}
-      #     # kubectl rollout status deployment/cart-actor-x86 -n cart
-
-  # BuildAndDeployArm64:
-  #   needs: Metadata
-  #   runs-on: arm64
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: Build arm64 image
-  #       run: |
-  #         docker build \
-  #           --build-arg VERSION=${{ needs.Metadata.outputs.version }} \
-  #           --build-arg GIT_COMMIT=${{ needs.Metadata.outputs.git_commit }} \
-  #           --build-arg BUILD_DATE=${{ needs.Metadata.outputs.build_date }} \
-  #           --progress=plain \
-  #           -t registry.knatofs.se/go-otel-debug:latest \
-  #           -t registry.knatofs.se/go-otel-debug:${{ needs.Metadata.outputs.version }} \
-  #           .
-  #     - name: Push arm64 images
-  #       run: |
-  #         docker push registry.knatofs.se/go-otel-debug:latest
-  #         docker push registry.knatofs.se/go-otel-debug:${{ needs.Metadata.outputs.version }}
-  #     - name: Rollout arm64 deployment (pin to version)
-  #       run: |
-  #         kubectl set image deployment/cart-actor-arm64 -n cart cart-actor-arm64=registry.knatofs.se/go-otel-debug:${{ needs.Metadata.outputs.version }}
-  #         # kubectl rollout status deployment/cart-actor-arm64 -n cart

alerts.json

@@ -0,0 +1,2 @@
+{"enabled":true,"dryRun":false,"channels":["telegram"],"lastRun":"2026-09-25T18:56:43.3470721Z","alerts":[{"id":"tlxng1-nginx","at":"2026-09-25T18:31:13.347300133Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:26:13.347300133Z","windowEnd":"2026-09-25T18:31:13.347300133Z","errorCount":69,"baselineMean":14.03448275862069,"excess":4.9655172413793105,"patterns":[{"signature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","count":4,"baselineMean":0.034482758620689655,"excess":3.9655172413793105,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:31:11.346792393Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"0751de83a2f0ce211f2b7ab1a64d8aba","PatternSignature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"auth.k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"34.177.93.136","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"fb485cd240e1dab0261891dc14170112bbc37c90189f276b45935afdedf534cb","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:31:06.252746363Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d05925e26f581fcba99d165ea5928114","PatternSignature":"fb485cd240e1dab0261891dc14170112bbc37c90189f276b45935afdedf534cb","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/oauth2/auth","Host":"users.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"auth.k6n.net","path":"/","services":["nginx"],"routes":[{"ingress":"dev/zitadel-ingress","host":"auth.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"zitadel-dev","port":"8080","deployments":["zitadel-dev"]}],"pods":[{"namespace":"dev","deployment":"zitadel-dev","found":false}],"rule":"status-5xx-upstream","new":false,"count":4,"baseline":0.034482758620689655,"clients":1,"what":"HTTP 503 | /"},{"category":"client_credential_issue","priority":"low","status":401,"host":"users.k6n.net","path":"/oauth2/auth","services":["nginx"],"routes":[{"ingress":"ecommerce/ecommerce-auth","host":"users.k6n.net","path":"/","pathType":"Prefix","namespace":"ecommerce","service":"auth","port":"8080","deployments":["auth"]}],"pods":[{"namespace":"ecommerce","deployment":"auth","pod":"auth-8f497c689-qfpzb","lines":["2026/09/25 18:13:06 oidc: device grant JHMJ-C23E started for client dev-console","2026/09/25 18:13:06 oidc: device grant EQ5R-UWRW approved by mats@tornberg.me","2026/09/25 18:23:19 oidc: device grant RHMN-6257 started for client dev-console"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /oauth2/auth"}]},{"id":"tlxn9d-nginx","at":"2026-09-25T18:27:13.347226389Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:22:13.347226389Z","windowEnd":"2026-09-25T18:27:13.347226389Z","errorCount":64,"baselineMean":13.931034482758621,"excess":4,"patterns":[{"signature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","count":4,"baselineMean":0,"excess":4,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:26:26.152736211Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d1de7de09ad342e7b0670cb78cac1430","PatternSignature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"cc.k6n.net","StatusCode":405,"ExceptionType":"","ClientIP":"35.189.233.235","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"needs_review","priority":"low","status":405,"host":"cc.k6n.net","path":"/","services":["nginx"],"rule":"no-rule","new":false,"count":4,"baseline":0,"clients":1,"what":"HTTP 405 | /"}]},{"id":"tlxn61-nginx","at":"2026-09-25T18:25:13.346661147Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:20:13.346661147Z","windowEnd":"2026-09-25T18:25:13.346661147Z","errorCount":60,"baselineMean":14.03448275862069,"excess":1,"patterns":[{"signature":"ea444cb83ff471aad7f3fab8a5b36dddb25f333f7dce2eb4ff27f6e9bb3969f7","count":1,"baselineMean":0,"excess":1,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:24:36.217473837Z","Service":"nginx","Release":"","Kind":"span","SourcLine truncated
+

build.sh

@@ -5,10 +5,6 @@ RELEASE="${RELEASE:-$(git rev-parse --short HEAD)}"
 PUBLIC_SOURCEMAPS="${PUBLIC_SOURCEMAPS:-false}"
 UPLOAD_SOURCEMAPS="${UPLOAD_SOURCEMAPS:-true}"
 
-echo "Building Needle 3 sidecar image..."
-docker build -t matst80/needle-sidecar:latest cmd/needle-sidecar
-docker push matst80/needle-sidecar:latest
-
 echo "Building otel-debug receiver image..."
 docker build \
   --build-arg "APP_RELEASE=${RELEASE}" \

cmd/laya-eval/main.go

@@ -0,0 +1,144 @@
+// laya-eval runs alert scenarios through laya and compares its shadow verdict
+// with what a person judged the right response to be, so the model can be
+// assessed before it is ever allowed to act.
+//
+//	go run ./cmd/laya-eval                        # built-in scenarios
+//	go run ./cmd/laya-eval -file alerts.json      # a GET /api/alerts dump (no expectations)
+//	go run ./cmd/laya-eval -send                  # also deliver each as a TEST alert
+//
+// -send uses the channels configured by OTEL_ALERT_* (see alerts.ConfigFromEnv).
+package main
+
+import (
+	"context"
+	_ "embed"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"text/tabwriter"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/alerts"
+)
+
+//go:embed scenarios.json
+var builtin []byte
+
+// Scenario is one alert and the response a person would pick for it.
+type Scenario struct {
+	Name   string       `json:"name"`
+	Expect *Expectation `json:"expect,omitempty"`
+	Alert  alerts.Alert `json:"alert"`
+}
+
+type Expectation struct {
+	Action string `json:"action"`
+	Human  bool   `json:"human"`
+	Why    string `json:"why"`
+}
+
+func main() {
+	url := flag.String("url", envOr("OTEL_ALERT_LAYA_URL", "https://laya.k6n.net"), "laya base URL")
+	file := flag.String("file", "", "scenarios JSON, or a GET /api/alerts response (default: built-in scenarios)")
+	send := flag.Bool("send", false, "deliver each scenario, with its verdict, as a TEST alert to the configured channels")
+	flag.Parse()
+
+	raw := builtin
+	if *file != "" {
+		var err error
+		if raw, err = os.ReadFile(*file); err != nil {
+			fail(err)
+		}
+	}
+	scenarios, err := parse(raw)
+	if err != nil {
+		fail(err)
+	}
+	var notifiers []alerts.Notifier
+	if *send {
+		cfg := alerts.ConfigFromEnv()
+		if notifiers = cfg.Notifiers(); len(notifiers) == 0 {
+			fail(fmt.Errorf("-send: no channel configured (OTEL_ALERT_TELEGRAM_TOKEN + OTEL_ALERT_TELEGRAM_CHAT_ID, OTEL_ALERT_WEBHOOK_URL or OTEL_ALERT_AOD_URL)"))
+		}
+	}
+
+	laya := &alerts.Laya{URL: *url}
+	tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(tw, "SCENARIO\tLAYA ACTION\tHUMAN\tURGENCY\tEXPECTED\tMATCH")
+	judged, actionOK, humanOK := 0, 0, 0
+	for _, s := range scenarios {
+		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+		v, err := laya.Classify(ctx, s.Alert)
+		cancel()
+		if err != nil {
+			fmt.Fprintf(tw, "%s\terror: %v\t\t\t\t\n", s.Name, err)
+			continue
+		}
+		exp, match := "-", "-"
+		if s.Expect != nil {
+			judged++
+			a, h := v.Action == s.Expect.Action, (v.Human >= 0.5) == s.Expect.Human
+			if a {
+				actionOK++
+			}
+			if h {
+				humanOK++
+			}
+			exp = fmt.Sprintf("%s, human=%v", s.Expect.Action, s.Expect.Human)
+			match = fmt.Sprintf("action %s, human %s", mark(a), mark(h))
+		}
+		fmt.Fprintf(tw, "%s\t%s %.0f%%\t%.0f%%\t%s %.0f%%\t%s\t%s\n", s.Name, v.Action, v.ActionConfidence*100, v.Human*100, v.Urgency, v.UrgencyConf*100, exp, match)
+
+		if *send {
+			a := s.Alert
+			a.Test, a.Laya = true, v
+			for _, n := range notifiers {
+				sctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+				if err := n.Send(sctx, a); err != nil {
+					fmt.Fprintf(os.Stderr, "%s: %s: %v\n", s.Name, n.Name(), err)
+				}
+				cancel()
+			}
+		}
+	}
+	tw.Flush()
+	if judged > 0 {
+		fmt.Printf("\nagreement over %d judged scenarios: action %d/%d, needs-human %d/%d\n", judged, actionOK, judged, humanOK, judged)
+	}
+}
+
+// parse accepts a scenario list or a raw GET /api/alerts response.
+func parse(raw []byte) ([]Scenario, error) {
+	var list []Scenario
+	if err := json.Unmarshal(raw, &list); err == nil {
+		return list, nil
+	}
+	var st alerts.Status
+	if err := json.Unmarshal(raw, &st); err != nil {
+		return nil, fmt.Errorf("neither a scenario list nor an /api/alerts response: %w", err)
+	}
+	for _, a := range st.Alerts {
+		list = append(list, Scenario{Name: a.ID + " " + a.Spike.Service, Alert: a})
+	}
+	return list, nil
+}
+
+func mark(ok bool) string {
+	if ok {
+		return "ok"
+	}
+	return "MISS"
+}
+
+func envOr(k, def string) string {
+	if v := os.Getenv(k); v != "" {
+		return v
+	}
+	return def
+}
+
+func fail(err error) {
+	fmt.Fprintln(os.Stderr, err)
+	os.Exit(1)
+}

cmd/needle-sidecar/Dockerfile

@@ -1,28 +0,0 @@
-FROM python:3.12-slim
-
-# Create non-root user with UID 1000 matching Kubernetes pod securityContext
-RUN groupadd -g 1000 appuser && useradd -u 1000 -g appuser -m -d /app appuser
-
-WORKDIR /app
-
-ENV HOME=/app
-ENV NEEDLE_PORT=5005
-ENV NEEDLE_HOST=0.0.0.0
-ENV NEEDLE_TELEMETRY=0
-
-# Install dependencies
-RUN pip install --no-cache-dir cactus-needle
-
-# Pre-cache model and shared library into /app/.cache as user 1000
-RUN python -c "import needle; needle.Needle()"
-
-COPY server.py /app/server.py
-
-# Ensure UID 1000 owns /app and all caches
-RUN chown -R appuser:appuser /app
-
-USER appuser
-
-EXPOSE 5005
-
-CMD ["python", "/app/server.py"]

cmd/needle-sidecar/eval/rules.py

@@ -1,42 +0,0 @@
-"""Deterministic baseline for the cause categories. Needle has to beat this."""
-import re
-
-PROBE = re.compile(r"(^|/)(\.env|\.git|wp-|wp/|wp$|wordpress|blog/wp|phpmyadmin|cgi-bin|\.aws|\.ssh|xmlrpc|userfiles|@fs/|etc/passwd)", re.I)
-DEP = re.compile(r"dial|connection refused|eof|no such host|upstream|unavailable|broken pipe|reset by peer", re.I)
-CAP = re.compile(r"timeout|deadline exceeded|pool|memory limit|oom|too many|queue full|exhaust", re.I)
-BUG = re.compile(r"panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault", re.I)
-LIFE = re.compile(r"healthz|readyz|/ping|retry|cancel|keepalive|vital|\bcls\b|ttfb|fcp|lcp|snapshot|replay|unchanged|session\.load", re.I)
-
-
-def fields(text):
-    d = {}
-    for part in text.split(" | "):
-        k, _, v = part.partition(": ")
-        d[k.lower()] = v
-    return d
-
-
-def classify(text):
-    f = fields(text)
-    status = int(f.get("status") or 0)
-    route = f.get("route", "")
-    blob = " ".join([f.get("title", ""), f.get("message", ""), route])
-    if PROBE.search(route):
-        return "scanner_probe"
-    if BUG.search(blob):
-        return "service_defect"
-    if CAP.search(blob):
-        return "capacity_saturation"
-    if DEP.search(blob):
-        return "dependency_unreachable"
-    if status == 401:
-        return "client_credential_issue"
-    if status in (502, 503, 504):
-        return "dependency_unreachable"
-    if status >= 500:
-        return "service_defect"
-    if status == 404:
-        return "stale_reference"
-    if LIFE.search(blob) or status in (200, 202, 204, 301, 302, 304, 308, 499) or (status == 0 and f.get("kind") in ("span", "log")):
-        return "expected_lifecycle"
-    return "needs_review"

cmd/needle-sidecar/eval/run_eval.py

@@ -1,79 +0,0 @@
-#!/usr/bin/env python3
-"""Evaluate the Needle sidecar against a reviewed golden set.
-
-Usage: run_eval.py [--url http://127.0.0.1:5005] [--golden eval/golden.jsonl] [--json out.json]
-
-Each golden row: {"id", "expected", "text"}; `expected` is a sidecar action name
-including "needs_review" for the fallback.
-"""
-import argparse, json, os, sys, time, urllib.request
-from collections import Counter, defaultdict
-
-HERE = os.path.dirname(os.path.abspath(__file__))
-
-
-def classify(url, text):
-    req = urllib.request.Request(
-        url.rstrip("/") + "/classify",
-        data=json.dumps({"text": text}).encode(),
-        headers={"Content-Type": "application/json"},
-    )
-    with urllib.request.urlopen(req, timeout=60) as r:
-        return json.load(r)
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--url", default=os.environ.get("OTEL_NEEDLE_ADDR", "http://127.0.0.1:5005"))
-    ap.add_argument("--golden", default=os.path.join(HERE, "golden.jsonl"))
-    ap.add_argument("--json", help="write per-row results here")
-    ap.add_argument("--min-conf", type=float, default=0.0,
-                    help="predictions below this confidence count as needs_review")
-    a = ap.parse_args()
-
-    rows = [json.loads(l) for l in open(a.golden) if l.strip()]
-    results, confusion = [], Counter()
-    lat = []
-    for row in rows:
-        try:
-            out = classify(a.url, row["text"])
-        except Exception as e:
-            print(f"sidecar error on {row['id']}: {e}", file=sys.stderr)
-            sys.exit(2)
-        pred = out.get("action", "needs_review")
-        conf = out.get("confidence", 0.0)
-        if conf < a.min_conf:
-            pred = "needs_review"
-        lat.append(out.get("latency_ms", 0.0))
-        confusion[(row["expected"], pred)] += 1
-        results.append({**row, "predicted": pred, "confidence": conf, "ok": pred == row["expected"]})
-
-    labels = sorted({r["expected"] for r in results} | {r["predicted"] for r in results})
-    print(f"{'label':<18}{'prec':>7}{'recall':>8}{'support':>9}")
-    for l in labels:
-        tp = confusion[(l, l)]
-        pp = sum(v for (e, p), v in confusion.items() if p == l)
-        sup = sum(v for (e, p), v in confusion.items() if e == l)
-        prec = tp / pp if pp else 0.0
-        rec = tp / sup if sup else 0.0
-        print(f"{l:<18}{prec:>7.2f}{rec:>8.2f}{sup:>9}")
-
-    acc = sum(r["ok"] for r in results) / len(results)
-    print(f"\naccuracy {acc:.2%} over {len(results)} rows; "
-          f"median latency {sorted(lat)[len(lat)//2]:.0f}ms")
-
-    for src in sorted({r.get("src", "?") for r in results}):
-        sub = [r for r in results if r.get("src", "?") == src]
-        print(f"  {src:<11} {sum(r['ok'] for r in sub)}/{len(sub)}")
-
-    wrong = [r for r in results if not r["ok"]]
-    if wrong:
-        print("\nmisses:")
-        for r in wrong:
-            print(f"  {r['id']:<8} expected={r['expected']:<16} got={r['predicted']:<16} conf={r['confidence']:.2f}")
-    if a.json:
-        json.dump(results, open(a.json, "w"), indent=1)
-
-
-if __name__ == "__main__":
-    main()

cmd/needle-sidecar/server.py

@@ -1,240 +0,0 @@
-#!/usr/bin/env python3
-"""
-Needle 3 AI Telemetry Triage Sidecar Service
-Accepts OTel incident summaries and uses Cactus-Compute/needle3 to classify and extract actionable insights.
-"""
-
-import os
-import sys
-import json
-import time
-import re
-import hashlib
-from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
-import needle
-
-PORT = int(os.environ.get("NEEDLE_PORT", 5005))
-HOST = os.environ.get("NEEDLE_HOST", "0.0.0.0")
-
-# --- Cause-level triage tools ---
-# These deliberately do not restate what the event already carries (status
-# code, is_error, severity). Each one is a judgement about *why* it happened
-# or *who should care*, which is what an alert reader needs.
-@needle.tool
-def report_scanner_probe(service: str, path: str, status_code: int):
-    """Automated scan or exploit crawler requesting sensitive or nonexistent paths such as .env, wp-admin, .git, phpmyadmin. Applies whatever status the probe received, including 500 or 502."""
-    return {"action": "scanner_probe", "service": service, "path": path, "status_code": status_code}
-
-@needle.tool
-def report_dependency_unreachable(service: str, dependency: str, error_message: str):
-    """A downstream or upstream service is down, refusing connections, closed the stream (EOF), or returned upstream unavailable."""
-    return {"action": "dependency_unreachable", "service": service, "dependency": dependency, "error": error_message}
-
-@needle.tool
-def report_client_credential_issue(service: str, route: str, reason: str):
-    """A caller lacks a valid or unexpired token, for example an unauthenticated client polling an API or a registry pull without login. The caller's fault, not a fault in this service."""
-    return {"action": "client_credential_issue", "service": service, "route": route, "reason": reason}
-
-@needle.tool
-def report_stale_reference(service: str, path: str):
-    """Harmless missing resource caused by an old link, removed asset, deleted commit, or dev tooling request."""
-    return {"action": "stale_reference", "service": service, "path": path}
-
-@needle.tool
-def report_service_defect(service: str, error_message: str):
-    """A bug in the service's own code: panic, nil dereference, unhandled exception, index out of range, deadlock, or a 500 from a real application route."""
-    return {"action": "service_defect", "service": service, "error": error_message}
-
-@needle.tool
-def report_capacity_saturation(service: str, resource: str, error_message: str):
-    """Resource exhaustion or overload: timeouts, connection pool exhausted, memory limit exceeded, OOM, queue full."""
-    return {"action": "capacity_saturation", "service": service, "resource": resource, "error": error_message}
-
-@needle.tool
-def report_expected_lifecycle(service: str, what: str):
-    """Normal behaviour that is not a problem: health and readiness checks, keepalive pings, retries, cancellations, client disconnects, redirects, successful requests, routine info logs, web vitals."""
-    return {"action": "expected_lifecycle", "service": service, "what": what}
-
-@needle.tool
-def report_needs_review(service: str, summary: str):
-    """Fallback when none of the other categories clearly applies."""
-    return {"action": "needs_review", "service": service, "summary": summary}
-
-# --- Redaction: nothing PII-bearing should reach the model or the cache ---
-_REDACTIONS = [
-    (re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+"), "<email>"),
-    (re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), "<ip>"),
-    (re.compile(r"\b[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}\b"), "<uuid>"),
-    (re.compile(r"\beyJ[\w-]+\.[\w-]+\.[\w-]+\b"), "<jwt>"),
-    (re.compile(r"\b[0-9a-fA-F]{16,}\b"), "<hex>"),
-    (re.compile(r"([?&][\w.-]+=)[^&\s|]+"), r"\1<v>"),
-]
-
-def redact(text: str) -> str:
-    for rx, sub in _REDACTIONS:
-        text = rx.sub(sub, text)
-    return text[:1000]
-
-# Cache keyed on the redacted text: repeated signatures cost nothing.
-CACHE_MAX = int(os.environ.get("NEEDLE_CACHE_MAX", 2000))
-_cache = {}
-_stats = {"hits": 0, "misses": 0}
-
-# Initialize Needle Agent once
-print(f"Loading Needle 3 model (Cactus-Compute/needle3)...")
-t0 = time.perf_counter()
-agent = needle.Needle(tools=[
-    report_scanner_probe,
-    report_dependency_unreachable,
-    report_client_credential_issue,
-    report_stale_reference,
-    report_service_defect,
-    report_capacity_saturation,
-    report_expected_lifecycle,
-    report_needs_review,
-])
-print(f"Needle 3 loaded in {(time.perf_counter() - t0):.2f}s")
-
-
-class NeedleHandler(BaseHTTPRequestHandler):
-    def _send_cors(self):
-        self.send_header("Access-Control-Allow-Origin", "*")
-        self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
-        self.send_header("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Otel-Agent-Token")
-
-    def do_OPTIONS(self):
-        self.send_response(204)
-        self._send_cors()
-        self.end_headers()
-
-    def do_GET(self):
-        if self.path == "/health" or self.path == "/api/needle/status":
-            self.send_response(200)
-            self._send_cors()
-            self.send_header("Content-Type", "application/json")
-            self.end_headers()
-            resp = {
-                "status": "ok",
-                "available": True,
-                "model": "Cactus-Compute/needle3",
-                "cache": {**_stats, "size": len(_cache)},
-                "tools": [
-                    "scanner_probe", "dependency_unreachable", "client_credential_issue", "stale_reference",
-                    "service_defect", "capacity_saturation", "expected_lifecycle", "needs_review"
-                ]
-            }
-            self.wfile.write(json.dumps(resp).encode("utf-8"))
-        else:
-            self.send_response(404)
-            self._send_cors()
-            self.end_headers()
-
-    def do_POST(self):
-        if self.path in ("/classify", "/api/needle/classify"):
-            content_len = int(self.headers.get("Content-Length", 0))
-            body = self.rfile.read(content_len)
-            try:
-                data = json.loads(body) if body else {}
-            except Exception as e:
-                self.send_response(400)
-                self._send_cors()
-                self.send_header("Content-Type", "application/json")
-                self.end_headers()
-                self.wfile.write(json.dumps({"error": f"Invalid JSON: {e}"}).encode("utf-8"))
-                return
-
-            text = data.get("text", "").strip()
-            # If text is not provided, build from structured fields:
-            if not text:
-                parts = []
-                if data.get("service"):
-                    parts.append(f"Service: {data['service']}")
-                if data.get("kind"):
-                    parts.append(f"Kind: {data['kind']}")
-                if data.get("title"):
-                    parts.append(f"Title: {data['title']}")
-                if data.get("message"):
-                    parts.append(f"Message: {data['message']}")
-                if data.get("url"):
-                    parts.append(f"URL: {data['url']}")
-                if data.get("status_code"):
-                    parts.append(f"Status: {data['status_code']}")
-                text = " | ".join(parts)
-
-            if not text:
-                self.send_response(400)
-                self._send_cors()
-                self.send_header("Content-Type", "application/json")
-                self.end_headers()
-                self.wfile.write(json.dumps({"error": "Missing 'text' or telemetry fields"}).encode("utf-8"))
-                return
-
-            text = redact(text)
-            key = hashlib.sha256(text.encode()).hexdigest()
-            cached = _cache.get(key)
-            if cached is not None:
-                _stats["hits"] += 1
-                out = {**cached, "cached": True}
-                self.send_response(200)
-                self._send_cors()
-                self.send_header("Content-Type", "application/json")
-                self.end_headers()
-                self.wfile.write(json.dumps(out).encode("utf-8"))
-                return
-            _stats["misses"] += 1
-
-            t_start = time.perf_counter()
-            agent.reset()
-            res = agent.run(text, strict=False)
-            latency_ms = (time.perf_counter() - t_start) * 1000
-
-            results = res.get("results", [])
-            first_result = results[0] if results and isinstance(results[0], dict) else {}
-            action = first_result.get("action", "needs_review")
-
-            calls = res.get("function_calls", [])
-            tool_name = calls[0].get("name") if calls else "triage_tool"
-
-            out = {
-                "success": True,
-                "input_text": text,
-                "action": action,
-                "tool": tool_name,
-                "arguments": first_result,
-                "confidence": float(res.get("confidence", 0.0)),
-                "reasoning": res.get("reasoning", ""),
-                "latency_ms": round(latency_ms, 2),
-                "model": "Cactus-Compute/needle3"
-            }
-
-            if len(_cache) >= CACHE_MAX:
-                _cache.pop(next(iter(_cache)))
-            _cache[key] = out
-
-            self.send_response(200)
-            self._send_cors()
-            self.send_header("Content-Type", "application/json")
-            self.end_headers()
-            self.wfile.write(json.dumps(out).encode("utf-8"))
-        else:
-            self.send_response(404)
-            self._send_cors()
-            self.end_headers()
-
-    def log_message(self, format, *args):
-        # Concise logging
-        print(f"[needle-sidecar] {args[0]} {args[1]} -> {args[2]}")
-
-
-def run():
-    server = ThreadingHTTPServer((HOST, PORT), NeedleHandler)
-    print(f"[needle-sidecar] Listening on http://{HOST}:{PORT}")
-    try:
-        server.serve_forever()
-    except KeyboardInterrupt:
-        print("\nShutting down needle sidecar...")
-        server.server_close()
-
-
-if __name__ == "__main__":
-    run()

cmd/server/main.go

@@ -490,8 +490,6 @@ func main() {
 	uiMux.HandleFunc("POST /api/sourcemaps", coll.HandleSourceMap)
 	uiMux.Handle("GET /api/sourcemaps", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleListSourceMaps)))
 	uiMux.Handle("POST /api/sourcemaps/resolve", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleResolveStacks)))
-	uiMux.HandleFunc("GET /api/needle/status", coll.HandleNeedleStatus)
-	uiMux.HandleFunc("POST /api/needle/classify", coll.HandleNeedleClassify)
 
 	// Protected API & WebSocket routes
 	uiMux.Handle("/api/history", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleHistory)))

cmd/spike-alerts/main.go

@@ -1,16 +1,13 @@
 // spike-alerts replays the deterministic spike rule over a bounded window of
-// normalized_events and, optionally, asks the Needle sidecar what kind of
-// failure each top pattern is. It only reads ClickHouse.
+// normalized_events. It only reads ClickHouse.
 package main
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"log"
-	"net/http"
 	"os"
 	"strings"
 	"time"
@@ -36,7 +33,6 @@ func main() {
 	minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert")
 	multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline")
 	cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles")
-	needle := flag.String("needle", "", "Needle sidecar URL (dormant: it is not accurate enough to gate alerts); empty skips it")
 	showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)")
 	asJSON := flag.Bool("json", false, "emit JSON lines instead of text")
 	flag.Parse()
@@ -75,7 +71,6 @@ func main() {
 	log.Printf("bad-actor events (ignored for alerts, kept as data): %v of %d", badActors, len(events))
 	log.Printf("loaded %d error events", len(events))
 
-	cache := map[string]classification{}
 	first := start.Add(time.Duration(*baselineWindows) * *window)
 	suppress := features.NewSuppressor(*cooldown)
 	muted := map[triage.Category]int{}
@@ -87,24 +82,11 @@ func main() {
 		}
 		for _, spike := range suppress.Filter(detected) {
 			fired++
-			var cls []classification
-			for _, p := range spike.Patterns {
-				c := classification{}
-				if *needle != "" {
-					if hit, ok := cache[p.Signature]; ok {
-						c = hit
-					} else {
-						c = classify(*needle, needleText(p.Example))
-						cache[p.Signature] = c
-					}
-				}
-				cls = append(cls, c)
-			}
 			if *asJSON {
-				_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike, "classifications": cls})
+				_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike})
 				continue
 			}
-			printSpike(spike, cls)
+			printSpike(spike)
 		}
 	}
 	for c, n := range muted {
@@ -113,28 +95,7 @@ func main() {
 	log.Printf("%d alert(s) between %s and %s", fired, first.Format(time.RFC3339), end.Format(time.RFC3339))
 }
 
-type classification struct {
-	Action     string  `json:"action"`
-	Confidence float64 `json:"confidence"`
-	Cached     bool    `json:"cached"`
-	Err        string  `json:"error,omitempty"`
-}
-
-func classify(base, text string) classification {
-	body, _ := json.Marshal(map[string]string{"text": text})
-	resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body))
-	if err != nil {
-		return classification{Action: "?", Err: err.Error()}
-	}
-	defer resp.Body.Close()
-	var out classification
-	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
-		return classification{Action: "?", Err: err.Error()}
-	}
-	return out
-}
-
-func needleText(e features.Event) string {
+func eventText(e features.Event) string {
 	parts := []string{"Service: " + e.Service, "Kind: " + e.Kind}
 	if e.SpanName != "" {
 		parts = append(parts, "Title: "+e.SpanName)
@@ -151,17 +112,14 @@ func needleText(e features.Event) string {
 	return strings.Join(parts, " | ")
 }
 
-func printSpike(s features.Spike, cls []classification) {
+func printSpike(s features.Spike) {
 	fmt.Printf("\n%s  %s  errors=%d (baseline %.1f/window, excess %.0f)\n", s.WindowEnd.Format("01-02 15:04"), s.Service, s.ErrorCount, s.BaselineMean, s.Excess)
-	for i, p := range s.Patterns {
+	for _, p := range s.Patterns {
 		tag := "rising"
 		if p.Novel {
 			tag = "NEW"
 		}
 		what := string(alerts.TriageOf(p.Example).Category)
-		if cls[i].Action != "" {
-			what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence)
-		}
 		clients := ""
 		if p.Clients > 0 {
 			clients = fmt.Sprintf(" [%d client%s]", p.Clients, map[bool]string{true: "", false: "s"}[p.Clients == 1])
@@ -169,7 +127,7 @@ func printSpike(s features.Spike, cls []classification) {
 		if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) {
 			clients += " [" + p.Example.UAClass + "]"
 		}
-		fmt.Printf("  %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example), clients)
+		fmt.Printf("  %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, eventText(p.Example), clients)
 	}
 }
 

docs/ML.md

@@ -1,7 +1,7 @@
 # Error detection model
 
 > Latest status, decisions and next steps: [STATUS.md](STATUS.md). Where they
-> differ (ClickHouse is now a LoadBalancer; Needle is dormant), STATUS.md wins.
+> differ (ClickHouse is now a LoadBalancer; Needle is removed), STATUS.md wins.
 
 ## Current handoff (2026-09-18)
 

docs/STATUS.md

@@ -10,7 +10,6 @@ unless stated. Greenfield, single user: radical changes are fine.
   patched live. `docs/ML.md` still says "keep ClusterIP"; that is superseded.
 - Credentials are in `.env` (gitignored, mode 600). Load with
   `set -a; . ./.env; set +a`. `clickhouse-client` is installed locally.
-- `.env` deliberately does not enable Needle; pass `-needle URL` explicitly.
 
 ## What exists now
 
@@ -21,7 +20,7 @@ unless stated. Greenfield, single user: radical changes are fine.
 | 404/410 are not errors | `isErrorStatus` in `internal/normalization/normalizer.go` | Done. Rows stored before this still count 404s as errors. |
 | Backfill | `cmd/normalized-backfill` | Ran 2026-09-19 12:00 to ~21:00 UTC. Queries need `FINAL` (duplicates until merge). |
 | Spike rule | `internal/features/spike.go` | Excess-error rule, trimmed-mean baseline, `Suppressor` (cooldown, 2x escalation). Tested. |
-| Replay CLI | `cmd/spike-alerts` | Replays a range; mutes non-actionable categories; Needle optional and off. |
+| Replay CLI | `cmd/spike-alerts` | Replays a range; mutes non-actionable categories. |
 | Cause categories | `internal/triage` | Deterministic rules, first match wins. Tested. |
 | Rules scoring | `cmd/triage-eval` | Scores a labelled JSONL; also prepares rows for labelling. |
 | Offender report | `cmd/bad-actors` | Read-only. Ranks IPs from raw nginx spans, marks `[BLOCK]` / `[INTERNAL]`. |
@@ -35,14 +34,14 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue`
 
 ## Findings worth remembering
 
-- **Needle scored 10% on the 40-row golden set** (about chance for 8 classes).
-  Reversing the tool order changed its answers wholesale (15%, 34/40 `service_defect`),
-  so it picks by position, not content. Confidence is uncorrelated with
-  correctness. Decision: **Needle stays dormant.** Intended later use: a gate that
-  decides whether a larger model should act on an issue (affects anyone? fixable by
-  a smart model?). Do not use it for categorization.
+- **Needle was removed (2026-11-01).** It scored 10% on the 40-row golden set (about
+  chance for 8 classes), picked by tool position rather than content, and its
+  confidence was uncorrelated with correctness. Sidecar, proxy handlers, UI and k8s
+  container are gone; the labelled sets live in `cmd/triage-eval/data/`.
+- **Edge 4xx never pages.** nginx 4xx (registry 401 challenges, `/oauth2/auth`,
+  scanner 405s) is dropped in `FilterActionable`; nginx 5xx still alerts.
 - **Rules scored 39/40 on the golden set, but that is a fit** (written after seeing
-  the rows). A fresh 60-row holdout is in `cmd/needle-sidecar/eval/holdout.jsonl`
+  the rows). A fresh 60-row holdout is in `cmd/triage-eval/data/holdout.jsonl`
   (from 2026-09-18, unseen); it still needs your `expected` labels. Known
   probe-like paths that still land in `stale_reference`: `/ssl/server.key`,
   `/_debugbar/open`, `/debug/pprof`, `/actuator/*`, `/graphql`, `/server-info`,
@@ -78,7 +77,7 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue`
 1. Deploy the collector so normalized rows stay current; verify with
    `SELECT max(timestamp) FROM otel.normalized_events`.
 2. Label `holdout.jsonl` (fill `expected`), run
-   `go run ./cmd/triage-eval -input cmd/needle-sidecar/eval/holdout.jsonl`, fix
+   `go run ./cmd/triage-eval -input cmd/triage-eval/data/holdout.jsonl`, fix
    the rule families that miss. Then re-export a fresh holdout so scoring stays
    honest.
 3. Warm-up guard in `DetectSpikes`: require baseline data before alerting (the
@@ -164,7 +163,7 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue`
 ```bash
 set -a; . ./.env; set +a
 
-# replay the spike rule over a range (Needle off)
+# replay the spike rule over a range
 go run ./cmd/spike-alerts -since 2026-09-19T12:00:00Z -until 2026-09-19T20:30:00Z -baseline-windows 12
 # add -show-muted to see probes / stale / lifecycle patterns too
 
@@ -172,12 +171,7 @@ go run ./cmd/spike-alerts -since 2026-09-19T12:00:00Z -until 2026-09-19T20:30:00
 go run ./cmd/bad-actors -since 2026-09-19T08:00:00Z -until 2026-09-19T20:30:00Z
 
 # score the rules against labelled rows
-go run ./cmd/triage-eval -input cmd/needle-sidecar/eval/holdout.jsonl
-
-# Needle eval (needs the sidecar running; venv was in scratch and is gone)
-pip install cactus-needle   # python 3.12 recommended
-NEEDLE_PORT=5005 NEEDLE_HOST=127.0.0.1 python cmd/needle-sidecar/server.py
-python cmd/needle-sidecar/eval/run_eval.py
+go run ./cmd/triage-eval -input cmd/triage-eval/data/holdout.jsonl
 ```
 
 Backfill in small windows only (10 to 30 minutes over the LAN); one-hour windows
@@ -187,14 +181,11 @@ through a `kubectl port-forward` hung and dropped.
 
 Everything is uncommitted. Files changed by this work: `internal/{storage,
 normalization,features,triage,honeypot,blocker}`, `cmd/{spike-alerts,
-triage-eval,bad-actors,normalized-backfill,server,needle-sidecar}`,
+triage-eval,bad-actors,normalized-backfill,server}`,
 `k8s/{clickhouse,deployment}.yaml`, `.gitignore`.
 
 **Not from this work; check before committing:** `AGENTS.md`,
-`internal/collector/{collector_test,issue_extract}.go`, `otel-ui/src/**`
-(including a deleted `GaugeMeter.tsx`). Also tracked `__pycache__/*.pyc` files
-under `cmd/needle-sidecar` show as modified; they should be untracked and
-gitignored.
+`internal/collector/{collector_test,issue_extract}.go`, `otel-ui/src/**` (including a deleted `GaugeMeter.tsx`).
 
 ## IP details (MaxMind)
 

gitops/README.md

@@ -0,0 +1,29 @@
+# Deploying go-otel with k8s-helper
+
+Gitea Actions builds and pushes versioned and `latest` `matst80/otel-debug`
+images on pushes to `main`, uploads the matching source maps, and then sends a
+signed reconcile webhook. The agent applies the explicit `k8s/kustomization.yaml`
+resource list into namespace `monitoring`.
+
+## One-time setup
+
+The k8s-helper GitOps CRDs and agent must be installed. The GitSource reuses
+`dev/gitea-k8s-helper`, whose token must have read access to this repository.
+Configure a Gitea push-only webhook for this repo at
+`https://ui.k6n.net/api/gitops/webhooks/gitea`, using the same secret as
+`K8SHELPER_GITOPS_WEBHOOK_SECRET` on the dashboard. Add that value as the Gitea
+Actions secret `GITOPS_WEBHOOK_SECRET` too; a user- or organization-level
+secret can be shared by other repos.
+
+The runner uses its existing Docker login. Bootstrap once with
+`kubectl apply -f gitops/` after confirming the monitoring namespace and the
+`otel-sourcemaps` PVC already exist. The PVC is deliberately excluded from the
+managed kustomization because its lifecycle was called out for separate review.
+The ClickHouse manifests, auth-dev resources, and ingress-catchall work are also
+excluded; this kustomization lists only the receiver Deployment, Service, and
+Ingress resources. Do not run Flux and k8s-helper against the same resources at
+the same time; check for an active `flux-system/go-otel` Kustomization before
+bootstrapping this controller.
+
+Prune is initially disabled. Inspect the GitOps inventory and applied revision
+before changing it.

gitops/deployment.yaml

@@ -0,0 +1,14 @@
+apiVersion: gitops.k6n.net/v1alpha1
+kind: Deployment
+metadata:
+  name: go-otel
+  namespace: dev
+spec:
+  sourceRef:
+    kind: GitSource
+    name: go-otel
+  path: ./k8s
+  targetNamespace: monitoring
+  prune: false
+  wait: true
+  deletionPolicy: Orphan

gitops/gitsource.yaml

@@ -0,0 +1,16 @@
+# Keep GitOps CRs outside the path they manage.
+apiVersion: gitops.k6n.net/v1alpha1
+kind: GitSource
+metadata:
+  name: go-otel
+  namespace: dev
+spec:
+  url: https://git.k6n.net/mats/go-otel.git
+  ref:
+    branch: main
+  interval: 5m
+  secretRef:
+    name: gitea-k8s-helper
+  ignore: |
+    .gitea/
+    docs/

index.html

@@ -141,45 +141,9 @@
           Clear Logs
         </button>
       </div>
-
-      <div class="data-section" style="border-left: 4px solid #10b981;">
-        <h2 style="border-bottom-color: #10b981; color: #065f46;">Needle 3 AI Telemetry Triage</h2>
-        <p style="font-size: 13px; color: #666; margin-top: 0;">Test on-device telemetry classification with Cactus-Compute/needle3:</p>
-        <div style="display: flex; gap: 8px; margin-bottom: 10px;">
-          <input id="needle-input" type="text" style="flex: 1; padding: 8px 12px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;" value="User reported client browser crash on service eventsourced-ai-web: Script error on url https://event-agent.k6n.net/#session=angry-koala-1" />
-          <button id="needle-btn" style="background: #10b981; color: white; border: none; padding: 8px 16px; border-radius: 4px; font-weight: bold; cursor: pointer;" onclick="classifyWithNeedle()">
-            Classify with Needle 3
-          </button>
-        </div>
-        <div id="needle-result" class="data-content" style="background: #ecfdf5; border: 1px solid #a7f3d0; min-height: 40px; color: #064e3b;">
-          Click "Classify with Needle 3" to test the sidecar...
-        </div>
-      </div>
     </div>
 
     <script>
-      async function classifyWithNeedle() {
-        const input = document.getElementById("needle-input").value;
-        const resultDiv = document.getElementById("needle-result");
-        const btn = document.getElementById("needle-btn");
-        if (!input.trim()) return;
-        btn.textContent = "Classifying...";
-        btn.disabled = true;
-        try {
-          const resp = await fetch("/api/needle/classify", {
-            method: "POST",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify({ text: input }),
-          });
-          const data = await resp.json();
-          resultDiv.innerHTML = `<pre style="margin: 0; font-family: monospace;">${JSON.stringify(data, null, 2)}</pre>`;
-        } catch (err) {
-          resultDiv.textContent = "Error: " + err;
-        } finally {
-          btn.textContent = "Classify with Needle 3";
-          btn.disabled = false;
-        }
-      }
       let ws;
       let tracesCount = 0;
       let metricsCount = 0;

internal/alerts/filter.go

@@ -11,6 +11,13 @@ func TriageOf(e features.Event) triage.Result {
 		IsTimeout: e.IsTimeout, IsCancellation: e.IsCancellation, IsRetry: e.IsRetry})
 }
 
+// edgeClientError reports a 4xx seen at the nginx edge: registry auth
+// challenges, /oauth2/auth subrequests, 405s from scanners. These are the
+// caller's problem and never page, even next to a real 5xx.
+func edgeClientError(e features.Event) bool {
+	return e.Service == "nginx" && e.StatusCode >= 400 && e.StatusCode < 500
+}
+
 // FilterActionable keeps only patterns whose category should page a human
 // (unless keepMuted), recomputing each spike's excess so a probe storm cannot
 // alert by itself. muted, when non-nil, counts what was dropped per category.
@@ -21,7 +28,7 @@ func FilterActionable(spikes []features.Spike, keepMuted bool, muted map[triage.
 		sp.Excess = 0
 		for _, p := range sp.Patterns {
 			c := TriageOf(p.Example).Category
-			if !keepMuted && !c.Actionable() {
+			if !keepMuted && (!c.Actionable() || edgeClientError(p.Example)) {
 				if muted != nil {
 					muted[c]++
 				}

internal/alerts/laya.go

@@ -0,0 +1,180 @@
+package alerts
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Laya asks the laya model (POST {URL}/predict) what it would do about an
+// alert. It runs in shadow mode: the verdict is recorded on the alert and
+// shown in notifications, but nothing is ever acted on, muted or dropped
+// because of it. A failed call leaves the alert exactly as it was.
+type Laya struct {
+	URL    string // base URL, e.g. https://laya.k6n.net
+	Client *http.Client
+}
+
+// LayaActions are the remediations laya may choose between. They are
+// suggestions only; none is wired to an automated action yet.
+var LayaActions = map[string]string{
+	"ignore":             "transient blip or noise, nobody needs to look at it",
+	"restart_deployment": "the service is stuck, leaking or crashed; restarting its pods is likely to clear it",
+	"check_capacity":     "requests time out or resources are exhausted; check load, connection pools, CPU/memory limits and scaling",
+	"fix_dependency":     "a database, upstream service or network path this service depends on is failing",
+	"investigate_code":   "a bug or bad release in this service; needs a code fix or a rollback",
+}
+
+var layaUrgency = []string{"cosmetic", "degraded performance", "service outage"}
+
+// LayaVerdict is laya's answer, reduced to what a person needs to judge it.
+type LayaVerdict struct {
+	Model            string             `json:"model,omitempty"`
+	Action           string             `json:"action"`
+	ActionConfidence float64            `json:"actionConfidence"` // probability of the chosen action
+	ActionProbs      map[string]float64 `json:"actionProbs,omitempty"`
+	Human            float64            `json:"human"` // probability a person must intervene
+	Urgency          string             `json:"urgency"`
+	UrgencyScore     float64            `json:"urgencyScore"` // expected value on 0 (cosmetic) .. 2 (outage)
+	UrgencyConf      float64            `json:"urgencyConfidence"`
+}
+
+func layaRequest(state string) map[string]any {
+	return map[string]any{
+		"state": state,
+		"questions": map[string]any{
+			"remediation_action": map[string]any{
+				"type":         "choice",
+				"instructions": "Which operational response fits this production error alert best?",
+				"criteria":     LayaActions,
+			},
+			"requires_human_attention": map[string]any{
+				"type":         "noul",
+				"instructions": "Does this alert need a human engineer to intervene, or will it resolve or matter little on its own?",
+				"criteria": map[string]string{
+					"true":  "errors are new or rising well above normal and users or requests are failing",
+					"false": "a transient blip, client noise or bot traffic",
+				},
+			},
+			"urgency": map[string]any{
+				"type":         "score",
+				"instructions": "Rate the operational severity of this alert.",
+				"criteria":     layaUrgency,
+			},
+		},
+	}
+}
+
+type layaAnswer struct {
+	Choice           string             `json:"choice"`
+	Probabilities    map[string]float64 `json:"probabilities"`
+	AnswerConfidence float64            `json:"answer_confidence"`
+	Noul             float64            `json:"noul"`
+	Score            float64            `json:"score"`
+	Legend           map[string]string  `json:"legend"`
+}
+
+type layaResponse struct {
+	Model   string                `json:"model"`
+	Answers map[string]layaAnswer `json:"answers"`
+}
+
+// Classify asks laya about the alert. The state is the alert's plain-text
+// message, so the model sees exactly the facts a person would, nothing more.
+func (l *Laya) Classify(ctx context.Context, a Alert) (*LayaVerdict, error) {
+	a.Laya = nil
+	return l.classifyState(ctx, a.Text(""))
+}
+
+func (l *Laya) classifyState(ctx context.Context, state string) (*LayaVerdict, error) {
+	raw, err := json.Marshal(layaRequest(state))
+	if err != nil {
+		return nil, err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(l.URL, "/")+"/predict", bytes.NewReader(raw))
+	if err != nil {
+		return nil, errors.New("laya: invalid URL")
+	}
+	req.Header.Set("Content-Type", "application/json")
+	client := l.Client
+	if client == nil {
+		client = &http.Client{Timeout: 10 * time.Second}
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("laya: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode/100 != 2 {
+		detail, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
+		return nil, fmt.Errorf("laya: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(detail)))
+	}
+	var lr layaResponse
+	if err := json.NewDecoder(resp.Body).Decode(&lr); err != nil {
+		return nil, fmt.Errorf("laya: invalid response: %w", err)
+	}
+	return lr.verdict()
+}
+
+func (lr layaResponse) verdict() (*LayaVerdict, error) {
+	act, ok1 := lr.Answers["remediation_action"]
+	hum, ok2 := lr.Answers["requires_human_attention"]
+	urg, ok3 := lr.Answers["urgency"]
+	if !ok1 || !ok2 || !ok3 || act.Choice == "" {
+		return nil, errors.New("laya: response is missing answers")
+	}
+	v := &LayaVerdict{Model: lr.Model, Action: act.Choice, ActionConfidence: act.Probabilities[act.Choice], ActionProbs: act.Probabilities,
+		Human: hum.Noul, UrgencyScore: urg.Score}
+	// The urgency label is the most probable bucket, not the rounded score:
+	// a 50/50 cosmetic/outage split must not read as "degraded".
+	best := -1.0
+	for k, p := range urg.Probabilities {
+		if p > best || (p == best && k < v.Urgency) {
+			best, v.Urgency = p, k
+		}
+	}
+	v.UrgencyConf = best
+	if label, ok := urg.Legend[v.Urgency]; ok {
+		v.Urgency = label
+	} else if i, err := strconv.Atoi(v.Urgency); err == nil && i >= 0 && i < len(layaUrgency) {
+		v.Urgency = layaUrgency[i]
+	}
+	return v, nil
+}
+
+// Summary is a one-line rendering, e.g.
+// "check_capacity 62% · human 71% · degraded performance 75%".
+func (v LayaVerdict) Summary() string {
+	return fmt.Sprintf("%s %s · human %s · %s %s", v.Action, pct(v.ActionConfidence), pct(v.Human), v.Urgency, pct(v.UrgencyConf))
+}
+
+// RunnersUp lists the other actions by probability, for judging how close
+// the call was.
+func (v LayaVerdict) RunnersUp() string {
+	type kv struct {
+		k string
+		p float64
+	}
+	var rest []kv
+	for k, p := range v.ActionProbs {
+		if k != v.Action {
+			rest = append(rest, kv{k, p})
+		}
+	}
+	sort.Slice(rest, func(i, j int) bool { return rest[i].p > rest[j].p })
+	parts := make([]string, 0, len(rest))
+	for _, r := range rest {
+		parts = append(parts, r.k+" "+pct(r.p))
+	}
+	return strings.Join(parts, ", ")
+}
+
+func pct(p float64) string { return strconv.Itoa(int(p*100+0.5)) + "%" }

internal/alerts/laya_test.go

@@ -0,0 +1,123 @@
+package alerts
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/features"
+)
+
+// layaReply is the shape laya.k6n.net/predict returned when probed.
+const layaReply = `{"model":"laya-rl-agent","answers":{
+ "remediation_action":{"type":"choice","choice":"check_capacity","probabilities":{"ignore":0.1,"restart_deployment":0.12,"check_capacity":0.62,"fix_dependency":0.1,"investigate_code":0.06},"confidence":0.3,"answer_confidence":0.62},
+ "requires_human_attention":{"type":"noul","noul":0.7123,"confidence":0.7123,"answer_confidence":0.7123},
+ "urgency":{"type":"score","score":1.03,"legend":{"0":"cosmetic","1":"degraded performance","2":"service outage"},"probabilities":{"0":0.11,"1":0.75,"2":0.14},"confidence":0.32,"answer_confidence":0.75}},
+ "usage":{"input_tokens":141,"output_tokens":0}}`
+
+func layaServer(t *testing.T, status int, body string, got *map[string]any) *httptest.Server {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/predict" {
+			t.Errorf("path %q", r.URL.Path)
+		}
+		if got != nil {
+			_ = json.NewDecoder(r.Body).Decode(got)
+		}
+		w.WriteHeader(status)
+		_, _ = w.Write([]byte(body))
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func timeoutAlert() Alert {
+	now := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	return NewAlert(features.Spike{Service: "codesearch", WindowStart: now.Add(-5 * time.Minute), WindowEnd: now, ErrorCount: 15, BaselineMean: 0, Excess: 15,
+		Patterns: []features.PatternDelta{{Signature: "t", Count: 15, Excess: 15, Novel: true,
+			Example: features.Event{Service: "codesearch", Kind: "span", SpanName: "upload", Message: "context deadline exceeded", IsTimeout: true}}}}, now)
+}
+
+func TestLayaClassifySendsAlertTextAndParsesVerdict(t *testing.T) {
+	var req map[string]any
+	srv := layaServer(t, 200, layaReply, &req)
+	a := timeoutAlert()
+	v, err := (&Laya{URL: srv.URL + "/"}).Classify(context.Background(), a)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req["state"] != a.Text("") {
+		t.Fatalf("state is not the alert text:\n%v", req["state"])
+	}
+	qs := req["questions"].(map[string]any)
+	for _, k := range []string{"remediation_action", "requires_human_attention", "urgency"} {
+		if qs[k] == nil {
+			t.Fatalf("question %s missing", k)
+		}
+	}
+	if v.Action != "check_capacity" || v.ActionConfidence != 0.62 || v.Human != 0.7123 || v.Urgency != "degraded performance" || v.UrgencyConf != 0.75 {
+		t.Fatalf("verdict %+v", v)
+	}
+	if got := v.Summary(); got != "check_capacity 62% · human 71% · degraded performance 75%" {
+		t.Fatalf("summary %q", got)
+	}
+	if got := v.RunnersUp(); !strings.HasPrefix(got, "restart_deployment 12%, ") {
+		t.Fatalf("runners-up %q", got)
+	}
+}
+
+func TestLayaErrors(t *testing.T) {
+	for name, tc := range map[string]struct {
+		status int
+		body   string
+	}{
+		"http":    {502, "bad gateway"},
+		"json":    {200, "not json"},
+		"missing": {200, `{"answers":{"urgency":{}}}`},
+	} {
+		srv := layaServer(t, tc.status, tc.body, nil)
+		if _, err := (&Laya{URL: srv.URL}).Classify(context.Background(), timeoutAlert()); err == nil {
+			t.Errorf("%s: want error", name)
+		}
+	}
+}
+
+func TestLayaVerdictInNotifications(t *testing.T) {
+	a := timeoutAlert()
+	a.Laya = &LayaVerdict{Action: "restart_deployment", ActionConfidence: 0.8, ActionProbs: map[string]float64{"restart_deployment": 0.8, "ignore": 0.2}, Human: 0.4, Urgency: "service outage", UrgencyConf: 0.6}
+	h := a.HTML("https://otel.example", telegramMax)
+	if !strings.Contains(h, "🤖 <b>laya</b> <i>(shadow, no action taken)</i>: <code>restart_deployment</code> 80% · human 40% · service outage 60%") ||
+		!strings.Contains(h, "other: ignore 20%") {
+		t.Fatalf("html:\n%s", h)
+	}
+	if txt := a.Text(""); !strings.Contains(txt, "laya (shadow, no action taken): restart_deployment 80%") {
+		t.Fatalf("text:\n%s", txt)
+	}
+	a.Laya = nil
+	if strings.Contains(a.HTML("", telegramMax), "laya") {
+		t.Fatal("no verdict must render no laya line")
+	}
+}
+
+// A laya outage must never cost a page; the alert just goes out without a verdict.
+func TestRunnerAttachesVerdictAndSurvivesLayaFailure(t *testing.T) {
+	for _, tc := range []struct {
+		status int
+		want   bool
+	}{{200, true}, {500, false}} {
+		srv := layaServer(t, tc.status, layaReply, nil)
+		f := &fake{}
+		r := newRunner(nil, f)
+		r.laya = &Laya{URL: srv.URL}
+		r.deliver(context.Background(), timeoutAlert())
+		if len(f.sent) != 1 {
+			t.Fatalf("status %d: sent %d", tc.status, len(f.sent))
+		}
+		if got := f.sent[0].Laya != nil; got != tc.want {
+			t.Fatalf("status %d: verdict attached=%v", tc.status, got)
+		}
+	}
+}

internal/alerts/notify.go

@@ -29,6 +29,9 @@ type Alert struct {
 	Spike    features.Spike  `json:"spike"`
 	Patterns []PatternReport `json:"patterns"`
 	Test     bool            `json:"test,omitempty"`
+	// Laya is the laya model's shadow verdict, when it was asked. Advisory
+	// only: nothing is acted on or suppressed because of it.
+	Laya *LayaVerdict `json:"laya,omitempty"`
 }
 
 // PatternReport is a spike pattern with its triage verdict, ready to show.
@@ -214,6 +217,9 @@ func (a Alert) Text(baseURL string) string {
 	window := a.Spike.WindowEnd.Sub(a.Spike.WindowStart).Round(time.Minute)
 	fmt.Fprintf(&b, "%s: %s is failing (%d errors in the last %s, normally about %.0f)\n",
 		prefix, a.Spike.Service, a.Spike.ErrorCount, window, a.Spike.BaselineMean)
+	if a.Laya != nil {
+		fmt.Fprintf(&b, "laya (shadow, no action taken): %s\n", a.Laya.Summary())
+	}
 	base := strings.TrimRight(baseURL, "/")
 	for i, p := range a.Patterns {
 		fmt.Fprintf(&b, "\n%d. %s\n", i+1, p.What)
@@ -282,6 +288,13 @@ func (a Alert) HTML(baseURL string, max int) string {
 	if base != "" {
 		foot = fmt.Sprintf("\n<a href=\"%s\">All issues</a>", esc(base+"/issues"))
 	}
+	if a.Laya != nil {
+		head += fmt.Sprintf("🤖 <b>laya</b> <i>(shadow, no action taken)</i>: <code>%s</code> %s · human %s · %s %s\n",
+			esc(a.Laya.Action), pct(a.Laya.ActionConfidence), pct(a.Laya.Human), esc(a.Laya.Urgency), pct(a.Laya.UrgencyConf))
+		if r := a.Laya.RunnersUp(); r != "" {
+			head += "<i>   other: " + esc(r) + "</i>\n"
+		}
+	}
 	var b strings.Builder
 	b.WriteString(head)
 	for i, p := range a.Patterns {

internal/alerts/runner.go

@@ -34,6 +34,9 @@ type Config struct {
 	Muted                     func(issueID string) bool
 	WebhookURL, WebhookSecret string
 	AODURL                    string
+	// LayaURL, when set, asks laya (POST {LayaURL}/predict) for a shadow
+	// verdict on every alert and shows it in the notification.
+	LayaURL string
 }
 
 func ConfigFromEnv() Config {
@@ -54,6 +57,7 @@ func ConfigFromEnv() Config {
 		WebhookURL:       os.Getenv("OTEL_ALERT_WEBHOOK_URL"),
 		WebhookSecret:    os.Getenv("OTEL_ALERT_WEBHOOK_SECRET"),
 		AODURL:           os.Getenv("OTEL_ALERT_AOD_URL"),
+		LayaURL:          os.Getenv("OTEL_ALERT_LAYA_URL"),
 	}
 }
 
@@ -106,6 +110,7 @@ type Runner struct {
 	dataSince func(ctx context.Context) time.Time
 	notifiers []Notifier
 	resolver  Resolver // optional: ingress -> service -> deployment for edge errors
+	laya      *Laya    // optional: shadow verdict attached to each alert
 	sup       *features.Suppressor
 
 	mu        sync.Mutex
@@ -137,6 +142,10 @@ func Start(ctx context.Context, cfg Config, conn driver.Conn, database string, r
 		},
 		dataSince: func(ctx context.Context) time.Time { return DataSince(ctx, conn, database) },
 	}
+	if cfg.LayaURL != "" {
+		r.laya = &Laya{URL: cfg.LayaURL}
+		log.Printf("alerts: laya shadow verdicts from %s", cfg.LayaURL)
+	}
 	names := make([]string, 0, len(notifiers))
 	for _, n := range notifiers {
 		names = append(names, n.Name())
@@ -239,6 +248,7 @@ func (r *Runner) deliver(ctx context.Context, a Alert) {
 	if pr, ok := r.resolver.(PodResolver); ok {
 		a.EnrichPods(ctx, pr)
 	}
+	r.askLaya(ctx, &a)
 	r.mu.Lock()
 	r.recent = append(r.recent, a)
 	if len(r.recent) > maxRecent {
@@ -261,6 +271,23 @@ func (r *Runner) deliver(ctx context.Context, a Alert) {
 	}
 }
 
+// askLaya attaches laya's shadow verdict. It is bounded so a slow model
+// delays a page by seconds at most, and a failure only costs the verdict.
+func (r *Runner) askLaya(ctx context.Context, a *Alert) {
+	if r.laya == nil {
+		return
+	}
+	lctx, cancel := context.WithTimeout(ctx, 8*time.Second)
+	defer cancel()
+	v, err := r.laya.Classify(lctx, *a)
+	if err != nil {
+		log.Printf("alerts: laya verdict for %s failed: %v", a.ID, err)
+		return
+	}
+	a.Laya = v
+	log.Printf("alerts: laya[shadow] %s: %s", a.ID, v.Summary())
+}
+
 // Status is the JSON served by GET /api/alerts.
 type Status struct {
 	Enabled   bool      `json:"enabled"`
@@ -305,6 +332,7 @@ func (r *Runner) HandleTest(w http.ResponseWriter, req *http.Request) {
 		Example: features.Event{Service: "alert-test", Kind: "span", StatusCode: 500, Route: "/api/example", Message: "this is a test alert"},
 	}}}, now)
 	a.Test = true
+	r.askLaya(req.Context(), &a)
 	results := map[string]string{}
 	for _, n := range r.notifiers {
 		if err := n.Send(req.Context(), a); err != nil {

internal/collector/handlers.go

@@ -13,7 +13,6 @@ import (
 	"io"
 	"log"
 	"net/http"
-	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -1007,110 +1006,6 @@ func (c *Collector) HandleStatsEndpoints(w http.ResponseWriter, r *http.Request)
 	})
 }
 
-// needleAddr returns the configured or default address for the Needle 3 sidecar.
-func needleAddr() string {
-	addr := os.Getenv("OTEL_NEEDLE_ADDR")
-	if addr == "" {
-		addr = "http://127.0.0.1:5005"
-	}
-	return strings.TrimRight(addr, "/")
-}
-
-// HandleNeedleStatus checks if the Needle 3 sidecar is reachable and reports its health.
-func (c *Collector) HandleNeedleStatus(w http.ResponseWriter, r *http.Request) {
-	if r.Method != http.MethodGet {
-		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
-		return
-	}
-
-	target := needleAddr() + "/health"
-	client := &http.Client{Timeout: 2 * time.Second}
-	resp, err := client.Get(target)
-	w.Header().Set("Content-Type", "application/json")
-	if err != nil {
-		w.WriteHeader(http.StatusServiceUnavailable)
-		_ = json.NewEncoder(w).Encode(map[string]interface{}{
-			"available": false,
-			"error":     fmt.Sprintf("Needle sidecar unreachable at %s: %v", target, err),
-		})
-		return
-	}
-	defer resp.Body.Close()
-	w.WriteHeader(resp.StatusCode)
-	_, _ = io.Copy(w, resp.Body)
-}
-
-// HandleNeedleClassify proxies telemetry triage requests to the Needle 3 sidecar.
-func (c *Collector) HandleNeedleClassify(w http.ResponseWriter, r *http.Request) {
-	if r.Method != http.MethodPost {
-		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
-		return
-	}
-
-	body, err := io.ReadAll(r.Body)
-	if err != nil {
-		http.Error(w, "failed to read request body", http.StatusBadRequest)
-		return
-	}
-
-	// If the payload specifies an issue_id and no text, look up the issue
-	var reqMap map[string]interface{}
-	if err := json.Unmarshal(body, &reqMap); err == nil {
-		if issueID, ok := reqMap["issue_id"].(string); ok && issueID != "" {
-			if issue, ok := c.GetIssue(issueID); ok {
-				var promptParts []string
-				promptParts = append(promptParts, fmt.Sprintf("Service: %s", issue.Service))
-				promptParts = append(promptParts, fmt.Sprintf("Title: %s", issue.Title))
-				promptParts = append(promptParts, fmt.Sprintf("Kind: %s", issue.Kind))
-				if len(issue.Samples) > 0 {
-					latest := issue.Samples[len(issue.Samples)-1]
-					if latest.Message != "" {
-						promptParts = append(promptParts, fmt.Sprintf("Message: %s", latest.Message))
-					}
-					if latest.URL != "" {
-						promptParts = append(promptParts, fmt.Sprintf("URL: %s", latest.URL))
-					}
-					if latest.HTTPStatus > 0 {
-						promptParts = append(promptParts, fmt.Sprintf("Status: %d", latest.HTTPStatus))
-					}
-					if latest.HTTPRoute != "" {
-						promptParts = append(promptParts, fmt.Sprintf("Route: %s", latest.HTTPRoute))
-					}
-				}
-				reqMap["text"] = strings.Join(promptParts, " | ")
-				reqMap["service"] = issue.Service
-				reqMap["title"] = issue.Title
-				body, _ = json.Marshal(reqMap)
-			}
-		}
-	}
-
-	target := needleAddr() + "/classify"
-	client := &http.Client{Timeout: 60 * time.Second}
-	proxyReq, err := http.NewRequestWithContext(r.Context(), http.MethodPost, target, strings.NewReader(string(body)))
-	if err != nil {
-		http.Error(w, "failed to create proxy request", http.StatusInternalServerError)
-		return
-	}
-	proxyReq.Header.Set("Content-Type", "application/json")
-
-	resp, err := client.Do(proxyReq)
-	if err != nil {
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(http.StatusServiceUnavailable)
-		_ = json.NewEncoder(w).Encode(map[string]interface{}{
-			"success": false,
-			"error":   fmt.Sprintf("Needle sidecar unreachable at %s: %v", target, err),
-		})
-		return
-	}
-	defer resp.Body.Close()
-
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(resp.StatusCode)
-	_, _ = io.Copy(w, resp.Body)
-}
-
 // stampFrontendClient fills in client.address and user_agent.original on
 // browser-originated log records (source=frontend) from the ingest request,
 // since the browser cannot know its own public IP. Values the SDK already

k8s/deployment.yaml

@@ -15,6 +15,9 @@ spec:
     metadata:
       labels:
         app: otel-debug-receiver
+      annotations:
+        k8s-helper.io/git-url: "https://git.k6n.net/mats/go-otel"
+        k8s-helper.io/git-ssh: "git@git.k6n.net:mats/go-otel.git"
     spec:
       serviceAccountName: otel-debug-receiver
       securityContext:
@@ -138,6 +141,9 @@ spec:
         # Optional Always-On Display session API.
         - name: OTEL_ALERT_AOD_URL
           value: ""
+        # laya shadow verdicts: shown in each notification, never acted on.
+        - name: OTEL_ALERT_LAYA_URL
+          value: "https://laya.k6n.net"
         - name: OTEL_STORE_WORKERS
           value: "4"
         # Optional rollout observation. The watcher is read-only and uses the
@@ -167,8 +173,6 @@ spec:
           value: "15m"
         - name: OTEL_HONEYPOT_TTL
           value: "1h"
-        - name: OTEL_NEEDLE_ADDR
-          value: "http://127.0.0.1:5005"
         # ~80% of the memory limit so the GC works harder before an OOMKill.
         - name: GOMEMLIMIT
           value: "800MiB"
@@ -196,58 +200,12 @@ spec:
           limits:
             memory: "1Gi"
             cpu: "500m"
-      - name: needle-sidecar
-        image: matst80/needle-sidecar:latest
-        imagePullPolicy: Always
-        ports:
-        - containerPort: 5005
-          name: needle-http
-        env:
-        - name: HOME
-          value: "/app"
-        - name: NEEDLE_PORT
-          value: "5005"
-        - name: NEEDLE_HOST
-          value: "0.0.0.0"
-        - name: NEEDLE_TELEMETRY
-          value: "0"
-        livenessProbe:
-          httpGet:
-            path: /health
-            port: needle-http
-          initialDelaySeconds: 5
-          periodSeconds: 15
-        readinessProbe:
-          httpGet:
-            path: /health
-            port: needle-http
-          initialDelaySeconds: 3
-          periodSeconds: 10
-        resources:
-          requests:
-            memory: "512Mi"
-            cpu: "200m"
-          limits:
-            memory: "2Gi"
-            cpu: "4000m"
       volumes:
       - name: log-dir
         emptyDir: {}
       - name: sourcemap-dir
         persistentVolumeClaim:
           claimName: otel-sourcemaps
-
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: otel-sourcemaps
-  namespace: monitoring
-spec:
-  accessModes: ["ReadWriteOnce"]
-  resources:
-    requests:
-      storage: 1Gi
 ---
 apiVersion: v1
 kind: ServiceAccount

k8s/flux/gitops-dashboard.yaml

@@ -0,0 +1,120 @@
+# OIDC Registration in ecommerce namespace
+apiVersion: v1
+kind: Secret
+metadata:
+  name: flux-dashboard-oidc-client-secret
+  namespace: ecommerce
+type: Opaque
+stringData:
+  secret: "f9f44c88d9fda647e4abbb4479f1116611ecc14624bd67ec"
+---
+apiVersion: auth.k6n.net/v1alpha1
+kind: AuthClient
+metadata:
+  name: flux-dashboard
+  namespace: ecommerce
+  labels:
+    auth.k6n.net/server: auth
+spec:
+  clientID: flux-dashboard
+  displayName: Flux GitOps Dashboard
+  description: Web UI for Flux CD (Weave GitOps)
+  redirectURIs:
+    - https://flux.k6n.net/oauth2/callback
+  scopes:
+    - openid
+    - profile
+    - email
+    - groups
+  secretRef:
+    name: flux-dashboard-oidc-client-secret
+    key: secret
+---
+# Weave GitOps OIDC Secret in flux-system
+apiVersion: v1
+kind: Secret
+metadata:
+  name: oidc-auth
+  namespace: flux-system
+type: Opaque
+stringData:
+  clientID: "flux-dashboard"
+  clientSecret: "f9f44c88d9fda647e4abbb4479f1116611ecc14624bd67ec"
+  issuerURL: "https://users.k6n.net"
+  redirectURL: "https://flux.k6n.net/oauth2/callback"
+  claimUsername: "email"
+  claimGroups: "roles"
+---
+# Weave GitOps Helm OCI Source and HelmRelease
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: weave-gitops
+  namespace: flux-system
+spec:
+  type: oci
+  url: oci://ghcr.io/weaveworks/charts
+  interval: 1h
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: weave-gitops
+  namespace: flux-system
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: weave-gitops
+      version: "4.0.36"
+      sourceRef:
+        kind: HelmRepository
+        name: weave-gitops
+  values:
+    envVars:
+      - name: KUBERNETES_SERVICE_HOST
+        value: "10.10.1.102"
+      - name: KUBERNETES_SERVICE_PORT
+        value: "6443"
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: kubernetes.io/hostname
+                  operator: NotIn
+                  values:
+                    - serber
+    # Use existing oidc-auth secret created above
+    oidcSecret:
+      create: false
+    adminUser:
+      create: false
+    ingress:
+      enabled: true
+      className: nginx
+      annotations:
+        cert-manager.io/cluster-issuer: letsencrypt-prod
+      hosts:
+        - host: flux.k6n.net
+          paths:
+            - path: /
+              pathType: ImplementationSpecific
+      tls:
+        - hosts:
+            - flux.k6n.net
+          secretName: flux-k6n-tls
+---
+# ClusterRoleBinding so authenticated users can view/manage GitOps resources
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: weave-gitops-admin
+subjects:
+  - kind: Group
+    name: admin
+    apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io

k8s/flux/sync-go-otel.yaml

@@ -0,0 +1,28 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: GitRepository
+metadata:
+  name: go-otel
+  namespace: flux-system
+spec:
+  interval: 1m
+  url: ssh://git@git.k6n.net/mats/go-otel.git
+  ref:
+    branch: main
+  secretRef:
+    name: go-otel-auth
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: go-otel
+  namespace: flux-system
+spec:
+  interval: 5m
+  targetNamespace: monitoring
+  sourceRef:
+    kind: GitRepository
+    name: go-otel
+  path: "./k8s"
+  prune: true
+  wait: true
+  timeout: 3m

k8s/ingress-catchall.yaml

@@ -0,0 +1,173 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ingress-catchall
+  namespace: ingress-nginx
+data:
+  nginx.conf: |
+    worker_processes  1;
+    pid /tmp/nginx.pid;
+
+    events {
+      worker_connections  1024;
+    }
+
+    http {
+      include       /etc/nginx/mime.types;
+      default_type  application/octet-stream;
+      sendfile      on;
+      keepalive_timeout  65;
+      client_body_temp_path /tmp/client_temp;
+      proxy_temp_path /tmp/proxy_temp;
+      fastcgi_temp_path /tmp/fastcgi_temp;
+      uwsgi_temp_path /tmp/uwsgi_temp;
+      scgi_temp_path /tmp/scgi_temp;
+
+      server {
+        listen 8080;
+      server_name _;
+      root /usr/share/nginx/html;
+      error_page 403 =404 /index.html;
+
+      # Only GET is intentionally accepted by this catch-all.
+      # The "if" is safe here because it only performs a return.
+        # Serve robots.txt directly with Disallow: /
+        location = /robots.txt {
+          default_type text/plain;
+          return 200 "User-agent: *\nDisallow: /\n";
+        }
+
+        # Deliberately trigger an internal 403 so the static error page is
+        # returned with the externally visible status still set to 404.
+        location / {
+          if ($request_method != GET) {
+            add_header Allow GET always;
+            return 405;
+          }
+
+          deny all;
+        }
+
+        # Keep the error document internal so /index.html also follows the
+        # catch-all policy instead of becoming a successful public endpoint.
+        location = /index.html {
+          internal;
+          allow all;
+        }
+      }
+    }
+  index.html: |
+    <!doctype html>
+    <html lang="en">
+    <head>
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width, initial-scale=1">
+      <title>Not found</title>
+      <style>
+        :root { color-scheme: dark; }
+        * { box-sizing: border-box; }
+        body {
+          min-height: 100vh;
+          margin: 0;
+          display: grid;
+          place-items: center;
+          padding: 2rem;
+          color: #e8ecf2;
+          background: #0d1014;
+          font: 16px/1.5 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+        }
+        main { width: min(100%, 42rem); }
+        .code {
+          margin: 0 0 .75rem;
+          color: #7b8cff;
+          font: 700 clamp(4rem, 14vw, 8rem)/.9 ui-monospace, SFMono-Regular, Menlo, monospace;
+          letter-spacing: -.08em;
+        }
+        h1 { margin: 0 0 .75rem; font-size: clamp(1.5rem, 4vw, 2.25rem); }
+        p { margin: 0; max-width: 34rem; color: #98a3b2; }
+        code { color: #e8ecf2; }
+      </style>
+    </head>
+    <body>
+      <main>
+        <p class="code">404</p>
+        <h1>There is nothing here.</h1>
+        <p>The requested resource does not exist on this ingress endpoint.</p>
+      </main>
+    </body>
+    </html>
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ingress-catchall
+  namespace: ingress-nginx
+  labels:
+    app.kubernetes.io/name: ingress-catchall
+    app.kubernetes.io/part-of: ingress-nginx
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ingress-catchall
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: ingress-catchall
+        app.kubernetes.io/part-of: ingress-nginx
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: nginx
+          image: nginx:1.27-alpine
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 8080
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            runAsNonRoot: true
+            runAsUser: 101
+          volumeMounts:
+            - name: config
+              mountPath: /etc/nginx/nginx.conf
+              subPath: nginx.conf
+              readOnly: true
+            - name: config
+              mountPath: /usr/share/nginx/html/index.html
+              subPath: index.html
+              readOnly: true
+          readinessProbe:
+            tcpSocket:
+              port: http
+            periodSeconds: 5
+          livenessProbe:
+            tcpSocket:
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+      volumes:
+        - name: config
+          configMap:
+            name: ingress-catchall
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ingress-catchall
+  namespace: ingress-nginx
+  labels:
+    app.kubernetes.io/name: ingress-catchall
+    app.kubernetes.io/part-of: ingress-nginx
+spec:
+  selector:
+    app.kubernetes.io/name: ingress-catchall
+  ports:
+    - name: http
+      port: 80
+      targetPort: http

k8s/ingress.yaml

@@ -85,6 +85,10 @@ metadata:
   name: otel-debug-ingress
   namespace: monitoring
   annotations:
+    k8s-helper.io/name: "Go OTel"
+    k8s-helper.io/description: "OpenTelemetry debugging dashboard"
+    k8s-helper.io/group: "Observability"
+    k8s-helper.io/icon: "activity"
     cert-manager.io/cluster-issuer: letsencrypt-prod
     # OIDC protection via the platform auth service (users.k6n.net):
     # /oauth2/auth resolves the session cookie / bearer token and answers

k8s/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - deployment.yaml
+  - service.yaml
+  - ingress.yaml

k8s/otel-sourcemaps-pvc.example.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: otel-sourcemaps
+  namespace: monitoring
+spec:
+  accessModes: ["ReadWriteOnce"]
+  resources:
+    requests:
+      storage: 1Gi

otel-ui/src/components/IssuesVisualizer.tsx

@@ -16,7 +16,6 @@ import {
   ScrollText,
   ShieldBan,
   Bot,
-  Sparkles,
   Loader2,
   X,
 } from "lucide-react"
@@ -38,7 +37,6 @@ import { useSubRoute } from "../lib/routes"
 import { CopyRef } from "./common/CopyRef"
 import { IpButton } from "./common/IpButton"
 import { IssueOccurrences } from "./IssueOccurrences"
-import { NeedleTestModal } from "./NeedleTestModal"
 import { FilterBar, type FilterChip } from "./FilterBar"
 import { CRAWLER_CLASSES, UA_CLASSES, uaClassLabel } from "../lib/ua-class"
 import type { Issue, IssueSample } from "../lib/types"
@@ -94,16 +92,6 @@ interface DetectorTestResult {
   error?: string
 }
 
-interface NeedleResult {
-  action?: string
-  tool?: string
-  confidence?: number
-  reasoning?: string
-  latency_ms?: number
-  arguments?: Record<string, unknown>
-  error?: string
-}
-
 const statusTabs: { value: StatusFilter; label: string }[] = [
   { value: "all", label: "All" },
   { value: "open", label: "Open" },
@@ -315,26 +303,8 @@ const IssueCard = ({
   detectedIPByAddress: Map<string, DetectedIP>
 }) => {
   const [expanded, setExpanded] = useState(false)
-  const [needleResult, setNeedleResult] = useState<NeedleResult | null>(null)
-  const [needleLoading, setNeedleLoading] = useState(false)
   const { sources, loading: sourcesLoading, error: sourcesError } = useIssueSources(issue.id, expanded)
 
-  const handleNeedleTriage = async () => {
-    setNeedleLoading(true)
-    try {
-      const response = await authFetch("/api/needle/classify", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ issue_id: issue.id }),
-      })
-      setNeedleResult(await response.json())
-    } catch (err) {
-      setNeedleResult({ error: err instanceof Error ? err.message : String(err) })
-    } finally {
-      setNeedleLoading(false)
-    }
-  }
-
   const meta = kindMeta[issue.kind] ?? { label: issue.kind, Icon: Bug }
   const tone = issueSeverityTone(issue)
   const latestSample = issue.samples[issue.samples.length - 1]
@@ -505,57 +475,6 @@ const IssueCard = ({
             )}
           </div>
 
-          <div className="space-y-2">
-            <button
-              type="button"
-              onClick={() => void handleNeedleTriage()}
-              disabled={needleLoading}
-              title="Classify this issue using local Needle 3 model"
-              className="inline-flex h-8 items-center gap-2 rounded-[9px] border border-ds-line-strong px-3 text-[12px] font-medium text-ds-text-2 hover:text-ds-text disabled:opacity-50"
-            >
-              {needleLoading ? (
-                <Loader2 className="h-3.5 w-3.5 animate-spin" aria-hidden="true" />
-              ) : (
-                <Sparkles className="h-3.5 w-3.5" aria-hidden="true" />
-              )}
-              {needleLoading ? "Needle 3 triaging…" : "Needle 3 triage"}
-            </button>
-            {needleResult && (
-              <div className="space-y-2 rounded-[12px] border border-ds-line bg-ds-surface-2 p-3 text-[12px]">
-                <div className="flex flex-wrap items-center justify-between gap-2 border-b border-ds-line pb-2">
-                  <span className="flex flex-wrap items-center gap-2">
-                    <span className="text-[10.5px] font-bold uppercase tracking-wider text-ds-text-3">Needle 3 action</span>
-                    <span className="rounded-[6px] border border-ds-line-strong px-2 py-0.5 font-[family-name:var(--font-data)] text-ds-text">
-                      {needleResult.action || needleResult.tool || "Processed"}
-                    </span>
-                  </span>
-                  {needleResult.confidence !== undefined && (
-                    <span className="font-[family-name:var(--font-data)] text-[11px] text-ds-text-3">
-                      confidence{" "}
-                      <span className={needleResult.confidence >= 0.6 ? "text-ds-ok" : "text-ds-warn"}>
-                        {(needleResult.confidence * 100).toFixed(0)}%
-                      </span>
-                      {needleResult.latency_ms ? ` (${needleResult.latency_ms}ms)` : ""}
-                    </span>
-                  )}
-                </div>
-                {needleResult.reasoning && <p className="italic text-ds-text-2">&ldquo;{needleResult.reasoning}&rdquo;</p>}
-                {needleResult.arguments && Object.keys(needleResult.arguments).length > 0 && (
-                  <div className="flex flex-wrap gap-1.5 font-[family-name:var(--font-data)] text-[11px]">
-                    {Object.entries(needleResult.arguments).map(([k, v]) => (
-                      <span key={k} className="rounded-[6px] border border-ds-line bg-ds-surface px-2 py-0.5 text-ds-text-2">
-                        <span className="text-ds-text-3">{k}:</span> {String(v)}
-                      </span>
-                    ))}
-                  </div>
-                )}
-                {needleResult.error && (
-                  <p className="font-[family-name:var(--font-data)] text-[11px] text-ds-critical">{needleResult.error}</p>
-                )}
-              </div>
-            )}
-          </div>
-
           <div className="space-y-2">
             <div className="flex items-baseline justify-between gap-3">
               <p className="text-[10.5px] font-bold uppercase tracking-widest text-ds-text-3">
@@ -750,7 +669,6 @@ export const IssuesVisualizer = ({
   const [statusFilter, setStatusFilter] = useState<StatusFilter>("all")
   const [kindFilter, setKindFilter] = useState<KindFilter>("all")
   const [groupBy, setGroupBy] = useState<GroupBy>("none")
-  const [needleModalOpen, setNeedleModalOpen] = useState(false)
   const [detectorTest, setDetectorTest] = useState<DetectorTestResult | null>(null)
   const [detectorTestLoading, setDetectorTestLoading] = useState(false)
   const [excludeSpamIPs, setExcludeSpamIPs] = useState(false)
@@ -1119,15 +1037,6 @@ export const IssuesVisualizer = ({
               <Bot className="h-3.5 w-3.5" aria-hidden="true" />
               {crawlersHidden ? "Crawlers hidden" : "Hide crawlers"}
             </button>
-            <button
-              type="button"
-              onClick={() => setNeedleModalOpen(true)}
-              title="Open interactive Needle 3 telemetry classifier"
-              className="inline-flex h-9 items-center gap-1.5 rounded-[9px] border border-ds-line-strong px-2.5 text-[12px] font-medium text-ds-text-2 hover:text-ds-text"
-            >
-              <Sparkles className="h-3.5 w-3.5" aria-hidden="true" />
-              Test Needle 3
-            </button>
             <button
               type="button"
               onClick={() => void runDetectorTest()}
@@ -1295,7 +1204,6 @@ export const IssuesVisualizer = ({
         </aside>
       </div>
 
-      <NeedleTestModal isOpen={needleModalOpen} onClose={() => setNeedleModalOpen(false)} />
     </div>
   )
 }

otel-ui/src/components/NeedleTestModal.tsx

@@ -1,236 +0,0 @@
-import React, { useState } from "react"
-import { X, Sparkles, Loader2, Cpu, CheckCircle2, AlertTriangle } from "lucide-react"
-import { authFetch } from "../lib/auth"
-
-interface NeedleTestModalProps {
-  isOpen: boolean
-  onClose: () => void
-}
-
-interface NeedleResponse {
-  success?: boolean
-  action?: string
-  tool?: string
-  confidence?: number
-  reasoning?: string
-  latency_ms?: number
-  arguments?: Record<string, unknown>
-  error?: string
-}
-
-const PRESETS = [
-  {
-    label: "Frontend RUM Crash",
-    service: "eventsourced-ai-web",
-    text: "User reported client browser crash on service eventsourced-ai-web: Script error on url https://event-agent.k6n.net/#session=angry-koala-1",
-  },
-  {
-    label: "Security CVE Probe",
-    service: "nginx",
-    text: "Alert security team: scanner metabase-cve-2026-72898-detect/1.0 probed target /api/session/properties with status code 503",
-  },
-  {
-    label: "Web Vital Metric",
-    service: "storefront-web",
-    text: "Store RUM metric on service storefront-web: record TTFB of 84 ms",
-  },
-  {
-    label: "Benign 404 Route",
-    service: "git.k6n.net",
-    text: "Client requested missing git resource: 404 on path /mats/go-cart-actor/blame/commit/81569b on host git.k6n.net",
-  },
-  {
-    label: "Database Pool Deadlock",
-    service: "checkout-service",
-    text: "Critical incident: Escalate severe backend database timeout on service checkout-service: deadlocked while acquiring connection pool",
-  },
-]
-
-export const NeedleTestModal: React.FC<NeedleTestModalProps> = ({ isOpen, onClose }) => {
-  const [inputText, setInputText] = useState(PRESETS[0].text)
-  const [loading, setLoading] = useState(false)
-  const [result, setResult] = useState<NeedleResponse | null>(null)
-
-  if (!isOpen) return null
-
-  const handleClassify = async () => {
-    if (!inputText.trim()) return
-    setLoading(true)
-    setResult(null)
-    try {
-      const resp = await authFetch("/api/needle/classify", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ text: inputText }),
-      })
-      const data = await resp.json()
-      setResult(data)
-    } catch (err) {
-      setResult({ error: err instanceof Error ? err.message : String(err) })
-    } finally {
-      setLoading(false)
-    }
-  }
-
-  return (
-    <div className="fixed inset-0 z-50 flex items-center justify-center bg-ds-bg/80 p-4 animate-fade-in">
-      <div className="relative flex flex-col w-full max-w-2xl rounded-[14px] border border-ds-ok/30 bg-ds-bg/95 p-6 max-h-[90vh] overflow-y-auto">
-        {/* Header */}
-        <div className="flex items-center justify-between border-b border-ds-line pb-4">
-          <div className="flex items-center gap-3">
-            <div className="flex h-10 w-10 items-center justify-center rounded-xl bg-ds-ok/15 border border-ds-ok/30 text-ds-ok">
-              <Cpu className="h-5 w-5" />
-            </div>
-            <div>
-              <h2 className="text-base font-bold text-ds-text flex items-center gap-2">
-                Needle 3 AI Telemetry Classifier
-                <span className="rounded-full bg-ds-ok/15 border border-ds-ok/30 px-2 py-0.5 text-[10px] font-mono text-ds-ok">
-                  Cactus-Compute
-                </span>
-              </h2>
-              <p className="text-xs text-ds-text-2">
-                On-device ~35MB foundation model executing real-time tool calling and extraction on local CPU.
-              </p>
-            </div>
-          </div>
-          <button
-            type="button"
-            onClick={onClose}
-            className="rounded-lg p-1.5 text-ds-text-2 hover:bg-ds-fill hover:text-ds-text transition-colors"
-          >
-            <X className="h-5 w-5" />
-          </button>
-        </div>
-
-        {/* Preset Selector */}
-        <div className="mt-4">
-          <label className="text-[11px] font-bold uppercase tracking-wider text-ds-text-2">
-            Quick Telemetry Presets (from ClickHouse)
-          </label>
-          <div className="flex flex-wrap gap-2 mt-2">
-            {PRESETS.map((p) => (
-              <button
-                key={p.label}
-                type="button"
-                onClick={() => {
-                  setInputText(p.text)
-                  setResult(null)
-                }}
-                className={`rounded-lg border px-2.5 py-1 text-[11px] font-medium transition-all ${
-                  inputText === p.text
-                    ? "border-ds-ok bg-ds-ok/15 text-ds-ok"
-                    : "border-ds-line bg-ds-surface-2 text-ds-text-2 hover:border-ds-line-strong hover:bg-ds-fill"
-                }`}
-              >
-                {p.label}
-              </button>
-            ))}
-          </div>
-        </div>
-
-        {/* Input Textarea */}
-        <div className="mt-4">
-          <label className="text-[11px] font-bold uppercase tracking-wider text-ds-text-2">
-            Telemetry Event / Log / Trace Summary
-          </label>
-          <textarea
-            value={inputText}
-            onChange={(e) => setInputText(e.target.value)}
-            rows={3}
-            className="mt-1.5 w-full rounded-xl border border-ds-line bg-ds-surface-2 p-3 font-mono text-xs text-ds-text placeholder:text-ds-text-3 focus:border-ds-ok/50 focus:outline-none focus:ring-1 focus:ring-ds-ok/40"
-            placeholder="Type or paste telemetry event..."
-          />
-        </div>
-
-        {/* Classify Button */}
-        <div className="mt-3 flex justify-end">
-          <button
-            type="button"
-            onClick={handleClassify}
-            disabled={loading || !inputText.trim()}
-            className="inline-flex items-center gap-2 rounded-xl bg-ds-surface-2 px-4 py-2 text-xs font-bold text-ds-on-accent transition-all hover:opacity-95 active:scale-95 disabled:opacity-50"
-          >
-            {loading ? (
-              <>
-                <Loader2 className="h-4 w-4 animate-spin" />
-                Classifying on CPU...
-              </>
-            ) : (
-              <>
-                <Sparkles className="h-4 w-4" />
-                Classify with Needle 3
-              </>
-            )}
-          </button>
-        </div>
-
-        {/* Results Area */}
-        {result && (
-          <div className="mt-5 rounded-xl border border-ds-ok/30 bg-ds-ok/15 p-4 space-y-3">
-            <div className="flex items-center justify-between border-b border-ds-ok/20 pb-3">
-              <div className="flex items-center gap-2">
-                {result.error ? (
-                  <AlertTriangle className="h-4 w-4 text-ds-critical" />
-                ) : (
-                  <CheckCircle2 className="h-4 w-4 text-ds-ok" />
-                )}
-                <span className="text-xs font-bold uppercase tracking-wider text-ds-text-2">
-                  {result.error ? "Error" : "Triage Result"}
-                </span>
-                {result.action && (
-                  <span className="rounded-lg bg-ds-ok/15 border border-ds-ok/40 px-2 py-0.5 font-mono text-xs font-bold text-ds-ok">
-                    {result.action}
-                  </span>
-                )}
-              </div>
-              {result.confidence !== undefined && (
-                <div className="flex items-center gap-3 text-xs font-mono">
-                  <span className="text-ds-text-2">
-                    Confidence:{" "}
-                    <span className="font-bold text-ds-ok">
-                      {(result.confidence * 100).toFixed(0)}%
-                    </span>
-                  </span>
-                  {result.latency_ms !== undefined && (
-                    <span className="text-ds-text-3">({result.latency_ms}ms)</span>
-                  )}
-                </div>
-              )}
-            </div>
-
-            {result.reasoning && (
-              <p className="text-xs text-ds-text-2 italic">
-                &ldquo;{result.reasoning}&rdquo;
-              </p>
-            )}
-
-            {result.arguments && Object.keys(result.arguments).length > 0 && (
-              <div className="space-y-1.5 pt-1">
-                <span className="text-[10px] font-bold uppercase tracking-wider text-ds-text-2">
-                  Extracted Typed Arguments:
-                </span>
-                <div className="grid grid-cols-2 gap-2 font-mono text-xs">
-                  {Object.entries(result.arguments).map(([key, value]) => (
-                    <div
-                      key={key}
-                      className="rounded-lg border border-ds-line bg-ds-surface-2 p-2 text-ds-text break-all"
-                    >
-                      <span className="text-ds-text-3">{key}: </span>
-                      <span className="text-ds-ok font-semibold">{String(value)}</span>
-                    </div>
-                  ))}
-                </div>
-              </div>
-            )}
-
-            {result.error && (
-              <div className="rounded-lg bg-ds-critical-bg border border-ds-critical/20 p-3 text-xs text-ds-critical">
-                {result.error}
-              </div>
-            )}
-          </div>
-        )}
-      </div>
-    </div>
-  )
-}

scripts/run_needle_test.sh

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
-
-# Use pre-configured virtual environment or fall back to local .venv
-VENV_DIR="/home/mats/.gemini/antigravity-cli/brain/8e7a2c5c-81aa-487e-81b1-bf0cf790d055/scratch/needle_venv"
-if [ ! -d "${VENV_DIR}" ]; then
-  VENV_DIR="${ROOT_DIR}/.venv"
-fi
-
-if [ ! -f "${VENV_DIR}/bin/python" ]; then
-  echo "Setting up virtual environment at ${VENV_DIR}..."
-  python3 -m venv "${VENV_DIR}"
-  "${VENV_DIR}/bin/pip" install --upgrade pip
-  "${VENV_DIR}/bin/pip" install cactus-needle pydantic
-fi
-
-echo "Running Needle 3 OTel Verification Suite..."
-"${VENV_DIR}/bin/python" "${SCRIPT_DIR}/test_needle_classifier.py"

scripts/test_needle_classifier.py

@@ -1,168 +0,0 @@
-#!/usr/bin/env python3
-"""
-Test and Verification Suite for Cactus-Compute/needle3 with OpenTelemetry data.
-Tests:
-1. Action/Tool Dispatch Accuracy (Ground Truth Verification)
-2. Field Extraction & Grounding
-3. Confidence Calibrations (Act vs. Escalate)
-4. Semantic Vector Embeddings (Issue Clustering)
-5. Execution Performance & Latency (Tokens/sec)
-"""
-
-import sys
-import time
-import json
-import math
-import subprocess
-
-try:
-    import needle
-except ImportError:
-    print("Error: cactus-needle not found. Run with the virtualenv python:")
-    print("  /home/mats/.gemini/antigravity-cli/brain/8e7a2c5c-81aa-487e-81b1-bf0cf790d055/scratch/needle_venv/bin/python " + __file__)
-    sys.exit(1)
-
-# --- Define Triage Tools ---
-@needle.tool
-def report_client_javascript_crash(service: str, message: str, url: str):
-    """Report client-side JavaScript crash or error logged by browser RUM."""
-    return {"action": "client_js_crash", "service": service, "message": message, "url": url}
-
-@needle.tool
-def alert_vulnerability_probe(scanner: str, target: str, status_code: int):
-    """Alert on malicious vulnerability scans or exploit crawlers."""
-    return {"action": "security_alert", "scanner": scanner, "target": target, "status_code": status_code}
-
-@needle.tool
-def record_frontend_web_vital(service: str, metric_name: str, duration_ms: int):
-    """Record RUM performance metrics like TTFB or FCP."""
-    return {"action": "record_vital", "service": service, "metric": metric_name, "ms": duration_ms}
-
-@needle.tool
-def log_missing_resource_404(path: str, host: str):
-    """Log benign 404 Not Found requests for missing pages or git commits."""
-    return {"action": "log_404", "path": path, "host": host}
-
-@needle.tool
-def escalate_backend_outage(service: str, error_message: str):
-    """Escalate severe backend database timeout or service crash to on-call."""
-    return {"action": "escalate_outage", "service": service, "error": error_message}
-
-
-def run_verification():
-    print("=" * 70)
-    print("  NEEDLE 3 OPENTELEMETRY CLASSIFIER - VERIFICATION SUITE")
-    print("=" * 70)
-
-    # 1. Initialize Agent
-    print("\n[Step 1] Initializing Needle 3 Engine...")
-    t0 = time.perf_counter()
-    agent = needle.Needle(tools=[
-        report_client_javascript_crash,
-        alert_vulnerability_probe,
-        record_frontend_web_vital,
-        log_missing_resource_404,
-        escalate_backend_outage
-    ])
-    init_ms = (time.perf_counter() - t0) * 1000
-    print(f"✓ Needle 3 Engine initialized in {init_ms:.1f}ms")
-
-    # 2. Ground-Truth Test Cases (from actual ClickHouse telemetry)
-    test_cases = [
-        {
-            "name": "Frontend Browser RUM Crash",
-            "input": "User reported client browser crash on service eventsourced-ai-web: Script error on url https://event-agent.k6n.net/#session=angry-koala-1",
-            "expected_tool": "report_client_javascript_crash",
-            "expected_fields": ["service", "url"]
-        },
-        {
-            "name": "Malicious CVE Exploit Probe",
-            "input": "Alert security team: scanner metabase-cve-2026-72898-detect/1.0 probed target /api/session/properties with status code 503",
-            "expected_tool": "alert_vulnerability_probe",
-            "expected_fields": ["target", "status_code"]
-        },
-        {
-            "name": "Web Vital RUM Metric",
-            "input": "Store RUM metric on service storefront-web: record TTFB of 84 ms",
-            "expected_tool": "record_frontend_web_vital",
-            "expected_fields": ["service", "metric", "ms"]
-        },
-        {
-            "name": "Nginx Benign 404 Route",
-            "input": "Client requested missing git resource: 404 on path /mats/go-cart-actor/blame/commit/81569b on host git.k6n.net",
-            "expected_tool": "log_missing_resource_404",
-            "expected_fields": ["path", "host"]
-        },
-        {
-            "name": "Critical Database Outage",
-            "input": "Critical incident: Escalate severe backend database timeout on service checkout-service: deadlocked while acquiring connection pool",
-            "expected_tool": "escalate_backend_outage",
-            "expected_fields": ["service"]
-        }
-    ]
-
-    print("\n[Step 2] Testing Classification & Tool Dispatch Accuracy...")
-    passed = 0
-    total = len(test_cases)
-    latencies = []
-
-    for i, tc in enumerate(test_cases, 1):
-        agent.reset()
-        t_start = time.perf_counter()
-        res = agent.run(tc["input"], strict=False)
-        dur_ms = (time.perf_counter() - t_start) * 1000
-        latencies.append(dur_ms)
-
-        results = res.get("results", [])
-        matched_action = results[0].get("action") if results and isinstance(results[0], dict) else None
-        
-        # Tool name to action map
-        expected_action = tc["expected_tool"].replace("report_", "").replace("alert_", "").replace("record_", "").replace("log_", "").replace("escalate_", "")
-        # Check match
-        is_match = False
-        if results and any(tc["expected_tool"] in str(c) or matched_action for c in results):
-            is_match = True
-
-        status_str = "PASS" if is_match else "FAIL"
-        if is_match:
-            passed += 1
-
-        print(f"\n  Test {i}/{total}: {tc['name']} -> [{status_str}] ({dur_ms:.1f}ms)")
-        print(f"    Confidence: {res.get('confidence', 0.0):.3f}")
-        print(f"    Reasoning : {res.get('reasoning')}")
-        print(f"    Dispatched: {results}")
-
-    print("\n" + "-" * 70)
-    print(f"Accuracy Score: {passed}/{total} ({passed/total*100:.0f}%) | Avg Latency: {sum(latencies)/len(latencies):.1f}ms")
-    print("-" * 70)
-
-    # 3. Embedding & Semantic Search Verification
-    print("\n[Step 3] Testing Semantic Vector Embeddings for Near-Match Clustering...")
-    emb_e1 = agent.embed("database connection timeout after 3000ms")
-    emb_e2 = agent.embed("postgres pool exhausted: connection timed out")
-    emb_e3 = agent.embed("invalid JWT signature token expired")
-
-    def cosine_similarity(v1, v2):
-        dot = sum(a * b for a, b in zip(v1, v2))
-        m1 = math.sqrt(sum(a * a for a in v1))
-        m2 = math.sqrt(sum(b * b for b in v2))
-        return dot / (m1 * m2)
-
-    sim_similar = cosine_similarity(emb_e1, emb_e2)
-    sim_different = cosine_similarity(emb_e1, emb_e3)
-
-    print(f"  Vector Dimensions: {len(emb_e1)}")
-    print(f"  Similarity between 2 DB timeouts : {sim_similar:.4f}")
-    print(f"  Similarity between DB and Auth   : {sim_different:.4f}")
-
-    if sim_similar > sim_different:
-        print("✓ Semantic clustering validated: related issues score higher similarity.")
-    else:
-        print("✗ Warning: semantic separation failed.")
-
-    print("\n" + "=" * 70)
-    print("  VERIFICATION COMPLETE")
-    print("=" * 70)
-
-if __name__ == "__main__":
-    run_verification()