Merge origin/main into feature/issues-tui

Remove Needle; stop paging on nginx 4xx
Needle scored ~chance on the golden set and never gated alerts, so drop the sidecar, proxy handlers, UI buttons, k8s container and build step. The labelled eval sets move to cmd/triage-eval/data. nginx 4xx (registry 401 challenges, /oauth2/auth, scanner 405s) is now dropped in FilterActionable instead of only being discounted, so it cannot ride along with a real 5xx. nginx 5xx still alerts. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-11-01 11:40:57 +01:00 · 2026-11-01 11:40:53 +01:00 · 2026-09-27 16:47:09 +02:00 · 2026-09-27 16:46:37 +02:00 · 2026-09-27 16:45:10 +02:00
44 changed files with 3051 additions and 2562 deletions
@@ -1,6 +1,9 @@
 name: Build and Publish
 run-name: ${{ gitea.actor }} build 🚀
-on: [push]
+on:
  push:
    branches: [main]
  workflow_dispatch:
 jobs:
  Metadata:
@@ -26,7 +29,7 @@ jobs:
          echo "version=$VERSION" >> $GITHUB_OUTPUT
          echo "build_date=$BUILD_DATE" >> $GITHUB_OUTPUT
-  BuildAndDeployAmd64:
+  BuildAndPushAmd64:
    needs: Metadata
    runs-on: amd64
    steps:
@@ -39,15 +42,32 @@ jobs:
            --build-arg GIT_COMMIT=${{ needs.Metadata.outputs.git_commit }} \
            --build-arg BUILD_DATE=${{ needs.Metadata.outputs.build_date }} \
            --progress=plain \
            -t matst80/otel-debug:${{ needs.Metadata.outputs.version }} \
            -t matst80/otel-debug:latest \
            .
-      - name: Push amd64 images
+      - name: Push images
        run: |
          docker push matst80/otel-debug:${{ needs.Metadata.outputs.version }}
          docker push matst80/otel-debug:latest
-      - name: Rollout amd64 otel deployment
+      - name: Reconcile after image is published
        env:
          GITOPS_WEBHOOK_SECRET: ${{ secrets.GITOPS_WEBHOOK_SECRET }}
        run: |
-          kubectl rollout restart deployment/otel-debug-receiver -n monitoring
+          if [ -z "$GITOPS_WEBHOOK_SECRET" ]; then
-          kubectl rollout status deployment/otel-debug-receiver -n monitoring
+            echo "GITOPS_WEBHOOK_SECRET must match the k8s-helper Gitea webhook secret" >&2
            exit 1
          fi
          payload="$(mktemp)"
          trap 'rm -f "$payload"' EXIT
          printf '{"ref":"%s","after":"%s","repository":{"clone_url":"https://git.k6n.net/mats/go-otel.git","ssh_url":"git@git.k6n.net:mats/go-otel.git"}}' \
            "$GITHUB_REF" "$GITHUB_SHA" > "$payload"
          signature="$(openssl dgst -sha256 -hmac "$GITOPS_WEBHOOK_SECRET" "$payload" | awk '{print $NF}')"
          curl --fail --silent --show-error \
            -X POST \
            -H "Content-Type: application/json" \
            -H "X-Gitea-Signature: $signature" \
            --data-binary "@$payload" \
            https://ui.k6n.net/api/gitops/webhooks/gitea
      - name: Upload source maps
        env:
          RELEASE: ${{ needs.Metadata.outputs.version }}
@@ -69,31 +89,3 @@ jobs:
          done < <(find "$maps_dir" -type f -name '*.js.map' -print0)
          rm -rf "$maps_dir"
          [ "$found" = 1 ] || { echo "no source maps found in UI build" >&2; exit 1; }
      # - name: Rollout amd64 deployment (pin to version)
      #   run: |
      #     kubectl set image deployment/cart-actor-x86 -n cart cart-actor-amd64=registry.knatofs.se/go-otel-debug-amd64:${{ needs.Metadata.outputs.version }}
      #     # kubectl rollout status deployment/cart-actor-x86 -n cart
  # BuildAndDeployArm64:
  #   needs: Metadata
  #   runs-on: arm64
  #   steps:
  #     - uses: actions/checkout@v4
  #     - name: Build arm64 image
  #       run: |
  #         docker build \
  #           --build-arg VERSION=${{ needs.Metadata.outputs.version }} \
  #           --build-arg GIT_COMMIT=${{ needs.Metadata.outputs.git_commit }} \
  #           --build-arg BUILD_DATE=${{ needs.Metadata.outputs.build_date }} \
  #           --progress=plain \
  #           -t registry.knatofs.se/go-otel-debug:latest \
  #           -t registry.knatofs.se/go-otel-debug:${{ needs.Metadata.outputs.version }} \
  #           .
  #     - name: Push arm64 images
  #       run: |
  #         docker push registry.knatofs.se/go-otel-debug:latest
  #         docker push registry.knatofs.se/go-otel-debug:${{ needs.Metadata.outputs.version }}
  #     - name: Rollout arm64 deployment (pin to version)
  #       run: |
  #         kubectl set image deployment/cart-actor-arm64 -n cart cart-actor-arm64=registry.knatofs.se/go-otel-debug:${{ needs.Metadata.outputs.version }}
  #         # kubectl rollout status deployment/cart-actor-arm64 -n cart
@@ -0,0 +1,2 @@
 {"enabled":true,"dryRun":false,"channels":["telegram"],"lastRun":"2026-09-25T18:56:43.3470721Z","alerts":[{"id":"tlxng1-nginx","at":"2026-09-25T18:31:13.347300133Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:26:13.347300133Z","windowEnd":"2026-09-25T18:31:13.347300133Z","errorCount":69,"baselineMean":14.03448275862069,"excess":4.9655172413793105,"patterns":[{"signature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","count":4,"baselineMean":0.034482758620689655,"excess":3.9655172413793105,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:31:11.346792393Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"0751de83a2f0ce211f2b7ab1a64d8aba","PatternSignature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"auth.k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"34.177.93.136","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"fb485cd240e1dab0261891dc14170112bbc37c90189f276b45935afdedf534cb","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:31:06.252746363Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d05925e26f581fcba99d165ea5928114","PatternSignature":"fb485cd240e1dab0261891dc14170112bbc37c90189f276b45935afdedf534cb","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/oauth2/auth","Host":"users.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"auth.k6n.net","path":"/","services":["nginx"],"routes":[{"ingress":"dev/zitadel-ingress","host":"auth.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"zitadel-dev","port":"8080","deployments":["zitadel-dev"]}],"pods":[{"namespace":"dev","deployment":"zitadel-dev","found":false}],"rule":"status-5xx-upstream","new":false,"count":4,"baseline":0.034482758620689655,"clients":1,"what":"HTTP 503 | /"},{"category":"client_credential_issue","priority":"low","status":401,"host":"users.k6n.net","path":"/oauth2/auth","services":["nginx"],"routes":[{"ingress":"ecommerce/ecommerce-auth","host":"users.k6n.net","path":"/","pathType":"Prefix","namespace":"ecommerce","service":"auth","port":"8080","deployments":["auth"]}],"pods":[{"namespace":"ecommerce","deployment":"auth","pod":"auth-8f497c689-qfpzb","lines":["2026/09/25 18:13:06 oidc: device grant JHMJ-C23E started for client dev-console","2026/09/25 18:13:06 oidc: device grant EQ5R-UWRW approved by mats@tornberg.me","2026/09/25 18:23:19 oidc: device grant RHMN-6257 started for client dev-console"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /oauth2/auth"}]},{"id":"tlxn9d-nginx","at":"2026-09-25T18:27:13.347226389Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:22:13.347226389Z","windowEnd":"2026-09-25T18:27:13.347226389Z","errorCount":64,"baselineMean":13.931034482758621,"excess":4,"patterns":[{"signature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","count":4,"baselineMean":0,"excess":4,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:26:26.152736211Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d1de7de09ad342e7b0670cb78cac1430","PatternSignature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"cc.k6n.net","StatusCode":405,"ExceptionType":"","ClientIP":"35.189.233.235","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"needs_review","priority":"low","status":405,"host":"cc.k6n.net","path":"/","services":["nginx"],"rule":"no-rule","new":false,"count":4,"baseline":0,"clients":1,"what":"HTTP 405 | /"}]},{"id":"tlxn61-nginx","at":"2026-09-25T18:25:13.346661147Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:20:13.346661147Z","windowEnd":"2026-09-25T18:25:13.346661147Z","errorCount":60,"baselineMean":14.03448275862069,"excess":1,"patterns":[{"signature":"ea444cb83ff471aad7f3fab8a5b36dddb25f333f7dce2eb4ff27f6e9bb3969f7","count":1,"baselineMean":0,"excess":1,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:24:36.217473837Z","Service":"nginx","Release":"","Kind":"span","SourcLine truncated
@@ -5,10 +5,6 @@ RELEASE="${RELEASE:-$(git rev-parse --short HEAD)}"
 PUBLIC_SOURCEMAPS="${PUBLIC_SOURCEMAPS:-false}"
 UPLOAD_SOURCEMAPS="${UPLOAD_SOURCEMAPS:-true}"
 echo "Building Needle 3 sidecar image..."
 docker build -t matst80/needle-sidecar:latest cmd/needle-sidecar
 docker push matst80/needle-sidecar:latest
 echo "Building otel-debug receiver image..."
 docker build \
  --build-arg "APP_RELEASE=${RELEASE}" \
@@ -0,0 +1,144 @@
 // laya-eval runs alert scenarios through laya and compares its shadow verdict
 // with what a person judged the right response to be, so the model can be
 // assessed before it is ever allowed to act.
 //
 //	go run ./cmd/laya-eval                        # built-in scenarios
 //	go run ./cmd/laya-eval -file alerts.json      # a GET /api/alerts dump (no expectations)
 //	go run ./cmd/laya-eval -send                  # also deliver each as a TEST alert
 //
 // -send uses the channels configured by OTEL_ALERT_* (see alerts.ConfigFromEnv).
 package main
 import (
 	"context"
 	_ "embed"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"os"
 	"text/tabwriter"
 	"time"
 	"git.tornberg.me/go-otel/internal/alerts"
 )
 //go:embed scenarios.json
 var builtin []byte
 // Scenario is one alert and the response a person would pick for it.
 type Scenario struct {
 	Name   string       `json:"name"`
 	Expect *Expectation `json:"expect,omitempty"`
 	Alert  alerts.Alert `json:"alert"`
 }
 type Expectation struct {
 	Action string `json:"action"`
 	Human  bool   `json:"human"`
 	Why    string `json:"why"`
 }
 func main() {
 	url := flag.String("url", envOr("OTEL_ALERT_LAYA_URL", "https://laya.k6n.net"), "laya base URL")
 	file := flag.String("file", "", "scenarios JSON, or a GET /api/alerts response (default: built-in scenarios)")
 	send := flag.Bool("send", false, "deliver each scenario, with its verdict, as a TEST alert to the configured channels")
 	flag.Parse()
 	raw := builtin
 	if *file != "" {
 		var err error
 		if raw, err = os.ReadFile(*file); err != nil {
 			fail(err)
 		}
 	}
 	scenarios, err := parse(raw)
 	if err != nil {
 		fail(err)
 	}
 	var notifiers []alerts.Notifier
 	if *send {
 		cfg := alerts.ConfigFromEnv()
 		if notifiers = cfg.Notifiers(); len(notifiers) == 0 {
 			fail(fmt.Errorf("-send: no channel configured (OTEL_ALERT_TELEGRAM_TOKEN + OTEL_ALERT_TELEGRAM_CHAT_ID, OTEL_ALERT_WEBHOOK_URL or OTEL_ALERT_AOD_URL)"))
 		}
 	}
 	laya := &alerts.Laya{URL: *url}
 	tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
 	fmt.Fprintln(tw, "SCENARIO\tLAYA ACTION\tHUMAN\tURGENCY\tEXPECTED\tMATCH")
 	judged, actionOK, humanOK := 0, 0, 0
 	for _, s := range scenarios {
 		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
 		v, err := laya.Classify(ctx, s.Alert)
 		cancel()
 		if err != nil {
 			fmt.Fprintf(tw, "%s\terror: %v\t\t\t\t\n", s.Name, err)
 			continue
 		}
 		exp, match := "-", "-"
 		if s.Expect != nil {
 			judged++
 			a, h := v.Action == s.Expect.Action, (v.Human >= 0.5) == s.Expect.Human
 			if a {
 				actionOK++
 			}
 			if h {
 				humanOK++
 			}
 			exp = fmt.Sprintf("%s, human=%v", s.Expect.Action, s.Expect.Human)
 			match = fmt.Sprintf("action %s, human %s", mark(a), mark(h))
 		}
 		fmt.Fprintf(tw, "%s\t%s %.0f%%\t%.0f%%\t%s %.0f%%\t%s\t%s\n", s.Name, v.Action, v.ActionConfidence*100, v.Human*100, v.Urgency, v.UrgencyConf*100, exp, match)
 		if *send {
 			a := s.Alert
 			a.Test, a.Laya = true, v
 			for _, n := range notifiers {
 				sctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
 				if err := n.Send(sctx, a); err != nil {
 					fmt.Fprintf(os.Stderr, "%s: %s: %v\n", s.Name, n.Name(), err)
 				}
 				cancel()
 			}
 		}
 	}
 	tw.Flush()
 	if judged > 0 {
 		fmt.Printf("\nagreement over %d judged scenarios: action %d/%d, needs-human %d/%d\n", judged, actionOK, judged, humanOK, judged)
 	}
 }
 // parse accepts a scenario list or a raw GET /api/alerts response.
 func parse(raw []byte) ([]Scenario, error) {
 	var list []Scenario
 	if err := json.Unmarshal(raw, &list); err == nil {
 		return list, nil
 	}
 	var st alerts.Status
 	if err := json.Unmarshal(raw, &st); err != nil {
 		return nil, fmt.Errorf("neither a scenario list nor an /api/alerts response: %w", err)
 	}
 	for _, a := range st.Alerts {
 		list = append(list, Scenario{Name: a.ID + " " + a.Spike.Service, Alert: a})
 	}
 	return list, nil
 }
 func mark(ok bool) string {
 	if ok {
 		return "ok"
 	}
 	return "MISS"
 }
 func envOr(k, def string) string {
 	if v := os.Getenv(k); v != "" {
 		return v
 	}
 	return def
 }
 func fail(err error) {
 	fmt.Fprintln(os.Stderr, err)
 	os.Exit(1)
 }
@@ -1,28 +0,0 @@
 FROM python:3.12-slim
 # Create non-root user with UID 1000 matching Kubernetes pod securityContext
 RUN groupadd -g 1000 appuser && useradd -u 1000 -g appuser -m -d /app appuser
 WORKDIR /app
 ENV HOME=/app
 ENV NEEDLE_PORT=5005
 ENV NEEDLE_HOST=0.0.0.0
 ENV NEEDLE_TELEMETRY=0
 # Install dependencies
 RUN pip install --no-cache-dir cactus-needle
 # Pre-cache model and shared library into /app/.cache as user 1000
 RUN python -c "import needle; needle.Needle()"
 COPY server.py /app/server.py
 # Ensure UID 1000 owns /app and all caches
 RUN chown -R appuser:appuser /app
 USER appuser
 EXPOSE 5005
 CMD ["python", "/app/server.py"]
@@ -1,42 +0,0 @@
 """Deterministic baseline for the cause categories. Needle has to beat this."""
 import re
 PROBE = re.compile(r"(^|/)(\.env|\.git|wp-|wp/|wp$|wordpress|blog/wp|phpmyadmin|cgi-bin|\.aws|\.ssh|xmlrpc|userfiles|@fs/|etc/passwd)", re.I)
 DEP = re.compile(r"dial|connection refused|eof|no such host|upstream|unavailable|broken pipe|reset by peer", re.I)
 CAP = re.compile(r"timeout|deadline exceeded|pool|memory limit|oom|too many|queue full|exhaust", re.I)
 BUG = re.compile(r"panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault", re.I)
 LIFE = re.compile(r"healthz|readyz|/ping|retry|cancel|keepalive|vital|\bcls\b|ttfb|fcp|lcp|snapshot|replay|unchanged|session\.load", re.I)
 def fields(text):
    d = {}
    for part in text.split(" | "):
        k, _, v = part.partition(": ")
        d[k.lower()] = v
    return d
 def classify(text):
    f = fields(text)
    status = int(f.get("status") or 0)
    route = f.get("route", "")
    blob = " ".join([f.get("title", ""), f.get("message", ""), route])
    if PROBE.search(route):
        return "scanner_probe"
    if BUG.search(blob):
        return "service_defect"
    if CAP.search(blob):
        return "capacity_saturation"
    if DEP.search(blob):
        return "dependency_unreachable"
    if status == 401:
        return "client_credential_issue"
    if status in (502, 503, 504):
        return "dependency_unreachable"
    if status >= 500:
        return "service_defect"
    if status == 404:
        return "stale_reference"
    if LIFE.search(blob) or status in (200, 202, 204, 301, 302, 304, 308, 499) or (status == 0 and f.get("kind") in ("span", "log")):
        return "expected_lifecycle"
    return "needs_review"
@@ -1,79 +0,0 @@
 #!/usr/bin/env python3
 """Evaluate the Needle sidecar against a reviewed golden set.
 Usage: run_eval.py [--url http://127.0.0.1:5005] [--golden eval/golden.jsonl] [--json out.json]
 Each golden row: {"id", "expected", "text"}; `expected` is a sidecar action name
 including "needs_review" for the fallback.
 """
 import argparse, json, os, sys, time, urllib.request
 from collections import Counter, defaultdict
 HERE = os.path.dirname(os.path.abspath(__file__))
 def classify(url, text):
    req = urllib.request.Request(
        url.rstrip("/") + "/classify",
        data=json.dumps({"text": text}).encode(),
        headers={"Content-Type": "application/json"},
    )
    with urllib.request.urlopen(req, timeout=60) as r:
        return json.load(r)
 def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--url", default=os.environ.get("OTEL_NEEDLE_ADDR", "http://127.0.0.1:5005"))
    ap.add_argument("--golden", default=os.path.join(HERE, "golden.jsonl"))
    ap.add_argument("--json", help="write per-row results here")
    ap.add_argument("--min-conf", type=float, default=0.0,
                    help="predictions below this confidence count as needs_review")
    a = ap.parse_args()
    rows = [json.loads(l) for l in open(a.golden) if l.strip()]
    results, confusion = [], Counter()
    lat = []
    for row in rows:
        try:
            out = classify(a.url, row["text"])
        except Exception as e:
            print(f"sidecar error on {row['id']}: {e}", file=sys.stderr)
            sys.exit(2)
        pred = out.get("action", "needs_review")
        conf = out.get("confidence", 0.0)
        if conf < a.min_conf:
            pred = "needs_review"
        lat.append(out.get("latency_ms", 0.0))
        confusion[(row["expected"], pred)] += 1
        results.append({**row, "predicted": pred, "confidence": conf, "ok": pred == row["expected"]})
    labels = sorted({r["expected"] for r in results} | {r["predicted"] for r in results})
    print(f"{'label':<18}{'prec':>7}{'recall':>8}{'support':>9}")
    for l in labels:
        tp = confusion[(l, l)]
        pp = sum(v for (e, p), v in confusion.items() if p == l)
        sup = sum(v for (e, p), v in confusion.items() if e == l)
        prec = tp / pp if pp else 0.0
        rec = tp / sup if sup else 0.0
        print(f"{l:<18}{prec:>7.2f}{rec:>8.2f}{sup:>9}")
    acc = sum(r["ok"] for r in results) / len(results)
    print(f"\naccuracy {acc:.2%} over {len(results)} rows; "
          f"median latency {sorted(lat)[len(lat)//2]:.0f}ms")
    for src in sorted({r.get("src", "?") for r in results}):
        sub = [r for r in results if r.get("src", "?") == src]
        print(f"  {src:<11} {sum(r['ok'] for r in sub)}/{len(sub)}")
    wrong = [r for r in results if not r["ok"]]
    if wrong:
        print("\nmisses:")
        for r in wrong:
            print(f"  {r['id']:<8} expected={r['expected']:<16} got={r['predicted']:<16} conf={r['confidence']:.2f}")
    if a.json:
        json.dump(results, open(a.json, "w"), indent=1)
 if __name__ == "__main__":
    main()
@@ -1,240 +0,0 @@
 #!/usr/bin/env python3
 """
 Needle 3 AI Telemetry Triage Sidecar Service
 Accepts OTel incident summaries and uses Cactus-Compute/needle3 to classify and extract actionable insights.
 """
 import os
 import sys
 import json
 import time
 import re
 import hashlib
 from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
 import needle
 PORT = int(os.environ.get("NEEDLE_PORT", 5005))
 HOST = os.environ.get("NEEDLE_HOST", "0.0.0.0")
 # --- Cause-level triage tools ---
 # These deliberately do not restate what the event already carries (status
 # code, is_error, severity). Each one is a judgement about *why* it happened
 # or *who should care*, which is what an alert reader needs.
@needle.tool
 def report_scanner_probe(service: str, path: str, status_code: int):
    """Automated scan or exploit crawler requesting sensitive or nonexistent paths such as .env, wp-admin, .git, phpmyadmin. Applies whatever status the probe received, including 500 or 502."""
    return {"action": "scanner_probe", "service": service, "path": path, "status_code": status_code}
@needle.tool
 def report_dependency_unreachable(service: str, dependency: str, error_message: str):
    """A downstream or upstream service is down, refusing connections, closed the stream (EOF), or returned upstream unavailable."""
    return {"action": "dependency_unreachable", "service": service, "dependency": dependency, "error": error_message}
@needle.tool
 def report_client_credential_issue(service: str, route: str, reason: str):
    """A caller lacks a valid or unexpired token, for example an unauthenticated client polling an API or a registry pull without login. The caller's fault, not a fault in this service."""
    return {"action": "client_credential_issue", "service": service, "route": route, "reason": reason}
@needle.tool
 def report_stale_reference(service: str, path: str):
    """Harmless missing resource caused by an old link, removed asset, deleted commit, or dev tooling request."""
    return {"action": "stale_reference", "service": service, "path": path}
@needle.tool
 def report_service_defect(service: str, error_message: str):
    """A bug in the service's own code: panic, nil dereference, unhandled exception, index out of range, deadlock, or a 500 from a real application route."""
    return {"action": "service_defect", "service": service, "error": error_message}
@needle.tool
 def report_capacity_saturation(service: str, resource: str, error_message: str):
    """Resource exhaustion or overload: timeouts, connection pool exhausted, memory limit exceeded, OOM, queue full."""
    return {"action": "capacity_saturation", "service": service, "resource": resource, "error": error_message}
@needle.tool
 def report_expected_lifecycle(service: str, what: str):
    """Normal behaviour that is not a problem: health and readiness checks, keepalive pings, retries, cancellations, client disconnects, redirects, successful requests, routine info logs, web vitals."""
    return {"action": "expected_lifecycle", "service": service, "what": what}
@needle.tool
 def report_needs_review(service: str, summary: str):
    """Fallback when none of the other categories clearly applies."""
    return {"action": "needs_review", "service": service, "summary": summary}
 # --- Redaction: nothing PII-bearing should reach the model or the cache ---
 _REDACTIONS = [
    (re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+"), "<email>"),
    (re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), "<ip>"),
    (re.compile(r"\b[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}\b"), "<uuid>"),
    (re.compile(r"\beyJ[\w-]+\.[\w-]+\.[\w-]+\b"), "<jwt>"),
    (re.compile(r"\b[0-9a-fA-F]{16,}\b"), "<hex>"),
    (re.compile(r"([?&][\w.-]+=)[^&\s|]+"), r"\1<v>"),
 ]
 def redact(text: str) -> str:
    for rx, sub in _REDACTIONS:
        text = rx.sub(sub, text)
    return text[:1000]
 # Cache keyed on the redacted text: repeated signatures cost nothing.
 CACHE_MAX = int(os.environ.get("NEEDLE_CACHE_MAX", 2000))
 _cache = {}
 _stats = {"hits": 0, "misses": 0}
 # Initialize Needle Agent once
 print(f"Loading Needle 3 model (Cactus-Compute/needle3)...")
 t0 = time.perf_counter()
 agent = needle.Needle(tools=[
    report_scanner_probe,
    report_dependency_unreachable,
    report_client_credential_issue,
    report_stale_reference,
    report_service_defect,
    report_capacity_saturation,
    report_expected_lifecycle,
    report_needs_review,
 ])
 print(f"Needle 3 loaded in {(time.perf_counter() - t0):.2f}s")
 class NeedleHandler(BaseHTTPRequestHandler):
    def _send_cors(self):
        self.send_header("Access-Control-Allow-Origin", "*")
        self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
        self.send_header("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Otel-Agent-Token")
    def do_OPTIONS(self):
        self.send_response(204)
        self._send_cors()
        self.end_headers()
    def do_GET(self):
        if self.path == "/health" or self.path == "/api/needle/status":
            self.send_response(200)
            self._send_cors()
            self.send_header("Content-Type", "application/json")
            self.end_headers()
            resp = {
                "status": "ok",
                "available": True,
                "model": "Cactus-Compute/needle3",
                "cache": {**_stats, "size": len(_cache)},
                "tools": [
                    "scanner_probe", "dependency_unreachable", "client_credential_issue", "stale_reference",
                    "service_defect", "capacity_saturation", "expected_lifecycle", "needs_review"
                ]
            }
            self.wfile.write(json.dumps(resp).encode("utf-8"))
        else:
            self.send_response(404)
            self._send_cors()
            self.end_headers()
    def do_POST(self):
        if self.path in ("/classify", "/api/needle/classify"):
            content_len = int(self.headers.get("Content-Length", 0))
            body = self.rfile.read(content_len)
            try:
                data = json.loads(body) if body else {}
            except Exception as e:
                self.send_response(400)
                self._send_cors()
                self.send_header("Content-Type", "application/json")
                self.end_headers()
                self.wfile.write(json.dumps({"error": f"Invalid JSON: {e}"}).encode("utf-8"))
                return
            text = data.get("text", "").strip()
            # If text is not provided, build from structured fields:
            if not text:
                parts = []
                if data.get("service"):
                    parts.append(f"Service: {data['service']}")
                if data.get("kind"):
                    parts.append(f"Kind: {data['kind']}")
                if data.get("title"):
                    parts.append(f"Title: {data['title']}")
                if data.get("message"):
                    parts.append(f"Message: {data['message']}")
                if data.get("url"):
                    parts.append(f"URL: {data['url']}")
                if data.get("status_code"):
                    parts.append(f"Status: {data['status_code']}")
                text = " | ".join(parts)
            if not text:
                self.send_response(400)
                self._send_cors()
                self.send_header("Content-Type", "application/json")
                self.end_headers()
                self.wfile.write(json.dumps({"error": "Missing 'text' or telemetry fields"}).encode("utf-8"))
                return
            text = redact(text)
            key = hashlib.sha256(text.encode()).hexdigest()
            cached = _cache.get(key)
            if cached is not None:
                _stats["hits"] += 1
                out = {**cached, "cached": True}
                self.send_response(200)
                self._send_cors()
                self.send_header("Content-Type", "application/json")
                self.end_headers()
                self.wfile.write(json.dumps(out).encode("utf-8"))
                return
            _stats["misses"] += 1
            t_start = time.perf_counter()
            agent.reset()
            res = agent.run(text, strict=False)
            latency_ms = (time.perf_counter() - t_start) * 1000
            results = res.get("results", [])
            first_result = results[0] if results and isinstance(results[0], dict) else {}
            action = first_result.get("action", "needs_review")
            calls = res.get("function_calls", [])
            tool_name = calls[0].get("name") if calls else "triage_tool"
            out = {
                "success": True,
                "input_text": text,
                "action": action,
                "tool": tool_name,
                "arguments": first_result,
                "confidence": float(res.get("confidence", 0.0)),
                "reasoning": res.get("reasoning", ""),
                "latency_ms": round(latency_ms, 2),
                "model": "Cactus-Compute/needle3"
            }
            if len(_cache) >= CACHE_MAX:
                _cache.pop(next(iter(_cache)))
            _cache[key] = out
            self.send_response(200)
            self._send_cors()
            self.send_header("Content-Type", "application/json")
            self.end_headers()
            self.wfile.write(json.dumps(out).encode("utf-8"))
        else:
            self.send_response(404)
            self._send_cors()
            self.end_headers()
    def log_message(self, format, *args):
        # Concise logging
        print(f"[needle-sidecar] {args[0]} {args[1]} -> {args[2]}")
 def run():
    server = ThreadingHTTPServer((HOST, PORT), NeedleHandler)
    print(f"[needle-sidecar] Listening on http://{HOST}:{PORT}")
    try:
        server.serve_forever()
    except KeyboardInterrupt:
        print("\nShutting down needle sidecar...")
        server.server_close()
 if __name__ == "__main__":
    run()
@@ -490,8 +490,6 @@ func main() {
 	uiMux.HandleFunc("POST /api/sourcemaps", coll.HandleSourceMap)
 	uiMux.Handle("GET /api/sourcemaps", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleListSourceMaps)))
 	uiMux.Handle("POST /api/sourcemaps/resolve", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleResolveStacks)))
 	uiMux.HandleFunc("GET /api/needle/status", coll.HandleNeedleStatus)
 	uiMux.HandleFunc("POST /api/needle/classify", coll.HandleNeedleClassify)
 	// Protected API & WebSocket routes
 	uiMux.Handle("/api/history", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleHistory)))
@@ -1,16 +1,13 @@
 // spike-alerts replays the deterministic spike rule over a bounded window of
-// normalized_events and, optionally, asks the Needle sidecar what kind of
+// normalized_events. It only reads ClickHouse.
 // failure each top pattern is. It only reads ClickHouse.
 package main
 import (
 	"bytes"
 	"context"
 	"encoding/json"
 	"flag"
 	"fmt"
 	"log"
 	"net/http"
 	"os"
 	"strings"
 	"time"
@@ -36,7 +33,6 @@ func main() {
 	minExcess := flag.Int("min-excess", 5, "minimum excess errors to alert")
 	multiple := flag.Float64("multiple", 3, "pattern must be novel or >= multiple * its baseline")
 	cooldown := flag.Duration("cooldown", 30*time.Minute, "suppress repeats of a pattern for this long unless its count doubles")
 	needle := flag.String("needle", "", "Needle sidecar URL (dormant: it is not accurate enough to gate alerts); empty skips it")
 	showMuted := flag.Bool("show-muted", false, "also report non-actionable categories (probes, stale references, expected lifecycle)")
 	asJSON := flag.Bool("json", false, "emit JSON lines instead of text")
 	flag.Parse()
@@ -75,7 +71,6 @@ func main() {
 	log.Printf("bad-actor events (ignored for alerts, kept as data): %v of %d", badActors, len(events))
 	log.Printf("loaded %d error events", len(events))
 	cache := map[string]classification{}
 	first := start.Add(time.Duration(*baselineWindows) * *window)
 	suppress := features.NewSuppressor(*cooldown)
 	muted := map[triage.Category]int{}
@@ -87,24 +82,11 @@ func main() {
 		}
 		for _, spike := range suppress.Filter(detected) {
 			fired++
 			var cls []classification
 			for _, p := range spike.Patterns {
 				c := classification{}
 				if *needle != "" {
 					if hit, ok := cache[p.Signature]; ok {
 						c = hit
 					} else {
 						c = classify(*needle, needleText(p.Example))
 						cache[p.Signature] = c
 					}
 				}
 				cls = append(cls, c)
 			}
 			if *asJSON {
-				_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike, "classifications": cls})
+				_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"spike": spike})
 				continue
 			}
-			printSpike(spike, cls)
+			printSpike(spike)
 		}
 	}
 	for c, n := range muted {
@@ -113,28 +95,7 @@ func main() {
 	log.Printf("%d alert(s) between %s and %s", fired, first.Format(time.RFC3339), end.Format(time.RFC3339))
 }
-type classification struct {
+func eventText(e features.Event) string {
 	Action     string  `json:"action"`
 	Confidence float64 `json:"confidence"`
 	Cached     bool    `json:"cached"`
 	Err        string  `json:"error,omitempty"`
 }
 func classify(base, text string) classification {
 	body, _ := json.Marshal(map[string]string{"text": text})
 	resp, err := (&http.Client{Timeout: 60 * time.Second}).Post(strings.TrimRight(base, "/")+"/classify", "application/json", bytes.NewReader(body))
 	if err != nil {
 		return classification{Action: "?", Err: err.Error()}
 	}
 	defer resp.Body.Close()
 	var out classification
 	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
 		return classification{Action: "?", Err: err.Error()}
 	}
 	return out
 }
 func needleText(e features.Event) string {
 	parts := []string{"Service: " + e.Service, "Kind: " + e.Kind}
 	if e.SpanName != "" {
 		parts = append(parts, "Title: "+e.SpanName)
@@ -151,17 +112,14 @@ func needleText(e features.Event) string {
 	return strings.Join(parts, " | ")
 }
-func printSpike(s features.Spike, cls []classification) {
+func printSpike(s features.Spike) {
 	fmt.Printf("\n%s  %s  errors=%d (baseline %.1f/window, excess %.0f)\n", s.WindowEnd.Format("01-02 15:04"), s.Service, s.ErrorCount, s.BaselineMean, s.Excess)
-	for i, p := range s.Patterns {
+	for _, p := range s.Patterns {
 		tag := "rising"
 		if p.Novel {
 			tag = "NEW"
 		}
 		what := string(alerts.TriageOf(p.Example).Category)
 		if cls[i].Action != "" {
 			what += fmt.Sprintf(" [needle:%s %.2f]", cls[i].Action, cls[i].Confidence)
 		}
 		clients := ""
 		if p.Clients > 0 {
 			clients = fmt.Sprintf(" [%d client%s]", p.Clients, map[bool]string{true: "", false: "s"}[p.Clients == 1])
@@ -169,7 +127,7 @@ func printSpike(s features.Spike, cls []classification) {
 		if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) {
 			clients += " [" + p.Example.UAClass + "]"
 		}
-		fmt.Printf("  %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, needleText(p.Example), clients)
+		fmt.Printf("  %-6s x%-4d base %-6.1f %-24s %s%s\n", tag, p.Count, p.BaselineMean, what, eventText(p.Example), clients)
 	}
 }
@@ -1,7 +1,7 @@
 # Error detection model
 > Latest status, decisions and next steps: [STATUS.md](STATUS.md). Where they
-> differ (ClickHouse is now a LoadBalancer; Needle is dormant), STATUS.md wins.
+> differ (ClickHouse is now a LoadBalancer; Needle is removed), STATUS.md wins.
 ## Current handoff (2026-09-18)
@@ -10,7 +10,6 @@ unless stated. Greenfield, single user: radical changes are fine.
  patched live. `docs/ML.md` still says "keep ClusterIP"; that is superseded.
 - Credentials are in `.env` (gitignored, mode 600). Load with
  `set -a; . ./.env; set +a`. `clickhouse-client` is installed locally.
 - `.env` deliberately does not enable Needle; pass `-needle URL` explicitly.
 ## What exists now
@@ -21,7 +20,7 @@ unless stated. Greenfield, single user: radical changes are fine.
 | 404/410 are not errors | `isErrorStatus` in `internal/normalization/normalizer.go` | Done. Rows stored before this still count 404s as errors. |
 | Backfill | `cmd/normalized-backfill` | Ran 2026-09-19 12:00 to ~21:00 UTC. Queries need `FINAL` (duplicates until merge). |
 | Spike rule | `internal/features/spike.go` | Excess-error rule, trimmed-mean baseline, `Suppressor` (cooldown, 2x escalation). Tested. |
-| Replay CLI | `cmd/spike-alerts` | Replays a range; mutes non-actionable categories; Needle optional and off. |
+| Replay CLI | `cmd/spike-alerts` | Replays a range; mutes non-actionable categories. |
 | Cause categories | `internal/triage` | Deterministic rules, first match wins. Tested. |
 | Rules scoring | `cmd/triage-eval` | Scores a labelled JSONL; also prepares rows for labelling. |
 | Offender report | `cmd/bad-actors` | Read-only. Ranks IPs from raw nginx spans, marks `[BLOCK]` / `[INTERNAL]`. |
@@ -35,14 +34,14 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue`
 ## Findings worth remembering
- **Needle scored 10% on the 40-row golden set** (about chance for 8 classes).
+- **Needle was removed (2026-11-01).** It scored 10% on the 40-row golden set (about
-  Reversing the tool order changed its answers wholesale (15%, 34/40 `service_defect`),
+  chance for 8 classes), picked by tool position rather than content, and its
-  so it picks by position, not content. Confidence is uncorrelated with
+  confidence was uncorrelated with correctness. Sidecar, proxy handlers, UI and k8s
-  correctness. Decision: **Needle stays dormant.** Intended later use: a gate that
+  container are gone; the labelled sets live in `cmd/triage-eval/data/`.
-  decides whether a larger model should act on an issue (affects anyone? fixable by
+- **Edge 4xx never pages.** nginx 4xx (registry 401 challenges, `/oauth2/auth`,
-  a smart model?). Do not use it for categorization.
+  scanner 405s) is dropped in `FilterActionable`; nginx 5xx still alerts.
 - **Rules scored 39/40 on the golden set, but that is a fit** (written after seeing
-  the rows). A fresh 60-row holdout is in `cmd/needle-sidecar/eval/holdout.jsonl`
+  the rows). A fresh 60-row holdout is in `cmd/triage-eval/data/holdout.jsonl`
  (from 2026-09-18, unseen); it still needs your `expected` labels. Known
  probe-like paths that still land in `stale_reference`: `/ssl/server.key`,
  `/_debugbar/open`, `/debug/pprof`, `/actuator/*`, `/graphql`, `/server-info`,
@@ -78,7 +77,7 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue`
 1. Deploy the collector so normalized rows stay current; verify with
   `SELECT max(timestamp) FROM otel.normalized_events`.
 2. Label `holdout.jsonl` (fill `expected`), run
-   `go run ./cmd/triage-eval -input cmd/needle-sidecar/eval/holdout.jsonl`, fix
+   `go run ./cmd/triage-eval -input cmd/triage-eval/data/holdout.jsonl`, fix
   the rule families that miss. Then re-export a fresh holdout so scoring stays
   honest.
 3. Warm-up guard in `DetectSpikes`: require baseline data before alerting (the
@@ -164,7 +163,7 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue`
 ```bash
 set -a; . ./.env; set +a
-# replay the spike rule over a range (Needle off)
+# replay the spike rule over a range
 go run ./cmd/spike-alerts -since 2026-09-19T12:00:00Z -until 2026-09-19T20:30:00Z -baseline-windows 12
 # add -show-muted to see probes / stale / lifecycle patterns too
@@ -172,12 +171,7 @@ go run ./cmd/spike-alerts -since 2026-09-19T12:00:00Z -until 2026-09-19T20:30:00
 go run ./cmd/bad-actors -since 2026-09-19T08:00:00Z -until 2026-09-19T20:30:00Z
 # score the rules against labelled rows
-go run ./cmd/triage-eval -input cmd/needle-sidecar/eval/holdout.jsonl
+go run ./cmd/triage-eval -input cmd/triage-eval/data/holdout.jsonl
 # Needle eval (needs the sidecar running; venv was in scratch and is gone)
 pip install cactus-needle   # python 3.12 recommended
 NEEDLE_PORT=5005 NEEDLE_HOST=127.0.0.1 python cmd/needle-sidecar/server.py
 python cmd/needle-sidecar/eval/run_eval.py
 ```
 Backfill in small windows only (10 to 30 minutes over the LAN); one-hour windows
@@ -187,14 +181,11 @@ through a `kubectl port-forward` hung and dropped.
 Everything is uncommitted. Files changed by this work: `internal/{storage,
 normalization,features,triage,honeypot,blocker}`, `cmd/{spike-alerts,
-triage-eval,bad-actors,normalized-backfill,server,needle-sidecar}`,
+triage-eval,bad-actors,normalized-backfill,server}`,
 `k8s/{clickhouse,deployment}.yaml`, `.gitignore`.
 **Not from this work; check before committing:** `AGENTS.md`,
-`internal/collector/{collector_test,issue_extract}.go`, `otel-ui/src/**`
+`internal/collector/{collector_test,issue_extract}.go`, `otel-ui/src/**` (including a deleted `GaugeMeter.tsx`).
 (including a deleted `GaugeMeter.tsx`). Also tracked `__pycache__/*.pyc` files
 under `cmd/needle-sidecar` show as modified; they should be untracked and
 gitignored.
 ## IP details (MaxMind)
@@ -0,0 +1,29 @@
 # Deploying go-otel with k8s-helper
 Gitea Actions builds and pushes versioned and `latest` `matst80/otel-debug`
 images on pushes to `main`, uploads the matching source maps, and then sends a
 signed reconcile webhook. The agent applies the explicit `k8s/kustomization.yaml`
 resource list into namespace `monitoring`.
 ## One-time setup
 The k8s-helper GitOps CRDs and agent must be installed. The GitSource reuses
 `dev/gitea-k8s-helper`, whose token must have read access to this repository.
 Configure a Gitea push-only webhook for this repo at
 `https://ui.k6n.net/api/gitops/webhooks/gitea`, using the same secret as
 `K8SHELPER_GITOPS_WEBHOOK_SECRET` on the dashboard. Add that value as the Gitea
 Actions secret `GITOPS_WEBHOOK_SECRET` too; a user- or organization-level
 secret can be shared by other repos.
 The runner uses its existing Docker login. Bootstrap once with
 `kubectl apply -f gitops/` after confirming the monitoring namespace and the
 `otel-sourcemaps` PVC already exist. The PVC is deliberately excluded from the
 managed kustomization because its lifecycle was called out for separate review.
 The ClickHouse manifests, auth-dev resources, and ingress-catchall work are also
 excluded; this kustomization lists only the receiver Deployment, Service, and
 Ingress resources. Do not run Flux and k8s-helper against the same resources at
 the same time; check for an active `flux-system/go-otel` Kustomization before
 bootstrapping this controller.
 Prune is initially disabled. Inspect the GitOps inventory and applied revision
 before changing it.
@@ -0,0 +1,14 @@
 apiVersion: gitops.k6n.net/v1alpha1
 kind: Deployment
 metadata:
  name: go-otel
  namespace: dev
 spec:
  sourceRef:
    kind: GitSource
    name: go-otel
  path: ./k8s
  targetNamespace: monitoring
  prune: false
  wait: true
  deletionPolicy: Orphan
@@ -0,0 +1,16 @@
 # Keep GitOps CRs outside the path they manage.
 apiVersion: gitops.k6n.net/v1alpha1
 kind: GitSource
 metadata:
  name: go-otel
  namespace: dev
 spec:
  url: https://git.k6n.net/mats/go-otel.git
  ref:
    branch: main
  interval: 5m
  secretRef:
    name: gitea-k8s-helper
  ignore: |
    .gitea/
    docs/
@@ -141,45 +141,9 @@
          Clear Logs
        </button>
      </div>
      <div class="data-section" style="border-left: 4px solid #10b981;">
        <h2 style="border-bottom-color: #10b981; color: #065f46;">Needle 3 AI Telemetry Triage</h2>
        <p style="font-size: 13px; color: #666; margin-top: 0;">Test on-device telemetry classification with Cactus-Compute/needle3:</p>
        <div style="display: flex; gap: 8px; margin-bottom: 10px;">
          <input id="needle-input" type="text" style="flex: 1; padding: 8px 12px; border: 1px solid #ccc; border-radius: 4px; font-family: monospace; font-size: 12px;" value="User reported client browser crash on service eventsourced-ai-web: Script error on url https://event-agent.k6n.net/#session=angry-koala-1" />
          <button id="needle-btn" style="background: #10b981; color: white; border: none; padding: 8px 16px; border-radius: 4px; font-weight: bold; cursor: pointer;" onclick="classifyWithNeedle()">
            Classify with Needle 3
          </button>
        </div>
        <div id="needle-result" class="data-content" style="background: #ecfdf5; border: 1px solid #a7f3d0; min-height: 40px; color: #064e3b;">
          Click "Classify with Needle 3" to test the sidecar...
        </div>
      </div>
    </div>
    <script>
      async function classifyWithNeedle() {
        const input = document.getElementById("needle-input").value;
        const resultDiv = document.getElementById("needle-result");
        const btn = document.getElementById("needle-btn");
        if (!input.trim()) return;
        btn.textContent = "Classifying...";
        btn.disabled = true;
        try {
          const resp = await fetch("/api/needle/classify", {
            method: "POST",
            headers: { "Content-Type": "application/json" },
            body: JSON.stringify({ text: input }),
          });
          const data = await resp.json();
          resultDiv.innerHTML = `<pre style="margin: 0; font-family: monospace;">${JSON.stringify(data, null, 2)}</pre>`;
        } catch (err) {
          resultDiv.textContent = "Error: " + err;
        } finally {
          btn.textContent = "Classify with Needle 3";
          btn.disabled = false;
        }
      }
      let ws;
      let tracesCount = 0;
      let metricsCount = 0;
@@ -11,6 +11,13 @@ func TriageOf(e features.Event) triage.Result {
 		IsTimeout: e.IsTimeout, IsCancellation: e.IsCancellation, IsRetry: e.IsRetry})
 }
 // edgeClientError reports a 4xx seen at the nginx edge: registry auth
 // challenges, /oauth2/auth subrequests, 405s from scanners. These are the
 // caller's problem and never page, even next to a real 5xx.
 func edgeClientError(e features.Event) bool {
 	return e.Service == "nginx" && e.StatusCode >= 400 && e.StatusCode < 500
 }
 // FilterActionable keeps only patterns whose category should page a human
 // (unless keepMuted), recomputing each spike's excess so a probe storm cannot
 // alert by itself. muted, when non-nil, counts what was dropped per category.
@@ -21,7 +28,7 @@ func FilterActionable(spikes []features.Spike, keepMuted bool, muted map[triage.
 		sp.Excess = 0
 		for _, p := range sp.Patterns {
 			c := TriageOf(p.Example).Category
-			if !keepMuted && !c.Actionable() {
+			if !keepMuted && (!c.Actionable() || edgeClientError(p.Example)) {
 				if muted != nil {
 					muted[c]++
 				}
@@ -0,0 +1,180 @@
 package alerts
 import (
 	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
 )
 // Laya asks the laya model (POST {URL}/predict) what it would do about an
 // alert. It runs in shadow mode: the verdict is recorded on the alert and
 // shown in notifications, but nothing is ever acted on, muted or dropped
 // because of it. A failed call leaves the alert exactly as it was.
 type Laya struct {
 	URL    string // base URL, e.g. https://laya.k6n.net
 	Client *http.Client
 }
 // LayaActions are the remediations laya may choose between. They are
 // suggestions only; none is wired to an automated action yet.
 var LayaActions = map[string]string{
 	"ignore":             "transient blip or noise, nobody needs to look at it",
 	"restart_deployment": "the service is stuck, leaking or crashed; restarting its pods is likely to clear it",
 	"check_capacity":     "requests time out or resources are exhausted; check load, connection pools, CPU/memory limits and scaling",
 	"fix_dependency":     "a database, upstream service or network path this service depends on is failing",
 	"investigate_code":   "a bug or bad release in this service; needs a code fix or a rollback",
 }
 var layaUrgency = []string{"cosmetic", "degraded performance", "service outage"}
 // LayaVerdict is laya's answer, reduced to what a person needs to judge it.
 type LayaVerdict struct {
 	Model            string             `json:"model,omitempty"`
 	Action           string             `json:"action"`
 	ActionConfidence float64            `json:"actionConfidence"` // probability of the chosen action
 	ActionProbs      map[string]float64 `json:"actionProbs,omitempty"`
 	Human            float64            `json:"human"` // probability a person must intervene
 	Urgency          string             `json:"urgency"`
 	UrgencyScore     float64            `json:"urgencyScore"` // expected value on 0 (cosmetic) .. 2 (outage)
 	UrgencyConf      float64            `json:"urgencyConfidence"`
 }
 func layaRequest(state string) map[string]any {
 	return map[string]any{
 		"state": state,
 		"questions": map[string]any{
 			"remediation_action": map[string]any{
 				"type":         "choice",
 				"instructions": "Which operational response fits this production error alert best?",
 				"criteria":     LayaActions,
 			},
 			"requires_human_attention": map[string]any{
 				"type":         "noul",
 				"instructions": "Does this alert need a human engineer to intervene, or will it resolve or matter little on its own?",
 				"criteria": map[string]string{
 					"true":  "errors are new or rising well above normal and users or requests are failing",
 					"false": "a transient blip, client noise or bot traffic",
 				},
 			},
 			"urgency": map[string]any{
 				"type":         "score",
 				"instructions": "Rate the operational severity of this alert.",
 				"criteria":     layaUrgency,
 			},
 		},
 	}
 }
 type layaAnswer struct {
 	Choice           string             `json:"choice"`
 	Probabilities    map[string]float64 `json:"probabilities"`
 	AnswerConfidence float64            `json:"answer_confidence"`
 	Noul             float64            `json:"noul"`
 	Score            float64            `json:"score"`
 	Legend           map[string]string  `json:"legend"`
 }
 type layaResponse struct {
 	Model   string                `json:"model"`
 	Answers map[string]layaAnswer `json:"answers"`
 }
 // Classify asks laya about the alert. The state is the alert's plain-text
 // message, so the model sees exactly the facts a person would, nothing more.
 func (l *Laya) Classify(ctx context.Context, a Alert) (*LayaVerdict, error) {
 	a.Laya = nil
 	return l.classifyState(ctx, a.Text(""))
 }
 func (l *Laya) classifyState(ctx context.Context, state string) (*LayaVerdict, error) {
 	raw, err := json.Marshal(layaRequest(state))
 	if err != nil {
 		return nil, err
 	}
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(l.URL, "/")+"/predict", bytes.NewReader(raw))
 	if err != nil {
 		return nil, errors.New("laya: invalid URL")
 	}
 	req.Header.Set("Content-Type", "application/json")
 	client := l.Client
 	if client == nil {
 		client = &http.Client{Timeout: 10 * time.Second}
 	}
 	resp, err := client.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("laya: %w", err)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode/100 != 2 {
 		detail, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
 		return nil, fmt.Errorf("laya: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(detail)))
 	}
 	var lr layaResponse
 	if err := json.NewDecoder(resp.Body).Decode(&lr); err != nil {
 		return nil, fmt.Errorf("laya: invalid response: %w", err)
 	}
 	return lr.verdict()
 }
 func (lr layaResponse) verdict() (*LayaVerdict, error) {
 	act, ok1 := lr.Answers["remediation_action"]
 	hum, ok2 := lr.Answers["requires_human_attention"]
 	urg, ok3 := lr.Answers["urgency"]
 	if !ok1 || !ok2 || !ok3 || act.Choice == "" {
 		return nil, errors.New("laya: response is missing answers")
 	}
 	v := &LayaVerdict{Model: lr.Model, Action: act.Choice, ActionConfidence: act.Probabilities[act.Choice], ActionProbs: act.Probabilities,
 		Human: hum.Noul, UrgencyScore: urg.Score}
 	// The urgency label is the most probable bucket, not the rounded score:
 	// a 50/50 cosmetic/outage split must not read as "degraded".
 	best := -1.0
 	for k, p := range urg.Probabilities {
 		if p > best || (p == best && k < v.Urgency) {
 			best, v.Urgency = p, k
 		}
 	}
 	v.UrgencyConf = best
 	if label, ok := urg.Legend[v.Urgency]; ok {
 		v.Urgency = label
 	} else if i, err := strconv.Atoi(v.Urgency); err == nil && i >= 0 && i < len(layaUrgency) {
 		v.Urgency = layaUrgency[i]
 	}
 	return v, nil
 }
 // Summary is a one-line rendering, e.g.
 // "check_capacity 62% · human 71% · degraded performance 75%".
 func (v LayaVerdict) Summary() string {
 	return fmt.Sprintf("%s %s · human %s · %s %s", v.Action, pct(v.ActionConfidence), pct(v.Human), v.Urgency, pct(v.UrgencyConf))
 }
 // RunnersUp lists the other actions by probability, for judging how close
 // the call was.
 func (v LayaVerdict) RunnersUp() string {
 	type kv struct {
 		k string
 		p float64
 	}
 	var rest []kv
 	for k, p := range v.ActionProbs {
 		if k != v.Action {
 			rest = append(rest, kv{k, p})
 		}
 	}
 	sort.Slice(rest, func(i, j int) bool { return rest[i].p > rest[j].p })
 	parts := make([]string, 0, len(rest))
 	for _, r := range rest {
 		parts = append(parts, r.k+" "+pct(r.p))
 	}
 	return strings.Join(parts, ", ")
 }
 func pct(p float64) string { return strconv.Itoa(int(p*100+0.5)) + "%" }
@@ -0,0 +1,123 @@
 package alerts
 import (
 	"context"
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 	"testing"
 	"time"
 	"git.tornberg.me/go-otel/internal/features"
 )
 // layaReply is the shape laya.k6n.net/predict returned when probed.
 const layaReply = `{"model":"laya-rl-agent","answers":{
 "remediation_action":{"type":"choice","choice":"check_capacity","probabilities":{"ignore":0.1,"restart_deployment":0.12,"check_capacity":0.62,"fix_dependency":0.1,"investigate_code":0.06},"confidence":0.3,"answer_confidence":0.62},
 "requires_human_attention":{"type":"noul","noul":0.7123,"confidence":0.7123,"answer_confidence":0.7123},
 "urgency":{"type":"score","score":1.03,"legend":{"0":"cosmetic","1":"degraded performance","2":"service outage"},"probabilities":{"0":0.11,"1":0.75,"2":0.14},"confidence":0.32,"answer_confidence":0.75}},
 "usage":{"input_tokens":141,"output_tokens":0}}`
 func layaServer(t *testing.T, status int, body string, got *map[string]any) *httptest.Server {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.URL.Path != "/predict" {
 			t.Errorf("path %q", r.URL.Path)
 		}
 		if got != nil {
 			_ = json.NewDecoder(r.Body).Decode(got)
 		}
 		w.WriteHeader(status)
 		_, _ = w.Write([]byte(body))
 	}))
 	t.Cleanup(srv.Close)
 	return srv
 }
 func timeoutAlert() Alert {
 	now := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
 	return NewAlert(features.Spike{Service: "codesearch", WindowStart: now.Add(-5 * time.Minute), WindowEnd: now, ErrorCount: 15, BaselineMean: 0, Excess: 15,
 		Patterns: []features.PatternDelta{{Signature: "t", Count: 15, Excess: 15, Novel: true,
 			Example: features.Event{Service: "codesearch", Kind: "span", SpanName: "upload", Message: "context deadline exceeded", IsTimeout: true}}}}, now)
 }
 func TestLayaClassifySendsAlertTextAndParsesVerdict(t *testing.T) {
 	var req map[string]any
 	srv := layaServer(t, 200, layaReply, &req)
 	a := timeoutAlert()
 	v, err := (&Laya{URL: srv.URL + "/"}).Classify(context.Background(), a)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if req["state"] != a.Text("") {
 		t.Fatalf("state is not the alert text:\n%v", req["state"])
 	}
 	qs := req["questions"].(map[string]any)
 	for _, k := range []string{"remediation_action", "requires_human_attention", "urgency"} {
 		if qs[k] == nil {
 			t.Fatalf("question %s missing", k)
 		}
 	}
 	if v.Action != "check_capacity" || v.ActionConfidence != 0.62 || v.Human != 0.7123 || v.Urgency != "degraded performance" || v.UrgencyConf != 0.75 {
 		t.Fatalf("verdict %+v", v)
 	}
 	if got := v.Summary(); got != "check_capacity 62% · human 71% · degraded performance 75%" {
 		t.Fatalf("summary %q", got)
 	}
 	if got := v.RunnersUp(); !strings.HasPrefix(got, "restart_deployment 12%, ") {
 		t.Fatalf("runners-up %q", got)
 	}
 }
 func TestLayaErrors(t *testing.T) {
 	for name, tc := range map[string]struct {
 		status int
 		body   string
 	}{
 		"http":    {502, "bad gateway"},
 		"json":    {200, "not json"},
 		"missing": {200, `{"answers":{"urgency":{}}}`},
 	} {
 		srv := layaServer(t, tc.status, tc.body, nil)
 		if _, err := (&Laya{URL: srv.URL}).Classify(context.Background(), timeoutAlert()); err == nil {
 			t.Errorf("%s: want error", name)
 		}
 	}
 }
 func TestLayaVerdictInNotifications(t *testing.T) {
 	a := timeoutAlert()
 	a.Laya = &LayaVerdict{Action: "restart_deployment", ActionConfidence: 0.8, ActionProbs: map[string]float64{"restart_deployment": 0.8, "ignore": 0.2}, Human: 0.4, Urgency: "service outage", UrgencyConf: 0.6}
 	h := a.HTML("https://otel.example", telegramMax)
 	if !strings.Contains(h, "🤖 <b>laya</b> <i>(shadow, no action taken)</i>: <code>restart_deployment</code> 80% · human 40% · service outage 60%") ||
 		!strings.Contains(h, "other: ignore 20%") {
 		t.Fatalf("html:\n%s", h)
 	}
 	if txt := a.Text(""); !strings.Contains(txt, "laya (shadow, no action taken): restart_deployment 80%") {
 		t.Fatalf("text:\n%s", txt)
 	}
 	a.Laya = nil
 	if strings.Contains(a.HTML("", telegramMax), "laya") {
 		t.Fatal("no verdict must render no laya line")
 	}
 }
 // A laya outage must never cost a page; the alert just goes out without a verdict.
 func TestRunnerAttachesVerdictAndSurvivesLayaFailure(t *testing.T) {
 	for _, tc := range []struct {
 		status int
 		want   bool
 	}{{200, true}, {500, false}} {
 		srv := layaServer(t, tc.status, layaReply, nil)
 		f := &fake{}
 		r := newRunner(nil, f)
 		r.laya = &Laya{URL: srv.URL}
 		r.deliver(context.Background(), timeoutAlert())
 		if len(f.sent) != 1 {
 			t.Fatalf("status %d: sent %d", tc.status, len(f.sent))
 		}
 		if got := f.sent[0].Laya != nil; got != tc.want {
 			t.Fatalf("status %d: verdict attached=%v", tc.status, got)
 		}
 	}
 }
@@ -29,6 +29,9 @@ type Alert struct {
 	Spike    features.Spike  `json:"spike"`
 	Patterns []PatternReport `json:"patterns"`
 	Test     bool            `json:"test,omitempty"`
 	// Laya is the laya model's shadow verdict, when it was asked. Advisory
 	// only: nothing is acted on or suppressed because of it.
 	Laya *LayaVerdict `json:"laya,omitempty"`
 }
 // PatternReport is a spike pattern with its triage verdict, ready to show.
@@ -214,6 +217,9 @@ func (a Alert) Text(baseURL string) string {
 	window := a.Spike.WindowEnd.Sub(a.Spike.WindowStart).Round(time.Minute)
 	fmt.Fprintf(&b, "%s: %s is failing (%d errors in the last %s, normally about %.0f)\n",
 		prefix, a.Spike.Service, a.Spike.ErrorCount, window, a.Spike.BaselineMean)
 	if a.Laya != nil {
 		fmt.Fprintf(&b, "laya (shadow, no action taken): %s\n", a.Laya.Summary())
 	}
 	base := strings.TrimRight(baseURL, "/")
 	for i, p := range a.Patterns {
 		fmt.Fprintf(&b, "\n%d. %s\n", i+1, p.What)
@@ -282,6 +288,13 @@ func (a Alert) HTML(baseURL string, max int) string {
 	if base != "" {
 		foot = fmt.Sprintf("\n<a href=\"%s\">All issues</a>", esc(base+"/issues"))
 	}
 	if a.Laya != nil {
 		head += fmt.Sprintf("🤖 <b>laya</b> <i>(shadow, no action taken)</i>: <code>%s</code> %s · human %s · %s %s\n",
 			esc(a.Laya.Action), pct(a.Laya.ActionConfidence), pct(a.Laya.Human), esc(a.Laya.Urgency), pct(a.Laya.UrgencyConf))
 		if r := a.Laya.RunnersUp(); r != "" {
 			head += "<i>   other: " + esc(r) + "</i>\n"
 		}
 	}
 	var b strings.Builder
 	b.WriteString(head)
 	for i, p := range a.Patterns {
@@ -34,6 +34,9 @@ type Config struct {
 	Muted                     func(issueID string) bool
 	WebhookURL, WebhookSecret string
 	AODURL                    string
 	// LayaURL, when set, asks laya (POST {LayaURL}/predict) for a shadow
 	// verdict on every alert and shows it in the notification.
 	LayaURL string
 }
 func ConfigFromEnv() Config {
@@ -54,6 +57,7 @@ func ConfigFromEnv() Config {
 		WebhookURL:       os.Getenv("OTEL_ALERT_WEBHOOK_URL"),
 		WebhookSecret:    os.Getenv("OTEL_ALERT_WEBHOOK_SECRET"),
 		AODURL:           os.Getenv("OTEL_ALERT_AOD_URL"),
 		LayaURL:          os.Getenv("OTEL_ALERT_LAYA_URL"),
 	}
 }
@@ -106,6 +110,7 @@ type Runner struct {
 	dataSince func(ctx context.Context) time.Time
 	notifiers []Notifier
 	resolver  Resolver // optional: ingress -> service -> deployment for edge errors
 	laya      *Laya    // optional: shadow verdict attached to each alert
 	sup       *features.Suppressor
 	mu        sync.Mutex
@@ -137,6 +142,10 @@ func Start(ctx context.Context, cfg Config, conn driver.Conn, database string, r
 		},
 		dataSince: func(ctx context.Context) time.Time { return DataSince(ctx, conn, database) },
 	}
 	if cfg.LayaURL != "" {
 		r.laya = &Laya{URL: cfg.LayaURL}
 		log.Printf("alerts: laya shadow verdicts from %s", cfg.LayaURL)
 	}
 	names := make([]string, 0, len(notifiers))
 	for _, n := range notifiers {
 		names = append(names, n.Name())
@@ -239,6 +248,7 @@ func (r *Runner) deliver(ctx context.Context, a Alert) {
 	if pr, ok := r.resolver.(PodResolver); ok {
 		a.EnrichPods(ctx, pr)
 	}
 	r.askLaya(ctx, &a)
 	r.mu.Lock()
 	r.recent = append(r.recent, a)
 	if len(r.recent) > maxRecent {
@@ -261,6 +271,23 @@ func (r *Runner) deliver(ctx context.Context, a Alert) {
 	}
 }
 // askLaya attaches laya's shadow verdict. It is bounded so a slow model
 // delays a page by seconds at most, and a failure only costs the verdict.
 func (r *Runner) askLaya(ctx context.Context, a *Alert) {
 	if r.laya == nil {
 		return
 	}
 	lctx, cancel := context.WithTimeout(ctx, 8*time.Second)
 	defer cancel()
 	v, err := r.laya.Classify(lctx, *a)
 	if err != nil {
 		log.Printf("alerts: laya verdict for %s failed: %v", a.ID, err)
 		return
 	}
 	a.Laya = v
 	log.Printf("alerts: laya[shadow] %s: %s", a.ID, v.Summary())
 }
 // Status is the JSON served by GET /api/alerts.
 type Status struct {
 	Enabled   bool      `json:"enabled"`
@@ -305,6 +332,7 @@ func (r *Runner) HandleTest(w http.ResponseWriter, req *http.Request) {
 		Example: features.Event{Service: "alert-test", Kind: "span", StatusCode: 500, Route: "/api/example", Message: "this is a test alert"},
 	}}}, now)
 	a.Test = true
 	r.askLaya(req.Context(), &a)
 	results := map[string]string{}
 	for _, n := range r.notifiers {
 		if err := n.Send(req.Context(), a); err != nil {
@@ -13,7 +13,6 @@ import (
 	"io"
 	"log"
 	"net/http"
 	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -1007,110 +1006,6 @@ func (c *Collector) HandleStatsEndpoints(w http.ResponseWriter, r *http.Request)
 	})
 }
 // needleAddr returns the configured or default address for the Needle 3 sidecar.
 func needleAddr() string {
 	addr := os.Getenv("OTEL_NEEDLE_ADDR")
 	if addr == "" {
 		addr = "http://127.0.0.1:5005"
 	}
 	return strings.TrimRight(addr, "/")
 }
 // HandleNeedleStatus checks if the Needle 3 sidecar is reachable and reports its health.
 func (c *Collector) HandleNeedleStatus(w http.ResponseWriter, r *http.Request) {
 	if r.Method != http.MethodGet {
 		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
 		return
 	}
 	target := needleAddr() + "/health"
 	client := &http.Client{Timeout: 2 * time.Second}
 	resp, err := client.Get(target)
 	w.Header().Set("Content-Type", "application/json")
 	if err != nil {
 		w.WriteHeader(http.StatusServiceUnavailable)
 		_ = json.NewEncoder(w).Encode(map[string]interface{}{
 			"available": false,
 			"error":     fmt.Sprintf("Needle sidecar unreachable at %s: %v", target, err),
 		})
 		return
 	}
 	defer resp.Body.Close()
 	w.WriteHeader(resp.StatusCode)
 	_, _ = io.Copy(w, resp.Body)
 }
 // HandleNeedleClassify proxies telemetry triage requests to the Needle 3 sidecar.
 func (c *Collector) HandleNeedleClassify(w http.ResponseWriter, r *http.Request) {
 	if r.Method != http.MethodPost {
 		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
 		return
 	}
 	body, err := io.ReadAll(r.Body)
 	if err != nil {
 		http.Error(w, "failed to read request body", http.StatusBadRequest)
 		return
 	}
 	// If the payload specifies an issue_id and no text, look up the issue
 	var reqMap map[string]interface{}
 	if err := json.Unmarshal(body, &reqMap); err == nil {
 		if issueID, ok := reqMap["issue_id"].(string); ok && issueID != "" {
 			if issue, ok := c.GetIssue(issueID); ok {
 				var promptParts []string
 				promptParts = append(promptParts, fmt.Sprintf("Service: %s", issue.Service))
 				promptParts = append(promptParts, fmt.Sprintf("Title: %s", issue.Title))
 				promptParts = append(promptParts, fmt.Sprintf("Kind: %s", issue.Kind))
 				if len(issue.Samples) > 0 {
 					latest := issue.Samples[len(issue.Samples)-1]
 					if latest.Message != "" {
 						promptParts = append(promptParts, fmt.Sprintf("Message: %s", latest.Message))
 					}
 					if latest.URL != "" {
 						promptParts = append(promptParts, fmt.Sprintf("URL: %s", latest.URL))
 					}
 					if latest.HTTPStatus > 0 {
 						promptParts = append(promptParts, fmt.Sprintf("Status: %d", latest.HTTPStatus))
 					}
 					if latest.HTTPRoute != "" {
 						promptParts = append(promptParts, fmt.Sprintf("Route: %s", latest.HTTPRoute))
 					}
 				}
 				reqMap["text"] = strings.Join(promptParts, " | ")
 				reqMap["service"] = issue.Service
 				reqMap["title"] = issue.Title
 				body, _ = json.Marshal(reqMap)
 			}
 		}
 	}
 	target := needleAddr() + "/classify"
 	client := &http.Client{Timeout: 60 * time.Second}
 	proxyReq, err := http.NewRequestWithContext(r.Context(), http.MethodPost, target, strings.NewReader(string(body)))
 	if err != nil {
 		http.Error(w, "failed to create proxy request", http.StatusInternalServerError)
 		return
 	}
 	proxyReq.Header.Set("Content-Type", "application/json")
 	resp, err := client.Do(proxyReq)
 	if err != nil {
 		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(http.StatusServiceUnavailable)
 		_ = json.NewEncoder(w).Encode(map[string]interface{}{
 			"success": false,
 			"error":   fmt.Sprintf("Needle sidecar unreachable at %s: %v", target, err),
 		})
 		return
 	}
 	defer resp.Body.Close()
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(resp.StatusCode)
 	_, _ = io.Copy(w, resp.Body)
 }
 // stampFrontendClient fills in client.address and user_agent.original on
 // browser-originated log records (source=frontend) from the ingest request,
 // since the browser cannot know its own public IP. Values the SDK already
@@ -1,289 +0,0 @@
 // Package laya talks to the Laya decision API (https://laya.k6n.net) - a fast
 // non-autoregressive "System 1" model that answers typed questions about a
 // state in one forward pass. It is used here as an incident triage brain: give
 // it a compact textual state of a firing spike and it returns a calibrated
 // choice (restart / ignore / notify / escalate) with probabilities.
 //
 // The API is described by https://laya.k6n.net/openapi.json. Only the parts
 // this repo uses are modelled; the wire format is deliberately loose on the
 // response side (answers are a map) because the model decides which questions
 // get answers.
 package laya
 import (
 	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"strings"
 	"time"
 )
 // Question kinds understood by the API.
 const (
 	KindChoice = "choice" // pick one label from Criteria
 	KindScore  = "score"  // pick a level index from Levels
 	KindNoul   = "noul"   // a single 0..1 non-choice answer
 )
 // Question is one typed question. Use the ChoiceQuestion, ScoreQuestion or
 // NoulQuestion constructors rather than building it by hand, because
 // "criteria" is a JSON object for choice questions and a JSON array for score
 // ones - both are rendered by MarshalJSON.
 type Question struct {
 	kind         string
 	instructions string
 	criteria     map[string]string // choice: label -> what that label means
 	levels       []string          // score: level descriptions, index 0 first
 }
 func (q Question) MarshalJSON() ([]byte, error) {
 	out := map[string]any{"type": q.kind, "instructions": q.instructions}
 	switch q.kind {
 	case KindChoice:
 		if len(q.criteria) == 0 {
 			return nil, errors.New("laya: choice question needs criteria")
 		}
 		out["criteria"] = q.criteria
 	case KindScore:
 		if len(q.levels) == 0 {
 			return nil, errors.New("laya: score question needs criteria levels")
 		}
 		out["criteria"] = q.levels
 	}
 	return json.Marshal(out)
 }
 func (q Question) Kind() string { return q.kind }
 // ChoiceQuestion asks the model to pick one of criteria. The label -> text map
 // matters: the model reads the descriptions, not just the labels, so "restart"
 // should say what restarting means here.
 func ChoiceQuestion(instructions string, criteria map[string]string) Question {
 	return Question{kind: KindChoice, instructions: instructions, criteria: criteria}
 }
 // ScoreQuestion asks for a level between 0 and len(levels)-1.
 func ScoreQuestion(instructions string, levels []string) Question {
 	return Question{kind: KindScore, instructions: instructions, levels: levels}
 }
 // NoulQuestion asks a single open question answered with one 0..1 value.
 func NoulQuestion(instructions string) Question {
 	return Question{kind: KindNoul, instructions: instructions}
 }
 // PredictRequest is the body of POST /v1/predict. State is a string, a map or
 // a list; this repo always sends a rendered text state.
 type PredictRequest struct {
 	State     any                 `json:"state"`
 	Questions map[string]Question `json:"questions"`
 	Lang      string              `json:"lang,omitempty"`
 	MaxLen    int                 `json:"max_len,omitempty"`
 	BatchSize int                 `json:"batch_size,omitempty"` // batch endpoint only
 }
 // BatchPredictRequest is the body of POST /v1/predict/batch: several states
 // evaluated against the same questions, packed into shared forward passes.
 type BatchPredictRequest struct {
 	States    []any               `json:"states"`
 	Questions map[string]Question `json:"questions"`
 	Lang      string              `json:"lang,omitempty"`
 	BatchSize int                 `json:"batch_size,omitempty"`
 	MaxLen    int                 `json:"max_len,omitempty"`
 }
 // Answer is one answered question. Only the fields matching Type are set.
 type Answer struct {
 	Type          string             `json:"type"`
 	Choice        string             `json:"choice,omitempty"`
 	Probabilities map[string]float64 `json:"probabilities,omitempty"`
 	Score         float64            `json:"score,omitempty"`
 	Legend        map[string]string  `json:"legend,omitempty"`
 	Noul          float64            `json:"noul,omitempty"`
 	Confidence    float64            `json:"confidence,omitempty"`
 	// AnswerConfidence is the model's confidence in the answer it picked,
 	// as opposed to Confidence, which measures how decisive the forward pass
 	// was overall.
 	AnswerConfidence float64 `json:"answer_confidence,omitempty"`
 }
 // Probability returns the model's probability for one choice label, 0 if the
 // label was not offered or not answered.
 func (a Answer) Probability(label string) float64 { return a.Probabilities[label] }
 // Margin is how far the chosen answer leads the next best one. The label
 // probabilities are diffuse - even a correct answer rarely exceeds 0.5 - so
 // the lead over the runner-up is a better confidence signal than the raw
 // probability.
 func (a Answer) Margin() float64 {
 	best, second := 0.0, 0.0
 	for _, p := range a.Probabilities {
 		switch {
 		case p > best:
 			second, best = best, p
 		case p > second:
 			second = p
 		}
 	}
 	return best - second
 }
 type Usage struct {
 	InputTokens  int `json:"input_tokens"`
 	OutputTokens int `json:"output_tokens"`
 }
 type Response struct {
 	Model   string            `json:"model"`
 	Answers map[string]Answer `json:"answers"`
 	Usage   Usage             `json:"usage"`
 }
 // Health is the body of GET /healthz.
 type Health struct {
 	Status  string `json:"status"`
 	Model   string `json:"model"`
 	Device  string `json:"device"`
 	Version string `json:"version"`
 }
 // Client is a Laya API client. BaseURL defaults to https://laya.k6n.net.
 type Client struct {
 	BaseURL   string
 	HTTP      *http.Client
 	Lang      string
 	MaxLen    int
 	BatchSize int // forward-pass chunking for the batch endpoint; 0 = server default
 }
 // NewClient returns a client for base ("https://laya.k6n.net" when empty).
 func NewClient(base string) *Client {
 	return &Client{BaseURL: strings.TrimRight(base, "/"), HTTP: &http.Client{Timeout: 30 * time.Second}, Lang: "en"}
 }
 func (c *Client) url(path string) string {
 	base := c.BaseURL
 	if base == "" {
 		base = "https://laya.k6n.net"
 	}
 	return base + path
 }
 func (c *Client) client() *http.Client {
 	if c.HTTP != nil {
 		return c.HTTP
 	}
 	return &http.Client{Timeout: 30 * time.Second}
 }
 // Health reports the model behind the API, for a setup probe.
 func (c *Client) Health(ctx context.Context) (Health, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url("/healthz"), nil)
 	if err != nil {
 		return Health{}, err
 	}
 	resp, err := c.client().Do(req)
 	if err != nil {
 		return Health{}, unwrap(err)
 	}
 	defer resp.Body.Close()
 	if err := checkStatus(resp); err != nil {
 		return Health{}, err
 	}
 	var h Health
 	if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&h); err != nil {
 		return Health{}, fmt.Errorf("laya: decode health: %w", err)
 	}
 	return h, nil
 }
 // Predict evaluates one state against the questions in a single pass.
 func (c *Client) Predict(ctx context.Context, state any, questions map[string]Question) (Response, error) {
 	body := PredictRequest{State: state, Questions: questions, Lang: c.Lang, MaxLen: c.MaxLen}
 	return c.post(ctx, "/v1/predict", body)
 }
 // PredictBatch evaluates several states against the same questions. Results
 // are returned in request order, one entry per state.
 func (c *Client) PredictBatch(ctx context.Context, states []any, questions map[string]Question) ([]Response, error) {
 	body := BatchPredictRequest{States: states, Questions: questions, Lang: c.Lang, BatchSize: c.BatchSize, MaxLen: c.MaxLen}
 	var out struct {
 		Results []Response `json:"results"`
 	}
 	if err := c.postInto(ctx, "/v1/predict/batch", body, &out); err != nil {
 		return nil, err
 	}
 	if len(out.Results) != len(states) {
 		return nil, fmt.Errorf("laya: batch returned %d results for %d states", len(out.Results), len(states))
 	}
 	return out.Results, nil
 }
 func (c *Client) post(ctx context.Context, path string, body any) (Response, error) {
 	var out Response
 	if err := c.postInto(ctx, path, body, &out); err != nil {
 		return Response{}, err
 	}
 	return out, nil
 }
 func (c *Client) postInto(ctx context.Context, path string, body, out any) error {
 	raw, err := json.Marshal(body)
 	if err != nil {
 		return err
 	}
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url(path), bytes.NewReader(raw))
 	if err != nil {
 		return errors.New("laya: invalid request")
 	}
 	req.Header.Set("Content-Type", "application/json")
 	resp, err := c.client().Do(req)
 	if err != nil {
 		return unwrap(err)
 	}
 	defer resp.Body.Close()
 	if err := checkStatus(resp); err != nil {
 		return err
 	}
 	return json.NewDecoder(io.LimitReader(resp.Body, 1<<22)).Decode(out)
 }
 // unwrap drops the URL the transport puts in its error, which may carry a
 // token; the returned error must be safe to log.
 func unwrap(err error) error {
 	var ue interface{ Unwrap() error }
 	if errors.As(err, &ue) {
 		return errors.New(scrub(ue.Unwrap().Error()))
 	}
 	return errors.New(scrub(err.Error()))
 }
 // scrub removes anything that looks like a credential from a URL in an error.
 func scrub(msg string) string {
 	for _, part := range strings.Fields(msg) {
 		if i := strings.Index(part, "://"); i >= 0 {
 			rest := part[i+3:]
 			if at := strings.IndexByte(rest, '@'); at >= 0 {
 				host := rest[at+1:]
 				msg = strings.ReplaceAll(msg, part, "laya://"+host)
 			}
 		}
 	}
 	return msg
 }
 func checkStatus(resp *http.Response) error {
 	if resp.StatusCode/100 == 2 {
 		return nil
 	}
 	detail, _ := io.ReadAll(io.LimitReader(resp.Body, 500))
 	msg := strings.TrimSpace(string(detail))
 	if msg == "" {
 		msg = http.StatusText(resp.StatusCode)
 	}
 	return fmt.Errorf("laya: HTTP %d: %s", resp.StatusCode, scrub(msg))
 }
@@ -1,579 +0,0 @@
 package laya
 import (
 	"context"
 	"fmt"
 	"sort"
 	"strings"
 	"time"
 )
 // Action is what the decision engine wants done with a firing alert. These are
 // the labels offered to the model, so they must stay stable: a rename changes
 // what the model was trained to answer.
 type Action string
 const (
 	ActionRestart  Action = "restart"  // roll the Deployment's pods
 	ActionIgnore   Action = "ignore"   // known noise, do not page
 	ActionNotify   Action = "notify"   // tell the on-call
 	ActionEscalate Action = "escalate" // tell a human who owns the service
 )
 // Actions is the full ladder, least to most forceful.
 var Actions = []Action{ActionIgnore, ActionNotify, ActionEscalate, ActionRestart}
 // Pattern is one reduced error pattern of an alert, as the triage package
 // classifies it. It is deliberately plain data: the engine can be fed from
 // anywhere without importing the alerting pipeline.
 type Pattern struct {
 	What     string // "HTTP 500 | /api/cart | panic: ..."
 	Category string // triage category, e.g. service-defect
 	Status   int    // HTTP status, 0 when the error was not an HTTP one
 	Count    int
 	Baseline float64
 	New      bool     // never seen in the baseline windows
 	Clients  int      // distinct clients affected
 	Services []string // every service failing in these requests
 	Host     string   // ingress host, for edge errors
 	Path     string   // request path
 	Routes   []string // "host/path -> service ns/name -> deployment ns/name"
 }
 // Pod is a pod behind one of the alert's deployments, with the facts that
 // explain errors the log lines alone cannot: a pod being replaced right now
 // (so a 503 burst is expected) or a crash loop (so a restart changes nothing).
 type Pod struct {
 	Namespace, Deployment, Name string
 	Found                       bool
 	Ready                       bool
 	Phase                       string
 	Restarts                    int32  // restarts of this pod
 	PodRestarts                 int32  // restarts across the Deployment
 	Reason                      string // CrashLoopBackOff, OOMKilled, ...
 	ExitCode                    *int32
 	Message                     string
 	LastTerminatedAt            time.Time
 	LastEvents                  []string // last log lines, oldest first
 	Err                         string
 }
 // Deploy is a rollout that overlaps the alert window.
 type Deploy struct {
 	Namespace, Workload string
 	Release             string
 	Age                 time.Duration // how long ago the rollout started
 	Completed           bool
 	Ready, Desired      int32
 }
 // Input is everything known about a firing alert, rendered into the state text
 // the model reasons over.
 type Input struct {
 	AlertID      string
 	Service      string
 	Priority     string // high | medium | low
 	ErrorCount   int
 	Window       time.Duration
 	BaselineMean float64
 	NewErrors    bool
 	Clients      int
 	Patterns     []Pattern
 	Pods         []Pod
 	Deploys      []Deploy
 	// KnownNoise marks a pattern a human has already marked as ignorable, e.g.
 	// a scanner or a muted issue. The model is told so it need not guess.
 	KnownNoise bool
 }
 // maxStateChars keeps the rendered state inside the model's context. The tail
 // (pods, deploys) is what the model most often misses, so patterns are cut
 // first, never the pod events.
 const maxStateChars = 4000
 // Render turns an alert into a compact factual state. It states only what was
 // measured - the model supplies judgement, this supplies facts - and always
 // includes the pods' last events, because a 503 burst while a pod is being
 // replaced is expected and indistinguishable from an outage without them.
 func (in Input) Render(now time.Time) string {
 	var b strings.Builder
 	multiple := 0.0
 	if in.BaselineMean > 0 {
 		multiple = float64(in.ErrorCount) / in.BaselineMean
 	}
 	fmt.Fprintf(&b, "Firing error spike in service %s (priority %s).\n", or(in.Service, "unknown"), or(in.Priority, "unknown"))
 	fmt.Fprintf(&b, "%d errors in the last %s, normally about %.0f per window (%.0fx normal).\n",
 		in.ErrorCount, in.Window.Round(time.Second), in.BaselineMean, multiple)
 	if in.NewErrors {
 		b.WriteString("At least one error is new: it was never seen in the baseline windows.\n")
 	}
 	if in.Clients > 0 {
 		fmt.Fprintf(&b, "%d distinct clients affected.\n", in.Clients)
 	}
 	if in.KnownNoise {
 		b.WriteString("A human has already marked this pattern as ignorable noise.\n")
 	}
 	b.WriteString("\nError patterns, most urgent first:\n")
 	for i, p := range in.Patterns {
 		fmt.Fprintf(&b, "%d. [%s] %s\n", i+1, or(p.Category, "unclassified"), or(p.What, "no message"))
 		fmt.Fprintf(&b, "   %d in the window, normally %.0f. %s. clients affected: %d.",
 			p.Count, p.Baseline, newness(p.New), p.Clients)
 		if p.Status > 0 {
 			fmt.Fprintf(&b, " HTTP %d.", p.Status)
 		}
 		b.WriteString("\n")
 		if len(p.Services) > 1 {
 			fmt.Fprintf(&b, "   Failing together: %s.\n", strings.Join(p.Services, ", "))
 		}
 		for _, r := range p.Routes {
 			fmt.Fprintf(&b, "   Route: %s.\n", r)
 		}
 	}
 	if len(in.Pods) == 0 {
 		b.WriteString("\nPod state: unknown, the deployment behind these errors was not resolved.\n")
 	}
 	for _, p := range in.Pods {
 		b.WriteString("\nPod events:\n")
 		fmt.Fprintf(&b, "- %s/%s pod %s: %s\n", p.Namespace, p.Deployment, or(p.Name, "unknown"), podStateLine(p, now))
 		for _, line := range p.LastEvents {
 			fmt.Fprintf(&b, "  log: %s\n", clip(line, 160))
 		}
 	}
 	for _, d := range in.Deploys {
 		state := "rollout in progress"
 		if d.Completed {
 			state = "rollout completed"
 		}
 		fmt.Fprintf(&b, "\nDeploy: %s/%s %s started %s ago, %d/%d replicas ready",
 			d.Namespace, d.Workload, state, d.Age.Round(time.Second), d.Ready, d.Desired)
 		if d.Release != "" {
 			fmt.Fprintf(&b, " (release %s)", d.Release)
 		}
 		b.WriteString("\n")
 	}
 	return clip(b.String(), maxStateChars)
 }
 // podStateLine says what the pod is doing now, in the words an operator would
 // use: crash-looping, being replaced, up but not ready, or healthy.
 func podStateLine(p Pod, now time.Time) string {
 	switch {
 	case p.Err != "":
 		return "could not be inspected (" + clip(p.Err, 80) + ")"
 	case !p.Found:
 		return "no pod running, the Deployment is scaled to 0 or has nothing ready"
 	}
 	var parts []string
 	phase := or(p.Phase, "unknown")
 	if p.Ready {
 		parts = append(parts, "phase "+phase+", passing its readiness probe")
 	} else {
 		parts = append(parts, "phase "+phase+", NOT passing its readiness probe")
 	}
 	switch {
 	case p.Restarting(now):
 		parts = append(parts, fmt.Sprintf("its container restarted %s ago and is still coming up", since(p.LastTerminatedAt, now)))
 	case p.CrashLooping(now):
 		parts = append(parts, fmt.Sprintf("crash-looping: %d restarts", p.Restarts))
 	case p.Restarts > 0:
 		parts = append(parts, fmt.Sprintf("%d restarts, none recent", p.Restarts))
 	}
 	if p.PodRestarts > p.Restarts {
 		parts = append(parts, fmt.Sprintf("%d restarts across the Deployment", p.PodRestarts))
 	}
 	if p.Reason != "" {
 		parts = append(parts, "last container termination: "+p.Reason)
 	}
 	if p.ExitCode != nil {
 		parts = append(parts, fmt.Sprintf("exit code %d", *p.ExitCode))
 	}
 	if p.Message != "" {
 		parts = append(parts, clip(p.Message, 120))
 	}
 	return strings.Join(parts, "; ")
 }
 // Restarting reports a pod that is right now being replaced, so the errors
 // around it are the rollout's, not a new defect.
 func (p Pod) Restarting(now time.Time) bool {
 	return !p.LastTerminatedAt.IsZero() && !now.Before(p.LastTerminatedAt) && now.Sub(p.LastTerminatedAt) < rollingWindow
 }
 // CrashLooping reports a pod stuck restarting: a restart cannot fix it.
 func (p Pod) CrashLooping(now time.Time) bool {
 	if p.Reason == "CrashLoopBackOff" {
 		return true
 	}
 	return p.Restarts >= 2 && !p.LastTerminatedAt.IsZero() && now.Sub(p.LastTerminatedAt) < rollingWindow
 }
 func newness(isNew bool) string {
 	if isNew {
 		return "never seen before"
 	}
 	return "seen before"
 }
 func since(t, now time.Time) string {
 	d := now.Sub(t)
 	switch {
 	case d < time.Minute:
 		return fmt.Sprintf("%ds", int(d.Seconds()))
 	case d < time.Hour:
 		return fmt.Sprintf("%dm", int(d.Minutes()))
 	default:
 		return fmt.Sprintf("%.1fh", d.Hours())
 	}
 }
 func or(v, fallback string) string {
 	if strings.TrimSpace(v) == "" {
 		return fallback
 	}
 	return v
 }
 func clip(s string, n int) string {
 	s = strings.TrimSpace(s)
 	if len(s) <= n {
 		return s
 	}
 	return strings.TrimSpace(s[:n-1]) + "…"
 }
 // Policy decides which of the model's answers may be executed. The model
 // proposes; the policy disposes. Nothing is acted on unless it is in Actions,
 // clears MinProbability, and survives the gates below.
 type Policy struct {
 	// Actions allowed to be executed automatically. An empty list means
 	// "decide and report, never act", which is the safe default.
 	Actions []Action
 	// MinProbability is the model's probability for its chosen action below
 	// which the decision is downgraded to a notification.
 	MinProbability float64
 	// MinMargin is the lead over the runner-up required to act. The label
 	// probabilities are diffuse, so this is the more meaningful of the two.
 	MinMargin float64
 	// MinClients: a pattern affecting fewer clients than this is never
 	// auto-acted on, however confident the model is.
 	MinClients int
 	// MaxRestartsPerHour caps automatic restarts per deployment, so a flapping
 	// service cannot be restarted into a loop.
 	MaxRestartsPerHour int
 	// Now is the clock, overridable in tests.
 	Now func() time.Time
 }
 func (p Policy) now() time.Time {
 	if p.Now != nil {
 		return p.Now()
 	}
 	return time.Now()
 }
 func (p Policy) allows(a Action) bool {
 	for _, allowed := range p.Actions {
 		if allowed == a {
 			return true
 		}
 	}
 	return false
 }
 func (p Policy) minProbability() float64 {
 	if p.MinProbability <= 0 {
 		return 0.35 // below this the leading label barely beats the field
 	}
 	return p.MinProbability
 }
 func (p Policy) minMargin() float64 {
 	if p.MinMargin <= 0 {
 		return 0.05
 	}
 	return p.MinMargin
 }
 // Verdict is the outcome for one alert: what the model asked for, what was
 // allowed, and why it differs.
 type Verdict struct {
 	AlertID       string
 	Action        Action  // what is actually done
 	Requested     Action  // what the model chose
 	Probability   float64 // model's probability for Requested
 	Margin        float64 // how far Requested leads the runner-up
 	Confidence    float64 // how decisive the forward pass was
 	Probabilities map[string]float64
 	Severity      string
 	Urgency       int
 	Model         string
 	State         string   // the text the model was given, kept for the UI
 	Gates         []string // why the action was changed or withheld, in order
 	// Targets are the deployments a restart was cleared for, after the
 	// hourly budget. Executable means exactly this list may be acted on.
 	Targets []string
 	Tokens  int
 	At      time.Time
 	Error   string // set when the model could not be reached
 }
 // Executable reports whether the action may be carried out automatically.
 func (v Verdict) Executable() bool {
 	return v.Error == "" && v.Action != ActionNotify && v.Action != ActionEscalate
 }
 // Changed reports whether a gate altered the model's answer.
 func (v Verdict) Changed() bool { return v.Error == "" && v.Requested != "" && v.Action != v.Requested }
 // Explain renders the verdict as one line for a notification or a log.
 func (v Verdict) Explain() string {
 	if v.Error != "" {
 		return "decision unavailable: " + v.Error + " (notifying as usual)"
 	}
 	msg := fmt.Sprintf("laya %s: %s (%.0f%%, margin %.0f%%, decisiveness %.2f)", v.Action, v.Requested, v.Probability*100, v.Margin*100, v.Confidence)
 	if v.Severity != "" {
 		msg += ", severity " + v.Severity
 	}
 	for _, g := range v.Gates {
 		msg += "; " + g
 	}
 	return msg
 }
 // Decider is the part of an Engine the alerting pipeline talks to, so the
 // pipeline can be tested without a model.
 type Decider interface {
 	Decide(ctx context.Context, in Input) Verdict
 }
 // Engine renders an alert, asks Laya what to do, and applies the policy gates.
 type Engine struct {
 	Client *Client
 	Policy Policy
 	// restarts counts automatic restarts per deployment within the hour.
 	restarts map[string][]time.Time
 }
 func NewEngine(client *Client, policy Policy) *Engine {
 	return &Engine{Client: client, Policy: policy, restarts: map[string][]time.Time{}}
 }
 // Questions are the typed questions asked of the model. The criteria spell
 // out what each action means here, because the model reads the descriptions
 // rather than the labels.
 func Questions() map[string]Question {
 	return map[string]Question{
 		"action": ChoiceQuestion("What is the right operational response to this error spike right now?", map[string]string{
 			"restart":  "Restart the pods of the deployment: only right when the pod itself is wedged or crash-looping and a fresh process would fix it",
 			"ignore":   "Ignore: known noise, an expected consequence of a deploy or a scanner, nothing for a human to do",
 			"notify":   "Notify the on-call: a real problem that needs a human to look, but not self-fixable",
 			"escalate": "Escalate to the service owner: serious, ongoing, and likely to need a code change or a rollback",
 		}),
 		"severity": ChoiceQuestion("How severe is this for users of the service?", map[string]string{
 			"low":      "cosmetic, a tiny fraction of requests, no user-visible effect",
 			"medium":   "degraded, some users are affected but the service still works",
 			"high":     "users are visibly broken, a large share of requests fail",
 			"critical": "the service is down or unusable for most of its users",
 		}),
 		"urgency": ScoreQuestion("How soon does a human need to act? 0 = nobody needs to act at all, 4 = a human is needed right now.",
 			[]string{"0 nobody needs to act", "1 low priority", "2 today", "3 within the hour", "4 needed right now"}),
 	}
 }
 // Decide renders the state, asks the model, and gates the answer. It never
 // returns an error: a decision that could not be made degrades to a plain
 // notification, which is what the pipeline would have done anyway.
 func (e *Engine) Decide(ctx context.Context, in Input) Verdict {
 	now := e.Policy.now()
 	v := Verdict{AlertID: in.AlertID, Requested: ActionNotify, Action: ActionNotify, At: now, State: in.Render(now)}
 	if e == nil || e.Client == nil {
 		return v
 	}
 	resp, err := e.Client.Predict(ctx, v.State, Questions())
 	if err != nil {
 		v.Error = err.Error()
 		return v
 	}
 	v.Model, v.Tokens = resp.Model, resp.Usage.InputTokens
 	answer := resp.Answers["action"]
 	v.Probabilities = answer.Probabilities
 	v.Probability = answer.Probability(answer.Choice)
 	v.Margin = answer.Margin()
 	v.Confidence = answer.Confidence
 	v.Severity = resp.Answers["severity"].Choice
 	v.Urgency = int(resp.Answers["urgency"].Score + 0.5)
 	requested := Action(answer.Choice)
 	if !knownAction(requested) {
 		v.Error = "model answered with an unknown action " + answer.Choice
 		return v
 	}
 	v.Requested = requested
 	v.Action = requested
 	e.gate(ctx, &v, in, now)
 	return v
 }
 func knownAction(a Action) bool {
 	for _, k := range Actions {
 		if k == a {
 			return true
 		}
 	}
 	return false
 }
 // gate is the safety envelope. Each check either leaves the action alone or
 // downgrades it to a notification, always recording why.
 func (e *Engine) gate(ctx context.Context, v *Verdict, in Input, now time.Time) {
 	downgrade := func(to Action, why string) {
 		if v.Action != to {
 			v.Gates = append(v.Gates, why)
 			v.Action = to
 		}
 	}
 	// 1. The model has to lead the field by a real margin. Both the absolute
 	//    probability and the lead are checked: a diffuse answer that only just
 	//    wins is a guess, and must not move a cluster on its own.
 	if v.Probability < e.Policy.minProbability() {
 		downgrade(ActionNotify, fmt.Sprintf("model was only %.0f%% sure of %s", v.Probability*100, v.Requested))
 	}
 	if v.Margin < e.Policy.minMargin() {
 		downgrade(ActionNotify, fmt.Sprintf("%s only just beat the alternatives (margin %.0f%%)", v.Requested, v.Margin*100))
 	}
 	// 2. Restarts are the one action that can make things worse, so the pod
 	//    state has to support it rather than merely allow it.
 	if v.Action == ActionRestart {
 		switch reason := restartRefused(in, now); {
 		case reason != "":
 			downgrade(ActionNotify, reason)
 		case !e.Policy.allows(ActionRestart):
 			downgrade(ActionNotify, "automatic restarts are not enabled")
 		case !e.allowRestart(v, in, now):
 			downgrade(ActionNotify, "restart budget exhausted for this deployment")
 		}
 	}
 	// 3. Ignoring is only safe when the errors are explained: a human's mute,
 	//    a crawler, or a rollout that is still replacing pods.
 	if v.Action == ActionIgnore {
 		if reason := ignoreRefused(in); reason != "" {
 			downgrade(ActionNotify, reason)
 		}
 	}
 	// 4. Anything not on the allowlist is reported, never executed.
 	if v.Action != ActionNotify && v.Action != ActionEscalate && !e.Policy.allows(v.Action) {
 		downgrade(ActionNotify, "action "+string(v.Action)+" is not in the allowed set")
 	}
 }
 // restartRefused explains why a restart would not help, using the pods' last
 // events. A pod the kubelet is already restarting will just be restarted again,
 // and an expected 503 burst during a rollout is not an incident at all.
 func restartRefused(in Input, now time.Time) string {
 	var live, crashLooping, restarting int
 	for _, p := range in.Pods {
 		if p.Err != "" || !p.Found {
 			continue
 		}
 		live++
 		switch {
 		case p.CrashLooping(now):
 			crashLooping++
 		case p.Restarting(now):
 			restarting++
 		}
 	}
 	if live == 0 {
 		return "no pod state to justify a restart"
 	}
 	if crashLooping == live {
 		return "pods are already crash-looping, a restart repeats what the kubelet already does"
 	}
 	// Every pod died moments ago, or a rollout is replacing them: the 503s
 	// around it are the rollout's own, and restarting would restart the
 	// rollout as well.
 	if restarting == live || (notReady(in) && rolloutOngoing(in, now)) {
 		return "every pod is being replaced by a rollout, the 503s are the rollout's own"
 	}
 	return ""
 }
 // notReady reports whether no pod behind the alert is serving traffic.
 func notReady(in Input) bool {
 	ready := false
 	for _, p := range in.Pods {
 		if p.Err == "" && p.Found && p.Ready {
 			ready = true
 		}
 	}
 	return !ready
 }
 // ignoreRefused explains why a spike may not be dropped silently. A brand new
 // error is never ignored: nobody has seen it before, so nobody can know it is
 // harmless.
 func ignoreRefused(in Input) string {
 	if in.KnownNoise {
 		return ""
 	}
 	if in.NewErrors {
 		return "the error is new, so nobody has confirmed it is harmless"
 	}
 	if in.Priority == "high" && in.ErrorCount > 0 {
 		return "a high-priority spike is never ignored without a human's confirmation"
 	}
 	for _, p := range in.Patterns {
 		if p.New {
 			return "pattern " + p.What + " is new, so it cannot be assumed harmless"
 		}
 	}
 	return ""
 }
 func rolloutOngoing(in Input, now time.Time) bool {
 	for _, d := range in.Deploys {
 		if !d.Completed && d.Age > 0 && d.Age < rollingWindow {
 			return true
 		}
 	}
 	return false
 }
 // allowRestart enforces the hourly budget per deployment, so a service that
 // flaps cannot be restarted over and over.
 func (e *Engine) allowRestart(v *Verdict, in Input, now time.Time) bool {
 	limit := e.Policy.MaxRestartsPerHour
 	if limit <= 0 {
 		limit = 3
 	}
 	cleared := 0
 	for _, p := range in.Pods {
 		key := p.Namespace + "/" + p.Deployment
 		cut := now.Add(-time.Hour)
 		kept := e.restarts[key][:0]
 		for _, at := range e.restarts[key] {
 			if at.After(cut) {
 				kept = append(kept, at)
 			}
 		}
 		e.restarts[key] = kept
 		if len(kept) >= limit {
 			continue
 		}
 		e.restarts[key] = append(kept, now)
 		v.Targets = append(v.Targets, key)
 		cleared++
 	}
 	return cleared > 0
 }
 // RollingWindow is how recently a container must have died for the errors
 // around it to be attributed to the pod still coming up.
 const rollingWindow = 5 * time.Minute
 // sortedDeploys is used by the pipeline to feed Deploys newest first.
 func sortedDeploys(in []Deploy) []Deploy {
 	out := append([]Deploy(nil), in...)
 	sort.SliceStable(out, func(i, j int) bool { return out[i].Age < out[j].Age })
 	return out
 }
@@ -1,344 +0,0 @@
 package laya
 import (
 	"context"
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
 	"strings"
 	"testing"
 	"time"
 )
 // fake serves the Laya API with a fixed answer per action label, and records
 // the last state and questions it was asked.
 type fake struct {
 	choice     string
 	prob       map[string]float64
 	lastState  string
 	lastQs     map[string]json.RawMessage
 	status     int
 	batchSizes int
 }
 func (f *fake) server(t *testing.T) *httptest.Server {
 	t.Helper()
 	mux := http.NewServeMux()
 	mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
 		writeJSON(w, Health{Status: "ok", Model: "test-model"})
 	})
 	mux.HandleFunc("/v1/predict", func(w http.ResponseWriter, r *http.Request) {
 		var req struct {
 			State     json.RawMessage            `json:"state"`
 			Questions map[string]json.RawMessage `json:"questions"`
 		}
 		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
 			t.Errorf("decode request: %v", err)
 		}
 		f.lastState = string(req.State)
 		f.lastQs = req.Questions
 		f.respond(w, map[string]Answer{
 			"action":   {Type: KindChoice, Choice: f.choice, Probabilities: f.prob},
 			"severity": {Type: KindChoice, Choice: "high", Probabilities: map[string]float64{"high": 0.8}},
 			"urgency":  {Type: KindScore, Score: 3},
 		})
 	})
 	mux.HandleFunc("/v1/predict/batch", func(w http.ResponseWriter, r *http.Request) {
 		var req struct {
 			States []json.RawMessage `json:"states"`
 		}
 		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
 			t.Errorf("decode batch request: %v", err)
 		}
 		f.batchSizes = len(req.States)
 		results := make([]Response, 0, len(req.States))
 		for range req.States {
 			results = append(results, Response{Model: "test-model", Answers: map[string]Answer{
 				"action": {Type: KindChoice, Choice: f.choice, Probabilities: f.prob},
 			}})
 		}
 		writeJSON(w, map[string]any{"results": results})
 	})
 	mux.HandleFunc("/boom", func(w http.ResponseWriter, _ *http.Request) {
 		w.WriteHeader(http.StatusBadGateway)
 		_, _ = w.Write([]byte("upstream exploded"))
 	})
 	srv := httptest.NewServer(mux)
 	t.Cleanup(srv.Close)
 	return srv
 }
 func (f *fake) respond(w http.ResponseWriter, answers map[string]Answer) {
 	if f.status != 0 {
 		w.WriteHeader(f.status)
 		return
 	}
 	writeJSON(w, Response{Model: "test-model", Answers: answers, Usage: Usage{InputTokens: 128}})
 }
 func writeJSON(w http.ResponseWriter, v any) {
 	w.Header().Set("Content-Type", "application/json")
 	_ = json.NewEncoder(w).Encode(v)
 }
 func TestPredictSendsTypedQuestionsAndReadsAnswers(t *testing.T) {
 	f := &fake{choice: "restart", prob: map[string]float64{"restart": 0.9, "notify": 0.1}}
 	srv := f.server(t)
 	c := NewClient(srv.URL)
 	resp, err := c.Predict(context.Background(), "checkout-api is 500ing", Questions())
 	if err != nil {
 		t.Fatalf("Predict: %v", err)
 	}
 	if got := resp.Answers["action"].Choice; got != "restart" {
 		t.Errorf("choice = %q, want restart", got)
 	}
 	if got := resp.Answers["urgency"].Score; got != 3 {
 		t.Errorf("urgency score = %v, want 3", got)
 	}
 	if got := resp.Usage.InputTokens; got != 128 {
 		t.Errorf("input tokens = %d, want 128", got)
 	}
 	// The choice question must carry a criteria object and the score question an
 	// array, from the same "criteria" key.
 	if q := string(f.lastQs["action"]); !strings.Contains(q, `"criteria":{"escalate"`) {
 		t.Errorf("action question = %s, want a criteria object", q)
 	}
 	if q := string(f.lastQs["urgency"]); !strings.Contains(q, `"criteria":[`) {
 		t.Errorf("urgency question = %s, want a criteria array", q)
 	}
 	if f.lastState != `"checkout-api is 500ing"` {
 		t.Errorf("state = %s, want the JSON-encoded state", f.lastState)
 	}
 }
 func TestPredictBatchKeepsOrder(t *testing.T) {
 	srv := (&fake{choice: "notify", prob: map[string]float64{"notify": 1}}).server(t)
 	got, err := NewClient(srv.URL).PredictBatch(context.Background(), []any{"a", "b", "c"}, Questions())
 	if err != nil {
 		t.Fatalf("PredictBatch: %v", err)
 	}
 	if len(got) != 3 {
 		t.Fatalf("got %d results, want 3", len(got))
 	}
 	for i, r := range got {
 		if r.Answers["action"].Choice != "notify" {
 			t.Errorf("result %d choice = %q", i, r.Answers["action"].Choice)
 		}
 	}
 }
 func TestPredictErrorsAreSafeToLog(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.WriteHeader(http.StatusServiceUnavailable)
 	}))
 	defer srv.Close()
 	_, err := NewClient(srv.URL).Predict(context.Background(), "x", Questions())
 	if err == nil {
 		t.Fatal("expected an error")
 	}
 	if !strings.Contains(err.Error(), "503") {
 		t.Errorf("error = %v, want the status code", err)
 	}
 }
 func TestScrubRemovesCredentialsFromURL(t *testing.T) {
 	got := scrub(`Post "https://user:hunter2@laya.k6n.net/v1/predict": timeout`)
 	if strings.Contains(got, "hunter2") {
 		t.Errorf("scrub(%q) leaked the password: %s", "user:hunter2@", got)
 	}
 	if !strings.Contains(got, "laya.k6n.net") {
 		t.Errorf("scrub(%q) dropped the host: %s", "user:hunter2@", got)
 	}
 }
 func TestHealth(t *testing.T) {
 	srv := (&fake{}).server(t)
 	h, err := NewClient(srv.URL).Health(context.Background())
 	if err != nil {
 		t.Fatalf("Health: %v", err)
 	}
 	if h.Model != "test-model" || h.Status != "ok" {
 		t.Errorf("health = %+v, want status ok from test-model", h)
 	}
 }
 // --- state rendering and gates ---
 func nowFixture() time.Time { return time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) }
 func healthyPod() Pod {
 	return Pod{Namespace: "shop", Deployment: "checkout-api", Name: "checkout-api-7d9", Found: true, Ready: true, Phase: "Running"}
 }
 func engineFor(t *testing.T, policy Policy, choice string, prob float64) (*Engine, *fake) {
 	t.Helper()
 	f := &fake{choice: choice, prob: map[string]float64{choice: prob}}
 	srv := f.server(t)
 	policy.Now = nowFixture
 	return NewEngine(NewClient(srv.URL), policy), f
 }
 func TestRenderStatesPodEvents(t *testing.T) {
 	exit := int32(137)
 	in := Input{
 		AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 412, Window: 5 * time.Minute,
 		BaselineMean: 3, NewErrors: true, Clients: 42,
 		Patterns: []Pattern{{What: "HTTP 500 | /api/cart | panic", Category: "service-defect", Status: 500, Count: 300, Baseline: 2, New: true, Clients: 42}},
 		Pods: []Pod{{
 			Namespace: "shop", Deployment: "checkout-api", Name: "checkout-api-7d9", Found: true, Phase: "Running",
 			Restarts: 3, PodRestarts: 9, Reason: "OOMKilled", ExitCode: &exit, LastTerminatedAt: nowFixture().Add(-30 * time.Second),
 			LastEvents: []string{"panic: runtime error: invalid memory address"},
 		}},
 		Deploys: []Deploy{{Namespace: "shop", Workload: "checkout-api", Age: 4 * time.Minute, Ready: 1, Desired: 3}},
 	}
 	state := in.Render(nowFixture())
 	for _, want := range []string{
 		"shop/checkout-api pod checkout-api-7d9",
 		"OOMKilled",
 		"exit code 137",
 		"its container restarted 30s ago and is still coming up",
 		"9 restarts across the Deployment",
 		"panic: runtime error: invalid memory address",
 		"rollout in progress started 4m0s ago",
 		"137x normal",
 	} {
 		if !strings.Contains(state, want) {
 			t.Errorf("state is missing %q:\n%s", want, state)
 		}
 	}
 }
 func TestDecideExecutesRestartOnlyWhenPodsSupportIt(t *testing.T) {
 	e, f := engineFor(t, Policy{Actions: []Action{ActionRestart, ActionIgnore, ActionNotify, ActionEscalate}, MinProbability: 0.5}, "restart", 0.82)
 	v := e.Decide(context.Background(), Input{
 		AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 100, Window: 5 * time.Minute, BaselineMean: 2, Clients: 30,
 		Patterns: []Pattern{{What: "HTTP 500 | /api/cart", Category: "service-defect", Count: 90, Baseline: 1, New: true, Clients: 30}},
 		Pods:     []Pod{{Namespace: "shop", Deployment: "checkout-api", Name: "p1", Found: true, Ready: false, Phase: "Running"}},
 	})
 	if !v.Executable() || v.Action != ActionRestart {
 		t.Fatalf("action = %q executable=%v gates=%v, want an executed restart", v.Action, v.Executable(), v.Gates)
 	}
 	if len(v.Targets) != 1 || v.Targets[0] != "shop/checkout-api" {
 		t.Errorf("targets = %v, want shop/checkout-api", v.Targets)
 	}
 	if !strings.Contains(f.lastState, "pod p1") {
 		t.Errorf("model was not given the pod state:\n%s", f.lastState)
 	}
 }
 func TestDecideRefusesRestartDuringCrashLoop(t *testing.T) {
 	e, _ := engineFor(t, Policy{Actions: []Action{ActionRestart}, MinProbability: 0.5}, "restart", 0.95)
 	v := e.Decide(context.Background(), Input{
 		AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 300, Window: 5 * time.Minute, BaselineMean: 2,
 		Patterns: []Pattern{{What: "HTTP 500 | panic", Category: "service-defect", Count: 300, New: true}},
 		Pods:     []Pod{{Namespace: "shop", Deployment: "checkout-api", Name: "p1", Found: true, Phase: "Running", Restarts: 5, LastTerminatedAt: nowFixture().Add(-20 * time.Second)}},
 	})
 	if v.Action != ActionNotify {
 		t.Errorf("action = %q, want notify: a crash loop needs a new release, not a restart", v.Action)
 	}
 	if len(v.Gates) == 0 || !strings.Contains(strings.Join(v.Gates, " "), "crash-looping") {
 		t.Errorf("gates = %v, want the crash-loop reason", v.Gates)
 	}
 }
 // A short 503 burst while every pod is being replaced is the rollout's own, so
 // a restart must never be fired into it.
 func TestDecideRefusesRestartDuringRollout(t *testing.T) {
 	e, _ := engineFor(t, Policy{Actions: []Action{ActionRestart, ActionIgnore}, MinProbability: 0.5}, "restart", 0.9)
 	v := e.Decide(context.Background(), Input{
 		AlertID: "a1", Service: "checkout-api", Priority: "medium", ErrorCount: 40, Window: 5 * time.Minute, BaselineMean: 2,
 		Patterns: []Pattern{{What: "HTTP 503 | /api/cart", Category: "capacity-saturation", Status: 503, Count: 40}},
 		Pods:     []Pod{{Namespace: "shop", Deployment: "checkout-api", Name: "p1", Found: true, Phase: "Pending", LastTerminatedAt: nowFixture().Add(-40 * time.Second)}},
 		Deploys:  []Deploy{{Namespace: "shop", Workload: "checkout-api", Age: 2 * time.Minute, Ready: 1, Desired: 3}},
 	})
 	if v.Action == ActionRestart {
 		t.Fatalf("action = restart during a rollout, want it withheld (gates=%v)", v.Gates)
 	}
 	if !strings.Contains(strings.Join(v.Gates, " "), "replaced by a rollout") {
 		t.Errorf("gates = %v, want the rollout reason", v.Gates)
 	}
 }
 func TestDecideRefusesIgnoreOfANewError(t *testing.T) {
 	e, _ := engineFor(t, Policy{Actions: []Action{ActionIgnore}, MinProbability: 0.5}, "ignore", 0.9)
 	v := e.Decide(context.Background(), Input{
 		AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 200, Window: 5 * time.Minute, BaselineMean: 2, NewErrors: true,
 		Patterns: []Pattern{{What: "HTTP 500 | /api/cart", Count: 200, New: true}},
 		Pods:     []Pod{healthyPod()},
 	})
 	if v.Action != ActionNotify {
 		t.Errorf("action = %q, want notify: a new error was never confirmed harmless", v.Action)
 	}
 }
 func TestDecideAllowsIgnoreOfKnownNoise(t *testing.T) {
 	e, _ := engineFor(t, Policy{Actions: []Action{ActionIgnore}, MinProbability: 0.5}, "ignore", 0.88)
 	v := e.Decide(context.Background(), Input{
 		AlertID: "a1", Service: "blog", Priority: "low", ErrorCount: 1800, Window: 5 * time.Minute, BaselineMean: 20, KnownNoise: true,
 		Patterns: []Pattern{{What: "HTTP 404 | /old-post", Status: 404, Count: 1800}},
 	})
 	if v.Action != ActionIgnore || !v.Executable() {
 		t.Errorf("action = %q executable = %v, want an executable ignore", v.Action, v.Executable())
 	}
 }
 func TestDecideWithholdsActionsOutsideTheAllowlist(t *testing.T) {
 	e, _ := engineFor(t, Policy{Actions: []Action{ActionNotify}, MinProbability: 0.5}, "ignore", 0.99)
 	v := e.Decide(context.Background(), Input{
 		AlertID: "a1", Service: "blog", Priority: "low", ErrorCount: 100, Window: 5 * time.Minute, BaselineMean: 20,
 		Patterns: []Pattern{{What: "HTTP 404", Status: 404, Count: 100}},
 		Pods:     []Pod{healthyPod()},
 	})
 	if v.Action != ActionNotify || v.Requested != ActionIgnore || !v.Changed() {
 		t.Errorf("action = %q requested = %q changed = %v, want ignore downgraded to notify", v.Action, v.Requested, v.Changed())
 	}
 }
 func TestDecideDowngradesALowConfidenceChoice(t *testing.T) {
 	e, _ := engineFor(t, Policy{Actions: []Action{ActionRestart, ActionIgnore}, MinProbability: 0.6}, "restart", 0.41)
 	v := e.Decide(context.Background(), Input{
 		AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 90, Window: 5 * time.Minute, BaselineMean: 2,
 		Patterns: []Pattern{{What: "HTTP 500", Count: 90, New: true}},
 		Pods:     []Pod{{Namespace: "shop", Deployment: "checkout-api", Found: true, Ready: false, Phase: "Running"}},
 	})
 	if v.Action != ActionNotify {
 		t.Errorf("action = %q, want notify: 41%% is a guess", v.Action)
 	}
 }
 func TestRestartBudgetIsEnforced(t *testing.T) {
 	e, _ := engineFor(t, Policy{Actions: []Action{ActionRestart}, MinProbability: 0.5, MaxRestartsPerHour: 1}, "restart", 0.9)
 	in := Input{
 		AlertID: "a1", Service: "checkout-api", Priority: "high", ErrorCount: 90, Window: 5 * time.Minute, BaselineMean: 2,
 		Patterns: []Pattern{{What: "HTTP 500", Count: 90, New: true}},
 		Pods:     []Pod{{Namespace: "shop", Deployment: "checkout-api", Found: true, Ready: false, Phase: "Running"}},
 	}
 	if v := e.Decide(context.Background(), in); v.Action != ActionRestart {
 		t.Fatalf("first decision = %q, want restart", v.Action)
 	}
 	second := e.Decide(context.Background(), in)
 	if second.Action != ActionNotify {
 		t.Errorf("second decision = %q, want notify: the hourly restart budget is spent", second.Action)
 	}
 }
 func TestDecideDegradesToNotifyWhenTheModelIsDown(t *testing.T) {
 	f := &fake{status: http.StatusServiceUnavailable}
 	srv := f.server(t)
 	e := NewEngine(NewClient(srv.URL), Policy{Actions: []Action{ActionRestart}, Now: nowFixture})
 	v := e.Decide(context.Background(), Input{AlertID: "a1", Service: "checkout-api", ErrorCount: 90, Window: 5 * time.Minute})
 	if v.Action != ActionNotify || v.Executable() || v.Error == "" {
 		t.Errorf("action = %q executable = %v error = %q, want a plain notification", v.Action, v.Executable(), v.Error)
 	}
 }
 func TestDecideRejectsAnUnknownAction(t *testing.T) {
 	f := &fake{choice: "delete-the-cluster", prob: map[string]float64{"delete-the-cluster": 1}}
 	srv := f.server(t)
 	e := NewEngine(NewClient(srv.URL), Policy{Actions: []Action{ActionRestart}, Now: nowFixture})
 	v := e.Decide(context.Background(), Input{AlertID: "a1", Service: "x", ErrorCount: 1, Window: time.Minute})
 	if v.Error == "" || v.Executable() {
 		t.Errorf("error = %q executable = %v, want an unknown action to be rejected", v.Error, v.Executable())
 	}
 }
@@ -1,59 +0,0 @@
 package laya
 import (
 	"context"
 	"os"
 	"testing"
 	"time"
 )
 // TestLiveModel is a smoke test against the real Laya API, so the state
 // rendering and the questions stay in step with what the model actually
 // understands. It is opt-in: LAYA_URL=https://laya.k6n.net go test ./internal/laya -run TestLiveModel
 func TestLiveModel(t *testing.T) {
 	base := os.Getenv("LAYA_URL")
 	if base == "" {
 		t.Skip("set LAYA_URL to run against the live model")
 	}
 	cases := []struct {
 		name string
 		in   Input
 	}{
 		{
 			name: "crash loop after a release",
 			in: Input{
 				Service: "checkout-api", Priority: "high", ErrorCount: 412, Window: 5 * time.Minute, BaselineMean: 3, NewErrors: true, Clients: 42,
 				Patterns: []Pattern{{What: "HTTP 500 | /api/cart | panic: runtime error: invalid memory address", Category: "service-defect", Status: 500, Count: 300, Baseline: 2, New: true, Clients: 42, Services: []string{"checkout-api", "cart-service"}}},
 				Pods:     []Pod{{Namespace: "shop", Deployment: "checkout-api", Name: "checkout-api-7d9", Found: true, Phase: "Running", Restarts: 3, PodRestarts: 9, Reason: "CrashLoopBackOff", LastTerminatedAt: time.Now().Add(-30 * time.Second), LastEvents: []string{"panic: runtime error: invalid memory address", "SIGTERM received"}}},
 				Deploys:  []Deploy{{Namespace: "shop", Workload: "checkout-api", Release: "v2.31", Age: 4 * time.Minute, Ready: 1, Desired: 3}},
 			},
 		},
 		{
 			name: "503 burst while a pod is being replaced",
 			in: Input{
 				Service: "checkout-api", Priority: "medium", ErrorCount: 40, Window: 5 * time.Minute, BaselineMean: 2, Clients: 8,
 				Patterns: []Pattern{{What: "HTTP 503 | /api/cart | no healthy upstream", Category: "capacity-saturation", Status: 503, Count: 40, Baseline: 1, Clients: 8}},
 				Pods:     []Pod{{Namespace: "shop", Deployment: "checkout-api", Name: "checkout-api-2k1", Found: true, Phase: "Pending", LastTerminatedAt: time.Now().Add(-40 * time.Second), LastEvents: []string{"terminated"}}},
 				Deploys:  []Deploy{{Namespace: "shop", Workload: "checkout-api", Age: 2 * time.Minute, Ready: 1, Desired: 3}},
 			},
 		},
 		{
 			name: "crawler 404s, nobody affected",
 			in: Input{
 				Service: "blog", Priority: "low", ErrorCount: 1800, Window: 5 * time.Minute, BaselineMean: 20, KnownNoise: true,
 				Patterns: []Pattern{{What: "HTTP 404 | /old-post-2019 | UA looks like a crawler", Status: 404, Count: 1800, Baseline: 15}},
 				Pods:     []Pod{{Namespace: "shop", Deployment: "blog", Name: "blog-6f2", Found: true, Ready: true, Phase: "Running", LastEvents: []string{"GET /old-post-2019 404"}}},
 			},
 		},
 	}
 	// Engine defaults, as deployed: the label probabilities are diffuse, so the
 	// gate is the lead over the runner-up.
 	engine := NewEngine(NewClient(base), Policy{Actions: Actions, Now: time.Now})
 	for _, tc := range cases {
 		v := engine.Decide(context.Background(), tc.in)
 		t.Logf("%s:\n%s\n  -> %s\n", tc.name, v.State, v.Explain())
 		if v.Error != "" {
 			t.Errorf("%s: %v", tc.name, v.Error)
 		}
 	}
 }
@@ -16,38 +16,6 @@ type PodLog struct {
 	Lines []string // most recent log lines, oldest first
 	Found bool     // a pod exists and was queried (Lines may still be empty: a quiet pod)
 	Err   string   // set when the lookup itself failed; Found is false
 	// The pod's own lifecycle facts, which explain errors the log lines alone
 	// cannot: a 503 burst while a replica is being replaced, a crash loop the
 	// kubelet is already retrying, or a pod that runs but never becomes ready.
 	Ready            bool      // container passing its readiness probe
 	Phase            string    // Pending | Running | Succeeded | Failed | Unknown
 	Restarts         int32     // restarts of the selected pod's containers
 	PodRestarts      int32     // restarts summed over every pod of the Deployment
 	Reason           string    // CrashLoopBackOff, OOMKilled, ImagePullBackOff, ... ("" while healthy)
 	ExitCode         *int32    // exit code of the last container termination
 	Message          string    // message of the last container termination
 	LastTerminatedAt time.Time // when the last container terminated
 }
 // rollingWindow is how recently a container must have died for the resulting
 // connection errors to be attributed to its replacement still coming up.
 const rollingWindow = 5 * time.Minute
 // CrashLooping reports whether the pod is stuck restarting instead of serving.
 // Restarting the Deployment then only repeats what the kubelet is already
 // doing, so the answer is a new release, not a restart.
 func (p PodLog) CrashLooping() bool {
 	if p.Reason == "CrashLoopBackOff" {
 		return true
 	}
 	return p.Restarts > 0 && !p.LastTerminatedAt.IsZero()
 }
 // Restarting reports whether a pod of this Deployment is right now being
 // replaced, which is what makes a short 503 burst expected rather than new.
 func (p PodLog) Restarting() bool {
 	return !p.LastTerminatedAt.IsZero() && time.Since(p.LastTerminatedAt) < rollingWindow
 }
 // podLogTimeout bounds one lookup so a slow or unreachable API server never
@@ -96,7 +64,6 @@ func (r *Registry) PodLogs(ctx context.Context, namespace, deployment string, ta
 		return live[i].CreationTimestamp.After(live[j].CreationTimestamp.Time)
 	})
 	pod := live[0]
 	state := podState(pod, live)
 	container := ""
 	if len(pod.Spec.Containers) > 0 {
 		container = pod.Spec.Containers[0].Name
@@ -104,7 +71,7 @@ func (r *Registry) PodLogs(ctx context.Context, namespace, deployment string, ta
 	tailLines := int64(tail)
 	stream, err := r.client.CoreV1().Pods(namespace).GetLogs(pod.Name, &corev1.PodLogOptions{Container: container, TailLines: &tailLines}).Stream(ctx)
 	if err != nil {
-		return withState(state, PodLog{Pod: pod.Name, Found: true, Err: err.Error()})
+		return PodLog{Pod: pod.Name, Found: true, Err: err.Error()}
 	}
 	defer stream.Close()
 	var lines []string
@@ -116,53 +83,5 @@ func (r *Registry) PodLogs(ctx context.Context, namespace, deployment string, ta
 	if len(lines) > tail {
 		lines = lines[len(lines)-tail:]
 	}
-	return withState(state, PodLog{Pod: pod.Name, Lines: lines, Found: true})
+	return PodLog{Pod: pod.Name, Lines: lines, Found: true}
 }
 // withState keeps the query result but carries the lifecycle facts along.
 func withState(state, result PodLog) PodLog {
 	result.Ready, result.Phase, result.Restarts, result.PodRestarts = state.Ready, state.Phase, state.Restarts, state.PodRestarts
 	result.Reason, result.ExitCode, result.Message, result.LastTerminatedAt = state.Reason, state.ExitCode, state.Message, state.LastTerminatedAt
 	return result
 }
 // podState reads the lifecycle facts off the selected pod and totals the
 // restarts of its siblings, so one unhealthy replica is not mistaken for a
 // healthy deployment.
 func podState(selected corev1.Pod, all []corev1.Pod) PodLog {
 	state := PodLog{Found: true, Ready: len(selected.Status.ContainerStatuses) > 0, Phase: string(selected.Status.Phase)}
 	// The most recent termination across the pod's containers is the
 	// interesting one: a pod being replaced is explained by its last death.
 	keepTermination := func(term *corev1.ContainerStateTerminated) {
 		if term == nil || !term.FinishedAt.Time.After(state.LastTerminatedAt) {
 			return
 		}
 		state.LastTerminatedAt = term.FinishedAt.Time
 		state.Message = term.Message
 		code := term.ExitCode
 		state.ExitCode = &code
 		switch {
 		case term.Reason != "":
 			state.Reason = term.Reason
 		case term.Message != "":
 			state.Reason = "Error"
 		}
 	}
 	for i := range selected.Status.ContainerStatuses {
 		cs := &selected.Status.ContainerStatuses[i]
 		state.Restarts += cs.RestartCount
 		if !cs.Ready {
 			state.Ready = false
 		}
 		keepTermination(cs.LastTerminationState.Terminated)
 	}
 	if selected.Status.Reason != "" {
 		state.Reason = selected.Status.Reason
 	}
 	for _, p := range all {
 		for _, cs := range p.Status.ContainerStatuses {
 			state.PodRestarts += cs.RestartCount
 		}
 	}
 	return state
 }
@@ -15,6 +15,9 @@ spec:
    metadata:
      labels:
        app: otel-debug-receiver
      annotations:
        k8s-helper.io/git-url: "https://git.k6n.net/mats/go-otel"
        k8s-helper.io/git-ssh: "git@git.k6n.net:mats/go-otel.git"
    spec:
      serviceAccountName: otel-debug-receiver
      securityContext:
@@ -138,6 +141,9 @@ spec:
        # Optional Always-On Display session API.
        - name: OTEL_ALERT_AOD_URL
          value: ""
        # laya shadow verdicts: shown in each notification, never acted on.
        - name: OTEL_ALERT_LAYA_URL
          value: "https://laya.k6n.net"
        - name: OTEL_STORE_WORKERS
          value: "4"
        # Optional rollout observation. The watcher is read-only and uses the
@@ -167,8 +173,6 @@ spec:
          value: "15m"
        - name: OTEL_HONEYPOT_TTL
          value: "1h"
        - name: OTEL_NEEDLE_ADDR
          value: "http://127.0.0.1:5005"
        # ~80% of the memory limit so the GC works harder before an OOMKill.
        - name: GOMEMLIMIT
          value: "800MiB"
@@ -196,58 +200,12 @@ spec:
          limits:
            memory: "1Gi"
            cpu: "500m"
      - name: needle-sidecar
        image: matst80/needle-sidecar:latest
        imagePullPolicy: Always
        ports:
        - containerPort: 5005
          name: needle-http
        env:
        - name: HOME
          value: "/app"
        - name: NEEDLE_PORT
          value: "5005"
        - name: NEEDLE_HOST
          value: "0.0.0.0"
        - name: NEEDLE_TELEMETRY
          value: "0"
        livenessProbe:
          httpGet:
            path: /health
            port: needle-http
          initialDelaySeconds: 5
          periodSeconds: 15
        readinessProbe:
          httpGet:
            path: /health
            port: needle-http
          initialDelaySeconds: 3
          periodSeconds: 10
        resources:
          requests:
            memory: "512Mi"
            cpu: "200m"
          limits:
            memory: "2Gi"
            cpu: "4000m"
      volumes:
      - name: log-dir
        emptyDir: {}
      - name: sourcemap-dir
        persistentVolumeClaim:
          claimName: otel-sourcemaps
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
  name: otel-sourcemaps
  namespace: monitoring
 spec:
  accessModes: ["ReadWriteOnce"]
  resources:
    requests:
      storage: 1Gi
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -0,0 +1,120 @@
 # OIDC Registration in ecommerce namespace
 apiVersion: v1
 kind: Secret
 metadata:
  name: flux-dashboard-oidc-client-secret
  namespace: ecommerce
 type: Opaque
 stringData:
  secret: "f9f44c88d9fda647e4abbb4479f1116611ecc14624bd67ec"
 ---
 apiVersion: auth.k6n.net/v1alpha1
 kind: AuthClient
 metadata:
  name: flux-dashboard
  namespace: ecommerce
  labels:
    auth.k6n.net/server: auth
 spec:
  clientID: flux-dashboard
  displayName: Flux GitOps Dashboard
  description: Web UI for Flux CD (Weave GitOps)
  redirectURIs:
    - https://flux.k6n.net/oauth2/callback
  scopes:
    - openid
    - profile
    - email
    - groups
  secretRef:
    name: flux-dashboard-oidc-client-secret
    key: secret
 ---
 # Weave GitOps OIDC Secret in flux-system
 apiVersion: v1
 kind: Secret
 metadata:
  name: oidc-auth
  namespace: flux-system
 type: Opaque
 stringData:
  clientID: "flux-dashboard"
  clientSecret: "f9f44c88d9fda647e4abbb4479f1116611ecc14624bd67ec"
  issuerURL: "https://users.k6n.net"
  redirectURL: "https://flux.k6n.net/oauth2/callback"
  claimUsername: "email"
  claimGroups: "roles"
 ---
 # Weave GitOps Helm OCI Source and HelmRelease
 apiVersion: source.toolkit.fluxcd.io/v1
 kind: HelmRepository
 metadata:
  name: weave-gitops
  namespace: flux-system
 spec:
  type: oci
  url: oci://ghcr.io/weaveworks/charts
  interval: 1h
 ---
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
  name: weave-gitops
  namespace: flux-system
 spec:
  interval: 10m
  chart:
    spec:
      chart: weave-gitops
      version: "4.0.36"
      sourceRef:
        kind: HelmRepository
        name: weave-gitops
  values:
    envVars:
      - name: KUBERNETES_SERVICE_HOST
        value: "10.10.1.102"
      - name: KUBERNETES_SERVICE_PORT
        value: "6443"
    affinity:
      nodeAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          nodeSelectorTerms:
            - matchExpressions:
                - key: kubernetes.io/hostname
                  operator: NotIn
                  values:
                    - serber
    # Use existing oidc-auth secret created above
    oidcSecret:
      create: false
    adminUser:
      create: false
    ingress:
      enabled: true
      className: nginx
      annotations:
        cert-manager.io/cluster-issuer: letsencrypt-prod
      hosts:
        - host: flux.k6n.net
          paths:
            - path: /
              pathType: ImplementationSpecific
      tls:
        - hosts:
            - flux.k6n.net
          secretName: flux-k6n-tls
 ---
 # ClusterRoleBinding so authenticated users can view/manage GitOps resources
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
  name: weave-gitops-admin
 subjects:
  - kind: Group
    name: admin
    apiGroup: rbac.authorization.k8s.io
 roleRef:
  kind: ClusterRole
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io
@@ -0,0 +1,28 @@
 apiVersion: source.toolkit.fluxcd.io/v1
 kind: GitRepository
 metadata:
  name: go-otel
  namespace: flux-system
 spec:
  interval: 1m
  url: ssh://git@git.k6n.net/mats/go-otel.git
  ref:
    branch: main
  secretRef:
    name: go-otel-auth
 ---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
  name: go-otel
  namespace: flux-system
 spec:
  interval: 5m
  targetNamespace: monitoring
  sourceRef:
    kind: GitRepository
    name: go-otel
  path: "./k8s"
  prune: true
  wait: true
  timeout: 3m
@@ -0,0 +1,173 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: ingress-catchall
  namespace: ingress-nginx
 data:
  nginx.conf: |
    worker_processes  1;
    pid /tmp/nginx.pid;
    events {
      worker_connections  1024;
    }
    http {
      include       /etc/nginx/mime.types;
      default_type  application/octet-stream;
      sendfile      on;
      keepalive_timeout  65;
      client_body_temp_path /tmp/client_temp;
      proxy_temp_path /tmp/proxy_temp;
      fastcgi_temp_path /tmp/fastcgi_temp;
      uwsgi_temp_path /tmp/uwsgi_temp;
      scgi_temp_path /tmp/scgi_temp;
      server {
        listen 8080;
      server_name _;
      root /usr/share/nginx/html;
      error_page 403 =404 /index.html;
      # Only GET is intentionally accepted by this catch-all.
      # The "if" is safe here because it only performs a return.
        # Serve robots.txt directly with Disallow: /
        location = /robots.txt {
          default_type text/plain;
          return 200 "User-agent: *\nDisallow: /\n";
        }
        # Deliberately trigger an internal 403 so the static error page is
        # returned with the externally visible status still set to 404.
        location / {
          if ($request_method != GET) {
            add_header Allow GET always;
            return 405;
          }
          deny all;
        }
        # Keep the error document internal so /index.html also follows the
        # catch-all policy instead of becoming a successful public endpoint.
        location = /index.html {
          internal;
          allow all;
        }
      }
    }
  index.html: |
    <!doctype html>
    <html lang="en">
    <head>
      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width, initial-scale=1">
      <title>Not found</title>
      <style>
        :root { color-scheme: dark; }
        * { box-sizing: border-box; }
        body {
          min-height: 100vh;
          margin: 0;
          display: grid;
          place-items: center;
          padding: 2rem;
          color: #e8ecf2;
          background: #0d1014;
          font: 16px/1.5 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
        }
        main { width: min(100%, 42rem); }
        .code {
          margin: 0 0 .75rem;
          color: #7b8cff;
          font: 700 clamp(4rem, 14vw, 8rem)/.9 ui-monospace, SFMono-Regular, Menlo, monospace;
          letter-spacing: -.08em;
        }
        h1 { margin: 0 0 .75rem; font-size: clamp(1.5rem, 4vw, 2.25rem); }
        p { margin: 0; max-width: 34rem; color: #98a3b2; }
        code { color: #e8ecf2; }
      </style>
    </head>
    <body>
      <main>
        <p class="code">404</p>
        <h1>There is nothing here.</h1>
        <p>The requested resource does not exist on this ingress endpoint.</p>
      </main>
    </body>
    </html>
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: ingress-catchall
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-catchall
    app.kubernetes.io/part-of: ingress-nginx
 spec:
  replicas: 2
  selector:
    matchLabels:
      app.kubernetes.io/name: ingress-catchall
  template:
    metadata:
      labels:
        app.kubernetes.io/name: ingress-catchall
        app.kubernetes.io/part-of: ingress-nginx
    spec:
      automountServiceAccountToken: false
      securityContext:
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: nginx
          image: nginx:1.27-alpine
          imagePullPolicy: IfNotPresent
          ports:
            - name: http
              containerPort: 8080
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop: ["ALL"]
            runAsNonRoot: true
            runAsUser: 101
          volumeMounts:
            - name: config
              mountPath: /etc/nginx/nginx.conf
              subPath: nginx.conf
              readOnly: true
            - name: config
              mountPath: /usr/share/nginx/html/index.html
              subPath: index.html
              readOnly: true
          readinessProbe:
            tcpSocket:
              port: http
            periodSeconds: 5
          livenessProbe:
            tcpSocket:
              port: http
            initialDelaySeconds: 5
            periodSeconds: 10
      volumes:
        - name: config
          configMap:
            name: ingress-catchall
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: ingress-catchall
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-catchall
    app.kubernetes.io/part-of: ingress-nginx
 spec:
  selector:
    app.kubernetes.io/name: ingress-catchall
  ports:
    - name: http
      port: 80
      targetPort: http
@@ -85,6 +85,10 @@ metadata:
  name: otel-debug-ingress
  namespace: monitoring
  annotations:
    k8s-helper.io/name: "Go OTel"
    k8s-helper.io/description: "OpenTelemetry debugging dashboard"
    k8s-helper.io/group: "Observability"
    k8s-helper.io/icon: "activity"
    cert-manager.io/cluster-issuer: letsencrypt-prod
    # OIDC protection via the platform auth service (users.k6n.net):
    # /oauth2/auth resolves the session cookie / bearer token and answers
@@ -0,0 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - deployment.yaml
  - service.yaml
  - ingress.yaml
@@ -0,0 +1,10 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
  name: otel-sourcemaps
  namespace: monitoring
 spec:
  accessModes: ["ReadWriteOnce"]
  resources:
    requests:
      storage: 1Gi
@@ -16,7 +16,6 @@ import {
  ScrollText,
  ShieldBan,
  Bot,
  Sparkles,
  Loader2,
  X,
 } from "lucide-react"
@@ -38,7 +37,6 @@ import { useSubRoute } from "../lib/routes"
 import { CopyRef } from "./common/CopyRef"
 import { IpButton } from "./common/IpButton"
 import { IssueOccurrences } from "./IssueOccurrences"
 import { NeedleTestModal } from "./NeedleTestModal"
 import { FilterBar, type FilterChip } from "./FilterBar"
 import { CRAWLER_CLASSES, UA_CLASSES, uaClassLabel } from "../lib/ua-class"
 import type { Issue, IssueSample } from "../lib/types"
@@ -94,16 +92,6 @@ interface DetectorTestResult {
  error?: string
 }
 interface NeedleResult {
  action?: string
  tool?: string
  confidence?: number
  reasoning?: string
  latency_ms?: number
  arguments?: Record<string, unknown>
  error?: string
 }
 const statusTabs: { value: StatusFilter; label: string }[] = [
  { value: "all", label: "All" },
  { value: "open", label: "Open" },
@@ -315,26 +303,8 @@ const IssueCard = ({
  detectedIPByAddress: Map<string, DetectedIP>
 }) => {
  const [expanded, setExpanded] = useState(false)
  const [needleResult, setNeedleResult] = useState<NeedleResult | null>(null)
  const [needleLoading, setNeedleLoading] = useState(false)
  const { sources, loading: sourcesLoading, error: sourcesError } = useIssueSources(issue.id, expanded)
  const handleNeedleTriage = async () => {
    setNeedleLoading(true)
    try {
      const response = await authFetch("/api/needle/classify", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({ issue_id: issue.id }),
      })
      setNeedleResult(await response.json())
    } catch (err) {
      setNeedleResult({ error: err instanceof Error ? err.message : String(err) })
    } finally {
      setNeedleLoading(false)
    }
  }
  const meta = kindMeta[issue.kind] ?? { label: issue.kind, Icon: Bug }
  const tone = issueSeverityTone(issue)
  const latestSample = issue.samples[issue.samples.length - 1]
@@ -505,57 +475,6 @@ const IssueCard = ({
            )}
          </div>
          <div className="space-y-2">
            <button
              type="button"
              onClick={() => void handleNeedleTriage()}
              disabled={needleLoading}
              title="Classify this issue using local Needle 3 model"
              className="inline-flex h-8 items-center gap-2 rounded-[9px] border border-ds-line-strong px-3 text-[12px] font-medium text-ds-text-2 hover:text-ds-text disabled:opacity-50"
            >
              {needleLoading ? (
                <Loader2 className="h-3.5 w-3.5 animate-spin" aria-hidden="true" />
              ) : (
                <Sparkles className="h-3.5 w-3.5" aria-hidden="true" />
              )}
              {needleLoading ? "Needle 3 triaging…" : "Needle 3 triage"}
            </button>
            {needleResult && (
              <div className="space-y-2 rounded-[12px] border border-ds-line bg-ds-surface-2 p-3 text-[12px]">
                <div className="flex flex-wrap items-center justify-between gap-2 border-b border-ds-line pb-2">
                  <span className="flex flex-wrap items-center gap-2">
                    <span className="text-[10.5px] font-bold uppercase tracking-wider text-ds-text-3">Needle 3 action</span>
                    <span className="rounded-[6px] border border-ds-line-strong px-2 py-0.5 font-[family-name:var(--font-data)] text-ds-text">
                      {needleResult.action || needleResult.tool || "Processed"}
                    </span>
                  </span>
                  {needleResult.confidence !== undefined && (
                    <span className="font-[family-name:var(--font-data)] text-[11px] text-ds-text-3">
                      confidence{" "}
                      <span className={needleResult.confidence >= 0.6 ? "text-ds-ok" : "text-ds-warn"}>
                        {(needleResult.confidence * 100).toFixed(0)}%
                      </span>
                      {needleResult.latency_ms ? ` (${needleResult.latency_ms}ms)` : ""}
                    </span>
                  )}
                </div>
                {needleResult.reasoning && <p className="italic text-ds-text-2">&ldquo;{needleResult.reasoning}&rdquo;</p>}
                {needleResult.arguments && Object.keys(needleResult.arguments).length > 0 && (
                  <div className="flex flex-wrap gap-1.5 font-[family-name:var(--font-data)] text-[11px]">
                    {Object.entries(needleResult.arguments).map(([k, v]) => (
                      <span key={k} className="rounded-[6px] border border-ds-line bg-ds-surface px-2 py-0.5 text-ds-text-2">
                        <span className="text-ds-text-3">{k}:</span> {String(v)}
                      </span>
                    ))}
                  </div>
                )}
                {needleResult.error && (
                  <p className="font-[family-name:var(--font-data)] text-[11px] text-ds-critical">{needleResult.error}</p>
                )}
              </div>
            )}
          </div>
          <div className="space-y-2">
            <div className="flex items-baseline justify-between gap-3">
              <p className="text-[10.5px] font-bold uppercase tracking-widest text-ds-text-3">
@@ -750,7 +669,6 @@ export const IssuesVisualizer = ({
  const [statusFilter, setStatusFilter] = useState<StatusFilter>("all")
  const [kindFilter, setKindFilter] = useState<KindFilter>("all")
  const [groupBy, setGroupBy] = useState<GroupBy>("none")
  const [needleModalOpen, setNeedleModalOpen] = useState(false)
  const [detectorTest, setDetectorTest] = useState<DetectorTestResult | null>(null)
  const [detectorTestLoading, setDetectorTestLoading] = useState(false)
  const [excludeSpamIPs, setExcludeSpamIPs] = useState(false)
@@ -1119,15 +1037,6 @@ export const IssuesVisualizer = ({
              <Bot className="h-3.5 w-3.5" aria-hidden="true" />
              {crawlersHidden ? "Crawlers hidden" : "Hide crawlers"}
            </button>
            <button
              type="button"
              onClick={() => setNeedleModalOpen(true)}
              title="Open interactive Needle 3 telemetry classifier"
              className="inline-flex h-9 items-center gap-1.5 rounded-[9px] border border-ds-line-strong px-2.5 text-[12px] font-medium text-ds-text-2 hover:text-ds-text"
            >
              <Sparkles className="h-3.5 w-3.5" aria-hidden="true" />
              Test Needle 3
            </button>
            <button
              type="button"
              onClick={() => void runDetectorTest()}
@@ -1295,7 +1204,6 @@ export const IssuesVisualizer = ({
        </aside>
      </div>
      <NeedleTestModal isOpen={needleModalOpen} onClose={() => setNeedleModalOpen(false)} />
    </div>
  )
 }
@@ -1,236 +0,0 @@
 import React, { useState } from "react"
 import { X, Sparkles, Loader2, Cpu, CheckCircle2, AlertTriangle } from "lucide-react"
 import { authFetch } from "../lib/auth"
 interface NeedleTestModalProps {
  isOpen: boolean
  onClose: () => void
 }
 interface NeedleResponse {
  success?: boolean
  action?: string
  tool?: string
  confidence?: number
  reasoning?: string
  latency_ms?: number
  arguments?: Record<string, unknown>
  error?: string
 }
 const PRESETS = [
  {
    label: "Frontend RUM Crash",
    service: "eventsourced-ai-web",
    text: "User reported client browser crash on service eventsourced-ai-web: Script error on url https://event-agent.k6n.net/#session=angry-koala-1",
  },
  {
    label: "Security CVE Probe",
    service: "nginx",
    text: "Alert security team: scanner metabase-cve-2026-72898-detect/1.0 probed target /api/session/properties with status code 503",
  },
  {
    label: "Web Vital Metric",
    service: "storefront-web",
    text: "Store RUM metric on service storefront-web: record TTFB of 84 ms",
  },
  {
    label: "Benign 404 Route",
    service: "git.k6n.net",
    text: "Client requested missing git resource: 404 on path /mats/go-cart-actor/blame/commit/81569b on host git.k6n.net",
  },
  {
    label: "Database Pool Deadlock",
    service: "checkout-service",
    text: "Critical incident: Escalate severe backend database timeout on service checkout-service: deadlocked while acquiring connection pool",
  },
 ]
 export const NeedleTestModal: React.FC<NeedleTestModalProps> = ({ isOpen, onClose }) => {
  const [inputText, setInputText] = useState(PRESETS[0].text)
  const [loading, setLoading] = useState(false)
  const [result, setResult] = useState<NeedleResponse | null>(null)
  if (!isOpen) return null
  const handleClassify = async () => {
    if (!inputText.trim()) return
    setLoading(true)
    setResult(null)
    try {
      const resp = await authFetch("/api/needle/classify", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({ text: inputText }),
      })
      const data = await resp.json()
      setResult(data)
    } catch (err) {
      setResult({ error: err instanceof Error ? err.message : String(err) })
    } finally {
      setLoading(false)
    }
  }
  return (
    <div className="fixed inset-0 z-50 flex items-center justify-center bg-ds-bg/80 p-4 animate-fade-in">
      <div className="relative flex flex-col w-full max-w-2xl rounded-[14px] border border-ds-ok/30 bg-ds-bg/95 p-6 max-h-[90vh] overflow-y-auto">
        {/* Header */}
        <div className="flex items-center justify-between border-b border-ds-line pb-4">
          <div className="flex items-center gap-3">
            <div className="flex h-10 w-10 items-center justify-center rounded-xl bg-ds-ok/15 border border-ds-ok/30 text-ds-ok">
              <Cpu className="h-5 w-5" />
            </div>
            <div>
              <h2 className="text-base font-bold text-ds-text flex items-center gap-2">
                Needle 3 AI Telemetry Classifier
                <span className="rounded-full bg-ds-ok/15 border border-ds-ok/30 px-2 py-0.5 text-[10px] font-mono text-ds-ok">
                  Cactus-Compute
                </span>
              </h2>
              <p className="text-xs text-ds-text-2">
                On-device ~35MB foundation model executing real-time tool calling and extraction on local CPU.
              </p>
            </div>
          </div>
          <button
            type="button"
            onClick={onClose}
            className="rounded-lg p-1.5 text-ds-text-2 hover:bg-ds-fill hover:text-ds-text transition-colors"
          >
            <X className="h-5 w-5" />
          </button>
        </div>
        {/* Preset Selector */}
        <div className="mt-4">
          <label className="text-[11px] font-bold uppercase tracking-wider text-ds-text-2">
            Quick Telemetry Presets (from ClickHouse)
          </label>
          <div className="flex flex-wrap gap-2 mt-2">
            {PRESETS.map((p) => (
              <button
                key={p.label}
                type="button"
                onClick={() => {
                  setInputText(p.text)
                  setResult(null)
                }}
                className={`rounded-lg border px-2.5 py-1 text-[11px] font-medium transition-all ${
                  inputText === p.text
                    ? "border-ds-ok bg-ds-ok/15 text-ds-ok"
                    : "border-ds-line bg-ds-surface-2 text-ds-text-2 hover:border-ds-line-strong hover:bg-ds-fill"
                }`}
              >
                {p.label}
              </button>
            ))}
          </div>
        </div>
        {/* Input Textarea */}
        <div className="mt-4">
          <label className="text-[11px] font-bold uppercase tracking-wider text-ds-text-2">
            Telemetry Event / Log / Trace Summary
          </label>
          <textarea
            value={inputText}
            onChange={(e) => setInputText(e.target.value)}
            rows={3}
            className="mt-1.5 w-full rounded-xl border border-ds-line bg-ds-surface-2 p-3 font-mono text-xs text-ds-text placeholder:text-ds-text-3 focus:border-ds-ok/50 focus:outline-none focus:ring-1 focus:ring-ds-ok/40"
            placeholder="Type or paste telemetry event..."
          />
        </div>
        {/* Classify Button */}
        <div className="mt-3 flex justify-end">
          <button
            type="button"
            onClick={handleClassify}
            disabled={loading || !inputText.trim()}
            className="inline-flex items-center gap-2 rounded-xl bg-ds-surface-2 px-4 py-2 text-xs font-bold text-ds-on-accent transition-all hover:opacity-95 active:scale-95 disabled:opacity-50"
          >
            {loading ? (
              <>
                <Loader2 className="h-4 w-4 animate-spin" />
                Classifying on CPU...
              </>
            ) : (
              <>
                <Sparkles className="h-4 w-4" />
                Classify with Needle 3
              </>
            )}
          </button>
        </div>
        {/* Results Area */}
        {result && (
          <div className="mt-5 rounded-xl border border-ds-ok/30 bg-ds-ok/15 p-4 space-y-3">
            <div className="flex items-center justify-between border-b border-ds-ok/20 pb-3">
              <div className="flex items-center gap-2">
                {result.error ? (
                  <AlertTriangle className="h-4 w-4 text-ds-critical" />
                ) : (
                  <CheckCircle2 className="h-4 w-4 text-ds-ok" />
                )}
                <span className="text-xs font-bold uppercase tracking-wider text-ds-text-2">
                  {result.error ? "Error" : "Triage Result"}
                </span>
                {result.action && (
                  <span className="rounded-lg bg-ds-ok/15 border border-ds-ok/40 px-2 py-0.5 font-mono text-xs font-bold text-ds-ok">
                    {result.action}
                  </span>
                )}
              </div>
              {result.confidence !== undefined && (
                <div className="flex items-center gap-3 text-xs font-mono">
                  <span className="text-ds-text-2">
                    Confidence:{" "}
                    <span className="font-bold text-ds-ok">
                      {(result.confidence * 100).toFixed(0)}%
                    </span>
                  </span>
                  {result.latency_ms !== undefined && (
                    <span className="text-ds-text-3">({result.latency_ms}ms)</span>
                  )}
                </div>
              )}
            </div>
            {result.reasoning && (
              <p className="text-xs text-ds-text-2 italic">
                &ldquo;{result.reasoning}&rdquo;
              </p>
            )}
            {result.arguments && Object.keys(result.arguments).length > 0 && (
              <div className="space-y-1.5 pt-1">
                <span className="text-[10px] font-bold uppercase tracking-wider text-ds-text-2">
                  Extracted Typed Arguments:
                </span>
                <div className="grid grid-cols-2 gap-2 font-mono text-xs">
                  {Object.entries(result.arguments).map(([key, value]) => (
                    <div
                      key={key}
                      className="rounded-lg border border-ds-line bg-ds-surface-2 p-2 text-ds-text break-all"
                    >
                      <span className="text-ds-text-3">{key}: </span>
                      <span className="text-ds-ok font-semibold">{String(value)}</span>
                    </div>
                  ))}
                </div>
              </div>
            )}
            {result.error && (
              <div className="rounded-lg bg-ds-critical-bg border border-ds-critical/20 p-3 text-xs text-ds-critical">
                {result.error}
              </div>
            )}
          </div>
        )}
      </div>
    </div>
  )
 }
@@ -1,21 +0,0 @@
 #!/usr/bin/env bash
 set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 ROOT_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
 # Use pre-configured virtual environment or fall back to local .venv
 VENV_DIR="/home/mats/.gemini/antigravity-cli/brain/8e7a2c5c-81aa-487e-81b1-bf0cf790d055/scratch/needle_venv"
 if [ ! -d "${VENV_DIR}" ]; then
  VENV_DIR="${ROOT_DIR}/.venv"
 fi
 if [ ! -f "${VENV_DIR}/bin/python" ]; then
  echo "Setting up virtual environment at ${VENV_DIR}..."
  python3 -m venv "${VENV_DIR}"
  "${VENV_DIR}/bin/pip" install --upgrade pip
  "${VENV_DIR}/bin/pip" install cactus-needle pydantic
 fi
 echo "Running Needle 3 OTel Verification Suite..."
 "${VENV_DIR}/bin/python" "${SCRIPT_DIR}/test_needle_classifier.py"
@@ -1,168 +0,0 @@
 #!/usr/bin/env python3
 """
 Test and Verification Suite for Cactus-Compute/needle3 with OpenTelemetry data.
 Tests:
 1. Action/Tool Dispatch Accuracy (Ground Truth Verification)
 2. Field Extraction & Grounding
 3. Confidence Calibrations (Act vs. Escalate)
 4. Semantic Vector Embeddings (Issue Clustering)
 5. Execution Performance & Latency (Tokens/sec)
 """
 import sys
 import time
 import json
 import math
 import subprocess
 try:
    import needle
 except ImportError:
    print("Error: cactus-needle not found. Run with the virtualenv python:")
    print("  /home/mats/.gemini/antigravity-cli/brain/8e7a2c5c-81aa-487e-81b1-bf0cf790d055/scratch/needle_venv/bin/python " + __file__)
    sys.exit(1)
 # --- Define Triage Tools ---
@needle.tool
 def report_client_javascript_crash(service: str, message: str, url: str):
    """Report client-side JavaScript crash or error logged by browser RUM."""
    return {"action": "client_js_crash", "service": service, "message": message, "url": url}
@needle.tool
 def alert_vulnerability_probe(scanner: str, target: str, status_code: int):
    """Alert on malicious vulnerability scans or exploit crawlers."""
    return {"action": "security_alert", "scanner": scanner, "target": target, "status_code": status_code}
@needle.tool
 def record_frontend_web_vital(service: str, metric_name: str, duration_ms: int):
    """Record RUM performance metrics like TTFB or FCP."""
    return {"action": "record_vital", "service": service, "metric": metric_name, "ms": duration_ms}
@needle.tool
 def log_missing_resource_404(path: str, host: str):
    """Log benign 404 Not Found requests for missing pages or git commits."""
    return {"action": "log_404", "path": path, "host": host}
@needle.tool
 def escalate_backend_outage(service: str, error_message: str):
    """Escalate severe backend database timeout or service crash to on-call."""
    return {"action": "escalate_outage", "service": service, "error": error_message}
 def run_verification():
    print("=" * 70)
    print("  NEEDLE 3 OPENTELEMETRY CLASSIFIER - VERIFICATION SUITE")
    print("=" * 70)
    # 1. Initialize Agent
    print("\n[Step 1] Initializing Needle 3 Engine...")
    t0 = time.perf_counter()
    agent = needle.Needle(tools=[
        report_client_javascript_crash,
        alert_vulnerability_probe,
        record_frontend_web_vital,
        log_missing_resource_404,
        escalate_backend_outage
    ])
    init_ms = (time.perf_counter() - t0) * 1000
    print(f"✓ Needle 3 Engine initialized in {init_ms:.1f}ms")
    # 2. Ground-Truth Test Cases (from actual ClickHouse telemetry)
    test_cases = [
        {
            "name": "Frontend Browser RUM Crash",
            "input": "User reported client browser crash on service eventsourced-ai-web: Script error on url https://event-agent.k6n.net/#session=angry-koala-1",
            "expected_tool": "report_client_javascript_crash",
            "expected_fields": ["service", "url"]
        },
        {
            "name": "Malicious CVE Exploit Probe",
            "input": "Alert security team: scanner metabase-cve-2026-72898-detect/1.0 probed target /api/session/properties with status code 503",
            "expected_tool": "alert_vulnerability_probe",
            "expected_fields": ["target", "status_code"]
        },
        {
            "name": "Web Vital RUM Metric",
            "input": "Store RUM metric on service storefront-web: record TTFB of 84 ms",
            "expected_tool": "record_frontend_web_vital",
            "expected_fields": ["service", "metric", "ms"]
        },
        {
            "name": "Nginx Benign 404 Route",
            "input": "Client requested missing git resource: 404 on path /mats/go-cart-actor/blame/commit/81569b on host git.k6n.net",
            "expected_tool": "log_missing_resource_404",
            "expected_fields": ["path", "host"]
        },
        {
            "name": "Critical Database Outage",
            "input": "Critical incident: Escalate severe backend database timeout on service checkout-service: deadlocked while acquiring connection pool",
            "expected_tool": "escalate_backend_outage",
            "expected_fields": ["service"]
        }
    ]
    print("\n[Step 2] Testing Classification & Tool Dispatch Accuracy...")
    passed = 0
    total = len(test_cases)
    latencies = []
    for i, tc in enumerate(test_cases, 1):
        agent.reset()
        t_start = time.perf_counter()
        res = agent.run(tc["input"], strict=False)
        dur_ms = (time.perf_counter() - t_start) * 1000
        latencies.append(dur_ms)
        results = res.get("results", [])
        matched_action = results[0].get("action") if results and isinstance(results[0], dict) else None
        # Tool name to action map
        expected_action = tc["expected_tool"].replace("report_", "").replace("alert_", "").replace("record_", "").replace("log_", "").replace("escalate_", "")
        # Check match
        is_match = False
        if results and any(tc["expected_tool"] in str(c) or matched_action for c in results):
            is_match = True
        status_str = "PASS" if is_match else "FAIL"
        if is_match:
            passed += 1
        print(f"\n  Test {i}/{total}: {tc['name']} -> [{status_str}] ({dur_ms:.1f}ms)")
        print(f"    Confidence: {res.get('confidence', 0.0):.3f}")
        print(f"    Reasoning : {res.get('reasoning')}")
        print(f"    Dispatched: {results}")
    print("\n" + "-" * 70)
    print(f"Accuracy Score: {passed}/{total} ({passed/total*100:.0f}%) | Avg Latency: {sum(latencies)/len(latencies):.1f}ms")
    print("-" * 70)
    # 3. Embedding & Semantic Search Verification
    print("\n[Step 3] Testing Semantic Vector Embeddings for Near-Match Clustering...")
    emb_e1 = agent.embed("database connection timeout after 3000ms")
    emb_e2 = agent.embed("postgres pool exhausted: connection timed out")
    emb_e3 = agent.embed("invalid JWT signature token expired")
    def cosine_similarity(v1, v2):
        dot = sum(a * b for a, b in zip(v1, v2))
        m1 = math.sqrt(sum(a * a for a in v1))
        m2 = math.sqrt(sum(b * b for b in v2))
        return dot / (m1 * m2)
    sim_similar = cosine_similarity(emb_e1, emb_e2)
    sim_different = cosine_similarity(emb_e1, emb_e3)
    print(f"  Vector Dimensions: {len(emb_e1)}")
    print(f"  Similarity between 2 DB timeouts : {sim_similar:.4f}")
    print(f"  Similarity between DB and Auth   : {sim_different:.4f}")
    if sim_similar > sim_different:
        print("✓ Semantic clustering validated: related issues score higher similarity.")
    else:
        print("✗ Warning: semantic separation failed.")
    print("\n" + "=" * 70)
    print("  VERIFICATION COMPLETE")
    print("=" * 70)
 if __name__ == "__main__":
    run_verification()
Author	SHA1	Message	Date
mats	7b68f60202	Merge origin/main into feature/issues-tui Build and Publish / Metadata (push) Successful in 5s Details Build and Publish / BuildAndPushAmd64 (push) Failing after 1m42s Details	2026-11-01 11:40:57 +01:00
matsandClaude Sonnet 5.5	51a8412ae1	Remove Needle; stop paging on nginx 4xx Needle scored ~chance on the golden set and never gated alerts, so drop the sidecar, proxy handlers, UI buttons, k8s container and build step. The labelled eval sets move to cmd/triage-eval/data. nginx 4xx (registry 401 challenges, /oauth2/auth, scanner 405s) is now dropped in FilterActionable instead of only being discounted, so it cannot ride along with a real 5xx. nginx 5xx still alerts. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>	2026-11-01 11:40:53 +01:00
mats	601f382af6	Merge pull request 'eval' (#2 ) from feature/issues-tui into main Build and Publish / Metadata (push) Successful in 3s Details Build and Publish / BuildAndPushAmd64 (push) Failing after 1m19s Details Reviewed-on: #2	2026-09-27 16:47:09 +02:00
mats	71184e0823	Merge branch 'main' into feature/issues-tui	2026-09-27 16:46:37 +02:00
mats	e046bd86d1	eval	2026-09-27 16:45:10 +02:00
		`@@ -0,0 +1,2 @@`
							{"enabled":true,"dryRun":false,"channels":["telegram"],"lastRun":"2026-09-25T18:56:43.3470721Z","alerts":[{"id":"tlxng1-nginx","at":"2026-09-25T18:31:13.347300133Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:26:13.347300133Z","windowEnd":"2026-09-25T18:31:13.347300133Z","errorCount":69,"baselineMean":14.03448275862069,"excess":4.9655172413793105,"patterns":[{"signature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","count":4,"baselineMean":0.034482758620689655,"excess":3.9655172413793105,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:31:11.346792393Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"0751de83a2f0ce211f2b7ab1a64d8aba","PatternSignature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"auth.k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"34.177.93.136","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"fb485cd240e1dab0261891dc14170112bbc37c90189f276b45935afdedf534cb","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:31:06.252746363Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d05925e26f581fcba99d165ea5928114","PatternSignature":"fb485cd240e1dab0261891dc14170112bbc37c90189f276b45935afdedf534cb","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/oauth2/auth","Host":"users.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"auth.k6n.net","path":"/","services":["nginx"],"routes":[{"ingress":"dev/zitadel-ingress","host":"auth.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"zitadel-dev","port":"8080","deployments":["zitadel-dev"]}],"pods":[{"namespace":"dev","deployment":"zitadel-dev","found":false}],"rule":"status-5xx-upstream","new":false,"count":4,"baseline":0.034482758620689655,"clients":1,"what":"HTTP 503 \| /"},{"category":"client_credential_issue","priority":"low","status":401,"host":"users.k6n.net","path":"/oauth2/auth","services":["nginx"],"routes":[{"ingress":"ecommerce/ecommerce-auth","host":"users.k6n.net","path":"/","pathType":"Prefix","namespace":"ecommerce","service":"auth","port":"8080","deployments":["auth"]}],"pods":[{"namespace":"ecommerce","deployment":"auth","pod":"auth-8f497c689-qfpzb","lines":["2026/09/25 18:13:06 oidc: device grant JHMJ-C23E started for client dev-console","2026/09/25 18:13:06 oidc: device grant EQ5R-UWRW approved by mats@tornberg.me","2026/09/25 18:23:19 oidc: device grant RHMN-6257 started for client dev-console"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 \| /oauth2/auth"}]},{"id":"tlxn9d-nginx","at":"2026-09-25T18:27:13.347226389Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:22:13.347226389Z","windowEnd":"2026-09-25T18:27:13.347226389Z","errorCount":64,"baselineMean":13.931034482758621,"excess":4,"patterns":[{"signature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","count":4,"baselineMean":0,"excess":4,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:26:26.152736211Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d1de7de09ad342e7b0670cb78cac1430","PatternSignature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"cc.k6n.net","StatusCode":405,"ExceptionType":"","ClientIP":"35.189.233.235","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"needs_review","priority":"low","status":405,"host":"cc.k6n.net","path":"/","services":["nginx"],"rule":"no-rule","new":false,"count":4,"baseline":0,"clients":1,"what":"HTTP 405 \| /"}]},{"id":"tlxn61-nginx","at":"2026-09-25T18:25:13.346661147Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:20:13.346661147Z","windowEnd":"2026-09-25T18:25:13.346661147Z","errorCount":60,"baselineMean":14.03448275862069,"excess":1,"patterns":[{"signature":"ea444cb83ff471aad7f3fab8a5b36dddb25f333f7dce2eb4ff27f6e9bb3969f7","count":1,"baselineMean":0,"excess":1,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:24:36.217473837Z","Service":"nginx","Release":"","Kind":"span","SourcLine truncated