2026-09-27 16:47:09 +02:00
18 changed files with 3020 additions and 46 deletions
@@ -1,6 +1,9 @@
 name: Build and Publish
 run-name: ${{ gitea.actor }} build 🚀
-on: [push]
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:

 jobs:
  Metadata:
@@ -26,7 +29,7 @@ jobs:
          echo "version=$VERSION" >> $GITHUB_OUTPUT
          echo "build_date=$BUILD_DATE" >> $GITHUB_OUTPUT

-  BuildAndDeployAmd64:
+  BuildAndPushAmd64:
    needs: Metadata
    runs-on: amd64
    steps:
@@ -39,15 +42,32 @@ jobs:
            --build-arg GIT_COMMIT=${{ needs.Metadata.outputs.git_commit }} \
            --build-arg BUILD_DATE=${{ needs.Metadata.outputs.build_date }} \
            --progress=plain \
+            -t matst80/otel-debug:${{ needs.Metadata.outputs.version }} \
            -t matst80/otel-debug:latest \
            .
-      - name: Push amd64 images
+      - name: Push images
        run: |
+          docker push matst80/otel-debug:${{ needs.Metadata.outputs.version }}
          docker push matst80/otel-debug:latest
-      - name: Rollout amd64 otel deployment
+      - name: Reconcile after image is published
+        env:
+          GITOPS_WEBHOOK_SECRET: ${{ secrets.GITOPS_WEBHOOK_SECRET }}
        run: |
-          kubectl rollout restart deployment/otel-debug-receiver -n monitoring
-          kubectl rollout status deployment/otel-debug-receiver -n monitoring
+          if [ -z "$GITOPS_WEBHOOK_SECRET" ]; then
+            echo "GITOPS_WEBHOOK_SECRET must match the k8s-helper Gitea webhook secret" >&2
+            exit 1
+          fi
+          payload="$(mktemp)"
+          trap 'rm -f "$payload"' EXIT
+          printf '{"ref":"%s","after":"%s","repository":{"clone_url":"https://git.k6n.net/mats/go-otel.git","ssh_url":"git@git.k6n.net:mats/go-otel.git"}}' \
+            "$GITHUB_REF" "$GITHUB_SHA" > "$payload"
+          signature="$(openssl dgst -sha256 -hmac "$GITOPS_WEBHOOK_SECRET" "$payload" | awk '{print $NF}')"
+          curl --fail --silent --show-error \
+            -X POST \
+            -H "Content-Type: application/json" \
+            -H "X-Gitea-Signature: $signature" \
+            --data-binary "@$payload" \
+            https://ui.k6n.net/api/gitops/webhooks/gitea
      - name: Upload source maps
        env:
          RELEASE: ${{ needs.Metadata.outputs.version }}
@@ -69,31 +89,3 @@ jobs:
          done < <(find "$maps_dir" -type f -name '*.js.map' -print0)
          rm -rf "$maps_dir"
          [ "$found" = 1 ] || { echo "no source maps found in UI build" >&2; exit 1; }
-      # - name: Rollout amd64 deployment (pin to version)
-      #   run: |
-      #     kubectl set image deployment/cart-actor-x86 -n cart cart-actor-amd64=registry.knatofs.se/go-otel-debug-amd64:${{ needs.Metadata.outputs.version }}
-      #     # kubectl rollout status deployment/cart-actor-x86 -n cart
-
-  # BuildAndDeployArm64:
-  #   needs: Metadata
-  #   runs-on: arm64
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: Build arm64 image
-  #       run: |
-  #         docker build \
-  #           --build-arg VERSION=${{ needs.Metadata.outputs.version }} \
-  #           --build-arg GIT_COMMIT=${{ needs.Metadata.outputs.git_commit }} \
-  #           --build-arg BUILD_DATE=${{ needs.Metadata.outputs.build_date }} \
-  #           --progress=plain \
-  #           -t registry.knatofs.se/go-otel-debug:latest \
-  #           -t registry.knatofs.se/go-otel-debug:${{ needs.Metadata.outputs.version }} \
-  #           .
-  #     - name: Push arm64 images
-  #       run: |
-  #         docker push registry.knatofs.se/go-otel-debug:latest
-  #         docker push registry.knatofs.se/go-otel-debug:${{ needs.Metadata.outputs.version }}
-  #     - name: Rollout arm64 deployment (pin to version)
-  #       run: |
-  #         kubectl set image deployment/cart-actor-arm64 -n cart cart-actor-arm64=registry.knatofs.se/go-otel-debug:${{ needs.Metadata.outputs.version }}
-  #         # kubectl rollout status deployment/cart-actor-arm64 -n cart
@@ -0,0 +1,2 @@
+{"enabled":true,"dryRun":false,"channels":["telegram"],"lastRun":"2026-09-25T18:56:43.3470721Z","alerts":[{"id":"tlxng1-nginx","at":"2026-09-25T18:31:13.347300133Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:26:13.347300133Z","windowEnd":"2026-09-25T18:31:13.347300133Z","errorCount":69,"baselineMean":14.03448275862069,"excess":4.9655172413793105,"patterns":[{"signature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","count":4,"baselineMean":0.034482758620689655,"excess":3.9655172413793105,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:31:11.346792393Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"0751de83a2f0ce211f2b7ab1a64d8aba","PatternSignature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"auth.k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"34.177.93.136","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"fb485cd240e1dab0261891dc14170112bbc37c90189f276b45935afdedf534cb","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:31:06.252746363Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d05925e26f581fcba99d165ea5928114","PatternSignature":"fb485cd240e1dab0261891dc14170112bbc37c90189f276b45935afdedf534cb","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/oauth2/auth","Host":"users.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"auth.k6n.net","path":"/","services":["nginx"],"routes":[{"ingress":"dev/zitadel-ingress","host":"auth.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"zitadel-dev","port":"8080","deployments":["zitadel-dev"]}],"pods":[{"namespace":"dev","deployment":"zitadel-dev","found":false}],"rule":"status-5xx-upstream","new":false,"count":4,"baseline":0.034482758620689655,"clients":1,"what":"HTTP 503 | /"},{"category":"client_credential_issue","priority":"low","status":401,"host":"users.k6n.net","path":"/oauth2/auth","services":["nginx"],"routes":[{"ingress":"ecommerce/ecommerce-auth","host":"users.k6n.net","path":"/","pathType":"Prefix","namespace":"ecommerce","service":"auth","port":"8080","deployments":["auth"]}],"pods":[{"namespace":"ecommerce","deployment":"auth","pod":"auth-8f497c689-qfpzb","lines":["2026/09/25 18:13:06 oidc: device grant JHMJ-C23E started for client dev-console","2026/09/25 18:13:06 oidc: device grant EQ5R-UWRW approved by mats@tornberg.me","2026/09/25 18:23:19 oidc: device grant RHMN-6257 started for client dev-console"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /oauth2/auth"}]},{"id":"tlxn9d-nginx","at":"2026-09-25T18:27:13.347226389Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:22:13.347226389Z","windowEnd":"2026-09-25T18:27:13.347226389Z","errorCount":64,"baselineMean":13.931034482758621,"excess":4,"patterns":[{"signature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","count":4,"baselineMean":0,"excess":4,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:26:26.152736211Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d1de7de09ad342e7b0670cb78cac1430","PatternSignature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"cc.k6n.net","StatusCode":405,"ExceptionType":"","ClientIP":"35.189.233.235","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"needs_review","priority":"low","status":405,"host":"cc.k6n.net","path":"/","services":["nginx"],"rule":"no-rule","new":false,"count":4,"baseline":0,"clients":1,"what":"HTTP 405 | /"}]},{"id":"tlxn61-nginx","at":"2026-09-25T18:25:13.346661147Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:20:13.346661147Z","windowEnd":"2026-09-25T18:25:13.346661147Z","errorCount":60,"baselineMean":14.03448275862069,"excess":1,"patterns":[{"signature":"ea444cb83ff471aad7f3fab8a5b36dddb25f333f7dce2eb4ff27f6e9bb3969f7","count":1,"baselineMean":0,"excess":1,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:24:36.217473837Z","Service":"nginx","Release":"","Kind":"span","SourcLine truncated
+
@@ -0,0 +1,144 @@
+// laya-eval runs alert scenarios through laya and compares its shadow verdict
+// with what a person judged the right response to be, so the model can be
+// assessed before it is ever allowed to act.
+//
+//	go run ./cmd/laya-eval                        # built-in scenarios
+//	go run ./cmd/laya-eval -file alerts.json      # a GET /api/alerts dump (no expectations)
+//	go run ./cmd/laya-eval -send                  # also deliver each as a TEST alert
+//
+// -send uses the channels configured by OTEL_ALERT_* (see alerts.ConfigFromEnv).
+package main
+
+import (
+	"context"
+	_ "embed"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"text/tabwriter"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/alerts"
+)
+
+//go:embed scenarios.json
+var builtin []byte
+
+// Scenario is one alert and the response a person would pick for it.
+type Scenario struct {
+	Name   string       `json:"name"`
+	Expect *Expectation `json:"expect,omitempty"`
+	Alert  alerts.Alert `json:"alert"`
+}
+
+type Expectation struct {
+	Action string `json:"action"`
+	Human  bool   `json:"human"`
+	Why    string `json:"why"`
+}
+
+func main() {
+	url := flag.String("url", envOr("OTEL_ALERT_LAYA_URL", "https://laya.k6n.net"), "laya base URL")
+	file := flag.String("file", "", "scenarios JSON, or a GET /api/alerts response (default: built-in scenarios)")
+	send := flag.Bool("send", false, "deliver each scenario, with its verdict, as a TEST alert to the configured channels")
+	flag.Parse()
+
+	raw := builtin
+	if *file != "" {
+		var err error
+		if raw, err = os.ReadFile(*file); err != nil {
+			fail(err)
+		}
+	}
+	scenarios, err := parse(raw)
+	if err != nil {
+		fail(err)
+	}
+	var notifiers []alerts.Notifier
+	if *send {
+		cfg := alerts.ConfigFromEnv()
+		if notifiers = cfg.Notifiers(); len(notifiers) == 0 {
+			fail(fmt.Errorf("-send: no channel configured (OTEL_ALERT_TELEGRAM_TOKEN + OTEL_ALERT_TELEGRAM_CHAT_ID, OTEL_ALERT_WEBHOOK_URL or OTEL_ALERT_AOD_URL)"))
+		}
+	}
+
+	laya := &alerts.Laya{URL: *url}
+	tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(tw, "SCENARIO\tLAYA ACTION\tHUMAN\tURGENCY\tEXPECTED\tMATCH")
+	judged, actionOK, humanOK := 0, 0, 0
+	for _, s := range scenarios {
+		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+		v, err := laya.Classify(ctx, s.Alert)
+		cancel()
+		if err != nil {
+			fmt.Fprintf(tw, "%s\terror: %v\t\t\t\t\n", s.Name, err)
+			continue
+		}
+		exp, match := "-", "-"
+		if s.Expect != nil {
+			judged++
+			a, h := v.Action == s.Expect.Action, (v.Human >= 0.5) == s.Expect.Human
+			if a {
+				actionOK++
+			}
+			if h {
+				humanOK++
+			}
+			exp = fmt.Sprintf("%s, human=%v", s.Expect.Action, s.Expect.Human)
+			match = fmt.Sprintf("action %s, human %s", mark(a), mark(h))
+		}
+		fmt.Fprintf(tw, "%s\t%s %.0f%%\t%.0f%%\t%s %.0f%%\t%s\t%s\n", s.Name, v.Action, v.ActionConfidence*100, v.Human*100, v.Urgency, v.UrgencyConf*100, exp, match)
+
+		if *send {
+			a := s.Alert
+			a.Test, a.Laya = true, v
+			for _, n := range notifiers {
+				sctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+				if err := n.Send(sctx, a); err != nil {
+					fmt.Fprintf(os.Stderr, "%s: %s: %v\n", s.Name, n.Name(), err)
+				}
+				cancel()
+			}
+		}
+	}
+	tw.Flush()
+	if judged > 0 {
+		fmt.Printf("\nagreement over %d judged scenarios: action %d/%d, needs-human %d/%d\n", judged, actionOK, judged, humanOK, judged)
+	}
+}
+
+// parse accepts a scenario list or a raw GET /api/alerts response.
+func parse(raw []byte) ([]Scenario, error) {
+	var list []Scenario
+	if err := json.Unmarshal(raw, &list); err == nil {
+		return list, nil
+	}
+	var st alerts.Status
+	if err := json.Unmarshal(raw, &st); err != nil {
+		return nil, fmt.Errorf("neither a scenario list nor an /api/alerts response: %w", err)
+	}
+	for _, a := range st.Alerts {
+		list = append(list, Scenario{Name: a.ID + " " + a.Spike.Service, Alert: a})
+	}
+	return list, nil
+}
+
+func mark(ok bool) string {
+	if ok {
+		return "ok"
+	}
+	return "MISS"
+}
+
+func envOr(k, def string) string {
+	if v := os.Getenv(k); v != "" {
+		return v
+	}
+	return def
+}
+
+func fail(err error) {
+	fmt.Fprintln(os.Stderr, err)
+	os.Exit(1)
+}
@@ -0,0 +1,29 @@
+# Deploying go-otel with k8s-helper
+
+Gitea Actions builds and pushes versioned and `latest` `matst80/otel-debug`
+images on pushes to `main`, uploads the matching source maps, and then sends a
+signed reconcile webhook. The agent applies the explicit `k8s/kustomization.yaml`
+resource list into namespace `monitoring`.
+
+## One-time setup
+
+The k8s-helper GitOps CRDs and agent must be installed. The GitSource reuses
+`dev/gitea-k8s-helper`, whose token must have read access to this repository.
+Configure a Gitea push-only webhook for this repo at
+`https://ui.k6n.net/api/gitops/webhooks/gitea`, using the same secret as
+`K8SHELPER_GITOPS_WEBHOOK_SECRET` on the dashboard. Add that value as the Gitea
+Actions secret `GITOPS_WEBHOOK_SECRET` too; a user- or organization-level
+secret can be shared by other repos.
+
+The runner uses its existing Docker login. Bootstrap once with
+`kubectl apply -f gitops/` after confirming the monitoring namespace and the
+`otel-sourcemaps` PVC already exist. The PVC is deliberately excluded from the
+managed kustomization because its lifecycle was called out for separate review.
+The ClickHouse manifests, auth-dev resources, and ingress-catchall work are also
+excluded; this kustomization lists only the receiver Deployment, Service, and
+Ingress resources. Do not run Flux and k8s-helper against the same resources at
+the same time; check for an active `flux-system/go-otel` Kustomization before
+bootstrapping this controller.
+
+Prune is initially disabled. Inspect the GitOps inventory and applied revision
+before changing it.
@@ -0,0 +1,14 @@
+apiVersion: gitops.k6n.net/v1alpha1
+kind: Deployment
+metadata:
+  name: go-otel
+  namespace: dev
+spec:
+  sourceRef:
+    kind: GitSource
+    name: go-otel
+  path: ./k8s
+  targetNamespace: monitoring
+  prune: false
+  wait: true
+  deletionPolicy: Orphan
@@ -0,0 +1,16 @@
+# Keep GitOps CRs outside the path they manage.
+apiVersion: gitops.k6n.net/v1alpha1
+kind: GitSource
+metadata:
+  name: go-otel
+  namespace: dev
+spec:
+  url: https://git.k6n.net/mats/go-otel.git
+  ref:
+    branch: main
+  interval: 5m
+  secretRef:
+    name: gitea-k8s-helper
+  ignore: |
+    .gitea/
+    docs/
@@ -0,0 +1,180 @@
+package alerts
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Laya asks the laya model (POST {URL}/predict) what it would do about an
+// alert. It runs in shadow mode: the verdict is recorded on the alert and
+// shown in notifications, but nothing is ever acted on, muted or dropped
+// because of it. A failed call leaves the alert exactly as it was.
+type Laya struct {
+	URL    string // base URL, e.g. https://laya.k6n.net
+	Client *http.Client
+}
+
+// LayaActions are the remediations laya may choose between. They are
+// suggestions only; none is wired to an automated action yet.
+var LayaActions = map[string]string{
+	"ignore":             "transient blip or noise, nobody needs to look at it",
+	"restart_deployment": "the service is stuck, leaking or crashed; restarting its pods is likely to clear it",
+	"check_capacity":     "requests time out or resources are exhausted; check load, connection pools, CPU/memory limits and scaling",
+	"fix_dependency":     "a database, upstream service or network path this service depends on is failing",
+	"investigate_code":   "a bug or bad release in this service; needs a code fix or a rollback",
+}
+
+var layaUrgency = []string{"cosmetic", "degraded performance", "service outage"}
+
+// LayaVerdict is laya's answer, reduced to what a person needs to judge it.
+type LayaVerdict struct {
+	Model            string             `json:"model,omitempty"`
+	Action           string             `json:"action"`
+	ActionConfidence float64            `json:"actionConfidence"` // probability of the chosen action
+	ActionProbs      map[string]float64 `json:"actionProbs,omitempty"`
+	Human            float64            `json:"human"` // probability a person must intervene
+	Urgency          string             `json:"urgency"`
+	UrgencyScore     float64            `json:"urgencyScore"` // expected value on 0 (cosmetic) .. 2 (outage)
+	UrgencyConf      float64            `json:"urgencyConfidence"`
+}
+
+func layaRequest(state string) map[string]any {
+	return map[string]any{
+		"state": state,
+		"questions": map[string]any{
+			"remediation_action": map[string]any{
+				"type":         "choice",
+				"instructions": "Which operational response fits this production error alert best?",
+				"criteria":     LayaActions,
+			},
+			"requires_human_attention": map[string]any{
+				"type":         "noul",
+				"instructions": "Does this alert need a human engineer to intervene, or will it resolve or matter little on its own?",
+				"criteria": map[string]string{
+					"true":  "errors are new or rising well above normal and users or requests are failing",
+					"false": "a transient blip, client noise or bot traffic",
+				},
+			},
+			"urgency": map[string]any{
+				"type":         "score",
+				"instructions": "Rate the operational severity of this alert.",
+				"criteria":     layaUrgency,
+			},
+		},
+	}
+}
+
+type layaAnswer struct {
+	Choice           string             `json:"choice"`
+	Probabilities    map[string]float64 `json:"probabilities"`
+	AnswerConfidence float64            `json:"answer_confidence"`
+	Noul             float64            `json:"noul"`
+	Score            float64            `json:"score"`
+	Legend           map[string]string  `json:"legend"`
+}
+
+type layaResponse struct {
+	Model   string                `json:"model"`
+	Answers map[string]layaAnswer `json:"answers"`
+}
+
+// Classify asks laya about the alert. The state is the alert's plain-text
+// message, so the model sees exactly the facts a person would, nothing more.
+func (l *Laya) Classify(ctx context.Context, a Alert) (*LayaVerdict, error) {
+	a.Laya = nil
+	return l.classifyState(ctx, a.Text(""))
+}
+
+func (l *Laya) classifyState(ctx context.Context, state string) (*LayaVerdict, error) {
+	raw, err := json.Marshal(layaRequest(state))
+	if err != nil {
+		return nil, err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(l.URL, "/")+"/predict", bytes.NewReader(raw))
+	if err != nil {
+		return nil, errors.New("laya: invalid URL")
+	}
+	req.Header.Set("Content-Type", "application/json")
+	client := l.Client
+	if client == nil {
+		client = &http.Client{Timeout: 10 * time.Second}
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("laya: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode/100 != 2 {
+		detail, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
+		return nil, fmt.Errorf("laya: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(detail)))
+	}
+	var lr layaResponse
+	if err := json.NewDecoder(resp.Body).Decode(&lr); err != nil {
+		return nil, fmt.Errorf("laya: invalid response: %w", err)
+	}
+	return lr.verdict()
+}
+
+func (lr layaResponse) verdict() (*LayaVerdict, error) {
+	act, ok1 := lr.Answers["remediation_action"]
+	hum, ok2 := lr.Answers["requires_human_attention"]
+	urg, ok3 := lr.Answers["urgency"]
+	if !ok1 || !ok2 || !ok3 || act.Choice == "" {
+		return nil, errors.New("laya: response is missing answers")
+	}
+	v := &LayaVerdict{Model: lr.Model, Action: act.Choice, ActionConfidence: act.Probabilities[act.Choice], ActionProbs: act.Probabilities,
+		Human: hum.Noul, UrgencyScore: urg.Score}
+	// The urgency label is the most probable bucket, not the rounded score:
+	// a 50/50 cosmetic/outage split must not read as "degraded".
+	best := -1.0
+	for k, p := range urg.Probabilities {
+		if p > best || (p == best && k < v.Urgency) {
+			best, v.Urgency = p, k
+		}
+	}
+	v.UrgencyConf = best
+	if label, ok := urg.Legend[v.Urgency]; ok {
+		v.Urgency = label
+	} else if i, err := strconv.Atoi(v.Urgency); err == nil && i >= 0 && i < len(layaUrgency) {
+		v.Urgency = layaUrgency[i]
+	}
+	return v, nil
+}
+
+// Summary is a one-line rendering, e.g.
+// "check_capacity 62% · human 71% · degraded performance 75%".
+func (v LayaVerdict) Summary() string {
+	return fmt.Sprintf("%s %s · human %s · %s %s", v.Action, pct(v.ActionConfidence), pct(v.Human), v.Urgency, pct(v.UrgencyConf))
+}
+
+// RunnersUp lists the other actions by probability, for judging how close
+// the call was.
+func (v LayaVerdict) RunnersUp() string {
+	type kv struct {
+		k string
+		p float64
+	}
+	var rest []kv
+	for k, p := range v.ActionProbs {
+		if k != v.Action {
+			rest = append(rest, kv{k, p})
+		}
+	}
+	sort.Slice(rest, func(i, j int) bool { return rest[i].p > rest[j].p })
+	parts := make([]string, 0, len(rest))
+	for _, r := range rest {
+		parts = append(parts, r.k+" "+pct(r.p))
+	}
+	return strings.Join(parts, ", ")
+}
+
+func pct(p float64) string { return strconv.Itoa(int(p*100+0.5)) + "%" }
@@ -0,0 +1,123 @@
+package alerts
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/features"
+)
+
+// layaReply is the shape laya.k6n.net/predict returned when probed.
+const layaReply = `{"model":"laya-rl-agent","answers":{
+ "remediation_action":{"type":"choice","choice":"check_capacity","probabilities":{"ignore":0.1,"restart_deployment":0.12,"check_capacity":0.62,"fix_dependency":0.1,"investigate_code":0.06},"confidence":0.3,"answer_confidence":0.62},
+ "requires_human_attention":{"type":"noul","noul":0.7123,"confidence":0.7123,"answer_confidence":0.7123},
+ "urgency":{"type":"score","score":1.03,"legend":{"0":"cosmetic","1":"degraded performance","2":"service outage"},"probabilities":{"0":0.11,"1":0.75,"2":0.14},"confidence":0.32,"answer_confidence":0.75}},
+ "usage":{"input_tokens":141,"output_tokens":0}}`
+
+func layaServer(t *testing.T, status int, body string, got *map[string]any) *httptest.Server {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/predict" {
+			t.Errorf("path %q", r.URL.Path)
+		}
+		if got != nil {
+			_ = json.NewDecoder(r.Body).Decode(got)
+		}
+		w.WriteHeader(status)
+		_, _ = w.Write([]byte(body))
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func timeoutAlert() Alert {
+	now := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	return NewAlert(features.Spike{Service: "codesearch", WindowStart: now.Add(-5 * time.Minute), WindowEnd: now, ErrorCount: 15, BaselineMean: 0, Excess: 15,
+		Patterns: []features.PatternDelta{{Signature: "t", Count: 15, Excess: 15, Novel: true,
+			Example: features.Event{Service: "codesearch", Kind: "span", SpanName: "upload", Message: "context deadline exceeded", IsTimeout: true}}}}, now)
+}
+
+func TestLayaClassifySendsAlertTextAndParsesVerdict(t *testing.T) {
+	var req map[string]any
+	srv := layaServer(t, 200, layaReply, &req)
+	a := timeoutAlert()
+	v, err := (&Laya{URL: srv.URL + "/"}).Classify(context.Background(), a)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req["state"] != a.Text("") {
+		t.Fatalf("state is not the alert text:\n%v", req["state"])
+	}
+	qs := req["questions"].(map[string]any)
+	for _, k := range []string{"remediation_action", "requires_human_attention", "urgency"} {
+		if qs[k] == nil {
+			t.Fatalf("question %s missing", k)
+		}
+	}
+	if v.Action != "check_capacity" || v.ActionConfidence != 0.62 || v.Human != 0.7123 || v.Urgency != "degraded performance" || v.UrgencyConf != 0.75 {
+		t.Fatalf("verdict %+v", v)
+	}
+	if got := v.Summary(); got != "check_capacity 62% · human 71% · degraded performance 75%" {
+		t.Fatalf("summary %q", got)
+	}
+	if got := v.RunnersUp(); !strings.HasPrefix(got, "restart_deployment 12%, ") {
+		t.Fatalf("runners-up %q", got)
+	}
+}
+
+func TestLayaErrors(t *testing.T) {
+	for name, tc := range map[string]struct {
+		status int
+		body   string
+	}{
+		"http":    {502, "bad gateway"},
+		"json":    {200, "not json"},
+		"missing": {200, `{"answers":{"urgency":{}}}`},
+	} {
+		srv := layaServer(t, tc.status, tc.body, nil)
+		if _, err := (&Laya{URL: srv.URL}).Classify(context.Background(), timeoutAlert()); err == nil {
+			t.Errorf("%s: want error", name)
+		}
+	}
+}
+
+func TestLayaVerdictInNotifications(t *testing.T) {
+	a := timeoutAlert()
+	a.Laya = &LayaVerdict{Action: "restart_deployment", ActionConfidence: 0.8, ActionProbs: map[string]float64{"restart_deployment": 0.8, "ignore": 0.2}, Human: 0.4, Urgency: "service outage", UrgencyConf: 0.6}
+	h := a.HTML("https://otel.example", telegramMax)
+	if !strings.Contains(h, "🤖 <b>laya</b> <i>(shadow, no action taken)</i>: <code>restart_deployment</code> 80% · human 40% · service outage 60%") ||
+		!strings.Contains(h, "other: ignore 20%") {
+		t.Fatalf("html:\n%s", h)
+	}
+	if txt := a.Text(""); !strings.Contains(txt, "laya (shadow, no action taken): restart_deployment 80%") {
+		t.Fatalf("text:\n%s", txt)
+	}
+	a.Laya = nil
+	if strings.Contains(a.HTML("", telegramMax), "laya") {
+		t.Fatal("no verdict must render no laya line")
+	}
+}
+
+// A laya outage must never cost a page; the alert just goes out without a verdict.
+func TestRunnerAttachesVerdictAndSurvivesLayaFailure(t *testing.T) {
+	for _, tc := range []struct {
+		status int
+		want   bool
+	}{{200, true}, {500, false}} {
+		srv := layaServer(t, tc.status, layaReply, nil)
+		f := &fake{}
+		r := newRunner(nil, f)
+		r.laya = &Laya{URL: srv.URL}
+		r.deliver(context.Background(), timeoutAlert())
+		if len(f.sent) != 1 {
+			t.Fatalf("status %d: sent %d", tc.status, len(f.sent))
+		}
+		if got := f.sent[0].Laya != nil; got != tc.want {
+			t.Fatalf("status %d: verdict attached=%v", tc.status, got)
+		}
+	}
+}
@@ -29,6 +29,9 @@ type Alert struct {
 	Spike    features.Spike  `json:"spike"`
 	Patterns []PatternReport `json:"patterns"`
 	Test     bool            `json:"test,omitempty"`
+	// Laya is the laya model's shadow verdict, when it was asked. Advisory
+	// only: nothing is acted on or suppressed because of it.
+	Laya *LayaVerdict `json:"laya,omitempty"`
 }

 // PatternReport is a spike pattern with its triage verdict, ready to show.
@@ -214,6 +217,9 @@ func (a Alert) Text(baseURL string) string {
 	window := a.Spike.WindowEnd.Sub(a.Spike.WindowStart).Round(time.Minute)
 	fmt.Fprintf(&b, "%s: %s is failing (%d errors in the last %s, normally about %.0f)\n",
 		prefix, a.Spike.Service, a.Spike.ErrorCount, window, a.Spike.BaselineMean)
+	if a.Laya != nil {
+		fmt.Fprintf(&b, "laya (shadow, no action taken): %s\n", a.Laya.Summary())
+	}
 	base := strings.TrimRight(baseURL, "/")
 	for i, p := range a.Patterns {
 		fmt.Fprintf(&b, "\n%d. %s\n", i+1, p.What)
@@ -282,6 +288,13 @@ func (a Alert) HTML(baseURL string, max int) string {
 	if base != "" {
 		foot = fmt.Sprintf("\n<a href=\"%s\">All issues</a>", esc(base+"/issues"))
 	}
+	if a.Laya != nil {
+		head += fmt.Sprintf("🤖 <b>laya</b> <i>(shadow, no action taken)</i>: <code>%s</code> %s · human %s · %s %s\n",
+			esc(a.Laya.Action), pct(a.Laya.ActionConfidence), pct(a.Laya.Human), esc(a.Laya.Urgency), pct(a.Laya.UrgencyConf))
+		if r := a.Laya.RunnersUp(); r != "" {
+			head += "<i>   other: " + esc(r) + "</i>\n"
+		}
+	}
 	var b strings.Builder
 	b.WriteString(head)
 	for i, p := range a.Patterns {
@@ -34,6 +34,9 @@ type Config struct {
 	Muted                     func(issueID string) bool
 	WebhookURL, WebhookSecret string
 	AODURL                    string
+	// LayaURL, when set, asks laya (POST {LayaURL}/predict) for a shadow
+	// verdict on every alert and shows it in the notification.
+	LayaURL string
 }

 func ConfigFromEnv() Config {
@@ -54,6 +57,7 @@ func ConfigFromEnv() Config {
 		WebhookURL:       os.Getenv("OTEL_ALERT_WEBHOOK_URL"),
 		WebhookSecret:    os.Getenv("OTEL_ALERT_WEBHOOK_SECRET"),
 		AODURL:           os.Getenv("OTEL_ALERT_AOD_URL"),
+		LayaURL:          os.Getenv("OTEL_ALERT_LAYA_URL"),
 	}
 }

@@ -106,6 +110,7 @@ type Runner struct {
 	dataSince func(ctx context.Context) time.Time
 	notifiers []Notifier
 	resolver  Resolver // optional: ingress -> service -> deployment for edge errors
+	laya      *Laya    // optional: shadow verdict attached to each alert
 	sup       *features.Suppressor

 	mu        sync.Mutex
@@ -137,6 +142,10 @@ func Start(ctx context.Context, cfg Config, conn driver.Conn, database string, r
 		},
 		dataSince: func(ctx context.Context) time.Time { return DataSince(ctx, conn, database) },
 	}
+	if cfg.LayaURL != "" {
+		r.laya = &Laya{URL: cfg.LayaURL}
+		log.Printf("alerts: laya shadow verdicts from %s", cfg.LayaURL)
+	}
 	names := make([]string, 0, len(notifiers))
 	for _, n := range notifiers {
 		names = append(names, n.Name())
@@ -239,6 +248,7 @@ func (r *Runner) deliver(ctx context.Context, a Alert) {
 	if pr, ok := r.resolver.(PodResolver); ok {
 		a.EnrichPods(ctx, pr)
 	}
+	r.askLaya(ctx, &a)
 	r.mu.Lock()
 	r.recent = append(r.recent, a)
 	if len(r.recent) > maxRecent {
@@ -261,6 +271,23 @@ func (r *Runner) deliver(ctx context.Context, a Alert) {
 	}
 }

+// askLaya attaches laya's shadow verdict. It is bounded so a slow model
+// delays a page by seconds at most, and a failure only costs the verdict.
+func (r *Runner) askLaya(ctx context.Context, a *Alert) {
+	if r.laya == nil {
+		return
+	}
+	lctx, cancel := context.WithTimeout(ctx, 8*time.Second)
+	defer cancel()
+	v, err := r.laya.Classify(lctx, *a)
+	if err != nil {
+		log.Printf("alerts: laya verdict for %s failed: %v", a.ID, err)
+		return
+	}
+	a.Laya = v
+	log.Printf("alerts: laya[shadow] %s: %s", a.ID, v.Summary())
+}
+
 // Status is the JSON served by GET /api/alerts.
 type Status struct {
 	Enabled   bool      `json:"enabled"`
@@ -305,6 +332,7 @@ func (r *Runner) HandleTest(w http.ResponseWriter, req *http.Request) {
 		Example: features.Event{Service: "alert-test", Kind: "span", StatusCode: 500, Route: "/api/example", Message: "this is a test alert"},
 	}}}, now)
 	a.Test = true
+	r.askLaya(req.Context(), &a)
 	results := map[string]string{}
 	for _, n := range r.notifiers {
 		if err := n.Send(req.Context(), a); err != nil {
@@ -15,6 +15,9 @@ spec:
    metadata:
      labels:
        app: otel-debug-receiver
+      annotations:
+        k8s-helper.io/git-url: "https://git.k6n.net/mats/go-otel"
+        k8s-helper.io/git-ssh: "git@git.k6n.net:mats/go-otel.git"
    spec:
      serviceAccountName: otel-debug-receiver
      securityContext:
@@ -138,6 +141,9 @@ spec:
        # Optional Always-On Display session API.
        - name: OTEL_ALERT_AOD_URL
          value: ""
+        # laya shadow verdicts: shown in each notification, never acted on.
+        - name: OTEL_ALERT_LAYA_URL
+          value: "https://laya.k6n.net"
        - name: OTEL_STORE_WORKERS
          value: "4"
        # Optional rollout observation. The watcher is read-only and uses the
@@ -236,18 +242,6 @@ spec:
      - name: sourcemap-dir
        persistentVolumeClaim:
          claimName: otel-sourcemaps
-
---
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: otel-sourcemaps
-  namespace: monitoring
-spec:
-  accessModes: ["ReadWriteOnce"]
-  resources:
-    requests:
-      storage: 1Gi
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -0,0 +1,120 @@
+# OIDC Registration in ecommerce namespace
+apiVersion: v1
+kind: Secret
+metadata:
+  name: flux-dashboard-oidc-client-secret
+  namespace: ecommerce
+type: Opaque
+stringData:
+  secret: "f9f44c88d9fda647e4abbb4479f1116611ecc14624bd67ec"
+---
+apiVersion: auth.k6n.net/v1alpha1
+kind: AuthClient
+metadata:
+  name: flux-dashboard
+  namespace: ecommerce
+  labels:
+    auth.k6n.net/server: auth
+spec:
+  clientID: flux-dashboard
+  displayName: Flux GitOps Dashboard
+  description: Web UI for Flux CD (Weave GitOps)
+  redirectURIs:
+    - https://flux.k6n.net/oauth2/callback
+  scopes:
+    - openid
+    - profile
+    - email
+    - groups
+  secretRef:
+    name: flux-dashboard-oidc-client-secret
+    key: secret
+---
+# Weave GitOps OIDC Secret in flux-system
+apiVersion: v1
+kind: Secret
+metadata:
+  name: oidc-auth
+  namespace: flux-system
+type: Opaque
+stringData:
+  clientID: "flux-dashboard"
+  clientSecret: "f9f44c88d9fda647e4abbb4479f1116611ecc14624bd67ec"
+  issuerURL: "https://users.k6n.net"
+  redirectURL: "https://flux.k6n.net/oauth2/callback"
+  claimUsername: "email"
+  claimGroups: "roles"
+---
+# Weave GitOps Helm OCI Source and HelmRelease
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: weave-gitops
+  namespace: flux-system
+spec:
+  type: oci
+  url: oci://ghcr.io/weaveworks/charts
+  interval: 1h
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: weave-gitops
+  namespace: flux-system
+spec:
+  interval: 10m
+  chart:
+    spec:
+      chart: weave-gitops
+      version: "4.0.36"
+      sourceRef:
+        kind: HelmRepository
+        name: weave-gitops
+  values:
+    envVars:
+      - name: KUBERNETES_SERVICE_HOST
+        value: "10.10.1.102"
+      - name: KUBERNETES_SERVICE_PORT
+        value: "6443"
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: kubernetes.io/hostname
+                  operator: NotIn
+                  values:
+                    - serber
+    # Use existing oidc-auth secret created above
+    oidcSecret:
+      create: false
+    adminUser:
+      create: false
+    ingress:
+      enabled: true
+      className: nginx
+      annotations:
+        cert-manager.io/cluster-issuer: letsencrypt-prod
+      hosts:
+        - host: flux.k6n.net
+          paths:
+            - path: /
+              pathType: ImplementationSpecific
+      tls:
+        - hosts:
+            - flux.k6n.net
+          secretName: flux-k6n-tls
+---
+# ClusterRoleBinding so authenticated users can view/manage GitOps resources
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: weave-gitops-admin
+subjects:
+  - kind: Group
+    name: admin
+    apiGroup: rbac.authorization.k8s.io
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io
@@ -0,0 +1,28 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: GitRepository
+metadata:
+  name: go-otel
+  namespace: flux-system
+spec:
+  interval: 1m
+  url: ssh://git@git.k6n.net/mats/go-otel.git
+  ref:
+    branch: main
+  secretRef:
+    name: go-otel-auth
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: go-otel
+  namespace: flux-system
+spec:
+  interval: 5m
+  targetNamespace: monitoring
+  sourceRef:
+    kind: GitRepository
+    name: go-otel
+  path: "./k8s"
+  prune: true
+  wait: true
+  timeout: 3m
@@ -0,0 +1,173 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ingress-catchall
+  namespace: ingress-nginx
+data:
+  nginx.conf: |
+    worker_processes  1;
+    pid /tmp/nginx.pid;
+
+    events {
+      worker_connections  1024;
+    }
+
+    http {
+      include       /etc/nginx/mime.types;
+      default_type  application/octet-stream;
+      sendfile      on;
+      keepalive_timeout  65;
+      client_body_temp_path /tmp/client_temp;
+      proxy_temp_path /tmp/proxy_temp;
+      fastcgi_temp_path /tmp/fastcgi_temp;
+      uwsgi_temp_path /tmp/uwsgi_temp;
+      scgi_temp_path /tmp/scgi_temp;
+
+      server {
+        listen 8080;
+      server_name _;
+      root /usr/share/nginx/html;
+      error_page 403 =404 /index.html;
+
+      # Only GET is intentionally accepted by this catch-all.
+      # The "if" is safe here because it only performs a return.
+        # Serve robots.txt directly with Disallow: /
+        location = /robots.txt {
+          default_type text/plain;
+          return 200 "User-agent: *\nDisallow: /\n";
+        }
+
+        # Deliberately trigger an internal 403 so the static error page is
+        # returned with the externally visible status still set to 404.
+        location / {
+          if ($request_method != GET) {
+            add_header Allow GET always;
+            return 405;
+          }
+
+          deny all;
+        }
+
+        # Keep the error document internal so /index.html also follows the
+        # catch-all policy instead of becoming a successful public endpoint.
+        location = /index.html {
+          internal;
+          allow all;
+        }
+      }
+    }
+  index.html: |
+    <!doctype html>
+    <html lang="en">
+    <head>
+      <meta charset="utf-8">
+      <meta name="viewport" content="width=device-width, initial-scale=1">
+      <title>Not found</title>
+      <style>
+        :root { color-scheme: dark; }
+        * { box-sizing: border-box; }
+        body {
+          min-height: 100vh;
+          margin: 0;
+          display: grid;
+          place-items: center;
+          padding: 2rem;
+          color: #e8ecf2;
+          background: #0d1014;
+          font: 16px/1.5 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
+        }
+        main { width: min(100%, 42rem); }
+        .code {
+          margin: 0 0 .75rem;
+          color: #7b8cff;
+          font: 700 clamp(4rem, 14vw, 8rem)/.9 ui-monospace, SFMono-Regular, Menlo, monospace;
+          letter-spacing: -.08em;
+        }
+        h1 { margin: 0 0 .75rem; font-size: clamp(1.5rem, 4vw, 2.25rem); }
+        p { margin: 0; max-width: 34rem; color: #98a3b2; }
+        code { color: #e8ecf2; }
+      </style>
+    </head>
+    <body>
+      <main>
+        <p class="code">404</p>
+        <h1>There is nothing here.</h1>
+        <p>The requested resource does not exist on this ingress endpoint.</p>
+      </main>
+    </body>
+    </html>
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ingress-catchall
+  namespace: ingress-nginx
+  labels:
+    app.kubernetes.io/name: ingress-catchall
+    app.kubernetes.io/part-of: ingress-nginx
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ingress-catchall
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: ingress-catchall
+        app.kubernetes.io/part-of: ingress-nginx
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: nginx
+          image: nginx:1.27-alpine
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 8080
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            runAsNonRoot: true
+            runAsUser: 101
+          volumeMounts:
+            - name: config
+              mountPath: /etc/nginx/nginx.conf
+              subPath: nginx.conf
+              readOnly: true
+            - name: config
+              mountPath: /usr/share/nginx/html/index.html
+              subPath: index.html
+              readOnly: true
+          readinessProbe:
+            tcpSocket:
+              port: http
+            periodSeconds: 5
+          livenessProbe:
+            tcpSocket:
+              port: http
+            initialDelaySeconds: 5
+            periodSeconds: 10
+      volumes:
+        - name: config
+          configMap:
+            name: ingress-catchall
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ingress-catchall
+  namespace: ingress-nginx
+  labels:
+    app.kubernetes.io/name: ingress-catchall
+    app.kubernetes.io/part-of: ingress-nginx
+spec:
+  selector:
+    app.kubernetes.io/name: ingress-catchall
+  ports:
+    - name: http
+      port: 80
+      targetPort: http
@@ -85,6 +85,10 @@ metadata:
  name: otel-debug-ingress
  namespace: monitoring
  annotations:
+    k8s-helper.io/name: "Go OTel"
+    k8s-helper.io/description: "OpenTelemetry debugging dashboard"
+    k8s-helper.io/group: "Observability"
+    k8s-helper.io/icon: "activity"
    cert-manager.io/cluster-issuer: letsencrypt-prod
    # OIDC protection via the platform auth service (users.k6n.net):
    # /oauth2/auth resolves the session cookie / bearer token and answers
@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - deployment.yaml
+  - service.yaml
+  - ingress.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: otel-sourcemaps
+  namespace: monitoring
+spec:
+  accessModes: ["ReadWriteOnce"]
+  resources:
+    requests:
+      storage: 1Gi
				`@@ -0,0 +1,2 @@`
				{"enabled":true,"dryRun":false,"channels":["telegram"],"lastRun":"2026-09-25T18:56:43.3470721Z","alerts":[{"id":"tlxng1-nginx","at":"2026-09-25T18:31:13.347300133Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:26:13.347300133Z","windowEnd":"2026-09-25T18:31:13.347300133Z","errorCount":69,"baselineMean":14.03448275862069,"excess":4.9655172413793105,"patterns":[{"signature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","count":4,"baselineMean":0.034482758620689655,"excess":3.9655172413793105,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:31:11.346792393Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"0751de83a2f0ce211f2b7ab1a64d8aba","PatternSignature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"auth.k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"34.177.93.136","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"fb485cd240e1dab0261891dc14170112bbc37c90189f276b45935afdedf534cb","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:31:06.252746363Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d05925e26f581fcba99d165ea5928114","PatternSignature":"fb485cd240e1dab0261891dc14170112bbc37c90189f276b45935afdedf534cb","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/oauth2/auth","Host":"users.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"auth.k6n.net","path":"/","services":["nginx"],"routes":[{"ingress":"dev/zitadel-ingress","host":"auth.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"zitadel-dev","port":"8080","deployments":["zitadel-dev"]}],"pods":[{"namespace":"dev","deployment":"zitadel-dev","found":false}],"rule":"status-5xx-upstream","new":false,"count":4,"baseline":0.034482758620689655,"clients":1,"what":"HTTP 503 \| /"},{"category":"client_credential_issue","priority":"low","status":401,"host":"users.k6n.net","path":"/oauth2/auth","services":["nginx"],"routes":[{"ingress":"ecommerce/ecommerce-auth","host":"users.k6n.net","path":"/","pathType":"Prefix","namespace":"ecommerce","service":"auth","port":"8080","deployments":["auth"]}],"pods":[{"namespace":"ecommerce","deployment":"auth","pod":"auth-8f497c689-qfpzb","lines":["2026/09/25 18:13:06 oidc: device grant JHMJ-C23E started for client dev-console","2026/09/25 18:13:06 oidc: device grant EQ5R-UWRW approved by mats@tornberg.me","2026/09/25 18:23:19 oidc: device grant RHMN-6257 started for client dev-console"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 \| /oauth2/auth"}]},{"id":"tlxn9d-nginx","at":"2026-09-25T18:27:13.347226389Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:22:13.347226389Z","windowEnd":"2026-09-25T18:27:13.347226389Z","errorCount":64,"baselineMean":13.931034482758621,"excess":4,"patterns":[{"signature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","count":4,"baselineMean":0,"excess":4,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:26:26.152736211Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d1de7de09ad342e7b0670cb78cac1430","PatternSignature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"cc.k6n.net","StatusCode":405,"ExceptionType":"","ClientIP":"35.189.233.235","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"needs_review","priority":"low","status":405,"host":"cc.k6n.net","path":"/","services":["nginx"],"rule":"no-rule","new":false,"count":4,"baseline":0,"clients":1,"what":"HTTP 405 \| /"}]},{"id":"tlxn61-nginx","at":"2026-09-25T18:25:13.346661147Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:20:13.346661147Z","windowEnd":"2026-09-25T18:25:13.346661147Z","errorCount":60,"baselineMean":14.03448275862069,"excess":1,"patterns":[{"signature":"ea444cb83ff471aad7f3fab8a5b36dddb25f333f7dce2eb4ff27f6e9bb3969f7","count":1,"baselineMean":0,"excess":1,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:24:36.217473837Z","Service":"nginx","Release":"","Kind":"span","SourcLine truncated