From e046bd86d1198711dad9780c1dcf5b12872623ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats=20T=C3=B6rnberg?= Date: Sun, 27 Sep 2026 16:45:10 +0200 Subject: [PATCH] eval --- .gitea/workflows/build.yaml | 60 +- alerts.json | 2 + cmd/laya-eval/main.go | 144 ++ cmd/laya-eval/scenarios.json | 2098 ++++++++++++++++++++++++++ gitops/README.md | 29 + gitops/deployment.yaml | 14 + gitops/gitsource.yaml | 16 + internal/alerts/laya.go | 180 +++ internal/alerts/laya_test.go | 123 ++ internal/alerts/notify.go | 13 + internal/alerts/runner.go | 28 + k8s/deployment.yaml | 18 +- k8s/flux/gitops-dashboard.yaml | 120 ++ k8s/flux/sync-go-otel.yaml | 28 + k8s/ingress-catchall.yaml | 173 +++ k8s/ingress.yaml | 4 + k8s/kustomization.yaml | 6 + k8s/otel-sourcemaps-pvc.example.yaml | 10 + 18 files changed, 3020 insertions(+), 46 deletions(-) create mode 100644 alerts.json create mode 100644 cmd/laya-eval/main.go create mode 100644 cmd/laya-eval/scenarios.json create mode 100644 gitops/README.md create mode 100644 gitops/deployment.yaml create mode 100644 gitops/gitsource.yaml create mode 100644 internal/alerts/laya.go create mode 100644 internal/alerts/laya_test.go create mode 100644 k8s/flux/gitops-dashboard.yaml create mode 100644 k8s/flux/sync-go-otel.yaml create mode 100644 k8s/ingress-catchall.yaml create mode 100644 k8s/kustomization.yaml create mode 100644 k8s/otel-sourcemaps-pvc.example.yaml diff --git a/.gitea/workflows/build.yaml b/.gitea/workflows/build.yaml index 90c41fd..1a5d783 100644 --- a/.gitea/workflows/build.yaml +++ b/.gitea/workflows/build.yaml @@ -1,6 +1,9 @@ name: Build and Publish run-name: ${{ gitea.actor }} build 🚀 -on: [push] +on: + push: + branches: [main] + workflow_dispatch: jobs: Metadata: @@ -26,7 +29,7 @@ jobs: echo "version=$VERSION" >> $GITHUB_OUTPUT echo "build_date=$BUILD_DATE" >> $GITHUB_OUTPUT - BuildAndDeployAmd64: + BuildAndPushAmd64: needs: Metadata runs-on: amd64 steps: @@ -39,15 +42,32 @@ jobs: --build-arg GIT_COMMIT=${{ needs.Metadata.outputs.git_commit }} \ --build-arg BUILD_DATE=${{ needs.Metadata.outputs.build_date }} \ --progress=plain \ + -t matst80/otel-debug:${{ needs.Metadata.outputs.version }} \ -t matst80/otel-debug:latest \ . - - name: Push amd64 images + - name: Push images run: | + docker push matst80/otel-debug:${{ needs.Metadata.outputs.version }} docker push matst80/otel-debug:latest - - name: Rollout amd64 otel deployment + - name: Reconcile after image is published + env: + GITOPS_WEBHOOK_SECRET: ${{ secrets.GITOPS_WEBHOOK_SECRET }} run: | - kubectl rollout restart deployment/otel-debug-receiver -n monitoring - kubectl rollout status deployment/otel-debug-receiver -n monitoring + if [ -z "$GITOPS_WEBHOOK_SECRET" ]; then + echo "GITOPS_WEBHOOK_SECRET must match the k8s-helper Gitea webhook secret" >&2 + exit 1 + fi + payload="$(mktemp)" + trap 'rm -f "$payload"' EXIT + printf '{"ref":"%s","after":"%s","repository":{"clone_url":"https://git.k6n.net/mats/go-otel.git","ssh_url":"git@git.k6n.net:mats/go-otel.git"}}' \ + "$GITHUB_REF" "$GITHUB_SHA" > "$payload" + signature="$(openssl dgst -sha256 -hmac "$GITOPS_WEBHOOK_SECRET" "$payload" | awk '{print $NF}')" + curl --fail --silent --show-error \ + -X POST \ + -H "Content-Type: application/json" \ + -H "X-Gitea-Signature: $signature" \ + --data-binary "@$payload" \ + https://ui.k6n.net/api/gitops/webhooks/gitea - name: Upload source maps env: RELEASE: ${{ needs.Metadata.outputs.version }} @@ -69,31 +89,3 @@ jobs: done < <(find "$maps_dir" -type f -name '*.js.map' -print0) rm -rf "$maps_dir" [ "$found" = 1 ] || { echo "no source maps found in UI build" >&2; exit 1; } - # - name: Rollout amd64 deployment (pin to version) - # run: | - # kubectl set image deployment/cart-actor-x86 -n cart cart-actor-amd64=registry.knatofs.se/go-otel-debug-amd64:${{ needs.Metadata.outputs.version }} - # # kubectl rollout status deployment/cart-actor-x86 -n cart - - # BuildAndDeployArm64: - # needs: Metadata - # runs-on: arm64 - # steps: - # - uses: actions/checkout@v4 - # - name: Build arm64 image - # run: | - # docker build \ - # --build-arg VERSION=${{ needs.Metadata.outputs.version }} \ - # --build-arg GIT_COMMIT=${{ needs.Metadata.outputs.git_commit }} \ - # --build-arg BUILD_DATE=${{ needs.Metadata.outputs.build_date }} \ - # --progress=plain \ - # -t registry.knatofs.se/go-otel-debug:latest \ - # -t registry.knatofs.se/go-otel-debug:${{ needs.Metadata.outputs.version }} \ - # . - # - name: Push arm64 images - # run: | - # docker push registry.knatofs.se/go-otel-debug:latest - # docker push registry.knatofs.se/go-otel-debug:${{ needs.Metadata.outputs.version }} - # - name: Rollout arm64 deployment (pin to version) - # run: | - # kubectl set image deployment/cart-actor-arm64 -n cart cart-actor-arm64=registry.knatofs.se/go-otel-debug:${{ needs.Metadata.outputs.version }} - # # kubectl rollout status deployment/cart-actor-arm64 -n cart diff --git a/alerts.json b/alerts.json new file mode 100644 index 0000000..3fc3cad --- /dev/null +++ b/alerts.json @@ -0,0 +1,2 @@ +{"enabled":true,"dryRun":false,"channels":["telegram"],"lastRun":"2026-09-25T18:56:43.3470721Z","alerts":[{"id":"tlxng1-nginx","at":"2026-09-25T18:31:13.347300133Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:26:13.347300133Z","windowEnd":"2026-09-25T18:31:13.347300133Z","errorCount":69,"baselineMean":14.03448275862069,"excess":4.9655172413793105,"patterns":[{"signature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","count":4,"baselineMean":0.034482758620689655,"excess":3.9655172413793105,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:31:11.346792393Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"0751de83a2f0ce211f2b7ab1a64d8aba","PatternSignature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"auth.k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"34.177.93.136","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"fb485cd240e1dab0261891dc14170112bbc37c90189f276b45935afdedf534cb","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:31:06.252746363Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d05925e26f581fcba99d165ea5928114","PatternSignature":"fb485cd240e1dab0261891dc14170112bbc37c90189f276b45935afdedf534cb","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/oauth2/auth","Host":"users.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"auth.k6n.net","path":"/","services":["nginx"],"routes":[{"ingress":"dev/zitadel-ingress","host":"auth.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"zitadel-dev","port":"8080","deployments":["zitadel-dev"]}],"pods":[{"namespace":"dev","deployment":"zitadel-dev","found":false}],"rule":"status-5xx-upstream","new":false,"count":4,"baseline":0.034482758620689655,"clients":1,"what":"HTTP 503 | /"},{"category":"client_credential_issue","priority":"low","status":401,"host":"users.k6n.net","path":"/oauth2/auth","services":["nginx"],"routes":[{"ingress":"ecommerce/ecommerce-auth","host":"users.k6n.net","path":"/","pathType":"Prefix","namespace":"ecommerce","service":"auth","port":"8080","deployments":["auth"]}],"pods":[{"namespace":"ecommerce","deployment":"auth","pod":"auth-8f497c689-qfpzb","lines":["2026/09/25 18:13:06 oidc: device grant JHMJ-C23E started for client dev-console","2026/09/25 18:13:06 oidc: device grant EQ5R-UWRW approved by mats@tornberg.me","2026/09/25 18:23:19 oidc: device grant RHMN-6257 started for client dev-console"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /oauth2/auth"}]},{"id":"tlxn9d-nginx","at":"2026-09-25T18:27:13.347226389Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:22:13.347226389Z","windowEnd":"2026-09-25T18:27:13.347226389Z","errorCount":64,"baselineMean":13.931034482758621,"excess":4,"patterns":[{"signature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","count":4,"baselineMean":0,"excess":4,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:26:26.152736211Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d1de7de09ad342e7b0670cb78cac1430","PatternSignature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"cc.k6n.net","StatusCode":405,"ExceptionType":"","ClientIP":"35.189.233.235","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"needs_review","priority":"low","status":405,"host":"cc.k6n.net","path":"/","services":["nginx"],"rule":"no-rule","new":false,"count":4,"baseline":0,"clients":1,"what":"HTTP 405 | /"}]},{"id":"tlxn61-nginx","at":"2026-09-25T18:25:13.346661147Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:20:13.346661147Z","windowEnd":"2026-09-25T18:25:13.346661147Z","errorCount":60,"baselineMean":14.03448275862069,"excess":1,"patterns":[{"signature":"ea444cb83ff471aad7f3fab8a5b36dddb25f333f7dce2eb4ff27f6e9bb3969f7","count":1,"baselineMean":0,"excess":1,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:24:36.217473837Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"b52e540166ef48a42f0c0c1c504a6d49","PatternSignature":"ea444cb83ff471aad7f3fab8a5b36dddb25f333f7dce2eb4ff27f6e9bb3969f7","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/robots.txt","Host":"bot.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"74.7.241.187","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":502,"host":"bot.k6n.net","path":"/robots.txt","services":["nginx"],"routes":[{"ingress":"home/telegram-ai-bot-ingress","host":"bot.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"telegram-ai-bot-service","port":"8092"}],"rule":"status-5xx-upstream","new":true,"count":1,"baseline":0,"clients":1,"uaClass":"ai_crawler","what":"HTTP 502 | /robots.txt"}]},{"id":"tlxmxp-nginx","at":"2026-09-25T18:20:13.346876969Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:15:13.346876969Z","windowEnd":"2026-09-25T18:20:13.346876969Z","errorCount":63,"baselineMean":13.96551724137931,"excess":2,"patterns":[{"signature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","count":2,"baselineMean":0,"excess":2,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:19:16.012002529Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"073248aba0d908063aeff887fbd8061e","PatternSignature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"94-255-139-35.cust.bredband2.com","StatusCode":405,"ExceptionType":"","ClientIP":"195.178.110.247","UAClass":"unknown","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"needs_review","priority":"low","status":405,"host":"94-255-139-35.cust.bredband2.com","path":"/","services":["nginx"],"rule":"no-rule","new":false,"count":2,"baseline":0,"clients":1,"what":"HTTP 405 | /"}]},{"id":"tlxmsp-nginx","at":"2026-09-25T18:17:13.346624365Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:12:13.346624365Z","windowEnd":"2026-09-25T18:17:13.346624365Z","errorCount":60,"baselineMean":13.793103448275861,"excess":59,"patterns":[{"signature":"55c1fa26cf30fc45c2011897bca61d2a94dbc04444d973a94a4b3bc668527ccf","count":57,"baselineMean":0,"excess":57,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:17:11.164791167Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"1e1afe400019c5197996dab1603c2d3d","PatternSignature":"55c1fa26cf30fc45c2011897bca61d2a94dbc04444d973a94a4b3bc668527ccf","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/auth/device/token","Host":"users.k6n.net","StatusCode":400,"ExceptionType":"","ClientIP":"","UAClass":"library","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"1f3b735bd556874994fd713d10b6d65e38e4a8f13fb5b644f383cfbf020ee4d9","count":2,"baselineMean":0,"excess":2,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:13:10.98074347Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"f14f9846a455d0a2ada6516226285a81","PatternSignature":"1f3b735bd556874994fd713d10b6d65e38e4a8f13fb5b644f383cfbf020ee4d9","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/favicon.ico","Host":"users.k6n.net","StatusCode":405,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"needs_review","priority":"low","status":400,"host":"users.k6n.net","path":"/api/auth/device/token","services":["nginx"],"routes":[{"ingress":"ecommerce/ecommerce-auth","host":"users.k6n.net","path":"/","pathType":"Prefix","namespace":"ecommerce","service":"auth","port":"8080","deployments":["auth"]}],"pods":[{"namespace":"ecommerce","deployment":"auth","pod":"auth-8f497c689-qfpzb","lines":["2026/09/25 18:11:59 oidc: device grant EQ5R-UWRW started for client dev-console","2026/09/25 18:13:06 oidc: device grant JHMJ-C23E started for client dev-console","2026/09/25 18:13:06 oidc: device grant EQ5R-UWRW approved by mats@tornberg.me"],"found":true}],"rule":"no-rule","new":false,"count":57,"baseline":0,"uaClass":"library","what":"HTTP 400 | /api/auth/device/token"},{"category":"needs_review","priority":"low","status":405,"host":"users.k6n.net","path":"/favicon.ico","services":["nginx"],"routes":[{"ingress":"ecommerce/ecommerce-auth","host":"users.k6n.net","path":"/","pathType":"Prefix","namespace":"ecommerce","service":"auth","port":"8080","deployments":["auth"]}],"pods":[{"namespace":"ecommerce","deployment":"auth","pod":"auth-8f497c689-qfpzb","lines":["2026/09/25 18:11:59 oidc: device grant EQ5R-UWRW started for client dev-console","2026/09/25 18:13:06 oidc: device grant JHMJ-C23E started for client dev-console","2026/09/25 18:13:06 oidc: device grant EQ5R-UWRW approved by mats@tornberg.me"],"found":true}],"rule":"no-rule","new":false,"count":2,"baseline":0,"what":"HTTP 405 | /favicon.ico"}]},{"id":"tlxmh1-eventsourced-orchestrator","at":"2026-09-25T18:10:13.34664129Z","spike":{"service":"eventsourced-orchestrator","windowStart":"2026-09-25T18:05:13.34664129Z","windowEnd":"2026-09-25T18:10:13.34664129Z","errorCount":16,"baselineMean":19.93103448275862,"excess":16,"patterns":[{"signature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","count":16,"baselineMean":0,"excess":16,"novel":false,"services":["eventsourced-orchestrator"],"example":{"Timestamp":"2026-09-25T18:10:08.793065336Z","Service":"eventsourced-orchestrator","Release":"","Kind":"log","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"","PatternSignature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","IssueFingerprint":"b57379b9dd3a6a9d","IsRetry":false,"IsTimeout":true,"IsCancellation":false,"Message":"codesearch upload failed","SpanName":"","Route":"","Host":"","StatusCode":0,"ExceptionType":"context.deadlineExceededError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":false,"HasUpstream":false,"TraceServices":null,"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","issueId":"b57379b9dd3a6a9d","services":["eventsourced-orchestrator"],"rule":"timeout-flag","new":false,"count":16,"baseline":0,"what":"context.deadlineExceededError | codesearch upload failed"}]},{"id":"tlxmdp-eventsourced-worker-code","at":"2026-09-25T18:08:13.346954479Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T18:03:13.346954479Z","windowEnd":"2026-09-25T18:08:13.346954479Z","errorCount":36,"baselineMean":0.13793103448275862,"excess":36,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":29,"baselineMean":0,"excess":29,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T18:08:12.308751715Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"34f7c7c1d9c07bb48b3241454d10a7bf","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}},{"signature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","count":7,"baselineMean":0,"excess":7,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T18:07:50.177441418Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"72e0cbf2c415b129eab8f4aadca42c9f","PatternSignature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","IssueFingerprint":"b40901e8fbfcf66e","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: eof","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":29,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"},{"category":"dependency_unreachable","priority":"high","issueId":"b40901e8fbfcf66e","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":7,"baseline":0,"what":"worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"}]},{"id":"tlxjp1-eventsourced-worker-code","at":"2026-09-25T17:10:13.347080773Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T17:05:13.347080773Z","windowEnd":"2026-09-25T17:10:13.347080773Z","errorCount":66,"baselineMean":0.4482758620689655,"excess":60,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":60,"baselineMean":0,"excess":60,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T17:10:12.613732024Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"a9d858213f84b6dcf310c4701d1c0a39","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":60,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"}]},{"id":"tlxitd-eventsourced-orchestrator","at":"2026-09-25T16:51:13.346883436Z","spike":{"service":"eventsourced-orchestrator","windowStart":"2026-09-25T16:46:13.346883436Z","windowEnd":"2026-09-25T16:51:13.346883436Z","errorCount":15,"baselineMean":8.310344827586206,"excess":15,"patterns":[{"signature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","count":15,"baselineMean":0,"excess":15,"novel":false,"services":["eventsourced-orchestrator"],"example":{"Timestamp":"2026-09-25T16:50:38.820074629Z","Service":"eventsourced-orchestrator","Release":"","Kind":"log","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"","PatternSignature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","IssueFingerprint":"b57379b9dd3a6a9d","IsRetry":false,"IsTimeout":true,"IsCancellation":false,"Message":"codesearch upload failed","SpanName":"","Route":"","Host":"","StatusCode":0,"ExceptionType":"context.deadlineExceededError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":false,"HasUpstream":false,"TraceServices":null,"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","issueId":"b57379b9dd3a6a9d","services":["eventsourced-orchestrator"],"rule":"timeout-flag","new":false,"count":15,"baseline":0,"what":"context.deadlineExceededError | codesearch upload failed"}]},{"id":"tlxirp-nginx","at":"2026-09-25T16:50:13.347044487Z","spike":{"service":"nginx","windowStart":"2026-09-25T16:45:13.347044487Z","windowEnd":"2026-09-25T16:50:13.347044487Z","errorCount":62,"baselineMean":18.93103448275862,"excess":41.96551724137931,"patterns":[{"signature":"507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d","count":17,"baselineMean":0,"excess":17,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T16:50:05.564200851Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"51f8278d03367c21a21f76835ec204b2","PatternSignature":"507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai-worker/manifests/latest","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}},{"signature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","count":14,"baselineMean":0,"excess":14,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T16:48:10.547871686Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"6664346e0c22d3a9396460163f0b0d6b","PatternSignature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"library","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}},{"signature":"ec8f49cca245ee954c6f7a7839235c13a8cfaabc1f0bcac8ecd29c88a5533891","count":5,"baselineMean":0.034482758620689655,"excess":4.9655172413793105,"novel":false,"clients":1,"services":["eventsourced-orchestrator","nginx"],"example":{"Timestamp":"2026-09-25T16:49:50.586320711Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"2b5d0ed3c205922143afa2bc0c5b0b8e","PatternSignature":"ec8f49cca245ee954c6f7a7839235c13a8cfaabc1f0bcac8ecd29c88a5533891","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/auth/me","Host":"event-agent.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"170.62.100.17","UAClass":"browser","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["eventsourced-orchestrator","nginx"],"BadActor":""}},{"signature":"10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b","count":3,"baselineMean":0,"excess":3,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T16:49:50.586320711Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"44416b3fb21e6f57b9bba5ac0eccd68b","PatternSignature":"10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b","IssueFingerprint":"","IsRetry":false,"IsTimeout":true,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/stream","Host":"event-agent.k6n.net","StatusCode":504,"ExceptionType":"","ClientIP":"170.62.100.17","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"f524391f412efceb1222fd1ccb3c5fcb673ace4787f214acaf300dea6cc88115","count":3,"baselineMean":0,"excess":3,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T16:48:10.577418006Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"b870baab8dd2de51be3f9da1663d239e","PatternSignature":"f524391f412efceb1222fd1ccb3c5fcb673ace4787f214acaf300dea6cc88115","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/auth/me","Host":"event-agent.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"170.62.100.17","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","status":504,"host":"event-agent.k6n.net","path":"/api/stream","services":["nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-5865685999-xtxdz","lines":["10.244.9.1 - - [25/Sep/2026:16:50:20 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:16:50:30 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:16:50:40 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"timeout-flag","new":false,"count":3,"baseline":0,"clients":1,"what":"HTTP 504 | /api/stream"},{"category":"dependency_unreachable","priority":"high","status":502,"host":"event-agent.k6n.net","path":"/auth/me","services":["nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-5865685999-xtxdz","lines":["10.244.9.1 - - [25/Sep/2026:16:50:20 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:16:50:30 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:16:50:40 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"status-5xx-upstream","new":false,"count":3,"baseline":0,"clients":1,"what":"HTTP 502 | /auth/me"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai-worker/manifests/latest","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:16:50:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T16:50:39.183971552Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"85d74dfb059f5fbff7018c85c990e2bc\\\",\\\"SpanID\\\":\\\"dfd025e674cceb15\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T16:50:38.934252898Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:38.934346788Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":54180}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T16:50:39.184168228Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"b389a9064776af9f4a23cc817ee4231f\\\",\\\"SpanID\\\":\\\"16fa1d622c943f61\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T16:50:39.102298404Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:39.105804949Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":17,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai-worker/manifests/latest"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:16:50:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T16:50:39.183971552Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"85d74dfb059f5fbff7018c85c990e2bc\\\",\\\"SpanID\\\":\\\"dfd025e674cceb15\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T16:50:38.934252898Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:38.934346788Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":54180}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T16:50:39.184168228Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"b389a9064776af9f4a23cc817ee4231f\\\",\\\"SpanID\\\":\\\"16fa1d622c943f61\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T16:50:39.102298404Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:39.105804949Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":14,"baseline":0,"uaClass":"library","what":"HTTP 401 | /v2"},{"category":"client_credential_issue","priority":"low","status":401,"host":"event-agent.k6n.net","path":"/auth/me","services":["eventsourced-orchestrator","nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-5865685999-xtxdz","lines":["10.244.9.1 - - [25/Sep/2026:16:50:20 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:16:50:30 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:16:50:40 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"status-401","new":false,"count":5,"baseline":0.034482758620689655,"clients":1,"what":"HTTP 401 | /auth/me"}]},{"id":"tlxiq1-eventsourced-worker-code","at":"2026-09-25T16:49:13.347152978Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T16:44:13.347152978Z","windowEnd":"2026-09-25T16:49:13.347152978Z","errorCount":35,"baselineMean":0.6206896551724138,"excess":35,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":28,"baselineMean":0,"excess":28,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T16:48:57.813919594Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"ae2f322da11ef7dd36f14ffc6e391321","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}},{"signature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","count":7,"baselineMean":0,"excess":7,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T16:48:10.959364475Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"b7adf673cf07369996548673a85a9060","PatternSignature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","IssueFingerprint":"b40901e8fbfcf66e","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: eof","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":28,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"},{"category":"dependency_unreachable","priority":"high","issueId":"b40901e8fbfcf66e","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":7,"baseline":0,"what":"worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"}]},{"id":"tlxgbd-nginx","at":"2026-09-25T15:57:13.347060965Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:52:13.347060965Z","windowEnd":"2026-09-25T15:57:13.347060965Z","errorCount":72,"baselineMean":18.413793103448278,"excess":31.448275862068968,"patterns":[{"signature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","count":40,"baselineMean":8.551724137931034,"excess":31.448275862068968,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:57:06.60535739Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"7654e99020c8f9de172e1bf4cbd65aa3","PatternSignature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/metrics/query","Host":"ui.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":502,"host":"ui.k6n.net","path":"/api/metrics/query","services":["nginx"],"routes":[{"ingress":"dev/k8s-helper-ingress","host":"ui.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"k8s-helper","port":"80","deployments":["k8s-helper"]}],"pods":[{"namespace":"dev","deployment":"k8s-helper","pod":"k8s-helper-5665cd69c9-kjd2t","lines":["I0925 15:56:34.658960 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 15:57:20.162724 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 15:57:24.894618 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\""],"found":true}],"rule":"status-5xx-upstream","new":false,"count":40,"baseline":8.551724137931034,"what":"HTTP 502 | /api/metrics/query"}]},{"id":"tlxg6d-nginx","at":"2026-09-25T15:54:13.346717682Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:49:13.346717682Z","windowEnd":"2026-09-25T15:54:13.346717682Z","errorCount":74,"baselineMean":18.448275862068964,"excess":12,"patterns":[{"signature":"0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e","count":4,"baselineMean":0,"excess":4,"novel":false,"clients":2,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:53:46.266195024Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"ecdf390536970c3339612fb2a201df51","PatternSignature":"0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/designer/v1/file-content","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"104.23.160.37","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6","count":4,"baselineMean":0,"excess":4,"novel":false,"clients":2,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:53:46.266195024Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"744692761138c480f27fac58b3aa1fda","PatternSignature":"b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/templates/preview","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"104.23.160.37","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab","count":2,"baselineMean":0,"excess":2,"novel":false,"clients":2,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:53:46.266195024Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"7ffa5f428b4de59811dd86ded001a33c","PatternSignature":"004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/fs/exec","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"104.23.160.37","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"7281b16d4395d124b0fac7a82520ee2b9e179df689e8f4eb6d515a6b567aa256","count":2,"baselineMean":0,"excess":2,"novel":false,"clients":2,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:53:46.266195024Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"942867286e5ba5cac5ec2102cd059106","PatternSignature":"7281b16d4395d124b0fac7a82520ee2b9e179df689e8f4eb6d515a6b567aa256","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/v1/build_public_tmp/{id}/flow","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"104.23.160.37","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/designer/v1/file-content","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":4,"baseline":0,"clients":2,"uaClass":"ai_crawler","what":"HTTP 503 | /api/designer/v1/file-content"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/templates/preview","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":4,"baseline":0,"clients":2,"uaClass":"ai_crawler","what":"HTTP 503 | /api/templates/preview"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/fs/exec","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":2,"baseline":0,"clients":2,"uaClass":"ai_crawler","what":"HTTP 503 | /api/fs/exec"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/v1/build_public_tmp/{id}/flow","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":2,"baseline":0,"clients":2,"uaClass":"ai_crawler","what":"HTTP 503 | /api/v1/build_public_tmp/{id}/flow"}]},{"id":"tlxg31-nginx","at":"2026-09-25T15:52:13.34677222Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:47:13.34677222Z","windowEnd":"2026-09-25T15:52:13.34677222Z","errorCount":44,"baselineMean":18.413793103448278,"excess":1,"patterns":[{"signature":"4f0026d193a2d63deed6f864410365d22dc0a941000e445e1bd3340a553df1c6","count":1,"baselineMean":0,"excess":1,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:52:10.518612164Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"ecf6036348e1d53bf4a441e861f14ca3","PatternSignature":"4f0026d193a2d63deed6f864410365d22dc0a941000e445e1bd3340a553df1c6","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/auth/session","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"172.69.130.15","UAClass":"other_bot","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/auth/session","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":1,"baseline":0,"clients":1,"uaClass":"other_bot","what":"HTTP 503 | /api/auth/session"}]},{"id":"tlxfkp-nginx","at":"2026-09-25T15:41:13.346872997Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:36:13.346872997Z","windowEnd":"2026-09-25T15:41:13.346872997Z","errorCount":73,"baselineMean":16.896551724137932,"excess":5,"patterns":[{"signature":"cfb79f6686e37386a47e979141249f6e98eb12a891beb2cc2f9c6ef4a03e546a","count":5,"baselineMean":0,"excess":5,"novel":true,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:40:16.478551828Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"0adee7ffe614204eab8ee2af9893a095","PatternSignature":"cfb79f6686e37386a47e979141249f6e98eb12a891beb2cc2f9c6ef4a03e546a","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/laya-api/blobs/{id}","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/laya-api/blobs/{id}","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:15:41:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T15:41:42.894679186Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f41801a22b3d7f5900080d1a819bf56f\\\",\\\"SpanID\\\":\\\"3eb10e4e058eff12\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:41:38.993750605Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:41:38.993853614Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":40190}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:41:42.894853982Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"6975180cdef6ad17b9efbdbe1b7e80e4\\\",\\\"SpanID\\\":\\\"3b0dec7a3a297e39\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:41:39.112850054Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:41:39.151798112Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":true,"count":5,"baseline":0,"what":"HTTP 401 | /v2/laya-api/blobs/{id}"}]},{"id":"tlxffp-nginx","at":"2026-09-25T15:38:13.346714157Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:33:13.346714157Z","windowEnd":"2026-09-25T15:38:13.346714157Z","errorCount":68,"baselineMean":15.551724137931034,"excess":7,"patterns":[{"signature":"0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e","count":2,"baselineMean":0,"excess":2,"novel":false,"clients":2,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:37:36.21665375Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"3d9b70200ecac81e31ffeecfa86054e1","PatternSignature":"0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/designer/v1/file-content","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"162.158.217.104","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"5a9d99056a99f32d99bbbe116c5aa8521d6ed732e15513bbf8053db6d96913bf","count":2,"baselineMean":0,"excess":2,"novel":false,"clients":2,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:37:30.409837455Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"c3153b7817a4cbfc3b6a2357e5885a7b","PatternSignature":"5a9d99056a99f32d99bbbe116c5aa8521d6ed732e15513bbf8053db6d96913bf","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/config","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"162.158.217.105","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6","count":2,"baselineMean":0,"excess":2,"novel":false,"clients":2,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:37:36.21665375Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"b36f6a36de27af8bd9dea775a1dd181e","PatternSignature":"b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/templates/preview","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"162.158.217.104","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab","count":1,"baselineMean":0,"excess":1,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:37:36.21665375Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"5a8f88bfa7368f5346ab25b13ca1b472","PatternSignature":"004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/fs/exec","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"162.158.217.104","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/designer/v1/file-content","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":2,"baseline":0,"clients":2,"uaClass":"ai_crawler","what":"HTTP 503 | /api/designer/v1/file-content"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/config","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":2,"baseline":0,"clients":2,"what":"HTTP 503 | /api/config"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/templates/preview","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":2,"baseline":0,"clients":2,"uaClass":"ai_crawler","what":"HTTP 503 | /api/templates/preview"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/fs/exec","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":1,"baseline":0,"clients":1,"uaClass":"ai_crawler","what":"HTTP 503 | /api/fs/exec"}]},{"id":"tlxexd-nginx","at":"2026-09-25T15:27:13.346952193Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:22:13.346952193Z","windowEnd":"2026-09-25T15:27:13.346952193Z","errorCount":44,"baselineMean":14,"excess":38.89655172413793,"patterns":[{"signature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","count":40,"baselineMean":1.103448275862069,"excess":38.89655172413793,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:27:10.338499832Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"8f0569d8963e8d96b1d3e497700ce25b","PatternSignature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/metrics/query","Host":"ui.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":502,"host":"ui.k6n.net","path":"/api/metrics/query","services":["nginx"],"routes":[{"ingress":"dev/k8s-helper-ingress","host":"ui.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"k8s-helper","port":"80","deployments":["k8s-helper"]}],"pods":[{"namespace":"dev","deployment":"k8s-helper","pod":"k8s-helper-5665cd69c9-kjd2t","lines":["I0925 15:26:24.888952 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 15:26:51.998835 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 15:27:24.895350 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\""],"found":true}],"rule":"status-5xx-upstream","new":false,"count":40,"baseline":1.103448275862069,"what":"HTTP 502 | /api/metrics/query"}]},{"id":"tlxevp-nginx","at":"2026-09-25T15:26:13.346647513Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:21:13.346647513Z","windowEnd":"2026-09-25T15:26:13.346647513Z","errorCount":44,"baselineMean":13.724137931034482,"excess":2,"patterns":[{"signature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","count":2,"baselineMean":0,"excess":2,"novel":false,"clients":2,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:25:31.359710287Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"ea2a5c59210e1c1076b971328b6688c0","PatternSignature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"rcon-cs.tornberg.me","StatusCode":503,"ExceptionType":"","ClientIP":"172.69.194.250","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"rcon-cs.tornberg.me","path":"/","services":["nginx"],"routes":[{"ingress":"game/cs-rcon2000","host":"rcon-cs.tornberg.me","path":"/","pathType":"Prefix","namespace":"game","service":"cs-rcon2000","port":"80","deployments":["cs-rcon2000"]}],"pods":[{"namespace":"game","deployment":"cs-rcon2000","found":false}],"rule":"status-5xx-upstream","new":false,"count":2,"baseline":0,"clients":2,"what":"HTTP 503 | /"}]},{"id":"tlxea1-nginx","at":"2026-09-25T15:13:13.346941264Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:08:13.346941264Z","windowEnd":"2026-09-25T15:13:13.346941264Z","errorCount":44,"baselineMean":9.586206896551724,"excess":2.9310344827586206,"patterns":[{"signature":"3c1f3fe95cf095ba60cb1377d69a6a1c2cfe5fb12394303c6d557c8ce3b1fb4c","count":1,"baselineMean":0,"excess":1,"novel":true,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:11:25.232506557Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"138fbe9ca4bbe5aefcb3e012d9aebc86","PatternSignature":"3c1f3fe95cf095ba60cb1377d69a6a1c2cfe5fb12394303c6d557c8ce3b1fb4c","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/resources","Host":"ui.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"dd10eb359ab4e657a6db632468d4e5d1025e9c824562fa1737f1fb0a2c51a512","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["eventsourced-orchestrator","nginx"],"example":{"Timestamp":"2026-09-25T15:12:41.040181855Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"914660c641fd99e4b6c10b3d4b2b325d","PatternSignature":"dd10eb359ab4e657a6db632468d4e5d1025e9c824562fa1737f1fb0a2c51a512","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/overview","Host":"event-agent.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["eventsourced-orchestrator","nginx"],"BadActor":""}},{"signature":"ec8f49cca245ee954c6f7a7839235c13a8cfaabc1f0bcac8ecd29c88a5533891","count":1,"baselineMean":0.06896551724137931,"excess":0.9310344827586207,"novel":false,"services":["eventsourced-orchestrator","nginx"],"example":{"Timestamp":"2026-09-25T15:12:41.040181855Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"8999cc489544cd7ff668240c6ffb1b74","PatternSignature":"ec8f49cca245ee954c6f7a7839235c13a8cfaabc1f0bcac8ecd29c88a5533891","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/auth/me","Host":"event-agent.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["eventsourced-orchestrator","nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":502,"host":"ui.k6n.net","path":"/api/resources","services":["nginx"],"routes":[{"ingress":"dev/k8s-helper-ingress","host":"ui.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"k8s-helper","port":"80","deployments":["k8s-helper"]}],"pods":[{"namespace":"dev","deployment":"k8s-helper","pod":"k8s-helper-5665cd69c9-kjd2t","lines":["I0925 15:12:24.895836 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 15:13:09.988989 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 15:13:24.891348 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\""],"found":true}],"rule":"status-5xx-upstream","new":true,"count":1,"baseline":0,"what":"HTTP 502 | /api/resources"},{"category":"client_credential_issue","priority":"low","status":401,"host":"event-agent.k6n.net","path":"/api/overview","services":["eventsourced-orchestrator","nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-5db7c4c548-r54xs","lines":["10.244.9.1 - - [25/Sep/2026:15:13:31 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:15:13:41 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.5.162 - - [25/Sep/2026:15:13:43 +0000] \"GET /api/overview HTTP/1.1\" 200 279064 \"https://event-agent.k6n.net/session/crying-rabbit-92\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36\" \"10.10.0.1\""],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /api/overview"},{"category":"client_credential_issue","priority":"low","status":401,"host":"event-agent.k6n.net","path":"/auth/me","services":["eventsourced-orchestrator","nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-5db7c4c548-r54xs","lines":["10.244.9.1 - - [25/Sep/2026:15:13:31 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:15:13:41 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.5.162 - - [25/Sep/2026:15:13:43 +0000] \"GET /api/overview HTTP/1.1\" 200 279064 \"https://event-agent.k6n.net/session/crying-rabbit-92\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36\" \"10.10.0.1\""],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0.06896551724137931,"what":"HTTP 401 | /auth/me"}]},{"id":"tlxe8d-nginx","at":"2026-09-25T15:12:13.346987906Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:07:13.346987906Z","windowEnd":"2026-09-25T15:12:13.346987906Z","errorCount":49,"baselineMean":9.96551724137931,"excess":1,"patterns":[{"signature":"382021ec653c3879cceb5c06de218d7bf25435b52cb45899720aa54dfd422bf2","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:07:26.21663347Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"c36b7b4d86d2fc34a2e92dca980d630e","PatternSignature":"382021ec653c3879cceb5c06de218d7bf25435b52cb45899720aa54dfd422bf2","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai-worker-web/manifests/{id}","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai-worker-web/manifests/{id}","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:15:12:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T15:12:41.724259905Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"3fc51021cde7f61e31590c3a815cf30b\\\",\\\"SpanID\\\":\\\"0ec55ee4060e641f\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:12:38.935591926Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:12:38.935694744Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":58652}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:12:41.724462705Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"78ad11ca20f16a7adc51ea89a17af451\\\",\\\"SpanID\\\":\\\"9b3c073f369525ee\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:12:39.101643343Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:12:39.10432783Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai-worker-web/manifests/{id}"}]},{"id":"tlxe6p-nginx","at":"2026-09-25T15:11:13.346878467Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:06:13.346878467Z","windowEnd":"2026-09-25T15:11:13.346878467Z","errorCount":50,"baselineMean":9.551724137931034,"excess":3,"patterns":[{"signature":"0b9d38585ec8cbdca191445a42ebbab0b254811df06aec7d26679e5cded6b4e6","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:07:25.201694002Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"ae870ad54232c9689315ca3ca81891dd","PatternSignature":"0b9d38585ec8cbdca191445a42ebbab0b254811df06aec7d26679e5cded6b4e6","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/mcp","Host":"rag.k6n.net","StatusCode":400,"ExceptionType":"","ClientIP":"","UAClass":"library","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:06:46.212551042Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"0a8bdff037a04864cdbc911cd7a0251d","PatternSignature":"10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b","IssueFingerprint":"","IsRetry":false,"IsTimeout":true,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/stream","Host":"event-agent.k6n.net","StatusCode":504,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"2b2c0c74b5d8ee0e6e3814f70aa1654b3d4f43bb8663a2552102ba1cb7457a8b","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:07:25.201694002Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"2db0875de2536941bdd5e4d7836c8748","PatternSignature":"2b2c0c74b5d8ee0e6e3814f70aa1654b3d4f43bb8663a2552102ba1cb7457a8b","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai-kubeworker/manifests/latest","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","status":504,"host":"event-agent.k6n.net","path":"/api/stream","services":["nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-5db7c4c548-r54xs","lines":["10.244.9.1 - - [25/Sep/2026:15:11:21 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:15:11:31 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:15:11:41 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"timeout-flag","new":false,"count":1,"baseline":0,"what":"HTTP 504 | /api/stream"},{"category":"needs_review","priority":"low","status":400,"host":"rag.k6n.net","path":"/mcp","services":["nginx"],"routes":[{"ingress":"home/rag-mcp-ingress","host":"rag.k6n.net","path":"/mcp","pathType":"Prefix","namespace":"home","service":"rag-service-cuda","port":"4001","deployments":["rust-rag-cuda"]}],"pods":[{"namespace":"home","deployment":"rust-rag-cuda","pod":"rust-rag-cuda-7c7454f95b-rl4np","lines":["\u001b[2m2026-09-25T15:10:48.879055Z\u001b[0m \u001b[32m INFO\u001b[0m \u001b[2mrust_rag::db::code_store\u001b[0m\u001b[2m:\u001b[0m codesearch: repo snapshot merged \u001b[3mrepo\u001b[0m\u001b[2m=\u001b[0mgit@git.k6n.net:mats/eventsourced-ai.git \u001b[3mfiles\u001b[0m\u001b[2m=\u001b[0m1164 \u001b[3msymbols\u001b[0m\u001b[2m=\u001b[0m0 \u001b[3mcalls\u001b[0m\u001b[2m=\u001b[0m76358","\u001b[2m2026-09-25T15:11:08.781298Z\u001b[0m \u001b[32m INFO\u001b[0m \u001b[2mrust_rag::db::code_store\u001b[0m\u001b[2m:\u001b[0m codesearch: repo snapshot merged \u001b[3mrepo\u001b[0m\u001b[2m=\u001b[0mgit@git.k6n.net:mats/eventsourced-ai.git \u001b[3mfiles\u001b[0m\u001b[2m=\u001b[0m1164 \u001b[3msymbols\u001b[0m\u001b[2m=\u001b[0m0 \u001b[3mcalls\u001b[0m\u001b[2m=\u001b[0m76358","\u001b[2m2026-09-25T15:11:26.781932Z\u001b[0m \u001b[32m INFO\u001b[0m \u001b[2mrust_rag::db::code_store\u001b[0m\u001b[2m:\u001b[0m codesearch: repo snapshot merged \u001b[3mrepo\u001b[0m\u001b[2m=\u001b[0mgit@git.k6n.net:mats/eventsourced-ai.git \u001b[3mfiles\u001b[0m\u001b[2m=\u001b[0m1164 \u001b[3msymbols\u001b[0m\u001b[2m=\u001b[0m0 \u001b[3mcalls\u001b[0m\u001b[2m=\u001b[0m76358"],"found":true}],"rule":"no-rule","new":false,"count":1,"baseline":0,"uaClass":"library","what":"HTTP 400 | /mcp"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai-kubeworker/manifests/latest","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:15:11:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T15:11:41.690355357Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"5f01dc3778125e823c9e23e1cacbe7f6\\\",\\\"SpanID\\\":\\\"d76882ecc4500e22\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:11:38.934372333Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:11:38.934484718Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":46224}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:11:41.690525348Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"4f549d5b9e73732825879869fe477ae9\\\",\\\"SpanID\\\":\\\"33d6d34949f1017c\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:11:39.102557984Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:11:39.146578046Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai-kubeworker/manifests/latest"}]},{"id":"tlxe51-nginx","at":"2026-09-25T15:10:13.346769076Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:05:13.346769076Z","windowEnd":"2026-09-25T15:10:13.346769076Z","errorCount":67,"baselineMean":9.931034482758621,"excess":2,"patterns":[{"signature":"bc548cc4f9f3a04d38c7be8915ced890a998b00f1af00a79d36e0d32037af40d","count":2,"baselineMean":0,"excess":2,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:07:31.011384824Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"cf55bcead348adb96b029075181f9a91","PatternSignature":"bc548cc4f9f3a04d38c7be8915ced890a998b00f1af00a79d36e0d32037af40d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai-worker-mcp/manifests/latest","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["nginx","docker-registry"],"BadActor":""}}]},"patterns":[{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai-worker-mcp/manifests/latest","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:15:10:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T15:10:41.658410004Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"396027739ba44124a08bf5a3ce582499\\\",\\\"SpanID\\\":\\\"2296151ddf383168\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:10:38.934291682Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:10:38.934400004Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":58652}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:10:41.658574145Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"4062afeab1d8d70ef4bc5922ad82b8f4\\\",\\\"SpanID\\\":\\\"1113a5878ca748cf\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:10:39.102513029Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:10:39.104615689Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":2,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai-worker-mcp/manifests/latest"}]},{"id":"tlxe1p-eventsourced-orchestrator","at":"2026-09-25T15:08:13.346626294Z","spike":{"service":"eventsourced-orchestrator","windowStart":"2026-09-25T15:03:13.346626294Z","windowEnd":"2026-09-25T15:08:13.346626294Z","errorCount":22,"baselineMean":0.27586206896551724,"excess":22,"patterns":[{"signature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","count":22,"baselineMean":0,"excess":22,"novel":false,"services":["eventsourced-orchestrator"],"example":{"Timestamp":"2026-09-25T15:07:30.033409716Z","Service":"eventsourced-orchestrator","Release":"","Kind":"log","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"","PatternSignature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","IssueFingerprint":"b57379b9dd3a6a9d","IsRetry":false,"IsTimeout":true,"IsCancellation":false,"Message":"codesearch upload failed","SpanName":"","Route":"","Host":"","StatusCode":0,"ExceptionType":"context.deadlineExceededError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":false,"HasUpstream":false,"TraceServices":null,"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","issueId":"b57379b9dd3a6a9d","services":["eventsourced-orchestrator"],"rule":"timeout-flag","new":false,"count":22,"baseline":0,"what":"context.deadlineExceededError | codesearch upload failed"}]},{"id":"tlxe01-eventsourced-worker-code","at":"2026-09-25T15:07:13.346691368Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T15:02:13.346691368Z","windowEnd":"2026-09-25T15:07:13.346691368Z","errorCount":45,"baselineMean":1.8620689655172413,"excess":44.310344827586206,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":38,"baselineMean":0.6896551724137931,"excess":37.310344827586206,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T15:05:54.340775446Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"67fa6def37f0feda16f801e8f40194b2","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}},{"signature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","count":7,"baselineMean":0,"excess":7,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T15:05:01.828115407Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"a9475ecf3ae5a4ca0e8e73a7630cb89f","PatternSignature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","IssueFingerprint":"b40901e8fbfcf66e","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: eof","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":38,"baseline":0.6896551724137931,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"},{"category":"dependency_unreachable","priority":"high","issueId":"b40901e8fbfcf66e","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":7,"baseline":0,"what":"worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"}]},{"id":"tlxdyd-nginx","at":"2026-09-25T15:06:13.347093859Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:01:13.347093859Z","windowEnd":"2026-09-25T15:06:13.347093859Z","errorCount":121,"baselineMean":8.379310344827585,"excess":16,"patterns":[{"signature":"507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d","count":14,"baselineMean":0,"excess":14,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:05:36.202277428Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"a4da26a4a32edf925be5858fb04f23fe","PatternSignature":"507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai-worker/manifests/latest","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}},{"signature":"63eff8b04309c018ea6fe0604d6e3b0885a94f3252cdf65ac9e777ae320a3af7","count":2,"baselineMean":0,"excess":2,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:05:16.201179908Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"bef5078df057b841700020d2d7128086","PatternSignature":"63eff8b04309c018ea6fe0604d6e3b0885a94f3252cdf65ac9e777ae320a3af7","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/network-speed","Host":"tornberg.me","StatusCode":500,"ExceptionType":"","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"service_defect","priority":"high","status":500,"host":"tornberg.me","path":"/api/network-speed","services":["nginx"],"routes":[{"ingress":"dev/static-web","host":"tornberg.me","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-api","port":"3000","deployments":["go-api"]}],"pods":[{"namespace":"dev","deployment":"go-api","pod":"go-api-85d88c9bd-qp8p4","found":true}],"rule":"status-5xx","new":false,"count":2,"baseline":0,"what":"HTTP 500 | /api/network-speed"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai-worker/manifests/latest","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:15:06:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T15:06:41.534921099Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"5fa8dd0ea0a6e33d84a5d5cdce9938fb\\\",\\\"SpanID\\\":\\\"d195e207dfbf1775\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:06:38.93415321Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:06:38.934250546Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":42508}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:06:41.535080838Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"8db442a0165869c57a316a8faa1f8fb8\\\",\\\"SpanID\\\":\\\"e498e0a764f76319\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:06:39.102209346Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:06:39.10428595Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":14,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai-worker/manifests/latest"}]},{"id":"tlxdwp-nginx","at":"2026-09-25T15:05:13.346979342Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:00:13.346979342Z","windowEnd":"2026-09-25T15:05:13.346979342Z","errorCount":113,"baselineMean":8.89655172413793,"excess":5,"patterns":[{"signature":"507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d","count":2,"baselineMean":0,"excess":2,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:05:06.200527764Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"171ceb5b0a017957cc0510541f80fd07","PatternSignature":"507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai-worker/manifests/latest","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}},{"signature":"645f263c238fbbac7ee018af53104cade809f58f461e3939c252e097381e265c","count":2,"baselineMean":0,"excess":2,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:05:06.200527764Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"f90437c17a24182429ec1dbb33fb1209","PatternSignature":"645f263c238fbbac7ee018af53104cade809f58f461e3939c252e097381e265c","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai-worker/manifests/{id}","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}},{"signature":"38f693f9028998b8f34bc954af6be996e8b6add9ad53bbbae8c7f2ae5456586f","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:05:01.199520113Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"b341496525b8a9346fd0ddc5fa26498c","PatternSignature":"38f693f9028998b8f34bc954af6be996e8b6add9ad53bbbae8c7f2ae5456586f","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai-worker-browser/manifests/latest","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai-worker/manifests/latest","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:15:05:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T15:05:41.507526902Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"758d99c1d694e7816fbf337d231dc906\\\",\\\"SpanID\\\":\\\"7cb6d7f33c9cf9e7\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:05:38.934630694Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:38.934733409Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36062}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:05:41.507683183Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f4b89e61b17217cfa6bda198b2b91f46\\\",\\\"SpanID\\\":\\\"d95c0c7e1b5008ba\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:05:39.10176949Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:39.103919224Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":2,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai-worker/manifests/latest"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai-worker/manifests/{id}","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:15:05:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T15:05:41.507526902Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"758d99c1d694e7816fbf337d231dc906\\\",\\\"SpanID\\\":\\\"7cb6d7f33c9cf9e7\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:05:38.934630694Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:38.934733409Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36062}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:05:41.507683183Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f4b89e61b17217cfa6bda198b2b91f46\\\",\\\"SpanID\\\":\\\"d95c0c7e1b5008ba\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:05:39.10176949Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:39.103919224Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":2,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai-worker/manifests/{id}"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai-worker-browser/manifests/latest","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:15:05:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T15:05:41.507526902Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"758d99c1d694e7816fbf337d231dc906\\\",\\\"SpanID\\\":\\\"7cb6d7f33c9cf9e7\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:05:38.934630694Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:38.934733409Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36062}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:05:41.507683183Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f4b89e61b17217cfa6bda198b2b91f46\\\",\\\"SpanID\\\":\\\"d95c0c7e1b5008ba\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:05:39.10176949Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:39.103919224Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai-worker-browser/manifests/latest"}]},{"id":"tlxdv1-nginx","at":"2026-09-25T15:04:13.347024215Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:59:13.347024215Z","windowEnd":"2026-09-25T15:04:13.347024215Z","errorCount":91,"baselineMean":8.10344827586207,"excess":50,"patterns":[{"signature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","count":49,"baselineMean":0,"excess":49,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:04:05.974352097Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"2ecf7a7bbc8ea827b2dc41694939ce36","PatternSignature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"monitor","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}},{"signature":"fb977ed57f4e204d7992f54a0ef1d8a216ea60662250029db6570e708aaae4da","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:02:46.184100608Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"a7bf46f1b94a51a50950320b18a40e30","PatternSignature":"fb977ed57f4e204d7992f54a0ef1d8a216ea60662250029db6570e708aaae4da","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/dev-machine/manifests/latest","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["time=\"2026-09-25T15:04:41.482860568Z\" level=debug msg=\"{\\\"Name\\\":\\\"GetContent\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"ed26c8d64c3d4819\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"27eef151ac9e36e2\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:04:39.178516808Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:04:39.181030949Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/docker/registry/v2/repositories/eventsourced-ai-worker-mcp/_layers/sha256/3a28460d299da20e81866def93a4d1cd7af8b918597dcddc3e36afc6499f71c8/link\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:04:41.482994542Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"6f84b5597bc3b047\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"27eef151ac9e36e2\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:04:39.181088148Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:04:39.183677221Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/docker/registry/v2/blobs/sha256/3a/3a28460d299da20e81866def93a4d1cd7af8b918597dcddc3e36afc6499f71c8/data\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:04:41.483176512Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /v2/eventsourced-ai-worker-mcp/blobs/sha256:3a28460d299da20e81866def93a4d1cd7af8b918597dcddc3e36afc6499f71c8\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"27eef151ac9e36e2\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"b4d5d67576275b23\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":true},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:04:39.106399223Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:04:39.183981801Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"registry.k6n.net\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.5.162\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36364}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker/28.1.1 go/go1.23.8 git-commit/01f442b kernel/6.8.0-142-generic os/linux arch/amd64 UpstreamClient(Go-http-client/1.1)\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.10.11.135\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/v2/eventsourced-ai-worker-mcp/blobs/sha256:3a28460d299da20e81866def93a4d1cd7af8b918597dcddc3e36afc6499f71c8\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"1.1\\\"}},{\\\"Key\\\":\\\"http.response.body.size\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":2783}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":8,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":49,"baseline":0,"uaClass":"monitor","what":"HTTP 401 | /v2"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/dev-machine/manifests/latest","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["time=\"2026-09-25T15:04:41.482860568Z\" level=debug msg=\"{\\\"Name\\\":\\\"GetContent\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"ed26c8d64c3d4819\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"27eef151ac9e36e2\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:04:39.178516808Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:04:39.181030949Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/docker/registry/v2/repositories/eventsourced-ai-worker-mcp/_layers/sha256/3a28460d299da20e81866def93a4d1cd7af8b918597dcddc3e36afc6499f71c8/link\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:04:41.482994542Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"6f84b5597bc3b047\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"27eef151ac9e36e2\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:04:39.181088148Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:04:39.183677221Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/docker/registry/v2/blobs/sha256/3a/3a28460d299da20e81866def93a4d1cd7af8b918597dcddc3e36afc6499f71c8/data\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:04:41.483176512Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /v2/eventsourced-ai-worker-mcp/blobs/sha256:3a28460d299da20e81866def93a4d1cd7af8b918597dcddc3e36afc6499f71c8\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"27eef151ac9e36e2\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"b4d5d67576275b23\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":true},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:04:39.106399223Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:04:39.183981801Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"registry.k6n.net\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.5.162\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36364}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker/28.1.1 go/go1.23.8 git-commit/01f442b kernel/6.8.0-142-generic os/linux arch/amd64 UpstreamClient(Go-http-client/1.1)\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.10.11.135\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/v2/eventsourced-ai-worker-mcp/blobs/sha256:3a28460d299da20e81866def93a4d1cd7af8b918597dcddc3e36afc6499f71c8\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"1.1\\\"}},{\\\"Key\\\":\\\"http.response.body.size\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":2783}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":8,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /v2/dev-machine/manifests/latest"}]},{"id":"tlxdrp-nginx","at":"2026-09-25T15:02:13.346726581Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:57:13.346726581Z","windowEnd":"2026-09-25T15:02:13.346726581Z","errorCount":81,"baselineMean":8,"excess":24,"patterns":[{"signature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","count":24,"baselineMean":0,"excess":24,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:02:10.960575775Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"dd7c8820f20ffa5e56e77b4af727d26b","PatternSignature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"monitor","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["time=\"2026-09-25T15:02:43.940293986Z\" level=debug msg=\"(*schema2ManifestHandler).Unmarshal\" auth.user.name=mats environment=development go.version=go1.25.9 http.request.host=registry.k6n.net http.request.id=01a0d917-15d4-7c71-89ce-2e064874e3fd http.request.method=HEAD http.request.remoteaddr=10.10.11.135 http.request.uri=/v2/eventsourced-ai/manifests/latest http.request.useragent=\"docker/28.1.1 go/go1.23.8 git-commit/01f442b kernel/6.8.0-142-generic os/linux arch/amd64 UpstreamClient(Go-http-client/1.1)\" instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry vars.name=eventsourced-ai vars.reference=latest version=3.1.1","time=\"2026-09-25T15:02:43.940781825Z\" level=info msg=\"response completed\" auth.user.name=mats environment=development go.version=go1.25.9 http.request.host=registry.k6n.net http.request.id=01a0d917-15d4-7c71-89ce-2e064874e3fd http.request.method=HEAD http.request.remoteaddr=10.10.11.135 http.request.uri=/v2/eventsourced-ai/manifests/latest http.request.useragent=\"docker/28.1.1 go/go1.23.8 git-commit/01f442b kernel/6.8.0-142-generic os/linux arch/amd64 UpstreamClient(Go-http-client/1.1)\" http.response.contenttype=application/vnd.docker.distribution.manifest.v2+json http.response.duration=15.948049ms http.response.status=200 http.response.written=0 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry vars.name=eventsourced-ai vars.reference=latest version=3.1.1","10.244.5.162 - - [25/Sep/2026:15:02:43 +0000] \"HEAD /v2/eventsourced-ai/manifests/latest HTTP/1.1\" 200 0 \"\" \"docker/28.1.1 go/go1.23.8 git-commit/01f442b kernel/6.8.0-142-generic os/linux arch/amd64 UpstreamClient(Go-http-client/1.1)\""],"found":true}],"rule":"status-401","new":false,"count":24,"baseline":0,"uaClass":"monitor","what":"HTTP 401 | /v2"}]},{"id":"tlxdq1-nginx","at":"2026-09-25T15:01:13.346685668Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:56:13.346685668Z","windowEnd":"2026-09-25T15:01:13.346685668Z","errorCount":66,"baselineMean":7.379310344827586,"excess":9,"patterns":[{"signature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","count":9,"baselineMean":0,"excess":9,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:01:10.965147823Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"b90df686739eee8b90af562fb7d5fa0a","PatternSignature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"monitor","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["time=\"2026-09-25T15:01:41.670782168Z\" level=debug msg=GetBlob auth.user.name=mats environment=development go.version=go1.25.9 http.request.host=registry.k6n.net http.request.id=01a0d916-229f-7e53-8c88-7150eaf7c907 http.request.method=GET http.request.remoteaddr=10.10.0.1 http.request.uri=\"/v2/ecommerce/redirector/blobs/sha256:b0e3a4fe5d17450f527d2774392458ca2a0bd1722c4fc9affbdaa74fbbb68386\" http.request.useragent=trivy/0.74.0 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry vars.digest=\"sha256:b0e3a4fe5d17450f527d2774392458ca2a0bd1722c4fc9affbdaa74fbbb68386\" vars.name=ecommerce/redirector version=3.1.1","time=\"2026-09-25T15:01:41.766015511Z\" level=info msg=\"response completed\" auth.user.name=mats environment=development go.version=go1.25.9 http.request.host=registry.k6n.net http.request.id=01a0d916-229f-7e53-8c88-7150eaf7c907 http.request.method=GET http.request.remoteaddr=10.10.0.1 http.request.uri=\"/v2/ecommerce/redirector/blobs/sha256:b0e3a4fe5d17450f527d2774392458ca2a0bd1722c4fc9affbdaa74fbbb68386\" http.request.useragent=trivy/0.74.0 http.response.contenttype=application/octet-stream http.response.duration=102.063279ms http.response.status=200 http.response.written=1038 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry vars.digest=\"sha256:b0e3a4fe5d17450f527d2774392458ca2a0bd1722c4fc9affbdaa74fbbb68386\" vars.name=ecommerce/redirector version=3.1.1","10.244.5.162 - - [25/Sep/2026:15:01:41 +0000] \"GET /v2/ecommerce/redirector/blobs/sha256:b0e3a4fe5d17450f527d2774392458ca2a0bd1722c4fc9affbdaa74fbbb68386 HTTP/1.1\" 200 1038 \"\" \"trivy/0.74.0\""],"found":true}],"rule":"status-401","new":false,"count":9,"baseline":0,"uaClass":"monitor","what":"HTTP 401 | /v2"}]},{"id":"tlxdmp-nginx","at":"2026-09-25T14:59:13.347107325Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:54:13.347107325Z","windowEnd":"2026-09-25T14:59:13.347107325Z","errorCount":61,"baselineMean":7.241379310344827,"excess":5,"patterns":[{"signature":"0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e","count":2,"baselineMean":0,"excess":2,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:58:21.152985505Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"b3fb9ac861025eeb7e06e81f072cdc9a","PatternSignature":"0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/designer/v1/file-content","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"172.71.120.24","UAClass":"search_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6","count":2,"baselineMean":0,"excess":2,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:58:21.152985505Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"7352d1f5a6b9f720cde3a0716ea19a6c","PatternSignature":"b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/templates/preview","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"172.71.120.24","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab","count":1,"baselineMean":0,"excess":1,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:58:21.152985505Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"05fc1e480d330b915927c8d90b82520f","PatternSignature":"004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/fs/exec","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"172.71.120.24","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/designer/v1/file-content","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":true,"count":2,"baseline":0,"clients":1,"uaClass":"search_crawler","what":"HTTP 503 | /api/designer/v1/file-content"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/templates/preview","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":true,"count":2,"baseline":0,"clients":1,"uaClass":"ai_crawler","what":"HTTP 503 | /api/templates/preview"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/fs/exec","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":true,"count":1,"baseline":0,"clients":1,"uaClass":"ai_crawler","what":"HTTP 503 | /api/fs/exec"}]},{"id":"tlxdl1-nginx","at":"2026-09-25T14:58:13.347295198Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:53:13.347295198Z","windowEnd":"2026-09-25T14:58:13.347295198Z","errorCount":52,"baselineMean":6.862068965517241,"excess":2,"patterns":[{"signature":"176140cc93493ec68f0ac58d27d0285f38e9e5b3324cc884123aa732e7594753","count":1,"baselineMean":0,"excess":1,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:58:06.153720966Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"2d4718c5e62a66b9b95b55433986c38f","PatternSignature":"176140cc93493ec68f0ac58d27d0285f38e9e5b3324cc884123aa732e7594753","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/{id}/config","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"172.71.120.24","UAClass":"other_bot","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"489772f6cee5972487f0fc121924a67472a534df12901ea2afd0539ddaadaf21","count":1,"baselineMean":0,"excess":1,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:58:06.153720966Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"932e810264dc1287789c21767583242c","PatternSignature":"489772f6cee5972487f0fc121924a67472a534df12901ea2afd0539ddaadaf21","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/account","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"172.71.120.24","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/{id}/config","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":true,"count":1,"baseline":0,"clients":1,"uaClass":"other_bot","what":"HTTP 503 | /api/{id}/config"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/account","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":true,"count":1,"baseline":0,"clients":1,"uaClass":"ai_crawler","what":"HTTP 503 | /api/account"}]},{"id":"tlxdjd-nginx","at":"2026-09-25T14:57:13.346722313Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:52:13.346722313Z","windowEnd":"2026-09-25T14:57:13.346722313Z","errorCount":41,"baselineMean":7.137931034482759,"excess":1,"patterns":[{"signature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","count":1,"baselineMean":0,"excess":1,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:56:51.1417847Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"f87713e8faead3287a112e09a450f9d3","PatternSignature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"knatofs.se","StatusCode":405,"ExceptionType":"","ClientIP":"188.245.62.219","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"needs_review","priority":"low","status":405,"host":"knatofs.se","path":"/","services":["nginx"],"rule":"no-rule","new":false,"count":1,"baseline":0,"clients":1,"what":"HTTP 405 | /"}]},{"id":"tlxdhp-nginx","at":"2026-09-25T14:56:13.346780665Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:51:13.346780665Z","windowEnd":"2026-09-25T14:56:13.346780665Z","errorCount":36,"baselineMean":6.448275862068965,"excess":32,"patterns":[{"signature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","count":32,"baselineMean":0,"excess":32,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:56:05.932078547Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d2a655791d6b4e0786c37a1a52a057be","PatternSignature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/metrics/query","Host":"ui.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":502,"host":"ui.k6n.net","path":"/api/metrics/query","services":["nginx"],"routes":[{"ingress":"dev/k8s-helper-ingress","host":"ui.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"k8s-helper","port":"80","deployments":["k8s-helper"]}],"pods":[{"namespace":"dev","deployment":"k8s-helper","pod":"k8s-helper-5665cd69c9-kjd2t","lines":["2026/09/25 14:56:36 scan: started pod trivy-scan-39e0ce2743afe7d6 for matst80/go-mail2telegram","2026/09/25 14:56:40 scan cycle: matst80/go-mail2telegram: 58 vulns","2026/09/25 14:56:40 scan: started pod trivy-scan-860a2822ca25fb18 for matst80/info-server:latest"],"found":true}],"rule":"status-5xx-upstream","new":false,"count":32,"baseline":0,"what":"HTTP 502 | /api/metrics/query"}]},{"id":"tlxded-nginx","at":"2026-09-25T14:54:13.347301316Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:49:13.347301316Z","windowEnd":"2026-09-25T14:54:13.347301316Z","errorCount":24,"baselineMean":6.413793103448276,"excess":4,"patterns":[{"signature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","count":2,"baselineMean":0,"excess":2,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T14:53:26.116399324Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"5bf4af948bf803fe849f2b826f3404ce","PatternSignature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}},{"signature":"b9a1368dda50f9d0d082c6783302b9386936009289826c4e6a5d3aa8389982cd","count":2,"baselineMean":0,"excess":2,"novel":true,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T14:53:26.116399324Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"133b5a4d5b71fce2cb959468fd870351","PatternSignature":"b9a1368dda50f9d0d082c6783302b9386936009289826c4e6a5d3aa8389982cd","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/k8s-helper/manifests/{id}","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:14:54:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T14:54:41.13845619Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"4b9dce625eb455364bc6550e0df58c65\\\",\\\"SpanID\\\":\\\"7409de8192221aca\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T14:54:38.93455149Z\\\",\\\"EndTime\\\":\\\"2026-09-25T14:54:38.934659304Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":57204}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T14:54:41.138612963Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"03ce02d69ea60153fac32c157d3e21b9\\\",\\\"SpanID\\\":\\\"efa040ef04b76133\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T14:54:39.101671401Z\\\",\\\"EndTime\\\":\\\"2026-09-25T14:54:39.103906648Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":2,"baseline":0,"what":"HTTP 401 | /v2"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/k8s-helper/manifests/{id}","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:14:54:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T14:54:41.13845619Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"4b9dce625eb455364bc6550e0df58c65\\\",\\\"SpanID\\\":\\\"7409de8192221aca\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T14:54:38.93455149Z\\\",\\\"EndTime\\\":\\\"2026-09-25T14:54:38.934659304Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":57204}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T14:54:41.138612963Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"03ce02d69ea60153fac32c157d3e21b9\\\",\\\"SpanID\\\":\\\"efa040ef04b76133\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T14:54:39.101671401Z\\\",\\\"EndTime\\\":\\\"2026-09-25T14:54:39.103906648Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":true,"count":2,"baseline":0,"what":"HTTP 401 | /v2/k8s-helper/manifests/{id}"}]},{"id":"tlxd11-nginx","at":"2026-09-25T14:46:13.34705123Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:41:13.34705123Z","windowEnd":"2026-09-25T14:46:13.34705123Z","errorCount":23,"baselineMean":5.137931034482759,"excess":16,"patterns":[{"signature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","count":16,"baselineMean":0,"excess":16,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:45:45.86299307Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"e3c39c7524075625f201873a72e9b911","PatternSignature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/metrics/query","Host":"ui.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":502,"host":"ui.k6n.net","path":"/api/metrics/query","services":["nginx"],"routes":[{"ingress":"dev/k8s-helper-ingress","host":"ui.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"k8s-helper","port":"80","deployments":["k8s-helper"]}],"pods":[{"namespace":"dev","deployment":"k8s-helper","pod":"k8s-helper-75474c4b8b-zj7fx","lines":["I0925 14:45:22.941450 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 14:46:21.193030 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 14:46:22.949899 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\""],"found":true}],"rule":"status-5xx-upstream","new":false,"count":16,"baseline":0,"what":"HTTP 502 | /api/metrics/query"}]},{"id":"tlxckd-eventsourced-worker-code","at":"2026-09-25T14:36:13.347443969Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T14:31:13.347443969Z","windowEnd":"2026-09-25T14:36:13.347443969Z","errorCount":77,"baselineMean":0.9310344827586207,"excess":77,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":63,"baselineMean":0,"excess":63,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T14:36:11.69601872Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"e42ab0493268396096f54d37942f2db8","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}},{"signature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","count":14,"baselineMean":0,"excess":14,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T14:35:54.760751892Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"f4ec4924e09c57fd686a1443958948af","PatternSignature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","IssueFingerprint":"b40901e8fbfcf66e","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: eof","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":63,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"},{"category":"dependency_unreachable","priority":"high","issueId":"b40901e8fbfcf66e","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":14,"baseline":0,"what":"worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"}]},{"id":"tlxc21-nginx","at":"2026-09-25T14:25:13.346654632Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:20:13.346654632Z","windowEnd":"2026-09-25T14:25:13.346654632Z","errorCount":31,"baselineMean":4.137931034482759,"excess":4,"patterns":[{"signature":"efb1d02de6ef2fbc3e1f234fa547da4e31652733a4c6f8aababe8da1900871f9","count":4,"baselineMean":0,"excess":4,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:24:15.768783875Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"fcd10da95cccfebe9e02e43df79d7570","PatternSignature":"efb1d02de6ef2fbc3e1f234fa547da4e31652733a4c6f8aababe8da1900871f9","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/transmission/rpc","Host":"torrent.k6n.net","StatusCode":409,"ExceptionType":"","ClientIP":"209.206.25.124","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"needs_review","priority":"low","status":409,"host":"torrent.k6n.net","path":"/transmission/rpc","services":["nginx"],"routes":[{"ingress":"home/torrent-ingress","host":"torrent.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"torrent-service","port":"9091"}],"rule":"no-rule","new":true,"count":4,"baseline":0,"clients":1,"what":"HTTP 409 | /transmission/rpc"}]},{"id":"tlxc0d-nginx","at":"2026-09-25T14:24:13.347064057Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:19:13.347064057Z","windowEnd":"2026-09-25T14:24:13.347064057Z","errorCount":27,"baselineMean":3.7241379310344827,"excess":3,"patterns":[{"signature":"41b6e057dd561fe6bf91eb3d028dbfe48f6333be61488ed661ea8bdcfb379a96","count":2,"baselineMean":0,"excess":2,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:23:20.761650796Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"2beaa3d1c2cbcc00449161ed313fedba","PatternSignature":"41b6e057dd561fe6bf91eb3d028dbfe48f6333be61488ed661ea8bdcfb379a96","IssueFingerprint":"","IsRetry":false,"IsTimeout":true,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/sessions/{id}/state","Host":"event-agent.k6n.net","StatusCode":504,"ExceptionType":"","ClientIP":"209.206.25.124","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"990205581d87cfdec48d07f37464355a8cd1b918827d9e18511bb8da7b0c1dd6","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T14:22:10.872293248Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"fe25b634337b1ef0773ab4ea3954d973","PatternSignature":"990205581d87cfdec48d07f37464355a8cd1b918827d9e18511bb8da7b0c1dd6","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai/manifests/latest","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","status":504,"host":"event-agent.k6n.net","path":"/api/sessions/{id}/state","services":["nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-56fdfc6cbc-9rjzb","lines":["10.244.9.1 - - [25/Sep/2026:14:24:32 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.5.162 - - [25/Sep/2026:14:24:37 +0000] \"GET /api/overview HTTP/1.1\" 200 153207 \"https://event-agent.k6n.net/session/crying-monkey-50\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Safari/537.36\" \"209.206.25.124\"","10.244.9.1 - - [25/Sep/2026:14:24:42 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"timeout-flag","new":false,"count":2,"baseline":0,"clients":1,"what":"HTTP 504 | /api/sessions/{id}/state"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai/manifests/latest","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:14:24:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T14:24:40.388398143Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"9a7320d962c760aed339162de5df76c9\\\",\\\"SpanID\\\":\\\"b0e4e7da1c3e808b\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T14:24:38.934703547Z\\\",\\\"EndTime\\\":\\\"2026-09-25T14:24:38.934805825Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":42970}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T14:24:40.388564842Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"3d8f2ed158e81d5e7197b33d3e04dc1a\\\",\\\"SpanID\\\":\\\"b127d729aece6baf\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T14:24:39.101781941Z\\\",\\\"EndTime\\\":\\\"2026-09-25T14:24:39.104725835Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai/manifests/latest"}]},{"id":"tlxbyp-nginx","at":"2026-09-25T14:23:13.347056801Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:18:13.347056801Z","windowEnd":"2026-09-25T14:23:13.347056801Z","errorCount":28,"baselineMean":3.4827586206896552,"excess":1,"patterns":[{"signature":"10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b","count":1,"baselineMean":0,"excess":1,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:22:25.756704937Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"c73381a9fccb067af1c61cd923bfc28a","PatternSignature":"10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/stream","Host":"event-agent.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"209.206.25.124","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":502,"host":"event-agent.k6n.net","path":"/api/stream","services":["nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-56fdfc6cbc-9rjzb","lines":["10.244.5.162 - - [25/Sep/2026:14:23:41 +0000] \"GET /sw.js HTTP/1.1\" 304 0 \"https://event-agent.k6n.net/sw.js\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Safari/537.36\" \"209.206.25.124\"","10.244.5.162 - - [25/Sep/2026:14:23:42 +0000] \"GET /auth/me HTTP/1.1\" 200 111 \"https://event-agent.k6n.net/session/crying-monkey-50\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Safari/537.36\" \"209.206.25.124\"","10.244.9.1 - - [25/Sep/2026:14:23:42 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"status-5xx-upstream","new":false,"count":1,"baseline":0,"clients":1,"what":"HTTP 502 | /api/stream"}]},{"id":"tlxbx1-nginx","at":"2026-09-25T14:22:13.347454307Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:17:13.347454307Z","windowEnd":"2026-09-25T14:22:13.347454307Z","errorCount":23,"baselineMean":3.3448275862068964,"excess":14,"patterns":[{"signature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","count":8,"baselineMean":0,"excess":8,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:22:00.791722367Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d709c813d8dc8f4780ce8ba77b711ad9","PatternSignature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/metrics/query","Host":"ui.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"209.206.25.124","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"6fd42b8e91d39c7666d3c140cdca5aa7711ce1e2d21f0cf6b0f67535b3f68dac","count":3,"baselineMean":0,"excess":3,"novel":true,"clients":1,"services":["eventsourced-orchestrator","nginx"],"example":{"Timestamp":"2026-09-25T14:18:20.845671196Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d2aff32d937667f8013c63ca8d494317","PatternSignature":"6fd42b8e91d39c7666d3c140cdca5aa7711ce1e2d21f0cf6b0f67535b3f68dac","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/session/reset_password","Host":"event-agent.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"103.206.253.106","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["nginx","eventsourced-orchestrator"],"BadActor":""}},{"signature":"5c87cc1fcbcc6c723041e1eee7b41eef12f1f99c88d2a1e98804404024a291c1","count":2,"baselineMean":0,"excess":2,"novel":true,"clients":1,"services":["eventsourced-orchestrator","nginx"],"example":{"Timestamp":"2026-09-25T14:18:05.729846843Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"f55568b95bd1b356c01bad8e3f349a3a","PatternSignature":"5c87cc1fcbcc6c723041e1eee7b41eef12f1f99c88d2a1e98804404024a291c1","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/session/properties","Host":"event-agent.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"103.206.253.106","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["eventsourced-orchestrator","nginx"],"BadActor":""}},{"signature":"41b6e057dd561fe6bf91eb3d028dbfe48f6333be61488ed661ea8bdcfb379a96","count":1,"baselineMean":0,"excess":1,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:22:10.755531343Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"35e68f3ccb4a275c735de2dcf51b35e6","PatternSignature":"41b6e057dd561fe6bf91eb3d028dbfe48f6333be61488ed661ea8bdcfb379a96","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/sessions/{id}/state","Host":"event-agent.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"209.206.25.124","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":502,"host":"ui.k6n.net","path":"/api/metrics/query","services":["nginx"],"routes":[{"ingress":"dev/k8s-helper-ingress","host":"ui.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"k8s-helper","port":"80","deployments":["k8s-helper"]}],"pods":[{"namespace":"dev","deployment":"k8s-helper","pod":"k8s-helper-75474c4b8b-zj7fx","lines":["I0925 14:21:34.050281 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 14:22:04.282513 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 14:22:22.964288 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\""],"found":true}],"rule":"status-5xx-upstream","new":true,"count":8,"baseline":0,"clients":1,"what":"HTTP 502 | /api/metrics/query"},{"category":"dependency_unreachable","priority":"high","status":502,"host":"event-agent.k6n.net","path":"/api/sessions/{id}/state","services":["nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-56fdfc6cbc-9rjzb","lines":["10.244.9.1 - - [25/Sep/2026:14:22:22 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:14:22:32 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:14:22:42 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"status-5xx-upstream","new":false,"count":1,"baseline":0,"clients":1,"what":"HTTP 502 | /api/sessions/{id}/state"},{"category":"client_credential_issue","priority":"low","status":401,"host":"event-agent.k6n.net","path":"/api/session/reset_password","services":["eventsourced-orchestrator","nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-56fdfc6cbc-9rjzb","lines":["10.244.9.1 - - [25/Sep/2026:14:22:22 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:14:22:32 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:14:22:42 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"status-401","new":true,"count":3,"baseline":0,"clients":1,"what":"HTTP 401 | /api/session/reset_password"},{"category":"client_credential_issue","priority":"low","status":401,"host":"event-agent.k6n.net","path":"/api/session/properties","services":["eventsourced-orchestrator","nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-56fdfc6cbc-9rjzb","lines":["10.244.9.1 - - [25/Sep/2026:14:22:22 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:14:22:32 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:14:22:42 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"status-401","new":true,"count":2,"baseline":0,"clients":1,"what":"HTTP 401 | /api/session/properties"}]},{"id":"tlxbx1-eventsourced-worker-code","at":"2026-09-25T14:22:13.347454307Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T14:17:13.347454307Z","windowEnd":"2026-09-25T14:22:13.347454307Z","errorCount":35,"baselineMean":0.41379310344827586,"excess":8,"patterns":[{"signature":"c79ebdd574e5a9103bcc3c2faaa06abee7f041701094b24e883949dcc5371eab","count":8,"baselineMean":0,"excess":8,"novel":true,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T14:22:02.949148899Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"cb422f420e0e5e387e57384138fa6f42","PatternSignature":"c79ebdd574e5a9103bcc3c2faaa06abee7f041701094b24e883949dcc5371eab","IssueFingerprint":"b082dde6853f6053","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*worker.error: bash: invalid arguments: json: cannot unmarshal string into go struct field bashargs.timeout_seconds of type int","SpanName":"worker.fulfill","Route":"","Host":"","StatusCode":0,"ExceptionType":"*worker.Error","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","issueId":"b082dde6853f6053","services":["eventsourced-worker-code"],"rule":"capacity-text","new":true,"count":8,"baseline":0,"what":"worker.fulfill | *worker.Error | *worker.error: bash: invalid arguments: json: cannot unmarshal string into go struct field bashargs.timeout_seconds of …"}]},{"id":"tlxbd1-eventsourced-worker-code","at":"2026-09-25T14:10:13.346727499Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T14:05:13.346727499Z","windowEnd":"2026-09-25T14:10:13.346727499Z","errorCount":37,"baselineMean":0.3448275862068966,"excess":35,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":28,"baselineMean":0,"excess":28,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T14:09:49.577703801Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"14ad7fa4b0d513f8fa057a4216eafb83","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}},{"signature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","count":7,"baselineMean":0,"excess":7,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T14:09:41.534950695Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"8584decc043b76b0347c329d4e7d9e97","PatternSignature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","IssueFingerprint":"b40901e8fbfcf66e","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: eof","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":28,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"},{"category":"dependency_unreachable","priority":"high","issueId":"b40901e8fbfcf66e","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":7,"baseline":0,"what":"worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"}]},{"id":"tlx951-eventsourced-worker-code","at":"2026-09-25T13:22:13.347013465Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T13:17:13.347013465Z","windowEnd":"2026-09-25T13:22:13.347013465Z","errorCount":37,"baselineMean":0,"excess":37,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":30,"baselineMean":0,"excess":30,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T13:22:11.585302542Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"08659ce24a8ed07494d7cdcf4cc5eb82","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}},{"signature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","count":7,"baselineMean":0,"excess":7,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T13:21:46.49062247Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"7902028e7f4807da43236b784641ba03","PatternSignature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","IssueFingerprint":"b40901e8fbfcf66e","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: eof","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":30,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"},{"category":"dependency_unreachable","priority":"high","issueId":"b40901e8fbfcf66e","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":7,"baseline":0,"what":"worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"}]},{"id":"tlx7pd-eventsourced-orchestrator","at":"2026-09-25T12:51:13.347152151Z","spike":{"service":"eventsourced-orchestrator","windowStart":"2026-09-25T12:46:13.347152151Z","windowEnd":"2026-09-25T12:51:13.347152151Z","errorCount":15,"baselineMean":0,"excess":1,"patterns":[{"signature":"29f5030badef82d221f8a79563cf2c9dbf4d653551ca54b081bc0780c3b35c37","count":1,"baselineMean":0,"excess":1,"novel":true,"services":["eventsourced-orchestrator"],"example":{"Timestamp":"2026-09-25T12:50:45.113996545Z","Service":"eventsourced-orchestrator","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"31ff3a0f713c612d62db59873507090e","PatternSignature":"29f5030badef82d221f8a79563cf2c9dbf4d653551ca54b081bc0780c3b35c37","IssueFingerprint":"260fc07492b5c2f8","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: provider stream error: {\"code\":\u003cnum\u003e,\"message\":\"provider returned an empty response\",\"metadata\":{\"error_type\":\"provider_unavailable\"}}","SpanName":"orchestrator.llm.attempt","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":false,"HasUpstream":false,"TraceServices":["eventsourced-orchestrator"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"260fc07492b5c2f8","services":["eventsourced-orchestrator"],"rule":"dependency-text","new":true,"count":1,"baseline":0,"what":"orchestrator.llm.attempt | *errors.errorString | *errors.errorstring: provider stream error: {\"code\":\u003cnum\u003e,\"message\":\"provider returned an empty response\",\"metadata\":{\"…"}]},{"id":"tlx7m1-eventsourced-orchestrator","at":"2026-09-25T12:49:13.346915639Z","spike":{"service":"eventsourced-orchestrator","windowStart":"2026-09-25T12:44:13.346915639Z","windowEnd":"2026-09-25T12:49:13.346915639Z","errorCount":13,"baselineMean":0,"excess":13,"patterns":[{"signature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","count":13,"baselineMean":0,"excess":13,"novel":false,"services":["eventsourced-orchestrator"],"example":{"Timestamp":"2026-09-25T12:48:32.179425433Z","Service":"eventsourced-orchestrator","Release":"","Kind":"log","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"","PatternSignature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","IssueFingerprint":"b57379b9dd3a6a9d","IsRetry":false,"IsTimeout":true,"IsCancellation":false,"Message":"codesearch upload failed","SpanName":"","Route":"","Host":"","StatusCode":0,"ExceptionType":"context.deadlineExceededError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":false,"HasUpstream":false,"TraceServices":null,"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","issueId":"b57379b9dd3a6a9d","services":["eventsourced-orchestrator"],"rule":"timeout-flag","new":false,"count":13,"baseline":0,"what":"context.deadlineExceededError | codesearch upload failed"}]},{"id":"tlx7ip-eventsourced-worker-code","at":"2026-09-25T12:47:13.346939505Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T12:42:13.346939505Z","windowEnd":"2026-09-25T12:47:13.346939505Z","errorCount":47,"baselineMean":0,"excess":40,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":40,"baselineMean":0,"excess":40,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T12:46:57.647725048Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"a646787952ca743875ed75a4943fd6b0","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":40,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"}]},{"id":"tlx7h1-eventsourced-worker-code","at":"2026-09-25T12:46:13.347336983Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T12:41:13.347336983Z","windowEnd":"2026-09-25T12:46:13.347336983Z","errorCount":10,"baselineMean":0,"excess":10,"patterns":[{"signature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","count":7,"baselineMean":0,"excess":7,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T12:46:03.649627264Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"e49ca924b6a716cb0d737e870cf7d8ec","PatternSignature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","IssueFingerprint":"b40901e8fbfcf66e","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: eof","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}},{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":3,"baselineMean":0,"excess":3,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T12:46:03.649627264Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"cb600304e20a370fd4c4eeb13d87536a","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"b40901e8fbfcf66e","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":7,"baseline":0,"what":"worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"},{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":3,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"}]},{"id":"tlx3b1-eventsourced-orchestrator","at":"2026-09-25T11:16:13.346973966Z","spike":{"service":"eventsourced-orchestrator","windowStart":"2026-09-25T11:11:13.346973966Z","windowEnd":"2026-09-25T11:16:13.346973966Z","errorCount":23,"baselineMean":0,"excess":1,"patterns":[{"signature":"ed8d4fd7b78d58f7f14c9fbec7d67d550e8a40f236126cd4964d72ad3645dd9f","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["eventsourced-orchestrator"],"example":{"Timestamp":"2026-09-25T11:15:34.045242671Z","Service":"eventsourced-orchestrator","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"a951dfed8b514f48f773df0889f33d4f","PatternSignature":"ed8d4fd7b78d58f7f14c9fbec7d67d550e8a40f236126cd4964d72ad3645dd9f","IssueFingerprint":"869896e00332d212","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*fmt.wraperror: read llm response: context deadline exceeded (client.timeout or context cancellation while reading body)","SpanName":"llm.provider_fallback","Route":"","Host":"","StatusCode":0,"ExceptionType":"*fmt.wrapError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":false,"HasUpstream":false,"TraceServices":["eventsourced-orchestrator"],"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","issueId":"869896e00332d212","services":["eventsourced-orchestrator"],"rule":"capacity-text","new":false,"count":1,"baseline":0,"what":"llm.provider_fallback | *fmt.wrapError | *fmt.wraperror: read llm response: context deadline exceeded (client.timeout or context cancellation while reading body)"}]},{"id":"tlx34d-eventsourced-orchestrator","at":"2026-09-25T11:12:13.347409701Z","spike":{"service":"eventsourced-orchestrator","windowStart":"2026-09-25T11:07:13.347409701Z","windowEnd":"2026-09-25T11:12:13.347409701Z","errorCount":19,"baselineMean":0,"excess":19,"patterns":[{"signature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","count":19,"baselineMean":0,"excess":19,"novel":false,"services":["eventsourced-orchestrator"],"example":{"Timestamp":"2026-09-25T11:12:01.123433759Z","Service":"eventsourced-orchestrator","Release":"","Kind":"log","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"","PatternSignature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","IssueFingerprint":"b57379b9dd3a6a9d","IsRetry":false,"IsTimeout":true,"IsCancellation":false,"Message":"codesearch upload failed","SpanName":"","Route":"","Host":"","StatusCode":0,"ExceptionType":"context.deadlineExceededError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":false,"HasUpstream":false,"TraceServices":null,"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","issueId":"b57379b9dd3a6a9d","services":["eventsourced-orchestrator"],"rule":"timeout-flag","new":false,"count":19,"baseline":0,"what":"context.deadlineExceededError | codesearch upload failed"}]},{"id":"tlx311-eventsourced-worker-code","at":"2026-09-25T11:10:13.346712877Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T11:05:13.346712877Z","windowEnd":"2026-09-25T11:10:13.346712877Z","errorCount":39,"baselineMean":0.20689655172413793,"excess":39,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":32,"baselineMean":0,"excess":32,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T11:10:12.495433233Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"7e73ad1291b81c8250f0bd529d41c514","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}},{"signature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","count":7,"baselineMean":0,"excess":7,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T11:09:29.914245587Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"95d11690cfe3f8a3d370fc001394503d","PatternSignature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","IssueFingerprint":"b40901e8fbfcf66e","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: eof","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":32,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"},{"category":"dependency_unreachable","priority":"high","issueId":"b40901e8fbfcf66e","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":7,"baseline":0,"what":"worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"}]}]} + diff --git a/cmd/laya-eval/main.go b/cmd/laya-eval/main.go new file mode 100644 index 0000000..e6c3135 --- /dev/null +++ b/cmd/laya-eval/main.go @@ -0,0 +1,144 @@ +// laya-eval runs alert scenarios through laya and compares its shadow verdict +// with what a person judged the right response to be, so the model can be +// assessed before it is ever allowed to act. +// +// go run ./cmd/laya-eval # built-in scenarios +// go run ./cmd/laya-eval -file alerts.json # a GET /api/alerts dump (no expectations) +// go run ./cmd/laya-eval -send # also deliver each as a TEST alert +// +// -send uses the channels configured by OTEL_ALERT_* (see alerts.ConfigFromEnv). +package main + +import ( + "context" + _ "embed" + "encoding/json" + "flag" + "fmt" + "os" + "text/tabwriter" + "time" + + "git.tornberg.me/go-otel/internal/alerts" +) + +//go:embed scenarios.json +var builtin []byte + +// Scenario is one alert and the response a person would pick for it. +type Scenario struct { + Name string `json:"name"` + Expect *Expectation `json:"expect,omitempty"` + Alert alerts.Alert `json:"alert"` +} + +type Expectation struct { + Action string `json:"action"` + Human bool `json:"human"` + Why string `json:"why"` +} + +func main() { + url := flag.String("url", envOr("OTEL_ALERT_LAYA_URL", "https://laya.k6n.net"), "laya base URL") + file := flag.String("file", "", "scenarios JSON, or a GET /api/alerts response (default: built-in scenarios)") + send := flag.Bool("send", false, "deliver each scenario, with its verdict, as a TEST alert to the configured channels") + flag.Parse() + + raw := builtin + if *file != "" { + var err error + if raw, err = os.ReadFile(*file); err != nil { + fail(err) + } + } + scenarios, err := parse(raw) + if err != nil { + fail(err) + } + var notifiers []alerts.Notifier + if *send { + cfg := alerts.ConfigFromEnv() + if notifiers = cfg.Notifiers(); len(notifiers) == 0 { + fail(fmt.Errorf("-send: no channel configured (OTEL_ALERT_TELEGRAM_TOKEN + OTEL_ALERT_TELEGRAM_CHAT_ID, OTEL_ALERT_WEBHOOK_URL or OTEL_ALERT_AOD_URL)")) + } + } + + laya := &alerts.Laya{URL: *url} + tw := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(tw, "SCENARIO\tLAYA ACTION\tHUMAN\tURGENCY\tEXPECTED\tMATCH") + judged, actionOK, humanOK := 0, 0, 0 + for _, s := range scenarios { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + v, err := laya.Classify(ctx, s.Alert) + cancel() + if err != nil { + fmt.Fprintf(tw, "%s\terror: %v\t\t\t\t\n", s.Name, err) + continue + } + exp, match := "-", "-" + if s.Expect != nil { + judged++ + a, h := v.Action == s.Expect.Action, (v.Human >= 0.5) == s.Expect.Human + if a { + actionOK++ + } + if h { + humanOK++ + } + exp = fmt.Sprintf("%s, human=%v", s.Expect.Action, s.Expect.Human) + match = fmt.Sprintf("action %s, human %s", mark(a), mark(h)) + } + fmt.Fprintf(tw, "%s\t%s %.0f%%\t%.0f%%\t%s %.0f%%\t%s\t%s\n", s.Name, v.Action, v.ActionConfidence*100, v.Human*100, v.Urgency, v.UrgencyConf*100, exp, match) + + if *send { + a := s.Alert + a.Test, a.Laya = true, v + for _, n := range notifiers { + sctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + if err := n.Send(sctx, a); err != nil { + fmt.Fprintf(os.Stderr, "%s: %s: %v\n", s.Name, n.Name(), err) + } + cancel() + } + } + } + tw.Flush() + if judged > 0 { + fmt.Printf("\nagreement over %d judged scenarios: action %d/%d, needs-human %d/%d\n", judged, actionOK, judged, humanOK, judged) + } +} + +// parse accepts a scenario list or a raw GET /api/alerts response. +func parse(raw []byte) ([]Scenario, error) { + var list []Scenario + if err := json.Unmarshal(raw, &list); err == nil { + return list, nil + } + var st alerts.Status + if err := json.Unmarshal(raw, &st); err != nil { + return nil, fmt.Errorf("neither a scenario list nor an /api/alerts response: %w", err) + } + for _, a := range st.Alerts { + list = append(list, Scenario{Name: a.ID + " " + a.Spike.Service, Alert: a}) + } + return list, nil +} + +func mark(ok bool) string { + if ok { + return "ok" + } + return "MISS" +} + +func envOr(k, def string) string { + if v := os.Getenv(k); v != "" { + return v + } + return def +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} diff --git a/cmd/laya-eval/scenarios.json b/cmd/laya-eval/scenarios.json new file mode 100644 index 0000000..f593114 --- /dev/null +++ b/cmd/laya-eval/scenarios.json @@ -0,0 +1,2098 @@ +[ + { + "name": "codesearch upload deadline exceeded", + "expect": { + "action": "check_capacity", + "human": true, + "why": "uploads time out repeatedly; the codesearch path is saturated or too slow" + }, + "alert": { + "id": "tlxmh1-eventsourced-orchestrator", + "at": "2026-09-25T18:10:13.34664129Z", + "spike": { + "service": "eventsourced-orchestrator", + "windowStart": "2026-09-25T18:05:13.34664129Z", + "windowEnd": "2026-09-25T18:10:13.34664129Z", + "errorCount": 16, + "baselineMean": 19.93103448275862, + "excess": 16, + "patterns": [ + { + "signature": "8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef", + "count": 16, + "baselineMean": 0, + "excess": 16, + "novel": false, + "services": [ + "eventsourced-orchestrator" + ], + "example": { + "Timestamp": "2026-09-25T18:10:08.793065336Z", + "Service": "eventsourced-orchestrator", + "Release": "", + "Kind": "log", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "", + "PatternSignature": "8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef", + "IssueFingerprint": "b57379b9dd3a6a9d", + "IsRetry": false, + "IsTimeout": true, + "IsCancellation": false, + "Message": "codesearch upload failed", + "SpanName": "", + "Route": "", + "Host": "", + "StatusCode": 0, + "ExceptionType": "context.deadlineExceededError", + "ClientIP": "", + "UAClass": "", + "IsDuplicate": false, + "IsRootCause": false, + "HasUpstream": false, + "TraceServices": null, + "BadActor": "" + } + } + ] + }, + "patterns": [ + { + "category": "capacity_saturation", + "priority": "high", + "issueId": "b57379b9dd3a6a9d", + "services": [ + "eventsourced-orchestrator" + ], + "rule": "timeout-flag", + "new": false, + "count": 16, + "baseline": 0, + "what": "context.deadlineExceededError | codesearch upload failed" + } + ] + } + }, + { + "name": "worker cannot dial orchestrator websocket", + "expect": { + "action": "fix_dependency", + "human": true, + "why": "workers cannot reach the in-cluster orchestrator; the dependency is down or unreachable" + }, + "alert": { + "id": "tlxmdp-eventsourced-worker-code", + "at": "2026-09-25T18:08:13.346954479Z", + "spike": { + "service": "eventsourced-worker-code", + "windowStart": "2026-09-25T18:03:13.346954479Z", + "windowEnd": "2026-09-25T18:08:13.346954479Z", + "errorCount": 36, + "baselineMean": 0.13793103448275862, + "excess": 36, + "patterns": [ + { + "signature": "fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885", + "count": 29, + "baselineMean": 0, + "excess": 29, + "novel": false, + "services": [ + "eventsourced-worker-code" + ], + "example": { + "Timestamp": "2026-09-25T18:08:12.308751715Z", + "Service": "eventsourced-worker-code", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "34f7c7c1d9c07bb48b3241454d10a7bf", + "PatternSignature": "fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885", + "IssueFingerprint": "4580ae864fa21049", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:/internal/worker-terminal: dial tcp ...:: connect: connection refused", + "SpanName": "worker.terminal.connection", + "Route": "", + "Host": "", + "StatusCode": 0, + "ExceptionType": "*websocket.DialError", + "ClientIP": "", + "UAClass": "", + "IsDuplicate": false, + "IsRootCause": true, + "HasUpstream": false, + "TraceServices": [ + "eventsourced-worker-code" + ], + "BadActor": "" + } + }, + { + "signature": "2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7", + "count": 7, + "baselineMean": 0, + "excess": 7, + "novel": false, + "services": [ + "eventsourced-worker-code" + ], + "example": { + "Timestamp": "2026-09-25T18:07:50.177441418Z", + "Service": "eventsourced-worker-code", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "72e0cbf2c415b129eab8f4aadca42c9f", + "PatternSignature": "2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7", + "IssueFingerprint": "b40901e8fbfcf66e", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "*errors.errorstring: eof", + "SpanName": "worker.terminal.connection", + "Route": "", + "Host": "", + "StatusCode": 0, + "ExceptionType": "*errors.errorString", + "ClientIP": "", + "UAClass": "", + "IsDuplicate": false, + "IsRootCause": true, + "HasUpstream": false, + "TraceServices": [ + "eventsourced-worker-code" + ], + "BadActor": "" + } + } + ] + }, + "patterns": [ + { + "category": "dependency_unreachable", + "priority": "high", + "issueId": "4580ae864fa21049", + "services": [ + "eventsourced-worker-code" + ], + "rule": "dependency-text", + "new": false, + "count": 29, + "baseline": 0, + "what": "worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:/internal/worker-termina…" + }, + { + "category": "dependency_unreachable", + "priority": "high", + "issueId": "b40901e8fbfcf66e", + "services": [ + "eventsourced-worker-code" + ], + "rule": "dependency-text", + "new": false, + "count": 7, + "baseline": 0, + "what": "worker.terminal.connection | *errors.errorString | *errors.errorstring: eof" + } + ] + } + }, + { + "name": "bash tool args fail to unmarshal", + "expect": { + "action": "investigate_code", + "human": true, + "why": "a new json decode error on timeout_seconds is a code/contract bug, not capacity (triage mislabels it via the word 'timeout')" + }, + "alert": { + "id": "tlxbx1-eventsourced-worker-code", + "at": "2026-09-25T14:22:13.347454307Z", + "spike": { + "service": "eventsourced-worker-code", + "windowStart": "2026-09-25T14:17:13.347454307Z", + "windowEnd": "2026-09-25T14:22:13.347454307Z", + "errorCount": 35, + "baselineMean": 0.41379310344827586, + "excess": 8, + "patterns": [ + { + "signature": "c79ebdd574e5a9103bcc3c2faaa06abee7f041701094b24e883949dcc5371eab", + "count": 8, + "baselineMean": 0, + "excess": 8, + "novel": true, + "services": [ + "eventsourced-worker-code" + ], + "example": { + "Timestamp": "2026-09-25T14:22:02.949148899Z", + "Service": "eventsourced-worker-code", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "cb422f420e0e5e387e57384138fa6f42", + "PatternSignature": "c79ebdd574e5a9103bcc3c2faaa06abee7f041701094b24e883949dcc5371eab", + "IssueFingerprint": "b082dde6853f6053", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "*worker.error: bash: invalid arguments: json: cannot unmarshal string into go struct field bashargs.timeout_seconds of type int", + "SpanName": "worker.fulfill", + "Route": "", + "Host": "", + "StatusCode": 0, + "ExceptionType": "*worker.Error", + "ClientIP": "", + "UAClass": "", + "IsDuplicate": false, + "IsRootCause": true, + "HasUpstream": false, + "TraceServices": [ + "eventsourced-worker-code" + ], + "BadActor": "" + } + } + ] + }, + "patterns": [ + { + "category": "capacity_saturation", + "priority": "high", + "issueId": "b082dde6853f6053", + "services": [ + "eventsourced-worker-code" + ], + "rule": "capacity-text", + "new": true, + "count": 8, + "baseline": 0, + "what": "worker.fulfill | *worker.Error | *worker.error: bash: invalid arguments: json: cannot unmarshal string into go struct field bashargs.timeout_seconds of …" + } + ] + } + }, + { + "name": "go-api 500 on /api/network-speed", + "expect": { + "action": "investigate_code", + "human": true, + "why": "500 from a running pod with no logs: a defect in go-api" + }, + "alert": { + "id": "tlxdyd-nginx", + "at": "2026-09-25T15:06:13.347093859Z", + "spike": { + "service": "nginx", + "windowStart": "2026-09-25T15:01:13.347093859Z", + "windowEnd": "2026-09-25T15:06:13.347093859Z", + "errorCount": 121, + "baselineMean": 8.379310344827585, + "excess": 16, + "patterns": [ + { + "signature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d", + "count": 14, + "baselineMean": 0, + "excess": 14, + "novel": false, + "services": [ + "docker-registry", + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T15:05:36.202277428Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "a4da26a4a32edf925be5858fb04f23fe", + "PatternSignature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/v2/eventsourced-ai-worker/manifests/latest", + "Host": "registry.k6n.net", + "StatusCode": 401, + "ExceptionType": "", + "ClientIP": "", + "UAClass": "unknown", + "IsDuplicate": false, + "IsRootCause": false, + "HasUpstream": true, + "TraceServices": [ + "docker-registry", + "nginx" + ], + "BadActor": "" + } + }, + { + "signature": "63eff8b04309c018ea6fe0604d6e3b0885a94f3252cdf65ac9e777ae320a3af7", + "count": 2, + "baselineMean": 0, + "excess": 2, + "novel": false, + "services": [ + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T15:05:16.201179908Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "bef5078df057b841700020d2d7128086", + "PatternSignature": "63eff8b04309c018ea6fe0604d6e3b0885a94f3252cdf65ac9e777ae320a3af7", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/api/network-speed", + "Host": "tornberg.me", + "StatusCode": 500, + "ExceptionType": "", + "ClientIP": "", + "UAClass": "", + "IsDuplicate": false, + "IsRootCause": true, + "HasUpstream": false, + "TraceServices": [ + "nginx" + ], + "BadActor": "" + } + } + ] + }, + "patterns": [ + { + "category": "service_defect", + "priority": "high", + "status": 500, + "host": "tornberg.me", + "path": "/api/network-speed", + "services": [ + "nginx" + ], + "routes": [ + { + "ingress": "dev/static-web", + "host": "tornberg.me", + "path": "/api", + "pathType": "Prefix", + "namespace": "dev", + "service": "go-api", + "port": "3000", + "deployments": [ + "go-api" + ] + } + ], + "pods": [ + { + "namespace": "dev", + "deployment": "go-api", + "pod": "go-api-85d88c9bd-qp8p4", + "found": true + } + ], + "rule": "status-5xx", + "new": false, + "count": 2, + "baseline": 0, + "what": "HTTP 500 | /api/network-speed" + }, + { + "category": "client_credential_issue", + "priority": "low", + "status": 401, + "host": "registry.k6n.net", + "path": "/v2/eventsourced-ai-worker/manifests/latest", + "services": [ + "docker-registry", + "nginx" + ], + "routes": [ + { + "ingress": "home/registry", + "host": "registry.k6n.net", + "path": "/", + "pathType": "Prefix", + "namespace": "home", + "service": "registry", + "port": "5000", + "deployments": [ + "registry" + ] + } + ], + "pods": [ + { + "namespace": "home", + "deployment": "registry", + "pod": "registry-559bf85884-sxbrw", + "lines": [ + "10.244.2.1 - - [25/Sep/2026:15:06:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"", + "time=\"2026-09-25T15:06:41.534921099Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"5fa8dd0ea0a6e33d84a5d5cdce9938fb\\\",\\\"SpanID\\\":\\\"d195e207dfbf1775\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:06:38.93415321Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:06:38.934250546Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":42508}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1", + "time=\"2026-09-25T15:06:41.535080838Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"8db442a0165869c57a316a8faa1f8fb8\\\",\\\"SpanID\\\":\\\"e498e0a764f76319\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:06:39.102209346Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:06:39.10428595Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1" + ], + "found": true + } + ], + "rule": "status-401", + "new": false, + "count": 14, + "baseline": 0, + "what": "HTTP 401 | /v2/eventsourced-ai-worker/manifests/latest" + } + ] + } + }, + { + "name": "k8s-helper 502 on /api/metrics/query", + "expect": { + "action": "fix_dependency", + "human": true, + "why": "40 bad gateways against a running k8s-helper: its upstream (Prometheus/metrics) or the pod path is failing" + }, + "alert": { + "id": "tlxgbd-nginx", + "at": "2026-09-25T15:57:13.347060965Z", + "spike": { + "service": "nginx", + "windowStart": "2026-09-25T15:52:13.347060965Z", + "windowEnd": "2026-09-25T15:57:13.347060965Z", + "errorCount": 72, + "baselineMean": 18.413793103448278, + "excess": 31.448275862068968, + "patterns": [ + { + "signature": "77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e", + "count": 40, + "baselineMean": 8.551724137931034, + "excess": 31.448275862068968, + "novel": false, + "services": [ + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T15:57:06.60535739Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "7654e99020c8f9de172e1bf4cbd65aa3", + "PatternSignature": "77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/api/metrics/query", + "Host": "ui.k6n.net", + "StatusCode": 502, + "ExceptionType": "", + "ClientIP": "", + "UAClass": "browser", + "IsDuplicate": false, + "IsRootCause": true, + "HasUpstream": false, + "TraceServices": [ + "nginx" + ], + "BadActor": "" + } + } + ] + }, + "patterns": [ + { + "category": "dependency_unreachable", + "priority": "high", + "status": 502, + "host": "ui.k6n.net", + "path": "/api/metrics/query", + "services": [ + "nginx" + ], + "routes": [ + { + "ingress": "dev/k8s-helper-ingress", + "host": "ui.k6n.net", + "path": "/", + "pathType": "Prefix", + "namespace": "dev", + "service": "k8s-helper", + "port": "80", + "deployments": [ + "k8s-helper" + ] + } + ], + "pods": [ + { + "namespace": "dev", + "deployment": "k8s-helper", + "pod": "k8s-helper-5665cd69c9-kjd2t", + "lines": [ + "I0925 15:56:34.658960 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"", + "I0925 15:57:20.162724 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"", + "I0925 15:57:24.894618 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"" + ], + "found": true + } + ], + "rule": "status-5xx-upstream", + "new": false, + "count": 40, + "baseline": 8.551724137931034, + "what": "HTTP 502 | /api/metrics/query" + } + ] + } + }, + { + "name": "eventsourced-web 504 on /api/stream", + "expect": { + "action": "check_capacity", + "human": true, + "why": "gateway timeouts on the streaming endpoint while the pod only logs probes" + }, + "alert": { + "id": "tlxirp-nginx", + "at": "2026-09-25T16:50:13.347044487Z", + "spike": { + "service": "nginx", + "windowStart": "2026-09-25T16:45:13.347044487Z", + "windowEnd": "2026-09-25T16:50:13.347044487Z", + "errorCount": 62, + "baselineMean": 18.93103448275862, + "excess": 41.96551724137931, + "patterns": [ + { + "signature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d", + "count": 17, + "baselineMean": 0, + "excess": 17, + "novel": false, + "services": [ + "docker-registry", + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T16:50:05.564200851Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "51f8278d03367c21a21f76835ec204b2", + "PatternSignature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/v2/eventsourced-ai-worker/manifests/latest", + "Host": "registry.k6n.net", + "StatusCode": 401, + "ExceptionType": "", + "ClientIP": "", + "UAClass": "unknown", + "IsDuplicate": false, + "IsRootCause": false, + "HasUpstream": true, + "TraceServices": [ + "docker-registry", + "nginx" + ], + "BadActor": "" + } + }, + { + "signature": "2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda", + "count": 14, + "baselineMean": 0, + "excess": 14, + "novel": false, + "services": [ + "docker-registry", + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T16:48:10.547871686Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "6664346e0c22d3a9396460163f0b0d6b", + "PatternSignature": "2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/v2", + "Host": "registry.k6n.net", + "StatusCode": 401, + "ExceptionType": "", + "ClientIP": "", + "UAClass": "library", + "IsDuplicate": false, + "IsRootCause": false, + "HasUpstream": true, + "TraceServices": [ + "docker-registry", + "nginx" + ], + "BadActor": "" + } + }, + { + "signature": "ec8f49cca245ee954c6f7a7839235c13a8cfaabc1f0bcac8ecd29c88a5533891", + "count": 5, + "baselineMean": 0.034482758620689655, + "excess": 4.9655172413793105, + "novel": false, + "clients": 1, + "services": [ + "eventsourced-orchestrator", + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T16:49:50.586320711Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "2b5d0ed3c205922143afa2bc0c5b0b8e", + "PatternSignature": "ec8f49cca245ee954c6f7a7839235c13a8cfaabc1f0bcac8ecd29c88a5533891", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/auth/me", + "Host": "event-agent.k6n.net", + "StatusCode": 401, + "ExceptionType": "", + "ClientIP": "170.62.100.17", + "UAClass": "browser", + "IsDuplicate": false, + "IsRootCause": false, + "HasUpstream": true, + "TraceServices": [ + "eventsourced-orchestrator", + "nginx" + ], + "BadActor": "" + } + }, + { + "signature": "10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b", + "count": 3, + "baselineMean": 0, + "excess": 3, + "novel": false, + "clients": 1, + "services": [ + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T16:49:50.586320711Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "44416b3fb21e6f57b9bba5ac0eccd68b", + "PatternSignature": "10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": true, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/api/stream", + "Host": "event-agent.k6n.net", + "StatusCode": 504, + "ExceptionType": "", + "ClientIP": "170.62.100.17", + "UAClass": "browser", + "IsDuplicate": false, + "IsRootCause": true, + "HasUpstream": false, + "TraceServices": [ + "nginx" + ], + "BadActor": "" + } + }, + { + "signature": "f524391f412efceb1222fd1ccb3c5fcb673ace4787f214acaf300dea6cc88115", + "count": 3, + "baselineMean": 0, + "excess": 3, + "novel": false, + "clients": 1, + "services": [ + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T16:48:10.577418006Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "b870baab8dd2de51be3f9da1663d239e", + "PatternSignature": "f524391f412efceb1222fd1ccb3c5fcb673ace4787f214acaf300dea6cc88115", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/auth/me", + "Host": "event-agent.k6n.net", + "StatusCode": 502, + "ExceptionType": "", + "ClientIP": "170.62.100.17", + "UAClass": "browser", + "IsDuplicate": false, + "IsRootCause": true, + "HasUpstream": false, + "TraceServices": [ + "nginx" + ], + "BadActor": "" + } + } + ] + }, + "patterns": [ + { + "category": "capacity_saturation", + "priority": "high", + "status": 504, + "host": "event-agent.k6n.net", + "path": "/api/stream", + "services": [ + "nginx" + ], + "routes": [ + { + "ingress": "dev/eventsourced-web", + "host": "event-agent.k6n.net", + "path": "/", + "pathType": "Prefix", + "namespace": "dev", + "service": "eventsourced-web", + "port": "80", + "deployments": [ + "eventsourced-web" + ] + } + ], + "pods": [ + { + "namespace": "dev", + "deployment": "eventsourced-web", + "pod": "eventsourced-web-5865685999-xtxdz", + "lines": [ + "10.244.9.1 - - [25/Sep/2026:16:50:20 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"", + "10.244.9.1 - - [25/Sep/2026:16:50:30 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"", + "10.244.9.1 - - [25/Sep/2026:16:50:40 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"" + ], + "found": true + } + ], + "rule": "timeout-flag", + "new": false, + "count": 3, + "baseline": 0, + "clients": 1, + "what": "HTTP 504 | /api/stream" + }, + { + "category": "dependency_unreachable", + "priority": "high", + "status": 502, + "host": "event-agent.k6n.net", + "path": "/auth/me", + "services": [ + "nginx" + ], + "routes": [ + { + "ingress": "dev/eventsourced-web", + "host": "event-agent.k6n.net", + "path": "/", + "pathType": "Prefix", + "namespace": "dev", + "service": "eventsourced-web", + "port": "80", + "deployments": [ + "eventsourced-web" + ] + } + ], + "pods": [ + { + "namespace": "dev", + "deployment": "eventsourced-web", + "pod": "eventsourced-web-5865685999-xtxdz", + "lines": [ + "10.244.9.1 - - [25/Sep/2026:16:50:20 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"", + "10.244.9.1 - - [25/Sep/2026:16:50:30 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"", + "10.244.9.1 - - [25/Sep/2026:16:50:40 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"" + ], + "found": true + } + ], + "rule": "status-5xx-upstream", + "new": false, + "count": 3, + "baseline": 0, + "clients": 1, + "what": "HTTP 502 | /auth/me" + }, + { + "category": "client_credential_issue", + "priority": "low", + "status": 401, + "host": "registry.k6n.net", + "path": "/v2/eventsourced-ai-worker/manifests/latest", + "services": [ + "docker-registry", + "nginx" + ], + "routes": [ + { + "ingress": "home/registry", + "host": "registry.k6n.net", + "path": "/", + "pathType": "Prefix", + "namespace": "home", + "service": "registry", + "port": "5000", + "deployments": [ + "registry" + ] + } + ], + "pods": [ + { + "namespace": "home", + "deployment": "registry", + "pod": "registry-559bf85884-sxbrw", + "lines": [ + "10.244.2.1 - - [25/Sep/2026:16:50:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"", + "time=\"2026-09-25T16:50:39.183971552Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"85d74dfb059f5fbff7018c85c990e2bc\\\",\\\"SpanID\\\":\\\"dfd025e674cceb15\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T16:50:38.934252898Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:38.934346788Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":54180}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1", + "time=\"2026-09-25T16:50:39.184168228Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"b389a9064776af9f4a23cc817ee4231f\\\",\\\"SpanID\\\":\\\"16fa1d622c943f61\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T16:50:39.102298404Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:39.105804949Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1" + ], + "found": true + } + ], + "rule": "status-401", + "new": false, + "count": 17, + "baseline": 0, + "what": "HTTP 401 | /v2/eventsourced-ai-worker/manifests/latest" + }, + { + "category": "client_credential_issue", + "priority": "low", + "status": 401, + "host": "registry.k6n.net", + "path": "/v2", + "services": [ + "docker-registry", + "nginx" + ], + "routes": [ + { + "ingress": "home/registry", + "host": "registry.k6n.net", + "path": "/", + "pathType": "Prefix", + "namespace": "home", + "service": "registry", + "port": "5000", + "deployments": [ + "registry" + ] + } + ], + "pods": [ + { + "namespace": "home", + "deployment": "registry", + "pod": "registry-559bf85884-sxbrw", + "lines": [ + "10.244.2.1 - - [25/Sep/2026:16:50:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"", + "time=\"2026-09-25T16:50:39.183971552Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"85d74dfb059f5fbff7018c85c990e2bc\\\",\\\"SpanID\\\":\\\"dfd025e674cceb15\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T16:50:38.934252898Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:38.934346788Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":54180}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1", + "time=\"2026-09-25T16:50:39.184168228Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"b389a9064776af9f4a23cc817ee4231f\\\",\\\"SpanID\\\":\\\"16fa1d622c943f61\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T16:50:39.102298404Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:39.105804949Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1" + ], + "found": true + } + ], + "rule": "status-401", + "new": false, + "count": 14, + "baseline": 0, + "uaClass": "library", + "what": "HTTP 401 | /v2" + }, + { + "category": "client_credential_issue", + "priority": "low", + "status": 401, + "host": "event-agent.k6n.net", + "path": "/auth/me", + "services": [ + "eventsourced-orchestrator", + "nginx" + ], + "routes": [ + { + "ingress": "dev/eventsourced-web", + "host": "event-agent.k6n.net", + "path": "/", + "pathType": "Prefix", + "namespace": "dev", + "service": "eventsourced-web", + "port": "80", + "deployments": [ + "eventsourced-web" + ] + } + ], + "pods": [ + { + "namespace": "dev", + "deployment": "eventsourced-web", + "pod": "eventsourced-web-5865685999-xtxdz", + "lines": [ + "10.244.9.1 - - [25/Sep/2026:16:50:20 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"", + "10.244.9.1 - - [25/Sep/2026:16:50:30 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"", + "10.244.9.1 - - [25/Sep/2026:16:50:40 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"" + ], + "found": true + } + ], + "rule": "status-401", + "new": false, + "count": 5, + "baseline": 0.034482758620689655, + "clients": 1, + "what": "HTTP 401 | /auth/me" + } + ] + } + }, + { + "name": "LLM provider empty responses", + "expect": { + "action": "fix_dependency", + "human": false, + "why": "external LLM provider misbehaving; fallback exists, nothing in-cluster to fix unless it persists" + }, + "alert": { + "id": "tlx7pd-eventsourced-orchestrator", + "at": "2026-09-25T12:51:13.347152151Z", + "spike": { + "service": "eventsourced-orchestrator", + "windowStart": "2026-09-25T12:46:13.347152151Z", + "windowEnd": "2026-09-25T12:51:13.347152151Z", + "errorCount": 15, + "baselineMean": 0, + "excess": 1, + "patterns": [ + { + "signature": "29f5030badef82d221f8a79563cf2c9dbf4d653551ca54b081bc0780c3b35c37", + "count": 1, + "baselineMean": 0, + "excess": 1, + "novel": true, + "services": [ + "eventsourced-orchestrator" + ], + "example": { + "Timestamp": "2026-09-25T12:50:45.113996545Z", + "Service": "eventsourced-orchestrator", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "31ff3a0f713c612d62db59873507090e", + "PatternSignature": "29f5030badef82d221f8a79563cf2c9dbf4d653551ca54b081bc0780c3b35c37", + "IssueFingerprint": "260fc07492b5c2f8", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "*errors.errorstring: provider stream error: {\"code\":,\"message\":\"provider returned an empty response\",\"metadata\":{\"error_type\":\"provider_unavailable\"}}", + "SpanName": "orchestrator.llm.attempt", + "Route": "", + "Host": "", + "StatusCode": 0, + "ExceptionType": "*errors.errorString", + "ClientIP": "", + "UAClass": "", + "IsDuplicate": false, + "IsRootCause": false, + "HasUpstream": false, + "TraceServices": [ + "eventsourced-orchestrator" + ], + "BadActor": "" + } + } + ] + }, + "patterns": [ + { + "category": "dependency_unreachable", + "priority": "high", + "issueId": "260fc07492b5c2f8", + "services": [ + "eventsourced-orchestrator" + ], + "rule": "dependency-text", + "new": true, + "count": 1, + "baseline": 0, + "what": "orchestrator.llm.attempt | *errors.errorString | *errors.errorstring: provider stream error: {\"code\":,\"message\":\"provider returned an empty response\",\"metadata\":{\"…" + } + ] + } + }, + { + "name": "LLM response read deadline", + "expect": { + "action": "fix_dependency", + "human": false, + "why": "slow external LLM provider; a single pattern count of 1" + }, + "alert": { + "id": "tlx3b1-eventsourced-orchestrator", + "at": "2026-09-25T11:16:13.346973966Z", + "spike": { + "service": "eventsourced-orchestrator", + "windowStart": "2026-09-25T11:11:13.346973966Z", + "windowEnd": "2026-09-25T11:16:13.346973966Z", + "errorCount": 23, + "baselineMean": 0, + "excess": 1, + "patterns": [ + { + "signature": "ed8d4fd7b78d58f7f14c9fbec7d67d550e8a40f236126cd4964d72ad3645dd9f", + "count": 1, + "baselineMean": 0, + "excess": 1, + "novel": false, + "services": [ + "eventsourced-orchestrator" + ], + "example": { + "Timestamp": "2026-09-25T11:15:34.045242671Z", + "Service": "eventsourced-orchestrator", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "a951dfed8b514f48f773df0889f33d4f", + "PatternSignature": "ed8d4fd7b78d58f7f14c9fbec7d67d550e8a40f236126cd4964d72ad3645dd9f", + "IssueFingerprint": "869896e00332d212", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "*fmt.wraperror: read llm response: context deadline exceeded (client.timeout or context cancellation while reading body)", + "SpanName": "llm.provider_fallback", + "Route": "", + "Host": "", + "StatusCode": 0, + "ExceptionType": "*fmt.wrapError", + "ClientIP": "", + "UAClass": "", + "IsDuplicate": false, + "IsRootCause": false, + "HasUpstream": false, + "TraceServices": [ + "eventsourced-orchestrator" + ], + "BadActor": "" + } + } + ] + }, + "patterns": [ + { + "category": "capacity_saturation", + "priority": "high", + "issueId": "869896e00332d212", + "services": [ + "eventsourced-orchestrator" + ], + "rule": "capacity-text", + "new": false, + "count": 1, + "baseline": 0, + "what": "llm.provider_fallback | *fmt.wrapError | *fmt.wraperror: read llm response: context deadline exceeded (client.timeout or context cancellation while reading body)" + } + ] + } + }, + { + "name": "crawler hits scaled-to-zero go-html-edit", + "expect": { + "action": "ignore", + "human": false, + "why": "503s only because the dev deployment is scaled to 0 and only AI crawlers asked" + }, + "alert": { + "id": "tlxg6d-nginx", + "at": "2026-09-25T15:54:13.346717682Z", + "spike": { + "service": "nginx", + "windowStart": "2026-09-25T15:49:13.346717682Z", + "windowEnd": "2026-09-25T15:54:13.346717682Z", + "errorCount": 74, + "baselineMean": 18.448275862068964, + "excess": 12, + "patterns": [ + { + "signature": "0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e", + "count": 4, + "baselineMean": 0, + "excess": 4, + "novel": false, + "clients": 2, + "services": [ + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T15:53:46.266195024Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "ecdf390536970c3339612fb2a201df51", + "PatternSignature": "0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/api/designer/v1/file-content", + "Host": "k6n.net", + "StatusCode": 503, + "ExceptionType": "", + "ClientIP": "104.23.160.37", + "UAClass": "ai_crawler", + "IsDuplicate": false, + "IsRootCause": true, + "HasUpstream": false, + "TraceServices": [ + "nginx" + ], + "BadActor": "" + } + }, + { + "signature": "b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6", + "count": 4, + "baselineMean": 0, + "excess": 4, + "novel": false, + "clients": 2, + "services": [ + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T15:53:46.266195024Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "744692761138c480f27fac58b3aa1fda", + "PatternSignature": "b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/api/templates/preview", + "Host": "k6n.net", + "StatusCode": 503, + "ExceptionType": "", + "ClientIP": "104.23.160.37", + "UAClass": "ai_crawler", + "IsDuplicate": false, + "IsRootCause": true, + "HasUpstream": false, + "TraceServices": [ + "nginx" + ], + "BadActor": "" + } + }, + { + "signature": "004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab", + "count": 2, + "baselineMean": 0, + "excess": 2, + "novel": false, + "clients": 2, + "services": [ + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T15:53:46.266195024Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "7ffa5f428b4de59811dd86ded001a33c", + "PatternSignature": "004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/api/fs/exec", + "Host": "k6n.net", + "StatusCode": 503, + "ExceptionType": "", + "ClientIP": "104.23.160.37", + "UAClass": "ai_crawler", + "IsDuplicate": false, + "IsRootCause": true, + "HasUpstream": false, + "TraceServices": [ + "nginx" + ], + "BadActor": "" + } + }, + { + "signature": "7281b16d4395d124b0fac7a82520ee2b9e179df689e8f4eb6d515a6b567aa256", + "count": 2, + "baselineMean": 0, + "excess": 2, + "novel": false, + "clients": 2, + "services": [ + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T15:53:46.266195024Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "942867286e5ba5cac5ec2102cd059106", + "PatternSignature": "7281b16d4395d124b0fac7a82520ee2b9e179df689e8f4eb6d515a6b567aa256", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/api/v1/build_public_tmp/{id}/flow", + "Host": "k6n.net", + "StatusCode": 503, + "ExceptionType": "", + "ClientIP": "104.23.160.37", + "UAClass": "ai_crawler", + "IsDuplicate": false, + "IsRootCause": true, + "HasUpstream": false, + "TraceServices": [ + "nginx" + ], + "BadActor": "" + } + } + ] + }, + "patterns": [ + { + "category": "dependency_unreachable", + "priority": "high", + "status": 503, + "host": "k6n.net", + "path": "/api/designer/v1/file-content", + "services": [ + "nginx" + ], + "routes": [ + { + "ingress": "dev/static-knatofs", + "host": "k6n.net", + "path": "/api", + "pathType": "Prefix", + "namespace": "dev", + "service": "go-html-edit", + "port": "8080", + "deployments": [ + "go-html-edit" + ] + } + ], + "pods": [ + { + "namespace": "dev", + "deployment": "go-html-edit", + "found": false + } + ], + "rule": "status-5xx-upstream", + "new": false, + "count": 4, + "baseline": 0, + "clients": 2, + "uaClass": "ai_crawler", + "what": "HTTP 503 | /api/designer/v1/file-content" + }, + { + "category": "dependency_unreachable", + "priority": "high", + "status": 503, + "host": "k6n.net", + "path": "/api/templates/preview", + "services": [ + "nginx" + ], + "routes": [ + { + "ingress": "dev/static-knatofs", + "host": "k6n.net", + "path": "/api", + "pathType": "Prefix", + "namespace": "dev", + "service": "go-html-edit", + "port": "8080", + "deployments": [ + "go-html-edit" + ] + } + ], + "pods": [ + { + "namespace": "dev", + "deployment": "go-html-edit", + "found": false + } + ], + "rule": "status-5xx-upstream", + "new": false, + "count": 4, + "baseline": 0, + "clients": 2, + "uaClass": "ai_crawler", + "what": "HTTP 503 | /api/templates/preview" + }, + { + "category": "dependency_unreachable", + "priority": "high", + "status": 503, + "host": "k6n.net", + "path": "/api/fs/exec", + "services": [ + "nginx" + ], + "routes": [ + { + "ingress": "dev/static-knatofs", + "host": "k6n.net", + "path": "/api", + "pathType": "Prefix", + "namespace": "dev", + "service": "go-html-edit", + "port": "8080", + "deployments": [ + "go-html-edit" + ] + } + ], + "pods": [ + { + "namespace": "dev", + "deployment": "go-html-edit", + "found": false + } + ], + "rule": "status-5xx-upstream", + "new": false, + "count": 2, + "baseline": 0, + "clients": 2, + "uaClass": "ai_crawler", + "what": "HTTP 503 | /api/fs/exec" + }, + { + "category": "dependency_unreachable", + "priority": "high", + "status": 503, + "host": "k6n.net", + "path": "/api/v1/build_public_tmp/{id}/flow", + "services": [ + "nginx" + ], + "routes": [ + { + "ingress": "dev/static-knatofs", + "host": "k6n.net", + "path": "/api", + "pathType": "Prefix", + "namespace": "dev", + "service": "go-html-edit", + "port": "8080", + "deployments": [ + "go-html-edit" + ] + } + ], + "pods": [ + { + "namespace": "dev", + "deployment": "go-html-edit", + "found": false + } + ], + "rule": "status-5xx-upstream", + "new": false, + "count": 2, + "baseline": 0, + "clients": 2, + "uaClass": "ai_crawler", + "what": "HTTP 503 | /api/v1/build_public_tmp/{id}/flow" + } + ] + } + }, + { + "name": "scaled-to-zero game server", + "expect": { + "action": "ignore", + "human": false, + "why": "cs-rcon2000 intentionally has no pod; 2 requests" + }, + "alert": { + "id": "tlxevp-nginx", + "at": "2026-09-25T15:26:13.346647513Z", + "spike": { + "service": "nginx", + "windowStart": "2026-09-25T15:21:13.346647513Z", + "windowEnd": "2026-09-25T15:26:13.346647513Z", + "errorCount": 44, + "baselineMean": 13.724137931034482, + "excess": 2, + "patterns": [ + { + "signature": "3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d", + "count": 2, + "baselineMean": 0, + "excess": 2, + "novel": false, + "clients": 2, + "services": [ + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T15:25:31.359710287Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "ea2a5c59210e1c1076b971328b6688c0", + "PatternSignature": "3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/", + "Host": "rcon-cs.tornberg.me", + "StatusCode": 503, + "ExceptionType": "", + "ClientIP": "172.69.194.250", + "UAClass": "browser", + "IsDuplicate": false, + "IsRootCause": true, + "HasUpstream": false, + "TraceServices": [ + "nginx" + ], + "BadActor": "" + } + } + ] + }, + "patterns": [ + { + "category": "dependency_unreachable", + "priority": "high", + "status": 503, + "host": "rcon-cs.tornberg.me", + "path": "/", + "services": [ + "nginx" + ], + "routes": [ + { + "ingress": "game/cs-rcon2000", + "host": "rcon-cs.tornberg.me", + "path": "/", + "pathType": "Prefix", + "namespace": "game", + "service": "cs-rcon2000", + "port": "80", + "deployments": [ + "cs-rcon2000" + ] + } + ], + "pods": [ + { + "namespace": "game", + "deployment": "cs-rcon2000", + "found": false + } + ], + "rule": "status-5xx-upstream", + "new": false, + "count": 2, + "baseline": 0, + "clients": 2, + "what": "HTTP 503 | /" + } + ] + } + }, + { + "name": "registry 401 on manifest pulls", + "expect": { + "action": "ignore", + "human": false, + "why": "docker clients probe anonymously first; 401 is the normal auth handshake" + }, + "alert": { + "id": "tlxdwp-nginx", + "at": "2026-09-25T15:05:13.346979342Z", + "spike": { + "service": "nginx", + "windowStart": "2026-09-25T15:00:13.346979342Z", + "windowEnd": "2026-09-25T15:05:13.346979342Z", + "errorCount": 113, + "baselineMean": 8.89655172413793, + "excess": 5, + "patterns": [ + { + "signature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d", + "count": 2, + "baselineMean": 0, + "excess": 2, + "novel": false, + "services": [ + "docker-registry", + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T15:05:06.200527764Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "171ceb5b0a017957cc0510541f80fd07", + "PatternSignature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/v2/eventsourced-ai-worker/manifests/latest", + "Host": "registry.k6n.net", + "StatusCode": 401, + "ExceptionType": "", + "ClientIP": "", + "UAClass": "unknown", + "IsDuplicate": false, + "IsRootCause": false, + "HasUpstream": true, + "TraceServices": [ + "docker-registry", + "nginx" + ], + "BadActor": "" + } + }, + { + "signature": "645f263c238fbbac7ee018af53104cade809f58f461e3939c252e097381e265c", + "count": 2, + "baselineMean": 0, + "excess": 2, + "novel": false, + "services": [ + "docker-registry", + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T15:05:06.200527764Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "f90437c17a24182429ec1dbb33fb1209", + "PatternSignature": "645f263c238fbbac7ee018af53104cade809f58f461e3939c252e097381e265c", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/v2/eventsourced-ai-worker/manifests/{id}", + "Host": "registry.k6n.net", + "StatusCode": 401, + "ExceptionType": "", + "ClientIP": "", + "UAClass": "unknown", + "IsDuplicate": false, + "IsRootCause": false, + "HasUpstream": true, + "TraceServices": [ + "docker-registry", + "nginx" + ], + "BadActor": "" + } + }, + { + "signature": "38f693f9028998b8f34bc954af6be996e8b6add9ad53bbbae8c7f2ae5456586f", + "count": 1, + "baselineMean": 0, + "excess": 1, + "novel": false, + "services": [ + "docker-registry", + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T15:05:01.199520113Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "b341496525b8a9346fd0ddc5fa26498c", + "PatternSignature": "38f693f9028998b8f34bc954af6be996e8b6add9ad53bbbae8c7f2ae5456586f", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/v2/eventsourced-ai-worker-browser/manifests/latest", + "Host": "registry.k6n.net", + "StatusCode": 401, + "ExceptionType": "", + "ClientIP": "", + "UAClass": "unknown", + "IsDuplicate": false, + "IsRootCause": false, + "HasUpstream": true, + "TraceServices": [ + "docker-registry", + "nginx" + ], + "BadActor": "" + } + } + ] + }, + "patterns": [ + { + "category": "client_credential_issue", + "priority": "low", + "status": 401, + "host": "registry.k6n.net", + "path": "/v2/eventsourced-ai-worker/manifests/latest", + "services": [ + "docker-registry", + "nginx" + ], + "routes": [ + { + "ingress": "home/registry", + "host": "registry.k6n.net", + "path": "/", + "pathType": "Prefix", + "namespace": "home", + "service": "registry", + "port": "5000", + "deployments": [ + "registry" + ] + } + ], + "pods": [ + { + "namespace": "home", + "deployment": "registry", + "pod": "registry-559bf85884-sxbrw", + "lines": [ + "10.244.2.1 - - [25/Sep/2026:15:05:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"", + "time=\"2026-09-25T15:05:41.507526902Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"758d99c1d694e7816fbf337d231dc906\\\",\\\"SpanID\\\":\\\"7cb6d7f33c9cf9e7\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:05:38.934630694Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:38.934733409Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36062}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1", + "time=\"2026-09-25T15:05:41.507683183Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f4b89e61b17217cfa6bda198b2b91f46\\\",\\\"SpanID\\\":\\\"d95c0c7e1b5008ba\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:05:39.10176949Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:39.103919224Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1" + ], + "found": true + } + ], + "rule": "status-401", + "new": false, + "count": 2, + "baseline": 0, + "what": "HTTP 401 | /v2/eventsourced-ai-worker/manifests/latest" + }, + { + "category": "client_credential_issue", + "priority": "low", + "status": 401, + "host": "registry.k6n.net", + "path": "/v2/eventsourced-ai-worker/manifests/{id}", + "services": [ + "docker-registry", + "nginx" + ], + "routes": [ + { + "ingress": "home/registry", + "host": "registry.k6n.net", + "path": "/", + "pathType": "Prefix", + "namespace": "home", + "service": "registry", + "port": "5000", + "deployments": [ + "registry" + ] + } + ], + "pods": [ + { + "namespace": "home", + "deployment": "registry", + "pod": "registry-559bf85884-sxbrw", + "lines": [ + "10.244.2.1 - - [25/Sep/2026:15:05:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"", + "time=\"2026-09-25T15:05:41.507526902Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"758d99c1d694e7816fbf337d231dc906\\\",\\\"SpanID\\\":\\\"7cb6d7f33c9cf9e7\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:05:38.934630694Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:38.934733409Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36062}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1", + "time=\"2026-09-25T15:05:41.507683183Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f4b89e61b17217cfa6bda198b2b91f46\\\",\\\"SpanID\\\":\\\"d95c0c7e1b5008ba\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:05:39.10176949Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:39.103919224Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1" + ], + "found": true + } + ], + "rule": "status-401", + "new": false, + "count": 2, + "baseline": 0, + "what": "HTTP 401 | /v2/eventsourced-ai-worker/manifests/{id}" + }, + { + "category": "client_credential_issue", + "priority": "low", + "status": 401, + "host": "registry.k6n.net", + "path": "/v2/eventsourced-ai-worker-browser/manifests/latest", + "services": [ + "docker-registry", + "nginx" + ], + "routes": [ + { + "ingress": "home/registry", + "host": "registry.k6n.net", + "path": "/", + "pathType": "Prefix", + "namespace": "home", + "service": "registry", + "port": "5000", + "deployments": [ + "registry" + ] + } + ], + "pods": [ + { + "namespace": "home", + "deployment": "registry", + "pod": "registry-559bf85884-sxbrw", + "lines": [ + "10.244.2.1 - - [25/Sep/2026:15:05:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"", + "time=\"2026-09-25T15:05:41.507526902Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"758d99c1d694e7816fbf337d231dc906\\\",\\\"SpanID\\\":\\\"7cb6d7f33c9cf9e7\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:05:38.934630694Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:38.934733409Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36062}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1", + "time=\"2026-09-25T15:05:41.507683183Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f4b89e61b17217cfa6bda198b2b91f46\\\",\\\"SpanID\\\":\\\"d95c0c7e1b5008ba\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:05:39.10176949Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:39.103919224Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1" + ], + "found": true + } + ], + "rule": "status-401", + "new": false, + "count": 1, + "baseline": 0, + "what": "HTTP 401 | /v2/eventsourced-ai-worker-browser/manifests/latest" + } + ] + } + }, + { + "name": "transmission 409 session handshake", + "expect": { + "action": "ignore", + "human": false, + "why": "Transmission RPC answers 409 to hand out its session id; expected protocol" + }, + "alert": { + "id": "tlxc21-nginx", + "at": "2026-09-25T14:25:13.346654632Z", + "spike": { + "service": "nginx", + "windowStart": "2026-09-25T14:20:13.346654632Z", + "windowEnd": "2026-09-25T14:25:13.346654632Z", + "errorCount": 31, + "baselineMean": 4.137931034482759, + "excess": 4, + "patterns": [ + { + "signature": "efb1d02de6ef2fbc3e1f234fa547da4e31652733a4c6f8aababe8da1900871f9", + "count": 4, + "baselineMean": 0, + "excess": 4, + "novel": true, + "clients": 1, + "services": [ + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T14:24:15.768783875Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "fcd10da95cccfebe9e02e43df79d7570", + "PatternSignature": "efb1d02de6ef2fbc3e1f234fa547da4e31652733a4c6f8aababe8da1900871f9", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/transmission/rpc", + "Host": "torrent.k6n.net", + "StatusCode": 409, + "ExceptionType": "", + "ClientIP": "209.206.25.124", + "UAClass": "browser", + "IsDuplicate": false, + "IsRootCause": true, + "HasUpstream": false, + "TraceServices": [ + "nginx" + ], + "BadActor": "" + } + } + ] + }, + "patterns": [ + { + "category": "needs_review", + "priority": "low", + "status": 409, + "host": "torrent.k6n.net", + "path": "/transmission/rpc", + "services": [ + "nginx" + ], + "routes": [ + { + "ingress": "home/torrent-ingress", + "host": "torrent.k6n.net", + "path": "/", + "pathType": "Prefix", + "namespace": "home", + "service": "torrent-service", + "port": "9091" + } + ], + "rule": "no-rule", + "new": true, + "count": 4, + "baseline": 0, + "clients": 1, + "what": "HTTP 409 | /transmission/rpc" + } + ] + } + }, + { + "name": "405 on /", + "expect": { + "action": "ignore", + "human": false, + "why": "a client using the wrong method on the root; 4 requests from 1 client" + }, + "alert": { + "id": "tlxn9d-nginx", + "at": "2026-09-25T18:27:13.347226389Z", + "spike": { + "service": "nginx", + "windowStart": "2026-09-25T18:22:13.347226389Z", + "windowEnd": "2026-09-25T18:27:13.347226389Z", + "errorCount": 64, + "baselineMean": 13.931034482758621, + "excess": 4, + "patterns": [ + { + "signature": "ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f", + "count": 4, + "baselineMean": 0, + "excess": 4, + "novel": false, + "clients": 1, + "services": [ + "nginx" + ], + "example": { + "Timestamp": "2026-09-25T18:26:26.152736211Z", + "Service": "nginx", + "Release": "", + "Kind": "span", + "Source": "backend", + "IsError": true, + "StatusClass": "", + "DurationMS": 0, + "TraceID": "d1de7de09ad342e7b0670cb78cac1430", + "PatternSignature": "ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f", + "IssueFingerprint": "", + "IsRetry": false, + "IsTimeout": false, + "IsCancellation": false, + "Message": "", + "SpanName": "", + "Route": "/", + "Host": "cc.k6n.net", + "StatusCode": 405, + "ExceptionType": "", + "ClientIP": "35.189.233.235", + "UAClass": "browser", + "IsDuplicate": false, + "IsRootCause": true, + "HasUpstream": false, + "TraceServices": [ + "nginx" + ], + "BadActor": "" + } + } + ] + }, + "patterns": [ + { + "category": "needs_review", + "priority": "low", + "status": 405, + "host": "cc.k6n.net", + "path": "/", + "services": [ + "nginx" + ], + "rule": "no-rule", + "new": false, + "count": 4, + "baseline": 0, + "clients": 1, + "what": "HTTP 405 | /" + } + ] + } + } +] diff --git a/gitops/README.md b/gitops/README.md new file mode 100644 index 0000000..1e27d17 --- /dev/null +++ b/gitops/README.md @@ -0,0 +1,29 @@ +# Deploying go-otel with k8s-helper + +Gitea Actions builds and pushes versioned and `latest` `matst80/otel-debug` +images on pushes to `main`, uploads the matching source maps, and then sends a +signed reconcile webhook. The agent applies the explicit `k8s/kustomization.yaml` +resource list into namespace `monitoring`. + +## One-time setup + +The k8s-helper GitOps CRDs and agent must be installed. The GitSource reuses +`dev/gitea-k8s-helper`, whose token must have read access to this repository. +Configure a Gitea push-only webhook for this repo at +`https://ui.k6n.net/api/gitops/webhooks/gitea`, using the same secret as +`K8SHELPER_GITOPS_WEBHOOK_SECRET` on the dashboard. Add that value as the Gitea +Actions secret `GITOPS_WEBHOOK_SECRET` too; a user- or organization-level +secret can be shared by other repos. + +The runner uses its existing Docker login. Bootstrap once with +`kubectl apply -f gitops/` after confirming the monitoring namespace and the +`otel-sourcemaps` PVC already exist. The PVC is deliberately excluded from the +managed kustomization because its lifecycle was called out for separate review. +The ClickHouse manifests, auth-dev resources, and ingress-catchall work are also +excluded; this kustomization lists only the receiver Deployment, Service, and +Ingress resources. Do not run Flux and k8s-helper against the same resources at +the same time; check for an active `flux-system/go-otel` Kustomization before +bootstrapping this controller. + +Prune is initially disabled. Inspect the GitOps inventory and applied revision +before changing it. diff --git a/gitops/deployment.yaml b/gitops/deployment.yaml new file mode 100644 index 0000000..01604be --- /dev/null +++ b/gitops/deployment.yaml @@ -0,0 +1,14 @@ +apiVersion: gitops.k6n.net/v1alpha1 +kind: Deployment +metadata: + name: go-otel + namespace: dev +spec: + sourceRef: + kind: GitSource + name: go-otel + path: ./k8s + targetNamespace: monitoring + prune: false + wait: true + deletionPolicy: Orphan diff --git a/gitops/gitsource.yaml b/gitops/gitsource.yaml new file mode 100644 index 0000000..174677a --- /dev/null +++ b/gitops/gitsource.yaml @@ -0,0 +1,16 @@ +# Keep GitOps CRs outside the path they manage. +apiVersion: gitops.k6n.net/v1alpha1 +kind: GitSource +metadata: + name: go-otel + namespace: dev +spec: + url: https://git.k6n.net/mats/go-otel.git + ref: + branch: main + interval: 5m + secretRef: + name: gitea-k8s-helper + ignore: | + .gitea/ + docs/ diff --git a/internal/alerts/laya.go b/internal/alerts/laya.go new file mode 100644 index 0000000..49d4b96 --- /dev/null +++ b/internal/alerts/laya.go @@ -0,0 +1,180 @@ +package alerts + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "sort" + "strconv" + "strings" + "time" +) + +// Laya asks the laya model (POST {URL}/predict) what it would do about an +// alert. It runs in shadow mode: the verdict is recorded on the alert and +// shown in notifications, but nothing is ever acted on, muted or dropped +// because of it. A failed call leaves the alert exactly as it was. +type Laya struct { + URL string // base URL, e.g. https://laya.k6n.net + Client *http.Client +} + +// LayaActions are the remediations laya may choose between. They are +// suggestions only; none is wired to an automated action yet. +var LayaActions = map[string]string{ + "ignore": "transient blip or noise, nobody needs to look at it", + "restart_deployment": "the service is stuck, leaking or crashed; restarting its pods is likely to clear it", + "check_capacity": "requests time out or resources are exhausted; check load, connection pools, CPU/memory limits and scaling", + "fix_dependency": "a database, upstream service or network path this service depends on is failing", + "investigate_code": "a bug or bad release in this service; needs a code fix or a rollback", +} + +var layaUrgency = []string{"cosmetic", "degraded performance", "service outage"} + +// LayaVerdict is laya's answer, reduced to what a person needs to judge it. +type LayaVerdict struct { + Model string `json:"model,omitempty"` + Action string `json:"action"` + ActionConfidence float64 `json:"actionConfidence"` // probability of the chosen action + ActionProbs map[string]float64 `json:"actionProbs,omitempty"` + Human float64 `json:"human"` // probability a person must intervene + Urgency string `json:"urgency"` + UrgencyScore float64 `json:"urgencyScore"` // expected value on 0 (cosmetic) .. 2 (outage) + UrgencyConf float64 `json:"urgencyConfidence"` +} + +func layaRequest(state string) map[string]any { + return map[string]any{ + "state": state, + "questions": map[string]any{ + "remediation_action": map[string]any{ + "type": "choice", + "instructions": "Which operational response fits this production error alert best?", + "criteria": LayaActions, + }, + "requires_human_attention": map[string]any{ + "type": "noul", + "instructions": "Does this alert need a human engineer to intervene, or will it resolve or matter little on its own?", + "criteria": map[string]string{ + "true": "errors are new or rising well above normal and users or requests are failing", + "false": "a transient blip, client noise or bot traffic", + }, + }, + "urgency": map[string]any{ + "type": "score", + "instructions": "Rate the operational severity of this alert.", + "criteria": layaUrgency, + }, + }, + } +} + +type layaAnswer struct { + Choice string `json:"choice"` + Probabilities map[string]float64 `json:"probabilities"` + AnswerConfidence float64 `json:"answer_confidence"` + Noul float64 `json:"noul"` + Score float64 `json:"score"` + Legend map[string]string `json:"legend"` +} + +type layaResponse struct { + Model string `json:"model"` + Answers map[string]layaAnswer `json:"answers"` +} + +// Classify asks laya about the alert. The state is the alert's plain-text +// message, so the model sees exactly the facts a person would, nothing more. +func (l *Laya) Classify(ctx context.Context, a Alert) (*LayaVerdict, error) { + a.Laya = nil + return l.classifyState(ctx, a.Text("")) +} + +func (l *Laya) classifyState(ctx context.Context, state string) (*LayaVerdict, error) { + raw, err := json.Marshal(layaRequest(state)) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, strings.TrimRight(l.URL, "/")+"/predict", bytes.NewReader(raw)) + if err != nil { + return nil, errors.New("laya: invalid URL") + } + req.Header.Set("Content-Type", "application/json") + client := l.Client + if client == nil { + client = &http.Client{Timeout: 10 * time.Second} + } + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("laya: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + detail, _ := io.ReadAll(io.LimitReader(resp.Body, 300)) + return nil, fmt.Errorf("laya: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(detail))) + } + var lr layaResponse + if err := json.NewDecoder(resp.Body).Decode(&lr); err != nil { + return nil, fmt.Errorf("laya: invalid response: %w", err) + } + return lr.verdict() +} + +func (lr layaResponse) verdict() (*LayaVerdict, error) { + act, ok1 := lr.Answers["remediation_action"] + hum, ok2 := lr.Answers["requires_human_attention"] + urg, ok3 := lr.Answers["urgency"] + if !ok1 || !ok2 || !ok3 || act.Choice == "" { + return nil, errors.New("laya: response is missing answers") + } + v := &LayaVerdict{Model: lr.Model, Action: act.Choice, ActionConfidence: act.Probabilities[act.Choice], ActionProbs: act.Probabilities, + Human: hum.Noul, UrgencyScore: urg.Score} + // The urgency label is the most probable bucket, not the rounded score: + // a 50/50 cosmetic/outage split must not read as "degraded". + best := -1.0 + for k, p := range urg.Probabilities { + if p > best || (p == best && k < v.Urgency) { + best, v.Urgency = p, k + } + } + v.UrgencyConf = best + if label, ok := urg.Legend[v.Urgency]; ok { + v.Urgency = label + } else if i, err := strconv.Atoi(v.Urgency); err == nil && i >= 0 && i < len(layaUrgency) { + v.Urgency = layaUrgency[i] + } + return v, nil +} + +// Summary is a one-line rendering, e.g. +// "check_capacity 62% · human 71% · degraded performance 75%". +func (v LayaVerdict) Summary() string { + return fmt.Sprintf("%s %s · human %s · %s %s", v.Action, pct(v.ActionConfidence), pct(v.Human), v.Urgency, pct(v.UrgencyConf)) +} + +// RunnersUp lists the other actions by probability, for judging how close +// the call was. +func (v LayaVerdict) RunnersUp() string { + type kv struct { + k string + p float64 + } + var rest []kv + for k, p := range v.ActionProbs { + if k != v.Action { + rest = append(rest, kv{k, p}) + } + } + sort.Slice(rest, func(i, j int) bool { return rest[i].p > rest[j].p }) + parts := make([]string, 0, len(rest)) + for _, r := range rest { + parts = append(parts, r.k+" "+pct(r.p)) + } + return strings.Join(parts, ", ") +} + +func pct(p float64) string { return strconv.Itoa(int(p*100+0.5)) + "%" } diff --git a/internal/alerts/laya_test.go b/internal/alerts/laya_test.go new file mode 100644 index 0000000..bde6320 --- /dev/null +++ b/internal/alerts/laya_test.go @@ -0,0 +1,123 @@ +package alerts + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "git.tornberg.me/go-otel/internal/features" +) + +// layaReply is the shape laya.k6n.net/predict returned when probed. +const layaReply = `{"model":"laya-rl-agent","answers":{ + "remediation_action":{"type":"choice","choice":"check_capacity","probabilities":{"ignore":0.1,"restart_deployment":0.12,"check_capacity":0.62,"fix_dependency":0.1,"investigate_code":0.06},"confidence":0.3,"answer_confidence":0.62}, + "requires_human_attention":{"type":"noul","noul":0.7123,"confidence":0.7123,"answer_confidence":0.7123}, + "urgency":{"type":"score","score":1.03,"legend":{"0":"cosmetic","1":"degraded performance","2":"service outage"},"probabilities":{"0":0.11,"1":0.75,"2":0.14},"confidence":0.32,"answer_confidence":0.75}}, + "usage":{"input_tokens":141,"output_tokens":0}}` + +func layaServer(t *testing.T, status int, body string, got *map[string]any) *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/predict" { + t.Errorf("path %q", r.URL.Path) + } + if got != nil { + _ = json.NewDecoder(r.Body).Decode(got) + } + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + return srv +} + +func timeoutAlert() Alert { + now := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC) + return NewAlert(features.Spike{Service: "codesearch", WindowStart: now.Add(-5 * time.Minute), WindowEnd: now, ErrorCount: 15, BaselineMean: 0, Excess: 15, + Patterns: []features.PatternDelta{{Signature: "t", Count: 15, Excess: 15, Novel: true, + Example: features.Event{Service: "codesearch", Kind: "span", SpanName: "upload", Message: "context deadline exceeded", IsTimeout: true}}}}, now) +} + +func TestLayaClassifySendsAlertTextAndParsesVerdict(t *testing.T) { + var req map[string]any + srv := layaServer(t, 200, layaReply, &req) + a := timeoutAlert() + v, err := (&Laya{URL: srv.URL + "/"}).Classify(context.Background(), a) + if err != nil { + t.Fatal(err) + } + if req["state"] != a.Text("") { + t.Fatalf("state is not the alert text:\n%v", req["state"]) + } + qs := req["questions"].(map[string]any) + for _, k := range []string{"remediation_action", "requires_human_attention", "urgency"} { + if qs[k] == nil { + t.Fatalf("question %s missing", k) + } + } + if v.Action != "check_capacity" || v.ActionConfidence != 0.62 || v.Human != 0.7123 || v.Urgency != "degraded performance" || v.UrgencyConf != 0.75 { + t.Fatalf("verdict %+v", v) + } + if got := v.Summary(); got != "check_capacity 62% · human 71% · degraded performance 75%" { + t.Fatalf("summary %q", got) + } + if got := v.RunnersUp(); !strings.HasPrefix(got, "restart_deployment 12%, ") { + t.Fatalf("runners-up %q", got) + } +} + +func TestLayaErrors(t *testing.T) { + for name, tc := range map[string]struct { + status int + body string + }{ + "http": {502, "bad gateway"}, + "json": {200, "not json"}, + "missing": {200, `{"answers":{"urgency":{}}}`}, + } { + srv := layaServer(t, tc.status, tc.body, nil) + if _, err := (&Laya{URL: srv.URL}).Classify(context.Background(), timeoutAlert()); err == nil { + t.Errorf("%s: want error", name) + } + } +} + +func TestLayaVerdictInNotifications(t *testing.T) { + a := timeoutAlert() + a.Laya = &LayaVerdict{Action: "restart_deployment", ActionConfidence: 0.8, ActionProbs: map[string]float64{"restart_deployment": 0.8, "ignore": 0.2}, Human: 0.4, Urgency: "service outage", UrgencyConf: 0.6} + h := a.HTML("https://otel.example", telegramMax) + if !strings.Contains(h, "🤖 laya (shadow, no action taken): restart_deployment 80% · human 40% · service outage 60%") || + !strings.Contains(h, "other: ignore 20%") { + t.Fatalf("html:\n%s", h) + } + if txt := a.Text(""); !strings.Contains(txt, "laya (shadow, no action taken): restart_deployment 80%") { + t.Fatalf("text:\n%s", txt) + } + a.Laya = nil + if strings.Contains(a.HTML("", telegramMax), "laya") { + t.Fatal("no verdict must render no laya line") + } +} + +// A laya outage must never cost a page; the alert just goes out without a verdict. +func TestRunnerAttachesVerdictAndSurvivesLayaFailure(t *testing.T) { + for _, tc := range []struct { + status int + want bool + }{{200, true}, {500, false}} { + srv := layaServer(t, tc.status, layaReply, nil) + f := &fake{} + r := newRunner(nil, f) + r.laya = &Laya{URL: srv.URL} + r.deliver(context.Background(), timeoutAlert()) + if len(f.sent) != 1 { + t.Fatalf("status %d: sent %d", tc.status, len(f.sent)) + } + if got := f.sent[0].Laya != nil; got != tc.want { + t.Fatalf("status %d: verdict attached=%v", tc.status, got) + } + } +} diff --git a/internal/alerts/notify.go b/internal/alerts/notify.go index 07215c2..7e04f33 100644 --- a/internal/alerts/notify.go +++ b/internal/alerts/notify.go @@ -29,6 +29,9 @@ type Alert struct { Spike features.Spike `json:"spike"` Patterns []PatternReport `json:"patterns"` Test bool `json:"test,omitempty"` + // Laya is the laya model's shadow verdict, when it was asked. Advisory + // only: nothing is acted on or suppressed because of it. + Laya *LayaVerdict `json:"laya,omitempty"` } // PatternReport is a spike pattern with its triage verdict, ready to show. @@ -214,6 +217,9 @@ func (a Alert) Text(baseURL string) string { window := a.Spike.WindowEnd.Sub(a.Spike.WindowStart).Round(time.Minute) fmt.Fprintf(&b, "%s: %s is failing (%d errors in the last %s, normally about %.0f)\n", prefix, a.Spike.Service, a.Spike.ErrorCount, window, a.Spike.BaselineMean) + if a.Laya != nil { + fmt.Fprintf(&b, "laya (shadow, no action taken): %s\n", a.Laya.Summary()) + } base := strings.TrimRight(baseURL, "/") for i, p := range a.Patterns { fmt.Fprintf(&b, "\n%d. %s\n", i+1, p.What) @@ -282,6 +288,13 @@ func (a Alert) HTML(baseURL string, max int) string { if base != "" { foot = fmt.Sprintf("\nAll issues", esc(base+"/issues")) } + if a.Laya != nil { + head += fmt.Sprintf("🤖 laya (shadow, no action taken): %s %s · human %s · %s %s\n", + esc(a.Laya.Action), pct(a.Laya.ActionConfidence), pct(a.Laya.Human), esc(a.Laya.Urgency), pct(a.Laya.UrgencyConf)) + if r := a.Laya.RunnersUp(); r != "" { + head += " other: " + esc(r) + "\n" + } + } var b strings.Builder b.WriteString(head) for i, p := range a.Patterns { diff --git a/internal/alerts/runner.go b/internal/alerts/runner.go index e2aecad..93ec8ca 100644 --- a/internal/alerts/runner.go +++ b/internal/alerts/runner.go @@ -34,6 +34,9 @@ type Config struct { Muted func(issueID string) bool WebhookURL, WebhookSecret string AODURL string + // LayaURL, when set, asks laya (POST {LayaURL}/predict) for a shadow + // verdict on every alert and shows it in the notification. + LayaURL string } func ConfigFromEnv() Config { @@ -54,6 +57,7 @@ func ConfigFromEnv() Config { WebhookURL: os.Getenv("OTEL_ALERT_WEBHOOK_URL"), WebhookSecret: os.Getenv("OTEL_ALERT_WEBHOOK_SECRET"), AODURL: os.Getenv("OTEL_ALERT_AOD_URL"), + LayaURL: os.Getenv("OTEL_ALERT_LAYA_URL"), } } @@ -106,6 +110,7 @@ type Runner struct { dataSince func(ctx context.Context) time.Time notifiers []Notifier resolver Resolver // optional: ingress -> service -> deployment for edge errors + laya *Laya // optional: shadow verdict attached to each alert sup *features.Suppressor mu sync.Mutex @@ -137,6 +142,10 @@ func Start(ctx context.Context, cfg Config, conn driver.Conn, database string, r }, dataSince: func(ctx context.Context) time.Time { return DataSince(ctx, conn, database) }, } + if cfg.LayaURL != "" { + r.laya = &Laya{URL: cfg.LayaURL} + log.Printf("alerts: laya shadow verdicts from %s", cfg.LayaURL) + } names := make([]string, 0, len(notifiers)) for _, n := range notifiers { names = append(names, n.Name()) @@ -239,6 +248,7 @@ func (r *Runner) deliver(ctx context.Context, a Alert) { if pr, ok := r.resolver.(PodResolver); ok { a.EnrichPods(ctx, pr) } + r.askLaya(ctx, &a) r.mu.Lock() r.recent = append(r.recent, a) if len(r.recent) > maxRecent { @@ -261,6 +271,23 @@ func (r *Runner) deliver(ctx context.Context, a Alert) { } } +// askLaya attaches laya's shadow verdict. It is bounded so a slow model +// delays a page by seconds at most, and a failure only costs the verdict. +func (r *Runner) askLaya(ctx context.Context, a *Alert) { + if r.laya == nil { + return + } + lctx, cancel := context.WithTimeout(ctx, 8*time.Second) + defer cancel() + v, err := r.laya.Classify(lctx, *a) + if err != nil { + log.Printf("alerts: laya verdict for %s failed: %v", a.ID, err) + return + } + a.Laya = v + log.Printf("alerts: laya[shadow] %s: %s", a.ID, v.Summary()) +} + // Status is the JSON served by GET /api/alerts. type Status struct { Enabled bool `json:"enabled"` @@ -305,6 +332,7 @@ func (r *Runner) HandleTest(w http.ResponseWriter, req *http.Request) { Example: features.Event{Service: "alert-test", Kind: "span", StatusCode: 500, Route: "/api/example", Message: "this is a test alert"}, }}}, now) a.Test = true + r.askLaya(req.Context(), &a) results := map[string]string{} for _, n := range r.notifiers { if err := n.Send(req.Context(), a); err != nil { diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index f8d569d..201b195 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -15,6 +15,9 @@ spec: metadata: labels: app: otel-debug-receiver + annotations: + k8s-helper.io/git-url: "https://git.k6n.net/mats/go-otel" + k8s-helper.io/git-ssh: "git@git.k6n.net:mats/go-otel.git" spec: serviceAccountName: otel-debug-receiver securityContext: @@ -138,6 +141,9 @@ spec: # Optional Always-On Display session API. - name: OTEL_ALERT_AOD_URL value: "" + # laya shadow verdicts: shown in each notification, never acted on. + - name: OTEL_ALERT_LAYA_URL + value: "https://laya.k6n.net" - name: OTEL_STORE_WORKERS value: "4" # Optional rollout observation. The watcher is read-only and uses the @@ -236,18 +242,6 @@ spec: - name: sourcemap-dir persistentVolumeClaim: claimName: otel-sourcemaps - ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: otel-sourcemaps - namespace: monitoring -spec: - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: 1Gi --- apiVersion: v1 kind: ServiceAccount diff --git a/k8s/flux/gitops-dashboard.yaml b/k8s/flux/gitops-dashboard.yaml new file mode 100644 index 0000000..229f03a --- /dev/null +++ b/k8s/flux/gitops-dashboard.yaml @@ -0,0 +1,120 @@ +# OIDC Registration in ecommerce namespace +apiVersion: v1 +kind: Secret +metadata: + name: flux-dashboard-oidc-client-secret + namespace: ecommerce +type: Opaque +stringData: + secret: "f9f44c88d9fda647e4abbb4479f1116611ecc14624bd67ec" +--- +apiVersion: auth.k6n.net/v1alpha1 +kind: AuthClient +metadata: + name: flux-dashboard + namespace: ecommerce + labels: + auth.k6n.net/server: auth +spec: + clientID: flux-dashboard + displayName: Flux GitOps Dashboard + description: Web UI for Flux CD (Weave GitOps) + redirectURIs: + - https://flux.k6n.net/oauth2/callback + scopes: + - openid + - profile + - email + - groups + secretRef: + name: flux-dashboard-oidc-client-secret + key: secret +--- +# Weave GitOps OIDC Secret in flux-system +apiVersion: v1 +kind: Secret +metadata: + name: oidc-auth + namespace: flux-system +type: Opaque +stringData: + clientID: "flux-dashboard" + clientSecret: "f9f44c88d9fda647e4abbb4479f1116611ecc14624bd67ec" + issuerURL: "https://users.k6n.net" + redirectURL: "https://flux.k6n.net/oauth2/callback" + claimUsername: "email" + claimGroups: "roles" +--- +# Weave GitOps Helm OCI Source and HelmRelease +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: weave-gitops + namespace: flux-system +spec: + type: oci + url: oci://ghcr.io/weaveworks/charts + interval: 1h +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: weave-gitops + namespace: flux-system +spec: + interval: 10m + chart: + spec: + chart: weave-gitops + version: "4.0.36" + sourceRef: + kind: HelmRepository + name: weave-gitops + values: + envVars: + - name: KUBERNETES_SERVICE_HOST + value: "10.10.1.102" + - name: KUBERNETES_SERVICE_PORT + value: "6443" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: + - serber + # Use existing oidc-auth secret created above + oidcSecret: + create: false + adminUser: + create: false + ingress: + enabled: true + className: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod + hosts: + - host: flux.k6n.net + paths: + - path: / + pathType: ImplementationSpecific + tls: + - hosts: + - flux.k6n.net + secretName: flux-k6n-tls +--- +# ClusterRoleBinding so authenticated users can view/manage GitOps resources +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: weave-gitops-admin +subjects: + - kind: Group + name: admin + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io diff --git a/k8s/flux/sync-go-otel.yaml b/k8s/flux/sync-go-otel.yaml new file mode 100644 index 0000000..59de9f8 --- /dev/null +++ b/k8s/flux/sync-go-otel.yaml @@ -0,0 +1,28 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: go-otel + namespace: flux-system +spec: + interval: 1m + url: ssh://git@git.k6n.net/mats/go-otel.git + ref: + branch: main + secretRef: + name: go-otel-auth +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: go-otel + namespace: flux-system +spec: + interval: 5m + targetNamespace: monitoring + sourceRef: + kind: GitRepository + name: go-otel + path: "./k8s" + prune: true + wait: true + timeout: 3m diff --git a/k8s/ingress-catchall.yaml b/k8s/ingress-catchall.yaml new file mode 100644 index 0000000..de21cf4 --- /dev/null +++ b/k8s/ingress-catchall.yaml @@ -0,0 +1,173 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ingress-catchall + namespace: ingress-nginx +data: + nginx.conf: | + worker_processes 1; + pid /tmp/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + client_body_temp_path /tmp/client_temp; + proxy_temp_path /tmp/proxy_temp; + fastcgi_temp_path /tmp/fastcgi_temp; + uwsgi_temp_path /tmp/uwsgi_temp; + scgi_temp_path /tmp/scgi_temp; + + server { + listen 8080; + server_name _; + root /usr/share/nginx/html; + error_page 403 =404 /index.html; + + # Only GET is intentionally accepted by this catch-all. + # The "if" is safe here because it only performs a return. + # Serve robots.txt directly with Disallow: / + location = /robots.txt { + default_type text/plain; + return 200 "User-agent: *\nDisallow: /\n"; + } + + # Deliberately trigger an internal 403 so the static error page is + # returned with the externally visible status still set to 404. + location / { + if ($request_method != GET) { + add_header Allow GET always; + return 405; + } + + deny all; + } + + # Keep the error document internal so /index.html also follows the + # catch-all policy instead of becoming a successful public endpoint. + location = /index.html { + internal; + allow all; + } + } + } + index.html: | + + + + + + Not found + + + +
+

404

+

There is nothing here.

+

The requested resource does not exist on this ingress endpoint.

+
+ + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ingress-catchall + namespace: ingress-nginx + labels: + app.kubernetes.io/name: ingress-catchall + app.kubernetes.io/part-of: ingress-nginx +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: ingress-catchall + template: + metadata: + labels: + app.kubernetes.io/name: ingress-catchall + app.kubernetes.io/part-of: ingress-nginx + spec: + automountServiceAccountToken: false + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: nginx + image: nginx:1.27-alpine + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + runAsUser: 101 + volumeMounts: + - name: config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + readOnly: true + - name: config + mountPath: /usr/share/nginx/html/index.html + subPath: index.html + readOnly: true + readinessProbe: + tcpSocket: + port: http + periodSeconds: 5 + livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + volumes: + - name: config + configMap: + name: ingress-catchall +--- +apiVersion: v1 +kind: Service +metadata: + name: ingress-catchall + namespace: ingress-nginx + labels: + app.kubernetes.io/name: ingress-catchall + app.kubernetes.io/part-of: ingress-nginx +spec: + selector: + app.kubernetes.io/name: ingress-catchall + ports: + - name: http + port: 80 + targetPort: http diff --git a/k8s/ingress.yaml b/k8s/ingress.yaml index f66ce65..913d78d 100644 --- a/k8s/ingress.yaml +++ b/k8s/ingress.yaml @@ -85,6 +85,10 @@ metadata: name: otel-debug-ingress namespace: monitoring annotations: + k8s-helper.io/name: "Go OTel" + k8s-helper.io/description: "OpenTelemetry debugging dashboard" + k8s-helper.io/group: "Observability" + k8s-helper.io/icon: "activity" cert-manager.io/cluster-issuer: letsencrypt-prod # OIDC protection via the platform auth service (users.k6n.net): # /oauth2/auth resolves the session cookie / bearer token and answers diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml new file mode 100644 index 0000000..94bdff3 --- /dev/null +++ b/k8s/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - deployment.yaml + - service.yaml + - ingress.yaml diff --git a/k8s/otel-sourcemaps-pvc.example.yaml b/k8s/otel-sourcemaps-pvc.example.yaml new file mode 100644 index 0000000..9e81ee5 --- /dev/null +++ b/k8s/otel-sourcemaps-pvc.example.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: otel-sourcemaps + namespace: monitoring +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi -- 2.54.0