3 lines
212 KiB
JSON
3 lines
212 KiB
JSON
{"enabled":true,"dryRun":false,"channels":["telegram"],"lastRun":"2026-09-25T18:56:43.3470721Z","alerts":[{"id":"tlxng1-nginx","at":"2026-09-25T18:31:13.347300133Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:26:13.347300133Z","windowEnd":"2026-09-25T18:31:13.347300133Z","errorCount":69,"baselineMean":14.03448275862069,"excess":4.9655172413793105,"patterns":[{"signature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","count":4,"baselineMean":0.034482758620689655,"excess":3.9655172413793105,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:31:11.346792393Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"0751de83a2f0ce211f2b7ab1a64d8aba","PatternSignature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"auth.k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"34.177.93.136","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"fb485cd240e1dab0261891dc14170112bbc37c90189f276b45935afdedf534cb","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:31:06.252746363Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d05925e26f581fcba99d165ea5928114","PatternSignature":"fb485cd240e1dab0261891dc14170112bbc37c90189f276b45935afdedf534cb","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/oauth2/auth","Host":"users.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"auth.k6n.net","path":"/","services":["nginx"],"routes":[{"ingress":"dev/zitadel-ingress","host":"auth.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"zitadel-dev","port":"8080","deployments":["zitadel-dev"]}],"pods":[{"namespace":"dev","deployment":"zitadel-dev","found":false}],"rule":"status-5xx-upstream","new":false,"count":4,"baseline":0.034482758620689655,"clients":1,"what":"HTTP 503 | /"},{"category":"client_credential_issue","priority":"low","status":401,"host":"users.k6n.net","path":"/oauth2/auth","services":["nginx"],"routes":[{"ingress":"ecommerce/ecommerce-auth","host":"users.k6n.net","path":"/","pathType":"Prefix","namespace":"ecommerce","service":"auth","port":"8080","deployments":["auth"]}],"pods":[{"namespace":"ecommerce","deployment":"auth","pod":"auth-8f497c689-qfpzb","lines":["2026/09/25 18:13:06 oidc: device grant JHMJ-C23E started for client dev-console","2026/09/25 18:13:06 oidc: device grant EQ5R-UWRW approved by mats@tornberg.me","2026/09/25 18:23:19 oidc: device grant RHMN-6257 started for client dev-console"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /oauth2/auth"}]},{"id":"tlxn9d-nginx","at":"2026-09-25T18:27:13.347226389Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:22:13.347226389Z","windowEnd":"2026-09-25T18:27:13.347226389Z","errorCount":64,"baselineMean":13.931034482758621,"excess":4,"patterns":[{"signature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","count":4,"baselineMean":0,"excess":4,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:26:26.152736211Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d1de7de09ad342e7b0670cb78cac1430","PatternSignature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"cc.k6n.net","StatusCode":405,"ExceptionType":"","ClientIP":"35.189.233.235","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"needs_review","priority":"low","status":405,"host":"cc.k6n.net","path":"/","services":["nginx"],"rule":"no-rule","new":false,"count":4,"baseline":0,"clients":1,"what":"HTTP 405 | /"}]},{"id":"tlxn61-nginx","at":"2026-09-25T18:25:13.346661147Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:20:13.346661147Z","windowEnd":"2026-09-25T18:25:13.346661147Z","errorCount":60,"baselineMean":14.03448275862069,"excess":1,"patterns":[{"signature":"ea444cb83ff471aad7f3fab8a5b36dddb25f333f7dce2eb4ff27f6e9bb3969f7","count":1,"baselineMean":0,"excess":1,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:24:36.217473837Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"b52e540166ef48a42f0c0c1c504a6d49","PatternSignature":"ea444cb83ff471aad7f3fab8a5b36dddb25f333f7dce2eb4ff27f6e9bb3969f7","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/robots.txt","Host":"bot.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"74.7.241.187","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":502,"host":"bot.k6n.net","path":"/robots.txt","services":["nginx"],"routes":[{"ingress":"home/telegram-ai-bot-ingress","host":"bot.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"telegram-ai-bot-service","port":"8092"}],"rule":"status-5xx-upstream","new":true,"count":1,"baseline":0,"clients":1,"uaClass":"ai_crawler","what":"HTTP 502 | /robots.txt"}]},{"id":"tlxmxp-nginx","at":"2026-09-25T18:20:13.346876969Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:15:13.346876969Z","windowEnd":"2026-09-25T18:20:13.346876969Z","errorCount":63,"baselineMean":13.96551724137931,"excess":2,"patterns":[{"signature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","count":2,"baselineMean":0,"excess":2,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:19:16.012002529Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"073248aba0d908063aeff887fbd8061e","PatternSignature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"94-255-139-35.cust.bredband2.com","StatusCode":405,"ExceptionType":"","ClientIP":"195.178.110.247","UAClass":"unknown","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"needs_review","priority":"low","status":405,"host":"94-255-139-35.cust.bredband2.com","path":"/","services":["nginx"],"rule":"no-rule","new":false,"count":2,"baseline":0,"clients":1,"what":"HTTP 405 | /"}]},{"id":"tlxmsp-nginx","at":"2026-09-25T18:17:13.346624365Z","spike":{"service":"nginx","windowStart":"2026-09-25T18:12:13.346624365Z","windowEnd":"2026-09-25T18:17:13.346624365Z","errorCount":60,"baselineMean":13.793103448275861,"excess":59,"patterns":[{"signature":"55c1fa26cf30fc45c2011897bca61d2a94dbc04444d973a94a4b3bc668527ccf","count":57,"baselineMean":0,"excess":57,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:17:11.164791167Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"1e1afe400019c5197996dab1603c2d3d","PatternSignature":"55c1fa26cf30fc45c2011897bca61d2a94dbc04444d973a94a4b3bc668527ccf","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/auth/device/token","Host":"users.k6n.net","StatusCode":400,"ExceptionType":"","ClientIP":"","UAClass":"library","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"1f3b735bd556874994fd713d10b6d65e38e4a8f13fb5b644f383cfbf020ee4d9","count":2,"baselineMean":0,"excess":2,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T18:13:10.98074347Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"f14f9846a455d0a2ada6516226285a81","PatternSignature":"1f3b735bd556874994fd713d10b6d65e38e4a8f13fb5b644f383cfbf020ee4d9","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/favicon.ico","Host":"users.k6n.net","StatusCode":405,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"needs_review","priority":"low","status":400,"host":"users.k6n.net","path":"/api/auth/device/token","services":["nginx"],"routes":[{"ingress":"ecommerce/ecommerce-auth","host":"users.k6n.net","path":"/","pathType":"Prefix","namespace":"ecommerce","service":"auth","port":"8080","deployments":["auth"]}],"pods":[{"namespace":"ecommerce","deployment":"auth","pod":"auth-8f497c689-qfpzb","lines":["2026/09/25 18:11:59 oidc: device grant EQ5R-UWRW started for client dev-console","2026/09/25 18:13:06 oidc: device grant JHMJ-C23E started for client dev-console","2026/09/25 18:13:06 oidc: device grant EQ5R-UWRW approved by mats@tornberg.me"],"found":true}],"rule":"no-rule","new":false,"count":57,"baseline":0,"uaClass":"library","what":"HTTP 400 | /api/auth/device/token"},{"category":"needs_review","priority":"low","status":405,"host":"users.k6n.net","path":"/favicon.ico","services":["nginx"],"routes":[{"ingress":"ecommerce/ecommerce-auth","host":"users.k6n.net","path":"/","pathType":"Prefix","namespace":"ecommerce","service":"auth","port":"8080","deployments":["auth"]}],"pods":[{"namespace":"ecommerce","deployment":"auth","pod":"auth-8f497c689-qfpzb","lines":["2026/09/25 18:11:59 oidc: device grant EQ5R-UWRW started for client dev-console","2026/09/25 18:13:06 oidc: device grant JHMJ-C23E started for client dev-console","2026/09/25 18:13:06 oidc: device grant EQ5R-UWRW approved by mats@tornberg.me"],"found":true}],"rule":"no-rule","new":false,"count":2,"baseline":0,"what":"HTTP 405 | /favicon.ico"}]},{"id":"tlxmh1-eventsourced-orchestrator","at":"2026-09-25T18:10:13.34664129Z","spike":{"service":"eventsourced-orchestrator","windowStart":"2026-09-25T18:05:13.34664129Z","windowEnd":"2026-09-25T18:10:13.34664129Z","errorCount":16,"baselineMean":19.93103448275862,"excess":16,"patterns":[{"signature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","count":16,"baselineMean":0,"excess":16,"novel":false,"services":["eventsourced-orchestrator"],"example":{"Timestamp":"2026-09-25T18:10:08.793065336Z","Service":"eventsourced-orchestrator","Release":"","Kind":"log","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"","PatternSignature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","IssueFingerprint":"b57379b9dd3a6a9d","IsRetry":false,"IsTimeout":true,"IsCancellation":false,"Message":"codesearch upload failed","SpanName":"","Route":"","Host":"","StatusCode":0,"ExceptionType":"context.deadlineExceededError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":false,"HasUpstream":false,"TraceServices":null,"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","issueId":"b57379b9dd3a6a9d","services":["eventsourced-orchestrator"],"rule":"timeout-flag","new":false,"count":16,"baseline":0,"what":"context.deadlineExceededError | codesearch upload failed"}]},{"id":"tlxmdp-eventsourced-worker-code","at":"2026-09-25T18:08:13.346954479Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T18:03:13.346954479Z","windowEnd":"2026-09-25T18:08:13.346954479Z","errorCount":36,"baselineMean":0.13793103448275862,"excess":36,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":29,"baselineMean":0,"excess":29,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T18:08:12.308751715Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"34f7c7c1d9c07bb48b3241454d10a7bf","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}},{"signature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","count":7,"baselineMean":0,"excess":7,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T18:07:50.177441418Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"72e0cbf2c415b129eab8f4aadca42c9f","PatternSignature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","IssueFingerprint":"b40901e8fbfcf66e","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: eof","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":29,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"},{"category":"dependency_unreachable","priority":"high","issueId":"b40901e8fbfcf66e","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":7,"baseline":0,"what":"worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"}]},{"id":"tlxjp1-eventsourced-worker-code","at":"2026-09-25T17:10:13.347080773Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T17:05:13.347080773Z","windowEnd":"2026-09-25T17:10:13.347080773Z","errorCount":66,"baselineMean":0.4482758620689655,"excess":60,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":60,"baselineMean":0,"excess":60,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T17:10:12.613732024Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"a9d858213f84b6dcf310c4701d1c0a39","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":60,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"}]},{"id":"tlxitd-eventsourced-orchestrator","at":"2026-09-25T16:51:13.346883436Z","spike":{"service":"eventsourced-orchestrator","windowStart":"2026-09-25T16:46:13.346883436Z","windowEnd":"2026-09-25T16:51:13.346883436Z","errorCount":15,"baselineMean":8.310344827586206,"excess":15,"patterns":[{"signature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","count":15,"baselineMean":0,"excess":15,"novel":false,"services":["eventsourced-orchestrator"],"example":{"Timestamp":"2026-09-25T16:50:38.820074629Z","Service":"eventsourced-orchestrator","Release":"","Kind":"log","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"","PatternSignature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","IssueFingerprint":"b57379b9dd3a6a9d","IsRetry":false,"IsTimeout":true,"IsCancellation":false,"Message":"codesearch upload failed","SpanName":"","Route":"","Host":"","StatusCode":0,"ExceptionType":"context.deadlineExceededError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":false,"HasUpstream":false,"TraceServices":null,"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","issueId":"b57379b9dd3a6a9d","services":["eventsourced-orchestrator"],"rule":"timeout-flag","new":false,"count":15,"baseline":0,"what":"context.deadlineExceededError | codesearch upload failed"}]},{"id":"tlxirp-nginx","at":"2026-09-25T16:50:13.347044487Z","spike":{"service":"nginx","windowStart":"2026-09-25T16:45:13.347044487Z","windowEnd":"2026-09-25T16:50:13.347044487Z","errorCount":62,"baselineMean":18.93103448275862,"excess":41.96551724137931,"patterns":[{"signature":"507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d","count":17,"baselineMean":0,"excess":17,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T16:50:05.564200851Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"51f8278d03367c21a21f76835ec204b2","PatternSignature":"507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai-worker/manifests/latest","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}},{"signature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","count":14,"baselineMean":0,"excess":14,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T16:48:10.547871686Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"6664346e0c22d3a9396460163f0b0d6b","PatternSignature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"library","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}},{"signature":"ec8f49cca245ee954c6f7a7839235c13a8cfaabc1f0bcac8ecd29c88a5533891","count":5,"baselineMean":0.034482758620689655,"excess":4.9655172413793105,"novel":false,"clients":1,"services":["eventsourced-orchestrator","nginx"],"example":{"Timestamp":"2026-09-25T16:49:50.586320711Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"2b5d0ed3c205922143afa2bc0c5b0b8e","PatternSignature":"ec8f49cca245ee954c6f7a7839235c13a8cfaabc1f0bcac8ecd29c88a5533891","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/auth/me","Host":"event-agent.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"170.62.100.17","UAClass":"browser","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["eventsourced-orchestrator","nginx"],"BadActor":""}},{"signature":"10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b","count":3,"baselineMean":0,"excess":3,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T16:49:50.586320711Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"44416b3fb21e6f57b9bba5ac0eccd68b","PatternSignature":"10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b","IssueFingerprint":"","IsRetry":false,"IsTimeout":true,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/stream","Host":"event-agent.k6n.net","StatusCode":504,"ExceptionType":"","ClientIP":"170.62.100.17","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"f524391f412efceb1222fd1ccb3c5fcb673ace4787f214acaf300dea6cc88115","count":3,"baselineMean":0,"excess":3,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T16:48:10.577418006Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"b870baab8dd2de51be3f9da1663d239e","PatternSignature":"f524391f412efceb1222fd1ccb3c5fcb673ace4787f214acaf300dea6cc88115","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/auth/me","Host":"event-agent.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"170.62.100.17","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","status":504,"host":"event-agent.k6n.net","path":"/api/stream","services":["nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-5865685999-xtxdz","lines":["10.244.9.1 - - [25/Sep/2026:16:50:20 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:16:50:30 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:16:50:40 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"timeout-flag","new":false,"count":3,"baseline":0,"clients":1,"what":"HTTP 504 | /api/stream"},{"category":"dependency_unreachable","priority":"high","status":502,"host":"event-agent.k6n.net","path":"/auth/me","services":["nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-5865685999-xtxdz","lines":["10.244.9.1 - - [25/Sep/2026:16:50:20 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:16:50:30 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:16:50:40 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"status-5xx-upstream","new":false,"count":3,"baseline":0,"clients":1,"what":"HTTP 502 | /auth/me"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai-worker/manifests/latest","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:16:50:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T16:50:39.183971552Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"85d74dfb059f5fbff7018c85c990e2bc\\\",\\\"SpanID\\\":\\\"dfd025e674cceb15\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T16:50:38.934252898Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:38.934346788Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":54180}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T16:50:39.184168228Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"b389a9064776af9f4a23cc817ee4231f\\\",\\\"SpanID\\\":\\\"16fa1d622c943f61\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T16:50:39.102298404Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:39.105804949Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":17,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai-worker/manifests/latest"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:16:50:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T16:50:39.183971552Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"85d74dfb059f5fbff7018c85c990e2bc\\\",\\\"SpanID\\\":\\\"dfd025e674cceb15\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T16:50:38.934252898Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:38.934346788Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":54180}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T16:50:39.184168228Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"b389a9064776af9f4a23cc817ee4231f\\\",\\\"SpanID\\\":\\\"16fa1d622c943f61\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T16:50:39.102298404Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:39.105804949Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":14,"baseline":0,"uaClass":"library","what":"HTTP 401 | /v2"},{"category":"client_credential_issue","priority":"low","status":401,"host":"event-agent.k6n.net","path":"/auth/me","services":["eventsourced-orchestrator","nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-5865685999-xtxdz","lines":["10.244.9.1 - - [25/Sep/2026:16:50:20 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:16:50:30 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:16:50:40 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"status-401","new":false,"count":5,"baseline":0.034482758620689655,"clients":1,"what":"HTTP 401 | /auth/me"}]},{"id":"tlxiq1-eventsourced-worker-code","at":"2026-09-25T16:49:13.347152978Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T16:44:13.347152978Z","windowEnd":"2026-09-25T16:49:13.347152978Z","errorCount":35,"baselineMean":0.6206896551724138,"excess":35,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":28,"baselineMean":0,"excess":28,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T16:48:57.813919594Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"ae2f322da11ef7dd36f14ffc6e391321","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}},{"signature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","count":7,"baselineMean":0,"excess":7,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T16:48:10.959364475Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"b7adf673cf07369996548673a85a9060","PatternSignature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","IssueFingerprint":"b40901e8fbfcf66e","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: eof","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":28,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"},{"category":"dependency_unreachable","priority":"high","issueId":"b40901e8fbfcf66e","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":7,"baseline":0,"what":"worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"}]},{"id":"tlxgbd-nginx","at":"2026-09-25T15:57:13.347060965Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:52:13.347060965Z","windowEnd":"2026-09-25T15:57:13.347060965Z","errorCount":72,"baselineMean":18.413793103448278,"excess":31.448275862068968,"patterns":[{"signature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","count":40,"baselineMean":8.551724137931034,"excess":31.448275862068968,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:57:06.60535739Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"7654e99020c8f9de172e1bf4cbd65aa3","PatternSignature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/metrics/query","Host":"ui.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":502,"host":"ui.k6n.net","path":"/api/metrics/query","services":["nginx"],"routes":[{"ingress":"dev/k8s-helper-ingress","host":"ui.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"k8s-helper","port":"80","deployments":["k8s-helper"]}],"pods":[{"namespace":"dev","deployment":"k8s-helper","pod":"k8s-helper-5665cd69c9-kjd2t","lines":["I0925 15:56:34.658960 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 15:57:20.162724 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 15:57:24.894618 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\""],"found":true}],"rule":"status-5xx-upstream","new":false,"count":40,"baseline":8.551724137931034,"what":"HTTP 502 | /api/metrics/query"}]},{"id":"tlxg6d-nginx","at":"2026-09-25T15:54:13.346717682Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:49:13.346717682Z","windowEnd":"2026-09-25T15:54:13.346717682Z","errorCount":74,"baselineMean":18.448275862068964,"excess":12,"patterns":[{"signature":"0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e","count":4,"baselineMean":0,"excess":4,"novel":false,"clients":2,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:53:46.266195024Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"ecdf390536970c3339612fb2a201df51","PatternSignature":"0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/designer/v1/file-content","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"104.23.160.37","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6","count":4,"baselineMean":0,"excess":4,"novel":false,"clients":2,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:53:46.266195024Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"744692761138c480f27fac58b3aa1fda","PatternSignature":"b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/templates/preview","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"104.23.160.37","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab","count":2,"baselineMean":0,"excess":2,"novel":false,"clients":2,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:53:46.266195024Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"7ffa5f428b4de59811dd86ded001a33c","PatternSignature":"004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/fs/exec","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"104.23.160.37","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"7281b16d4395d124b0fac7a82520ee2b9e179df689e8f4eb6d515a6b567aa256","count":2,"baselineMean":0,"excess":2,"novel":false,"clients":2,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:53:46.266195024Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"942867286e5ba5cac5ec2102cd059106","PatternSignature":"7281b16d4395d124b0fac7a82520ee2b9e179df689e8f4eb6d515a6b567aa256","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/v1/build_public_tmp/{id}/flow","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"104.23.160.37","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/designer/v1/file-content","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":4,"baseline":0,"clients":2,"uaClass":"ai_crawler","what":"HTTP 503 | /api/designer/v1/file-content"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/templates/preview","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":4,"baseline":0,"clients":2,"uaClass":"ai_crawler","what":"HTTP 503 | /api/templates/preview"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/fs/exec","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":2,"baseline":0,"clients":2,"uaClass":"ai_crawler","what":"HTTP 503 | /api/fs/exec"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/v1/build_public_tmp/{id}/flow","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":2,"baseline":0,"clients":2,"uaClass":"ai_crawler","what":"HTTP 503 | /api/v1/build_public_tmp/{id}/flow"}]},{"id":"tlxg31-nginx","at":"2026-09-25T15:52:13.34677222Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:47:13.34677222Z","windowEnd":"2026-09-25T15:52:13.34677222Z","errorCount":44,"baselineMean":18.413793103448278,"excess":1,"patterns":[{"signature":"4f0026d193a2d63deed6f864410365d22dc0a941000e445e1bd3340a553df1c6","count":1,"baselineMean":0,"excess":1,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:52:10.518612164Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"ecf6036348e1d53bf4a441e861f14ca3","PatternSignature":"4f0026d193a2d63deed6f864410365d22dc0a941000e445e1bd3340a553df1c6","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/auth/session","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"172.69.130.15","UAClass":"other_bot","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/auth/session","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":1,"baseline":0,"clients":1,"uaClass":"other_bot","what":"HTTP 503 | /api/auth/session"}]},{"id":"tlxfkp-nginx","at":"2026-09-25T15:41:13.346872997Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:36:13.346872997Z","windowEnd":"2026-09-25T15:41:13.346872997Z","errorCount":73,"baselineMean":16.896551724137932,"excess":5,"patterns":[{"signature":"cfb79f6686e37386a47e979141249f6e98eb12a891beb2cc2f9c6ef4a03e546a","count":5,"baselineMean":0,"excess":5,"novel":true,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:40:16.478551828Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"0adee7ffe614204eab8ee2af9893a095","PatternSignature":"cfb79f6686e37386a47e979141249f6e98eb12a891beb2cc2f9c6ef4a03e546a","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/laya-api/blobs/{id}","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/laya-api/blobs/{id}","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:15:41:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T15:41:42.894679186Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f41801a22b3d7f5900080d1a819bf56f\\\",\\\"SpanID\\\":\\\"3eb10e4e058eff12\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:41:38.993750605Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:41:38.993853614Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":40190}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:41:42.894853982Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"6975180cdef6ad17b9efbdbe1b7e80e4\\\",\\\"SpanID\\\":\\\"3b0dec7a3a297e39\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:41:39.112850054Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:41:39.151798112Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":true,"count":5,"baseline":0,"what":"HTTP 401 | /v2/laya-api/blobs/{id}"}]},{"id":"tlxffp-nginx","at":"2026-09-25T15:38:13.346714157Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:33:13.346714157Z","windowEnd":"2026-09-25T15:38:13.346714157Z","errorCount":68,"baselineMean":15.551724137931034,"excess":7,"patterns":[{"signature":"0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e","count":2,"baselineMean":0,"excess":2,"novel":false,"clients":2,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:37:36.21665375Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"3d9b70200ecac81e31ffeecfa86054e1","PatternSignature":"0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/designer/v1/file-content","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"162.158.217.104","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"5a9d99056a99f32d99bbbe116c5aa8521d6ed732e15513bbf8053db6d96913bf","count":2,"baselineMean":0,"excess":2,"novel":false,"clients":2,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:37:30.409837455Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"c3153b7817a4cbfc3b6a2357e5885a7b","PatternSignature":"5a9d99056a99f32d99bbbe116c5aa8521d6ed732e15513bbf8053db6d96913bf","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/config","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"162.158.217.105","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6","count":2,"baselineMean":0,"excess":2,"novel":false,"clients":2,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:37:36.21665375Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"b36f6a36de27af8bd9dea775a1dd181e","PatternSignature":"b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/templates/preview","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"162.158.217.104","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab","count":1,"baselineMean":0,"excess":1,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:37:36.21665375Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"5a8f88bfa7368f5346ab25b13ca1b472","PatternSignature":"004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/fs/exec","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"162.158.217.104","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/designer/v1/file-content","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":2,"baseline":0,"clients":2,"uaClass":"ai_crawler","what":"HTTP 503 | /api/designer/v1/file-content"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/config","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":2,"baseline":0,"clients":2,"what":"HTTP 503 | /api/config"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/templates/preview","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":2,"baseline":0,"clients":2,"uaClass":"ai_crawler","what":"HTTP 503 | /api/templates/preview"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/fs/exec","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":false,"count":1,"baseline":0,"clients":1,"uaClass":"ai_crawler","what":"HTTP 503 | /api/fs/exec"}]},{"id":"tlxexd-nginx","at":"2026-09-25T15:27:13.346952193Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:22:13.346952193Z","windowEnd":"2026-09-25T15:27:13.346952193Z","errorCount":44,"baselineMean":14,"excess":38.89655172413793,"patterns":[{"signature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","count":40,"baselineMean":1.103448275862069,"excess":38.89655172413793,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:27:10.338499832Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"8f0569d8963e8d96b1d3e497700ce25b","PatternSignature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/metrics/query","Host":"ui.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":502,"host":"ui.k6n.net","path":"/api/metrics/query","services":["nginx"],"routes":[{"ingress":"dev/k8s-helper-ingress","host":"ui.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"k8s-helper","port":"80","deployments":["k8s-helper"]}],"pods":[{"namespace":"dev","deployment":"k8s-helper","pod":"k8s-helper-5665cd69c9-kjd2t","lines":["I0925 15:26:24.888952 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 15:26:51.998835 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 15:27:24.895350 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\""],"found":true}],"rule":"status-5xx-upstream","new":false,"count":40,"baseline":1.103448275862069,"what":"HTTP 502 | /api/metrics/query"}]},{"id":"tlxevp-nginx","at":"2026-09-25T15:26:13.346647513Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:21:13.346647513Z","windowEnd":"2026-09-25T15:26:13.346647513Z","errorCount":44,"baselineMean":13.724137931034482,"excess":2,"patterns":[{"signature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","count":2,"baselineMean":0,"excess":2,"novel":false,"clients":2,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:25:31.359710287Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"ea2a5c59210e1c1076b971328b6688c0","PatternSignature":"3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"rcon-cs.tornberg.me","StatusCode":503,"ExceptionType":"","ClientIP":"172.69.194.250","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"rcon-cs.tornberg.me","path":"/","services":["nginx"],"routes":[{"ingress":"game/cs-rcon2000","host":"rcon-cs.tornberg.me","path":"/","pathType":"Prefix","namespace":"game","service":"cs-rcon2000","port":"80","deployments":["cs-rcon2000"]}],"pods":[{"namespace":"game","deployment":"cs-rcon2000","found":false}],"rule":"status-5xx-upstream","new":false,"count":2,"baseline":0,"clients":2,"what":"HTTP 503 | /"}]},{"id":"tlxea1-nginx","at":"2026-09-25T15:13:13.346941264Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:08:13.346941264Z","windowEnd":"2026-09-25T15:13:13.346941264Z","errorCount":44,"baselineMean":9.586206896551724,"excess":2.9310344827586206,"patterns":[{"signature":"3c1f3fe95cf095ba60cb1377d69a6a1c2cfe5fb12394303c6d557c8ce3b1fb4c","count":1,"baselineMean":0,"excess":1,"novel":true,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:11:25.232506557Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"138fbe9ca4bbe5aefcb3e012d9aebc86","PatternSignature":"3c1f3fe95cf095ba60cb1377d69a6a1c2cfe5fb12394303c6d557c8ce3b1fb4c","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/resources","Host":"ui.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"dd10eb359ab4e657a6db632468d4e5d1025e9c824562fa1737f1fb0a2c51a512","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["eventsourced-orchestrator","nginx"],"example":{"Timestamp":"2026-09-25T15:12:41.040181855Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"914660c641fd99e4b6c10b3d4b2b325d","PatternSignature":"dd10eb359ab4e657a6db632468d4e5d1025e9c824562fa1737f1fb0a2c51a512","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/overview","Host":"event-agent.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["eventsourced-orchestrator","nginx"],"BadActor":""}},{"signature":"ec8f49cca245ee954c6f7a7839235c13a8cfaabc1f0bcac8ecd29c88a5533891","count":1,"baselineMean":0.06896551724137931,"excess":0.9310344827586207,"novel":false,"services":["eventsourced-orchestrator","nginx"],"example":{"Timestamp":"2026-09-25T15:12:41.040181855Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"8999cc489544cd7ff668240c6ffb1b74","PatternSignature":"ec8f49cca245ee954c6f7a7839235c13a8cfaabc1f0bcac8ecd29c88a5533891","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/auth/me","Host":"event-agent.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["eventsourced-orchestrator","nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":502,"host":"ui.k6n.net","path":"/api/resources","services":["nginx"],"routes":[{"ingress":"dev/k8s-helper-ingress","host":"ui.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"k8s-helper","port":"80","deployments":["k8s-helper"]}],"pods":[{"namespace":"dev","deployment":"k8s-helper","pod":"k8s-helper-5665cd69c9-kjd2t","lines":["I0925 15:12:24.895836 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 15:13:09.988989 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 15:13:24.891348 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\""],"found":true}],"rule":"status-5xx-upstream","new":true,"count":1,"baseline":0,"what":"HTTP 502 | /api/resources"},{"category":"client_credential_issue","priority":"low","status":401,"host":"event-agent.k6n.net","path":"/api/overview","services":["eventsourced-orchestrator","nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-5db7c4c548-r54xs","lines":["10.244.9.1 - - [25/Sep/2026:15:13:31 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:15:13:41 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.5.162 - - [25/Sep/2026:15:13:43 +0000] \"GET /api/overview HTTP/1.1\" 200 279064 \"https://event-agent.k6n.net/session/crying-rabbit-92\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36\" \"10.10.0.1\""],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /api/overview"},{"category":"client_credential_issue","priority":"low","status":401,"host":"event-agent.k6n.net","path":"/auth/me","services":["eventsourced-orchestrator","nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-5db7c4c548-r54xs","lines":["10.244.9.1 - - [25/Sep/2026:15:13:31 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:15:13:41 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.5.162 - - [25/Sep/2026:15:13:43 +0000] \"GET /api/overview HTTP/1.1\" 200 279064 \"https://event-agent.k6n.net/session/crying-rabbit-92\" \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36\" \"10.10.0.1\""],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0.06896551724137931,"what":"HTTP 401 | /auth/me"}]},{"id":"tlxe8d-nginx","at":"2026-09-25T15:12:13.346987906Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:07:13.346987906Z","windowEnd":"2026-09-25T15:12:13.346987906Z","errorCount":49,"baselineMean":9.96551724137931,"excess":1,"patterns":[{"signature":"382021ec653c3879cceb5c06de218d7bf25435b52cb45899720aa54dfd422bf2","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:07:26.21663347Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"c36b7b4d86d2fc34a2e92dca980d630e","PatternSignature":"382021ec653c3879cceb5c06de218d7bf25435b52cb45899720aa54dfd422bf2","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai-worker-web/manifests/{id}","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai-worker-web/manifests/{id}","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:15:12:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T15:12:41.724259905Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"3fc51021cde7f61e31590c3a815cf30b\\\",\\\"SpanID\\\":\\\"0ec55ee4060e641f\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:12:38.935591926Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:12:38.935694744Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":58652}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:12:41.724462705Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"78ad11ca20f16a7adc51ea89a17af451\\\",\\\"SpanID\\\":\\\"9b3c073f369525ee\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:12:39.101643343Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:12:39.10432783Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai-worker-web/manifests/{id}"}]},{"id":"tlxe6p-nginx","at":"2026-09-25T15:11:13.346878467Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:06:13.346878467Z","windowEnd":"2026-09-25T15:11:13.346878467Z","errorCount":50,"baselineMean":9.551724137931034,"excess":3,"patterns":[{"signature":"0b9d38585ec8cbdca191445a42ebbab0b254811df06aec7d26679e5cded6b4e6","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:07:25.201694002Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"ae870ad54232c9689315ca3ca81891dd","PatternSignature":"0b9d38585ec8cbdca191445a42ebbab0b254811df06aec7d26679e5cded6b4e6","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/mcp","Host":"rag.k6n.net","StatusCode":400,"ExceptionType":"","ClientIP":"","UAClass":"library","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:06:46.212551042Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"0a8bdff037a04864cdbc911cd7a0251d","PatternSignature":"10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b","IssueFingerprint":"","IsRetry":false,"IsTimeout":true,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/stream","Host":"event-agent.k6n.net","StatusCode":504,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"2b2c0c74b5d8ee0e6e3814f70aa1654b3d4f43bb8663a2552102ba1cb7457a8b","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:07:25.201694002Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"2db0875de2536941bdd5e4d7836c8748","PatternSignature":"2b2c0c74b5d8ee0e6e3814f70aa1654b3d4f43bb8663a2552102ba1cb7457a8b","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai-kubeworker/manifests/latest","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","status":504,"host":"event-agent.k6n.net","path":"/api/stream","services":["nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-5db7c4c548-r54xs","lines":["10.244.9.1 - - [25/Sep/2026:15:11:21 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:15:11:31 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:15:11:41 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"timeout-flag","new":false,"count":1,"baseline":0,"what":"HTTP 504 | /api/stream"},{"category":"needs_review","priority":"low","status":400,"host":"rag.k6n.net","path":"/mcp","services":["nginx"],"routes":[{"ingress":"home/rag-mcp-ingress","host":"rag.k6n.net","path":"/mcp","pathType":"Prefix","namespace":"home","service":"rag-service-cuda","port":"4001","deployments":["rust-rag-cuda"]}],"pods":[{"namespace":"home","deployment":"rust-rag-cuda","pod":"rust-rag-cuda-7c7454f95b-rl4np","lines":["\u001b[2m2026-09-25T15:10:48.879055Z\u001b[0m \u001b[32m INFO\u001b[0m \u001b[2mrust_rag::db::code_store\u001b[0m\u001b[2m:\u001b[0m codesearch: repo snapshot merged \u001b[3mrepo\u001b[0m\u001b[2m=\u001b[0mgit@git.k6n.net:mats/eventsourced-ai.git \u001b[3mfiles\u001b[0m\u001b[2m=\u001b[0m1164 \u001b[3msymbols\u001b[0m\u001b[2m=\u001b[0m0 \u001b[3mcalls\u001b[0m\u001b[2m=\u001b[0m76358","\u001b[2m2026-09-25T15:11:08.781298Z\u001b[0m \u001b[32m INFO\u001b[0m \u001b[2mrust_rag::db::code_store\u001b[0m\u001b[2m:\u001b[0m codesearch: repo snapshot merged \u001b[3mrepo\u001b[0m\u001b[2m=\u001b[0mgit@git.k6n.net:mats/eventsourced-ai.git \u001b[3mfiles\u001b[0m\u001b[2m=\u001b[0m1164 \u001b[3msymbols\u001b[0m\u001b[2m=\u001b[0m0 \u001b[3mcalls\u001b[0m\u001b[2m=\u001b[0m76358","\u001b[2m2026-09-25T15:11:26.781932Z\u001b[0m \u001b[32m INFO\u001b[0m \u001b[2mrust_rag::db::code_store\u001b[0m\u001b[2m:\u001b[0m codesearch: repo snapshot merged \u001b[3mrepo\u001b[0m\u001b[2m=\u001b[0mgit@git.k6n.net:mats/eventsourced-ai.git \u001b[3mfiles\u001b[0m\u001b[2m=\u001b[0m1164 \u001b[3msymbols\u001b[0m\u001b[2m=\u001b[0m0 \u001b[3mcalls\u001b[0m\u001b[2m=\u001b[0m76358"],"found":true}],"rule":"no-rule","new":false,"count":1,"baseline":0,"uaClass":"library","what":"HTTP 400 | /mcp"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai-kubeworker/manifests/latest","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:15:11:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T15:11:41.690355357Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"5f01dc3778125e823c9e23e1cacbe7f6\\\",\\\"SpanID\\\":\\\"d76882ecc4500e22\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:11:38.934372333Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:11:38.934484718Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":46224}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:11:41.690525348Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"4f549d5b9e73732825879869fe477ae9\\\",\\\"SpanID\\\":\\\"33d6d34949f1017c\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:11:39.102557984Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:11:39.146578046Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai-kubeworker/manifests/latest"}]},{"id":"tlxe51-nginx","at":"2026-09-25T15:10:13.346769076Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:05:13.346769076Z","windowEnd":"2026-09-25T15:10:13.346769076Z","errorCount":67,"baselineMean":9.931034482758621,"excess":2,"patterns":[{"signature":"bc548cc4f9f3a04d38c7be8915ced890a998b00f1af00a79d36e0d32037af40d","count":2,"baselineMean":0,"excess":2,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:07:31.011384824Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"cf55bcead348adb96b029075181f9a91","PatternSignature":"bc548cc4f9f3a04d38c7be8915ced890a998b00f1af00a79d36e0d32037af40d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai-worker-mcp/manifests/latest","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["nginx","docker-registry"],"BadActor":""}}]},"patterns":[{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai-worker-mcp/manifests/latest","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:15:10:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T15:10:41.658410004Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"396027739ba44124a08bf5a3ce582499\\\",\\\"SpanID\\\":\\\"2296151ddf383168\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:10:38.934291682Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:10:38.934400004Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":58652}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:10:41.658574145Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"4062afeab1d8d70ef4bc5922ad82b8f4\\\",\\\"SpanID\\\":\\\"1113a5878ca748cf\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:10:39.102513029Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:10:39.104615689Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":2,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai-worker-mcp/manifests/latest"}]},{"id":"tlxe1p-eventsourced-orchestrator","at":"2026-09-25T15:08:13.346626294Z","spike":{"service":"eventsourced-orchestrator","windowStart":"2026-09-25T15:03:13.346626294Z","windowEnd":"2026-09-25T15:08:13.346626294Z","errorCount":22,"baselineMean":0.27586206896551724,"excess":22,"patterns":[{"signature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","count":22,"baselineMean":0,"excess":22,"novel":false,"services":["eventsourced-orchestrator"],"example":{"Timestamp":"2026-09-25T15:07:30.033409716Z","Service":"eventsourced-orchestrator","Release":"","Kind":"log","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"","PatternSignature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","IssueFingerprint":"b57379b9dd3a6a9d","IsRetry":false,"IsTimeout":true,"IsCancellation":false,"Message":"codesearch upload failed","SpanName":"","Route":"","Host":"","StatusCode":0,"ExceptionType":"context.deadlineExceededError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":false,"HasUpstream":false,"TraceServices":null,"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","issueId":"b57379b9dd3a6a9d","services":["eventsourced-orchestrator"],"rule":"timeout-flag","new":false,"count":22,"baseline":0,"what":"context.deadlineExceededError | codesearch upload failed"}]},{"id":"tlxe01-eventsourced-worker-code","at":"2026-09-25T15:07:13.346691368Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T15:02:13.346691368Z","windowEnd":"2026-09-25T15:07:13.346691368Z","errorCount":45,"baselineMean":1.8620689655172413,"excess":44.310344827586206,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":38,"baselineMean":0.6896551724137931,"excess":37.310344827586206,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T15:05:54.340775446Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"67fa6def37f0feda16f801e8f40194b2","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}},{"signature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","count":7,"baselineMean":0,"excess":7,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T15:05:01.828115407Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"a9475ecf3ae5a4ca0e8e73a7630cb89f","PatternSignature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","IssueFingerprint":"b40901e8fbfcf66e","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: eof","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":38,"baseline":0.6896551724137931,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"},{"category":"dependency_unreachable","priority":"high","issueId":"b40901e8fbfcf66e","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":7,"baseline":0,"what":"worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"}]},{"id":"tlxdyd-nginx","at":"2026-09-25T15:06:13.347093859Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:01:13.347093859Z","windowEnd":"2026-09-25T15:06:13.347093859Z","errorCount":121,"baselineMean":8.379310344827585,"excess":16,"patterns":[{"signature":"507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d","count":14,"baselineMean":0,"excess":14,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:05:36.202277428Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"a4da26a4a32edf925be5858fb04f23fe","PatternSignature":"507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai-worker/manifests/latest","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}},{"signature":"63eff8b04309c018ea6fe0604d6e3b0885a94f3252cdf65ac9e777ae320a3af7","count":2,"baselineMean":0,"excess":2,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T15:05:16.201179908Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"bef5078df057b841700020d2d7128086","PatternSignature":"63eff8b04309c018ea6fe0604d6e3b0885a94f3252cdf65ac9e777ae320a3af7","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/network-speed","Host":"tornberg.me","StatusCode":500,"ExceptionType":"","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"service_defect","priority":"high","status":500,"host":"tornberg.me","path":"/api/network-speed","services":["nginx"],"routes":[{"ingress":"dev/static-web","host":"tornberg.me","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-api","port":"3000","deployments":["go-api"]}],"pods":[{"namespace":"dev","deployment":"go-api","pod":"go-api-85d88c9bd-qp8p4","found":true}],"rule":"status-5xx","new":false,"count":2,"baseline":0,"what":"HTTP 500 | /api/network-speed"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai-worker/manifests/latest","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:15:06:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T15:06:41.534921099Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"5fa8dd0ea0a6e33d84a5d5cdce9938fb\\\",\\\"SpanID\\\":\\\"d195e207dfbf1775\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:06:38.93415321Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:06:38.934250546Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":42508}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:06:41.535080838Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"8db442a0165869c57a316a8faa1f8fb8\\\",\\\"SpanID\\\":\\\"e498e0a764f76319\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:06:39.102209346Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:06:39.10428595Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":14,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai-worker/manifests/latest"}]},{"id":"tlxdwp-nginx","at":"2026-09-25T15:05:13.346979342Z","spike":{"service":"nginx","windowStart":"2026-09-25T15:00:13.346979342Z","windowEnd":"2026-09-25T15:05:13.346979342Z","errorCount":113,"baselineMean":8.89655172413793,"excess":5,"patterns":[{"signature":"507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d","count":2,"baselineMean":0,"excess":2,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:05:06.200527764Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"171ceb5b0a017957cc0510541f80fd07","PatternSignature":"507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai-worker/manifests/latest","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}},{"signature":"645f263c238fbbac7ee018af53104cade809f58f461e3939c252e097381e265c","count":2,"baselineMean":0,"excess":2,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:05:06.200527764Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"f90437c17a24182429ec1dbb33fb1209","PatternSignature":"645f263c238fbbac7ee018af53104cade809f58f461e3939c252e097381e265c","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai-worker/manifests/{id}","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}},{"signature":"38f693f9028998b8f34bc954af6be996e8b6add9ad53bbbae8c7f2ae5456586f","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:05:01.199520113Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"b341496525b8a9346fd0ddc5fa26498c","PatternSignature":"38f693f9028998b8f34bc954af6be996e8b6add9ad53bbbae8c7f2ae5456586f","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai-worker-browser/manifests/latest","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai-worker/manifests/latest","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:15:05:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T15:05:41.507526902Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"758d99c1d694e7816fbf337d231dc906\\\",\\\"SpanID\\\":\\\"7cb6d7f33c9cf9e7\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:05:38.934630694Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:38.934733409Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36062}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:05:41.507683183Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f4b89e61b17217cfa6bda198b2b91f46\\\",\\\"SpanID\\\":\\\"d95c0c7e1b5008ba\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:05:39.10176949Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:39.103919224Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":2,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai-worker/manifests/latest"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai-worker/manifests/{id}","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:15:05:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T15:05:41.507526902Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"758d99c1d694e7816fbf337d231dc906\\\",\\\"SpanID\\\":\\\"7cb6d7f33c9cf9e7\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:05:38.934630694Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:38.934733409Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36062}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:05:41.507683183Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f4b89e61b17217cfa6bda198b2b91f46\\\",\\\"SpanID\\\":\\\"d95c0c7e1b5008ba\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:05:39.10176949Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:39.103919224Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":2,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai-worker/manifests/{id}"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai-worker-browser/manifests/latest","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:15:05:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T15:05:41.507526902Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"758d99c1d694e7816fbf337d231dc906\\\",\\\"SpanID\\\":\\\"7cb6d7f33c9cf9e7\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:05:38.934630694Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:38.934733409Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36062}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:05:41.507683183Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f4b89e61b17217cfa6bda198b2b91f46\\\",\\\"SpanID\\\":\\\"d95c0c7e1b5008ba\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:05:39.10176949Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:39.103919224Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai-worker-browser/manifests/latest"}]},{"id":"tlxdv1-nginx","at":"2026-09-25T15:04:13.347024215Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:59:13.347024215Z","windowEnd":"2026-09-25T15:04:13.347024215Z","errorCount":91,"baselineMean":8.10344827586207,"excess":50,"patterns":[{"signature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","count":49,"baselineMean":0,"excess":49,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:04:05.974352097Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"2ecf7a7bbc8ea827b2dc41694939ce36","PatternSignature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"monitor","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}},{"signature":"fb977ed57f4e204d7992f54a0ef1d8a216ea60662250029db6570e708aaae4da","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:02:46.184100608Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"a7bf46f1b94a51a50950320b18a40e30","PatternSignature":"fb977ed57f4e204d7992f54a0ef1d8a216ea60662250029db6570e708aaae4da","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/dev-machine/manifests/latest","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["time=\"2026-09-25T15:04:41.482860568Z\" level=debug msg=\"{\\\"Name\\\":\\\"GetContent\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"ed26c8d64c3d4819\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"27eef151ac9e36e2\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:04:39.178516808Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:04:39.181030949Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/docker/registry/v2/repositories/eventsourced-ai-worker-mcp/_layers/sha256/3a28460d299da20e81866def93a4d1cd7af8b918597dcddc3e36afc6499f71c8/link\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:04:41.482994542Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"6f84b5597bc3b047\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"27eef151ac9e36e2\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:04:39.181088148Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:04:39.183677221Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/docker/registry/v2/blobs/sha256/3a/3a28460d299da20e81866def93a4d1cd7af8b918597dcddc3e36afc6499f71c8/data\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:04:41.483176512Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /v2/eventsourced-ai-worker-mcp/blobs/sha256:3a28460d299da20e81866def93a4d1cd7af8b918597dcddc3e36afc6499f71c8\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"27eef151ac9e36e2\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"b4d5d67576275b23\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":true},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:04:39.106399223Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:04:39.183981801Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"registry.k6n.net\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.5.162\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36364}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker/28.1.1 go/go1.23.8 git-commit/01f442b kernel/6.8.0-142-generic os/linux arch/amd64 UpstreamClient(Go-http-client/1.1)\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.10.11.135\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/v2/eventsourced-ai-worker-mcp/blobs/sha256:3a28460d299da20e81866def93a4d1cd7af8b918597dcddc3e36afc6499f71c8\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"1.1\\\"}},{\\\"Key\\\":\\\"http.response.body.size\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":2783}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":8,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":49,"baseline":0,"uaClass":"monitor","what":"HTTP 401 | /v2"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/dev-machine/manifests/latest","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["time=\"2026-09-25T15:04:41.482860568Z\" level=debug msg=\"{\\\"Name\\\":\\\"GetContent\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"ed26c8d64c3d4819\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"27eef151ac9e36e2\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:04:39.178516808Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:04:39.181030949Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/docker/registry/v2/repositories/eventsourced-ai-worker-mcp/_layers/sha256/3a28460d299da20e81866def93a4d1cd7af8b918597dcddc3e36afc6499f71c8/link\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:04:41.482994542Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"6f84b5597bc3b047\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"27eef151ac9e36e2\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:04:39.181088148Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:04:39.183677221Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/docker/registry/v2/blobs/sha256/3a/3a28460d299da20e81866def93a4d1cd7af8b918597dcddc3e36afc6499f71c8/data\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T15:04:41.483176512Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /v2/eventsourced-ai-worker-mcp/blobs/sha256:3a28460d299da20e81866def93a4d1cd7af8b918597dcddc3e36afc6499f71c8\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"27eef151ac9e36e2\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"33e4a4029129a138e597de4df7258dc8\\\",\\\"SpanID\\\":\\\"b4d5d67576275b23\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":true},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:04:39.106399223Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:04:39.183981801Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"registry.k6n.net\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.5.162\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36364}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker/28.1.1 go/go1.23.8 git-commit/01f442b kernel/6.8.0-142-generic os/linux arch/amd64 UpstreamClient(Go-http-client/1.1)\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.10.11.135\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/v2/eventsourced-ai-worker-mcp/blobs/sha256:3a28460d299da20e81866def93a4d1cd7af8b918597dcddc3e36afc6499f71c8\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"1.1\\\"}},{\\\"Key\\\":\\\"http.response.body.size\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":2783}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":8,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /v2/dev-machine/manifests/latest"}]},{"id":"tlxdrp-nginx","at":"2026-09-25T15:02:13.346726581Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:57:13.346726581Z","windowEnd":"2026-09-25T15:02:13.346726581Z","errorCount":81,"baselineMean":8,"excess":24,"patterns":[{"signature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","count":24,"baselineMean":0,"excess":24,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:02:10.960575775Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"dd7c8820f20ffa5e56e77b4af727d26b","PatternSignature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"monitor","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["time=\"2026-09-25T15:02:43.940293986Z\" level=debug msg=\"(*schema2ManifestHandler).Unmarshal\" auth.user.name=mats environment=development go.version=go1.25.9 http.request.host=registry.k6n.net http.request.id=01a0d917-15d4-7c71-89ce-2e064874e3fd http.request.method=HEAD http.request.remoteaddr=10.10.11.135 http.request.uri=/v2/eventsourced-ai/manifests/latest http.request.useragent=\"docker/28.1.1 go/go1.23.8 git-commit/01f442b kernel/6.8.0-142-generic os/linux arch/amd64 UpstreamClient(Go-http-client/1.1)\" instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry vars.name=eventsourced-ai vars.reference=latest version=3.1.1","time=\"2026-09-25T15:02:43.940781825Z\" level=info msg=\"response completed\" auth.user.name=mats environment=development go.version=go1.25.9 http.request.host=registry.k6n.net http.request.id=01a0d917-15d4-7c71-89ce-2e064874e3fd http.request.method=HEAD http.request.remoteaddr=10.10.11.135 http.request.uri=/v2/eventsourced-ai/manifests/latest http.request.useragent=\"docker/28.1.1 go/go1.23.8 git-commit/01f442b kernel/6.8.0-142-generic os/linux arch/amd64 UpstreamClient(Go-http-client/1.1)\" http.response.contenttype=application/vnd.docker.distribution.manifest.v2+json http.response.duration=15.948049ms http.response.status=200 http.response.written=0 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry vars.name=eventsourced-ai vars.reference=latest version=3.1.1","10.244.5.162 - - [25/Sep/2026:15:02:43 +0000] \"HEAD /v2/eventsourced-ai/manifests/latest HTTP/1.1\" 200 0 \"\" \"docker/28.1.1 go/go1.23.8 git-commit/01f442b kernel/6.8.0-142-generic os/linux arch/amd64 UpstreamClient(Go-http-client/1.1)\""],"found":true}],"rule":"status-401","new":false,"count":24,"baseline":0,"uaClass":"monitor","what":"HTTP 401 | /v2"}]},{"id":"tlxdq1-nginx","at":"2026-09-25T15:01:13.346685668Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:56:13.346685668Z","windowEnd":"2026-09-25T15:01:13.346685668Z","errorCount":66,"baselineMean":7.379310344827586,"excess":9,"patterns":[{"signature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","count":9,"baselineMean":0,"excess":9,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T15:01:10.965147823Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"b90df686739eee8b90af562fb7d5fa0a","PatternSignature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"monitor","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["time=\"2026-09-25T15:01:41.670782168Z\" level=debug msg=GetBlob auth.user.name=mats environment=development go.version=go1.25.9 http.request.host=registry.k6n.net http.request.id=01a0d916-229f-7e53-8c88-7150eaf7c907 http.request.method=GET http.request.remoteaddr=10.10.0.1 http.request.uri=\"/v2/ecommerce/redirector/blobs/sha256:b0e3a4fe5d17450f527d2774392458ca2a0bd1722c4fc9affbdaa74fbbb68386\" http.request.useragent=trivy/0.74.0 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry vars.digest=\"sha256:b0e3a4fe5d17450f527d2774392458ca2a0bd1722c4fc9affbdaa74fbbb68386\" vars.name=ecommerce/redirector version=3.1.1","time=\"2026-09-25T15:01:41.766015511Z\" level=info msg=\"response completed\" auth.user.name=mats environment=development go.version=go1.25.9 http.request.host=registry.k6n.net http.request.id=01a0d916-229f-7e53-8c88-7150eaf7c907 http.request.method=GET http.request.remoteaddr=10.10.0.1 http.request.uri=\"/v2/ecommerce/redirector/blobs/sha256:b0e3a4fe5d17450f527d2774392458ca2a0bd1722c4fc9affbdaa74fbbb68386\" http.request.useragent=trivy/0.74.0 http.response.contenttype=application/octet-stream http.response.duration=102.063279ms http.response.status=200 http.response.written=1038 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry vars.digest=\"sha256:b0e3a4fe5d17450f527d2774392458ca2a0bd1722c4fc9affbdaa74fbbb68386\" vars.name=ecommerce/redirector version=3.1.1","10.244.5.162 - - [25/Sep/2026:15:01:41 +0000] \"GET /v2/ecommerce/redirector/blobs/sha256:b0e3a4fe5d17450f527d2774392458ca2a0bd1722c4fc9affbdaa74fbbb68386 HTTP/1.1\" 200 1038 \"\" \"trivy/0.74.0\""],"found":true}],"rule":"status-401","new":false,"count":9,"baseline":0,"uaClass":"monitor","what":"HTTP 401 | /v2"}]},{"id":"tlxdmp-nginx","at":"2026-09-25T14:59:13.347107325Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:54:13.347107325Z","windowEnd":"2026-09-25T14:59:13.347107325Z","errorCount":61,"baselineMean":7.241379310344827,"excess":5,"patterns":[{"signature":"0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e","count":2,"baselineMean":0,"excess":2,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:58:21.152985505Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"b3fb9ac861025eeb7e06e81f072cdc9a","PatternSignature":"0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/designer/v1/file-content","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"172.71.120.24","UAClass":"search_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6","count":2,"baselineMean":0,"excess":2,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:58:21.152985505Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"7352d1f5a6b9f720cde3a0716ea19a6c","PatternSignature":"b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/templates/preview","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"172.71.120.24","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab","count":1,"baselineMean":0,"excess":1,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:58:21.152985505Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"05fc1e480d330b915927c8d90b82520f","PatternSignature":"004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/fs/exec","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"172.71.120.24","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/designer/v1/file-content","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":true,"count":2,"baseline":0,"clients":1,"uaClass":"search_crawler","what":"HTTP 503 | /api/designer/v1/file-content"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/templates/preview","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":true,"count":2,"baseline":0,"clients":1,"uaClass":"ai_crawler","what":"HTTP 503 | /api/templates/preview"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/fs/exec","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":true,"count":1,"baseline":0,"clients":1,"uaClass":"ai_crawler","what":"HTTP 503 | /api/fs/exec"}]},{"id":"tlxdl1-nginx","at":"2026-09-25T14:58:13.347295198Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:53:13.347295198Z","windowEnd":"2026-09-25T14:58:13.347295198Z","errorCount":52,"baselineMean":6.862068965517241,"excess":2,"patterns":[{"signature":"176140cc93493ec68f0ac58d27d0285f38e9e5b3324cc884123aa732e7594753","count":1,"baselineMean":0,"excess":1,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:58:06.153720966Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"2d4718c5e62a66b9b95b55433986c38f","PatternSignature":"176140cc93493ec68f0ac58d27d0285f38e9e5b3324cc884123aa732e7594753","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/{id}/config","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"172.71.120.24","UAClass":"other_bot","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"489772f6cee5972487f0fc121924a67472a534df12901ea2afd0539ddaadaf21","count":1,"baselineMean":0,"excess":1,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:58:06.153720966Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"932e810264dc1287789c21767583242c","PatternSignature":"489772f6cee5972487f0fc121924a67472a534df12901ea2afd0539ddaadaf21","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/account","Host":"k6n.net","StatusCode":503,"ExceptionType":"","ClientIP":"172.71.120.24","UAClass":"ai_crawler","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/{id}/config","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":true,"count":1,"baseline":0,"clients":1,"uaClass":"other_bot","what":"HTTP 503 | /api/{id}/config"},{"category":"dependency_unreachable","priority":"high","status":503,"host":"k6n.net","path":"/api/account","services":["nginx"],"routes":[{"ingress":"dev/static-knatofs","host":"k6n.net","path":"/api","pathType":"Prefix","namespace":"dev","service":"go-html-edit","port":"8080","deployments":["go-html-edit"]}],"pods":[{"namespace":"dev","deployment":"go-html-edit","found":false}],"rule":"status-5xx-upstream","new":true,"count":1,"baseline":0,"clients":1,"uaClass":"ai_crawler","what":"HTTP 503 | /api/account"}]},{"id":"tlxdjd-nginx","at":"2026-09-25T14:57:13.346722313Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:52:13.346722313Z","windowEnd":"2026-09-25T14:57:13.346722313Z","errorCount":41,"baselineMean":7.137931034482759,"excess":1,"patterns":[{"signature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","count":1,"baselineMean":0,"excess":1,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:56:51.1417847Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"f87713e8faead3287a112e09a450f9d3","PatternSignature":"ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/","Host":"knatofs.se","StatusCode":405,"ExceptionType":"","ClientIP":"188.245.62.219","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"needs_review","priority":"low","status":405,"host":"knatofs.se","path":"/","services":["nginx"],"rule":"no-rule","new":false,"count":1,"baseline":0,"clients":1,"what":"HTTP 405 | /"}]},{"id":"tlxdhp-nginx","at":"2026-09-25T14:56:13.346780665Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:51:13.346780665Z","windowEnd":"2026-09-25T14:56:13.346780665Z","errorCount":36,"baselineMean":6.448275862068965,"excess":32,"patterns":[{"signature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","count":32,"baselineMean":0,"excess":32,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:56:05.932078547Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d2a655791d6b4e0786c37a1a52a057be","PatternSignature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/metrics/query","Host":"ui.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":502,"host":"ui.k6n.net","path":"/api/metrics/query","services":["nginx"],"routes":[{"ingress":"dev/k8s-helper-ingress","host":"ui.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"k8s-helper","port":"80","deployments":["k8s-helper"]}],"pods":[{"namespace":"dev","deployment":"k8s-helper","pod":"k8s-helper-5665cd69c9-kjd2t","lines":["2026/09/25 14:56:36 scan: started pod trivy-scan-39e0ce2743afe7d6 for matst80/go-mail2telegram","2026/09/25 14:56:40 scan cycle: matst80/go-mail2telegram: 58 vulns","2026/09/25 14:56:40 scan: started pod trivy-scan-860a2822ca25fb18 for matst80/info-server:latest"],"found":true}],"rule":"status-5xx-upstream","new":false,"count":32,"baseline":0,"what":"HTTP 502 | /api/metrics/query"}]},{"id":"tlxded-nginx","at":"2026-09-25T14:54:13.347301316Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:49:13.347301316Z","windowEnd":"2026-09-25T14:54:13.347301316Z","errorCount":24,"baselineMean":6.413793103448276,"excess":4,"patterns":[{"signature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","count":2,"baselineMean":0,"excess":2,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T14:53:26.116399324Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"5bf4af948bf803fe849f2b826f3404ce","PatternSignature":"2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}},{"signature":"b9a1368dda50f9d0d082c6783302b9386936009289826c4e6a5d3aa8389982cd","count":2,"baselineMean":0,"excess":2,"novel":true,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T14:53:26.116399324Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"133b5a4d5b71fce2cb959468fd870351","PatternSignature":"b9a1368dda50f9d0d082c6783302b9386936009289826c4e6a5d3aa8389982cd","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/k8s-helper/manifests/{id}","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:14:54:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T14:54:41.13845619Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"4b9dce625eb455364bc6550e0df58c65\\\",\\\"SpanID\\\":\\\"7409de8192221aca\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T14:54:38.93455149Z\\\",\\\"EndTime\\\":\\\"2026-09-25T14:54:38.934659304Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":57204}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T14:54:41.138612963Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"03ce02d69ea60153fac32c157d3e21b9\\\",\\\"SpanID\\\":\\\"efa040ef04b76133\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T14:54:39.101671401Z\\\",\\\"EndTime\\\":\\\"2026-09-25T14:54:39.103906648Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":2,"baseline":0,"what":"HTTP 401 | /v2"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/k8s-helper/manifests/{id}","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:14:54:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T14:54:41.13845619Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"4b9dce625eb455364bc6550e0df58c65\\\",\\\"SpanID\\\":\\\"7409de8192221aca\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T14:54:38.93455149Z\\\",\\\"EndTime\\\":\\\"2026-09-25T14:54:38.934659304Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":57204}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T14:54:41.138612963Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"03ce02d69ea60153fac32c157d3e21b9\\\",\\\"SpanID\\\":\\\"efa040ef04b76133\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T14:54:39.101671401Z\\\",\\\"EndTime\\\":\\\"2026-09-25T14:54:39.103906648Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":true,"count":2,"baseline":0,"what":"HTTP 401 | /v2/k8s-helper/manifests/{id}"}]},{"id":"tlxd11-nginx","at":"2026-09-25T14:46:13.34705123Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:41:13.34705123Z","windowEnd":"2026-09-25T14:46:13.34705123Z","errorCount":23,"baselineMean":5.137931034482759,"excess":16,"patterns":[{"signature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","count":16,"baselineMean":0,"excess":16,"novel":false,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:45:45.86299307Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"e3c39c7524075625f201873a72e9b911","PatternSignature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/metrics/query","Host":"ui.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":502,"host":"ui.k6n.net","path":"/api/metrics/query","services":["nginx"],"routes":[{"ingress":"dev/k8s-helper-ingress","host":"ui.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"k8s-helper","port":"80","deployments":["k8s-helper"]}],"pods":[{"namespace":"dev","deployment":"k8s-helper","pod":"k8s-helper-75474c4b8b-zj7fx","lines":["I0925 14:45:22.941450 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 14:46:21.193030 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 14:46:22.949899 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\""],"found":true}],"rule":"status-5xx-upstream","new":false,"count":16,"baseline":0,"what":"HTTP 502 | /api/metrics/query"}]},{"id":"tlxckd-eventsourced-worker-code","at":"2026-09-25T14:36:13.347443969Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T14:31:13.347443969Z","windowEnd":"2026-09-25T14:36:13.347443969Z","errorCount":77,"baselineMean":0.9310344827586207,"excess":77,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":63,"baselineMean":0,"excess":63,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T14:36:11.69601872Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"e42ab0493268396096f54d37942f2db8","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}},{"signature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","count":14,"baselineMean":0,"excess":14,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T14:35:54.760751892Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"f4ec4924e09c57fd686a1443958948af","PatternSignature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","IssueFingerprint":"b40901e8fbfcf66e","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: eof","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":63,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"},{"category":"dependency_unreachable","priority":"high","issueId":"b40901e8fbfcf66e","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":14,"baseline":0,"what":"worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"}]},{"id":"tlxc21-nginx","at":"2026-09-25T14:25:13.346654632Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:20:13.346654632Z","windowEnd":"2026-09-25T14:25:13.346654632Z","errorCount":31,"baselineMean":4.137931034482759,"excess":4,"patterns":[{"signature":"efb1d02de6ef2fbc3e1f234fa547da4e31652733a4c6f8aababe8da1900871f9","count":4,"baselineMean":0,"excess":4,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:24:15.768783875Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"fcd10da95cccfebe9e02e43df79d7570","PatternSignature":"efb1d02de6ef2fbc3e1f234fa547da4e31652733a4c6f8aababe8da1900871f9","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/transmission/rpc","Host":"torrent.k6n.net","StatusCode":409,"ExceptionType":"","ClientIP":"209.206.25.124","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"needs_review","priority":"low","status":409,"host":"torrent.k6n.net","path":"/transmission/rpc","services":["nginx"],"routes":[{"ingress":"home/torrent-ingress","host":"torrent.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"torrent-service","port":"9091"}],"rule":"no-rule","new":true,"count":4,"baseline":0,"clients":1,"what":"HTTP 409 | /transmission/rpc"}]},{"id":"tlxc0d-nginx","at":"2026-09-25T14:24:13.347064057Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:19:13.347064057Z","windowEnd":"2026-09-25T14:24:13.347064057Z","errorCount":27,"baselineMean":3.7241379310344827,"excess":3,"patterns":[{"signature":"41b6e057dd561fe6bf91eb3d028dbfe48f6333be61488ed661ea8bdcfb379a96","count":2,"baselineMean":0,"excess":2,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:23:20.761650796Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"2beaa3d1c2cbcc00449161ed313fedba","PatternSignature":"41b6e057dd561fe6bf91eb3d028dbfe48f6333be61488ed661ea8bdcfb379a96","IssueFingerprint":"","IsRetry":false,"IsTimeout":true,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/sessions/{id}/state","Host":"event-agent.k6n.net","StatusCode":504,"ExceptionType":"","ClientIP":"209.206.25.124","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"990205581d87cfdec48d07f37464355a8cd1b918827d9e18511bb8da7b0c1dd6","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["docker-registry","nginx"],"example":{"Timestamp":"2026-09-25T14:22:10.872293248Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"fe25b634337b1ef0773ab4ea3954d973","PatternSignature":"990205581d87cfdec48d07f37464355a8cd1b918827d9e18511bb8da7b0c1dd6","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/v2/eventsourced-ai/manifests/latest","Host":"registry.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["docker-registry","nginx"],"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","status":504,"host":"event-agent.k6n.net","path":"/api/sessions/{id}/state","services":["nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-56fdfc6cbc-9rjzb","lines":["10.244.9.1 - - [25/Sep/2026:14:24:32 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.5.162 - - [25/Sep/2026:14:24:37 +0000] \"GET /api/overview HTTP/1.1\" 200 153207 \"https://event-agent.k6n.net/session/crying-monkey-50\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Safari/537.36\" \"209.206.25.124\"","10.244.9.1 - - [25/Sep/2026:14:24:42 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"timeout-flag","new":false,"count":2,"baseline":0,"clients":1,"what":"HTTP 504 | /api/sessions/{id}/state"},{"category":"client_credential_issue","priority":"low","status":401,"host":"registry.k6n.net","path":"/v2/eventsourced-ai/manifests/latest","services":["docker-registry","nginx"],"routes":[{"ingress":"home/registry","host":"registry.k6n.net","path":"/","pathType":"Prefix","namespace":"home","service":"registry","port":"5000","deployments":["registry"]}],"pods":[{"namespace":"home","deployment":"registry","pod":"registry-559bf85884-sxbrw","lines":["10.244.2.1 - - [25/Sep/2026:14:24:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"","time=\"2026-09-25T14:24:40.388398143Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"9a7320d962c760aed339162de5df76c9\\\",\\\"SpanID\\\":\\\"b0e4e7da1c3e808b\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T14:24:38.934703547Z\\\",\\\"EndTime\\\":\\\"2026-09-25T14:24:38.934805825Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":42970}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1","time=\"2026-09-25T14:24:40.388564842Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"3d8f2ed158e81d5e7197b33d3e04dc1a\\\",\\\"SpanID\\\":\\\"b127d729aece6baf\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T14:24:39.101781941Z\\\",\\\"EndTime\\\":\\\"2026-09-25T14:24:39.104725835Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"],"found":true}],"rule":"status-401","new":false,"count":1,"baseline":0,"what":"HTTP 401 | /v2/eventsourced-ai/manifests/latest"}]},{"id":"tlxbyp-nginx","at":"2026-09-25T14:23:13.347056801Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:18:13.347056801Z","windowEnd":"2026-09-25T14:23:13.347056801Z","errorCount":28,"baselineMean":3.4827586206896552,"excess":1,"patterns":[{"signature":"10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b","count":1,"baselineMean":0,"excess":1,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:22:25.756704937Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"c73381a9fccb067af1c61cd923bfc28a","PatternSignature":"10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/stream","Host":"event-agent.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"209.206.25.124","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":502,"host":"event-agent.k6n.net","path":"/api/stream","services":["nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-56fdfc6cbc-9rjzb","lines":["10.244.5.162 - - [25/Sep/2026:14:23:41 +0000] \"GET /sw.js HTTP/1.1\" 304 0 \"https://event-agent.k6n.net/sw.js\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Safari/537.36\" \"209.206.25.124\"","10.244.5.162 - - [25/Sep/2026:14:23:42 +0000] \"GET /auth/me HTTP/1.1\" 200 111 \"https://event-agent.k6n.net/session/crying-monkey-50\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Safari/537.36\" \"209.206.25.124\"","10.244.9.1 - - [25/Sep/2026:14:23:42 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"status-5xx-upstream","new":false,"count":1,"baseline":0,"clients":1,"what":"HTTP 502 | /api/stream"}]},{"id":"tlxbx1-nginx","at":"2026-09-25T14:22:13.347454307Z","spike":{"service":"nginx","windowStart":"2026-09-25T14:17:13.347454307Z","windowEnd":"2026-09-25T14:22:13.347454307Z","errorCount":23,"baselineMean":3.3448275862068964,"excess":14,"patterns":[{"signature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","count":8,"baselineMean":0,"excess":8,"novel":true,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:22:00.791722367Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d709c813d8dc8f4780ce8ba77b711ad9","PatternSignature":"77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/metrics/query","Host":"ui.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"209.206.25.124","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}},{"signature":"6fd42b8e91d39c7666d3c140cdca5aa7711ce1e2d21f0cf6b0f67535b3f68dac","count":3,"baselineMean":0,"excess":3,"novel":true,"clients":1,"services":["eventsourced-orchestrator","nginx"],"example":{"Timestamp":"2026-09-25T14:18:20.845671196Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"d2aff32d937667f8013c63ca8d494317","PatternSignature":"6fd42b8e91d39c7666d3c140cdca5aa7711ce1e2d21f0cf6b0f67535b3f68dac","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/session/reset_password","Host":"event-agent.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"103.206.253.106","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["nginx","eventsourced-orchestrator"],"BadActor":""}},{"signature":"5c87cc1fcbcc6c723041e1eee7b41eef12f1f99c88d2a1e98804404024a291c1","count":2,"baselineMean":0,"excess":2,"novel":true,"clients":1,"services":["eventsourced-orchestrator","nginx"],"example":{"Timestamp":"2026-09-25T14:18:05.729846843Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"f55568b95bd1b356c01bad8e3f349a3a","PatternSignature":"5c87cc1fcbcc6c723041e1eee7b41eef12f1f99c88d2a1e98804404024a291c1","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/session/properties","Host":"event-agent.k6n.net","StatusCode":401,"ExceptionType":"","ClientIP":"103.206.253.106","UAClass":"unknown","IsDuplicate":false,"IsRootCause":false,"HasUpstream":true,"TraceServices":["eventsourced-orchestrator","nginx"],"BadActor":""}},{"signature":"41b6e057dd561fe6bf91eb3d028dbfe48f6333be61488ed661ea8bdcfb379a96","count":1,"baselineMean":0,"excess":1,"novel":false,"clients":1,"services":["nginx"],"example":{"Timestamp":"2026-09-25T14:22:10.755531343Z","Service":"nginx","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"35e68f3ccb4a275c735de2dcf51b35e6","PatternSignature":"41b6e057dd561fe6bf91eb3d028dbfe48f6333be61488ed661ea8bdcfb379a96","IssueFingerprint":"","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"","SpanName":"","Route":"/api/sessions/{id}/state","Host":"event-agent.k6n.net","StatusCode":502,"ExceptionType":"","ClientIP":"209.206.25.124","UAClass":"browser","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["nginx"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","status":502,"host":"ui.k6n.net","path":"/api/metrics/query","services":["nginx"],"routes":[{"ingress":"dev/k8s-helper-ingress","host":"ui.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"k8s-helper","port":"80","deployments":["k8s-helper"]}],"pods":[{"namespace":"dev","deployment":"k8s-helper","pod":"k8s-helper-75474c4b8b-zj7fx","lines":["I0925 14:21:34.050281 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 14:22:04.282513 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"","I0925 14:22:22.964288 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\""],"found":true}],"rule":"status-5xx-upstream","new":true,"count":8,"baseline":0,"clients":1,"what":"HTTP 502 | /api/metrics/query"},{"category":"dependency_unreachable","priority":"high","status":502,"host":"event-agent.k6n.net","path":"/api/sessions/{id}/state","services":["nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-56fdfc6cbc-9rjzb","lines":["10.244.9.1 - - [25/Sep/2026:14:22:22 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:14:22:32 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:14:22:42 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"status-5xx-upstream","new":false,"count":1,"baseline":0,"clients":1,"what":"HTTP 502 | /api/sessions/{id}/state"},{"category":"client_credential_issue","priority":"low","status":401,"host":"event-agent.k6n.net","path":"/api/session/reset_password","services":["eventsourced-orchestrator","nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-56fdfc6cbc-9rjzb","lines":["10.244.9.1 - - [25/Sep/2026:14:22:22 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:14:22:32 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:14:22:42 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"status-401","new":true,"count":3,"baseline":0,"clients":1,"what":"HTTP 401 | /api/session/reset_password"},{"category":"client_credential_issue","priority":"low","status":401,"host":"event-agent.k6n.net","path":"/api/session/properties","services":["eventsourced-orchestrator","nginx"],"routes":[{"ingress":"dev/eventsourced-web","host":"event-agent.k6n.net","path":"/","pathType":"Prefix","namespace":"dev","service":"eventsourced-web","port":"80","deployments":["eventsourced-web"]}],"pods":[{"namespace":"dev","deployment":"eventsourced-web","pod":"eventsourced-web-56fdfc6cbc-9rjzb","lines":["10.244.9.1 - - [25/Sep/2026:14:22:22 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:14:22:32 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"","10.244.9.1 - - [25/Sep/2026:14:22:42 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""],"found":true}],"rule":"status-401","new":true,"count":2,"baseline":0,"clients":1,"what":"HTTP 401 | /api/session/properties"}]},{"id":"tlxbx1-eventsourced-worker-code","at":"2026-09-25T14:22:13.347454307Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T14:17:13.347454307Z","windowEnd":"2026-09-25T14:22:13.347454307Z","errorCount":35,"baselineMean":0.41379310344827586,"excess":8,"patterns":[{"signature":"c79ebdd574e5a9103bcc3c2faaa06abee7f041701094b24e883949dcc5371eab","count":8,"baselineMean":0,"excess":8,"novel":true,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T14:22:02.949148899Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"cb422f420e0e5e387e57384138fa6f42","PatternSignature":"c79ebdd574e5a9103bcc3c2faaa06abee7f041701094b24e883949dcc5371eab","IssueFingerprint":"b082dde6853f6053","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*worker.error: bash: invalid arguments: json: cannot unmarshal string into go struct field bashargs.timeout_seconds of type int","SpanName":"worker.fulfill","Route":"","Host":"","StatusCode":0,"ExceptionType":"*worker.Error","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","issueId":"b082dde6853f6053","services":["eventsourced-worker-code"],"rule":"capacity-text","new":true,"count":8,"baseline":0,"what":"worker.fulfill | *worker.Error | *worker.error: bash: invalid arguments: json: cannot unmarshal string into go struct field bashargs.timeout_seconds of …"}]},{"id":"tlxbd1-eventsourced-worker-code","at":"2026-09-25T14:10:13.346727499Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T14:05:13.346727499Z","windowEnd":"2026-09-25T14:10:13.346727499Z","errorCount":37,"baselineMean":0.3448275862068966,"excess":35,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":28,"baselineMean":0,"excess":28,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T14:09:49.577703801Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"14ad7fa4b0d513f8fa057a4216eafb83","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}},{"signature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","count":7,"baselineMean":0,"excess":7,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T14:09:41.534950695Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"8584decc043b76b0347c329d4e7d9e97","PatternSignature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","IssueFingerprint":"b40901e8fbfcf66e","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: eof","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":28,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"},{"category":"dependency_unreachable","priority":"high","issueId":"b40901e8fbfcf66e","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":7,"baseline":0,"what":"worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"}]},{"id":"tlx951-eventsourced-worker-code","at":"2026-09-25T13:22:13.347013465Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T13:17:13.347013465Z","windowEnd":"2026-09-25T13:22:13.347013465Z","errorCount":37,"baselineMean":0,"excess":37,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":30,"baselineMean":0,"excess":30,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T13:22:11.585302542Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"08659ce24a8ed07494d7cdcf4cc5eb82","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}},{"signature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","count":7,"baselineMean":0,"excess":7,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T13:21:46.49062247Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"7902028e7f4807da43236b784641ba03","PatternSignature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","IssueFingerprint":"b40901e8fbfcf66e","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: eof","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":30,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"},{"category":"dependency_unreachable","priority":"high","issueId":"b40901e8fbfcf66e","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":7,"baseline":0,"what":"worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"}]},{"id":"tlx7pd-eventsourced-orchestrator","at":"2026-09-25T12:51:13.347152151Z","spike":{"service":"eventsourced-orchestrator","windowStart":"2026-09-25T12:46:13.347152151Z","windowEnd":"2026-09-25T12:51:13.347152151Z","errorCount":15,"baselineMean":0,"excess":1,"patterns":[{"signature":"29f5030badef82d221f8a79563cf2c9dbf4d653551ca54b081bc0780c3b35c37","count":1,"baselineMean":0,"excess":1,"novel":true,"services":["eventsourced-orchestrator"],"example":{"Timestamp":"2026-09-25T12:50:45.113996545Z","Service":"eventsourced-orchestrator","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"31ff3a0f713c612d62db59873507090e","PatternSignature":"29f5030badef82d221f8a79563cf2c9dbf4d653551ca54b081bc0780c3b35c37","IssueFingerprint":"260fc07492b5c2f8","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: provider stream error: {\"code\":\u003cnum\u003e,\"message\":\"provider returned an empty response\",\"metadata\":{\"error_type\":\"provider_unavailable\"}}","SpanName":"orchestrator.llm.attempt","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":false,"HasUpstream":false,"TraceServices":["eventsourced-orchestrator"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"260fc07492b5c2f8","services":["eventsourced-orchestrator"],"rule":"dependency-text","new":true,"count":1,"baseline":0,"what":"orchestrator.llm.attempt | *errors.errorString | *errors.errorstring: provider stream error: {\"code\":\u003cnum\u003e,\"message\":\"provider returned an empty response\",\"metadata\":{\"…"}]},{"id":"tlx7m1-eventsourced-orchestrator","at":"2026-09-25T12:49:13.346915639Z","spike":{"service":"eventsourced-orchestrator","windowStart":"2026-09-25T12:44:13.346915639Z","windowEnd":"2026-09-25T12:49:13.346915639Z","errorCount":13,"baselineMean":0,"excess":13,"patterns":[{"signature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","count":13,"baselineMean":0,"excess":13,"novel":false,"services":["eventsourced-orchestrator"],"example":{"Timestamp":"2026-09-25T12:48:32.179425433Z","Service":"eventsourced-orchestrator","Release":"","Kind":"log","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"","PatternSignature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","IssueFingerprint":"b57379b9dd3a6a9d","IsRetry":false,"IsTimeout":true,"IsCancellation":false,"Message":"codesearch upload failed","SpanName":"","Route":"","Host":"","StatusCode":0,"ExceptionType":"context.deadlineExceededError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":false,"HasUpstream":false,"TraceServices":null,"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","issueId":"b57379b9dd3a6a9d","services":["eventsourced-orchestrator"],"rule":"timeout-flag","new":false,"count":13,"baseline":0,"what":"context.deadlineExceededError | codesearch upload failed"}]},{"id":"tlx7ip-eventsourced-worker-code","at":"2026-09-25T12:47:13.346939505Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T12:42:13.346939505Z","windowEnd":"2026-09-25T12:47:13.346939505Z","errorCount":47,"baselineMean":0,"excess":40,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":40,"baselineMean":0,"excess":40,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T12:46:57.647725048Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"a646787952ca743875ed75a4943fd6b0","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":40,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"}]},{"id":"tlx7h1-eventsourced-worker-code","at":"2026-09-25T12:46:13.347336983Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T12:41:13.347336983Z","windowEnd":"2026-09-25T12:46:13.347336983Z","errorCount":10,"baselineMean":0,"excess":10,"patterns":[{"signature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","count":7,"baselineMean":0,"excess":7,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T12:46:03.649627264Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"e49ca924b6a716cb0d737e870cf7d8ec","PatternSignature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","IssueFingerprint":"b40901e8fbfcf66e","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: eof","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}},{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":3,"baselineMean":0,"excess":3,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T12:46:03.649627264Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"cb600304e20a370fd4c4eeb13d87536a","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"b40901e8fbfcf66e","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":7,"baseline":0,"what":"worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"},{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":3,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"}]},{"id":"tlx3b1-eventsourced-orchestrator","at":"2026-09-25T11:16:13.346973966Z","spike":{"service":"eventsourced-orchestrator","windowStart":"2026-09-25T11:11:13.346973966Z","windowEnd":"2026-09-25T11:16:13.346973966Z","errorCount":23,"baselineMean":0,"excess":1,"patterns":[{"signature":"ed8d4fd7b78d58f7f14c9fbec7d67d550e8a40f236126cd4964d72ad3645dd9f","count":1,"baselineMean":0,"excess":1,"novel":false,"services":["eventsourced-orchestrator"],"example":{"Timestamp":"2026-09-25T11:15:34.045242671Z","Service":"eventsourced-orchestrator","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"a951dfed8b514f48f773df0889f33d4f","PatternSignature":"ed8d4fd7b78d58f7f14c9fbec7d67d550e8a40f236126cd4964d72ad3645dd9f","IssueFingerprint":"869896e00332d212","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*fmt.wraperror: read llm response: context deadline exceeded (client.timeout or context cancellation while reading body)","SpanName":"llm.provider_fallback","Route":"","Host":"","StatusCode":0,"ExceptionType":"*fmt.wrapError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":false,"HasUpstream":false,"TraceServices":["eventsourced-orchestrator"],"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","issueId":"869896e00332d212","services":["eventsourced-orchestrator"],"rule":"capacity-text","new":false,"count":1,"baseline":0,"what":"llm.provider_fallback | *fmt.wrapError | *fmt.wraperror: read llm response: context deadline exceeded (client.timeout or context cancellation while reading body)"}]},{"id":"tlx34d-eventsourced-orchestrator","at":"2026-09-25T11:12:13.347409701Z","spike":{"service":"eventsourced-orchestrator","windowStart":"2026-09-25T11:07:13.347409701Z","windowEnd":"2026-09-25T11:12:13.347409701Z","errorCount":19,"baselineMean":0,"excess":19,"patterns":[{"signature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","count":19,"baselineMean":0,"excess":19,"novel":false,"services":["eventsourced-orchestrator"],"example":{"Timestamp":"2026-09-25T11:12:01.123433759Z","Service":"eventsourced-orchestrator","Release":"","Kind":"log","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"","PatternSignature":"8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef","IssueFingerprint":"b57379b9dd3a6a9d","IsRetry":false,"IsTimeout":true,"IsCancellation":false,"Message":"codesearch upload failed","SpanName":"","Route":"","Host":"","StatusCode":0,"ExceptionType":"context.deadlineExceededError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":false,"HasUpstream":false,"TraceServices":null,"BadActor":""}}]},"patterns":[{"category":"capacity_saturation","priority":"high","issueId":"b57379b9dd3a6a9d","services":["eventsourced-orchestrator"],"rule":"timeout-flag","new":false,"count":19,"baseline":0,"what":"context.deadlineExceededError | codesearch upload failed"}]},{"id":"tlx311-eventsourced-worker-code","at":"2026-09-25T11:10:13.346712877Z","spike":{"service":"eventsourced-worker-code","windowStart":"2026-09-25T11:05:13.346712877Z","windowEnd":"2026-09-25T11:10:13.346712877Z","errorCount":39,"baselineMean":0.20689655172413793,"excess":39,"patterns":[{"signature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","count":32,"baselineMean":0,"excess":32,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T11:10:12.495433233Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"7e73ad1291b81c8250f0bd529d41c514","PatternSignature":"fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885","IssueFingerprint":"4580ae864fa21049","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-terminal: dial tcp \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e:\u003cnum\u003e: connect: connection refused","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*websocket.DialError","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}},{"signature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","count":7,"baselineMean":0,"excess":7,"novel":false,"services":["eventsourced-worker-code"],"example":{"Timestamp":"2026-09-25T11:09:29.914245587Z","Service":"eventsourced-worker-code","Release":"","Kind":"span","Source":"backend","IsError":true,"StatusClass":"","DurationMS":0,"TraceID":"95d11690cfe3f8a3d370fc001394503d","PatternSignature":"2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7","IssueFingerprint":"b40901e8fbfcf66e","IsRetry":false,"IsTimeout":false,"IsCancellation":false,"Message":"*errors.errorstring: eof","SpanName":"worker.terminal.connection","Route":"","Host":"","StatusCode":0,"ExceptionType":"*errors.errorString","ClientIP":"","UAClass":"","IsDuplicate":false,"IsRootCause":true,"HasUpstream":false,"TraceServices":["eventsourced-worker-code"],"BadActor":""}}]},"patterns":[{"category":"dependency_unreachable","priority":"high","issueId":"4580ae864fa21049","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":32,"baseline":0,"what":"worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:\u003cnum\u003e/internal/worker-termina…"},{"category":"dependency_unreachable","priority":"high","issueId":"b40901e8fbfcf66e","services":["eventsourced-worker-code"],"rule":"dependency-text","new":false,"count":7,"baseline":0,"what":"worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"}]}]}
|
|
|