[ { "name": "codesearch upload deadline exceeded", "expect": { "action": "check_capacity", "human": true, "why": "uploads time out repeatedly; the codesearch path is saturated or too slow" }, "alert": { "id": "tlxmh1-eventsourced-orchestrator", "at": "2026-09-25T18:10:13.34664129Z", "spike": { "service": "eventsourced-orchestrator", "windowStart": "2026-09-25T18:05:13.34664129Z", "windowEnd": "2026-09-25T18:10:13.34664129Z", "errorCount": 16, "baselineMean": 19.93103448275862, "excess": 16, "patterns": [ { "signature": "8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef", "count": 16, "baselineMean": 0, "excess": 16, "novel": false, "services": [ "eventsourced-orchestrator" ], "example": { "Timestamp": "2026-09-25T18:10:08.793065336Z", "Service": "eventsourced-orchestrator", "Release": "", "Kind": "log", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "", "PatternSignature": "8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef", "IssueFingerprint": "b57379b9dd3a6a9d", "IsRetry": false, "IsTimeout": true, "IsCancellation": false, "Message": "codesearch upload failed", "SpanName": "", "Route": "", "Host": "", "StatusCode": 0, "ExceptionType": "context.deadlineExceededError", "ClientIP": "", "UAClass": "", "IsDuplicate": false, "IsRootCause": false, "HasUpstream": false, "TraceServices": null, "BadActor": "" } } ] }, "patterns": [ { "category": "capacity_saturation", "priority": "high", "issueId": "b57379b9dd3a6a9d", "services": [ "eventsourced-orchestrator" ], "rule": "timeout-flag", "new": false, "count": 16, "baseline": 0, "what": "context.deadlineExceededError | codesearch upload failed" } ] } }, { "name": "worker cannot dial orchestrator websocket", "expect": { "action": "fix_dependency", "human": true, "why": "workers cannot reach the in-cluster orchestrator; the dependency is down or unreachable" }, "alert": { "id": "tlxmdp-eventsourced-worker-code", "at": "2026-09-25T18:08:13.346954479Z", "spike": { "service": "eventsourced-worker-code", "windowStart": "2026-09-25T18:03:13.346954479Z", "windowEnd": "2026-09-25T18:08:13.346954479Z", "errorCount": 36, "baselineMean": 0.13793103448275862, "excess": 36, "patterns": [ { "signature": "fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885", "count": 29, "baselineMean": 0, "excess": 29, "novel": false, "services": [ "eventsourced-worker-code" ], "example": { "Timestamp": "2026-09-25T18:08:12.308751715Z", "Service": "eventsourced-worker-code", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "34f7c7c1d9c07bb48b3241454d10a7bf", "PatternSignature": "fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885", "IssueFingerprint": "4580ae864fa21049", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:/internal/worker-terminal: dial tcp ...:: connect: connection refused", "SpanName": "worker.terminal.connection", "Route": "", "Host": "", "StatusCode": 0, "ExceptionType": "*websocket.DialError", "ClientIP": "", "UAClass": "", "IsDuplicate": false, "IsRootCause": true, "HasUpstream": false, "TraceServices": [ "eventsourced-worker-code" ], "BadActor": "" } }, { "signature": "2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7", "count": 7, "baselineMean": 0, "excess": 7, "novel": false, "services": [ "eventsourced-worker-code" ], "example": { "Timestamp": "2026-09-25T18:07:50.177441418Z", "Service": "eventsourced-worker-code", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "72e0cbf2c415b129eab8f4aadca42c9f", "PatternSignature": "2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7", "IssueFingerprint": "b40901e8fbfcf66e", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "*errors.errorstring: eof", "SpanName": "worker.terminal.connection", "Route": "", "Host": "", "StatusCode": 0, "ExceptionType": "*errors.errorString", "ClientIP": "", "UAClass": "", "IsDuplicate": false, "IsRootCause": true, "HasUpstream": false, "TraceServices": [ "eventsourced-worker-code" ], "BadActor": "" } } ] }, "patterns": [ { "category": "dependency_unreachable", "priority": "high", "issueId": "4580ae864fa21049", "services": [ "eventsourced-worker-code" ], "rule": "dependency-text", "new": false, "count": 29, "baseline": 0, "what": "worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:/internal/worker-termina…" }, { "category": "dependency_unreachable", "priority": "high", "issueId": "b40901e8fbfcf66e", "services": [ "eventsourced-worker-code" ], "rule": "dependency-text", "new": false, "count": 7, "baseline": 0, "what": "worker.terminal.connection | *errors.errorString | *errors.errorstring: eof" } ] } }, { "name": "bash tool args fail to unmarshal", "expect": { "action": "investigate_code", "human": true, "why": "a new json decode error on timeout_seconds is a code/contract bug, not capacity (triage mislabels it via the word 'timeout')" }, "alert": { "id": "tlxbx1-eventsourced-worker-code", "at": "2026-09-25T14:22:13.347454307Z", "spike": { "service": "eventsourced-worker-code", "windowStart": "2026-09-25T14:17:13.347454307Z", "windowEnd": "2026-09-25T14:22:13.347454307Z", "errorCount": 35, "baselineMean": 0.41379310344827586, "excess": 8, "patterns": [ { "signature": "c79ebdd574e5a9103bcc3c2faaa06abee7f041701094b24e883949dcc5371eab", "count": 8, "baselineMean": 0, "excess": 8, "novel": true, "services": [ "eventsourced-worker-code" ], "example": { "Timestamp": "2026-09-25T14:22:02.949148899Z", "Service": "eventsourced-worker-code", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "cb422f420e0e5e387e57384138fa6f42", "PatternSignature": "c79ebdd574e5a9103bcc3c2faaa06abee7f041701094b24e883949dcc5371eab", "IssueFingerprint": "b082dde6853f6053", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "*worker.error: bash: invalid arguments: json: cannot unmarshal string into go struct field bashargs.timeout_seconds of type int", "SpanName": "worker.fulfill", "Route": "", "Host": "", "StatusCode": 0, "ExceptionType": "*worker.Error", "ClientIP": "", "UAClass": "", "IsDuplicate": false, "IsRootCause": true, "HasUpstream": false, "TraceServices": [ "eventsourced-worker-code" ], "BadActor": "" } } ] }, "patterns": [ { "category": "capacity_saturation", "priority": "high", "issueId": "b082dde6853f6053", "services": [ "eventsourced-worker-code" ], "rule": "capacity-text", "new": true, "count": 8, "baseline": 0, "what": "worker.fulfill | *worker.Error | *worker.error: bash: invalid arguments: json: cannot unmarshal string into go struct field bashargs.timeout_seconds of …" } ] } }, { "name": "go-api 500 on /api/network-speed", "expect": { "action": "investigate_code", "human": true, "why": "500 from a running pod with no logs: a defect in go-api" }, "alert": { "id": "tlxdyd-nginx", "at": "2026-09-25T15:06:13.347093859Z", "spike": { "service": "nginx", "windowStart": "2026-09-25T15:01:13.347093859Z", "windowEnd": "2026-09-25T15:06:13.347093859Z", "errorCount": 121, "baselineMean": 8.379310344827585, "excess": 16, "patterns": [ { "signature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d", "count": 14, "baselineMean": 0, "excess": 14, "novel": false, "services": [ "docker-registry", "nginx" ], "example": { "Timestamp": "2026-09-25T15:05:36.202277428Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "a4da26a4a32edf925be5858fb04f23fe", "PatternSignature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/v2/eventsourced-ai-worker/manifests/latest", "Host": "registry.k6n.net", "StatusCode": 401, "ExceptionType": "", "ClientIP": "", "UAClass": "unknown", "IsDuplicate": false, "IsRootCause": false, "HasUpstream": true, "TraceServices": [ "docker-registry", "nginx" ], "BadActor": "" } }, { "signature": "63eff8b04309c018ea6fe0604d6e3b0885a94f3252cdf65ac9e777ae320a3af7", "count": 2, "baselineMean": 0, "excess": 2, "novel": false, "services": [ "nginx" ], "example": { "Timestamp": "2026-09-25T15:05:16.201179908Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "bef5078df057b841700020d2d7128086", "PatternSignature": "63eff8b04309c018ea6fe0604d6e3b0885a94f3252cdf65ac9e777ae320a3af7", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/api/network-speed", "Host": "tornberg.me", "StatusCode": 500, "ExceptionType": "", "ClientIP": "", "UAClass": "", "IsDuplicate": false, "IsRootCause": true, "HasUpstream": false, "TraceServices": [ "nginx" ], "BadActor": "" } } ] }, "patterns": [ { "category": "service_defect", "priority": "high", "status": 500, "host": "tornberg.me", "path": "/api/network-speed", "services": [ "nginx" ], "routes": [ { "ingress": "dev/static-web", "host": "tornberg.me", "path": "/api", "pathType": "Prefix", "namespace": "dev", "service": "go-api", "port": "3000", "deployments": [ "go-api" ] } ], "pods": [ { "namespace": "dev", "deployment": "go-api", "pod": "go-api-85d88c9bd-qp8p4", "found": true } ], "rule": "status-5xx", "new": false, "count": 2, "baseline": 0, "what": "HTTP 500 | /api/network-speed" }, { "category": "client_credential_issue", "priority": "low", "status": 401, "host": "registry.k6n.net", "path": "/v2/eventsourced-ai-worker/manifests/latest", "services": [ "docker-registry", "nginx" ], "routes": [ { "ingress": "home/registry", "host": "registry.k6n.net", "path": "/", "pathType": "Prefix", "namespace": "home", "service": "registry", "port": "5000", "deployments": [ "registry" ] } ], "pods": [ { "namespace": "home", "deployment": "registry", "pod": "registry-559bf85884-sxbrw", "lines": [ "10.244.2.1 - - [25/Sep/2026:15:06:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"", "time=\"2026-09-25T15:06:41.534921099Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"5fa8dd0ea0a6e33d84a5d5cdce9938fb\\\",\\\"SpanID\\\":\\\"d195e207dfbf1775\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:06:38.93415321Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:06:38.934250546Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":42508}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1", "time=\"2026-09-25T15:06:41.535080838Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"8db442a0165869c57a316a8faa1f8fb8\\\",\\\"SpanID\\\":\\\"e498e0a764f76319\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:06:39.102209346Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:06:39.10428595Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1" ], "found": true } ], "rule": "status-401", "new": false, "count": 14, "baseline": 0, "what": "HTTP 401 | /v2/eventsourced-ai-worker/manifests/latest" } ] } }, { "name": "k8s-helper 502 on /api/metrics/query", "expect": { "action": "fix_dependency", "human": true, "why": "40 bad gateways against a running k8s-helper: its upstream (Prometheus/metrics) or the pod path is failing" }, "alert": { "id": "tlxgbd-nginx", "at": "2026-09-25T15:57:13.347060965Z", "spike": { "service": "nginx", "windowStart": "2026-09-25T15:52:13.347060965Z", "windowEnd": "2026-09-25T15:57:13.347060965Z", "errorCount": 72, "baselineMean": 18.413793103448278, "excess": 31.448275862068968, "patterns": [ { "signature": "77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e", "count": 40, "baselineMean": 8.551724137931034, "excess": 31.448275862068968, "novel": false, "services": [ "nginx" ], "example": { "Timestamp": "2026-09-25T15:57:06.60535739Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "7654e99020c8f9de172e1bf4cbd65aa3", "PatternSignature": "77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/api/metrics/query", "Host": "ui.k6n.net", "StatusCode": 502, "ExceptionType": "", "ClientIP": "", "UAClass": "browser", "IsDuplicate": false, "IsRootCause": true, "HasUpstream": false, "TraceServices": [ "nginx" ], "BadActor": "" } } ] }, "patterns": [ { "category": "dependency_unreachable", "priority": "high", "status": 502, "host": "ui.k6n.net", "path": "/api/metrics/query", "services": [ "nginx" ], "routes": [ { "ingress": "dev/k8s-helper-ingress", "host": "ui.k6n.net", "path": "/", "pathType": "Prefix", "namespace": "dev", "service": "k8s-helper", "port": "80", "deployments": [ "k8s-helper" ] } ], "pods": [ { "namespace": "dev", "deployment": "k8s-helper", "pod": "k8s-helper-5665cd69c9-kjd2t", "lines": [ "I0925 15:56:34.658960 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"", "I0925 15:57:20.162724 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"", "I0925 15:57:24.894618 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"" ], "found": true } ], "rule": "status-5xx-upstream", "new": false, "count": 40, "baseline": 8.551724137931034, "what": "HTTP 502 | /api/metrics/query" } ] } }, { "name": "eventsourced-web 504 on /api/stream", "expect": { "action": "check_capacity", "human": true, "why": "gateway timeouts on the streaming endpoint while the pod only logs probes" }, "alert": { "id": "tlxirp-nginx", "at": "2026-09-25T16:50:13.347044487Z", "spike": { "service": "nginx", "windowStart": "2026-09-25T16:45:13.347044487Z", "windowEnd": "2026-09-25T16:50:13.347044487Z", "errorCount": 62, "baselineMean": 18.93103448275862, "excess": 41.96551724137931, "patterns": [ { "signature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d", "count": 17, "baselineMean": 0, "excess": 17, "novel": false, "services": [ "docker-registry", "nginx" ], "example": { "Timestamp": "2026-09-25T16:50:05.564200851Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "51f8278d03367c21a21f76835ec204b2", "PatternSignature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/v2/eventsourced-ai-worker/manifests/latest", "Host": "registry.k6n.net", "StatusCode": 401, "ExceptionType": "", "ClientIP": "", "UAClass": "unknown", "IsDuplicate": false, "IsRootCause": false, "HasUpstream": true, "TraceServices": [ "docker-registry", "nginx" ], "BadActor": "" } }, { "signature": "2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda", "count": 14, "baselineMean": 0, "excess": 14, "novel": false, "services": [ "docker-registry", "nginx" ], "example": { "Timestamp": "2026-09-25T16:48:10.547871686Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "6664346e0c22d3a9396460163f0b0d6b", "PatternSignature": "2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/v2", "Host": "registry.k6n.net", "StatusCode": 401, "ExceptionType": "", "ClientIP": "", "UAClass": "library", "IsDuplicate": false, "IsRootCause": false, "HasUpstream": true, "TraceServices": [ "docker-registry", "nginx" ], "BadActor": "" } }, { "signature": "ec8f49cca245ee954c6f7a7839235c13a8cfaabc1f0bcac8ecd29c88a5533891", "count": 5, "baselineMean": 0.034482758620689655, "excess": 4.9655172413793105, "novel": false, "clients": 1, "services": [ "eventsourced-orchestrator", "nginx" ], "example": { "Timestamp": "2026-09-25T16:49:50.586320711Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "2b5d0ed3c205922143afa2bc0c5b0b8e", "PatternSignature": "ec8f49cca245ee954c6f7a7839235c13a8cfaabc1f0bcac8ecd29c88a5533891", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/auth/me", "Host": "event-agent.k6n.net", "StatusCode": 401, "ExceptionType": "", "ClientIP": "170.62.100.17", "UAClass": "browser", "IsDuplicate": false, "IsRootCause": false, "HasUpstream": true, "TraceServices": [ "eventsourced-orchestrator", "nginx" ], "BadActor": "" } }, { "signature": "10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b", "count": 3, "baselineMean": 0, "excess": 3, "novel": false, "clients": 1, "services": [ "nginx" ], "example": { "Timestamp": "2026-09-25T16:49:50.586320711Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "44416b3fb21e6f57b9bba5ac0eccd68b", "PatternSignature": "10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": true, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/api/stream", "Host": "event-agent.k6n.net", "StatusCode": 504, "ExceptionType": "", "ClientIP": "170.62.100.17", "UAClass": "browser", "IsDuplicate": false, "IsRootCause": true, "HasUpstream": false, "TraceServices": [ "nginx" ], "BadActor": "" } }, { "signature": "f524391f412efceb1222fd1ccb3c5fcb673ace4787f214acaf300dea6cc88115", "count": 3, "baselineMean": 0, "excess": 3, "novel": false, "clients": 1, "services": [ "nginx" ], "example": { "Timestamp": "2026-09-25T16:48:10.577418006Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "b870baab8dd2de51be3f9da1663d239e", "PatternSignature": "f524391f412efceb1222fd1ccb3c5fcb673ace4787f214acaf300dea6cc88115", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/auth/me", "Host": "event-agent.k6n.net", "StatusCode": 502, "ExceptionType": "", "ClientIP": "170.62.100.17", "UAClass": "browser", "IsDuplicate": false, "IsRootCause": true, "HasUpstream": false, "TraceServices": [ "nginx" ], "BadActor": "" } } ] }, "patterns": [ { "category": "capacity_saturation", "priority": "high", "status": 504, "host": "event-agent.k6n.net", "path": "/api/stream", "services": [ "nginx" ], "routes": [ { "ingress": "dev/eventsourced-web", "host": "event-agent.k6n.net", "path": "/", "pathType": "Prefix", "namespace": "dev", "service": "eventsourced-web", "port": "80", "deployments": [ "eventsourced-web" ] } ], "pods": [ { "namespace": "dev", "deployment": "eventsourced-web", "pod": "eventsourced-web-5865685999-xtxdz", "lines": [ "10.244.9.1 - - [25/Sep/2026:16:50:20 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"", "10.244.9.1 - - [25/Sep/2026:16:50:30 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"", "10.244.9.1 - - [25/Sep/2026:16:50:40 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"" ], "found": true } ], "rule": "timeout-flag", "new": false, "count": 3, "baseline": 0, "clients": 1, "what": "HTTP 504 | /api/stream" }, { "category": "dependency_unreachable", "priority": "high", "status": 502, "host": "event-agent.k6n.net", "path": "/auth/me", "services": [ "nginx" ], "routes": [ { "ingress": "dev/eventsourced-web", "host": "event-agent.k6n.net", "path": "/", "pathType": "Prefix", "namespace": "dev", "service": "eventsourced-web", "port": "80", "deployments": [ "eventsourced-web" ] } ], "pods": [ { "namespace": "dev", "deployment": "eventsourced-web", "pod": "eventsourced-web-5865685999-xtxdz", "lines": [ "10.244.9.1 - - [25/Sep/2026:16:50:20 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"", "10.244.9.1 - - [25/Sep/2026:16:50:30 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"", "10.244.9.1 - - [25/Sep/2026:16:50:40 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"" ], "found": true } ], "rule": "status-5xx-upstream", "new": false, "count": 3, "baseline": 0, "clients": 1, "what": "HTTP 502 | /auth/me" }, { "category": "client_credential_issue", "priority": "low", "status": 401, "host": "registry.k6n.net", "path": "/v2/eventsourced-ai-worker/manifests/latest", "services": [ "docker-registry", "nginx" ], "routes": [ { "ingress": "home/registry", "host": "registry.k6n.net", "path": "/", "pathType": "Prefix", "namespace": "home", "service": "registry", "port": "5000", "deployments": [ "registry" ] } ], "pods": [ { "namespace": "home", "deployment": "registry", "pod": "registry-559bf85884-sxbrw", "lines": [ "10.244.2.1 - - [25/Sep/2026:16:50:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"", "time=\"2026-09-25T16:50:39.183971552Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"85d74dfb059f5fbff7018c85c990e2bc\\\",\\\"SpanID\\\":\\\"dfd025e674cceb15\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T16:50:38.934252898Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:38.934346788Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":54180}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1", "time=\"2026-09-25T16:50:39.184168228Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"b389a9064776af9f4a23cc817ee4231f\\\",\\\"SpanID\\\":\\\"16fa1d622c943f61\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T16:50:39.102298404Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:39.105804949Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1" ], "found": true } ], "rule": "status-401", "new": false, "count": 17, "baseline": 0, "what": "HTTP 401 | /v2/eventsourced-ai-worker/manifests/latest" }, { "category": "client_credential_issue", "priority": "low", "status": 401, "host": "registry.k6n.net", "path": "/v2", "services": [ "docker-registry", "nginx" ], "routes": [ { "ingress": "home/registry", "host": "registry.k6n.net", "path": "/", "pathType": "Prefix", "namespace": "home", "service": "registry", "port": "5000", "deployments": [ "registry" ] } ], "pods": [ { "namespace": "home", "deployment": "registry", "pod": "registry-559bf85884-sxbrw", "lines": [ "10.244.2.1 - - [25/Sep/2026:16:50:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"", "time=\"2026-09-25T16:50:39.183971552Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"85d74dfb059f5fbff7018c85c990e2bc\\\",\\\"SpanID\\\":\\\"dfd025e674cceb15\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T16:50:38.934252898Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:38.934346788Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":54180}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1", "time=\"2026-09-25T16:50:39.184168228Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"b389a9064776af9f4a23cc817ee4231f\\\",\\\"SpanID\\\":\\\"16fa1d622c943f61\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T16:50:39.102298404Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:39.105804949Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1" ], "found": true } ], "rule": "status-401", "new": false, "count": 14, "baseline": 0, "uaClass": "library", "what": "HTTP 401 | /v2" }, { "category": "client_credential_issue", "priority": "low", "status": 401, "host": "event-agent.k6n.net", "path": "/auth/me", "services": [ "eventsourced-orchestrator", "nginx" ], "routes": [ { "ingress": "dev/eventsourced-web", "host": "event-agent.k6n.net", "path": "/", "pathType": "Prefix", "namespace": "dev", "service": "eventsourced-web", "port": "80", "deployments": [ "eventsourced-web" ] } ], "pods": [ { "namespace": "dev", "deployment": "eventsourced-web", "pod": "eventsourced-web-5865685999-xtxdz", "lines": [ "10.244.9.1 - - [25/Sep/2026:16:50:20 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"", "10.244.9.1 - - [25/Sep/2026:16:50:30 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"", "10.244.9.1 - - [25/Sep/2026:16:50:40 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"" ], "found": true } ], "rule": "status-401", "new": false, "count": 5, "baseline": 0.034482758620689655, "clients": 1, "what": "HTTP 401 | /auth/me" } ] } }, { "name": "LLM provider empty responses", "expect": { "action": "fix_dependency", "human": false, "why": "external LLM provider misbehaving; fallback exists, nothing in-cluster to fix unless it persists" }, "alert": { "id": "tlx7pd-eventsourced-orchestrator", "at": "2026-09-25T12:51:13.347152151Z", "spike": { "service": "eventsourced-orchestrator", "windowStart": "2026-09-25T12:46:13.347152151Z", "windowEnd": "2026-09-25T12:51:13.347152151Z", "errorCount": 15, "baselineMean": 0, "excess": 1, "patterns": [ { "signature": "29f5030badef82d221f8a79563cf2c9dbf4d653551ca54b081bc0780c3b35c37", "count": 1, "baselineMean": 0, "excess": 1, "novel": true, "services": [ "eventsourced-orchestrator" ], "example": { "Timestamp": "2026-09-25T12:50:45.113996545Z", "Service": "eventsourced-orchestrator", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "31ff3a0f713c612d62db59873507090e", "PatternSignature": "29f5030badef82d221f8a79563cf2c9dbf4d653551ca54b081bc0780c3b35c37", "IssueFingerprint": "260fc07492b5c2f8", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "*errors.errorstring: provider stream error: {\"code\":,\"message\":\"provider returned an empty response\",\"metadata\":{\"error_type\":\"provider_unavailable\"}}", "SpanName": "orchestrator.llm.attempt", "Route": "", "Host": "", "StatusCode": 0, "ExceptionType": "*errors.errorString", "ClientIP": "", "UAClass": "", "IsDuplicate": false, "IsRootCause": false, "HasUpstream": false, "TraceServices": [ "eventsourced-orchestrator" ], "BadActor": "" } } ] }, "patterns": [ { "category": "dependency_unreachable", "priority": "high", "issueId": "260fc07492b5c2f8", "services": [ "eventsourced-orchestrator" ], "rule": "dependency-text", "new": true, "count": 1, "baseline": 0, "what": "orchestrator.llm.attempt | *errors.errorString | *errors.errorstring: provider stream error: {\"code\":,\"message\":\"provider returned an empty response\",\"metadata\":{\"…" } ] } }, { "name": "LLM response read deadline", "expect": { "action": "fix_dependency", "human": false, "why": "slow external LLM provider; a single pattern count of 1" }, "alert": { "id": "tlx3b1-eventsourced-orchestrator", "at": "2026-09-25T11:16:13.346973966Z", "spike": { "service": "eventsourced-orchestrator", "windowStart": "2026-09-25T11:11:13.346973966Z", "windowEnd": "2026-09-25T11:16:13.346973966Z", "errorCount": 23, "baselineMean": 0, "excess": 1, "patterns": [ { "signature": "ed8d4fd7b78d58f7f14c9fbec7d67d550e8a40f236126cd4964d72ad3645dd9f", "count": 1, "baselineMean": 0, "excess": 1, "novel": false, "services": [ "eventsourced-orchestrator" ], "example": { "Timestamp": "2026-09-25T11:15:34.045242671Z", "Service": "eventsourced-orchestrator", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "a951dfed8b514f48f773df0889f33d4f", "PatternSignature": "ed8d4fd7b78d58f7f14c9fbec7d67d550e8a40f236126cd4964d72ad3645dd9f", "IssueFingerprint": "869896e00332d212", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "*fmt.wraperror: read llm response: context deadline exceeded (client.timeout or context cancellation while reading body)", "SpanName": "llm.provider_fallback", "Route": "", "Host": "", "StatusCode": 0, "ExceptionType": "*fmt.wrapError", "ClientIP": "", "UAClass": "", "IsDuplicate": false, "IsRootCause": false, "HasUpstream": false, "TraceServices": [ "eventsourced-orchestrator" ], "BadActor": "" } } ] }, "patterns": [ { "category": "capacity_saturation", "priority": "high", "issueId": "869896e00332d212", "services": [ "eventsourced-orchestrator" ], "rule": "capacity-text", "new": false, "count": 1, "baseline": 0, "what": "llm.provider_fallback | *fmt.wrapError | *fmt.wraperror: read llm response: context deadline exceeded (client.timeout or context cancellation while reading body)" } ] } }, { "name": "crawler hits scaled-to-zero go-html-edit", "expect": { "action": "ignore", "human": false, "why": "503s only because the dev deployment is scaled to 0 and only AI crawlers asked" }, "alert": { "id": "tlxg6d-nginx", "at": "2026-09-25T15:54:13.346717682Z", "spike": { "service": "nginx", "windowStart": "2026-09-25T15:49:13.346717682Z", "windowEnd": "2026-09-25T15:54:13.346717682Z", "errorCount": 74, "baselineMean": 18.448275862068964, "excess": 12, "patterns": [ { "signature": "0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e", "count": 4, "baselineMean": 0, "excess": 4, "novel": false, "clients": 2, "services": [ "nginx" ], "example": { "Timestamp": "2026-09-25T15:53:46.266195024Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "ecdf390536970c3339612fb2a201df51", "PatternSignature": "0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/api/designer/v1/file-content", "Host": "k6n.net", "StatusCode": 503, "ExceptionType": "", "ClientIP": "104.23.160.37", "UAClass": "ai_crawler", "IsDuplicate": false, "IsRootCause": true, "HasUpstream": false, "TraceServices": [ "nginx" ], "BadActor": "" } }, { "signature": "b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6", "count": 4, "baselineMean": 0, "excess": 4, "novel": false, "clients": 2, "services": [ "nginx" ], "example": { "Timestamp": "2026-09-25T15:53:46.266195024Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "744692761138c480f27fac58b3aa1fda", "PatternSignature": "b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/api/templates/preview", "Host": "k6n.net", "StatusCode": 503, "ExceptionType": "", "ClientIP": "104.23.160.37", "UAClass": "ai_crawler", "IsDuplicate": false, "IsRootCause": true, "HasUpstream": false, "TraceServices": [ "nginx" ], "BadActor": "" } }, { "signature": "004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab", "count": 2, "baselineMean": 0, "excess": 2, "novel": false, "clients": 2, "services": [ "nginx" ], "example": { "Timestamp": "2026-09-25T15:53:46.266195024Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "7ffa5f428b4de59811dd86ded001a33c", "PatternSignature": "004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/api/fs/exec", "Host": "k6n.net", "StatusCode": 503, "ExceptionType": "", "ClientIP": "104.23.160.37", "UAClass": "ai_crawler", "IsDuplicate": false, "IsRootCause": true, "HasUpstream": false, "TraceServices": [ "nginx" ], "BadActor": "" } }, { "signature": "7281b16d4395d124b0fac7a82520ee2b9e179df689e8f4eb6d515a6b567aa256", "count": 2, "baselineMean": 0, "excess": 2, "novel": false, "clients": 2, "services": [ "nginx" ], "example": { "Timestamp": "2026-09-25T15:53:46.266195024Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "942867286e5ba5cac5ec2102cd059106", "PatternSignature": "7281b16d4395d124b0fac7a82520ee2b9e179df689e8f4eb6d515a6b567aa256", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/api/v1/build_public_tmp/{id}/flow", "Host": "k6n.net", "StatusCode": 503, "ExceptionType": "", "ClientIP": "104.23.160.37", "UAClass": "ai_crawler", "IsDuplicate": false, "IsRootCause": true, "HasUpstream": false, "TraceServices": [ "nginx" ], "BadActor": "" } } ] }, "patterns": [ { "category": "dependency_unreachable", "priority": "high", "status": 503, "host": "k6n.net", "path": "/api/designer/v1/file-content", "services": [ "nginx" ], "routes": [ { "ingress": "dev/static-knatofs", "host": "k6n.net", "path": "/api", "pathType": "Prefix", "namespace": "dev", "service": "go-html-edit", "port": "8080", "deployments": [ "go-html-edit" ] } ], "pods": [ { "namespace": "dev", "deployment": "go-html-edit", "found": false } ], "rule": "status-5xx-upstream", "new": false, "count": 4, "baseline": 0, "clients": 2, "uaClass": "ai_crawler", "what": "HTTP 503 | /api/designer/v1/file-content" }, { "category": "dependency_unreachable", "priority": "high", "status": 503, "host": "k6n.net", "path": "/api/templates/preview", "services": [ "nginx" ], "routes": [ { "ingress": "dev/static-knatofs", "host": "k6n.net", "path": "/api", "pathType": "Prefix", "namespace": "dev", "service": "go-html-edit", "port": "8080", "deployments": [ "go-html-edit" ] } ], "pods": [ { "namespace": "dev", "deployment": "go-html-edit", "found": false } ], "rule": "status-5xx-upstream", "new": false, "count": 4, "baseline": 0, "clients": 2, "uaClass": "ai_crawler", "what": "HTTP 503 | /api/templates/preview" }, { "category": "dependency_unreachable", "priority": "high", "status": 503, "host": "k6n.net", "path": "/api/fs/exec", "services": [ "nginx" ], "routes": [ { "ingress": "dev/static-knatofs", "host": "k6n.net", "path": "/api", "pathType": "Prefix", "namespace": "dev", "service": "go-html-edit", "port": "8080", "deployments": [ "go-html-edit" ] } ], "pods": [ { "namespace": "dev", "deployment": "go-html-edit", "found": false } ], "rule": "status-5xx-upstream", "new": false, "count": 2, "baseline": 0, "clients": 2, "uaClass": "ai_crawler", "what": "HTTP 503 | /api/fs/exec" }, { "category": "dependency_unreachable", "priority": "high", "status": 503, "host": "k6n.net", "path": "/api/v1/build_public_tmp/{id}/flow", "services": [ "nginx" ], "routes": [ { "ingress": "dev/static-knatofs", "host": "k6n.net", "path": "/api", "pathType": "Prefix", "namespace": "dev", "service": "go-html-edit", "port": "8080", "deployments": [ "go-html-edit" ] } ], "pods": [ { "namespace": "dev", "deployment": "go-html-edit", "found": false } ], "rule": "status-5xx-upstream", "new": false, "count": 2, "baseline": 0, "clients": 2, "uaClass": "ai_crawler", "what": "HTTP 503 | /api/v1/build_public_tmp/{id}/flow" } ] } }, { "name": "scaled-to-zero game server", "expect": { "action": "ignore", "human": false, "why": "cs-rcon2000 intentionally has no pod; 2 requests" }, "alert": { "id": "tlxevp-nginx", "at": "2026-09-25T15:26:13.346647513Z", "spike": { "service": "nginx", "windowStart": "2026-09-25T15:21:13.346647513Z", "windowEnd": "2026-09-25T15:26:13.346647513Z", "errorCount": 44, "baselineMean": 13.724137931034482, "excess": 2, "patterns": [ { "signature": "3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d", "count": 2, "baselineMean": 0, "excess": 2, "novel": false, "clients": 2, "services": [ "nginx" ], "example": { "Timestamp": "2026-09-25T15:25:31.359710287Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "ea2a5c59210e1c1076b971328b6688c0", "PatternSignature": "3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/", "Host": "rcon-cs.tornberg.me", "StatusCode": 503, "ExceptionType": "", "ClientIP": "172.69.194.250", "UAClass": "browser", "IsDuplicate": false, "IsRootCause": true, "HasUpstream": false, "TraceServices": [ "nginx" ], "BadActor": "" } } ] }, "patterns": [ { "category": "dependency_unreachable", "priority": "high", "status": 503, "host": "rcon-cs.tornberg.me", "path": "/", "services": [ "nginx" ], "routes": [ { "ingress": "game/cs-rcon2000", "host": "rcon-cs.tornberg.me", "path": "/", "pathType": "Prefix", "namespace": "game", "service": "cs-rcon2000", "port": "80", "deployments": [ "cs-rcon2000" ] } ], "pods": [ { "namespace": "game", "deployment": "cs-rcon2000", "found": false } ], "rule": "status-5xx-upstream", "new": false, "count": 2, "baseline": 0, "clients": 2, "what": "HTTP 503 | /" } ] } }, { "name": "registry 401 on manifest pulls", "expect": { "action": "ignore", "human": false, "why": "docker clients probe anonymously first; 401 is the normal auth handshake" }, "alert": { "id": "tlxdwp-nginx", "at": "2026-09-25T15:05:13.346979342Z", "spike": { "service": "nginx", "windowStart": "2026-09-25T15:00:13.346979342Z", "windowEnd": "2026-09-25T15:05:13.346979342Z", "errorCount": 113, "baselineMean": 8.89655172413793, "excess": 5, "patterns": [ { "signature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d", "count": 2, "baselineMean": 0, "excess": 2, "novel": false, "services": [ "docker-registry", "nginx" ], "example": { "Timestamp": "2026-09-25T15:05:06.200527764Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "171ceb5b0a017957cc0510541f80fd07", "PatternSignature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/v2/eventsourced-ai-worker/manifests/latest", "Host": "registry.k6n.net", "StatusCode": 401, "ExceptionType": "", "ClientIP": "", "UAClass": "unknown", "IsDuplicate": false, "IsRootCause": false, "HasUpstream": true, "TraceServices": [ "docker-registry", "nginx" ], "BadActor": "" } }, { "signature": "645f263c238fbbac7ee018af53104cade809f58f461e3939c252e097381e265c", "count": 2, "baselineMean": 0, "excess": 2, "novel": false, "services": [ "docker-registry", "nginx" ], "example": { "Timestamp": "2026-09-25T15:05:06.200527764Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "f90437c17a24182429ec1dbb33fb1209", "PatternSignature": "645f263c238fbbac7ee018af53104cade809f58f461e3939c252e097381e265c", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/v2/eventsourced-ai-worker/manifests/{id}", "Host": "registry.k6n.net", "StatusCode": 401, "ExceptionType": "", "ClientIP": "", "UAClass": "unknown", "IsDuplicate": false, "IsRootCause": false, "HasUpstream": true, "TraceServices": [ "docker-registry", "nginx" ], "BadActor": "" } }, { "signature": "38f693f9028998b8f34bc954af6be996e8b6add9ad53bbbae8c7f2ae5456586f", "count": 1, "baselineMean": 0, "excess": 1, "novel": false, "services": [ "docker-registry", "nginx" ], "example": { "Timestamp": "2026-09-25T15:05:01.199520113Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "b341496525b8a9346fd0ddc5fa26498c", "PatternSignature": "38f693f9028998b8f34bc954af6be996e8b6add9ad53bbbae8c7f2ae5456586f", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/v2/eventsourced-ai-worker-browser/manifests/latest", "Host": "registry.k6n.net", "StatusCode": 401, "ExceptionType": "", "ClientIP": "", "UAClass": "unknown", "IsDuplicate": false, "IsRootCause": false, "HasUpstream": true, "TraceServices": [ "docker-registry", "nginx" ], "BadActor": "" } } ] }, "patterns": [ { "category": "client_credential_issue", "priority": "low", "status": 401, "host": "registry.k6n.net", "path": "/v2/eventsourced-ai-worker/manifests/latest", "services": [ "docker-registry", "nginx" ], "routes": [ { "ingress": "home/registry", "host": "registry.k6n.net", "path": "/", "pathType": "Prefix", "namespace": "home", "service": "registry", "port": "5000", "deployments": [ "registry" ] } ], "pods": [ { "namespace": "home", "deployment": "registry", "pod": "registry-559bf85884-sxbrw", "lines": [ "10.244.2.1 - - [25/Sep/2026:15:05:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"", "time=\"2026-09-25T15:05:41.507526902Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"758d99c1d694e7816fbf337d231dc906\\\",\\\"SpanID\\\":\\\"7cb6d7f33c9cf9e7\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:05:38.934630694Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:38.934733409Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36062}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1", "time=\"2026-09-25T15:05:41.507683183Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f4b89e61b17217cfa6bda198b2b91f46\\\",\\\"SpanID\\\":\\\"d95c0c7e1b5008ba\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:05:39.10176949Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:39.103919224Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1" ], "found": true } ], "rule": "status-401", "new": false, "count": 2, "baseline": 0, "what": "HTTP 401 | /v2/eventsourced-ai-worker/manifests/latest" }, { "category": "client_credential_issue", "priority": "low", "status": 401, "host": "registry.k6n.net", "path": "/v2/eventsourced-ai-worker/manifests/{id}", "services": [ "docker-registry", "nginx" ], "routes": [ { "ingress": "home/registry", "host": "registry.k6n.net", "path": "/", "pathType": "Prefix", "namespace": "home", "service": "registry", "port": "5000", "deployments": [ "registry" ] } ], "pods": [ { "namespace": "home", "deployment": "registry", "pod": "registry-559bf85884-sxbrw", "lines": [ "10.244.2.1 - - [25/Sep/2026:15:05:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"", "time=\"2026-09-25T15:05:41.507526902Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"758d99c1d694e7816fbf337d231dc906\\\",\\\"SpanID\\\":\\\"7cb6d7f33c9cf9e7\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:05:38.934630694Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:38.934733409Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36062}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1", "time=\"2026-09-25T15:05:41.507683183Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f4b89e61b17217cfa6bda198b2b91f46\\\",\\\"SpanID\\\":\\\"d95c0c7e1b5008ba\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:05:39.10176949Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:39.103919224Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1" ], "found": true } ], "rule": "status-401", "new": false, "count": 2, "baseline": 0, "what": "HTTP 401 | /v2/eventsourced-ai-worker/manifests/{id}" }, { "category": "client_credential_issue", "priority": "low", "status": 401, "host": "registry.k6n.net", "path": "/v2/eventsourced-ai-worker-browser/manifests/latest", "services": [ "docker-registry", "nginx" ], "routes": [ { "ingress": "home/registry", "host": "registry.k6n.net", "path": "/", "pathType": "Prefix", "namespace": "home", "service": "registry", "port": "5000", "deployments": [ "registry" ] } ], "pods": [ { "namespace": "home", "deployment": "registry", "pod": "registry-559bf85884-sxbrw", "lines": [ "10.244.2.1 - - [25/Sep/2026:15:05:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"", "time=\"2026-09-25T15:05:41.507526902Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"758d99c1d694e7816fbf337d231dc906\\\",\\\"SpanID\\\":\\\"7cb6d7f33c9cf9e7\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:05:38.934630694Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:38.934733409Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36062}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1", "time=\"2026-09-25T15:05:41.507683183Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f4b89e61b17217cfa6bda198b2b91f46\\\",\\\"SpanID\\\":\\\"d95c0c7e1b5008ba\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:05:39.10176949Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:39.103919224Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1" ], "found": true } ], "rule": "status-401", "new": false, "count": 1, "baseline": 0, "what": "HTTP 401 | /v2/eventsourced-ai-worker-browser/manifests/latest" } ] } }, { "name": "transmission 409 session handshake", "expect": { "action": "ignore", "human": false, "why": "Transmission RPC answers 409 to hand out its session id; expected protocol" }, "alert": { "id": "tlxc21-nginx", "at": "2026-09-25T14:25:13.346654632Z", "spike": { "service": "nginx", "windowStart": "2026-09-25T14:20:13.346654632Z", "windowEnd": "2026-09-25T14:25:13.346654632Z", "errorCount": 31, "baselineMean": 4.137931034482759, "excess": 4, "patterns": [ { "signature": "efb1d02de6ef2fbc3e1f234fa547da4e31652733a4c6f8aababe8da1900871f9", "count": 4, "baselineMean": 0, "excess": 4, "novel": true, "clients": 1, "services": [ "nginx" ], "example": { "Timestamp": "2026-09-25T14:24:15.768783875Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "fcd10da95cccfebe9e02e43df79d7570", "PatternSignature": "efb1d02de6ef2fbc3e1f234fa547da4e31652733a4c6f8aababe8da1900871f9", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/transmission/rpc", "Host": "torrent.k6n.net", "StatusCode": 409, "ExceptionType": "", "ClientIP": "209.206.25.124", "UAClass": "browser", "IsDuplicate": false, "IsRootCause": true, "HasUpstream": false, "TraceServices": [ "nginx" ], "BadActor": "" } } ] }, "patterns": [ { "category": "needs_review", "priority": "low", "status": 409, "host": "torrent.k6n.net", "path": "/transmission/rpc", "services": [ "nginx" ], "routes": [ { "ingress": "home/torrent-ingress", "host": "torrent.k6n.net", "path": "/", "pathType": "Prefix", "namespace": "home", "service": "torrent-service", "port": "9091" } ], "rule": "no-rule", "new": true, "count": 4, "baseline": 0, "clients": 1, "what": "HTTP 409 | /transmission/rpc" } ] } }, { "name": "405 on /", "expect": { "action": "ignore", "human": false, "why": "a client using the wrong method on the root; 4 requests from 1 client" }, "alert": { "id": "tlxn9d-nginx", "at": "2026-09-25T18:27:13.347226389Z", "spike": { "service": "nginx", "windowStart": "2026-09-25T18:22:13.347226389Z", "windowEnd": "2026-09-25T18:27:13.347226389Z", "errorCount": 64, "baselineMean": 13.931034482758621, "excess": 4, "patterns": [ { "signature": "ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f", "count": 4, "baselineMean": 0, "excess": 4, "novel": false, "clients": 1, "services": [ "nginx" ], "example": { "Timestamp": "2026-09-25T18:26:26.152736211Z", "Service": "nginx", "Release": "", "Kind": "span", "Source": "backend", "IsError": true, "StatusClass": "", "DurationMS": 0, "TraceID": "d1de7de09ad342e7b0670cb78cac1430", "PatternSignature": "ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f", "IssueFingerprint": "", "IsRetry": false, "IsTimeout": false, "IsCancellation": false, "Message": "", "SpanName": "", "Route": "/", "Host": "cc.k6n.net", "StatusCode": 405, "ExceptionType": "", "ClientIP": "35.189.233.235", "UAClass": "browser", "IsDuplicate": false, "IsRootCause": true, "HasUpstream": false, "TraceServices": [ "nginx" ], "BadActor": "" } } ] }, "patterns": [ { "category": "needs_review", "priority": "low", "status": 405, "host": "cc.k6n.net", "path": "/", "services": [ "nginx" ], "rule": "no-rule", "new": false, "count": 4, "baseline": 0, "clients": 1, "what": "HTTP 405 | /" } ] } } ]