mats/go-otel
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
601f382af6d1b0be11cdff070e2e6d79f4259e96
go-otel/cmd/laya-eval/scenarios.json
T
mats e046bd86d1 eval
2026-09-27 16:45:10 +02:00

2099 lines
82 KiB
JSON
Raw Blame History

[
{
"name": "codesearch upload deadline exceeded",
"expect": {
"action": "check_capacity",
"human": true,
"why": "uploads time out repeatedly; the codesearch path is saturated or too slow"
},
"alert": {
"id": "tlxmh1-eventsourced-orchestrator",
"at": "2026-09-25T18:10:13.34664129Z",
"spike": {
"service": "eventsourced-orchestrator",
"windowStart": "2026-09-25T18:05:13.34664129Z",
"windowEnd": "2026-09-25T18:10:13.34664129Z",
"errorCount": 16,
"baselineMean": 19.93103448275862,
"excess": 16,
"patterns": [
{
"signature": "8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef",
"count": 16,
"baselineMean": 0,
"excess": 16,
"novel": false,
"services": [
"eventsourced-orchestrator"
],
"example": {
"Timestamp": "2026-09-25T18:10:08.793065336Z",
"Service": "eventsourced-orchestrator",
"Release": "",
"Kind": "log",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "",
"PatternSignature": "8d66d20152ce8563ce2ec70cc41706fa5e15b94ba1a343713b6823cf3ffe01ef",
"IssueFingerprint": "b57379b9dd3a6a9d",
"IsRetry": false,
"IsTimeout": true,
"IsCancellation": false,
"Message": "codesearch upload failed",
"SpanName": "",
"Route": "",
"Host": "",
"StatusCode": 0,
"ExceptionType": "context.deadlineExceededError",
"ClientIP": "",
"UAClass": "",
"IsDuplicate": false,
"IsRootCause": false,
"HasUpstream": false,
"TraceServices": null,
"BadActor": ""
}
}
]
},
"patterns": [
{
"category": "capacity_saturation",
"priority": "high",
"issueId": "b57379b9dd3a6a9d",
"services": [
"eventsourced-orchestrator"
],
"rule": "timeout-flag",
"new": false,
"count": 16,
"baseline": 0,
"what": "context.deadlineExceededError | codesearch upload failed"
}
]
}
},
{
"name": "worker cannot dial orchestrator websocket",
"expect": {
"action": "fix_dependency",
"human": true,
"why": "workers cannot reach the in-cluster orchestrator; the dependency is down or unreachable"
},
"alert": {
"id": "tlxmdp-eventsourced-worker-code",
"at": "2026-09-25T18:08:13.346954479Z",
"spike": {
"service": "eventsourced-worker-code",
"windowStart": "2026-09-25T18:03:13.346954479Z",
"windowEnd": "2026-09-25T18:08:13.346954479Z",
"errorCount": 36,
"baselineMean": 0.13793103448275862,
"excess": 36,
"patterns": [
{
"signature": "fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885",
"count": 29,
"baselineMean": 0,
"excess": 29,
"novel": false,
"services": [
"eventsourced-worker-code"
],
"example": {
"Timestamp": "2026-09-25T18:08:12.308751715Z",
"Service": "eventsourced-worker-code",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "34f7c7c1d9c07bb48b3241454d10a7bf",
"PatternSignature": "fc789999d6af6a1c511f7d606f60852b06878f5efda8bb388f44569eb26d4885",
"IssueFingerprint": "4580ae864fa21049",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "*websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:<num>/internal/worker-terminal: dial tcp <num>.<num>.<num>.<num>:<num>: connect: connection refused",
"SpanName": "worker.terminal.connection",
"Route": "",
"Host": "",
"StatusCode": 0,
"ExceptionType": "*websocket.DialError",
"ClientIP": "",
"UAClass": "",
"IsDuplicate": false,
"IsRootCause": true,
"HasUpstream": false,
"TraceServices": [
"eventsourced-worker-code"
],
"BadActor": ""
}
},
{
"signature": "2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7",
"count": 7,
"baselineMean": 0,
"excess": 7,
"novel": false,
"services": [
"eventsourced-worker-code"
],
"example": {
"Timestamp": "2026-09-25T18:07:50.177441418Z",
"Service": "eventsourced-worker-code",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "72e0cbf2c415b129eab8f4aadca42c9f",
"PatternSignature": "2de02c9635a7e371aa97960a82c3dd98f0e360f74cb7f7448aaeae31c86605a7",
"IssueFingerprint": "b40901e8fbfcf66e",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "*errors.errorstring: eof",
"SpanName": "worker.terminal.connection",
"Route": "",
"Host": "",
"StatusCode": 0,
"ExceptionType": "*errors.errorString",
"ClientIP": "",
"UAClass": "",
"IsDuplicate": false,
"IsRootCause": true,
"HasUpstream": false,
"TraceServices": [
"eventsourced-worker-code"
],
"BadActor": ""
}
}
]
},
"patterns": [
{
"category": "dependency_unreachable",
"priority": "high",
"issueId": "4580ae864fa21049",
"services": [
"eventsourced-worker-code"
],
"rule": "dependency-text",
"new": false,
"count": 29,
"baseline": 0,
"what": "worker.terminal.connection | *websocket.DialError | *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:<num>/internal/worker-termina…"
},
{
"category": "dependency_unreachable",
"priority": "high",
"issueId": "b40901e8fbfcf66e",
"services": [
"eventsourced-worker-code"
],
"rule": "dependency-text",
"new": false,
"count": 7,
"baseline": 0,
"what": "worker.terminal.connection | *errors.errorString | *errors.errorstring: eof"
}
]
}
},
{
"name": "bash tool args fail to unmarshal",
"expect": {
"action": "investigate_code",
"human": true,
"why": "a new json decode error on timeout_seconds is a code/contract bug, not capacity (triage mislabels it via the word 'timeout')"
},
"alert": {
"id": "tlxbx1-eventsourced-worker-code",
"at": "2026-09-25T14:22:13.347454307Z",
"spike": {
"service": "eventsourced-worker-code",
"windowStart": "2026-09-25T14:17:13.347454307Z",
"windowEnd": "2026-09-25T14:22:13.347454307Z",
"errorCount": 35,
"baselineMean": 0.41379310344827586,
"excess": 8,
"patterns": [
{
"signature": "c79ebdd574e5a9103bcc3c2faaa06abee7f041701094b24e883949dcc5371eab",
"count": 8,
"baselineMean": 0,
"excess": 8,
"novel": true,
"services": [
"eventsourced-worker-code"
],
"example": {
"Timestamp": "2026-09-25T14:22:02.949148899Z",
"Service": "eventsourced-worker-code",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "cb422f420e0e5e387e57384138fa6f42",
"PatternSignature": "c79ebdd574e5a9103bcc3c2faaa06abee7f041701094b24e883949dcc5371eab",
"IssueFingerprint": "b082dde6853f6053",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "*worker.error: bash: invalid arguments: json: cannot unmarshal string into go struct field bashargs.timeout_seconds of type int",
"SpanName": "worker.fulfill",
"Route": "",
"Host": "",
"StatusCode": 0,
"ExceptionType": "*worker.Error",
"ClientIP": "",
"UAClass": "",
"IsDuplicate": false,
"IsRootCause": true,
"HasUpstream": false,
"TraceServices": [
"eventsourced-worker-code"
],
"BadActor": ""
}
}
]
},
"patterns": [
{
"category": "capacity_saturation",
"priority": "high",
"issueId": "b082dde6853f6053",
"services": [
"eventsourced-worker-code"
],
"rule": "capacity-text",
"new": true,
"count": 8,
"baseline": 0,
"what": "worker.fulfill | *worker.Error | *worker.error: bash: invalid arguments: json: cannot unmarshal string into go struct field bashargs.timeout_seconds of …"
}
]
}
},
{
"name": "go-api 500 on /api/network-speed",
"expect": {
"action": "investigate_code",
"human": true,
"why": "500 from a running pod with no logs: a defect in go-api"
},
"alert": {
"id": "tlxdyd-nginx",
"at": "2026-09-25T15:06:13.347093859Z",
"spike": {
"service": "nginx",
"windowStart": "2026-09-25T15:01:13.347093859Z",
"windowEnd": "2026-09-25T15:06:13.347093859Z",
"errorCount": 121,
"baselineMean": 8.379310344827585,
"excess": 16,
"patterns": [
{
"signature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d",
"count": 14,
"baselineMean": 0,
"excess": 14,
"novel": false,
"services": [
"docker-registry",
"nginx"
],
"example": {
"Timestamp": "2026-09-25T15:05:36.202277428Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "a4da26a4a32edf925be5858fb04f23fe",
"PatternSignature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/v2/eventsourced-ai-worker/manifests/latest",
"Host": "registry.k6n.net",
"StatusCode": 401,
"ExceptionType": "",
"ClientIP": "",
"UAClass": "unknown",
"IsDuplicate": false,
"IsRootCause": false,
"HasUpstream": true,
"TraceServices": [
"docker-registry",
"nginx"
],
"BadActor": ""
}
},
{
"signature": "63eff8b04309c018ea6fe0604d6e3b0885a94f3252cdf65ac9e777ae320a3af7",
"count": 2,
"baselineMean": 0,
"excess": 2,
"novel": false,
"services": [
"nginx"
],
"example": {
"Timestamp": "2026-09-25T15:05:16.201179908Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "bef5078df057b841700020d2d7128086",
"PatternSignature": "63eff8b04309c018ea6fe0604d6e3b0885a94f3252cdf65ac9e777ae320a3af7",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/api/network-speed",
"Host": "tornberg.me",
"StatusCode": 500,
"ExceptionType": "",
"ClientIP": "",
"UAClass": "",
"IsDuplicate": false,
"IsRootCause": true,
"HasUpstream": false,
"TraceServices": [
"nginx"
],
"BadActor": ""
}
}
]
},
"patterns": [
{
"category": "service_defect",
"priority": "high",
"status": 500,
"host": "tornberg.me",
"path": "/api/network-speed",
"services": [
"nginx"
],
"routes": [
{
"ingress": "dev/static-web",
"host": "tornberg.me",
"path": "/api",
"pathType": "Prefix",
"namespace": "dev",
"service": "go-api",
"port": "3000",
"deployments": [
"go-api"
]
}
],
"pods": [
{
"namespace": "dev",
"deployment": "go-api",
"pod": "go-api-85d88c9bd-qp8p4",
"found": true
}
],
"rule": "status-5xx",
"new": false,
"count": 2,
"baseline": 0,
"what": "HTTP 500 | /api/network-speed"
},
{
"category": "client_credential_issue",
"priority": "low",
"status": 401,
"host": "registry.k6n.net",
"path": "/v2/eventsourced-ai-worker/manifests/latest",
"services": [
"docker-registry",
"nginx"
],
"routes": [
{
"ingress": "home/registry",
"host": "registry.k6n.net",
"path": "/",
"pathType": "Prefix",
"namespace": "home",
"service": "registry",
"port": "5000",
"deployments": [
"registry"
]
}
],
"pods": [
{
"namespace": "home",
"deployment": "registry",
"pod": "registry-559bf85884-sxbrw",
"lines": [
"10.244.2.1 - - [25/Sep/2026:15:06:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"",
"time=\"2026-09-25T15:06:41.534921099Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"5fa8dd0ea0a6e33d84a5d5cdce9938fb\\\",\\\"SpanID\\\":\\\"d195e207dfbf1775\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:06:38.93415321Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:06:38.934250546Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":42508}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1",
"time=\"2026-09-25T15:06:41.535080838Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"8db442a0165869c57a316a8faa1f8fb8\\\",\\\"SpanID\\\":\\\"e498e0a764f76319\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:06:39.102209346Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:06:39.10428595Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"
],
"found": true
}
],
"rule": "status-401",
"new": false,
"count": 14,
"baseline": 0,
"what": "HTTP 401 | /v2/eventsourced-ai-worker/manifests/latest"
}
]
}
},
{
"name": "k8s-helper 502 on /api/metrics/query",
"expect": {
"action": "fix_dependency",
"human": true,
"why": "40 bad gateways against a running k8s-helper: its upstream (Prometheus/metrics) or the pod path is failing"
},
"alert": {
"id": "tlxgbd-nginx",
"at": "2026-09-25T15:57:13.347060965Z",
"spike": {
"service": "nginx",
"windowStart": "2026-09-25T15:52:13.347060965Z",
"windowEnd": "2026-09-25T15:57:13.347060965Z",
"errorCount": 72,
"baselineMean": 18.413793103448278,
"excess": 31.448275862068968,
"patterns": [
{
"signature": "77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e",
"count": 40,
"baselineMean": 8.551724137931034,
"excess": 31.448275862068968,
"novel": false,
"services": [
"nginx"
],
"example": {
"Timestamp": "2026-09-25T15:57:06.60535739Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "7654e99020c8f9de172e1bf4cbd65aa3",
"PatternSignature": "77be697bb9fd8f44d138b4fe65ead1a4fbb1dd3f573acbb7eadd78cd2f52372e",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/api/metrics/query",
"Host": "ui.k6n.net",
"StatusCode": 502,
"ExceptionType": "",
"ClientIP": "",
"UAClass": "browser",
"IsDuplicate": false,
"IsRootCause": true,
"HasUpstream": false,
"TraceServices": [
"nginx"
],
"BadActor": ""
}
}
]
},
"patterns": [
{
"category": "dependency_unreachable",
"priority": "high",
"status": 502,
"host": "ui.k6n.net",
"path": "/api/metrics/query",
"services": [
"nginx"
],
"routes": [
{
"ingress": "dev/k8s-helper-ingress",
"host": "ui.k6n.net",
"path": "/",
"pathType": "Prefix",
"namespace": "dev",
"service": "k8s-helper",
"port": "80",
"deployments": [
"k8s-helper"
]
}
],
"pods": [
{
"namespace": "dev",
"deployment": "k8s-helper",
"pod": "k8s-helper-5665cd69c9-kjd2t",
"lines": [
"I0925 15:56:34.658960 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"",
"I0925 15:57:20.162724 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\"",
"I0925 15:57:24.894618 1 warnings.go:107] \"Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\""
],
"found": true
}
],
"rule": "status-5xx-upstream",
"new": false,
"count": 40,
"baseline": 8.551724137931034,
"what": "HTTP 502 | /api/metrics/query"
}
]
}
},
{
"name": "eventsourced-web 504 on /api/stream",
"expect": {
"action": "check_capacity",
"human": true,
"why": "gateway timeouts on the streaming endpoint while the pod only logs probes"
},
"alert": {
"id": "tlxirp-nginx",
"at": "2026-09-25T16:50:13.347044487Z",
"spike": {
"service": "nginx",
"windowStart": "2026-09-25T16:45:13.347044487Z",
"windowEnd": "2026-09-25T16:50:13.347044487Z",
"errorCount": 62,
"baselineMean": 18.93103448275862,
"excess": 41.96551724137931,
"patterns": [
{
"signature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d",
"count": 17,
"baselineMean": 0,
"excess": 17,
"novel": false,
"services": [
"docker-registry",
"nginx"
],
"example": {
"Timestamp": "2026-09-25T16:50:05.564200851Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "51f8278d03367c21a21f76835ec204b2",
"PatternSignature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/v2/eventsourced-ai-worker/manifests/latest",
"Host": "registry.k6n.net",
"StatusCode": 401,
"ExceptionType": "",
"ClientIP": "",
"UAClass": "unknown",
"IsDuplicate": false,
"IsRootCause": false,
"HasUpstream": true,
"TraceServices": [
"docker-registry",
"nginx"
],
"BadActor": ""
}
},
{
"signature": "2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda",
"count": 14,
"baselineMean": 0,
"excess": 14,
"novel": false,
"services": [
"docker-registry",
"nginx"
],
"example": {
"Timestamp": "2026-09-25T16:48:10.547871686Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "6664346e0c22d3a9396460163f0b0d6b",
"PatternSignature": "2ba6d4b4d01f995ee5087f0fea1db074efddba4b8c1a423dbdc1ec5014652dda",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/v2",
"Host": "registry.k6n.net",
"StatusCode": 401,
"ExceptionType": "",
"ClientIP": "",
"UAClass": "library",
"IsDuplicate": false,
"IsRootCause": false,
"HasUpstream": true,
"TraceServices": [
"docker-registry",
"nginx"
],
"BadActor": ""
}
},
{
"signature": "ec8f49cca245ee954c6f7a7839235c13a8cfaabc1f0bcac8ecd29c88a5533891",
"count": 5,
"baselineMean": 0.034482758620689655,
"excess": 4.9655172413793105,
"novel": false,
"clients": 1,
"services": [
"eventsourced-orchestrator",
"nginx"
],
"example": {
"Timestamp": "2026-09-25T16:49:50.586320711Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "2b5d0ed3c205922143afa2bc0c5b0b8e",
"PatternSignature": "ec8f49cca245ee954c6f7a7839235c13a8cfaabc1f0bcac8ecd29c88a5533891",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/auth/me",
"Host": "event-agent.k6n.net",
"StatusCode": 401,
"ExceptionType": "",
"ClientIP": "170.62.100.17",
"UAClass": "browser",
"IsDuplicate": false,
"IsRootCause": false,
"HasUpstream": true,
"TraceServices": [
"eventsourced-orchestrator",
"nginx"
],
"BadActor": ""
}
},
{
"signature": "10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b",
"count": 3,
"baselineMean": 0,
"excess": 3,
"novel": false,
"clients": 1,
"services": [
"nginx"
],
"example": {
"Timestamp": "2026-09-25T16:49:50.586320711Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "44416b3fb21e6f57b9bba5ac0eccd68b",
"PatternSignature": "10bbcf3f9283b4c5344a0007e528062e5258836a955379783837e1b0f9c9533b",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": true,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/api/stream",
"Host": "event-agent.k6n.net",
"StatusCode": 504,
"ExceptionType": "",
"ClientIP": "170.62.100.17",
"UAClass": "browser",
"IsDuplicate": false,
"IsRootCause": true,
"HasUpstream": false,
"TraceServices": [
"nginx"
],
"BadActor": ""
}
},
{
"signature": "f524391f412efceb1222fd1ccb3c5fcb673ace4787f214acaf300dea6cc88115",
"count": 3,
"baselineMean": 0,
"excess": 3,
"novel": false,
"clients": 1,
"services": [
"nginx"
],
"example": {
"Timestamp": "2026-09-25T16:48:10.577418006Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "b870baab8dd2de51be3f9da1663d239e",
"PatternSignature": "f524391f412efceb1222fd1ccb3c5fcb673ace4787f214acaf300dea6cc88115",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/auth/me",
"Host": "event-agent.k6n.net",
"StatusCode": 502,
"ExceptionType": "",
"ClientIP": "170.62.100.17",
"UAClass": "browser",
"IsDuplicate": false,
"IsRootCause": true,
"HasUpstream": false,
"TraceServices": [
"nginx"
],
"BadActor": ""
}
}
]
},
"patterns": [
{
"category": "capacity_saturation",
"priority": "high",
"status": 504,
"host": "event-agent.k6n.net",
"path": "/api/stream",
"services": [
"nginx"
],
"routes": [
{
"ingress": "dev/eventsourced-web",
"host": "event-agent.k6n.net",
"path": "/",
"pathType": "Prefix",
"namespace": "dev",
"service": "eventsourced-web",
"port": "80",
"deployments": [
"eventsourced-web"
]
}
],
"pods": [
{
"namespace": "dev",
"deployment": "eventsourced-web",
"pod": "eventsourced-web-5865685999-xtxdz",
"lines": [
"10.244.9.1 - - [25/Sep/2026:16:50:20 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"",
"10.244.9.1 - - [25/Sep/2026:16:50:30 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"",
"10.244.9.1 - - [25/Sep/2026:16:50:40 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""
],
"found": true
}
],
"rule": "timeout-flag",
"new": false,
"count": 3,
"baseline": 0,
"clients": 1,
"what": "HTTP 504 | /api/stream"
},
{
"category": "dependency_unreachable",
"priority": "high",
"status": 502,
"host": "event-agent.k6n.net",
"path": "/auth/me",
"services": [
"nginx"
],
"routes": [
{
"ingress": "dev/eventsourced-web",
"host": "event-agent.k6n.net",
"path": "/",
"pathType": "Prefix",
"namespace": "dev",
"service": "eventsourced-web",
"port": "80",
"deployments": [
"eventsourced-web"
]
}
],
"pods": [
{
"namespace": "dev",
"deployment": "eventsourced-web",
"pod": "eventsourced-web-5865685999-xtxdz",
"lines": [
"10.244.9.1 - - [25/Sep/2026:16:50:20 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"",
"10.244.9.1 - - [25/Sep/2026:16:50:30 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"",
"10.244.9.1 - - [25/Sep/2026:16:50:40 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""
],
"found": true
}
],
"rule": "status-5xx-upstream",
"new": false,
"count": 3,
"baseline": 0,
"clients": 1,
"what": "HTTP 502 | /auth/me"
},
{
"category": "client_credential_issue",
"priority": "low",
"status": 401,
"host": "registry.k6n.net",
"path": "/v2/eventsourced-ai-worker/manifests/latest",
"services": [
"docker-registry",
"nginx"
],
"routes": [
{
"ingress": "home/registry",
"host": "registry.k6n.net",
"path": "/",
"pathType": "Prefix",
"namespace": "home",
"service": "registry",
"port": "5000",
"deployments": [
"registry"
]
}
],
"pods": [
{
"namespace": "home",
"deployment": "registry",
"pod": "registry-559bf85884-sxbrw",
"lines": [
"10.244.2.1 - - [25/Sep/2026:16:50:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"",
"time=\"2026-09-25T16:50:39.183971552Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"85d74dfb059f5fbff7018c85c990e2bc\\\",\\\"SpanID\\\":\\\"dfd025e674cceb15\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T16:50:38.934252898Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:38.934346788Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":54180}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1",
"time=\"2026-09-25T16:50:39.184168228Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"b389a9064776af9f4a23cc817ee4231f\\\",\\\"SpanID\\\":\\\"16fa1d622c943f61\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T16:50:39.102298404Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:39.105804949Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"
],
"found": true
}
],
"rule": "status-401",
"new": false,
"count": 17,
"baseline": 0,
"what": "HTTP 401 | /v2/eventsourced-ai-worker/manifests/latest"
},
{
"category": "client_credential_issue",
"priority": "low",
"status": 401,
"host": "registry.k6n.net",
"path": "/v2",
"services": [
"docker-registry",
"nginx"
],
"routes": [
{
"ingress": "home/registry",
"host": "registry.k6n.net",
"path": "/",
"pathType": "Prefix",
"namespace": "home",
"service": "registry",
"port": "5000",
"deployments": [
"registry"
]
}
],
"pods": [
{
"namespace": "home",
"deployment": "registry",
"pod": "registry-559bf85884-sxbrw",
"lines": [
"10.244.2.1 - - [25/Sep/2026:16:50:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"",
"time=\"2026-09-25T16:50:39.183971552Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"85d74dfb059f5fbff7018c85c990e2bc\\\",\\\"SpanID\\\":\\\"dfd025e674cceb15\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T16:50:38.934252898Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:38.934346788Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":54180}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1",
"time=\"2026-09-25T16:50:39.184168228Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"b389a9064776af9f4a23cc817ee4231f\\\",\\\"SpanID\\\":\\\"16fa1d622c943f61\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T16:50:39.102298404Z\\\",\\\"EndTime\\\":\\\"2026-09-25T16:50:39.105804949Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"
],
"found": true
}
],
"rule": "status-401",
"new": false,
"count": 14,
"baseline": 0,
"uaClass": "library",
"what": "HTTP 401 | /v2"
},
{
"category": "client_credential_issue",
"priority": "low",
"status": 401,
"host": "event-agent.k6n.net",
"path": "/auth/me",
"services": [
"eventsourced-orchestrator",
"nginx"
],
"routes": [
{
"ingress": "dev/eventsourced-web",
"host": "event-agent.k6n.net",
"path": "/",
"pathType": "Prefix",
"namespace": "dev",
"service": "eventsourced-web",
"port": "80",
"deployments": [
"eventsourced-web"
]
}
],
"pods": [
{
"namespace": "dev",
"deployment": "eventsourced-web",
"pod": "eventsourced-web-5865685999-xtxdz",
"lines": [
"10.244.9.1 - - [25/Sep/2026:16:50:20 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"",
"10.244.9.1 - - [25/Sep/2026:16:50:30 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\"",
"10.244.9.1 - - [25/Sep/2026:16:50:40 +0000] \"GET / HTTP/1.1\" 200 3020 \"-\" \"kube-probe/1.36\" \"-\""
],
"found": true
}
],
"rule": "status-401",
"new": false,
"count": 5,
"baseline": 0.034482758620689655,
"clients": 1,
"what": "HTTP 401 | /auth/me"
}
]
}
},
{
"name": "LLM provider empty responses",
"expect": {
"action": "fix_dependency",
"human": false,
"why": "external LLM provider misbehaving; fallback exists, nothing in-cluster to fix unless it persists"
},
"alert": {
"id": "tlx7pd-eventsourced-orchestrator",
"at": "2026-09-25T12:51:13.347152151Z",
"spike": {
"service": "eventsourced-orchestrator",
"windowStart": "2026-09-25T12:46:13.347152151Z",
"windowEnd": "2026-09-25T12:51:13.347152151Z",
"errorCount": 15,
"baselineMean": 0,
"excess": 1,
"patterns": [
{
"signature": "29f5030badef82d221f8a79563cf2c9dbf4d653551ca54b081bc0780c3b35c37",
"count": 1,
"baselineMean": 0,
"excess": 1,
"novel": true,
"services": [
"eventsourced-orchestrator"
],
"example": {
"Timestamp": "2026-09-25T12:50:45.113996545Z",
"Service": "eventsourced-orchestrator",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "31ff3a0f713c612d62db59873507090e",
"PatternSignature": "29f5030badef82d221f8a79563cf2c9dbf4d653551ca54b081bc0780c3b35c37",
"IssueFingerprint": "260fc07492b5c2f8",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "*errors.errorstring: provider stream error: {\"code\":<num>,\"message\":\"provider returned an empty response\",\"metadata\":{\"error_type\":\"provider_unavailable\"}}",
"SpanName": "orchestrator.llm.attempt",
"Route": "",
"Host": "",
"StatusCode": 0,
"ExceptionType": "*errors.errorString",
"ClientIP": "",
"UAClass": "",
"IsDuplicate": false,
"IsRootCause": false,
"HasUpstream": false,
"TraceServices": [
"eventsourced-orchestrator"
],
"BadActor": ""
}
}
]
},
"patterns": [
{
"category": "dependency_unreachable",
"priority": "high",
"issueId": "260fc07492b5c2f8",
"services": [
"eventsourced-orchestrator"
],
"rule": "dependency-text",
"new": true,
"count": 1,
"baseline": 0,
"what": "orchestrator.llm.attempt | *errors.errorString | *errors.errorstring: provider stream error: {\"code\":<num>,\"message\":\"provider returned an empty response\",\"metadata\":{\"…"
}
]
}
},
{
"name": "LLM response read deadline",
"expect": {
"action": "fix_dependency",
"human": false,
"why": "slow external LLM provider; a single pattern count of 1"
},
"alert": {
"id": "tlx3b1-eventsourced-orchestrator",
"at": "2026-09-25T11:16:13.346973966Z",
"spike": {
"service": "eventsourced-orchestrator",
"windowStart": "2026-09-25T11:11:13.346973966Z",
"windowEnd": "2026-09-25T11:16:13.346973966Z",
"errorCount": 23,
"baselineMean": 0,
"excess": 1,
"patterns": [
{
"signature": "ed8d4fd7b78d58f7f14c9fbec7d67d550e8a40f236126cd4964d72ad3645dd9f",
"count": 1,
"baselineMean": 0,
"excess": 1,
"novel": false,
"services": [
"eventsourced-orchestrator"
],
"example": {
"Timestamp": "2026-09-25T11:15:34.045242671Z",
"Service": "eventsourced-orchestrator",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "a951dfed8b514f48f773df0889f33d4f",
"PatternSignature": "ed8d4fd7b78d58f7f14c9fbec7d67d550e8a40f236126cd4964d72ad3645dd9f",
"IssueFingerprint": "869896e00332d212",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "*fmt.wraperror: read llm response: context deadline exceeded (client.timeout or context cancellation while reading body)",
"SpanName": "llm.provider_fallback",
"Route": "",
"Host": "",
"StatusCode": 0,
"ExceptionType": "*fmt.wrapError",
"ClientIP": "",
"UAClass": "",
"IsDuplicate": false,
"IsRootCause": false,
"HasUpstream": false,
"TraceServices": [
"eventsourced-orchestrator"
],
"BadActor": ""
}
}
]
},
"patterns": [
{
"category": "capacity_saturation",
"priority": "high",
"issueId": "869896e00332d212",
"services": [
"eventsourced-orchestrator"
],
"rule": "capacity-text",
"new": false,
"count": 1,
"baseline": 0,
"what": "llm.provider_fallback | *fmt.wrapError | *fmt.wraperror: read llm response: context deadline exceeded (client.timeout or context cancellation while reading body)"
}
]
}
},
{
"name": "crawler hits scaled-to-zero go-html-edit",
"expect": {
"action": "ignore",
"human": false,
"why": "503s only because the dev deployment is scaled to 0 and only AI crawlers asked"
},
"alert": {
"id": "tlxg6d-nginx",
"at": "2026-09-25T15:54:13.346717682Z",
"spike": {
"service": "nginx",
"windowStart": "2026-09-25T15:49:13.346717682Z",
"windowEnd": "2026-09-25T15:54:13.346717682Z",
"errorCount": 74,
"baselineMean": 18.448275862068964,
"excess": 12,
"patterns": [
{
"signature": "0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e",
"count": 4,
"baselineMean": 0,
"excess": 4,
"novel": false,
"clients": 2,
"services": [
"nginx"
],
"example": {
"Timestamp": "2026-09-25T15:53:46.266195024Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "ecdf390536970c3339612fb2a201df51",
"PatternSignature": "0cf49ad761cba3613a35803d84df5b05860c1df45163d5557cf70b8c6c3af21e",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/api/designer/v1/file-content",
"Host": "k6n.net",
"StatusCode": 503,
"ExceptionType": "",
"ClientIP": "104.23.160.37",
"UAClass": "ai_crawler",
"IsDuplicate": false,
"IsRootCause": true,
"HasUpstream": false,
"TraceServices": [
"nginx"
],
"BadActor": ""
}
},
{
"signature": "b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6",
"count": 4,
"baselineMean": 0,
"excess": 4,
"novel": false,
"clients": 2,
"services": [
"nginx"
],
"example": {
"Timestamp": "2026-09-25T15:53:46.266195024Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "744692761138c480f27fac58b3aa1fda",
"PatternSignature": "b83e0bcb8892e29d01e6c27e40e51b64a997799b885b09496d0a533b4fc9cea6",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/api/templates/preview",
"Host": "k6n.net",
"StatusCode": 503,
"ExceptionType": "",
"ClientIP": "104.23.160.37",
"UAClass": "ai_crawler",
"IsDuplicate": false,
"IsRootCause": true,
"HasUpstream": false,
"TraceServices": [
"nginx"
],
"BadActor": ""
}
},
{
"signature": "004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab",
"count": 2,
"baselineMean": 0,
"excess": 2,
"novel": false,
"clients": 2,
"services": [
"nginx"
],
"example": {
"Timestamp": "2026-09-25T15:53:46.266195024Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "7ffa5f428b4de59811dd86ded001a33c",
"PatternSignature": "004819b8dce17bf8dd20998fc2b65cbd117a694b0c83a695896a73215feda4ab",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/api/fs/exec",
"Host": "k6n.net",
"StatusCode": 503,
"ExceptionType": "",
"ClientIP": "104.23.160.37",
"UAClass": "ai_crawler",
"IsDuplicate": false,
"IsRootCause": true,
"HasUpstream": false,
"TraceServices": [
"nginx"
],
"BadActor": ""
}
},
{
"signature": "7281b16d4395d124b0fac7a82520ee2b9e179df689e8f4eb6d515a6b567aa256",
"count": 2,
"baselineMean": 0,
"excess": 2,
"novel": false,
"clients": 2,
"services": [
"nginx"
],
"example": {
"Timestamp": "2026-09-25T15:53:46.266195024Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "942867286e5ba5cac5ec2102cd059106",
"PatternSignature": "7281b16d4395d124b0fac7a82520ee2b9e179df689e8f4eb6d515a6b567aa256",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/api/v1/build_public_tmp/{id}/flow",
"Host": "k6n.net",
"StatusCode": 503,
"ExceptionType": "",
"ClientIP": "104.23.160.37",
"UAClass": "ai_crawler",
"IsDuplicate": false,
"IsRootCause": true,
"HasUpstream": false,
"TraceServices": [
"nginx"
],
"BadActor": ""
}
}
]
},
"patterns": [
{
"category": "dependency_unreachable",
"priority": "high",
"status": 503,
"host": "k6n.net",
"path": "/api/designer/v1/file-content",
"services": [
"nginx"
],
"routes": [
{
"ingress": "dev/static-knatofs",
"host": "k6n.net",
"path": "/api",
"pathType": "Prefix",
"namespace": "dev",
"service": "go-html-edit",
"port": "8080",
"deployments": [
"go-html-edit"
]
}
],
"pods": [
{
"namespace": "dev",
"deployment": "go-html-edit",
"found": false
}
],
"rule": "status-5xx-upstream",
"new": false,
"count": 4,
"baseline": 0,
"clients": 2,
"uaClass": "ai_crawler",
"what": "HTTP 503 | /api/designer/v1/file-content"
},
{
"category": "dependency_unreachable",
"priority": "high",
"status": 503,
"host": "k6n.net",
"path": "/api/templates/preview",
"services": [
"nginx"
],
"routes": [
{
"ingress": "dev/static-knatofs",
"host": "k6n.net",
"path": "/api",
"pathType": "Prefix",
"namespace": "dev",
"service": "go-html-edit",
"port": "8080",
"deployments": [
"go-html-edit"
]
}
],
"pods": [
{
"namespace": "dev",
"deployment": "go-html-edit",
"found": false
}
],
"rule": "status-5xx-upstream",
"new": false,
"count": 4,
"baseline": 0,
"clients": 2,
"uaClass": "ai_crawler",
"what": "HTTP 503 | /api/templates/preview"
},
{
"category": "dependency_unreachable",
"priority": "high",
"status": 503,
"host": "k6n.net",
"path": "/api/fs/exec",
"services": [
"nginx"
],
"routes": [
{
"ingress": "dev/static-knatofs",
"host": "k6n.net",
"path": "/api",
"pathType": "Prefix",
"namespace": "dev",
"service": "go-html-edit",
"port": "8080",
"deployments": [
"go-html-edit"
]
}
],
"pods": [
{
"namespace": "dev",
"deployment": "go-html-edit",
"found": false
}
],
"rule": "status-5xx-upstream",
"new": false,
"count": 2,
"baseline": 0,
"clients": 2,
"uaClass": "ai_crawler",
"what": "HTTP 503 | /api/fs/exec"
},
{
"category": "dependency_unreachable",
"priority": "high",
"status": 503,
"host": "k6n.net",
"path": "/api/v1/build_public_tmp/{id}/flow",
"services": [
"nginx"
],
"routes": [
{
"ingress": "dev/static-knatofs",
"host": "k6n.net",
"path": "/api",
"pathType": "Prefix",
"namespace": "dev",
"service": "go-html-edit",
"port": "8080",
"deployments": [
"go-html-edit"
]
}
],
"pods": [
{
"namespace": "dev",
"deployment": "go-html-edit",
"found": false
}
],
"rule": "status-5xx-upstream",
"new": false,
"count": 2,
"baseline": 0,
"clients": 2,
"uaClass": "ai_crawler",
"what": "HTTP 503 | /api/v1/build_public_tmp/{id}/flow"
}
]
}
},
{
"name": "scaled-to-zero game server",
"expect": {
"action": "ignore",
"human": false,
"why": "cs-rcon2000 intentionally has no pod; 2 requests"
},
"alert": {
"id": "tlxevp-nginx",
"at": "2026-09-25T15:26:13.346647513Z",
"spike": {
"service": "nginx",
"windowStart": "2026-09-25T15:21:13.346647513Z",
"windowEnd": "2026-09-25T15:26:13.346647513Z",
"errorCount": 44,
"baselineMean": 13.724137931034482,
"excess": 2,
"patterns": [
{
"signature": "3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d",
"count": 2,
"baselineMean": 0,
"excess": 2,
"novel": false,
"clients": 2,
"services": [
"nginx"
],
"example": {
"Timestamp": "2026-09-25T15:25:31.359710287Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "ea2a5c59210e1c1076b971328b6688c0",
"PatternSignature": "3c59646ff131a8ccb8f4d1931f3536112cac2eb59d5a5dbdcbe3f08873d23f0d",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/",
"Host": "rcon-cs.tornberg.me",
"StatusCode": 503,
"ExceptionType": "",
"ClientIP": "172.69.194.250",
"UAClass": "browser",
"IsDuplicate": false,
"IsRootCause": true,
"HasUpstream": false,
"TraceServices": [
"nginx"
],
"BadActor": ""
}
}
]
},
"patterns": [
{
"category": "dependency_unreachable",
"priority": "high",
"status": 503,
"host": "rcon-cs.tornberg.me",
"path": "/",
"services": [
"nginx"
],
"routes": [
{
"ingress": "game/cs-rcon2000",
"host": "rcon-cs.tornberg.me",
"path": "/",
"pathType": "Prefix",
"namespace": "game",
"service": "cs-rcon2000",
"port": "80",
"deployments": [
"cs-rcon2000"
]
}
],
"pods": [
{
"namespace": "game",
"deployment": "cs-rcon2000",
"found": false
}
],
"rule": "status-5xx-upstream",
"new": false,
"count": 2,
"baseline": 0,
"clients": 2,
"what": "HTTP 503 | /"
}
]
}
},
{
"name": "registry 401 on manifest pulls",
"expect": {
"action": "ignore",
"human": false,
"why": "docker clients probe anonymously first; 401 is the normal auth handshake"
},
"alert": {
"id": "tlxdwp-nginx",
"at": "2026-09-25T15:05:13.346979342Z",
"spike": {
"service": "nginx",
"windowStart": "2026-09-25T15:00:13.346979342Z",
"windowEnd": "2026-09-25T15:05:13.346979342Z",
"errorCount": 113,
"baselineMean": 8.89655172413793,
"excess": 5,
"patterns": [
{
"signature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d",
"count": 2,
"baselineMean": 0,
"excess": 2,
"novel": false,
"services": [
"docker-registry",
"nginx"
],
"example": {
"Timestamp": "2026-09-25T15:05:06.200527764Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "171ceb5b0a017957cc0510541f80fd07",
"PatternSignature": "507e896d155db5576b01b577127b54f75a8909089be9b04a82926c7b0deb5a0d",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/v2/eventsourced-ai-worker/manifests/latest",
"Host": "registry.k6n.net",
"StatusCode": 401,
"ExceptionType": "",
"ClientIP": "",
"UAClass": "unknown",
"IsDuplicate": false,
"IsRootCause": false,
"HasUpstream": true,
"TraceServices": [
"docker-registry",
"nginx"
],
"BadActor": ""
}
},
{
"signature": "645f263c238fbbac7ee018af53104cade809f58f461e3939c252e097381e265c",
"count": 2,
"baselineMean": 0,
"excess": 2,
"novel": false,
"services": [
"docker-registry",
"nginx"
],
"example": {
"Timestamp": "2026-09-25T15:05:06.200527764Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "f90437c17a24182429ec1dbb33fb1209",
"PatternSignature": "645f263c238fbbac7ee018af53104cade809f58f461e3939c252e097381e265c",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/v2/eventsourced-ai-worker/manifests/{id}",
"Host": "registry.k6n.net",
"StatusCode": 401,
"ExceptionType": "",
"ClientIP": "",
"UAClass": "unknown",
"IsDuplicate": false,
"IsRootCause": false,
"HasUpstream": true,
"TraceServices": [
"docker-registry",
"nginx"
],
"BadActor": ""
}
},
{
"signature": "38f693f9028998b8f34bc954af6be996e8b6add9ad53bbbae8c7f2ae5456586f",
"count": 1,
"baselineMean": 0,
"excess": 1,
"novel": false,
"services": [
"docker-registry",
"nginx"
],
"example": {
"Timestamp": "2026-09-25T15:05:01.199520113Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "b341496525b8a9346fd0ddc5fa26498c",
"PatternSignature": "38f693f9028998b8f34bc954af6be996e8b6add9ad53bbbae8c7f2ae5456586f",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/v2/eventsourced-ai-worker-browser/manifests/latest",
"Host": "registry.k6n.net",
"StatusCode": 401,
"ExceptionType": "",
"ClientIP": "",
"UAClass": "unknown",
"IsDuplicate": false,
"IsRootCause": false,
"HasUpstream": true,
"TraceServices": [
"docker-registry",
"nginx"
],
"BadActor": ""
}
}
]
},
"patterns": [
{
"category": "client_credential_issue",
"priority": "low",
"status": 401,
"host": "registry.k6n.net",
"path": "/v2/eventsourced-ai-worker/manifests/latest",
"services": [
"docker-registry",
"nginx"
],
"routes": [
{
"ingress": "home/registry",
"host": "registry.k6n.net",
"path": "/",
"pathType": "Prefix",
"namespace": "home",
"service": "registry",
"port": "5000",
"deployments": [
"registry"
]
}
],
"pods": [
{
"namespace": "home",
"deployment": "registry",
"pod": "registry-559bf85884-sxbrw",
"lines": [
"10.244.2.1 - - [25/Sep/2026:15:05:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"",
"time=\"2026-09-25T15:05:41.507526902Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"758d99c1d694e7816fbf337d231dc906\\\",\\\"SpanID\\\":\\\"7cb6d7f33c9cf9e7\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:05:38.934630694Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:38.934733409Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36062}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1",
"time=\"2026-09-25T15:05:41.507683183Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f4b89e61b17217cfa6bda198b2b91f46\\\",\\\"SpanID\\\":\\\"d95c0c7e1b5008ba\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:05:39.10176949Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:39.103919224Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"
],
"found": true
}
],
"rule": "status-401",
"new": false,
"count": 2,
"baseline": 0,
"what": "HTTP 401 | /v2/eventsourced-ai-worker/manifests/latest"
},
{
"category": "client_credential_issue",
"priority": "low",
"status": 401,
"host": "registry.k6n.net",
"path": "/v2/eventsourced-ai-worker/manifests/{id}",
"services": [
"docker-registry",
"nginx"
],
"routes": [
{
"ingress": "home/registry",
"host": "registry.k6n.net",
"path": "/",
"pathType": "Prefix",
"namespace": "home",
"service": "registry",
"port": "5000",
"deployments": [
"registry"
]
}
],
"pods": [
{
"namespace": "home",
"deployment": "registry",
"pod": "registry-559bf85884-sxbrw",
"lines": [
"10.244.2.1 - - [25/Sep/2026:15:05:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"",
"time=\"2026-09-25T15:05:41.507526902Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"758d99c1d694e7816fbf337d231dc906\\\",\\\"SpanID\\\":\\\"7cb6d7f33c9cf9e7\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:05:38.934630694Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:38.934733409Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36062}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1",
"time=\"2026-09-25T15:05:41.507683183Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f4b89e61b17217cfa6bda198b2b91f46\\\",\\\"SpanID\\\":\\\"d95c0c7e1b5008ba\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:05:39.10176949Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:39.103919224Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"
],
"found": true
}
],
"rule": "status-401",
"new": false,
"count": 2,
"baseline": 0,
"what": "HTTP 401 | /v2/eventsourced-ai-worker/manifests/{id}"
},
{
"category": "client_credential_issue",
"priority": "low",
"status": 401,
"host": "registry.k6n.net",
"path": "/v2/eventsourced-ai-worker-browser/manifests/latest",
"services": [
"docker-registry",
"nginx"
],
"routes": [
{
"ingress": "home/registry",
"host": "registry.k6n.net",
"path": "/",
"pathType": "Prefix",
"namespace": "home",
"service": "registry",
"port": "5000",
"deployments": [
"registry"
]
}
],
"pods": [
{
"namespace": "home",
"deployment": "registry",
"pod": "registry-559bf85884-sxbrw",
"lines": [
"10.244.2.1 - - [25/Sep/2026:15:05:38 +0000] \"GET / HTTP/2.0\" 200 0 \"\" \"kube-probe/1.36\"",
"time=\"2026-09-25T15:05:41.507526902Z\" level=debug msg=\"{\\\"Name\\\":\\\"GET /\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"758d99c1d694e7816fbf337d231dc906\\\",\\\"SpanID\\\":\\\"7cb6d7f33c9cf9e7\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":2,\\\"StartTime\\\":\\\"2026-09-25T15:05:38.934630694Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:38.934733409Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"server.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.194\\\"}},{\\\"Key\\\":\\\"http.request.method\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"GET\\\"}},{\\\"Key\\\":\\\"url.scheme\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"https\\\"}},{\\\"Key\\\":\\\"server.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":5000}},{\\\"Key\\\":\\\"network.peer.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"network.peer.port\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":36062}},{\\\"Key\\\":\\\"user_agent.original\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"kube-probe/1.36\\\"}},{\\\"Key\\\":\\\"client.address\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"10.244.2.1\\\"}},{\\\"Key\\\":\\\"url.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}},{\\\"Key\\\":\\\"network.protocol.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"2.0\\\"}},{\\\"Key\\\":\\\"http.response.status_code\\\",\\\"Value\\\":{\\\"Type\\\":\\\"INT64\\\",\\\"Value\\\":200}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp\\\",\\\"Version\\\":\\\"0.67.0\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1",
"time=\"2026-09-25T15:05:41.507683183Z\" level=debug msg=\"{\\\"Name\\\":\\\"Stat\\\",\\\"SpanContext\\\":{\\\"TraceID\\\":\\\"f4b89e61b17217cfa6bda198b2b91f46\\\",\\\"SpanID\\\":\\\"d95c0c7e1b5008ba\\\",\\\"TraceFlags\\\":\\\"01\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"Parent\\\":{\\\"TraceID\\\":\\\"00000000000000000000000000000000\\\",\\\"SpanID\\\":\\\"0000000000000000\\\",\\\"TraceFlags\\\":\\\"00\\\",\\\"TraceState\\\":\\\"\\\",\\\"Remote\\\":false},\\\"SpanKind\\\":1,\\\"StartTime\\\":\\\"2026-09-25T15:05:39.10176949Z\\\",\\\"EndTime\\\":\\\"2026-09-25T15:05:39.103919224Z\\\",\\\"Attributes\\\":[{\\\"Key\\\":\\\"io.cncf.distribution.storage.driver.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"s3aws\\\"}},{\\\"Key\\\":\\\"io.cncf.distribution.storage.path\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"/\\\"}}],\\\"Events\\\":null,\\\"Links\\\":null,\\\"Status\\\":{\\\"Code\\\":\\\"Unset\\\",\\\"Description\\\":\\\"\\\"},\\\"DroppedAttributes\\\":0,\\\"DroppedEvents\\\":0,\\\"DroppedLinks\\\":0,\\\"ChildSpanCount\\\":0,\\\"Resource\\\":[{\\\"Key\\\":\\\"service.name\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"docker-registry\\\"}},{\\\"Key\\\":\\\"service.version\\\",\\\"Value\\\":{\\\"Type\\\":\\\"STRING\\\",\\\"Value\\\":\\\"3.1.1\\\"}}],\\\"InstrumentationScope\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null},\\\"InstrumentationLibrary\\\":{\\\"Name\\\":\\\"github.com/distribution/distribution/v3/registry/storage/driver/base\\\",\\\"Version\\\":\\\"\\\",\\\"SchemaURL\\\":\\\"\\\",\\\"Attributes\\\":null}}\\n\" environment=development go.version=go1.25.9 instance.id=01a0d3ba-0689-7ee7-945f-0e9c97950869 service=registry version=3.1.1"
],
"found": true
}
],
"rule": "status-401",
"new": false,
"count": 1,
"baseline": 0,
"what": "HTTP 401 | /v2/eventsourced-ai-worker-browser/manifests/latest"
}
]
}
},
{
"name": "transmission 409 session handshake",
"expect": {
"action": "ignore",
"human": false,
"why": "Transmission RPC answers 409 to hand out its session id; expected protocol"
},
"alert": {
"id": "tlxc21-nginx",
"at": "2026-09-25T14:25:13.346654632Z",
"spike": {
"service": "nginx",
"windowStart": "2026-09-25T14:20:13.346654632Z",
"windowEnd": "2026-09-25T14:25:13.346654632Z",
"errorCount": 31,
"baselineMean": 4.137931034482759,
"excess": 4,
"patterns": [
{
"signature": "efb1d02de6ef2fbc3e1f234fa547da4e31652733a4c6f8aababe8da1900871f9",
"count": 4,
"baselineMean": 0,
"excess": 4,
"novel": true,
"clients": 1,
"services": [
"nginx"
],
"example": {
"Timestamp": "2026-09-25T14:24:15.768783875Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "fcd10da95cccfebe9e02e43df79d7570",
"PatternSignature": "efb1d02de6ef2fbc3e1f234fa547da4e31652733a4c6f8aababe8da1900871f9",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/transmission/rpc",
"Host": "torrent.k6n.net",
"StatusCode": 409,
"ExceptionType": "",
"ClientIP": "209.206.25.124",
"UAClass": "browser",
"IsDuplicate": false,
"IsRootCause": true,
"HasUpstream": false,
"TraceServices": [
"nginx"
],
"BadActor": ""
}
}
]
},
"patterns": [
{
"category": "needs_review",
"priority": "low",
"status": 409,
"host": "torrent.k6n.net",
"path": "/transmission/rpc",
"services": [
"nginx"
],
"routes": [
{
"ingress": "home/torrent-ingress",
"host": "torrent.k6n.net",
"path": "/",
"pathType": "Prefix",
"namespace": "home",
"service": "torrent-service",
"port": "9091"
}
],
"rule": "no-rule",
"new": true,
"count": 4,
"baseline": 0,
"clients": 1,
"what": "HTTP 409 | /transmission/rpc"
}
]
}
},
{
"name": "405 on /",
"expect": {
"action": "ignore",
"human": false,
"why": "a client using the wrong method on the root; 4 requests from 1 client"
},
"alert": {
"id": "tlxn9d-nginx",
"at": "2026-09-25T18:27:13.347226389Z",
"spike": {
"service": "nginx",
"windowStart": "2026-09-25T18:22:13.347226389Z",
"windowEnd": "2026-09-25T18:27:13.347226389Z",
"errorCount": 64,
"baselineMean": 13.931034482758621,
"excess": 4,
"patterns": [
{
"signature": "ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f",
"count": 4,
"baselineMean": 0,
"excess": 4,
"novel": false,
"clients": 1,
"services": [
"nginx"
],
"example": {
"Timestamp": "2026-09-25T18:26:26.152736211Z",
"Service": "nginx",
"Release": "",
"Kind": "span",
"Source": "backend",
"IsError": true,
"StatusClass": "",
"DurationMS": 0,
"TraceID": "d1de7de09ad342e7b0670cb78cac1430",
"PatternSignature": "ce6a670c8bf24372ec95cd45548edf7c8e219f3b8dcdd37852800956555ac46f",
"IssueFingerprint": "",
"IsRetry": false,
"IsTimeout": false,
"IsCancellation": false,
"Message": "",
"SpanName": "",
"Route": "/",
"Host": "cc.k6n.net",
"StatusCode": 405,
"ExceptionType": "",
"ClientIP": "35.189.233.235",
"UAClass": "browser",
"IsDuplicate": false,
"IsRootCause": true,
"HasUpstream": false,
"TraceServices": [
"nginx"
],
"BadActor": ""
}
}
]
},
"patterns": [
{
"category": "needs_review",
"priority": "low",
"status": 405,
"host": "cc.k6n.net",
"path": "/",
"services": [
"nginx"
],
"rule": "no-rule",
"new": false,
"count": 4,
"baseline": 0,
"clients": 1,
"what": "HTTP 405 | /"
}
]
}
}
]
Reference in new issue View Git Blame Copy Permalink