mats/go-otel
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove Needle; stop paging on nginx 4xx

Browse Source 
Needle scored ~chance on the golden set and never gated alerts, so drop the
sidecar, proxy handlers, UI buttons, k8s container and build step. The labelled
eval sets move to cmd/triage-eval/data.

nginx 4xx (registry 401 challenges, /oauth2/auth, scanner 405s) is now dropped
in FilterActionable instead of only being discounted, so it cannot ride along
with a real 5xx. nginx 5xx still alerts.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
matsandClaude Sonnet 5.5 committed 2026-11-01 11:40:53 +01:00
1 parent e046bd86d1
commit 51a8412ae1
22 files changed
+29 -1162
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

cmd/triage-eval/data/golden.jsonl
+40
View File
@@ -0,0 +1,40 @@
{"id": "ng-404-root", "src": "clickhouse", "expected": "stale_reference", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /"}
{"id": "ng-401-repos", "src": "clickhouse", "expected": "client_credential_issue", "note": "polled ~700x/h by an unauthenticated client", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/repos/available"}
{"id": "ng-401-ctrl", "src": "clickhouse", "expected": "client_credential_issue", "note": "", "text": "Service: nginx | Kind: span | Status: 401 | Route: /api/controller/instances"}
{"id": "orch-401-repos", "src": "clickhouse", "expected": "client_credential_issue", "note": "same request as ng-401-repos, second span", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/repos/available"}
{"id": "orch-401-chat", "src": "clickhouse", "expected": "client_credential_issue", "note": "", "text": "Service: eventsourced-orchestrator | Kind: span | Title: GET / | Message: get / | Status: 401 | Route: /api/sessions/{id}/collab/chat"}
{"id": "ng-401-manifest", "src": "clickhouse", "expected": "client_credential_issue", "note": "registry pull without token", "text": "Service: nginx | Kind: span | Status: 401 | Route: /v2/eventsourced-ai-worker/manifests/latest"}
{"id": "ng-503-store", "src": "clickhouse", "expected": "dependency_unreachable", "note": "nginx 503 burst: upstream down is the likeliest cause", "text": "Service: nginx | Kind: span | Status: 503 | Route: /store/regions"}
{"id": "ng-500-root", "src": "clickhouse", "expected": "service_defect", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /"}
{"id": "ng-500-wp", "src": "clickhouse", "expected": "scanner_probe", "note": "WordPress path that does not exist here", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp"}
{"id": "ng-500-blog", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /blog"}
{"id": "ng-500-wpjson", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wp-json/batch/v1"}
{"id": "ng-500-wordpress", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 500 | Route: /wordpress"}
{"id": "ng-404-env", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env"}
{"id": "ng-404-envlocal", "src": "clickhouse", "expected": "scanner_probe", "note": "", "text": "Service: nginx | Kind: span | Status: 404 | Route: /.env.local"}
{"id": "ng-404-userfiles", "src": "clickhouse", "expected": "scanner_probe", "note": "ambiguous: probe or stale link", "text": "Service: nginx | Kind: span | Status: 404 | Route: /userfiles"}
{"id": "ng-404-commit", "src": "clickhouse", "expected": "stale_reference", "note": "git UI link to missing commit", "text": "Service: nginx | Kind: span | Status: 404 | Route: /mats/go-cart-actor/commit/{id}"}
{"id": "ng-404-vite", "src": "clickhouse", "expected": "stale_reference", "note": "dev tooling request", "text": "Service: nginx | Kind: span | Status: 404 | Route: /__vite_rsc_findSourceMapURL"}
{"id": "ng-404-blob", "src": "clickhouse", "expected": "stale_reference", "note": "registry blob existence check", "text": "Service: nginx | Kind: span | Status: 404 | Route: /v2/eventsourced-ai/blobs/{id}"}
{"id": "ng-499-repos", "src": "clickhouse", "expected": "expected_lifecycle", "note": "client closed request", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/repos/available"}
{"id": "ng-499-ctrl", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: nginx | Kind: span | Status: 499 | Route: /api/controller/instances"}
{"id": "ng-308-blog", "src": "clickhouse", "expected": "expected_lifecycle", "note": "redirect", "text": "Service: nginx | Kind: span | Status: 308 | Route: /blog/"}
{"id": "ng-200-speed", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/network-speed"}
{"id": "ng-200-matrix", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /_matrix/federation/v1/send/{id}"}
{"id": "ng-200-overview", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: nginx | Kind: span | Status: 200 | Route: /api/overview"}
{"id": "sched-readyz", "src": "clickhouse", "expected": "expected_lifecycle", "note": "health check", "text": "Service: scheduling | Kind: span | Title: GET /readyz | Message: get /readyz | Status: 200 | Route: /readyz"}
{"id": "cart-ping", "src": "clickhouse", "expected": "expected_lifecycle", "note": "gRPC keepalive", "text": "Service: cart | Kind: span | Title: /control_plane_messages.ControlPlane/Ping | Message: /control_plane_messages.controlplane/ping"}
{"id": "rag-request", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: rust-rag | Kind: span | Title: request | Message: request"}
{"id": "orch-session-load", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: eventsourced-orchestrator | Kind: log | Message: session snapshot/replay loaded"}
{"id": "worker-dialerr", "src": "clickhouse", "expected": "dependency_unreachable", "note": "", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *websocket.dialerror: websocket.dial ws://eventsourced-orchestrator.dev.svc.cluster.local:<n>/...: connection refused"}
{"id": "worker-eof", "src": "clickhouse", "expected": "dependency_unreachable", "note": "", "text": "Service: eventsourced-worker-code | Kind: span | Title: worker.terminal.connection | Message: *errors.errorstring: eof"}
{"id": "ledger-skip", "src": "clickhouse", "expected": "expected_lifecycle", "note": "", "text": "Service: ledger | Kind: log | Message: offsite replicate: content unchanged since last push - skipping"}
{"id": "web-cls", "src": "clickhouse", "expected": "expected_lifecycle", "note": "real frontend log; is this a vital or a console line?", "text": "Service: eventsourced-ai-web | Kind: log | Message: cls <num>"}
{"id": "syn-js-1", "src": "synthetic", "expected": "service_defect", "note": "", "text": "Service: web-frontend | Kind: log | Title: TypeError: Cannot read properties of undefined | Message: Uncaught TypeError: Cannot read properties of undefined (reading 'map') | Route: /checkout"}
{"id": "syn-js-2", "src": "synthetic", "expected": "service_defect", "note": "", "text": "Service: shop-ui | Kind: log | Title: ChunkLoadError | Message: Loading chunk 42 failed | Route: /products/<id>"}
{"id": "syn-vital-1", "src": "synthetic", "expected": "expected_lifecycle", "note": "", "text": "Service: web-frontend | Kind: metric | Title: web_vital TTFB | Message: TTFB 812ms"}
{"id": "syn-db-1", "src": "synthetic", "expected": "capacity_saturation", "note": "", "text": "Service: checkout | Kind: log | Title: database timeout after <n>ms | Message: context deadline exceeded querying orders"}
{"id": "syn-db-2", "src": "synthetic", "expected": "service_defect", "note": "", "text": "Service: inventory | Kind: log | Title: deadlock detected | Message: pq: deadlock detected while updating stock"}
{"id": "syn-db-3", "src": "synthetic", "expected": "capacity_saturation", "note": "", "text": "Service: orders | Kind: log | Title: connection pool exhausted | Message: too many connections in pool"}
{"id": "syn-oom", "src": "synthetic", "expected": "capacity_saturation", "note": "", "text": "Service: kubelet-proxy | Kind: log | Title: OOMKilled | Message: container exceeded memory limit and was restarted"}
{"id": "syn-retry", "src": "synthetic", "expected": "expected_lifecycle", "note": "", "text": "Service: worker | Kind: span | Title: retry attempt 2 | Message: retrying after transient error"}
cmd/triage-eval/data/holdout.jsonl
+60
View File
@@ -0,0 +1,60 @@
{"service":"nginx","kind":"span","status_code":404,"route":"/ses.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":404,"route":"/api/v1/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":404,"route":"/api/w/admins/jobs_u/get_log_file/../../../../proc/self/environ","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"eventsourced-ai-web","kind":"log","status_code":0,"route":"","title":"","message":"cls <num>","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":16,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"lifecycle-text"}
{"service":"nginx","kind":"span","status_code":500,"route":"/wordpress","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":30,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":404,"route":"/private/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":404,"route":"/settings.js","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":200,"route":"/","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":17,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"benign-status"}
{"service":"eventsourced-orchestrator","kind":"span","status_code":200,"route":"/api/memories","title":"GET /api/memories","message":"get /api/memories","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":5,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"benign-status"}
{"service":"cart","kind":"log","status_code":0,"route":"","title":"","message":"handleownershipchange from <num>.<num>.<num>.<num>: accepted_remote=<num> kept_local=<num> legacy_no_stamp=<num>","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"no-status-signal"}
{"service":"eventsourced-orchestrator","kind":"span","status_code":0,"route":"","title":"streamproc.replay","message":"streamproc.replay","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"lifecycle-text"}
{"service":"nginx","kind":"span","status_code":404,"route":"/.htpasswd","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":401,"route":"/v2/eventsourced-ai-worker/manifests/6d2e5b3-sandbox","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":16,"expected":"client_credential_issue","predicted":"client_credential_issue","rule":"status-401"}
{"service":"nginx","kind":"span","status_code":404,"route":"/api/config","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"}
{"service":"nginx","kind":"span","status_code":404,"route":"/ssl/server.key","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"}
{"service":"nginx","kind":"span","status_code":404,"route":"/_debugbar/open","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"}
{"service":"nginx","kind":"span","status_code":500,"route":"/register","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"service_defect","predicted":"service_defect","rule":"status-5xx"}
{"service":"nginx","kind":"span","status_code":500,"route":"/wp/wordpress/wp-json/batch/v1","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":404,"route":"/old/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":404,"route":"/mcp","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"}
{"service":"nginx","kind":"span","status_code":401,"route":"/mcp","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"client_credential_issue","predicted":"client_credential_issue","rule":"status-401"}
{"service":"nginx","kind":"span","status_code":404,"route":"/graphql/console","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"}
{"service":"nginx","kind":"span","status_code":404,"route":"/debug/pprof","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"}
{"service":"nginx","kind":"span","status_code":404,"route":"/panel/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":200,"route":"/api/actions/runner.v1.RunnerService/FetchTask","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":729,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"benign-status"}
{"service":"nginx","kind":"span","status_code":404,"route":"/server-info","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"}
{"service":"eventsourced-orchestrator","kind":"span","status_code":401,"route":"/api/overview","title":"GET /","message":"get /","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"client_credential_issue","predicted":"client_credential_issue","rule":"status-401"}
{"service":"nginx","kind":"span","status_code":101,"route":"/api/sessions/{id}/collab/chat-prompt/ws","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"expected_lifecycle","predicted":"needs_review","rule":"no-rule"}
{"service":"nginx","kind":"span","status_code":401,"route":"/v2/eventsourced-ai/blobs/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"client_credential_issue","predicted":"client_credential_issue","rule":"status-401"}
{"service":"nginx","kind":"span","status_code":404,"route":"/v2/eventsourced-ai/blobs/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":17,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"}
{"service":"nginx","kind":"span","status_code":404,"route":"/api/v2/settings","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"}
{"service":"nginx","kind":"span","status_code":404,"route":"/.well-known/jwks.json","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"}
{"service":"eventsourced-ai-web","kind":"log","status_code":0,"route":"","title":"","message":"ttfb <num>","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":5,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"lifecycle-text"}
{"service":"nginx","kind":"span","status_code":200,"route":"/api/stream","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":9,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"benign-status"}
{"service":"eventsourced-orchestrator","kind":"span","status_code":200,"route":"/api/sessions/{id}/replay","title":"GET /api/sessions/{id}/replay","message":"get /api/sessions/{id}/replay","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"lifecycle-text"}
{"service":"nginx","kind":"span","status_code":201,"route":"/v2/ecommerce/auth/blobs/uploads/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"expected_lifecycle","predicted":"needs_review","rule":"no-rule"}
{"service":"nginx","kind":"span","status_code":404,"route":"/cms/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":404,"route":"/__env.js","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"}
{"service":"nginx","kind":"span","status_code":504,"route":"/api/stream","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"dependency_unreachable","predicted":"dependency_unreachable","rule":"status-5xx-upstream"}
{"service":"nginx","kind":"span","status_code":404,"route":"/api/v2/config","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"}
{"service":"nginx","kind":"span","status_code":404,"route":"/actuator/threaddump","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"}
{"service":"nginx","kind":"span","status_code":404,"route":"/scripts/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":404,"route":"/.streamlit/secrets.toml","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":404,"route":"/web/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"cart","kind":"log","status_code":0,"route":"","title":"","message":"<num>.<num>.<num>.<num> taking ownership of <num> ids","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"no-status-signal"}
{"service":"nginx","kind":"span","status_code":200,"route":"/api/schemas/harness_evidence","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"benign-status"}
{"service":"nginx","kind":"span","status_code":404,"route":"/agent/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":404,"route":"/settings.py","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"}
{"service":"nginx","kind":"span","status_code":404,"route":"/conf/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":404,"route":"/dev/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"eventsourced-orchestrator","kind":"span","status_code":302,"route":"/auth/login","title":"GET /auth/login","message":"get /auth/login","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"benign-status"}
{"service":"nginx","kind":"span","status_code":404,"route":"/favicon.ico","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"}
{"service":"nginx","kind":"span","status_code":200,"route":"/api/dev-machines","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":120,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"benign-status"}
{"service":"nginx","kind":"span","status_code":308,"route":"/wp-json/batch/v1/","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":404,"route":"/.bash_profile","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"}
{"service":"nginx","kind":"span","status_code":404,"route":"/@fs/home/ubuntu/.aws/credentials","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":404,"route":"/serviceAccountKey.json","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"}
{"service":"nginx","kind":"span","status_code":404,"route":"/graphql","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"}
{"service":"nginx","kind":"span","status_code":404,"route":"/api","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"}
{"service":"nginx","kind":"span","status_code":200,"route":"/v2/ecommerce/auth/manifests/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"benign-status"}