diff --git a/.gitignore b/.gitignore index f13d797..949fc48 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ otel-agent.log # Local credentials and scratch .env + +# MaxMind GeoLite2 databases (large, licensed; baked into the image at build time) +/db/*.mmdb diff --git a/Dockerfile b/Dockerfile index 6e9cddc..ad2b5fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,6 +46,7 @@ WORKDIR /home/appuser/ COPY --from=go-builder /go-otel . COPY --from=go-builder /app/otel-ui/dist ./dist COPY static/ ./static +COPY db/GeoLite2-City.mmdb db/GeoLite2-ASN.mmdb ./db/ COPY agents_example.md . COPY instrumentation.md . @@ -58,6 +59,9 @@ RUN chown -R appuser:appgroup /home/appuser/ USER appuser +ENV OTEL_GEOIP_CITY_DB=/home/appuser/db/GeoLite2-City.mmdb \ + OTEL_GEOIP_ASN_DB=/home/appuser/db/GeoLite2-ASN.mmdb + EXPOSE 8080 CMD ["./go-otel"] diff --git a/cmd/needle-sidecar/eval/holdout.jsonl b/cmd/needle-sidecar/eval/holdout.jsonl index 9ae830f..e4f410a 100644 --- a/cmd/needle-sidecar/eval/holdout.jsonl +++ b/cmd/needle-sidecar/eval/holdout.jsonl @@ -1,60 +1,60 @@ -{"service":"nginx","kind":"span","status_code":404,"route":"/ses.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":404,"route":"/api/v1/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":404,"route":"/api/w/admins/jobs_u/get_log_file/../../../../proc/self/environ","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"eventsourced-ai-web","kind":"log","status_code":0,"route":"","title":"","message":"cls \u003cnum\u003e","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":16,"expected":"","predicted":"expected_lifecycle","rule":"lifecycle-text"} -{"service":"nginx","kind":"span","status_code":500,"route":"/wordpress","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":30,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":404,"route":"/private/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":404,"route":"/settings.js","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":200,"route":"/","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":17,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"} -{"service":"eventsourced-orchestrator","kind":"span","status_code":200,"route":"/api/memories","title":"GET /api/memories","message":"get /api/memories","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":5,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"} -{"service":"cart","kind":"log","status_code":0,"route":"","title":"","message":"handleownershipchange from \u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e: accepted_remote=\u003cnum\u003e kept_local=\u003cnum\u003e legacy_no_stamp=\u003cnum\u003e","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"expected_lifecycle","rule":"no-status-signal"} -{"service":"eventsourced-orchestrator","kind":"span","status_code":0,"route":"","title":"streamproc.replay","message":"streamproc.replay","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"","predicted":"expected_lifecycle","rule":"lifecycle-text"} -{"service":"nginx","kind":"span","status_code":404,"route":"/.htpasswd","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":401,"route":"/v2/eventsourced-ai-worker/manifests/6d2e5b3-sandbox","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":16,"expected":"","predicted":"client_credential_issue","rule":"status-401"} -{"service":"nginx","kind":"span","status_code":404,"route":"/api/config","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"nginx","kind":"span","status_code":404,"route":"/ssl/server.key","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"nginx","kind":"span","status_code":404,"route":"/_debugbar/open","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"nginx","kind":"span","status_code":500,"route":"/register","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"","predicted":"service_defect","rule":"status-5xx"} -{"service":"nginx","kind":"span","status_code":500,"route":"/wp/wordpress/wp-json/batch/v1","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":404,"route":"/old/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":404,"route":"/mcp","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"nginx","kind":"span","status_code":401,"route":"/mcp","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"client_credential_issue","rule":"status-401"} -{"service":"nginx","kind":"span","status_code":404,"route":"/graphql/console","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"nginx","kind":"span","status_code":404,"route":"/debug/pprof","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"nginx","kind":"span","status_code":404,"route":"/panel/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":200,"route":"/api/actions/runner.v1.RunnerService/FetchTask","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":729,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"} -{"service":"nginx","kind":"span","status_code":404,"route":"/server-info","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"eventsourced-orchestrator","kind":"span","status_code":401,"route":"/api/overview","title":"GET /","message":"get /","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"","predicted":"client_credential_issue","rule":"status-401"} -{"service":"nginx","kind":"span","status_code":101,"route":"/api/sessions/{id}/collab/chat-prompt/ws","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"needs_review","rule":"no-rule"} -{"service":"nginx","kind":"span","status_code":401,"route":"/v2/eventsourced-ai/blobs/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"","predicted":"client_credential_issue","rule":"status-401"} -{"service":"nginx","kind":"span","status_code":404,"route":"/v2/eventsourced-ai/blobs/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":17,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"nginx","kind":"span","status_code":404,"route":"/api/v2/settings","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"nginx","kind":"span","status_code":404,"route":"/.well-known/jwks.json","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"eventsourced-ai-web","kind":"log","status_code":0,"route":"","title":"","message":"ttfb \u003cnum\u003e","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":5,"expected":"","predicted":"expected_lifecycle","rule":"lifecycle-text"} -{"service":"nginx","kind":"span","status_code":200,"route":"/api/stream","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":9,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"} -{"service":"eventsourced-orchestrator","kind":"span","status_code":200,"route":"/api/sessions/{id}/replay","title":"GET /api/sessions/{id}/replay","message":"get /api/sessions/{id}/replay","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"","predicted":"expected_lifecycle","rule":"lifecycle-text"} -{"service":"nginx","kind":"span","status_code":201,"route":"/v2/ecommerce/auth/blobs/uploads/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"needs_review","rule":"no-rule"} -{"service":"nginx","kind":"span","status_code":404,"route":"/cms/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":404,"route":"/__env.js","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"nginx","kind":"span","status_code":504,"route":"/api/stream","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"","predicted":"dependency_unreachable","rule":"status-5xx-upstream"} -{"service":"nginx","kind":"span","status_code":404,"route":"/api/v2/config","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"nginx","kind":"span","status_code":404,"route":"/actuator/threaddump","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"nginx","kind":"span","status_code":404,"route":"/scripts/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":404,"route":"/.streamlit/secrets.toml","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":404,"route":"/web/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"cart","kind":"log","status_code":0,"route":"","title":"","message":"\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e.\u003cnum\u003e taking ownership of \u003cnum\u003e ids","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"expected_lifecycle","rule":"no-status-signal"} -{"service":"nginx","kind":"span","status_code":200,"route":"/api/schemas/harness_evidence","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"} -{"service":"nginx","kind":"span","status_code":404,"route":"/agent/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":404,"route":"/settings.py","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"nginx","kind":"span","status_code":404,"route":"/conf/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":404,"route":"/dev/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"eventsourced-orchestrator","kind":"span","status_code":302,"route":"/auth/login","title":"GET /auth/login","message":"get /auth/login","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"} -{"service":"nginx","kind":"span","status_code":404,"route":"/favicon.ico","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"nginx","kind":"span","status_code":200,"route":"/api/dev-machines","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":120,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"} -{"service":"nginx","kind":"span","status_code":308,"route":"/wp-json/batch/v1/","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":404,"route":"/.bash_profile","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"nginx","kind":"span","status_code":404,"route":"/@fs/home/ubuntu/.aws/credentials","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":404,"route":"/serviceAccountKey.json","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"scanner_probe","rule":"probe-route"} -{"service":"nginx","kind":"span","status_code":404,"route":"/graphql","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"nginx","kind":"span","status_code":404,"route":"/api","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"stale_reference","rule":"status-404"} -{"service":"nginx","kind":"span","status_code":200,"route":"/v2/ecommerce/auth/manifests/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"","predicted":"expected_lifecycle","rule":"benign-status"} +{"service":"nginx","kind":"span","status_code":404,"route":"/ses.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/api/v1/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/api/w/admins/jobs_u/get_log_file/../../../../proc/self/environ","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"eventsourced-ai-web","kind":"log","status_code":0,"route":"","title":"","message":"cls ","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":16,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"lifecycle-text"} +{"service":"nginx","kind":"span","status_code":500,"route":"/wordpress","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":30,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/private/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/settings.js","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":200,"route":"/","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":17,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"benign-status"} +{"service":"eventsourced-orchestrator","kind":"span","status_code":200,"route":"/api/memories","title":"GET /api/memories","message":"get /api/memories","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":5,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"benign-status"} +{"service":"cart","kind":"log","status_code":0,"route":"","title":"","message":"handleownershipchange from ...: accepted_remote= kept_local= legacy_no_stamp=","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"no-status-signal"} +{"service":"eventsourced-orchestrator","kind":"span","status_code":0,"route":"","title":"streamproc.replay","message":"streamproc.replay","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"lifecycle-text"} +{"service":"nginx","kind":"span","status_code":404,"route":"/.htpasswd","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":401,"route":"/v2/eventsourced-ai-worker/manifests/6d2e5b3-sandbox","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":16,"expected":"client_credential_issue","predicted":"client_credential_issue","rule":"status-401"} +{"service":"nginx","kind":"span","status_code":404,"route":"/api/config","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/ssl/server.key","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/_debugbar/open","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":500,"route":"/register","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"service_defect","predicted":"service_defect","rule":"status-5xx"} +{"service":"nginx","kind":"span","status_code":500,"route":"/wp/wordpress/wp-json/batch/v1","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/old/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/mcp","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":401,"route":"/mcp","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"client_credential_issue","predicted":"client_credential_issue","rule":"status-401"} +{"service":"nginx","kind":"span","status_code":404,"route":"/graphql/console","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/debug/pprof","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/panel/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":200,"route":"/api/actions/runner.v1.RunnerService/FetchTask","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":729,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"benign-status"} +{"service":"nginx","kind":"span","status_code":404,"route":"/server-info","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"} +{"service":"eventsourced-orchestrator","kind":"span","status_code":401,"route":"/api/overview","title":"GET /","message":"get /","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"client_credential_issue","predicted":"client_credential_issue","rule":"status-401"} +{"service":"nginx","kind":"span","status_code":101,"route":"/api/sessions/{id}/collab/chat-prompt/ws","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"expected_lifecycle","predicted":"needs_review","rule":"no-rule"} +{"service":"nginx","kind":"span","status_code":401,"route":"/v2/eventsourced-ai/blobs/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":12,"expected":"client_credential_issue","predicted":"client_credential_issue","rule":"status-401"} +{"service":"nginx","kind":"span","status_code":404,"route":"/v2/eventsourced-ai/blobs/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":17,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/api/v2/settings","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/.well-known/jwks.json","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"} +{"service":"eventsourced-ai-web","kind":"log","status_code":0,"route":"","title":"","message":"ttfb ","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":5,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"lifecycle-text"} +{"service":"nginx","kind":"span","status_code":200,"route":"/api/stream","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":9,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"benign-status"} +{"service":"eventsourced-orchestrator","kind":"span","status_code":200,"route":"/api/sessions/{id}/replay","title":"GET /api/sessions/{id}/replay","message":"get /api/sessions/{id}/replay","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"lifecycle-text"} +{"service":"nginx","kind":"span","status_code":201,"route":"/v2/ecommerce/auth/blobs/uploads/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"expected_lifecycle","predicted":"needs_review","rule":"no-rule"} +{"service":"nginx","kind":"span","status_code":404,"route":"/cms/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/__env.js","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":504,"route":"/api/stream","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"dependency_unreachable","predicted":"dependency_unreachable","rule":"status-5xx-upstream"} +{"service":"nginx","kind":"span","status_code":404,"route":"/api/v2/config","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/actuator/threaddump","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/scripts/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/.streamlit/secrets.toml","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/web/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"cart","kind":"log","status_code":0,"route":"","title":"","message":"... taking ownership of ids","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"no-status-signal"} +{"service":"nginx","kind":"span","status_code":200,"route":"/api/schemas/harness_evidence","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":3,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"benign-status"} +{"service":"nginx","kind":"span","status_code":404,"route":"/agent/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/settings.py","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/conf/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/dev/.env","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":6,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"eventsourced-orchestrator","kind":"span","status_code":302,"route":"/auth/login","title":"GET /auth/login","message":"get /auth/login","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"benign-status"} +{"service":"nginx","kind":"span","status_code":404,"route":"/favicon.ico","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":200,"route":"/api/dev-machines","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":120,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"benign-status"} +{"service":"nginx","kind":"span","status_code":308,"route":"/wp-json/batch/v1/","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":7,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/.bash_profile","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/@fs/home/ubuntu/.aws/credentials","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/serviceAccountKey.json","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"scanner_probe","predicted":"scanner_probe","rule":"probe-route"} +{"service":"nginx","kind":"span","status_code":404,"route":"/graphql","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":8,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":404,"route":"/api","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"stale_reference","predicted":"stale_reference","rule":"status-404"} +{"service":"nginx","kind":"span","status_code":200,"route":"/v2/ecommerce/auth/manifests/{id}","title":"","message":"","is_timeout":0,"is_cancellation":0,"is_retry":0,"n":4,"expected":"expected_lifecycle","predicted":"expected_lifecycle","rule":"benign-status"} diff --git a/cmd/server/main.go b/cmd/server/main.go index ba06d30..1e812d9 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -11,6 +11,7 @@ import ( "git.tornberg.me/go-otel/internal/auth" "git.tornberg.me/go-otel/internal/collector" + "git.tornberg.me/go-otel/internal/geoip" "git.tornberg.me/go-otel/internal/blocker" "git.tornberg.me/go-otel/internal/honeypot" kube "git.tornberg.me/go-otel/internal/kubernetes" @@ -244,6 +245,13 @@ func main() { } // Update collector with wsServer coll.SetBroadcaster(wsServer) + // Optional MaxMind databases for the UI's IP details dialog. + geo, err := geoip.Open(os.Getenv("OTEL_GEOIP_CITY_DB"), os.Getenv("OTEL_GEOIP_ASN_DB")) + if err != nil { + log.Fatalf("failed to open geoip database: %v", err) + } + defer geo.Close() + ingestKey := os.Getenv("OTEL_INGEST_KEY") coll.SetIngestKey(ingestKey) oidcAuth := auth.NewOidcAuth(ingestKey) @@ -299,6 +307,7 @@ func main() { // Public OIDC endpoints uiMux.HandleFunc("POST /api/auth/token", oidcAuth.HandleTokenExchange) uiMux.HandleFunc("GET /api/auth/config", oidcAuth.HandleConfig) + uiMux.Handle("GET /api/ip/{ip}", oidcAuth.RequireAuth(http.HandlerFunc(geo.Handler))) uiMux.Handle("/api/honeypot/ips", oidcAuth.RequireAuth(http.HandlerFunc(honeypotTracker.HandleIPsForUser))) if ipBlocker != nil { uiMux.Handle("GET /api/blocker/status", oidcAuth.RequireAuth(http.HandlerFunc(ipBlocker.HandleStatus))) diff --git a/cmd/spike-alerts/main.go b/cmd/spike-alerts/main.go index 5e5d049..56e1851 100644 --- a/cmd/spike-alerts/main.go +++ b/cmd/spike-alerts/main.go @@ -52,6 +52,7 @@ func main() { cfg := features.SpikeConfig{Window: *window, BaselineWindows: *baselineWindows, Multiple: *multiple, MinExcess: *minExcess} events := load(*addr, *database, *user, *password, start, end) + cfg.DataSince = dataSince(*addr, *database, *user, *password) log.Printf("loaded %d error events", len(events)) cache := map[string]classification{} @@ -212,3 +213,23 @@ func envOr(k, f string) string { } return f } + +// dataSince returns the earliest normalized row, used as the warm-up guard. +func dataSince(addr, database, user, password string) time.Time { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{addr}, Auth: clickhouse.Auth{Database: database, Username: user, Password: password}}) + if err != nil { + log.Fatal(err) + } + defer conn.Close() + var t time.Time + if err := conn.QueryRow(ctx, fmt.Sprintf(`SELECT min(timestamp) FROM %s.normalized_events`, database)).Scan(&t); err != nil { + log.Printf("warm-up guard disabled: %v", err) + return time.Time{} + } + if t.Year() < 2000 { // empty table + return time.Time{} + } + return t +} diff --git a/docs/STATUS.md b/docs/STATUS.md index edb1533..a923b8f 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -94,6 +94,32 @@ Categories: `scanner_probe`, `dependency_unreachable`, `client_credential_issue` logistic regression / MLP from `docs/ML.md`; the rule detector is what they must beat. +## Update 2026-09-20 + +- Warm-up guard: `SpikeConfig.DataSince` / `WarmupWindows` (default 6). No alert + until that much baseline exists; `spike-alerts` sets it from + `min(timestamp)` of `normalized_events`. Test: `TestDetectSpikesWarmupGuard`. +- `holdout.jsonl` labelled (by Claude, review it; ambiguous 404s such as `/mcp`, + `/graphql`, `/api`, `/api/v2/*` were labelled `stale_reference`). Rules fixed for + the misses: more probe paths (`_debugbar`, `debug/pprof`, `server-info`, + `actuator`, `__env`, `.bash_*`, `*.key|pem|py`, `graphql/console`) and all + 1xx/2xx/3xx are `expected_lifecycle`. 60/60, but the set is now tuned + against, so re-export a fresh holdout before trusting the number. + `IsProbePath` is shared with the blocker, so these also widen what counts as a probe. +- Client IP gap (last 6h of nginx spans): public IPs 10.3k spans (2192 IPs), + `10.10.0.1` 25k (the user's firewall: traffic it NATs to ingress arrives with the firewall as source), loopback 4.1k. The loopback rows include + real external scanners (CameraScannerV3, MSIE8 UAs, raw-IP hosts), so ~10% of + spans lose the attacker IP. Ingress has `externalTrafficPolicy: Local`, no + `use-proxy-protocol`, and `x-real-ip` is never attached. Cause: something on the + node forwards to ingress from 127.0.0.1; not yet identified. + +- UA classes: `internal/uaclass` holds the known crawler / AI agent / monitor / + scan-tool list (first match wins; UA is spoofable, so a class is a positive + signal only). Stored as `normalized_events.ua_class` (added by + `EnsureNormalizedSchema` on next start; old rows are `''`), served as + `uaClass` on issue samples and endpoint stats. Issues UI: add-filter offers + `client: X` and `hide client: X`. Not yet a triage/spike feature. + ## Commands ```bash @@ -130,3 +156,11 @@ triage-eval,bad-actors,normalized-backfill,server,needle-sidecar}`, (including a deleted `GaugeMeter.tsx`). Also tracked `__pycache__/*.pyc` files under `cmd/needle-sidecar` show as modified; they should be untracked and gitignored. + +## IP details (MaxMind) + +The issues view's source IP opens a dialog (`GET /api/ip/{ip}`, authed) with +country/region/city, timezone, ASN and network. Lookups are local. Enable by +mounting GeoLite2 files and setting `OTEL_GEOIP_CITY_DB` and/or +`OTEL_GEOIP_ASN_DB` (the Dockerfile copies `db/*.mmdb` into the image and sets both); without them the dialog says so. Private/loopback +addresses are labelled instead of looked up. diff --git a/go.mod b/go.mod index 3194520..117d596 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,8 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/oschwald/geoip2-golang/v2 v2.4.0 // indirect + github.com/oschwald/maxminddb-golang/v2 v2.6.0 // indirect github.com/paulmach/orb v0.13.0 // indirect github.com/pierrec/lz4/v4 v4.1.27 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -59,7 +61,7 @@ require ( go.opentelemetry.io/otel/trace v1.44.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/net v0.57.0 // indirect golang.org/x/oauth2 v0.35.0 // indirect golang.org/x/sys v0.47.0 // indirect diff --git a/go.sum b/go.sum index 0cca551..cd3eb7b 100644 --- a/go.sum +++ b/go.sum @@ -1,29 +1,58 @@ +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/ClickHouse/ch-go v0.74.0 h1:uYs2m4wIt0ZHSM1E72rg0maCfzhR2V3xWb/vZEgpeWE= github.com/ClickHouse/ch-go v0.74.0/go.mod h1:sZ/r+8ttZMjyrP9PuFbgoVbth1ywIu2LIQNA2vgko6M= github.com/ClickHouse/clickhouse-go/v2 v2.48.0 h1:auzd4VkapQYhQF8F2Gog7s3x78Bi1JZmByxGbrw3C+4= github.com/ClickHouse/clickhouse-go/v2 v2.48.0/go.mod h1:lBjUCPRG6RpRQdMbkXq+JV8rY0/O5lw+Z7jShgReFjM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dmarkham/enumer v1.6.3/go.mod h1:DyjXaqCglj4GhELF73oWiparNkYkXvmOBLza/o4kO74= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= +github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= @@ -38,8 +67,10 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -51,10 +82,12 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= +github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -70,8 +103,21 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= +github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/spdystream v0.5.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -80,29 +126,46 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/oschwald/geoip2-golang/v2 v2.4.0 h1:JdVymxpwFf7o+3o53Sw2gCYBX8maA5DWxcgzNb14yJU= +github.com/oschwald/geoip2-golang/v2 v2.4.0/go.mod h1:VJW7lAC5Dw4WH42mjhUFkxf7+v3K1YOafLD8iBiszsc= +github.com/oschwald/maxminddb-golang/v2 v2.6.0 h1:pRlHCdJmc+4uxMOSthmKDt5HOw3JTX8TJZlhyP5ew0w= +github.com/oschwald/maxminddb-golang/v2 v2.6.0/go.mod h1:sjqpB3z2BZrMduDp9TAUTCkZDoT3nDhixUc4Dge2qRQ= +github.com/pascaldekloe/name v1.0.1/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM= github.com/paulmach/orb v0.13.0 h1:r7n7mQGGF+cj/CbcivEj9J3HGK+XR+yXnvzRdq9saIw= github.com/paulmach/orb v0.13.0/go.mod h1:6scRWINywA2Jf05dcjOfLfxrUIMECvTSG2MVbRLxu/k= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk= github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil/v4 v4.26.5/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -114,12 +177,19 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/testcontainers/testcontainers-go v0.43.0/go.mod h1:+VxkT2NQnKOZPKi6praMuMKYHYyOGXr0XSBSlSMCzFo= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/collector/consumer v1.55.0 h1:7Per8P4J0nlBrFVSXb+nwZ+egiel1BRtggZngyykGsM= @@ -130,6 +200,8 @@ go.opentelemetry.io/collector/internal/testutil v0.149.0 h1:OWfUPO3NFKSaJtz/SBZp go.opentelemetry.io/collector/internal/testutil v0.149.0/go.mod h1:Jkjs6rkqs973LqgZ0Fe3zrokQRKULYXPIf4HuqStiEE= go.opentelemetry.io/collector/pdata v1.55.0 h1:WBgye8bo8koUyV9Vmp/r2Q3lgDezdsgfKDQAaM1oT2I= go.opentelemetry.io/collector/pdata v1.55.0/go.mod h1:6jPrbM4tuliCPACDznjFtxnnHisfKfzwrBVoeuESYuk= +go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= @@ -152,15 +224,20 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.39.0/go.mod h1:bvIbwjQ0HUFFf5AKukeeYQG4ZBUG9yxQbR9aEweIwYY= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -172,6 +249,7 @@ golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwE golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -191,6 +269,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -225,6 +305,7 @@ k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4= k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY= k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8= +k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= diff --git a/internal/collector/collector.go b/internal/collector/collector.go index c8f9556..f10465c 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -18,6 +18,7 @@ import ( "git.tornberg.me/go-otel/internal/prometheus" "git.tornberg.me/go-otel/internal/sourcemaps" "git.tornberg.me/go-otel/internal/storage" + "git.tornberg.me/go-otel/internal/uaclass" "git.tornberg.me/go-otel/internal/websocket" "go.opentelemetry.io/collector/consumer" @@ -163,8 +164,9 @@ func (c *Collector) ListIssues(limit int) []issues.Issue { // ListIssuesFiltered returns a filtered page of deduplicated issues. func (c *Collector) ListIssuesFiltered(opts issues.ListOptions) issues.ListPage { page := c.issues.ListFiltered(opts) - if c.sourceMaps != nil { - for i := range page.Issues { + for i := range page.Issues { + classifyIssue(&page.Issues[i]) + if c.sourceMaps != nil { resolveIssue(&page.Issues[i], c.sourceMaps) } } @@ -174,8 +176,11 @@ func (c *Collector) ListIssuesFiltered(opts issues.ListOptions) issues.ListPage // GetIssue returns a single issue by id. func (c *Collector) GetIssue(id string) (issues.Issue, bool) { issue, ok := c.issues.Get(id) - if ok && c.sourceMaps != nil { - resolveIssue(&issue, c.sourceMaps) + if ok { + classifyIssue(&issue) + if c.sourceMaps != nil { + resolveIssue(&issue, c.sourceMaps) + } } return issue, ok } @@ -194,6 +199,16 @@ func resolveIssue(issue *issues.Issue, resolver *sourcemaps.Resolver) { } } +// classifyIssue sets each sample's UAClass from its User-Agent. Samples with +// no UA (backend logs, most spans) stay unclassified rather than "none". +func classifyIssue(issue *issues.Issue) { + for i := range issue.Samples { + if ua := issue.Samples[i].UserAgent; ua != "" { + issue.Samples[i].UAClass = uaclass.Of(ua) + } + } +} + // SetIssueStatus updates the triage state of an issue. func (c *Collector) SetIssueStatus(id, status string) (issues.Issue, bool) { return c.issues.SetStatus(id, status) diff --git a/internal/collector/handlers.go b/internal/collector/handlers.go index f882856..c814ff8 100644 --- a/internal/collector/handlers.go +++ b/internal/collector/handlers.go @@ -175,6 +175,8 @@ func (c *Collector) HandleLogs(w http.ResponseWriter, r *http.Request) { return } + stampFrontendClient(logs, r) + if err := c.ConsumeLogs(context.Background(), logs); err != nil { log.Printf("failed to consume logs: %v", err) http.Error(w, "failed to consume logs", http.StatusInternalServerError) @@ -990,3 +992,31 @@ func (c *Collector) HandleNeedleClassify(w http.ResponseWriter, r *http.Request) w.WriteHeader(resp.StatusCode) _, _ = io.Copy(w, resp.Body) } + +// stampFrontendClient fills in client.address and user_agent.original on +// browser-originated log records (source=frontend) from the ingest request, +// since the browser cannot know its own public IP. Values the SDK already +// sent are kept, and backend services are left alone. +func stampFrontendClient(logs plog.Logs, r *http.Request) { + ip := strings.TrimSpace(clientIP(r)) + ua := r.Header.Get("User-Agent") + rls := logs.ResourceLogs() + for i := 0; i < rls.Len(); i++ { + sls := rls.At(i).ScopeLogs() + for j := 0; j < sls.Len(); j++ { + recs := sls.At(j).LogRecords() + for k := 0; k < recs.Len(); k++ { + a := recs.At(k).Attributes() + if v, ok := a.Get("source"); !ok || v.Str() != "frontend" { + continue + } + if _, ok := a.Get("client.address"); !ok && ip != "" { + a.PutStr("client.address", ip) + } + if _, ok := a.Get("browser.userAgent"); !ok && ua != "" { + a.PutStr("user_agent.original", ua) + } + } + } + } +} diff --git a/internal/features/spike.go b/internal/features/spike.go index c4d92b9..f8e617d 100644 --- a/internal/features/spike.go +++ b/internal/features/spike.go @@ -15,6 +15,12 @@ type SpikeConfig struct { Multiple float64 // a pattern must be novel or >= Multiple * its own baseline mean MinExcess int // and excess errors must be >= MinExcess TopPatterns int + // DataSince is when data collection began (earliest row of any kind). When + // set, no alert fires until WarmupWindows windows of baseline exist, + // otherwise the first window after data begins reports everything as NEW. + // Zero disables the guard. + DataSince time.Time + WarmupWindows int } func DefaultSpikeConfig() SpikeConfig { @@ -64,6 +70,15 @@ func DetectSpikes(events []Event, at time.Time, cfg SpikeConfig) []Spike { at = at.UTC() start := at.Add(-cfg.Window) baseStart := start.Add(-time.Duration(cfg.BaselineWindows) * cfg.Window) + if !cfg.DataSince.IsZero() { + warm := cfg.WarmupWindows + if warm <= 0 { + warm = min(cfg.BaselineWindows, 6) + } + if start.Sub(cfg.DataSince.UTC()) < time.Duration(warm)*cfg.Window { + return nil + } + } type agg struct { cur int diff --git a/internal/features/spike_test.go b/internal/features/spike_test.go index 3571262..f9ae709 100644 --- a/internal/features/spike_test.go +++ b/internal/features/spike_test.go @@ -77,3 +77,24 @@ func TestSuppressorCooldownAndEscalation(t *testing.T) { t.Fatal("alert after cooldown should pass") } } + +func TestDetectSpikesWarmupGuard(t *testing.T) { + at := time.Date(2026, 9, 19, 12, 0, 0, 0, time.UTC) + cfg := SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, WarmupWindows: 6} + var events []Event + for i := 0; i < 20; i++ { + events = append(events, Event{Timestamp: at.Add(-time.Minute).Add(time.Duration(i) * time.Second), Service: "api", IsError: true, PatternSignature: "new"}) + } + cfg.DataSince = at.Add(-10 * time.Minute) // data began 10 min ago: cold + if got := DetectSpikes(events, at, cfg); len(got) != 0 { + t.Fatalf("alerted during warm-up: %+v", got) + } + cfg.DataSince = at.Add(-2 * time.Hour) // warm + if got := DetectSpikes(events, at, cfg); len(got) != 1 { + t.Fatalf("expected alert once warm, got %+v", got) + } + cfg.DataSince = time.Time{} // guard disabled + if got := DetectSpikes(events, at, cfg); len(got) != 1 { + t.Fatalf("guard should be off when DataSince is zero") + } +} diff --git a/internal/geoip/geoip.go b/internal/geoip/geoip.go new file mode 100644 index 0000000..02171f9 --- /dev/null +++ b/internal/geoip/geoip.go @@ -0,0 +1,140 @@ +// Package geoip resolves IP addresses against local MaxMind (GeoLite2 / GeoIP2) +// databases so the UI can show where an issue's client came from. Lookups are +// purely local; no IP ever leaves the process. +package geoip + +import ( + "encoding/json" + "log" + "net/http" + "net/netip" + "strings" + + geoip2 "github.com/oschwald/geoip2-golang/v2" +) + +// Lookup holds the opened databases. Either reader may be nil; a nil *Lookup +// means no database is configured. +type Lookup struct { + city *geoip2.Reader + asn *geoip2.Reader +} + +// Result is the JSON shape returned to the UI. Fields the databases don't +// know are omitted. +type Result struct { + IP string `json:"ip"` + Configured bool `json:"configured"` + // Scope is set for addresses that can't be geolocated: "private", + // "loopback", "link-local" or "unspecified". + Scope string `json:"scope,omitempty"` + + Country string `json:"country,omitempty"` + CountryCode string `json:"countryCode,omitempty"` + InEU bool `json:"inEu,omitempty"` + Region string `json:"region,omitempty"` + City string `json:"city,omitempty"` + Postal string `json:"postal,omitempty"` + Timezone string `json:"timezone,omitempty"` + Latitude *float64 `json:"latitude,omitempty"` + Longitude *float64 `json:"longitude,omitempty"` + Network string `json:"network,omitempty"` + ASN uint `json:"asn,omitempty"` + ASOrg string `json:"asOrg,omitempty"` +} + +// Open opens the given database files; empty paths are skipped. It returns nil +// (and no error) when both are empty. +func Open(cityPath, asnPath string) (*Lookup, error) { + if cityPath == "" && asnPath == "" { + return nil, nil + } + l := &Lookup{} + var err error + if cityPath != "" { + if l.city, err = geoip2.Open(cityPath); err != nil { + return nil, err + } + } + if asnPath != "" { + if l.asn, err = geoip2.Open(asnPath); err != nil { + l.Close() + return nil, err + } + } + return l, nil +} + +// Close releases the memory-mapped databases. +func (l *Lookup) Close() { + if l == nil { + return + } + if l.city != nil { + l.city.Close() + } + if l.asn != nil { + l.asn.Close() + } +} + +// Lookup resolves ip. It never fails: an unparsable or unroutable address just +// yields a Result without geo fields. +func (l *Lookup) Lookup(ip netip.Addr) Result { + ip = ip.Unmap() + res := Result{IP: ip.String(), Configured: l != nil} + switch { + case ip.IsLoopback(): + res.Scope = "loopback" + case ip.IsPrivate(): + res.Scope = "private" + case ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast(): + res.Scope = "link-local" + case ip.IsUnspecified(): + res.Scope = "unspecified" + } + if l == nil || res.Scope != "" { + return res + } + if l.city != nil { + if c, err := l.city.City(ip); err == nil && c.HasData() { + res.Country = c.Country.Names.English + res.CountryCode = c.Country.ISOCode + res.InEU = c.Country.IsInEuropeanUnion + res.City = c.City.Names.English + res.Postal = c.Postal.Code + res.Timezone = c.Location.TimeZone + res.Latitude, res.Longitude = c.Location.Latitude, c.Location.Longitude + if n := len(c.Subdivisions); n > 0 { + res.Region = c.Subdivisions[0].Names.English + } + if c.Traits.Network.IsValid() { + res.Network = c.Traits.Network.String() + } + } + } + if l.asn != nil { + if a, err := l.asn.ASN(ip); err == nil && a.HasData() { + res.ASN, res.ASOrg = a.AutonomousSystemNumber, a.AutonomousSystemOrganization + if res.Network == "" && a.Network.IsValid() { + res.Network = a.Network.String() + } + } + } + return res +} + +// Handler serves GET /api/ip/{ip}. Without a configured database it still +// answers (Configured=false) so the UI can explain how to enable it. +func (l *Lookup) Handler(w http.ResponseWriter, r *http.Request) { + ip, err := netip.ParseAddr(strings.TrimSpace(r.PathValue("ip"))) + if err != nil { + http.Error(w, "invalid ip address", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "private, max-age=3600") + if err := json.NewEncoder(w).Encode(l.Lookup(ip)); err != nil { + log.Printf("geoip: encode: %v", err) + } +} diff --git a/internal/geoip/geoip_test.go b/internal/geoip/geoip_test.go new file mode 100644 index 0000000..9c8061c --- /dev/null +++ b/internal/geoip/geoip_test.go @@ -0,0 +1,25 @@ +package geoip + +import ( + "net/netip" + "testing" +) + +func TestLookupWithoutDB(t *testing.T) { + var l *Lookup + if r := l.Lookup(netip.MustParseAddr("8.8.8.8")); r.Configured || r.Country != "" { + t.Fatalf("unexpected result %+v", r) + } + if r := l.Lookup(netip.MustParseAddr("10.1.2.3")); r.Scope != "private" { + t.Fatalf("want private scope, got %+v", r) + } + if r := l.Lookup(netip.MustParseAddr("::ffff:127.0.0.1")); r.Scope != "loopback" || r.IP != "127.0.0.1" { + t.Fatalf("want unmapped loopback, got %+v", r) + } +} + +func TestOpenEmpty(t *testing.T) { + if l, err := Open("", ""); l != nil || err != nil { + t.Fatalf("want nil,nil got %v,%v", l, err) + } +} diff --git a/internal/issues/tracker.go b/internal/issues/tracker.go index 0500647..bc6730a 100644 --- a/internal/issues/tracker.go +++ b/internal/issues/tracker.go @@ -51,7 +51,10 @@ type Sample struct { HTTPRoute string `json:"httpRoute,omitempty"` HTTPStatus int `json:"httpStatus,omitempty"` // Client/release context. - UserAgent string `json:"userAgent,omitempty"` + UserAgent string `json:"userAgent,omitempty"` + // UAClass is derived from UserAgent when issues are served (see + // internal/uaclass); it is not part of the stored sample. + UAClass string `json:"uaClass,omitempty"` SourceIP string `json:"sourceIp,omitempty"` Release string `json:"release,omitempty"` Environment string `json:"environment,omitempty"` diff --git a/internal/normalization/normalizer.go b/internal/normalization/normalizer.go index 0f193fa..406989b 100644 --- a/internal/normalization/normalizer.go +++ b/internal/normalization/normalizer.go @@ -12,9 +12,10 @@ import ( "git.tornberg.me/go-otel/internal/httpattr" "git.tornberg.me/go-otel/internal/issues" + "git.tornberg.me/go-otel/internal/uaclass" ) -const Version = "v1" +const Version = "v2" type Event struct { EventID string `json:"eventId"` @@ -29,6 +30,7 @@ type Event struct { Method string `json:"method,omitempty"` Host string `json:"host,omitempty"` UserAgent string `json:"userAgent,omitempty"` + UAClass string `json:"uaClass,omitempty"` OriginHost string `json:"originHost,omitempty"` RefererHost string `json:"refererHost,omitempty"` FetchSite string `json:"fetchSite,omitempty"` @@ -53,6 +55,15 @@ type Event struct { MessageNormalized string `json:"messageNormalized,omitempty"` StackSignature string `json:"stackSignature,omitempty"` TraceID string `json:"traceId,omitempty"` + SpanID string `json:"spanId,omitempty"` + ParentSpanID string `json:"parentSpanId,omitempty"` + SpanKind int `json:"spanKind,omitempty"` + PeerIP string `json:"peerIp,omitempty"` + PeerIPSource string `json:"peerIpSource,omitempty"` + EdgePeerIP string `json:"edgePeerIp,omitempty"` + EdgePeerSource string `json:"edgePeerSource,omitempty"` + EdgeService string `json:"edgeService,omitempty"` + EdgeSpanID string `json:"edgeSpanId,omitempty"` IssueFingerprint string `json:"issueFingerprint,omitempty"` IsRetry bool `json:"isRetry"` IsTimeout bool `json:"isTimeout"` @@ -96,35 +107,67 @@ func normalizeTraces(ingest int64, envelope map[string]json.RawMessage) []Event } `json:"scopeSpans"` } _ = json.Unmarshal(envelope["resourceSpans"], &resources) - result := []Event{} + type traceSpan struct { + resource object + item span + attrs []attribute + service string + } + var spans []traceSpan for _, resource := range resources { for _, scope := range resource.ScopeSpans { for _, item := range scope.Spans { attrs := mergeAttrs(resource.Resource.Attributes, item.Attributes) - status := intValue(item.Status.Code) - statusCode := intAttr(attrs, "http.response.status_code", "http.status_code") - route := normalizeHTTPRoute(attrs) - message := item.Status.Message - exceptionType, exceptionMessage, stack := exception(item.Events) - if exceptionMessage != "" { - message = exceptionMessage - } - if message == "" { - message = item.Name - } - if exceptionType != "" && message != "" { - message = exceptionType + ": " + message - } - isError := status == 2 || isErrorStatus(statusCode) || intAttr(attrs, "rpc.grpc.status_code") > 0 || exceptionType != "" - e := baseEvent(ingest, attrs, "span", item.TraceID) - e.IsError, e.StatusCode, e.StatusClass = isError, statusCode, statusClass(statusCode) - e.DurationMS = durationMS(item.Start, item.End) - e.Route, e.SpanName, e.ExceptionType, e.MessageNormalized, e.StackSignature = route, item.Name, exceptionType, normalizeMessage(message), firstFrame(stack) - e.IsRetry, e.IsTimeout, e.IsCancellation = operationFlags(attrs, statusCode, intAttr(attrs, "rpc.grpc.status_code")) - result = append(result, finish(e)) + spans = append(spans, traceSpan{ + resource: resource.Resource, + item: item, + attrs: attrs, + service: defaultString(stringAttr(attrs, "service.name"), "unknown"), + }) } } } + contexts := make(map[string]spanContext, len(spans)) + for _, current := range spans { + if current.item.SpanID == "" || current.item.TraceID == "" { + continue + } + peer, peerSource := peerAddress(current.attrs) + contexts[spanKey(current.item.TraceID, current.item.SpanID)] = spanContext{ + TraceID: current.item.TraceID, SpanID: current.item.SpanID, + ParentSpanID: current.item.ParentSpanID, Service: current.service, + PeerIP: peer, PeerSource: peerSource, + Edge: isEdgeService(current.service), + } + } + + result := make([]Event, 0, len(spans)) + for _, current := range spans { + item, attrs := current.item, current.attrs + status := intValue(item.Status.Code) + statusCode := intAttr(attrs, "http.response.status_code", "http.status_code") + route := normalizeHTTPRoute(attrs) + message := item.Status.Message + exceptionType, exceptionMessage, stack := exception(item.Events) + if exceptionMessage != "" { + message = exceptionMessage + } + if message == "" { + message = item.Name + } + if exceptionType != "" && message != "" { + message = exceptionType + ": " + message + } + isError := status == 2 || isErrorStatus(statusCode) || intAttr(attrs, "rpc.grpc.status_code") > 0 || exceptionType != "" + e := baseEvent(ingest, attrs, "span", item.TraceID) + e.SpanID, e.ParentSpanID, e.SpanKind = item.SpanID, item.ParentSpanID, item.Kind + applyTraceContext(&e, contexts[spanKey(item.TraceID, item.SpanID)], contexts) + e.IsError, e.StatusCode, e.StatusClass = isError, statusCode, statusClass(statusCode) + e.DurationMS = durationMS(item.Start, item.End) + e.Route, e.SpanName, e.ExceptionType, e.MessageNormalized, e.StackSignature = route, item.Name, exceptionType, normalizeMessage(message), firstFrame(stack) + e.IsRetry, e.IsTimeout, e.IsCancellation = operationFlags(attrs, statusCode, intAttr(attrs, "rpc.grpc.status_code")) + result = append(result, finish(e)) + } return result } @@ -198,17 +241,71 @@ type value struct { BoolValue bool `json:"boolValue"` } type span struct { - TraceID string `json:"traceId"` - Name string `json:"name"` - Start string `json:"startTimeUnixNano"` - End string `json:"endTimeUnixNano"` - Attributes []attribute `json:"attributes"` - Events []spanEvent `json:"events"` - Status struct { + TraceID string `json:"traceId"` + SpanID string `json:"spanId"` + ParentSpanID string `json:"parentSpanId"` + Name string `json:"name"` + Kind int `json:"kind"` + Start string `json:"startTimeUnixNano"` + End string `json:"endTimeUnixNano"` + Attributes []attribute `json:"attributes"` + Events []spanEvent `json:"events"` + Status struct { Code json.Number `json:"code"` Message string `json:"message"` } `json:"status"` } + +type spanContext struct { + TraceID, SpanID, ParentSpanID string + Service string + PeerIP, PeerSource string + Edge bool +} + +func spanKey(traceID, spanID string) string { return traceID + "\x00" + spanID } + +// applyTraceContext keeps the immediate peer separate from the peer observed +// at an edge ancestor. In particular, a firewall/NAT address is never +// relabeled as the original client address. +func applyTraceContext(e *Event, current spanContext, contexts map[string]spanContext) { + e.PeerIP, e.PeerIPSource = current.PeerIP, current.PeerSource + edge := current + found := current.Edge && current.PeerIP != "" + for hops := 0; !found && edge.ParentSpanID != "" && hops < 32; hops++ { + parent, ok := contexts[spanKey(edge.TraceID, edge.ParentSpanID)] + if !ok { + break + } + edge = parent + if edge.Edge && edge.PeerIP != "" { + found = true + } + } + if found { + e.EdgePeerIP, e.EdgePeerSource = edge.PeerIP, edge.PeerSource + e.EdgeService, e.EdgeSpanID = edge.Service, edge.SpanID + } +} + +func peerAddress(attrs []attribute) (string, string) { + for _, key := range []string{"net.peer.ip", "client.address", "net.sock.peer.addr"} { + if value := stringAttr(attrs, key); value != "" { + return value, key + } + } + return "", "" +} + +func isEdgeService(service string) bool { + switch strings.ToLower(service) { + case "nginx", "ingress-nginx", "envoy", "traefik", "haproxy", "gateway": + return true + default: + return false + } +} + type spanEvent struct { Name string `json:"name"` Attributes []attribute `json:"attributes"` @@ -309,13 +406,18 @@ func baseEvent(timestamp int64, attrs []attribute, kind, traceID string) Event { source = "backend" } ep := httpattr.Extract(func(keys ...string) string { return stringAttr(attrs, keys...) }) + ua := stringAttr(attrs, "http.user_agent", "user_agent.original") + uaClass := "" + if ua != "" { + uaClass = uaclass.Of(ua) + } return Event{ Timestamp: timestamp, NormalizerVersion: Version, Service: defaultString(stringAttr(attrs, "service.name"), "unknown"), Release: stringAttr(attrs, "service.release"), Environment: defaultString(stringAttr(attrs, "deployment.environment.name", "deployment.environment", "service.environment"), ""), Method: ep.Method, Host: ep.Host, - UserAgent: stringAttr(attrs, "http.user_agent", "user_agent.original"), + UserAgent: ua, UAClass: uaClass, OriginHost: attrHost(stringAttr(attrs, "http.request.header.origin", "http.origin", "origin")), RefererHost: attrHost(stringAttr(attrs, "http.request.header.referer", "http.referer")), FetchSite: stringAttr(attrs, "http.request.header.sec_fetch_site"), diff --git a/internal/normalization/normalizer_test.go b/internal/normalization/normalizer_test.go index fc08836..866885a 100644 --- a/internal/normalization/normalizer_test.go +++ b/internal/normalization/normalizer_test.go @@ -46,6 +46,31 @@ func TestNormalizeTraceRedactsValuesAndCreatesStableSignatures(t *testing.T) { } } +func TestNormalizeTraceCarriesPeerAndEdgeContext(t *testing.T) { + data := []byte(`{"resourceSpans":[ + {"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"nginx"}}]},"scopeSpans":[{"spans":[ + {"traceId":"trace-1","spanId":"edge-1","kind":2,"name":"GET /","startTimeUnixNano":"100","endTimeUnixNano":"200","attributes":[{"key":"net.peer.ip","value":{"stringValue":"10.10.0.1"}}]} + ]}]}, + {"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"checkout"}}]},"scopeSpans":[{"spans":[ + {"traceId":"trace-1","spanId":"app-1","parentSpanId":"edge-1","kind":1,"name":"checkout","startTimeUnixNano":"120","endTimeUnixNano":"180","attributes":[{"key":"rpc.system","value":{"stringValue":"grpc"}}]} + ]}]} + ]}`) + events, err := NormalizeEntry("traces", 99, data) + if err != nil || len(events) != 2 { + t.Fatalf("normalize failed: %v %+v", err, events) + } + if events[0].PeerIP != "10.10.0.1" || events[0].PeerIPSource != "net.peer.ip" { + t.Fatalf("edge peer was not preserved: %+v", events[0]) + } + child := events[1] + if child.PeerIP != "" || child.EdgePeerIP != "10.10.0.1" || child.EdgeService != "nginx" || child.EdgeSpanID != "edge-1" { + t.Fatalf("unexpected inherited edge context: %+v", child) + } + if child.SpanID != "app-1" || child.ParentSpanID != "edge-1" || child.SpanKind != 1 { + t.Fatalf("missing span relationship fields: %+v", child) + } +} + func TestNormalizeLogUsesEventTimestampAndFrontendSource(t *testing.T) { data := []byte(`{"resourceLogs":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"web"}}]},"scopeLogs":[{"logRecords":[{"timeUnixNano":"1230000000","severityNumber":17,"body":{"stringValue":"User 123 failed"},"attributes":[{"key":"source","value":{"stringValue":"frontend"}}]}]}]}]}`) events, err := NormalizeEntry("logs", 99, data) diff --git a/internal/storage/clickhouse_endpoints.go b/internal/storage/clickhouse_endpoints.go index 8944264..b004a17 100644 --- a/internal/storage/clickhouse_endpoints.go +++ b/internal/storage/clickhouse_endpoints.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "time" + + "git.tornberg.me/go-otel/internal/uaclass" ) // EndpointStats reads raw trace spans because the production entries table is @@ -85,6 +87,9 @@ func (s *ClickHouseStore) EndpointStats(ctx context.Context, host, service strin ); err != nil { return nil, err } + if stat.UserAgent != "" { + stat.UAClass = uaclass.Of(stat.UserAgent) + } if stat.Route == "/" && stat.Path != "/" { stat.Route = "" } diff --git a/internal/storage/normalized.go b/internal/storage/normalized.go index 0a94d82..a081df4 100644 --- a/internal/storage/normalized.go +++ b/internal/storage/normalized.go @@ -19,8 +19,9 @@ import ( const normalizedColumns = `event_id, entry_type, event_index, ingest_timestamp, timestamp, service, release, environment, kind, source, is_error, status_class, status_code, duration_ms, route, span_name, metric_name, metric_type, metric_unit, metric_value, metric_count, metric_sum, metric_min, metric_max, exception_type, message_normalized, stack_signature, - trace_id, issue_fingerprint, is_retry, is_timeout, is_cancellation, exact_signature, pattern_signature, version, - method, host, user_agent, origin_host, referer_host, fetch_site, fetch_mode` + trace_id, span_id, parent_span_id, span_kind, peer_ip, peer_ip_source, edge_peer_ip, edge_peer_source, edge_service, edge_span_id, + issue_fingerprint, is_retry, is_timeout, is_cancellation, exact_signature, pattern_signature, version, + method, host, user_agent, origin_host, referer_host, fetch_site, fetch_mode, ua_class` // NormalizedInsertSQL is the INSERT prefix for a normalized_events batch. func NormalizedInsertSQL(db string) string { @@ -41,11 +42,15 @@ func EnsureNormalizedSchema(ctx context.Context, conn driver.Conn, db string) er status_class LowCardinality(String), status_code UInt16, duration_ms Float64, route String, span_name String, metric_name String, metric_type LowCardinality(String), metric_unit String, metric_value Float64, metric_count UInt64, metric_sum Float64, metric_min Float64, metric_max Float64, - exception_type String, message_normalized String, stack_signature String, trace_id String, issue_fingerprint String, + exception_type String, message_normalized String, stack_signature String, trace_id String, + span_id String DEFAULT '', parent_span_id String DEFAULT '', span_kind UInt8 DEFAULT 0, + peer_ip String DEFAULT '', peer_ip_source LowCardinality(String) DEFAULT '', + edge_peer_ip String DEFAULT '', edge_peer_source LowCardinality(String) DEFAULT '', + edge_service LowCardinality(String) DEFAULT '', edge_span_id String DEFAULT '', issue_fingerprint String, is_retry UInt8, is_timeout UInt8, is_cancellation UInt8, exact_signature String, pattern_signature String, version UInt8 DEFAULT 1, method String DEFAULT '', host String DEFAULT '', user_agent String DEFAULT '', - origin_host String DEFAULT '', referer_host String DEFAULT '', fetch_site String DEFAULT '', fetch_mode String DEFAULT '' + origin_host String DEFAULT '', referer_host String DEFAULT '', fetch_site String DEFAULT '', fetch_mode String DEFAULT '', ua_class LowCardinality(String) DEFAULT '' ) ENGINE = ReplacingMergeTree(version) PARTITION BY toYYYYMM(timestamp) ORDER BY (service, kind, pattern_signature, timestamp, event_id) @@ -55,10 +60,14 @@ func EnsureNormalizedSchema(ctx context.Context, conn driver.Conn, db string) er } // Developer tables predate these columns; historical rows get safe defaults. for _, col := range []string{ + "span_id String DEFAULT ''", "parent_span_id String DEFAULT ''", "span_kind UInt8 DEFAULT 0", + "peer_ip String DEFAULT ''", "peer_ip_source LowCardinality(String) DEFAULT ''", + "edge_peer_ip String DEFAULT ''", "edge_peer_source LowCardinality(String) DEFAULT ''", + "edge_service LowCardinality(String) DEFAULT ''", "edge_span_id String DEFAULT ''", "issue_fingerprint String DEFAULT ''", "is_retry UInt8 DEFAULT 0", "is_timeout UInt8 DEFAULT 0", "is_cancellation UInt8 DEFAULT 0", "method String DEFAULT ''", "host String DEFAULT ''", "user_agent String DEFAULT ''", "origin_host String DEFAULT ''", "referer_host String DEFAULT ''", - "fetch_site String DEFAULT ''", "fetch_mode String DEFAULT ''", + "fetch_site String DEFAULT ''", "fetch_mode String DEFAULT ''", "ua_class LowCardinality(String) DEFAULT ''", } { if err := conn.Exec(ctx, fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS %s", db, col)); err != nil { return err @@ -74,9 +83,10 @@ func AppendNormalizedEvent(batch driver.Batch, e normalization.Event) error { e.Service, e.Release, e.Environment, e.Kind, e.Source, boolToUint8(e.IsError), e.StatusClass, uint16(e.StatusCode), e.DurationMS, e.Route, e.SpanName, e.MetricName, e.MetricType, e.MetricUnit, e.MetricValue, e.MetricCount, e.MetricSum, e.MetricMin, e.MetricMax, - e.ExceptionType, e.MessageNormalized, e.StackSignature, e.TraceID, e.IssueFingerprint, + e.ExceptionType, e.MessageNormalized, e.StackSignature, e.TraceID, e.SpanID, e.ParentSpanID, uint8(e.SpanKind), + e.PeerIP, e.PeerIPSource, e.EdgePeerIP, e.EdgePeerSource, e.EdgeService, e.EdgeSpanID, e.IssueFingerprint, boolToUint8(e.IsRetry), boolToUint8(e.IsTimeout), boolToUint8(e.IsCancellation), e.ExactSignature, e.PatternSignature, - uint8(1), e.Method, e.Host, e.UserAgent, e.OriginHost, e.RefererHost, e.FetchSite, e.FetchMode, + uint8(2), e.Method, e.Host, e.UserAgent, e.OriginHost, e.RefererHost, e.FetchSite, e.FetchMode, e.UAClass, ) } diff --git a/internal/storage/store.go b/internal/storage/store.go index cf75519..c2565b8 100644 --- a/internal/storage/store.go +++ b/internal/storage/store.go @@ -56,6 +56,7 @@ type EndpointStat struct { Route string `json:"route,omitempty"` StatusCode uint16 `json:"statusCode"` UserAgent string `json:"userAgent,omitempty"` + UAClass string `json:"uaClass,omitempty"` OriginHost string `json:"originHost,omitempty"` RefererHost string `json:"refererHost,omitempty"` FetchSite string `json:"fetchSite,omitempty"` diff --git a/internal/triage/triage.go b/internal/triage/triage.go index 9a31f45..57772fd 100644 --- a/internal/triage/triage.go +++ b/internal/triage/triage.go @@ -47,8 +47,9 @@ var ( // control, CMS/admin paths, path traversal. Deliberately path-only. probeRoute = regexp.MustCompile(`(?i)(\.env\b|\.git\b|\.htpasswd|\.htaccess|\.npmrc|\.claude|\.docker|\.kube|\.aws|\.ssh|` + `wp-|/wp/|/wp$|wordpress|phpmyadmin|phpinfo|cgi-bin|xmlrpc|userfiles|@fs/|etc/passwd|proc/self|\.\./|%2e%2e|` + - `\.php\b|credentials|secrets?\b|firebase|serviceaccount|/config(/|\.)|settings\.(js|json)$|bootstrap\.properties|` + - `\.(ya?ml|toml|properties|sql|bak|old|swp|zip|tar|tgz|gz|ini|conf|cfg)$)`) + `\.php\b|_debugbar|debug/pprof|server-info|actuator|__env|\.bash_|\.profile$|graphql/console|` + + `credentials|secrets?\b|firebase|serviceaccount|/config(/|\.)|settings\.(js|json|py)$|bootstrap\.properties|` + + `\.(ya?ml|toml|properties|sql|bak|old|swp|zip|tar|tgz|gz|ini|conf|cfg|key|pem)$)`) bugText = regexp.MustCompile(`(?i)panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault`) capText = regexp.MustCompile(`(?i)timeout|deadline exceeded|\bpool\b|memory limit|\boom(killed)?\b|too many|queue full|exhaust`) depText = regexp.MustCompile(`(?i)dial|connection refused|\beof\b|no such host|upstream|unavailable|broken pipe|reset by peer`) @@ -87,7 +88,7 @@ func Classify(in Input) Result { return Result{StaleReference, "status-404"} case lifeText.MatchString(blob): return Result{ExpectedLifecycle, "lifecycle-text"} - case in.Status == 499 || in.Status == 200 || in.Status == 202 || in.Status == 204 || (in.Status >= 300 && in.Status < 400): + case in.Status == 499 || (in.Status >= 100 && in.Status < 400): return Result{ExpectedLifecycle, "benign-status"} case in.Status == 0 && (in.Kind == "span" || in.Kind == "log"): return Result{ExpectedLifecycle, "no-status-signal"} diff --git a/internal/uaclass/uaclass.go b/internal/uaclass/uaclass.go new file mode 100644 index 0000000..98d2ddf --- /dev/null +++ b/internal/uaclass/uaclass.go @@ -0,0 +1,267 @@ +// Package uaclass classifies a User-Agent string into a coarse client class +// using a list of known crawlers, AI agents, monitors and scan tools. +// +// A User-Agent is self-declared and trivially spoofed, so the class is a +// positive signal only: "ai_crawler" means the client says it is one, not that +// it is. A missing or unrecognised UA is never evidence of a browser. +package uaclass + +import "strings" + +// Class values are stable strings: they are stored in ClickHouse and used as +// UI filter values. +const ( + None = "none" // no User-Agent sent + AICrawler = "ai_crawler" // model-training / AI search fetchers + AIAgent = "ai_agent" // agent harnesses and user-triggered AI fetchers + SearchCrawler = "search_crawler" // classic search engines and link previewers + SEOCrawler = "seo_crawler" // SEO / marketing crawlers + Monitor = "monitor" // uptime checks and cluster probes + ScanTool = "scan_tool" // known scanners and exploit tooling + OtherBot = "other_bot" // declares bot/crawler/spider but is not listed + Library = "library" // HTTP client libraries and CLIs + Browser = "browser" // looks like a real browser + Unknown = "unknown" +) + +// Classes lists every class in display order, for UI filters and docs. +var Classes = []string{AICrawler, AIAgent, SearchCrawler, SEOCrawler, Monitor, ScanTool, OtherBot, Library, Browser, Unknown, None} + +// Result is the class plus the matched product token, when one matched. +type Result struct { + Class string `json:"class"` + Name string `json:"name,omitempty"` +} + +type rule struct { + class string + name string + token string // lower-case substring of the lower-cased UA +} + +// Order matters: the first match wins, and specific products come before the +// generic bot regexes and the "Mozilla/" browser fallback (most crawlers +// prefix their UA with Mozilla/5.0). +var rules = []rule{ + // AI training and AI search crawlers. + {AICrawler, "GPTBot", "gptbot"}, + {AICrawler, "OAI-SearchBot", "oai-searchbot"}, + {AICrawler, "ClaudeBot", "claudebot"}, + {AICrawler, "Claude-SearchBot", "claude-searchbot"}, + {AICrawler, "anthropic-ai", "anthropic-ai"}, + {AICrawler, "PerplexityBot", "perplexitybot"}, + {AICrawler, "Google-Extended", "google-extended"}, + {AICrawler, "GoogleOther", "googleother"}, + {AICrawler, "Google-CloudVertexBot", "google-cloudvertexbot"}, + {AICrawler, "CCBot", "ccbot"}, + {AICrawler, "Bytespider", "bytespider"}, + {AICrawler, "Amazonbot", "amazonbot"}, + {AICrawler, "Applebot-Extended", "applebot-extended"}, + {AICrawler, "meta-externalagent", "meta-externalagent"}, + {AICrawler, "meta-externalfetcher", "meta-externalfetcher"}, + {AICrawler, "FacebookBot", "facebookbot"}, + {AICrawler, "cohere-ai", "cohere-ai"}, + {AICrawler, "cohere-training-data-crawler", "cohere-training-data-crawler"}, + {AICrawler, "Diffbot", "diffbot"}, + {AICrawler, "YouBot", "youbot"}, + {AICrawler, "DuckAssistBot", "duckassistbot"}, + {AICrawler, "MistralAI-User", "mistralai-user"}, + {AICrawler, "Timpibot", "timpibot"}, + {AICrawler, "ImagesiftBot", "imagesiftbot"}, + {AICrawler, "Omgilibot", "omgili"}, + {AICrawler, "Kangaroo Bot", "kangaroo bot"}, + {AICrawler, "AI2Bot", "ai2bot"}, + {AICrawler, "Webzio", "webzio"}, + {AICrawler, "PanguBot", "pangubot"}, + {AICrawler, "SemrushBot-OCOB", "semrushbot-ocob"}, + + // User-triggered AI fetchers and agent / coding harnesses. + {AIAgent, "ChatGPT-User", "chatgpt-user"}, + {AIAgent, "Claude-User", "claude-user"}, + {AIAgent, "Perplexity-User", "perplexity-user"}, + {AIAgent, "Claude Code", "claude-code"}, + {AIAgent, "Claude CLI", "claude-cli"}, + {AIAgent, "Claude Agent SDK", "claude-agent-sdk"}, + {AIAgent, "Codex", "codex"}, + {AIAgent, "Cursor", "cursor/"}, + {AIAgent, "Windsurf", "windsurf"}, + {AIAgent, "opencode", "opencode"}, + {AIAgent, "aider", "aider/"}, + {AIAgent, "Cline", "cline/"}, + {AIAgent, "Continue", "continue/"}, + {AIAgent, "Gemini CLI", "gemini-cli"}, + {AIAgent, "Copilot", "copilot"}, + {AIAgent, "browser-use", "browser-use"}, + {AIAgent, "Playwright MCP", "playwright-mcp"}, + {AIAgent, "Manus", "manus-user"}, + {AIAgent, "LangChain", "langchain"}, + {AIAgent, "LlamaIndex", "llamaindex"}, + {AIAgent, "Anthropic SDK", "anthropic/"}, + {AIAgent, "OpenAI SDK", "openai/"}, + + // Search engines and link-preview fetchers. + {SearchCrawler, "Googlebot", "googlebot"}, + {SearchCrawler, "Google-InspectionTool", "google-inspectiontool"}, + {SearchCrawler, "AdsBot-Google", "adsbot-google"}, + {SearchCrawler, "Mediapartners-Google", "mediapartners-google"}, + {SearchCrawler, "Storebot-Google", "storebot-google"}, + {SearchCrawler, "bingbot", "bingbot"}, + {SearchCrawler, "BingPreview", "bingpreview"}, + {SearchCrawler, "DuckDuckBot", "duckduckbot"}, + {SearchCrawler, "YandexBot", "yandex"}, + {SearchCrawler, "Baiduspider", "baiduspider"}, + {SearchCrawler, "Slurp", "slurp"}, + {SearchCrawler, "Applebot", "applebot"}, + {SearchCrawler, "PetalBot", "petalbot"}, + {SearchCrawler, "Sogou", "sogou"}, + {SearchCrawler, "Qwantify", "qwantify"}, + {SearchCrawler, "SeznamBot", "seznambot"}, + {SearchCrawler, "Naver Yeti", "yeti/"}, + {SearchCrawler, "Exabot", "exabot"}, + {SearchCrawler, "Ecosia", "ecosiabot"}, + {SearchCrawler, "Kagi", "kagibot"}, + {SearchCrawler, "Brave Search", "bravebot"}, + {SearchCrawler, "facebookexternalhit", "facebookexternalhit"}, + {SearchCrawler, "Twitterbot", "twitterbot"}, + {SearchCrawler, "LinkedInBot", "linkedinbot"}, + {SearchCrawler, "Slackbot", "slackbot"}, + {SearchCrawler, "Discordbot", "discordbot"}, + {SearchCrawler, "TelegramBot", "telegrambot"}, + {SearchCrawler, "WhatsApp", "whatsapp/"}, + {SearchCrawler, "Pinterestbot", "pinterestbot"}, + {SearchCrawler, "redditbot", "redditbot"}, + {SearchCrawler, "Embedly", "embedly"}, + + // SEO and marketing crawlers. + {SEOCrawler, "AhrefsBot", "ahrefsbot"}, + {SEOCrawler, "AhrefsSiteAudit", "ahrefssiteaudit"}, + {SEOCrawler, "SemrushBot", "semrushbot"}, + {SEOCrawler, "MJ12bot", "mj12bot"}, + {SEOCrawler, "DotBot", "dotbot"}, + {SEOCrawler, "rogerbot", "rogerbot"}, + {SEOCrawler, "BLEXBot", "blexbot"}, + {SEOCrawler, "DataForSeoBot", "dataforseobot"}, + {SEOCrawler, "serpstatbot", "serpstatbot"}, + {SEOCrawler, "Screaming Frog", "screaming frog"}, + {SEOCrawler, "MegaIndex", "megaindex"}, + {SEOCrawler, "Barkrowler", "barkrowler"}, + {SEOCrawler, "SISTRIX", "sistrix"}, + {SEOCrawler, "Seekport", "seekport"}, + {SEOCrawler, "BuiltWith", "builtwith"}, + {SEOCrawler, "Netcraft", "netcraft"}, + {SEOCrawler, "Expanse", "expanse, a palo alto networks"}, + + // Uptime checks and cluster probes. + {Monitor, "Uptime-Kuma", "uptime-kuma"}, + {Monitor, "UptimeRobot", "uptimerobot"}, + {Monitor, "Pingdom", "pingdom"}, + {Monitor, "StatusCake", "statuscake"}, + {Monitor, "Better Uptime", "betteruptime"}, + {Monitor, "Site24x7", "site24x7"}, + {Monitor, "Datadog", "datadog"}, + {Monitor, "NewRelic", "newrelicpinger"}, + {Monitor, "kube-probe", "kube-probe"}, + {Monitor, "Prometheus", "prometheus/"}, + {Monitor, "Blackbox Exporter", "blackbox exporter"}, + {Monitor, "GoogleHC", "googlehc"}, + {Monitor, "ELB-HealthChecker", "elb-healthchecker"}, + {Monitor, "Trivy", "trivy/"}, + + // Scanners and exploit tooling. + {ScanTool, "CameraScanner", "camerascanner"}, + {ScanTool, "zgrab", "zgrab"}, + {ScanTool, "masscan", "masscan"}, + {ScanTool, "Nmap", "nmap scripting engine"}, + {ScanTool, "Nuclei", "nuclei"}, + {ScanTool, "sqlmap", "sqlmap"}, + {ScanTool, "Nikto", "nikto"}, + {ScanTool, "WPScan", "wpscan"}, + {ScanTool, "dirbuster", "dirbuster"}, + {ScanTool, "gobuster", "gobuster"}, + {ScanTool, "feroxbuster", "feroxbuster"}, + {ScanTool, "ffuf", "fuzz faster u fool"}, + {ScanTool, "Acunetix", "acunetix"}, + {ScanTool, "Nessus", "nessus"}, + {ScanTool, "OpenVAS", "openvas"}, + {ScanTool, "Burp", "burp"}, + {ScanTool, "Censys", "censys"}, + {ScanTool, "Shodan", "shodan"}, + {ScanTool, "Palo Alto Xpanse", "xpanse"}, + {ScanTool, "Internet Measurement", "internet-measurement"}, + {ScanTool, "ALittle Client", "alittle client"}, + {ScanTool, "l9explore", "l9explore"}, + {ScanTool, "LeakIX", "leakix"}, + {ScanTool, "NetSystemsResearch", "netsystemsresearch"}, + {ScanTool, "Hello, World", "hello, world"}, + + // Generic HTTP clients and CLIs. + {Library, "curl", "curl/"}, + {Library, "Wget", "wget/"}, + {Library, "python-requests", "python-requests"}, + {Library, "python-httpx", "python-httpx"}, + {Library, "aiohttp", "aiohttp"}, + {Library, "urllib", "python-urllib"}, + {Library, "Go-http-client", "go-http-client"}, + {Library, "Scrapy", "scrapy"}, + {Library, "Java", "java/"}, + {Library, "Apache-HttpClient", "apache-httpclient"}, + {Library, "okhttp", "okhttp"}, + {Library, "axios", "axios/"}, + {Library, "node-fetch", "node-fetch"}, + {Library, "undici", "undici"}, + {Library, "node", "node"}, + {Library, "libwww-perl", "libwww-perl"}, + {Library, "PostmanRuntime", "postmanruntime"}, + {Library, "insomnia", "insomnia/"}, + {Library, "HTTPie", "httpie"}, + {Library, "Ruby", "ruby"}, + {Library, "PHP", "php/"}, + {Library, "Dart", "dart:"}, + {Library, "Faraday", "faraday"}, + {Library, "Deno", "deno/"}, + {Library, "Bun", "bun/"}, +} + +// Classify returns the class of ua. +func Classify(ua string) Result { + ua = strings.TrimSpace(ua) + if ua == "" || ua == "-" { + return Result{Class: None} + } + l := strings.ToLower(ua) + for _, r := range rules { + if r.name == "node" || r.name == "Ruby" { + // Short tokens only count when they are the whole product name, + // not a substring of a browser UA. + if l == r.token || strings.HasPrefix(l, r.token+"/") || strings.HasPrefix(l, r.token+" ") { + return Result{Class: r.class, Name: r.name} + } + continue + } + if strings.Contains(l, r.token) { + return Result{Class: r.class, Name: r.name} + } + } + for _, t := range []string{"bot", "crawler", "spider", "scraper", "fetcher", "headlesschrome", "phantomjs"} { + if strings.Contains(l, t) { + return Result{Class: OtherBot} + } + } + if strings.HasPrefix(l, "mozilla/") || strings.HasPrefix(l, "opera/") { + return Result{Class: Browser} + } + return Result{Class: Unknown} +} + +// Of is Classify(ua).Class. +func Of(ua string) string { return Classify(ua).Class } + +// IsBot reports whether the class is a self-declared non-human client +// (everything except a browser, a missing UA or an unrecognised one). +func IsBot(class string) bool { + switch class { + case AICrawler, AIAgent, SearchCrawler, SEOCrawler, Monitor, ScanTool, OtherBot, Library: + return true + } + return false +} diff --git a/internal/uaclass/uaclass_test.go b/internal/uaclass/uaclass_test.go new file mode 100644 index 0000000..ffdf684 --- /dev/null +++ b/internal/uaclass/uaclass_test.go @@ -0,0 +1,37 @@ +package uaclass + +import "testing" + +func TestClassify(t *testing.T) { + cases := []struct{ ua, class, name string }{ + {"", None, ""}, + {"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)", AICrawler, "GPTBot"}, + {"Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)", AICrawler, "ClaudeBot"}, + {"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0", AIAgent, "ChatGPT-User"}, + {"claude-code/2.1.0 (external, cli)", AIAgent, "Claude Code"}, + {"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", SearchCrawler, "Googlebot"}, + {"Mozilla/5.0 (compatible; AhrefsBot/7.0)", SEOCrawler, "AhrefsBot"}, + {"Uptime-Kuma/1.23.11", Monitor, "Uptime-Kuma"}, + {"kube-probe/1.31", Monitor, "kube-probe"}, + {"CameraScannerV3/1.0", ScanTool, "CameraScanner"}, + {"Go-http-client/2.0", Library, "Go-http-client"}, + {"node", Library, "node"}, + {"curl/8.5.0", Library, "curl"}, + {"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36", Browser, ""}, + {"Mozilla/5.0 (compatible; SomeNewFooBot/3.1)", OtherBot, ""}, + {"Mozilla/5.0 (Macintosh) Nodeblock", Browser, ""}, // "node" must not match inside a browser UA + {"weird-thing", Unknown, ""}, + } + for _, c := range cases { + got := Classify(c.ua) + if got.Class != c.class || got.Name != c.name { + t.Errorf("Classify(%q) = %+v, want %s/%s", c.ua, got, c.class, c.name) + } + } +} + +func TestIsBot(t *testing.T) { + if !IsBot(AICrawler) || IsBot(Browser) || IsBot(None) || IsBot(Unknown) { + t.Fatal("IsBot mismatch") + } +} diff --git a/otel-ui/src/components/IssuesVisualizer.tsx b/otel-ui/src/components/IssuesVisualizer.tsx index 0c16c01..75cdee5 100644 --- a/otel-ui/src/components/IssuesVisualizer.tsx +++ b/otel-ui/src/components/IssuesVisualizer.tsx @@ -34,9 +34,11 @@ import { } from "../lib/issue-utils" import { useSubRoute } from "../lib/routes" import { CopyRef } from "./common/CopyRef" +import { IpButton } from "./common/IpButton" import { IssueOccurrences } from "./IssueOccurrences" import { NeedleTestModal } from "./NeedleTestModal" import { FilterBar, type FilterChip } from "./FilterBar" +import { UA_CLASSES, uaClassLabel } from "../lib/ua-class" import type { Issue, IssueSample } from "../lib/types" export const kindMeta: Record = { @@ -46,6 +48,12 @@ export const kindMeta: Record } type StatusFilter = "all" | "open" | "resolved" | "ignored" +const without = (set: Set, value: string) => { + const next = new Set(set) + next.delete(value) + return next +} + type KindFilter = "all" | "span_error" | "log_error" | "frontend_error" interface HoneypotIP { @@ -438,6 +446,9 @@ const IssueCard = ({ {(contextSample?.userAgent || latestSample?.userAgent)?.slice(0, 80)} )} + {(contextSample?.sourceIp || latestSample?.sourceIp) && ( + + )} {(contextSample?.release || latestSample?.release) && ( release: {contextSample?.release || latestSample?.release} )} @@ -666,6 +677,11 @@ export const IssuesVisualizer = ({ const [detectorTest, setDetectorTest] = useState(null) const [detectorTestLoading, setDetectorTestLoading] = useState(false) const [excludeSpamIPs, setExcludeSpamIPs] = useState(false) + // Client-class filters: an issue matches "client" when any sample came from + // one of these classes, and is hidden by "not client" when every classified + // sample did (a mixed human + bot issue stays visible). + const [clientInclude, setClientInclude] = useState>(new Set()) + const [clientExclude, setClientExclude] = useState>(new Set()) const [showSpamIPs, setShowSpamIPs] = useState(false) const [spamIPs, setSpamIPs] = useState([]) const [spamIPsLoading, setSpamIPsLoading] = useState(false) @@ -710,6 +726,11 @@ export const IssuesVisualizer = ({ ) { return false } + if (clientInclude.size > 0 || clientExclude.size > 0) { + const classes = issue.samples.map((s) => s.uaClass).filter((c): c is string => Boolean(c)) + if (clientInclude.size > 0 && !classes.some((c) => clientInclude.has(c))) return false + if (clientExclude.size > 0 && classes.length > 0 && classes.every((c) => clientExclude.has(c))) return false + } if (selectedServices.size > 0) { const matches = selectedServices.has(issue.service) || @@ -724,7 +745,7 @@ export const IssuesVisualizer = ({ } return true }) - }, [issues, selectedServices, searchQuery, statusFilter, kindFilter, excludeSpamIPs, spamIPSet]) + }, [issues, selectedServices, searchQuery, statusFilter, kindFilter, excludeSpamIPs, spamIPSet, clientInclude, clientExclude]) const { primaryIssues, secondaryIssues } = useMemo(() => { const sorted = [...filteredIssues].sort((a, b) => b.count - a.count) @@ -753,7 +774,12 @@ export const IssuesVisualizer = ({ } const activeFilterCount = - selectedServices.size + (statusFilter !== "all" ? 1 : 0) + (excludeSpamIPs ? 1 : 0) + (searchQuery ? 1 : 0) + selectedServices.size + + clientInclude.size + + clientExclude.size + + (statusFilter !== "all" ? 1 : 0) + + (excludeSpamIPs ? 1 : 0) + + (searchQuery ? 1 : 0) const chips: FilterChip[] = [ ...Array.from(selectedServices).map((service) => ({ @@ -763,6 +789,20 @@ export const IssuesVisualizer = ({ tone: "neutral" as const, onRemove: () => toggleService(service), })), + ...Array.from(clientInclude).map((c) => ({ + id: `client-${c}`, + filterKey: "client", + value: uaClassLabel(c), + tone: "neutral" as const, + onRemove: () => setClientInclude((prev) => without(prev, c)), + })), + ...Array.from(clientExclude).map((c) => ({ + id: `not-client-${c}`, + filterKey: "not client", + value: uaClassLabel(c), + tone: "neutral" as const, + onRemove: () => setClientExclude((prev) => without(prev, c)), + })), ...(statusFilter !== "all" ? [ { @@ -798,15 +838,39 @@ export const IssuesVisualizer = ({ : []), ] - const addFilterOptions = uniqueServices - .filter((s) => !selectedServices.has(s)) - .map((s) => ({ value: s, label: s })) + const addFilterOptions = [ + ...uniqueServices.filter((s) => !selectedServices.has(s)).map((s) => ({ value: s, label: s })), + ...UA_CLASSES.filter((c) => !clientInclude.has(c.value)).map((c) => ({ + value: `client:${c.value}`, + label: `client: ${c.label}`, + })), + ...UA_CLASSES.filter((c) => !clientExclude.has(c.value)).map((c) => ({ + value: `not-client:${c.value}`, + label: `hide client: ${c.label}`, + })), + ] + + const addFilter = (value: string) => { + if (value.startsWith("client:")) { + const c = value.slice("client:".length) + setClientInclude((prev) => new Set(prev).add(c)) + setClientExclude((prev) => without(prev, c)) + } else if (value.startsWith("not-client:")) { + const c = value.slice("not-client:".length) + setClientExclude((prev) => new Set(prev).add(c)) + setClientInclude((prev) => without(prev, c)) + } else { + toggleService(value) + } + } const clearAll = () => { resetFilters() setSearchQuery("") setStatusFilter("all") setExcludeSpamIPs(false) + setClientInclude(new Set()) + setClientExclude(new Set()) } if (loading && issues.length === 0) { @@ -852,7 +916,7 @@ export const IssuesVisualizer = ({ }} chips={chips} addFilterOptions={addFilterOptions} - onAddFilter={(v) => toggleService(v)} + onAddFilter={addFilter} matchText={`Matching ${filteredIssues.length} of ${issues.length} issues · ${activeFilterCount} filter${activeFilterCount === 1 ? "" : "s"}`} onClearAll={activeFilterCount > 0 ? clearAll : undefined} /> diff --git a/otel-ui/src/components/common/IpButton.tsx b/otel-ui/src/components/common/IpButton.tsx new file mode 100644 index 0000000..7c4bf70 --- /dev/null +++ b/otel-ui/src/components/common/IpButton.tsx @@ -0,0 +1,137 @@ +import { useEffect, useRef, useState } from "react" +import { Globe, X } from "lucide-react" +import { authFetch } from "../../lib/auth" +import { DetailField } from "./DetailField" + +interface IpInfo { + ip: string + configured: boolean + scope?: string + country?: string + countryCode?: string + inEu?: boolean + region?: string + city?: string + postal?: string + timezone?: string + latitude?: number + longitude?: number + network?: string + asn?: number + asOrg?: string +} + +type State = { status: "loading" } | { status: "error"; message: string } | { status: "ok"; info: IpInfo } + +const IpDialog = ({ ip, onClose }: { ip: string; onClose: () => void }) => { + const ref = useRef(null) + const [state, setState] = useState({ status: "loading" }) + + useEffect(() => { + const dialog = ref.current + if (dialog && !dialog.open) dialog.showModal() + }, []) + + useEffect(() => { + let cancelled = false + authFetch(`/api/ip/${encodeURIComponent(ip)}`) + .then(async (resp) => { + if (!resp.ok) throw new Error(`HTTP ${resp.status}`) + return (await resp.json()) as IpInfo + }) + .then((info) => !cancelled && setState({ status: "ok", info })) + .catch((e: unknown) => !cancelled && setState({ status: "error", message: e instanceof Error ? e.message : String(e) })) + return () => { + cancelled = true + } + }, [ip]) + + const info = state.status === "ok" ? state.info : null + const place = info ? [info.city, info.region, info.country].filter(Boolean).join(", ") : "" + const hasGeo = Boolean(place || info?.asn || info?.network) + + return ( + e.target === ref.current && ref.current?.close()} + aria-labelledby="ip-dialog-title" + className="m-auto w-[min(92vw,460px)] rounded-[14px] border border-ds-line bg-ds-surface p-0 text-ds-text backdrop:bg-ds-bg/70" + > +
+

+ {ip} +

+ +
+
+ {state.status === "loading" &&

Looking up…

} + {state.status === "error" &&

Lookup failed: {state.message}

} + {info && !info.configured && ( +

+ No MaxMind database is configured. Set OTEL_GEOIP_CITY_DB{" "} + and/or OTEL_GEOIP_ASN_DB to the paths of GeoLite2 + .mmdb files. +

+ )} + {info?.scope &&

This is a {info.scope} address, so it has no public location.

} + {info?.configured && !info.scope && !hasGeo &&

The database has no record for this address.

} + {info && hasGeo && ( +
+ + + + + + + + +
+ )} + {hasGeo && ( +

Approximate, from the local MaxMind database. Not precise enough to identify a person.

+ )} +
+ + ) +} + +/** An IP address that opens a details dialog (geo + ASN from MaxMind) on click. */ +export const IpButton = ({ ip, prefix }: { ip: string; prefix?: string }) => { + const [open, setOpen] = useState(false) + return ( + <> + + {open && setOpen(false)} />} + + ) +} diff --git a/otel-ui/src/components/issues/OccurrenceDetail.tsx b/otel-ui/src/components/issues/OccurrenceDetail.tsx index b176294..8a6b189 100644 --- a/otel-ui/src/components/issues/OccurrenceDetail.tsx +++ b/otel-ui/src/components/issues/OccurrenceDetail.tsx @@ -1,8 +1,10 @@ import { ExternalLink } from "lucide-react" import { DetailField } from "../common/DetailField" +import { IpButton } from "../common/IpButton" import { formatRelativeTime } from "../../lib/timeUtils" import { parseBreadcrumbs } from "../../lib/issue-utils" import type { IssueSample } from "../../lib/types" +import { uaClassLabel } from "../../lib/ua-class" export const OccurrenceDetail = ({ sample }: { sample: IssueSample }) => { const stack = sample.resolvedStacktrace || sample.stacktrace @@ -21,8 +23,16 @@ export const OccurrenceDetail = ({ sample }: { sample: IssueSample }) => { - + {sample.sourceIp && ( +
+
Source IP
+
+ +
+
+ )} + {sample.url && ( diff --git a/otel-ui/src/lib/routes.ts b/otel-ui/src/lib/routes.ts index 264780a..64c48fd 100644 --- a/otel-ui/src/lib/routes.ts +++ b/otel-ui/src/lib/routes.ts @@ -57,7 +57,10 @@ export function useSubRoute(): { go: (segments: string[], options?: { replace?: boolean }) => void } { const [location, navigate] = useLocation() - const { base, rest } = splitPath(location) + const split = splitPath(location) + // "/" is the default (issues) tab; building "//" from it would be parsed as a host. + const base = split.base === "/" ? "/issues" : split.base + const rest = split.rest const key = rest.join("/") // Stable identity per URL so effects can depend on `segments` directly. // eslint-disable-next-line react-hooks/exhaustive-deps diff --git a/otel-ui/src/lib/types.ts b/otel-ui/src/lib/types.ts index 1095d15..9f2ae9e 100644 --- a/otel-ui/src/lib/types.ts +++ b/otel-ui/src/lib/types.ts @@ -178,6 +178,8 @@ export interface IssueSample { httpRoute?: string; httpStatus?: number; userAgent?: string; + /** Client class derived from userAgent by the backend (see lib/ua-class.ts). */ + uaClass?: string; sourceIp?: string; release?: string; environment?: string; diff --git a/otel-ui/src/lib/ua-class.ts b/otel-ui/src/lib/ua-class.ts new file mode 100644 index 0000000..9bd20df --- /dev/null +++ b/otel-ui/src/lib/ua-class.ts @@ -0,0 +1,19 @@ +// Client classes derived server-side from the User-Agent (internal/uaclass in +// the Go backend owns the crawler list; this is only the display side). +// A UA is self-declared, so a class means "says it is", not "is". +export const UA_CLASSES: { value: string; label: string }[] = [ + { value: "ai_crawler", label: "AI crawler" }, + { value: "ai_agent", label: "AI agent" }, + { value: "search_crawler", label: "Search crawler" }, + { value: "seo_crawler", label: "SEO crawler" }, + { value: "monitor", label: "Monitor" }, + { value: "scan_tool", label: "Scan tool" }, + { value: "other_bot", label: "Other bot" }, + { value: "library", label: "HTTP library" }, + { value: "browser", label: "Browser" }, + { value: "unknown", label: "Unknown UA" }, + { value: "none", label: "No UA" }, +] + +export const uaClassLabel = (value?: string): string => + UA_CLASSES.find((c) => c.value === value)?.label ?? value ?? ""