more otel

.gitignore

@@ -2,6 +2,7 @@
 .DS_Store
 otel-agent.log
 ./semgrep-results.json
-# Separate frontend repo, checked out locally for the Dockerfile's UI build
+# Frontend dependencies and generated build output.
-# stage; not nested into this repo's history.
+/otel-ui/node_modules/
-/otel-ui/
+/otel-ui/dist/
+/otel-ui/dist-ssr/

Dockerfile

@@ -1,8 +1,14 @@
+ARG APP_RELEASE=dev
+ARG PUBLIC_SOURCEMAPS=false
+
 # UI build stage
 ARG TARGETOS
 ARG TARGETARCH
 FROM node:alpine AS ui-builder
 
+ARG APP_RELEASE
+ENV VITE_APP_RELEASE=${APP_RELEASE}
+
 WORKDIR /app
 
 COPY otel-ui/ .
@@ -28,6 +34,8 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -installsuff
 # Final stage
 FROM alpine:latest
 
+ARG PUBLIC_SOURCEMAPS
+
 RUN apk --no-cache add ca-certificates
 
 # Create non-root user
@@ -39,6 +47,11 @@ COPY --from=go-builder /go-otel .
 COPY --from=go-builder /app/otel-ui/dist ./dist
 COPY static/ ./static
 COPY agents_example.md .
+COPY instrumentation.md .
+
+# Private server-side maps are the default. Staging/dev can opt into public
+# maps for Chrome DevTools with --build-arg PUBLIC_SOURCEMAPS=true.
+RUN if [ "${PUBLIC_SOURCEMAPS}" != "true" ]; then find ./dist -type f -name '*.map' -delete; fi
 
 # Ensure the appuser owns the files
 RUN chown -R appuser:appgroup /home/appuser/

README.md

@@ -103,8 +103,26 @@ gRPC services for traces, metrics, and logs using standard OTLP protobuf definit
 - `GET /api/traces/{traceId}` - Retrieve all stored entries (spans and
   correlated logs) referencing a given trace id.
 
-All endpoints above search the full persistent store (when `OTELLOG_PATH`
+All endpoints above search the full persistent store, not just the
-storage is enabled), not just the in-memory ring buffer.
+in-memory ring buffer.
+
+### Stats API Endpoints (Port 8080, requires ClickHouse)
+
+Aggregate time-series over the stored telemetry, computed server-side from
+an hourly rollup table. Available only when `OTEL_CLICKHOUSE_ADDR` is
+configured (see Configuration below); otherwise they respond `501` with an
+explanatory JSON body and the UI hides its "Trends" tab.
+
+- `GET /api/stats/errors?hours=72&service=` - Total/error entry counts per
+  hour (per day when `hours > 336`). Response:
+  `{"granularity":"hour","points":[{"bucket":"2026-09-17 14:00:00","total":42,"errors":3}]}`.
+- `GET /api/stats/releases?hours=72&service=` - Per-release error-count
+  series: `{"series":{"v1.3":[{bucket,total,errors}...],"v1.4":[...]}}`
+  for release-over-release comparison. Entries need a `service.release`
+  attribute to appear here.
+- `GET /api/stats/services` - Distinct services seen in the rollup.
+
+All accept `hours` (1..2160, default 72) and an optional `service` filter.
 
 ### Issues API Endpoints (Port 8080)
 
@@ -115,6 +133,20 @@ Errors seen in traces (spans with an error status) and logs (severity
   Supports `limit`.
 - `GET /api/issues/{id}` - Fetch a single issue, including up to 5 recent
   sample occurrences (timestamp, trace/span id, message).
+- `GET /api/patterns` - List deterministic groups of related exact issues.
+  Pattern grouping is explainable and does not merge issue IDs.
+- `GET /api/issues/{id}/similar` - Return bounded, deterministic related-issue
+  suggestions with a score and matching reasons.
+- `POST /api/issues/{id}/similar` - Record a review label with
+  `{"issueId":"...","decision":"same|different"}` for future training data.
+- `GET /api/rollouts` - List read-only Kubernetes Deployment rollout state.
+- `GET /api/rollouts/{id}` - Fetch one normalized rollout.
+- `GET /api/rollouts/{id}/impact` - Return optional Prometheus error-rate points
+  around the rollout.
+- `POST /api/sourcemaps` - Upload a JavaScript source map. Public multipart upload;
+  the release build script sends the source-map file contents directly.
+  multipart fields are `release`, `source` (the generated bundle URL/path),
+  and `file`. Uploaded maps are used server-side to resolve frontend stacks.
 
 ### Web Interface
 
@@ -127,6 +159,9 @@ Errors seen in traces (spans with an error status) and logs (severity
 - `GET /agents_example.md` - Serves this repo's agent instructions doc
   directly, so an AI agent can fetch it from the base URL before making any
   API calls.
+- `GET /instrumentation.md` - Serves the frontend/backend instrumentation
+  guide, including React error boundaries and source-map deployment.
+- `GET /static/instrumentation.md` - Static/public alias for the same guide.
 
 ## Usage
 
@@ -175,7 +210,12 @@ through the same history/search/issue pipeline as backend telemetry, tagged
 ```html
 <script src="https://otel.k6n.net/static/otel-rum.js"></script>
 <script>
-  var rum = OtelRUM.init({ serviceName: "my-web-app" });
+  var rum = OtelRUM.init({
+    serviceName: "my-web-app",
+    release: "2026.09.18-abc123",
+    // Optional; defaults to https://otel.k6n.net.
+    baseUrl: "https://otel.example.com",
+  });
 
   // Inject a W3C traceparent header into any HTTP client so a backend span
   // (if the service is OTel-instrumented) shares this request's trace id -
@@ -188,23 +228,128 @@ through the same history/search/issue pipeline as backend telemetry, tagged
 </script>
 ```
 
+React render failures should be reported from an error boundary as well. The
+project UI uses this pattern in `otel-ui/src/components/ViewErrorBoundary.tsx`:
+
+```tsx
+class AppBoundary extends React.Component<{ children: React.ReactNode }> {
+  componentDidCatch(error: Error, info: React.ErrorInfo) {
+    OtelRUM.last?.captureException(error, {
+      "exception.type": "ReactErrorBoundary",
+      "react.component_stack": info.componentStack,
+    });
+  }
+
+  render() { return this.props.children; }
+}
+```
+
+The boundary should also render a fallback UI; otherwise a broken component
+can leave the whole application blank while still being logged.
+
 `fetch` is auto-instrumented by default (same-origin requests only) to
 inject `traceparent` and record request/response breadcrumbs; this can be
 disabled via `injectTraceHeaders: false` / `autoInstrumentFetch: false`.
-Out of scope for v1: source map resolution (stack traces stay minified),
+The log ingest endpoint supports unauthenticated cross-origin POSTs; no OIDC
-session replay, PII scrubbing - don't route sensitive data through
+session or `Authorization` header is needed.
-breadcrumbs or attributes.
+Source maps are kept server-side. When a captured frontend error has a
+matching `service.release` and generated bundle path, the Issues view displays
+the resolved original source frame while retaining the generated frame.
+
+#### Source-map deploy workflow
+
+Build the UI with a stable release identifier and keep the generated map next
+to the bundle. Do not publish the map publicly:
+
+```sh
+export RELEASE="2026.09.18-$(git rev-parse --short HEAD)"
+npm --prefix otel-ui run build
+
+# Upload after the new UI is deployed. Use the generated URL/path exactly as
+# it appears in browser stack traces.
+curl -fS -X POST https://otel.example.com/api/sourcemaps \
+  -F "release=$RELEASE" \
+  -F "source=/assets/index-ABC123.js" \
+  -F "file=@otel-ui/dist/assets/index-ABC123.js.map"
+```
+
+For this project, `otel-ui/src/main.tsx` sends `VITE_APP_RELEASE` as the RUM
+`service.release` value; set it during the build, for example:
+
+```sh
+VITE_APP_RELEASE="$RELEASE" npm --prefix otel-ui run build
+```
+
+The repository also includes `scripts/upload-sourcemap.sh`; after deploying,
+run it against the emitted bundle, for example:
+
+```sh
+RELEASE="$RELEASE" OTEL_URL="https://otel.example.com" \
+  ./scripts/upload-sourcemap.sh otel-ui/dist/assets/index-ABC123.js
+```
+
+The receiver stores uploaded maps under `OTEL_SOURCEMAP_DIR` (default
+`./sourcemaps`, `/sourcemaps` in the Kubernetes deployment). Upload maps for
+each release before investigating its errors. The API never returns map
+contents or original source files.
+
+`build.sh` passes the release into the Docker UI build. If `OTEL_URL` is set,
+it also builds the matching UI locally and uploads all generated JavaScript
+maps. The Dockerfile removes `.map` files from the
+final image by default; set `PUBLIC_SOURCEMAPS=true` for a development or
+staging image where Chrome DevTools should fetch the original sources:
+
+```sh
+PUBLIC_SOURCEMAPS=true ./build.sh
+```
+
+Public maps can expose application source code, so keep this disabled for
+production unless that exposure is intentional. Credentials are never passed
+to Docker or stored in the image.
 
 ## Configuration
 
 - `OTELLOG_PATH` - path to the persistent telemetry log file (default
-  `otel-agent.log`).
+  `otel-agent.log`). Only used when ClickHouse is not configured.
+- `OTEL_SOURCEMAP_DIR` - directory for uploaded release source maps (default
+  `./sourcemaps`; mount persistent storage there in production).
+- `OTEL_KUBERNETES_ENABLED` - set to `false` to disable rollout observation;
+  otherwise it activates in-cluster or `KUBECONFIG` observation when available.
+- `OTEL_KUBERNETES_NAMESPACE` - optional namespace restriction; empty watches
+  all namespaces permitted by the service account.
+- `OTEL_KUBERNETES_LABEL_SELECTOR` - optional Deployment label selector.
+- `OTEL_PROMETHEUS_URL` - optional Prometheus base URL for rollout impact.
+- `OTEL_PROMETHEUS_ERROR_QUERY` - optional `fmt.Sprintf`-style PromQL template;
+  `%q` placeholders receive service and release in that order.
+- `OTEL_CLICKHOUSE_ADDR` - optional `host:port` (native protocol, usually
+  `9000`). When set, telemetry is stored in ClickHouse instead of the JSONL
+  file: history search stays fast and unbounded, and the aggregate stats
+  endpoints (`/api/stats/*`) and the UI "Trends" tab are enabled. The
+  schema (`otel` database, raw `entries` table + materialized hourly
+  rollup) is created automatically on first connect with a 90-day TTL; if
+  ClickHouse is unreachable at startup the server falls back to the file
+  store.
+- `OTEL_CLICKHOUSE_DB` / `OTEL_CLICKHOUSE_USER` /
+  `OTEL_CLICKHOUSE_PASSWORD` - optional ClickHouse connection settings
+  (defaults: `otel` / `default` / empty).
 - `OTEL_INGEST_KEY` - optional shared key required (via `?key=` or the
   `X-Otel-Ingest-Key` header) on `/v1/traces`, `/v1/metrics` and `/v1/logs`
   over HTTP. Unset by default (no key required). This is meant to
   scope/rate-limit a public ingest endpoint fed by a browser script, not to
   provide real authentication - pair it with the built-in per-IP rate
   limiting on those endpoints if the endpoint is public.
+- `OTEL_AGENT_TOKEN` - optional service token for non-interactive debugging
+  agents. Send it as `X-Otel-Agent-Token` or `Authorization: Bearer ...` when
+  calling protected `/api/history`, `/api/issues`, `/api/traces`, and stats
+  endpoints. Keep it in a Kubernetes Secret and never expose it to browsers.
+
+A single-node ClickHouse instance for Kubernetes is provided in
+`k8s/clickhouse.yaml` (StatefulSet + Service + PVC, deployed in the
+`monitoring` namespace); the receiver Deployment in `k8s/deployment.yaml`
+already points at it. The UI's Trends tab (and live issue updates) appear
+automatically once the stats endpoints answer; on file-store deployments the
+tab hides itself. SPA routes (/issues, /traces, /logs, /metrics, /trends,
+/setup) are served client-side by wouter and survive the OIDC login redirect.
 
 Currently, the history buffer size is hardcoded to 100 entries. This can be made configurable in future versions.
 

agents_example.md

@@ -4,6 +4,8 @@ This document provides instructions for AI agents on how to utilize the OTEL Col
 
 ## Base Configuration
 - **Base URL**: `https://otel.k6n.net`
+- **Authentication**: set the service token in `OTEL_AGENT_TOKEN` and send it
+  as `X-Otel-Agent-Token: $OTEL_AGENT_TOKEN` on every protected debug request.
 - **Endpoints**:
     - `GET /api/history/search?q={query}&type={type}&since={ns}&until={ns}&limit={limit}`: Search all historical telemetry data.
         - **Default Limit**: 50 (max 100 recommended for context).
@@ -16,8 +18,11 @@ This document provides instructions for AI agents on how to utilize the OTEL Col
         - **Ideal for Automation**: Use this after triggering an action to wait for the exact result.
         - **Rate limited**: capped per client IP; back off if you receive `429`.
     - `GET /api/traces/{traceId}`: Fetch every stored entry (spans and correlated logs) referencing a specific trace id in one call. Prefer this over substring-searching for a trace id.
-    - `GET /api/issues?limit={limit}`: List deduplicated error issues (from error-status spans and error/fatal-severity logs), most recently seen first. Use this to triage what's currently broken before diving into raw search.
+    - `GET /api/issues`: List deduplicated error issues (from error-status spans and error/fatal-severity logs), most recently seen first. Supported filters are `limit` (default 100, max 200), `cursor`, `service`, `kind`, `status`, `q` (title substring), `sourceIp`, `route`, `since`, and `until` (the time bounds are unix nanoseconds). Results remain a JSON array for compatibility; when another page exists, use the `X-Next-Cursor` response header or the `Link: ...; rel=next` header in the next request. The cursor is opaque and must be reused unchanged. Use this to triage what's currently broken before diving into raw search.
-    - `GET /api/issues/{id}`: Fetch a single issue's detail, including sample occurrences with trace ids you can feed into `/api/traces/{traceId}`.
+    - `GET /api/issues/{id}`: Fetch a single issue's detail, including sample occurrences with trace ids you can feed into `/api/traces/{traceId}`. `PATCH` the same path with `{"status":"resolved"|"ignored"|"open"}` to update triage state (a resolved issue that fires again is auto-reopened).
+    - `GET /api/stats/errors?hours=72&service=`: Hourly (daily when `hours > 336`) total/error entry counts, ClickHouse-backed. Responds `501` when the server runs on file storage. Use for "when did errors spike?" and trend questions.
+    - `GET /api/stats/releases?hours=72&service=`: Per-release error-count series (`{"series":{"v1.3":[...],"v1.4":[...]}}`) to compare error rates before/after a deploy. Only entries carrying a `service.release` attribute appear.
+    - `GET /api/stats/services`: Distinct services present in the stats rollup.
     - `GET /api/history`: Fetch recent history of all telemetry (searches the full persistent store, not just recent in-memory entries).
     - `GET /api/history/traces`: Fetch recent traces.
     - `GET /api/history/logs`: Fetch recent logs.
@@ -31,7 +36,8 @@ When debugging an issue, start by searching for relevant keywords in the logs an
 **Example Request:**
 ```bash
 # Search for recent "ERROR" occurrences (default limit 50, latest first)
-curl "https://otel.k6n.net/api/history/search?q=error"
+curl "https://otel.k6n.net/api/history/search?q=error" \
+  -H "X-Otel-Agent-Token: $OTEL_AGENT_TOKEN"
 ```
 
 ### 2. Waiting for a Signal
@@ -40,7 +46,8 @@ Use the `wait` endpoint to synchronize your debugging flow. This is perfect for
 **Example Request:**
 ```bash
 # Wait up to 60s for a specific traceId to show up in the collector
-curl "https://otel.k6n.net/api/history/wait?q=5682138e4a9b4629852899d45e542456&timeout=60"
+curl "https://otel.k6n.net/api/history/wait?q=5682138e4a9b4629852899d45e542456&timeout=60" \
+  -H "X-Otel-Agent-Token: $OTEL_AGENT_TOKEN"
 ```
 
 ### 3. Identifying Trace Chains
@@ -49,7 +56,8 @@ If you find a log entry with a `traceId`, use the dedicated trace endpoint to fe
 **Example Request:**
 ```bash
 # Fetch all activity related to a specific trace
-curl "https://otel.k6n.net/api/traces/5682138e4a9b4629852899d45e542456"
+curl "https://otel.k6n.net/api/traces/5682138e4a9b4629852899d45e542456" \
+  -H "X-Otel-Agent-Token: $OTEL_AGENT_TOKEN"
 ```
 
 ### 3. Fetching Recent Telemetry
@@ -57,7 +65,8 @@ To see the most recent activity without specific keywords, use the search endpoi
 
 **Example Request:**
 ```bash
-curl "https://otel.k6n.net/api/history/search?limit=20"
+curl "https://otel.k6n.net/api/history/search?limit=20" \
+  -H "X-Otel-Agent-Token: $OTEL_AGENT_TOKEN"
 ```
 
 ### 4. Data Format

build.sh

@@ -1,8 +1,30 @@
 #!/bin/bash
+set -euo pipefail
+
+RELEASE="${RELEASE:-$(git rev-parse --short HEAD)}"
+PUBLIC_SOURCEMAPS="${PUBLIC_SOURCEMAPS:-false}"
+
+docker build \
+  --build-arg "APP_RELEASE=${RELEASE}" \
+  --build-arg "PUBLIC_SOURCEMAPS=${PUBLIC_SOURCEMAPS}" \
+  -t matst80/otel-debug:latest .
 
-docker build -t matst80/otel-debug:latest .
 docker push matst80/otel-debug:latest
-
 kubectl rollout restart deployment otel-debug-receiver -n monitoring
+kubectl rollout status deployment/otel-debug-receiver -n monitoring
 
-kubectl logs -f deployment/otel-debug-receiver -n monitoring
+# Docker builds intentionally omit public source maps. If the public receiver
+# URL is present, build the same release locally and upload maps to the running
+# receiver. The Kubernetes source-map volume is currently
+# emptyDir, so upload after the rollout rather than before it.
+if [[ -n "${OTEL_URL:-}" ]]; then
+  VITE_APP_RELEASE="$RELEASE" npm --prefix otel-ui run build
+  while IFS= read -r -d '' map; do
+    bundle="${map%.map}"
+    RELEASE="$RELEASE" OTEL_URL="$OTEL_URL" \
+      ./scripts/upload-sourcemap.sh "$bundle"
+  done < <(find otel-ui/dist -type f -name '*.js.map' -print0)
+  find otel-ui/dist -type f -name '*.map' -delete
+fi
+
+kubectl logs -f deployment/otel-debug-receiver -n monitoring

cmd/server/main.go

@@ -9,7 +9,12 @@ import (
 	"os"
 	"strings"
 
+	"git.tornberg.me/go-otel/internal/auth"
 	"git.tornberg.me/go-otel/internal/collector"
+	"git.tornberg.me/go-otel/internal/honeypot"
+	kube "git.tornberg.me/go-otel/internal/kubernetes"
+	"git.tornberg.me/go-otel/internal/prometheus"
+	"git.tornberg.me/go-otel/internal/sourcemaps"
 	"git.tornberg.me/go-otel/internal/storage"
 	"git.tornberg.me/go-otel/internal/websocket"
 	"go.opentelemetry.io/collector/pdata/plog"
@@ -150,14 +155,20 @@ func (s *logsService) Export(ctx context.Context, req *logsgrpc.ExportLogsServic
 }
 
 // withCORS allows cross-origin ingestion (e.g. from the otel-rum.js browser
-// script running on a third-party site) into the OTLP HTTP endpoints. It's
+// script running on a third-party site like event-agent.k6n.net) into the
-// deliberately permissive on origin, since this is a public ingest endpoint
+// OTLP HTTP endpoints. It is deliberately permissive on origin and headers,
 // scoped by the ingest key/rate limiting instead of same-origin policy.
 func withCORS(next http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Access-Control-Allow-Origin", "*")
-		w.Header().Set("Access-Control-Allow-Methods", "POST, OPTIONS")
+		w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS")
-		w.Header().Set("Access-Control-Allow-Headers", "Content-Type, X-Otel-Ingest-Key")
+		reqHeaders := r.Header.Get("Access-Control-Request-Headers")
+		if reqHeaders != "" {
+			w.Header().Set("Access-Control-Allow-Headers", reqHeaders)
+		} else {
+			w.Header().Set("Access-Control-Allow-Headers", "Content-Type, X-Otel-Ingest-Key, Authorization, Baggage, Traceparent, Tracestate, X-Requested-With")
+		}
+		w.Header().Set("Access-Control-Max-Age", "86400")
 		if r.Method == http.MethodOptions {
 			w.WriteHeader(http.StatusNoContent)
 			return
@@ -166,20 +177,69 @@ func withCORS(next http.HandlerFunc) http.HandlerFunc {
 	}
 }
 
+// setupStore returns the telemetry store: ClickHouse when OTEL_CLICKHOUSE_ADDR
+// is set (host:port, native protocol), otherwise the JSONL file store.
+func setupStore(ctx context.Context) storage.Store {
+	addr := os.Getenv("OTEL_CLICKHOUSE_ADDR")
+	if addr == "" {
+		logPath := os.Getenv("OTELLOG_PATH")
+		if logPath == "" {
+			logPath = "otel-agent.log"
+		}
+		log.Printf("storage: file store at %s (set OTEL_CLICKHOUSE_ADDR for ClickHouse)", logPath)
+		return storage.NewFileStore(logPath)
+	}
+
+	chStore, err := storage.NewClickHouseStore(ctx, storage.ClickHouseConfig{
+		Addr:     addr,
+		Database: os.Getenv("OTEL_CLICKHOUSE_DB"),
+		Username: os.Getenv("OTEL_CLICKHOUSE_USER"),
+		Password: os.Getenv("OTEL_CLICKHOUSE_PASSWORD"),
+	})
+	if err != nil {
+		log.Printf("storage: ClickHouse unavailable (%v), falling back to file store", err)
+		logPath := os.Getenv("OTELLOG_PATH")
+		if logPath == "" {
+			logPath = "otel-agent.log"
+		}
+		return storage.NewFileStore(logPath)
+	}
+	log.Printf("storage: ClickHouse at %s", addr)
+	return chStore
+}
+
 func main() {
 
 	// Configure history size (default 100, can be made configurable)
 	historySize := 100
-	logPath := os.Getenv("OTELLOG_PATH")
+	store := setupStore(context.Background())
-	if logPath == "" {
-		logPath = "otel-agent.log"
-	}
-	store := storage.NewFileStore(logPath)
 	coll := collector.New(nil, historySize, store) // Temporary nil for wsServer
+	kubeWatcher, kubeErr := kube.NewFromEnv(context.Background())
+	if kubeErr != nil {
+		log.Printf("kubernetes: rollout watcher disabled: %v", kubeErr)
+	} else if kubeWatcher != nil {
+		coll.SetKubernetesWatcher(kubeWatcher)
+		log.Printf("kubernetes: rollout watcher enabled")
+	}
+	coll.SetPrometheusClient(prometheus.NewFromEnv())
+	sourceMaps, err := sourcemaps.New(os.Getenv("OTEL_SOURCEMAP_DIR"))
+	if err != nil {
+		log.Fatalf("failed to initialize source maps: %v", err)
+	}
+	coll.SetSourceMaps(sourceMaps)
+	// Aggregate stats endpoints are enabled only for stores that support
+	// them (ClickHouse); they 501 politely on the file store.
+	if sq, ok := store.(storage.StatsQuerier); ok {
+		coll.SetStatsQuerier(sq)
+	}
 	wsServer := websocket.New(coll)
+	honeypotTracker := honeypot.New(honeypot.ConfigFromEnv())
 	// Update collector with wsServer
 	coll.SetBroadcaster(wsServer)
-	coll.SetIngestKey(os.Getenv("OTEL_INGEST_KEY"))
+	ingestKey := os.Getenv("OTEL_INGEST_KEY")
+	coll.SetIngestKey(ingestKey)
+	oidcAuth := auth.NewOidcAuth(ingestKey)
+	log.Printf("auth: OIDC issuer=%s client=%s disabled=%v", oidcAuth.Issuer(), oidcAuth.ClientID(), oidcAuth.IsDisabled())
 
 	// Start gRPC server
 	go func() {
@@ -205,8 +265,11 @@ func main() {
 	// OTLP HTTP mux (standard OTel port 4318)
 	otelMux := http.NewServeMux()
 	otelMux.HandleFunc("/v1/traces", withCORS(coll.HandleTraces))
+	otelMux.HandleFunc("/v1/traces/", withCORS(coll.HandleTraces))
 	otelMux.HandleFunc("/v1/metrics", withCORS(coll.HandleMetrics))
+	otelMux.HandleFunc("/v1/metrics/", withCORS(coll.HandleMetrics))
 	otelMux.HandleFunc("/v1/logs", withCORS(coll.HandleLogs))
+	otelMux.HandleFunc("/v1/logs/", withCORS(coll.HandleLogs))
 	otelMux.HandleFunc("/health", coll.HandleHealth)
 
 	// UI and API mux (port 8080)
@@ -215,19 +278,43 @@ func main() {
 	// CORS is enabled here since this is the publicly exposed port (via the
 	// ingress) that a browser script such as static/otel-rum.js posts to.
 	uiMux.HandleFunc("/v1/traces", withCORS(coll.HandleTraces))
+	uiMux.HandleFunc("/v1/traces/", withCORS(coll.HandleTraces))
 	uiMux.HandleFunc("/v1/metrics", withCORS(coll.HandleMetrics))
+	uiMux.HandleFunc("/v1/metrics/", withCORS(coll.HandleMetrics))
 	uiMux.HandleFunc("/v1/logs", withCORS(coll.HandleLogs))
+	uiMux.HandleFunc("/v1/logs/", withCORS(coll.HandleLogs))
 	uiMux.HandleFunc("/health", coll.HandleHealth)
+	uiMux.HandleFunc("GET /internal/honeypot/ips", honeypotTracker.HandleIPs)
 
-	uiMux.HandleFunc("/api/history", coll.HandleHistory)
+	// Public OIDC endpoints
-	uiMux.HandleFunc("/api/history/traces", coll.HandleHistoryByType("traces"))
+	uiMux.HandleFunc("POST /api/auth/token", oidcAuth.HandleTokenExchange)
-	uiMux.HandleFunc("/api/history/metrics", coll.HandleHistoryByType("metrics"))
+	uiMux.HandleFunc("GET /api/auth/config", oidcAuth.HandleConfig)
-	uiMux.HandleFunc("/api/history/logs", coll.HandleHistoryByType("logs"))
+	uiMux.Handle("/api/honeypot/ips", oidcAuth.RequireAuth(http.HandlerFunc(honeypotTracker.HandleIPsForUser)))
-	uiMux.HandleFunc("/api/history/search", coll.HandleSearch)
+	// Source-map uploads are intentionally public so the release build script
-	uiMux.HandleFunc("/api/history/wait", coll.HandleWait)
+	// can upload maps without an OIDC session or bearer token.
-	uiMux.HandleFunc("GET /api/traces/{traceId}", coll.HandleTraceByID)
+	uiMux.HandleFunc("POST /api/sourcemaps", coll.HandleSourceMap)
-	uiMux.HandleFunc("/api/issues", coll.HandleIssues)
+
-	uiMux.HandleFunc("GET /api/issues/{id}", coll.HandleIssueByID)
+	// Protected API & WebSocket routes
+	uiMux.Handle("/api/history", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleHistory)))
+	uiMux.Handle("/api/history/traces", oidcAuth.RequireAuth(coll.HandleHistoryByType("traces")))
+	uiMux.Handle("/api/history/metrics", oidcAuth.RequireAuth(coll.HandleHistoryByType("metrics")))
+	uiMux.Handle("/api/history/logs", oidcAuth.RequireAuth(coll.HandleHistoryByType("logs")))
+	uiMux.Handle("/api/history/search", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleSearch)))
+	uiMux.Handle("/api/history/wait", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleWait)))
+	uiMux.Handle("GET /api/traces/{traceId}", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleTraceByID)))
+	uiMux.Handle("/api/issues", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleIssues)))
+	uiMux.Handle("GET /api/issues/{id}", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleIssueByID)))
+	uiMux.Handle("GET /api/issues/{id}/similar", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleIssueSimilar)))
+	uiMux.Handle("POST /api/issues/{id}/similar", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleSimilarityFeedback)))
+	uiMux.Handle("GET /api/patterns", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandlePatterns)))
+	uiMux.Handle("GET /api/rollouts", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleRollouts)))
+	uiMux.Handle("GET /api/rollouts/{id}", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleRolloutByID)))
+	uiMux.Handle("GET /api/rollouts/{id}/impact", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleRolloutImpact)))
+	// Aggregate stats (ClickHouse-backed); return 501 with an explanatory
+	// JSON body when only the file store is configured.
+	uiMux.Handle("/api/stats/errors", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleStatsErrors)))
+	uiMux.Handle("/api/stats/releases", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleStatsReleases)))
+	uiMux.Handle("/api/stats/services", oidcAuth.RequireAuth(http.HandlerFunc(coll.HandleStatsServices)))
 	uiMux.HandleFunc("/ws", wsServer.HandleConnections)
 	uiMux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("./static"))))
 	// Serve the agent instructions doc as a static route so an AI agent can
@@ -235,7 +322,23 @@ func main() {
 	uiMux.HandleFunc("GET /agents_example.md", func(w http.ResponseWriter, r *http.Request) {
 		http.ServeFile(w, r, "agents_example.md")
 	})
-	uiMux.Handle("/", spaHandler(http.Dir("./dist")))
+	uiMux.HandleFunc("GET /instrumentation.md", func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "instrumentation.md")
+	})
+	uiMux.HandleFunc("GET /static/instrumentation.md", func(w http.ResponseWriter, r *http.Request) {
+		http.ServeFile(w, r, "instrumentation.md")
+	})
+	distDir := "./dist"
+	if _, err := os.Stat(distDir); os.IsNotExist(err) {
+		if _, err := os.Stat("./otel-ui/dist"); err == nil {
+			distDir = "./otel-ui/dist"
+		} else if _, err := os.Stat("./index.html"); err == nil {
+			// Development fallback: serve the checked-in visualizer when no
+			// separately built frontend exists.
+			distDir = "."
+		}
+	}
+	uiMux.Handle("/", spaHandler(http.Dir(distDir)))
 
 	log.Printf("Starting HTTP server on port 8080 (UI/API)")
 	log.Printf("OTLP HTTP endpoints (also on port 8080):")
@@ -254,7 +357,8 @@ func main() {
 		}
 	}()
 
-	if err := http.ListenAndServe(":8080", uiMux); err != nil {
+	uiHandler := honeypotTracker.Middleware(uiMux)
+	if err := http.ListenAndServe(":8080", uiHandler); err != nil {
 		log.Fatalf("failed to start HTTP server: %v", err)
 	}
 }

docs/ML.md

@@ -0,0 +1,145 @@
+# Error detection model
+
+## Current decision
+
+Keep the original OTLP payloads as the source of truth, but derive a stable
+feature row for each service/release/time window. The model should not consume
+raw OTLP JSON, and it does not need to classify every span or log message into
+a semantic taxonomy first.
+
+The first target is an **error-spike predictor**:
+
+> Given the telemetry observed for a service in a five-minute window, predict
+> whether the following fifteen-minute window contains an abnormal error rate.
+
+This is more useful than classifying an event that the collector already knows
+is an error.
+
+## Normalized event contract
+
+The extraction layer should reduce spans, logs, and metrics to these stable
+dimensions before aggregation:
+
+| Field | Meaning |
+| --- | --- |
+| `window_start` | UTC five-minute bucket |
+| `service` | `service.name`, or `unknown` |
+| `release` | `service.release`, or empty |
+| `environment` | deployment environment, when present |
+| `kind` | `span`, `log`, or `metric` |
+| `is_error` | explicit error status, error HTTP/gRPC status, or error severity |
+| `status_code` | HTTP status when present |
+| `severity_number` | OTLP log severity, when present |
+| `duration_ms` | span duration, when present |
+| `route` | normalized HTTP route, when present |
+| `source` | `frontend` or `backend` |
+| `trace_id` | correlation identifier, when present |
+| `issue_fingerprint` | existing issue fingerprint, when available |
+
+PII-bearing fields such as raw messages, URLs, user agents, source IPs, and
+stack traces must not be model features. They can remain in the raw history and
+issue views subject to the existing retention and access controls.
+
+## First aggregate feature row
+
+For each service/release/window, derive:
+
+- total event count and error count
+- error rate
+- p50/p95 span duration
+- counts by kind (`span`, `log`, `metric`)
+- counts by HTTP status class (`2xx`, `4xx`, `5xx`)
+- unique issue count and newly observed issue count
+- frontend and backend error counts
+- retry, timeout, and cancellation counts where attributes identify them
+- release age, if deployment time is available
+
+The model input should be numeric and fixed-width. Categorical service and
+release values can initially be filtered per service; one global model can be
+considered after there is enough multi-service data.
+
+## Finding similar errors and spans
+
+Similarity should be a separate concern from the primary issue fingerprint.
+Use deterministic grouping for the canonical issue ID because it is stable,
+explainable, cheap at ingest time, and safe for alert counts. The current
+`issues.NormalizeTitle` already removes common variable values such as UUIDs,
+hex identifiers, and request numbers.
+
+The recommended matching stages are:
+
+1. **Canonical normalization**: lowercase and normalize IDs, numbers, quoted
+   values, URLs, and whitespace in messages and span names.
+2. **Structured signature**: combine service, event kind, normalized route or
+   span name, exception type, HTTP/gRPC status class, and the top application
+   stack frames. This avoids merging unrelated errors that happen to contain
+   the same words.
+3. **Exact fingerprint**: use the structured signature for the canonical issue
+   ID and deterministic deduplication.
+4. **Near-match suggestions**: compare only within a bounded candidate group,
+   such as the same service and event kind. Token n-gram similarity, SimHash,
+   or weighted field similarity can suggest related issues without changing
+   their IDs.
+
+For example, these should usually be suggested as similar but remain separate
+until confirmed:
+
+```text
+checkout: database timeout after 2000ms
+checkout: database timeout after 5000ms
+```
+
+They may represent the same underlying failure, but the timeout value could be
+operationally important. A review UI can offer “merge” or “not related”; those
+decisions become valuable labels later.
+
+Embeddings or a neural similarity model are reasonable as a second phase for
+semantic discovery, especially when messages use different wording. They
+should not be the first grouping mechanism: they add model/version drift,
+latency, explainability, and possible sensitive-data leakage. If introduced,
+embed redacted normalized text and stack signatures, retrieve nearest
+neighbors, and present the result as a suggestion rather than silently
+merging issues.
+
+For spans, the strongest similarity fields are usually the normalized span
+name or route, service, status class, exception type, and stack-frame signature.
+Trace IDs, span IDs, timestamps, and request-specific values should never
+contribute to similarity.
+
+## Label
+
+For a row at time `t`, label it positive when the future interval `[t+5m,
+t+20m)` satisfies both conditions:
+
+1. error rate is at least three times the trailing seven-day baseline for that
+   service and time-of-day; and
+2. the future error count is at least 5.
+
+The count floor prevents a single error in a very low-volume service from
+becoming an incident. The baseline must be calculated using data before `t`
+only.
+
+## Baseline before neural network
+
+Start with two baselines:
+
+1. a trailing-rate/spike rule; and
+2. logistic regression using the same aggregate features.
+
+Then compare a small MLP (`32 -> 16 -> 1`, ReLU plus sigmoid) against them.
+Use a chronological train/validation/test split, and report precision, recall,
+PR-AUC, and false alerts per service. Accuracy is not meaningful for this
+imbalanced target.
+
+## Data readiness
+
+The current cluster ClickHouse instance has approximately 168k entries across
+18 services, but only about 13 hours of history at the time this document was
+created. It is sufficient to test normalization and aggregation, but not to
+train the seven-day-baseline detector. Let the normalized feature table collect
+at least two weeks of data before evaluating a model; four weeks is preferable.
+
+ClickHouse should remain a ClusterIP service while this work is developed. A
+LoadBalancer would expose the receiver's OTLP and debug surfaces and is not
+needed for model development; use a temporary `kubectl port-forward` or an
+authenticated internal job for inspection and training data export.

go.mod

@@ -3,7 +3,9 @@ module git.tornberg.me/go-otel
 go 1.27.0
 
 require (
-	github.com/gorilla/websocket v1.5.3
+	github.com/ClickHouse/clickhouse-go/v2 v2.48.0
+	github.com/go-sourcemap/sourcemap v2.1.4+incompatible
+	github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674
 	github.com/natefinch/lumberjack v2.0.0+incompatible
 	go.opentelemetry.io/collector/consumer v1.55.0
 	go.opentelemetry.io/collector/pdata v1.55.0
@@ -11,24 +13,70 @@ require (
 	golang.org/x/time v0.16.0
 	google.golang.org/grpc v1.80.0
 	google.golang.org/protobuf v1.36.11
+	k8s.io/api v0.34.1
+	k8s.io/apimachinery v0.34.1
+	k8s.io/client-go v0.34.1
 )
 
 require (
 	github.com/BurntSushi/toml v1.6.0 // indirect
+	github.com/ClickHouse/ch-go v0.74.0 // indirect
+	github.com/andybalholm/brotli v1.2.2 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.12.2 // indirect
+	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
+	github.com/go-faster/city v1.0.1 // indirect
+	github.com/go-faster/errors v0.7.1 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/gnostic-models v0.7.0 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
 	github.com/hashicorp/go-version v1.9.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.19.1 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/paulmach/orb v0.13.0 // indirect
+	github.com/pierrec/lz4/v4 v4.1.27 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/segmentio/asm v1.2.1 // indirect
+	github.com/shopspring/decimal v1.4.0 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/collector/featuregate v1.55.0 // indirect
-	go.opentelemetry.io/otel v1.43.0 // indirect
+	go.opentelemetry.io/otel v1.44.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
+	go.opentelemetry.io/otel/trace v1.44.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/net v0.52.0 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
-	golang.org/x/sys v0.42.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/text v0.35.0 // indirect
+	golang.org/x/net v0.57.0 // indirect
+	golang.org/x/oauth2 v0.35.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	golang.org/x/term v0.45.0 // indirect
+	golang.org/x/text v0.40.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/klog/v2 v2.130.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
+	k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect
+	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
+	sigs.k8s.io/yaml v1.6.0 // indirect
 )

go.sum

@@ -1,43 +1,125 @@
 github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
 github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/ClickHouse/ch-go v0.74.0 h1:uYs2m4wIt0ZHSM1E72rg0maCfzhR2V3xWb/vZEgpeWE=
+github.com/ClickHouse/ch-go v0.74.0/go.mod h1:sZ/r+8ttZMjyrP9PuFbgoVbth1ywIu2LIQNA2vgko6M=
+github.com/ClickHouse/clickhouse-go/v2 v2.48.0 h1:auzd4VkapQYhQF8F2Gog7s3x78Bi1JZmByxGbrw3C+4=
+github.com/ClickHouse/clickhouse-go/v2 v2.48.0/go.mod h1:lBjUCPRG6RpRQdMbkXq+JV8rY0/O5lw+Z7jShgReFjM=
+github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM=
+github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU=
+github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
+github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
+github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
+github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
+github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
+github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
+github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
+github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
+github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
+github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
+github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
+github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
+github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
+github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo=
+github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo=
+github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo=
-github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c=
 github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA=
 github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
+github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
 github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM=
+github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
+github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4=
+github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
+github.com/paulmach/orb v0.13.0 h1:r7n7mQGGF+cj/CbcivEj9J3HGK+XR+yXnvzRdq9saIw=
+github.com/paulmach/orb v0.13.0/go.mod h1:6scRWINywA2Jf05dcjOfLfxrUIMECvTSG2MVbRLxu/k=
+github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk=
+github.com/pierrec/lz4/v4 v4.1.27/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0=
+github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
+github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
+github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/collector/consumer v1.55.0 h1:7Per8P4J0nlBrFVSXb+nwZ+egiel1BRtggZngyykGsM=
@@ -48,16 +130,16 @@ go.opentelemetry.io/collector/internal/testutil v0.149.0 h1:OWfUPO3NFKSaJtz/SBZp
 go.opentelemetry.io/collector/internal/testutil v0.149.0/go.mod h1:Jkjs6rkqs973LqgZ0Fe3zrokQRKULYXPIf4HuqStiEE=
 go.opentelemetry.io/collector/pdata v1.55.0 h1:WBgye8bo8koUyV9Vmp/r2Q3lgDezdsgfKDQAaM1oT2I=
 go.opentelemetry.io/collector/pdata v1.55.0/go.mod h1:6jPrbM4tuliCPACDznjFtxnnHisfKfzwrBVoeuESYuk=
-go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
+go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU=
-go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
+go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc=
-go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
+go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc=
-go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
+go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo=
-go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
+go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58=
-go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
+go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0=
 go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
 go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
-go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
+go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk=
-go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
+go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE=
 go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g=
 go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk=
 go.opentelemetry.io/proto/slim/otlp v1.10.0 h1:iR97Vs/ZDR+y9TfuP9b1XBtdPWeC+OMslIBmhcLU7jM=
@@ -70,14 +152,49 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
-golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
+go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
-golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
+go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
-golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
-golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
-golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
+golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
+golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
+golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
 golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE=
 golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
+golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
 gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
 google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA=
@@ -89,9 +206,36 @@ google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
 google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4=
+gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+k8s.io/api v0.34.1 h1:jC+153630BMdlFukegoEL8E/yT7aLyQkIVuwhmwDgJM=
+k8s.io/api v0.34.1/go.mod h1:SB80FxFtXn5/gwzCoN6QCtPD7Vbu5w2n1S0J5gFfTYk=
+k8s.io/apimachinery v0.34.1 h1:dTlxFls/eikpJxmAC7MVE8oOeP1zryV7iRyIjB0gky4=
+k8s.io/apimachinery v0.34.1/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw=
+k8s.io/client-go v0.34.1 h1:ZUPJKgXsnKwVwmKKdPfw4tB58+7/Ik3CrjOEhsiZ7mY=
+k8s.io/client-go v0.34.1/go.mod h1:kA8v0FP+tk6sZA0yKLRG67LWjqufAoSHA2xVGKw9Of8=
+k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
+k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
+k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA=
+k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts=
+k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y=
+k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
+sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=
+sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
+sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
+sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
+sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco=
+sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE=
+sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs=
+sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4=

instrumentation.md

@@ -0,0 +1,175 @@
+# Instrumentation guide
+
+This document describes the recommended instrumentation contract for this
+receiver. It is available at `/instrumentation.md` so frontend and backend
+agents can read it before integrating telemetry.
+
+## Shared identity
+
+Use the same values across all signals from one deployed application:
+
+- `service.name`: stable application name, for example `checkout-web`.
+- `service.release`: immutable deploy identifier, for example
+  `2026.09.18-a1b2c3`.
+- `service.environment`: `production`, `staging`, or `development`.
+
+The release must exactly match the source-map upload release.
+
+## Browser frontend
+
+Load the RUM script and initialize it once, before the application starts:
+
+```html
+<script src="https://otel.example.com/static/otel-rum.js"></script>
+<script>
+  const rum = OtelRUM.init({
+    serviceName: "checkout-web",
+    release: "2026.09.18-a1b2c3",
+    environment: "production",
+    baseUrl: "https://otel.example.com"
+  });
+</script>
+```
+
+The collector captures uncaught errors, unhandled promise rejections, failed
+HTTP requests, breadcrumbs, Web Vitals, and optional trace context. Correlate
+frontend requests with backend spans when possible:
+
+```js
+fetch("/api/orders", { headers: rum.traceHeaders() });
+```
+
+For manually handled errors:
+
+```js
+try {
+  riskyOperation();
+} catch (error) {
+  rum.captureException(error, { "feature.name": "checkout" });
+}
+```
+
+Do not put secrets, tokens, or sensitive customer data in breadcrumbs or
+custom attributes.
+
+## React error boundaries
+
+Rendering errors are not reliably caught by global `window.onerror` handlers.
+Report them from an error boundary and render a recovery screen:
+
+```tsx
+import React from "react";
+
+export class AppErrorBoundary extends React.Component<
+  { children: React.ReactNode },
+  { error: Error | null }
+> {
+  state = { error: null as Error | null };
+
+  static getDerivedStateFromError(error: Error) {
+    return { error };
+  }
+
+  componentDidCatch(error: Error, info: React.ErrorInfo) {
+    window.OtelRUM?.last?.captureException(error, {
+      "exception.type": "ReactErrorBoundary",
+      "react.component_stack": info.componentStack,
+    });
+  }
+
+  render() {
+    if (this.state.error) {
+      return <button onClick={() => this.setState({ error: null })}>Retry</button>;
+    }
+    return this.props.children;
+  }
+}
+```
+
+Use the same pattern in your application, including a user-visible fallback
+for individual views and the application root.
+
+## Backend instrumentation
+
+Instrument backend services with an OpenTelemetry SDK or auto-instrumentation
+and export OTLP to this receiver:
+
+- OTLP/gRPC: `otel.example.com:4317`
+- OTLP/HTTP traces: `https://otel.example.com/v1/traces`
+- OTLP/HTTP metrics: `https://otel.example.com/v1/metrics`
+- OTLP/HTTP logs: `https://otel.example.com/v1/logs`
+
+Set the resource attributes on every provider/exporter:
+
+```text
+service.name=checkout-api
+service.release=2026.09.18-a1b2c3
+service.environment=production
+```
+
+Create spans around inbound requests, database calls, external HTTP calls,
+queue operations, and important business operations. Record HTTP status,
+route, method, and exception details. Mark failed spans with an error status.
+The receiver derives backend issues from error spans and ERROR/FATAL logs.
+
+If HTTP ingest authentication is enabled, provide `OTEL_INGEST_KEY` as the
+`X-Otel-Ingest-Key` header or `key` query parameter. Keep that key server-side;
+browser clients should not receive it.
+
+For non-interactive agents that need to query protected debug endpoints, use a
+separate `OTEL_AGENT_TOKEN` service token. Send it as
+`X-Otel-Agent-Token: $OTEL_AGENT_TOKEN`; do not put it in browser code or
+telemetry payloads.
+
+## Source maps
+
+Build the frontend with the same immutable release used by RUM. The command
+and output directory depend on your build tool; the example below assumes a
+standard `dist/` output:
+
+```sh
+export RELEASE="2026.09.18-a1b2c3"
+VITE_APP_RELEASE="$RELEASE" npm run build
+```
+
+After deployment, upload the contents of each generated JavaScript map to the
+public multipart endpoint. No repository checkout or upload script is needed.
+`source` must match the generated bundle URL/path captured in the browser
+stack, while `file=@...` reads the local source-map file contents:
+
+```sh
+curl --fail-with-body --silent --show-error \
+  -X POST "https://otel.example.com/api/sourcemaps" \
+  -F "release=$RELEASE" \
+  -F "source=/assets/app-HASH.js" \
+  -F "file=@dist/assets/app-HASH.js.map;type=application/json"
+```
+
+For example, if the deployed JavaScript is available at
+`https://app.example.com/assets/main-abc123.js`, upload its matching map with
+`source=/assets/main-abc123.js`. The source-map filename on the build machine
+does not need to match the URL, but the `source` field must.
+
+Maps are parsed and retained server-side. Issue responses include a resolved
+stack when `service.release` and the generated bundle path match. Original
+source files and map contents are never returned by the API.
+
+The Docker build removes `.map` files from production images by default. For
+development or staging, public maps can be enabled for Chrome DevTools:
+
+```sh
+PUBLIC_SOURCEMAPS=true ./build.sh
+```
+
+Public maps expose application source code and should not be enabled in
+production unless that is intentional.
+
+## Verification checklist
+
+1. Trigger a controlled frontend exception.
+2. Trigger a controlled backend error span or ERROR log.
+3. Confirm both appear under Issues as separate frontend/backend issues.
+4. Confirm the issue sample contains the expected release.
+5. Confirm a mapped frontend stack shows original file and line information.
+6. Confirm trace IDs connect frontend requests to backend traces where headers
+   were propagated.

internal/auth/oidc.go

@@ -0,0 +1,380 @@
+package auth
+
+import (
+	"context"
+	"crypto"
+	"crypto/rsa"
+	"crypto/sha256"
+	"crypto/subtle"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"math/big"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"time"
+)
+
+type contextKey string
+
+// UserKey is the context key for authenticated user claims.
+const UserKey contextKey = "user"
+
+// Audience handles either a single string or an array of strings in JWT audience claim.
+type Audience []string
+
+func (a *Audience) UnmarshalJSON(b []byte) error {
+	if len(b) > 0 && b[0] == '"' {
+		var s string
+		if err := json.Unmarshal(b, &s); err != nil {
+			return err
+		}
+		*a = Audience{s}
+		return nil
+	}
+	return json.Unmarshal(b, (*[]string)(a))
+}
+
+func (a Audience) Contains(s string) bool {
+	for _, v := range a {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+// IDClaims represents verified claims in an OIDC token.
+type IDClaims struct {
+	Iss   string   `json:"iss"`
+	Sub   string   `json:"sub"`
+	Aud   Audience `json:"aud"`
+	Email string   `json:"email,omitempty"`
+	Name  string   `json:"name,omitempty"`
+	Roles []string `json:"roles,omitempty"`
+	Scope string   `json:"scope,omitempty"`
+	Exp   int64    `json:"exp"`
+}
+
+type jwksDoc struct {
+	Keys []struct {
+		Kid string `json:"kid"`
+		Kty string `json:"kty"`
+		Alg string `json:"alg"`
+		N   string `json:"n"`
+		E   string `json:"e"`
+	} `json:"keys"`
+}
+
+// OidcAuth provides OIDC token verification and authentication middleware.
+type OidcAuth struct {
+	issuer     string
+	client     string
+	disabled   bool
+	ingestKey  string
+	agentToken string
+
+	mu      sync.Mutex
+	keys    map[string]*rsa.PublicKey
+	fetched time.Time
+}
+
+// NewOidcAuth constructs a new OidcAuth instance.
+func NewOidcAuth(ingestKey string) *OidcAuth {
+	issuer := os.Getenv("OTEL_OIDC_ISSUER")
+	if issuer == "" {
+		issuer = "https://users.k6n.net"
+	}
+	client := os.Getenv("OTEL_OIDC_CLIENT_ID")
+	if client == "" {
+		client = "otel-debug"
+	}
+	disabled := os.Getenv("OTEL_AUTH_DISABLED") == "true"
+
+	return &OidcAuth{
+		issuer:     strings.TrimSuffix(issuer, "/"),
+		client:     client,
+		disabled:   disabled,
+		ingestKey:  ingestKey,
+		agentToken: os.Getenv("OTEL_AGENT_TOKEN"),
+	}
+}
+
+func (a *OidcAuth) Issuer() string {
+	return a.issuer
+}
+
+func (a *OidcAuth) ClientID() string {
+	return a.client
+}
+
+func (a *OidcAuth) IsDisabled() bool {
+	return a.disabled
+}
+
+func (a *OidcAuth) fetchKeys(ctx context.Context) (map[string]*rsa.PublicKey, error) {
+	url := a.issuer + "/oidc/v1/keys"
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("jwks endpoint returned %d", resp.StatusCode)
+	}
+
+	var doc jwksDoc
+	if err := json.NewDecoder(resp.Body).Decode(&doc); err != nil {
+		return nil, err
+	}
+
+	keys := make(map[string]*rsa.PublicKey, len(doc.Keys))
+	for _, k := range doc.Keys {
+		if k.Kty != "RSA" {
+			continue
+		}
+		n, err := base64.RawURLEncoding.DecodeString(k.N)
+		if err != nil {
+			continue
+		}
+		eBytes, err := base64.RawURLEncoding.DecodeString(k.E)
+		if err != nil || len(eBytes) > 4 {
+			continue
+		}
+		var eInt uint32
+		for _, b := range eBytes {
+			eInt = eInt<<8 | uint32(b)
+		}
+		keys[k.Kid] = &rsa.PublicKey{N: new(big.Int).SetBytes(n), E: int(eInt)}
+	}
+
+	if len(keys) == 0 {
+		return nil, errors.New("jwks contained no RSA keys")
+	}
+	return keys, nil
+}
+
+func (a *OidcAuth) publicKey(ctx context.Context, kid string) (*rsa.PublicKey, error) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	if key, ok := a.keys[kid]; ok {
+		return key, nil
+	}
+
+	// Throttle refreshes to at most once per minute
+	if a.keys == nil || time.Since(a.fetched) > time.Minute {
+		keys, err := a.fetchKeys(ctx)
+		if err != nil {
+			return nil, err
+		}
+		a.keys = keys
+		a.fetched = time.Now()
+	}
+
+	key, ok := a.keys[kid]
+	if !ok {
+		return nil, fmt.Errorf("unknown key id %q", kid)
+	}
+	return key, nil
+}
+
+// VerifyToken verifies an RS256 JWT against the cached JWKS public keys.
+func (a *OidcAuth) VerifyToken(ctx context.Context, token string) (*IDClaims, error) {
+	parts := strings.Split(token, ".")
+	if len(parts) != 3 {
+		return nil, errors.New("malformed token")
+	}
+
+	var header struct {
+		Kid string `json:"kid"`
+		Alg string `json:"alg"`
+	}
+	h, err := base64.RawURLEncoding.DecodeString(parts[0])
+	if err != nil {
+		return nil, fmt.Errorf("decode header: %w", err)
+	}
+	if err := json.Unmarshal(h, &header); err != nil {
+		return nil, fmt.Errorf("parse header: %w", err)
+	}
+	if header.Alg != "RS256" {
+		return nil, fmt.Errorf("unexpected alg %q", header.Alg)
+	}
+
+	key, err := a.publicKey(ctx, header.Kid)
+	if err != nil {
+		return nil, err
+	}
+
+	sig, err := base64.RawURLEncoding.DecodeString(parts[2])
+	if err != nil {
+		return nil, fmt.Errorf("decode signature: %w", err)
+	}
+
+	sum := sha256.Sum256([]byte(parts[0] + "." + parts[1]))
+	if err := rsa.VerifyPKCS1v15(key, crypto.SHA256, sum[:], sig); err != nil {
+		return nil, errors.New("invalid signature")
+	}
+
+	payload, err := base64.RawURLEncoding.DecodeString(parts[1])
+	if err != nil {
+		return nil, fmt.Errorf("decode claims: %w", err)
+	}
+
+	var claims IDClaims
+	if err := json.Unmarshal(payload, &claims); err != nil {
+		return nil, fmt.Errorf("parse claims: %w", err)
+	}
+
+	if claims.Iss != a.issuer {
+		return nil, fmt.Errorf("unexpected issuer %q", claims.Iss)
+	}
+	if !claims.Aud.Contains(a.client) {
+		return nil, errors.New("token not issued for this client")
+	}
+	if claims.Exp != 0 && time.Now().Unix() > claims.Exp {
+		return nil, errors.New("token expired")
+	}
+
+	return &claims, nil
+}
+
+// HandleTokenExchange proxies the OIDC token exchange endpoint so the SPA
+// doesn't face CORS issues.
+func (a *OidcAuth) HandleTokenExchange(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "POST required", http.StatusMethodNotAllowed)
+		return
+	}
+	if err := r.ParseForm(); err != nil {
+		http.Error(w, "invalid form body", http.StatusBadRequest)
+		return
+	}
+
+	// Always ensure client_id is set
+	if r.PostForm.Get("client_id") == "" {
+		r.PostForm.Set("client_id", a.client)
+	}
+
+	resp, err := http.PostForm(a.issuer+"/oauth/v2/token", r.PostForm)
+	if err != nil {
+		log.Printf("token endpoint unreachable: %v", err)
+		http.Error(w, "token endpoint unreachable", http.StatusBadGateway)
+		return
+	}
+	defer resp.Body.Close()
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(resp.StatusCode)
+	_, _ = io.Copy(w, resp.Body)
+}
+
+// HandleConfig returns OIDC client configuration for the SPA frontend.
+func (a *OidcAuth) HandleConfig(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{
+		"issuer":   a.issuer,
+		"clientId": a.client,
+		"disabled": a.disabled,
+	})
+}
+
+// RequireAuth wraps an HTTP handler and ensures valid authentication.
+func (a *OidcAuth) RequireAuth(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if a.disabled {
+			next.ServeHTTP(w, r)
+			return
+		}
+
+		// 1. Allow if upstream proxy (ingress) already verified via auth-url
+		if authUser := r.Header.Get("X-Auth-User"); authUser != "" {
+			ctx := context.WithValue(r.Context(), UserKey, &IDClaims{Sub: authUser, Email: authUser})
+			next.ServeHTTP(w, r.WithContext(ctx))
+			return
+		}
+
+		// 2. Allow if valid ingest key is provided (for background automation / agents)
+		if a.ingestKey != "" {
+			k := r.Header.Get("X-Otel-Ingest-Key")
+			if k == "" {
+				k = r.URL.Query().Get("key")
+			}
+			if k != "" && k == a.ingestKey {
+				next.ServeHTTP(w, r)
+				return
+			}
+		}
+
+		// Service-to-service credential for debugging agents. It is separate
+		// from the ingest key because it grants access to protected debug data.
+		if a.agentToken != "" && constantTimeTokenMatch(r, a.agentToken) {
+			ctx := context.WithValue(r.Context(), UserKey, &IDClaims{Sub: "service-agent", Name: "debug agent"})
+			next.ServeHTTP(w, r.WithContext(ctx))
+			return
+		}
+
+		// 3. Check Bearer token (or access_token query param for WebSocket)
+		token := bearerToken(r)
+		if token != "" {
+			claims, err := a.VerifyToken(r.Context(), token)
+			if err == nil {
+				ctx := context.WithValue(r.Context(), UserKey, claims)
+				next.ServeHTTP(w, r.WithContext(ctx))
+				return
+			}
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusUnauthorized)
+			_ = json.NewEncoder(w).Encode(map[string]string{
+				"error": "invalid token: " + err.Error(),
+			})
+			return
+		}
+
+		// 4. Unauthorized
+		w.Header().Set("WWW-Authenticate", fmt.Sprintf(`Bearer realm="%s"`, a.client))
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusUnauthorized)
+		_ = json.NewEncoder(w).Encode(map[string]string{
+			"error": "authentication required",
+		})
+	})
+}
+
+func constantTimeTokenMatch(r *http.Request, expected string) bool {
+	token := r.Header.Get("X-Otel-Agent-Token")
+	if token == "" {
+		auth := r.Header.Get("Authorization")
+		if len(auth) > 7 && strings.EqualFold(auth[:7], "bearer ") {
+			token = strings.TrimSpace(auth[7:])
+		}
+	}
+	return len(token) == len(expected) && subtle.ConstantTimeCompare([]byte(token), []byte(expected)) == 1
+}
+
+func bearerToken(r *http.Request) string {
+	if t := r.URL.Query().Get("access_token"); t != "" {
+		return t
+	}
+	auth := r.Header.Get("Authorization")
+	if len(auth) > 7 && strings.EqualFold(auth[:7], "bearer ") {
+		return strings.TrimSpace(auth[7:])
+	}
+	return ""
+}
+
+// CurrentUser retrieves the authenticated user claims from the context, if present.
+func CurrentUser(ctx context.Context) *IDClaims {
+	claims, _ := ctx.Value(UserKey).(*IDClaims)
+	return claims
+}

internal/auth/oidc_test.go

@@ -0,0 +1,165 @@
+package auth
+
+import (
+	"context"
+	"crypto"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestAudienceUnmarshal(t *testing.T) {
+	var a1 Audience
+	if err := json.Unmarshal([]byte(`"single-aud"`), &a1); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !a1.Contains("single-aud") || a1.Contains("other") {
+		t.Fatalf("unexpected audience content: %v", a1)
+	}
+
+	var a2 Audience
+	if err := json.Unmarshal([]byte(`["aud1", "aud2"]`), &a2); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !a2.Contains("aud1") || !a2.Contains("aud2") || a2.Contains("aud3") {
+		t.Fatalf("unexpected audience content: %v", a2)
+	}
+}
+
+func TestVerifyToken(t *testing.T) {
+	privKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("failed to generate RSA key: %v", err)
+	}
+
+	auth := &OidcAuth{
+		issuer: "https://test.issuer",
+		client: "test-client",
+		keys: map[string]*rsa.PublicKey{
+			"key-1": &privKey.PublicKey,
+		},
+		fetched: time.Now(),
+	}
+
+	// Helper to mint test token
+	mintToken := func(kid string, claims IDClaims) string {
+		headerJSON, _ := json.Marshal(map[string]string{
+			"kid": kid,
+			"alg": "RS256",
+		})
+		claimsJSON, _ := json.Marshal(claims)
+
+		headerB64 := base64.RawURLEncoding.EncodeToString(headerJSON)
+		payloadB64 := base64.RawURLEncoding.EncodeToString(claimsJSON)
+
+		sum := sha256.Sum256([]byte(headerB64 + "." + payloadB64))
+		sig, _ := rsa.SignPKCS1v15(rand.Reader, privKey, crypto.SHA256, sum[:])
+		sigB64 := base64.RawURLEncoding.EncodeToString(sig)
+
+		return headerB64 + "." + payloadB64 + "." + sigB64
+	}
+
+	// Valid token
+	validToken := mintToken("key-1", IDClaims{
+		Iss: "https://test.issuer",
+		Sub: "user-123",
+		Aud: Audience{"test-client"},
+		Exp: time.Now().Add(time.Hour).Unix(),
+	})
+
+	claims, err := auth.VerifyToken(context.Background(), validToken)
+	if err != nil {
+		t.Fatalf("expected valid token to pass, got: %v", err)
+	}
+	if claims.Sub != "user-123" {
+		t.Fatalf("expected sub user-123, got %s", claims.Sub)
+	}
+
+	// Expired token
+	expiredToken := mintToken("key-1", IDClaims{
+		Iss: "https://test.issuer",
+		Sub: "user-123",
+		Aud: Audience{"test-client"},
+		Exp: time.Now().Add(-time.Hour).Unix(),
+	})
+	if _, err := auth.VerifyToken(context.Background(), expiredToken); err == nil {
+		t.Fatalf("expected expired token to fail")
+	}
+
+	// Wrong audience
+	wrongAudToken := mintToken("key-1", IDClaims{
+		Iss: "https://test.issuer",
+		Sub: "user-123",
+		Aud: Audience{"other-client"},
+		Exp: time.Now().Add(time.Hour).Unix(),
+	})
+	if _, err := auth.VerifyToken(context.Background(), wrongAudToken); err == nil {
+		t.Fatalf("expected wrong audience to fail")
+	}
+
+	// Wrong issuer
+	wrongIssToken := mintToken("key-1", IDClaims{
+		Iss: "https://wrong.issuer",
+		Sub: "user-123",
+		Aud: Audience{"test-client"},
+		Exp: time.Now().Add(time.Hour).Unix(),
+	})
+	if _, err := auth.VerifyToken(context.Background(), wrongIssToken); err == nil {
+		t.Fatalf("expected wrong issuer to fail")
+	}
+}
+
+func TestRequireAuthMiddleware(t *testing.T) {
+	auth := &OidcAuth{
+		issuer:     "https://test.issuer",
+		client:     "test-client",
+		ingestKey:  "secret-key",
+		agentToken: "agent-secret",
+	}
+
+	handler := auth.RequireAuth(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("ok"))
+	}))
+
+	// 1. Unauthenticated request -> 401
+	req := httptest.NewRequest("GET", "/api/history", nil)
+	rec := httptest.NewRecorder()
+	handler.ServeHTTP(rec, req)
+	if rec.Code != http.StatusUnauthorized {
+		t.Fatalf("expected 401 for unauthenticated request, got %d", rec.Code)
+	}
+
+	// 2. Ingest key in header -> 200
+	reqKey := httptest.NewRequest("GET", "/api/history", nil)
+	reqKey.Header.Set("X-Otel-Ingest-Key", "secret-key")
+	recKey := httptest.NewRecorder()
+	handler.ServeHTTP(recKey, reqKey)
+	if recKey.Code != http.StatusOK {
+		t.Fatalf("expected 200 with valid ingest key, got %d", recKey.Code)
+	}
+
+	// 3. Dedicated service token in a header -> 200
+	reqAgent := httptest.NewRequest("GET", "/api/history", nil)
+	reqAgent.Header.Set("X-Otel-Agent-Token", "agent-secret")
+	recAgent := httptest.NewRecorder()
+	handler.ServeHTTP(recAgent, reqAgent)
+	if recAgent.Code != http.StatusOK {
+		t.Fatalf("expected 200 with valid agent token, got %d", recAgent.Code)
+	}
+
+	// 4. X-Auth-User from upstream ingress -> 200
+	reqAuthUser := httptest.NewRequest("GET", "/api/history", nil)
+	reqAuthUser.Header.Set("X-Auth-User", "mats@tornberg.me")
+	recAuthUser := httptest.NewRecorder()
+	handler.ServeHTTP(recAuthUser, reqAuthUser)
+	if recAuthUser.Code != http.StatusOK {
+		t.Fatalf("expected 200 with X-Auth-User, got %d", recAuthUser.Code)
+	}
+}

internal/collector/collector.go

@@ -7,11 +7,16 @@ import (
 	"context"
 	"encoding/json"
 	"log"
+	"os"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 
 	"git.tornberg.me/go-otel/internal/issues"
+	kube "git.tornberg.me/go-otel/internal/kubernetes"
+	"git.tornberg.me/go-otel/internal/prometheus"
+	"git.tornberg.me/go-otel/internal/sourcemaps"
 	"git.tornberg.me/go-otel/internal/storage"
 	"git.tornberg.me/go-otel/internal/websocket"
 
@@ -28,6 +33,7 @@ type Collector struct {
 	history      *ring.Ring
 	historyMutex sync.RWMutex
 	store        storage.Store
+	storeChan    chan websocket.HistoryEntry
 	subscribers  map[chan websocket.HistoryEntry]func(websocket.HistoryEntry) bool
 	subMutex     sync.Mutex
 	waitLimiter  *ipRateLimiter
@@ -35,15 +41,26 @@ type Collector struct {
 
 	ingestLimiter *ipRateLimiter
 	ingestKey     string
+
+	notifyIssuesFn func()
+
+	// statsQuerier is the ClickHouse store when configured; nil otherwise.
+	// The stats endpoints return 501 when it's nil so the UI can hide those
+	// views instead of erroring.
+	statsQuerier storage.StatsQuerier
+	sourceMaps   *sourcemaps.Resolver
+	kubeWatcher  *kube.Watcher
+	prometheus   *prometheus.Client
 }
 
 // New creates a new Collector instance.
 func New(broadcaster websocket.Broadcaster, historySize int, store storage.Store) *Collector {
-	return &Collector{
+	c := &Collector{
 		broadcaster: broadcaster,
 		historySize: historySize,
 		history:     ring.New(historySize),
 		store:       store,
+		storeChan:   make(chan websocket.HistoryEntry, 10000),
 		subscribers: make(map[chan websocket.HistoryEntry]func(websocket.HistoryEntry) bool),
 		// Allow bursts of a few immediate waits, then steady-state ~1 req/sec per IP.
 		waitLimiter: newIPRateLimiter(1, 5),
@@ -53,6 +70,26 @@ func New(broadcaster websocket.Broadcaster, historySize int, store storage.Store
 		// looping) from hammering the collector.
 		ingestLimiter: newIPRateLimiter(20, 50),
 	}
+
+	if store != nil {
+		// Bounded worker pool prevents concurrent ClickHouse INSERTs from
+		// multiplying query and merge memory under bursty ingest traffic.
+		numWorkers := 4
+		if value, err := strconv.Atoi(os.Getenv("OTEL_STORE_WORKERS")); err == nil && value > 0 && value <= 32 {
+			numWorkers = value
+		}
+		for i := 0; i < numWorkers; i++ {
+			go func() {
+				for entry := range c.storeChan {
+					if err := c.store.Append(context.Background(), entry); err != nil {
+						log.Printf("failed to append to store: %v", err)
+					}
+				}
+			}()
+		}
+	}
+
+	return c
 }
 
 // SetIngestKey configures an optional shared key that HTTP OTLP ingestion
@@ -64,19 +101,102 @@ func (c *Collector) SetIngestKey(key string) {
 	c.ingestKey = key
 }
 
+// SetStatsQuerier enables the aggregate stats endpoints (error trends,
+// release comparisons) when a store supporting them is configured.
+func (c *Collector) SetStatsQuerier(q storage.StatsQuerier) {
+	c.statsQuerier = q
+}
+
+func (c *Collector) SetKubernetesWatcher(w *kube.Watcher)     { c.kubeWatcher = w }
+func (c *Collector) SetPrometheusClient(p *prometheus.Client) { c.prometheus = p }
+func (c *Collector) ListRollouts() []kube.Rollout {
+	if c.kubeWatcher == nil {
+		return nil
+	}
+	return c.kubeWatcher.List()
+}
+func (c *Collector) GetRollout(id string) (kube.Rollout, bool) {
+	if c.kubeWatcher == nil {
+		return kube.Rollout{}, false
+	}
+	return c.kubeWatcher.Get(id)
+}
+func (c *Collector) KubernetesEnabled() bool { return c.kubeWatcher != nil && c.kubeWatcher.Enabled() }
+func (c *Collector) KubernetesError() string {
+	if c.kubeWatcher == nil {
+		return "kubernetes integration is not configured"
+	}
+	return c.kubeWatcher.Error()
+}
+
 // ListIssues returns deduplicated error issues, most recently seen first.
 func (c *Collector) ListIssues(limit int) []issues.Issue {
-	return c.issues.List(limit)
+	list := c.issues.ListFiltered(issues.ListOptions{Limit: limit}).Issues
+	if c.sourceMaps != nil {
+		for i := range list {
+			resolveIssue(&list[i], c.sourceMaps)
+		}
+	}
+	return list
+}
+
+// ListIssuesFiltered returns a filtered page of deduplicated issues.
+func (c *Collector) ListIssuesFiltered(opts issues.ListOptions) issues.ListPage {
+	page := c.issues.ListFiltered(opts)
+	if c.sourceMaps != nil {
+		for i := range page.Issues {
+			resolveIssue(&page.Issues[i], c.sourceMaps)
+		}
+	}
+	return page
 }
 
 // GetIssue returns a single issue by id.
 func (c *Collector) GetIssue(id string) (issues.Issue, bool) {
-	return c.issues.Get(id)
+	issue, ok := c.issues.Get(id)
+	if ok && c.sourceMaps != nil {
+		resolveIssue(&issue, c.sourceMaps)
+	}
+	return issue, ok
+}
+
+func (c *Collector) SetSourceMaps(resolver *sourcemaps.Resolver) { c.sourceMaps = resolver }
+
+func resolveIssue(issue *issues.Issue, resolver *sourcemaps.Resolver) {
+	for i := range issue.Samples {
+		issue.Samples[i].ResolvedStacktrace = resolver.ResolveStack(issue.Samples[i].Release, issue.Samples[i].Stacktrace)
+	}
+}
+
+// SetIssueStatus updates the triage state of an issue.
+func (c *Collector) SetIssueStatus(id, status string) (issues.Issue, bool) {
+	return c.issues.SetStatus(id, status)
+}
+
+func (c *Collector) SimilarIssues(id string, limit int) ([]issues.SimilarIssue, bool) {
+	return c.issues.Similar(id, limit)
+}
+
+func (c *Collector) ListPatterns() []issues.Pattern { return c.issues.Patterns() }
+
+func (c *Collector) SetSimilarityFeedback(a, b, decision string) bool {
+	return c.issues.SetSimilarityFeedback(a, b, decision)
 }
 
 // SetBroadcaster updates the broadcaster used by the collector.
 func (c *Collector) SetBroadcaster(broadcaster websocket.Broadcaster) {
 	c.broadcaster = broadcaster
+	if notifier, ok := broadcaster.(interface{ NotifyIssues() }); ok {
+		c.notifyIssuesFn = notifier.NotifyIssues
+	}
+}
+
+// notifyIssues pings WebSocket clients that new error telemetry arrived, so
+// the UI can refresh its Issues view without waiting for its poll interval.
+func (c *Collector) notifyIssues() {
+	if c.notifyIssuesFn != nil {
+		c.notifyIssuesFn()
+	}
 }
 
 func (c *Collector) addToHistory(dataType websocket.DataType, data json.RawMessage) {
@@ -92,12 +212,11 @@ func (c *Collector) addToHistory(dataType websocket.DataType, data json.RawMessa
 	c.historyMutex.Unlock()
 
 	if c.store != nil {
-		// Asynchronously append to store to avoid blocking ingestion
+		select {
-		go func() {
+		case c.storeChan <- entry:
-			if err := c.store.Append(context.Background(), entry); err != nil {
+		default:
-				log.Printf("failed to append to store: %v", err)
+			log.Printf("collector: store write queue full, dropping entry to preserve ingestion")
-			}
+		}
-		}()
 	}
 
 	c.notifySubscribers(entry)
@@ -219,8 +338,10 @@ func (c *Collector) ConsumeTraces(ctx context.Context, td ptrace.Traces) error {
 	}
 
 	// Store and broadcast raw bytes as json.RawMessage to avoid unsafe interface{} usage
-		c.addToHistory(websocket.DataTypeTraces, buf)
+	c.addToHistory(websocket.DataTypeTraces, buf)
-	extractSpanErrors(c.issues, td, time.Now().UnixNano())
+	if extractSpanErrors(c.issues, td, time.Now().UnixNano()) {
+		c.notifyIssues()
+	}
 
 	entry := websocket.Entry{
 		Type: string(websocket.DataTypeTraces),
@@ -241,7 +362,7 @@ func (c *Collector) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) erro
 	}
 
 	// Store and broadcast raw bytes as json.RawMessage to avoid unsafe interface{} usage
-		c.addToHistory(websocket.DataTypeMetrics, buf)
+	c.addToHistory(websocket.DataTypeMetrics, buf)
 
 	entry := websocket.Entry{
 		Type: string(websocket.DataTypeMetrics),
@@ -262,8 +383,10 @@ func (c *Collector) ConsumeLogs(ctx context.Context, ld plog.Logs) error {
 	}
 
 	// Store and broadcast raw bytes as json.RawMessage to avoid unsafe interface{} usage
-		c.addToHistory(websocket.DataTypeLogs, buf)
+	c.addToHistory(websocket.DataTypeLogs, buf)
-	extractLogErrors(c.issues, ld, time.Now().UnixNano())
+	if extractLogErrors(c.issues, ld, time.Now().UnixNano()) {
+		c.notifyIssues()
+	}
 
 	entry := websocket.Entry{
 		Type: string(websocket.DataTypeLogs),

internal/collector/collector_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"git.tornberg.me/go-otel/internal/issues"
 	"git.tornberg.me/go-otel/internal/storage"
 	"git.tornberg.me/go-otel/internal/websocket"
 	"go.opentelemetry.io/collector/pdata/pcommon"
@@ -242,6 +243,77 @@ func TestConsumeTracesRecordsIssue(t *testing.T) {
 	}
 }
 
+func TestConsumeTracesHTTPErrorsAndNaming(t *testing.T) {
+	coll := New(noopBroadcaster{}, 10, nil)
+
+	td := ptrace.NewTraces()
+
+	// 1. Nginx span with 404 and empty name (common for ingress/reverse proxy)
+	rs1 := td.ResourceSpans().AppendEmpty()
+	rs1.Resource().Attributes().PutStr("service.name", "nginx")
+	s1 := rs1.ScopeSpans().AppendEmpty().Spans().AppendEmpty()
+	s1.SetName("") // empty name
+	s1.Attributes().PutInt("http.status_code", 404)
+	s1.Attributes().PutStr("http.method", "GET")
+	s1.Attributes().PutStr("http.target", "/missing-path")
+
+	// 2. e2e-verify span with 500 and service name in span attributes
+	rs2 := td.ResourceSpans().AppendEmpty()
+	// Resource has no service.name, fallback to app attribute on span
+	s2 := rs2.ScopeSpans().AppendEmpty().Spans().AppendEmpty()
+	s2.SetName("test-endpoint")
+	s2.Attributes().PutStr("app.kubernetes.io/name", "e2e-verify")
+	s2.Attributes().PutInt("http.response.status_code", 500)
+	s2.Attributes().PutStr("http.request.method", "POST")
+	s2.Attributes().PutStr("http.route", "/api/v1/test")
+
+	if err := coll.ConsumeTraces(context.Background(), td); err != nil {
+		t.Fatalf("ConsumeTraces failed: %v", err)
+	}
+
+	issuesList := coll.ListIssues(10)
+	if len(issuesList) != 2 {
+		t.Fatalf("expected 2 distinct issues (404 and 500), got %d: %+v", len(issuesList), issuesList)
+	}
+
+	foundNginx := false
+	foundE2E := false
+
+	for _, iss := range issuesList {
+		if iss.Title == "(error)" {
+			t.Errorf("issue title must not be '(error)', got %q", iss.Title)
+		}
+		if iss.Service == "unknown" {
+			t.Errorf("service must not be 'unknown', got %s", iss.Service)
+		}
+		if iss.Service == "nginx" {
+			foundNginx = true
+			if !strings.Contains(iss.Title, "HTTP 404 (Not Found): GET /missing-path") {
+				t.Errorf("expected nginx 404 title, got %q", iss.Title)
+			}
+			if iss.Samples[0].HTTPStatus != 404 {
+				t.Errorf("expected sample HTTPStatus 404, got %d", iss.Samples[0].HTTPStatus)
+			}
+		}
+		if iss.Service == "e2e-verify" {
+			foundE2E = true
+			if !strings.Contains(iss.Title, "HTTP 500 (Internal Server Error): POST /api/v1/test") {
+				t.Errorf("expected e2e-verify 500 title, got %q", iss.Title)
+			}
+			if iss.Samples[0].HTTPStatus != 500 {
+				t.Errorf("expected sample HTTPStatus 500, got %d", iss.Samples[0].HTTPStatus)
+			}
+		}
+	}
+
+	if !foundNginx {
+		t.Errorf("expected to find nginx issue")
+	}
+	if !foundE2E {
+		t.Errorf("expected to find e2e-verify issue")
+	}
+}
+
 func TestConsumeLogsRecordsIssue(t *testing.T) {
 	coll := New(noopBroadcaster{}, 10, nil)
 
@@ -320,6 +392,41 @@ func TestHandleIssuesAndByID(t *testing.T) {
 	}
 }
 
+func TestHandleIssuesFiltersAndPaginates(t *testing.T) {
+	coll := New(noopBroadcaster{}, 10, nil)
+	coll.issues.Record("span_error", "nginx", "GET /.env", issues.Sample{Timestamp: 300, SourceIP: "203.0.113.10", HTTPRoute: "/.env"})
+	coll.issues.Record("span_error", "nginx", "GET /.git/HEAD", issues.Sample{Timestamp: 200, SourceIP: "203.0.113.11", HTTPRoute: "/.git/HEAD"})
+	coll.issues.Record("log_error", "api", "database down", issues.Sample{Timestamp: 100})
+
+	req := httptest.NewRequest(http.MethodGet, "/api/issues?service=nginx&kind=span_error&limit=1", nil)
+	rec := httptest.NewRecorder()
+	coll.HandleIssues(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("expected 200, got %d: %s", rec.Code, rec.Body.String())
+	}
+	var page []issues.Issue
+	if err := json.Unmarshal(rec.Body.Bytes(), &page); err != nil {
+		t.Fatalf("failed to decode issues page: %v", err)
+	}
+	if len(page) != 1 || page[0].Service != "nginx" {
+		t.Fatalf("unexpected filtered page: %+v", page)
+	}
+	cursor := rec.Header().Get("X-Next-Cursor")
+	if cursor == "" {
+		t.Fatal("expected next cursor header")
+	}
+
+	req = httptest.NewRequest(http.MethodGet, "/api/issues?service=nginx&limit=1&cursor="+cursor, nil)
+	rec = httptest.NewRecorder()
+	coll.HandleIssues(rec, req)
+	if err := json.Unmarshal(rec.Body.Bytes(), &page); err != nil {
+		t.Fatalf("failed to decode cursor page: %v", err)
+	}
+	if len(page) != 1 || page[0].Title != "GET /.git/HEAD" {
+		t.Fatalf("unexpected cursor page: %+v", page)
+	}
+}
+
 func TestHandleLogsIngestKey(t *testing.T) {
 	coll := New(noopBroadcaster{}, 10, nil)
 	coll.SetIngestKey("secret")

internal/collector/handlers.go

@@ -4,7 +4,9 @@ package collector
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"io"
 	"log"
 	"net/http"
@@ -12,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	"git.tornberg.me/go-otel/internal/issues"
 	"git.tornberg.me/go-otel/internal/storage"
 	"git.tornberg.me/go-otel/internal/websocket"
 	"go.opentelemetry.io/collector/pdata/plog"
@@ -378,35 +381,282 @@ func (c *Collector) HandleIssues(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	list := c.ListIssues(parseLimit(r, 100))
+	opts, err := parseIssueListOptions(r)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	page := c.ListIssuesFiltered(opts)
 
 	w.Header().Set("Content-Type", "application/json")
-	if err := json.NewEncoder(w).Encode(list); err != nil {
+	if page.HasMore && len(page.Issues) > 0 {
+		last := page.Issues[len(page.Issues)-1]
+		next := encodeIssueCursor(last.LastSeen, last.ID)
+		w.Header().Set("X-Next-Cursor", next)
+		nextQuery := r.URL.Query()
+		nextQuery.Set("cursor", next)
+		w.Header().Set("Link", fmt.Sprintf("<%s?%s>; rel=next", r.URL.Path, nextQuery.Encode()))
+	}
+	// Keep the response as a JSON array for compatibility. Pagination metadata
+	// is exposed in headers so existing UI clients do not need a migration.
+	if err := json.NewEncoder(w).Encode(page.Issues); err != nil {
 		log.Printf("failed to encode issues: %v", err)
 		http.Error(w, "failed to encode issues", http.StatusInternalServerError)
 		return
 	}
 }
 
+const maxIssuePageSize = 200
+
+type issueCursor struct {
+	LastSeen int64  `json:"lastSeen"`
+	ID       string `json:"id"`
+}
+
+func parseIssueListOptions(r *http.Request) (issues.ListOptions, error) {
+	q := r.URL.Query()
+	opts := issues.ListOptions{
+		Service:  q.Get("service"),
+		Kind:     q.Get("kind"),
+		Status:   q.Get("status"),
+		Title:    q.Get("q"),
+		SourceIP: q.Get("sourceIp"),
+		Route:    q.Get("route"),
+		Limit:    parseLimit(r, 100),
+	}
+	if opts.Limit <= 0 {
+		opts.Limit = 100
+	}
+	if opts.Limit > maxIssuePageSize {
+		opts.Limit = maxIssuePageSize
+	}
+	for name, dst := range map[string]*int64{"since": &opts.Since, "until": &opts.Until} {
+		if value := q.Get(name); value != "" {
+			parsed, err := strconv.ParseInt(value, 10, 64)
+			if err != nil || parsed < 0 {
+				return issues.ListOptions{}, fmt.Errorf("invalid %s; expected unix nanoseconds", name)
+			}
+			*dst = parsed
+		}
+	}
+	if opts.Since != 0 && opts.Until != 0 && opts.Since > opts.Until {
+		return issues.ListOptions{}, fmt.Errorf("since must be less than or equal to until")
+	}
+	if value := q.Get("cursor"); value != "" {
+		decoded, err := base64.RawURLEncoding.DecodeString(value)
+		if err != nil {
+			return issues.ListOptions{}, fmt.Errorf("invalid cursor")
+		}
+		var cursor issueCursor
+		if err := json.Unmarshal(decoded, &cursor); err != nil || cursor.LastSeen <= 0 || cursor.ID == "" {
+			return issues.ListOptions{}, fmt.Errorf("invalid cursor")
+		}
+		opts.AfterLastSeen = cursor.LastSeen
+		opts.AfterID = cursor.ID
+	}
+	return opts, nil
+}
+
+func encodeIssueCursor(lastSeen int64, id string) string {
+	data, _ := json.Marshal(issueCursor{LastSeen: lastSeen, ID: id})
+	return base64.RawURLEncoding.EncodeToString(data)
+}
+
+// HandleSourceMap accepts a JavaScript source map for a deployed release.
+// Multipart fields: release, source, file.
+func (c *Collector) HandleSourceMap(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if c.sourceMaps == nil {
+		http.Error(w, "source maps are not configured", http.StatusNotImplemented)
+		return
+	}
+	if err := r.ParseMultipartForm(50 << 20); err != nil {
+		http.Error(w, "invalid multipart body", http.StatusBadRequest)
+		return
+	}
+	release, source := r.FormValue("release"), r.FormValue("source")
+	file, _, err := r.FormFile("file")
+	if err != nil {
+		http.Error(w, "file is required", http.StatusBadRequest)
+		return
+	}
+	defer file.Close()
+	if err := c.sourceMaps.Upload(release, source, file); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	w.WriteHeader(http.StatusCreated)
+}
+
 // HandleIssueByID returns a single issue by id, including its samples.
 func (c *Collector) HandleIssueByID(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	switch r.Method {
+	case http.MethodGet:
+		issue, ok := c.GetIssue(id)
+		if !ok {
+			http.Error(w, "issue not found", http.StatusNotFound)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		if err := json.NewEncoder(w).Encode(issue); err != nil {
+			log.Printf("failed to encode issue: %v", err)
+			http.Error(w, "failed to encode issue", http.StatusInternalServerError)
+		}
+	case http.MethodPatch:
+		c.HandleIssueStatus(w, r, id)
+	default:
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+// HandlePatterns returns deterministic related-issue groups for visualization.
+func (c *Collector) HandlePatterns(w http.ResponseWriter, r *http.Request) {
 	if r.Method != http.MethodGet {
 		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
 		return
 	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(c.ListPatterns())
+}
 
-	id := r.PathValue("id")
+// HandleRollouts returns normalized Kubernetes Deployment rollout history.
-	issue, ok := c.GetIssue(id)
+func (c *Collector) HandleRollouts(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.KubernetesEnabled() {
+		jsonError(w, http.StatusNotImplemented, c.KubernetesError())
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(c.ListRollouts())
+}
+
+func (c *Collector) HandleRolloutByID(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.KubernetesEnabled() {
+		jsonError(w, http.StatusNotImplemented, c.KubernetesError())
+		return
+	}
+	rollout, ok := c.GetRollout(r.PathValue("id"))
+	if !ok {
+		http.Error(w, "rollout not found", http.StatusNotFound)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(rollout)
+}
+
+func (c *Collector) HandleRolloutImpact(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.KubernetesEnabled() {
+		jsonError(w, http.StatusNotImplemented, c.KubernetesError())
+		return
+	}
+	rollout, ok := c.GetRollout(r.PathValue("id"))
+	if !ok {
+		http.Error(w, "rollout not found", http.StatusNotFound)
+		return
+	}
+	response := map[string]interface{}{"configured": c.prometheus != nil, "rollout": rollout, "points": []interface{}{}}
+	if c.prometheus != nil {
+		start := time.Unix(0, rollout.StartedAt).Add(-15 * time.Minute)
+		end := time.Now()
+		if start.Before(end.Add(-24 * time.Hour)) {
+			start = end.Add(-24 * time.Hour)
+		}
+		points, err := c.prometheus.ErrorRate(r.Context(), rollout.Service, rollout.Release, start, end)
+		if err != nil {
+			jsonError(w, http.StatusBadGateway, err.Error())
+			return
+		}
+		response["points"] = points
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(response)
+}
+
+func jsonError(w http.ResponseWriter, status int, message string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(map[string]string{"error": message})
+}
+
+// HandleIssueSimilar returns explainable, non-merging related-issue suggestions.
+func (c *Collector) HandleIssueSimilar(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	limit := parseLimit(r, 10)
+	result, ok := c.SimilarIssues(r.PathValue("id"), limit)
 	if !ok {
 		http.Error(w, "issue not found", http.StatusNotFound)
 		return
 	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(result)
+}
+
+// HandleSimilarityFeedback records a human review label for a pair of issues.
+// Body: {"issueId":"...", "decision":"same|different"}.
+func (c *Collector) HandleSimilarityFeedback(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	var req struct {
+		IssueID  string `json:"issueId"`
+		Decision string `json:"decision"`
+	}
+	if err := json.NewDecoder(io.LimitReader(r.Body, 4096)).Decode(&req); err != nil {
+		http.Error(w, "invalid JSON body", http.StatusBadRequest)
+		return
+	}
+	if !c.SetSimilarityFeedback(r.PathValue("id"), req.IssueID, req.Decision) {
+		http.Error(w, "invalid issue pair or decision", http.StatusBadRequest)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]string{"status": "recorded", "decision": req.Decision})
+}
+
+// HandleIssueStatus updates the triage state (open/resolved/ignored) of an
+// issue. Body: {"status": "resolved"}.
+func (c *Collector) HandleIssueStatus(w http.ResponseWriter, r *http.Request, id string) {
+	var req struct {
+		Status string `json:"status"`
+	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, 4096))
+	if err != nil {
+		http.Error(w, "failed to read request body", http.StatusBadRequest)
+		return
+	}
+	if err := json.Unmarshal(body, &req); err != nil {
+		http.Error(w, "invalid JSON body", http.StatusBadRequest)
+		return
+	}
+
+	issue, ok := c.SetIssueStatus(id, req.Status)
+	if !ok {
+		http.Error(w, "issue not found or invalid status", http.StatusBadRequest)
+		return
+	}
 
 	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(issue); err != nil {
 		log.Printf("failed to encode issue: %v", err)
-		http.Error(w, "failed to encode issue", http.StatusInternalServerError)
-		return
 	}
 }
 
@@ -418,3 +668,102 @@ func (c *Collector) HandleHealth(w http.ResponseWriter, r *http.Request) {
 		log.Printf("failed to encode health check: %v", err)
 	}
 }
+
+// statsEnabled reports whether an aggregate-capable store is configured.
+func (c *Collector) statsEnabled() bool { return c.statsQuerier != nil }
+
+// statsNotEnabled writes the response used when the file store is active:
+// a stable 501 the UI treats as "view not available".
+func statsNotEnabled(w http.ResponseWriter) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusNotImplemented)
+	_ = json.NewEncoder(w).Encode(map[string]string{
+		"error": "stats require OTEL_CLICKHOUSE_ADDR (ClickHouse storage) to be configured",
+	})
+}
+
+// parseStatsCommon reads the shared service/hours params for stats endpoints.
+func parseStatsCommon(r *http.Request) (service string, hours int) {
+	service = r.URL.Query().Get("service")
+	hours = 72
+	if h, err := strconv.Atoi(r.URL.Query().Get("hours")); err == nil && h > 0 && h <= 24*90 {
+		hours = h
+	}
+	return service, hours
+}
+
+// HandleStatsErrors returns hourly/daily total and error counts for the
+// trends graphs. Only available with ClickHouse storage.
+func (c *Collector) HandleStatsErrors(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.statsEnabled() {
+		statsNotEnabled(w)
+		return
+	}
+
+	service, hours := parseStatsCommon(r)
+	daily := hours > 24*14 // beyond two weeks, daily buckets read better
+	points, err := c.statsQuerier.ErrorStats(r.Context(), service, hours, daily)
+	if err != nil {
+		log.Printf("failed to compute error stats: %v", err)
+		http.Error(w, "failed to compute error stats", http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{
+		"granularity": map[bool]string{true: "day", false: "hour"}[daily],
+		"points":      points,
+	})
+}
+
+// HandleStatsReleases returns per-release error-count time series so two
+// deploys can be compared line-by-line. Only available with ClickHouse.
+func (c *Collector) HandleStatsReleases(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.statsEnabled() {
+		statsNotEnabled(w)
+		return
+	}
+
+	service, hours := parseStatsCommon(r)
+	series, err := c.statsQuerier.ReleaseSeries(r.Context(), service, hours)
+	if err != nil {
+		log.Printf("failed to compute release stats: %v", err)
+		http.Error(w, "failed to compute release stats", http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{
+		"series": series,
+	})
+}
+
+// HandleStatsServices lists the distinct services present in the rollup.
+func (c *Collector) HandleStatsServices(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.statsEnabled() {
+		statsNotEnabled(w)
+		return
+	}
+
+	services, err := c.statsQuerier.Services(r.Context())
+	if err != nil {
+		log.Printf("failed to list services: %v", err)
+		http.Error(w, "failed to list services", http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{"services": services})
+}

internal/collector/issue_extract.go

@@ -1,6 +1,8 @@
 package collector
 
 import (
+	"fmt"
+	"strconv"
 	"strings"
 
 	"git.tornberg.me/go-otel/internal/issues"
@@ -9,46 +11,46 @@ import (
 	"go.opentelemetry.io/collector/pdata/ptrace"
 )
 
-// maxIssueTitleLen bounds how much of a log body is kept as an issue title,
+// maxIssueTitleLen bounds how much of a log body or span title is kept,
-// so a single noisy log line can't blow up issue payload size.
+// so a single noisy error can't blow up issue payload size.
 const maxIssueTitleLen = 200
 
-// extractSpanErrors scans traces for spans with an error status and records
+// extractSpanErrors scans traces for spans with an error status or error HTTP/gRPC
-// one issue occurrence per error found.
+// status code, recording one issue occurrence per error found. It reports
-func extractSpanErrors(tracker *issues.Tracker, td ptrace.Traces, timestamp int64) {
+// whether any error was seen, so the caller can notify live clients.
+func extractSpanErrors(tracker *issues.Tracker, td ptrace.Traces, timestamp int64) bool {
+	found := false
 	resourceSpans := td.ResourceSpans()
 	for i := 0; i < resourceSpans.Len(); i++ {
 		rs := resourceSpans.At(i)
-		service := resourceServiceName(rs.Resource())
 
 		scopeSpans := rs.ScopeSpans()
 		for j := 0; j < scopeSpans.Len(); j++ {
 			spans := scopeSpans.At(j).Spans()
 			for k := 0; k < spans.Len(); k++ {
 				span := spans.At(k)
-				if span.Status().Code() != ptrace.StatusCodeError {
+				if !isSpanError(span) {
 					continue
 				}
 
+				service := spanServiceName(rs.Resource(), span)
 				title := spanErrorTitle(span)
-				tracker.Record("span_error", service, title, issues.Sample{
+				found = true
-					Timestamp: timestamp,
+				tracker.Record("span_error", service, title, spanSample(span, rs.Resource(), timestamp, title))
-					TraceID:   idString(span.TraceID()),
-					SpanID:    idString(span.SpanID()),
-					Message:   title,
-				})
 			}
 		}
 	}
+	return found
 }
 
 // extractLogErrors scans logs for error/fatal severity records and records
-// one issue occurrence per error found.
+// one issue occurrence per error found. It reports whether any error was
-func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) {
+// seen, so the caller can notify live clients.
+func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) bool {
+	found := false
 	resourceLogs := ld.ResourceLogs()
 	for i := 0; i < resourceLogs.Len(); i++ {
 		rl := resourceLogs.At(i)
-		service := resourceServiceName(rl.Resource())
 
 		scopeLogs := rl.ScopeLogs()
 		for j := 0; j < scopeLogs.Len(); j++ {
@@ -59,23 +61,223 @@ func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) {
 					continue
 				}
 
+				service := logServiceName(rl.Resource(), record)
 				title := logErrorTitle(record)
+				found = true
 				kind := "log_error"
-				var url string
+				sample := logSample(record, rl.Resource(), timestamp, title)
 				if attrString(record.Attributes(), "source") == "frontend" {
 					kind = "frontend_error"
-					url = attrString(record.Attributes(), "browser.url")
+					sample.URL = attrString(record.Attributes(), "browser.url")
 				}
-				tracker.Record(kind, service, title, issues.Sample{
+				tracker.Record(kind, service, title, sample)
-					Timestamp: timestamp,
-					TraceID:   idString(record.TraceID()),
-					SpanID:    idString(record.SpanID()),
-					Message:   title,
-					URL:       url,
-				})
 			}
 		}
 	}
+	return found
+}
+
+// spanSample builds an enriched issue sample from a span's attributes:
+// stacktrace from exception events, HTTP request context from semconv
+// attributes (old and new names), and client/release metadata.
+func spanSample(span ptrace.Span, res pcommon.Resource, timestamp int64, title string) issues.Sample {
+	sample := issues.Sample{
+		Timestamp:   timestamp,
+		TraceID:     idString(span.TraceID()),
+		SpanID:      idString(span.SpanID()),
+		Message:     title,
+		HTTPMethod:  firstAttr(span.Attributes(), "http.request.method", "http.method"),
+		HTTPRoute:   firstAttr(span.Attributes(), "http.route", "http.target", "url.path"),
+		UserAgent:   firstAttr(span.Attributes(), "user_agent.original", "http.user_agent"),
+		SourceIP:    sourceIP(span.Attributes(), res.Attributes()),
+		Release:     firstAttr(span.Attributes(), "service.release"),
+		Environment: resourceEnvironment(res),
+	}
+	if status, ok := getHTTPStatusCode(span.Attributes()); ok {
+		sample.HTTPStatus = status
+	}
+	sample.Stacktrace = spanStacktrace(span)
+	return sample
+}
+
+// logSample builds an enriched issue sample from a log record's attributes.
+func logSample(record plog.LogRecord, res pcommon.Resource, timestamp int64, title string) issues.Sample {
+	sample := issues.Sample{
+		Timestamp:   timestamp,
+		TraceID:     idString(record.TraceID()),
+		SpanID:      idString(record.SpanID()),
+		Message:     title,
+		Stacktrace:  attrString(record.Attributes(), "exception.stacktrace"),
+		HTTPMethod:  firstAttr(record.Attributes(), "http.request.method", "http.method"),
+		HTTPRoute:   firstAttr(record.Attributes(), "http.route", "http.target", "url.path"),
+		UserAgent:   firstAttr(record.Attributes(), "user_agent.original", "http.user_agent", "browser.userAgent"),
+		SourceIP:    sourceIP(record.Attributes(), res.Attributes()),
+		Release:     firstAttr(record.Attributes(), "service.release"),
+		Environment: firstAttr(record.Attributes(), "service.environment"),
+		Breadcrumbs: attrString(record.Attributes(), "breadcrumbs"),
+	}
+	if status, ok := getHTTPStatusCode(record.Attributes()); ok {
+		sample.HTTPStatus = status
+	}
+	if sample.Environment == "" {
+		sample.Environment = resourceEnvironment(res)
+	}
+	return sample
+}
+
+// sourceIP extracts common client-IP attributes without trusting arbitrary
+// request headers. Instrumentation may attach these to either the span/log or
+// its resource.
+func sourceIP(attrs, resource pcommon.Map) string {
+	ip := firstAttr(attrs,
+		"client.address",
+		"client.ip",
+		"http.client_ip",
+		"net.peer.ip",
+		"network.peer.address",
+		"http.request.header.x-real-ip",
+		"http.request.header.x-forwarded-for",
+		"http.request.header.x_real_ip",
+		"http.request.header.x_forwarded_for",
+	)
+	if ip == "" {
+		ip = firstAttr(resource,
+			"client.address",
+			"client.ip",
+			"http.client_ip",
+			"net.peer.ip",
+			"network.peer.address",
+		)
+	}
+	return strings.TrimSpace(strings.SplitN(ip, ",", 2)[0])
+}
+
+// spanStacktrace returns exception.stacktrace from the span's exception
+// events, falling back to the span status message when it looks like an
+// exception ("type: message" is how spanErrorTitle renders one).
+func spanStacktrace(span ptrace.Span) string {
+	events := span.Events()
+	for i := 0; i < events.Len(); i++ {
+		event := events.At(i)
+		if event.Name() != "exception" {
+			continue
+		}
+		if st := attrString(event.Attributes(), "exception.stacktrace"); st != "" {
+			return truncate(st)
+		}
+	}
+	return ""
+}
+
+// firstAttr returns the first present attribute among the given keys.
+func firstAttr(attrs pcommon.Map, keys ...string) string {
+	for _, key := range keys {
+		if v, ok := attrs.Get(key); ok {
+			return v.AsString()
+		}
+	}
+	return ""
+}
+
+// getHTTPStatusCode extracts an HTTP status code from attributes, supporting
+// both new (http.response.status_code) and old (http.status_code) semconv keys,
+// stored as integer or string.
+func getHTTPStatusCode(attrs pcommon.Map) (int, bool) {
+	for _, key := range []string{"http.response.status_code", "http.status_code"} {
+		if v, ok := attrs.Get(key); ok {
+			switch v.Type() {
+			case pcommon.ValueTypeInt:
+				return int(v.Int()), true
+			case pcommon.ValueTypeStr:
+				if code, err := strconv.Atoi(v.Str()); err == nil {
+					return code, true
+				}
+			default:
+				if code, err := strconv.Atoi(v.AsString()); err == nil {
+					return code, true
+				}
+			}
+		}
+	}
+	return 0, false
+}
+
+// httpStatusText maps common HTTP error status codes to their canonical reason phrase.
+func httpStatusText(code int) string {
+	switch code {
+	case 400:
+		return "Bad Request"
+	case 401:
+		return "Unauthorized"
+	case 402:
+		return "Payment Required"
+	case 403:
+		return "Forbidden"
+	case 404:
+		return "Not Found"
+	case 405:
+		return "Method Not Allowed"
+	case 406:
+		return "Not Acceptable"
+	case 408:
+		return "Request Timeout"
+	case 409:
+		return "Conflict"
+	case 410:
+		return "Gone"
+	case 415:
+		return "Unsupported Media Type"
+	case 422:
+		return "Unprocessable Entity"
+	case 429:
+		return "Too Many Requests"
+	case 500:
+		return "Internal Server Error"
+	case 501:
+		return "Not Implemented"
+	case 502:
+		return "Bad Gateway"
+	case 503:
+		return "Service Unavailable"
+	case 504:
+		return "Gateway Timeout"
+	default:
+		if code >= 400 && code < 500 {
+			return "Client Error"
+		}
+		if code >= 500 && code < 600 {
+			return "Server Error"
+		}
+		return ""
+	}
+}
+
+// isSpanError determines if a span represents an error, checking:
+// 1. Explicit StatusCodeError
+// 2. HTTP response status >= 400
+// 3. gRPC error code (> 0)
+func isSpanError(span ptrace.Span) bool {
+	if span.Status().Code() == ptrace.StatusCodeError {
+		return true
+	}
+	if status, ok := getHTTPStatusCode(span.Attributes()); ok && status >= 400 {
+		return true
+	}
+	if grpcStr := firstAttr(span.Attributes(), "rpc.grpc.status_code"); grpcStr != "" {
+		if code, err := strconv.Atoi(grpcStr); err == nil && code > 0 {
+			return true
+		}
+	}
+	return false
+}
+
+// resourceEnvironment reads deployment.environment (new semconv) or
+// deployment.environment.name from a resource.
+func resourceEnvironment(res pcommon.Resource) string {
+	if v := attrString(res.Attributes(), "deployment.environment.name"); v != "" {
+		return v
+	}
+	return attrString(res.Attributes(), "deployment.environment")
 }
 
 func isErrorLog(record plog.LogRecord) bool {
@@ -91,6 +293,7 @@ func isErrorLog(record plog.LogRecord) bool {
 }
 
 func spanErrorTitle(span ptrace.Span) string {
+	// 1. Exception events take top priority (panics, exceptions with stacktrace).
 	events := span.Events()
 	for i := 0; i < events.Len(); i++ {
 		event := events.At(i)
@@ -109,10 +312,72 @@ func spanErrorTitle(span ptrace.Span) string {
 		}
 	}
 
-	if msg := span.Status().Message(); msg != "" {
+	// 2. HTTP error titles: "HTTP 404 (Not Found): GET /path" or "HTTP 500 (Internal Server Error): /api"
-		return span.Name() + ": " + msg
+	if status, ok := getHTTPStatusCode(span.Attributes()); ok && status >= 400 {
+		statusDesc := fmt.Sprintf("HTTP %d", status)
+		if txt := httpStatusText(status); txt != "" {
+			statusDesc = fmt.Sprintf("HTTP %d (%s)", status, txt)
+		}
+
+		method := firstAttr(span.Attributes(), "http.request.method", "http.method")
+		route := firstAttr(span.Attributes(), "http.route", "http.target", "url.path")
+
+		var endpoint string
+		if method != "" && route != "" {
+			endpoint = method + " " + route
+		} else if route != "" {
+			endpoint = route
+		} else if span.Name() != "" && !strings.EqualFold(span.Name(), "http") {
+			endpoint = span.Name()
+		}
+
+		statusMsg := span.Status().Message()
+		switch {
+		case endpoint != "" && statusMsg != "" && !strings.EqualFold(statusMsg, httpStatusText(status)):
+			return truncate(statusDesc + ": " + endpoint + " (" + statusMsg + ")")
+		case endpoint != "":
+			return truncate(statusDesc + ": " + endpoint)
+		case statusMsg != "":
+			return truncate(statusDesc + ": " + statusMsg)
+		default:
+			return statusDesc
+		}
 	}
-	return span.Name() + " (error)"
+
+	// 3. gRPC error titles
+	if grpcStr := firstAttr(span.Attributes(), "rpc.grpc.status_code"); grpcStr != "" {
+		if code, err := strconv.Atoi(grpcStr); err == nil && code > 0 {
+			method := firstAttr(span.Attributes(), "rpc.method")
+			service := firstAttr(span.Attributes(), "rpc.service")
+			target := span.Name()
+			if service != "" && method != "" {
+				target = service + "/" + method
+			}
+			msg := span.Status().Message()
+			if msg != "" {
+				return truncate(fmt.Sprintf("gRPC error %d: %s (%s)", code, target, msg))
+			}
+			if target != "" {
+				return truncate(fmt.Sprintf("gRPC error %d: %s", code, target))
+			}
+			return fmt.Sprintf("gRPC error %d", code)
+		}
+	}
+
+	// 4. Status message with span name
+	if msg := span.Status().Message(); msg != "" {
+		if span.Name() != "" {
+			return truncate(span.Name() + ": " + msg)
+		}
+		return truncate(msg)
+	}
+
+	// 5. Fallback to span name
+	if span.Name() != "" {
+		return truncate(span.Name() + " (error)")
+	}
+
+	return "span error"
 }
 
 func logErrorTitle(record plog.LogRecord) string {
@@ -139,8 +404,28 @@ func truncate(s string) string {
 	return s[:maxIssueTitleLen] + "…"
 }
 
+func spanServiceName(res pcommon.Resource, span ptrace.Span) string {
+	if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
+		return name
+	}
+	if name := firstAttr(span.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
+		return name
+	}
+	return "unknown"
+}
+
+func logServiceName(res pcommon.Resource, record plog.LogRecord) string {
+	if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
+		return name
+	}
+	if name := firstAttr(record.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
+		return name
+	}
+	return "unknown"
+}
+
 func resourceServiceName(res pcommon.Resource) string {
-	if name := attrString(res.Attributes(), "service.name"); name != "" {
+	if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
 		return name
 	}
 	return "unknown"

internal/honeypot/honeypot.go

@@ -0,0 +1,245 @@
+// Package honeypot tracks common vulnerability-scanner requests and serves an
+// inert decoy response. Its promoted IP list can be consumed by an ingress
+// controller or a separate synchronizer.
+package honeypot
+
+import (
+	"crypto/subtle"
+	"encoding/json"
+	"net"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	defaultThreshold = 3
+	defaultWindow    = 15 * time.Minute
+	defaultTTL       = 1 * time.Hour
+	maxTracked       = 10000
+)
+
+type Config struct {
+	Threshold int
+	Window    time.Duration
+	TTL       time.Duration
+	Delay     time.Duration
+	Token     string
+}
+
+type entry struct {
+	Count         int
+	FirstSeen     time.Time
+	LastSeen      time.Time
+	PromotedUntil time.Time
+}
+
+// IP describes an IP promoted for ingress honeypot routing.
+type IP struct {
+	Address       string    `json:"ip"`
+	Count         int       `json:"count"`
+	FirstSeen     time.Time `json:"first_seen"`
+	LastSeen      time.Time `json:"last_seen"`
+	PromotedUntil time.Time `json:"promoted_until"`
+}
+
+type Tracker struct {
+	threshold int
+	window    time.Duration
+	ttl       time.Duration
+	delay     time.Duration
+	token     string
+
+	mu      sync.Mutex
+	clients map[string]entry
+}
+
+func New(cfg Config) *Tracker {
+	if cfg.Threshold <= 0 {
+		cfg.Threshold = defaultThreshold
+	}
+	if cfg.Window <= 0 {
+		cfg.Window = defaultWindow
+	}
+	if cfg.TTL <= 0 {
+		cfg.TTL = defaultTTL
+	}
+	if cfg.Delay < 0 {
+		cfg.Delay = 0
+	}
+	if cfg.Delay > 10*time.Second {
+		cfg.Delay = 10 * time.Second
+	}
+	return &Tracker{threshold: cfg.Threshold, window: cfg.Window, ttl: cfg.TTL, delay: cfg.Delay, token: cfg.Token, clients: make(map[string]entry)}
+}
+
+func ConfigFromEnv() Config {
+	cfg := Config{Threshold: defaultThreshold, Window: defaultWindow, TTL: defaultTTL, Delay: 0, Token: os.Getenv("OTEL_HONEYPOT_TOKEN")}
+	if n, err := strconv.Atoi(os.Getenv("OTEL_HONEYPOT_THRESHOLD")); err == nil && n > 0 {
+		cfg.Threshold = n
+	}
+	if d, err := time.ParseDuration(os.Getenv("OTEL_HONEYPOT_WINDOW")); err == nil && d > 0 {
+		cfg.Window = d
+	}
+	if d, err := time.ParseDuration(os.Getenv("OTEL_HONEYPOT_TTL")); err == nil && d > 0 {
+		cfg.TTL = d
+	}
+	if d, err := time.ParseDuration(os.Getenv("OTEL_HONEYPOT_DELAY")); err == nil && d >= 0 {
+		cfg.Delay = d
+	}
+	return cfg
+}
+
+// Middleware records scanner paths and returns decoys. All other requests
+// continue to the wrapped application unchanged.
+func (t *Tracker) Middleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !isSuspiciousPath(r.URL.Path) {
+			next.ServeHTTP(w, r)
+			return
+		}
+		ip := clientIP(r)
+		promoted := t.record(ip)
+		if promoted && t.delay > 0 {
+			timer := time.NewTimer(t.delay)
+			select {
+			case <-timer.C:
+			case <-r.Context().Done():
+				if !timer.Stop() {
+					<-timer.C
+				}
+				return
+			}
+		}
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		w.Header().Set("Cache-Control", "no-store")
+		w.WriteHeader(http.StatusOK)
+		if strings.Contains(strings.ToLower(r.URL.Path), ".env") {
+			_, _ = w.Write([]byte("# decoy configuration - no real secrets\nAPP_ENV=production\nAPP_DEBUG=false\nDB_HOST=127.0.0.1\nDB_DATABASE=example_decoy\nDB_USERNAME=decoy_reader\nDB_PASSWORD=not-a-real-password\n"))
+			return
+		}
+		_, _ = w.Write([]byte("-- decoy diagnostic response\n-- no database connection was made\n-- DROP TABLE users;\n-- DROP TABLE sessions;\n-- inert text only\n"))
+	})
+}
+
+func (t *Tracker) record(ip string) bool {
+	now := time.Now()
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	state, ok := t.clients[ip]
+	if !ok || now.Sub(state.LastSeen) > t.window {
+		state = entry{FirstSeen: now}
+	}
+	state.Count++
+	state.LastSeen = now
+	if state.Count > t.threshold {
+		state.PromotedUntil = now.Add(t.ttl)
+	}
+	t.clients[ip] = state
+	if len(t.clients) > maxTracked {
+		t.evictOldest()
+	}
+	return !state.PromotedUntil.IsZero() && now.Before(state.PromotedUntil)
+}
+
+// PromotedIPs returns a snapshot suitable for an ingress synchronizer.
+func (t *Tracker) PromotedIPs() []IP {
+	now := time.Now()
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	result := make([]IP, 0)
+	for address, state := range t.clients {
+		if state.PromotedUntil.IsZero() || !now.Before(state.PromotedUntil) {
+			continue
+		}
+		result = append(result, IP{Address: address, Count: state.Count, FirstSeen: state.FirstSeen, LastSeen: state.LastSeen, PromotedUntil: state.PromotedUntil})
+	}
+	return result
+}
+
+// HandleIPs exposes the promoted list only when OTEL_HONEYPOT_TOKEN is set.
+// It is intended for a private ingress-side synchronizer, not public clients.
+func (t *Tracker) HandleIPs(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet || t.token == "" || !validToken(r, t.token) {
+		w.Header().Set("WWW-Authenticate", `Bearer realm="honeypot"`)
+		http.Error(w, "unauthorized", http.StatusUnauthorized)
+		return
+	}
+	t.writeIPs(w)
+}
+
+// HandleIPsForUser is used behind the application's normal authenticated UI
+// middleware. The internal machine endpoint remains protected by Token.
+func (t *Tracker) HandleIPsForUser(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	t.writeIPs(w)
+}
+
+func (t *Tracker) writeIPs(w http.ResponseWriter) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Cache-Control", "no-store")
+	_ = json.NewEncoder(w).Encode(t.PromotedIPs())
+}
+
+func validToken(r *http.Request, token string) bool {
+	value := strings.TrimSpace(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "))
+	if value == "" {
+		value = r.Header.Get("X-Honeypot-Token")
+	}
+	return len(value) == len(token) && subtle.ConstantTimeCompare([]byte(value), []byte(token)) == 1
+}
+
+func (t *Tracker) evictOldest() {
+	var address string
+	var oldest time.Time
+	for ip, state := range t.clients {
+		if address == "" || state.LastSeen.Before(oldest) {
+			address, oldest = ip, state.LastSeen
+		}
+	}
+	if address != "" {
+		delete(t.clients, address)
+	}
+}
+
+func isSuspiciousPath(path string) bool {
+	path = strings.ToLower(path)
+	for _, marker := range []string{
+		"/.env", "/.git/", "/.svn/", "/wp-", "/xmlrpc.php", "/phpmyadmin",
+		"/pma/", "/vendor/phpunit", "/eval-stdin.php", "/cgi-bin/", "/shell.php",
+		"/cmd.php", "/upload.php", "/v2/keys", "/var/task/", "/dashboard/env-", "/app.js",
+	} {
+		if strings.Contains(path, marker) {
+			return true
+		}
+	}
+	for _, suffix := range []string{".php", ".ini", ".bak", ".swp", ".yml", ".yaml", ".sql", ".json", ".config", ".cfg"} {
+		if strings.HasSuffix(path, suffix) {
+			return true
+		}
+	}
+	return false
+}
+
+func clientIP(r *http.Request) string {
+	// Prefer the single-value header normally overwritten by ingress-nginx.
+	// X-Forwarded-For can contain client-supplied values when a proxy appends
+	// rather than replaces it, so it is only a fallback.
+	if real := strings.TrimSpace(r.Header.Get("X-Real-IP")); real != "" {
+		return real
+	}
+	if forwarded := r.Header.Get("X-Forwarded-For"); forwarded != "" {
+		return strings.TrimSpace(strings.SplitN(forwarded, ",", 2)[0])
+	}
+	host, _, err := net.SplitHostPort(r.RemoteAddr)
+	if err == nil {
+		return host
+	}
+	return r.RemoteAddr
+}

internal/honeypot/honeypot_test.go

@@ -0,0 +1,41 @@
+package honeypot
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestPromotesAfterThresholdAndExpires(t *testing.T) {
+	tracker := New(Config{Threshold: 2, Window: time.Minute, TTL: time.Millisecond, Token: "secret"})
+	for i := 0; i < 3; i++ {
+		req := httptest.NewRequest(http.MethodGet, "https://example.test/.env", nil)
+		req.RemoteAddr = "192.0.2.10:1234"
+		tracker.Middleware(http.HandlerFunc(http.NotFound)).ServeHTTP(httptest.NewRecorder(), req)
+	}
+	if got := len(tracker.PromotedIPs()); got != 1 {
+		t.Fatalf("got %d promoted IPs, want 1", got)
+	}
+	time.Sleep(2 * time.Millisecond)
+	if got := len(tracker.PromotedIPs()); got != 0 {
+		t.Fatalf("got %d expired IPs, want 0", got)
+	}
+}
+
+func TestHandleIPsRequiresToken(t *testing.T) {
+	tracker := New(Config{Threshold: 1, Token: "secret"})
+	recorder := httptest.NewRecorder()
+	tracker.HandleIPs(recorder, httptest.NewRequest(http.MethodGet, "https://example.test/internal/honeypot/ips", nil))
+	if recorder.Code != http.StatusUnauthorized {
+		t.Fatalf("got %d, want 401", recorder.Code)
+	}
+	recorder = httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodGet, "https://example.test/internal/honeypot/ips", nil)
+	req.Header.Set("Authorization", "Bearer secret")
+	tracker.HandleIPs(recorder, req)
+	if recorder.Code != http.StatusOK || !strings.HasPrefix(recorder.Body.String(), "[]") {
+		t.Fatalf("got %d and %q", recorder.Code, recorder.Body.String())
+	}
+}

internal/issues/tracker.go

@@ -5,9 +5,11 @@ package issues
 import (
 	"fmt"
 	"hash/fnv"
+	"regexp"
 	"sort"
 	"strings"
 	"sync"
+	"time"
 )
 
 // maxIssues bounds memory use: once exceeded, the least-recently-seen issue
@@ -17,6 +19,18 @@ const maxIssues = 2000
 // maxSamples bounds how many sample occurrences are kept per issue.
 const maxSamples = 5
 
+// trendWindow bounds how far back the per-hour trend buckets are kept.
+const trendWindow = 48 * time.Hour
+
+// Issue lifecycle states. New occurrences automatically reopen resolved
+// issues (a regression), but stay merged into ignored ones until the user
+// explicitly sets the state back to open.
+const (
+	StatusOpen     = "open"
+	StatusResolved = "resolved"
+	StatusIgnored  = "ignored"
+)
+
 // Sample is one occurrence of an issue.
 type Sample struct {
 	Timestamp int64  `json:"timestamp"`
@@ -26,38 +40,121 @@ type Sample struct {
 	// URL is the page URL an occurrence happened on, populated only for
 	// browser-originated (otel-rum) errors.
 	URL string `json:"url,omitempty"`
+	// Stacktrace is the exception.stacktrace attribute of the occurrence,
+	// when the source telemetry carried one.
+	Stacktrace         string `json:"stacktrace,omitempty"`
+	ResolvedStacktrace string `json:"resolvedStacktrace,omitempty"`
+	// HTTP request context, extracted from the span/log attributes when the
+	// error happened inside an HTTP handler (semconv v1.x old and new names
+	// are both handled by the extractor).
+	HTTPMethod string `json:"httpMethod,omitempty"`
+	HTTPRoute  string `json:"httpRoute,omitempty"`
+	HTTPStatus int    `json:"httpStatus,omitempty"`
+	// Client/release context.
+	UserAgent   string `json:"userAgent,omitempty"`
+	SourceIP    string `json:"sourceIp,omitempty"`
+	Release     string `json:"release,omitempty"`
+	Environment string `json:"environment,omitempty"`
+	// Breadcrumbs is the raw JSON captured by otel-rum.js before the error
+	// (fetch/console events), present only on frontend issues.
+	Breadcrumbs string `json:"breadcrumbs,omitempty"`
+	// Deterministic, non-PII features used for grouping and review.
+	NormalizedMessage string `json:"normalizedMessage,omitempty"`
+	NormalizedRoute   string `json:"normalizedRoute,omitempty"`
+	StatusClass       string `json:"statusClass,omitempty"`
+	StackSignature    string `json:"stackSignature,omitempty"`
 }
 
 // Issue is a deduplicated group of related errors, fingerprinted by kind,
-// service and title.
+// service and a normalized form of the title.
 type Issue struct {
 	ID               string   `json:"id"`
 	Title            string   `json:"title"`
-	Kind             string   `json:"kind"` // "span_error" or "log_error"
+	Kind             string   `json:"kind"` // "span_error", "log_error" or "frontend_error"
 	Service          string   `json:"service"`
 	Count            int      `json:"count"`
 	FirstSeen        int64    `json:"firstSeen"`
 	LastSeen         int64    `json:"lastSeen"`
 	AffectedServices []string `json:"affectedServices"`
 	Samples          []Sample `json:"samples"`
+	// Trend maps UTC hour buckets ("2006-01-02T15:00") to occurrence counts
+	// over the last trendWindow, for sparklines and spike detection.
+	Trend map[string]int `json:"trend,omitempty"`
+	// Status is the triage state: open, resolved or ignored.
+	Status     string `json:"status"`
+	ResolvedAt int64  `json:"resolvedAt,omitempty"`
+	// NormalizedTitle is the deterministic, redacted form used for exact grouping.
+	NormalizedTitle string `json:"normalizedTitle,omitempty"`
+	// PatternID groups related exact issues without merging their operational IDs.
+	PatternID        string `json:"patternId,omitempty"`
+	PatternSignature string `json:"patternSignature,omitempty"`
+}
+
+// SimilarIssue is a deterministic related-issue suggestion. Suggestions never
+// change issue IDs and include the reasons that produced the score.
+type SimilarIssue struct {
+	Issue    Issue    `json:"issue"`
+	Score    float64  `json:"score"`
+	Reasons  []string `json:"reasons"`
+	Decision string   `json:"decision,omitempty"`
+}
+
+// Pattern is a UI-friendly aggregate of exact issues.
+type Pattern struct {
+	ID              string   `json:"id"`
+	Signature       string   `json:"signature"`
+	Service         string   `json:"service"`
+	Kind            string   `json:"kind"`
+	IssueCount      int      `json:"issueCount"`
+	OccurrenceCount int      `json:"occurrenceCount"`
+	LastSeen        int64    `json:"lastSeen"`
+	IssueIDs        []string `json:"issueIds"`
 }
 
 // Tracker incrementally fingerprints and deduplicates errors as they're
 // recorded. It is safe for concurrent use.
 type Tracker struct {
-	mu     sync.Mutex
+	mu       sync.Mutex
-	issues map[string]*Issue
+	issues   map[string]*Issue
+	feedback map[string]string
+}
+
+// ListOptions filters and paginates the issue list. Issues remain ordered by
+// LastSeen descending; AfterLastSeen/AfterID form an opaque-to-callers cursor
+// boundary and make pagination stable when multiple issues share a timestamp.
+type ListOptions struct {
+	Service       string
+	Kind          string
+	Status        string
+	Title         string
+	SourceIP      string
+	Route         string
+	Since         int64
+	Until         int64
+	AfterLastSeen int64
+	AfterID       string
+	Limit         int
+}
+
+// ListPage is a filtered page of issues. HasMore indicates that another page
+// can be requested using the final issue as the cursor boundary.
+type ListPage struct {
+	Issues  []Issue
+	HasMore bool
 }
 
 // NewTracker creates an empty Tracker.
 func NewTracker() *Tracker {
-	return &Tracker{issues: make(map[string]*Issue)}
+	return &Tracker{issues: make(map[string]*Issue), feedback: make(map[string]string)}
 }
 
 // Record adds an occurrence of an error, creating a new issue or merging into
-// an existing one with a matching (kind, service, title) fingerprint.
+// an existing one with a matching (kind, service, normalized title)
+// fingerprint.
 func (t *Tracker) Record(kind, service, title string, sample Sample) {
-	id := fingerprint(kind, service, title)
+	normalizeSample(&sample, title)
+	id := fingerprint(kind, service, NormalizeTitle(title))
+	bucket := HourBucket(sample.Timestamp)
 
 	t.mu.Lock()
 	defer t.mu.Unlock()
@@ -68,24 +165,282 @@ func (t *Tracker) Record(kind, service, title string, sample Sample) {
 			t.evictOldestLocked()
 		}
 		issue = &Issue{
-			ID:        id,
+			ID:              id,
-			Title:     title,
+			Title:           title,
-			Kind:      kind,
+			Kind:            kind,
-			Service:   service,
+			Service:         service,
-			FirstSeen: sample.Timestamp,
+			FirstSeen:       sample.Timestamp,
+			Status:          StatusOpen,
+			NormalizedTitle: NormalizeTitle(title),
 		}
+		issue.PatternSignature = patternSignature(kind, service, title, sample)
+		issue.PatternID = fingerprint("pattern", issue.PatternSignature)
 		t.issues[id] = issue
 	}
 
 	issue.Count++
 	issue.LastSeen = sample.Timestamp
 	issue.AffectedServices = addUnique(issue.AffectedServices, service)
+	if issue.Trend == nil {
+		issue.Trend = make(map[string]int)
+	}
+	issue.Trend[bucket]++
+	pruneTrendLocked(issue.Trend, sample.Timestamp)
+	// A resolved issue that fires again is a regression: reopen it. Ignored
+	// issues keep their state (that's the point of ignoring a noisy one).
+	if issue.Status == StatusResolved {
+		issue.Status = StatusOpen
+		issue.ResolvedAt = 0
+	}
 	issue.Samples = append(issue.Samples, sample)
 	if len(issue.Samples) > maxSamples {
 		issue.Samples = issue.Samples[len(issue.Samples)-maxSamples:]
 	}
 }
 
+// Similar returns bounded, deterministic near-match suggestions for an issue.
+func (t *Tracker) Similar(id string, limit int) ([]SimilarIssue, bool) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	target, ok := t.issues[id]
+	if !ok {
+		return nil, false
+	}
+	if limit <= 0 || limit > 50 {
+		limit = 10
+	}
+	result := make([]SimilarIssue, 0)
+	for otherID, other := range t.issues {
+		if otherID == id || other.Service != target.Service || other.Kind != target.Kind {
+			continue
+		}
+		score, reasons := similarity(*target, *other)
+		if score < 0.45 {
+			continue
+		}
+		decision := t.feedback[feedbackKey(id, otherID)]
+		result = append(result, SimilarIssue{Issue: cloneIssue(other), Score: score, Reasons: reasons, Decision: decision})
+	}
+	sort.Slice(result, func(i, j int) bool {
+		if result[i].Score != result[j].Score {
+			return result[i].Score > result[j].Score
+		}
+		return result[i].Issue.ID < result[j].Issue.ID
+	})
+	if len(result) > limit {
+		result = result[:limit]
+	}
+	return result, true
+}
+
+// Patterns returns deterministic groups built from exact issue IDs.
+func (t *Tracker) Patterns() []Pattern {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	groups := make(map[string]*Pattern)
+	for _, issue := range t.issues {
+		p := groups[issue.PatternID]
+		if p == nil {
+			p = &Pattern{ID: issue.PatternID, Signature: issue.PatternSignature, Service: issue.Service, Kind: issue.Kind}
+			groups[issue.PatternID] = p
+		}
+		p.IssueCount++
+		p.OccurrenceCount += issue.Count
+		p.IssueIDs = append(p.IssueIDs, issue.ID)
+		if issue.LastSeen > p.LastSeen {
+			p.LastSeen = issue.LastSeen
+		}
+	}
+	result := make([]Pattern, 0, len(groups))
+	for _, p := range groups {
+		sort.Strings(p.IssueIDs)
+		result = append(result, *p)
+	}
+	sort.Slice(result, func(i, j int) bool {
+		if result[i].LastSeen != result[j].LastSeen {
+			return result[i].LastSeen > result[j].LastSeen
+		}
+		return result[i].ID < result[j].ID
+	})
+	return result
+}
+
+// SetSimilarityFeedback records a human label for later training data.
+func (t *Tracker) SetSimilarityFeedback(a, b, decision string) bool {
+	if a == "" || b == "" || a == b || (decision != "same" && decision != "different") {
+		return false
+	}
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if _, ok := t.issues[a]; !ok {
+		return false
+	}
+	if _, ok := t.issues[b]; !ok {
+		return false
+	}
+	t.feedback[feedbackKey(a, b)] = decision
+	return true
+}
+
+func feedbackKey(a, b string) string {
+	if a > b {
+		a, b = b, a
+	}
+	return a + "\x00" + b
+}
+
+func patternSignature(kind, service, title string, sample Sample) string {
+	route := sample.NormalizedRoute
+	status := sample.StatusClass
+	stack := sample.StackSignature
+	// The title is deliberately broad here: the exact normalized title remains
+	// the canonical issue identity, while this signature supports UI grouping.
+	title = normalizePatternTitle(title)
+	return strings.Join([]string{service, kind, route, status, title, stack}, "\x00")
+}
+
+func normalizeSample(sample *Sample, title string) {
+	sample.NormalizedMessage = NormalizeTitle(title)
+	sample.NormalizedRoute = NormalizeRoute(sample.HTTPRoute)
+	sample.StatusClass = statusClass(sample.HTTPStatus)
+	sample.StackSignature = firstStackFrame(sample.Stacktrace)
+}
+
+func normalizePatternTitle(title string) string {
+	s := NormalizeTitle(title)
+	s = regexp.MustCompile(`(?i)http\s+[1-5]\d{2}`).ReplaceAllString(s, "http <status>")
+	return s
+}
+
+func NormalizeRoute(route string) string {
+	route = strings.TrimSpace(strings.SplitN(route, "?", 2)[0])
+	if route == "" {
+		return ""
+	}
+	parts := strings.Split(route, "/")
+	for i, part := range parts {
+		if part == "" {
+			continue
+		}
+		if uuidRe.MatchString(part) || hexIDRe.MatchString(part) || numRe.MatchString(part) {
+			parts[i] = "{id}"
+		}
+	}
+	return strings.Join(parts, "/")
+}
+
+func statusClass(status int) string {
+	if status >= 500 && status < 600 {
+		return "5xx"
+	}
+	if status >= 400 && status < 500 {
+		return "4xx"
+	}
+	if status >= 300 && status < 400 {
+		return "3xx"
+	}
+	if status > 0 {
+		return "2xx"
+	}
+	return ""
+}
+
+func firstStackFrame(stack string) string {
+	if i := strings.IndexByte(stack, '\n'); i >= 0 {
+		stack = stack[:i]
+	}
+	return strings.TrimSpace(stack)
+}
+
+func similarity(a, b Issue) (float64, []string) {
+	score := 0.0
+	reasons := []string{}
+	if a.Service == b.Service {
+		score += .25
+	}
+	if a.Kind == b.Kind {
+		score += .15
+	}
+	aSample, bSample := latestSample(a), latestSample(b)
+	if NormalizeRoute(aSample.HTTPRoute) != "" && NormalizeRoute(aSample.HTTPRoute) == NormalizeRoute(bSample.HTTPRoute) {
+		score += .20
+		reasons = append(reasons, "same route template")
+	}
+	if statusClass(aSample.HTTPStatus) != "" && statusClass(aSample.HTTPStatus) == statusClass(bSample.HTTPStatus) {
+		score += .15
+		reasons = append(reasons, "same HTTP status class")
+	}
+	if firstStackFrame(aSample.Stacktrace) != "" && firstStackFrame(aSample.Stacktrace) == firstStackFrame(bSample.Stacktrace) {
+		score += .10
+		reasons = append(reasons, "same top stack frame")
+	}
+	if a.NormalizedTitle != "" && a.NormalizedTitle == b.NormalizedTitle {
+		score += .15
+		reasons = append(reasons, "same normalized message")
+	} else if tokenSimilarity(a.NormalizedTitle, b.NormalizedTitle) >= .5 {
+		score += .10
+		reasons = append(reasons, "similar normalized message")
+	}
+	if len(reasons) == 0 {
+		reasons = append(reasons, "same service and event kind")
+	}
+	return score, reasons
+}
+
+func latestSample(issue Issue) Sample {
+	if len(issue.Samples) == 0 {
+		return Sample{}
+	}
+	return issue.Samples[len(issue.Samples)-1]
+}
+
+func tokenSimilarity(a, b string) float64 {
+	aa, bb := strings.Fields(a), strings.Fields(b)
+	if len(aa) == 0 || len(bb) == 0 {
+		return 0
+	}
+	set := map[string]bool{}
+	for _, v := range aa {
+		set[v] = true
+	}
+	other := map[string]bool{}
+	for _, v := range bb {
+		other[v] = true
+	}
+	common := 0
+	for v := range other {
+		if set[v] {
+			common++
+		}
+	}
+	return float64(common) / float64(len(set)+len(other)-common)
+}
+
+// SetStatus updates the triage state of an issue. It returns the updated
+// issue, or false when the id is unknown or the status is invalid.
+func (t *Tracker) SetStatus(id, status string) (Issue, bool) {
+	switch status {
+	case StatusOpen, StatusResolved, StatusIgnored:
+	default:
+		return Issue{}, false
+	}
+
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	issue, ok := t.issues[id]
+	if !ok {
+		return Issue{}, false
+	}
+	issue.Status = status
+	if status == StatusOpen {
+		issue.ResolvedAt = 0
+	} else {
+		issue.ResolvedAt = time.Now().UnixNano()
+	}
+	return cloneIssue(issue), true
+}
+
 // evictOldestLocked removes the issue with the smallest LastSeen. Callers
 // must hold t.mu.
 func (t *Tracker) evictOldestLocked() {
@@ -107,22 +462,89 @@ func (t *Tracker) evictOldestLocked() {
 // List returns issues sorted by LastSeen descending (most recently seen
 // first). A non-positive limit returns all issues.
 func (t *Tracker) List(limit int) []Issue {
+	return t.ListFiltered(ListOptions{Limit: limit}).Issues
+}
+
+// ListFiltered returns a filtered, stable page of issues.
+func (t *Tracker) ListFiltered(opts ListOptions) ListPage {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
 	result := make([]Issue, 0, len(t.issues))
 	for _, issue := range t.issues {
+		if !matches(issue, opts) || beforeCursor(issue, opts) {
+			continue
+		}
 		result = append(result, cloneIssue(issue))
 	}
 
 	sort.Slice(result, func(i, j int) bool {
-		return result[i].LastSeen > result[j].LastSeen
+		if result[i].LastSeen != result[j].LastSeen {
+			return result[i].LastSeen > result[j].LastSeen
+		}
+		return result[i].ID > result[j].ID
 	})
 
-	if limit > 0 && len(result) > limit {
+	limit := opts.Limit
-		result = result[:limit]
+	if limit <= 0 {
+		limit = len(result)
 	}
-	return result
+	page := ListPage{Issues: result}
+	if len(page.Issues) > limit {
+		page.Issues = page.Issues[:limit]
+		page.HasMore = true
+	}
+	return page
+}
+
+func matches(issue *Issue, opts ListOptions) bool {
+	if opts.Service != "" && issue.Service != opts.Service {
+		return false
+	}
+	if opts.Kind != "" && issue.Kind != opts.Kind {
+		return false
+	}
+	if opts.Status != "" && issue.Status != opts.Status {
+		return false
+	}
+	if opts.Title != "" && !strings.Contains(strings.ToLower(issue.Title), strings.ToLower(opts.Title)) {
+		return false
+	}
+	if opts.Since != 0 && issue.LastSeen < opts.Since {
+		return false
+	}
+	if opts.Until != 0 && issue.LastSeen > opts.Until {
+		return false
+	}
+	if opts.SourceIP != "" && !issueHasSample(issue, func(sample Sample) bool { return sample.SourceIP == opts.SourceIP }) {
+		return false
+	}
+	if opts.Route != "" && !issueHasSample(issue, func(sample Sample) bool { return sample.HTTPRoute == opts.Route }) {
+		return false
+	}
+	return true
+}
+
+func issueHasSample(issue *Issue, predicate func(Sample) bool) bool {
+	for _, sample := range issue.Samples {
+		if predicate(sample) {
+			return true
+		}
+	}
+	return false
+}
+
+func beforeCursor(issue *Issue, opts ListOptions) bool {
+	if opts.AfterLastSeen == 0 {
+		return false
+	}
+	if issue.LastSeen < opts.AfterLastSeen {
+		return false
+	}
+	if issue.LastSeen > opts.AfterLastSeen {
+		return true
+	}
+	return issue.ID >= opts.AfterID
 }
 
 // Get returns a single issue by id.
@@ -137,15 +559,100 @@ func (t *Tracker) Get(id string) (Issue, bool) {
 	return cloneIssue(issue), true
 }
 
-// cloneIssue deep-copies the slice fields so returned issues don't share
+// cloneIssue deep-copies the slice/map fields so returned issues don't share
 // backing arrays with ones still being mutated by Record.
 func cloneIssue(i *Issue) Issue {
 	c := *i
 	c.AffectedServices = append([]string(nil), i.AffectedServices...)
 	c.Samples = append([]Sample(nil), i.Samples...)
+	if i.Trend != nil {
+		c.Trend = make(map[string]int, len(i.Trend))
+		for k, v := range i.Trend {
+			c.Trend[k] = v
+		}
+	}
 	return c
 }
 
+// HourBucket renders a unix-nanosecond timestamp as a UTC hour key used in
+// Issue.Trend. Keys sort lexicographically, so the UI can sort them as-is.
+func HourBucket(ts int64) string {
+	return time.Unix(0, ts).UTC().Format("2006-01-02T15:00")
+}
+
+// pruneTrendLocked drops trend buckets older than trendWindow. Callers must
+// hold the tracker lock.
+func pruneTrendLocked(trend map[string]int, now int64) {
+	if len(trend) <= 48 {
+		return
+	}
+	cutoff := HourBucket(time.Unix(0, now).Add(-trendWindow).UnixNano())
+	for k := range trend {
+		if k < cutoff {
+			delete(trend, k)
+		}
+	}
+}
+
+var (
+	uuidRe       = regexp.MustCompile(`(?i)\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b`)
+	hexIDRe      = regexp.MustCompile(`(?i)\b[0-9a-f]{8,}\b`)
+	httpStatusRe = regexp.MustCompile(`(?i)\bhttp\s+([1-5]\d{2})\b`)
+	numRe        = regexp.MustCompile(`\b\d+\b`)
+	spaceRe      = regexp.MustCompile(`\s+`)
+)
+
+// NormalizeTitle strips variable content (UUIDs, hex ids, numbers, whitespace,
+// casing) from an error title so that "User 123 not found" and "User 456 not
+// found" fingerprint into the same issue. The raw title is kept as the
+// display title of the first-seen occurrence. HTTP status codes (e.g. HTTP 404)
+// are preserved so different HTTP error types don't collapse together.
+func NormalizeTitle(title string) string {
+	s := uuidRe.ReplaceAllString(title, "<id>")
+	// Protect HTTP status codes from being replaced by <num>
+	s = httpStatusRe.ReplaceAllStringFunc(s, func(match string) string {
+		parts := strings.Fields(match)
+		if len(parts) != 2 {
+			return match
+		}
+		return "__HTTP_STATUS_" + encodeStatus(parts[1]) + "__"
+	})
+	s = numRe.ReplaceAllString(s, "<num>")
+	s = hexIDRe.ReplaceAllStringFunc(s, func(match string) string {
+		// Mixed hex runs (letters + digits, e.g. trace/span ids) become ids.
+		// Pure-digit runs were already handled by numRe above, and words
+		// spelled entirely from a-f (rare at this length) survive so distinct
+		// error messages don't collapse.
+		if strings.ContainsAny(match, "0123456789") {
+			return "<id>"
+		}
+		return match
+	})
+	s = regexp.MustCompile(`__HTTP_STATUS_([a-z]+)__`).ReplaceAllStringFunc(s, func(match string) string {
+		encoded := strings.TrimSuffix(strings.TrimPrefix(match, "__HTTP_STATUS_"), "__")
+		return "http " + decodeStatus(encoded)
+	})
+	s = strings.ReplaceAll(s, "__", "")
+	s = spaceRe.ReplaceAllString(s, " ")
+	return strings.ToLower(strings.TrimSpace(s))
+}
+
+func encodeStatus(code string) string {
+	var b strings.Builder
+	for _, digit := range code {
+		b.WriteByte('a' + byte(digit-'0'))
+	}
+	return b.String()
+}
+
+func decodeStatus(encoded string) string {
+	var b strings.Builder
+	for _, letter := range encoded {
+		b.WriteByte('0' + byte(letter-'a'))
+	}
+	return b.String()
+}
+
 func fingerprint(parts ...string) string {
 	h := fnv.New64a()
 	_, _ = h.Write([]byte(strings.Join(parts, "\x00")))

internal/issues/tracker_test.go

@@ -73,3 +73,157 @@ func TestTrackerListRespectsLimit(t *testing.T) {
 		t.Fatalf("expected limit to cap results at 2, got %d", len(list))
 	}
 }
+
+func TestTrackerFilteredCursorPagination(t *testing.T) {
+	tracker := NewTracker()
+	tracker.Record("span_error", "nginx", "GET /.env", Sample{Timestamp: 300, SourceIP: "203.0.113.10", HTTPRoute: "/.env"})
+	tracker.Record("span_error", "nginx", "GET /.git/HEAD", Sample{Timestamp: 200, SourceIP: "203.0.113.11", HTTPRoute: "/.git/HEAD"})
+	tracker.Record("log_error", "api", "database down", Sample{Timestamp: 100})
+
+	page := tracker.ListFiltered(ListOptions{Service: "nginx", Kind: "span_error", Route: "/.env", Limit: 1})
+	if len(page.Issues) != 1 || page.Issues[0].Title != "GET /.env" {
+		t.Fatalf("unexpected filtered page: %+v", page)
+	}
+	if page.HasMore {
+		t.Error("single matching issue should not have another page")
+	}
+
+	all := tracker.ListFiltered(ListOptions{Service: "nginx", Limit: 1})
+	if len(all.Issues) != 1 || !all.HasMore {
+		t.Fatalf("expected first paginated page with more results, got %+v", all)
+	}
+	next := tracker.ListFiltered(ListOptions{
+		Service:       "nginx",
+		AfterLastSeen: all.Issues[0].LastSeen,
+		AfterID:       all.Issues[0].ID,
+		Limit:         1,
+	})
+	if len(next.Issues) != 1 || next.Issues[0].Title != "GET /.git/HEAD" || next.HasMore {
+		t.Fatalf("unexpected cursor page: %+v", next)
+	}
+}
+
+func TestNormalizeTitleGroupsVariableMessages(t *testing.T) {
+	cases := [][3]string{
+		// same, same, different: [a, b, expectedEqual(a,b)]
+		{"User 123 not found", "User 456 not found", "user <num> not found"},
+		{"order 550e8400-e29b-41d4-a716-446655440000 failed", "order 550e8400-e29b-41d4-a716-446655440001 failed", "order <id> failed"},
+		{"request 0a1b2c3d4e5f6a7b timed out", "request ff11aa22bb33cc44 timed out", "request <id> timed out"},
+	}
+	for _, c := range cases {
+		if got := NormalizeTitle(c[0]); got != c[2] {
+			t.Errorf("NormalizeTitle(%q) = %q, want %q", c[0], got, c[2])
+		}
+		if NormalizeTitle(c[0]) != NormalizeTitle(c[1]) {
+			t.Errorf("expected %q and %q to normalize identically", c[0], c[1])
+		}
+	}
+	// Words spelled from hex letters without digits must survive.
+	if got := NormalizeTitle("bad failed deadbeef badge"); got != "bad failed deadbeef badge" {
+		t.Errorf("word-like hex incorrectly replaced: %q", got)
+	}
+}
+
+func TestTrackerNormalizedFingerprinting(t *testing.T) {
+	tracker := NewTracker()
+	tracker.Record("log_error", "api", "User 123 not found", Sample{Timestamp: 100, Message: "User 123 not found"})
+	tracker.Record("log_error", "api", "User 98765432 not found", Sample{Timestamp: 200, Message: "User 98765432 not found"})
+
+	list := tracker.List(0)
+	if len(list) != 1 {
+		t.Fatalf("expected variable messages to merge into 1 issue, got %d", len(list))
+	}
+	if list[0].Count != 2 {
+		t.Errorf("expected count 2, got %d", list[0].Count)
+	}
+	// The display title is the first-seen raw message.
+	if list[0].Title != "User 123 not found" {
+		t.Errorf("expected first-seen raw title kept, got %q", list[0].Title)
+	}
+}
+
+func TestTrackerTrendBuckets(t *testing.T) {
+	tracker := NewTracker()
+	// Two occurrences in the same hour.
+	tracker.Record("log_error", "api", "boom", Sample{Timestamp: 100})
+	tracker.Record("log_error", "api", "boom", Sample{Timestamp: 200})
+
+	issue, ok := tracker.Get(tracker.List(1)[0].ID)
+	if !ok {
+		t.Fatal("expected issue to exist")
+	}
+	bucket := HourBucket(100)
+	if issue.Trend[bucket] != 2 {
+		t.Errorf("expected trend[%s]=2, got %d", bucket, issue.Trend[bucket])
+	}
+}
+
+func TestTrackerStatusLifecycle(t *testing.T) {
+	tracker := NewTracker()
+	tracker.Record("log_error", "api", "boom", Sample{Timestamp: 100})
+	id := tracker.List(1)[0].ID
+
+	// Invalid status rejected.
+	if _, ok := tracker.SetStatus(id, "bogus"); ok {
+		t.Error("expected invalid status to be rejected")
+	}
+	// Unknown id rejected.
+	if _, ok := tracker.SetStatus("nope", StatusResolved); ok {
+		t.Error("expected unknown id to be rejected")
+	}
+	// Resolve.
+	resolved, ok := tracker.SetStatus(id, StatusResolved)
+	if !ok || resolved.Status != StatusResolved || resolved.ResolvedAt == 0 {
+		t.Fatalf("expected resolved issue, got %+v ok=%v", resolved, ok)
+	}
+	// A new occurrence reopens it (regression).
+	tracker.Record("log_error", "api", "boom", Sample{Timestamp: 200})
+	issue, _ := tracker.Get(id)
+	if issue.Status != StatusOpen || issue.ResolvedAt != 0 {
+		t.Errorf("expected regression to reopen issue, got status=%s resolvedAt=%d", issue.Status, issue.ResolvedAt)
+	}
+	// Ignored survives new occurrences.
+	if _, ok := tracker.SetStatus(id, StatusIgnored); !ok {
+		t.Fatal("expected ignore to succeed")
+	}
+	tracker.Record("log_error", "api", "boom", Sample{Timestamp: 300})
+	issue, _ = tracker.Get(id)
+	if issue.Status != StatusIgnored {
+		t.Errorf("expected ignored state to survive new occurrence, got %s", issue.Status)
+	}
+}
+
+func TestTrackerPatternsAndSimilarSuggestions(t *testing.T) {
+	tracker := NewTracker()
+	tracker.Record("span_error", "checkout", "HTTP 504: database timeout for order 123", Sample{
+		Timestamp: 100, HTTPRoute: "/orders/123", HTTPStatus: 504, Stacktrace: "db.go:42\nhandler.go:10",
+	})
+	tracker.Record("span_error", "checkout", "HTTP 502: database timeout for order 456", Sample{
+		Timestamp: 200, HTTPRoute: "/orders/456", HTTPStatus: 502, Stacktrace: "db.go:42\nhandler.go:10",
+	})
+	issuesList := tracker.List(0)
+	if len(issuesList) != 2 {
+		t.Fatalf("expected separate exact issues, got %d", len(issuesList))
+	}
+	if issuesList[0].PatternID == "" || issuesList[0].PatternID != issuesList[1].PatternID {
+		t.Fatalf("expected related HTTP variants to share a pattern: %+v", issuesList)
+	}
+	if issuesList[0].Samples[0].NormalizedRoute != "/orders/{id}" {
+		t.Fatalf("expected normalized route, got %q", issuesList[0].Samples[0].NormalizedRoute)
+	}
+	related, ok := tracker.Similar(issuesList[0].ID, 5)
+	if !ok || len(related) != 1 || related[0].Score < .45 {
+		t.Fatalf("expected deterministic related suggestion, got ok=%v related=%+v", ok, related)
+	}
+	if !tracker.SetSimilarityFeedback(issuesList[0].ID, related[0].Issue.ID, "same") {
+		t.Fatal("expected feedback to be accepted")
+	}
+	related, _ = tracker.Similar(issuesList[0].ID, 5)
+	if related[0].Decision != "same" {
+		t.Fatalf("expected feedback decision, got %q", related[0].Decision)
+	}
+	patterns := tracker.Patterns()
+	if len(patterns) != 1 || patterns[0].IssueCount != 2 || patterns[0].OccurrenceCount != 2 {
+		t.Fatalf("expected one two-issue pattern, got %+v", patterns)
+	}
+}

internal/kubernetes/types.go

@@ -0,0 +1,23 @@
+package kubernetes
+
+// Rollout is the normalized, UI-safe view of a Kubernetes Deployment rollout.
+type Rollout struct {
+	ID                string `json:"id"`
+	Namespace         string `json:"namespace"`
+	Workload          string `json:"workload"`
+	Service           string `json:"service,omitempty"`
+	Release           string `json:"release,omitempty"`
+	Revision          string `json:"revision,omitempty"`
+	Image             string `json:"image,omitempty"`
+	Phase             string `json:"phase"`
+	Reason            string `json:"reason,omitempty"`
+	Message           string `json:"message,omitempty"`
+	StartedAt         int64  `json:"startedAt"`
+	CompletedAt       int64  `json:"completedAt,omitempty"`
+	LastSeen          int64  `json:"lastSeen"`
+	Generation        int64  `json:"generation"`
+	DesiredReplicas   int32  `json:"desiredReplicas"`
+	UpdatedReplicas   int32  `json:"updatedReplicas"`
+	AvailableReplicas int32  `json:"availableReplicas"`
+	ReadyReplicas     int32  `json:"readyReplicas"`
+}

internal/kubernetes/watcher.go

@@ -0,0 +1,239 @@
+// Package kubernetes observes read-only workload state for rollout correlation.
+package kubernetes
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+const maxRollouts = 500
+
+type Watcher struct {
+	mu       sync.RWMutex
+	rollouts map[string]Rollout
+	enabled  bool
+	err      string
+}
+
+// NewFromEnv enables observation in-cluster, or with KUBECONFIG locally. It is
+// disabled by default outside Kubernetes so local development needs no kubeconfig.
+func NewFromEnv(ctx context.Context) (*Watcher, error) {
+	if strings.EqualFold(os.Getenv("OTEL_KUBERNETES_ENABLED"), "false") {
+		return nil, nil
+	}
+	config, err := kubeConfig()
+	if err != nil {
+		if os.Getenv("OTEL_KUBERNETES_ENABLED") == "true" {
+			return nil, err
+		}
+		return nil, nil
+	}
+	client, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("create kubernetes client: %w", err)
+	}
+
+	w := &Watcher{rollouts: make(map[string]Rollout), enabled: true}
+	namespace := os.Getenv("OTEL_KUBERNETES_NAMESPACE")
+	selector := os.Getenv("OTEL_KUBERNETES_LABEL_SELECTOR")
+	factory := informers.NewSharedInformerFactoryWithOptions(client, 30*time.Second,
+		informers.WithTweakListOptions(func(options *metav1.ListOptions) {
+			options.LabelSelector = selector
+		}),
+	)
+	if namespace != "" {
+		factory = informers.NewSharedInformerFactoryWithOptions(client, 30*time.Second,
+			informers.WithNamespace(namespace), informers.WithTweakListOptions(func(options *metav1.ListOptions) {
+				options.LabelSelector = selector
+			}))
+	}
+	informer := factory.Apps().V1().Deployments().Informer()
+	_, _ = informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    func(obj interface{}) { w.observe(obj) },
+		UpdateFunc: func(_, obj interface{}) { w.observe(obj) },
+	})
+	go func() {
+		factory.Start(ctx.Done())
+		if !cache.WaitForCacheSync(ctx.Done(), informer.HasSynced) {
+			w.setError("deployment informer did not sync")
+			return
+		}
+		<-ctx.Done()
+	}()
+	return w, nil
+}
+
+func kubeConfig() (*rest.Config, error) {
+	if config, err := rest.InClusterConfig(); err == nil {
+		return config, nil
+	}
+	kubeconfig := os.Getenv("KUBECONFIG")
+	if kubeconfig == "" {
+		home, err := os.UserHomeDir()
+		if err != nil {
+			return nil, err
+		}
+		kubeconfig = home + string(os.PathSeparator) + ".kube" + string(os.PathSeparator) + "config"
+	}
+	return clientcmd.BuildConfigFromFlags("", kubeconfig)
+}
+
+func (w *Watcher) observe(obj interface{}) {
+	deployment, ok := obj.(*appsv1.Deployment)
+	if !ok {
+		return
+	}
+	now := time.Now().UnixNano()
+	revision := deployment.Annotations["deployment.kubernetes.io/revision"]
+	if revision == "" {
+		revision = strconv.FormatInt(deployment.Generation, 10)
+	}
+	key := deployment.Namespace + "/" + deployment.Name + "/" + revision
+	phase, reason, message, completed := rolloutStatus(deployment)
+	started := now
+	for _, condition := range deployment.Status.Conditions {
+		if condition.Type == appsv1.DeploymentProgressing && !condition.LastUpdateTime.IsZero() {
+			started = condition.LastUpdateTime.UnixNano()
+		}
+		if condition.Type == appsv1.DeploymentAvailable && !condition.LastUpdateTime.IsZero() && completed {
+			completed = condition.LastUpdateTime.UnixNano() != 0
+		}
+	}
+	if old, exists := w.rollouts[key]; exists {
+		started = old.StartedAt
+		if old.CompletedAt != 0 {
+			completed = true
+		}
+	}
+	rollout := Rollout{
+		ID: rolloutID(key), Namespace: deployment.Namespace, Workload: deployment.Name,
+		Service:  metadataValue(deployment, "app.kubernetes.io/name", "app", "service.name"),
+		Release:  metadataValue(deployment, "otel.k6n.net/release", "app.kubernetes.io/version", "service.release"),
+		Revision: revision, Image: deploymentImage(deployment), Phase: phase, Reason: reason,
+		Message: message, StartedAt: started, LastSeen: now, Generation: deployment.Generation,
+		DesiredReplicas: desiredReplicas(deployment), UpdatedReplicas: deployment.Status.UpdatedReplicas,
+		AvailableReplicas: deployment.Status.AvailableReplicas, ReadyReplicas: deployment.Status.ReadyReplicas,
+	}
+	if completed {
+		rollout.CompletedAt = now
+	}
+	w.mu.Lock()
+	w.rollouts[key] = rollout
+	if len(w.rollouts) > maxRollouts {
+		w.evictOldestLocked()
+	}
+	w.mu.Unlock()
+}
+
+func (w *Watcher) evictOldestLocked() {
+	var oldest string
+	var ts int64
+	for key, item := range w.rollouts {
+		if oldest == "" || item.LastSeen < ts {
+			oldest, ts = key, item.LastSeen
+		}
+	}
+	if oldest != "" {
+		delete(w.rollouts, oldest)
+	}
+}
+func (w *Watcher) setError(message string) { w.mu.Lock(); defer w.mu.Unlock(); w.err = message }
+
+func (w *Watcher) List() []Rollout {
+	w.mu.RLock()
+	defer w.mu.RUnlock()
+	result := make([]Rollout, 0, len(w.rollouts))
+	for _, item := range w.rollouts {
+		result = append(result, item)
+	}
+	sort.Slice(result, func(i, j int) bool { return result[i].LastSeen > result[j].LastSeen })
+	return result
+}
+func (w *Watcher) Get(id string) (Rollout, bool) {
+	w.mu.RLock()
+	defer w.mu.RUnlock()
+	for _, item := range w.rollouts {
+		if item.ID == id {
+			return item, true
+		}
+	}
+	return Rollout{}, false
+}
+func (w *Watcher) Enabled() bool { return w != nil && w.enabled }
+func (w *Watcher) Error() string {
+	if w == nil {
+		return "kubernetes integration is not configured"
+	}
+	w.mu.RLock()
+	defer w.mu.RUnlock()
+	return w.err
+}
+
+func rolloutID(key string) string {
+	sum := sha256.Sum256([]byte(key))
+	return hex.EncodeToString(sum[:8])
+}
+func metadataValue(deployment *appsv1.Deployment, keys ...string) string {
+	for _, key := range keys {
+		if value := deployment.Labels[key]; value != "" {
+			return value
+		}
+		if value := deployment.Annotations[key]; value != "" {
+			return value
+		}
+		if value := deployment.Spec.Template.Labels[key]; value != "" {
+			return value
+		}
+		if value := deployment.Spec.Template.Annotations[key]; value != "" {
+			return value
+		}
+	}
+	return ""
+}
+func deploymentImage(deployment *appsv1.Deployment) string {
+	if len(deployment.Spec.Template.Spec.Containers) == 0 {
+		return ""
+	}
+	return deployment.Spec.Template.Spec.Containers[0].Image
+}
+func desiredReplicas(deployment *appsv1.Deployment) int32 {
+	if deployment.Spec.Replicas == nil {
+		return 1
+	}
+	return *deployment.Spec.Replicas
+}
+
+func rolloutStatus(deployment *appsv1.Deployment) (phase, reason, message string, completed bool) {
+	phase = "progressing"
+	for _, condition := range deployment.Status.Conditions {
+		switch condition.Type {
+		case appsv1.DeploymentProgressing:
+			reason, message = condition.Reason, condition.Message
+			if condition.Status == corev1.ConditionFalse {
+				return "failed", reason, message, false
+			}
+		case appsv1.DeploymentAvailable:
+			if condition.Status == corev1.ConditionTrue && deployment.Status.UpdatedReplicas == desiredReplicas(deployment) && deployment.Status.AvailableReplicas >= desiredReplicas(deployment) {
+				return "completed", reason, message, true
+			}
+		}
+	}
+	return phase, reason, message, false
+}

internal/prometheus/client.go

@@ -0,0 +1,103 @@
+// Package prometheus provides optional Prometheus range queries for rollout impact.
+package prometheus
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type Point struct {
+	Timestamp float64 `json:"timestamp"`
+	Value     string  `json:"value"`
+}
+
+type Client struct {
+	baseURL       string
+	queryTemplate string
+	httpClient    *http.Client
+}
+
+func NewFromEnv() *Client {
+	baseURL := strings.TrimRight(os.Getenv("OTEL_PROMETHEUS_URL"), "/")
+	if baseURL == "" {
+		return nil
+	}
+	query := os.Getenv("OTEL_PROMETHEUS_ERROR_QUERY")
+	if query == "" {
+		query = `sum(rate(http_server_requests_total{service_name=%q,http_response_status_code=~"5.."}[5m]))`
+	}
+	return &Client{baseURL: baseURL, queryTemplate: query, httpClient: &http.Client{Timeout: 10 * time.Second}}
+}
+
+func (c *Client) Configured() bool { return c != nil }
+
+func (c *Client) ErrorRate(ctx context.Context, service, release string, start, end time.Time) ([]Point, error) {
+	if c == nil {
+		return nil, fmt.Errorf("prometheus is not configured")
+	}
+	query := renderQuery(c.queryTemplate, service, release)
+	params := url.Values{
+		"query": {query},
+		"start": {strconv.FormatInt(start.Unix(), 10)},
+		"end":   {strconv.FormatInt(end.Unix(), 10)},
+		"step":  {"300"},
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/api/v1/query_range?"+params.Encode(), nil)
+	if err != nil {
+		return nil, err
+	}
+	response, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+	if response.StatusCode/100 != 2 {
+		return nil, fmt.Errorf("prometheus returned HTTP %s", response.Status)
+	}
+	var payload struct {
+		Status string `json:"status"`
+		Error  string `json:"error"`
+		Data   struct {
+			Result []struct {
+				Values [][]json.RawMessage `json:"values"`
+			} `json:"result"`
+		} `json:"data"`
+	}
+	if err := json.NewDecoder(response.Body).Decode(&payload); err != nil {
+		return nil, err
+	}
+	if payload.Status != "success" {
+		return nil, fmt.Errorf("prometheus query failed: %s", payload.Error)
+	}
+	points := make([]Point, 0)
+	for _, series := range payload.Data.Result {
+		for _, value := range series.Values {
+			if len(value) != 2 {
+				continue
+			}
+			var timestamp float64
+			var sample string
+			if json.Unmarshal(value[0], &timestamp) == nil && json.Unmarshal(value[1], &sample) == nil {
+				points = append(points, Point{Timestamp: timestamp, Value: sample})
+			}
+		}
+	}
+	return points, nil
+}
+
+func renderQuery(template, service, release string) string {
+	values := []string{strconv.Quote(service), strconv.Quote(release)}
+	for _, value := range values {
+		if strings.Contains(template, "%q") {
+			template = strings.Replace(template, "%q", value, 1)
+		}
+	}
+	return template
+}

internal/sourcemaps/store.go

@@ -0,0 +1,106 @@
+package sourcemaps
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+
+	sm "github.com/go-sourcemap/sourcemap"
+)
+
+var frameRE = regexp.MustCompile(`(.*?)(https?://[^ )]+|[^ )]+):(\d+):(\d+)(\)?\s*)$`)
+
+type Resolver struct {
+	dir  string
+	mu   sync.RWMutex
+	maps map[string]*sm.Consumer
+}
+
+func New(dir string) (*Resolver, error) {
+	if dir == "" {
+		dir = "./sourcemaps"
+	}
+	if err := os.MkdirAll(dir, 0750); err != nil {
+		return nil, err
+	}
+	return &Resolver{dir: dir, maps: make(map[string]*sm.Consumer)}, nil
+}
+
+func key(release, source string) string { return release + "\x00" + normalize(source) }
+
+func normalize(source string) string {
+	source = strings.Split(source, "?")[0]
+	if i := strings.Index(source, "/assets/"); i >= 0 {
+		return source[i:]
+	}
+	return strings.TrimPrefix(source, "./")
+}
+
+func (r *Resolver) Upload(release, source string, body io.Reader) error {
+	if strings.TrimSpace(release) == "" || strings.TrimSpace(source) == "" {
+		return fmt.Errorf("release and source are required")
+	}
+	b, err := io.ReadAll(io.LimitReader(body, 50<<20))
+	if err != nil {
+		return err
+	}
+	consumer, err := sm.Parse(source+".map", b)
+	if err != nil {
+		return fmt.Errorf("invalid source map: %w", err)
+	}
+	name := filepath.Join(r.dir, safe(release)+"-"+safe(normalize(source))+".map")
+	if err := os.WriteFile(name, b, 0600); err != nil {
+		return err
+	}
+	r.mu.Lock()
+	r.maps[key(release, source)] = consumer
+	r.mu.Unlock()
+	return nil
+}
+
+func safe(s string) string {
+	s = strings.ReplaceAll(s, "/", "_")
+	s = strings.ReplaceAll(s, "\\", "_")
+	return strings.Map(func(r rune) rune {
+		if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || strings.ContainsRune("._-", r) {
+			return r
+		}
+		return '_'
+	}, s)
+}
+
+func (r *Resolver) ResolveStack(release, stack string) string {
+	if release == "" || stack == "" {
+		return stack
+	}
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	lines := strings.Split(stack, "\n")
+	for i, line := range lines {
+		m := frameRE.FindStringSubmatch(line)
+		if m == nil {
+			continue
+		}
+		genLine, _ := strconv.Atoi(m[3])
+		genCol, _ := strconv.Atoi(m[4])
+		consumer := r.maps[key(release, m[2])]
+		if consumer == nil {
+			continue
+		}
+		source, name, srcLine, srcCol, ok := consumer.Source(genLine, genCol)
+		if !ok {
+			continue
+		}
+		label := source
+		if name != "" {
+			label += " (" + name + ")"
+		}
+		lines[i] = fmt.Sprintf("%s%s:%d:%d%s  [mapped from %s:%d:%d]", m[1], label, srcLine, srcCol, m[5], m[2], genLine, genCol)
+	}
+	return strings.Join(lines, "\n")
+}

internal/storage/clickhouse.go

@@ -0,0 +1,528 @@
+package storage
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+
+	"git.tornberg.me/go-otel/internal/websocket"
+)
+
+// ClickHouseStore implements the Store interface backed by ClickHouse, so
+// history search stays fast and unbounded (the FileStore rescans the whole
+// log file on every query) and aggregate time-series - error trends, release
+// comparisons - can be computed server-side instead of in the browser.
+type ClickHouseStore struct {
+	conn driver.Conn
+	db   string
+}
+
+// ClickHouseConfig controls how the store connects.
+type ClickHouseConfig struct {
+	Addr            string        // host:port, native protocol (default port 9000)
+	Database        string        // defaults to "otel"
+	Username        string        // defaults to "default"
+	Password        string
+	MaxOpenConns    int           // defaults to 50
+	MaxIdleConns    int           // defaults to 25
+	DialTimeout     time.Duration // defaults to 30s (controls connection acquire timeout in driver pool)
+	ConnMaxLifetime time.Duration // defaults to 10m
+}
+
+func parseEnvInt(keys []string, def int) int {
+	for _, k := range keys {
+		if v := os.Getenv(k); v != "" {
+			if n, err := strconv.Atoi(v); err == nil && n > 0 {
+				return n
+			}
+		}
+	}
+	return def
+}
+
+func parseEnvDuration(keys []string, def time.Duration) time.Duration {
+	for _, k := range keys {
+		if v := os.Getenv(k); v != "" {
+			if d, err := time.ParseDuration(v); err == nil && d > 0 {
+				return d
+			}
+		}
+	}
+	return def
+}
+
+// NewClickHouseStore connects, creates the schema if needed, and returns a
+// ready store. Non-table errors during schema init are logged, not fatal:
+// raw storage works even if the rollup view can't be created.
+func NewClickHouseStore(ctx context.Context, cfg ClickHouseConfig) (*ClickHouseStore, error) {
+	if cfg.Database == "" {
+		cfg.Database = "otel"
+	}
+	if cfg.Username == "" {
+		cfg.Username = "default"
+	}
+
+	maxOpen := cfg.MaxOpenConns
+	if maxOpen <= 0 {
+		maxOpen = parseEnvInt([]string{"OTEL_CLICKHOUSE_MAX_OPEN_CONNS", "CLICKHOUSE_MAX_OPEN_CONNS"}, 50)
+	}
+	maxIdle := cfg.MaxIdleConns
+	if maxIdle <= 0 {
+		maxIdle = parseEnvInt([]string{"OTEL_CLICKHOUSE_MAX_IDLE_CONNS", "CLICKHOUSE_MAX_IDLE_CONNS"}, 25)
+	}
+	dialTimeout := cfg.DialTimeout
+	if dialTimeout <= 0 {
+		dialTimeout = parseEnvDuration([]string{"OTEL_CLICKHOUSE_DIAL_TIMEOUT", "CLICKHOUSE_DIAL_TIMEOUT"}, 30*time.Second)
+	}
+	connMaxLifetime := cfg.ConnMaxLifetime
+	if connMaxLifetime <= 0 {
+		connMaxLifetime = parseEnvDuration([]string{"OTEL_CLICKHOUSE_CONN_MAX_LIFETIME", "CLICKHOUSE_CONN_MAX_LIFETIME"}, 10*time.Minute)
+	}
+
+	// 1. Connect to default database with short timeout to ensure the target database exists.
+	bootConn, err := clickhouse.Open(&clickhouse.Options{
+		Addr: []string{cfg.Addr},
+		Auth: clickhouse.Auth{
+			Database: "default",
+			Username: cfg.Username,
+			Password: cfg.Password,
+		},
+		DialTimeout: 5 * time.Second,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("clickhouse dial %s: %w", cfg.Addr, err)
+	}
+	if err := bootConn.Ping(ctx); err != nil {
+		_ = bootConn.Close()
+		return nil, fmt.Errorf("clickhouse ping %s: %w", cfg.Addr, err)
+	}
+
+	if err := bootConn.Exec(ctx,
+		fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", cfg.Database)); err != nil {
+		_ = bootConn.Close()
+		return nil, fmt.Errorf("clickhouse create database %s: %w", cfg.Database, err)
+	}
+	// Close bootstrap connection so it does not leak idle connections.
+	_ = bootConn.Close()
+
+	// 2. Open production connection pool targeting the actual database with full pool settings.
+	conn, err := clickhouse.Open(&clickhouse.Options{
+		Addr: []string{cfg.Addr},
+		Auth: clickhouse.Auth{
+			Database: cfg.Database,
+			Username: cfg.Username,
+			Password: cfg.Password,
+		},
+		MaxOpenConns:     maxOpen,
+		MaxIdleConns:     maxIdle,
+		DialTimeout:      dialTimeout,
+		ConnMaxLifetime:  connMaxLifetime,
+		ConnOpenStrategy: clickhouse.ConnOpenRoundRobin,
+		ClientInfo: clickhouse.ClientInfo{
+			Products: []struct {
+				Name    string
+				Version string
+			}{
+				{Name: "go-otel", Version: "1.0"},
+			},
+		},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("clickhouse open %s/%s: %w", cfg.Addr, cfg.Database, err)
+	}
+	if err := conn.Ping(ctx); err != nil {
+		_ = conn.Close()
+		return nil, fmt.Errorf("clickhouse ping %s/%s: %w", cfg.Addr, cfg.Database, err)
+	}
+
+	s := &ClickHouseStore{conn: conn, db: cfg.Database}
+	if err := s.initSchema(ctx); err != nil {
+		_ = conn.Close()
+		return nil, err
+	}
+	return s, nil
+}
+
+// initSchema creates the raw entries table and the materialized hourly
+// rollup used by the stats endpoints. Idempotent.
+func (s *ClickHouseStore) initSchema(ctx context.Context) error {
+	if err := s.conn.Exec(ctx, fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.entries (
+		timestamp DateTime64(9, 'UTC') NOT NULL,
+		type      LowCardinality(String) NOT NULL,
+		service   LowCardinality(String) NOT NULL DEFAULT 'unknown',
+		is_error  UInt8 NOT NULL DEFAULT 0,
+		release   LowCardinality(String) NOT NULL DEFAULT '',
+		trace_id  String NOT NULL DEFAULT '',
+		data      String NOT NULL
+	) ENGINE = MergeTree
+	PARTITION BY toYYYYMM(timestamp)
+	ORDER BY (type, timestamp)
+	TTL toDateTime(timestamp) + INTERVAL 90 DAY`, s.db)); err != nil {
+		return fmt.Errorf("clickhouse create entries: %w", err)
+	}
+
+	// Hourly per-service/release rollup maintained on insert. is_error is a
+	// dimension (0/1), so errors = sum(count) WHERE is_error=1 and
+	// total = sum(count) unfiltered.
+	if err := s.conn.Exec(ctx, fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.hourly (
+		hour     DateTime('UTC') NOT NULL,
+		type     LowCardinality(String) NOT NULL,
+		service  LowCardinality(String) NOT NULL,
+		release  LowCardinality(String) NOT NULL DEFAULT '',
+		is_error UInt8 NOT NULL DEFAULT 0,
+		count    UInt64
+	) ENGINE = SummingMergeTree
+	ORDER BY (hour, type, service, release, is_error)`, s.db)); err != nil {
+		return fmt.Errorf("clickhouse create hourly: %w", err)
+	}
+
+	mv := fmt.Sprintf(`CREATE MATERIALIZED VIEW IF NOT EXISTS %s.hourly_mv TO %s.hourly AS
+		SELECT
+			toStartOfHour(timestamp) AS hour,
+			type,
+			service,
+			release,
+			is_error,
+			count() AS count
+		FROM %s.entries
+		GROUP BY hour, type, service, release, is_error`, s.db, s.db, s.db)
+	if err := s.conn.Exec(ctx, mv); err != nil {
+		// Raw storage works without the rollup; stats endpoints will just be
+		// empty until this succeeds.
+		fmt.Printf("clickhouse: failed to create rollup view (stats endpoints stay empty): %v\n", err)
+	}
+	return nil
+}
+
+// chPayload extracts rollup columns from the marshaled OTLP JSON. Field
+// structs are deliberately minimal so unmarshal cost stays low.
+type chPayload struct {
+	ResourceSpans []chResourceSpan `json:"resourceSpans"`
+	ResourceLogs  []chResourceLog  `json:"resourceLogs"`
+}
+
+type chResourceSpan struct {
+	Resource   chResource `json:"resource"`
+	ScopeSpans []struct {
+		Spans []chSpan `json:"spans"`
+	} `json:"scopeSpans"`
+}
+
+type chResourceLog struct {
+	Resource  chResource `json:"resource"`
+	ScopeLogs []struct {
+		LogRecords []chLogRecord `json:"logRecords"`
+	} `json:"scopeLogs"`
+}
+
+type chResource struct {
+	Attributes []chAttr `json:"attributes"`
+}
+
+type chAttr struct {
+	Key   string `json:"key"`
+	Value struct {
+		StringValue string `json:"stringValue"`
+	} `json:"value"`
+}
+
+type chSpan struct {
+	TraceID string `json:"traceId"`
+	Status  struct {
+		Code int `json:"code"`
+	} `json:"status"`
+	Attributes []chAttr `json:"attributes"`
+}
+
+type chLogRecord struct {
+	SeverityNumber int  `json:"severityNumber"`
+	SeverityText   string `json:"severityText"`
+	TraceID        string `json:"traceId"`
+	Attributes     []chAttr `json:"attributes"`
+}
+
+// extractHead pulls (service, is_error, release, trace_id) out of a raw OTLP
+// JSON payload for the rollup columns. Best-effort: on parse failure the
+// entry is still stored with defaults.
+func extractHead(data []byte) (service string, isErr bool, release, traceID string) {
+	service, release = "unknown", ""
+	var p chPayload
+	if err := json.Unmarshal(data, &p); err != nil {
+		return service, false, release, traceID
+	}
+
+	attrString := func(attrs []chAttr, key string) string {
+		for _, a := range attrs {
+			if a.Key == key {
+				return a.Value.StringValue
+			}
+		}
+		return ""
+	}
+
+	// Traces: error if any span has status code 2 (StatusCodeError).
+	for _, rs := range p.ResourceSpans {
+		if s := attrString(rs.Resource.Attributes, "service.name"); s != "" && service == "unknown" {
+			service = s
+		}
+		if r := attrString(rs.Resource.Attributes, "service.release"); r != "" && release == "" {
+			release = r
+		}
+		for _, ss := range rs.ScopeSpans {
+			for _, span := range ss.Spans {
+				if span.TraceID != "" && traceID == "" {
+					traceID = span.TraceID
+				}
+				if span.Status.Code == 2 {
+					isErr = true
+				}
+				if r := attrString(span.Attributes, "service.release"); r != "" && release == "" {
+					release = r
+				}
+			}
+		}
+	}
+
+	// Logs: error if severity number >= 17 (Error) or text is error-ish.
+	for _, rl := range p.ResourceLogs {
+		if s := attrString(rl.Resource.Attributes, "service.name"); s != "" && service == "unknown" {
+			service = s
+		}
+		if r := attrString(rl.Resource.Attributes, "service.release"); r != "" && release == "" {
+			release = r
+		}
+		for _, sl := range rl.ScopeLogs {
+			for _, rec := range sl.LogRecords {
+				if rec.TraceID != "" && traceID == "" {
+					traceID = rec.TraceID
+				}
+				if r := attrString(rec.Attributes, "service.release"); r != "" && release == "" {
+					release = r
+				}
+				if rec.SeverityNumber >= 17 {
+					isErr = true
+				}
+				switch strings.ToUpper(strings.TrimSpace(rec.SeverityText)) {
+				case "ERROR", "FATAL", "CRITICAL":
+					isErr = true
+				}
+			}
+		}
+	}
+	return service, isErr, release, traceID
+}
+
+// Append stores one history entry with its extracted rollup columns.
+// AsyncInsert lets ClickHouse batch the writes server-side. The severity
+// column is intentionally omitted: one payload can hold many records with
+// differing severities, so error-ness is a per-entry flag instead.
+func (s *ClickHouseStore) Append(ctx context.Context, entry websocket.HistoryEntry) error {
+	service, isErr, release, traceID := extractHead(entry.Data)
+
+	ts := time.Unix(0, entry.Timestamp).UTC()
+	err := s.conn.AsyncInsert(ctx, fmt.Sprintf(
+		`INSERT INTO %s.entries
+		 (timestamp, type, service, is_error, release, trace_id, data)
+		 VALUES (?, ?, ?, ?, ?, ?, ?)`, s.db),
+		false,
+		ts, entry.Type, service, boolToUint8(isErr), release, traceID, string(entry.Data),
+	)
+	if err != nil {
+		return fmt.Errorf("clickhouse append: %w", err)
+	}
+	return nil
+}
+
+func boolToUint8(b bool) uint8 {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+// GetHistory returns the most recent limit entries, newest first.
+func (s *ClickHouseStore) GetHistory(ctx context.Context, limit int) ([]websocket.HistoryEntry, error) {
+	return s.Search(ctx, SearchOptions{}, limit)
+}
+
+// Search implements the same semantics as FileStore.Search: case-insensitive
+// substring query over the raw payload, type/time filters, newest first,
+// capped at limit.
+func (s *ClickHouseStore) Search(ctx context.Context, opts SearchOptions, limit int) ([]websocket.HistoryEntry, error) {
+	if limit <= 0 {
+		limit = 50
+	}
+
+	where := []string{"1=1"}
+	args := []interface{}{}
+	if opts.Query != "" {
+		where = append(where, "positionCaseInsensitive(data, ?) > 0")
+		args = append(args, opts.Query)
+	}
+	if opts.Type != "" {
+		where = append(where, "type = ?")
+		args = append(args, opts.Type)
+	}
+	if opts.Since != 0 {
+		where = append(where, "timestamp >= ?")
+		args = append(args, time.Unix(0, opts.Since).UTC())
+	}
+	if opts.Until != 0 {
+		where = append(where, "timestamp <= ?")
+		args = append(args, time.Unix(0, opts.Until).UTC())
+	}
+
+	query := fmt.Sprintf(
+		`SELECT timestamp, type, data
+		 FROM %s.entries
+		 WHERE %s
+		 ORDER BY timestamp DESC
+		 LIMIT ?`, s.db, strings.Join(where, " AND "))
+	args = append(args, limit)
+
+	rows, err := s.conn.Query(ctx, query, args...)
+	if err != nil {
+		return nil, fmt.Errorf("clickhouse search: %w", err)
+	}
+	defer rows.Close()
+
+	var results []websocket.HistoryEntry
+	for rows.Next() {
+		var (
+			ts  time.Time
+			typ string
+			raw string
+		)
+		if err := rows.Scan(&ts, &typ, &raw); err != nil {
+			return nil, err
+		}
+		results = append(results, websocket.HistoryEntry{
+			Type:      typ,
+			Timestamp: ts.UnixNano(),
+			Data:      json.RawMessage(raw),
+		})
+	}
+	return results, rows.Err()
+}
+
+// StatsPoint is one bucket of a stats time-series.
+type StatsPoint struct {
+	Bucket string `json:"bucket"`
+	Total  uint64 `json:"total"`
+	Errors uint64 `json:"errors"`
+}
+
+// ErrorStats returns hourly (or daily) total/error entry counts over the
+// lookback window, for the trends graphs. Empty buckets are absent; the UI
+// fills gaps for display.
+func (s *ClickHouseStore) ErrorStats(ctx context.Context, service string, hours int, daily bool) ([]StatsPoint, error) {
+	bucket := "toStartOfHour(hour)"
+	if daily {
+		bucket = "toStartOfDay(hour)"
+	}
+	where := []string{"hour >= ?"}
+	args := []interface{}{time.Now().UTC().Add(-time.Duration(hours) * time.Hour)}
+	if service != "" {
+		where = append(where, "service = ?")
+		args = append(args, service)
+	}
+
+	query := fmt.Sprintf(
+		`SELECT
+			toString(%s) AS bucket,
+			sum(count) AS total,
+			sumIf(count, is_error = 1) AS errors
+		 FROM %s.hourly
+		 WHERE %s
+		 GROUP BY bucket
+		 ORDER BY bucket`, bucket, s.db, strings.Join(where, " AND "))
+
+	rows, err := s.conn.Query(ctx, query, args...)
+	if err != nil {
+		return nil, fmt.Errorf("clickhouse error stats: %w", err)
+	}
+	defer rows.Close()
+
+	var out []StatsPoint
+	for rows.Next() {
+		var p StatsPoint
+		if err := rows.Scan(&p.Bucket, &p.Total, &p.Errors); err != nil {
+			return nil, err
+		}
+		out = append(out, p)
+	}
+	return out, rows.Err()
+}
+
+// ReleaseSeries returns one error-count series per release:
+// {release -> [{bucket, total, errors}]}, so the UI can overlay deploys.
+func (s *ClickHouseStore) ReleaseSeries(ctx context.Context, service string, hours int) (map[string][]StatsPoint, error) {
+	where := []string{"hour >= ?", "release != ''"}
+	args := []interface{}{time.Now().UTC().Add(-time.Duration(hours) * time.Hour)}
+	if service != "" {
+		where = append(where, "service = ?")
+		args = append(args, service)
+	}
+
+	query := fmt.Sprintf(
+		`SELECT
+			release,
+			toString(toStartOfHour(hour)) AS bucket,
+			sum(count) AS total,
+			sumIf(count, is_error = 1) AS errors
+		 FROM %s.hourly
+		 WHERE %s
+		 GROUP BY release, bucket
+		 ORDER BY bucket`, s.db, strings.Join(where, " AND "))
+
+	rows, err := s.conn.Query(ctx, query, args...)
+	if err != nil {
+		return nil, fmt.Errorf("clickhouse release series: %w", err)
+	}
+	defer rows.Close()
+
+	out := map[string][]StatsPoint{}
+	for rows.Next() {
+		var (
+			release string
+			p       StatsPoint
+		)
+		if err := rows.Scan(&release, &p.Bucket, &p.Total, &p.Errors); err != nil {
+			return nil, err
+		}
+		out[release] = append(out[release], p)
+	}
+	return out, rows.Err()
+}
+
+// Services returns the distinct services seen in the rollup, for filter UIs.
+func (s *ClickHouseStore) Services(ctx context.Context) ([]string, error) {
+	rows, err := s.conn.Query(ctx, fmt.Sprintf(
+		`SELECT DISTINCT service FROM %s.hourly ORDER BY service`, s.db))
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var out []string
+	for rows.Next() {
+		var svc string
+		if err := rows.Scan(&svc); err != nil {
+			return nil, err
+		}
+		out = append(out, svc)
+	}
+	return out, rows.Err()
+}
+
+// Close closes the underlying connection.
+func (s *ClickHouseStore) Close() error {
+	return s.conn.Close()
+}

internal/storage/clickhouse_test.go

@@ -0,0 +1,121 @@
+package storage
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"testing"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/websocket"
+)
+
+// TestClickHouseStoreIntegration is skipped unless OTEL_CLICKHOUSE_TEST_ADDR
+// points at a disposable ClickHouse (native port 9000). It exercises the real
+// SQL: schema creation, Append, rollup-triggered stats and Search filters.
+//
+// Placeholder round-trips through AsyncInsert use wait_for_async_insert so
+// reads below see the writes without sleeps.
+func TestClickHouseStoreIntegration(t *testing.T) {
+	addr := os.Getenv("OTEL_CLICKHOUSE_TEST_ADDR")
+	if addr == "" {
+		t.Skip("set OTEL_CLICKHOUSE_TEST_ADDR=host:9000 (with a disposable DB) to run")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// Unique database per run so repeated runs don't collide.
+	db := "otel_test_" + time.Now().UTC().Format("150405")
+	cfg := ClickHouseConfig{Addr: addr, Database: db, Username: os.Getenv("OTEL_CLICKHOUSE_TEST_USER"), Password: os.Getenv("OTEL_CLICKHOUSE_TEST_PASSWORD")}
+	s, err := NewClickHouseStore(ctx, cfg)
+	if err != nil {
+		t.Skipf("ClickHouse not reachable at %s: %v", addr, err)
+	}
+	defer func() {
+		_ = s.conn.Exec(ctx, "DROP DATABASE IF EXISTS "+db)
+		_ = s.Close()
+	}()
+
+	now := time.Now().UTC()
+	mk := func(minsAgo int, typ, service string, isErr bool, release string) websocket.HistoryEntry {
+		raw, _ := json.Marshal(map[string]interface{}{"n": minsAgo})
+		return websocket.HistoryEntry{
+			Type:      typ,
+			Timestamp: now.Add(-time.Duration(minsAgo) * time.Minute).UnixNano(),
+			Data:      raw,
+		}
+	}
+	entries := []websocket.HistoryEntry{
+		mk(200, "logs", "api", true, "v1.3"),
+		mk(190, "logs", "api", false, "v1.3"),
+		mk(100, "logs", "api", true, "v1.4"),
+		mk(90, "logs", "api", false, "v1.4"),
+		mk(30, "logs", "api", false, "v1.4"),
+	}
+	for i, e := range entries {
+		if err := s.Append(ctx, e); err != nil {
+			t.Fatalf("Append[%d]: %v", i, err)
+		}
+	}
+	// Flush async insert buffer before reading.
+	if err := s.conn.AsyncInsert(ctx, "INSERT INTO FUNCTION null('x UInt8') VALUES (0)", true); err != nil {
+		t.Fatalf("flush: %v", err)
+	}
+
+	// Search: newest first, limit respected.
+	res, err := s.Search(ctx, SearchOptions{Type: "logs"}, 3)
+	if err != nil {
+		t.Fatalf("Search: %v", err)
+	}
+	if len(res) != 3 {
+		t.Fatalf("expected 3 search results, got %d", len(res))
+	}
+	if res[0].Timestamp < res[2].Timestamp {
+		t.Errorf("expected newest first")
+	}
+
+	// Search: substring filter.
+	res, err = s.Search(ctx, SearchOptions{Query: "\"n\":200"}, 10)
+	if err != nil {
+		t.Fatalf("Search query: %v", err)
+	}
+	if len(res) != 1 {
+		t.Errorf("expected substring filter to match 1 entry, got %d", len(res))
+	}
+
+	// ErrorStats via the rollup MV: two hours with data, 2 errors total.
+	points, err := s.ErrorStats(ctx, "api", 24, false)
+	if err != nil {
+		t.Fatalf("ErrorStats: %v", err)
+	}
+	errSum, totSum := uint64(0), uint64(0)
+	for _, p := range points {
+		errSum += p.Errors
+		totSum += p.Total
+	}
+	if errSum != 2 || totSum != 5 {
+		t.Errorf("expected rollup totals errors=2 total=5, got errors=%d total=%d", errSum, totSum)
+	}
+
+	// ReleaseSeries: v1.3 has 2 entries (1 error), v1.4 has 3 (1 error).
+	series, err := s.ReleaseSeries(ctx, "api", 24)
+	if err != nil {
+		t.Fatalf("ReleaseSeries: %v", err)
+	}
+	var v13, v14 uint64
+	for _, p := range series["v1.3"] {
+		v13 += p.Total
+	}
+	for _, p := range series["v1.4"] {
+		v14 += p.Total
+	}
+	if v13 != 2 || v14 != 3 {
+		t.Errorf("expected per-release totals v1.3=2 v1.4=3, got %d/%d", v13, v14)
+	}
+
+	// Services.
+	svcs, err := s.Services(ctx)
+	if err != nil || len(svcs) == 0 || svcs[0] != "api" {
+		t.Errorf("expected services [api], got %v (err=%v)", svcs, err)
+	}
+}

internal/storage/store.go

@@ -26,3 +26,15 @@ type Store interface {
 	GetHistory(ctx context.Context, limit int) ([]websocket.HistoryEntry, error)
 	Search(ctx context.Context, opts SearchOptions, limit int) ([]websocket.HistoryEntry, error)
 }
+
+// StatsQuerier is implemented by stores that can answer aggregate
+// time-series questions (currently ClickHouseStore). The stats HTTP
+// endpoints are only enabled when the configured store implements it.
+type StatsQuerier interface {
+	// ErrorStats returns hourly/daily total and error entry counts.
+	ErrorStats(ctx context.Context, service string, hours int, daily bool) ([]StatsPoint, error)
+	// ReleaseSeries returns per-release error-count time series.
+	ReleaseSeries(ctx context.Context, service string, hours int) (map[string][]StatsPoint, error)
+	// Services lists distinct services seen in the rollup.
+	Services(ctx context.Context) ([]string, error)
+}

internal/websocket/server.go

@@ -38,6 +38,28 @@ type Entry struct {
 	Data json.RawMessage `json:"data"`
 }
 
+// issuesNotification is broadcast when new error telemetry has been ingested,
+// so UI clients can refresh their Issues view immediately. It deliberately
+// carries no payload - clients re-fetch GET /api/issues.
+type issuesNotification struct {
+	Type string `json:"type"` // always "issues"
+}
+
+// NotifyIssues broadcasts an issues-refresh hint to all connected clients.
+func (s *Server) NotifyIssues() {
+	msg, err := json.Marshal(issuesNotification{Type: "issues"})
+	if err != nil {
+		return
+	}
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	for client := range s.clients {
+		if err := client.WriteMessage(websocket.TextMessage, msg); err != nil {
+			log.Printf("write error: %v", err)
+		}
+	}
+}
+
 // Broadcaster interface for sending telemetry updates to all connected clients.
 type Broadcaster interface {
 	Broadcast(data *Entry)

k8s/clickhouse.yaml

@@ -0,0 +1,134 @@
+# Single-node ClickHouse for the otel-debug-receiver, applied as one file:
+#   kubectl apply -f k8s/clickhouse.yaml
+#
+# A single instance (per request): one StatefulSet replica with a PVC so
+# telemetry survives pod restarts. Resources sized for a small cluster;
+# adjust before production use.
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: otel-debug
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: clickhouse-auth
+  namespace: monitoring
+type: Opaque
+stringData:
+  # Change these before applying. ClickHouse will honor the values from
+  # the users.d override mounted below, so no post-install SQL is needed.
+  username: otel
+  password: CHANGE_ME
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: clickhouse-config
+  namespace: monitoring
+data:
+  users.d-otel.xml: |
+    <clickhouse>
+      <users>
+        <otel>
+          <password from_env="CLICKHOUSE_PASSWORD"/>
+          <networks><ip>::/0</ip></networks>
+          <grants>
+            <grant><query>GRANT ALL ON otel.*</query></grant>
+          </grants>
+        </otel>
+      </users>
+    </clickhouse>
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: clickhouse
+  namespace: monitoring
+spec:
+  serviceName: clickhouse
+  replicas: 1
+  selector:
+    matchLabels:
+      app: clickhouse
+  template:
+    metadata:
+      labels:
+        app: clickhouse
+    spec:
+      securityContext:
+        fsGroup: 101
+      containers:
+      - name: clickhouse
+        image: clickhouse/clickhouse-server:24.8
+        ports:
+        - containerPort: 9000
+          name: native
+        - containerPort: 8123
+          name: http
+        env:
+        - name: CLICKHOUSE_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: clickhouse-auth
+              key: password
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/clickhouse
+        # Mount the user override as a single file: ConfigMap volumes are
+        # read-only, and the entrypoint needs to write its own
+        # default-user.xml into the users.d directory on first start.
+        - name: user-config
+          mountPath: /etc/clickhouse-server/users.d/otel.xml
+          subPath: users.d-otel.xml
+        readinessProbe:
+          httpGet:
+            path: /ping
+            port: http
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: http
+          initialDelaySeconds: 30
+          periodSeconds: 30
+        resources:
+          requests:
+            memory: "512Mi"
+            cpu: "250m"
+          limits:
+            memory: "2Gi"
+            cpu: "1"
+      volumes:
+      - name: user-config
+        configMap:
+          name: clickhouse-config
+          items:
+          - key: users.d-otel.xml
+            path: users.d-otel.xml
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      accessModes: ["ReadWriteOnce"]
+      resources:
+        requests:
+          storage: 20Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: clickhouse
+  namespace: monitoring
+spec:
+  selector:
+    app: clickhouse
+  ports:
+  - name: native
+    port: 9000
+    targetPort: 9000
+  - name: http
+    port: 8123
+    targetPort: 8123
+  type: ClusterIP

k8s/deployment.yaml

@@ -2,6 +2,10 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: otel-debug-receiver
+  namespace: monitoring
+  annotations:
+    k8s-helper.io/git-ssh: git@git.k6n.net:mats/go-otel.git
+    k8s-helper.io/git-url: https://git.k6n.net/mats/go-otel
 spec:
   replicas: 1
   selector:
@@ -12,6 +16,7 @@ spec:
       labels:
         app: otel-debug-receiver
     spec:
+      serviceAccountName: otel-debug-receiver
       securityContext:
         runAsNonRoot: true
         runAsUser: 1000
@@ -40,9 +45,77 @@ spec:
           value: "info"
         - name: OTELLOG_PATH
           value: "/logs/otel-agent.log"
+        - name: OTEL_SOURCEMAP_DIR
+          value: "/sourcemaps"
+        - name: OTEL_OIDC_ISSUER
+          value: "https://users.k6n.net"
+        - name: OTEL_OIDC_CLIENT_ID
+          value: "otel-debug"
+        # Service credential for non-interactive debugging agents. It grants
+        # access to protected /api debug endpoints, not OTLP ingest.
+        - name: OTEL_AGENT_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: otel-agent-auth
+              key: token
+              # The debug API must not start without its service credential.
+              # Leaving this optional makes the pod appear healthy while every
+              # protected API request can only return 401.
+              optional: false
+        # ClickHouse storage: enables the aggregate stats endpoints
+        # (/api/stats/*). Falls back to the file store if unreachable.
+        - name: OTEL_CLICKHOUSE_ADDR
+          value: "clickhouse.monitoring.svc.cluster.local:9000"
+        - name: OTEL_CLICKHOUSE_DB
+          value: "otel"
+        - name: OTEL_CLICKHOUSE_USER
+          valueFrom:
+            secretKeyRef:
+              name: clickhouse-auth
+              key: username
+              optional: true
+        - name: OTEL_CLICKHOUSE_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: clickhouse-auth
+              key: password
+              optional: true
+        # Connection pool tuning to prevent "acquire conn timeout" under ingest bursts
+        - name: OTEL_CLICKHOUSE_MAX_OPEN_CONNS
+          value: "10"
+        - name: OTEL_CLICKHOUSE_MAX_IDLE_CONNS
+          value: "5"
+        - name: OTEL_CLICKHOUSE_DIAL_TIMEOUT
+          value: "30s"
+        - name: OTEL_CLICKHOUSE_CONN_MAX_LIFETIME
+          value: "10m"
+        - name: OTEL_STORE_WORKERS
+          value: "4"
+        # Optional rollout observation. The watcher is read-only and uses the
+        # service account below. Leave Prometheus unset when unavailable.
+        - name: OTEL_KUBERNETES_NAMESPACE
+          value: ""
+        # - name: OTEL_PROMETHEUS_URL
+        #   value: "http://prometheus.monitoring.svc.cluster.local:9090"
+        # Private token for an ingress-side synchronizer. If absent, the
+        # promoted-IP endpoint stays disabled.
+        - name: OTEL_HONEYPOT_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: honeypot-auth
+              key: token
+              optional: true
+        - name: OTEL_HONEYPOT_THRESHOLD
+          value: "3"
+        - name: OTEL_HONEYPOT_WINDOW
+          value: "15m"
+        - name: OTEL_HONEYPOT_TTL
+          value: "1h"
         volumeMounts:
         - name: log-dir
           mountPath: /logs
+        - name: sourcemap-dir
+          mountPath: /sourcemaps
         livenessProbe:
           httpGet:
             path: /health
@@ -65,3 +138,39 @@ spec:
       volumes:
       - name: log-dir
         emptyDir: {}
+      - name: sourcemap-dir
+        emptyDir: {}
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: otel-debug-receiver
+  namespace: monitoring
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: otel-debug-receiver-rollout-reader
+rules:
+- apiGroups: ["apps"]
+  resources: ["deployments", "replicasets"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  resources: ["pods", "events"]
+  verbs: ["get", "list", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: otel-debug-receiver-rollout-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: otel-debug-receiver-rollout-reader
+subjects:
+- kind: ServiceAccount
+  name: otel-debug-receiver
+  namespace: monitoring

k8s/ingress.yaml

@@ -1,11 +1,99 @@
 apiVersion: networking.k8s.io/v1
 kind: Ingress
+metadata:
+  name: otel-debug-ingress-public
+  namespace: monitoring
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+spec:
+  ingressClassName: nginx
+  tls:
+    - hosts:
+        - otel.k6n.net
+      secretName: otel-k6n-tls
+  rules:
+    - host: otel.k6n.net
+      http:
+        paths:
+          - path: /api
+            pathType: Prefix
+            backend:
+              service:
+                name: otel-debug-service
+                port:
+                  number: 80
+          - path: /v1
+            pathType: Prefix
+            backend:
+              service:
+                name: otel-debug-service
+                port:
+                  number: 80
+          - path: /health
+            pathType: Prefix
+            backend:
+              service:
+                name: otel-debug-service
+                port:
+                  number: 80
+          # The handler has its own static token authentication. This route
+          # lets an in-cluster ingress synchronizer fetch promoted IPs without
+          # requiring a browser OIDC session.
+          - path: /internal/honeypot
+            pathType: Prefix
+            backend:
+              service:
+                name: otel-debug-service
+                port:
+                  number: 80
+          - path: /ws
+            pathType: Prefix
+            backend:
+              service:
+                name: otel-debug-service
+                port:
+                  number: 80
+          - path: /agents_example.md
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: otel-debug-service
+                port:
+                  number: 80
+          - path: /instrumentation.md
+            pathType: ImplementationSpecific
+            backend:
+              service:
+                name: otel-debug-service
+                port:
+                  number: 80
+          - path: /static
+            pathType: Prefix
+            backend:
+              service:
+                name: otel-debug-service
+                port:
+                  number: 80
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
 metadata:
   name: otel-debug-ingress
+  namespace: monitoring
   annotations:
     cert-manager.io/cluster-issuer: letsencrypt-prod
+    # OIDC protection via the platform auth service (users.k6n.net):
+    # /oauth2/auth resolves the session cookie / bearer token and answers
+    # 200 (allow), 401 (challenge -> browser_redirect to /login) or 403.
+    # otel.k6n.net is registered as a realm host in the auth service's
+    # realms.json. Public API and agent endpoints are routed via otel-debug-ingress-public.
+    nginx.ingress.kubernetes.io/auth-url: https://users.k6n.net/oauth2/auth
+    nginx.ingress.kubernetes.io/auth-signin: https://users.k6n.net/login?rd=$scheme://$host$escaped_request_uri
+    nginx.ingress.kubernetes.io/auth-response-headers: X-Auth-User
+    nginx.ingress.kubernetes.io/auth-cache-duration: 200 60s, 401 5s, 403 5s
+    nginx.ingress.kubernetes.io/auth-proxy-set-headers: monitoring/auth-subrequest-headers
 spec:
-  ingressClassName: nginx # Assuming nginx ingress controller
+  ingressClassName: nginx
   tls:
     - hosts:
         - otel.k6n.net
@@ -21,3 +109,13 @@ spec:
                 name: otel-debug-service
                 port:
                   number: 80
+---
+# Header map for the auth subrequest (same pattern as home/auth-subrequest-headers).
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: auth-subrequest-headers
+  namespace: monitoring
+data:
+  X-Original-Method: $request_method
+  X-Original-URL: $scheme://$host$request_uri

otel-ui/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

otel-ui/README.md

@@ -0,0 +1,73 @@
+# React + TypeScript + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Babel](https://babeljs.io/) (or [oxc](https://oxc.rs) when used in [rolldown-vite](https://vite.dev/guide/rolldown)) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
+
+## React Compiler
+
+The React Compiler is not enabled on this template because of its impact on dev & build performances. To add it, see [this documentation](https://react.dev/learn/react-compiler/installation).
+
+## Expanding the ESLint configuration
+
+If you are developing a production application, we recommend updating the configuration to enable type-aware lint rules:
+
+```js
+export default defineConfig([
+  globalIgnores(['dist']),
+  {
+    files: ['**/*.{ts,tsx}'],
+    extends: [
+      // Other configs...
+
+      // Remove tseslint.configs.recommended and replace with this
+      tseslint.configs.recommendedTypeChecked,
+      // Alternatively, use this for stricter rules
+      tseslint.configs.strictTypeChecked,
+      // Optionally, add this for stylistic rules
+      tseslint.configs.stylisticTypeChecked,
+
+      // Other configs...
+    ],
+    languageOptions: {
+      parserOptions: {
+        project: ['./tsconfig.node.json', './tsconfig.app.json'],
+        tsconfigRootDir: import.meta.dirname,
+      },
+      // other options...
+    },
+  },
+])
+```
+
+You can also install [eslint-plugin-react-x](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-x) and [eslint-plugin-react-dom](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-dom) for React-specific lint rules:
+
+```js
+// eslint.config.js
+import reactX from 'eslint-plugin-react-x'
+import reactDom from 'eslint-plugin-react-dom'
+
+export default defineConfig([
+  globalIgnores(['dist']),
+  {
+    files: ['**/*.{ts,tsx}'],
+    extends: [
+      // Other configs...
+      // Enable lint rules for React
+      reactX.configs['recommended-typescript'],
+      // Enable lint rules for React DOM
+      reactDom.configs.recommended,
+    ],
+    languageOptions: {
+      parserOptions: {
+        project: ['./tsconfig.node.json', './tsconfig.app.json'],
+        tsconfigRootDir: import.meta.dirname,
+      },
+      // other options...
+    },
+  },
+])
+```

otel-ui/index.html

@@ -0,0 +1,20 @@
+<!doctype html>
+<html class="dark" lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>HyperScope OTel — Distributed Telemetry</title>
+    <link rel="preconnect" href="https://fonts.googleapis.com" />
+    <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
+    <link
+      href="https://fonts.googleapis.com/css2?family=Geist:wght@400;500;600;700&family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500;600;700&family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200&display=swap"
+      rel="stylesheet"
+    />
+  </head>
+  <body class="bg-background text-on-surface min-h-screen antialiased">
+    <div id="root"></div>
+    <script src="/static/otel-rum.js"></script>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>

otel-ui/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "otel-ui",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc -b && vite build",
+    "lint": "eslint .",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@radix-ui/react-accordion": "1.2.2",
+    "@radix-ui/react-alert-dialog": "1.1.4",
+    "@radix-ui/react-aspect-ratio": "1.1.1",
+    "@radix-ui/react-avatar": "1.1.2",
+    "@radix-ui/react-checkbox": "1.1.3",
+    "@radix-ui/react-collapsible": "1.1.2",
+    "@radix-ui/react-context-menu": "2.2.4",
+    "@radix-ui/react-dialog": "1.1.4",
+    "@radix-ui/react-dropdown-menu": "2.1.4",
+    "@radix-ui/react-hover-card": "1.1.4",
+    "@radix-ui/react-label": "2.1.1",
+    "@radix-ui/react-menubar": "1.1.4",
+    "@radix-ui/react-navigation-menu": "1.2.3",
+    "@radix-ui/react-popover": "1.1.4",
+    "@radix-ui/react-progress": "1.1.1",
+    "@radix-ui/react-radio-group": "1.2.2",
+    "@radix-ui/react-scroll-area": "1.2.2",
+    "@radix-ui/react-select": "2.1.4",
+    "@radix-ui/react-separator": "1.1.1",
+    "@radix-ui/react-slider": "1.2.2",
+    "@radix-ui/react-slot": "1.1.1",
+    "@radix-ui/react-switch": "1.1.2",
+    "@radix-ui/react-tabs": "1.1.2",
+    "@radix-ui/react-toast": "1.2.4",
+    "@radix-ui/react-toggle": "1.1.1",
+    "@radix-ui/react-toggle-group": "1.1.1",
+    "@radix-ui/react-tooltip": "1.1.6",
+    "@tailwindcss/postcss": "^4.1.16",
+    "@tailwindcss/vite": "^4.1.17",
+    "@tanstack/react-virtual": "^3.14.13",
+    "react": "^19.1.1",
+    "react-dom": "^19.1.1",
+    "recharts": "^3.8.1",
+    "swr": "^2.5.1",
+    "tailwind-merge": "^2.5.5",
+    "wouter": "^3.11.0"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.36.0",
+    "@types/node": "^24.6.0",
+    "@types/react": "^19.1.16",
+    "@types/react-dom": "^19.1.9",
+    "@vitejs/plugin-react": "^5.0.4",
+    "biome": "^0.3.3",
+    "globals": "^16.4.0",
+    "lucide-react": "^0.454.0",
+    "postcss": "^8.5",
+    "tailwindcss": "^4.1.17",
+    "tw-animate-css": "1.3.3",
+    "typescript": "~5.9.3",
+    "vite": "^7.1.7"
+  }
+}

otel-ui/postcss.config.mjs

@@ -0,0 +1,8 @@
+/** @type {import('postcss-load-config').Config} */
+const config = {
+  plugins: {
+    '@tailwindcss/postcss': {},
+  },
+}
+
+export default config

otel-ui/public/vite.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="31.88" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 257"><defs><linearGradient id="IconifyId1813088fe1fbc01fb466" x1="-.828%" x2="57.636%" y1="7.652%" y2="78.411%"><stop offset="0%" stop-color="#41D1FF"></stop><stop offset="100%" stop-color="#BD34FE"></stop></linearGradient><linearGradient id="IconifyId1813088fe1fbc01fb467" x1="43.376%" x2="50.316%" y1="2.242%" y2="89.03%"><stop offset="0%" stop-color="#FFEA83"></stop><stop offset="8.333%" stop-color="#FFDD35"></stop><stop offset="100%" stop-color="#FFA800"></stop></linearGradient></defs><path fill="url(#IconifyId1813088fe1fbc01fb466)" d="M255.153 37.938L134.897 252.976c-2.483 4.44-8.862 4.466-11.382.048L.875 37.958c-2.746-4.814 1.371-10.646 6.827-9.67l120.385 21.517a6.537 6.537 0 0 0 2.322-.004l117.867-21.483c5.438-.991 9.574 4.796 6.877 9.62Z"></path><path fill="url(#IconifyId1813088fe1fbc01fb467)" d="M185.432.063L96.44 17.501a3.268 3.268 0 0 0-2.634 3.014l-5.474 92.456a3.268 3.268 0 0 0 3.997 3.378l24.777-5.718c2.318-.535 4.413 1.507 3.936 3.838l-7.361 36.047c-.495 2.426 1.782 4.5 4.151 3.78l15.304-4.649c2.372-.72 4.652 1.36 4.15 3.788l-11.698 56.621c-.732 3.542 3.979 5.473 5.943 2.437l1.313-2.028l72.516-144.72c1.215-2.423-.88-5.186-3.54-4.672l-25.505 4.922c-2.396.462-4.435-1.77-3.759-4.114l16.646-57.705c.677-2.35-1.37-4.583-3.769-4.113Z"></path></svg>

otel-ui/src/App.tsx

@@ -0,0 +1,126 @@
+import { useState, useEffect, useRef } from "react";
+import { useObservabilityData } from "./hooks/useObservabilityData";
+import { Sidebar } from "./components/Sidebar";
+import { TopHeader } from "./components/TopHeader";
+import { CommandAndQueryBar } from "./components/CommandAndQueryBar";
+import { WaveformHistogram } from "./components/WaveformHistogram";
+import { EntryVisualization } from "./components/EntryVisualization";
+import { handleCallback, login } from "./lib/auth";
+
+function AuthCallbackView() {
+  const [error, setError] = useState<string | null>(null);
+  const handledRef = useRef(false);
+
+  useEffect(() => {
+    if (handledRef.current) return;
+    handledRef.current = true;
+
+    handleCallback()
+      .then(() => {
+        window.location.href = "/";
+      })
+      .catch((err) => {
+        console.error("Authentication callback error:", err);
+        setError(err instanceof Error ? err.message : String(err));
+      });
+  }, []);
+
+  return (
+    <div className="min-h-screen bg-background text-on-surface flex items-center justify-center p-6">
+      <div className="max-w-md w-full rounded-2xl border border-surface-container bg-surface-container-low p-8 text-center shadow-xl">
+        {error ? (
+          <div className="flex flex-col items-center gap-4">
+            <div className="w-14 h-14 rounded-full bg-error-container text-on-error-container flex items-center justify-center">
+              <span className="material-symbols-outlined text-[32px]">error</span>
+            </div>
+            <div>
+              <h2 className="text-lg font-bold text-error">Authentication Failed</h2>
+              <p className="text-xs text-outline mt-1">{error}</p>
+            </div>
+            <button
+              type="button"
+              onClick={() => login()}
+              className="mt-2 px-4 py-2 rounded bg-primary text-on-primary font-mono text-xs hover:bg-primary/90 transition-colors"
+            >
+              Try again
+            </button>
+          </div>
+        ) : (
+          <div className="flex flex-col items-center gap-4">
+            <div className="w-14 h-14 rounded-full bg-primary/10 text-primary flex items-center justify-center animate-pulse">
+              <span className="material-symbols-outlined text-[32px]">lock</span>
+            </div>
+            <div>
+              <h2 className="text-lg font-semibold text-on-surface">Completing sign in</h2>
+              <p className="text-xs text-outline mt-1">Exchanging authorization code with OIDC provider…</p>
+            </div>
+          </div>
+        )}
+      </div>
+    </div>
+  );
+}
+
+function App() {
+  const { isConnected, traces, logs, metrics } = useObservabilityData();
+
+  const hasLoadedAnyData = traces.length > 0 || logs.length > 0 || metrics.length > 0;
+  const isCallback =
+    typeof window !== "undefined" &&
+    (window.location.pathname === "/auth_callback" ||
+      window.location.search.includes("code="));
+
+  if (isCallback) {
+    return <AuthCallbackView />;
+  }
+
+  return (
+    <div className="bg-background font-body text-on-surface min-h-screen antialiased selection:bg-primary-container selection:text-on-primary-container">
+      {/* Fixed Left Navigation Sidebar (w-72) */}
+      <Sidebar />
+
+      {/* Main Content Area offset by pl-72 */}
+      <div className="pl-72">
+        {/* Fixed Top Header Bar */}
+        <TopHeader />
+
+        {/* Main Telemetry Console View */}
+        <main className="relative pt-18 w-full px-6 bg-background min-h-screen pb-12">
+          <div className="flex flex-col w-full">
+            {/* Real Anomaly Radar & Filter Bar */}
+            <CommandAndQueryBar />
+
+            {/* Ingest & Incident Waveform Histogram */}
+            <WaveformHistogram />
+
+            {/* Telemetry Visualizations & Views */}
+            <section className="relative flex-1 flex flex-col min-h-0">
+              {isConnected || hasLoadedAnyData ? (
+                <EntryVisualization />
+              ) : (
+                <div className="flex h-full min-h-[360px] flex-col items-center justify-center gap-4 text-center rounded-2xl border border-dashed border-surface-container-high p-12 bg-surface-container-lowest">
+                  <div className="flex h-16 w-16 items-center justify-center rounded-full bg-surface-container-high text-primary-container shadow-lg animate-pulse">
+                    <span className="material-symbols-outlined text-[32px]">
+                      cloud_sync
+                    </span>
+                  </div>
+                  <div>
+                    <p className="text-base font-semibold text-on-surface">
+                      Establishing telemetry pipeline
+                    </p>
+                    <p className="text-xs text-outline mt-1 max-w-sm mx-auto">
+                      Connecting to OTLP collector. Listening for incoming traces, logs, and metrics.
+                    </p>
+                  </div>
+                </div>
+              )}
+            </section>
+          </div>
+        </main>
+      </div>
+    </div>
+  );
+
+}
+
+export default App;

otel-ui/src/assets/react.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" class="iconify iconify--logos" width="35.93" height="32" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 228"><path fill="#00D8FF" d="M210.483 73.824a171.49 171.49 0 0 0-8.24-2.597c.465-1.9.893-3.777 1.273-5.621c6.238-30.281 2.16-54.676-11.769-62.708c-13.355-7.7-35.196.329-57.254 19.526a171.23 171.23 0 0 0-6.375 5.848a155.866 155.866 0 0 0-4.241-3.917C100.759 3.829 77.587-4.822 63.673 3.233C50.33 10.957 46.379 33.89 51.995 62.588a170.974 170.974 0 0 0 1.892 8.48c-3.28.932-6.445 1.924-9.474 2.98C17.309 83.498 0 98.307 0 113.668c0 15.865 18.582 31.778 46.812 41.427a145.52 145.52 0 0 0 6.921 2.165a167.467 167.467 0 0 0-2.01 9.138c-5.354 28.2-1.173 50.591 12.134 58.266c13.744 7.926 36.812-.22 59.273-19.855a145.567 145.567 0 0 0 5.342-4.923a168.064 168.064 0 0 0 6.92 6.314c21.758 18.722 43.246 26.282 56.54 18.586c13.731-7.949 18.194-32.003 12.4-61.268a145.016 145.016 0 0 0-1.535-6.842c1.62-.48 3.21-.974 4.76-1.488c29.348-9.723 48.443-25.443 48.443-41.52c0-15.417-17.868-30.326-45.517-39.844Zm-6.365 70.984c-1.4.463-2.836.91-4.3 1.345c-3.24-10.257-7.612-21.163-12.963-32.432c5.106-11 9.31-21.767 12.459-31.957c2.619.758 5.16 1.557 7.61 2.4c23.69 8.156 38.14 20.213 38.14 29.504c0 9.896-15.606 22.743-40.946 31.14Zm-10.514 20.834c2.562 12.94 2.927 24.64 1.23 33.787c-1.524 8.219-4.59 13.698-8.382 15.893c-8.067 4.67-25.32-1.4-43.927-17.412a156.726 156.726 0 0 1-6.437-5.87c7.214-7.889 14.423-17.06 21.459-27.246c12.376-1.098 24.068-2.894 34.671-5.345a134.17 134.17 0 0 1 1.386 6.193ZM87.276 214.515c-7.882 2.783-14.16 2.863-17.955.675c-8.075-4.657-11.432-22.636-6.853-46.752a156.923 156.923 0 0 1 1.869-8.499c10.486 2.32 22.093 3.988 34.498 4.994c7.084 9.967 14.501 19.128 21.976 27.15a134.668 134.668 0 0 1-4.877 4.492c-9.933 8.682-19.886 14.842-28.658 17.94ZM50.35 144.747c-12.483-4.267-22.792-9.812-29.858-15.863c-6.35-5.437-9.555-10.836-9.555-15.216c0-9.322 13.897-21.212 37.076-29.293c2.813-.98 5.757-1.905 8.812-2.773c3.204 10.42 7.406 21.315 12.477 32.332c-5.137 11.18-9.399 22.249-12.634 32.792a134.718 134.718 0 0 1-6.318-1.979Zm12.378-84.26c-4.811-24.587-1.616-43.134 6.425-47.789c8.564-4.958 27.502 2.111 47.463 19.835a144.318 144.318 0 0 1 3.841 3.545c-7.438 7.987-14.787 17.08-21.808 26.988c-12.04 1.116-23.565 2.908-34.161 5.309a160.342 160.342 0 0 1-1.76-7.887Zm110.427 27.268a347.8 347.8 0 0 0-7.785-12.803c8.168 1.033 15.994 2.404 23.343 4.08c-2.206 7.072-4.956 14.465-8.193 22.045a381.151 381.151 0 0 0-7.365-13.322Zm-45.032-43.861c5.044 5.465 10.096 11.566 15.065 18.186a322.04 322.04 0 0 0-30.257-.006c4.974-6.559 10.069-12.652 15.192-18.18ZM82.802 87.83a323.167 323.167 0 0 0-7.227 13.238c-3.184-7.553-5.909-14.98-8.134-22.152c7.304-1.634 15.093-2.97 23.209-3.984a321.524 321.524 0 0 0-7.848 12.897Zm8.081 65.352c-8.385-.936-16.291-2.203-23.593-3.793c2.26-7.3 5.045-14.885 8.298-22.6a321.187 321.187 0 0 0 7.257 13.246c2.594 4.48 5.28 8.868 8.038 13.147Zm37.542 31.03c-5.184-5.592-10.354-11.779-15.403-18.433c4.902.192 9.899.29 14.978.29c5.218 0 10.376-.117 15.453-.343c-4.985 6.774-10.018 12.97-15.028 18.486Zm52.198-57.817c3.422 7.8 6.306 15.345 8.596 22.52c-7.422 1.694-15.436 3.058-23.88 4.071a382.417 382.417 0 0 0 7.859-13.026a347.403 347.403 0 0 0 7.425-13.565Zm-16.898 8.101a358.557 358.557 0 0 1-12.281 19.815a329.4 329.4 0 0 1-23.444.823c-7.967 0-15.716-.248-23.178-.732a310.202 310.202 0 0 1-12.513-19.846h.001a307.41 307.41 0 0 1-10.923-20.627a310.278 310.278 0 0 1 10.89-20.637l-.001.001a307.318 307.318 0 0 1 12.413-19.761c7.613-.576 15.42-.876 23.31-.876H128c7.926 0 15.743.303 23.354.883a329.357 329.357 0 0 1 12.335 19.695a358.489 358.489 0 0 1 11.036 20.54a329.472 329.472 0 0 1-11 20.722Zm22.56-122.124c8.572 4.944 11.906 24.881 6.52 51.026c-.344 1.668-.73 3.367-1.15 5.09c-10.622-2.452-22.155-4.275-34.23-5.408c-7.034-10.017-14.323-19.124-21.64-27.008a160.789 160.789 0 0 1 5.888-5.4c18.9-16.447 36.564-22.941 44.612-18.3ZM128 90.808c12.625 0 22.86 10.235 22.86 22.86s-10.235 22.86-22.86 22.86s-22.86-10.235-22.86-22.86s10.235-22.86 22.86-22.86Z"></path></svg>

otel-ui/src/components/CommandAndQueryBar.tsx

@@ -0,0 +1,251 @@
+import React from "react";
+import { useObservabilityData } from "../hooks/useObservabilityData";
+import { useIssues } from "../hooks/useIssues";
+import { getServiceName, isSpanError } from "../lib/otel-utils";
+
+export const CommandAndQueryBar: React.FC = () => {
+  const {
+    logs,
+    traces,
+    metrics,
+    searchQuery,
+    setSearchQuery,
+
+    showErrorsOnly,
+    setShowErrorsOnly,
+    selectedServices,
+    toggleService,
+    resetFilters,
+    minDurationMs,
+    setMinDurationMs,
+    timeWindow,
+    setTimeWindow,
+    setActiveTab,
+  } = useObservabilityData();
+
+  const { issues } = useIssues();
+
+  // Compute real error metrics from filtered signals
+  const errorStats = React.useMemo(() => {
+    let errorCount = 0;
+    const servicesWithErrors = new Set<string>();
+
+    logs.forEach((l) =>
+      (l.scopeLogs ?? []).forEach((sl) =>
+        (sl.logRecords ?? []).forEach((r) => {
+          if ((r.severityNumber ?? 0) >= 17) {
+            errorCount++;
+            const sName = getServiceName(l.resource?.attributes ?? [], r.attributes ?? []);
+            if (sName !== "unknown-service") {
+              servicesWithErrors.add(sName);
+            }
+          }
+        }),
+      ),
+    );
+
+    traces.forEach((t) =>
+      (t.resourceSpans ?? []).forEach((rs) =>
+        (rs.scopeSpans ?? []).forEach((ss) =>
+          (ss.spans ?? []).forEach((s) => {
+            if (isSpanError(s)) {
+              errorCount++;
+              const sName = getServiceName(rs.resource?.attributes ?? [], s.attributes ?? []);
+              if (sName !== "unknown-service") {
+                servicesWithErrors.add(sName);
+              }
+            }
+          }),
+        ),
+      ),
+    );
+
+    return {
+      errorCount,
+      services: Array.from(servicesWithErrors),
+    };
+  }, [logs, traces]);
+
+  // Top open issue if available
+  const topOpenIssue = React.useMemo(() => {
+    const open = issues.filter((i) => !i.status || i.status === "open");
+    return open.sort((a, b) => b.count - a.count)[0] || null;
+  }, [issues]);
+
+  const totalFilteredEvents = logs.length + traces.length + metrics.length;
+
+  const exportNDJSON = () => {
+    const records = [
+      ...traces.map((t) => JSON.stringify({ type: "traces", ...t })),
+      ...logs.map((l) => JSON.stringify({ type: "logs", ...l })),
+      ...metrics.map((m) => JSON.stringify({ type: "metrics", ...m })),
+    ].join("\n");
+
+    const blob = new Blob([records], { type: "application/x-ndjson" });
+    const url = URL.createObjectURL(blob);
+    const a = document.createElement("a");
+    a.href = url;
+    a.download = `hyperscope-filtered-telemetry-${new Date().toISOString()}.ndjson`;
+    a.click();
+    URL.revokeObjectURL(url);
+  };
+
+  const hasActiveFilters =
+    Boolean(searchQuery) ||
+    showErrorsOnly ||
+    selectedServices.size > 0 ||
+    minDurationMs > 0 ||
+    timeWindow !== "all";
+
+  return (
+    <div className="flex flex-col gap-2.5 mb-4 font-body">
+      {/* Real Anomaly Radar Banner */}
+      {(errorStats.errorCount > 0 || topOpenIssue) && (
+        <div className="rounded-xl border border-rose-500/30 bg-gradient-to-r from-rose-950/40 via-slate-950/60 to-rose-950/20 p-3 shadow-lg flex flex-col sm:flex-row sm:items-center justify-between gap-3 animate-in fade-in duration-300">
+          <div className="flex items-center gap-2.5 min-w-0">
+            <div className="flex h-7 w-7 shrink-0 items-center justify-center rounded-lg bg-rose-500/20 text-rose-400 border border-rose-500/40">
+              <span className="material-symbols-outlined text-[18px]">warning</span>
+            </div>
+            <div className="flex flex-col min-w-0">
+              <div className="flex items-center gap-2">
+                <span className="font-mono text-xs font-bold text-rose-300 uppercase tracking-wider">
+                  Anomaly Radar:
+                </span>
+                <span className="font-mono text-xs text-white truncate">
+                  {errorStats.errorCount > 0
+                    ? `${errorStats.errorCount} errors in active window (${errorStats.services.join(", ") || "various"})`
+                    : `${topOpenIssue?.count} occurrences of "${topOpenIssue?.title}"`}
+                </span>
+              </div>
+              {topOpenIssue && (
+                <span className="text-[11px] text-slate-400 truncate">
+                  Top Issue: <span className="text-slate-200 font-semibold">{topOpenIssue.title}</span> (Service: {topOpenIssue.service}, ×{topOpenIssue.count})
+                </span>
+              )}
+            </div>
+          </div>
+
+          <div className="flex items-center gap-2 shrink-0">
+            <button
+              type="button"
+              onClick={() => setActiveTab("tab0")}
+              className="px-3 py-1 rounded-lg bg-rose-500 hover:bg-rose-600 text-white font-mono text-xs font-bold transition-colors shadow-sm"
+            >
+              Triage Issues
+            </button>
+            {!showErrorsOnly && (
+              <button
+                type="button"
+                onClick={() => setShowErrorsOnly(true)}
+                className="px-2.5 py-1 rounded-lg bg-white/10 hover:bg-white/20 text-slate-200 font-mono text-xs transition-colors"
+              >
+                Isolate Errors
+              </button>
+            )}
+          </div>
+        </div>
+      )}
+
+      {/* Filter and Telemetry Status Bar */}
+      <div className="flex flex-wrap items-center justify-between gap-2.5 p-3 rounded-xl bg-surface-container-lowest border border-surface-container/50 shadow-sm">
+        <div className="flex items-center gap-2 flex-wrap">
+          <span className="font-mono text-[10px] text-outline uppercase tracking-wider font-semibold shrink-0">
+            Active Context:
+          </span>
+
+          <span className="font-mono text-xs text-secondary font-semibold">
+            {totalFilteredEvents} Signals ({timeWindow === "all" ? "All Time" : `Last ${timeWindow}`})
+          </span>
+
+          {/* Selected Service Pills */}
+          {Array.from(selectedServices).map((service) => (
+            <span
+              key={service}
+              className="inline-flex items-center gap-1 rounded-full bg-primary-container/20 text-primary-container border border-primary-container/40 px-2.5 py-0.5 font-mono text-xs font-semibold"
+            >
+              <span>service:{service}</span>
+              <button
+                type="button"
+                onClick={() => toggleService(service)}
+                className="hover:text-white"
+              >
+                ×
+              </button>
+            </span>
+          ))}
+
+          {/* Errors Only Pill */}
+          {showErrorsOnly && (
+            <span className="inline-flex items-center gap-1 rounded-full bg-error-container text-on-error-container px-2.5 py-0.5 font-mono text-xs font-semibold">
+              <span>Errors Only</span>
+              <button
+                type="button"
+                onClick={() => setShowErrorsOnly(false)}
+                className="hover:text-white"
+              >
+                ×
+              </button>
+            </span>
+          )}
+
+          {/* Slow Spans Pill */}
+          {minDurationMs > 0 && (
+            <span className="inline-flex items-center gap-1 rounded-full bg-amber-500/20 text-amber-300 border border-amber-500/40 px-2.5 py-0.5 font-mono text-xs font-semibold">
+              <span>Latency &gt;{minDurationMs}ms</span>
+              <button
+                type="button"
+                onClick={() => setMinDurationMs(0)}
+                className="hover:text-white"
+              >
+                ×
+              </button>
+            </span>
+          )}
+
+          {/* Search Query Pill */}
+          {searchQuery && (
+            <span className="inline-flex items-center gap-1 rounded-full bg-surface-container text-on-surface border border-surface-container-high px-2.5 py-0.5 font-mono text-xs">
+              <span>query: &quot;{searchQuery}&quot;</span>
+              <button
+                type="button"
+                onClick={() => setSearchQuery("")}
+                className="hover:text-white"
+              >
+                ×
+              </button>
+            </span>
+          )}
+
+          {hasActiveFilters && (
+            <button
+              type="button"
+              onClick={() => {
+                resetFilters();
+                setMinDurationMs(0);
+                setTimeWindow("all");
+              }}
+              className="text-outline hover:text-white font-mono text-xs underline ml-1"
+            >
+              Clear all
+            </button>
+          )}
+        </div>
+
+        {/* Export and Action buttons */}
+        <div className="flex items-center gap-2">
+          <button
+            type="button"
+            onClick={exportNDJSON}
+            className="flex items-center gap-1 px-2.5 py-1 bg-surface-container-low hover:bg-surface-container text-on-surface font-mono text-xs rounded-lg transition-colors border border-surface-container/40 shadow-sm"
+            title="Export currently filtered telemetry signals as NDJSON"
+          >
+            <span className="material-symbols-outlined text-[15px]">download</span>
+            <span>Export NDJSON</span>
+          </button>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default CommandAndQueryBar;

otel-ui/src/components/EntryVisualization.tsx

@@ -0,0 +1,9 @@
+import { TabsComponent } from "./Tabs";
+
+export const EntryVisualization = () => {
+  return (
+    <div className="animate-in fade-in slide-in-from-bottom-4 duration-700">
+       <TabsComponent />
+    </div>
+  );
+};

otel-ui/src/components/GaugeMeter.tsx

@@ -0,0 +1,372 @@
+import { useMemo } from "react";
+import { ArrowDownRight, ArrowUpRight, Minus } from "lucide-react";
+import { getServicePalette } from "../lib/otel-utils";
+
+export interface GaugeMeterProps {
+  name: string;
+  service?: string;
+  value: number;
+  min?: number;
+  max?: number;
+  avg?: number;
+  unit?: string;
+  size?: "sm" | "md" | "lg";
+  thresholds?: {
+    warn: number;
+    danger: number;
+  };
+  subtitle?: string;
+  history?: number[];
+  showTrend?: boolean;
+  isSum?: boolean;
+  aggLabel?: string;
+  soloPod?: string;
+  podCount?: number;
+  podSummaries?: Array<{ pod: string; value: number }>;
+}
+
+const formatNumber = (val: number, maxDecimals = 2) => {
+  if (Math.abs(val) >= 1_000_000) {
+    return `${(val / 1_000_000).toFixed(1)}M`;
+  }
+  if (Math.abs(val) >= 1_000) {
+    return `${(val / 1_000).toFixed(1)}k`;
+  }
+  if (Number.isInteger(val)) {
+    return val.toString();
+  }
+  return val.toFixed(maxDecimals);
+};
+
+// SVG arc calculations
+const polarToCartesian = (centerX: number, centerY: number, radius: number, angleInDegrees: number) => {
+  const angleInRadians = ((angleInDegrees - 90) * Math.PI) / 180.0;
+  return {
+    x: centerX + radius * Math.cos(angleInRadians),
+    y: centerY + radius * Math.sin(angleInRadians),
+  };
+};
+
+const describeArc = (x: number, y: number, radius: number, startAngle: number, endAngle: number) => {
+  const start = polarToCartesian(x, y, radius, endAngle);
+  const end = polarToCartesian(x, y, radius, startAngle);
+  const largeArcFlag = endAngle - startAngle <= 180 ? "0" : "1";
+  return ["M", start.x, start.y, "A", radius, radius, 0, largeArcFlag, 0, end.x, end.y].join(" ");
+};
+
+export const GaugeMeter = ({
+  name,
+  service,
+  value,
+  min: customMin,
+  max: customMax,
+  avg,
+  unit = "",
+  size = "md",
+  thresholds,
+  subtitle,
+  history = [],
+  showTrend = true,
+  isSum = false,
+  aggLabel,
+  soloPod,
+  podCount,
+  podSummaries,
+}: GaugeMeterProps) => {
+  const servicePalette = service ? getServicePalette(service) : null;
+  // Determine min/max range
+  const { min, max } = useMemo(() => {
+    let computedMin = customMin !== undefined ? customMin : 0;
+    let computedMax = customMax !== undefined ? customMax : 100;
+
+    if (history.length > 0 && customMin === undefined && customMax === undefined) {
+      const histMin = Math.min(...history);
+      const histMax = Math.max(...history);
+      if (histMax <= 1 && histMin >= 0) {
+        // Percentage ratio 0..1
+        computedMin = 0;
+        computedMax = 1;
+      } else if (histMax <= 100 && histMin >= 0) {
+        // 0..100%
+        computedMin = 0;
+        computedMax = 100;
+      } else {
+        computedMin = Math.floor(Math.min(0, histMin));
+        computedMax = Math.ceil(histMax * 1.15) || 100;
+      }
+    }
+
+    if (computedMax <= computedMin) {
+      computedMax = computedMin + 1;
+    }
+
+    return { min: computedMin, max: computedMax };
+  }, [customMin, customMax, history]);
+
+  // Normalized percent 0..100
+  const percent = useMemo(() => {
+    const clamped = Math.max(min, Math.min(value, max));
+    return ((clamped - min) / (max - min)) * 100;
+  }, [value, min, max]);
+
+  // Color determination based on thresholds
+  const warnThreshold = thresholds?.warn ?? 70;
+  const dangerThreshold = thresholds?.danger ?? 90;
+
+  const statusColor = useMemo(() => {
+    if (percent >= dangerThreshold) {
+      return {
+        text: "text-rose-400",
+        stroke: "#f43f5e",
+        gradientStart: "#fb7185",
+        gradientEnd: "#e11d48",
+        bg: "bg-rose-500/10",
+        border: "border-rose-500/30",
+        glow: "rgba(244, 63, 94, 0.35)",
+        label: "Critical",
+      };
+    }
+    if (percent >= warnThreshold) {
+      return {
+        text: "text-amber-400",
+        stroke: "#f59e0b",
+        gradientStart: "#fbbf24",
+        gradientEnd: "#d97706",
+        bg: "bg-amber-500/10",
+        border: "border-amber-500/30",
+        glow: "rgba(245, 158, 11, 0.35)",
+        label: "Warning",
+      };
+    }
+    return {
+      text: "text-cyan-400",
+      stroke: "#06b6d4",
+      gradientStart: "#38bdf8",
+      gradientEnd: "#0284c7",
+      bg: "bg-cyan-500/10",
+      border: "border-cyan-500/30",
+      glow: "rgba(6, 182, 212, 0.35)",
+      label: "Nominal",
+    };
+  }, [percent, warnThreshold, dangerThreshold]);
+
+  // Trend calculation
+  const trend = useMemo(() => {
+    if (!showTrend || history.length < 2) return null;
+    const prev = history[history.length - 2];
+    const diff = value - prev;
+    if (Math.abs(diff) < 0.0001) return { direction: "flat", text: "steady", color: "text-slate-500" };
+    if (diff > 0) return { direction: "up", text: `+${formatNumber(diff)}`, color: "text-rose-400" };
+    return { direction: "down", text: `${formatNumber(diff)}`, color: "text-emerald-400" };
+  }, [value, history, showTrend]);
+
+  // Dimensions
+  const dim = size === "sm" ? 120 : size === "lg" ? 220 : 160;
+  const strokeWidth = size === "sm" ? 8 : size === "lg" ? 14 : 10;
+  const radius = (dim - strokeWidth * 2) / 2;
+  const centerX = dim / 2;
+  const centerY = dim / 2 + (size === "sm" ? 4 : 8);
+
+  // 240 degree arc from -120 to +120 degrees
+  const startAngle = -120;
+  const endAngle = 120;
+  const totalAngle = endAngle - startAngle;
+  const valueAngle = startAngle + (percent / 100) * totalAngle;
+
+  const bgArc = describeArc(centerX, centerY, radius, startAngle, endAngle);
+  const valueArc = describeArc(centerX, centerY, radius, startAngle, Math.min(valueAngle, endAngle));
+
+  // Ticks at 0, 50, 100
+  const tickAngles = [startAngle, startAngle + totalAngle * 0.5, endAngle];
+
+  const gaugeId = useMemo(
+    () => `${service ? service.replace(/[^a-zA-Z0-9]/g, "_") + "_" : ""}${name.replace(/[^a-zA-Z0-9]/g, "_")}`,
+    [service, name]
+  );
+
+  return (
+    <div className="flex flex-col items-center justify-between rounded-2xl border border-white/10 bg-slate-900/40 p-4 shadow-lg backdrop-blur-sm transition-all hover:border-white/20">
+      <div className="flex w-full items-start justify-between gap-2 border-b border-white/5 pb-2">
+        <div className="flex flex-col min-w-0 pr-1">
+          <div className="flex items-center gap-1 flex-wrap mb-1">
+            {service && (
+              <span
+                className={`inline-flex items-center gap-1 self-start rounded px-1.5 py-0.5 text-[9px] font-mono font-bold uppercase tracking-wider border ${servicePalette?.bg} ${servicePalette?.text} ${servicePalette?.border}`}
+                title={`Owning Service: ${service}`}
+              >
+                <span className="w-1.5 h-1.5 rounded-full bg-current opacity-80" />
+                <span className="truncate max-w-[120px]">{service}</span>
+              </span>
+            )}
+            {podCount !== undefined && podCount > 1 && (
+              <span
+                className="inline-flex items-center gap-1 rounded bg-sky-500/15 px-1.5 py-0.5 text-[9px] font-mono font-bold text-sky-300 border border-sky-500/30"
+                title={`Sum of values across ${podCount} pods`}
+              >
+                <span className="text-[11px] font-black leading-none text-sky-400">∑</span>
+                <span>{podCount} pods</span>
+              </span>
+            )}
+          </div>
+          <span className="truncate text-xs font-bold uppercase tracking-wider text-slate-300" title={name}>
+            {name}
+          </span>
+        </div>
+        <span className={`shrink-0 rounded-full px-2 py-0.5 text-[9px] font-black uppercase tracking-wider ${statusColor.bg} ${statusColor.text} ${statusColor.border} border`}>
+          {statusColor.label}
+        </span>
+      </div>
+
+      <div className="relative my-2 flex items-center justify-center">
+        <svg width={dim} height={dim * 0.85} viewBox={`0 0 ${dim} ${dim * 0.85}`} className="overflow-visible">
+          <defs>
+            <linearGradient id={`gauge-grad-${gaugeId}`} x1="0%" y1="0%" x2="100%" y2="0%">
+              <stop offset="0%" stopColor={statusColor.gradientStart} />
+              <stop offset="100%" stopColor={statusColor.gradientEnd} />
+            </linearGradient>
+            <filter id={`gauge-glow-${gaugeId}`} x="-20%" y="-20%" width="140%" height="140%">
+              <feDropShadow dx="0" dy="0" stdDeviation="3" floodColor={statusColor.glow} />
+            </filter>
+          </defs>
+
+          {/* Background Track */}
+          <path
+            d={bgArc}
+            fill="none"
+            stroke="rgba(255, 255, 255, 0.08)"
+            strokeWidth={strokeWidth}
+            strokeLinecap="round"
+          />
+
+          {/* Ticks */}
+          {tickAngles.map((angle, idx) => {
+            const inner = polarToCartesian(centerX, centerY, radius - strokeWidth * 0.8, angle);
+            const outer = polarToCartesian(centerX, centerY, radius + strokeWidth * 0.8, angle);
+            return (
+              <line
+                key={idx}
+                x1={inner.x}
+                y1={inner.y}
+                x2={outer.x}
+                y2={outer.y}
+                stroke="rgba(255, 255, 255, 0.2)"
+                strokeWidth={1.5}
+              />
+            );
+          })}
+
+          {/* Active Value Arc */}
+          {percent > 0 && (
+            <path
+              d={valueArc}
+              fill="none"
+              stroke={`url(#gauge-grad-${gaugeId})`}
+              strokeWidth={strokeWidth}
+              strokeLinecap="round"
+              filter={`url(#gauge-glow-${gaugeId})`}
+            />
+          )}
+
+          {/* Needle / Value dot indicator */}
+          {percent > 0 && (() => {
+            const tip = polarToCartesian(centerX, centerY, radius, valueAngle);
+            return (
+              <circle
+                cx={tip.x}
+                cy={tip.y}
+                r={strokeWidth * 0.6}
+                fill="#ffffff"
+                stroke={statusColor.stroke}
+                strokeWidth={2}
+                className="transition-all duration-300"
+              />
+            );
+          })()}
+        </svg>
+
+        {/* Center Readout */}
+        <div className="absolute inset-0 flex flex-col items-center justify-center pt-3 text-center pointer-events-none">
+          {soloPod ? (
+            <span className="text-[8px] font-mono uppercase font-black text-amber-400 tracking-wider truncate max-w-[100px]" title={`Solo: ${soloPod}`}>
+              {soloPod}
+            </span>
+          ) : aggLabel ? (
+            <span className="text-[8.5px] font-mono uppercase font-black text-sky-400 tracking-wider">
+              {aggLabel}
+            </span>
+          ) : podCount !== undefined && podCount > 1 ? (
+            <span className="text-[8.5px] font-mono uppercase font-black text-sky-400 tracking-wider">
+              {isSum ? "Sum (∑)" : "Avg (μ)"}
+            </span>
+          ) : null}
+          <div className="flex items-baseline gap-1">
+            <span className={`font-mono font-black tracking-tight ${size === "sm" ? "text-lg" : size === "lg" ? "text-3xl" : "text-2xl"} ${statusColor.text}`}>
+              {formatNumber(value)}
+            </span>
+            {unit && (
+              <span className="text-[10px] font-bold uppercase text-slate-400">
+                {unit}
+              </span>
+            )}
+          </div>
+          {podCount !== undefined && podCount > 1 ? (
+            <span
+              className="text-[9px] font-mono text-slate-400 truncate max-w-[100px]"
+              title={isSum ? `Average: ${formatNumber(value / podCount)} per pod` : `Fleet average across ${podCount} pods`}
+            >
+              {isSum ? `~${formatNumber(value / podCount)}/pod` : `${podCount} pods avg`}
+            </span>
+          ) : subtitle ? (
+            <span className="text-[10px] text-slate-500 truncate max-w-[90px]">
+              {subtitle}
+            </span>
+          ) : null}
+        </div>
+      </div>
+
+      {/* Footer range indicators & trend */}
+      <div className="flex w-full items-center justify-between border-t border-white/5 pt-2 text-[10px] text-slate-400">
+        <span className="font-mono">
+          <span className="text-slate-600">min</span> {formatNumber(min)}
+        </span>
+        {trend && (
+          <span className={`inline-flex items-center gap-0.5 font-mono text-[9px] font-bold ${trend.color}`}>
+            {trend.direction === "up" && <ArrowUpRight className="h-2.5 w-2.5" />}
+            {trend.direction === "down" && <ArrowDownRight className="h-2.5 w-2.5" />}
+            {trend.direction === "flat" && <Minus className="h-2.5 w-2.5" />}
+            {trend.text}
+          </span>
+        )}
+        {avg !== undefined && (
+          <span className="font-mono text-slate-400">
+            <span className="text-slate-600">avg</span> {formatNumber(avg)}
+          </span>
+        )}
+        <span className="font-mono">
+          <span className="text-slate-600">max</span> {formatNumber(max)}
+        </span>
+      </div>
+
+      {/* Pod Breakdown pills if available */}
+      {podSummaries && podSummaries.length > 1 && (
+        <div className="mt-2 flex w-full flex-wrap items-center gap-1 border-t border-white/5 pt-1.5">
+          {podSummaries.slice(0, 3).map((ps) => (
+            <div
+              key={ps.pod}
+              className="flex items-center gap-1 rounded bg-white/5 px-1.5 py-0.5 text-[8.5px] font-mono text-slate-400"
+              title={`${ps.pod}: ${formatNumber(ps.value)} ${unit || ""}`}
+            >
+              <span className="truncate max-w-[65px] text-slate-300">{ps.pod}</span>
+              <span className="font-bold text-sky-300">{formatNumber(ps.value)}</span>
+            </div>
+          ))}
+          {podSummaries.length > 3 && (
+            <span className="text-[8px] font-mono text-slate-500 self-center">
+              +{podSummaries.length - 3}
+            </span>
+          )}
+        </div>
+      )}
+    </div>
+  );
+};

otel-ui/src/components/IssuesVisualizer.tsx

@@ -0,0 +1,601 @@
+import { useMemo, useState } from "react"
+import {
+  Activity,
+  AlertOctagon,
+  Bug,
+  CheckCircle2,
+  ChevronDown,
+  ChevronUp,
+  Clock,
+  EyeOff,
+  ExternalLink,
+  Globe,
+  MonitorSmartphone,
+  RefreshCw,
+  RotateCcw,
+  ScrollText,
+  ShieldBan,
+  X,
+} from "lucide-react"
+import { useObservabilityData } from "../hooks/useObservabilityData"
+import { authFetch } from "../lib/auth"
+import { formatRelativeTime } from "../lib/timeUtils"
+import { getServicePalette } from "../lib/otel-utils"
+import {
+  issueStatusMeta,
+  issueStatusOf,
+  latestBreadcrumbs,
+  latestStacktrace,
+  sampleContext,
+  trendPoints,
+} from "../lib/issue-utils"
+import type { Issue, IssueSample } from "../lib/types"
+
+const kindMeta: Record<string, { label: string; Icon: typeof Activity }> = {
+  span_error: { label: "Trace Error", Icon: Activity },
+  log_error: { label: "Log Error", Icon: ScrollText },
+  frontend_error: { label: "Frontend Error", Icon: Globe },
+}
+
+type StatusFilter = "all" | "open" | "resolved" | "ignored"
+type KindFilter = "all" | "span_error" | "log_error" | "frontend_error"
+
+interface HoneypotIP {
+  ip: string
+  count: number
+  first_seen: string
+  last_seen: string
+  promoted_until: string
+}
+
+const statusFilters: { value: StatusFilter; label: string }[] = [
+  { value: "all", label: "All" },
+  { value: "open", label: "Open" },
+  { value: "resolved", label: "Resolved" },
+  { value: "ignored", label: "Ignored" },
+]
+
+const kindFilters: { value: KindFilter; label: string; Icon: typeof Activity }[] = [
+  { value: "all", label: "All kinds", Icon: Bug },
+  { value: "span_error", label: "Traces", Icon: Activity },
+  { value: "log_error", label: "Logs", Icon: ScrollText },
+  { value: "frontend_error", label: "Frontend", Icon: Globe },
+]
+
+// Lightweight inline-SVG sparkline of the issue's hourly occurrence trend.
+// Deliberately not recharts: issue lists can hold dozens of cards and each
+// recharts instance carries meaningful render cost.
+const TrendSparkline = ({ issue }: { issue: Issue }) => {
+  const points = trendPoints(issue, 24)
+  const max = Math.max(...points.map((p) => p.count), 1)
+  const width = 96
+  const height = 24
+  const step = width / (points.length - 1)
+  const path = points
+    .map((p, i) => `${i === 0 ? "M" : "L"}${(i * step).toFixed(1)},${(height - (p.count / max) * (height - 2) - 1).toFixed(1)}`)
+    .join(" ")
+  const gradientId = `sparkFill-${issue.id}`
+
+  return (
+    <span
+      className="inline-flex shrink-0 flex-col items-end gap-0.5"
+      title="Occurrences per hour (last 24h)"
+    >
+      <svg width={width} height={height} className="overflow-visible">
+        <path d={`${path} L${width},${height} L0,${height} Z`} fill={`url(#${gradientId})`} opacity="0.35" />
+        <path d={path} fill="none" stroke="currentColor" strokeWidth="1.5" strokeLinejoin="round" className="text-rose-400/80" />
+        <defs>
+          <linearGradient id={gradientId} x1="0" y1="0" x2="0" y2="1">
+            <stop offset="0%" stopColor="currentColor" className="text-rose-400" />
+            <stop offset="100%" stopColor="currentColor" stopOpacity="0" className="text-rose-400" />
+          </linearGradient>
+        </defs>
+      </svg>
+    </span>
+  )
+}
+
+
+const StatusBadge = ({ issue }: { issue: Issue }) => {
+  const status = issueStatusOf(issue)
+  const meta = issueStatusMeta[status]
+  return (
+    <span className={`shrink-0 rounded border px-1.5 py-0.5 text-[9px] font-bold uppercase tracking-wider ${meta.className}`}>
+      {meta.label}
+    </span>
+  )
+}
+
+const StacktraceView = ({ stacktrace }: { stacktrace: string }) => (
+  <pre className="max-h-64 overflow-auto rounded-xl border border-white/5 bg-black/60 p-3 font-mono text-[10px] leading-relaxed text-slate-300 whitespace-pre-wrap break-all">
+    {stacktrace}
+  </pre>
+)
+
+const BreadcrumbsView = ({ issue }: { issue: Issue }) => {
+  const crumbs = latestBreadcrumbs(issue)
+  if (!crumbs.length) return null
+  return (
+    <div className="space-y-1.5">
+      <p className="flex items-center gap-1.5 text-[10px] font-bold uppercase tracking-widest text-slate-500">
+        <Clock className="h-3 w-3" />
+        Breadcrumbs before error
+      </p>
+      <div className="max-h-48 space-y-1 overflow-auto rounded-xl border border-white/5 bg-black/40 p-3">
+        {crumbs.map((crumb, idx) => (
+          <div key={`${crumb.t}-${idx}`} className="flex items-baseline gap-2 font-mono text-[10px]">
+            <span className="shrink-0 text-slate-600">
+              {new Date(crumb.t).toLocaleTimeString([], { hour12: false })}
+            </span>
+            <span className="shrink-0 rounded bg-white/5 px-1.5 py-0.5 text-[9px] font-bold uppercase text-sky-400/80">
+              {crumb.category}
+            </span>
+            <span className="min-w-0 break-all text-slate-300">{crumb.message.slice(0, 200)}</span>
+          </div>
+        ))}
+      </div>
+    </div>
+  )
+}
+
+const SampleContextChips = ({ sample }: { sample: IssueSample }) => {
+  const ctx = sampleContext(sample)
+  if (!ctx.length) return null
+  return (
+    <span className="inline-flex shrink-0 items-center gap-1 rounded-full border border-amber-400/20 bg-amber-400/10 px-2 py-0.5 font-mono text-[10px] font-bold text-amber-300">
+      {ctx.join(" · ")}
+    </span>
+  )
+}
+
+const StatusActions = ({
+  issue,
+  onStatusChange,
+  busy,
+}: {
+  issue: Issue
+  onStatusChange: (issue: Issue, status: string) => void
+  busy: boolean
+}) => {
+  const status = issueStatusOf(issue)
+  const actions: { to: string; label: string; Icon: typeof CheckCircle2 }[] =
+    status === "open"
+      ? [
+          { to: "resolved", label: "Resolve", Icon: CheckCircle2 },
+          { to: "ignored", label: "Ignore", Icon: EyeOff },
+        ]
+      : [{ to: "open", label: "Reopen", Icon: RotateCcw }]
+
+  return (
+    <div className="flex items-center gap-2">
+      {actions.map(({ to, label, Icon }) => (
+        <button
+          key={to}
+          disabled={busy}
+          onClick={(e) => {
+            e.stopPropagation()
+            onStatusChange(issue, to)
+          }}
+          className="inline-flex items-center gap-1.5 rounded-lg border border-white/10 bg-white/5 px-2.5 py-1 text-[10px] font-bold uppercase tracking-wider text-slate-400 transition-all hover:border-white/20 hover:text-white disabled:opacity-50"
+        >
+          <Icon className="h-3 w-3" />
+          {label}
+        </button>
+      ))}
+    </div>
+  )
+}
+
+const IssueCard = ({
+  issue,
+  onStatusChange,
+  statusBusy,
+}: {
+  issue: Issue
+  onStatusChange: (issue: Issue, status: string) => void
+  statusBusy: boolean
+}) => {
+  const { setActiveTraceId, setActiveSpanId, setActiveTab } = useObservabilityData()
+  const [expanded, setExpanded] = useState(false)
+
+  const meta = kindMeta[issue.kind] ?? { label: issue.kind, Icon: Bug }
+  const isFrontend = issue.kind === "frontend_error"
+  const badgeGradient = isFrontend
+    ? "from-violet-500 via-sky-500 to-cyan-400"
+    : "from-rose-500 via-red-500 to-orange-400"
+  const palette = getServicePalette(issue.service)
+  const latestSample = issue.samples[issue.samples.length - 1]
+  const stacktrace = latestStacktrace(issue)
+  const hasMappedStacktrace = issue.samples.some((sample) => Boolean(sample.resolvedStacktrace))
+  const contextSample = issue.samples.find((s) => s.httpMethod || s.httpRoute || s.httpStatus)
+
+  const viewTrace = (traceId: string, spanId?: string) => {
+    setActiveTraceId(traceId)
+    if (spanId) setActiveSpanId(spanId)
+    setActiveTab("tab1")
+  }
+
+  return (
+    <div className="group relative flex flex-col overflow-hidden rounded-xl border border-rose-500/10 bg-slate-950/40 transition-all hover:border-rose-500/25 hover:bg-slate-900/50">
+      <div className={`absolute left-0 top-0 bottom-0 w-1 bg-gradient-to-b ${badgeGradient} opacity-50 transition-opacity group-hover:opacity-100`} />
+
+      <div
+        onClick={() => setExpanded((prev) => !prev)}
+        className="flex cursor-pointer flex-wrap items-center gap-x-4 gap-y-2 p-3 pl-4"
+      >
+        <span className={`inline-flex shrink-0 items-center gap-1.5 rounded px-1.5 py-0.5 text-[10px] font-bold uppercase tracking-wider bg-gradient-to-r ${badgeGradient} text-slate-950`}>
+          <meta.Icon className="h-3 w-3" />
+          {meta.label}
+        </span>
+
+        <span className={`shrink-0 rounded px-1.5 py-0.5 text-[10px] font-bold uppercase tracking-wider ${palette.bg} ${palette.text} ${palette.border} border`}>
+          {issue.service}
+        </span>
+
+        <span className="flex-1 min-w-[240px] break-all font-mono text-[12px] text-slate-200">
+          {issue.title}
+        </span>
+
+        <TrendSparkline issue={issue} />
+
+        <span className="shrink-0 rounded-full border border-rose-500/20 bg-rose-500/10 px-2 py-0.5 text-[10px] font-bold text-rose-300">
+          ×{issue.count}
+        </span>
+
+        <StatusBadge issue={issue} />
+
+        <span className="shrink-0 text-[10px] text-slate-500 whitespace-nowrap">
+          last seen {formatRelativeTime(issue.lastSeen / 1e6)}
+        </span>
+
+        {expanded ? <ChevronUp className="h-3 w-3 text-slate-500" /> : <ChevronDown className="h-3 w-3 text-slate-500" />}
+      </div>
+
+      {expanded && (
+        <div className="space-y-4 border-t border-white/5 bg-black/40 p-4 text-[12px]">
+          <div className="flex flex-wrap items-center justify-between gap-3">
+            <div className="flex flex-wrap items-center gap-4 text-[10px] text-slate-500">
+              <span>First seen {formatRelativeTime(issue.firstSeen / 1e6)}</span>
+              {issue.affectedServices.length > 1 && (
+                <span>Affects: {issue.affectedServices.join(", ")}</span>
+              )}
+              {issue.status === "resolved" && issue.resolvedAt ? (
+                <span>Resolved {formatRelativeTime(issue.resolvedAt / 1e6)}</span>
+              ) : null}
+            </div>
+            <StatusActions issue={issue} onStatusChange={onStatusChange} busy={statusBusy} />
+          </div>
+
+          {stacktrace && (
+            <div className="space-y-1.5">
+              <p className="text-[10px] font-bold uppercase tracking-widest text-slate-500">
+                {hasMappedStacktrace ? "Source-mapped stacktrace" : "Stacktrace"}
+              </p>
+              <StacktraceView stacktrace={stacktrace} />
+            </div>
+          )}
+
+          <BreadcrumbsView issue={issue} />
+
+          <div className="flex flex-wrap gap-4 text-[10px] text-slate-500">
+            {(contextSample?.userAgent || latestSample?.userAgent) && (
+              <span className="inline-flex items-center gap-1.5 max-w-[360px] truncate" title={contextSample?.userAgent || latestSample?.userAgent}>
+                <MonitorSmartphone className="h-3 w-3 shrink-0" />
+                {(contextSample?.userAgent || latestSample?.userAgent)?.slice(0, 80)}
+              </span>
+            )}
+            {(contextSample?.release || latestSample?.release) && (
+              <span>release: {contextSample?.release || latestSample?.release}</span>
+            )}
+            {(contextSample?.environment || latestSample?.environment) && (
+              <span>env: {contextSample?.environment || latestSample?.environment}</span>
+            )}
+          </div>
+
+          <div className="space-y-2">
+            <p className="text-[10px] font-bold uppercase tracking-widest text-slate-500">
+              Recent occurrences ({issue.samples.length})
+            </p>
+            <div className="space-y-1.5">
+              {[...issue.samples].reverse().map((sample, idx) => (
+                <div
+                  key={`${sample.timestamp}-${idx}`}
+                  className="flex flex-wrap items-center gap-3 rounded-lg border border-white/5 bg-white/5 px-2.5 py-1.5"
+                >
+                  <span className="shrink-0 text-slate-500 font-mono text-[10px]">
+                    {new Date(sample.timestamp / 1e6).toLocaleTimeString([], {
+                      hour12: false,
+                      hour: "2-digit",
+                      minute: "2-digit",
+                      second: "2-digit",
+                    })}
+                  </span>
+                  <SampleContextChips sample={sample} />
+                  <span className="flex-1 min-w-[160px] break-all text-slate-300">{sample.message}</span>
+                  {sample.url && (
+                    <a
+                      href={sample.url}
+                      target="_blank"
+                      rel="noreferrer"
+                      onClick={(e) => e.stopPropagation()}
+                      className="flex shrink-0 items-center gap-1 rounded-full bg-violet-500/10 border border-violet-400/20 px-2.5 py-0.5 text-[10px] font-bold text-violet-300 hover:bg-violet-500/20 hover:border-violet-400/30 transition-all max-w-[220px] truncate"
+                      title={sample.url}
+                    >
+                      <ExternalLink className="h-2.5 w-2.5 shrink-0" />
+                      <span className="truncate">{sample.url.replace(/^https?:\/\//, "")}</span>
+                    </a>
+                  )}
+                  {sample.traceId && (
+                    <button
+                      onClick={(e) => {
+                        e.stopPropagation()
+                        viewTrace(sample.traceId!, sample.spanId)
+                      }}
+                      className="group/trace flex shrink-0 items-center gap-1.5 rounded-full bg-sky-500/10 border border-sky-400/20 px-2.5 py-0.5 text-[10px] font-bold text-sky-400 hover:bg-sky-500/20 hover:border-sky-400/30 transition-all"
+                    >
+                      <Activity className="h-2.5 w-2.5 group-hover/trace:animate-pulse" />
+                      TRACE:{sample.traceId.slice(0, 6)}
+                    </button>
+                  )}
+                </div>
+              ))}
+            </div>
+          </div>
+
+          {latestSample?.traceId && (
+            <div className="mt-1 pt-4 border-t border-white/5 flex justify-end">
+              <button
+                onClick={() => viewTrace(latestSample.traceId!, latestSample.spanId)}
+                className="inline-flex items-center gap-2 rounded-xl bg-sky-500 px-4 py-2 text-[10px] font-black uppercase tracking-widest text-slate-950 transition-all hover:bg-sky-400 hover:scale-[1.02] active:scale-[0.98] shadow-lg shadow-sky-500/20"
+              >
+                <Activity className="h-3 w-3" />
+                View Latest Trace
+              </button>
+            </div>
+          )}
+        </div>
+      )}
+    </div>
+  )
+}
+
+interface IssuesVisualizerProps {
+  issues: Issue[]
+  loading: boolean
+  error: string | null
+  refresh: () => void
+  onStatusChange: (issue: Issue, status: string) => void
+  statusBusy: boolean
+}
+
+export const IssuesVisualizer = ({
+  issues,
+  loading,
+  error,
+  refresh,
+  onStatusChange,
+  statusBusy,
+}: IssuesVisualizerProps) => {
+  const { selectedServices, searchQuery } = useObservabilityData()
+  const [statusFilter, setStatusFilter] = useState<StatusFilter>("all")
+  const [kindFilter, setKindFilter] = useState<KindFilter>("all")
+  const [excludeSpamIPs, setExcludeSpamIPs] = useState(false)
+  const [showSpamIPs, setShowSpamIPs] = useState(false)
+  const [spamIPs, setSpamIPs] = useState<HoneypotIP[]>([])
+  const [spamIPsLoading, setSpamIPsLoading] = useState(false)
+  const [spamIPsError, setSpamIPsError] = useState<string | null>(null)
+
+  const loadSpamIPs = async () => {
+    setSpamIPsLoading(true)
+    setSpamIPsError(null)
+    try {
+      const response = await authFetch("/api/honeypot/ips")
+      if (!response.ok) throw new Error(`HTTP ${response.status}`)
+      setSpamIPs((await response.json()) as HoneypotIP[])
+    } catch (err) {
+      setSpamIPsError(err instanceof Error ? err.message : String(err))
+    } finally {
+      setSpamIPsLoading(false)
+    }
+  }
+
+  const spamIPSet = useMemo(() => new Set(spamIPs.map((entry) => entry.ip)), [spamIPs])
+
+  const statusCounts = useMemo(() => {
+    const counts = { all: issues.length, open: 0, resolved: 0, ignored: 0 }
+    issues.forEach((issue) => {
+      counts[issueStatusOf(issue)] += 1
+    })
+    return counts
+  }, [issues])
+
+  const filteredIssues = useMemo(() => {
+    return issues.filter((issue) => {
+      if (statusFilter !== "all" && issueStatusOf(issue) !== statusFilter) return false
+      if (kindFilter !== "all" && issue.kind !== kindFilter) return false
+      if (
+        excludeSpamIPs &&
+        issue.samples.some(
+          (sample) =>
+            (sample.httpStatus ?? 0) >= 400 &&
+            Boolean(sample.sourceIp) &&
+            spamIPSet.has(sample.sourceIp!),
+        )
+      ) {
+        return false
+      }
+      if (selectedServices.size > 0) {
+        const matches =
+          selectedServices.has(issue.service) ||
+          issue.affectedServices.some((s) => selectedServices.has(s))
+        if (!matches) return false
+      }
+      if (searchQuery) {
+        const q = searchQuery.toLowerCase()
+        const matches =
+          issue.title.toLowerCase().includes(q) || issue.service.toLowerCase().includes(q)
+        if (!matches) return false
+      }
+      return true
+    })
+  }, [issues, selectedServices, searchQuery, statusFilter, kindFilter, excludeSpamIPs, spamIPSet])
+
+  if (loading && issues.length === 0) {
+    return (
+      <div className="py-12 text-center text-sm text-slate-400 flex flex-col items-center justify-center">
+        <AlertOctagon className="h-10 w-10 mx-auto mb-4 opacity-20 animate-pulse" />
+        Loading issues...
+      </div>
+    )
+  }
+
+  return (
+    <div className="space-y-4">
+      <div className="sticky top-0 z-20 flex flex-wrap items-center justify-between gap-3 py-2 border-b border-white/5 mb-2 bg-slate-950/80 backdrop-blur">
+        <div className="flex flex-wrap items-center gap-3">
+          <span className="inline-flex items-center gap-2 text-[10px] font-black uppercase tracking-[0.2em] text-slate-500">
+            <AlertOctagon className="h-3 w-3 text-rose-400" />
+            Deduplicated Issues
+          </span>
+
+          {/* Status filter chips */}
+          <div className="flex items-center gap-1 rounded-lg border border-white/5 bg-white/5 p-0.5">
+            {statusFilters.map(({ value, label }) => (
+              <button
+                key={value}
+                onClick={() => setStatusFilter(value)}
+                className={`rounded px-2 py-0.5 text-[10px] font-bold uppercase tracking-wider transition-all ${
+                  statusFilter === value
+                    ? "bg-sky-500/20 text-sky-300 ring-1 ring-sky-500/30"
+                    : "text-slate-500 hover:text-slate-300"
+                }`}
+              >
+                {label}
+                <span className="ml-1 opacity-60">{statusCounts[value]}</span>
+              </button>
+            ))}
+          </div>
+
+          <button
+            onClick={() => setExcludeSpamIPs((value) => !value)}
+            className={`inline-flex items-center gap-1.5 rounded-lg border px-2.5 py-1 text-[10px] font-bold uppercase tracking-wider transition-all ${
+              excludeSpamIPs
+                ? "border-amber-400/40 bg-amber-400/15 text-amber-200"
+                : "border-white/10 text-slate-500 hover:border-white/20 hover:text-slate-300"
+            }`}
+            title="Hide HTTP errors whose recorded source IP is on the honeypot list"
+          >
+            <ShieldBan className="h-3 w-3" />
+            {excludeSpamIPs ? "Spam IPs hidden" : "Hide spam IPs"}
+          </button>
+
+          {/* Kind filter chips */}
+          <div className="flex items-center gap-1 rounded-lg border border-white/5 bg-white/5 p-0.5">
+            {kindFilters.map(({ value, label, Icon }) => (
+              <button
+                key={value}
+                onClick={() => setKindFilter(value)}
+                className={`inline-flex items-center gap-1 rounded px-2 py-0.5 text-[10px] font-bold uppercase tracking-wider transition-all ${
+                  kindFilter === value
+                    ? "bg-violet-500/20 text-violet-300 ring-1 ring-violet-500/30"
+                    : "text-slate-500 hover:text-slate-300"
+                }`}
+              >
+                <Icon className="h-3 w-3" />
+                {label}
+              </button>
+            ))}
+          </div>
+
+          <span className="text-[10px] text-slate-500">
+            {filteredIssues.length} shown
+          </span>
+        </div>
+
+        <div className="flex items-center gap-3">
+          {error && <span className="text-[10px] text-rose-400">Failed to load: {error}</span>}
+          <button
+            onClick={() => {
+              const next = !showSpamIPs
+              setShowSpamIPs(next)
+              if (next && spamIPs.length === 0) void loadSpamIPs()
+            }}
+            className="flex items-center gap-1.5 rounded-lg border border-white/10 px-2.5 py-1 text-[10px] font-bold uppercase tracking-wider text-slate-400 hover:text-white hover:border-white/20 transition-all"
+          >
+            <ShieldBan className="h-3 w-3" />
+            Spam IPs ({spamIPs.length})
+          </button>
+          <button
+            onClick={() => refresh()}
+            className="flex items-center gap-1.5 rounded-lg border border-white/10 px-2.5 py-1 text-[10px] font-bold uppercase tracking-wider text-slate-400 hover:text-white hover:border-white/20 transition-all"
+          >
+            <RefreshCw className={`h-3 w-3 ${loading ? "animate-spin" : ""}`} />
+            Refresh
+          </button>
+        </div>
+      </div>
+
+      {showSpamIPs && (
+        <div className="rounded-xl border border-amber-400/20 bg-amber-400/5 p-3 text-xs text-slate-300">
+          <div className="flex items-center justify-between gap-3 mb-2">
+            <div>
+              <div className="font-bold text-amber-200">Promoted honeypot IPs</div>
+              <div className="text-[10px] text-slate-500">Used by the ingress tarpit synchronizer.</div>
+            </div>
+            <button
+              onClick={() => setShowSpamIPs(false)}
+              className="rounded p-1 text-slate-500 hover:text-white"
+              aria-label="Close spam IP list"
+            >
+              <X className="h-4 w-4" />
+            </button>
+          </div>
+          {spamIPsLoading && <div className="text-slate-500">Loading…</div>}
+          {spamIPsError && <div className="text-rose-300">Failed to load: {spamIPsError}</div>}
+          {!spamIPsLoading && !spamIPsError && spamIPs.length === 0 && (
+            <div className="text-slate-500">No promoted IPs.</div>
+          )}
+          {spamIPs.length > 0 && (
+            <div className="grid gap-1 sm:grid-cols-2 lg:grid-cols-3">
+              {spamIPs.map((entry) => (
+                <div key={entry.ip} className="flex items-center justify-between rounded-lg bg-black/20 px-2.5 py-1.5 font-mono">
+                  <span className="text-slate-200">{entry.ip}</span>
+                  <span className="text-[10px] text-amber-300">×{entry.count}</span>
+                </div>
+              ))}
+            </div>
+          )}
+        </div>
+      )}
+
+      {filteredIssues.length === 0 ? (
+        <div className="flex flex-col items-center justify-center py-20 text-slate-500">
+          <div className="h-12 w-12 rounded-full bg-emerald-500/10 flex items-center justify-center mb-4">
+            <AlertOctagon className="h-6 w-6 text-emerald-400/60" />
+          </div>
+          <p className="text-sm font-medium text-slate-300">
+            {issues.length === 0 ? "No issues detected" : "No issues match your filters"}
+          </p>
+          <p className="text-xs mt-1 text-slate-600">
+            {issues.length === 0
+              ? "Error-status spans and error/fatal-severity logs will show up here as they arrive."
+              : "Try adjusting your filters or search query."}
+          </p>
+        </div>
+      ) : (
+        <div className="space-y-2 animate-in fade-in slide-in-from-bottom-2 duration-500">
+          {filteredIssues.map((issue) => (
+            <IssueCard
+              key={issue.id}
+              issue={issue}
+              onStatusChange={onStatusChange}
+              statusBusy={statusBusy}
+            />
+          ))}
+        </div>
+      )}
+    </div>
+  )
+}
+
+export default IssuesVisualizer

otel-ui/src/components/PatternsVisualizer.tsx

@@ -0,0 +1,76 @@
+import { useMemo, useState } from "react"
+import { GitBranch, RefreshCw, ThumbsDown, ThumbsUp } from "lucide-react"
+import { usePatterns } from "../hooks/usePatterns"
+import { formatRelativeTime } from "../lib/timeUtils"
+import type { Issue, SimilarIssue } from "../lib/types"
+
+const PatternIssues = ({ pattern, issues, similar, review }: {
+  pattern: { issueIds: string[] }
+  issues: Issue[]
+  similar: (id: string) => Promise<SimilarIssue[]>
+  review: (id: string, otherId: string, decision: "same" | "different") => Promise<void>
+}) => {
+  const [expanded, setExpanded] = useState(false)
+  const [related, setRelated] = useState<Record<string, SimilarIssue[]>>({})
+  const [busy, setBusy] = useState<string | null>(null)
+  const members = pattern.issueIds.map((id) => issues.find((issue) => issue.id === id)).filter(Boolean) as Issue[]
+
+  const toggle = async (issue: Issue) => {
+    if (expanded) { setExpanded(false); return }
+    setExpanded(true)
+    if (!related[issue.id]) {
+      const suggestions = await similar(issue.id)
+      setRelated((prev) => ({ ...prev, [issue.id]: suggestions }))
+    }
+  }
+
+  const submitReview = async (issue: Issue, item: SimilarIssue, decision: "same" | "different") => {
+    const key = `${issue.id}:${item.issue.id}`
+    setBusy(key)
+    try {
+      await review(issue.id, item.issue.id, decision)
+      setRelated((prev) => ({ ...prev, [issue.id]: (prev[issue.id] ?? []).map((entry) => entry.issue.id === item.issue.id ? { ...entry, decision } : entry) }))
+    } finally { setBusy(null) }
+  }
+
+  return (
+    <div className="mt-3 space-y-2 border-t border-white/5 pt-3">
+      <div className="flex flex-wrap gap-2">
+        {members.map((issue) => <button key={issue.id} onClick={() => void toggle(issue)} className="rounded-lg border border-white/10 bg-white/5 px-2 py-1 text-left text-[10px] text-slate-300 hover:border-sky-400/40">{issue.title} <span className="text-slate-500">×{issue.count}</span></button>)}
+      </div>
+      {expanded && members[0] && <div className="space-y-2 rounded-lg bg-black/20 p-3">
+        <div className="text-[10px] uppercase tracking-widest text-slate-500">Deterministic suggestions for {members[0].title}</div>
+        {(related[members[0].id] ?? []).map((item) => {
+          const key = `${members[0].id}:${item.issue.id}`
+          return <div key={item.issue.id} className="flex flex-wrap items-center gap-2 text-[11px]">
+            <span className="text-slate-300">{item.issue.title}</span><span className="text-sky-300">{Math.round(item.score * 100)}%</span><span className="text-slate-500">{item.reasons.join(" · ")}</span>
+            <button disabled={busy === key} onClick={() => void submitReview(members[0], item, "same")} className="ml-auto inline-flex items-center gap-1 rounded border border-emerald-400/20 px-2 py-1 text-emerald-300 hover:bg-emerald-400/10"><ThumbsUp className="h-3 w-3" /> same</button>
+            <button disabled={busy === key} onClick={() => void submitReview(members[0], item, "different")} className="inline-flex items-center gap-1 rounded border border-rose-400/20 px-2 py-1 text-rose-300 hover:bg-rose-400/10"><ThumbsDown className="h-3 w-3" /> separate</button>
+            {item.decision && <span className="text-slate-500">reviewed: {item.decision}</span>}
+          </div>
+        })}
+      </div>}
+    </div>
+  )
+}
+
+export function PatternsVisualizer() {
+  const { patterns, issues, loading, error, refresh, similar, review } = usePatterns()
+  const [query, setQuery] = useState("")
+  const visible = useMemo(() => patterns.filter((pattern) => `${pattern.service} ${pattern.kind} ${pattern.signature}`.toLowerCase().includes(query.toLowerCase())), [patterns, query])
+
+  if (loading && patterns.length === 0) return <div className="py-16 text-center text-sm text-slate-500">Loading deterministic patterns…</div>
+  return <div className="space-y-4">
+    <div className="flex flex-wrap items-center gap-3 border-b border-white/5 pb-3">
+      <div><div className="flex items-center gap-2 text-[10px] font-black uppercase tracking-[0.2em] text-slate-500"><GitBranch className="h-3 w-3 text-violet-400" /> Similarity Patterns</div><div className="mt-1 text-xs text-slate-500">Exact issues stay separate; broader deterministic patterns are reviewable.</div></div>
+      <input value={query} onChange={(event) => setQuery(event.target.value)} placeholder="Filter patterns…" className="ml-auto rounded-lg border border-white/10 bg-white/5 px-3 py-2 text-xs text-slate-200 outline-none placeholder:text-slate-600 focus:border-sky-400/40" />
+      <button onClick={() => void refresh()} className="inline-flex items-center gap-2 rounded-lg border border-white/10 px-3 py-2 text-[10px] font-bold uppercase tracking-wider text-slate-400 hover:text-white"><RefreshCw className="h-3 w-3" /> Refresh</button>
+    </div>
+    {error && <div className="rounded-lg border border-rose-400/20 bg-rose-400/5 p-3 text-xs text-rose-300">Failed to load patterns: {String(error)}</div>}
+    {visible.length === 0 ? <div className="py-16 text-center text-sm text-slate-500">No deterministic patterns observed yet.</div> : <div className="space-y-3">{visible.map((pattern) => <div key={pattern.id} className="rounded-xl border border-violet-400/15 bg-slate-950/40 p-4">
+      <div className="flex flex-wrap items-center gap-3"><span className="rounded bg-violet-500/15 px-2 py-1 text-[10px] font-bold uppercase text-violet-300">{pattern.service}</span><span className="text-xs text-slate-400">{pattern.kind}</span><span className="text-xs text-slate-300">{pattern.issueCount} exact issues · {pattern.occurrenceCount} occurrences</span><span className="ml-auto text-[10px] text-slate-600">last seen {formatRelativeTime(pattern.lastSeen / 1e6)}</span></div>
+      <div className="mt-3 break-all font-mono text-[10px] text-slate-500">{pattern.signature.split("\u0000").join(" · ")}</div>
+      <PatternIssues pattern={pattern} issues={issues} similar={similar} review={review} />
+    </div>)}</div>}
+  </div>
+}

otel-ui/src/components/RolloutsVisualizer.tsx

@@ -0,0 +1,48 @@
+import { useState } from "react"
+import { Activity, CheckCircle2, CloudCog, RefreshCw, XCircle } from "lucide-react"
+import { useRollouts } from "../hooks/useRollouts"
+import { formatRelativeTime } from "../lib/timeUtils"
+import type { RolloutImpact } from "../lib/types"
+
+const phaseStyle: Record<string, { className: string; Icon: typeof Activity }> = {
+  completed: { className: "text-emerald-300 bg-emerald-400/10 border-emerald-400/20", Icon: CheckCircle2 },
+  failed: { className: "text-rose-300 bg-rose-400/10 border-rose-400/20", Icon: XCircle },
+  progressing: { className: "text-amber-300 bg-amber-400/10 border-amber-400/20", Icon: Activity },
+}
+
+export function RolloutsVisualizer() {
+  const { rollouts, loading, error, refresh, impact } = useRollouts()
+  const [selected, setSelected] = useState<string | null>(null)
+  const [impactData, setImpactData] = useState<Record<string, RolloutImpact>>({})
+  const [impactError, setImpactError] = useState<string | null>(null)
+
+  const select = async (id: string) => {
+    setSelected(selected === id ? null : id)
+    if (impactData[id]) return
+    try {
+      setImpactError(null)
+      const result = await impact(id)
+      setImpactData((previous) => ({ ...previous, [id]: result }))
+    }
+    catch (cause) { setImpactError(cause instanceof Error ? cause.message : String(cause)) }
+  }
+
+  if (loading && rollouts.length === 0) return <div className="py-16 text-center text-sm text-slate-500">Loading cluster rollouts…</div>
+  return <div className="space-y-4">
+    <div className="flex flex-wrap items-center gap-3 border-b border-white/5 pb-3">
+      <div><div className="flex items-center gap-2 text-[10px] font-black uppercase tracking-[0.2em] text-slate-500"><CloudCog className="h-3 w-3 text-sky-400" /> Cluster Rollouts</div><div className="mt-1 text-xs text-slate-500">Read-only Kubernetes deployment state with optional Prometheus impact data.</div></div>
+      <button onClick={() => void refresh()} className="ml-auto inline-flex items-center gap-2 rounded-lg border border-white/10 px-3 py-2 text-[10px] font-bold uppercase tracking-wider text-slate-400 hover:text-white"><RefreshCw className="h-3 w-3" /> Refresh</button>
+    </div>
+    {error && <div className="rounded-lg border border-amber-400/20 bg-amber-400/5 p-3 text-xs text-amber-200">Rollout watcher unavailable: {String(error)}</div>}
+    {impactError && <div className="rounded-lg border border-rose-400/20 bg-rose-400/5 p-3 text-xs text-rose-300">Impact query failed: {impactError}</div>}
+    {rollouts.length === 0 ? <div className="py-16 text-center text-sm text-slate-500">No rollout data. Enable the Kubernetes watcher and grant its read-only RBAC permissions.</div> : <div className="space-y-2">{rollouts.map((rollout) => {
+      const status = phaseStyle[rollout.phase] ?? phaseStyle.progressing
+      const Icon = status.Icon
+      const data = impactData[rollout.id]
+      return <div key={rollout.id} className="rounded-xl border border-white/10 bg-slate-950/40 p-4">
+        <button onClick={() => void select(rollout.id)} className="w-full text-left"><div className="flex flex-wrap items-center gap-3"><span className={`inline-flex items-center gap-1 rounded border px-2 py-1 text-[10px] font-bold uppercase ${status.className}`}><Icon className="h-3 w-3" /> {rollout.phase}</span><span className="font-mono text-xs text-slate-200">{rollout.namespace}/{rollout.workload}</span><span className="text-xs text-slate-400">revision {rollout.revision || "unknown"}</span><span className="ml-auto text-[10px] text-slate-500">{formatRelativeTime(rollout.lastSeen / 1e6)}</span></div><div className="mt-3 grid gap-2 text-[11px] text-slate-400 sm:grid-cols-2 lg:grid-cols-4"><span>release: <b className="text-slate-200">{rollout.release || "unlabeled"}</b></span><span>image: <b className="font-mono text-slate-200">{rollout.image || "unknown"}</b></span><span>replicas: <b className="text-slate-200">{rollout.availableReplicas}/{rollout.desiredReplicas} ready</b></span><span>reason: <b className="text-slate-200">{rollout.reason || "unknown"}</b></span></div></button>
+        {selected === rollout.id && <div className="mt-4 border-t border-white/5 pt-3 text-xs text-slate-400"><div>Started {formatRelativeTime(rollout.startedAt / 1e6)}{rollout.completedAt ? ` · completed ${formatRelativeTime(rollout.completedAt / 1e6)}` : ""}</div><div className="mt-2 text-slate-500">{rollout.message || "No rollout message reported."}</div>{data && <div className="mt-3 rounded-lg bg-black/20 p-3"><div className="flex items-center gap-2 text-[10px] font-bold uppercase tracking-widest text-slate-500"><Activity className="h-3 w-3" /> Prometheus error-rate points</div>{!data.configured ? <div className="mt-2 text-slate-600">Prometheus is not configured.</div> : data.points.length === 0 ? <div className="mt-2 text-slate-600">No matching Prometheus series.</div> : <div className="mt-2 flex flex-wrap gap-2">{data.points.slice(-12).map((point) => <span key={`${point.timestamp}-${point.value}`} className="rounded bg-sky-400/10 px-2 py-1 font-mono text-[10px] text-sky-300">{new Date(point.timestamp * 1000).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" })}: {point.value}</span>)}</div>}</div>}</div>}
+      </div>
+    })}</div>}
+  </div>
+}

otel-ui/src/components/SearchAndWatcher.tsx

@@ -0,0 +1,504 @@
+import React, { useState, useRef } from "react";
+import {
+  Search,
+  Radar,
+  Play,
+  Square,
+  Copy,
+  Check,
+  ExternalLink,
+} from "lucide-react";
+import { authFetch } from "../lib/auth";
+import { useObservabilityData } from "../hooks/useObservabilityData";
+
+interface SearchResultEntry {
+
+  type: "traces" | "metrics" | "logs" | string;
+  timestamp: number;
+  data: any;
+}
+
+export const SearchAndWatcher: React.FC = () => {
+  const { setActiveTraceId, setActiveTab } = useObservabilityData();
+
+  // Search state
+  const [searchQuery, setSearchQuery] = useState("");
+  const [searchType, setSearchType] = useState<string>("");
+  const [searchLimit, setSearchLimit] = useState<number>(100);
+  const [searchTimeRange, setSearchTimeRange] = useState<string>("24h");
+  const [searchResults, setSearchResults] = useState<SearchResultEntry[] | null>(null);
+  const [searchLoading, setSearchLoading] = useState(false);
+  const [searchError, setSearchError] = useState<string | null>(null);
+
+  // Watcher state
+  const [watchQuery, setWatchQuery] = useState("");
+  const [watchType, setWatchType] = useState<string>("");
+  const [watchTimeout, setWatchTimeout] = useState<number>(30);
+  const [isWatching, setIsWatching] = useState(false);
+  const [caughtSignal, setCaughtSignal] = useState<SearchResultEntry | null>(null);
+  const [watchStatus, setWatchStatus] = useState<string | null>(null);
+  const abortControllerRef = useRef<AbortController | null>(null);
+
+  // UI helpers
+  const [copiedId, setCopiedId] = useState<string | null>(null);
+  const [expandedIndex, setExpandedIndex] = useState<number | null>(null);
+
+  const copyToClipboard = (text: string, id: string) => {
+    navigator.clipboard.writeText(text);
+    setCopiedId(id);
+    setTimeout(() => setCopiedId(null), 1500);
+  };
+
+  const handleRunSearch = async (e?: React.FormEvent) => {
+    if (e) e.preventDefault();
+    setSearchLoading(true);
+    setSearchError(null);
+
+    try {
+      const params = new URLSearchParams();
+      if (searchQuery.trim()) params.set("q", searchQuery.trim());
+      if (searchType) params.set("type", searchType);
+      params.set("limit", String(searchLimit));
+
+      if (searchTimeRange !== "all") {
+        const msMap: Record<string, number> = {
+          "1h": 60 * 60 * 1000,
+          "6h": 6 * 60 * 60 * 1000,
+          "24h": 24 * 60 * 60 * 1000,
+          "7d": 7 * 24 * 60 * 60 * 1000,
+        };
+        const ms = msMap[searchTimeRange] || 24 * 60 * 60 * 1000;
+        const sinceNano = BigInt(Date.now() - ms) * 1_000_000n;
+        params.set("since", sinceNano.toString());
+      }
+
+      const res = await authFetch(`/api/history/search?${params.toString()}`);
+      if (!res.ok) throw new Error(`Search failed: HTTP ${res.status}`);
+      const data = await res.json();
+      setSearchResults(Array.isArray(data) ? data : []);
+    } catch (err) {
+      setSearchError(err instanceof Error ? err.message : String(err));
+    } finally {
+      setSearchLoading(false);
+    }
+  };
+
+  const startWatcher = async () => {
+    if (isWatching) return;
+    setIsWatching(true);
+    setCaughtSignal(null);
+    setWatchStatus("Trap armed. Waiting for telemetry match...");
+
+    const controller = new AbortController();
+    abortControllerRef.current = controller;
+
+    try {
+      const params = new URLSearchParams();
+      if (watchQuery.trim()) params.set("q", watchQuery.trim());
+      if (watchType) params.set("type", watchType);
+      params.set("timeout", String(watchTimeout));
+
+      const res = await authFetch(`/api/history/wait?${params.toString()}`, {
+        signal: controller.signal,
+      });
+
+      if (!res.ok) {
+        if (res.status === 504) {
+          setWatchStatus(`Timeout reached (${watchTimeout}s) without match.`);
+        } else {
+          setWatchStatus(`Error: HTTP ${res.status}`);
+        }
+        return;
+      }
+
+      const signal = await res.json();
+      setCaughtSignal(signal);
+      setWatchStatus("Target signal trapped successfully!");
+    } catch (err: any) {
+      if (err.name === "AbortError") {
+        setWatchStatus("Trap disarmed by user.");
+      } else {
+        setWatchStatus(`Watcher error: ${err.message || String(err)}`);
+      }
+    } finally {
+      setIsWatching(false);
+      abortControllerRef.current = null;
+    }
+  };
+
+  const cancelWatcher = () => {
+    if (abortControllerRef.current) {
+      abortControllerRef.current.abort();
+    }
+    setIsWatching(false);
+  };
+
+  // Helper to extract trace ID from an entry
+  const extractTraceId = (entry: SearchResultEntry): string | null => {
+    const raw = JSON.stringify(entry.data);
+    const match = raw.match(/"traceId":"([a-f0-9]{16,32})"/i);
+    return match ? match[1] : null;
+  };
+
+  return (
+    <div className="space-y-8 max-w-6xl font-body">
+      {/* Top Banner */}
+      <div className="rounded-2xl border border-sky-500/20 bg-gradient-to-r from-sky-950/40 via-slate-900/60 to-slate-950/80 p-5 shadow-xl backdrop-blur-md">
+        <div className="flex flex-col sm:flex-row sm:items-center justify-between gap-4">
+          <div className="space-y-1">
+            <div className="flex items-center gap-2">
+              <div className="flex h-7 w-7 items-center justify-center rounded-lg bg-sky-500/20 text-sky-400">
+                <Radar className="h-4 w-4 animate-pulse" />
+              </div>
+              <h2 className="text-base font-bold text-white tracking-tight">
+                Archive Search &amp; Live Signal Trap
+              </h2>
+            </div>
+            <p className="text-xs text-slate-400 max-w-2xl leading-relaxed">
+              Expose deep telemetry data beyond the browser buffer. Query full ClickHouse &amp; persistent storage logs/traces, or arm an OTLP signal trap to catch anomalies as they hit the server.
+            </p>
+          </div>
+        </div>
+      </div>
+
+      {/* SECTION 1: Live Signal Trap */}
+      <section className="rounded-2xl border border-white/10 bg-slate-950/40 p-5 shadow-xl space-y-4">
+        <div className="flex items-center justify-between border-b border-white/5 pb-3">
+          <div className="flex items-center gap-2">
+            <Radar className="h-4 w-4 text-violet-400" />
+            <h3 className="text-xs font-black uppercase tracking-[0.2em] text-slate-400">
+              Live Signal Trap (/api/history/wait)
+            </h3>
+          </div>
+          <span className="text-[11px] font-mono text-slate-500">
+            Long-poll subscriber for active debugging
+          </span>
+        </div>
+
+        <div className="grid grid-cols-1 md:grid-cols-12 gap-3 items-center">
+          <div className="md:col-span-6">
+            <label className="block font-mono text-[10px] text-slate-400 uppercase tracking-wider mb-1">
+              Target Keyword / Pattern / Trace ID
+            </label>
+            <div className="relative flex items-center">
+              <input
+                type="text"
+                value={watchQuery}
+                onChange={(e) => setWatchQuery(e.target.value)}
+                placeholder="e.g. 'error', '500', 'db_timeout', or service name..."
+                disabled={isWatching}
+                className="w-full h-9 rounded-xl border border-white/10 bg-black/40 px-3 text-xs text-white placeholder-slate-500 focus:border-violet-500/50 focus:outline-none disabled:opacity-50 font-mono"
+              />
+            </div>
+          </div>
+
+          <div className="md:col-span-3">
+            <label className="block font-mono text-[10px] text-slate-400 uppercase tracking-wider mb-1">
+              Signal Type
+            </label>
+            <select
+              value={watchType}
+              onChange={(e) => setWatchType(e.target.value)}
+              disabled={isWatching}
+              className="w-full h-9 rounded-xl border border-white/10 bg-black/40 px-3 text-xs text-white focus:border-violet-500/50 focus:outline-none disabled:opacity-50 font-mono"
+            >
+              <option value="">Any Telemetry</option>
+              <option value="traces">Traces Only</option>
+              <option value="logs">Logs Only</option>
+              <option value="metrics">Metrics Only</option>
+            </select>
+          </div>
+
+          <div className="md:col-span-3 flex items-end gap-2">
+            <div className="flex-1">
+              <label className="block font-mono text-[10px] text-slate-400 uppercase tracking-wider mb-1">
+                Timeout
+              </label>
+              <select
+                value={watchTimeout}
+                onChange={(e) => setWatchTimeout(Number(e.target.value))}
+                disabled={isWatching}
+                className="w-full h-9 rounded-xl border border-white/10 bg-black/40 px-3 text-xs text-white focus:border-violet-500/50 focus:outline-none disabled:opacity-50 font-mono"
+              >
+                <option value={15}>15 seconds</option>
+                <option value={30}>30 seconds</option>
+                <option value={60}>60 seconds</option>
+              </select>
+            </div>
+
+            {isWatching ? (
+              <button
+                type="button"
+                onClick={cancelWatcher}
+                className="h-9 px-4 rounded-xl bg-rose-500 hover:bg-rose-600 text-white font-mono text-xs font-bold transition-colors flex items-center gap-1.5 shadow-lg shadow-rose-500/20"
+              >
+                <Square className="h-3.5 w-3.5 fill-current" />
+                <span>Cancel</span>
+              </button>
+            ) : (
+              <button
+                type="button"
+                onClick={startWatcher}
+                className="h-9 px-4 rounded-xl bg-violet-500 hover:bg-violet-600 text-white font-mono text-xs font-bold transition-colors flex items-center gap-1.5 shadow-lg shadow-violet-500/20"
+              >
+                <Play className="h-3.5 w-3.5 fill-current" />
+                <span>Arm Trap</span>
+              </button>
+            )}
+          </div>
+        </div>
+
+        {/* Watcher Status or Live Trigger Output */}
+        {watchStatus && (
+          <div
+            className={`rounded-xl p-3 text-xs font-mono flex items-center justify-between border ${
+              isWatching
+                ? "bg-violet-500/10 border-violet-500/30 text-violet-300 animate-pulse"
+                : caughtSignal
+                ? "bg-emerald-500/10 border-emerald-500/30 text-emerald-300"
+                : "bg-slate-900/60 border-white/5 text-slate-400"
+            }`}
+          >
+            <div className="flex items-center gap-2">
+              <span
+                className={`h-2 w-2 rounded-full ${
+                  isWatching ? "bg-violet-400 animate-ping" : caughtSignal ? "bg-emerald-400" : "bg-slate-500"
+                }`}
+              />
+              <span>{watchStatus}</span>
+            </div>
+          </div>
+        )}
+
+        {/* Captured Signal Preview */}
+        {caughtSignal && (
+          <div className="rounded-xl border border-emerald-500/20 bg-black/40 p-4 space-y-2 animate-in fade-in duration-300">
+            <div className="flex items-center justify-between">
+              <div className="flex items-center gap-2">
+                <span className="rounded bg-emerald-500/20 text-emerald-300 border border-emerald-500/40 px-2 py-0.5 text-[10px] font-bold uppercase">
+                  {caughtSignal.type}
+                </span>
+                <span className="font-mono text-xs text-slate-300">
+                  Captured at {new Date(caughtSignal.timestamp / 1_000_000).toLocaleTimeString()}
+                </span>
+              </div>
+
+              {extractTraceId(caughtSignal) && (
+                <button
+                  onClick={() => {
+                    const traceId = extractTraceId(caughtSignal);
+                    if (traceId) {
+                      setActiveTraceId(traceId);
+                      setActiveTab("tab1");
+                    }
+                  }}
+                  className="inline-flex items-center gap-1.5 rounded-lg bg-sky-500 px-3 py-1 text-[11px] font-bold text-slate-950 hover:bg-sky-400 transition-colors"
+                >
+                  <ExternalLink className="h-3 w-3" />
+                  Jump to Captured Trace
+                </button>
+              )}
+            </div>
+
+            <pre className="max-h-60 overflow-auto rounded-lg border border-white/5 bg-slate-950 p-3 font-mono text-[11px] text-slate-300">
+              <code>{JSON.stringify(caughtSignal.data, null, 2)}</code>
+            </pre>
+          </div>
+        )}
+      </section>
+
+      {/* SECTION 2: Persistent Storage Search */}
+      <section className="rounded-2xl border border-white/10 bg-slate-950/40 p-5 shadow-xl space-y-4">
+        <div className="flex items-center justify-between border-b border-white/5 pb-3">
+          <div className="flex items-center gap-2">
+            <Search className="h-4 w-4 text-sky-400" />
+            <h3 className="text-xs font-black uppercase tracking-[0.2em] text-slate-400">
+              Query Persistent Telemetry Store (/api/history/search)
+            </h3>
+          </div>
+          <span className="text-[11px] font-mono text-slate-500">
+            Searches full database history beyond active memory
+          </span>
+        </div>
+
+        <form onSubmit={handleRunSearch} className="space-y-3">
+          <div className="grid grid-cols-1 md:grid-cols-12 gap-3">
+            <div className="md:col-span-4">
+              <label className="block font-mono text-[10px] text-slate-400 uppercase tracking-wider mb-1">
+                Search Query
+              </label>
+              <input
+                type="text"
+                value={searchQuery}
+                onChange={(e) => setSearchQuery(e.target.value)}
+                placeholder="Free-text substring in telemetry JSON..."
+                className="w-full h-9 rounded-xl border border-white/10 bg-black/40 px-3 text-xs text-white placeholder-slate-500 focus:border-sky-500/50 focus:outline-none font-mono"
+              />
+            </div>
+
+            <div className="md:col-span-2">
+              <label className="block font-mono text-[10px] text-slate-400 uppercase tracking-wider mb-1">
+                Type
+              </label>
+              <select
+                value={searchType}
+                onChange={(e) => setSearchType(e.target.value)}
+                className="w-full h-9 rounded-xl border border-white/10 bg-black/40 px-3 text-xs text-white focus:border-sky-500/50 focus:outline-none font-mono"
+              >
+                <option value="">All Types</option>
+                <option value="traces">Traces</option>
+                <option value="logs">Logs</option>
+                <option value="metrics">Metrics</option>
+              </select>
+            </div>
+
+            <div className="md:col-span-2">
+              <label className="block font-mono text-[10px] text-slate-400 uppercase tracking-wider mb-1">
+                Time Window
+              </label>
+              <select
+                value={searchTimeRange}
+                onChange={(e) => setSearchTimeRange(e.target.value)}
+                className="w-full h-9 rounded-xl border border-white/10 bg-black/40 px-3 text-xs text-white focus:border-sky-500/50 focus:outline-none font-mono"
+              >
+                <option value="1h">Last 1 Hour</option>
+                <option value="6h">Last 6 Hours</option>
+                <option value="24h">Last 24 Hours</option>
+                <option value="7d">Last 7 Days</option>
+                <option value="all">All Storage</option>
+              </select>
+            </div>
+
+            <div className="md:col-span-2">
+              <label className="block font-mono text-[10px] text-slate-400 uppercase tracking-wider mb-1">
+                Limit
+              </label>
+              <select
+                value={searchLimit}
+                onChange={(e) => setSearchLimit(Number(e.target.value))}
+                className="w-full h-9 rounded-xl border border-white/10 bg-black/40 px-3 text-xs text-white focus:border-sky-500/50 focus:outline-none font-mono"
+              >
+                <option value={50}>50 entries</option>
+                <option value={100}>100 entries</option>
+                <option value={250}>250 entries</option>
+                <option value={500}>500 entries</option>
+              </select>
+            </div>
+
+            <div className="md:col-span-2 flex items-end">
+              <button
+                type="submit"
+                disabled={searchLoading}
+                className="w-full h-9 rounded-xl bg-sky-500 hover:bg-sky-400 text-slate-950 font-mono text-xs font-bold transition-colors flex items-center justify-center gap-1.5 shadow-lg shadow-sky-500/20 disabled:opacity-50"
+              >
+                <Search className="h-3.5 w-3.5" />
+                <span>{searchLoading ? "Searching..." : "Search"}</span>
+              </button>
+            </div>
+
+          </div>
+        </form>
+
+        {searchError && (
+          <div className="rounded-xl border border-rose-500/20 bg-rose-500/10 p-3 text-xs font-mono text-rose-300">
+            {searchError}
+          </div>
+        )}
+
+        {/* Results List */}
+        {searchResults !== null && (
+          <div className="space-y-3 pt-2">
+            <div className="flex items-center justify-between text-[11px] font-mono text-slate-400 border-b border-white/5 pb-2">
+              <span>Found {searchResults.length} persistent storage matches</span>
+            </div>
+
+            {searchResults.length === 0 ? (
+              <div className="py-12 text-center text-sm text-slate-500 font-mono">
+                No telemetry entries matched your search criteria in the selected time range.
+              </div>
+            ) : (
+              <div className="space-y-2 max-h-[500px] overflow-y-auto pr-1">
+                {searchResults.map((entry, idx) => {
+                  const traceId = extractTraceId(entry);
+                  const isExpanded = expandedIndex === idx;
+                  const timeFormatted = entry.timestamp
+                    ? new Date(entry.timestamp / 1_000_000).toLocaleString()
+                    : "—";
+
+                  return (
+                    <div
+                      key={idx}
+                      className="rounded-xl border border-white/5 bg-black/30 p-3 text-xs font-mono transition-colors hover:border-white/15 space-y-2"
+                    >
+                      <div
+                        onClick={() => setExpandedIndex(isExpanded ? null : idx)}
+                        className="flex cursor-pointer items-center justify-between gap-3"
+                      >
+                        <div className="flex items-center gap-2">
+                          <span
+                            className={`rounded px-1.5 py-0.5 text-[9px] font-bold uppercase tracking-wider ${
+                              entry.type === "traces"
+                                ? "bg-sky-500/20 text-sky-300 border border-sky-500/30"
+                                : entry.type === "logs"
+                                ? "bg-amber-500/20 text-amber-300 border border-amber-500/30"
+                                : "bg-emerald-500/20 text-emerald-300 border border-emerald-500/30"
+                            }`}
+                          >
+                            {entry.type}
+                          </span>
+                          <span className="text-slate-400 text-[11px]">{timeFormatted}</span>
+                          {traceId && (
+                            <span className="text-sky-400 font-bold">
+                              trace:{traceId.slice(0, 8)}...
+                            </span>
+                          )}
+                        </div>
+
+                        <div className="flex items-center gap-2">
+                          {traceId && (
+                            <button
+                              onClick={(e) => {
+                                e.stopPropagation();
+                                setActiveTraceId(traceId);
+                                setActiveTab("tab1");
+                              }}
+                              className="rounded bg-sky-500/10 border border-sky-500/30 px-2 py-0.5 text-[10px] text-sky-300 hover:bg-sky-500/20 transition-colors"
+                            >
+                              Open Trace
+                            </button>
+                          )}
+                          <button
+                            onClick={(e) => {
+                              e.stopPropagation();
+                              copyToClipboard(JSON.stringify(entry.data, null, 2), `res-${idx}`);
+                            }}
+                            className="p-1 text-slate-500 hover:text-white transition-colors"
+                            title="Copy JSON"
+                          >
+                            {copiedId === `res-${idx}` ? (
+                              <Check className="h-3 w-3 text-emerald-400" />
+                            ) : (
+                              <Copy className="h-3 w-3" />
+                            )}
+                          </button>
+                        </div>
+                      </div>
+
+                      {isExpanded && (
+                        <pre className="mt-2 max-h-56 overflow-auto rounded-lg border border-white/5 bg-slate-950 p-2.5 text-[10px] text-slate-300 leading-relaxed">
+                          <code>{JSON.stringify(entry.data, null, 2)}</code>
+                        </pre>
+                      )}
+                    </div>
+                  );
+                })}
+              </div>
+            )}
+          </div>
+        )}
+      </section>
+    </div>
+  );
+};
+
+export default SearchAndWatcher;

otel-ui/src/components/ServiceGraph.tsx

@@ -0,0 +1,495 @@
+import { useMemo, useState } from "react";
+import { Layers } from "lucide-react";
+import { formatDuration } from "../lib/timeUtils";
+
+export interface GraphSpan {
+  spanId: string;
+  parentSpanId?: string;
+  service: string;
+  duration: number;
+  hasError?: boolean;
+}
+
+interface ServiceNode {
+  id: string;
+  name: string;
+  spanCount: number;
+  errorCount: number;
+  totalDuration: number;
+  layer: number;
+  x: number;
+  y: number;
+}
+
+interface ServiceEdge {
+  id: string;
+  source: string;
+  target: string;
+  callCount: number;
+  avgDuration: number;
+  hasError: boolean;
+}
+
+interface ServiceGraphProps {
+  spans: GraphSpan[];
+  onSelectService?: (serviceName: string | null) => void;
+  selectedService?: string | null;
+}
+
+export const ServiceGraph = ({
+  spans,
+  onSelectService,
+  selectedService,
+}: ServiceGraphProps) => {
+  const [hoveredNode, setHoveredNode] = useState<string | null>(null);
+
+  const { nodes, edges, width, height } = useMemo(() => {
+    // 1. Map spans by ID for parent lookup
+    const spanMap = new Map<string, GraphSpan>();
+    spans.forEach((s) => spanMap.set(s.spanId, s));
+
+    // 2. Aggregate service metrics
+    const serviceStats = new Map<
+      string,
+      { spanCount: number; errorCount: number; totalDuration: number }
+    >();
+
+    spans.forEach((span) => {
+      const current = serviceStats.get(span.service) || {
+        spanCount: 0,
+        errorCount: 0,
+        totalDuration: 0,
+      };
+      current.spanCount += 1;
+      if (span.hasError) current.errorCount += 1;
+      current.totalDuration += span.duration;
+      serviceStats.set(span.service, current);
+    });
+
+    // 3. Aggregate cross-service edges
+    const edgeMap = new Map<
+      string,
+      { source: string; target: string; count: number; totalDur: number; errors: number }
+    >();
+
+    spans.forEach((span) => {
+      if (span.parentSpanId && spanMap.has(span.parentSpanId)) {
+        const parentSpan = spanMap.get(span.parentSpanId)!;
+        if (parentSpan.service !== span.service) {
+          const edgeKey = `${parentSpan.service}->${span.service}`;
+          const currentEdge = edgeMap.get(edgeKey) || {
+            source: parentSpan.service,
+            target: span.service,
+            count: 0,
+            totalDur: 0,
+            errors: 0,
+          };
+          currentEdge.count += 1;
+          currentEdge.totalDur += span.duration;
+          if (span.hasError) currentEdge.errors += 1;
+          edgeMap.set(edgeKey, currentEdge);
+        }
+      }
+    });
+
+    const uniqueServices = Array.from(serviceStats.keys());
+
+    // 4. Determine layer depth for each service (Topological ordering)
+    const inDegree = new Map<string, number>();
+    uniqueServices.forEach((s) => inDegree.set(s, 0));
+    edgeMap.forEach((edge) => {
+      inDegree.set(edge.target, (inDegree.get(edge.target) || 0) + 1);
+    });
+
+    const layers = new Map<string, number>();
+    // Roots have 0 in-degree
+    const queue: string[] = [];
+    uniqueServices.forEach((s) => {
+      if ((inDegree.get(s) || 0) === 0) {
+        layers.set(s, 0);
+        queue.push(s);
+      }
+    });
+
+    // If there's a cycle or no zero-in-degree root, fallback to first service
+    if (queue.length === 0 && uniqueServices.length > 0) {
+      layers.set(uniqueServices[0], 0);
+      queue.push(uniqueServices[0]);
+    }
+
+    while (queue.length > 0) {
+      const current = queue.shift()!;
+      const currentLayer = layers.get(current) || 0;
+      edgeMap.forEach((edge) => {
+        if (edge.source === current) {
+          const targetLayer = Math.max(layers.get(edge.target) || 0, currentLayer + 1);
+          layers.set(edge.target, targetLayer);
+          if (!queue.includes(edge.target)) {
+            queue.push(edge.target);
+          }
+        }
+      });
+    }
+
+    // Ensure all services have a layer
+    uniqueServices.forEach((s) => {
+      if (!layers.has(s)) layers.set(s, 0);
+    });
+
+    // Group services by layer
+    const layerGroups = new Map<number, string[]>();
+    uniqueServices.forEach((s) => {
+      const l = layers.get(s) || 0;
+      const group = layerGroups.get(l) || [];
+      group.push(s);
+      layerGroups.set(l, group);
+    });
+
+    const totalLayers = Math.max(1, layerGroups.size);
+    const nodeWidth = 200;
+    const nodeHeight = 84;
+    const xGap = 160;
+    const yGap = 36;
+
+    let maxNodesInLayer = 1;
+    layerGroups.forEach((group) => {
+      if (group.length > maxNodesInLayer) maxNodesInLayer = group.length;
+    });
+
+    const canvasWidth = Math.max(700, totalLayers * (nodeWidth + xGap));
+    const canvasHeight = Math.max(300, maxNodesInLayer * (nodeHeight + yGap) + 60);
+
+    // Compute positions
+    const computedNodes: ServiceNode[] = [];
+    const nodePosMap = new Map<string, { x: number; y: number }>();
+
+    Array.from(layerGroups.keys())
+      .sort((a, b) => a - b)
+      .forEach((layerIdx) => {
+        const group = layerGroups.get(layerIdx) || [];
+        const layerX = 40 + layerIdx * (nodeWidth + xGap);
+        const groupHeight = group.length * nodeHeight + (group.length - 1) * yGap;
+        const startY = Math.max(40, (canvasHeight - groupHeight) / 2);
+
+        group.forEach((serviceName, idxInGroup) => {
+          const layerY = startY + idxInGroup * (nodeHeight + yGap);
+          const stats = serviceStats.get(serviceName)!;
+          computedNodes.push({
+            id: serviceName,
+            name: serviceName,
+            spanCount: stats.spanCount,
+            errorCount: stats.errorCount,
+            totalDuration: stats.totalDuration,
+            layer: layerIdx,
+            x: layerX,
+            y: layerY,
+          });
+          nodePosMap.set(serviceName, { x: layerX, y: layerY });
+        });
+      });
+
+    // Computed edges with coordinates
+    const computedEdges: ServiceEdge[] = [];
+    edgeMap.forEach((edge, key) => {
+      computedEdges.push({
+        id: key,
+        source: edge.source,
+        target: edge.target,
+        callCount: edge.count,
+        avgDuration: edge.totalDur / edge.count,
+        hasError: edge.errors > 0,
+      });
+    });
+
+    return {
+      nodes: computedNodes,
+      edges: computedEdges,
+      width: canvasWidth,
+      height: canvasHeight,
+    };
+  }, [spans]);
+
+  if (nodes.length === 0) {
+    return (
+      <div className="py-12 text-center text-sm text-slate-500">
+        No service nodes available to visualize.
+      </div>
+    );
+  }
+
+  const nodeMap = new Map(nodes.map((n) => [n.id, n]));
+
+  return (
+    <div className="relative overflow-hidden rounded-2xl border border-white/10 bg-slate-950/60 p-4 shadow-2xl backdrop-blur-md">
+      <div className="mb-4 flex flex-wrap items-center justify-between gap-4 border-b border-white/5 pb-3">
+        <div className="flex items-center gap-2">
+          <Layers className="h-4 w-4 text-sky-400" />
+          <span className="text-xs font-bold uppercase tracking-widest text-slate-300">
+            Service Call Graph ({nodes.length} Services, {edges.length} Cross-Service Links)
+          </span>
+        </div>
+        <div className="flex items-center gap-2 text-[10px] text-slate-400">
+          <span className="inline-flex items-center gap-1">
+            <span className="h-2 w-2 rounded-full bg-emerald-400" /> Healthy
+          </span>
+          <span className="inline-flex items-center gap-1">
+            <span className="h-2 w-2 rounded-full bg-rose-400" /> Errors
+          </span>
+          {selectedService && (
+            <button
+              onClick={() => onSelectService?.(null)}
+              className="ml-2 rounded-lg bg-sky-500/20 px-2 py-0.5 text-sky-300 hover:bg-sky-500/30 transition-all font-semibold"
+            >
+              Clear filter ({selectedService})
+            </button>
+          )}
+        </div>
+      </div>
+
+      <div className="overflow-x-auto">
+        <svg
+          width={width}
+          height={height}
+          viewBox={`0 0 ${width} ${height}`}
+          className="mx-auto select-none"
+        >
+          <defs>
+            <marker
+              id="arrow-normal"
+              viewBox="0 0 10 10"
+              refX="8"
+              refY="5"
+              markerWidth="6"
+              markerHeight="6"
+              orient="auto-start-reverse"
+            >
+              <path d="M 0 1.5 L 8 5 L 0 8.5 z" fill="#38bdf8" />
+            </marker>
+            <marker
+              id="arrow-error"
+              viewBox="0 0 10 10"
+              refX="8"
+              refY="5"
+              markerWidth="6"
+              markerHeight="6"
+              orient="auto-start-reverse"
+            >
+              <path d="M 0 1.5 L 8 5 L 0 8.5 z" fill="#f43f5e" />
+            </marker>
+            <linearGradient id="edge-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+              <stop offset="0%" stopColor="#38bdf8" stopOpacity="0.8" />
+              <stop offset="100%" stopColor="#818cf8" stopOpacity="0.8" />
+            </linearGradient>
+            <linearGradient id="edge-error-grad" x1="0%" y1="0%" x2="100%" y2="0%">
+              <stop offset="0%" stopColor="#f43f5e" stopOpacity="0.9" />
+              <stop offset="100%" stopColor="#fb7185" stopOpacity="0.9" />
+            </linearGradient>
+          </defs>
+
+          {/* Render Connecting Edges */}
+          {edges.map((edge) => {
+            const src = nodeMap.get(edge.source);
+            const tgt = nodeMap.get(edge.target);
+            if (!src || !tgt) return null;
+
+            const startX = src.x + 200;
+            const startY = src.y + 42;
+            const endX = tgt.x;
+            const endY = tgt.y + 42;
+
+            const dx = endX - startX;
+            const cpx1 = startX + dx * 0.45;
+            const cpy1 = startY;
+            const cpx2 = endX - dx * 0.45;
+            const cpy2 = endY;
+
+            const pathD = `M ${startX} ${startY} C ${cpx1} ${cpy1}, ${cpx2} ${cpy2}, ${endX} ${endY}`;
+            const midX = (startX + endX) / 2;
+            const midY = (startY + endY) / 2;
+
+            const isEdgeHighlighted =
+              hoveredNode === edge.source ||
+              hoveredNode === edge.target ||
+              selectedService === edge.source ||
+              selectedService === edge.target;
+
+            return (
+              <g key={edge.id} className="transition-opacity duration-300">
+                {/* Background path for hit testing */}
+                <path
+                  d={pathD}
+                  fill="none"
+                  stroke="transparent"
+                  strokeWidth={20}
+                  className="cursor-pointer"
+                />
+                {/* Visible animated curve */}
+                <path
+                  d={pathD}
+                  fill="none"
+                  stroke={
+                    edge.hasError
+                      ? "url(#edge-error-grad)"
+                      : isEdgeHighlighted
+                      ? "#38bdf8"
+                      : "rgba(56, 189, 248, 0.4)"
+                  }
+                  strokeWidth={isEdgeHighlighted ? 3 : 2}
+                  strokeDasharray={edge.hasError ? "4 2" : undefined}
+                  markerEnd={edge.hasError ? "url(#arrow-error)" : "url(#arrow-normal)"}
+                />
+                {/* Edge Call Badge */}
+                <g transform={`translate(${midX - 35}, ${midY - 12})`}>
+                  <rect
+                    width={70}
+                    height={22}
+                    rx={6}
+                    fill="#030712"
+                    stroke={edge.hasError ? "rgba(244,63,94,0.4)" : "rgba(255,255,255,0.1)"}
+                    strokeWidth={1}
+                  />
+                  <text
+                    x={35}
+                    y={14}
+                    textAnchor="middle"
+                    fill={edge.hasError ? "#fb7185" : "#94a3b8"}
+                    fontSize={9}
+                    fontWeight="bold"
+                    fontFamily="monospace"
+                  >
+                    {edge.callCount}x · {formatDuration(edge.avgDuration)}
+                  </text>
+                </g>
+              </g>
+            );
+          })}
+
+          {/* Render Service Nodes */}
+          {nodes.map((node) => {
+            const isSelected = selectedService === node.name;
+            const isHovered = hoveredNode === node.name;
+            const hasError = node.errorCount > 0;
+
+            return (
+              <g
+                key={node.id}
+                transform={`translate(${node.x}, ${node.y})`}
+                onClick={() => onSelectService?.(isSelected ? null : node.name)}
+                onMouseEnter={() => setHoveredNode(node.name)}
+                onMouseLeave={() => setHoveredNode(null)}
+                className="cursor-pointer"
+              >
+                {/* Node Box */}
+                <rect
+                  width={200}
+                  height={84}
+                  rx={16}
+                  fill="#090d16"
+                  stroke={
+                    isSelected
+                      ? "#38bdf8"
+                      : hasError
+                      ? "#f43f5e"
+                      : isHovered
+                      ? "rgba(255, 255, 255, 0.3)"
+                      : "rgba(255, 255, 255, 0.1)"
+                  }
+                  strokeWidth={isSelected ? 2.5 : hasError ? 2 : 1}
+                  filter={isSelected ? "drop-shadow(0 0 12px rgba(56,189,248,0.4))" : undefined}
+                />
+
+                {/* Left Accent Stripe */}
+                <rect
+                  x={0}
+                  y={0}
+                  width={6}
+                  height={84}
+                  rx={3}
+                  fill={hasError ? "#f43f5e" : "#38bdf8"}
+                />
+
+                {/* Service Name Header */}
+                <g transform="translate(16, 24)">
+                  <text
+                    x={0}
+                    y={0}
+                    fill="#ffffff"
+                    fontSize={13}
+                    fontWeight="bold"
+                    fontFamily="sans-serif"
+                  >
+                    {node.name.length > 20 ? `${node.name.slice(0, 18)}...` : node.name}
+                  </text>
+                </g>
+
+                {/* Spans Count & Latency Pills */}
+                <g transform="translate(16, 44)">
+                  <text x={0} y={10} fill="#64748b" fontSize={10} fontFamily="monospace">
+                    Spans: <tspan fill="#e2e8f0">{node.spanCount}</tspan>
+                  </text>
+                  <text x={75} y={10} fill="#64748b" fontSize={10} fontFamily="monospace">
+                    Time: <tspan fill="#38bdf8">{formatDuration(node.totalDuration)}</tspan>
+                  </text>
+                </g>
+
+                {/* Status Badges */}
+                <g transform="translate(16, 68)">
+                  {hasError ? (
+                    <g>
+                      <rect width={80} height={16} rx={4} fill="rgba(244,63,94,0.15)" />
+                      <text
+                        x={40}
+                        y={11}
+                        textAnchor="middle"
+                        fill="#fb7185"
+                        fontSize={9}
+                        fontWeight="bold"
+                      >
+                        {node.errorCount} Error{node.errorCount > 1 ? "s" : ""}
+                      </text>
+                    </g>
+                  ) : (
+                    <g>
+                      <rect width={60} height={16} rx={4} fill="rgba(16,185,129,0.15)" />
+                      <text
+                        x={30}
+                        y={11}
+                        textAnchor="middle"
+                        fill="#34d399"
+                        fontSize={9}
+                        fontWeight="bold"
+                      >
+                        Healthy
+                      </text>
+                    </g>
+                  )}
+                  {isSelected && (
+                    <g transform="translate(90, 0)">
+                      <rect width={55} height={16} rx={4} fill="rgba(56,189,248,0.2)" />
+                      <text
+                        x={27.5}
+                        y={11}
+                        textAnchor="middle"
+                        fill="#38bdf8"
+                        fontSize={9}
+                        fontWeight="bold"
+                      >
+                        Active
+                      </text>
+                    </g>
+                  )}
+                </g>
+              </g>
+            );
+          })}
+        </svg>
+      </div>
+
+      {nodes.length === 1 && (
+        <div className="mt-3 text-center text-xs text-slate-500">
+          This trace was executed inside a single service ({nodes[0].name}). No cross-service network calls were made.
+        </div>
+      )}
+    </div>
+  );
+};

otel-ui/src/components/SetupPanel.tsx

@@ -0,0 +1,112 @@
+import { useState } from "react"
+import { Check, Copy, ExternalLink, FileText, Globe } from "lucide-react"
+import { useObservabilityData } from "../hooks/useObservabilityData"
+
+const scriptSnippet = `<script src="https://otel.k6n.net/static/otel-rum.js"></script>
+<script>
+  var rum = OtelRUM.init({
+    serviceName: "my-web-app",
+    baseUrl: "https://otel.k6n.net", // Target OTel collector (defaults to script origin)
+  });
+
+  // Optional: propagate trace context to your own API calls
+  fetch("/api/orders", { headers: rum.traceHeaders() });
+
+  // Optional: report a caught exception manually
+  try { risky(); } catch (err) { rum.captureException(err); }
+
+  // React render failures should be reported from an error boundary.
+  // componentDidCatch(error, info) {
+  //   rum.captureException(error, { "react.component_stack": info.componentStack });
+  // }
+</script>`
+
+const CopyButton = ({ text }: { text: string }) => {
+  const [copied, setCopied] = useState(false)
+  return (
+    <button
+      onClick={async () => {
+        try {
+          await navigator.clipboard.writeText(text)
+          setCopied(true)
+          setTimeout(() => setCopied(false), 1500)
+        } catch {
+          // clipboard API unavailable; nothing to fall back to here
+        }
+      }}
+      className="flex items-center gap-1.5 rounded-lg border border-white/10 px-2.5 py-1 text-[10px] font-bold uppercase tracking-wider text-slate-400 hover:text-white hover:border-white/20 transition-all"
+    >
+      {copied ? <Check className="h-3 w-3 text-emerald-400" /> : <Copy className="h-3 w-3" />}
+      {copied ? "Copied" : "Copy"}
+    </button>
+  )
+}
+
+export const SetupPanel = () => {
+  const { setActiveTab } = useObservabilityData()
+  return (
+    <div className="space-y-6 max-w-3xl">
+      <div className="rounded-xl border border-white/10 bg-slate-950/40 p-5">
+        <div className="flex items-center justify-between gap-4">
+          <span className="inline-flex items-center gap-2 text-[10px] font-black uppercase tracking-[0.2em] text-slate-500">
+            <Globe className="h-3.5 w-3.5 text-violet-400" />
+            Browser instrumentation (otel-rum.js)
+          </span>
+          <CopyButton text={scriptSnippet} />
+        </div>
+        <p className="mt-2 text-[12px] text-slate-400">
+          Drop this into any page to capture uncaught errors, unhandled rejections and failed
+          resource loads as OTLP logs. Errors show up here under the{" "}
+          <span className="text-slate-200 font-semibold">Issues</span> tab tagged{" "}
+          <span className="text-violet-300 font-semibold">Frontend Error</span>, and correlate with
+          backend traces via <code className="text-sky-300">traceHeaders()</code>.
+        </p>
+        <pre className="mt-3 overflow-x-auto rounded-lg border border-white/5 bg-black/50 p-3 text-[11px] leading-relaxed text-slate-300">
+          <code>{scriptSnippet}</code>
+        </pre>
+        <div className="mt-4 flex items-center gap-3">
+          <a
+            href="/static/otel-rum.js"
+            target="_blank"
+            rel="noreferrer"
+            className="inline-flex items-center gap-1.5 text-[11px] font-semibold text-sky-400 hover:text-sky-300"
+          >
+            <ExternalLink className="h-3 w-3" />
+            View raw script at /static/otel-rum.js
+          </a>
+          <span className="text-slate-600">•</span>
+          <button
+            type="button"
+            onClick={() => setActiveTab("tab7")}
+            className="inline-flex items-center gap-1.5 text-[11px] font-semibold text-indigo-400 hover:text-indigo-300"
+          >
+            <Globe className="h-3 w-3" />
+            Open Frontend & RUM Explorer →
+          </button>
+        </div>
+      </div>
+
+      <div className="rounded-xl border border-white/10 bg-slate-950/40 p-5">
+        <span className="inline-flex items-center gap-2 text-[10px] font-black uppercase tracking-[0.2em] text-slate-500">
+          <FileText className="h-3.5 w-3.5 text-sky-400" />
+          Agent instructions
+        </span>
+        <p className="mt-2 text-[12px] text-slate-400">
+          A single doc an AI coding agent can fetch before touching this API - endpoints, query
+          params and response shapes for traces, logs, metrics and issues.
+        </p>
+        <a
+          href="/agents_example.md"
+          target="_blank"
+          rel="noreferrer"
+          className="mt-3 inline-flex items-center gap-1.5 text-[11px] font-semibold text-sky-400 hover:text-sky-300"
+        >
+          <ExternalLink className="h-3 w-3" />
+          Open /agents_example.md
+        </a>
+      </div>
+    </div>
+  )
+}
+
+export default SetupPanel

otel-ui/src/components/Sidebar.tsx

@@ -0,0 +1,344 @@
+import React from "react";
+import { useObservabilityData } from "../hooks/useObservabilityData";
+import { useIssues } from "../hooks/useIssues";
+import { useStatsAvailability } from "../hooks/useStats";
+
+export const Sidebar: React.FC = () => {
+  const {
+    activeTab,
+    setActiveTab,
+    traces,
+    logs,
+    metrics,
+    isConnected,
+    uniqueServices,
+    serviceHealth,
+    selectedServices,
+    toggleService,
+    resetFilters,
+  } = useObservabilityData();
+
+  const { issues } = useIssues();
+  const statsAvailable = useStatsAvailability();
+
+  // Active open issues count
+  const openIssuesCount = React.useMemo(() => {
+    return issues.filter((i) => !i.status || i.status === "open").length;
+  }, [issues]);
+
+  const errorLogCount = React.useMemo(() => {
+    let errs = 0;
+    logs.forEach((log) => {
+      (log.scopeLogs ?? []).forEach((sl) => {
+        (sl.logRecords ?? []).forEach((r) => {
+          if ((r.severityNumber ?? 0) >= 17) errs += 1;
+        });
+      });
+    });
+    return errs;
+  }, [logs]);
+
+  const totalSpans = React.useMemo(() => {
+    let count = 0;
+    traces.forEach((t) => {
+      (t.resourceSpans ?? []).forEach((rs) => {
+        (rs.scopeSpans ?? []).forEach((ss) => {
+          count += (ss.spans ?? []).length;
+        });
+      });
+    });
+    return count;
+  }, [traces]);
+
+  const { frontendCount, frontendErrorCount } = React.useMemo(() => {
+    let count = 0;
+    let errors = 0;
+    logs.forEach((log) => {
+      (log.scopeLogs ?? []).forEach((sl) => {
+        (sl.logRecords ?? []).forEach((r) => {
+          const attrs = r.attributes ?? [];
+          const source = attrs.find((a) => a.key === "source")?.value.stringValue;
+          const isRum = sl.scope?.name === "otel-rum";
+          const vital = attrs.find((a) => a.key === "rum.vital")?.value.stringValue;
+          const isFrontend = source === "frontend" || isRum || Boolean(vital);
+          if (isFrontend) {
+            count++;
+            if ((r.severityNumber ?? 0) >= 17 || attrs.some((a) => a.key === "exception.type")) {
+              errors++;
+            }
+          }
+        });
+      });
+    });
+    return { frontendCount: count, frontendErrorCount: errors };
+  }, [logs]);
+
+  const navItems = [
+    {
+      tab: "tab0",
+      label: "Issues & Anomalies",
+      icon: "error",
+      badge: openIssuesCount > 0 ? `${openIssuesCount} active` : `${issues.length} total`,
+      badgeType: openIssuesCount > 0 ? ("error" as const) : ("outline" as const),
+    },
+    {
+      tab: "tab8",
+      label: "Similarity Patterns",
+      icon: "hub",
+      badge: undefined,
+      badgeType: "outline" as const,
+    },
+    {
+      tab: "tab9",
+      label: "Cluster Rollouts",
+      icon: "cloud_sync",
+      badge: undefined,
+      badgeType: "outline" as const,
+    },
+    {
+      tab: "tab1",
+      label: "Traces & Spans",
+      icon: "account_tree",
+      badge: totalSpans > 0 ? `${totalSpans} spans` : `${traces.length} traces`,
+      badgeType: totalSpans > 0 ? ("secondary" as const) : ("outline" as const),
+    },
+    {
+      tab: "tab3",
+      label: "Live Log Stream",
+      icon: "terminal",
+      badge: errorLogCount > 0 ? `${errorLogCount} err` : `${logs.length} logs`,
+      badgeType: errorLogCount > 0 ? ("error" as const) : ("secondary" as const),
+    },
+    {
+      tab: "tab2",
+      label: "Metrics & APM",
+      icon: "query_stats",
+      badge: `${metrics.length} metrics`,
+      badgeType: "outline" as const,
+    },
+    {
+      tab: "tab7",
+      label: "Frontend & RUM",
+      icon: "devices",
+      badge:
+        frontendCount > 0
+          ? frontendErrorCount > 0
+            ? `${frontendErrorCount} err`
+            : `${frontendCount} sig`
+          : "RUM",
+      badgeType: frontendErrorCount > 0 ? ("error" as const) : ("secondary" as const),
+    },
+    ...(statsAvailable
+      ? [
+          {
+            tab: "tab5",
+            label: "Historical Trends",
+            icon: "trending_up",
+            badge: "ClickHouse",
+            badgeType: "outline" as const,
+          },
+        ]
+      : []),
+    {
+      tab: "tab6",
+      label: "Archive & Trap",
+      icon: "radar",
+      badge: "Deep Query",
+      badgeType: "secondary" as const,
+    },
+    {
+      tab: "tab4",
+      label: "Setup & Agent API",
+      icon: "build",
+      badge: "RUM / Doc",
+      badgeType: "outline" as const,
+    },
+  ];
+
+  return (
+    <aside className="fixed left-0 top-0 h-full w-72 bg-surface-container-lowest z-50 flex flex-col justify-between select-none shadow-[0_1px_8px_rgba(0,0,0,0.04)] border-r border-surface-container/60">
+      <div className="flex flex-col flex-1 overflow-y-auto">
+        {/* Brand Header */}
+        <div className="h-14 px-4 flex items-center justify-between bg-surface-container-lowest border-b border-surface-container/40">
+          <div className="flex items-center gap-2.5">
+            <div className="flex h-8 w-8 items-center justify-center rounded-lg bg-gradient-to-br from-primary-container to-secondary text-on-primary font-black text-sm shadow-md">
+              <span className="material-symbols-outlined text-[18px]">hub</span>
+            </div>
+            <div className="flex flex-col">
+              <div className="flex items-center gap-1.5">
+                <span className="font-headline font-semibold text-base tracking-tight text-on-surface leading-none">
+                  HyperScope
+                </span>
+                <span className="font-mono text-[10px] px-1 py-0.5 rounded bg-surface-container-high text-primary-container leading-none uppercase font-bold">
+                  OTel
+                </span>
+              </div>
+              <span className="font-mono text-[11px] text-outline mt-0.5 leading-none">
+                Realtime Observability
+              </span>
+            </div>
+          </div>
+
+          <div
+            className={`flex items-center gap-1 px-2 py-0.5 rounded-full font-mono text-[9px] font-bold uppercase tracking-wider ${
+              isConnected
+                ? "bg-secondary-container/20 text-secondary border border-secondary-container/40"
+                : "bg-amber-500/15 text-amber-400 border border-amber-500/30 animate-pulse"
+            }`}
+          >
+            <span
+              className={`w-1.5 h-1.5 rounded-full ${
+                isConnected ? "bg-secondary animate-pulse" : "bg-amber-400"
+              }`}
+            />
+            <span>{isConnected ? "LIVE" : "RECONNECT"}</span>
+          </div>
+        </div>
+
+        {/* Telemetry Core Navigation */}
+        <div className="px-3 mt-3">
+          <div className="px-2 pb-1.5 font-mono text-[10px] text-outline uppercase tracking-wider font-semibold">
+            Telemetry Views
+          </div>
+          <nav className="space-y-1">
+            {navItems.map((item) => {
+              const isActive = activeTab === item.tab;
+              return (
+                <button
+                  key={item.tab}
+                  type="button"
+                  onClick={() => setActiveTab(item.tab)}
+                  className={`w-full flex items-center justify-between px-2.5 py-2 rounded-lg transition-colors group text-left ${
+                    isActive
+                      ? "bg-surface-container-high text-primary font-semibold shadow-sm"
+                      : "text-on-surface-variant hover:bg-surface-container hover:text-on-surface"
+                  }`}
+                >
+                  <div className="flex items-center gap-2.5">
+                    <span
+                      className={`material-symbols-outlined text-[18px] transition-colors ${
+                        isActive
+                          ? "text-primary-container"
+                          : "text-outline group-hover:text-primary-container"
+                      }`}
+                    >
+                      {item.icon}
+                    </span>
+                    <span className="text-xs font-body font-medium">{item.label}</span>
+                  </div>
+                  {item.badge && (
+                    <span
+                      className={`font-mono text-[10px] px-1.5 py-0.5 rounded font-medium ${
+                        item.badgeType === "error"
+                          ? "bg-error-container text-on-error-container font-semibold"
+                          : item.badgeType === "secondary"
+                          ? "bg-surface-container text-secondary"
+                          : "bg-surface-container text-outline"
+                      }`}
+                    >
+                      {item.badge}
+                    </span>
+                  )}
+                </button>
+              );
+            })}
+          </nav>
+        </div>
+
+        {/* Real Discovered Services Fleet List */}
+        <div className="px-3 mt-5 flex-1">
+          <div className="px-2 pb-1.5 flex items-center justify-between">
+            <span className="font-mono text-[10px] text-outline uppercase tracking-wider font-semibold">
+              Discovered Services ({uniqueServices.length})
+            </span>
+            {selectedServices.size > 0 && (
+              <button
+                type="button"
+                onClick={resetFilters}
+                className="font-mono text-[10px] text-primary-container hover:underline"
+              >
+                Clear
+              </button>
+            )}
+          </div>
+
+          <div className="space-y-1 font-mono text-xs">
+            {uniqueServices.length === 0 ? (
+              <div className="px-2.5 py-3 text-center rounded-lg bg-surface-container-low/50 text-[11px] text-outline italic">
+                Listening for service signals...
+              </div>
+            ) : (
+              uniqueServices.map((service) => {
+                const isSelected = selectedServices.has(service);
+                const health = serviceHealth[service];
+                const hasErrors = health?.hasErrors || false;
+                const errorCount = health?.errorCount || 0;
+
+                return (
+                  <button
+                    key={service}
+                    type="button"
+                    onClick={() => toggleService(service)}
+                    className={`w-full flex items-center justify-between px-2.5 py-1.5 rounded-lg text-left transition-colors ${
+                      isSelected
+                        ? "bg-primary-container/20 text-primary-container border border-primary-container/40 font-semibold"
+                        : "text-on-surface-variant hover:bg-surface-container hover:text-on-surface"
+                    }`}
+                  >
+                    <div className="flex items-center gap-2 min-w-0 pr-1">
+                      <span
+                        className={`w-2 h-2 rounded-full shrink-0 ${
+                          hasErrors
+                            ? "bg-error animate-pulse"
+                            : isSelected
+                            ? "bg-primary-container"
+                            : "bg-emerald-400/80"
+                        }`}
+                        title={hasErrors ? `${errorCount} errors in active window` : "Healthy"}
+                      />
+                      <span className="truncate text-xs">{service}</span>
+                    </div>
+
+                    {hasErrors ? (
+                      <span className="font-mono text-[10px] px-1.5 py-0.5 rounded bg-error-container text-on-error-container font-semibold shrink-0">
+                        {errorCount} err
+                      </span>
+                    ) : (
+                      <span className="font-mono text-[10px] text-outline shrink-0">
+                        {health?.totalSignals ?? 0}
+                      </span>
+                    )}
+                  </button>
+                );
+              })
+            )}
+          </div>
+        </div>
+      </div>
+
+      {/* Receiver Endpoints & WebSocket Health Status */}
+      <div className="p-3 bg-surface-container-lowest border-t border-surface-container/40">
+        <div className="p-2.5 rounded-lg bg-surface-container-low flex flex-col gap-1.5 border border-surface-container/40 font-mono text-[11px]">
+          <div className="flex items-center justify-between">
+            <span className="text-outline">Receiver Ports</span>
+            <span className="text-on-surface font-semibold">4317 / 4318</span>
+          </div>
+          <div className="flex items-center justify-between">
+            <span className="text-outline">Web UI &amp; API</span>
+            <span className="text-on-surface font-semibold">:8080</span>
+          </div>
+          <div className="flex items-center justify-between pt-1 border-t border-surface-container/40">
+            <span className="text-outline">Live Pipeline</span>
+            <span
+              className={`font-semibold ${
+                isConnected ? "text-secondary" : "text-amber-400"
+              }`}
+            >
+              {isConnected ? "Connected" : "Reconnecting..."}
+            </span>
+          </div>
+        </div>
+      </div>
+    </aside>
+  );
+};

otel-ui/src/components/StructuredLogTerminal.tsx

@@ -0,0 +1,582 @@
+import React, { useMemo, useState } from "react";
+import type { LogEntry, LogRecord, OtelAttribute } from "../lib/types";
+import { useObservabilityData } from "../hooks/useObservabilityData";
+import { formatAttributeValue, getServiceName } from "../lib/otel-utils";
+
+interface FlattenedRecord extends LogRecord {
+  scope?: string;
+  resourceAttributes: OtelAttribute[];
+  serviceName: string;
+}
+
+export const StructuredLogTerminal: React.FC<{ logs: LogEntry[] }> = ({ logs }) => {
+  const { setActiveTraceId, setActiveSpanId, setActiveTab, selectedServices, toggleService } =
+    useObservabilityData();
+
+  const [expandedRowId, setExpandedRowId] = useState<string | null>(null);
+  const [streamFilter, setStreamFilter] = useState("");
+  const [wrapText, setWrapText] = useState(false);
+  const [cleared, setCleared] = useState(false);
+
+  // Severity filters
+  const [selectedSeverities, setSelectedSeverities] = useState<Record<string, boolean>>({
+    FATAL: true,
+    ERROR: true,
+    WARN: true,
+    INFO: true,
+    DEBUG: false,
+  });
+
+  const toggleSeverity = (sev: string) => {
+    setSelectedSeverities((prev) => ({ ...prev, [sev]: !prev[sev] }));
+  };
+
+  // Flatten all log records
+  const allRecords: FlattenedRecord[] = useMemo(() => {
+    if (cleared) return [];
+    return logs.flatMap((log) =>
+      (log.scopeLogs ?? []).flatMap((scopeLog) =>
+        (scopeLog.logRecords ?? []).map((record) => {
+          const serviceName = getServiceName(
+            log.resource?.attributes ?? [],
+            record.attributes ?? [],
+          );
+          return {
+            ...record,
+            scope: scopeLog.scope?.name,
+            resourceAttributes: log.resource?.attributes ?? [],
+            serviceName,
+          };
+        }),
+      ),
+    );
+  }, [logs, cleared]);
+
+  // Counts by severity
+  const severityCounts = useMemo(() => {
+    const counts = { FATAL: 0, ERROR: 0, WARN: 0, INFO: 0, DEBUG: 0 };
+    allRecords.forEach((r) => {
+      const num = r.severityNumber ?? 0;
+      if (num >= 21) counts.FATAL++;
+      else if (num >= 17) counts.ERROR++;
+      else if (num >= 13) counts.WARN++;
+      else if (num >= 9) counts.INFO++;
+      else counts.DEBUG++;
+    });
+    return counts;
+  }, [allRecords]);
+
+  // Service facet counts
+  const serviceCounts = useMemo(() => {
+    const counts = new Map<string, number>();
+    allRecords.forEach((r) => {
+      counts.set(r.serviceName, (counts.get(r.serviceName) || 0) + 1);
+    });
+    return Array.from(counts.entries()).sort((a, b) => b[1] - a[1]);
+  }, [allRecords]);
+
+  // Real diagnostic heuristic calculated from actual logs
+  const errorHeuristic = useMemo(() => {
+    const errorRecords = allRecords.filter((r) => (r.severityNumber ?? 0) >= 17);
+    if (errorRecords.length === 0) {
+      return {
+        text: serviceCounts.length > 0
+          ? `All ${serviceCounts.length} active service(s) nominal. 0 error logs recorded in current buffer.`
+          : "No log signals received yet. Waiting for OTLP logs...",
+        hasErrors: false,
+      };
+    }
+
+    const errByService: Record<string, number> = {};
+    errorRecords.forEach((r) => {
+      errByService[r.serviceName] = (errByService[r.serviceName] || 0) + 1;
+    });
+
+    const sorted = Object.entries(errByService).sort((a, b) => b[1] - a[1]);
+    const topService = sorted[0]?.[0] || "unknown";
+    const topCount = sorted[0]?.[1] || 0;
+    const pct = Math.round((topCount / errorRecords.length) * 100);
+
+    return {
+      text: `${pct}% of current ERROR logs (${topCount} of ${errorRecords.length}) originated in "${topService}".`,
+      hasErrors: true,
+    };
+  }, [allRecords, serviceCounts.length]);
+
+  // Filter records based on selected severities, services, and local stream search
+  const visibleRecords = useMemo(() => {
+    return allRecords.filter((r) => {
+      const num = r.severityNumber ?? 0;
+      let sevKey = "DEBUG";
+      if (num >= 21) sevKey = "FATAL";
+      else if (num >= 17) sevKey = "ERROR";
+      else if (num >= 13) sevKey = "WARN";
+      else if (num >= 9) sevKey = "INFO";
+
+      if (!selectedSeverities[sevKey]) return false;
+
+      if (streamFilter.trim()) {
+        const query = streamFilter.toLowerCase().trim();
+        const msg = (r.body?.stringValue ?? "").toLowerCase();
+        const srv = r.serviceName.toLowerCase();
+        const trace = (r.traceId ?? "").toLowerCase();
+        const attrs = (r.attributes ?? []).some(
+          (a) =>
+            a.key.toLowerCase().includes(query) ||
+            formatAttributeValue(a.value).toLowerCase().includes(query),
+        );
+        if (!msg.includes(query) && !srv.includes(query) && !trace.includes(query) && !attrs) {
+          return false;
+        }
+      }
+
+      return true;
+    });
+  }, [allRecords, selectedSeverities, streamFilter]);
+
+  const copyLogStream = () => {
+    if (visibleRecords.length === 0) return;
+    const excerpt = visibleRecords
+      .slice(0, 50)
+      .map((r) => {
+        const time = new Date(Number(BigInt(r.timeUnixNano || "0") / 1_000_000n)).toISOString();
+        return `[${time}] ${r.severityText || "LOG"} ${r.serviceName}: ${r.body?.stringValue ?? ""}`;
+      })
+      .join("\n");
+    navigator.clipboard.writeText(excerpt);
+    alert("Copied visible log excerpt to clipboard.");
+  };
+
+  const navigateToTraceSpan = (traceId: string, spanId?: string) => {
+    setActiveTraceId(traceId);
+    if (spanId) setActiveSpanId(spanId);
+    setActiveTab("tab1");
+  };
+
+  const formatIsoTimestamp = (nanoStr?: string) => {
+    if (!nanoStr) return "—";
+    const millis = Number(BigInt(nanoStr) / 1_000_000n);
+    return new Date(millis).toISOString();
+  };
+
+  return (
+    <div className="grid grid-cols-1 lg:grid-cols-12 gap-4 items-start font-body">
+      {/* LEFT PANEL: Faceted Filter Breakdown (3 Cols) */}
+      <aside className="lg:col-span-3 flex flex-col gap-4">
+        {/* Severity Breakdown Card */}
+        <div className="bg-surface-container-lowest p-4 rounded-xl shadow-md flex flex-col gap-2 border border-surface-container/50">
+          <div className="flex items-center justify-between pb-1 border-b border-surface-container/40">
+            <span className="font-mono text-[10px] text-outline uppercase tracking-wider font-semibold">
+              Severity Breakdown
+            </span>
+            <span className="font-mono text-xs text-outline">{allRecords.length} Total</span>
+          </div>
+
+          <div className="space-y-1">
+            {/* FATAL */}
+            <label className="flex items-center justify-between p-1.5 rounded-lg bg-surface-container-low hover:bg-surface-container transition-colors cursor-pointer group">
+              <div className="flex items-center gap-2">
+                <input
+                  type="checkbox"
+                  checked={selectedSeverities.FATAL}
+                  onChange={() => toggleSeverity("FATAL")}
+                  className="rounded bg-surface-container-high border-outline text-error focus:ring-0"
+                />
+                <span className="font-mono text-xs text-error font-bold tracking-tight">FATAL</span>
+              </div>
+              <div className="flex items-center gap-1.5">
+                <span className="font-mono text-[10px] px-1.5 py-0.5 rounded bg-error-container text-on-error-container font-semibold">
+                  {severityCounts.FATAL}
+                </span>
+                <span className="w-1.5 h-1.5 rounded-full bg-error" />
+              </div>
+            </label>
+
+            {/* ERROR */}
+            <label className="flex items-center justify-between p-1.5 rounded-lg bg-surface-container-low hover:bg-surface-container transition-colors cursor-pointer group">
+              <div className="flex items-center gap-2">
+                <input
+                  type="checkbox"
+                  checked={selectedSeverities.ERROR}
+                  onChange={() => toggleSeverity("ERROR")}
+                  className="rounded bg-surface-container-high border-outline text-error focus:ring-0"
+                />
+                <span className="font-mono text-xs text-error font-medium">ERROR</span>
+              </div>
+              <div className="flex items-center gap-1.5">
+                <span className="font-mono text-[10px] px-1.5 py-0.5 rounded bg-error-container text-on-error-container font-semibold">
+                  {severityCounts.ERROR}
+                </span>
+                <span className="w-1.5 h-1.5 rounded-full bg-error" />
+              </div>
+            </label>
+
+            {/* WARN */}
+            <label className="flex items-center justify-between p-1.5 rounded-lg bg-surface-container-low hover:bg-surface-container transition-colors cursor-pointer group">
+              <div className="flex items-center gap-2">
+                <input
+                  type="checkbox"
+                  checked={selectedSeverities.WARN}
+                  onChange={() => toggleSeverity("WARN")}
+                  className="rounded bg-surface-container-high border-outline text-primary-fixed-dim focus:ring-0"
+                />
+                <span className="font-mono text-xs text-primary-fixed-dim">WARN</span>
+              </div>
+              <span className="font-mono text-[10px] px-1.5 py-0.5 rounded bg-surface-container-high text-on-surface font-medium">
+                {severityCounts.WARN}
+              </span>
+            </label>
+
+            {/* INFO */}
+            <label className="flex items-center justify-between p-1.5 rounded-lg bg-surface-container-low hover:bg-surface-container transition-colors cursor-pointer group">
+              <div className="flex items-center gap-2">
+                <input
+                  type="checkbox"
+                  checked={selectedSeverities.INFO}
+                  onChange={() => toggleSeverity("INFO")}
+                  className="rounded bg-surface-container-high border-outline text-secondary focus:ring-0"
+                />
+                <span className="font-mono text-xs text-secondary">INFO</span>
+              </div>
+              <span className="font-mono text-[10px] px-1.5 py-0.5 rounded bg-surface-container-high text-on-surface-variant">
+                {severityCounts.INFO}
+              </span>
+            </label>
+
+            {/* DEBUG */}
+            <label className="flex items-center justify-between p-1.5 rounded-lg bg-surface-container-low hover:bg-surface-container transition-colors cursor-pointer group">
+              <div className="flex items-center gap-2">
+                <input
+                  type="checkbox"
+                  checked={selectedSeverities.DEBUG}
+                  onChange={() => toggleSeverity("DEBUG")}
+                  className="rounded bg-surface-container-high border-outline text-outline focus:ring-0"
+                />
+                <span className="font-mono text-xs text-outline">DEBUG</span>
+              </div>
+              <span className="font-mono text-[10px] px-1.5 py-0.5 rounded bg-surface-container-high text-outline">
+                {severityCounts.DEBUG}
+              </span>
+            </label>
+          </div>
+        </div>
+
+        {/* Service Facet Card */}
+        <div className="bg-surface-container-lowest p-4 rounded-xl shadow-md flex flex-col gap-2 border border-surface-container/50">
+          <div className="flex items-center justify-between pb-1 border-b border-surface-container/40">
+            <span className="font-mono text-[10px] text-outline uppercase tracking-wider font-semibold">
+              Service Name
+            </span>
+            <span className="material-symbols-outlined text-outline text-[16px]">filter_alt</span>
+          </div>
+
+          <div className="space-y-1 font-mono text-xs">
+            {serviceCounts.slice(0, 6).map(([serviceName, count]) => {
+              const isSelected = selectedServices.has(serviceName);
+              return (
+                <div
+                  key={serviceName}
+                  onClick={() => toggleService(serviceName)}
+                  className={`flex items-center justify-between px-2.5 py-1.5 rounded-lg cursor-pointer transition-colors ${
+                    isSelected
+                      ? "bg-surface-container text-primary-container font-semibold"
+                      : "hover:bg-surface-container text-on-surface-variant"
+                  }`}
+                >
+                  <div className="flex items-center gap-2 min-w-0">
+                    <span
+                      className={`w-1.5 h-1.5 rounded-full shrink-0 ${
+                        isSelected ? "bg-primary-container" : "bg-outline"
+                      }`}
+                    />
+                    <span className="truncate">{serviceName}</span>
+                  </div>
+                  <span className="font-mono text-[10px] text-outline">{count}</span>
+                </div>
+              );
+            })}
+            {serviceCounts.length === 0 && (
+              <p className="text-center py-2 text-outline text-xs italic">No services detected</p>
+            )}
+          </div>
+        </div>
+
+        {/* AI Root Cause Diagnostic Hint Box */}
+        <div className="bg-surface-container-low p-4 rounded-xl shadow-md flex flex-col gap-1.5 border border-surface-container/60">
+          <div className="flex items-center gap-2 text-primary-container">
+            <span className="material-symbols-outlined text-[18px]">psychology</span>
+            <span className="font-headline font-semibold text-xs leading-none">
+              Diagnostic Heuristic
+            </span>
+          </div>
+          <p className="font-body text-xs text-on-surface-variant leading-relaxed">
+            {errorHeuristic.text}
+          </p>
+          <div className="pt-1 flex items-center justify-between">
+            <button
+              onClick={() => {
+                setSelectedSeverities({ FATAL: true, ERROR: true, WARN: false, INFO: false, DEBUG: false });
+              }}
+              className="font-mono text-[10px] text-primary-container hover:underline uppercase flex items-center gap-1 font-semibold"
+            >
+              <span>Filter Only Error Signatures</span>
+              <span className="material-symbols-outlined text-[14px]">arrow_forward</span>
+            </button>
+          </div>
+        </div>
+      </aside>
+
+      {/* RIGHT PANEL: High-Density Structured Live Log Stream (9 Cols) */}
+      <main className="lg:col-span-9 flex flex-col bg-surface-container-lowest rounded-xl shadow-md overflow-hidden border border-surface-container/50">
+        {/* Stream Controls Bar */}
+        <div className="flex flex-wrap items-center justify-between gap-2 p-2.5 bg-surface-container-low border-b border-surface-container/40">
+          <div className="flex items-center gap-2 flex-1 min-w-[240px]">
+            <div className="relative flex-1">
+              <span className="material-symbols-outlined absolute left-2.5 top-1/2 -translate-y-1/2 text-outline text-[16px] pointer-events-none">
+                filter_list
+              </span>
+              <input
+                type="text"
+                value={streamFilter}
+                onChange={(e) => setStreamFilter(e.target.value)}
+                placeholder="Filter current buffer (e.g. 'timeout', 'status:504')..."
+                className="w-full h-7 pl-8 pr-3 rounded bg-surface-container text-on-surface font-mono text-xs placeholder:text-outline focus:outline-none focus:bg-surface-container-high transition-colors border border-surface-container/50"
+              />
+            </div>
+          </div>
+
+          <div className="flex items-center gap-2">
+            <label className="flex items-center gap-1.5 font-mono text-xs text-outline cursor-pointer select-none">
+              <input
+                type="checkbox"
+                checked={wrapText}
+                onChange={(e) => setWrapText(e.target.checked)}
+                className="rounded bg-surface-container border-outline text-primary-container focus:ring-0"
+              />
+              <span>Wrap Text</span>
+            </label>
+
+            <div className="h-3 w-px bg-surface-container-high" />
+
+            <button
+              onClick={() => setCleared(true)}
+              className="text-outline hover:text-on-surface p-1 rounded hover:bg-surface-container transition-colors"
+              title="Clear Buffer"
+            >
+              <span className="material-symbols-outlined text-[18px]">clear_all</span>
+            </button>
+
+            <button
+              onClick={copyLogStream}
+              className="text-outline hover:text-on-surface p-1 rounded hover:bg-surface-container transition-colors"
+              title="Copy Visible Buffer"
+            >
+              <span className="material-symbols-outlined text-[18px]">content_copy</span>
+            </button>
+          </div>
+        </div>
+
+        {/* Log Terminal Headers */}
+        <div className="grid grid-cols-12 px-4 py-1.5 bg-surface-container text-outline font-mono uppercase tracking-wider select-none text-[11px] border-b border-surface-container/60">
+          <div className="col-span-2">Timestamp (UTC)</div>
+          <div className="col-span-1">Level</div>
+          <div className="col-span-2">Service &amp; Trace</div>
+          <div className="col-span-7">Log Message &amp; Structured Payload</div>
+        </div>
+
+        {/* Realtime Log Entries List */}
+        <div className="divide-y divide-surface-container-high/40 font-mono text-xs max-h-[640px] overflow-y-auto select-text">
+          {visibleRecords.length === 0 ? (
+            <div className="p-8 text-center text-outline text-xs">
+              {cleared
+                ? "Stream buffer cleared. Waiting for new telemetry records..."
+                : "No log records match the current filters."}
+            </div>
+          ) : (
+            visibleRecords.map((record) => {
+              const num = record.severityNumber ?? 0;
+              let sevLabel = record.severityText?.toUpperCase() || "INFO";
+              let sevClass = "bg-surface-container-high text-secondary";
+
+              if (num >= 21) {
+                sevLabel = "FATAL";
+                sevClass = "bg-error text-on-error font-extrabold";
+              } else if (num >= 17) {
+                sevLabel = "ERROR";
+                sevClass = "bg-error-container text-on-error-container font-bold";
+              } else if (num >= 13) {
+                sevLabel = "WARN";
+                sevClass = "bg-surface-container-high text-primary-fixed-dim font-medium";
+              }
+
+              const rowId = record.uid;
+              const isExpanded = expandedRowId === rowId;
+              const isoTime = formatIsoTimestamp(record.timeUnixNano);
+              const bodyMessage = record.body?.stringValue ?? "(empty body)";
+
+              return (
+                <div
+                  key={rowId}
+                  className="flex flex-col bg-surface-container-lowest hover:bg-surface-container/30 transition-colors"
+                >
+                  <div
+                    onClick={() => setExpandedRowId(isExpanded ? null : rowId)}
+                    className="grid grid-cols-12 px-4 py-2 items-start cursor-pointer group"
+                  >
+                    <div className="col-span-2 text-outline whitespace-nowrap pt-0.5 text-[11px]">
+                      {isoTime}
+                    </div>
+
+                    <div className="col-span-1">
+                      <span
+                        className={`font-mono text-[10px] px-1.5 py-0.5 rounded tracking-tight ${sevClass}`}
+                      >
+                        {sevLabel}
+                      </span>
+                    </div>
+
+                    <div className="col-span-2 flex flex-col gap-0.5 min-w-0 pr-2">
+                      <span className="text-primary truncate font-medium">{record.serviceName}</span>
+                      {record.traceId && (
+                        <div className="flex items-center gap-1">
+                          <span className="material-symbols-outlined text-[12px] text-primary-container">
+                            link
+                          </span>
+                          <button
+                            onClick={(e) => {
+                              e.stopPropagation();
+                              navigateToTraceSpan(record.traceId!, record.spanId);
+                            }}
+                            className="text-primary-container hover:underline truncate font-mono text-[10px]"
+                          >
+                            {record.traceId.slice(0, 12)}
+                          </button>
+                        </div>
+                      )}
+                    </div>
+
+                    <div className="col-span-7 flex items-start justify-between min-w-0">
+                      <div
+                        className={`text-on-surface font-normal ${
+                          wrapText ? "break-words" : "truncate"
+                        }`}
+                      >
+                        {num >= 17 && (
+                          <span className="text-error font-medium mr-1">Exception:</span>
+                        )}
+                        {bodyMessage}
+                      </div>
+
+                      <span className="material-symbols-outlined text-outline text-[16px] group-hover:text-on-surface transition-transform shrink-0 ml-2">
+                        {isExpanded ? "unfold_less" : "unfold_more"}
+                      </span>
+                    </div>
+                  </div>
+
+                  {/* Structured Payload Drawer (Expanded) */}
+                  {isExpanded && (
+                    <div className="px-4 py-3 bg-surface-container-low text-xs flex flex-col gap-2 border-t border-surface-container">
+                      <div className="flex items-center justify-between border-b border-surface-container pb-1">
+                        <span className="font-mono text-[10px] text-outline uppercase tracking-wider">
+                          OpenTelemetry Attributes &amp; Resource Scope
+                        </span>
+                        {record.traceId && (
+                          <button
+                            onClick={() => {
+                              navigator.clipboard.writeText(
+                                `trace:${record.traceId} span:${record.spanId || ""}`,
+                              );
+                              alert("Copied trace context to clipboard.");
+                            }}
+                            className="flex items-center gap-1 font-mono text-[10px] text-primary-container hover:text-primary transition-colors"
+                          >
+                            <span className="material-symbols-outlined text-[14px]">copy_all</span>
+                            <span>Copy Trace Context</span>
+                          </button>
+                        )}
+                      </div>
+
+                      {/* Attributes Grid */}
+                      <div className="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-4 gap-2 font-mono text-xs">
+                        {record.spanId && (
+                          <div className="p-1.5 rounded bg-surface-container flex flex-col">
+                            <span className="text-outline text-[10px] uppercase">span_id</span>
+                            <span className="text-primary-container truncate font-medium">
+                              {record.spanId}
+                            </span>
+                          </div>
+                        )}
+                        {record.traceId && (
+                          <div className="p-1.5 rounded bg-surface-container flex flex-col">
+                            <span className="text-outline text-[10px] uppercase">trace_id</span>
+                            <span className="text-on-surface truncate">{record.traceId}</span>
+                          </div>
+                        )}
+                        {record.scope && (
+                          <div className="p-1.5 rounded bg-surface-container flex flex-col">
+                            <span className="text-outline text-[10px] uppercase">scope</span>
+                            <span className="text-secondary truncate">{record.scope}</span>
+                          </div>
+                        )}
+                        <div className="p-1.5 rounded bg-surface-container flex flex-col">
+                          <span className="text-outline text-[10px] uppercase">service.name</span>
+                          <span className="text-primary truncate">{record.serviceName}</span>
+                        </div>
+                      </div>
+
+                      {/* Raw JSON block */}
+                      <pre className="p-2.5 rounded bg-surface-container text-on-surface overflow-x-auto text-[11px] leading-relaxed border border-surface-container-high/40">
+                        <code>
+                          {JSON.stringify(
+                            {
+                              timestamp: isoTime,
+                              severity: sevLabel,
+                              service_name: record.serviceName,
+                              trace_id: record.traceId,
+                              span_id: record.spanId,
+                              body: bodyMessage,
+                              attributes: Object.fromEntries(
+                                (record.attributes ?? []).map((a) => [
+                                  a.key,
+                                  formatAttributeValue(a.value),
+                                ]),
+                              ),
+                              resource_attributes: Object.fromEntries(
+                                (record.resourceAttributes ?? []).map((a) => [
+                                  a.key,
+                                  formatAttributeValue(a.value),
+                                ]),
+                              ),
+                            },
+                            null,
+                            2,
+                          )}
+                        </code>
+                      </pre>
+                    </div>
+                  )}
+                </div>
+              );
+            })
+          )}
+        </div>
+
+        {/* Log Terminal Footer Status */}
+        <div className="flex items-center justify-between px-4 py-2 bg-surface-container-low text-outline font-mono text-xs border-t border-surface-container">
+          <div className="flex items-center gap-3">
+            <span>
+              Showing {visibleRecords.length} of {allRecords.length} matched events
+            </span>
+            <span className="h-3 w-px bg-surface-container-high" />
+            <span>Buffer: Healthy</span>
+          </div>
+
+          <div className="flex items-center gap-2">
+            <span className="w-1.5 h-1.5 rounded-full bg-secondary" />
+            <span className="text-on-surface">Auto-scroll: LIVE STREAM</span>
+          </div>
+        </div>
+      </main>
+    </div>
+  );
+};

otel-ui/src/components/Tabs.tsx

@@ -0,0 +1,513 @@
+import { useCallback, useState } from "react"
+import * as Tabs from "@radix-ui/react-tabs"
+import {
+  Activity,
+  AlertOctagon,
+  LineChart,
+  ScrollText,
+  AlertTriangle,
+  Info,
+  CheckCircle2,
+  ChevronDown,
+  ChevronUp,
+  TrendingUp,
+  Wrench,
+  Radar,
+  Globe,
+  GitBranch,
+  CloudCog,
+} from "lucide-react"
+import type { LucideIcon } from "lucide-react"
+import { MetricsVisualizer } from "./MetricsVisualizer"
+import TraceVisualization from "./TraceVisualization"
+import { IssuesVisualizer } from "./IssuesVisualizer"
+import { SetupPanel } from "./SetupPanel"
+import { SearchAndWatcher } from "./SearchAndWatcher"
+import { FrontendVisualizer } from "./FrontendVisualizer"
+import { ViewErrorBoundary } from "./ViewErrorBoundary"
+import { useObservabilityData } from "../hooks/useObservabilityData"
+import { useIssues } from "../hooks/useIssues"
+import { useStatsAvailability } from "../hooks/useStats"
+import { TrendsVisualizer } from "./TrendsVisualizer"
+import { authFetch } from "../lib/auth"
+import type { Issue, LogEntry, OtelAttribute } from "../lib/types"
+import { formatAttributeValue, getServicePalette, getServiceName } from "../lib/otel-utils"
+import { TelemetryFilters } from "./TelemetryFilters"
+import { PatternsVisualizer } from "./PatternsVisualizer"
+import { RolloutsVisualizer } from "./RolloutsVisualizer"
+
+
+type SeverityMeta = {
+  baseLabel: string
+  gradient: string
+  ring: string
+  icon: LucideIcon
+}
+
+const severityStyles: Record<string, SeverityMeta> = {
+  ERROR: {
+    baseLabel: "Error",
+    gradient: "from-rose-500 via-red-500 to-orange-400",
+    ring: "ring-rose-400/30",
+    icon: AlertTriangle,
+  },
+  WARN: {
+    baseLabel: "Warning",
+    gradient: "from-amber-400 via-orange-400 to-rose-400",
+    ring: "ring-amber-300/30",
+    icon: AlertTriangle,
+  },
+  INFO: {
+    baseLabel: "Info",
+    gradient: "from-sky-500 via-blue-500 to-violet-500",
+    ring: "ring-sky-400/30",
+    icon: Info,
+  },
+  DEBUG: {
+    baseLabel: "Debug",
+    gradient: "from-emerald-400 via-teal-400 to-cyan-400",
+    ring: "ring-emerald-300/30",
+    icon: CheckCircle2,
+  },
+}
+
+const fallbackSeverity: SeverityMeta = {
+  baseLabel: "Log",
+  gradient: "from-slate-600 via-slate-500 to-slate-400",
+  ring: "ring-white/20",
+  icon: Info,
+}
+
+const getSeverityPresentation = (
+  severityText?: string,
+  severityNumber?: number,
+) => {
+  if (severityText) {
+    const normalized = severityText.toUpperCase()
+    if (severityStyles[normalized]) {
+      const meta = severityStyles[normalized]
+      return { ...meta, display: meta.baseLabel }
+    }
+    return { ...fallbackSeverity, display: severityText }
+  }
+
+  if (severityNumber !== undefined) {
+    if (severityNumber >= 17) {
+      const meta = severityStyles.ERROR
+      return { ...meta, display: meta.baseLabel }
+    }
+    if (severityNumber >= 13) {
+      const meta = severityStyles.WARN
+      return { ...meta, display: meta.baseLabel }
+    }
+    if (severityNumber >= 9) {
+      const meta = severityStyles.INFO
+      return { ...meta, display: meta.baseLabel }
+    }
+    const meta = severityStyles.DEBUG
+    return { ...meta, display: meta.baseLabel }
+  }
+
+  return { ...fallbackSeverity, display: fallbackSeverity.baseLabel }
+}
+
+// tabItems moved inside component to access live data
+
+export const LogEntryView = ({ log }: { log: LogEntry }) => {
+  const { setActiveTraceId, setActiveSpanId, setActiveTab } = useObservabilityData()
+  const [expandedId, setExpandedId] = useState<string | null>(null)
+
+  const logRecords = (log.scopeLogs ?? []).flatMap((scopeLog) =>
+    (scopeLog.logRecords ?? []).map((record) => ({
+      ...record,
+      scope: scopeLog.scope?.name,
+      resourceAttributes: log.resource?.attributes ?? [],
+    })),
+  )
+
+  const getRecordServiceName = (recordAttributes: OtelAttribute[]) => {
+    return getServiceName(log.resource?.attributes ?? [], recordAttributes)
+  }
+
+  const navigateToSpan = (traceId: string, spanId?: string) => {
+    setActiveTraceId(traceId)
+    if (spanId) setActiveSpanId(spanId)
+    setActiveTab("tab1")
+  }
+
+  const formatTimestamp = (timeUnixNano: string) => {
+    const timestamp = (parseInt(timeUnixNano ?? "0") || 0) / 1e9
+    const date = new Date(timestamp * 1000)
+    return date.toLocaleTimeString([], { hour12: false, hour: '2-digit', minute: '2-digit', second: '2-digit', fractionalSecondDigits: 3 })
+  }
+
+  if (logRecords.length === 0) {
+    return null
+  }
+
+  return (
+    <div className="flex flex-col gap-1.5">
+      {logRecords.map((record) => {
+        const severity = getSeverityPresentation(
+          record.severityText,
+          record.severityNumber,
+        )
+        const bodyMessage = record.body?.stringValue ?? "(no body)"
+        const recordKey = record.uid
+        const isExpanded = expandedId === recordKey
+
+        return (
+          <div
+            key={recordKey}
+            className="group relative flex flex-col overflow-hidden rounded-xl border border-white/5 bg-slate-950/40 transition-all hover:border-white/10 hover:bg-slate-900/50"
+          >
+            {/* The Main Log Row */}
+            <div
+              onClick={() => setExpandedId(isExpanded ? null : recordKey)}
+              className="flex cursor-pointer flex-wrap items-center gap-x-4 gap-y-2 p-2.5 font-mono text-[12px]"
+            >
+              <div className={`absolute left-0 top-0 bottom-0 w-1 bg-gradient-to-b ${severity.gradient} opacity-40 transition-opacity group-hover:opacity-100`} />
+
+              {/* Timestamp */}
+              <span className="shrink-0 text-slate-500 whitespace-nowrap ml-1">
+                {formatTimestamp(record.timeUnixNano)}
+              </span>
+
+              {/* Service Badge */}
+                  {(() => {
+                    const sName = getRecordServiceName(record.attributes ?? [])
+                const palette = getServicePalette(sName)
+                return (
+                  <span className={`shrink-0 rounded px-1.5 py-0.5 text-[10px] font-bold uppercase tracking-wider ${palette.bg} ${palette.text} ${palette.border} border`}>
+                    {sName}
+                  </span>
+                )
+              })()}
+
+              {/* Severity Badge */}
+              <span className={`shrink-0 rounded px-1.5 py-0.5 text-[10px] font-bold uppercase tracking-wider bg-gradient-to-r ${severity.gradient} text-slate-950`}>
+                {severity.display}
+              </span>
+
+              {/* Log Message */}
+              <span className="flex-1 min-w-[300px] break-all text-slate-200">
+                {bodyMessage}
+              </span>
+
+              {/* Action Icons */}
+              <div className="flex items-center gap-2">
+                {record.traceId && (
+                  <button
+                    onClick={(e) => {
+                      e.stopPropagation()
+                      navigateToSpan(record.traceId!, record.spanId)
+                    }}
+                    className="group/trace flex items-center gap-1.5 rounded-full bg-sky-500/10 border border-sky-400/20 px-2.5 py-0.5 text-[10px] font-bold text-sky-400 hover:bg-sky-500/20 hover:border-sky-400/30 transition-all"
+                  >
+                    <Activity className="h-2.5 w-2.5 group-hover/trace:animate-pulse" />
+                    TRACE:{record.traceId.slice(0, 6)}
+                  </button>
+                )}
+                {isExpanded ? <ChevronUp className="h-3 w-3 text-slate-500" /> : <ChevronDown className="h-3 w-3 text-slate-500" />}
+              </div>
+            </div>
+
+            {/* Expansion Content */}
+            {isExpanded && (
+              <div className="border-t border-white/5 bg-black/40 p-4 text-[12px]">
+                <div className="grid gap-6 md:grid-cols-2">
+                  {(record.attributes ?? []).length > 0 && (
+                    <div className="space-y-2">
+                      <p className="text-[10px] font-bold uppercase tracking-widest text-slate-500">Attributes</p>
+                      <div className="space-y-1">
+                        {(record.attributes ?? []).map(attr => (
+                          <div key={attr.key} className="flex gap-2">
+                            <span className="text-sky-400 shrink-0">{attr.key}:</span>
+                            <span className="text-slate-300 break-all">{formatAttributeValue(attr.value)}</span>
+                          </div>
+                        ))}
+                      </div>
+                    </div>
+                  )}
+                  {(record.resourceAttributes ?? []).length > 0 && (
+                    <div className="space-y-2">
+                      <p className="text-[10px] font-bold uppercase tracking-widest text-slate-500">Resource</p>
+                      <div className="space-y-1">
+                        {(record.resourceAttributes ?? []).map(attr => (
+                          <div key={attr.key} className="flex gap-2">
+                            <span className="text-emerald-400 shrink-0">{attr.key}:</span>
+                            <span className="text-slate-400 break-all">{formatAttributeValue(attr.value)}</span>
+                          </div>
+                        ))}
+                      </div>
+                    </div>
+                  )}
+                </div>
+                <div className="mt-4 pt-4 border-t border-white/5 flex flex-wrap items-center justify-between gap-4">
+                  <div className="flex gap-4 text-[10px] text-slate-500">
+                    <span>Scope: {record.scope || "unknown"}</span>
+                    {record.spanId && <span>Span ID: {record.spanId}</span>}
+                    {record.traceId && <span>Trace ID: {record.traceId}</span>}
+                  </div>
+
+                  {record.traceId && (
+                    <button
+                      onClick={() => navigateToSpan(record.traceId!, record.spanId)}
+                      className="inline-flex items-center gap-2 rounded-xl bg-sky-500 px-4 py-2 text-[10px] font-black uppercase tracking-widest text-slate-950 transition-all hover:bg-sky-400 hover:scale-[1.02] active:scale-[0.98] shadow-lg shadow-sky-500/20"
+                    >
+                      <Activity className="h-3 w-3" />
+                      Jump to Span in Trace
+                    </button>
+                  )}
+                </div>
+              </div>
+            )}
+          </div>
+        )
+      })}
+    </div>
+  )
+}
+
+import { StructuredLogTerminal } from "./StructuredLogTerminal"
+
+const LogVisualizer = ({ logs }: { logs: LogEntry[] }) => {
+  return <StructuredLogTerminal logs={logs} />
+}
+
+export const TabsComponent = () => {
+  const { traces, logs, metrics, activeTab, setActiveTab } = useObservabilityData()
+  const issuesState = useIssues()
+  const [statusBusy, setStatusBusy] = useState(false)
+  // Trends tab only exists when the backend has ClickHouse-backed stats;
+  // null while the capability check is in flight.
+  const statsAvailable = useStatsAvailability()
+
+  // Triage action: PATCH the issue status, then refresh so counts/badges
+  // reflect the server state. Failures surface via the issues error banner.
+  const handleIssueStatusChange = useCallback(
+    async (issue: Issue, status: string) => {
+      setStatusBusy(true)
+      try {
+        const res = await authFetch(`/api/issues/${issue.id}`, {
+          method: "PATCH",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify({ status }),
+        })
+        if (!res.ok) throw new Error(`HTTP ${res.status}`)
+      } catch (err) {
+        console.error("Failed to update issue status:", err)
+      } finally {
+        await issuesState.refresh()
+        setStatusBusy(false)
+      }
+    },
+    [issuesState],
+  )
+
+  const numberFormatter = new Intl.NumberFormat("en-US", {
+    notation: "compact",
+    maximumFractionDigits: 1,
+  })
+
+  const frontendCount = logs.reduce((acc, log) => {
+    return (
+      acc +
+      (log.scopeLogs ?? []).reduce((sAcc, sl) => {
+        return (
+          sAcc +
+          (sl.logRecords ?? []).filter((r) => {
+            const attrs = r.attributes ?? []
+            return (
+              attrs.some((a) => a.key === "source" && a.value?.stringValue === "frontend") ||
+              sl.scope?.name === "otel-rum" ||
+              attrs.some((a) => a.key === "rum.vital") ||
+              attrs.some((a) => a.key === "browser.url")
+            )
+          }).length
+        )
+      }, 0)
+    )
+  }, 0)
+
+  const tabItems = [
+    {
+      value: "tab0",
+      label: "Issues",
+      description: "Deduplicated errors",
+      Icon: AlertOctagon,
+      count: issuesState.issues.length
+    },
+    {
+      value: "tab8",
+      label: "Patterns",
+      description: "Deterministic similarity review",
+      Icon: GitBranch,
+      count: undefined as number | undefined,
+    },
+    {
+      value: "tab9",
+      label: "Rollouts",
+      description: "Cluster deploy correlation",
+      Icon: CloudCog,
+      count: undefined as number | undefined,
+    },
+    {
+      value: "tab1",
+      label: "Traces",
+      description: "Span timelines & focus",
+      Icon: Activity,
+      count: traces.length
+    },
+    {
+      value: "tab2",
+      label: "Metrics",
+      description: "Performance trends",
+      Icon: LineChart,
+      count: metrics.length
+    },
+    {
+      value: "tab3",
+      label: "Logs",
+      description: "System log streams",
+      Icon: ScrollText,
+      count: logs.length
+    },
+    {
+      value: "tab7",
+      label: "Frontend & RUM",
+      description: "Web Vitals & browser errors",
+      Icon: Globe,
+      count: frontendCount,
+    },
+    // Historical trends need the ClickHouse-backed stats endpoints.
+    ...(statsAvailable
+      ? [
+          {
+            value: "tab5",
+            label: "Trends",
+            description: "History graphs & release comparison",
+            Icon: TrendingUp,
+            count: undefined as number | undefined,
+          },
+        ]
+      : []),
+    {
+      value: "tab6",
+      label: "Archive & Trap",
+      description: "Persistent storage search & signal trap",
+      Icon: Radar,
+      count: undefined,
+    },
+    {
+      value: "tab4",
+      label: "Setup",
+      description: "Instrument a new service",
+      Icon: Wrench,
+      count: undefined
+    },
+  ]
+
+  return (
+    <div className="flex flex-col gap-6">
+      <TelemetryFilters />
+      <Tabs.Root
+        value={activeTab}
+        onValueChange={setActiveTab}
+        className="flex flex-col gap-4"
+      >
+        <Tabs.List
+          className="flex gap-8 border-b border-white/10 pb-2 mb-2 w-full overflow-x-auto"
+          aria-label="Telemetry explorer views"
+        >
+          {tabItems.map(({ value, label, Icon, count }) => (
+            <Tabs.Trigger
+              key={value}
+              value={value}
+              id={`tab-trigger-${value}`}
+              aria-label={`${label} tab`}
+              className="group relative flex items-center gap-2.5 pb-2 text-sm font-semibold text-slate-400 outline-none transition-colors hover:text-slate-200 data-[state=active]:text-white"
+            >
+              <div className="absolute -bottom-[2px] left-0 right-0 h-0.5 bg-sky-400 opacity-0 transition-opacity group-data-[state=active]:opacity-100" />
+              <Icon className="h-4 w-4 text-slate-500 group-data-[state=active]:text-sky-400" />
+              <span aria-hidden="true">{label}</span>
+              {count !== undefined && (
+                <span className="inline-flex rounded-full bg-slate-800 px-2 py-0.5 text-[10px] font-bold text-slate-300 group-data-[state=active]:bg-sky-500/20 group-data-[state=active]:text-sky-300">
+                  {numberFormatter.format(count)}
+                </span>
+              )}
+            </Tabs.Trigger>
+          ))}
+        </Tabs.List>
+        <Tabs.Content
+          className="outline-none py-2"
+          value="tab0"
+        >
+          {activeTab === "tab0" && (
+            <ViewErrorBoundary>
+              <IssuesVisualizer
+                issues={issuesState.issues}
+                loading={issuesState.loading}
+                error={issuesState.error}
+                refresh={issuesState.refresh}
+                onStatusChange={handleIssueStatusChange}
+                statusBusy={statusBusy}
+              />
+            </ViewErrorBoundary>
+          )}
+        </Tabs.Content>
+        <Tabs.Content
+          className="outline-none py-2"
+          value="tab1"
+        >
+          {activeTab === "tab1" && <ViewErrorBoundary><TraceVisualization traces={traces} /></ViewErrorBoundary>}
+        </Tabs.Content>
+        <Tabs.Content className="outline-none py-2" value="tab8">
+          {activeTab === "tab8" && <ViewErrorBoundary><PatternsVisualizer /></ViewErrorBoundary>}
+        </Tabs.Content>
+        <Tabs.Content className="outline-none py-2" value="tab9">
+          {activeTab === "tab9" && <ViewErrorBoundary><RolloutsVisualizer /></ViewErrorBoundary>}
+        </Tabs.Content>
+        <Tabs.Content
+          className="outline-none py-2"
+          value="tab2"
+        >
+          {activeTab === "tab2" && <ViewErrorBoundary><MetricsVisualizer /></ViewErrorBoundary>}
+        </Tabs.Content>
+        <Tabs.Content
+          className="outline-none py-2"
+          value="tab3"
+        >
+          {activeTab === "tab3" && <ViewErrorBoundary><LogVisualizer logs={logs} /></ViewErrorBoundary>}
+        </Tabs.Content>
+        <Tabs.Content
+          className="outline-none py-2"
+          value="tab7"
+        >
+          {activeTab === "tab7" && <ViewErrorBoundary><FrontendVisualizer /></ViewErrorBoundary>}
+        </Tabs.Content>
+        <Tabs.Content
+          className="outline-none py-2"
+          value="tab4"
+        >
+          {activeTab === "tab4" && <ViewErrorBoundary><SetupPanel /></ViewErrorBoundary>}
+        </Tabs.Content>
+        {statsAvailable && (
+          <Tabs.Content
+            className="outline-none py-2"
+            value="tab5"
+          >
+            {activeTab === "tab5" && <ViewErrorBoundary><TrendsVisualizer /></ViewErrorBoundary>}
+          </Tabs.Content>
+        )}
+        <Tabs.Content
+          className="outline-none py-2"
+          value="tab6"
+        >
+          {activeTab === "tab6" && <ViewErrorBoundary><SearchAndWatcher /></ViewErrorBoundary>}
+        </Tabs.Content>
+      </Tabs.Root>
+    </div>
+  )
+
+}

otel-ui/src/components/TelemetryFilters.tsx

@@ -0,0 +1,124 @@
+import { X, Activity, AlertTriangle } from "lucide-react"
+import { useObservabilityData } from "../hooks/useObservabilityData"
+import { getServicePalette } from "../lib/otel-utils"
+
+export const TelemetryFilters = () => {
+  const {
+    selectedServices,
+    toggleService,
+    excludedAttributes,
+    removeExcludedValue,
+    clearAttributeKey,
+    uniqueServices,
+    filtersExpanded: isExpanded,
+  } = useObservabilityData()
+
+  const hasExcludedKeys = Object.keys(excludedAttributes).length > 0
+
+  if (!isExpanded) return null
+
+  return (
+    <div className="border-b border-white/10 pb-8 mb-6 animate-in fade-in slide-in-from-top-2 duration-300">
+      <div className="grid gap-8 lg:grid-cols-2">
+          {/* Service Multi-Select */}
+          <div className="space-y-4">
+            <div className="flex items-center gap-2">
+              <Activity className="h-3 w-3 text-slate-500" />
+              <span className="text-[10px] font-black uppercase tracking-[0.2em] text-slate-500">
+                Service Focus
+              </span>
+            </div>
+            <div className="flex flex-wrap gap-2.5">
+              {uniqueServices.map((service) => {
+                const isActive = selectedServices.has(service)
+                const palette = getServicePalette(service)
+                return (
+                  <button
+                    key={service}
+                    type="button"
+                    onClick={() => toggleService(service)}
+                    className={`group relative inline-flex items-center gap-3 rounded-2xl border px-5 py-2.5 text-[10px] font-bold uppercase tracking-[0.2em] transition-all duration-300 ${isActive
+                        ? `border-transparent bg-gradient-to-br ${palette.gradient} text-slate-950 shadow-lg shadow-sky-500/20`
+                        : "border-white/10 bg-slate-900/40 text-slate-400 hover:border-white/20 hover:text-white"
+                      }`}
+                  >
+                    {!isActive && (
+                      <span
+                        className={`h-2 w-2 rounded-full bg-gradient-to-r ${palette.gradient} opacity-50 block`}
+                      />
+                    )}
+                    {service}
+                    {isActive && (
+                      <span className="ml-1 flex h-4 w-4 items-center justify-center rounded-full bg-black/10">
+                        <X className="h-3 w-3" />
+                      </span>
+                    )}
+                  </button>
+                )
+              })}
+              {uniqueServices.length === 0 && (
+                <span className="text-[11px] text-slate-600 italic font-medium px-1 underline decoration-white/5 underline-offset-4">
+                  Listening for service signals...
+                </span>
+              )}
+            </div>
+          </div>
+
+          {/* Attribute Exclusions */}
+          <div className="space-y-4">
+            <div className="flex items-center gap-2">
+              <AlertTriangle className="h-3 w-3 text-slate-500" />
+              <span className="text-[10px] font-black uppercase tracking-[0.2em] text-slate-500">
+                Active Noise Suppression
+              </span>
+            </div>
+            <div className="flex flex-wrap gap-3">
+              {hasExcludedKeys ? (
+                Object.entries(excludedAttributes).map(([key, valueSet]) => (
+                  <div
+                    key={key}
+                    className="flex flex-col gap-2 rounded-2xl border border-white/5 bg-slate-900/40 p-3 transition-all hover:border-white/10"
+                  >
+                    <div className="flex items-center justify-between mb-0.5">
+                      <span className="text-[10px] font-bold text-sky-400 font-mono">
+                        {key}
+                      </span>
+                      <button
+                        type="button"
+                        onClick={() => clearAttributeKey(key)}
+                        className="text-[9px] font-black uppercase tracking-widest text-slate-600 hover:text-white transition-colors ml-4"
+                      >
+                        Remove All
+                      </button>
+                    </div>
+                    <div className="flex flex-wrap gap-1.5">
+                      {Array.from(valueSet).map((value) => (
+                        <button
+                          key={value}
+                          type="button"
+                          onClick={() => removeExcludedValue(key, value)}
+                          className="group inline-flex items-center gap-2 rounded-full border border-white/5 bg-slate-950/80 px-2.5 py-0.5 text-[10px] text-slate-300 transition-all hover:border-rose-500/50 hover:text-rose-400"
+                        >
+                          <span className="max-w-[10rem] truncate">{value}</span>
+                          <X className="h-2.5 w-2.5 text-slate-600 group-hover:text-rose-500" />
+                        </button>
+                      ))}
+                    </div>
+                  </div>
+                ))
+              ) : (
+                <div className="flex flex-col items-center justify-center w-full py-4 border border-dashed border-white/5 rounded-3xl bg-slate-900/20">
+                  <p className="text-[11px] text-slate-600 italic font-medium">
+                    No active attribute exclusions.
+                  </p>
+                  <p className="text-[9px] text-slate-700 uppercase tracking-widest mt-1">
+                    Click attributes in details to suppress chatty items
+                  </p>
+                </div>
+              )}
+            </div>
+          </div>
+        </div>
+    </div>
+  )
+}

otel-ui/src/components/TopHeader.tsx

@@ -0,0 +1,232 @@
+import React, { useRef, useEffect } from "react";
+import { useObservabilityData } from "../hooks/useObservabilityData";
+import { useIssues } from "../hooks/useIssues";
+import { currentUser, login, logout } from "../lib/auth";
+
+export const TopHeader: React.FC = () => {
+  const user = currentUser();
+  const {
+    searchQuery,
+    setSearchQuery,
+    showErrorsOnly,
+    setShowErrorsOnly,
+    resetFilters,
+    setActiveTab,
+    timeWindow,
+    setTimeWindow,
+    isStreaming,
+    setIsStreaming,
+    bufferedCount,
+    flushBufferedStream,
+    minDurationMs,
+    setMinDurationMs,
+  } = useObservabilityData();
+
+  const { issues } = useIssues();
+  const searchInputRef = useRef<HTMLInputElement>(null);
+
+  const openIssuesCount = React.useMemo(() => {
+    return issues.filter((i) => !i.status || i.status === "open").length;
+  }, [issues]);
+
+  // ⌘K / Ctrl+K keyboard shortcut to focus search
+  useEffect(() => {
+    const handleKeyDown = (e: KeyboardEvent) => {
+      if ((e.metaKey || e.ctrlKey) && e.key === "k") {
+        e.preventDefault();
+        searchInputRef.current?.focus();
+      }
+    };
+    window.addEventListener("keydown", handleKeyDown);
+    return () => window.removeEventListener("keydown", handleKeyDown);
+  }, []);
+
+  return (
+    <header className="fixed top-0 left-72 right-0 h-14 bg-surface-container-lowest/90 backdrop-blur-xl z-40 flex items-center justify-between px-6 shadow-[0_1px_8px_rgba(0,0,0,0.04)] border-b border-surface-container/60 font-body">
+      {/* Search and Time Range */}
+      <div className="flex items-center gap-3 flex-1 max-w-3xl">
+        {/* Global Search Bar */}
+        <div className="relative flex-1">
+          <span className="material-symbols-outlined absolute left-2.5 top-1/2 -translate-y-1/2 text-outline text-[18px] pointer-events-none">
+            search
+          </span>
+          <input
+            ref={searchInputRef}
+            type="text"
+            value={searchQuery}
+            onChange={(e) => setSearchQuery(e.target.value)}
+            placeholder="⌘K search attributes, services, traceId, spanId, or log payload..."
+            className="w-full h-8 pl-8 pr-16 rounded-lg bg-surface-container-low text-on-surface font-mono text-xs placeholder:text-outline focus:outline-none focus:bg-surface-container transition-colors border border-surface-container/40"
+          />
+          <div className="absolute right-2.5 top-1/2 -translate-y-1/2 flex items-center gap-0.5 pointer-events-none select-none">
+            <kbd className="font-mono text-[10px] text-outline px-1 rounded bg-surface-container-high border border-white/5">
+              ⌘
+            </kbd>
+            <kbd className="font-mono text-[10px] text-outline px-1 rounded bg-surface-container-high border border-white/5">
+              K
+            </kbd>
+          </div>
+        </div>
+
+        {/* Real Time Window Selector */}
+        <div className="flex items-center bg-surface-container-low rounded-lg p-0.5 border border-surface-container/40 select-none">
+          <span className="material-symbols-outlined text-outline text-[14px] ml-1 mr-0.5">
+            schedule
+          </span>
+          {(["15m", "1h", "6h", "24h", "all"] as const).map((w) => (
+            <button
+              key={w}
+              type="button"
+              onClick={() => setTimeWindow(w)}
+              className={`px-2 py-0.5 font-mono text-[11px] rounded transition-colors ${
+                timeWindow === w
+                  ? "bg-surface-container text-primary-container font-bold shadow-sm"
+                  : "text-outline hover:text-on-surface"
+              }`}
+            >
+              {w === "all" ? "All" : w}
+            </button>
+          ))}
+        </div>
+      </div>
+
+      {/* Action Buttons & Streaming Controls */}
+      <div className="flex items-center gap-3">
+        {/* Real Streaming Pause/Resume Button */}
+        <button
+          type="button"
+          onClick={() => {
+            if (isStreaming) {
+              setIsStreaming(false);
+            } else {
+              setIsStreaming(true);
+              flushBufferedStream();
+            }
+          }}
+          className={`flex items-center gap-1.5 px-2.5 py-1 rounded-lg font-mono text-[11px] font-semibold transition-colors border ${
+            isStreaming
+              ? "bg-surface-container-low text-secondary border-surface-container hover:bg-surface-container"
+              : "bg-amber-500/20 text-amber-300 border-amber-500/40 animate-pulse"
+          }`}
+          title={isStreaming ? "Pause stream to freeze buffer for inspection" : "Resume live telemetry tail"}
+        >
+          <span
+            className={`w-2 h-2 rounded-full ${
+              isStreaming ? "bg-secondary animate-pulse" : "bg-amber-400"
+            }`}
+          />
+          <span>
+            {isStreaming
+              ? "Live Tail"
+              : `Paused ${bufferedCount > 0 ? `(${bufferedCount})` : ""}`}
+          </span>
+        </button>
+
+        {/* All vs Errors Toggle */}
+        <div className="flex rounded-lg bg-surface-container-low p-0.5 border border-surface-container/40">
+          <button
+            type="button"
+            onClick={() => setShowErrorsOnly(false)}
+            className={`px-2.5 py-1 text-[10px] font-mono font-bold uppercase tracking-wider rounded transition-colors ${
+              !showErrorsOnly
+                ? "bg-surface-container-high text-primary font-bold shadow-sm"
+                : "text-outline hover:text-on-surface"
+            }`}
+          >
+            All
+          </button>
+          <button
+            type="button"
+            onClick={() => setShowErrorsOnly(true)}
+            className={`flex items-center gap-1 px-2.5 py-1 text-[10px] font-mono font-bold uppercase tracking-wider rounded transition-colors ${
+              showErrorsOnly
+                ? "bg-error-container text-on-error-container font-bold shadow-sm"
+                : "text-outline hover:text-error"
+            }`}
+          >
+            <span className="material-symbols-outlined text-[12px]">warning</span>
+            Errors
+          </button>
+        </div>
+
+        {/* Slow Spans Toggle (>500ms) */}
+        <button
+          type="button"
+          onClick={() => setMinDurationMs(minDurationMs > 0 ? 0 : 500)}
+          className={`flex items-center gap-1 px-2 py-1 text-[10px] font-mono font-bold uppercase tracking-wider rounded-lg border transition-colors ${
+            minDurationMs > 0
+              ? "bg-amber-500/20 text-amber-300 border-amber-500/40 font-bold"
+              : "bg-surface-container-low text-outline hover:text-on-surface border-surface-container/40"
+          }`}
+          title="Filter to slow spans (>500ms latency)"
+        >
+          <span className="material-symbols-outlined text-[13px]">speed</span>
+          <span>&gt;500ms</span>
+        </button>
+
+        <button
+          type="button"
+          onClick={() => {
+            resetFilters();
+            setMinDurationMs(0);
+          }}
+          className="px-2 py-1 text-[10px] font-mono font-bold uppercase tracking-wider text-outline hover:text-on-surface rounded-lg bg-surface-container-low hover:bg-surface-container border border-surface-container/40 transition-colors"
+          title="Reset active filters"
+        >
+          Reset
+        </button>
+
+        <div className="h-4 w-px bg-surface-container-high" />
+
+        {/* Active Issues / Incident Counter Button */}
+        <button
+          type="button"
+          onClick={() => setActiveTab("tab0")}
+          aria-label="Active issues"
+          className={`relative flex items-center gap-1.5 px-2.5 py-1 rounded-lg font-mono text-xs font-bold transition-colors border ${
+            openIssuesCount > 0
+              ? "bg-error-container text-on-error-container border-error/30 animate-pulse"
+              : "bg-surface-container-low text-outline hover:text-on-surface border-surface-container/40"
+          }`}
+          title="Open Issues & Anomalies"
+        >
+          <span className="material-symbols-outlined text-[16px]">notifications_active</span>
+          <span>{openIssuesCount}</span>
+        </button>
+
+        {/* User Account / Auth */}
+        {user ? (
+          <div className="flex items-center gap-2 pl-1 select-none">
+            <div
+              className="w-8 h-8 rounded-full bg-primary/15 text-primary flex items-center justify-center font-bold text-xs border border-primary/20 shadow-sm"
+              title={user.email || user.sub}
+            >
+              {(user.name || user.email || user.sub).charAt(0).toUpperCase()}
+            </div>
+            <div className="hidden xl:flex flex-col text-left">
+              <span className="font-body text-xs text-on-surface leading-tight font-medium truncate max-w-[120px]">
+                {user.name || user.email?.split("@")[0] || user.sub}
+              </span>
+              <button
+                type="button"
+                onClick={() => logout()}
+                className="font-mono text-[10px] text-outline hover:text-error text-left leading-none transition-colors"
+              >
+                Sign out
+              </button>
+            </div>
+          </div>
+        ) : (
+          <button
+            type="button"
+            onClick={() => login()}
+            className="flex items-center gap-1 px-2.5 py-1 rounded-lg bg-primary/10 text-primary hover:bg-primary/20 font-mono text-xs transition-colors border border-primary/20 select-none"
+          >
+            <span className="material-symbols-outlined text-[15px]">login</span>
+            <span>Sign in</span>
+          </button>
+        )}
+      </div>
+    </header>
+  );
+};

otel-ui/src/components/TrendsVisualizer.tsx

@@ -0,0 +1,323 @@
+import { useMemo, useState } from "react"
+import {
+  Area,
+  AreaChart,
+  CartesianGrid,
+  Legend,
+  Line,
+  LineChart,
+  ResponsiveContainer,
+  Tooltip,
+  XAxis,
+  YAxis,
+} from "recharts"
+import { AlertOctagon, GitCompareArrows, RefreshCw, Server } from "lucide-react"
+import { useStats } from "../hooks/useStats"
+import type { StatsPoint } from "../lib/types"
+
+const compactFormatter = new Intl.NumberFormat("en-US", {
+  notation: "compact",
+  maximumFractionDigits: 1,
+})
+
+// Stable palette for release lines; cycles when there are many releases.
+const RELEASE_COLORS = ["#38bdf8", "#f472b6", "#34d399", "#fbbf24", "#a78bfa", "#fb7185"]
+
+const label = (bucket: string, granularity: "hour" | "day") => {
+  const d = new Date(bucket.replace(" ", "T") + "Z")
+  if (Number.isNaN(d.getTime())) return bucket
+  return granularity === "day"
+    ? d.toLocaleDateString([], { month: "short", day: "numeric" })
+    : d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", hour12: false })
+}
+
+const ChartTooltip = ({ active, payload, granularity }: any) => {
+  if (!active || !payload?.length) return null
+  const p = payload[0].payload
+  const d = new Date(String(p.bucket).replace(" ", "T") + "Z")
+  return (
+    <div className="rounded-xl border border-white/10 bg-slate-950/95 p-3 text-xs shadow-2xl backdrop-blur-md">
+      <div className="mb-1 font-mono text-[10px] text-slate-400">
+        {d.toLocaleString([], {
+          month: "short",
+          day: "numeric",
+          hour: granularity === "hour" ? "2-digit" : undefined,
+          minute: granularity === "hour" ? "2-digit" : undefined,
+          hour12: false,
+        })}
+      </div>
+      {payload.map((entry: any) => (
+        <div key={entry.name} className="flex items-center justify-between gap-4">
+          <span className="flex items-center gap-1.5 text-slate-300">
+            <span className="h-1.5 w-1.5 rounded-full" style={{ background: entry.color }} />
+            {entry.name}
+          </span>
+          <span className="font-mono font-bold text-white">{entry.value}</span>
+        </div>
+      ))}
+    </div>
+  )
+}
+
+const rangeOptions = [24, 72, 168, 720] // 1d, 3d, 7d, 30d
+
+export const TrendsVisualizer = () => {
+  const [hours, setHours] = useState(72)
+  const [service, setService] = useState("")
+  const { errorPoints, granularity, releaseSeries, services, loading, error, refresh } = useStats({
+    hours,
+    service,
+  })
+
+  const errorData = useMemo(
+    () =>
+      errorPoints.map((p: StatsPoint) => ({
+        bucket: p.bucket,
+        total: p.total,
+        errors: p.errors,
+      })),
+    [errorPoints],
+  )
+
+  const releases = useMemo(() => Object.keys(releaseSeries).sort(), [releaseSeries])
+  const releaseData = useMemo(() => {
+    // Recharts wants rows, not per-series arrays: bucket -> {release: errors}.
+    const byBucket: Record<string, Record<string, number | string>> = {}
+    releases.forEach((rel) => {
+      releaseSeries[rel].forEach((p) => {
+        byBucket[p.bucket] = byBucket[p.bucket] || { bucket: p.bucket }
+        byBucket[p.bucket][rel] = p.errors
+      })
+    })
+    return Object.values(byBucket).sort((a, b) =>
+      String(a.bucket).localeCompare(String(b.bucket)),
+    )
+  }, [releaseSeries, releases])
+
+  const totals = useMemo(() => {
+    let total = 0
+    let errors = 0
+    errorPoints.forEach((p) => {
+      total += p.total
+      errors += p.errors
+    })
+    return { total, errors }
+  }, [errorPoints])
+
+  if (loading && errorPoints.length === 0) {
+    return (
+      <div className="flex min-h-[240px] flex-col items-center justify-center gap-3 text-slate-400">
+        <RefreshCw className="h-6 w-6 animate-spin opacity-40" />
+        <p className="text-sm">Loading trends…</p>
+      </div>
+    )
+  }
+
+  return (
+    <div className="space-y-6">
+      {/* Toolbar */}
+      <div className="sticky top-0 z-20 flex flex-wrap items-center justify-between gap-3 border-b border-white/5 bg-slate-950/80 py-2 backdrop-blur">
+        <div className="flex flex-wrap items-center gap-3">
+          <span className="inline-flex items-center gap-2 text-[10px] font-black uppercase tracking-[0.2em] text-slate-500">
+            <GitCompareArrows className="h-3 w-3 text-sky-400" />
+            Historical Trends
+          </span>
+          <div className="flex items-center gap-1 rounded-lg border border-white/5 bg-white/5 p-0.5">
+            {rangeOptions.map((h) => (
+              <button
+                key={h}
+                onClick={() => setHours(h)}
+                className={`rounded px-2 py-0.5 text-[10px] font-bold uppercase tracking-wider transition-all ${
+                  hours === h
+                    ? "bg-sky-500/20 text-sky-300 ring-1 ring-sky-500/30"
+                    : "text-slate-500 hover:text-slate-300"
+                }`}
+              >
+                {h < 24 ? `${h}h` : `${h / 24}d`}
+              </button>
+            ))}
+          </div>
+          {services.length > 0 && (
+            <select
+              value={service}
+              onChange={(e) => setService(e.target.value)}
+              className="rounded-lg border border-white/10 bg-slate-900/60 px-2 py-1 text-[10px] font-bold uppercase tracking-wider text-slate-300 outline-none"
+              aria-label="Filter by service"
+            >
+              <option value="">All services</option>
+              {services.map((s) => (
+                <option key={s} value={s}>
+                  {s}
+                </option>
+              ))}
+            </select>
+          )}
+        </div>
+        <div className="flex items-center gap-3">
+          <span className="font-mono text-[10px] text-slate-500">
+            {compactFormatter.format(totals.total)} entries ·{" "}
+            <span className={totals.errors > 0 ? "text-rose-400" : "text-emerald-400"}>
+              {compactFormatter.format(totals.errors)} errors
+            </span>
+          </span>
+          <button
+            onClick={() => refresh()}
+            className="flex items-center gap-1.5 rounded-lg border border-white/10 px-2.5 py-1 text-[10px] font-bold uppercase tracking-wider text-slate-400 transition-all hover:border-white/20 hover:text-white"
+          >
+            <RefreshCw className={`h-3 w-3 ${loading ? "animate-spin" : ""}`} />
+            Refresh
+          </button>
+        </div>
+      </div>
+
+      {error && (
+        <div className="flex items-center gap-2 rounded-xl border border-rose-500/20 bg-rose-500/5 p-4 text-sm text-rose-300">
+          <AlertOctagon className="h-4 w-4 shrink-0" />
+          {error}
+        </div>
+      )}
+
+      {/* Error trend */}
+      <section className="rounded-2xl border border-white/5 bg-slate-950/40 p-4">
+        <header className="mb-3 flex items-center justify-between">
+          <h3 className="flex items-center gap-2 text-xs font-bold uppercase tracking-[0.2em] text-slate-400">
+            <AlertOctagon className="h-3.5 w-3.5 text-rose-400" />
+            Entries &amp; errors over time
+          </h3>
+          <span className="text-[10px] uppercase tracking-wider text-slate-600">
+            per {granularity}
+          </span>
+        </header>
+        {errorData.length === 0 ? (
+          <p className="py-12 text-center text-sm text-slate-500">
+            No data in this range yet.
+          </p>
+        ) : (
+          <div className="h-64 w-full">
+            <ResponsiveContainer width="100%" height="100%">
+              <AreaChart data={errorData} margin={{ top: 5, right: 10, left: 0, bottom: 0 }}>
+                <defs>
+                  <linearGradient id="trendFill" x1="0" y1="0" x2="0" y2="100%">
+                    <stop offset="5%" stopColor="#38bdf8" stopOpacity={0.35} />
+                    <stop offset="95%" stopColor="#38bdf8" stopOpacity={0} />
+                  </linearGradient>
+                  <linearGradient id="trendErrFill" x1="0" y1="0" x2="0" y2="100%">
+                    <stop offset="5%" stopColor="#fb7185" stopOpacity={0.45} />
+                    <stop offset="95%" stopColor="#fb7185" stopOpacity={0} />
+                  </linearGradient>
+                </defs>
+                <CartesianGrid strokeDasharray="3 3" vertical={false} stroke="rgba(255,255,255,0.05)" />
+                <XAxis
+                  dataKey="bucket"
+                  tickFormatter={(b) => label(b, granularity)}
+                  stroke="#475569"
+                  fontSize={10}
+                  tickLine={false}
+                  axisLine={false}
+                  minTickGap={40}
+                />
+                <YAxis
+                  stroke="#475569"
+                  fontSize={10}
+                  tickLine={false}
+                  axisLine={false}
+                  width={36}
+                  tickFormatter={(v) => compactFormatter.format(v)}
+                />
+                <Tooltip content={<ChartTooltip granularity={granularity} />} />
+                <Legend
+                  wrapperStyle={{ fontSize: 10, textTransform: "uppercase", letterSpacing: "0.1em" }}
+                />
+                <Area
+                  type="monotone"
+                  dataKey="total"
+                  name="Total"
+                  stroke="#38bdf8"
+                  strokeWidth={2}
+                  fill="url(#trendFill)"
+                  isAnimationActive={false}
+                />
+                <Area
+                  type="monotone"
+                  dataKey="errors"
+                  name="Errors"
+                  stroke="#fb7185"
+                  strokeWidth={2}
+                  fill="url(#trendErrFill)"
+                  isAnimationActive={false}
+                />
+              </AreaChart>
+            </ResponsiveContainer>
+          </div>
+        )}
+      </section>
+
+      {/* Release comparison */}
+      <section className="rounded-2xl border border-white/5 bg-slate-950/40 p-4">
+        <header className="mb-3 flex items-center justify-between">
+          <h3 className="flex items-center gap-2 text-xs font-bold uppercase tracking-[0.2em] text-slate-400">
+            <GitCompareArrows className="h-3.5 w-3.5 text-sky-400" />
+            Errors by release
+          </h3>
+          <span className="text-[10px] uppercase tracking-wider text-slate-600">
+            {releases.length} release{releases.length === 1 ? "" : "s"} · overlay
+          </span>
+        </header>
+        {releaseData.length === 0 || releases.length === 0 ? (
+          <p className="py-12 text-center text-sm text-slate-500">
+            No release-tagged telemetry in this range. Have services send{" "}
+            <code className="rounded bg-white/5 px-1 font-mono text-[10px]">service.release</code>{" "}
+            to compare deploys here.
+          </p>
+        ) : (
+          <div className="h-64 w-full">
+            <ResponsiveContainer width="100%" height="100%">
+              <LineChart data={releaseData} margin={{ top: 5, right: 10, left: 0, bottom: 0 }}>
+                <CartesianGrid strokeDasharray="3 3" vertical={false} stroke="rgba(255,255,255,0.05)" />
+                <XAxis
+                  dataKey="bucket"
+                  tickFormatter={(b) => label(b, "hour")}
+                  stroke="#475569"
+                  fontSize={10}
+                  tickLine={false}
+                  axisLine={false}
+                  minTickGap={40}
+                />
+                <YAxis
+                  stroke="#475569"
+                  fontSize={10}
+                  tickLine={false}
+                  axisLine={false}
+                  width={36}
+                  tickFormatter={(v) => compactFormatter.format(v)}
+                />
+                <Tooltip content={<ChartTooltip granularity="hour" />} />
+                <Legend
+                  wrapperStyle={{ fontSize: 10, textTransform: "uppercase", letterSpacing: "0.1em" }}
+                />
+                {releases.map((rel, i) => (
+                  <Line
+                    key={rel}
+                    type="monotone"
+                    dataKey={rel}
+                    stroke={RELEASE_COLORS[i % RELEASE_COLORS.length]}
+                    strokeWidth={2}
+                    dot={false}
+                    isAnimationActive={false}
+                  />
+                ))}
+              </LineChart>
+            </ResponsiveContainer>
+          </div>
+        )}
+      </section>
+
+      <p className="flex items-center justify-center gap-1.5 pb-2 text-center text-[10px] text-slate-600">
+        <Server className="h-3 w-3" />
+        Served from the ClickHouse hourly rollup ({service || "all services"}, last {hours}h)
+      </p>
+    </div>
+  )
+}
+
+export default TrendsVisualizer

otel-ui/src/components/ViewErrorBoundary.tsx

@@ -0,0 +1,62 @@
+import { Component, type ErrorInfo, type ReactNode } from "react"
+import { AlertTriangle, RotateCcw } from "lucide-react"
+
+interface Props {
+  children: ReactNode
+}
+
+interface State {
+  error: Error | null
+}
+
+// Keeps a rendering bug in one telemetry view from blanking the whole app.
+export class ViewErrorBoundary extends Component<Props, State> {
+  state: State = { error: null }
+
+  static getDerivedStateFromError(error: Error): State {
+    return { error }
+  }
+
+  componentDidCatch(error: Error, info: ErrorInfo) {
+    console.error("View crashed:", error, info.componentStack)
+    const rum = (window as Window & {
+      OtelRUM?: {
+        last?: {
+          captureException: (error: Error, extra?: Record<string, unknown>) => void
+        }
+      }
+    }).OtelRUM?.last
+    rum?.captureException(error, {
+      "exception.type": "ReactErrorBoundary",
+      "react.component_stack": info.componentStack,
+    })
+  }
+
+  render() {
+    if (this.state.error) {
+      return (
+        <div className="flex min-h-[240px] flex-col items-center justify-center gap-4 rounded-3xl border border-rose-500/20 bg-rose-500/5 p-12 text-center">
+          <AlertTriangle className="h-10 w-10 text-rose-400" />
+          <div>
+            <p className="text-lg font-semibold text-white">
+              This view hit an unexpected error
+            </p>
+            <p className="mt-1 max-w-md truncate font-mono text-xs text-rose-300/80">
+              {this.state.error.message}
+            </p>
+          </div>
+          <button
+            type="button"
+            onClick={() => this.setState({ error: null })}
+            className="inline-flex items-center gap-2 rounded-xl border border-rose-500/30 bg-rose-500/10 px-4 py-2 text-[10px] font-black uppercase tracking-[0.2em] text-rose-300 transition-all hover:bg-rose-500/20"
+          >
+            <RotateCcw className="h-3 w-3" />
+            Retry
+          </button>
+        </div>
+      )
+    }
+
+    return this.props.children
+  }
+}

otel-ui/src/components/WaveformHistogram.tsx

@@ -0,0 +1,385 @@
+import { useMemo, useState } from "react";
+import { useObservabilityData } from "../hooks/useObservabilityData";
+import { isSpanError } from "../lib/otel-utils";
+
+export const WaveformHistogram = () => {
+  const {
+    logs,
+    traces,
+    unfocusedLogs,
+    unfocusedTraces,
+    selectedTimeRange,
+    setSelectedTimeRange,
+    setActiveTab,
+  } = useObservabilityData();
+
+  // Use unfocused data so that waveform buckets remain stable across the window
+  const sourceLogs = unfocusedLogs || logs;
+  const sourceTraces = unfocusedTraces || traces;
+
+  const { buckets, hasErrorSurge, surgePercent, timeLabels, totalCount } = useMemo(() => {
+    const numBuckets = 24;
+    const bucketStats = Array.from({ length: numBuckets }, () => ({
+      error: 0,
+      warn: 0,
+      info: 0,
+      total: 0,
+      traceCount: 0,
+      logCount: 0,
+    }));
+
+    // Collect timestamps and severity levels from real telemetry items
+    const items: Array<{ time: number; level: "error" | "warn" | "info"; isTrace?: boolean }> = [];
+
+    sourceLogs.forEach((log) => {
+      (log.scopeLogs ?? []).forEach((sl) => {
+        (sl.logRecords ?? []).forEach((r) => {
+          const t = Number(BigInt(r.timeUnixNano || "0") / 1_000_000n);
+          if (t > 0) {
+            const sev = r.severityNumber ?? 0;
+            const level = sev >= 17 ? "error" : sev >= 13 ? "warn" : "info";
+            items.push({ time: t, level, isTrace: false });
+          }
+        });
+      });
+    });
+
+    sourceTraces.forEach((trace) => {
+      (trace.resourceSpans ?? []).forEach((rs) => {
+        (rs.scopeSpans ?? []).forEach((ss) => {
+          (ss.spans ?? []).forEach((s) => {
+            const t = Number(BigInt(s.startTimeUnixNano || "0") / 1_000_000n);
+            if (t > 0) {
+              const hasErr = isSpanError(s);
+              items.push({ time: t, level: hasErr ? "error" : "info", isTrace: true });
+            }
+          });
+        });
+      });
+    });
+
+    if (items.length === 0) {
+      return {
+        buckets: [],
+        hasErrorSurge: false,
+        surgePercent: 0,
+        timeLabels: [],
+        totalCount: 0,
+      };
+    }
+
+    const minTime = Math.min(...items.map((it) => it.time));
+    const maxTime = Math.max(...items.map((it) => it.time));
+    const range = Math.max(maxTime - minTime, 1000);
+
+    items.forEach((it) => {
+      const idx = Math.min(numBuckets - 1, Math.floor(((it.time - minTime) / range) * numBuckets));
+      bucketStats[idx][it.level] += 1;
+      bucketStats[idx].total += 1;
+      if (it.isTrace) {
+        bucketStats[idx].traceCount += 1;
+      } else {
+        bucketStats[idx].logCount += 1;
+      }
+    });
+
+    // Check for surge in real errors
+    let maxErrors = 0;
+    let baselineErrors = 0;
+    bucketStats.forEach((b, idx) => {
+      if (b.error > maxErrors) maxErrors = b.error;
+      if (idx < numBuckets / 2) baselineErrors += b.error;
+    });
+
+    const avgBaseline = baselineErrors / (numBuckets / 2) || 1;
+    const computedSurge = Math.round(((maxErrors - avgBaseline) / avgBaseline) * 100);
+
+    const timeLabels = [
+      new Date(minTime).toLocaleTimeString([], { hour12: false, hour: "2-digit", minute: "2-digit", second: "2-digit" }),
+      new Date(minTime + range * 0.33).toLocaleTimeString([], { hour12: false, hour: "2-digit", minute: "2-digit", second: "2-digit" }),
+      new Date(minTime + range * 0.66).toLocaleTimeString([], { hour12: false, hour: "2-digit", minute: "2-digit", second: "2-digit" }),
+      new Date(maxTime).toLocaleTimeString([], { hour12: false, hour: "2-digit", minute: "2-digit", second: "2-digit" }),
+    ];
+
+    // Normalize heights to percentage
+    const maxBucketTotal = Math.max(...bucketStats.map((b) => b.total), 1);
+    const normalized = bucketStats.map((b, idx) => {
+      const scale = (b.total / maxBucketTotal) * 100;
+      const errorPct = b.total > 0 ? (b.error / b.total) * scale : 0;
+      const warnPct = b.total > 0 ? (b.warn / b.total) * scale : 0;
+      const infoPct = b.total > 0 ? (b.info / b.total) * scale : 0;
+
+      const startTime = minTime + Math.floor((idx * range) / numBuckets);
+      const endTime = minTime + Math.floor(((idx + 1) * range) / numBuckets);
+      const timeLabel = `${new Date(startTime).toLocaleTimeString([], { hour12: false, hour: "2-digit", minute: "2-digit", second: "2-digit" })} – ${new Date(endTime).toLocaleTimeString([], { hour12: false, hour: "2-digit", minute: "2-digit", second: "2-digit" })}`;
+
+      return {
+        idx,
+        error: errorPct,
+        warn: warnPct,
+        info: infoPct,
+        rawError: b.error,
+        rawWarn: b.warn,
+        rawInfo: b.info,
+        total: b.total,
+        traceCount: b.traceCount,
+        logCount: b.logCount,
+        startTime,
+        endTime,
+        timeLabel,
+      };
+    });
+
+    return {
+      buckets: normalized,
+      hasErrorSurge: maxErrors >= 5 && computedSurge > 50,
+      surgePercent: computedSurge,
+      timeLabels,
+      totalCount: items.length,
+    };
+  }, [sourceLogs, sourceTraces]);
+
+  const [isCollapsed, setIsCollapsed] = useState(false);
+
+  // Check which bucket matches the active selectedTimeRange
+  const activeBucket = useMemo(() => {
+    if (!selectedTimeRange || buckets.length === 0) return null;
+    return (
+      buckets.find(
+        (b) =>
+          selectedTimeRange.startMs <= b.endTime && selectedTimeRange.endMs >= b.startTime,
+      ) || null
+    );
+  }, [selectedTimeRange, buckets]);
+
+  const handleBucketClick = (b: (typeof buckets)[0]) => {
+    if (activeBucket && activeBucket.idx === b.idx) {
+      // Toggle off if clicking the already selected bucket
+      setSelectedTimeRange(null);
+    } else {
+      setSelectedTimeRange({
+        startMs: b.startTime,
+        endMs: b.endTime,
+        label: b.timeLabel,
+      });
+    }
+  };
+
+  if (buckets.length === 0) {
+    return (
+      <div className="relative bg-surface-container-lowest p-3.5 rounded-xl shadow-md mb-4 border border-surface-container/50">
+        <div className="flex items-center justify-between">
+          <div className="flex items-center gap-2">
+            <span className="material-symbols-outlined text-primary-container text-[18px]">
+              monitoring
+            </span>
+            <h2 className="font-headline font-semibold text-xs text-on-surface">
+              Telemetry Volume &amp; Error Waveform
+            </h2>
+          </div>
+          <span className="font-mono text-xs text-outline">Listening for OTLP traffic...</span>
+        </div>
+      </div>
+    );
+  }
+
+  return (
+    <div className="relative bg-surface-container-lowest p-3.5 rounded-xl shadow-md mb-4 overflow-hidden border border-surface-container/50 font-body">
+      <div className="flex flex-col sm:flex-row sm:items-center justify-between gap-2 mb-2">
+        <div className="flex items-center gap-3 flex-wrap">
+          <button
+            type="button"
+            onClick={() => setIsCollapsed(!isCollapsed)}
+            className="flex items-center gap-1.5 hover:text-white transition-colors"
+          >
+            <span className="material-symbols-outlined text-primary-container text-[18px]">
+              monitoring
+            </span>
+            <h2 className="font-headline font-semibold text-xs uppercase tracking-wider text-on-surface leading-none">
+              Telemetry Volume Waveform
+            </h2>
+            <span className="material-symbols-outlined text-outline text-[16px]">
+              {isCollapsed ? "expand_more" : "expand_less"}
+            </span>
+          </button>
+          <span className="font-mono text-xs text-outline">
+            {totalCount} events in current buffer
+          </span>
+          {hasErrorSurge && (
+            <span className="px-2 py-0.5 rounded-full bg-error-container text-on-error-container font-mono text-[10px] font-bold animate-pulse flex items-center gap-1">
+              <span className="w-1.5 h-1.5 rounded-full bg-error" />
+              SURGE +{surgePercent}%
+            </span>
+          )}
+          {selectedTimeRange && (
+            <span className="px-2 py-0.5 rounded-full bg-sky-500/20 text-sky-300 font-mono text-[10px] font-bold border border-sky-400/30 flex items-center gap-1">
+              <span className="w-1.5 h-1.5 rounded-full bg-sky-400 animate-ping" />
+              Time Focus Active
+              <button
+                type="button"
+                onClick={(e) => {
+                  e.stopPropagation();
+                  setSelectedTimeRange(null);
+                }}
+                className="ml-1 hover:text-white"
+                title="Clear time focus"
+              >
+                ✕
+              </button>
+            </span>
+          )}
+        </div>
+
+        {/* Legend */}
+        <div className="flex items-center gap-4 font-mono text-xs select-none">
+          <div className="flex items-center gap-1.5">
+            <span className="w-2.5 h-2.5 rounded-sm bg-error"></span>
+            <span className="text-on-surface">ERROR</span>
+          </div>
+          <div className="flex items-center gap-1.5">
+            <span className="w-2.5 h-2.5 rounded-sm bg-primary-fixed-dim"></span>
+            <span className="text-on-surface">WARN</span>
+          </div>
+          <div className="flex items-center gap-1.5">
+            <span className="w-2.5 h-2.5 rounded-sm bg-secondary"></span>
+            <span className="text-on-surface">INFO</span>
+          </div>
+        </div>
+      </div>
+
+      {!isCollapsed && (
+        <>
+          {/* Stacked SVG Histogram with Real Traffic */}
+          <div className="relative w-full h-24 flex items-end">
+            <div className="w-full h-full flex items-end justify-between gap-1 pt-3">
+              {buckets.map((b) => {
+                const isSurgeBar = hasErrorSurge && b.error > 40;
+                const isSelected = activeBucket?.idx === b.idx;
+                const hasAnySelected = activeBucket !== null;
+
+                return (
+                  <div
+                    key={b.idx}
+                    onClick={() => handleBucketClick(b)}
+                    className={`flex-1 flex flex-col justify-end h-full gap-0.5 group cursor-pointer relative transition-all rounded-sm ${
+                      isSelected
+                        ? "ring-2 ring-sky-400 bg-sky-400/25 scale-y-105 z-20 shadow-md shadow-sky-500/20"
+                        : hasAnySelected
+                        ? "opacity-40 hover:opacity-100 hover:brightness-125"
+                        : "hover:brightness-125"
+                    }`}
+                    title={`${b.timeLabel}\nTraces: ${b.traceCount} • Logs: ${b.logCount}\nErrors: ${b.rawError}, Warns: ${b.rawWarn}, Info: ${b.rawInfo}\nClick to ${isSelected ? "clear focus" : "focus and show events for this period"}`}
+                  >
+                    {/* Active Selection Focus Pin Indicator */}
+                    {isSelected && (
+                      <div className="absolute -top-4 left-1/2 -translate-x-1/2 flex flex-col items-center pointer-events-none z-20 whitespace-nowrap">
+                        <span className="px-1.5 py-0.5 bg-sky-500 text-slate-950 font-mono text-[9px] font-black rounded shadow">
+                          FOCUS
+                        </span>
+                        <span className="w-0.5 h-1 bg-sky-400"></span>
+                      </div>
+                    )}
+
+                    {/* Incident Flag Marker on Real Surge */}
+                    {!isSelected && isSurgeBar && b.idx === 15 && (
+                      <div className="absolute -top-3 left-1/2 -translate-x-1/2 flex flex-col items-center pointer-events-none z-10 whitespace-nowrap animate-bounce">
+                        <span className="px-1.5 py-0.5 bg-error-container text-on-error-container font-mono text-[10px] rounded shadow font-semibold flex items-center gap-1">
+                          <span className="w-1.5 h-1.5 rounded-full bg-error animate-ping"></span>
+                          SURGE +{surgePercent}%
+                        </span>
+                        <span className="w-0.5 h-2 bg-error"></span>
+                      </div>
+                    )}
+
+                    {/* Error segment */}
+                    {b.error > 0 && (
+                      <div
+                        className="w-full bg-error rounded-t-sm transition-all group-hover:brightness-125"
+                        style={{ height: `${b.error}%` }}
+                      />
+                    )}
+
+                    {/* Warn segment */}
+                    {b.warn > 0 && (
+                      <div
+                        className="w-full bg-primary-fixed-dim transition-all group-hover:brightness-125"
+                        style={{ height: `${b.warn}%` }}
+                      />
+                    )}
+
+                    {/* Info segment */}
+                    {b.info > 0 && (
+                      <div
+                        className="w-full bg-secondary/80 rounded-b-sm transition-all group-hover:brightness-125"
+                        style={{ height: `${b.info}%` }}
+                      />
+                    )}
+                  </div>
+                );
+              })}
+            </div>
+          </div>
+
+          {/* Real Time Scale Axes */}
+          <div className="flex justify-between items-center text-outline font-mono text-xs pt-2 select-none border-t border-surface-container/40 mt-1">
+            <span>{timeLabels[0]}</span>
+            <span>{timeLabels[1]}</span>
+            <span className={hasErrorSurge ? "text-error font-semibold" : ""}>
+              {timeLabels[2]} {hasErrorSurge ? "(SURGE)" : ""}
+            </span>
+            <span>{timeLabels[3]}</span>
+          </div>
+
+          {/* Time Focus Bar with quick tab navigation and clear button */}
+          {activeBucket && (
+            <div className="mt-2.5 flex flex-wrap items-center justify-between gap-3 rounded-xl bg-sky-500/10 border border-sky-500/30 px-3.5 py-2 animate-in fade-in slide-in-from-top-1 duration-200">
+              <div className="flex items-center gap-2.5 min-w-0">
+                <span className="flex h-2 w-2 rounded-full bg-sky-400 animate-ping" />
+                <div className="flex items-center gap-2 flex-wrap">
+                  <span className="font-semibold text-xs text-white">
+                    Focused Time Period:
+                  </span>
+                  <span className="font-mono text-xs text-sky-300 font-bold">
+                    {activeBucket.timeLabel}
+                  </span>
+                  <span className="text-[11px] text-slate-400 font-mono">
+                    ({activeBucket.traceCount} trace events • {activeBucket.logCount} log records
+                    {activeBucket.rawError > 0 ? ` • ${activeBucket.rawError} errors` : ""})
+                  </span>
+                </div>
+              </div>
+
+              <div className="flex items-center gap-2 font-mono text-xs">
+                <button
+                  type="button"
+                  onClick={() => setActiveTab("tab1")}
+                  className="flex items-center gap-1.5 px-2.5 py-1 rounded-lg bg-sky-500/20 hover:bg-sky-500/30 text-sky-300 font-bold transition-all border border-sky-400/30"
+                >
+                  <span className="material-symbols-outlined text-[14px]">timeline</span>
+                  Show Traces ({activeBucket.traceCount})
+                </button>
+                <button
+                  type="button"
+                  onClick={() => setActiveTab("tab3")}
+                  className="flex items-center gap-1.5 px-2.5 py-1 rounded-lg bg-sky-500/20 hover:bg-sky-500/30 text-sky-300 font-bold transition-all border border-sky-400/30"
+                >
+                  <span className="material-symbols-outlined text-[14px]">receipt_long</span>
+                  Show Logs ({activeBucket.logCount})
+                </button>
+                <button
+                  type="button"
+                  onClick={() => setSelectedTimeRange(null)}
+                  className="flex items-center gap-1 px-2 py-1 rounded-lg bg-white/10 hover:bg-white/20 text-slate-300 transition-colors"
+                  title="Clear time focus"
+                >
+                  <span className="material-symbols-outlined text-[14px]">close</span>
+                  Clear Focus
+                </button>
+              </div>
+            </div>
+          )}
+        </>
+      )}
+    </div>
+  );
+};

otel-ui/src/contexts/ObservabilityContextProvider.tsx

@@ -0,0 +1,799 @@
+"use client"
+
+import { useState, useEffect, useRef, useMemo, useCallback, type ReactNode } from "react"
+import { useLocation } from "wouter"
+import { tabForPath, pathForTab } from "../lib/routes"
+import type {
+  MetricData,
+  OtelHistoryMessage,
+  OtelMessage,
+  LogEntry,
+  TraceData,
+  ResourceSpan,
+  ScopeSpan,
+  Span,
+} from "../lib/types"
+import { ObservabilityContext, type TimeRangeFilter } from "./ObservabilityContextValue"
+import { getServiceName, formatAttributeValue, isSpanError } from "../lib/otel-utils"
+import { partitionMessages, mergeLogsByTraceId } from "../lib/ingest"
+import { authFetch, tokens } from "../lib/auth"
+
+interface ObservabilityProviderProps {
+  children: ReactNode
+}
+
+export function ObservabilityProvider({
+  children,
+}: ObservabilityProviderProps) {
+  const [logsByTraceId, setLogsByTraceId] = useState<
+    Record<string, LogEntry[]>
+  >({})
+  const [traces, setTraces] = useState<TraceData[]>([])
+  const [logs, setLogs] = useState<LogEntry[]>([])
+  const [metrics, setMetrics] = useState<MetricData[]>([])
+
+  const [isConnected, setIsConnected] = useState(false)
+  const wsRef = useRef<WebSocket | null>(null)
+  const reconnectTimeoutRef = useRef<number>(null)
+
+  // --- Real Streaming / Pause Buffer State ---
+  const [isStreaming, setIsStreaming] = useState(true)
+  const isStreamingRef = useRef(true)
+  const bufferedMessagesRef = useRef<OtelMessage[]>([])
+  const [bufferedCount, setBufferedCount] = useState(0)
+
+  // --- Issues live update counter ---
+  const [issuesVersion, setIssuesVersion] = useState(0)
+
+  // --- Latency filter (milliseconds) ---
+  const [minDurationMs, setMinDurationMs] = useState<number>(0)
+
+  // --- Persistence Keys ---
+  const SERVICES_STORAGE_KEY = "otel_selected_services"
+  const ATTRIBUTES_STORAGE_KEY = "otel_excluded_attributes"
+  const SHOW_ERRORS_STORAGE_KEY = "otel_show_errors_only"
+  const SEARCH_QUERY_STORAGE_KEY = "otel_search_query"
+  const TIME_WINDOW_STORAGE_KEY = "otel_time_window"
+
+  // --- Time Window Filter ---
+  const [timeWindow, setTimeWindow] = useState<"15m" | "1h" | "6h" | "24h" | "all">(() => {
+    if (typeof window === "undefined") return "15m"
+    return (localStorage.getItem(TIME_WINDOW_STORAGE_KEY) as any) || "15m"
+  })
+
+  // --- Selected Time Period Filter (e.g. from clicking waveform) ---
+  const [selectedTimeRange, setSelectedTimeRange] = useState<TimeRangeFilter | null>(null)
+
+  // --- Filtering State ---
+  const defaultExcludedAttributes = useCallback((): Record<string, Set<string>> => ({
+    "user_agent.original": new Set([
+      "kube-probe/1.33",
+      "kube-probe/1.31",
+      "Uptime-Kuma/1.23.11",
+    ]),
+    "http.target": new Set(["/healthz", "/readyz", "/livez"]),
+    "http.host": new Set(["git.k6n.net"]),
+    "http.user_agent": new Set(["Uptime-Kuma/1.23.11"]),
+  }), [])
+
+  const [selectedServices, setSelectedServices] = useState<Set<string>>(() => {
+    if (typeof window === "undefined") return new Set()
+    const saved = localStorage.getItem(SERVICES_STORAGE_KEY)
+    return saved ? new Set(JSON.parse(saved)) : new Set()
+  })
+
+  const [excludedAttributes, setExcludedAttributes] = useState<Record<string, Set<string>>>(() => {
+    if (typeof window === "undefined") return defaultExcludedAttributes()
+    const saved = localStorage.getItem(ATTRIBUTES_STORAGE_KEY)
+    if (!saved) return defaultExcludedAttributes()
+    try {
+      const parsed = JSON.parse(saved)
+      const restored: Record<string, Set<string>> = {}
+      Object.entries(parsed).forEach(([key, values]) => {
+        restored[key] = new Set(values as string[])
+      })
+      return restored
+    } catch (e) {
+      console.error("Failed to restore excluded attributes", e)
+      return defaultExcludedAttributes()
+    }
+  })
+
+  const [showErrorsOnly, setShowErrorsOnly] = useState<boolean>(() => {
+    if (typeof window === "undefined") return false
+    const saved = localStorage.getItem(SHOW_ERRORS_STORAGE_KEY)
+    return saved === "true"
+  })
+
+  const [searchQuery, setSearchQuery] = useState<string>(() => {
+    if (typeof window === "undefined") return ""
+    return localStorage.getItem(SEARCH_QUERY_STORAGE_KEY) || ""
+  })
+
+  // --- Persistence Effect ---
+  useEffect(() => {
+    if (typeof window === "undefined") return
+    localStorage.setItem(SERVICES_STORAGE_KEY, JSON.stringify(Array.from(selectedServices)))
+
+    const serializeable: Record<string, string[]> = {}
+    Object.entries(excludedAttributes).forEach(([key, values]) => {
+      serializeable[key] = Array.from(values)
+    })
+    localStorage.setItem(ATTRIBUTES_STORAGE_KEY, JSON.stringify(serializeable))
+    localStorage.setItem(SHOW_ERRORS_STORAGE_KEY, showErrorsOnly.toString())
+    localStorage.setItem(SEARCH_QUERY_STORAGE_KEY, searchQuery)
+    localStorage.setItem(TIME_WINDOW_STORAGE_KEY, timeWindow)
+  }, [selectedServices, excludedAttributes, showErrorsOnly, searchQuery, timeWindow])
+
+
+  // --- Selection State ---
+  const [activeTraceId, setActiveTraceId] = useState<string | null>(null)
+  const [activeSpanId, setActiveSpanId] = useState<string | null>(null)
+  // Tab state lives in the URL (wouter): /issues, /traces, ... so deep
+  // links survive the OIDC login redirect and views are shareable. The
+  // state mirror below keeps every existing setActiveTab call site working;
+  // it follows the URL on navigation and pushes the URL on programmatic
+  // setActiveTab calls.
+  const [location, navigate] = useLocation()
+  const [activeTab, setActiveTab] = useState<string>(
+    () => tabForPath(window.location.pathname) ?? "tab0",
+  )
+
+  // External navigation (back/forward, manual URL edit, login redirect):
+  // adopt the tab encoded in the new location.
+  useEffect(() => {
+    const tab = tabForPath(location)
+    if (tab && tab !== activeTab) setActiveTab(tab)
+    // eslint-disable-next-line react-hooks/exhaustive-deps -- activeTab is intentionally read stale here; see comment above
+  }, [location])
+
+  const setActiveTabWithRoute = useCallback(
+    (tab: string) => {
+      setActiveTab(tab)
+      const path = pathForTab(tab)
+      if (path && path !== window.location.pathname) navigate(path)
+    },
+    [navigate],
+  )
+  const [filtersExpanded, setFiltersExpanded] = useState<boolean>(false)
+  // Trace ids we already have spans for, so selecting a trace that fell
+  // outside the initial history window (or arrived before this session
+  // connected) triggers an on-demand GET /api/traces/{traceId} instead of
+  // silently showing nothing.
+  const [knownTraceIds, setKnownTraceIds] = useState<Set<string>>(new Set())
+
+  const toggleService = useCallback((service: string) => {
+    setSelectedServices(prev => {
+      const next = new Set(prev)
+      if (next.has(service)) {
+        next.delete(service)
+      } else {
+        next.add(service)
+      }
+      return next
+    })
+  }, [])
+
+  const toggleExcludedValue = useCallback((key: string, value: string) => {
+    setExcludedAttributes((prev) => {
+      const currentSet = prev[key] ?? new Set<string>()
+      const nextSet = new Set(currentSet)
+      if (nextSet.has(value)) {
+        nextSet.delete(value)
+      } else {
+        nextSet.add(value)
+      }
+      const next = { ...prev }
+      if (nextSet.size === 0) {
+        delete next[key]
+      } else {
+        next[key] = nextSet
+      }
+      return next
+    })
+  }, [])
+
+  const removeExcludedValue = useCallback((key: string, value: string) => {
+    setExcludedAttributes((prev) => {
+      const current = prev[key]
+      if (!current || !current.has(value)) {
+        return prev
+      }
+      const nextSet = new Set(current)
+      nextSet.delete(value)
+      const next = { ...prev }
+      if (nextSet.size === 0) {
+        delete next[key]
+      } else {
+        next[key] = nextSet
+      }
+      return next
+    })
+  }, [])
+
+  const clearAttributeKey = useCallback((key: string) => {
+    setExcludedAttributes((prev) => {
+      if (!prev[key]) {
+        return prev
+      }
+      const next = { ...prev }
+      delete next[key]
+      return next
+    })
+  }, [])
+
+  const resetFilters = useCallback(() => {
+    setSelectedServices(new Set())
+    setExcludedAttributes(defaultExcludedAttributes())
+    setSearchQuery("")
+    setSelectedTimeRange(null)
+  }, [defaultExcludedAttributes])
+
+  // Flush buffered WebSocket stream entries when resuming
+  const flushBufferedStream = useCallback(() => {
+    const queued = bufferedMessagesRef.current
+    bufferedMessagesRef.current = []
+    setBufferedCount(0)
+    if (queued.length > 0) {
+      const partitioned = partitionMessages(queued)
+      if (partitioned.traces.length) {
+        setTraces((prev) => [...prev, ...partitioned.traces])
+      }
+      if (partitioned.metrics.length) {
+        setMetrics((prev) => [...prev, ...partitioned.metrics])
+      }
+      if (partitioned.logs.length) {
+        setLogs((prev) => [...prev, ...partitioned.logs])
+        setLogsByTraceId((prev) => mergeLogsByTraceId(prev, partitioned.logsByTraceId))
+      }
+      if (partitioned.traceIds.size) {
+        setKnownTraceIds((prev) => {
+          const next = new Set(prev)
+          partitioned.traceIds.forEach((id) => next.add(id))
+          return next
+        })
+      }
+    }
+  }, [])
+
+  useEffect(() => {
+    isStreamingRef.current = isStreaming
+    if (isStreaming && bufferedMessagesRef.current.length > 0) {
+      flushBufferedStream()
+    }
+  }, [isStreaming, flushBufferedStream])
+
+  // Compute cutoff timestamp in nanoseconds for timeWindow
+  const cutoffNano = useMemo(() => {
+    if (timeWindow === "all") return 0n
+    const now = Date.now()
+    const windowMs = {
+      "15m": 15 * 60 * 1000,
+      "1h": 60 * 60 * 1000,
+      "6h": 6 * 60 * 60 * 1000,
+      "24h": 24 * 60 * 60 * 1000,
+      "all": 0,
+    }[timeWindow]
+    return BigInt(now - windowMs) * 1_000_000n
+  }, [timeWindow])
+
+  // Discover all unique services across all signals
+  const uniqueServices = useMemo(() => {
+    const services = new Set<string>()
+
+    // Check logs
+    logs.forEach(log => {
+      services.add(getServiceName(log.resource?.attributes ?? []))
+      ;(log.scopeLogs ?? []).forEach(sl => (sl.logRecords ?? []).forEach(record => {
+        services.add(getServiceName(log.resource?.attributes ?? [], record.attributes ?? []))
+      }))
+    })
+
+    // Check traces
+    traces.forEach(trace => {
+      trace.resourceSpans?.forEach(resourceSpan => {
+        services.add(getServiceName(resourceSpan.resource?.attributes ?? []))
+        resourceSpan.scopeSpans?.forEach(ss => {
+          ss.spans?.forEach(s => {
+            services.add(getServiceName(resourceSpan.resource?.attributes ?? [], s.attributes ?? []))
+          })
+        })
+      })
+    })
+
+    // Check metrics
+    metrics.forEach(metric => {
+      metric.resourceMetrics?.forEach(rm => {
+        services.add(getServiceName(rm.resource?.attributes ?? []))
+      })
+    })
+
+    return Array.from(services).sort().filter(s => s !== "unknown-service")
+  }, [logs, traces, metrics])
+
+  // --- Base Filtered Data (Global Filters: Service, TimeWindow, Search, Excluded Attributes) ---
+  const unfocusedLogs = useMemo(() => {
+    return logs.map(log => {
+      const filteredScopeLogs = (log.scopeLogs ?? []).map(sl => {
+        const filteredRecords = (sl.logRecords ?? []).filter(record => {
+          const serviceName = getServiceName(log.resource?.attributes ?? [], record.attributes ?? [])
+          // Filter by service
+          if (selectedServices.size > 0 && !selectedServices.has(serviceName)) return false
+
+          // Filter by time window
+          if (cutoffNano > 0n) {
+            const tNano = BigInt(record.timeUnixNano || "0")
+            if (tNano > 0n && tNano < cutoffNano) return false
+          }
+
+          // Filter by attributes
+          const recordAttrs = Object.fromEntries((record.attributes ?? []).map(a => [a.key, formatAttributeValue(a.value)]))
+          const isExcluded = Object.entries(excludedAttributes).some(([key, values]) => {
+            const val = recordAttrs[key]
+            return val && values.has(val)
+          })
+          if (isExcluded) return false
+
+          if (showErrorsOnly) {
+            return (record.severityNumber ?? 0) >= 13 // 13 is Warn, 17 is Error
+          }
+
+          if (searchQuery) {
+            const query = searchQuery.toLowerCase()
+            const matchInBody = record.body?.stringValue?.toLowerCase()?.includes(query) ?? false
+            const matchInAttrs = record.attributes?.some(a =>
+              a.key.toLowerCase()?.includes(query) ||
+              formatAttributeValue(a.value).toLowerCase()?.includes(query)
+            ) ?? false
+            const matchInServiceName = serviceName.toLowerCase()?.includes(query) ?? false
+            const matchInTrace = (record.traceId || "").toLowerCase().includes(query)
+            if (!matchInBody && !matchInAttrs && !matchInServiceName && !matchInTrace) return false
+          }
+
+          return true
+        })
+        if (filteredRecords.length === 0) return null
+        return { ...sl, logRecords: filteredRecords }
+      }).filter(Boolean)
+
+      if (filteredScopeLogs.length === 0) return null
+      return { ...log, scopeLogs: filteredScopeLogs as any }
+    }).filter(Boolean) as LogEntry[]
+  }, [logs, selectedServices, excludedAttributes, showErrorsOnly, searchQuery, cutoffNano])
+
+  const unfocusedTraces = useMemo(() => {
+    return traces.map(trace => {
+      const filteredResourceSpans = (trace.resourceSpans?.map(rs => {
+        const serviceName = getServiceName(rs.resource?.attributes ?? [])
+        // Filter by service
+        if (selectedServices.size > 0 && !selectedServices.has(serviceName)) return null
+
+        const filteredScopeSpans = (rs.scopeSpans?.map(ss => {
+          const filteredSpans = ss.spans?.filter(span => {
+            // Filter by time window
+            if (cutoffNano > 0n) {
+              const tNano = BigInt(span.startTimeUnixNano || "0")
+              if (tNano > 0n && tNano < cutoffNano) return false
+            }
+
+            // Filter by latency threshold (minDurationMs)
+            if (minDurationMs > 0) {
+              const start = Number(span.startTimeUnixNano || "0")
+              const end = Number(span.endTimeUnixNano || span.startTimeUnixNano || "0")
+              const durMs = (end - start) / 1_000_000
+              if (durMs < minDurationMs) return false
+            }
+
+            // 1. Excluded attributes
+            const spanAttrs = Object.fromEntries(span.attributes?.map(a => [a.key, formatAttributeValue(a.value)]) ?? [])
+            const isExcluded = Object.entries(excludedAttributes).some(([key, values]) => {
+              const val = spanAttrs[key]
+              return val && values.has(val)
+            })
+            if (isExcluded) return false
+
+            // 2. Search query
+            if (searchQuery) {
+              const query = searchQuery.toLowerCase()
+              const matchInName = (span.name || "").toLowerCase()?.includes(query) ?? false
+              const matchInTrace = (span.traceId || "").toLowerCase().includes(query)
+              const matchInAttrs = span.attributes?.some(a =>
+                a.key.toLowerCase()?.includes(query) ||
+                formatAttributeValue(a.value).toLowerCase()?.includes(query)
+              ) ?? false
+              if (!matchInName && !matchInAttrs && !matchInTrace) return false
+            }
+
+            return true
+          })
+          if (!filteredSpans || filteredSpans.length === 0) return null
+          return { ...ss, spans: filteredSpans }
+        }).filter(Boolean) as ScopeSpan[]) || []
+
+        if (!filteredScopeSpans || filteredScopeSpans.length === 0) return null
+        return { ...rs, scopeSpans: filteredScopeSpans }
+      }).filter(Boolean) as ResourceSpan[]) || []
+
+      if (!filteredResourceSpans || filteredResourceSpans.length === 0) return null
+
+      if (showErrorsOnly) {
+        const hasError = filteredResourceSpans?.some((rs: ResourceSpan) =>
+          rs?.scopeSpans?.some((ss: ScopeSpan) =>
+            ss?.spans?.some((span: Span) => isSpanError(span))
+          )
+        )
+        if (!hasError) return null
+      }
+
+      return { ...trace, resourceSpans: filteredResourceSpans }
+    }).filter(Boolean) as TraceData[]
+  }, [traces, selectedServices, excludedAttributes, showErrorsOnly, searchQuery, cutoffNano, minDurationMs])
+
+  // --- Selected Time Period Filtering (e.g. from waveform section click) ---
+  const filteredLogs = useMemo(() => {
+    if (!selectedTimeRange) return unfocusedLogs
+    const { startMs, endMs } = selectedTimeRange
+    return unfocusedLogs.map(log => {
+      const filteredScopeLogs = (log.scopeLogs ?? []).map(sl => {
+        const filteredRecords = (sl.logRecords ?? []).filter(record => {
+          const tMs = Number(BigInt(record.timeUnixNano || "0") / 1_000_000n)
+          return tMs >= startMs && tMs <= endMs
+        })
+        if (filteredRecords.length === 0) return null
+        return { ...sl, logRecords: filteredRecords }
+      }).filter(Boolean)
+
+      if (filteredScopeLogs.length === 0) return null
+      return { ...log, scopeLogs: filteredScopeLogs as any }
+    }).filter(Boolean) as LogEntry[]
+  }, [unfocusedLogs, selectedTimeRange])
+
+  const filteredTraces = useMemo(() => {
+    if (!selectedTimeRange) return unfocusedTraces
+    const { startMs, endMs } = selectedTimeRange
+    return unfocusedTraces.filter(trace => {
+      return (trace.resourceSpans ?? []).some(rs =>
+        (rs.scopeSpans ?? []).some(ss =>
+          (ss.spans ?? []).some(span => {
+            const spanStartMs = Number(BigInt(span.startTimeUnixNano || "0") / 1_000_000n)
+            const spanEndMs = Number(BigInt(span.endTimeUnixNano || span.startTimeUnixNano || "0") / 1_000_000n)
+            return spanStartMs <= endMs && spanEndMs >= startMs
+          })
+        )
+      )
+    })
+  }, [unfocusedTraces, selectedTimeRange])
+
+  const logCountBySpanId = useMemo(() => {
+    const counts: Record<string, number> = {}
+    filteredLogs.forEach(log => {
+      ;(log.scopeLogs ?? []).forEach(sl => {
+        ;(sl.logRecords ?? []).forEach(record => {
+          if (record.spanId) {
+            counts[record.spanId] = (counts[record.spanId] || 0) + 1
+          }
+        })
+      })
+    })
+    return counts
+  }, [filteredLogs])
+
+  const filteredMetrics = useMemo(() => {
+    return metrics.map(data => {
+      const filteredResourceMetrics = data.resourceMetrics?.map(rm => {
+        const serviceName = rm.resource?.attributes?.find(attr => attr.key === "service.name")?.value.stringValue || "unknown-service"
+        // Filter by service
+        if (selectedServices.size > 0 && !selectedServices.has(serviceName)) return null
+
+        const filteredScopeMetrics = rm.scopeMetrics?.map(sm => {
+          const filteredMetricsList = sm.metrics?.filter(_m => {
+            return true
+          })
+          if (filteredMetricsList?.length === 0) return null
+          return { ...sm, metrics: filteredMetricsList }
+        }).filter(Boolean)
+
+        if (filteredScopeMetrics?.length === 0) return null
+        return { ...rm, scopeMetrics: filteredScopeMetrics }
+      }).filter(Boolean)
+
+      if (filteredResourceMetrics?.length === 0) return null
+      return { ...data, resourceMetrics: filteredResourceMetrics }
+    }).filter(Boolean) as MetricData[]
+  }, [metrics, selectedServices])
+
+  // --- Real Service Health & Offender Metrics ---
+  const serviceHealth = useMemo(() => {
+    const health: Record<string, {
+      name: string;
+      totalSignals: number;
+      errorCount: number;
+      spansCount: number;
+      logsCount: number;
+      hasErrors: boolean;
+    }> = {}
+
+    const getOrInit = (srv: string) => {
+      if (!health[srv]) {
+        health[srv] = {
+          name: srv,
+          totalSignals: 0,
+          errorCount: 0,
+          spansCount: 0,
+          logsCount: 0,
+          hasErrors: false,
+        }
+      }
+      return health[srv]
+    }
+
+    filteredLogs.forEach((log) => {
+      (log.scopeLogs ?? []).forEach((sl) => {
+        (sl.logRecords ?? []).forEach((r) => {
+          const sName = getServiceName(log.resource?.attributes ?? [], r.attributes ?? [])
+          if (sName && sName !== "unknown-service") {
+            const h = getOrInit(sName)
+            h.totalSignals++
+            h.logsCount++
+            if ((r.severityNumber ?? 0) >= 17) {
+              h.errorCount++
+              h.hasErrors = true
+            }
+          }
+        })
+      })
+    })
+
+    filteredTraces.forEach((trace) => {
+      trace.resourceSpans?.forEach((rs) => {
+        rs.scopeSpans?.forEach((ss) => {
+          ss.spans?.forEach((s) => {
+            const sName = getServiceName(rs.resource?.attributes ?? [], s.attributes ?? [])
+            if (sName && sName !== "unknown-service") {
+              const h = getOrInit(sName)
+              h.totalSignals++
+              h.spansCount++
+              if (isSpanError(s)) {
+                h.errorCount++
+                h.hasErrors = true
+              }
+            }
+          })
+        })
+      })
+    })
+
+    return health
+  }, [filteredLogs, filteredTraces])
+
+
+  useEffect(() => {
+    // Do not start protected history/WebSocket traffic until a session exists.
+    // The auth callback reloads the app after storing tokens, so this is also
+    // sufficient for the normal login flow.
+    if (!tokens()) return
+
+    const fetchHistory = async () => {
+      try {
+        // Fetch each signal type separately with its own generous limit,
+        // rather than one combined /api/history call - otherwise a noisy
+        // signal type (metrics in particular) can crowd the others out of
+        // the initial window.
+        const [tracesRes, logsRes, metricsRes] = await Promise.all([
+          authFetch("/api/history/traces?limit=300"),
+          authFetch("/api/history/logs?limit=300"),
+          authFetch("/api/history/metrics?limit=150"),
+        ])
+        const [tracesData, logsData, metricsData] = await Promise.all([
+          tracesRes.json() as Promise<OtelHistoryMessage[] | null>,
+          logsRes.json() as Promise<OtelHistoryMessage[] | null>,
+          metricsRes.json() as Promise<OtelHistoryMessage[] | null>,
+        ])
+
+        const partitioned = partitionMessages([
+          ...(tracesData ?? []),
+          ...(logsData ?? []),
+          ...(metricsData ?? []),
+        ])
+
+        console.log(
+          "Fetched historical data:",
+          partitioned.traces.length, "traces,",
+          partitioned.logs.length, "logs,",
+          partitioned.metrics.length, "metrics",
+        )
+
+        setTraces(partitioned.traces)
+        setMetrics(partitioned.metrics)
+        setLogs(partitioned.logs)
+        setLogsByTraceId(partitioned.logsByTraceId)
+        setKnownTraceIds(partitioned.traceIds)
+      } catch (error) {
+        console.error("Error fetching history:", error)
+      }
+    }
+
+    const connect = () => {
+      const protocol = window.location.protocol === "https:" ? "wss:" : "ws:"
+      const t = tokens()
+      const tokenQuery = t?.accessToken ? `?access_token=${encodeURIComponent(t.accessToken)}` : ""
+      const wsUrl = `${protocol}//${window.location.host}/ws${tokenQuery}`
+      console.log(`Connecting to WebSocket at ${wsUrl}`)
+
+      const ws = new WebSocket(wsUrl)
+      wsRef.current = ws
+
+      ws.onopen = () => {
+        console.log("WebSocket connected")
+        setIsConnected(true)
+      }
+
+      ws.onmessage = (event) => {
+        try {
+          const data = JSON.parse(event.data)
+
+          // Server issues notification broadcast
+          if (data?.type === "issues") {
+            setIssuesVersion((v) => v + 1)
+            return
+          }
+
+          // If stream is paused, buffer the incoming signal
+          if (!isStreamingRef.current) {
+            bufferedMessagesRef.current.push(data as OtelMessage)
+            setBufferedCount(bufferedMessagesRef.current.length)
+            return
+          }
+
+          const partitioned = partitionMessages([data as OtelMessage])
+
+          if (partitioned.traces.length) {
+            setTraces((prev) => [...prev, ...partitioned.traces])
+          }
+          if (partitioned.metrics.length) {
+            setMetrics((prev) => [...prev, ...partitioned.metrics])
+          }
+          if (partitioned.logs.length) {
+            setLogs((prev) => [...prev, ...partitioned.logs])
+            setLogsByTraceId((prev) => mergeLogsByTraceId(prev, partitioned.logsByTraceId))
+          }
+          if (partitioned.traceIds.size) {
+            setKnownTraceIds((prev) => {
+              const next = new Set(prev)
+              partitioned.traceIds.forEach((id) => next.add(id))
+              return next
+            })
+          }
+        } catch (error) {
+          console.error("Error parsing WebSocket message:", error)
+        }
+      }
+
+      ws.onerror = (error) => {
+        console.error("WebSocket error:", error)
+        setIsConnected(false)
+      }
+
+      ws.onclose = () => {
+        console.log("WebSocket disconnected, reconnecting in 5s...")
+        setIsConnected(false)
+
+        // Reconnect after 5 seconds
+        reconnectTimeoutRef.current = setTimeout(() => {
+          connect()
+        }, 5000)
+      }
+    }
+    fetchHistory().then(() => connect())
+
+    return () => {
+      if (reconnectTimeoutRef.current) {
+        clearTimeout(reconnectTimeoutRef.current)
+      }
+      if (wsRef.current) {
+        wsRef.current.close()
+      }
+    }
+  }, [flushBufferedStream])
+
+  // On-demand trace fetch: if the selected trace id isn't one we already
+  // have spans for (e.g. it arrived before this session connected, or fell
+  // outside the initial history window), fetch it directly instead of
+  // silently showing an empty trace view.
+  useEffect(() => {
+    if (!tokens() || !activeTraceId || knownTraceIds.has(activeTraceId)) return
+
+    let cancelled = false
+    authFetch(`/api/traces/${encodeURIComponent(activeTraceId)}`)
+      .then((res) => {
+        if (!res.ok) throw new Error(`HTTP ${res.status}`)
+        return res.json() as Promise<OtelHistoryMessage[] | null>
+      })
+      .then((data) => {
+        if (cancelled) return
+        const partitioned = partitionMessages(data ?? [])
+
+        if (partitioned.traces.length) {
+          setTraces((prev) => [...prev, ...partitioned.traces])
+        }
+        if (partitioned.metrics.length) {
+          setMetrics((prev) => [...prev, ...partitioned.metrics])
+        }
+        if (partitioned.logs.length) {
+          setLogs((prev) => [...prev, ...partitioned.logs])
+          setLogsByTraceId((prev) => mergeLogsByTraceId(prev, partitioned.logsByTraceId))
+        }
+
+        setKnownTraceIds((prev) => {
+          const next = new Set(prev)
+          partitioned.traceIds.forEach((id) => next.add(id))
+          next.add(activeTraceId)
+          return next
+        })
+      })
+      .catch((error) => {
+        console.error(`Error fetching trace ${activeTraceId}:`, error)
+      })
+
+    return () => {
+      cancelled = true
+    }
+  }, [activeTraceId, knownTraceIds])
+
+  return (
+    <ObservabilityContext.Provider
+      value={{
+        logs: filteredLogs,
+        metrics: filteredMetrics,
+        traces: filteredTraces,
+        unfocusedLogs,
+        unfocusedTraces,
+        logsByTraceId,
+        logCountBySpanId,
+        isConnected,
+
+        // Real stream controls
+        isStreaming,
+        setIsStreaming,
+        bufferedCount,
+        flushBufferedStream,
+
+        // Real time window & latency
+        timeWindow,
+        setTimeWindow,
+        selectedTimeRange,
+        setSelectedTimeRange,
+        minDurationMs,
+        setMinDurationMs,
+
+        // Discovered service health & issues updates
+        serviceHealth,
+        issuesVersion,
+
+        selectedServices,
+        toggleService,
+        excludedAttributes,
+        toggleExcludedValue,
+        removeExcludedValue,
+        clearAttributeKey,
+        resetFilters,
+        uniqueServices,
+        showErrorsOnly,
+        setShowErrorsOnly,
+        searchQuery,
+        setSearchQuery,
+        activeTraceId,
+        setActiveTraceId,
+        activeSpanId,
+        setActiveSpanId,
+        activeTab,
+        setActiveTab: setActiveTabWithRoute,
+        filtersExpanded,
+        setFiltersExpanded,
+      }}
+    >
+      {children}
+    </ObservabilityContext.Provider>
+  )
+
+}

otel-ui/src/contexts/ObservabilityContextValue.ts

@@ -0,0 +1,88 @@
+"use client";
+
+import { createContext } from "react";
+import type { LogEntry, MetricData, TraceData } from "../lib/types";
+
+export interface FilterState {
+  selectedServices: Set<string>;
+  excludedAttributes: Record<string, Set<string>>;
+}
+
+export type TimeWindow = "15m" | "1h" | "6h" | "24h" | "all";
+
+export interface ServiceHealthSummary {
+  name: string;
+  totalSignals: number;
+  errorCount: number;
+  spansCount: number;
+  logsCount: number;
+  hasErrors: boolean;
+}
+
+export interface TimeRangeFilter {
+  startMs: number;
+  endMs: number;
+  label?: string;
+}
+
+interface ObservabilityContextType {
+  logs: LogEntry[];
+  metrics: MetricData[];
+  traces: TraceData[];
+  unfocusedLogs: LogEntry[];
+  unfocusedTraces: TraceData[];
+  logsByTraceId: Record<string, LogEntry[]>;
+  logCountBySpanId: Record<string, number>;
+  isConnected: boolean;
+
+  // Real Streaming / Live Tail Controls
+  isStreaming: boolean;
+  setIsStreaming: (streaming: boolean) => void;
+  bufferedCount: number;
+  flushBufferedStream: () => void;
+
+  // Real Time Window Filter
+  timeWindow: TimeWindow;
+  setTimeWindow: (window: TimeWindow) => void;
+
+  // Selected Time Period Filter (from waveform or custom selection)
+  selectedTimeRange: TimeRangeFilter | null;
+  setSelectedTimeRange: (range: TimeRangeFilter | null) => void;
+
+  // Latency / Anomaly Threshold Filter
+  minDurationMs: number;
+  setMinDurationMs: (ms: number) => void;
+
+  // Service Health & Facets
+  serviceHealth: Record<string, ServiceHealthSummary>;
+  issuesVersion: number;
+
+  // Global filters
+  selectedServices: Set<string>;
+  toggleService: (service: string) => void;
+  excludedAttributes: Record<string, Set<string>>;
+  toggleExcludedValue: (key: string, value: string) => void;
+  removeExcludedValue: (key: string, value: string) => void;
+  clearAttributeKey: (key: string) => void;
+  resetFilters: () => void;
+  uniqueServices: string[];
+  showErrorsOnly: boolean;
+  setShowErrorsOnly: (show: boolean) => void;
+  searchQuery: string;
+  setSearchQuery: (query: string) => void;
+
+  // Active selection
+  activeTraceId: string | null;
+  setActiveTraceId: (id: string | null) => void;
+  activeSpanId: string | null;
+  setActiveSpanId: (id: string | null) => void;
+  activeTab: string;
+  setActiveTab: (tab: string) => void;
+  filtersExpanded: boolean;
+  setFiltersExpanded: (expanded: boolean) => void;
+}
+
+export const ObservabilityContext = createContext<
+  ObservabilityContextType | undefined
+>(undefined);
+

otel-ui/src/hooks/use-mobile.ts

@@ -0,0 +1,19 @@
+import * as React from 'react'
+
+const MOBILE_BREAKPOINT = 768
+
+export function useIsMobile() {
+  const [isMobile, setIsMobile] = React.useState<boolean | undefined>(undefined)
+
+  React.useEffect(() => {
+    const mql = window.matchMedia(`(max-width: ${MOBILE_BREAKPOINT - 1}px)`)
+    const onChange = () => {
+      setIsMobile(window.innerWidth < MOBILE_BREAKPOINT)
+    }
+    mql.addEventListener('change', onChange)
+    setIsMobile(window.innerWidth < MOBILE_BREAKPOINT)
+    return () => mql.removeEventListener('change', onChange)
+  }, [])
+
+  return !!isMobile
+}

otel-ui/src/hooks/useIssues.ts

@@ -0,0 +1,50 @@
+import { useCallback, useEffect } from "react"
+import useSWR from "swr"
+import type { Issue } from "../lib/types"
+import { authSWRFetcher, tokens } from "../lib/auth"
+import { useObservabilityData } from "./useObservabilityData"
+
+
+const FALLBACK_POLL_INTERVAL_MS = 30_000
+const LIVE_REFRESH_DEBOUNCE_MS = 1_000
+let lastLiveRefreshAt = 0
+
+// Polls the backend's deduplicated error view (GET /api/issues) rather than
+// deriving "errors" from the raw telemetry stream in the browser, so counts,
+// first/last-seen and dedup match what the server has actually persisted -
+// not just whatever fits in this tab's in-memory window. Updates arrive live
+// over the WebSocket (useLiveIssues); the interval is only a fallback for
+// when the socket is down.
+export function useIssues() {
+  const key = tokens() ? "/api/issues?limit=200" : null
+  const { data, error, isLoading, mutate } = useSWR<Issue[] | null>(
+    key,
+    authSWRFetcher,
+    {
+      refreshInterval: FALLBACK_POLL_INTERVAL_MS,
+      revalidateOnFocus: false,
+      shouldRetryOnError: false,
+      isPaused: () => !tokens(),
+    },
+  )
+  const refresh = useCallback(() => mutate(), [mutate])
+
+  const { issuesVersion } = useObservabilityData()
+
+  // Live push over the primary WebSocket
+  useEffect(() => {
+    if (issuesVersion > 0 && tokens()) {
+      const now = Date.now()
+      if (now - lastLiveRefreshAt < LIVE_REFRESH_DEBOUNCE_MS) return
+      lastLiveRefreshAt = now
+      void refresh()
+    }
+  }, [issuesVersion, refresh])
+
+  return {
+    issues: data ?? [],
+    loading: isLoading,
+    error: error instanceof Error ? error.message : error ? String(error) : null,
+    refresh,
+  }
+}

otel-ui/src/hooks/useLiveIssues.ts

@@ -0,0 +1,84 @@
+// Live issue updates: the backend broadcasts "issues" events over the same
+// WebSocket as telemetry, so the Issues tab can refresh instantly when an
+// error is ingested instead of waiting for the next poll tick. Falls back to
+// polling if the socket misbehaves.
+import { useEffect, useState } from "react"
+
+export function useLiveIssues(onRefresh: () => void) {
+  const [live, setLive] = useState(false)
+
+  useEffect(() => {
+    const protocol = window.location.protocol === "https:" ? "wss:" : "ws:"
+    let ws: WebSocket | null = null
+    let retryTimer: number | null = null
+    let pollTimer: number | null = null
+    let disposed = false
+
+    const startPolling = () => {
+      if (pollTimer === null) {
+        pollTimer = window.setInterval(onRefresh, 30_000)
+      }
+    }
+    const stopPolling = () => {
+      if (pollTimer !== null) {
+        window.clearInterval(pollTimer)
+        pollTimer = null
+      }
+    }
+
+    const connect = () => {
+      if (disposed) return
+      try {
+        ws = new WebSocket(`${protocol}//${window.location.host}/ws`)
+      } catch {
+        startPolling()
+        return
+      }
+
+      ws.onopen = () => {
+        setLive(true)
+        stopPolling()
+      }
+      ws.onmessage = (event) => {
+        try {
+          const msg = JSON.parse(event.data)
+          if (msg?.type === "issues") onRefresh()
+        } catch {
+          /* ignore malformed frames */
+        }
+      }
+      ws.onclose = () => {
+        setLive(false)
+        startPolling()
+        // Reuse the telemetry socket's cadence for reconnects.
+        retryTimer = window.setTimeout(connect, 5000)
+      }
+      ws.onerror = () => {
+        try {
+          ws?.close()
+        } catch {
+          /* ignore */
+        }
+      }
+    }
+
+    connect()
+
+    return () => {
+      disposed = true
+      if (retryTimer !== null) window.clearTimeout(retryTimer)
+      stopPolling()
+      if (ws) {
+        ws.onclose = null
+        try {
+          ws.close()
+        } catch {
+          /* ignore */
+        }
+      }
+    }
+  }, [onRefresh])
+
+  // Exposed so the Issues header can show a "live" vs "polling" indicator.
+  return live
+}

otel-ui/src/hooks/useLogsForTraceId.ts

@@ -0,0 +1,7 @@
+import { useMemo } from "react";
+import { useObservabilityData } from "./useObservabilityData";
+
+export const useLogsForTraceId = (traceId: string) => {
+  const { logsByTraceId } = useObservabilityData();
+  return useMemo(() => logsByTraceId[traceId] || [], [logsByTraceId, traceId]);
+};

otel-ui/src/hooks/useMetrics.ts

@@ -0,0 +1,129 @@
+import { useMemo } from "react";
+import { useObservabilityData } from "./useObservabilityData";
+import type { MetricData, MetricDataPoint } from "../lib/types";
+import { getPodName } from "../lib/otel-utils";
+
+export interface MetricDetails {
+  id: string;
+  name: string;
+  service: string;
+  type: "gauge" | "sum" | "histogram";
+  points: MetricDataPoint[];
+  unit?: string;
+  pods: string[];
+}
+
+export const useMetricsDetails = (): Record<string, MetricDetails> => {
+  const { metrics } = useObservabilityData();
+  return useMemo(() => {
+    const allMetrics: Record<
+      string,
+      {
+        id: string;
+        name: string;
+        service: string;
+        type: "gauge" | "sum" | "histogram";
+        points: MetricDataPoint[];
+        unit?: string;
+        podsSet: Set<string>;
+      }
+    > = {};
+
+    metrics.forEach((metricData: MetricData) => {
+      (metricData.resourceMetrics ?? []).forEach((resourceMetric) => {
+        const resourceAttrs = resourceMetric.resource?.attributes ?? [];
+        const serviceName =
+          resourceAttrs.find((attr) => attr.key === "service.name")?.value.stringValue || "unknown-service";
+        const resourcePod = getPodName(resourceAttrs);
+
+        (resourceMetric.scopeMetrics ?? []).forEach((scopeMetric) => {
+          (scopeMetric.metrics ?? []).forEach((metric) => {
+            const key = `${serviceName}:${metric.name}`;
+            if (!allMetrics[key]) {
+              let detectedType: "gauge" | "sum" | "histogram" = "sum";
+              if (metric.gauge) detectedType = "gauge";
+              else if (metric.histogram) detectedType = "histogram";
+
+              allMetrics[key] = {
+                id: key,
+                name: metric.name,
+                service: serviceName,
+                type: detectedType,
+                points: [],
+                unit: (metric as any).unit || undefined,
+                podsSet: new Set<string>(),
+              };
+            }
+
+            const enrichPoint = (p: MetricDataPoint): MetricDataPoint => {
+              const pod = getPodName(undefined, p.attributes) || resourcePod;
+              if (pod) {
+                allMetrics[key].podsSet.add(pod);
+              }
+              return {
+                ...p,
+                pod: pod || undefined,
+              };
+            };
+
+            if (metric.gauge) {
+              allMetrics[key].type = "gauge";
+              allMetrics[key].points.push(...(metric.gauge.dataPoints ?? []).map(enrichPoint));
+            }
+            if (metric.sum) {
+              if (allMetrics[key].type !== "gauge") {
+                allMetrics[key].type = "sum";
+              }
+              allMetrics[key].points.push(...(metric.sum.dataPoints ?? []).map(enrichPoint));
+            }
+            if (metric.histogram) {
+              if (allMetrics[key].type !== "gauge") {
+                allMetrics[key].type = "histogram";
+              }
+              const histogramPoints = (metric.histogram.dataPoints ?? []).map((point) =>
+                enrichPoint({
+                  ...point,
+                  attributes: point.attributes ?? [],
+                  asDouble: point.sum ?? 0,
+                })
+              );
+              allMetrics[key].points.push(...histogramPoints);
+            }
+          });
+        });
+      });
+    });
+
+    const result: Record<string, MetricDetails> = {};
+    for (const key in allMetrics) {
+      const sorted = allMetrics[key].points.sort((a, b) => {
+        const aT = BigInt(a.timeUnixNano || "0");
+        const bT = BigInt(b.timeUnixNano || "0");
+        if (aT < bT) return -1;
+        if (aT > bT) return 1;
+        return 0;
+      });
+      result[key] = {
+        id: key,
+        name: allMetrics[key].name,
+        service: allMetrics[key].service,
+        type: allMetrics[key].type,
+        points: sorted.slice(-600),
+        unit: allMetrics[key].unit,
+        pods: Array.from(allMetrics[key].podsSet).sort(),
+      };
+    }
+    return result;
+  }, [metrics]);
+};
+
+export const useMetrics = () => {
+  const details = useMetricsDetails();
+  return useMemo(() => {
+    const grouped: Record<string, MetricDataPoint[]> = {};
+    for (const key in details) {
+      grouped[key] = details[key].points;
+    }
+    return grouped;
+  }, [details]);
+};

otel-ui/src/hooks/useObservabilityData.ts

@@ -0,0 +1,14 @@
+"use client";
+
+import { useContext } from "react";
+import { ObservabilityContext } from "../contexts/ObservabilityContextValue";
+
+export function useObservabilityData() {
+  const context = useContext(ObservabilityContext);
+  if (!context) {
+    throw new Error(
+      "useObservabilityData must be used within an ObservabilityProvider",
+    );
+  }
+  return context;
+}

otel-ui/src/hooks/usePatterns.ts

@@ -0,0 +1,41 @@
+import useSWR from "swr"
+import { authSWRFetcher, authFetch, tokens } from "../lib/auth"
+import type { Issue, IssuePattern, SimilarIssue } from "../lib/types"
+
+export function usePatterns() {
+  const enabled = Boolean(tokens())
+  const patterns = useSWR<IssuePattern[]>(enabled ? "/api/patterns" : null, authSWRFetcher, {
+    revalidateOnFocus: false,
+    refreshInterval: 30_000,
+  })
+  const issues = useSWR<Issue[]>(enabled ? "/api/issues?limit=200" : null, authSWRFetcher, {
+    revalidateOnFocus: false,
+    refreshInterval: 30_000,
+  })
+
+  const similar = async (id: string) => {
+    const response = await authFetch(`/api/issues/${encodeURIComponent(id)}/similar?limit=8`)
+    if (!response.ok) throw new Error(`HTTP ${response.status}`)
+    return (await response.json()) as SimilarIssue[]
+  }
+
+  const review = async (id: string, otherId: string, decision: "same" | "different") => {
+    const response = await authFetch(`/api/issues/${encodeURIComponent(id)}/similar`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ issueId: otherId, decision }),
+    })
+    if (!response.ok) throw new Error(`HTTP ${response.status}`)
+    await Promise.all([patterns.mutate(), issues.mutate()])
+  }
+
+  return {
+    patterns: patterns.data ?? [],
+    issues: issues.data ?? [],
+    loading: patterns.isLoading || issues.isLoading,
+    error: patterns.error ?? issues.error,
+    refresh: () => Promise.all([patterns.mutate(), issues.mutate()]),
+    similar,
+    review,
+  }
+}

otel-ui/src/hooks/useRollouts.ts

@@ -0,0 +1,14 @@
+import useSWR from "swr"
+import { authSWRFetcher, authFetch, tokens } from "../lib/auth"
+import type { Rollout, RolloutImpact } from "../lib/types"
+
+export function useRollouts() {
+  const key = tokens() ? "/api/rollouts" : null
+  const state = useSWR<Rollout[]>(key, authSWRFetcher, { refreshInterval: 30_000, revalidateOnFocus: false, shouldRetryOnError: false })
+  const impact = async (id: string) => {
+    const response = await authFetch(`/api/rollouts/${encodeURIComponent(id)}/impact`)
+    if (!response.ok) throw new Error(`HTTP ${response.status}`)
+    return (await response.json()) as RolloutImpact
+  }
+  return { rollouts: state.data ?? [], loading: state.isLoading, error: state.error, refresh: state.mutate, impact }
+}

otel-ui/src/hooks/useStats.ts

@@ -0,0 +1,60 @@
+import { useCallback } from "react"
+import useSWR from "swr"
+import type { ErrorStatsResponse, ReleaseStatsResponse } from "../lib/types"
+import { authFetch, authSWRFetcher, tokens } from "../lib/auth"
+
+/**
+ * True when the backend has ClickHouse-backed stats (the endpoints answer
+ * 200 instead of the documented 501). null while checking, so the Trends
+ * tab can hide itself entirely on file-store deployments.
+ */
+export function useStatsAvailability() {
+  const { data } = useSWR(tokens() ? "/api/stats/services" : null, async (key) => {
+    const res = await authFetch(key)
+    return res.ok
+  }, { revalidateOnFocus: false, shouldRetryOnError: false, isPaused: () => !tokens() })
+  return data ?? null
+}
+
+export interface StatsRange {
+  /** Lookback window in hours; server caps at 2160 (90 days). */
+  hours: number
+  service: string
+}
+
+const DEFAULT_RANGE: StatsRange = { hours: 72, service: "" }
+
+/**
+ * Fetches both error-trend and release-comparison series for the selected
+ * range. Both endpoints come from the same ClickHouse rollup, so a single
+ * refresh covers the tab.
+ */
+export function useStats(range: StatsRange = DEFAULT_RANGE) {
+  const params = new URLSearchParams({ hours: String(range.hours) })
+  if (range.service) params.set("service", range.service)
+  const key = tokens() ? `/api/stats/errors?${params}` : null
+  const { data, error, isLoading, mutate } = useSWR<{
+    errors: ErrorStatsResponse
+    releases: ReleaseStatsResponse | null
+    services: string[]
+  }>(key, async () => {
+    const [errors, releases, services] = await Promise.all([
+      authSWRFetcher<ErrorStatsResponse>(`/api/stats/errors?${params}`),
+      authSWRFetcher<ReleaseStatsResponse>(`/api/stats/releases?${params}`).catch(() => null),
+      authSWRFetcher<{ services?: string[] }>("/api/stats/services").catch(() => null),
+    ])
+    return { errors, releases, services: services?.services ?? [] }
+  }, { revalidateOnFocus: false, shouldRetryOnError: false, isPaused: () => !tokens() })
+
+  const refresh = useCallback(() => mutate(), [mutate])
+
+  return {
+    errorPoints: data?.errors.points ?? [],
+    granularity: data?.errors.granularity ?? "hour",
+    releaseSeries: data?.releases?.series ?? {},
+    services: data?.services ?? [],
+    loading: isLoading,
+    error: error instanceof Error ? error.message : error ? String(error) : null,
+    refresh,
+  }
+}

otel-ui/src/index.css

@@ -0,0 +1,90 @@
+@import "tailwindcss";
+
+@theme {
+  --color-background: #0f131c;
+  --color-surface: #0f131c;
+  --color-surface-dim: #0f131c;
+  --color-surface-bright: #353942;
+  --color-surface-container-lowest: #0a0e16;
+  --color-surface-container-low: #181c24;
+  --color-surface-container: #1c2028;
+  --color-surface-container-high: #262a33;
+  --color-surface-container-highest: #31353e;
+  --color-surface-variant: #31353e;
+  --color-surface-tint: #00dbe9;
+
+  --color-on-background: #dfe2ee;
+  --color-on-surface: #dfe2ee;
+  --color-on-surface-variant: #b9cacb;
+  --color-inverse-surface: #dfe2ee;
+  --color-inverse-on-surface: #2c3039;
+
+  --color-primary: #dbfcff;
+  --color-primary-container: #00f0ff;
+  --color-on-primary: #00363a;
+  --color-on-primary-container: #006970;
+  --color-primary-fixed: #7df4ff;
+  --color-primary-fixed-dim: #00dbe9;
+  --color-on-primary-fixed: #002022;
+  --color-on-primary-fixed-variant: #004f54;
+  --color-inverse-primary: #006970;
+
+  --color-secondary: #4edea3;
+  --color-secondary-container: #00a572;
+  --color-on-secondary: #003824;
+  --color-on-secondary-container: #00311f;
+  --color-secondary-fixed: #6ffbbe;
+  --color-secondary-fixed-dim: #4edea3;
+  --color-on-secondary-fixed: #002113;
+  --color-on-secondary-fixed-variant: #005236;
+
+  --color-tertiary: #fff3f3;
+  --color-tertiary-container: #ffcdd1;
+  --color-on-tertiary: #670020;
+  --color-on-tertiary-container: #be0042;
+  --color-tertiary-fixed: #ffd9dc;
+  --color-tertiary-fixed-dim: #ffb2ba;
+  --color-on-tertiary-fixed: #400011;
+  --color-on-tertiary-fixed-variant: #910030;
+
+  --color-error: #ffb4ab;
+  --color-error-container: #93000a;
+  --color-on-error: #690005;
+  --color-on-error-container: #ffdad6;
+
+  --color-outline: #849495;
+  --color-outline-variant: #3b494b;
+
+  --font-headline: "Geist", sans-serif;
+  --font-body: "Inter", sans-serif;
+  --font-mono: "JetBrains Mono", monospace;
+}
+
+@layer base {
+  html, body {
+    margin: 0;
+    padding: 0;
+    background-color: #0f131c;
+    color: #dfe2ee;
+    font-family: "Inter", -apple-system, BlinkMacSystemFont, sans-serif;
+  }
+  body {
+    overscroll-behavior: none;
+  }
+}
+
+.material-symbols-outlined {
+  font-family: 'Material Symbols Outlined';
+  font-weight: normal;
+  font-style: normal;
+  font-size: 20px;
+  line-height: 1;
+  letter-spacing: normal;
+  text-transform: none;
+  display: inline-block;
+  white-space: nowrap;
+  word-wrap: normal;
+  direction: ltr;
+  -webkit-font-feature-settings: 'liga';
+  -webkit-font-smoothing: antialiased;
+}

otel-ui/src/lib/auth.ts

@@ -0,0 +1,240 @@
+// OIDC authorization-code flow with PKCE against the go-auth service.
+// Client is public (no secret); tokens live in localStorage.
+
+const ISSUER =
+  (import.meta.env.VITE_OIDC_ISSUER as string | undefined) ??
+  "https://users.k6n.net";
+const CLIENT_ID =
+  (import.meta.env.VITE_OIDC_CLIENT_ID as string | undefined) ?? "otel-debug";
+const SCOPE = "openid profile email offline_access";
+
+const TOKEN_KEY = "otel.auth.tokens";
+
+export interface Tokens {
+  accessToken: string;
+  idToken: string;
+  refreshToken?: string;
+  expiresAt: number; // epoch ms
+}
+
+export interface UserClaims {
+  sub: string;
+  name?: string;
+  email?: string;
+  roles?: string[];
+}
+
+export class AuthRequiredError extends Error {
+  constructor() {
+    super("Authentication required")
+    this.name = "AuthRequiredError"
+  }
+}
+
+function b64url(bytes: Uint8Array): string {
+  return btoa(String.fromCharCode(...bytes))
+    .replace(/\+/g, "-")
+    .replace(/\//g, "_")
+    .replace(/=+$/, "");
+}
+
+function readTokens(): Tokens | null {
+  if (typeof window === "undefined") return null;
+  const raw = localStorage.getItem(TOKEN_KEY);
+  if (!raw) return null;
+  try {
+    return JSON.parse(raw) as Tokens;
+  } catch {
+    return null;
+  }
+}
+
+// tokens() returns tokens if the access token is currently valid.
+export function tokens(): Tokens | null {
+  const t = readTokens();
+  return t && t.expiresAt > Date.now() + 10_000 ? t : null;
+}
+
+export function hasSession(): boolean {
+  return readTokens() !== null;
+}
+
+let refreshing: Promise<Tokens | null> | null = null;
+
+// ensureTokens() returns valid tokens, refreshing if close to expiration.
+export async function ensureTokens(): Promise<Tokens | null> {
+  const t = readTokens();
+  if (!t) return null;
+  if (t.expiresAt > Date.now() + 30_000) return t;
+  if (!t.refreshToken) return null;
+
+  refreshing ??= doRefresh(t).finally(() => {
+    refreshing = null;
+  });
+  return refreshing;
+}
+
+async function doRefresh(t: Tokens): Promise<Tokens | null> {
+  try {
+    const res = await fetch("/api/auth/token", {
+      method: "POST",
+      headers: { "Content-Type": "application/x-www-form-urlencoded" },
+      body: new URLSearchParams({
+        grant_type: "refresh_token",
+        client_id: CLIENT_ID,
+        refresh_token: t.refreshToken!,
+      }),
+    });
+    if (!res.ok) {
+      localStorage.removeItem(TOKEN_KEY);
+      return null;
+    }
+    const body = (await res.json()) as TokenResponse;
+    saveTokens(body, t.refreshToken);
+    return readTokens();
+  } catch {
+    return null;
+  }
+}
+
+interface TokenResponse {
+  access_token: string;
+  id_token: string;
+  refresh_token?: string;
+  expires_in: number;
+}
+
+function saveTokens(body: TokenResponse, fallbackRefresh?: string) {
+  if (typeof window === "undefined") return;
+  localStorage.setItem(
+    TOKEN_KEY,
+    JSON.stringify({
+      accessToken: body.access_token,
+      idToken: body.id_token,
+      refreshToken: body.refresh_token ?? fallbackRefresh,
+      expiresAt: Date.now() + body.expires_in * 1000,
+    } satisfies Tokens),
+  );
+}
+
+export function currentUser(): UserClaims | null {
+  const t = tokens();
+  if (!t || !t.idToken) return null;
+  try {
+    const payload = t.idToken.split(".")[1];
+    return JSON.parse(
+      atob(payload.replace(/-/g, "+").replace(/_/g, "/")),
+    ) as UserClaims;
+  } catch {
+    return null;
+  }
+}
+
+export function logout() {
+  if (typeof window === "undefined") return;
+  localStorage.removeItem(TOKEN_KEY);
+  const end = `${ISSUER}/oidc/v1/end_session`;
+  window.location.assign(
+    `${end}?post_logout_redirect_uri=${encodeURIComponent(window.location.origin)}`,
+  );
+}
+
+function redirectUri(): string {
+  return `${window.location.origin}/auth_callback`;
+}
+
+export async function login() {
+  if (typeof window === "undefined") return;
+  const verifierBytes = crypto.getRandomValues(new Uint8Array(32));
+  const verifier = b64url(verifierBytes);
+  const challenge = b64url(
+    new Uint8Array(
+      await crypto.subtle.digest(
+        "SHA-256",
+        new TextEncoder().encode(verifier),
+      ),
+    ),
+  );
+  const state = b64url(crypto.getRandomValues(new Uint8Array(16)));
+
+  sessionStorage.setItem(
+    "otel.auth.pkce",
+    JSON.stringify({
+      verifier,
+      state,
+      back: window.location.pathname + window.location.search,
+    }),
+  );
+
+  const params = new URLSearchParams({
+    client_id: CLIENT_ID,
+    redirect_uri: redirectUri(),
+    response_type: "code",
+    scope: SCOPE,
+    code_challenge: challenge,
+    code_challenge_method: "S256",
+    state,
+  });
+
+  window.location.assign(`${ISSUER}/oauth/v2/authorize?${params}`);
+}
+
+export async function handleCallback(): Promise<string | null> {
+  if (typeof window === "undefined") return null;
+  const url = new URL(window.location.href);
+  const code = url.searchParams.get("code");
+  const state = url.searchParams.get("state");
+  if (!code) return null;
+
+  const saved = JSON.parse(
+    sessionStorage.getItem("otel.auth.pkce") ?? "null",
+  ) as { verifier: string; state: string; back: string } | null;
+  sessionStorage.removeItem("otel.auth.pkce");
+
+  if (!saved || state !== saved.state) {
+    throw new Error("OIDC state mismatch");
+  }
+
+  const res = await fetch("/api/auth/token", {
+    method: "POST",
+    headers: { "Content-Type": "application/x-www-form-urlencoded" },
+    body: new URLSearchParams({
+      grant_type: "authorization_code",
+      client_id: CLIENT_ID,
+      redirect_uri: redirectUri(),
+      code,
+      code_verifier: saved.verifier,
+    }),
+  });
+
+  if (!res.ok) {
+    throw new Error(`Token exchange failed with status ${res.status}`);
+  }
+
+  saveTokens((await res.json()) as TokenResponse);
+  const destination = saved.back || "/";
+  window.history.replaceState(null, "", destination);
+  return destination;
+}
+
+export async function authFetch(
+  input: RequestInfo | URL,
+  init?: RequestInit,
+): Promise<Response> {
+  const t = await ensureTokens();
+  // Protected API callers must never send anonymous requests. Apart from
+  // avoiding unnecessary 401s, this prevents the RUM error reporter from
+  // turning an expired session into an issues-refresh loop.
+  if (!t?.accessToken) throw new AuthRequiredError()
+  const headers = new Headers(init?.headers);
+  if (t?.accessToken && !headers.has("Authorization")) {
+    headers.set("Authorization", `Bearer ${t.accessToken}`);
+  }
+  return fetch(input, { ...init, headers });
+}
+
+export async function authSWRFetcher<T>(key: string): Promise<T> {
+  const response = await authFetch(key)
+  if (!response.ok) throw new Error(`HTTP ${response.status}`)
+  return response.json() as Promise<T>
+}

otel-ui/src/lib/ingest.ts

@@ -0,0 +1,85 @@
+// Shared message-partitioning logic used by every ingestion path: the
+// initial history fetch, live WebSocket messages, and on-demand trace
+// fetches (GET /api/traces/{traceId}). Keeping this in one place means all
+// three stay consistent instead of drifting.
+import type { LogEntry, MetricData, OtelMessage, TraceData } from "./types"
+import { normalizeLogEntry, normalizeMetricData, normalizeTrace } from "./normalize"
+
+export interface PartitionedMessages {
+  traces: TraceData[]
+  metrics: MetricData[]
+  logs: LogEntry[]
+  logsByTraceId: Record<string, LogEntry[]>
+  traceIds: Set<string>
+}
+
+export function partitionMessages(messages: OtelMessage[]): PartitionedMessages {
+  const traces: TraceData[] = []
+  const metrics: MetricData[] = []
+  const logs: LogEntry[] = []
+  const logsByTraceId: Record<string, LogEntry[]> = {}
+  const traceIds = new Set<string>()
+
+  messages.forEach((message) => {
+    switch (message.type) {
+      case "traces": {
+        const normalized = normalizeTrace(message.data)
+        traces.push(normalized)
+        normalized.resourceSpans.forEach((rs) =>
+          rs.scopeSpans.forEach((ss) =>
+            ss.spans.forEach((span) => {
+              if (span.traceId) traceIds.add(span.traceId)
+            }),
+          ),
+        )
+        break
+      }
+      case "metrics":
+        metrics.push(normalizeMetricData(message.data))
+        break
+      case "logs": {
+        const logsWithId = (message.data.resourceLogs ?? []).map((log) =>
+          normalizeLogEntry({
+            ...log,
+            id: crypto.randomUUID(),
+            scopeLogs: (log.scopeLogs ?? []).map((sl) => ({
+              ...sl,
+              logRecords: (sl.logRecords ?? []).map((record) => ({
+                ...record,
+                uid: record.uid ?? crypto.randomUUID(),
+              })),
+            })),
+          }),
+        )
+        logsWithId.forEach((log) => {
+          logs.push(log)
+          log.scopeLogs.forEach((scopeLog) => {
+            scopeLog.logRecords.forEach((record) => {
+              if (record.traceId) {
+                if (!logsByTraceId[record.traceId]) {
+                  logsByTraceId[record.traceId] = []
+                }
+                logsByTraceId[record.traceId].push(log)
+              }
+            })
+          })
+        })
+        break
+      }
+    }
+  })
+
+  return { traces, metrics, logs, logsByTraceId, traceIds }
+}
+
+export function mergeLogsByTraceId(
+  prev: Record<string, LogEntry[]>,
+  additions: Record<string, LogEntry[]>,
+): Record<string, LogEntry[]> {
+  if (Object.keys(additions).length === 0) return prev
+  const next = { ...prev }
+  Object.entries(additions).forEach(([traceId, entries]) => {
+    next[traceId] = [...(next[traceId] ?? []), ...entries]
+  })
+  return next
+}

otel-ui/src/lib/issue-utils.ts

@@ -0,0 +1,95 @@
+// Helpers for presenting backend issues in the UI: trend sparkline data,
+// breadcrumb parsing and triage-state presentation.
+import type { Issue, IssueSample } from "./types"
+
+export interface TrendPoint {
+  hour: string
+  count: number
+}
+
+/**
+ * Converts an issue's sparse hourly trend map into a dense series covering
+ * the full look-back window, oldest first - ready for a sparkline.
+ */
+export function trendPoints(issue: Issue, hours = 24): TrendPoint[] {
+  const now = new Date()
+  const currentHour = new Date(
+    Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate(), now.getUTCHours()),
+  )
+  const points: TrendPoint[] = []
+  for (let i = hours - 1; i >= 0; i--) {
+    const d = new Date(currentHour.getTime() - i * 3_600_000)
+    const hour = d.toISOString().slice(0, 13) + ":00"
+    points.push({ hour, count: issue.trend?.[hour] ?? 0 })
+  }
+  return points
+}
+
+/** True when the issue has at least one occurrence in the last hour. */
+export function isSpike(issue: Issue): boolean {
+  const points = trendPoints(issue, 1)
+  return (points[0]?.count ?? 0) > 1
+}
+
+export interface ParsedBreadcrumb {
+  t: number
+  category: string
+  message: string
+}
+
+/** Safely parses the raw breadcrumb JSON attached to a frontend sample. */
+export function parseBreadcrumbs(raw?: string): ParsedBreadcrumb[] {
+  if (!raw) return []
+  try {
+    const parsed = JSON.parse(raw)
+    if (!Array.isArray(parsed)) return []
+    return parsed.filter(
+      (b): b is ParsedBreadcrumb =>
+        b && typeof b === "object" && typeof b.category === "string" && typeof b.message === "string",
+    )
+  } catch {
+    return []
+  }
+}
+
+export type IssueStatus = "open" | "resolved" | "ignored"
+
+export const issueStatusMeta: Record<IssueStatus, { label: string; className: string }> = {
+  open: { label: "Open", className: "bg-rose-500/10 text-rose-300 border-rose-500/20" },
+  resolved: {
+    label: "Resolved",
+    className: "bg-emerald-500/10 text-emerald-300 border-emerald-500/20",
+  },
+  ignored: { label: "Ignored", className: "bg-slate-500/10 text-slate-400 border-white/10" },
+}
+
+export function issueStatusOf(issue: Issue): IssueStatus {
+  if (issue.status === "resolved" || issue.status === "ignored") return issue.status
+  return "open"
+}
+
+/** Most informative stacktrace across an issue's samples, if any. */
+export function latestStacktrace(issue: Issue): string | undefined {
+  for (let i = issue.samples.length - 1; i >= 0; i--) {
+    if (issue.samples[i]?.resolvedStacktrace) return issue.samples[i].resolvedStacktrace
+    if (issue.samples[i]?.stacktrace) return issue.samples[i].stacktrace
+  }
+  return undefined
+}
+
+/** Latest sample carrying breadcrumbs (frontend issues), if any. */
+export function latestBreadcrumbs(issue: Issue): ParsedBreadcrumb[] {
+  for (let i = issue.samples.length - 1; i >= 0; i--) {
+    const parsed = parseBreadcrumbs(issue.samples[i]?.breadcrumbs)
+    if (parsed.length) return parsed
+  }
+  return []
+}
+
+export function sampleContext(sample: IssueSample): string[] {
+  const parts: string[] = []
+  if (sample.httpMethod) parts.push(sample.httpMethod)
+  if (sample.httpRoute) parts.push(sample.httpRoute)
+  if (sample.httpStatus) parts.push(String(sample.httpStatus))
+  return parts
+}

otel-ui/src/lib/normalize.ts

@@ -0,0 +1,74 @@
+// OTLP JSON exporters omit fields when empty, so any of the arrays below can
+// be undefined at runtime even though the types say otherwise. Normalize every
+// message once at ingestion so consumers can rely on the declared types.
+import type { LogEntry, LogRecord, MetricData, TraceData } from "./types"
+
+const attrList = (value: unknown): LogRecord["attributes"] =>
+  Array.isArray(value) ? (value as LogRecord["attributes"]) : []
+
+const normalizeLogRecord = (record: any): LogRecord => ({
+  ...record,
+  attributes: attrList(record?.attributes),
+})
+
+export const normalizeLogEntry = (log: any): LogEntry => ({
+  ...log,
+  id: log?.id ?? crypto.randomUUID(),
+  resource: {
+    ...log?.resource,
+    attributes: attrList(log?.resource?.attributes),
+  },
+  scopeLogs: attrList(log?.scopeLogs).map((scopeLog: any) => ({
+    ...scopeLog,
+    scope: scopeLog?.scope ?? { name: "unknown" },
+    logRecords: attrList(scopeLog?.logRecords).map(normalizeLogRecord),
+  })),
+})
+
+export const normalizeTrace = (trace: any): TraceData => ({
+  ...trace,
+  resourceSpans: attrList(trace?.resourceSpans).map((resourceSpan: any) => ({
+    ...resourceSpan,
+    resource: {
+      ...resourceSpan?.resource,
+      attributes: attrList(resourceSpan?.resource?.attributes),
+    },
+    scopeSpans: attrList(resourceSpan?.scopeSpans).map((scopeSpan: any) => ({
+      ...scopeSpan,
+      scope: scopeSpan?.scope ?? { name: "unknown" },
+      spans: attrList(scopeSpan?.spans).map((span: any) => ({
+        ...span,
+        attributes: attrList(span?.attributes),
+        events: attrList(span?.events),
+        status: span?.status ?? {},
+      })),
+    })),
+  })),
+})
+
+export const normalizeMetricData = (data: any): MetricData => ({
+  ...data,
+  resourceMetrics: attrList(data?.resourceMetrics).map((resourceMetric: any) => ({
+    ...resourceMetric,
+    resource: {
+      ...resourceMetric?.resource,
+      attributes: attrList(resourceMetric?.resource?.attributes),
+    },
+    scopeMetrics: attrList(resourceMetric?.scopeMetrics).map((scopeMetric: any) => ({
+      ...scopeMetric,
+      scope: scopeMetric?.scope ?? { name: "unknown" },
+      metrics: attrList(scopeMetric?.metrics).map((metric: any) => ({
+        ...metric,
+        gauge: metric?.gauge
+          ? { ...metric.gauge, dataPoints: attrList(metric.gauge.dataPoints) }
+          : undefined,
+        sum: metric?.sum
+          ? { ...metric.sum, dataPoints: attrList(metric.sum.dataPoints) }
+          : undefined,
+        histogram: metric?.histogram
+          ? { ...metric.histogram, dataPoints: attrList(metric.histogram.dataPoints) }
+          : undefined,
+      })),
+    })),
+  })),
+})

otel-ui/src/lib/otel-utils.ts

@@ -0,0 +1,209 @@
+import type { AttributeValue, OtelAttribute } from "./types";
+
+export function getAttributeValue(
+  attributes: OtelAttribute[],
+  key: string,
+): string {
+  const lowerKey = key.toLowerCase();
+  const attr = attributes.find((a) => a.key === key || a.key.toLowerCase() === lowerKey);
+  if (!attr) return "";
+
+  const { value } = attr;
+  if (value.stringValue) return value.stringValue;
+  if (value.intValue) return value.intValue;
+  if (value.doubleValue !== undefined) return value.doubleValue.toString();
+  if (value.boolValue !== undefined) return value.boolValue.toString();
+
+  return "";
+}
+
+export function formatTimestamp(nanoTimestamp: string): string {
+  const ms = Number(nanoTimestamp) / 1_000_000;
+  return new Date(ms).toLocaleString();
+}
+
+export function formatDuration(startNano: string, endNano: string): string {
+  const durationMs = (Number(endNano) - Number(startNano)) / 1_000_000;
+
+  if (durationMs < 1) {
+    return `${(durationMs * 1000).toFixed(2)}μs`;
+  } else if (durationMs < 1000) {
+    return `${durationMs.toFixed(2)}ms`;
+  } else {
+    return `${(durationMs / 1000).toFixed(2)}s`;
+  }
+}
+
+export function getSpanKindLabel(kind: number): string {
+  const kinds = [
+    "UNSPECIFIED",
+    "INTERNAL",
+    "SERVER",
+    "CLIENT",
+    "PRODUCER",
+    "CONSUMER",
+  ];
+  return kinds[kind] || "UNKNOWN";
+}
+
+export function getServiceName(
+  resourceAttributes?: OtelAttribute[],
+  recordAttributes?: OtelAttribute[],
+): string {
+  const keys = ["service.name", "app.kubernetes.io/name", "app", "name"];
+  if (resourceAttributes) {
+    for (const k of keys) {
+      const v = getAttributeValue(resourceAttributes, k);
+      if (v && v !== "unknown" && v !== "unknown-service") return v;
+    }
+  }
+  if (recordAttributes) {
+    for (const k of keys) {
+      const v = getAttributeValue(recordAttributes, k);
+      if (v && v !== "unknown" && v !== "unknown-service") return v;
+    }
+  }
+  return "unknown-service";
+}
+
+export function isSpanError(span: {
+  status?: { code?: number };
+  attributes?: OtelAttribute[];
+}): boolean {
+  if (span.status?.code === 2) return true;
+  const statusStr =
+    getAttributeValue(span.attributes ?? [], "http.response.status_code") ||
+    getAttributeValue(span.attributes ?? [], "http.status_code");
+  if (statusStr) {
+    const code = parseInt(statusStr, 10);
+    if (!isNaN(code) && code >= 400) return true;
+  }
+  const grpcStr = getAttributeValue(span.attributes ?? [], "rpc.grpc.status_code");
+  if (grpcStr) {
+    const code = parseInt(grpcStr, 10);
+    if (!isNaN(code) && code > 0) return true;
+  }
+  return false;
+}
+
+export function getPodName(
+  resourceAttributes?: OtelAttribute[],
+  pointAttributes?: OtelAttribute[],
+): string {
+  const keys = [
+    "k8s.pod.name",
+    "pod.name",
+    "pod",
+    "pod_name",
+    "k8s_pod_name",
+    "service.instance.id",
+    "service_instance_id",
+    "instance.id",
+    "k8s.pod.uid",
+    "container.name",
+    "container_name",
+    "container.id",
+    "host.name",
+    "hostname",
+    "host",
+    "instance",
+    "process.pid",
+  ];
+  if (resourceAttributes) {
+    for (const k of keys) {
+      const v = getAttributeValue(resourceAttributes, k);
+      if (v) return v;
+    }
+  }
+  if (pointAttributes) {
+    for (const k of keys) {
+      const v = getAttributeValue(pointAttributes, k);
+      if (v) return v;
+    }
+  }
+  return "";
+}
+
+export const POD_COLORS = [
+  { stroke: "#38bdf8", fill: "#38bdf8", text: "text-sky-400", bg: "bg-sky-500/10", border: "border-sky-500/20" },
+  { stroke: "#a855f7", fill: "#a855f7", text: "text-purple-400", bg: "bg-purple-500/10", border: "border-purple-500/20" },
+  { stroke: "#34d399", fill: "#34d399", text: "text-emerald-400", bg: "bg-emerald-500/10", border: "border-emerald-500/20" },
+  { stroke: "#fbbf24", fill: "#fbbf24", text: "text-amber-400", bg: "bg-amber-500/10", border: "border-amber-500/20" },
+  { stroke: "#fb7185", fill: "#fb7185", text: "text-rose-400", bg: "bg-rose-500/10", border: "border-rose-500/20" },
+  { stroke: "#22d3ee", fill: "#22d3ee", text: "text-cyan-400", bg: "bg-cyan-500/10", border: "border-cyan-500/20" },
+  { stroke: "#f472b6", fill: "#f472b6", text: "text-pink-400", bg: "bg-pink-500/10", border: "border-pink-500/20" },
+  { stroke: "#818cf8", fill: "#818cf8", text: "text-indigo-400", bg: "bg-indigo-500/10", border: "border-indigo-500/20" },
+];
+
+export function getPodPalette(pod: string) {
+  return POD_COLORS[hashString(pod) % POD_COLORS.length] ?? POD_COLORS[0];
+}
+
+export const formatAttributeValue = (value: AttributeValue): string => {
+  if (value.stringValue !== undefined) return value.stringValue;
+  if (value.intValue !== undefined) return value.intValue.toString();
+  if (value.doubleValue !== undefined) return value.doubleValue.toString();
+  if (value.boolValue !== undefined) return value.boolValue.toString();
+  return "unknown";
+};
+
+export const servicePalettes = [
+  {
+    gradient: "from-violet-600 via-indigo-600 to-sky-600",
+    ring: "ring-violet-400/40",
+    text: "text-violet-400",
+    bg: "bg-violet-400/10",
+    border: "border-violet-400/20",
+  },
+  {
+    gradient: "from-emerald-500 via-teal-600 to-cyan-700",
+    ring: "ring-emerald-400/40",
+    text: "text-emerald-400",
+    bg: "bg-emerald-400/10",
+    border: "border-emerald-400/20",
+  },
+  {
+    gradient: "from-amber-600 via-orange-600 to-rose-600",
+    ring: "ring-amber-400/40",
+    text: "text-amber-400",
+    bg: "bg-amber-400/10",
+    border: "border-amber-400/20",
+  },
+  {
+    gradient: "from-fuchsia-600 via-purple-600 to-indigo-700",
+    ring: "ring-purple-400/40",
+    text: "text-fuchsia-400",
+    bg: "bg-fuchsia-400/10",
+    border: "border-fuchsia-400/20",
+  },
+  {
+    gradient: "from-sky-500 via-blue-600 to-indigo-700",
+    ring: "ring-sky-400/40",
+    text: "text-sky-400",
+    bg: "bg-sky-400/10",
+    border: "border-sky-400/20",
+  },
+  {
+    gradient: "from-lime-500 via-emerald-600 to-teal-700",
+    ring: "ring-lime-400/40",
+    text: "text-lime-400",
+    bg: "bg-lime-400/10",
+    border: "border-lime-400/20",
+  },
+];
+
+export const hashString = (value: string): number => {
+  let hash = 0;
+  for (let i = 0; i < value.length; i += 1) {
+    hash = (hash << 5) - hash + value.charCodeAt(i);
+    hash |= 0;
+  }
+  return Math.abs(hash);
+};
+
+export const getServicePalette = (service: string) => {
+  const palette =
+    servicePalettes[hashString(service) % servicePalettes.length] ??
+    servicePalettes[0];
+  return palette;
+};

otel-ui/src/lib/routes.ts

@@ -0,0 +1,31 @@
+// Route ↔ tab mapping. Tabs live in the URL (wouter) so deep links survive
+// the OIDC login redirect at the ingress: users.k6n.net sends you back to
+// exactly the view you requested, and URLs are shareable/bookmarkable.
+// The Go server's SPA fallback serves index.html for all of these paths.
+export const TAB_PATHS: Record<string, string> = {
+  "/issues": "tab0",
+  "/patterns": "tab8",
+  "/rollouts": "tab9",
+  "/traces": "tab1",
+  "/metrics": "tab2",
+  "/logs": "tab3",
+  "/setup": "tab4",
+  "/trends": "tab5",
+  "/search": "tab6",
+  "/frontend": "tab7",
+  "/rum": "tab7",
+}
+
+
+/** Tab value for a location pathname, or null when the path isn't a tab. */
+export function tabForPath(path: string): string | null {
+  return TAB_PATHS[path] ?? null
+}
+
+/** Canonical path for a tab value, or null when the tab has no route. */
+export function pathForTab(tab: string): string | null {
+  for (const [path, value] of Object.entries(TAB_PATHS)) {
+    if (value === tab) return path
+  }
+  return null
+}

otel-ui/src/lib/timeUtils.ts

@@ -0,0 +1,25 @@
+export function formatDuration(nano: number): string {
+  if (nano >= 3600000000000) { // 1 hour in nanoseconds
+    return `${(nano / 3600000000000).toFixed(1)}h`;
+  } else if (nano >= 60000000000) { // 1 minute
+    return `${(nano / 60000000000).toFixed(1)}m`;
+  } else if (nano >= 1000000000) { // 1 second
+    return `${(nano / 1000000000).toFixed(1)}s`;
+  } else if (nano >= 1000000) { // 1 millisecond
+    return `${(nano / 1000000).toFixed(1)}ms`;
+  } else if (nano >= 1000) { // 1 microsecond
+    return `${(nano / 1000).toFixed(1)}μs`;
+  } else {
+    return `${nano.toFixed(1)}ns`;
+  }
+}
+
+/** "just now", "2m ago", "3h ago", "5d ago" */
+export function formatRelativeTime(msEpoch: number): string {
+  const seconds = Math.max(0, (Date.now() - msEpoch) / 1000);
+  if (seconds < 60) return "just now";
+  if (seconds < 3600) return `${Math.floor(seconds / 60)}m ago`;
+  if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ago`;
+  return `${Math.floor(seconds / 86400)}d ago`;
+}
+

otel-ui/src/lib/types.ts

@@ -0,0 +1,268 @@
+// OpenTelemetry data types
+
+export interface OtelAttribute {
+  key: string;
+  value: AttributeValue;
+}
+
+export interface AttributeValue {
+  stringValue?: string;
+  intValue?: string;
+  doubleValue?: number;
+  boolValue?: boolean;
+}
+
+export interface SpanEvent {
+  name: string;
+  timeUnixNano: string;
+  attributes: OtelAttribute[];
+}
+
+export interface Span {
+  traceId: string;
+  spanId: string;
+  parentSpanId?: string;
+  name: string;
+  kind: number;
+  startTimeUnixNano: string;
+  endTimeUnixNano: string;
+  attributes: OtelAttribute[];
+  events: SpanEvent[];
+  status: Record<string, unknown>;
+  flags: number;
+}
+
+export interface ScopeSpan {
+  scope: {
+    name: string;
+  };
+  spans: Span[];
+}
+
+export interface ResourceSpan {
+  resource: {
+    attributes: OtelAttribute[];
+  };
+  schemaUrl?: string;
+  scopeSpans: ScopeSpan[];
+}
+
+export interface TraceData {
+  resourceSpans: ResourceSpan[];
+}
+
+export interface MetricDataPoint {
+  asInt?: string;
+  asDouble?: number;
+  attributes: OtelAttribute[];
+  timeUnixNano?: string;
+  pod?: string;
+  // Histogram-specific fields
+  bucketCounts?: string[];
+  count?: string;
+  exemplars?: Array<{
+    asInt?: string;
+    asDouble?: number;
+    spanId: string;
+    timeUnixNano: string;
+    traceId: string;
+  }>;
+  explicitBounds?: number[];
+  max?: number;
+  min?: number;
+  startTimeUnixNano?: string;
+  sum?: number;
+}
+
+export interface Metric {
+  name: string;
+  gauge?: {
+    dataPoints: MetricDataPoint[];
+  };
+  sum?: {
+    dataPoints: MetricDataPoint[];
+  };
+  histogram?: {
+    dataPoints: MetricDataPoint[];
+  };
+}
+
+export interface ScopeMetric {
+  scope: {
+    name: string;
+  };
+  metrics: Metric[];
+}
+
+export interface ResourceMetric {
+  resource: {
+    attributes: OtelAttribute[];
+  };
+  scopeMetrics: ScopeMetric[];
+}
+
+export interface MetricData {
+  resourceMetrics: ResourceMetric[];
+}
+
+export interface LogRecord {
+  uid: string;
+  timeUnixNano: string;
+  body: {
+    stringValue: string;
+  };
+  attributes: OtelAttribute[];
+  severityNumber?: number;
+  severityText?: string;
+  traceId?: string;
+  spanId?: string;
+}
+
+export interface ScopeLog {
+  scope: {
+    name: string;
+  };
+  logRecords: LogRecord[];
+}
+
+export interface ResourceLog {
+  resource: {
+    attributes: OtelAttribute[];
+  };
+  scopeLogs: ScopeLog[];
+}
+
+export type LogEntry = ResourceLog & {
+  id: string;
+}
+
+export interface LogData {
+  resourceLogs: ResourceLog[];
+}
+
+export interface OtelTraceData {
+  type: "traces";
+
+  data: TraceData;
+}
+
+export interface OtelMetricData {
+  type: "metrics";
+
+  data: MetricData;
+}
+
+export interface OtelLogData {
+  type: "logs";
+
+  data: LogData;
+}
+
+export type OtelHistoryMessage = OtelMessage & {
+  timestamp: number;
+};
+
+export type OtelMessage = OtelTraceData | OtelMetricData | OtelLogData;
+
+// Deduplicated errors derived server-side from traces/logs. See
+// GET /api/issues and GET /api/issues/{id} in the go-otel backend.
+export interface IssueSample {
+  timestamp: number;
+  traceId?: string;
+  spanId?: string;
+  message: string;
+  url?: string;
+  stacktrace?: string;
+  resolvedStacktrace?: string;
+  httpMethod?: string;
+  httpRoute?: string;
+  httpStatus?: number;
+  userAgent?: string;
+  sourceIp?: string;
+  release?: string;
+  environment?: string;
+  /** Raw JSON breadcrumb trail captured by otel-rum.js (frontend issues). */
+  breadcrumbs?: string;
+}
+
+export interface Issue {
+  id: string;
+  title: string;
+  kind: "span_error" | "log_error" | "frontend_error" | string;
+  service: string;
+  count: number;
+  firstSeen: number;
+  lastSeen: number;
+  affectedServices: string[];
+  samples: IssueSample[];
+  /** UTC hour-bucket ("2026-09-17T14:00") -> occurrence count, last 48h. */
+  trend?: Record<string, number>;
+  status?: "open" | "resolved" | "ignored" | string;
+  resolvedAt?: number;
+  normalizedTitle?: string;
+  patternId?: string;
+  patternSignature?: string;
+}
+
+export interface SimilarIssue {
+  issue: Issue;
+  score: number;
+  reasons: string[];
+  decision?: "same" | "different" | string;
+}
+
+export interface IssuePattern {
+  id: string;
+  signature: string;
+  service: string;
+  kind: string;
+  issueCount: number;
+  occurrenceCount: number;
+  lastSeen: number;
+  issueIds: string[];
+}
+
+export interface Rollout {
+  id: string;
+  namespace: string;
+  workload: string;
+  service?: string;
+  release?: string;
+  revision?: string;
+  image?: string;
+  phase: "progressing" | "completed" | "failed" | string;
+  reason?: string;
+  message?: string;
+  startedAt: number;
+  completedAt?: number;
+  lastSeen: number;
+  generation: number;
+  desiredReplicas: number;
+  updatedReplicas: number;
+  availableReplicas: number;
+  readyReplicas: number;
+}
+
+export interface RolloutImpact {
+  configured: boolean;
+  rollout: Rollout;
+  points: Array<{ timestamp: number; value: string }>;
+}
+
+// ---- Aggregate stats (ClickHouse-backed /api/stats/*) -------------------
+
+export interface StatsPoint {
+  /** ISO-ish bucket label from the server, e.g. "2026-09-17 14:00:00". */
+  bucket: string;
+  total: number;
+  errors: number;
+}
+
+export interface ErrorStatsResponse {
+  granularity: "hour" | "day";
+  points: StatsPoint[];
+}
+
+export interface ReleaseStatsResponse {
+  series: Record<string, StatsPoint[]>;
+}

otel-ui/src/main.tsx

@@ -0,0 +1,31 @@
+import { StrictMode } from "react";
+import { createRoot } from "react-dom/client";
+import { Router } from "wouter";
+import "./index.css";
+import App from "./App.tsx";
+import { ObservabilityProvider } from "./contexts/ObservabilityContextProvider.tsx";
+import { ViewErrorBoundary } from "./components/ViewErrorBoundary";
+
+// Use the project's own browser telemetry for UI errors, failed requests and
+// Web Vitals. The script is loaded before this module in index.html.
+const rum = (window as Window & {
+  OtelRUM?: { init: (config: Record<string, unknown>) => unknown }
+}).OtelRUM;
+rum?.init({
+  serviceName: "otel-ui",
+  release: (import.meta.env.VITE_APP_RELEASE as string | undefined) ?? "dev",
+  baseUrl: window.location.origin,
+  endpoint: "/v1/logs",
+});
+
+createRoot(document.getElementById("root")!).render(
+  <StrictMode>
+    <Router>
+      <ObservabilityProvider>
+        <ViewErrorBoundary>
+          <App />
+        </ViewErrorBoundary>
+      </ObservabilityProvider>
+    </Router>
+  </StrictMode>,
+);

otel-ui/src/styles/globals.css

@@ -0,0 +1,125 @@
+@import 'tailwindcss';
+@import 'tw-animate-css';
+
+@custom-variant dark (&:is(.dark *));
+
+:root {
+  --background: oklch(1 0 0);
+  --foreground: oklch(0.145 0 0);
+  --card: oklch(1 0 0);
+  --card-foreground: oklch(0.145 0 0);
+  --popover: oklch(1 0 0);
+  --popover-foreground: oklch(0.145 0 0);
+  --primary: oklch(0.205 0 0);
+  --primary-foreground: oklch(0.985 0 0);
+  --secondary: oklch(0.97 0 0);
+  --secondary-foreground: oklch(0.205 0 0);
+  --muted: oklch(0.97 0 0);
+  --muted-foreground: oklch(0.556 0 0);
+  --accent: oklch(0.97 0 0);
+  --accent-foreground: oklch(0.205 0 0);
+  --destructive: oklch(0.577 0.245 27.325);
+  --destructive-foreground: oklch(0.577 0.245 27.325);
+  --border: oklch(0.922 0 0);
+  --input: oklch(0.922 0 0);
+  --ring: oklch(0.708 0 0);
+  --chart-1: oklch(0.646 0.222 41.116);
+  --chart-2: oklch(0.6 0.118 184.704);
+  --chart-3: oklch(0.398 0.07 227.392);
+  --chart-4: oklch(0.828 0.189 84.429);
+  --chart-5: oklch(0.769 0.188 70.08);
+  --radius: 0.625rem;
+  --sidebar: oklch(0.985 0 0);
+  --sidebar-foreground: oklch(0.145 0 0);
+  --sidebar-primary: oklch(0.205 0 0);
+  --sidebar-primary-foreground: oklch(0.985 0 0);
+  --sidebar-accent: oklch(0.97 0 0);
+  --sidebar-accent-foreground: oklch(0.205 0 0);
+  --sidebar-border: oklch(0.922 0 0);
+  --sidebar-ring: oklch(0.708 0 0);
+}
+
+.dark {
+  --background: oklch(0.145 0 0);
+  --foreground: oklch(0.985 0 0);
+  --card: oklch(0.145 0 0);
+  --card-foreground: oklch(0.985 0 0);
+  --popover: oklch(0.145 0 0);
+  --popover-foreground: oklch(0.985 0 0);
+  --primary: oklch(0.985 0 0);
+  --primary-foreground: oklch(0.205 0 0);
+  --secondary: oklch(0.269 0 0);
+  --secondary-foreground: oklch(0.985 0 0);
+  --muted: oklch(0.269 0 0);
+  --muted-foreground: oklch(0.708 0 0);
+  --accent: oklch(0.269 0 0);
+  --accent-foreground: oklch(0.985 0 0);
+  --destructive: oklch(0.396 0.141 25.723);
+  --destructive-foreground: oklch(0.637 0.237 25.331);
+  --border: oklch(0.269 0 0);
+  --input: oklch(0.269 0 0);
+  --ring: oklch(0.439 0 0);
+  --chart-1: oklch(0.488 0.243 264.376);
+  --chart-2: oklch(0.696 0.17 162.48);
+  --chart-3: oklch(0.769 0.188 70.08);
+  --chart-4: oklch(0.627 0.265 303.9);
+  --chart-5: oklch(0.645 0.246 16.439);
+  --sidebar: oklch(0.205 0 0);
+  --sidebar-foreground: oklch(0.985 0 0);
+  --sidebar-primary: oklch(0.488 0.243 264.376);
+  --sidebar-primary-foreground: oklch(0.985 0 0);
+  --sidebar-accent: oklch(0.269 0 0);
+  --sidebar-accent-foreground: oklch(0.985 0 0);
+  --sidebar-border: oklch(0.269 0 0);
+  --sidebar-ring: oklch(0.439 0 0);
+}
+
+@theme inline {
+  --font-sans: 'Geist', 'Geist Fallback';
+  --font-mono: 'Geist Mono', 'Geist Mono Fallback';
+  --color-background: var(--background);
+  --color-foreground: var(--foreground);
+  --color-card: var(--card);
+  --color-card-foreground: var(--card-foreground);
+  --color-popover: var(--popover);
+  --color-popover-foreground: var(--popover-foreground);
+  --color-primary: var(--primary);
+  --color-primary-foreground: var(--primary-foreground);
+  --color-secondary: var(--secondary);
+  --color-secondary-foreground: var(--secondary-foreground);
+  --color-muted: var(--muted);
+  --color-muted-foreground: var(--muted-foreground);
+  --color-accent: var(--accent);
+  --color-accent-foreground: var(--accent-foreground);
+  --color-destructive: var(--destructive);
+  --color-destructive-foreground: var(--destructive-foreground);
+  --color-border: var(--border);
+  --color-input: var(--input);
+  --color-ring: var(--ring);
+  --color-chart-1: var(--chart-1);
+  --color-chart-2: var(--chart-2);
+  --color-chart-3: var(--chart-3);
+  --color-chart-4: var(--chart-4);
+  --color-chart-5: var(--chart-5);
+  --radius-sm: calc(var(--radius) - 4px);
+  --radius-md: calc(var(--radius) - 2px);
+  --radius-lg: var(--radius);
+  --radius-xl: calc(var(--radius) + 4px);
+  --color-sidebar: var(--sidebar);
+  --color-sidebar-foreground: var(--sidebar-foreground);
+  --color-sidebar-primary: var(--sidebar-primary);
+  --color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
+  --color-sidebar-accent: var(--sidebar-accent);
+  --color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
+  --color-sidebar-border: var(--sidebar-border);
+  --color-sidebar-ring: var(--sidebar-ring);
+}
+
+@layer base {
+  * {
+    @apply border-border outline-ring/50;
+  }
+  body {
+    @apply bg-background text-foreground;
+  }
+}

otel-ui/tsconfig.app.json

@@ -0,0 +1,28 @@
+{
+  "compilerOptions": {
+    "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo",
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "types": ["vite/client"],
+    "skipLibCheck": true,
+
+    /* Bundler mode */
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+    "jsx": "react-jsx",
+
+    /* Linting */
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "erasableSyntaxOnly": true,
+    "noFallthroughCasesInSwitch": true,
+    "noUncheckedSideEffectImports": true
+  },
+  "include": ["src"]
+}

otel-ui/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "files": [],
+  "references": [
+    { "path": "./tsconfig.app.json" },
+    { "path": "./tsconfig.node.json" }
+  ]
+}

otel-ui/tsconfig.node.json

@@ -0,0 +1,26 @@
+{
+  "compilerOptions": {
+    "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo",
+    "target": "ES2023",
+    "lib": ["ES2023"],
+    "module": "ESNext",
+    "types": ["node"],
+    "skipLibCheck": true,
+
+    /* Bundler mode */
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "moduleDetection": "force",
+    "noEmit": true,
+
+    /* Linting */
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "erasableSyntaxOnly": true,
+    "noFallthroughCasesInSwitch": true,
+    "noUncheckedSideEffectImports": true
+  },
+  "include": ["vite.config.ts"]
+}

otel-ui/vite.config.ts

@@ -0,0 +1,25 @@
+import { defineConfig } from "vite";
+import react from "@vitejs/plugin-react";
+import tailwindcss from "@tailwindcss/vite";
+
+// https://vite.dev/config/
+export default defineConfig({
+  plugins: [react(), tailwindcss()],
+  build: {
+    // Upload these maps to the receiver during deployment; do not publish
+    // them with the static UI.
+    sourcemap: true,
+  },
+  server: {
+    proxy: {
+      "/api": {
+        target: "http://localhost:8080",
+        changeOrigin: true,
+      },
+      "/ws": {
+        target: "ws://localhost:8080",
+        ws: true,
+      },
+    },
+  },
+});

scripts/upload-sourcemap.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${OTEL_URL:?set OTEL_URL, for example https://otel.example.com}"
+: "${RELEASE:?set RELEASE to the exact VITE_APP_RELEASE used at build time}"
+
+bundle="${1:?usage: RELEASE=... OTEL_URL=... $0 dist/assets/index-HASH.js}"
+map="${bundle}.map"
+test -f "$bundle" || { echo "bundle not found: $bundle" >&2; exit 1; }
+test -f "$map" || { echo "source map not found: $map" >&2; exit 1; }
+
+source_path="/${bundle#*/dist/}"
+curl --fail-with-body --silent --show-error \
+  -X POST "${OTEL_URL%/}/api/sourcemaps" \
+  -F "release=${RELEASE}" \
+  -F "source=${source_path}" \
+  -F "file=@${map};type=application/json"
+echo "uploaded ${source_path} for ${RELEASE}"

static/otel-rum.js

@@ -1,10 +1,11 @@
 /*!
- * otel-rum.js - minimal browser error/RUM collector for go-otel.
+ * otel-rum.js - browser error/RUM collector for go-otel.
  *
- * Captures uncaught errors, unhandled promise rejections and failed
+ * Captures uncaught errors, unhandled promise rejections, failed resource
- * resource loads, and ships them as OTLP log records to this server's
+ * loads, failing HTTP requests and Web Vitals (LCP/INP/CLS/TTFB/FCP), and
- * existing /v1/logs endpoint (no separate ingestion path needed - the
+ * ships them as OTLP log records to this server's existing /v1/logs endpoint
- * backend's issue view picks these up like any other error log).
+ * (no separate ingestion path needed - the backend's issue view picks these
+ * up like any other error log).
  *
  * Also exposes a `traceHeaders()` helper so outgoing `fetch`/XHR calls can
  * carry a W3C `traceparent` header. If the backend service is itself
@@ -16,7 +17,10 @@
  * Usage:
  *   <script src="https://otel.k6n.net/static/otel-rum.js"></script>
  *   <script>
- *     var rum = OtelRUM.init({ serviceName: 'my-web-app' });
+ *     var rum = OtelRUM.init({
+ *       serviceName: 'my-web-app',
+ *       baseUrl: 'https://otel.example.com', // optional collector override
+ *     });
  *
  *     // Manual correlation-id injection into any HTTP client:
  *     fetch('/api/orders', { headers: rum.traceHeaders() });
@@ -25,13 +29,32 @@
  *     try { risky(); } catch (err) { rum.captureException(err); }
  *   </script>
  *
- * Explicitly out of scope for v1: source map resolution (stack traces stay
+ * Events are batched client-side (flushed on pagehide via keepalive fetch/sendBeacon) and
- * minified), session replay, PII scrubbing. Don't route sensitive data
+ * rate-limited, so a runaway error loop cannot hammer the collector.
- * through breadcrumbs/attributes.
+ *
+ * Source-map resolution is performed server-side by go-otel when a matching
+ * release map has been uploaded. Session replay and PII scrubbing remain out
+ * of scope; don't route sensitive data through breadcrumbs/attributes.
  */
 (function (global) {
 	'use strict';
 
+	var DEFAULT_BASE_URL = 'https://otel.k6n.net';
+
+	// Find the loading script so declarative data-* configuration keeps working.
+	try {
+		var scriptEl = (typeof document !== 'undefined') ? (document.currentScript || (function() {
+			var scripts = document.getElementsByTagName('script');
+			for (var i = scripts.length - 1; i >= 0; i--) {
+				if (scripts[i].src && scripts[i].src.indexOf('otel-rum') !== -1) return scripts[i];
+			}
+			return null;
+		})()) : null;
+	} catch (e) {}
+
+	var rawFetch = typeof global.fetch === 'function' ? global.fetch : null;
+	var rawXHR = typeof global.XMLHttpRequest === 'function' ? global.XMLHttpRequest : null;
+
 	function randomHex(len) {
 		var chars = '0123456789abcdef';
 		var out = '';
@@ -73,17 +96,51 @@
 	function init(config) {
 		config = config || {};
 
-		var endpoint = config.endpoint || '/v1/logs';
+		var scriptServiceName = (scriptEl && (scriptEl.getAttribute('data-service-name') || scriptEl.getAttribute('data-service'))) || '';
-		var serviceName = config.serviceName || 'frontend';
+		var scriptBaseUrl = (scriptEl && scriptEl.getAttribute('data-base-url')) || '';
-		var environment = config.environment || '';
+		var scriptEndpoint = (scriptEl && scriptEl.getAttribute('data-endpoint')) || '';
-		var release = config.release || '';
+		var scriptIngestKey = (scriptEl && scriptEl.getAttribute('data-ingest-key')) || '';
-		var ingestKey = config.ingestKey || '';
+
+		// Collector base URL resolution:
+		// 1. Explicit config.baseUrl or scriptEl data-base-url
+		// 2. Production collector: https://otel.k6n.net
+		// Do not derive this from the page or script origin: the script may be
+		// bundled or self-hosted while logs still belong at the collector.
+		var baseUrl = (config.baseUrl || scriptBaseUrl || DEFAULT_BASE_URL).replace(/\/+$/, '');
+		var endpoint = config.endpoint || scriptEndpoint;
+		if (!endpoint) {
+			endpoint = baseUrl + '/v1/logs';
+		} else if (!/^https?:\/\//i.test(endpoint) && !endpoint.startsWith('//')) {
+			endpoint = baseUrl + '/' + endpoint.replace(/^\/+/, '');
+		}
+
+		function isCollectorUrl(u) {
+			if (!u) return false;
+			var s = String(u);
+			if (endpoint && s.indexOf(endpoint) === 0) return true;
+			if (s.indexOf('/v1/logs') !== -1 || s.indexOf('/v1/traces') !== -1 || s.indexOf('/v1/metrics') !== -1) return true;
+			return false;
+		}
+
+		var serviceName = config.serviceName || scriptServiceName || 'frontend';
+		var environment = config.environment || (scriptEl && scriptEl.getAttribute('data-environment')) || '';
+		var release = config.release || (scriptEl && scriptEl.getAttribute('data-release')) || '';
+		var ingestKey = config.ingestKey || scriptIngestKey || '';
 		var sampleRate = typeof config.sampleRate === 'number' ? config.sampleRate : 1;
 		var maxBreadcrumbs = config.maxBreadcrumbs || 20;
 		var captureConsoleErrors = config.captureConsoleErrors !== false;
 		var autoInstrumentFetch = config.autoInstrumentFetch !== false;
+		var autoInstrumentXhr = config.autoInstrumentXhr !== false;
+		var trackSpaRoutes = config.trackSpaRoutes !== false;
+		var reportHttpFailures = config.reportHttpFailures !== false;
+		var captureWebVitals = config.captureWebVitals !== false;
 		var injectTraceHeaders = config.injectTraceHeaders !== false;
 		var traceHeaderOrigins = config.traceHeaderOrigins; // optional array; defaults to same-origin only
+		// Client-side rate limiting: max events per rolling minute.
+		var maxEventsPerMinute = config.maxEventsPerMinute || 60;
+		// Batch size / flush interval for queued records.
+		var batchSize = config.batchSize || 10;
+		var flushIntervalMs = config.flushIntervalMs || 5000;
 
 		var breadcrumbs = [];
 		// Best-effort correlation: the most recently generated trace id is
@@ -92,37 +149,128 @@
 		// not a real per-async-context propagation mechanism.
 		var lastTraceId = '';
 
-		function addBreadcrumb(category, message) {
+		// Session id: survives reloads within the tab's lifetime, so server
-			breadcrumbs.push({ t: Date.now(), category: String(category), message: String(message) });
+		//-side issue views can group frontend errors by browsing session.
-			if (breadcrumbs.length > maxBreadcrumbs) {
+		var sessionId = '';
-				breadcrumbs.shift();
+		try {
+			sessionId = global.sessionStorage.getItem('otel-rum-session');
+			if (!sessionId) {
+				sessionId = randomHex(16);
+				global.sessionStorage.setItem('otel-rum-session', sessionId);
+			}
+		} catch (e) {
+			sessionId = randomHex(16); // storage blocked (privacy mode etc.)
+		}
+
+		// --- Client-side queue, batching and rate limiting -----------------
+
+		var queue = [];
+		var flushTimer = null;
+		var sentTimestamps = [];
+		var eventsDropped = 0;
+
+		function underRateLimit() {
+			var now = Date.now();
+			// Drop timestamps older than the rolling 60s window.
+			while (sentTimestamps.length && now - sentTimestamps[0] > 60000) {
+				sentTimestamps.shift();
+			}
+			if (sentTimestamps.length >= maxEventsPerMinute) return false;
+			sentTimestamps.push(now);
+			return true;
+		}
+
+		function enqueue(record) {
+			queue.push(record);
+			if (queue.length >= batchSize) {
+				flush();
+				return;
+			}
+			if (!flushTimer) {
+				flushTimer = global.setTimeout(flush, flushIntervalMs);
 			}
 		}
 
-		// traceHeaders returns a headers object carrying a fresh (or
+		function flush() {
-		// continued) W3C traceparent, ready to spread into any HTTP client's
+			if (flushTimer) {
-		// request headers - fetch, XHR, axios, etc.
+				global.clearTimeout(flushTimer);
-		function traceHeaders(existingTraceId) {
+				flushTimer = null;
-			var traceId = existingTraceId || newTraceId();
+			}
-			var spanId = newSpanId();
+			if (!queue.length) return;
-			lastTraceId = traceId;
+			var records = queue.splice(0, queue.length);
-			return { traceparent: '00-' + traceId + '-' + spanId + '-01' };
+			ship(records);
 		}
 
-		function send(severity, message, attrs, traceId) {
+		// ship sends already-built log records in one OTLP payload. Uses
-			if (sampleRate < 1 && Math.random() > sampleRate) return;
+		// sendBeacon when available (survives page unload), else keepalive
+		// fetch.
+		function ship(records) {
+			var payload = {
+				resourceLogs: [
+					{
+						resource: { attributes: toOtlpAttributes({ 'service.name': serviceName }) },
+						scopeLogs: [{ scope: { name: 'otel-rum' }, logRecords: records }]
+					}
+				]
+			};
 
+			var body = JSON.stringify(payload);
+			var url = endpoint;
+			if (ingestKey) {
+				url += (url.indexOf('?') === -1 ? '?' : '&') + 'key=' + encodeURIComponent(ingestKey);
+			}
+
+			// Use uninstrumented rawFetch with keepalive: true and mode: 'cors'
+			// to avoid intercepting and looping on our own telemetry requests.
+			var fetchFn = rawFetch || (typeof global.fetch === 'function' ? global.fetch : null);
+			if (fetchFn) {
+				var fetchHeaders = { 'Content-Type': 'application/json' };
+				if (ingestKey) {
+					fetchHeaders['X-Otel-Ingest-Key'] = ingestKey;
+				}
+				fetchFn.call(global, url, {
+					method: 'POST',
+					mode: 'cors',
+					headers: fetchHeaders,
+					body: body,
+					keepalive: true
+				}).catch(function () {
+					if (global.navigator && typeof global.navigator.sendBeacon === 'function') {
+						try {
+							var blob = new global.Blob([body], { type: 'text/plain;charset=UTF-8' });
+							global.navigator.sendBeacon(url, blob);
+						} catch (e) {}
+					}
+				});
+				return;
+			}
+
+			if (global.navigator && typeof global.navigator.sendBeacon === 'function') {
+				try {
+					var blob = new global.Blob([body], { type: 'text/plain;charset=UTF-8' });
+					global.navigator.sendBeacon(url, blob);
+				} catch (e) {}
+			}
+		}
+
+		var severityNumbers = { DEBUG: 5, INFO: 9, WARN: 13, ERROR: 17, FATAL: 21 };
+
+		function buildRecord(severity, message, attrs, traceId) {
 			var record = {
 				timeUnixNano: String(Date.now() * 1e6),
 				severityText: severity,
-				severityNumber: severity === 'FATAL' ? 21 : 17,
+				severityNumber: severityNumbers[severity] || 17,
 				body: { stringValue: String(message).slice(0, 2000) },
 				attributes: toOtlpAttributes(
 					Object.assign(
+						{},
+						contextAttrs,
 						{
 							source: 'frontend',
 							'browser.url': global.location ? global.location.href : '',
 							'browser.userAgent': global.navigator ? global.navigator.userAgent : '',
+							'browser.viewport': global.innerWidth ? global.innerWidth + 'x' + global.innerHeight : '',
+							'session.id': sessionId,
 							'service.release': release,
 							'service.environment': environment,
 							breadcrumbs: breadcrumbs.length ? JSON.stringify(breadcrumbs) : undefined
@@ -134,33 +282,114 @@
 			if (traceId) {
 				record.traceId = traceId;
 			}
+			return record;
+		}
 
-			var payload = {
+		function send(severity, message, attrs, traceId) {
-				resourceLogs: [
+			if (sampleRate < 1 && Math.random() > sampleRate) return;
-					{
+			if (!underRateLimit()) {
-						resource: { attributes: toOtlpAttributes({ 'service.name': serviceName }) },
+				eventsDropped++;
-						scopeLogs: [{ scope: { name: 'otel-rum' }, logRecords: [record] }]
+				return;
-					}
-				]
-			};
-
-			var body = JSON.stringify(payload);
-			var url = endpoint;
-			if (ingestKey) {
-				url += (url.indexOf('?') === -1 ? '?' : '&') + 'key=' + encodeURIComponent(ingestKey);
 			}
+			enqueue(buildRecord(severity, message, attrs, traceId));
+		}
+
+		// Web Vitals via PerformanceObserver. Values are reported as INFO logs
+		// tagged rum.vital=<name>; the server can chart/aggregate them like
+		// any other telemetry. Each vital fires once when it reaches its
+		// final value (plus a first-input proxy for INP on older browsers).
+		function observeVitals() {
+			if (typeof global.PerformanceObserver !== 'function') return;
+
+			function report(name, value, extra) {
+				var attrs = Object.assign({ 'rum.vital': name, 'rum.value': Math.round(value) }, extra || {});
+				// Vitals are telemetry, not errors: bypass the error rate limit
+				// but keep sampling.
+				if (sampleRate < 1 && Math.random() > sampleRate) return;
+				enqueue(buildRecord('INFO', name + ' ' + Math.round(value), attrs, ''));
+			}
+
+			function onPaint(list) {
+				list.getEntries().forEach(function (entry) {
+					if (entry.name === 'first-contentful-paint') report('FCP', entry.startTime);
+				});
+			}
+
+			function onLCP(list) {
+				var entries = list.getEntries();
+				var last = entries[entries.length - 1];
+				if (last) report('LCP', last.startTime, { 'rum.element': last.element && last.element.tagName });
+			}
+
+			function onCLS() {
+				var cls = 0;
+				var lastReport = 0;
+				var observer = new global.PerformanceObserver(function (list) {
+					list.getEntries().forEach(function (entry) {
+						if (!entry.hadRecentInput) cls += entry.value;
+					});
+					// Throttle: report at most once per 10s, plus a final value on
+					// pagehide, instead of once per layout shift.
+					var now = Date.now();
+					if (now - lastReport > 10000) {
+						lastReport = now;
+						report('CLS', Math.round(cls * 1000) / 1000);
+					}
+					clsFinal = function () {
+						report('CLS', Math.round(cls * 1000) / 1000);
+						clsFinal = null;
+					};
+				});
+				try {
+					observer.observe({ type: 'layout-shift', buffered: true });
+				} catch (e) {
+					/* unsupported */
+					clsFinal = null;
+				}
+			}
+
+			function onINP(entry) {
+				var duration = entry.duration;
+				var interactionId = entry.interactionId;
+				if (!interactionId) return;
+				if (duration > (observerState.maxINP || 0)) {
+					observerState.maxINP = duration;
+					report('INP', duration);
+				}
+			}
+
+			var observerState = {};
 
 			try {
-				if (global.navigator && typeof global.navigator.sendBeacon === 'function') {
+				new global.PerformanceObserver(onPaint).observe({ type: 'paint', buffered: true });
-					var blob = new global.Blob([body], { type: 'application/json' });
+			} catch (e) {}
-					if (global.navigator.sendBeacon(url, blob)) return;
+			try {
-				}
+				new global.PerformanceObserver(onLCP).observe({ type: 'largest-contentful-paint', buffered: true });
-			} catch (e) {
+			} catch (e) {}
-				// fall through to fetch
+			onCLS();
-			}
+			try {
-			if (typeof global.fetch === 'function') {
+				new global.PerformanceObserver(function (list) {
-				global.fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: body, keepalive: true }).catch(function () {});
+					list.getEntries().forEach(onINP);
-			}
+				}).observe({ type: 'event', buffered: true, durationThreshold: 40 });
+			} catch (e) {}
+
+			// Navigation timing: TTFB.
+				try {
+					var nav = performance.getEntriesByType('navigation')[0];
+					if (nav && nav.responseStart) {
+						report('TTFB', nav.responseStart);
+					}
+				} catch (e) {}
+		}
+
+		// traceHeaders returns a headers object carrying a fresh (or
+		// continued) W3C traceparent, ready to spread into any HTTP client's
+		// request headers - fetch, XHR, axios, etc.
+		function traceHeaders(existingTraceId) {
+			var traceId = existingTraceId || newTraceId();
+			var spanId = newSpanId();
+			lastTraceId = traceId;
+			return { traceparent: '00-' + traceId + '-' + spanId + '-01' };
 		}
 
 		function captureException(err, extra) {
@@ -184,6 +413,16 @@
 			send(severity || 'ERROR', message, extra, lastTraceId);
 		}
 
+		function setContext(context) {
+			// Attach stable context (e.g. { user: 'u-123', plan: 'pro' }) to
+			// every subsequent event. Values become OTLP attributes prefixed
+			// with "context.".
+			contextAttrs = Object.assign({}, contextAttrs, context || {});
+			flush(); // ensure earlier queued events don't mix contexts ambiguously
+		}
+		var contextAttrs = {};
+		var clsFinal = null;
+
 		function onError(event) {
 			// Resource load errors (img/script/link) surface as an ErrorEvent-less
 			// Event targeting the failed element; script errors carry a message.
@@ -233,45 +472,214 @@
 			};
 		}
 
+		function shouldInject(url) {
+			if (!injectTraceHeaders) return false;
+			if (!traceHeaderOrigins) return isSameOrigin(url);
+			return traceHeaderOrigins.some(function (o) {
+				return url.indexOf(o) === 0;
+			});
+		}
+
+		// httpFailureReport reports a failed HTTP round-trip as an ERROR event,
+		// so backend 5xx/4xx responses show up in the Issues view even when
+		// the app code swallows them.
+		function httpFailureReport(kind, method, url, status, traceId, extra) {
+			// Issue refresh failures must not create new issues notifications:
+			// otherwise an expired session can form an API-error feedback loop.
+			try {
+				if (new global.URL(url, global.location.href).pathname === '/api/issues') return;
+			} catch (e) {}
+			var attrs = Object.assign(
+				{
+					'exception.type': 'HttpFailure',
+					'http.request.method': method,
+					'url.full': String(url).slice(0, 500),
+					'http.response.status_code': status === undefined || status === 0 ? '' : String(status)
+				},
+				extra || {}
+			);
+			captureMessage(
+				method + ' ' + url + ' failed with ' + (status === undefined || status === 0 ? 'network error' : 'HTTP ' + status),
+				'ERROR',
+				attrs
+			);
+		}
+
 		if (autoInstrumentFetch && typeof global.fetch === 'function') {
 			var originalFetch = global.fetch;
 			global.fetch = function (input, requestInit) {
 				var url = typeof input === 'string' ? input : (input && input.url) || '';
+				if (isCollectorUrl(url)) {
+					return originalFetch.call(global, input, requestInit);
+				}
 				var method = (requestInit && requestInit.method) || (input && input.method) || 'GET';
+				var requestTraceId = '';
 
-				if (injectTraceHeaders) {
+				if (shouldInject(url)) {
-					var allowed = traceHeaderOrigins ? traceHeaderOrigins.some(function (o) { return url.indexOf(o) === 0; }) : isSameOrigin(url);
+					requestInit = requestInit || {};
-					if (allowed) {
+					var headers = new global.Headers(requestInit.headers || (input && input.headers) || {});
-						requestInit = requestInit || {};
+					var th = traceHeaders();
-						var headers = new global.Headers(requestInit.headers || (input && input.headers) || {});
+					headers.set('traceparent', th.traceparent);
-						var th = traceHeaders();
+					requestTraceId = th.traceparent.slice(3, 35); // trace id portion
-						headers.set('traceparent', th.traceparent);
+					requestInit = Object.assign({}, requestInit, { headers: headers });
-						requestInit = Object.assign({}, requestInit, { headers: headers });
-					}
 				}
 
 				return originalFetch.call(global, input, requestInit).then(
 					function (res) {
 						addBreadcrumb('fetch', method + ' ' + url + ' -> ' + res.status);
+						if (reportHttpFailures && res.status >= 400) {
+							httpFailureReport('fetch', method, url, res.status, requestTraceId);
+						}
 						return res;
 					},
 					function (err) {
 						addBreadcrumb('fetch', method + ' ' + url + ' -> error');
+						if (reportHttpFailures) {
+							httpFailureReport('fetch', method, url, 0, requestTraceId);
+						}
 						throw err;
 					}
 				);
 			};
 		}
 
-		return {
+		// XHR instrumentation: trace-header injection + breadcrumbs + failure
+		// reporting, so legacy code paths (axios older versions, jQuery, etc.)
+		// get the same visibility as fetch.
+		if (autoInstrumentXhr && typeof global.XMLHttpRequest === 'function') {
+			var originalOpen = global.XMLHttpRequest.prototype.open;
+			var originalSend = global.XMLHttpRequest.prototype.send;
+			var originalSetRequestHeader = global.XMLHttpRequest.prototype.setRequestHeader;
+
+			global.XMLHttpRequest.prototype.open = function (method, url) {
+				if (isCollectorUrl(url)) {
+					return originalOpen.apply(this, arguments);
+				}
+				try {
+					this.__rum = { method: String(method), url: String(url) };
+					if (shouldInject(String(url))) {
+						var th = traceHeaders();
+						this.__rum.traceId = th.traceparent.slice(3, 35);
+						originalSetRequestHeader.call(this, 'traceparent', th.traceparent);
+					}
+				} catch (e) {
+					/* ignore */
+				}
+				return originalOpen.apply(this, arguments);
+			};
+
+			global.XMLHttpRequest.prototype.send = function () {
+				var xhr = this;
+				var info = xhr.__rum || {};
+				if (info.url) {
+					var origHandler = xhr.onreadystatechange;
+					xhr.onreadystatechange = function () {
+						try {
+							if (xhr.readyState === 4) {
+								addBreadcrumb('xhr', info.method + ' ' + info.url + ' -> ' + xhr.status);
+								if (reportHttpFailures && (xhr.status === 0 || xhr.status >= 400)) {
+									httpFailureReport('xhr', info.method, info.url, xhr.status, info.traceId);
+								}
+							}
+						} catch (e) {
+							/* ignore */
+						}
+						if (origHandler) return origHandler.apply(this, arguments);
+					};
+				}
+				return originalSend.apply(this, arguments);
+			};
+		}
+
+		// SPA route-change breadcrumbs: history.pushState/replaceState and
+		// popstate/hashes get recorded so an error report shows which route
+		// transitions preceded it.
+		if (trackSpaRoutes && global.history) {
+			function recordRoute(kind) {
+				addBreadcrumb('navigation', kind + ' ' + (global.location ? global.location.pathname + global.location.search : ''));
+			}
+			var originalPush = global.history.pushState;
+			var originalReplace = global.history.replaceState;
+			if (typeof originalPush === 'function') {
+				global.history.pushState = function () {
+					var result = originalPush.apply(this, arguments);
+					try {
+						recordRoute('pushState');
+					} catch (e) {}
+					return result;
+				};
+			}
+			if (typeof originalReplace === 'function') {
+				global.history.replaceState = function () {
+					var result = originalReplace.apply(this, arguments);
+					try {
+						recordRoute('replaceState');
+					} catch (e) {}
+					return result;
+				};
+			}
+			global.addEventListener('popstate', function () {
+				try {
+					recordRoute('popstate');
+				} catch (e) {}
+			});
+		}
+
+		// Flush pending events when the page is being hidden or unloaded so
+		// errors that happen right before navigation are not lost, and emit
+		// the final CLS value.
+		global.addEventListener('pagehide', function () {
+			if (clsFinal) clsFinal();
+			flush();
+		});
+		global.addEventListener('visibilitychange', function () {
+			if (global.document && global.document.visibilityState === 'hidden') {
+				if (clsFinal) clsFinal();
+				flush();
+			}
+		});
+
+		// First paint: report vitals after observers have a chance to buffer.
+		if (captureWebVitals) {
+			observeVitals();
+		}
+
+		function addBreadcrumb(category, message) {
+			breadcrumbs.push({ t: Date.now(), category: String(category), message: String(message) });
+			if (breadcrumbs.length > maxBreadcrumbs) {
+				breadcrumbs.shift();
+			}
+		}
+
+		var instance = {
 			captureException: captureException,
 			captureMessage: captureMessage,
 			addBreadcrumb: addBreadcrumb,
+			setContext: setContext,
+			flush: flush,
 			newTraceId: newTraceId,
 			newSpanId: newSpanId,
 			traceHeaders: traceHeaders
 		};
+
+		// Convenience handle for manual/console verification:
+		// OtelRUM.last.captureMessage('test from console').
+		global.OtelRUM.last = instance;
+		return instance;
 	}
 
-	global.OtelRUM = { init: init, newTraceId: newTraceId, newSpanId: newSpanId };
+	global.OtelRUM = { init: init, newTraceId: newTraceId, newSpanId: newSpanId, last: undefined };
+
+	// Auto-initialize if data attributes are declared on the <script> element
+	try {
+		if (typeof document !== 'undefined' && scriptEl && (scriptEl.getAttribute('data-auto-init') === 'true' || scriptEl.getAttribute('data-service-name') || scriptEl.getAttribute('data-service'))) {
+			if (document.readyState === 'loading') {
+				document.addEventListener('DOMContentLoaded', function () {
+					global.OtelRUM.init();
+				});
+			} else {
+				global.OtelRUM.init();
+			}
+		}
+	} catch (e) {}
 })(typeof window !== 'undefined' ? window : this);