mreo ui

AGENTS.md

@@ -223,3 +223,23 @@ plus `AuthServer.spec.deviceFlow`.
   (a non-fatal "spec.issuer: Required value" warning for the partial AuthServer
   apply is expected).
 - Never commit tokens, persist them in a production build, or log them.
+
+### Drill-down screens: `MasterDetail` + `ResizablePanes`
+
+Navigation here drills list -> detail -> extended detail, so use the shared
+pane components instead of laying out columns by hand.
+
+- `components/panes/ResizablePanes.tsx` - generic stack of panes with
+  draggable dividers (arrow keys when focused, double-click resets, widths
+  persisted per `storageKey`). Fixed panes take a `size`; exactly one pane is
+  flexible. Below `lg` panes stack and dividers disappear.
+- `components/MasterDetail.tsx` - list | detail [| extended] on top of it.
+  The caller owns selection; `renderDetail(item, drill)` gets
+  `drill.openExtended(key)` / `closeExtended()` and `renderExtended(item, key)`
+  fills the third pane, which resets when the selection changes.
+  `layout="stacked"` gives a full-width table with the detail below.
+- Building blocks: `trace/TraceListItem` (narrow rows), `trace/TraceOverviewRow`
+  (table rows), `trace/TraceDetail`, `common/*` (Stat, StatHeader, Segmented,
+  DetailField, BackButton). The waterfall takes `selectedSpanId`/`onSelectSpan`
+  to drive an external inspector pane.
+- Local UI against a local server: `OTEL_URL=http://localhost:8080 npm run dev`.

internal/collector/issue_extract.go

@@ -5,6 +5,7 @@ import (
 	"strconv"
 	"strings"
 
+	"git.tornberg.me/go-otel/internal/httpattr"
 	"git.tornberg.me/go-otel/internal/issues"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/plog"
@@ -183,12 +184,11 @@ func firstAttr(attrs pcommon.Map, keys ...string) string {
 // as a generic router mount. This keeps API endpoints distinct in issue
 // fingerprints and avoids collapsing unrelated requests into one root route.
 func httpRoute(attrs pcommon.Map) string {
-	route := firstAttr(attrs, "http.route")
-	path := firstAttr(attrs, "url.path", "http.target")
-	if path != "" && (route == "" || route == "/" && path != "/") {
-		return path
+	ep := httpattr.Extract(func(keys ...string) string { return firstAttr(attrs, keys...) })
+	if ep.Route != "" {
+		return ep.Route
 	}
-	return route
+	return ep.Path
 }
 
 // getHTTPStatusCode extracts an HTTP status code from attributes, supporting
@@ -332,7 +332,7 @@ func spanErrorTitle(span ptrace.Span) string {
 		}
 
 		method := firstAttr(span.Attributes(), "http.request.method", "http.method")
-		route := firstAttr(span.Attributes(), "http.route", "http.target", "url.path")
+		route := httpRoute(span.Attributes())
 
 		var endpoint string
 		if method != "" && route != "" {

internal/httpattr/httpattr.go

@@ -0,0 +1,136 @@
+// Package httpattr collapses the many attribute spellings used for "which
+// HTTP endpoint was this" (OTel stable semconv, legacy semconv, nginx, Go
+// net/http, browser RUM) into one Endpoint, so spans from different
+// instrumentations can be compared and joined.
+package httpattr
+
+import (
+	"net/url"
+	"strings"
+)
+
+// Endpoint is the normalized identity of one HTTP request.
+type Endpoint struct {
+	Method string // upper-case, e.g. "GET"
+	Host   string // host without port, lower-case; "" if unknown
+	Path   string // concrete path, no query/fragment, no trailing slash
+	Route  string // instrumentation-provided template ("/orders/:id"), may be ""
+}
+
+// Key is the join key between a client-side (RUM) request and a server-side
+// span: method plus the concrete path. Route is deliberately not part of it -
+// a browser only knows the concrete URL.
+func (e Endpoint) Key() string {
+	if e.Path == "" {
+		return ""
+	}
+	return e.Method + " " + e.Path
+}
+
+// Lookup returns the first non-empty value among keys.
+type Lookup func(keys ...string) string
+
+// Extract reads an Endpoint out of span/log attributes.
+//
+// Sources, in order of preference:
+//
+//	method: http.request.method, http.method
+//	path:   url.path, http.target (path+query), url.full, http.url
+//	host:   server.address, http.host, net.host.name, url.full/http.url host
+//	route:  http.route
+//
+// nginx's OTel module emits the stable keys; older nginx/Go/Java agents emit
+// http.target/http.url; browsers only know url.full (absolute or relative).
+func Extract(get Lookup) Endpoint {
+	e := Endpoint{
+		Method: strings.ToUpper(get("http.request.method", "http.method")),
+		Route:  get("http.route"),
+	}
+	e.Host = stripPort(strings.ToLower(get("server.address", "http.host", "net.host.name", "server.host")))
+	if p := get("url.path"); p != "" {
+		e.Path = CleanPath(p)
+	} else if t := get("http.target"); t != "" {
+		e.Path = CleanPath(t)
+	}
+	if full := get("url.full", "http.url"); full != "" {
+		host, path := SplitURL(full)
+		if e.Path == "" {
+			e.Path = path
+		}
+		if e.Host == "" {
+			e.Host = host
+		}
+	}
+	// Some instrumentations report the router mount ("/") as the route.
+	if e.Route == "/" && e.Path != "" && e.Path != "/" {
+		e.Route = ""
+	}
+	return e
+}
+
+// SplitURL splits an absolute or relative URL into (host without port, clean path).
+func SplitURL(raw string) (host, path string) {
+	u, err := url.Parse(raw)
+	if err != nil {
+		return "", CleanPath(raw)
+	}
+	return stripPort(strings.ToLower(u.Host)), CleanPath(u.Path)
+}
+
+// CleanPath drops query/fragment and a trailing slash, and guarantees a
+// leading slash. "" stays "".
+func CleanPath(p string) string {
+	if i := strings.IndexAny(p, "?#"); i >= 0 {
+		p = p[:i]
+	}
+	p = strings.TrimSpace(p)
+	if p == "" {
+		return ""
+	}
+	if !strings.HasPrefix(p, "/") {
+		p = "/" + p
+	}
+	if len(p) > 1 {
+		p = strings.TrimRight(p, "/")
+		if p == "" {
+			p = "/"
+		}
+	}
+	return p
+}
+
+// StripPrefix removes a reverse-proxy mount prefix (e.g. nginx
+// `location /api/ { proxy_pass http://backend/; }` makes the browser see
+// /api/orders while the backend sees /orders). The longest matching prefix wins.
+func StripPrefix(path string, prefixes []string) string {
+	best := ""
+	for _, p := range prefixes {
+		p = CleanPath(p)
+		if p == "" || p == "/" {
+			continue
+		}
+		if (path == p || strings.HasPrefix(path, p+"/")) && len(p) > len(best) {
+			best = p
+		}
+	}
+	if best == "" {
+		return path
+	}
+	if rest := path[len(best):]; rest != "" {
+		return rest
+	}
+	return "/"
+}
+
+func stripPort(host string) string {
+	if strings.HasPrefix(host, "[") { // IPv6 literal
+		if i := strings.Index(host, "]"); i > 0 {
+			return host[:i+1]
+		}
+		return host
+	}
+	if i := strings.LastIndexByte(host, ':'); i >= 0 {
+		return host[:i]
+	}
+	return host
+}

internal/httpattr/httpattr_test.go

@@ -0,0 +1,49 @@
+package httpattr
+
+import "testing"
+
+func lookup(m map[string]string) Lookup {
+	return func(keys ...string) string {
+		for _, k := range keys {
+			if v := m[k]; v != "" {
+				return v
+			}
+		}
+		return ""
+	}
+}
+
+func TestExtractVariants(t *testing.T) {
+	cases := []struct {
+		name  string
+		attrs map[string]string
+		want  string
+	}{
+		{"stable semconv", map[string]string{"http.request.method": "get", "url.path": "/api/orders/"}, "GET /api/orders"},
+		{"legacy nginx target", map[string]string{"http.method": "GET", "http.target": "/api/orders?x=1"}, "GET /api/orders"},
+		{"go http.url", map[string]string{"http.method": "POST", "http.url": "https://shop.example.com:8443/api/cart?a=b"}, "POST /api/cart"},
+		{"browser url.full", map[string]string{"http.request.method": "GET", "url.full": "https://shop.example.com/api/orders#x"}, "GET /api/orders"},
+		{"relative url.full", map[string]string{"http.request.method": "GET", "url.full": "api/orders"}, "GET /api/orders"},
+	}
+	for _, c := range cases {
+		if got := Extract(lookup(c.attrs)).Key(); got != c.want {
+			t.Errorf("%s: got %q want %q", c.name, got, c.want)
+		}
+	}
+}
+
+func TestExtractHostAndMountRoute(t *testing.T) {
+	e := Extract(lookup(map[string]string{"http.route": "/", "url.path": "/x", "server.address": "API.example.com:443"}))
+	if e.Host != "api.example.com" || e.Route != "" {
+		t.Fatalf("got %+v", e)
+	}
+}
+
+func TestStripPrefix(t *testing.T) {
+	p := []string{"/api", "/api/v2"}
+	for in, want := range map[string]string{"/api/orders": "/orders", "/api/v2/orders": "/orders", "/api": "/", "/apix": "/apix", "/other": "/other"} {
+		if got := StripPrefix(in, p); got != want {
+			t.Errorf("%s: got %q want %q", in, got, want)
+		}
+	}
+}

internal/normalization/normalizer.go

@@ -10,6 +10,7 @@ import (
 	"strconv"
 	"strings"
 
+	"git.tornberg.me/go-otel/internal/httpattr"
 	"git.tornberg.me/go-otel/internal/issues"
 )
 
@@ -397,14 +398,13 @@ func defaultString(value, fallback string) string {
 func normalizeMessage(message string) string { return issues.NormalizeTitle(message) }
 
 func normalizeHTTPRoute(attrs []attribute) string {
-	route := stringAttr(attrs, "http.route")
-	path := stringAttr(attrs, "url.path", "http.target")
-	// Some HTTP instrumentations report the router mount point (often "/")
-	// as http.route while url.path contains the actual endpoint. Prefer the
-	// specific path in that case so unrelated API calls do not collapse into
-	// one pattern.
-	if path != "" && (route == "" || route == "/" && path != "/") {
-		route = path
+	ep := httpattr.Extract(func(keys ...string) string { return stringAttr(attrs, keys...) })
+	// Prefer the template when the instrumentation gave a meaningful one,
+	// otherwise the concrete path. A mount-point route ("/") is dropped by
+	// Extract so unrelated API calls do not collapse into one pattern.
+	route := ep.Route
+	if route == "" {
+		route = ep.Path
 	}
 	return issues.NormalizeRoute(route)
 }

k8s/clickhouse.yaml

@@ -1,9 +1,12 @@
 # Single-node ClickHouse for the otel-debug-receiver, applied as one file:
 #   kubectl apply -f k8s/clickhouse.yaml
 #
-# A single instance (per request): one StatefulSet replica with a PVC so
-# telemetry survives pod restarts. Resources sized for a small cluster;
-# adjust before production use.
+# A single instance (per request): one StatefulSet replica. Data lives on
+# serber's local disk at /srv/monitoring-db (hostPath), so the pod is pinned to
+# that node. It used to be an nfs-client PVC on a spinning-disk NAS, where
+# synchronous writes cost ~120 ms and every query paid that latency; see
+# k8s/migrate/clickhouse-nfs-to-local.yaml for the one-off data move.
+# Resources sized for a small cluster; adjust before production use.
 apiVersion: v1
 kind: Namespace
 metadata:
@@ -56,8 +59,21 @@ spec:
       labels:
         app: clickhouse
     spec:
+      nodeSelector:
+        kubernetes.io/hostname: serber
       securityContext:
         fsGroup: 101
+      initContainers:
+      # hostPath directories are created root-owned and fsGroup does not apply
+      # to them; ClickHouse runs as uid/gid 101.
+      - name: fix-data-perms
+        image: busybox:1.36
+        command: ["sh", "-c", "chown -R 101:101 /var/lib/clickhouse"]
+        securityContext:
+          runAsUser: 0
+        volumeMounts:
+        - name: data
+          mountPath: /var/lib/clickhouse
       containers:
       - name: clickhouse
         image: clickhouse/clickhouse-server:24.8
@@ -101,20 +117,16 @@ spec:
             memory: "2Gi"
             cpu: "1"
       volumes:
+      - name: data
+        hostPath:
+          path: /srv/monitoring-db
+          type: DirectoryOrCreate
       - name: user-config
         configMap:
           name: clickhouse-config
           items:
           - key: users.d-otel.xml
             path: users.d-otel.xml
-  volumeClaimTemplates:
-  - metadata:
-      name: data
-    spec:
-      accessModes: ["ReadWriteOnce"]
-      resources:
-        requests:
-          storage: 20Gi
 ---
 apiVersion: v1
 kind: Service

k8s/migrate/clickhouse-nfs-to-local.yaml

@@ -0,0 +1,57 @@
+# One-off: copy ClickHouse data from the old NFS PVC to serber:/srv/monitoring-db.
+# NOT applied by `kubectl apply -f k8s/` (subdirectory). Run it while the
+# StatefulSet is scaled to 0 so the copy is consistent:
+#
+#   kubectl scale sts clickhouse -n monitoring --replicas=0
+#   kubectl apply -f k8s/migrate/clickhouse-nfs-to-local.yaml
+#   kubectl wait --for=condition=complete job/clickhouse-nfs-to-local -n monitoring --timeout=30m
+#   kubectl delete sts clickhouse -n monitoring          # PVC is retained (rollback copy)
+#   kubectl apply -f k8s/clickhouse.yaml                 # new hostPath StatefulSet
+#   kubectl delete job clickhouse-nfs-to-local -n monitoring
+#
+# The old PVC (data-clickhouse-0) is left untouched until you delete it.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: clickhouse-nfs-to-local
+  namespace: monitoring
+spec:
+  backoffLimit: 0
+  template:
+    spec:
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: serber
+      containers:
+      - name: copy
+        image: busybox:1.36
+        securityContext:
+          runAsUser: 0
+        command:
+        - sh
+        - -c
+        - |
+          set -eu
+          if [ -n "$(ls -A /new 2>/dev/null)" ]; then
+            echo "refusing to copy: /new (serber:/srv/monitoring-db) is not empty"; exit 1
+          fi
+          du -sh /old
+          cp -a /old/. /new/
+          chown -R 101:101 /new
+          echo "old: $(find /old -type f | wc -l) files, new: $(find /new -type f | wc -l) files"
+          [ "$(find /old -type f | wc -l)" = "$(find /new -type f | wc -l)" ]
+        volumeMounts:
+        - name: old
+          mountPath: /old
+          readOnly: true
+        - name: new
+          mountPath: /new
+      volumes:
+      - name: old
+        persistentVolumeClaim:
+          claimName: data-clickhouse-0
+          readOnly: true
+      - name: new
+        hostPath:
+          path: /srv/monitoring-db
+          type: DirectoryOrCreate

otel-ui/src/components/IssueOccurrences.tsx

@@ -4,7 +4,9 @@ import { BackButton } from "./common/BackButton"
 import { SectionLabel } from "./common/DetailField"
 import { Stat, StatHeader } from "./common/Stat"
 import { TraceDetail } from "./trace/TraceDetail"
-import { TraceOverviewHeader, TraceOverviewRow } from "./trace/TraceOverviewRow"
+import { SpanInspector } from "./TraceVisualization"
+import { Segmented } from "./common/Segmented"
+import { TraceListItem } from "./trace/TraceListItem"
 import { OccurrenceDetail } from "./issues/OccurrenceDetail"
 import { useIssueTraces } from "../hooks/useIssueTraces"
 import { useProcessedTraces } from "../hooks/useProcessedTraces"
@@ -37,6 +39,8 @@ export const IssueOccurrences = ({
 }) => {
   const { traces, untraced, total, complete, hasMore, loading, error, loadMore } = useIssueTraces(issue.id)
 
+  const [sort, setSort] = useState<"recent" | "slowest" | "errors">("recent")
+
   const rows: Row[] = useMemo(() => {
     const traced = traces.map((t) => ({
       key: t.traceId,
@@ -47,8 +51,13 @@ export const IssueOccurrences = ({
       summary: t.summary,
     }))
     const loose = untraced.map((s, i) => ({ key: `s-${s.timestamp}-${i}`, samples: [s], ts: s.timestamp, hits: 1 }))
-    return [...traced, ...loose].sort((a, b) => b.ts - a.ts)
-  }, [traces, untraced])
+    const all = [...traced, ...loose]
+    const dur = (r: Row) => r.summary?.duration ?? 0
+    const errs = (r: Row) => r.summary?.errorCount ?? 0
+    return all.sort((a, b) =>
+      sort === "slowest" ? dur(b) - dur(a) : sort === "errors" ? errs(b) - errs(a) || b.ts - a.ts : b.ts - a.ts,
+    )
+  }, [traces, untraced, sort])
 
   const [picked, setPicked] = useState<string | null>(initialTraceId ?? null)
   const selected = rows.find((r) => r.key === picked) ?? rows[0]
@@ -94,15 +103,28 @@ export const IssueOccurrences = ({
       {loading && rows.length === 0 && <p className="text-[13px] text-ds-text-3">Loading traces…</p>}
 
       <MasterDetail
-        layout="stacked"
         items={rows}
         getKey={(r) => r.key}
         selectedKey={selected?.key ?? null}
         onSelect={setPicked}
         label="Traces containing this issue"
-        listHeader={<TraceOverviewHeader />}
+        listHeader={
+          <div className="flex items-center justify-between gap-2">
+            <SectionLabel>Traces ({rows.length}{hasMore ? "+" : ""})</SectionLabel>
+            <Segmented
+              label="Sort traces"
+              value={sort}
+              onChange={setSort}
+              options={[
+                { value: "recent", label: "Newest" },
+                { value: "slowest", label: "Slowest" },
+                { value: "errors", label: "Errors" },
+              ]}
+            />
+          </div>
+        }
         renderItem={(r) => (
-          <TraceOverviewRow
+          <TraceListItem
             summary={r.summary}
             traceId={r.traceId}
             timestampNs={r.ts}
@@ -111,7 +133,16 @@ export const IssueOccurrences = ({
             hits={r.hits}
           />
         )}
-        renderDetail={(r) => (
+        storageKey="issue-traces"
+        extendedTitle="Span"
+        renderExtended={(r, spanId) => {
+          const trace = r.traceId ? byId.get(r.traceId) : undefined
+          const span = trace?.spans.find((sp) => sp.spanId === spanId)
+          return span && trace ? (
+            <SpanInspector span={span} traceTotalRange={trace.duration} />
+          ) : null
+        }}
+        renderDetail={(r, drill) => (
           <div className="flex flex-col gap-4">
             {r.traceId && (
               <TraceDetail
@@ -119,6 +150,8 @@ export const IssueOccurrences = ({
                 trace={byId.get(r.traceId)}
                 loading={spansLoading && !failed.has(r.traceId)}
                 spanId={r.samples[0]?.spanId}
+                selectedSpanId={drill.extendedKey}
+                onSelectSpan={(id) => (id ? drill.openExtended(id) : drill.closeExtended())}
               />
             )}
             <SectionLabel>

otel-ui/src/components/MasterDetail.tsx

@@ -1,4 +1,13 @@
-import { useEffect, useRef, type KeyboardEvent, type ReactNode } from "react"
+import { useEffect, useRef, useState, type KeyboardEvent, type ReactNode } from "react"
+import { ResizablePanes, type PaneSize } from "./panes/ResizablePanes"
+
+/** Handed to renderDetail so the detail can open (and close) the extended pane. */
+export interface DrillApi {
+  /** Key of the currently open extended item, or null. */
+  extendedKey: string | null
+  openExtended: (key: string) => void
+  closeExtended: () => void
+}
 
 interface MasterDetailProps<T> {
   items: T[]
@@ -6,28 +15,33 @@ interface MasterDetailProps<T> {
   selectedKey: string | null
   onSelect: (key: string) => void
   renderItem: (item: T, selected: boolean) => ReactNode
-  renderDetail: (item: T) => ReactNode
-  /** Accessible name for the list, e.g. "Occurrences". */
+  renderDetail: (item: T, drill: DrillApi) => ReactNode
+  /**
+   * Third pane, opened from inside the detail via drill.openExtended(key).
+   * Omit for a plain two-pane list/detail.
+   */
+  renderExtended?: (item: T, key: string) => ReactNode
+  extendedTitle?: ReactNode
+  /** Accessible name for the list, e.g. "Traces". */
   label: string
-  /** Optional heading rendered above the list. */
+  /** Optional heading/controls rendered above the list. */
   listHeader?: ReactNode
   emptyDetail?: ReactNode
   /**
-   * "split": list left, detail right (lists of cards). "stacked": full-width
-   * list on top (tables/overviews), detail below.
+   * "split": resizable list | detail [| extended] panes (default).
+   * "stacked": full-width list on top (tables), detail below.
    */
   layout?: "split" | "stacked"
-  /** Split layout: Tailwind width class for the master column. */
-  listWidth?: string
-  /** Tailwind max-height class for the list (and, in split, the detail). */
-  maxHeight?: string
+  listSize?: PaneSize
+  extendedSize?: PaneSize
+  /** Persists pane widths per screen. */
+  storageKey?: string
 }
 
 /**
- * Reusable master-detail layout: a selectable list on the left, the selected
- * item's detail on the right (stacked on narrow screens). Arrow keys move the
- * selection. Callers own selection state so it can be driven from elsewhere
- * (e.g. a deep link or "focus this trace").
+ * Reusable drill-down layout. Selection is owned by the caller so it can be
+ * driven from elsewhere (deep links, "focus this trace"); the extended pane's
+ * state lives here and resets whenever the selection changes.
  */
 export function MasterDetail<T>({
   items,
@@ -36,78 +50,112 @@ export function MasterDetail<T>({
   onSelect,
   renderItem,
   renderDetail,
+  renderExtended,
+  extendedTitle = "Details",
   label,
   listHeader,
   emptyDetail,
   layout = "split",
-  listWidth = "lg:w-[380px]",
-  maxHeight,
+  listSize = { default: 320, min: 240, max: 560 },
+  extendedSize = { default: 380, min: 280, max: 720 },
+  storageKey,
 }: MasterDetailProps<T>) {
   const stacked = layout === "stacked"
-  const maxH = maxHeight ?? (stacked ? "max-h-[300px]" : "lg:max-h-[calc(100vh-260px)]")
   const listRef = useRef<HTMLDivElement>(null)
+  const [extendedKey, setExtendedKey] = useState<string | null>(null)
   const selected = items.find((item) => getKey(item) === selectedKey) ?? null
 
   useEffect(() => {
-    listRef.current
-      ?.querySelector<HTMLElement>('[aria-selected="true"]')
-      ?.scrollIntoView({ block: "nearest" })
+    listRef.current?.querySelector<HTMLElement>('[aria-selected="true"]')?.scrollIntoView({ block: "nearest" })
   }, [selectedKey])
 
+  // A different selection invalidates whatever was open in the extended pane.
+  useEffect(() => setExtendedKey(null), [selectedKey])
+
+  const select = (key: string) => {
+    setExtendedKey(null)
+    onSelect(key)
+  }
+
   const onKeyDown = (e: KeyboardEvent) => {
     if (e.key !== "ArrowDown" && e.key !== "ArrowUp") return
     e.preventDefault()
     const idx = items.findIndex((item) => getKey(item) === selectedKey)
     const next = items[Math.min(items.length - 1, Math.max(0, idx + (e.key === "ArrowDown" ? 1 : -1)))]
-    if (next) onSelect(getKey(next))
+    if (next) select(getKey(next))
+  }
+
+  const list = (
+    <div className="flex flex-col gap-2">
+      {listHeader}
+      <div
+        ref={listRef}
+        role="listbox"
+        aria-label={label}
+        tabIndex={0}
+        onKeyDown={onKeyDown}
+        className={`flex flex-col gap-1.5 overflow-auto rounded-[14px] border border-ds-line bg-ds-surface p-1.5 focus:outline-none focus-visible:border-ds-accent ${
+          stacked ? "max-h-[300px]" : "lg:max-h-[calc(100vh-240px)]"
+        }`}
+      >
+        {items.map((item) => {
+          const key = getKey(item)
+          const isSelected = key === selectedKey
+          return (
+            <div
+              key={key}
+              role="option"
+              aria-selected={isSelected}
+              onClick={() => select(key)}
+              className={`cursor-pointer rounded-[9px] border px-3 py-2 ${
+                isSelected ? "border-ds-accent/40 bg-ds-surface-2" : "border-transparent hover:bg-ds-surface-2"
+              }`}
+            >
+              {renderItem(item, isSelected)}
+            </div>
+          )
+        })}
+      </div>
+    </div>
+  )
+
+  const drill: DrillApi = { extendedKey, openExtended: setExtendedKey, closeExtended: () => setExtendedKey(null) }
+  const detail = selected ? (
+    renderDetail(selected, drill)
+  ) : (
+    <p className="py-10 text-center text-[13px] text-ds-text-3">{emptyDetail ?? "Select an item to see its details."}</p>
+  )
+
+  if (stacked) {
+    return (
+      <div className="flex flex-col gap-4">
+        {list}
+        <section aria-live="polite">{detail}</section>
+      </div>
+    )
   }
 
   return (
-    <div className={`flex flex-col gap-4 ${stacked ? "" : "lg:flex-row lg:items-start"}`}>
-      <div className={`flex w-full shrink-0 flex-col gap-2 ${stacked ? "" : listWidth}`}>
-        {listHeader}
-        <div
-          ref={listRef}
-          role="listbox"
-          aria-label={label}
-          tabIndex={0}
-          onKeyDown={onKeyDown}
-          className={`flex max-h-[360px] flex-col gap-1.5 overflow-auto rounded-[14px] border border-ds-line bg-ds-surface p-1.5 focus:outline-none focus-visible:border-ds-accent ${maxH}`}
-        >
-          {items.map((item) => {
-            const key = getKey(item)
-            const isSelected = key === selectedKey
-            return (
-              <div
-                key={key}
-                role="option"
-                aria-selected={isSelected}
-                onClick={() => onSelect(key)}
-                className={`cursor-pointer rounded-[9px] border px-3 py-2 ${
-                  isSelected
-                    ? "border-ds-accent/40 bg-ds-surface-2"
-                    : "border-transparent hover:bg-ds-surface-2"
-                }`}
-              >
-                {renderItem(item, isSelected)}
-              </div>
-            )
-          })}
-        </div>
-      </div>
-      <section
-        aria-live="polite"
-        className={`min-w-0 flex-1 ${stacked ? "" : `overflow-auto rounded-[14px] border border-ds-line bg-ds-surface p-5 ${maxH}`}`}
-      >
-        {selected ? (
-          renderDetail(selected)
-        ) : (
-          <p className="py-10 text-center text-[13px] text-ds-text-3">
-            {emptyDetail ?? "Select an item to see its details."}
-          </p>
-        )}
-      </section>
-    </div>
+    <ResizablePanes
+      storageKey={storageKey}
+      panes={[
+        { id: "list", label, node: list, size: listSize, sticky: true },
+        { id: "detail", label: "Details", node: detail },
+        ...(selected && renderExtended && extendedKey
+          ? [
+              {
+                id: "extended",
+                label: "Extended details",
+                node: renderExtended(selected, extendedKey),
+                size: extendedSize,
+                sticky: true,
+                title: extendedTitle,
+                onClose: drill.closeExtended,
+              },
+            ]
+          : []),
+      ]}
+    />
   )
 }
 

otel-ui/src/components/RumSessionsVisualizer.tsx

@@ -10,6 +10,12 @@ import {
 } from "lucide-react"
 import { useObservabilityData } from "../hooks/useObservabilityData"
 import { formatRelativeTime } from "../lib/timeUtils"
+import {
+  backendRequestsFromTraces,
+  linkRumRequest,
+  parseRequestCrumb,
+  type RumBackendLink,
+} from "../lib/http-endpoint"
 import {
   VITAL_THRESHOLDS,
   aggregateRum,
@@ -162,7 +168,8 @@ const SessionRow = ({
 }
 
 const SessionDetail = ({ session }: { session: RumSession | null }) => {
-  const { setActiveTraceId, setActiveSpanId, setActiveTab } = useObservabilityData()
+  const { setActiveTraceId, setActiveSpanId, setActiveTab, allTraces } = useObservabilityData()
+  const backend = useMemo(() => backendRequestsFromTraces(allTraces), [allTraces])
   const [copied, setCopied] = useState(false)
 
   if (!session) {
@@ -174,6 +181,28 @@ const SessionDetail = ({ session }: { session: RumSession | null }) => {
     )
   }
 
+  // Match each browser request to the backend span that served it. Falls back
+  // to method+path+time when the trace id doesn't survive the hop (e.g. an
+  // uninstrumented nginx in front of the API).
+  const links = new Map<number, RumBackendLink>()
+  session.timeline.forEach((entry, idx) => {
+    if (entry.category !== "fetch" && entry.category !== "xhr") return
+    const req = parseRequestCrumb(entry.message)
+    if (!req) return
+    const link = linkRumRequest(
+      {
+        // Breadcrumbs are stamped on completion; the request started duration earlier.
+        timestamp: entry.timestamp - (entry.durationMs ?? 0),
+        durationMs: entry.durationMs,
+        traceId: entry.traceId,
+        method: req.method,
+        url: req.url,
+      },
+      backend,
+    )
+    if (link) links.set(idx, link)
+  })
+
   const latestTracedError = session.errors.find((e) => e.traceId)
 
   const copySessionId = () => {
@@ -266,6 +295,22 @@ const SessionDetail = ({ session }: { session: RumSession | null }) => {
               <span className="break-all font-[family-name:var(--font-data)] text-[12px] text-ds-text-2">
                 {entry.message}
               </span>
+              {links.get(idx) && (
+                <button
+                  type="button"
+                  onClick={() => viewTrace(links.get(idx)!.backend.traceId)}
+                  className="inline-flex w-fit items-center gap-1 text-[12px] text-ds-accent"
+                  title={
+                    links.get(idx)!.confidence === "trace-id"
+                      ? "Linked by trace id"
+                      : "Inferred from method, path and timing - no trace id reached the backend"
+                  }
+                >
+                  <ExternalLink className="h-3 w-3" aria-hidden="true" />
+                  {links.get(idx)!.backend.service || "backend"} trace
+                  {links.get(idx)!.confidence === "path+time" && " (inferred)"}
+                </button>
+              )}
             </span>
           </li>
         ))}

otel-ui/src/components/TraceVisualization.tsx

@@ -148,13 +148,14 @@ const getOperationDisplayName = (span: ProcessedSpan) => {
 };
 
 // Span Detail Inspector Drawer
-const SpanInspector = ({
+export const SpanInspector = ({
   span,
   onClose,
   traceTotalRange,
 }: {
   span: ProcessedSpan;
-  onClose: () => void;
+  /** Omit when the host (e.g. a pane) already provides a close control. */
+  onClose?: () => void;
   traceTotalRange: number;
 }) => {
   const [tab, setTab] = useState<"overview" | "attributes" | "events" | "logs">("overview");
@@ -219,13 +220,15 @@ const SpanInspector = ({
           </h2>
         </div>
 
-        <button
-          onClick={onClose}
-          className="rounded-xl p-1.5 text-ds-text-2 hover:bg-ds-fill hover:text-ds-text transition-colors"
-          title="Close Inspector"
-        >
-          <X className="h-4 w-4" />
-        </button>
+        {onClose && (
+          <button
+            onClick={onClose}
+            className="rounded-xl p-1.5 text-ds-text-2 hover:bg-ds-fill hover:text-ds-text transition-colors"
+            title="Close Inspector"
+          >
+            <X className="h-4 w-4" />
+          </button>
+        )}
       </div>
 
       {/* Navigation Tabs */}
@@ -678,7 +681,7 @@ const VirtualizedSpanWaterfall = ({
       {/* Virtualized Span Rows Container */}
       <div
         ref={parentRef}
-        className="relative h-[560px] overflow-y-auto divide-y divide-ds-line select-none"
+        className="relative max-h-[560px] overflow-y-auto divide-y divide-ds-line select-none"
       >
         {visibleItems.length === 0 ? (
           <div className="flex h-full flex-col items-center justify-center text-sm text-ds-text-3">
@@ -823,10 +826,21 @@ const VirtualizedSpanWaterfall = ({
 // Main Trace Box with Header, Visual Time Distribution Bar, View Toggle, and Detail Drawer
 export const SingleTraceView = ({
   trace,
+  selectedSpanId: controlledSpanId,
+  onSelectSpan,
+  externalInspector = false,
 }: {
   trace: ProcessedTrace;
+  /** Controlled span selection; omit (with onSelectSpan) for the built-in inline inspector. */
+  selectedSpanId?: string | null;
+  onSelectSpan?: (spanId: string | null) => void;
+  /** Don't render the inspector inline: the caller shows it elsewhere (e.g. an extended pane). */
+  externalInspector?: boolean;
 }) => {
-  const [selectedSpanId, setSelectedSpanId] = useState<string | null>(null);
+  const [internalSpanId, setInternalSpanId] = useState<string | null>(null);
+  const controlled = onSelectSpan !== undefined;
+  const selectedSpanId = controlled ? (controlledSpanId ?? null) : internalSpanId;
+  const setSelectedSpanId = controlled ? onSelectSpan : setInternalSpanId;
   const [viewMode, setViewMode] = useState<"waterfall" | "graph">("waterfall");
   const [filterService, setFilterService] = useState<string | null>(null);
 
@@ -867,11 +881,7 @@ export const SingleTraceView = ({
   return (
     <section
       id={trace.id}
-      className={`rounded-3xl border transition-all ${
-        trace.hasError
-          ? "border-ds-critical/20 bg-ds-critical-bg"
-          : "border-ds-line bg-ds-surface-2"
-      } p-5 space-y-4`}
+      className="rounded-[14px] border border-ds-line bg-ds-surface p-5 space-y-4"
     >
       {/* Trace Card Header */}
       <header className="flex flex-wrap items-center justify-between gap-4 border-b border-ds-line pb-4">
@@ -1004,17 +1014,17 @@ export const SingleTraceView = ({
       ) : (
         <div className="grid grid-cols-1 lg:grid-cols-12 gap-4 items-start">
           {/* Virtualized Waterfall Table */}
-          <div className={selectedSpan ? "lg:col-span-7 xl:col-span-8" : "col-span-12"}>
+          <div className={selectedSpan && !externalInspector ? "lg:col-span-7 xl:col-span-8" : "col-span-12"}>
             <VirtualizedSpanWaterfall
               trace={trace}
               selectedSpanId={selectedSpanId}
-              onSelectSpan={(id) => setSelectedSpanId((prev) => (prev === id ? null : id))}
+              onSelectSpan={(id) => setSelectedSpanId(selectedSpanId === id ? null : id)}
               filterService={filterService}
             />
           </div>
 
           {/* Span Detail Inspector Panel */}
-          {selectedSpan && (
+          {selectedSpan && !externalInspector && (
             <div className="lg:col-span-5 xl:col-span-4 sticky top-6">
               <SpanInspector
                 span={selectedSpan}

otel-ui/src/components/common/Segmented.tsx

@@ -0,0 +1,30 @@
+/** Small segmented control for switching between a few mutually exclusive options (sort, range, ...). */
+export function Segmented<T extends string>({
+  value,
+  options,
+  onChange,
+  label,
+}: {
+  value: T
+  options: { value: T; label: string }[]
+  onChange: (v: T) => void
+  label: string
+}) {
+  return (
+    <div role="group" aria-label={label} className="inline-flex rounded-[9px] border border-ds-line bg-ds-surface-2 p-0.5">
+      {options.map((o) => (
+        <button
+          key={o.value}
+          type="button"
+          aria-pressed={o.value === value}
+          onClick={() => onChange(o.value)}
+          className={`rounded-[7px] px-2.5 py-1 text-[11.5px] font-semibold ${
+            o.value === value ? "bg-ds-accent-bg text-ds-accent" : "text-ds-text-3 hover:text-ds-text-2"
+          }`}
+        >
+          {o.label}
+        </button>
+      ))}
+    </div>
+  )
+}

otel-ui/src/components/panes/ResizablePanes.tsx

@@ -0,0 +1,177 @@
+import { Fragment, useCallback, useRef, useState, type KeyboardEvent, type PointerEvent, type ReactNode } from "react"
+import { X } from "lucide-react"
+
+export interface PaneSize {
+  default: number
+  min: number
+  max: number
+}
+
+export interface PaneSpec {
+  /** Stable id: keys the pane and its persisted width. */
+  id: string
+  /** Accessible name for the pane's region. */
+  label: string
+  node: ReactNode
+  /** Fixed-width, user-resizable pane. Omit for the one flexible pane that takes the remaining space. */
+  size?: PaneSize
+  /** Pin the pane while the page scrolls (lists, inspectors). Its content scrolls internally. */
+  sticky?: boolean
+  /** Renders the pane as a titled card with a close button when either is set. */
+  title?: ReactNode
+  onClose?: () => void
+}
+
+const STEP = 16
+const stickyMax = "lg:max-h-[calc(100vh-190px)]"
+
+function loadWidths(key: string | undefined): Record<string, number> {
+  if (!key) return {}
+  try {
+    return JSON.parse(localStorage.getItem(`otel.panes.${key}`) ?? "{}") as Record<string, number>
+  } catch {
+    return {}
+  }
+}
+
+function saveWidths(key: string | undefined, widths: Record<string, number>) {
+  if (!key) return
+  try {
+    localStorage.setItem(`otel.panes.${key}`, JSON.stringify(widths))
+  } catch {
+    /* storage unavailable: widths just don't persist */
+  }
+}
+
+/**
+ * Horizontal stack of panes with draggable dividers, for drill-down screens
+ * (list -> detail -> extended detail). Panes come and go with the `panes`
+ * array, so a third pane simply appears when something is selected.
+ *
+ *  - Fixed panes (`size`) are resizable: drag a divider, or focus it and use
+ *    the arrow keys; double-click resets to the default. Widths persist per
+ *    `storageKey`.
+ *  - One pane without `size` is flexible and absorbs the remaining space.
+ *  - Below the `lg` breakpoint panes stack vertically and dividers disappear.
+ */
+export function ResizablePanes({ panes, storageKey }: { panes: PaneSpec[]; storageKey?: string }) {
+  const [widths, setWidths] = useState<Record<string, number>>(() => loadWidths(storageKey))
+  const drag = useRef<{ index: number; x: number; start: number } | null>(null)
+
+  const widthOf = (p: PaneSpec) => (p.size ? clamp(widths[p.id] ?? p.size.default, p.size) : 0)
+
+  /** The fixed pane a divider before `index` controls, and the direction dx moves it. */
+  const target = (index: number): { pane: PaneSpec; sign: 1 | -1 } | null => {
+    const left = panes[index - 1]
+    const right = panes[index]
+    if (left?.size) return { pane: left, sign: 1 }
+    if (right?.size) return { pane: right, sign: -1 }
+    return null
+  }
+
+  const setWidth = useCallback(
+    (pane: PaneSpec, px: number, persist: boolean) => {
+      setWidths((prev) => {
+        const next = { ...prev, [pane.id]: clamp(px, pane.size!) }
+        if (persist) saveWidths(storageKey, next)
+        return next
+      })
+    },
+    [storageKey],
+  )
+
+  const onPointerDown = (index: number) => (e: PointerEvent<HTMLDivElement>) => {
+    const t = target(index)
+    if (!t) return
+    e.currentTarget.setPointerCapture(e.pointerId)
+    drag.current = { index, x: e.clientX, start: widthOf(t.pane) }
+  }
+
+  const onPointerMove = (e: PointerEvent<HTMLDivElement>) => {
+    const d = drag.current
+    const t = d && target(d.index)
+    if (!d || !t) return
+    setWidth(t.pane, d.start + t.sign * (e.clientX - d.x), false)
+  }
+
+  const onPointerUp = () => {
+    if (drag.current) saveWidths(storageKey, widths)
+    drag.current = null
+  }
+
+  const onKeyDown = (index: number) => (e: KeyboardEvent<HTMLDivElement>) => {
+    const t = target(index)
+    if (!t) return
+    const dir = e.key === "ArrowRight" ? 1 : e.key === "ArrowLeft" ? -1 : 0
+    if (!dir) return
+    e.preventDefault()
+    setWidth(t.pane, widthOf(t.pane) + dir * t.sign * STEP, true)
+  }
+
+  const reset = (index: number) => {
+    const t = target(index)
+    if (t) setWidth(t.pane, t.pane.size!.default, true)
+  }
+
+  return (
+    <div className="flex flex-col gap-4 lg:flex-row lg:items-start lg:gap-0">
+      {panes.map((p, i) => {
+        const t = i > 0 ? target(i) : null
+        const boxed = Boolean(p.title || p.onClose)
+        return (
+          <Fragment key={p.id}>
+            {i > 0 && (
+              <div
+                role="separator"
+                aria-orientation="vertical"
+                aria-label={`Resize ${p.label}`}
+                aria-valuenow={t ? widthOf(t.pane) : undefined}
+                tabIndex={t ? 0 : -1}
+                onPointerDown={onPointerDown(i)}
+                onPointerMove={onPointerMove}
+                onPointerUp={onPointerUp}
+                onKeyDown={onKeyDown(i)}
+                onDoubleClick={() => reset(i)}
+                className="group hidden w-3 shrink-0 cursor-col-resize touch-none justify-center self-stretch focus:outline-none lg:flex"
+              >
+                <span className="w-px bg-transparent group-hover:bg-ds-line-strong group-focus-visible:bg-ds-accent" />
+              </div>
+            )}
+            <section
+              aria-label={p.label}
+              style={p.size ? ({ "--pane-w": `${widthOf(p)}px` } as React.CSSProperties) : undefined}
+              className={`min-w-0 ${p.size ? "w-full lg:w-[var(--pane-w)] lg:shrink-0" : "w-full flex-1"} ${
+                p.sticky ? "lg:sticky lg:top-4 lg:self-start" : ""
+              }`}
+            >
+              {boxed ? (
+                <div className={`flex flex-col overflow-hidden rounded-[14px] border border-ds-line bg-ds-surface ${p.sticky ? stickyMax : ""}`}>
+                  <header className="flex items-center justify-between gap-2 border-b border-ds-line px-4 py-2.5">
+                    <span className="min-w-0 truncate text-[10.5px] font-bold uppercase tracking-widest text-ds-text-3">{p.title}</span>
+                    {p.onClose && (
+                      <button
+                        type="button"
+                        onClick={p.onClose}
+                        aria-label={`Close ${p.label}`}
+                        className="rounded p-1 text-ds-text-3 hover:bg-ds-surface-2 hover:text-ds-text"
+                      >
+                        <X className="h-4 w-4" aria-hidden="true" />
+                      </button>
+                    )}
+                  </header>
+                  <div className="min-h-0 flex-1 overflow-auto p-4">{p.node}</div>
+                </div>
+              ) : (
+                <div className={p.sticky ? `${stickyMax} lg:overflow-auto` : ""}>{p.node}</div>
+              )}
+            </section>
+          </Fragment>
+        )
+      })}
+    </div>
+  )
+}
+
+function clamp(n: number, s: PaneSize) {
+  return Math.min(s.max, Math.max(s.min, n))
+}

otel-ui/src/components/trace/TraceDetail.tsx

@@ -10,12 +10,17 @@ export const TraceDetail = ({
   loading,
   spanId,
   label = "Selected trace",
+  selectedSpanId,
+  onSelectSpan,
 }: {
   traceId: string
   trace?: ProcessedTrace
   loading?: boolean
   spanId?: string
   label?: string
+  /** Controlled span selection: the inspector is then expected to render elsewhere (extended pane). */
+  selectedSpanId?: string | null
+  onSelectSpan?: (spanId: string | null) => void
 }) => {
   const { setActiveTraceId, setActiveSpanId, setActiveTab } = useObservabilityData()
   const open = () => {
@@ -37,7 +42,13 @@ export const TraceDetail = ({
         </button>
       </div>
       {trace ? (
-        <SingleTraceView key={trace.id} trace={trace} />
+        <SingleTraceView
+          key={trace.id}
+          trace={trace}
+          selectedSpanId={selectedSpanId}
+          onSelectSpan={onSelectSpan}
+          externalInspector={onSelectSpan !== undefined}
+        />
       ) : (
         <p className="rounded-[14px] border border-dashed border-ds-line py-10 text-center text-[13px] text-ds-text-3">
           {loading ? "Loading trace…" : "Trace spans are no longer available."}

otel-ui/src/components/trace/TraceListItem.tsx

@@ -0,0 +1,56 @@
+import { formatDuration, formatRelativeTime } from "../../lib/timeUtils"
+import type { TraceSummary } from "../../lib/types"
+
+/**
+ * Compact trace card for narrow master lists: age and duration up top, name,
+ * a duration bar scaled against the set, then span/error/service counts.
+ * Colour is used only for real state (errors present).
+ */
+export const TraceListItem = ({
+  summary,
+  traceId,
+  timestampNs,
+  fallbackTitle,
+  maxDuration,
+  hits,
+}: {
+  summary?: TraceSummary
+  traceId?: string
+  timestampNs: number
+  fallbackTitle: string
+  maxDuration: number
+  hits?: number
+}) => {
+  const errored = (summary?.errorCount ?? 0) > 0
+  return (
+    <div className="flex flex-col gap-1.5">
+      <div className="flex items-baseline justify-between gap-2">
+        <span className="text-[11.5px] text-ds-text-3">{formatRelativeTime(timestampNs / 1e6)}</span>
+        {summary && (
+          <span className={`font-[family-name:var(--font-data)] text-[12px] font-medium ${errored ? "text-ds-critical" : "text-ds-text-2"}`}>
+            {formatDuration(summary.duration)}
+          </span>
+        )}
+      </div>
+      <span className="truncate text-[13px] text-ds-text">{summary?.name ?? fallbackTitle}</span>
+      {summary && (
+        <span className="h-1.5 overflow-hidden rounded-full bg-ds-fill">
+          <span
+            className={`block h-full rounded-full ${errored ? "bg-ds-critical" : "bg-ds-accent"}`}
+            style={{ width: `${Math.max(2, (summary.duration / maxDuration) * 100)}%` }}
+          />
+        </span>
+      )}
+      <div className="flex items-center justify-between gap-2 font-[family-name:var(--font-data)] text-[10.5px] text-ds-text-3">
+        <span className="truncate">
+          {summary
+            ? `${summary.spanCount} spans · ${summary.services.length} svc${errored ? ` · ${summary.errorCount} err` : ""}`
+            : traceId
+              ? `trace:${traceId.slice(0, 8)}`
+              : "no trace attached"}
+        </span>
+        {hits !== undefined && hits > 1 && <span className="shrink-0">×{hits}</span>}
+      </div>
+    </div>
+  )
+}

otel-ui/src/contexts/ObservabilityContextProvider.tsx

@@ -746,6 +746,7 @@ export function ObservabilityProvider({
         logs: filteredLogs,
         metrics: filteredMetrics,
         traces: filteredTraces,
+        allTraces: traces,
         unfocusedLogs,
         unfocusedTraces,
         logsByTraceId,

otel-ui/src/contexts/ObservabilityContextValue.ts

@@ -29,6 +29,8 @@ interface ObservabilityContextType {
   logs: LogEntry[];
   metrics: MetricData[];
   traces: TraceData[];
+  /** Every loaded trace, ignoring the active filters (used for cross-signal linking). */
+  allTraces: TraceData[];
   unfocusedLogs: LogEntry[];
   unfocusedTraces: TraceData[];
   logsByTraceId: Record<string, LogEntry[]>;

otel-ui/src/lib/http-endpoint.ts

@@ -0,0 +1,162 @@
+import { getAttributeValue } from "./otel-utils"
+import type { TraceData } from "./types"
+
+/**
+ * Normalizes the many attribute spellings for "which HTTP endpoint" (stable
+ * semconv, legacy semconv, nginx, Go, browser RUM) into one shape. Mirrors
+ * internal/httpattr in the Go backend - keep the two in sync.
+ */
+
+export interface HttpEndpoint {
+  method: string
+  host: string
+  path: string
+  route: string
+}
+
+type Get = (...keys: string[]) => string
+
+export function cleanPath(p: string): string {
+  p = p.split(/[?#]/)[0].trim()
+  if (!p) return ""
+  if (!p.startsWith("/")) p = "/" + p
+  return p.length > 1 ? p.replace(/\/+$/, "") || "/" : p
+}
+
+function stripPort(host: string): string {
+  return host.replace(/:\d+$/, "")
+}
+
+export function splitUrl(raw: string): { host: string; path: string } {
+  try {
+    const u = new URL(raw, "http://relative.invalid")
+    const host = u.host === "relative.invalid" ? "" : stripPort(u.host.toLowerCase())
+    return { host, path: cleanPath(u.pathname) }
+  } catch {
+    return { host: "", path: cleanPath(raw) }
+  }
+}
+
+export function extractEndpoint(get: Get): HttpEndpoint {
+  const e: HttpEndpoint = {
+    method: get("http.request.method", "http.method").toUpperCase(),
+    route: get("http.route"),
+    host: stripPort(get("server.address", "http.host", "net.host.name", "server.host").toLowerCase()),
+    path: cleanPath(get("url.path") || get("http.target")),
+  }
+  const full = get("url.full", "http.url")
+  if (full) {
+    const u = splitUrl(full)
+    e.path ||= u.path
+    e.host ||= u.host
+  }
+  if (e.route === "/" && e.path && e.path !== "/") e.route = ""
+  return e
+}
+
+/** Removes a reverse-proxy mount prefix; longest match wins. */
+export function stripPrefix(path: string, prefixes: string[]): string {
+  let best = ""
+  for (const raw of prefixes) {
+    const p = cleanPath(raw)
+    if (p && p !== "/" && (path === p || path.startsWith(p + "/")) && p.length > best.length) best = p
+  }
+  return best ? path.slice(best.length) || "/" : path
+}
+
+export interface BackendRequest {
+  traceId: string
+  spanId: string
+  service: string
+  startMs: number
+  durationMs: number
+  endpoint: HttpEndpoint
+}
+
+export interface RumRequest {
+  timestamp: number
+  traceId?: string
+  method?: string
+  url?: string
+  durationMs?: number
+}
+
+export type LinkConfidence = "trace-id" | "path+time"
+
+export interface RumBackendLink {
+  backend: BackendRequest
+  confidence: LinkConfidence
+}
+
+export interface LinkOptions {
+  /** Proxy mount prefixes the browser sees but the backend does not, e.g. ["/api"]. */
+  stripPrefixes?: string[]
+  /** How far a backend span start may be from the browser request, in ms. */
+  windowMs?: number
+}
+
+/**
+ * Finds the backend span a browser request hit. Exact trace id wins; when the
+ * hop in between (e.g. an uninstrumented nginx that neither propagates
+ * traceparent nor emits spans) breaks the trace, fall back to method + path
+ * inside a time window. Returns null unless exactly one candidate is closest,
+ * so an ambiguous match is never presented as a link.
+ */
+export function linkRumRequest(
+  req: RumRequest,
+  backend: BackendRequest[],
+  { stripPrefixes = [], windowMs = 2000 }: LinkOptions = {},
+): RumBackendLink | null {
+  if (req.traceId) {
+    const hit = backend.find((b) => b.traceId === req.traceId)
+    if (hit) return { backend: hit, confidence: "trace-id" }
+  }
+  if (!req.url) return null
+  const { path } = splitUrl(req.url)
+  const method = (req.method || "GET").toUpperCase()
+  const wanted = new Set([path, stripPrefix(path, stripPrefixes)])
+  // The request can only reach the server after the browser started it, and
+  // must have started before the browser saw the response.
+  const end = req.timestamp + (req.durationMs ?? 0)
+  const candidates = backend
+    .filter((b) => b.endpoint.method === method && wanted.has(b.endpoint.path))
+    .filter((b) => b.startMs >= req.timestamp - windowMs && b.startMs <= Math.max(end, req.timestamp) + windowMs)
+    .map((b) => ({ b, d: Math.abs(b.startMs - req.timestamp) }))
+    .sort((x, y) => x.d - y.d)
+  if (candidates.length === 0) return null
+  if (candidates.length > 1 && candidates[0].d === candidates[1].d) return null
+  return { backend: candidates[0].b, confidence: "path+time" }
+}
+
+/** Server-side HTTP spans (kind SERVER) from loaded traces, normalized for linking. */
+export function backendRequestsFromTraces(traces: TraceData[]): BackendRequest[] {
+  const out: BackendRequest[] = []
+  for (const trace of traces) {
+    for (const rs of trace.resourceSpans ?? []) {
+      const service = getAttributeValue(rs.resource?.attributes ?? [], "service.name")
+      for (const ss of rs.scopeSpans ?? []) {
+        for (const s of ss.spans ?? []) {
+          if (s.kind !== 2) continue
+          const endpoint = extractEndpoint((...keys) => {
+            for (const k of keys) {
+              const v = getAttributeValue(s.attributes ?? [], k)
+              if (v) return v
+            }
+            return ""
+          })
+          if (!endpoint.path) continue
+          const start = Number(BigInt(s.startTimeUnixNano) / 1_000_000n)
+          const end = Number(BigInt(s.endTimeUnixNano) / 1_000_000n)
+          out.push({ traceId: s.traceId, spanId: s.spanId, service, startMs: start, durationMs: end - start, endpoint })
+        }
+      }
+    }
+  }
+  return out
+}
+
+/** Parses an otel-rum.js fetch/xhr breadcrumb: "GET /api/x -> 200". */
+export function parseRequestCrumb(message: string): { method: string; url: string } | null {
+  const m = /^([A-Z]+)\s+(\S+)\s+->\s+\S+$/.exec(message)
+  return m ? { method: m[1], url: m[2] } : null
+}

otel-ui/src/lib/issue-utils.ts

@@ -35,6 +35,10 @@ export interface ParsedBreadcrumb {
   t: number
   category: string
   message: string
+  /** Request duration in ms (fetch/xhr crumbs); the request started at t - d. */
+  d?: number
+  /** Trace id otel-rum.js sent on the request, if it injected traceparent. */
+  tid?: string
 }
 
 /** Safely parses the raw breadcrumb JSON attached to a frontend sample. */

otel-ui/src/lib/rum-utils.ts

@@ -74,6 +74,8 @@ export interface RumTimelineEntry {
   category: "session-start" | "navigation" | "fetch" | "xhr" | "console.error" | "js-error" | "http-failure"
   message: string
   severe: boolean
+  durationMs?: number
+  traceId?: string
 }
 
 export interface RumSession {
@@ -273,6 +275,8 @@ export function buildRumSessions(logs: LogEntry[]): RumSession[] {
           : "navigation",
         message: crumb.message,
         severe: crumb.category === "console.error" || isFailedRequestBreadcrumb(crumb.message),
+        durationMs: typeof crumb.d === "number" ? crumb.d : undefined,
+        traceId: crumb.tid || undefined,
       })
     }
     for (const err of errors) {

otel-ui/vite.config.ts

@@ -2,7 +2,7 @@ import { defineConfig } from "vite";
 import react from "@vitejs/plugin-react";
 import tailwindcss from "@tailwindcss/vite";
 
-const target = "https://otel.k6n.net";
+const target = process.env.OTEL_URL ?? "https://otel.k6n.net";
 // Device-grant issuer for local/agent development (see src/dev/device-auth.ts).
 // Proxied same-origin so the browser needs no CORS and no issuer URL.
 const authIssuer = process.env.AUTH_DEV_BASE ?? "https://users.k6n.net";

static/otel-rum.js

@@ -524,16 +524,17 @@
 					requestInit = Object.assign({}, requestInit, { headers: headers });
 				}
 
+				var startedAt = Date.now();
 				return originalFetch.call(global, input, requestInit).then(
 					function (res) {
-						addBreadcrumb('fetch', method + ' ' + url + ' -> ' + res.status);
+						addBreadcrumb('fetch', method + ' ' + url + ' -> ' + res.status, { d: Date.now() - startedAt, tid: requestTraceId });
 						if (reportHttpFailures && res.status >= 400) {
 							httpFailureReport('fetch', method, url, res.status, requestTraceId);
 						}
 						return res;
 					},
 					function (err) {
-						addBreadcrumb('fetch', method + ' ' + url + ' -> error');
+						addBreadcrumb('fetch', method + ' ' + url + ' -> error', { d: Date.now() - startedAt, tid: requestTraceId });
 						if (reportHttpFailures) {
 							httpFailureReport('fetch', method, url, 0, requestTraceId);
 						}
@@ -571,12 +572,13 @@
 			global.XMLHttpRequest.prototype.send = function () {
 				var xhr = this;
 				var info = xhr.__rum || {};
+				var xhrStartedAt = Date.now();
 				if (info.url) {
 					var origHandler = xhr.onreadystatechange;
 					xhr.onreadystatechange = function () {
 						try {
 							if (xhr.readyState === 4) {
-								addBreadcrumb('xhr', info.method + ' ' + info.url + ' -> ' + xhr.status);
+								addBreadcrumb('xhr', info.method + ' ' + info.url + ' -> ' + xhr.status, { d: Date.now() - xhrStartedAt, tid: info.traceId });
 								if (reportHttpFailures && (xhr.status === 0 || xhr.status >= 400)) {
 									httpFailureReport('xhr', info.method, info.url, xhr.status, info.traceId);
 								}
@@ -644,8 +646,13 @@
 			observeVitals();
 		}
 
-		function addBreadcrumb(category, message) {
-			breadcrumbs.push({ t: Date.now(), category: String(category), message: String(message) });
+		// extra.d = request duration in ms, extra.tid = trace id we sent on the
+		// request. t stays the completion time; the request started at t - d.
+		function addBreadcrumb(category, message, extra) {
+			var crumb = { t: Date.now(), category: String(category), message: String(message) };
+			if (extra && typeof extra.d === 'number') crumb.d = extra.d;
+			if (extra && extra.tid) crumb.tid = String(extra.tid);
+			breadcrumbs.push(crumb);
 			if (breadcrumbs.length > maxBreadcrumbs) {
 				breadcrumbs.shift();
 			}