Merge branch 'main' into feature/issues-tui

README.md

@@ -377,6 +377,8 @@ to Docker or stored in the image.
 The standalone `issues-tui` command is a terminal dashboard for currently
 firing alert spikes. It reads normalized error events directly from
 ClickHouse; it does not replace the web UI's persisted `/api/issues` view.
+When a spike stops firing, its card stays on the display marked resolved
+until something fires again.
 
 Build and run it on the display computer (or use `go run`):
 

cmd/issues-tui/main.go

@@ -7,7 +7,8 @@
 // A pattern shows as PAGED when this tick would have sent a notification
 // (new or escalated past its cooldown) and as ongoing while it is still
 // firing but already paged once. When a service's spike disappears it is
-// reported as recovered for that scan.
+// reported as recovered for that scan, and the last firing issue stays on
+// the display as resolved until something fires again.
 package main
 
 import (
@@ -96,6 +97,12 @@ type scanner struct {
 	// firing; it mirrors what a person watching notifications would remember.
 	active map[string]time.Time
 
+	// lastActive is the most recent firing issue set; when a later scan
+	// finds nothing firing, it is kept on the display as resolved instead
+	// of flipping to ALL CLEAR.
+	lastActive []issueView
+	resolvedAt time.Time // window end when lastActive stopped firing
+
 	since   time.Time
 	sinceAt time.Time
 }
@@ -121,10 +128,11 @@ func newClickHouseScanner(addr, database, user, password string, cfg features.Sp
 // issueView is one card on the dashboard: a firing spike reduced to an Alert
 // (the same model the notifications carry) plus display state.
 type issueView struct {
-	alert     alerts.Alert
-	key       string
-	paged     bool      // this scan would have notified (new or escalated)
-	firstSeen time.Time // window end when this issue first appeared
+	alert      alerts.Alert
+	key        string
+	paged      bool      // this scan would have notified (new or escalated)
+	firstSeen  time.Time // window end when this issue first appeared
+	resolvedAt time.Time // window end when it stopped firing; renders as resolved
 }
 
 // recoveredIssue is an issue that fired on the previous scan but not now.
@@ -134,7 +142,8 @@ type recoveredIssue struct {
 	LastSeen  time.Time
 }
 
-// snapshot is one scan's result: everything firing right now.
+// snapshot is one scan's result: everything firing right now, plus the last
+// firing issue kept as resolved when nothing fires any more.
 type snapshot struct {
 	ScannedAt   time.Time
 	WindowEnd   time.Time
@@ -142,6 +151,7 @@ type snapshot struct {
 	Events      int
 	Issues      []issueView
 	Recovered   []recoveredIssue
+	Resolved    []issueView
 	Err         error
 }
 
@@ -219,6 +229,22 @@ func (s *scanner) Scan(ctx context.Context, now time.Time) snapshot {
 			snap.Recovered = append(snap.Recovered, recoveredIssue{Service: serviceOfKey(key), FirstSeen: first, LastSeen: end})
 		}
 	}
+	// Keep the last firing issue on the display: once nothing fires any
+	// more, the cards stay up as resolved rather than the dashboard erasing
+	// the context with an ALL CLEAR panel.
+	if len(snap.Issues) > 0 {
+		s.lastActive = snap.Issues
+		s.resolvedAt = time.Time{}
+	} else if len(s.lastActive) > 0 {
+		if s.resolvedAt.IsZero() {
+			s.resolvedAt = end
+		}
+		for _, v := range s.lastActive {
+			v.resolvedAt = s.resolvedAt
+			snap.Resolved = append(snap.Resolved, v)
+		}
+		sortIssues(snap.Resolved)
+	}
 	sortIssues(snap.Issues)
 	return snap
 }

cmd/issues-tui/main_test.go

@@ -96,10 +96,20 @@ func TestScannerPagesOnceThenOngoing(t *testing.T) {
 	if len(fourth.Recovered) != 1 || fourth.Recovered[0].Service != "checkout" {
 		t.Fatalf("expected checkout in recovered, got %+v", fourth.Recovered)
 	}
+	// The last issue stays on the display as resolved instead of ALL CLEAR.
+	if len(fourth.Resolved) != 1 || fourth.Resolved[0].alert.Spike.Service != "checkout" {
+		t.Fatalf("expected checkout kept as resolved, got %+v", fourth.Resolved)
+	}
+	if got := fourth.Resolved[0].resolvedAt; !got.Equal(fourth.WindowEnd) {
+		t.Fatalf("resolvedAt should be the window that stopped firing, got %v want %v", got, fourth.WindowEnd)
+	}
 	again := sc.Scan(context.Background(), base.Add(4*time.Minute))
 	if len(again.Recovered) != 0 {
 		t.Fatalf("recovery should be reported once, got %+v", again.Recovered)
 	}
+	if len(again.Resolved) != 1 || !again.Resolved[0].resolvedAt.Equal(fourth.Resolved[0].resolvedAt) {
+		t.Fatalf("resolved issue should persist with a stable resolved time, got %+v", again.Resolved)
+	}
 }
 
 func TestScannerAllClearAndError(t *testing.T) {
@@ -108,6 +118,9 @@ func TestScannerAllClearAndError(t *testing.T) {
 	if len(empty.Issues) != 0 || empty.Err != nil {
 		t.Fatalf("empty loader should yield a clean snapshot, got %+v", empty)
 	}
+	if len(empty.Resolved) != 0 {
+		t.Fatalf("a fresh scanner has no last issue to keep, got %+v", empty.Resolved)
+	}
 
 	sc.load = func(context.Context, time.Time, time.Time) ([]features.Event, error) {
 		return nil, context.DeadlineExceeded
@@ -204,6 +217,22 @@ func TestRenderSnapshotAllClear(t *testing.T) {
 	}
 }
 
+func TestRenderSnapshotKeepsResolvedIssue(t *testing.T) {
+	now := time.Date(2025, 6, 1, 12, 45, 0, 0, time.UTC)
+	snap, o := cardAt(t, 1)
+	resolved := snap.Issues[0]
+	resolved.resolvedAt = now
+	snap.Issues = nil
+	snap.Resolved = []issueView{resolved}
+	out := renderSnapshot(snap, o)
+	if strings.Contains(out, "ALL CLEAR") {
+		t.Fatalf("a resolved issue should replace the ALL CLEAR panel:\n%s", out)
+	}
+	if !strings.Contains(out, "RESOLVED") || !strings.Contains(out, "checkout") {
+		t.Fatalf("dashboard should keep the last issue as resolved:\n%s", out)
+	}
+}
+
 func TestRenderSnapshotScanError(t *testing.T) {
 	snap := snapshot{Err: context.DeadlineExceeded}
 	out := renderSnapshot(snap, snapshotOptions{Width: 200, Height: 12, Title: "otel issues", Now: time.Now()})

cmd/issues-tui/view.go

@@ -91,6 +91,10 @@ func renderBody(snap snapshot, o snapshotOptions, w, avail int) string {
 	case o.ScanError != nil && len(snap.Issues) == 0:
 		return statusPanel(w, min(avail, 5), colCritical, "SCAN FAILED", o.ScanError.Error(),
 			"no successful scan yet; retrying automatically")
+	case len(snap.Issues) == 0 && len(snap.Resolved) > 0:
+		// Nothing fires any more: keep the last issue on the display,
+		// flipped to resolved, instead of an ALL CLEAR panel.
+		return renderGrid(snap.Resolved, o, w, avail)
 	case len(snap.Issues) == 0:
 		sub := fmt.Sprintf("no issues firing · window %s–%s UTC · scanned %s",
 			snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"), commas(snap.Events))
@@ -99,19 +103,19 @@ func renderBody(snap snapshot, o snapshotOptions, w, avail int) string {
 		}
 		return statusPanel(w, min(avail, 5), colOK, "ALL CLEAR", sub, "")
 	default:
-		return renderGrid(snap, o, w, avail)
+		return renderGrid(snap.Issues, o, w, avail)
 	}
 }
 
 // renderGrid lays issues into rows of cards that fill the width; only as many
 // rows as the height allows are drawn, the rest are counted in the footer.
-func renderGrid(snap snapshot, o snapshotOptions, w, avail int) string {
-	perRow, _, shown, cardW := gridLayout(len(snap.Issues), w, avail)
+func renderGrid(views []issueView, o snapshotOptions, w, avail int) string {
+	perRow, _, shown, cardW := gridLayout(len(views), w, avail)
 	var rendered []string
 	for start := 0; start < shown; start += perRow {
 		end := min(start+perRow, shown)
 		var row []string
-		for _, v := range snap.Issues[start:end] {
+		for _, v := range views[start:end] {
 			row = append(row, issueCard(v, cardW, o))
 		}
 		rendered = append(rendered, lipgloss.JoinHorizontal(lipgloss.Top, row...))
@@ -145,11 +149,19 @@ func gridLayout(n, width, avail int) (perRow, rows, shown, cardW int) {
 func issueCard(v issueView, w int, o snapshotOptions) string {
 	a := v.alert
 	fg := priorityColor(a.Priority())
+	resolved := !v.resolvedAt.IsZero()
+	if resolved {
+		// A resolved card keeps its place but reads as recovered: the same
+		// green as the footer's recovered line and the ALL CLEAR panel.
+		fg = colOK
+	}
 	inner := w - 4 // border plus padding on both sides
 
 	// Line 1: HIGH · checkout · for 12m
 	age := ""
-	if !v.firstSeen.IsZero() && !o.Now.IsZero() {
+	if !v.firstSeen.IsZero() && resolved {
+		age = " · for " + shortDur(v.resolvedAt.Sub(v.firstSeen).Round(time.Minute))
+	} else if !v.firstSeen.IsZero() && !o.Now.IsZero() {
 		age = " · for " + shortDur(o.Now.Sub(v.firstSeen).Round(time.Minute))
 	}
 	head := ansi.Truncate(
@@ -190,14 +202,20 @@ func issueCard(v issueView, w int, o snapshotOptions) string {
 	}
 	statsLine := ansi.Truncate(strings.Join(segs, "  "), inner, "…")
 
-	// Line 4: display state (paged now vs already paged) and the issue link.
+	// Line 4: display state (paged now vs already paged vs resolved) and the
+	// issue link.
 	state := style(colText3, false).Render("ongoing")
-	if v.paged {
+	when := a.Spike.WindowEnd
+	switch {
+	case resolved:
+		state = style(colOK, true).Render("RESOLVED")
+		when = v.resolvedAt
+	case v.paged:
 		state = style(fg, true).Render("PAGED")
 	}
 	var meta []string
-	if !a.Spike.WindowEnd.IsZero() {
-		meta = append(meta, style(colText3, false).Render(fmtTime(a.Spike.WindowEnd)))
+	if !when.IsZero() {
+		meta = append(meta, style(colText3, false).Render(fmtTime(when)))
 	}
 	meta = append(meta, state)
 	if o.BaseURL != "" && len(a.Patterns) > 0 && a.Patterns[0].IssueID != "" {
@@ -254,9 +272,12 @@ func renderHeader(snap snapshot, o snapshotOptions) string {
 			}
 		}
 		counts := fmt.Sprintf("%d firing", len(snap.Issues))
-		if paged > 0 {
+		switch {
+		case len(snap.Issues) == 0 && len(snap.Resolved) > 0:
+			left = append(left, style(colOK, true).Render(fmt.Sprintf("%d resolved", len(snap.Resolved))))
+		case paged > 0:
 			left = append(left, style(colCritical, true).Render(fmt.Sprintf("%d PAGED · %s", paged, counts)))
-		} else {
+		default:
 			left = append(left, style(colText2, false).Render(counts))
 		}
 	}
@@ -282,7 +303,7 @@ func renderHeader(snap snapshot, o snapshotOptions) string {
 func renderFooter(snap snapshot, o snapshotOptions) string {
 	var left []string
 	left = append(left, style(colText3, false).Render(footerHint))
-	if hidden := len(snap.Issues) - shownCount(snap, o); hidden > 0 {
+	if hidden := cardCount(snap) - shownCount(snap, o); hidden > 0 {
 		left = append(left, style(colWarn, false).Render(fmt.Sprintf("+%d more not shown (widen or raise the display)", hidden)))
 	}
 	var right []string
@@ -299,11 +320,20 @@ func renderFooter(snap snapshot, o snapshotOptions) string {
 	return bar(o.Width-2, strings.Join(left, "  "), strings.Join(right, "  "))
 }
 
+// cardCount is how many cards the body renders: the firing issues, or the
+// kept resolved ones when nothing fires.
+func cardCount(snap snapshot) int {
+	if n := len(snap.Issues); n > 0 {
+		return n
+	}
+	return len(snap.Resolved)
+}
+
 func shownCount(snap snapshot, o snapshotOptions) int {
 	if o.Height < minUsableHeight {
-		return len(snap.Issues)
+		return cardCount(snap)
 	}
-	_, _, shown, _ := gridLayout(len(snap.Issues), o.Width, o.Height-2)
+	_, _, shown, _ := gridLayout(cardCount(snap), o.Width, o.Height-2)
 	return shown
 }
 
@@ -323,14 +353,20 @@ func renderMinimal(snap snapshot, o snapshotOptions) string {
 	if snap.Err != nil {
 		return "SCAN FAILED: " + snap.Err.Error()
 	}
-	if len(snap.Issues) == 0 {
+	if len(snap.Issues) == 0 && len(snap.Resolved) == 0 {
 		return strings.ToUpper(o.Title) + ": all clear"
 	}
-	lines := make([]string, 0, len(snap.Issues)+1)
+	views := snap.Issues
+	if len(views) == 0 {
+		views = snap.Resolved
+	}
+	lines := make([]string, 0, len(views)+1)
 	lines = append(lines, strings.ToUpper(o.Title)+":")
-	for _, v := range snap.Issues {
+	for _, v := range views {
 		state := "ongoing"
-		if v.paged {
+		if !v.resolvedAt.IsZero() {
+			state = "RESOLVED"
+		} else if v.paged {
 			state = "PAGED"
 		}
 		what := ""

otel-ui/src/components/CspReportsVisualizer.tsx

@@ -183,8 +183,8 @@ export function CspReportsVisualizer() {
       <div className="grid grid-cols-2 gap-3 lg:grid-cols-4">
         <StatCard label="Reports in window" value={summary ? summary.total.toLocaleString() : "—"} />
         <StatCard label="Sites reporting" value={summary ? summary.sites.toLocaleString() : "—"} />
-        <StatCard label="Top directive" value={summary?.byDirective[0]?.name ?? "—"} />
-        <StatCard label="Top blocked host" value={summary?.byBlockedHost[0]?.name ?? "—"} />
+        <StatCard label="Top directive" value={summary?.byDirective?.[0]?.name ?? "—"} />
+        <StatCard label="Top blocked host" value={summary?.byBlockedHost?.[0]?.name ?? "—"} />
       </div>
 
       <FilterBar

otel-ui/src/components/TraceVisualization.tsx

@@ -1,4 +1,4 @@
-import { useMemo, useState, useRef, useEffect } from "react";
+import { useMemo, useState, useRef, useEffect, useDeferredValue } from "react";
 import { useVirtualizer } from "@tanstack/react-virtual";
 import {
   Sparkles,
@@ -54,6 +54,8 @@ export type ProcessedTrace = {
   endpoint: string;
 };
 
+const EMPTY_TRACES: TraceData[] = [];
+
 // Flattened item for virtualized tree rendering
 interface FlattenedSpanItem {
   span: ProcessedSpan;
@@ -96,14 +98,24 @@ const flattenSpanTree = (
   collapsedSet: Set<string>,
   traceMinTime: number,
   traceTotalRange: number,
-  depth = 0,
+  initialDepth = 0,
 ): FlattenedSpanItem[] => {
   const result: FlattenedSpanItem[] = [];
+  const pending: Array<{ span: ProcessedSpan; depth: number }> = [];
 
-  for (const span of nodes) {
-    const hasChildren = Boolean(span.children && span.children.length > 0);
+  for (let index = nodes.length - 1; index >= 0; index -= 1) {
+    pending.push({ span: nodes[index], depth: initialDepth });
+  }
+
+  // Avoid recursively spreading descendant arrays: deep parent/child chains
+  // otherwise copy the same nodes once per ancestor.
+  while (pending.length > 0) {
+    const current = pending.pop();
+    if (!current) break;
+    const { span, depth } = current;
+    const children = span.children ?? [];
+    const hasChildren = children.length > 0;
     const isCollapsed = collapsedSet.has(span.spanId);
-
     const offsetPercent =
       traceTotalRange === 0 ? 0 : ((span.startTime - traceMinTime) / traceTotalRange) * 100;
     const widthPercent =
@@ -118,16 +130,10 @@ const flattenSpanTree = (
       widthPercent: Math.max(0.5, Math.min(widthPercent, 100 - offsetPercent)),
     });
 
-    if (hasChildren && !isCollapsed && span.children) {
-      result.push(
-        ...flattenSpanTree(
-          span.children,
-          collapsedSet,
-          traceMinTime,
-          traceTotalRange,
-          depth + 1,
-        ),
-      );
+    if (hasChildren && !isCollapsed) {
+      for (let index = children.length - 1; index >= 0; index -= 1) {
+        pending.push({ span: children[index], depth: depth + 1 });
+      }
     }
   }
 
@@ -539,6 +545,7 @@ const VirtualizedSpanWaterfall = ({
   const [searchQuery, setSearchQuery] = useState("");
   const [errorsOnly, setErrorsOnly] = useState(false);
   const [minDurationMsFilter, setMinDurationMsFilter] = useState<number>(0);
+  const deferredSearchQuery = useDeferredValue(searchQuery);
   const parentRef = useRef<HTMLDivElement>(null);
 
   // Build tree
@@ -563,8 +570,8 @@ const VirtualizedSpanWaterfall = ({
         const durMs = item.span.duration / 1_000_000;
         if (durMs < minDurationMsFilter) return false;
       }
-      if (searchQuery.trim()) {
-        const query = searchQuery.toLowerCase().trim();
+      if (deferredSearchQuery.trim()) {
+        const query = deferredSearchQuery.toLowerCase().trim();
         const matchesName = item.span.name.toLowerCase().includes(query);
         const matchesService = item.span.service.toLowerCase().includes(query);
         const matchesAttrs = Object.values(item.span.formattedAttributes).some((v) =>
@@ -574,7 +581,7 @@ const VirtualizedSpanWaterfall = ({
       }
       return true;
     });
-  }, [flattenedItems, filterService, errorsOnly, minDurationMsFilter, searchQuery]);
+  }, [flattenedItems, filterService, errorsOnly, minDurationMsFilter, deferredSearchQuery]);
 
 
   // Virtualizer
@@ -1683,10 +1690,12 @@ export const processTraces = (
     ),
   );
 
-  const uniqueSpans = spanAccumulator.filter(
-    (span, index, arr) =>
-      arr.findIndex((candidate) => candidate.spanId === span.spanId) === index,
-  );
+  const seenSpanIds = new Set<string>();
+  const uniqueSpans = spanAccumulator.filter((span) => {
+    if (seenSpanIds.has(span.spanId)) return false;
+    seenSpanIds.add(span.spanId);
+    return true;
+  });
 
   const traceGroups = uniqueSpans.reduce<Record<string, ProcessedSpan[]>>((acc, span) => {
     if (!acc[span.traceId]) acc[span.traceId] = [];
@@ -1739,7 +1748,7 @@ export interface TraceVisualizationProps {
 }
 
 export const TraceVisualization = ({ traces }: TraceVisualizationProps) => {
-  const safeTraces = traces ?? [];
+  const safeTraces = traces ?? EMPTY_TRACES;
   const {
     logCountBySpanId,
     activeTraceId,
@@ -1756,6 +1765,7 @@ export const TraceVisualization = ({ traces }: TraceVisualizationProps) => {
  "recent" | "duration-desc" | "duration-asc" | "errors" | "spans"
   >("recent");
   const [groupByEndpoint, setGroupByEndpoint] = useState(false);
+  const deferredTraceSearchQuery = useDeferredValue(traceSearchQuery);
 
   const orderedTraces: ProcessedTrace[] = useMemo(
     () => processTraces(safeTraces, logCountBySpanId),
@@ -1774,8 +1784,8 @@ export const TraceVisualization = ({ traces }: TraceVisualizationProps) => {
         if (traceFilter === "errors" && !t.hasError) return false;
         if (traceFilter === "slow" && t.duration / 1_000_000 < 200) return false;
         if (traceServiceFilter !== "all" && !t.services.includes(traceServiceFilter)) return false;
-        if (traceSearchQuery.trim()) {
-          const q = traceSearchQuery.toLowerCase().trim();
+        if (deferredTraceSearchQuery.trim()) {
+          const q = deferredTraceSearchQuery.toLowerCase().trim();
           const matchName = t.name.toLowerCase().includes(q);
           const matchId = t.id.toLowerCase().includes(q);
           const matchService = t.services.some((s) => s.toLowerCase().includes(q));
@@ -1791,7 +1801,7 @@ export const TraceVisualization = ({ traces }: TraceVisualizationProps) => {
         if (traceSortBy === "spans") return b.spans.length - a.spans.length;
         return b.startTime - a.startTime; // recent
       });
-  }, [orderedTraces, traceFilter, traceServiceFilter, traceSearchQuery, traceSortBy]);
+  }, [orderedTraces, traceFilter, traceServiceFilter, deferredTraceSearchQuery, traceSortBy]);
 
   const groupCounts = useMemo(() => {
     const counts: Record<string, number> = {};