diff --git a/README.md b/README.md index e58cd0f..05df5e5 100644 --- a/README.md +++ b/README.md @@ -377,6 +377,8 @@ to Docker or stored in the image. The standalone `issues-tui` command is a terminal dashboard for currently firing alert spikes. It reads normalized error events directly from ClickHouse; it does not replace the web UI's persisted `/api/issues` view. +When a spike stops firing, its card stays on the display marked resolved +until something fires again. Build and run it on the display computer (or use `go run`): diff --git a/cmd/issues-tui/main.go b/cmd/issues-tui/main.go index dff243f..44ab0ed 100644 --- a/cmd/issues-tui/main.go +++ b/cmd/issues-tui/main.go @@ -7,7 +7,8 @@ // A pattern shows as PAGED when this tick would have sent a notification // (new or escalated past its cooldown) and as ongoing while it is still // firing but already paged once. When a service's spike disappears it is -// reported as recovered for that scan. +// reported as recovered for that scan, and the last firing issue stays on +// the display as resolved until something fires again. package main import ( @@ -96,6 +97,12 @@ type scanner struct { // firing; it mirrors what a person watching notifications would remember. active map[string]time.Time + // lastActive is the most recent firing issue set; when a later scan + // finds nothing firing, it is kept on the display as resolved instead + // of flipping to ALL CLEAR. + lastActive []issueView + resolvedAt time.Time // window end when lastActive stopped firing + since time.Time sinceAt time.Time } @@ -121,10 +128,11 @@ func newClickHouseScanner(addr, database, user, password string, cfg features.Sp // issueView is one card on the dashboard: a firing spike reduced to an Alert // (the same model the notifications carry) plus display state. type issueView struct { - alert alerts.Alert - key string - paged bool // this scan would have notified (new or escalated) - firstSeen time.Time // window end when this issue first appeared + alert alerts.Alert + key string + paged bool // this scan would have notified (new or escalated) + firstSeen time.Time // window end when this issue first appeared + resolvedAt time.Time // window end when it stopped firing; renders as resolved } // recoveredIssue is an issue that fired on the previous scan but not now. @@ -134,7 +142,8 @@ type recoveredIssue struct { LastSeen time.Time } -// snapshot is one scan's result: everything firing right now. +// snapshot is one scan's result: everything firing right now, plus the last +// firing issue kept as resolved when nothing fires any more. type snapshot struct { ScannedAt time.Time WindowEnd time.Time @@ -142,6 +151,7 @@ type snapshot struct { Events int Issues []issueView Recovered []recoveredIssue + Resolved []issueView Err error } @@ -219,6 +229,22 @@ func (s *scanner) Scan(ctx context.Context, now time.Time) snapshot { snap.Recovered = append(snap.Recovered, recoveredIssue{Service: serviceOfKey(key), FirstSeen: first, LastSeen: end}) } } + // Keep the last firing issue on the display: once nothing fires any + // more, the cards stay up as resolved rather than the dashboard erasing + // the context with an ALL CLEAR panel. + if len(snap.Issues) > 0 { + s.lastActive = snap.Issues + s.resolvedAt = time.Time{} + } else if len(s.lastActive) > 0 { + if s.resolvedAt.IsZero() { + s.resolvedAt = end + } + for _, v := range s.lastActive { + v.resolvedAt = s.resolvedAt + snap.Resolved = append(snap.Resolved, v) + } + sortIssues(snap.Resolved) + } sortIssues(snap.Issues) return snap } diff --git a/cmd/issues-tui/main_test.go b/cmd/issues-tui/main_test.go index e2547b9..2b6e04f 100644 --- a/cmd/issues-tui/main_test.go +++ b/cmd/issues-tui/main_test.go @@ -96,10 +96,20 @@ func TestScannerPagesOnceThenOngoing(t *testing.T) { if len(fourth.Recovered) != 1 || fourth.Recovered[0].Service != "checkout" { t.Fatalf("expected checkout in recovered, got %+v", fourth.Recovered) } + // The last issue stays on the display as resolved instead of ALL CLEAR. + if len(fourth.Resolved) != 1 || fourth.Resolved[0].alert.Spike.Service != "checkout" { + t.Fatalf("expected checkout kept as resolved, got %+v", fourth.Resolved) + } + if got := fourth.Resolved[0].resolvedAt; !got.Equal(fourth.WindowEnd) { + t.Fatalf("resolvedAt should be the window that stopped firing, got %v want %v", got, fourth.WindowEnd) + } again := sc.Scan(context.Background(), base.Add(4*time.Minute)) if len(again.Recovered) != 0 { t.Fatalf("recovery should be reported once, got %+v", again.Recovered) } + if len(again.Resolved) != 1 || !again.Resolved[0].resolvedAt.Equal(fourth.Resolved[0].resolvedAt) { + t.Fatalf("resolved issue should persist with a stable resolved time, got %+v", again.Resolved) + } } func TestScannerAllClearAndError(t *testing.T) { @@ -108,6 +118,9 @@ func TestScannerAllClearAndError(t *testing.T) { if len(empty.Issues) != 0 || empty.Err != nil { t.Fatalf("empty loader should yield a clean snapshot, got %+v", empty) } + if len(empty.Resolved) != 0 { + t.Fatalf("a fresh scanner has no last issue to keep, got %+v", empty.Resolved) + } sc.load = func(context.Context, time.Time, time.Time) ([]features.Event, error) { return nil, context.DeadlineExceeded @@ -204,6 +217,22 @@ func TestRenderSnapshotAllClear(t *testing.T) { } } +func TestRenderSnapshotKeepsResolvedIssue(t *testing.T) { + now := time.Date(2025, 6, 1, 12, 45, 0, 0, time.UTC) + snap, o := cardAt(t, 1) + resolved := snap.Issues[0] + resolved.resolvedAt = now + snap.Issues = nil + snap.Resolved = []issueView{resolved} + out := renderSnapshot(snap, o) + if strings.Contains(out, "ALL CLEAR") { + t.Fatalf("a resolved issue should replace the ALL CLEAR panel:\n%s", out) + } + if !strings.Contains(out, "RESOLVED") || !strings.Contains(out, "checkout") { + t.Fatalf("dashboard should keep the last issue as resolved:\n%s", out) + } +} + func TestRenderSnapshotScanError(t *testing.T) { snap := snapshot{Err: context.DeadlineExceeded} out := renderSnapshot(snap, snapshotOptions{Width: 200, Height: 12, Title: "otel issues", Now: time.Now()}) diff --git a/cmd/issues-tui/view.go b/cmd/issues-tui/view.go index 449f900..2d326e9 100644 --- a/cmd/issues-tui/view.go +++ b/cmd/issues-tui/view.go @@ -91,6 +91,10 @@ func renderBody(snap snapshot, o snapshotOptions, w, avail int) string { case o.ScanError != nil && len(snap.Issues) == 0: return statusPanel(w, min(avail, 5), colCritical, "SCAN FAILED", o.ScanError.Error(), "no successful scan yet; retrying automatically") + case len(snap.Issues) == 0 && len(snap.Resolved) > 0: + // Nothing fires any more: keep the last issue on the display, + // flipped to resolved, instead of an ALL CLEAR panel. + return renderGrid(snap.Resolved, o, w, avail) case len(snap.Issues) == 0: sub := fmt.Sprintf("no issues firing · window %s–%s UTC · scanned %s", snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"), commas(snap.Events)) @@ -99,19 +103,19 @@ func renderBody(snap snapshot, o snapshotOptions, w, avail int) string { } return statusPanel(w, min(avail, 5), colOK, "ALL CLEAR", sub, "") default: - return renderGrid(snap, o, w, avail) + return renderGrid(snap.Issues, o, w, avail) } } // renderGrid lays issues into rows of cards that fill the width; only as many // rows as the height allows are drawn, the rest are counted in the footer. -func renderGrid(snap snapshot, o snapshotOptions, w, avail int) string { - perRow, _, shown, cardW := gridLayout(len(snap.Issues), w, avail) +func renderGrid(views []issueView, o snapshotOptions, w, avail int) string { + perRow, _, shown, cardW := gridLayout(len(views), w, avail) var rendered []string for start := 0; start < shown; start += perRow { end := min(start+perRow, shown) var row []string - for _, v := range snap.Issues[start:end] { + for _, v := range views[start:end] { row = append(row, issueCard(v, cardW, o)) } rendered = append(rendered, lipgloss.JoinHorizontal(lipgloss.Top, row...)) @@ -145,11 +149,19 @@ func gridLayout(n, width, avail int) (perRow, rows, shown, cardW int) { func issueCard(v issueView, w int, o snapshotOptions) string { a := v.alert fg := priorityColor(a.Priority()) + resolved := !v.resolvedAt.IsZero() + if resolved { + // A resolved card keeps its place but reads as recovered: the same + // green as the footer's recovered line and the ALL CLEAR panel. + fg = colOK + } inner := w - 4 // border plus padding on both sides // Line 1: HIGH · checkout · for 12m age := "" - if !v.firstSeen.IsZero() && !o.Now.IsZero() { + if !v.firstSeen.IsZero() && resolved { + age = " · for " + shortDur(v.resolvedAt.Sub(v.firstSeen).Round(time.Minute)) + } else if !v.firstSeen.IsZero() && !o.Now.IsZero() { age = " · for " + shortDur(o.Now.Sub(v.firstSeen).Round(time.Minute)) } head := ansi.Truncate( @@ -190,14 +202,20 @@ func issueCard(v issueView, w int, o snapshotOptions) string { } statsLine := ansi.Truncate(strings.Join(segs, " "), inner, "…") - // Line 4: display state (paged now vs already paged) and the issue link. + // Line 4: display state (paged now vs already paged vs resolved) and the + // issue link. state := style(colText3, false).Render("ongoing") - if v.paged { + when := a.Spike.WindowEnd + switch { + case resolved: + state = style(colOK, true).Render("RESOLVED") + when = v.resolvedAt + case v.paged: state = style(fg, true).Render("PAGED") } var meta []string - if !a.Spike.WindowEnd.IsZero() { - meta = append(meta, style(colText3, false).Render(fmtTime(a.Spike.WindowEnd))) + if !when.IsZero() { + meta = append(meta, style(colText3, false).Render(fmtTime(when))) } meta = append(meta, state) if o.BaseURL != "" && len(a.Patterns) > 0 && a.Patterns[0].IssueID != "" { @@ -254,9 +272,12 @@ func renderHeader(snap snapshot, o snapshotOptions) string { } } counts := fmt.Sprintf("%d firing", len(snap.Issues)) - if paged > 0 { + switch { + case len(snap.Issues) == 0 && len(snap.Resolved) > 0: + left = append(left, style(colOK, true).Render(fmt.Sprintf("%d resolved", len(snap.Resolved)))) + case paged > 0: left = append(left, style(colCritical, true).Render(fmt.Sprintf("%d PAGED · %s", paged, counts))) - } else { + default: left = append(left, style(colText2, false).Render(counts)) } } @@ -282,7 +303,7 @@ func renderHeader(snap snapshot, o snapshotOptions) string { func renderFooter(snap snapshot, o snapshotOptions) string { var left []string left = append(left, style(colText3, false).Render(footerHint)) - if hidden := len(snap.Issues) - shownCount(snap, o); hidden > 0 { + if hidden := cardCount(snap) - shownCount(snap, o); hidden > 0 { left = append(left, style(colWarn, false).Render(fmt.Sprintf("+%d more not shown (widen or raise the display)", hidden))) } var right []string @@ -299,11 +320,20 @@ func renderFooter(snap snapshot, o snapshotOptions) string { return bar(o.Width-2, strings.Join(left, " "), strings.Join(right, " ")) } +// cardCount is how many cards the body renders: the firing issues, or the +// kept resolved ones when nothing fires. +func cardCount(snap snapshot) int { + if n := len(snap.Issues); n > 0 { + return n + } + return len(snap.Resolved) +} + func shownCount(snap snapshot, o snapshotOptions) int { if o.Height < minUsableHeight { - return len(snap.Issues) + return cardCount(snap) } - _, _, shown, _ := gridLayout(len(snap.Issues), o.Width, o.Height-2) + _, _, shown, _ := gridLayout(cardCount(snap), o.Width, o.Height-2) return shown } @@ -323,14 +353,20 @@ func renderMinimal(snap snapshot, o snapshotOptions) string { if snap.Err != nil { return "SCAN FAILED: " + snap.Err.Error() } - if len(snap.Issues) == 0 { + if len(snap.Issues) == 0 && len(snap.Resolved) == 0 { return strings.ToUpper(o.Title) + ": all clear" } - lines := make([]string, 0, len(snap.Issues)+1) + views := snap.Issues + if len(views) == 0 { + views = snap.Resolved + } + lines := make([]string, 0, len(views)+1) lines = append(lines, strings.ToUpper(o.Title)+":") - for _, v := range snap.Issues { + for _, v := range views { state := "ongoing" - if v.paged { + if !v.resolvedAt.IsZero() { + state = "RESOLVED" + } else if v.paged { state = "PAGED" } what := "" diff --git a/otel-ui/src/components/CspReportsVisualizer.tsx b/otel-ui/src/components/CspReportsVisualizer.tsx index 2391ed4..24cd6df 100644 --- a/otel-ui/src/components/CspReportsVisualizer.tsx +++ b/otel-ui/src/components/CspReportsVisualizer.tsx @@ -183,8 +183,8 @@ export function CspReportsVisualizer() {
- - + +
, traceMinTime: number, traceTotalRange: number, - depth = 0, + initialDepth = 0, ): FlattenedSpanItem[] => { const result: FlattenedSpanItem[] = []; + const pending: Array<{ span: ProcessedSpan; depth: number }> = []; - for (const span of nodes) { - const hasChildren = Boolean(span.children && span.children.length > 0); + for (let index = nodes.length - 1; index >= 0; index -= 1) { + pending.push({ span: nodes[index], depth: initialDepth }); + } + + // Avoid recursively spreading descendant arrays: deep parent/child chains + // otherwise copy the same nodes once per ancestor. + while (pending.length > 0) { + const current = pending.pop(); + if (!current) break; + const { span, depth } = current; + const children = span.children ?? []; + const hasChildren = children.length > 0; const isCollapsed = collapsedSet.has(span.spanId); - const offsetPercent = traceTotalRange === 0 ? 0 : ((span.startTime - traceMinTime) / traceTotalRange) * 100; const widthPercent = @@ -118,16 +130,10 @@ const flattenSpanTree = ( widthPercent: Math.max(0.5, Math.min(widthPercent, 100 - offsetPercent)), }); - if (hasChildren && !isCollapsed && span.children) { - result.push( - ...flattenSpanTree( - span.children, - collapsedSet, - traceMinTime, - traceTotalRange, - depth + 1, - ), - ); + if (hasChildren && !isCollapsed) { + for (let index = children.length - 1; index >= 0; index -= 1) { + pending.push({ span: children[index], depth: depth + 1 }); + } } } @@ -539,6 +545,7 @@ const VirtualizedSpanWaterfall = ({ const [searchQuery, setSearchQuery] = useState(""); const [errorsOnly, setErrorsOnly] = useState(false); const [minDurationMsFilter, setMinDurationMsFilter] = useState(0); + const deferredSearchQuery = useDeferredValue(searchQuery); const parentRef = useRef(null); // Build tree @@ -563,8 +570,8 @@ const VirtualizedSpanWaterfall = ({ const durMs = item.span.duration / 1_000_000; if (durMs < minDurationMsFilter) return false; } - if (searchQuery.trim()) { - const query = searchQuery.toLowerCase().trim(); + if (deferredSearchQuery.trim()) { + const query = deferredSearchQuery.toLowerCase().trim(); const matchesName = item.span.name.toLowerCase().includes(query); const matchesService = item.span.service.toLowerCase().includes(query); const matchesAttrs = Object.values(item.span.formattedAttributes).some((v) => @@ -574,7 +581,7 @@ const VirtualizedSpanWaterfall = ({ } return true; }); - }, [flattenedItems, filterService, errorsOnly, minDurationMsFilter, searchQuery]); + }, [flattenedItems, filterService, errorsOnly, minDurationMsFilter, deferredSearchQuery]); // Virtualizer @@ -1683,10 +1690,12 @@ export const processTraces = ( ), ); - const uniqueSpans = spanAccumulator.filter( - (span, index, arr) => - arr.findIndex((candidate) => candidate.spanId === span.spanId) === index, - ); + const seenSpanIds = new Set(); + const uniqueSpans = spanAccumulator.filter((span) => { + if (seenSpanIds.has(span.spanId)) return false; + seenSpanIds.add(span.spanId); + return true; + }); const traceGroups = uniqueSpans.reduce>((acc, span) => { if (!acc[span.traceId]) acc[span.traceId] = []; @@ -1739,7 +1748,7 @@ export interface TraceVisualizationProps { } export const TraceVisualization = ({ traces }: TraceVisualizationProps) => { - const safeTraces = traces ?? []; + const safeTraces = traces ?? EMPTY_TRACES; const { logCountBySpanId, activeTraceId, @@ -1756,6 +1765,7 @@ export const TraceVisualization = ({ traces }: TraceVisualizationProps) => { "recent" | "duration-desc" | "duration-asc" | "errors" | "spans" >("recent"); const [groupByEndpoint, setGroupByEndpoint] = useState(false); + const deferredTraceSearchQuery = useDeferredValue(traceSearchQuery); const orderedTraces: ProcessedTrace[] = useMemo( () => processTraces(safeTraces, logCountBySpanId), @@ -1774,8 +1784,8 @@ export const TraceVisualization = ({ traces }: TraceVisualizationProps) => { if (traceFilter === "errors" && !t.hasError) return false; if (traceFilter === "slow" && t.duration / 1_000_000 < 200) return false; if (traceServiceFilter !== "all" && !t.services.includes(traceServiceFilter)) return false; - if (traceSearchQuery.trim()) { - const q = traceSearchQuery.toLowerCase().trim(); + if (deferredTraceSearchQuery.trim()) { + const q = deferredTraceSearchQuery.toLowerCase().trim(); const matchName = t.name.toLowerCase().includes(q); const matchId = t.id.toLowerCase().includes(q); const matchService = t.services.some((s) => s.toLowerCase().includes(q)); @@ -1791,7 +1801,7 @@ export const TraceVisualization = ({ traces }: TraceVisualizationProps) => { if (traceSortBy === "spans") return b.spans.length - a.spans.length; return b.startTime - a.startTime; // recent }); - }, [orderedTraces, traceFilter, traceServiceFilter, traceSearchQuery, traceSortBy]); + }, [orderedTraces, traceFilter, traceServiceFilter, deferredTraceSearchQuery, traceSortBy]); const groupCounts = useMemo(() => { const counts: Record = {};