diff --git a/README.md b/README.md index e58cd0f..05df5e5 100644 --- a/README.md +++ b/README.md @@ -377,6 +377,8 @@ to Docker or stored in the image. The standalone `issues-tui` command is a terminal dashboard for currently firing alert spikes. It reads normalized error events directly from ClickHouse; it does not replace the web UI's persisted `/api/issues` view. +When a spike stops firing, its card stays on the display marked resolved +until something fires again. Build and run it on the display computer (or use `go run`): diff --git a/cmd/issues-tui/main.go b/cmd/issues-tui/main.go index dff243f..44ab0ed 100644 --- a/cmd/issues-tui/main.go +++ b/cmd/issues-tui/main.go @@ -7,7 +7,8 @@ // A pattern shows as PAGED when this tick would have sent a notification // (new or escalated past its cooldown) and as ongoing while it is still // firing but already paged once. When a service's spike disappears it is -// reported as recovered for that scan. +// reported as recovered for that scan, and the last firing issue stays on +// the display as resolved until something fires again. package main import ( @@ -96,6 +97,12 @@ type scanner struct { // firing; it mirrors what a person watching notifications would remember. active map[string]time.Time + // lastActive is the most recent firing issue set; when a later scan + // finds nothing firing, it is kept on the display as resolved instead + // of flipping to ALL CLEAR. + lastActive []issueView + resolvedAt time.Time // window end when lastActive stopped firing + since time.Time sinceAt time.Time } @@ -121,10 +128,11 @@ func newClickHouseScanner(addr, database, user, password string, cfg features.Sp // issueView is one card on the dashboard: a firing spike reduced to an Alert // (the same model the notifications carry) plus display state. type issueView struct { - alert alerts.Alert - key string - paged bool // this scan would have notified (new or escalated) - firstSeen time.Time // window end when this issue first appeared + alert alerts.Alert + key string + paged bool // this scan would have notified (new or escalated) + firstSeen time.Time // window end when this issue first appeared + resolvedAt time.Time // window end when it stopped firing; renders as resolved } // recoveredIssue is an issue that fired on the previous scan but not now. @@ -134,7 +142,8 @@ type recoveredIssue struct { LastSeen time.Time } -// snapshot is one scan's result: everything firing right now. +// snapshot is one scan's result: everything firing right now, plus the last +// firing issue kept as resolved when nothing fires any more. type snapshot struct { ScannedAt time.Time WindowEnd time.Time @@ -142,6 +151,7 @@ type snapshot struct { Events int Issues []issueView Recovered []recoveredIssue + Resolved []issueView Err error } @@ -219,6 +229,22 @@ func (s *scanner) Scan(ctx context.Context, now time.Time) snapshot { snap.Recovered = append(snap.Recovered, recoveredIssue{Service: serviceOfKey(key), FirstSeen: first, LastSeen: end}) } } + // Keep the last firing issue on the display: once nothing fires any + // more, the cards stay up as resolved rather than the dashboard erasing + // the context with an ALL CLEAR panel. + if len(snap.Issues) > 0 { + s.lastActive = snap.Issues + s.resolvedAt = time.Time{} + } else if len(s.lastActive) > 0 { + if s.resolvedAt.IsZero() { + s.resolvedAt = end + } + for _, v := range s.lastActive { + v.resolvedAt = s.resolvedAt + snap.Resolved = append(snap.Resolved, v) + } + sortIssues(snap.Resolved) + } sortIssues(snap.Issues) return snap } diff --git a/cmd/issues-tui/main_test.go b/cmd/issues-tui/main_test.go index e2547b9..2b6e04f 100644 --- a/cmd/issues-tui/main_test.go +++ b/cmd/issues-tui/main_test.go @@ -96,10 +96,20 @@ func TestScannerPagesOnceThenOngoing(t *testing.T) { if len(fourth.Recovered) != 1 || fourth.Recovered[0].Service != "checkout" { t.Fatalf("expected checkout in recovered, got %+v", fourth.Recovered) } + // The last issue stays on the display as resolved instead of ALL CLEAR. + if len(fourth.Resolved) != 1 || fourth.Resolved[0].alert.Spike.Service != "checkout" { + t.Fatalf("expected checkout kept as resolved, got %+v", fourth.Resolved) + } + if got := fourth.Resolved[0].resolvedAt; !got.Equal(fourth.WindowEnd) { + t.Fatalf("resolvedAt should be the window that stopped firing, got %v want %v", got, fourth.WindowEnd) + } again := sc.Scan(context.Background(), base.Add(4*time.Minute)) if len(again.Recovered) != 0 { t.Fatalf("recovery should be reported once, got %+v", again.Recovered) } + if len(again.Resolved) != 1 || !again.Resolved[0].resolvedAt.Equal(fourth.Resolved[0].resolvedAt) { + t.Fatalf("resolved issue should persist with a stable resolved time, got %+v", again.Resolved) + } } func TestScannerAllClearAndError(t *testing.T) { @@ -108,6 +118,9 @@ func TestScannerAllClearAndError(t *testing.T) { if len(empty.Issues) != 0 || empty.Err != nil { t.Fatalf("empty loader should yield a clean snapshot, got %+v", empty) } + if len(empty.Resolved) != 0 { + t.Fatalf("a fresh scanner has no last issue to keep, got %+v", empty.Resolved) + } sc.load = func(context.Context, time.Time, time.Time) ([]features.Event, error) { return nil, context.DeadlineExceeded @@ -204,6 +217,22 @@ func TestRenderSnapshotAllClear(t *testing.T) { } } +func TestRenderSnapshotKeepsResolvedIssue(t *testing.T) { + now := time.Date(2025, 6, 1, 12, 45, 0, 0, time.UTC) + snap, o := cardAt(t, 1) + resolved := snap.Issues[0] + resolved.resolvedAt = now + snap.Issues = nil + snap.Resolved = []issueView{resolved} + out := renderSnapshot(snap, o) + if strings.Contains(out, "ALL CLEAR") { + t.Fatalf("a resolved issue should replace the ALL CLEAR panel:\n%s", out) + } + if !strings.Contains(out, "RESOLVED") || !strings.Contains(out, "checkout") { + t.Fatalf("dashboard should keep the last issue as resolved:\n%s", out) + } +} + func TestRenderSnapshotScanError(t *testing.T) { snap := snapshot{Err: context.DeadlineExceeded} out := renderSnapshot(snap, snapshotOptions{Width: 200, Height: 12, Title: "otel issues", Now: time.Now()}) diff --git a/cmd/issues-tui/view.go b/cmd/issues-tui/view.go index 449f900..2d326e9 100644 --- a/cmd/issues-tui/view.go +++ b/cmd/issues-tui/view.go @@ -91,6 +91,10 @@ func renderBody(snap snapshot, o snapshotOptions, w, avail int) string { case o.ScanError != nil && len(snap.Issues) == 0: return statusPanel(w, min(avail, 5), colCritical, "SCAN FAILED", o.ScanError.Error(), "no successful scan yet; retrying automatically") + case len(snap.Issues) == 0 && len(snap.Resolved) > 0: + // Nothing fires any more: keep the last issue on the display, + // flipped to resolved, instead of an ALL CLEAR panel. + return renderGrid(snap.Resolved, o, w, avail) case len(snap.Issues) == 0: sub := fmt.Sprintf("no issues firing · window %s–%s UTC · scanned %s", snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"), commas(snap.Events)) @@ -99,19 +103,19 @@ func renderBody(snap snapshot, o snapshotOptions, w, avail int) string { } return statusPanel(w, min(avail, 5), colOK, "ALL CLEAR", sub, "") default: - return renderGrid(snap, o, w, avail) + return renderGrid(snap.Issues, o, w, avail) } } // renderGrid lays issues into rows of cards that fill the width; only as many // rows as the height allows are drawn, the rest are counted in the footer. -func renderGrid(snap snapshot, o snapshotOptions, w, avail int) string { - perRow, _, shown, cardW := gridLayout(len(snap.Issues), w, avail) +func renderGrid(views []issueView, o snapshotOptions, w, avail int) string { + perRow, _, shown, cardW := gridLayout(len(views), w, avail) var rendered []string for start := 0; start < shown; start += perRow { end := min(start+perRow, shown) var row []string - for _, v := range snap.Issues[start:end] { + for _, v := range views[start:end] { row = append(row, issueCard(v, cardW, o)) } rendered = append(rendered, lipgloss.JoinHorizontal(lipgloss.Top, row...)) @@ -145,11 +149,19 @@ func gridLayout(n, width, avail int) (perRow, rows, shown, cardW int) { func issueCard(v issueView, w int, o snapshotOptions) string { a := v.alert fg := priorityColor(a.Priority()) + resolved := !v.resolvedAt.IsZero() + if resolved { + // A resolved card keeps its place but reads as recovered: the same + // green as the footer's recovered line and the ALL CLEAR panel. + fg = colOK + } inner := w - 4 // border plus padding on both sides // Line 1: HIGH · checkout · for 12m age := "" - if !v.firstSeen.IsZero() && !o.Now.IsZero() { + if !v.firstSeen.IsZero() && resolved { + age = " · for " + shortDur(v.resolvedAt.Sub(v.firstSeen).Round(time.Minute)) + } else if !v.firstSeen.IsZero() && !o.Now.IsZero() { age = " · for " + shortDur(o.Now.Sub(v.firstSeen).Round(time.Minute)) } head := ansi.Truncate( @@ -190,14 +202,20 @@ func issueCard(v issueView, w int, o snapshotOptions) string { } statsLine := ansi.Truncate(strings.Join(segs, " "), inner, "…") - // Line 4: display state (paged now vs already paged) and the issue link. + // Line 4: display state (paged now vs already paged vs resolved) and the + // issue link. state := style(colText3, false).Render("ongoing") - if v.paged { + when := a.Spike.WindowEnd + switch { + case resolved: + state = style(colOK, true).Render("RESOLVED") + when = v.resolvedAt + case v.paged: state = style(fg, true).Render("PAGED") } var meta []string - if !a.Spike.WindowEnd.IsZero() { - meta = append(meta, style(colText3, false).Render(fmtTime(a.Spike.WindowEnd))) + if !when.IsZero() { + meta = append(meta, style(colText3, false).Render(fmtTime(when))) } meta = append(meta, state) if o.BaseURL != "" && len(a.Patterns) > 0 && a.Patterns[0].IssueID != "" { @@ -254,9 +272,12 @@ func renderHeader(snap snapshot, o snapshotOptions) string { } } counts := fmt.Sprintf("%d firing", len(snap.Issues)) - if paged > 0 { + switch { + case len(snap.Issues) == 0 && len(snap.Resolved) > 0: + left = append(left, style(colOK, true).Render(fmt.Sprintf("%d resolved", len(snap.Resolved)))) + case paged > 0: left = append(left, style(colCritical, true).Render(fmt.Sprintf("%d PAGED · %s", paged, counts))) - } else { + default: left = append(left, style(colText2, false).Render(counts)) } } @@ -282,7 +303,7 @@ func renderHeader(snap snapshot, o snapshotOptions) string { func renderFooter(snap snapshot, o snapshotOptions) string { var left []string left = append(left, style(colText3, false).Render(footerHint)) - if hidden := len(snap.Issues) - shownCount(snap, o); hidden > 0 { + if hidden := cardCount(snap) - shownCount(snap, o); hidden > 0 { left = append(left, style(colWarn, false).Render(fmt.Sprintf("+%d more not shown (widen or raise the display)", hidden))) } var right []string @@ -299,11 +320,20 @@ func renderFooter(snap snapshot, o snapshotOptions) string { return bar(o.Width-2, strings.Join(left, " "), strings.Join(right, " ")) } +// cardCount is how many cards the body renders: the firing issues, or the +// kept resolved ones when nothing fires. +func cardCount(snap snapshot) int { + if n := len(snap.Issues); n > 0 { + return n + } + return len(snap.Resolved) +} + func shownCount(snap snapshot, o snapshotOptions) int { if o.Height < minUsableHeight { - return len(snap.Issues) + return cardCount(snap) } - _, _, shown, _ := gridLayout(len(snap.Issues), o.Width, o.Height-2) + _, _, shown, _ := gridLayout(cardCount(snap), o.Width, o.Height-2) return shown } @@ -323,14 +353,20 @@ func renderMinimal(snap snapshot, o snapshotOptions) string { if snap.Err != nil { return "SCAN FAILED: " + snap.Err.Error() } - if len(snap.Issues) == 0 { + if len(snap.Issues) == 0 && len(snap.Resolved) == 0 { return strings.ToUpper(o.Title) + ": all clear" } - lines := make([]string, 0, len(snap.Issues)+1) + views := snap.Issues + if len(views) == 0 { + views = snap.Resolved + } + lines := make([]string, 0, len(views)+1) lines = append(lines, strings.ToUpper(o.Title)+":") - for _, v := range snap.Issues { + for _, v := range views { state := "ongoing" - if v.paged { + if !v.resolvedAt.IsZero() { + state = "RESOLVED" + } else if v.paged { state = "PAGED" } what := ""