fix
Build and Publish / Metadata (push) Successful in 10s
Build and Publish / BuildAndDeployAmd64 (push) Successful in 2m36s

This commit is contained in:
developer committed 2026-09-24 13:26:24 +00:00
1 parent 49c6c6cc76
commit 6dc2785f71
4 files changed
+117 -24

No files matched your search

+2
View File
@@ -377,6 +377,8 @@ to Docker or stored in the image.
The standalone `issues-tui` command is a terminal dashboard for currently The standalone `issues-tui` command is a terminal dashboard for currently
firing alert spikes. It reads normalized error events directly from firing alert spikes. It reads normalized error events directly from
ClickHouse; it does not replace the web UI's persisted `/api/issues` view. ClickHouse; it does not replace the web UI's persisted `/api/issues` view.
When a spike stops firing, its card stays on the display marked resolved
until something fires again.
Build and run it on the display computer (or use `go run`): Build and run it on the display computer (or use `go run`):
+32 -6
View File
@@ -7,7 +7,8 @@
// A pattern shows as PAGED when this tick would have sent a notification // A pattern shows as PAGED when this tick would have sent a notification
// (new or escalated past its cooldown) and as ongoing while it is still // (new or escalated past its cooldown) and as ongoing while it is still
// firing but already paged once. When a service's spike disappears it is // firing but already paged once. When a service's spike disappears it is
// reported as recovered for that scan. // reported as recovered for that scan, and the last firing issue stays on
// the display as resolved until something fires again.
package main package main
import ( import (
@@ -96,6 +97,12 @@ type scanner struct {
// firing; it mirrors what a person watching notifications would remember. // firing; it mirrors what a person watching notifications would remember.
active map[string]time.Time active map[string]time.Time
// lastActive is the most recent firing issue set; when a later scan
// finds nothing firing, it is kept on the display as resolved instead
// of flipping to ALL CLEAR.
lastActive []issueView
resolvedAt time.Time // window end when lastActive stopped firing
since time.Time since time.Time
sinceAt time.Time sinceAt time.Time
} }
@@ -121,10 +128,11 @@ func newClickHouseScanner(addr, database, user, password string, cfg features.Sp
// issueView is one card on the dashboard: a firing spike reduced to an Alert // issueView is one card on the dashboard: a firing spike reduced to an Alert
// (the same model the notifications carry) plus display state. // (the same model the notifications carry) plus display state.
type issueView struct { type issueView struct {
alert alerts.Alert alert alerts.Alert
key string key string
paged bool // this scan would have notified (new or escalated) paged bool // this scan would have notified (new or escalated)
firstSeen time.Time // window end when this issue first appeared firstSeen time.Time // window end when this issue first appeared
resolvedAt time.Time // window end when it stopped firing; renders as resolved
} }
// recoveredIssue is an issue that fired on the previous scan but not now. // recoveredIssue is an issue that fired on the previous scan but not now.
@@ -134,7 +142,8 @@ type recoveredIssue struct {
LastSeen time.Time LastSeen time.Time
} }
// snapshot is one scan's result: everything firing right now. // snapshot is one scan's result: everything firing right now, plus the last
// firing issue kept as resolved when nothing fires any more.
type snapshot struct { type snapshot struct {
ScannedAt time.Time ScannedAt time.Time
WindowEnd time.Time WindowEnd time.Time
@@ -142,6 +151,7 @@ type snapshot struct {
Events int Events int
Issues []issueView Issues []issueView
Recovered []recoveredIssue Recovered []recoveredIssue
Resolved []issueView
Err error Err error
} }
@@ -219,6 +229,22 @@ func (s *scanner) Scan(ctx context.Context, now time.Time) snapshot {
snap.Recovered = append(snap.Recovered, recoveredIssue{Service: serviceOfKey(key), FirstSeen: first, LastSeen: end}) snap.Recovered = append(snap.Recovered, recoveredIssue{Service: serviceOfKey(key), FirstSeen: first, LastSeen: end})
} }
} }
// Keep the last firing issue on the display: once nothing fires any
// more, the cards stay up as resolved rather than the dashboard erasing
// the context with an ALL CLEAR panel.
if len(snap.Issues) > 0 {
s.lastActive = snap.Issues
s.resolvedAt = time.Time{}
} else if len(s.lastActive) > 0 {
if s.resolvedAt.IsZero() {
s.resolvedAt = end
}
for _, v := range s.lastActive {
v.resolvedAt = s.resolvedAt
snap.Resolved = append(snap.Resolved, v)
}
sortIssues(snap.Resolved)
}
sortIssues(snap.Issues) sortIssues(snap.Issues)
return snap return snap
} }
+29
View File
@@ -96,10 +96,20 @@ func TestScannerPagesOnceThenOngoing(t *testing.T) {
if len(fourth.Recovered) != 1 || fourth.Recovered[0].Service != "checkout" { if len(fourth.Recovered) != 1 || fourth.Recovered[0].Service != "checkout" {
t.Fatalf("expected checkout in recovered, got %+v", fourth.Recovered) t.Fatalf("expected checkout in recovered, got %+v", fourth.Recovered)
} }
// The last issue stays on the display as resolved instead of ALL CLEAR.
if len(fourth.Resolved) != 1 || fourth.Resolved[0].alert.Spike.Service != "checkout" {
t.Fatalf("expected checkout kept as resolved, got %+v", fourth.Resolved)
}
if got := fourth.Resolved[0].resolvedAt; !got.Equal(fourth.WindowEnd) {
t.Fatalf("resolvedAt should be the window that stopped firing, got %v want %v", got, fourth.WindowEnd)
}
again := sc.Scan(context.Background(), base.Add(4*time.Minute)) again := sc.Scan(context.Background(), base.Add(4*time.Minute))
if len(again.Recovered) != 0 { if len(again.Recovered) != 0 {
t.Fatalf("recovery should be reported once, got %+v", again.Recovered) t.Fatalf("recovery should be reported once, got %+v", again.Recovered)
} }
if len(again.Resolved) != 1 || !again.Resolved[0].resolvedAt.Equal(fourth.Resolved[0].resolvedAt) {
t.Fatalf("resolved issue should persist with a stable resolved time, got %+v", again.Resolved)
}
} }
func TestScannerAllClearAndError(t *testing.T) { func TestScannerAllClearAndError(t *testing.T) {
@@ -108,6 +118,9 @@ func TestScannerAllClearAndError(t *testing.T) {
if len(empty.Issues) != 0 || empty.Err != nil { if len(empty.Issues) != 0 || empty.Err != nil {
t.Fatalf("empty loader should yield a clean snapshot, got %+v", empty) t.Fatalf("empty loader should yield a clean snapshot, got %+v", empty)
} }
if len(empty.Resolved) != 0 {
t.Fatalf("a fresh scanner has no last issue to keep, got %+v", empty.Resolved)
}
sc.load = func(context.Context, time.Time, time.Time) ([]features.Event, error) { sc.load = func(context.Context, time.Time, time.Time) ([]features.Event, error) {
return nil, context.DeadlineExceeded return nil, context.DeadlineExceeded
@@ -204,6 +217,22 @@ func TestRenderSnapshotAllClear(t *testing.T) {
} }
} }
func TestRenderSnapshotKeepsResolvedIssue(t *testing.T) {
now := time.Date(2025, 6, 1, 12, 45, 0, 0, time.UTC)
snap, o := cardAt(t, 1)
resolved := snap.Issues[0]
resolved.resolvedAt = now
snap.Issues = nil
snap.Resolved = []issueView{resolved}
out := renderSnapshot(snap, o)
if strings.Contains(out, "ALL CLEAR") {
t.Fatalf("a resolved issue should replace the ALL CLEAR panel:\n%s", out)
}
if !strings.Contains(out, "RESOLVED") || !strings.Contains(out, "checkout") {
t.Fatalf("dashboard should keep the last issue as resolved:\n%s", out)
}
}
func TestRenderSnapshotScanError(t *testing.T) { func TestRenderSnapshotScanError(t *testing.T) {
snap := snapshot{Err: context.DeadlineExceeded} snap := snapshot{Err: context.DeadlineExceeded}
out := renderSnapshot(snap, snapshotOptions{Width: 200, Height: 12, Title: "otel issues", Now: time.Now()}) out := renderSnapshot(snap, snapshotOptions{Width: 200, Height: 12, Title: "otel issues", Now: time.Now()})
+54 -18
View File
@@ -91,6 +91,10 @@ func renderBody(snap snapshot, o snapshotOptions, w, avail int) string {
case o.ScanError != nil && len(snap.Issues) == 0: case o.ScanError != nil && len(snap.Issues) == 0:
return statusPanel(w, min(avail, 5), colCritical, "SCAN FAILED", o.ScanError.Error(), return statusPanel(w, min(avail, 5), colCritical, "SCAN FAILED", o.ScanError.Error(),
"no successful scan yet; retrying automatically") "no successful scan yet; retrying automatically")
case len(snap.Issues) == 0 && len(snap.Resolved) > 0:
// Nothing fires any more: keep the last issue on the display,
// flipped to resolved, instead of an ALL CLEAR panel.
return renderGrid(snap.Resolved, o, w, avail)
case len(snap.Issues) == 0: case len(snap.Issues) == 0:
sub := fmt.Sprintf("no issues firing · window %s–%s UTC · scanned %s", sub := fmt.Sprintf("no issues firing · window %s–%s UTC · scanned %s",
snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"), commas(snap.Events)) snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"), commas(snap.Events))
@@ -99,19 +103,19 @@ func renderBody(snap snapshot, o snapshotOptions, w, avail int) string {
} }
return statusPanel(w, min(avail, 5), colOK, "ALL CLEAR", sub, "") return statusPanel(w, min(avail, 5), colOK, "ALL CLEAR", sub, "")
default: default:
return renderGrid(snap, o, w, avail) return renderGrid(snap.Issues, o, w, avail)
} }
} }
// renderGrid lays issues into rows of cards that fill the width; only as many // renderGrid lays issues into rows of cards that fill the width; only as many
// rows as the height allows are drawn, the rest are counted in the footer. // rows as the height allows are drawn, the rest are counted in the footer.
func renderGrid(snap snapshot, o snapshotOptions, w, avail int) string { func renderGrid(views []issueView, o snapshotOptions, w, avail int) string {
perRow, _, shown, cardW := gridLayout(len(snap.Issues), w, avail) perRow, _, shown, cardW := gridLayout(len(views), w, avail)
var rendered []string var rendered []string
for start := 0; start < shown; start += perRow { for start := 0; start < shown; start += perRow {
end := min(start+perRow, shown) end := min(start+perRow, shown)
var row []string var row []string
for _, v := range snap.Issues[start:end] { for _, v := range views[start:end] {
row = append(row, issueCard(v, cardW, o)) row = append(row, issueCard(v, cardW, o))
} }
rendered = append(rendered, lipgloss.JoinHorizontal(lipgloss.Top, row...)) rendered = append(rendered, lipgloss.JoinHorizontal(lipgloss.Top, row...))
@@ -145,11 +149,19 @@ func gridLayout(n, width, avail int) (perRow, rows, shown, cardW int) {
func issueCard(v issueView, w int, o snapshotOptions) string { func issueCard(v issueView, w int, o snapshotOptions) string {
a := v.alert a := v.alert
fg := priorityColor(a.Priority()) fg := priorityColor(a.Priority())
resolved := !v.resolvedAt.IsZero()
if resolved {
// A resolved card keeps its place but reads as recovered: the same
// green as the footer's recovered line and the ALL CLEAR panel.
fg = colOK
}
inner := w - 4 // border plus padding on both sides inner := w - 4 // border plus padding on both sides
// Line 1: HIGH · checkout · for 12m // Line 1: HIGH · checkout · for 12m
age := "" age := ""
if !v.firstSeen.IsZero() && !o.Now.IsZero() { if !v.firstSeen.IsZero() && resolved {
age = " · for " + shortDur(v.resolvedAt.Sub(v.firstSeen).Round(time.Minute))
} else if !v.firstSeen.IsZero() && !o.Now.IsZero() {
age = " · for " + shortDur(o.Now.Sub(v.firstSeen).Round(time.Minute)) age = " · for " + shortDur(o.Now.Sub(v.firstSeen).Round(time.Minute))
} }
head := ansi.Truncate( head := ansi.Truncate(
@@ -190,14 +202,20 @@ func issueCard(v issueView, w int, o snapshotOptions) string {
} }
statsLine := ansi.Truncate(strings.Join(segs, " "), inner, "…") statsLine := ansi.Truncate(strings.Join(segs, " "), inner, "…")
// Line 4: display state (paged now vs already paged) and the issue link. // Line 4: display state (paged now vs already paged vs resolved) and the
// issue link.
state := style(colText3, false).Render("ongoing") state := style(colText3, false).Render("ongoing")
if v.paged { when := a.Spike.WindowEnd
switch {
case resolved:
state = style(colOK, true).Render("RESOLVED")
when = v.resolvedAt
case v.paged:
state = style(fg, true).Render("PAGED") state = style(fg, true).Render("PAGED")
} }
var meta []string var meta []string
if !a.Spike.WindowEnd.IsZero() { if !when.IsZero() {
meta = append(meta, style(colText3, false).Render(fmtTime(a.Spike.WindowEnd))) meta = append(meta, style(colText3, false).Render(fmtTime(when)))
} }
meta = append(meta, state) meta = append(meta, state)
if o.BaseURL != "" && len(a.Patterns) > 0 && a.Patterns[0].IssueID != "" { if o.BaseURL != "" && len(a.Patterns) > 0 && a.Patterns[0].IssueID != "" {
@@ -254,9 +272,12 @@ func renderHeader(snap snapshot, o snapshotOptions) string {
} }
} }
counts := fmt.Sprintf("%d firing", len(snap.Issues)) counts := fmt.Sprintf("%d firing", len(snap.Issues))
if paged > 0 { switch {
case len(snap.Issues) == 0 && len(snap.Resolved) > 0:
left = append(left, style(colOK, true).Render(fmt.Sprintf("%d resolved", len(snap.Resolved))))
case paged > 0:
left = append(left, style(colCritical, true).Render(fmt.Sprintf("%d PAGED · %s", paged, counts))) left = append(left, style(colCritical, true).Render(fmt.Sprintf("%d PAGED · %s", paged, counts)))
} else { default:
left = append(left, style(colText2, false).Render(counts)) left = append(left, style(colText2, false).Render(counts))
} }
} }
@@ -282,7 +303,7 @@ func renderHeader(snap snapshot, o snapshotOptions) string {
func renderFooter(snap snapshot, o snapshotOptions) string { func renderFooter(snap snapshot, o snapshotOptions) string {
var left []string var left []string
left = append(left, style(colText3, false).Render(footerHint)) left = append(left, style(colText3, false).Render(footerHint))
if hidden := len(snap.Issues) - shownCount(snap, o); hidden > 0 { if hidden := cardCount(snap) - shownCount(snap, o); hidden > 0 {
left = append(left, style(colWarn, false).Render(fmt.Sprintf("+%d more not shown (widen or raise the display)", hidden))) left = append(left, style(colWarn, false).Render(fmt.Sprintf("+%d more not shown (widen or raise the display)", hidden)))
} }
var right []string var right []string
@@ -299,11 +320,20 @@ func renderFooter(snap snapshot, o snapshotOptions) string {
return bar(o.Width-2, strings.Join(left, " "), strings.Join(right, " ")) return bar(o.Width-2, strings.Join(left, " "), strings.Join(right, " "))
} }
// cardCount is how many cards the body renders: the firing issues, or the
// kept resolved ones when nothing fires.
func cardCount(snap snapshot) int {
if n := len(snap.Issues); n > 0 {
return n
}
return len(snap.Resolved)
}
func shownCount(snap snapshot, o snapshotOptions) int { func shownCount(snap snapshot, o snapshotOptions) int {
if o.Height < minUsableHeight { if o.Height < minUsableHeight {
return len(snap.Issues) return cardCount(snap)
} }
_, _, shown, _ := gridLayout(len(snap.Issues), o.Width, o.Height-2) _, _, shown, _ := gridLayout(cardCount(snap), o.Width, o.Height-2)
return shown return shown
} }
@@ -323,14 +353,20 @@ func renderMinimal(snap snapshot, o snapshotOptions) string {
if snap.Err != nil { if snap.Err != nil {
return "SCAN FAILED: " + snap.Err.Error() return "SCAN FAILED: " + snap.Err.Error()
} }
if len(snap.Issues) == 0 { if len(snap.Issues) == 0 && len(snap.Resolved) == 0 {
return strings.ToUpper(o.Title) + ": all clear" return strings.ToUpper(o.Title) + ": all clear"
} }
lines := make([]string, 0, len(snap.Issues)+1) views := snap.Issues
if len(views) == 0 {
views = snap.Resolved
}
lines := make([]string, 0, len(views)+1)
lines = append(lines, strings.ToUpper(o.Title)+":") lines = append(lines, strings.ToUpper(o.Title)+":")
for _, v := range snap.Issues { for _, v := range views {
state := "ongoing" state := "ongoing"
if v.paged { if !v.resolvedAt.IsZero() {
state = "RESOLVED"
} else if v.paged {
state = "PAGED" state = "PAGED"
} }
what := "" what := ""