fix
This commit is contained in:
1 parent
49c6c6cc76
commit
6dc2785f71
4 files changed
+117
-24
No files matched your search
@@ -377,6 +377,8 @@ to Docker or stored in the image.
|
|||||||
The standalone `issues-tui` command is a terminal dashboard for currently
|
The standalone `issues-tui` command is a terminal dashboard for currently
|
||||||
firing alert spikes. It reads normalized error events directly from
|
firing alert spikes. It reads normalized error events directly from
|
||||||
ClickHouse; it does not replace the web UI's persisted `/api/issues` view.
|
ClickHouse; it does not replace the web UI's persisted `/api/issues` view.
|
||||||
|
When a spike stops firing, its card stays on the display marked resolved
|
||||||
|
until something fires again.
|
||||||
|
|
||||||
Build and run it on the display computer (or use `go run`):
|
Build and run it on the display computer (or use `go run`):
|
||||||
|
|
||||||
|
|||||||
+32
-6
@@ -7,7 +7,8 @@
|
|||||||
// A pattern shows as PAGED when this tick would have sent a notification
|
// A pattern shows as PAGED when this tick would have sent a notification
|
||||||
// (new or escalated past its cooldown) and as ongoing while it is still
|
// (new or escalated past its cooldown) and as ongoing while it is still
|
||||||
// firing but already paged once. When a service's spike disappears it is
|
// firing but already paged once. When a service's spike disappears it is
|
||||||
// reported as recovered for that scan.
|
// reported as recovered for that scan, and the last firing issue stays on
|
||||||
|
// the display as resolved until something fires again.
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -96,6 +97,12 @@ type scanner struct {
|
|||||||
// firing; it mirrors what a person watching notifications would remember.
|
// firing; it mirrors what a person watching notifications would remember.
|
||||||
active map[string]time.Time
|
active map[string]time.Time
|
||||||
|
|
||||||
|
// lastActive is the most recent firing issue set; when a later scan
|
||||||
|
// finds nothing firing, it is kept on the display as resolved instead
|
||||||
|
// of flipping to ALL CLEAR.
|
||||||
|
lastActive []issueView
|
||||||
|
resolvedAt time.Time // window end when lastActive stopped firing
|
||||||
|
|
||||||
since time.Time
|
since time.Time
|
||||||
sinceAt time.Time
|
sinceAt time.Time
|
||||||
}
|
}
|
||||||
@@ -121,10 +128,11 @@ func newClickHouseScanner(addr, database, user, password string, cfg features.Sp
|
|||||||
// issueView is one card on the dashboard: a firing spike reduced to an Alert
|
// issueView is one card on the dashboard: a firing spike reduced to an Alert
|
||||||
// (the same model the notifications carry) plus display state.
|
// (the same model the notifications carry) plus display state.
|
||||||
type issueView struct {
|
type issueView struct {
|
||||||
alert alerts.Alert
|
alert alerts.Alert
|
||||||
key string
|
key string
|
||||||
paged bool // this scan would have notified (new or escalated)
|
paged bool // this scan would have notified (new or escalated)
|
||||||
firstSeen time.Time // window end when this issue first appeared
|
firstSeen time.Time // window end when this issue first appeared
|
||||||
|
resolvedAt time.Time // window end when it stopped firing; renders as resolved
|
||||||
}
|
}
|
||||||
|
|
||||||
// recoveredIssue is an issue that fired on the previous scan but not now.
|
// recoveredIssue is an issue that fired on the previous scan but not now.
|
||||||
@@ -134,7 +142,8 @@ type recoveredIssue struct {
|
|||||||
LastSeen time.Time
|
LastSeen time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// snapshot is one scan's result: everything firing right now.
|
// snapshot is one scan's result: everything firing right now, plus the last
|
||||||
|
// firing issue kept as resolved when nothing fires any more.
|
||||||
type snapshot struct {
|
type snapshot struct {
|
||||||
ScannedAt time.Time
|
ScannedAt time.Time
|
||||||
WindowEnd time.Time
|
WindowEnd time.Time
|
||||||
@@ -142,6 +151,7 @@ type snapshot struct {
|
|||||||
Events int
|
Events int
|
||||||
Issues []issueView
|
Issues []issueView
|
||||||
Recovered []recoveredIssue
|
Recovered []recoveredIssue
|
||||||
|
Resolved []issueView
|
||||||
Err error
|
Err error
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -219,6 +229,22 @@ func (s *scanner) Scan(ctx context.Context, now time.Time) snapshot {
|
|||||||
snap.Recovered = append(snap.Recovered, recoveredIssue{Service: serviceOfKey(key), FirstSeen: first, LastSeen: end})
|
snap.Recovered = append(snap.Recovered, recoveredIssue{Service: serviceOfKey(key), FirstSeen: first, LastSeen: end})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Keep the last firing issue on the display: once nothing fires any
|
||||||
|
// more, the cards stay up as resolved rather than the dashboard erasing
|
||||||
|
// the context with an ALL CLEAR panel.
|
||||||
|
if len(snap.Issues) > 0 {
|
||||||
|
s.lastActive = snap.Issues
|
||||||
|
s.resolvedAt = time.Time{}
|
||||||
|
} else if len(s.lastActive) > 0 {
|
||||||
|
if s.resolvedAt.IsZero() {
|
||||||
|
s.resolvedAt = end
|
||||||
|
}
|
||||||
|
for _, v := range s.lastActive {
|
||||||
|
v.resolvedAt = s.resolvedAt
|
||||||
|
snap.Resolved = append(snap.Resolved, v)
|
||||||
|
}
|
||||||
|
sortIssues(snap.Resolved)
|
||||||
|
}
|
||||||
sortIssues(snap.Issues)
|
sortIssues(snap.Issues)
|
||||||
return snap
|
return snap
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -96,10 +96,20 @@ func TestScannerPagesOnceThenOngoing(t *testing.T) {
|
|||||||
if len(fourth.Recovered) != 1 || fourth.Recovered[0].Service != "checkout" {
|
if len(fourth.Recovered) != 1 || fourth.Recovered[0].Service != "checkout" {
|
||||||
t.Fatalf("expected checkout in recovered, got %+v", fourth.Recovered)
|
t.Fatalf("expected checkout in recovered, got %+v", fourth.Recovered)
|
||||||
}
|
}
|
||||||
|
// The last issue stays on the display as resolved instead of ALL CLEAR.
|
||||||
|
if len(fourth.Resolved) != 1 || fourth.Resolved[0].alert.Spike.Service != "checkout" {
|
||||||
|
t.Fatalf("expected checkout kept as resolved, got %+v", fourth.Resolved)
|
||||||
|
}
|
||||||
|
if got := fourth.Resolved[0].resolvedAt; !got.Equal(fourth.WindowEnd) {
|
||||||
|
t.Fatalf("resolvedAt should be the window that stopped firing, got %v want %v", got, fourth.WindowEnd)
|
||||||
|
}
|
||||||
again := sc.Scan(context.Background(), base.Add(4*time.Minute))
|
again := sc.Scan(context.Background(), base.Add(4*time.Minute))
|
||||||
if len(again.Recovered) != 0 {
|
if len(again.Recovered) != 0 {
|
||||||
t.Fatalf("recovery should be reported once, got %+v", again.Recovered)
|
t.Fatalf("recovery should be reported once, got %+v", again.Recovered)
|
||||||
}
|
}
|
||||||
|
if len(again.Resolved) != 1 || !again.Resolved[0].resolvedAt.Equal(fourth.Resolved[0].resolvedAt) {
|
||||||
|
t.Fatalf("resolved issue should persist with a stable resolved time, got %+v", again.Resolved)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestScannerAllClearAndError(t *testing.T) {
|
func TestScannerAllClearAndError(t *testing.T) {
|
||||||
@@ -108,6 +118,9 @@ func TestScannerAllClearAndError(t *testing.T) {
|
|||||||
if len(empty.Issues) != 0 || empty.Err != nil {
|
if len(empty.Issues) != 0 || empty.Err != nil {
|
||||||
t.Fatalf("empty loader should yield a clean snapshot, got %+v", empty)
|
t.Fatalf("empty loader should yield a clean snapshot, got %+v", empty)
|
||||||
}
|
}
|
||||||
|
if len(empty.Resolved) != 0 {
|
||||||
|
t.Fatalf("a fresh scanner has no last issue to keep, got %+v", empty.Resolved)
|
||||||
|
}
|
||||||
|
|
||||||
sc.load = func(context.Context, time.Time, time.Time) ([]features.Event, error) {
|
sc.load = func(context.Context, time.Time, time.Time) ([]features.Event, error) {
|
||||||
return nil, context.DeadlineExceeded
|
return nil, context.DeadlineExceeded
|
||||||
@@ -204,6 +217,22 @@ func TestRenderSnapshotAllClear(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRenderSnapshotKeepsResolvedIssue(t *testing.T) {
|
||||||
|
now := time.Date(2025, 6, 1, 12, 45, 0, 0, time.UTC)
|
||||||
|
snap, o := cardAt(t, 1)
|
||||||
|
resolved := snap.Issues[0]
|
||||||
|
resolved.resolvedAt = now
|
||||||
|
snap.Issues = nil
|
||||||
|
snap.Resolved = []issueView{resolved}
|
||||||
|
out := renderSnapshot(snap, o)
|
||||||
|
if strings.Contains(out, "ALL CLEAR") {
|
||||||
|
t.Fatalf("a resolved issue should replace the ALL CLEAR panel:\n%s", out)
|
||||||
|
}
|
||||||
|
if !strings.Contains(out, "RESOLVED") || !strings.Contains(out, "checkout") {
|
||||||
|
t.Fatalf("dashboard should keep the last issue as resolved:\n%s", out)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRenderSnapshotScanError(t *testing.T) {
|
func TestRenderSnapshotScanError(t *testing.T) {
|
||||||
snap := snapshot{Err: context.DeadlineExceeded}
|
snap := snapshot{Err: context.DeadlineExceeded}
|
||||||
out := renderSnapshot(snap, snapshotOptions{Width: 200, Height: 12, Title: "otel issues", Now: time.Now()})
|
out := renderSnapshot(snap, snapshotOptions{Width: 200, Height: 12, Title: "otel issues", Now: time.Now()})
|
||||||
|
|||||||
+54
-18
@@ -91,6 +91,10 @@ func renderBody(snap snapshot, o snapshotOptions, w, avail int) string {
|
|||||||
case o.ScanError != nil && len(snap.Issues) == 0:
|
case o.ScanError != nil && len(snap.Issues) == 0:
|
||||||
return statusPanel(w, min(avail, 5), colCritical, "SCAN FAILED", o.ScanError.Error(),
|
return statusPanel(w, min(avail, 5), colCritical, "SCAN FAILED", o.ScanError.Error(),
|
||||||
"no successful scan yet; retrying automatically")
|
"no successful scan yet; retrying automatically")
|
||||||
|
case len(snap.Issues) == 0 && len(snap.Resolved) > 0:
|
||||||
|
// Nothing fires any more: keep the last issue on the display,
|
||||||
|
// flipped to resolved, instead of an ALL CLEAR panel.
|
||||||
|
return renderGrid(snap.Resolved, o, w, avail)
|
||||||
case len(snap.Issues) == 0:
|
case len(snap.Issues) == 0:
|
||||||
sub := fmt.Sprintf("no issues firing · window %s–%s UTC · scanned %s",
|
sub := fmt.Sprintf("no issues firing · window %s–%s UTC · scanned %s",
|
||||||
snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"), commas(snap.Events))
|
snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"), commas(snap.Events))
|
||||||
@@ -99,19 +103,19 @@ func renderBody(snap snapshot, o snapshotOptions, w, avail int) string {
|
|||||||
}
|
}
|
||||||
return statusPanel(w, min(avail, 5), colOK, "ALL CLEAR", sub, "")
|
return statusPanel(w, min(avail, 5), colOK, "ALL CLEAR", sub, "")
|
||||||
default:
|
default:
|
||||||
return renderGrid(snap, o, w, avail)
|
return renderGrid(snap.Issues, o, w, avail)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// renderGrid lays issues into rows of cards that fill the width; only as many
|
// renderGrid lays issues into rows of cards that fill the width; only as many
|
||||||
// rows as the height allows are drawn, the rest are counted in the footer.
|
// rows as the height allows are drawn, the rest are counted in the footer.
|
||||||
func renderGrid(snap snapshot, o snapshotOptions, w, avail int) string {
|
func renderGrid(views []issueView, o snapshotOptions, w, avail int) string {
|
||||||
perRow, _, shown, cardW := gridLayout(len(snap.Issues), w, avail)
|
perRow, _, shown, cardW := gridLayout(len(views), w, avail)
|
||||||
var rendered []string
|
var rendered []string
|
||||||
for start := 0; start < shown; start += perRow {
|
for start := 0; start < shown; start += perRow {
|
||||||
end := min(start+perRow, shown)
|
end := min(start+perRow, shown)
|
||||||
var row []string
|
var row []string
|
||||||
for _, v := range snap.Issues[start:end] {
|
for _, v := range views[start:end] {
|
||||||
row = append(row, issueCard(v, cardW, o))
|
row = append(row, issueCard(v, cardW, o))
|
||||||
}
|
}
|
||||||
rendered = append(rendered, lipgloss.JoinHorizontal(lipgloss.Top, row...))
|
rendered = append(rendered, lipgloss.JoinHorizontal(lipgloss.Top, row...))
|
||||||
@@ -145,11 +149,19 @@ func gridLayout(n, width, avail int) (perRow, rows, shown, cardW int) {
|
|||||||
func issueCard(v issueView, w int, o snapshotOptions) string {
|
func issueCard(v issueView, w int, o snapshotOptions) string {
|
||||||
a := v.alert
|
a := v.alert
|
||||||
fg := priorityColor(a.Priority())
|
fg := priorityColor(a.Priority())
|
||||||
|
resolved := !v.resolvedAt.IsZero()
|
||||||
|
if resolved {
|
||||||
|
// A resolved card keeps its place but reads as recovered: the same
|
||||||
|
// green as the footer's recovered line and the ALL CLEAR panel.
|
||||||
|
fg = colOK
|
||||||
|
}
|
||||||
inner := w - 4 // border plus padding on both sides
|
inner := w - 4 // border plus padding on both sides
|
||||||
|
|
||||||
// Line 1: HIGH · checkout · for 12m
|
// Line 1: HIGH · checkout · for 12m
|
||||||
age := ""
|
age := ""
|
||||||
if !v.firstSeen.IsZero() && !o.Now.IsZero() {
|
if !v.firstSeen.IsZero() && resolved {
|
||||||
|
age = " · for " + shortDur(v.resolvedAt.Sub(v.firstSeen).Round(time.Minute))
|
||||||
|
} else if !v.firstSeen.IsZero() && !o.Now.IsZero() {
|
||||||
age = " · for " + shortDur(o.Now.Sub(v.firstSeen).Round(time.Minute))
|
age = " · for " + shortDur(o.Now.Sub(v.firstSeen).Round(time.Minute))
|
||||||
}
|
}
|
||||||
head := ansi.Truncate(
|
head := ansi.Truncate(
|
||||||
@@ -190,14 +202,20 @@ func issueCard(v issueView, w int, o snapshotOptions) string {
|
|||||||
}
|
}
|
||||||
statsLine := ansi.Truncate(strings.Join(segs, " "), inner, "…")
|
statsLine := ansi.Truncate(strings.Join(segs, " "), inner, "…")
|
||||||
|
|
||||||
// Line 4: display state (paged now vs already paged) and the issue link.
|
// Line 4: display state (paged now vs already paged vs resolved) and the
|
||||||
|
// issue link.
|
||||||
state := style(colText3, false).Render("ongoing")
|
state := style(colText3, false).Render("ongoing")
|
||||||
if v.paged {
|
when := a.Spike.WindowEnd
|
||||||
|
switch {
|
||||||
|
case resolved:
|
||||||
|
state = style(colOK, true).Render("RESOLVED")
|
||||||
|
when = v.resolvedAt
|
||||||
|
case v.paged:
|
||||||
state = style(fg, true).Render("PAGED")
|
state = style(fg, true).Render("PAGED")
|
||||||
}
|
}
|
||||||
var meta []string
|
var meta []string
|
||||||
if !a.Spike.WindowEnd.IsZero() {
|
if !when.IsZero() {
|
||||||
meta = append(meta, style(colText3, false).Render(fmtTime(a.Spike.WindowEnd)))
|
meta = append(meta, style(colText3, false).Render(fmtTime(when)))
|
||||||
}
|
}
|
||||||
meta = append(meta, state)
|
meta = append(meta, state)
|
||||||
if o.BaseURL != "" && len(a.Patterns) > 0 && a.Patterns[0].IssueID != "" {
|
if o.BaseURL != "" && len(a.Patterns) > 0 && a.Patterns[0].IssueID != "" {
|
||||||
@@ -254,9 +272,12 @@ func renderHeader(snap snapshot, o snapshotOptions) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
counts := fmt.Sprintf("%d firing", len(snap.Issues))
|
counts := fmt.Sprintf("%d firing", len(snap.Issues))
|
||||||
if paged > 0 {
|
switch {
|
||||||
|
case len(snap.Issues) == 0 && len(snap.Resolved) > 0:
|
||||||
|
left = append(left, style(colOK, true).Render(fmt.Sprintf("%d resolved", len(snap.Resolved))))
|
||||||
|
case paged > 0:
|
||||||
left = append(left, style(colCritical, true).Render(fmt.Sprintf("%d PAGED · %s", paged, counts)))
|
left = append(left, style(colCritical, true).Render(fmt.Sprintf("%d PAGED · %s", paged, counts)))
|
||||||
} else {
|
default:
|
||||||
left = append(left, style(colText2, false).Render(counts))
|
left = append(left, style(colText2, false).Render(counts))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -282,7 +303,7 @@ func renderHeader(snap snapshot, o snapshotOptions) string {
|
|||||||
func renderFooter(snap snapshot, o snapshotOptions) string {
|
func renderFooter(snap snapshot, o snapshotOptions) string {
|
||||||
var left []string
|
var left []string
|
||||||
left = append(left, style(colText3, false).Render(footerHint))
|
left = append(left, style(colText3, false).Render(footerHint))
|
||||||
if hidden := len(snap.Issues) - shownCount(snap, o); hidden > 0 {
|
if hidden := cardCount(snap) - shownCount(snap, o); hidden > 0 {
|
||||||
left = append(left, style(colWarn, false).Render(fmt.Sprintf("+%d more not shown (widen or raise the display)", hidden)))
|
left = append(left, style(colWarn, false).Render(fmt.Sprintf("+%d more not shown (widen or raise the display)", hidden)))
|
||||||
}
|
}
|
||||||
var right []string
|
var right []string
|
||||||
@@ -299,11 +320,20 @@ func renderFooter(snap snapshot, o snapshotOptions) string {
|
|||||||
return bar(o.Width-2, strings.Join(left, " "), strings.Join(right, " "))
|
return bar(o.Width-2, strings.Join(left, " "), strings.Join(right, " "))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// cardCount is how many cards the body renders: the firing issues, or the
|
||||||
|
// kept resolved ones when nothing fires.
|
||||||
|
func cardCount(snap snapshot) int {
|
||||||
|
if n := len(snap.Issues); n > 0 {
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
return len(snap.Resolved)
|
||||||
|
}
|
||||||
|
|
||||||
func shownCount(snap snapshot, o snapshotOptions) int {
|
func shownCount(snap snapshot, o snapshotOptions) int {
|
||||||
if o.Height < minUsableHeight {
|
if o.Height < minUsableHeight {
|
||||||
return len(snap.Issues)
|
return cardCount(snap)
|
||||||
}
|
}
|
||||||
_, _, shown, _ := gridLayout(len(snap.Issues), o.Width, o.Height-2)
|
_, _, shown, _ := gridLayout(cardCount(snap), o.Width, o.Height-2)
|
||||||
return shown
|
return shown
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -323,14 +353,20 @@ func renderMinimal(snap snapshot, o snapshotOptions) string {
|
|||||||
if snap.Err != nil {
|
if snap.Err != nil {
|
||||||
return "SCAN FAILED: " + snap.Err.Error()
|
return "SCAN FAILED: " + snap.Err.Error()
|
||||||
}
|
}
|
||||||
if len(snap.Issues) == 0 {
|
if len(snap.Issues) == 0 && len(snap.Resolved) == 0 {
|
||||||
return strings.ToUpper(o.Title) + ": all clear"
|
return strings.ToUpper(o.Title) + ": all clear"
|
||||||
}
|
}
|
||||||
lines := make([]string, 0, len(snap.Issues)+1)
|
views := snap.Issues
|
||||||
|
if len(views) == 0 {
|
||||||
|
views = snap.Resolved
|
||||||
|
}
|
||||||
|
lines := make([]string, 0, len(views)+1)
|
||||||
lines = append(lines, strings.ToUpper(o.Title)+":")
|
lines = append(lines, strings.ToUpper(o.Title)+":")
|
||||||
for _, v := range snap.Issues {
|
for _, v := range views {
|
||||||
state := "ongoing"
|
state := "ongoing"
|
||||||
if v.paged {
|
if !v.resolvedAt.IsZero() {
|
||||||
|
state = "RESOLVED"
|
||||||
|
} else if v.paged {
|
||||||
state = "PAGED"
|
state = "PAGED"
|
||||||
}
|
}
|
||||||
what := ""
|
what := ""
|
||||||
|
|||||||
Reference in new issue
Block a user