fix issues TUI display reliability
This commit is contained in:
1 parent
a96e95212a
commit
f5015581a4
5 files changed
+84
-5
No files matched your search
@@ -371,6 +371,33 @@ to Docker or stored in the image.
|
|||||||
- `OTEL_CLICKHOUSE_DB` / `OTEL_CLICKHOUSE_USER` /
|
- `OTEL_CLICKHOUSE_DB` / `OTEL_CLICKHOUSE_USER` /
|
||||||
`OTEL_CLICKHOUSE_PASSWORD` - optional ClickHouse connection settings
|
`OTEL_CLICKHOUSE_PASSWORD` - optional ClickHouse connection settings
|
||||||
(defaults: `otel` / `default` / empty).
|
(defaults: `otel` / `default` / empty).
|
||||||
|
|
||||||
|
### Issues display
|
||||||
|
|
||||||
|
The standalone `issues-tui` command is a terminal dashboard for currently
|
||||||
|
firing alert spikes. It reads normalized error events directly from
|
||||||
|
ClickHouse; it does not replace the web UI's persisted `/api/issues` view.
|
||||||
|
|
||||||
|
Build and run it on the display computer (or use `go run`):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go build -o issues-tui ./cmd/issues-tui
|
||||||
|
OTEL_CLICKHOUSE_ADDR=clickhouse.example.net:9000 \
|
||||||
|
OTEL_CLICKHOUSE_DB=otel \
|
||||||
|
OTEL_CLICKHOUSE_USER=default \
|
||||||
|
OTEL_CLICKHOUSE_PASSWORD='…' \
|
||||||
|
OTEL_ALERT_BASE_URL=https://otel.example.net \
|
||||||
|
./issues-tui
|
||||||
|
```
|
||||||
|
|
||||||
|
The display computer must be able to reach ClickHouse's native TCP port. For a
|
||||||
|
Kubernetes-only ClickHouse service, use a private network path or tunnel; the
|
||||||
|
in-cluster service name is not reachable from an ordinary workstation. The
|
||||||
|
dashboard refreshes using the `OTEL_ALERT_*` interval, window, baseline,
|
||||||
|
cooldown and lag settings. Press `r` to scan immediately and `q` to quit.
|
||||||
|
Use `-once` for a one-shot, non-interactive output suitable for a process
|
||||||
|
supervisor health check.
|
||||||
|
|
||||||
- `OTEL_INGEST_KEY` - optional shared key required (via `?key=` or the
|
- `OTEL_INGEST_KEY` - optional shared key required (via `?key=` or the
|
||||||
`X-Otel-Ingest-Key` header) on `/v1/traces`, `/v1/metrics` and `/v1/logs`
|
`X-Otel-Ingest-Key` header) on `/v1/traces`, `/v1/metrics` and `/v1/logs`
|
||||||
over HTTP. Unset by default (no key required). This is meant to
|
over HTTP. Unset by default (no key required). This is meant to
|
||||||
|
|||||||
+16
-3
@@ -155,6 +155,10 @@ func spikeKey(s features.Spike) string {
|
|||||||
return s.Service + "\x00" + strings.Join(sigs, "\x00")
|
return s.Service + "\x00" + strings.Join(sigs, "\x00")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func patternKey(service, signature string) string {
|
||||||
|
return service + "\x00" + signature
|
||||||
|
}
|
||||||
|
|
||||||
// Scan runs one evaluation, mirroring the runner's Tick: load the window
|
// Scan runs one evaluation, mirroring the runner's Tick: load the window
|
||||||
// ending lag ago plus its baseline, detect exactly what the server pages on,
|
// ending lag ago plus its baseline, detect exactly what the server pages on,
|
||||||
// and use a cooldown suppressor to tell a fresh page from an ongoing one.
|
// and use a cooldown suppressor to tell a fresh page from an ongoing one.
|
||||||
@@ -184,9 +188,11 @@ func (s *scanner) Scan(ctx context.Context, now time.Time) snapshot {
|
|||||||
// The suppressor mutates only its own state and the copies Filter makes,
|
// The suppressor mutates only its own state and the copies Filter makes,
|
||||||
// so firing keeps every pattern for display while paged marks what would
|
// so firing keeps every pattern for display while paged marks what would
|
||||||
// have been sent to Telegram / the webhook.
|
// have been sent to Telegram / the webhook.
|
||||||
pagedKeys := map[string]bool{}
|
pagedPatterns := map[string]bool{}
|
||||||
for _, sp := range s.pager.Filter(firing) {
|
for _, sp := range s.pager.Filter(firing) {
|
||||||
pagedKeys[spikeKey(sp)] = true
|
for _, p := range sp.Patterns {
|
||||||
|
pagedPatterns[patternKey(sp.Service, p.Signature)] = true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
nowActive := map[string]bool{}
|
nowActive := map[string]bool{}
|
||||||
@@ -198,7 +204,14 @@ func (s *scanner) Scan(ctx context.Context, now time.Time) snapshot {
|
|||||||
first = end
|
first = end
|
||||||
s.active[key] = first
|
s.active[key] = first
|
||||||
}
|
}
|
||||||
snap.Issues = append(snap.Issues, issueView{alert: alerts.NewAlert(sp, end), key: key, paged: pagedKeys[key], firstSeen: first})
|
paged := false
|
||||||
|
for _, p := range sp.Patterns {
|
||||||
|
if pagedPatterns[patternKey(sp.Service, p.Signature)] {
|
||||||
|
paged = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
snap.Issues = append(snap.Issues, issueView{alert: alerts.NewAlert(sp, end), key: key, paged: paged, firstSeen: first})
|
||||||
}
|
}
|
||||||
for key, first := range s.active {
|
for key, first := range s.active {
|
||||||
if !nowActive[key] {
|
if !nowActive[key] {
|
||||||
|
|||||||
@@ -251,3 +251,26 @@ func TestBarAndMinimal(t *testing.T) {
|
|||||||
t.Fatalf("minimal render should surface scan errors, got %q", got)
|
t.Fatalf("minimal render should surface scan errors, got %q", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestModelKeepsLastGoodSnapshotOnScanError(t *testing.T) {
|
||||||
|
m := newModel(nil, modelConfig{Interval: time.Minute})
|
||||||
|
good := snapshot{ScannedAt: time.Now(), Issues: []issueView{{}}}
|
||||||
|
updated, _ := m.Update(snapshotMsg(good))
|
||||||
|
m = updated.(model)
|
||||||
|
|
||||||
|
failed := snapshot{Err: context.DeadlineExceeded}
|
||||||
|
updated, _ = m.Update(snapshotMsg(failed))
|
||||||
|
m = updated.(model)
|
||||||
|
if len(m.snap.Issues) != 1 {
|
||||||
|
t.Fatalf("failed scan should not erase the last good snapshot: %+v", m.snap)
|
||||||
|
}
|
||||||
|
if m.scanErr == nil {
|
||||||
|
t.Fatal("failed scan should be visible separately from the last good snapshot")
|
||||||
|
}
|
||||||
|
|
||||||
|
updated, _ = m.Update(snapshotMsg(snapshot{ScannedAt: time.Now()}))
|
||||||
|
m = updated.(model)
|
||||||
|
if m.scanErr != nil {
|
||||||
|
t.Fatal("successful scan should clear the previous scan error")
|
||||||
|
}
|
||||||
|
}
|
||||||
+11
-2
@@ -25,6 +25,9 @@ type model struct {
|
|||||||
sc *scanner
|
sc *scanner
|
||||||
cfg modelConfig
|
cfg modelConfig
|
||||||
snap snapshot
|
snap snapshot
|
||||||
|
// scanErr is kept separately so a failed refresh does not erase the last
|
||||||
|
// successful data from the display.
|
||||||
|
scanErr error
|
||||||
|
|
||||||
scanning bool
|
scanning bool
|
||||||
nextScanAt time.Time
|
nextScanAt time.Time
|
||||||
@@ -79,7 +82,13 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
return m, cmd
|
return m, cmd
|
||||||
|
|
||||||
case snapshotMsg:
|
case snapshotMsg:
|
||||||
m.snap = snapshot(msg)
|
snap := snapshot(msg)
|
||||||
|
if snap.Err != nil {
|
||||||
|
m.scanErr = snap.Err
|
||||||
|
} else {
|
||||||
|
m.snap = snap
|
||||||
|
m.scanErr = nil
|
||||||
|
}
|
||||||
m.scanning = false
|
m.scanning = false
|
||||||
m.nextScanAt = time.Now().Add(m.cfg.Interval)
|
m.nextScanAt = time.Now().Add(m.cfg.Interval)
|
||||||
return m, nil
|
return m, nil
|
||||||
@@ -90,6 +99,6 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
|||||||
func (m model) View() string {
|
func (m model) View() string {
|
||||||
return renderSnapshot(m.snap, snapshotOptions{
|
return renderSnapshot(m.snap, snapshotOptions{
|
||||||
Width: m.width, Height: m.height, Title: m.cfg.Title, BaseURL: m.cfg.BaseURL,
|
Width: m.width, Height: m.height, Title: m.cfg.Title, BaseURL: m.cfg.BaseURL,
|
||||||
Scanning: m.scanning, NextScanAt: m.nextScanAt, Now: time.Now(),
|
Scanning: m.scanning, NextScanAt: m.nextScanAt, Now: time.Now(), ScanError: m.scanErr,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -34,6 +34,7 @@ type snapshotOptions struct {
|
|||||||
Scanning bool
|
Scanning bool
|
||||||
NextScanAt time.Time
|
NextScanAt time.Time
|
||||||
Now time.Time
|
Now time.Time
|
||||||
|
ScanError error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Colors mirror the otel-ui design-system tokens (AGENTS.md) so the terminal
|
// Colors mirror the otel-ui design-system tokens (AGENTS.md) so the terminal
|
||||||
@@ -87,6 +88,9 @@ func renderBody(snap snapshot, o snapshotOptions, w, avail int) string {
|
|||||||
case snap.Err != nil:
|
case snap.Err != nil:
|
||||||
return statusPanel(w, min(avail, 5), colCritical, "SCAN FAILED", snap.Err.Error(),
|
return statusPanel(w, min(avail, 5), colCritical, "SCAN FAILED", snap.Err.Error(),
|
||||||
"values are from "+fmtTime(snap.ScannedAt)+"; the display keeps the last good scan")
|
"values are from "+fmtTime(snap.ScannedAt)+"; the display keeps the last good scan")
|
||||||
|
case o.ScanError != nil && len(snap.Issues) == 0:
|
||||||
|
return statusPanel(w, min(avail, 5), colCritical, "SCAN FAILED", o.ScanError.Error(),
|
||||||
|
"no successful scan yet; retrying automatically")
|
||||||
case len(snap.Issues) == 0:
|
case len(snap.Issues) == 0:
|
||||||
sub := fmt.Sprintf("no issues firing · window %s–%s UTC · scanned %s",
|
sub := fmt.Sprintf("no issues firing · window %s–%s UTC · scanned %s",
|
||||||
snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"), commas(snap.Events))
|
snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"), commas(snap.Events))
|
||||||
@@ -234,6 +238,9 @@ func renderHeader(snap snapshot, o snapshotOptions) string {
|
|||||||
title := style(colAccent, true).Render(strings.ToUpper(o.Title))
|
title := style(colAccent, true).Render(strings.ToUpper(o.Title))
|
||||||
|
|
||||||
left := []string{title}
|
left := []string{title}
|
||||||
|
if o.ScanError != nil {
|
||||||
|
left = append(left, style(colCritical, true).Render("SCAN FAILED"))
|
||||||
|
}
|
||||||
if !snap.WindowEnd.IsZero() {
|
if !snap.WindowEnd.IsZero() {
|
||||||
left = append(left, style(colText3, false).Render(
|
left = append(left, style(colText3, false).Render(
|
||||||
fmt.Sprintf("%s–%s UTC", snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"))))
|
fmt.Sprintf("%s–%s UTC", snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"))))
|
||||||
|
|||||||
Reference in new issue
Block a user