fix issues TUI display reliability
Build and Publish / Metadata (push) Successful in 10s
Build and Publish / BuildAndDeployAmd64 (push) Successful in 2m0s

This commit is contained in:
mats committed 2026-09-24 11:11:29 +02:00
1 parent a96e95212a
commit f5015581a4
5 files changed
+84 -5

No files matched your search

+27
View File
@@ -371,6 +371,33 @@ to Docker or stored in the image.
- `OTEL_CLICKHOUSE_DB` / `OTEL_CLICKHOUSE_USER` /
`OTEL_CLICKHOUSE_PASSWORD` - optional ClickHouse connection settings
(defaults: `otel` / `default` / empty).
### Issues display
The standalone `issues-tui` command is a terminal dashboard for currently
firing alert spikes. It reads normalized error events directly from
ClickHouse; it does not replace the web UI's persisted `/api/issues` view.
Build and run it on the display computer (or use `go run`):
```sh
go build -o issues-tui ./cmd/issues-tui
OTEL_CLICKHOUSE_ADDR=clickhouse.example.net:9000 \
OTEL_CLICKHOUSE_DB=otel \
OTEL_CLICKHOUSE_USER=default \
OTEL_CLICKHOUSE_PASSWORD='…' \
OTEL_ALERT_BASE_URL=https://otel.example.net \
./issues-tui
```
The display computer must be able to reach ClickHouse's native TCP port. For a
Kubernetes-only ClickHouse service, use a private network path or tunnel; the
in-cluster service name is not reachable from an ordinary workstation. The
dashboard refreshes using the `OTEL_ALERT_*` interval, window, baseline,
cooldown and lag settings. Press `r` to scan immediately and `q` to quit.
Use `-once` for a one-shot, non-interactive output suitable for a process
supervisor health check.
- `OTEL_INGEST_KEY` - optional shared key required (via `?key=` or the
`X-Otel-Ingest-Key` header) on `/v1/traces`, `/v1/metrics` and `/v1/logs`
over HTTP. Unset by default (no key required). This is meant to
+16 -3
View File
@@ -155,6 +155,10 @@ func spikeKey(s features.Spike) string {
return s.Service + "\x00" + strings.Join(sigs, "\x00")
}
func patternKey(service, signature string) string {
return service + "\x00" + signature
}
// Scan runs one evaluation, mirroring the runner's Tick: load the window
// ending lag ago plus its baseline, detect exactly what the server pages on,
// and use a cooldown suppressor to tell a fresh page from an ongoing one.
@@ -184,9 +188,11 @@ func (s *scanner) Scan(ctx context.Context, now time.Time) snapshot {
// The suppressor mutates only its own state and the copies Filter makes,
// so firing keeps every pattern for display while paged marks what would
// have been sent to Telegram / the webhook.
pagedKeys := map[string]bool{}
pagedPatterns := map[string]bool{}
for _, sp := range s.pager.Filter(firing) {
pagedKeys[spikeKey(sp)] = true
for _, p := range sp.Patterns {
pagedPatterns[patternKey(sp.Service, p.Signature)] = true
}
}
nowActive := map[string]bool{}
@@ -198,7 +204,14 @@ func (s *scanner) Scan(ctx context.Context, now time.Time) snapshot {
first = end
s.active[key] = first
}
snap.Issues = append(snap.Issues, issueView{alert: alerts.NewAlert(sp, end), key: key, paged: pagedKeys[key], firstSeen: first})
paged := false
for _, p := range sp.Patterns {
if pagedPatterns[patternKey(sp.Service, p.Signature)] {
paged = true
break
}
}
snap.Issues = append(snap.Issues, issueView{alert: alerts.NewAlert(sp, end), key: key, paged: paged, firstSeen: first})
}
for key, first := range s.active {
if !nowActive[key] {
+23
View File
@@ -251,3 +251,26 @@ func TestBarAndMinimal(t *testing.T) {
t.Fatalf("minimal render should surface scan errors, got %q", got)
}
}
func TestModelKeepsLastGoodSnapshotOnScanError(t *testing.T) {
m := newModel(nil, modelConfig{Interval: time.Minute})
good := snapshot{ScannedAt: time.Now(), Issues: []issueView{{}}}
updated, _ := m.Update(snapshotMsg(good))
m = updated.(model)
failed := snapshot{Err: context.DeadlineExceeded}
updated, _ = m.Update(snapshotMsg(failed))
m = updated.(model)
if len(m.snap.Issues) != 1 {
t.Fatalf("failed scan should not erase the last good snapshot: %+v", m.snap)
}
if m.scanErr == nil {
t.Fatal("failed scan should be visible separately from the last good snapshot")
}
updated, _ = m.Update(snapshotMsg(snapshot{ScannedAt: time.Now()}))
m = updated.(model)
if m.scanErr != nil {
t.Fatal("successful scan should clear the previous scan error")
}
}
+11 -2
View File
@@ -25,6 +25,9 @@ type model struct {
sc *scanner
cfg modelConfig
snap snapshot
// scanErr is kept separately so a failed refresh does not erase the last
// successful data from the display.
scanErr error
scanning bool
nextScanAt time.Time
@@ -79,7 +82,13 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
return m, cmd
case snapshotMsg:
m.snap = snapshot(msg)
snap := snapshot(msg)
if snap.Err != nil {
m.scanErr = snap.Err
} else {
m.snap = snap
m.scanErr = nil
}
m.scanning = false
m.nextScanAt = time.Now().Add(m.cfg.Interval)
return m, nil
@@ -90,6 +99,6 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
func (m model) View() string {
return renderSnapshot(m.snap, snapshotOptions{
Width: m.width, Height: m.height, Title: m.cfg.Title, BaseURL: m.cfg.BaseURL,
Scanning: m.scanning, NextScanAt: m.nextScanAt, Now: time.Now(),
Scanning: m.scanning, NextScanAt: m.nextScanAt, Now: time.Now(), ScanError: m.scanErr,
})
}
+7
View File
@@ -34,6 +34,7 @@ type snapshotOptions struct {
Scanning bool
NextScanAt time.Time
Now time.Time
ScanError error
}
// Colors mirror the otel-ui design-system tokens (AGENTS.md) so the terminal
@@ -87,6 +88,9 @@ func renderBody(snap snapshot, o snapshotOptions, w, avail int) string {
case snap.Err != nil:
return statusPanel(w, min(avail, 5), colCritical, "SCAN FAILED", snap.Err.Error(),
"values are from "+fmtTime(snap.ScannedAt)+"; the display keeps the last good scan")
case o.ScanError != nil && len(snap.Issues) == 0:
return statusPanel(w, min(avail, 5), colCritical, "SCAN FAILED", o.ScanError.Error(),
"no successful scan yet; retrying automatically")
case len(snap.Issues) == 0:
sub := fmt.Sprintf("no issues firing · window %s–%s UTC · scanned %s",
snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"), commas(snap.Events))
@@ -234,6 +238,9 @@ func renderHeader(snap snapshot, o snapshotOptions) string {
title := style(colAccent, true).Render(strings.ToUpper(o.Title))
left := []string{title}
if o.ScanError != nil {
left = append(left, style(colCritical, true).Render("SCAN FAILED"))
}
if !snap.WindowEnd.IsZero() {
left = append(left, style(colText3, false).Render(
fmt.Sprintf("%s–%s UTC", snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"))))