diff --git a/README.md b/README.md index 0cb3dd3..e58cd0f 100644 --- a/README.md +++ b/README.md @@ -371,6 +371,33 @@ to Docker or stored in the image. - `OTEL_CLICKHOUSE_DB` / `OTEL_CLICKHOUSE_USER` / `OTEL_CLICKHOUSE_PASSWORD` - optional ClickHouse connection settings (defaults: `otel` / `default` / empty). + +### Issues display + +The standalone `issues-tui` command is a terminal dashboard for currently +firing alert spikes. It reads normalized error events directly from +ClickHouse; it does not replace the web UI's persisted `/api/issues` view. + +Build and run it on the display computer (or use `go run`): + +```sh +go build -o issues-tui ./cmd/issues-tui +OTEL_CLICKHOUSE_ADDR=clickhouse.example.net:9000 \ +OTEL_CLICKHOUSE_DB=otel \ +OTEL_CLICKHOUSE_USER=default \ +OTEL_CLICKHOUSE_PASSWORD='…' \ +OTEL_ALERT_BASE_URL=https://otel.example.net \ +./issues-tui +``` + +The display computer must be able to reach ClickHouse's native TCP port. For a +Kubernetes-only ClickHouse service, use a private network path or tunnel; the +in-cluster service name is not reachable from an ordinary workstation. The +dashboard refreshes using the `OTEL_ALERT_*` interval, window, baseline, +cooldown and lag settings. Press `r` to scan immediately and `q` to quit. +Use `-once` for a one-shot, non-interactive output suitable for a process +supervisor health check. + - `OTEL_INGEST_KEY` - optional shared key required (via `?key=` or the `X-Otel-Ingest-Key` header) on `/v1/traces`, `/v1/metrics` and `/v1/logs` over HTTP. Unset by default (no key required). This is meant to diff --git a/cmd/issues-tui/main.go b/cmd/issues-tui/main.go index bc12590..dff243f 100644 --- a/cmd/issues-tui/main.go +++ b/cmd/issues-tui/main.go @@ -155,6 +155,10 @@ func spikeKey(s features.Spike) string { return s.Service + "\x00" + strings.Join(sigs, "\x00") } +func patternKey(service, signature string) string { + return service + "\x00" + signature +} + // Scan runs one evaluation, mirroring the runner's Tick: load the window // ending lag ago plus its baseline, detect exactly what the server pages on, // and use a cooldown suppressor to tell a fresh page from an ongoing one. @@ -184,9 +188,11 @@ func (s *scanner) Scan(ctx context.Context, now time.Time) snapshot { // The suppressor mutates only its own state and the copies Filter makes, // so firing keeps every pattern for display while paged marks what would // have been sent to Telegram / the webhook. - pagedKeys := map[string]bool{} + pagedPatterns := map[string]bool{} for _, sp := range s.pager.Filter(firing) { - pagedKeys[spikeKey(sp)] = true + for _, p := range sp.Patterns { + pagedPatterns[patternKey(sp.Service, p.Signature)] = true + } } nowActive := map[string]bool{} @@ -198,7 +204,14 @@ func (s *scanner) Scan(ctx context.Context, now time.Time) snapshot { first = end s.active[key] = first } - snap.Issues = append(snap.Issues, issueView{alert: alerts.NewAlert(sp, end), key: key, paged: pagedKeys[key], firstSeen: first}) + paged := false + for _, p := range sp.Patterns { + if pagedPatterns[patternKey(sp.Service, p.Signature)] { + paged = true + break + } + } + snap.Issues = append(snap.Issues, issueView{alert: alerts.NewAlert(sp, end), key: key, paged: paged, firstSeen: first}) } for key, first := range s.active { if !nowActive[key] { diff --git a/cmd/issues-tui/main_test.go b/cmd/issues-tui/main_test.go index c470560..e2547b9 100644 --- a/cmd/issues-tui/main_test.go +++ b/cmd/issues-tui/main_test.go @@ -251,3 +251,26 @@ func TestBarAndMinimal(t *testing.T) { t.Fatalf("minimal render should surface scan errors, got %q", got) } } + +func TestModelKeepsLastGoodSnapshotOnScanError(t *testing.T) { + m := newModel(nil, modelConfig{Interval: time.Minute}) + good := snapshot{ScannedAt: time.Now(), Issues: []issueView{{}}} + updated, _ := m.Update(snapshotMsg(good)) + m = updated.(model) + + failed := snapshot{Err: context.DeadlineExceeded} + updated, _ = m.Update(snapshotMsg(failed)) + m = updated.(model) + if len(m.snap.Issues) != 1 { + t.Fatalf("failed scan should not erase the last good snapshot: %+v", m.snap) + } + if m.scanErr == nil { + t.Fatal("failed scan should be visible separately from the last good snapshot") + } + + updated, _ = m.Update(snapshotMsg(snapshot{ScannedAt: time.Now()})) + m = updated.(model) + if m.scanErr != nil { + t.Fatal("successful scan should clear the previous scan error") + } +} diff --git a/cmd/issues-tui/tui.go b/cmd/issues-tui/tui.go index 18997ef..4029647 100644 --- a/cmd/issues-tui/tui.go +++ b/cmd/issues-tui/tui.go @@ -25,6 +25,9 @@ type model struct { sc *scanner cfg modelConfig snap snapshot + // scanErr is kept separately so a failed refresh does not erase the last + // successful data from the display. + scanErr error scanning bool nextScanAt time.Time @@ -79,7 +82,13 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m, cmd case snapshotMsg: - m.snap = snapshot(msg) + snap := snapshot(msg) + if snap.Err != nil { + m.scanErr = snap.Err + } else { + m.snap = snap + m.scanErr = nil + } m.scanning = false m.nextScanAt = time.Now().Add(m.cfg.Interval) return m, nil @@ -90,6 +99,6 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { func (m model) View() string { return renderSnapshot(m.snap, snapshotOptions{ Width: m.width, Height: m.height, Title: m.cfg.Title, BaseURL: m.cfg.BaseURL, - Scanning: m.scanning, NextScanAt: m.nextScanAt, Now: time.Now(), + Scanning: m.scanning, NextScanAt: m.nextScanAt, Now: time.Now(), ScanError: m.scanErr, }) } diff --git a/cmd/issues-tui/view.go b/cmd/issues-tui/view.go index 07fff0a..449f900 100644 --- a/cmd/issues-tui/view.go +++ b/cmd/issues-tui/view.go @@ -34,6 +34,7 @@ type snapshotOptions struct { Scanning bool NextScanAt time.Time Now time.Time + ScanError error } // Colors mirror the otel-ui design-system tokens (AGENTS.md) so the terminal @@ -87,6 +88,9 @@ func renderBody(snap snapshot, o snapshotOptions, w, avail int) string { case snap.Err != nil: return statusPanel(w, min(avail, 5), colCritical, "SCAN FAILED", snap.Err.Error(), "values are from "+fmtTime(snap.ScannedAt)+"; the display keeps the last good scan") + case o.ScanError != nil && len(snap.Issues) == 0: + return statusPanel(w, min(avail, 5), colCritical, "SCAN FAILED", o.ScanError.Error(), + "no successful scan yet; retrying automatically") case len(snap.Issues) == 0: sub := fmt.Sprintf("no issues firing · window %s–%s UTC · scanned %s", snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"), commas(snap.Events)) @@ -234,6 +238,9 @@ func renderHeader(snap snapshot, o snapshotOptions) string { title := style(colAccent, true).Render(strings.ToUpper(o.Title)) left := []string{title} + if o.ScanError != nil { + left = append(left, style(colCritical, true).Render("SCAN FAILED")) + } if !snap.WindowEnd.IsZero() { left = append(left, style(colText3, false).Render( fmt.Sprintf("%s–%s UTC", snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"))))