issues-tui: wide, short terminal dashboard for current issues
Standalone cmd that runs the same pipeline as the webhook notifications (alerts.LoadErrorEvents -> alerts.DetectAlerts -> cooldown suppressor) on a schedule and renders what is firing as a row of severity-colored cards - built for an ultra-wide, low display. A pattern shows PAGED when the scan would have notified and ongoing while suppressed; disappearing spikes are reported as recovered. -once prints one snapshot for cron/pipe use. Colors follow the otel-ui design-system tokens (dark and light).
This commit is contained in:
developer committed
1 parent
36f00d49b9
commit
a96e95212a
6 files changed
+1048
-78
No files matched your search
@@ -0,0 +1,249 @@
|
||||
// issues-tui is a standalone terminal dashboard for current issues. It runs
|
||||
// the exact detection pipeline behind the webhook notifications
|
||||
// (alerts.LoadErrorEvents -> alerts.DetectAlerts -> the cooldown suppressor)
|
||||
// on a schedule against ClickHouse, and renders what is firing right now as
|
||||
// wide, short cards - built for an ultra-wide, low display.
|
||||
//
|
||||
// A pattern shows as PAGED when this tick would have sent a notification
|
||||
// (new or escalated past its cooldown) and as ongoing while it is still
|
||||
// firing but already paged once. When a service's spike disappears it is
|
||||
// reported as recovered for that scan.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
|
||||
"git.tornberg.me/go-otel/internal/alerts"
|
||||
"git.tornberg.me/go-otel/internal/features"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Detection defaults come from the same environment the server's
|
||||
// alerting runner reads, so the dashboard and the notifications cannot
|
||||
// drift apart; every flag overrides.
|
||||
acfg := alerts.ConfigFromEnv()
|
||||
addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port")
|
||||
database := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "ClickHouse database")
|
||||
user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "ClickHouse user")
|
||||
password := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "ClickHouse password")
|
||||
interval := flag.Duration("interval", acfg.Interval, "how often to rescan")
|
||||
window := flag.Duration("window", acfg.Window, "observation window")
|
||||
baselineWindows := flag.Int("baseline-windows", acfg.BaselineWindows, "preceding windows used as baseline")
|
||||
cooldown := flag.Duration("cooldown", acfg.Cooldown, "a paged pattern stays ongoing this long unless it doubles")
|
||||
minExcess := flag.Int("min-excess", acfg.MinExcess, "minimum paging excess to report a spike")
|
||||
multiple := flag.Float64("multiple", acfg.Multiple, "pattern must be novel or >= multiple * its baseline")
|
||||
lag := flag.Duration("lag", acfg.Lag, "ingest settle time: windows end this long before now")
|
||||
baseURL := flag.String("base-url", acfg.BaseURL, "UI base URL for issue links, e.g. https://otel.k6n.net")
|
||||
title := flag.String("title", "otel issues", "dashboard header title")
|
||||
once := flag.Bool("once", false, "scan once, print the dashboard to stdout and exit")
|
||||
flag.Parse()
|
||||
|
||||
sc, err := newClickHouseScanner(*addr, *database, *user, *password, features.SpikeConfig{
|
||||
Window: *window, BaselineWindows: *baselineWindows, Multiple: *multiple, MinExcess: *minExcess,
|
||||
}, *cooldown, *lag)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer sc.close()
|
||||
|
||||
if *once {
|
||||
snap := sc.Scan(context.Background(), time.Now())
|
||||
if snap.Err != nil {
|
||||
fmt.Fprintln(os.Stderr, "issues-tui:", snap.Err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println(renderSnapshot(snap, snapshotOptions{Width: 200, Height: 12, Title: *title, BaseURL: *baseURL}))
|
||||
return
|
||||
}
|
||||
|
||||
p := tea.NewProgram(newModel(sc, modelConfig{
|
||||
Interval: *interval, Title: *title, BaseURL: *baseURL,
|
||||
}), tea.WithAltScreen())
|
||||
if _, err := p.Run(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func envOr(k, f string) string {
|
||||
if v := os.Getenv(k); v != "" {
|
||||
return v
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// scanner re-evaluates the notification pipeline on demand. All state
|
||||
// (cooldown suppressor, since an issue first fired) lives here, so Scan can
|
||||
// be called from the TUI, from -once mode or from tests with the same
|
||||
// behavior.
|
||||
type scanner struct {
|
||||
load alerts.Loader
|
||||
dataSince func(ctx context.Context) time.Time
|
||||
close func()
|
||||
cfg features.SpikeConfig
|
||||
lag time.Duration
|
||||
pager *features.Suppressor
|
||||
|
||||
// active maps an issue key to the window end when it was first seen
|
||||
// firing; it mirrors what a person watching notifications would remember.
|
||||
active map[string]time.Time
|
||||
|
||||
since time.Time
|
||||
sinceAt time.Time
|
||||
}
|
||||
|
||||
func newClickHouseScanner(addr, database, user, password string, cfg features.SpikeConfig, cooldown, lag time.Duration) (*scanner, error) {
|
||||
conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{addr}, Auth: clickhouse.Auth{Database: database, Username: user, Password: password}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &scanner{
|
||||
load: func(ctx context.Context, start, end time.Time) ([]features.Event, error) {
|
||||
return alerts.LoadErrorEvents(ctx, conn, database, start, end)
|
||||
},
|
||||
dataSince: func(ctx context.Context) time.Time { return alerts.DataSince(ctx, conn, database) },
|
||||
close: func() { _ = conn.Close() },
|
||||
cfg: cfg,
|
||||
lag: lag,
|
||||
pager: features.NewSuppressor(cooldown),
|
||||
active: map[string]time.Time{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// issueView is one card on the dashboard: a firing spike reduced to an Alert
|
||||
// (the same model the notifications carry) plus display state.
|
||||
type issueView struct {
|
||||
alert alerts.Alert
|
||||
key string
|
||||
paged bool // this scan would have notified (new or escalated)
|
||||
firstSeen time.Time // window end when this issue first appeared
|
||||
}
|
||||
|
||||
// recoveredIssue is an issue that fired on the previous scan but not now.
|
||||
type recoveredIssue struct {
|
||||
Service string
|
||||
FirstSeen time.Time
|
||||
LastSeen time.Time
|
||||
}
|
||||
|
||||
// snapshot is one scan's result: everything firing right now.
|
||||
type snapshot struct {
|
||||
ScannedAt time.Time
|
||||
WindowEnd time.Time
|
||||
WindowStart time.Time
|
||||
Events int
|
||||
Issues []issueView
|
||||
Recovered []recoveredIssue
|
||||
Err error
|
||||
}
|
||||
|
||||
// spikeKey identifies an issue across scans: a service plus the set of
|
||||
// patterns currently firing for it.
|
||||
func spikeKey(s features.Spike) string {
|
||||
sigs := make([]string, 0, len(s.Patterns))
|
||||
for _, p := range s.Patterns {
|
||||
sigs = append(sigs, p.Signature)
|
||||
}
|
||||
return s.Service + "\x00" + strings.Join(sigs, "\x00")
|
||||
}
|
||||
|
||||
// Scan runs one evaluation, mirroring the runner's Tick: load the window
|
||||
// ending lag ago plus its baseline, detect exactly what the server pages on,
|
||||
// and use a cooldown suppressor to tell a fresh page from an ongoing one.
|
||||
func (s *scanner) Scan(ctx context.Context, now time.Time) snapshot {
|
||||
cctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
end := now.Add(-s.lag).UTC()
|
||||
start := end.Add(-time.Duration(s.cfg.BaselineWindows+1) * s.cfg.Window)
|
||||
snap := snapshot{ScannedAt: now.UTC(), WindowEnd: end, WindowStart: start}
|
||||
|
||||
if s.dataSince != nil && (s.since.IsZero() || now.Sub(s.sinceAt) > time.Hour) {
|
||||
// Mirrors the runner: refresh where data begins at most hourly, so
|
||||
// the warmup guard does not suppress alerts after a quiet period.
|
||||
if ds := s.dataSince(cctx); !ds.IsZero() {
|
||||
s.since, s.sinceAt, s.cfg.DataSince = ds, now, ds
|
||||
}
|
||||
}
|
||||
|
||||
events, err := s.load(cctx, start, end)
|
||||
if err != nil {
|
||||
snap.Err = err
|
||||
return snap
|
||||
}
|
||||
snap.Events = len(events)
|
||||
|
||||
firing := alerts.DetectAlerts(events, end, s.cfg)
|
||||
// The suppressor mutates only its own state and the copies Filter makes,
|
||||
// so firing keeps every pattern for display while paged marks what would
|
||||
// have been sent to Telegram / the webhook.
|
||||
pagedKeys := map[string]bool{}
|
||||
for _, sp := range s.pager.Filter(firing) {
|
||||
pagedKeys[spikeKey(sp)] = true
|
||||
}
|
||||
|
||||
nowActive := map[string]bool{}
|
||||
for _, sp := range firing {
|
||||
key := spikeKey(sp)
|
||||
nowActive[key] = true
|
||||
first, ok := s.active[key]
|
||||
if !ok {
|
||||
first = end
|
||||
s.active[key] = first
|
||||
}
|
||||
snap.Issues = append(snap.Issues, issueView{alert: alerts.NewAlert(sp, end), key: key, paged: pagedKeys[key], firstSeen: first})
|
||||
}
|
||||
for key, first := range s.active {
|
||||
if !nowActive[key] {
|
||||
delete(s.active, key)
|
||||
snap.Recovered = append(snap.Recovered, recoveredIssue{Service: serviceOfKey(key), FirstSeen: first, LastSeen: end})
|
||||
}
|
||||
}
|
||||
sortIssues(snap.Issues)
|
||||
return snap
|
||||
}
|
||||
|
||||
func serviceOfKey(key string) string {
|
||||
service, _, _ := strings.Cut(key, "\x00")
|
||||
return service
|
||||
}
|
||||
|
||||
// sortIssues puts pages first, then the most urgent and largest spikes - the
|
||||
// order a person should read the display in.
|
||||
func sortIssues(issues []issueView) {
|
||||
// insertion sort keeps it stable without pulling in sort.Slice churn for
|
||||
// the tiny slices a dashboard shows.
|
||||
for i := 1; i < len(issues); i++ {
|
||||
for j := i; j > 0 && issueLess(issues[j], issues[j-1]); j-- {
|
||||
issues[j], issues[j-1] = issues[j-1], issues[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func issueLess(a, b issueView) bool {
|
||||
if a.paged != b.paged {
|
||||
return a.paged
|
||||
}
|
||||
ra, rb := prioRank(a.alert.Priority()), prioRank(b.alert.Priority())
|
||||
if ra != rb {
|
||||
return ra < rb
|
||||
}
|
||||
return a.alert.Spike.Excess > b.alert.Spike.Excess
|
||||
}
|
||||
|
||||
func prioRank(p alerts.Priority) int {
|
||||
switch p {
|
||||
case alerts.PriorityHigh:
|
||||
return 0
|
||||
case alerts.PriorityMedium:
|
||||
return 1
|
||||
}
|
||||
return 2
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
|
||||
"git.tornberg.me/go-otel/internal/alerts"
|
||||
"git.tornberg.me/go-otel/internal/features"
|
||||
)
|
||||
|
||||
// fakeClock builds a scanner with a canned loader and no ClickHouse.
|
||||
func fakeScanner(events []features.Event, dataSince time.Time) *scanner {
|
||||
return &scanner{
|
||||
load: func(context.Context, time.Time, time.Time) ([]features.Event, error) { return events, nil },
|
||||
dataSince: func(context.Context) time.Time {
|
||||
return dataSince
|
||||
},
|
||||
close: func() {},
|
||||
cfg: features.SpikeConfig{Window: time.Minute, BaselineWindows: 2, Multiple: 3, MinExcess: 5},
|
||||
pager: features.NewSuppressor(30 * time.Minute),
|
||||
active: map[string]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
func errs(n int, at time.Time) []features.Event {
|
||||
events := make([]features.Event, 0, n)
|
||||
for i := 0; i < n; i++ {
|
||||
events = append(events, features.Event{
|
||||
Timestamp: at,
|
||||
Service: "checkout",
|
||||
Kind: "span",
|
||||
IsError: true,
|
||||
StatusCode: 500,
|
||||
Route: "/api/payments",
|
||||
SpanName: "POST /api/payments",
|
||||
Message: "payment gateway timeout",
|
||||
PatternSignature: "checkout|500|/api/payments",
|
||||
IssueFingerprint: "issue-checkout-500",
|
||||
})
|
||||
}
|
||||
return events
|
||||
}
|
||||
|
||||
// TestScannerPagesOnceThenOngoing mirrors the notification lifecycle: the
|
||||
// first window past the threshold pages, later windows are ongoing until the
|
||||
// count doubles (escalation) or the spike disappears.
|
||||
func TestScannerPagesOnceThenOngoing(t *testing.T) {
|
||||
base := time.Date(2025, 6, 1, 12, 0, 0, 0, time.UTC)
|
||||
sc := fakeScanner(nil, base.Add(-time.Hour))
|
||||
|
||||
// The loader places the events inside whatever window is being
|
||||
// evaluated, so every scan sees a live spike in its current window.
|
||||
count := 12
|
||||
sc.load = func(_ context.Context, _, end time.Time) ([]features.Event, error) {
|
||||
return errs(count, end.Add(-30*time.Second)), nil
|
||||
}
|
||||
|
||||
first := sc.Scan(context.Background(), base)
|
||||
if len(first.Issues) != 1 || !first.Issues[0].paged {
|
||||
t.Fatalf("first scan should page one issue, got %+v", first.Issues)
|
||||
}
|
||||
if first.Issues[0].alert.Priority() != alerts.PriorityHigh {
|
||||
t.Fatalf("500 on a span should be high priority, got %s", first.Issues[0].alert.Priority())
|
||||
}
|
||||
if got := first.Issues[0].firstSeen; !got.Equal(first.WindowEnd) {
|
||||
t.Fatalf("firstSeen should be the first window end, got %v", got)
|
||||
}
|
||||
|
||||
// A little more traffic, still not double: ongoing, not paged again.
|
||||
count = 14
|
||||
second := sc.Scan(context.Background(), base.Add(time.Minute))
|
||||
if len(second.Issues) != 1 || second.Issues[0].paged {
|
||||
t.Fatalf("second scan should be ongoing, got %+v", second.Issues)
|
||||
}
|
||||
if got := second.Issues[0].firstSeen; !got.Equal(first.WindowEnd) {
|
||||
t.Fatalf("ongoing issue should keep its first-seen time, got %v want %v", got, first.WindowEnd)
|
||||
}
|
||||
|
||||
// Escalation: at least double the count pages again.
|
||||
count = 30
|
||||
third := sc.Scan(context.Background(), base.Add(2*time.Minute))
|
||||
if len(third.Issues) != 1 || !third.Issues[0].paged {
|
||||
t.Fatalf("doubled count should page again, got %+v", third.Issues)
|
||||
}
|
||||
|
||||
// Recovery: the spike disappears and is reported once.
|
||||
sc.load = func(_ context.Context, _, _ time.Time) ([]features.Event, error) { return nil, nil }
|
||||
fourth := sc.Scan(context.Background(), base.Add(3*time.Minute))
|
||||
if len(fourth.Issues) != 0 {
|
||||
t.Fatalf("no events should mean no issues, got %+v", fourth.Issues)
|
||||
}
|
||||
if len(fourth.Recovered) != 1 || fourth.Recovered[0].Service != "checkout" {
|
||||
t.Fatalf("expected checkout in recovered, got %+v", fourth.Recovered)
|
||||
}
|
||||
again := sc.Scan(context.Background(), base.Add(4*time.Minute))
|
||||
if len(again.Recovered) != 0 {
|
||||
t.Fatalf("recovery should be reported once, got %+v", again.Recovered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerAllClearAndError(t *testing.T) {
|
||||
sc := fakeScanner(nil, time.Time{})
|
||||
empty := sc.Scan(context.Background(), time.Now())
|
||||
if len(empty.Issues) != 0 || empty.Err != nil {
|
||||
t.Fatalf("empty loader should yield a clean snapshot, got %+v", empty)
|
||||
}
|
||||
|
||||
sc.load = func(context.Context, time.Time, time.Time) ([]features.Event, error) {
|
||||
return nil, context.DeadlineExceeded
|
||||
}
|
||||
failed := sc.Scan(context.Background(), time.Now())
|
||||
if failed.Err == nil {
|
||||
t.Fatal("loader error should surface in the snapshot")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGridLayoutFillsWideShortDisplays(t *testing.T) {
|
||||
// 300 columns, one card row of height: six issues, one row of wide cards.
|
||||
perRow, rows, shown, cardW := gridLayout(6, 300, 8)
|
||||
if perRow != 6 || rows != 1 || shown != 6 {
|
||||
t.Fatalf("want all 6 on one row, got perRow=%d rows=%d shown=%d", perRow, rows, shown)
|
||||
}
|
||||
if cardW < cardMinWidth || cardW > cardMaxWidth {
|
||||
t.Fatalf("card width %d out of range", cardW)
|
||||
}
|
||||
|
||||
// More issues than fit: cap at one row and let the footer count the rest.
|
||||
perRow, rows, shown, _ = gridLayout(20, 300, 8)
|
||||
if rows != 1 || shown != perRow {
|
||||
t.Fatalf("want a single row capped at width, got rows=%d shown=%d perRow=%d", rows, shown, perRow)
|
||||
}
|
||||
|
||||
// A taller display wraps to a second row.
|
||||
_, rows, shown, _ = gridLayout(20, 300, 15)
|
||||
if rows != 2 {
|
||||
t.Fatalf("15 rows of height should fit two card rows, got %d", rows)
|
||||
}
|
||||
if shown != perRow*2 {
|
||||
t.Fatalf("shown %d should fill two rows of %d", shown, perRow)
|
||||
}
|
||||
|
||||
// Narrow display: at least one card still renders.
|
||||
perRow, rows, shown, cardW = gridLayout(3, 36, 8)
|
||||
if perRow < 1 || rows != 1 || shown != 1 || cardW < cardMinWidth {
|
||||
t.Fatalf("narrow display should show one card, got perRow=%d rows=%d shown=%d cardW=%d", perRow, rows, shown, cardW)
|
||||
}
|
||||
}
|
||||
|
||||
func cardAt(t *testing.T, n int) (snapshot, snapshotOptions) {
|
||||
t.Helper()
|
||||
now := time.Date(2025, 6, 1, 12, 45, 0, 0, time.UTC)
|
||||
sp := features.Spike{
|
||||
Service: "checkout", WindowStart: now.Add(-5 * time.Minute), WindowEnd: now,
|
||||
ErrorCount: 38, BaselineMean: 0.3, Excess: 37.7,
|
||||
Patterns: []features.PatternDelta{{
|
||||
Signature: "checkout|500|/api/payments", Count: 38, Excess: 37.7, Novel: true, Clients: 12,
|
||||
Services: []string{"checkout", "payments-db"},
|
||||
Example: features.Event{
|
||||
Service: "checkout", Kind: "span", StatusCode: 500, Route: "/api/payments",
|
||||
SpanName: "POST /api/payments", Message: "payment gateway timeout",
|
||||
IssueFingerprint: "abcdef1234567890",
|
||||
},
|
||||
}},
|
||||
}
|
||||
snap := snapshot{ScannedAt: now, WindowStart: sp.WindowStart, WindowEnd: now, Events: 8213,
|
||||
Issues: make([]issueView, n)}
|
||||
for i := range snap.Issues {
|
||||
snap.Issues[i] = issueView{alert: alerts.NewAlert(sp, now), key: sp.Service, paged: i == 0, firstSeen: now.Add(-12 * time.Minute)}
|
||||
}
|
||||
return snap, snapshotOptions{Width: 200, Height: 12, Title: "otel issues", BaseURL: "https://otel.k6n.net",
|
||||
NextScanAt: now.Add(30 * time.Second), Now: now}
|
||||
}
|
||||
|
||||
func TestRenderSnapshotShowsCards(t *testing.T) {
|
||||
snap, o := cardAt(t, 8)
|
||||
out := renderSnapshot(snap, o)
|
||||
if !strings.Contains(out, "PAGED") || !strings.Contains(out, "checkout") {
|
||||
t.Fatalf("dashboard should show the paged checkout card:\n%s", out)
|
||||
}
|
||||
if !strings.Contains(out, "HTTP 500") {
|
||||
t.Fatalf("card should describe the failure:\n%s", out)
|
||||
}
|
||||
if lines := strings.Split(out, "\n"); len(lines) > 12 {
|
||||
t.Fatalf("dashboard must respect the height budget, got %d lines", len(lines))
|
||||
}
|
||||
|
||||
// The footer counts what the single card row could not show: five cards
|
||||
// fit in 200 columns, three are hidden.
|
||||
if !strings.Contains(out, "+3 more") {
|
||||
t.Fatalf("footer should count hidden cards:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderSnapshotAllClear(t *testing.T) {
|
||||
now := time.Date(2025, 6, 1, 12, 45, 0, 0, time.UTC)
|
||||
snap := snapshot{ScannedAt: now, WindowStart: now.Add(-5 * time.Minute), WindowEnd: now, Events: 8213}
|
||||
out := renderSnapshot(snap, snapshotOptions{Width: 200, Height: 12, Title: "otel issues", Now: now})
|
||||
if !strings.Contains(out, "ALL CLEAR") {
|
||||
t.Fatalf("empty scan should show all clear:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderSnapshotScanError(t *testing.T) {
|
||||
snap := snapshot{Err: context.DeadlineExceeded}
|
||||
out := renderSnapshot(snap, snapshotOptions{Width: 200, Height: 12, Title: "otel issues", Now: time.Now()})
|
||||
if !strings.Contains(out, "SCAN FAILED") {
|
||||
t.Fatalf("failed scan should be visible:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortIssuesPagedFirst(t *testing.T) {
|
||||
now := time.Now()
|
||||
mk := func(service string, paged bool, prio alerts.Priority, excess float64) issueView {
|
||||
pat := features.PatternDelta{Example: features.Event{Service: service, StatusCode: 500}}
|
||||
if prio == alerts.PriorityLow {
|
||||
pat.Example.StatusCode = 404
|
||||
}
|
||||
return issueView{paged: paged, alert: alerts.NewAlert(features.Spike{
|
||||
Service: service, Excess: excess, Patterns: []features.PatternDelta{pat},
|
||||
}, now)}
|
||||
}
|
||||
issues := []issueView{
|
||||
mk("quiet", false, alerts.PriorityHigh, 40),
|
||||
mk("loud", true, alerts.PriorityLow, 10),
|
||||
mk("burning", false, alerts.PriorityHigh, 99),
|
||||
}
|
||||
sortIssues(issues)
|
||||
if issues[0].alert.Spike.Service != "loud" {
|
||||
t.Fatalf("paged issue should come first, got %s", issues[0].alert.Spike.Service)
|
||||
}
|
||||
if issues[1].alert.Spike.Service != "burning" || issues[2].alert.Spike.Service != "quiet" {
|
||||
t.Fatalf("then by priority and excess, got %s, %s", issues[1].alert.Spike.Service, issues[2].alert.Spike.Service)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBarAndMinimal(t *testing.T) {
|
||||
out := bar(40, "left", "right")
|
||||
if lipgloss.Width(out) != 40 {
|
||||
t.Fatalf("bar should pad to width, got %q (%d)", out, lipgloss.Width(out))
|
||||
}
|
||||
if out2 := bar(8, "left", "right"); lipgloss.Width(out2) > 8 {
|
||||
t.Fatalf("bar should truncate when both sides do not fit, got %q", out2)
|
||||
}
|
||||
|
||||
snap := snapshot{Err: context.DeadlineExceeded}
|
||||
if got := renderMinimal(snap, snapshotOptions{}); !strings.Contains(got, "SCAN FAILED") {
|
||||
t.Fatalf("minimal render should surface scan errors, got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
)
|
||||
|
||||
// modelConfig is the static configuration the dashboard needs beyond the
|
||||
// scanner.
|
||||
type modelConfig struct {
|
||||
Interval time.Duration
|
||||
Title string
|
||||
BaseURL string
|
||||
}
|
||||
|
||||
// tickMsg drives the clock and the scan countdown.
|
||||
type tickMsg time.Time
|
||||
|
||||
// snapshotMsg carries a finished scan back into the model.
|
||||
type snapshotMsg snapshot
|
||||
|
||||
type model struct {
|
||||
sc *scanner
|
||||
cfg modelConfig
|
||||
snap snapshot
|
||||
|
||||
scanning bool
|
||||
nextScanAt time.Time
|
||||
|
||||
width, height int
|
||||
ready bool
|
||||
}
|
||||
|
||||
func newModel(sc *scanner, cfg modelConfig) model {
|
||||
return model{sc: sc, cfg: cfg, nextScanAt: time.Now()}
|
||||
}
|
||||
|
||||
func (m model) Init() tea.Cmd {
|
||||
return tea.Batch(tickClock(), m.scanCmd())
|
||||
}
|
||||
|
||||
func tickClock() tea.Cmd {
|
||||
return tea.Tick(time.Second, func(t time.Time) tea.Msg { return tickMsg(t) })
|
||||
}
|
||||
|
||||
func (m *model) scanCmd() tea.Cmd {
|
||||
sc := m.sc
|
||||
return func() tea.Msg { return snapshotMsg(sc.Scan(context.Background(), time.Now())) }
|
||||
}
|
||||
|
||||
func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
switch msg := msg.(type) {
|
||||
case tea.WindowSizeMsg:
|
||||
m.width, m.height, m.ready = msg.Width, msg.Height, true
|
||||
return m, nil
|
||||
|
||||
case tea.KeyMsg:
|
||||
switch msg.String() {
|
||||
case "q", "esc", "ctrl+c":
|
||||
return m, tea.Quit
|
||||
case "r":
|
||||
if !m.scanning {
|
||||
m.scanning = true
|
||||
return m, m.scanCmd()
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
return m, nil
|
||||
|
||||
case tickMsg:
|
||||
now := time.Time(msg)
|
||||
cmd := tickClock()
|
||||
if !m.scanning && !now.Before(m.nextScanAt) {
|
||||
m.scanning = true
|
||||
return m, tea.Batch(cmd, m.scanCmd())
|
||||
}
|
||||
return m, cmd
|
||||
|
||||
case snapshotMsg:
|
||||
m.snap = snapshot(msg)
|
||||
m.scanning = false
|
||||
m.nextScanAt = time.Now().Add(m.cfg.Interval)
|
||||
return m, nil
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m model) View() string {
|
||||
return renderSnapshot(m.snap, snapshotOptions{
|
||||
Width: m.width, Height: m.height, Title: m.cfg.Title, BaseURL: m.cfg.BaseURL,
|
||||
Scanning: m.scanning, NextScanAt: m.nextScanAt, Now: time.Now(),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,394 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
"github.com/charmbracelet/x/ansi"
|
||||
|
||||
"git.tornberg.me/go-otel/internal/alerts"
|
||||
)
|
||||
|
||||
// Layout budget. The display this dashboard targets is much wider than it is
|
||||
// tall, so cards flow left-to-right in one row first; a second row only
|
||||
// appears when the terminal is tall enough.
|
||||
const (
|
||||
cardContentLines = 4 // priority/what/stats/meta inside the border
|
||||
cardHeight = cardContentLines + 2 // border rows included
|
||||
cardGap = 1 // columns between cards
|
||||
cardMinWidth = 34
|
||||
cardMaxWidth = 64
|
||||
minUsableWidth = 40
|
||||
minUsableHeight = 3
|
||||
footerHint = "q quit · r scan now"
|
||||
)
|
||||
|
||||
// snapshotOptions carries everything the renderer needs beyond the scan.
|
||||
type snapshotOptions struct {
|
||||
Width, Height int
|
||||
Title string
|
||||
BaseURL string
|
||||
Scanning bool
|
||||
NextScanAt time.Time
|
||||
Now time.Time
|
||||
}
|
||||
|
||||
// Colors mirror the otel-ui design-system tokens (AGENTS.md) so the terminal
|
||||
// dashboard and the web UI read as one system: neutral chrome, color only as
|
||||
// a severity signal.
|
||||
var (
|
||||
colChrome = lipgloss.AdaptiveColor{Light: "#FFFFFF", Dark: "#0B0E12"}
|
||||
colSurface = lipgloss.AdaptiveColor{Light: "#FFFFFF", Dark: "#14181E"}
|
||||
colLine = lipgloss.AdaptiveColor{Light: "#E3E0D8", Dark: "#242B34"}
|
||||
colText = lipgloss.AdaptiveColor{Light: "#161A1F", Dark: "#E8ECF2"}
|
||||
colText2 = lipgloss.AdaptiveColor{Light: "#4F5964", Dark: "#98A3B2"}
|
||||
colText3 = lipgloss.AdaptiveColor{Light: "#6B7480", Dark: "#8B96A6"}
|
||||
colCritical = lipgloss.AdaptiveColor{Light: "#C4402C", Dark: "#FF6F5E"}
|
||||
colWarn = lipgloss.AdaptiveColor{Light: "#8A5D0E", Dark: "#F0B24A"}
|
||||
colOK = lipgloss.AdaptiveColor{Light: "#1F7A56", Dark: "#4FCF9A"}
|
||||
colAccent = lipgloss.AdaptiveColor{Light: "#3B4FC4", Dark: "#7B8CFF"}
|
||||
)
|
||||
|
||||
func priorityColor(p alerts.Priority) lipgloss.TerminalColor {
|
||||
switch p {
|
||||
case alerts.PriorityHigh:
|
||||
return colCritical
|
||||
case alerts.PriorityMedium:
|
||||
return colWarn
|
||||
}
|
||||
return colText3
|
||||
}
|
||||
|
||||
// renderSnapshot draws the whole dashboard: one header line, the card grid,
|
||||
// one footer line. It is shared by the live TUI and -once mode.
|
||||
func renderSnapshot(snap snapshot, o snapshotOptions) string {
|
||||
w, h := o.Width, o.Height
|
||||
if w <= 0 {
|
||||
w = 80
|
||||
}
|
||||
if h <= 0 {
|
||||
h = 8
|
||||
}
|
||||
if w < minUsableWidth || h < minUsableHeight {
|
||||
return renderMinimal(snap, o)
|
||||
}
|
||||
|
||||
header := renderHeader(snap, o)
|
||||
footer := renderFooter(snap, o)
|
||||
body := renderBody(snap, o, w, h-2)
|
||||
return strings.Join([]string{header, body, footer}, "\n")
|
||||
}
|
||||
|
||||
func renderBody(snap snapshot, o snapshotOptions, w, avail int) string {
|
||||
switch {
|
||||
case snap.Err != nil:
|
||||
return statusPanel(w, min(avail, 5), colCritical, "SCAN FAILED", snap.Err.Error(),
|
||||
"values are from "+fmtTime(snap.ScannedAt)+"; the display keeps the last good scan")
|
||||
case len(snap.Issues) == 0:
|
||||
sub := fmt.Sprintf("no issues firing · window %s–%s UTC · scanned %s",
|
||||
snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"), commas(snap.Events))
|
||||
if snap.ScannedAt.IsZero() {
|
||||
sub = "waiting for the first scan…"
|
||||
}
|
||||
return statusPanel(w, min(avail, 5), colOK, "ALL CLEAR", sub, "")
|
||||
default:
|
||||
return renderGrid(snap, o, w, avail)
|
||||
}
|
||||
}
|
||||
|
||||
// renderGrid lays issues into rows of cards that fill the width; only as many
|
||||
// rows as the height allows are drawn, the rest are counted in the footer.
|
||||
func renderGrid(snap snapshot, o snapshotOptions, w, avail int) string {
|
||||
perRow, _, shown, cardW := gridLayout(len(snap.Issues), w, avail)
|
||||
var rendered []string
|
||||
for start := 0; start < shown; start += perRow {
|
||||
end := min(start+perRow, shown)
|
||||
var row []string
|
||||
for _, v := range snap.Issues[start:end] {
|
||||
row = append(row, issueCard(v, cardW, o))
|
||||
}
|
||||
rendered = append(rendered, lipgloss.JoinHorizontal(lipgloss.Top, row...))
|
||||
}
|
||||
grid := strings.Join(rendered, "\n")
|
||||
if lipgloss.Width(grid) < w {
|
||||
grid = lipgloss.PlaceHorizontal(w, lipgloss.Center, grid)
|
||||
}
|
||||
return grid
|
||||
}
|
||||
|
||||
// gridLayout fits n cards into the width and at most `avail` height. It
|
||||
// returns cards per row, rows drawn, how many cards that shows, and the card
|
||||
// width; whatever does not fit is reported by the caller as hidden.
|
||||
func gridLayout(n, width, avail int) (perRow, rows, shown, cardW int) {
|
||||
if n <= 0 {
|
||||
return 0, 0, 0, 0
|
||||
}
|
||||
maxRows := max(1, (avail+cardGap)/(cardHeight+cardGap))
|
||||
perRow = max(1, (width+cardGap)/(cardMinWidth+cardGap))
|
||||
perRow = min(perRow, n)
|
||||
rows = min(maxRows, (n+perRow-1)/perRow)
|
||||
shown = min(n, rows*perRow)
|
||||
cardW = min(cardMaxWidth, (width-cardGap*(perRow-1))/perRow)
|
||||
return perRow, rows, shown, cardW
|
||||
}
|
||||
|
||||
// issueCard renders one issue as a bordered card of exactly cardHeight lines.
|
||||
// Line 1 identifies it (priority, service, age), line 2 is what is failing,
|
||||
// line 3 the numbers behind it, line 4 the display state and issue link.
|
||||
func issueCard(v issueView, w int, o snapshotOptions) string {
|
||||
a := v.alert
|
||||
fg := priorityColor(a.Priority())
|
||||
inner := w - 4 // border plus padding on both sides
|
||||
|
||||
// Line 1: HIGH · checkout · for 12m
|
||||
age := ""
|
||||
if !v.firstSeen.IsZero() && !o.Now.IsZero() {
|
||||
age = " · for " + shortDur(o.Now.Sub(v.firstSeen).Round(time.Minute))
|
||||
}
|
||||
head := ansi.Truncate(
|
||||
style(fg, true).Render(strings.ToUpper(string(a.Priority())))+
|
||||
style(colText, true).Render(" · "+a.Spike.Service)+
|
||||
style(colText3, false).Render(age), inner, "…")
|
||||
|
||||
// Line 2: what is failing - the top pattern's description, then any
|
||||
// additional pattern count.
|
||||
what := fmt.Sprintf("%d error(s), no pattern detail", a.Spike.ErrorCount)
|
||||
if len(a.Patterns) > 0 {
|
||||
what = a.Patterns[0].What
|
||||
if extra := len(a.Patterns) - 1; extra > 0 {
|
||||
what += fmt.Sprintf(" (+%d more pattern%s)", extra, plural(extra))
|
||||
}
|
||||
}
|
||||
whatLine := style(colText, false).Render(ansi.Truncate(what, inner, "…"))
|
||||
|
||||
// Line 3: the detector's numbers: count vs baseline, excess, and the
|
||||
// top pattern's tags.
|
||||
segs := []string{style(colText2, false).Render(
|
||||
fmt.Sprintf("%d err vs %.1f/w · excess +%.0f", a.Spike.ErrorCount, a.Spike.BaselineMean, a.Spike.Excess))}
|
||||
var plain []string
|
||||
if len(a.Patterns) > 0 {
|
||||
top := a.Patterns[0]
|
||||
if top.New {
|
||||
segs = append(segs, style(colAccent, true).Render("NEW"))
|
||||
}
|
||||
if top.Clients > 0 {
|
||||
plain = append(plain, fmt.Sprintf("%d client%s", top.Clients, plural(top.Clients)))
|
||||
}
|
||||
if len(top.Services) > 1 {
|
||||
plain = append(plain, "with: "+strings.Join(top.Services, ","))
|
||||
}
|
||||
}
|
||||
if len(plain) > 0 {
|
||||
segs = append(segs, style(colText3, false).Render(strings.Join(plain, " · ")))
|
||||
}
|
||||
statsLine := ansi.Truncate(strings.Join(segs, " "), inner, "…")
|
||||
|
||||
// Line 4: display state (paged now vs already paged) and the issue link.
|
||||
state := style(colText3, false).Render("ongoing")
|
||||
if v.paged {
|
||||
state = style(fg, true).Render("PAGED")
|
||||
}
|
||||
var meta []string
|
||||
if !a.Spike.WindowEnd.IsZero() {
|
||||
meta = append(meta, style(colText3, false).Render(fmtTime(a.Spike.WindowEnd)))
|
||||
}
|
||||
meta = append(meta, state)
|
||||
if o.BaseURL != "" && len(a.Patterns) > 0 && a.Patterns[0].IssueID != "" {
|
||||
meta = append(meta, style(colAccent, false).Render("⧉ "+shortID(a.Patterns[0].IssueID)))
|
||||
}
|
||||
metaLine := ansi.Truncate(strings.Join(meta, " "), inner, "…")
|
||||
|
||||
return lipgloss.NewStyle().
|
||||
Border(lipgloss.RoundedBorder()).
|
||||
BorderForeground(fg).
|
||||
Background(colSurface).
|
||||
Padding(0, 1).
|
||||
Width(w - 2).
|
||||
Height(cardContentLines).
|
||||
Render(strings.Join([]string{head, whatLine, statsLine, metaLine}, "\n"))
|
||||
}
|
||||
|
||||
// statusPanel is the single wide panel used for ALL CLEAR and scan failures.
|
||||
func statusPanel(w, h int, border lipgloss.TerminalColor, title, line1, line2 string) string {
|
||||
inner := max(w-6, 10)
|
||||
lines := []string{
|
||||
style(border, true).Render(ansi.Truncate(title, inner, "…")),
|
||||
style(colText, false).Render(ansi.Truncate(line1, inner, "…")),
|
||||
}
|
||||
if line2 != "" {
|
||||
lines = append(lines, style(colText3, false).Render(ansi.Truncate(line2, inner, "…")))
|
||||
}
|
||||
return lipgloss.Place(w, h, lipgloss.Center, lipgloss.Center,
|
||||
lipgloss.NewStyle().Border(lipgloss.RoundedBorder()).BorderForeground(border).
|
||||
Background(colSurface).Padding(0, 2).Width(inner).
|
||||
Render(strings.Join(lines, "\n")))
|
||||
}
|
||||
|
||||
// renderHeader draws the one-line status bar: title, scan window, counts,
|
||||
// clock and next-scan countdown, right-aligned.
|
||||
func renderHeader(snap snapshot, o snapshotOptions) string {
|
||||
w := o.Width - 2
|
||||
title := style(colAccent, true).Render(strings.ToUpper(o.Title))
|
||||
|
||||
left := []string{title}
|
||||
if !snap.WindowEnd.IsZero() {
|
||||
left = append(left, style(colText3, false).Render(
|
||||
fmt.Sprintf("%s–%s UTC", snap.WindowStart.Format("15:04"), snap.WindowEnd.Format("15:04"))))
|
||||
}
|
||||
if !snap.ScannedAt.IsZero() {
|
||||
left = append(left, style(colText3, false).Render(commas(snap.Events)+" events"))
|
||||
var paged int
|
||||
for _, v := range snap.Issues {
|
||||
if v.paged {
|
||||
paged++
|
||||
}
|
||||
}
|
||||
counts := fmt.Sprintf("%d firing", len(snap.Issues))
|
||||
if paged > 0 {
|
||||
left = append(left, style(colCritical, true).Render(fmt.Sprintf("%d PAGED · %s", paged, counts)))
|
||||
} else {
|
||||
left = append(left, style(colText2, false).Render(counts))
|
||||
}
|
||||
}
|
||||
|
||||
var right []string
|
||||
if o.Scanning {
|
||||
right = append(right, style(colAccent, false).Render("scanning…"))
|
||||
} else if !o.NextScanAt.IsZero() && !o.Now.IsZero() {
|
||||
d := max(o.NextScanAt.Sub(o.Now).Round(time.Second), 0)
|
||||
right = append(right, style(colText3, false).Render("next "+shortDur(d)))
|
||||
}
|
||||
if !snap.ScannedAt.IsZero() {
|
||||
right = append(right, style(colText3, false).Render("scanned "+fmtTime(snap.ScannedAt)))
|
||||
}
|
||||
if !o.Now.IsZero() {
|
||||
right = append(right, style(colText, false).Render(o.Now.Format("15:04:05")))
|
||||
}
|
||||
return bar(w, strings.Join(left, " "), strings.Join(right, " "))
|
||||
}
|
||||
|
||||
// renderFooter draws the one-line help/status bar: keys, hidden-card count
|
||||
// and recovered issues.
|
||||
func renderFooter(snap snapshot, o snapshotOptions) string {
|
||||
var left []string
|
||||
left = append(left, style(colText3, false).Render(footerHint))
|
||||
if hidden := len(snap.Issues) - shownCount(snap, o); hidden > 0 {
|
||||
left = append(left, style(colWarn, false).Render(fmt.Sprintf("+%d more not shown (widen or raise the display)", hidden)))
|
||||
}
|
||||
var right []string
|
||||
if len(snap.Recovered) > 0 {
|
||||
names := make([]string, 0, len(snap.Recovered))
|
||||
for _, r := range snap.Recovered {
|
||||
names = append(names, r.Service)
|
||||
}
|
||||
right = append(right, style(colOK, false).Render("recovered: "+strings.Join(names, ", ")))
|
||||
}
|
||||
if o.BaseURL != "" {
|
||||
right = append(right, style(colText3, false).Render(strings.TrimRight(o.BaseURL, "/")+"/issues"))
|
||||
}
|
||||
return bar(o.Width-2, strings.Join(left, " "), strings.Join(right, " "))
|
||||
}
|
||||
|
||||
func shownCount(snap snapshot, o snapshotOptions) int {
|
||||
if o.Height < minUsableHeight {
|
||||
return len(snap.Issues)
|
||||
}
|
||||
_, _, shown, _ := gridLayout(len(snap.Issues), o.Width, o.Height-2)
|
||||
return shown
|
||||
}
|
||||
|
||||
// bar places left and right text on one full-width line, truncating when the
|
||||
// terminal is too narrow for both.
|
||||
func bar(w int, left, right string) string {
|
||||
lw, rw := lipgloss.Width(left), lipgloss.Width(right)
|
||||
if lw+rw+1 > w {
|
||||
return ansi.Truncate(left+" "+right, w, "…")
|
||||
}
|
||||
return left + strings.Repeat(" ", max(w-lw-rw, 0)) + right
|
||||
}
|
||||
|
||||
// renderMinimal is the fallback for tiny terminals and -once without a size:
|
||||
// one line per issue, no boxes.
|
||||
func renderMinimal(snap snapshot, o snapshotOptions) string {
|
||||
if snap.Err != nil {
|
||||
return "SCAN FAILED: " + snap.Err.Error()
|
||||
}
|
||||
if len(snap.Issues) == 0 {
|
||||
return strings.ToUpper(o.Title) + ": all clear"
|
||||
}
|
||||
lines := make([]string, 0, len(snap.Issues)+1)
|
||||
lines = append(lines, strings.ToUpper(o.Title)+":")
|
||||
for _, v := range snap.Issues {
|
||||
state := "ongoing"
|
||||
if v.paged {
|
||||
state = "PAGED"
|
||||
}
|
||||
what := ""
|
||||
if len(v.alert.Patterns) > 0 {
|
||||
what = v.alert.Patterns[0].What
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("%s %s %s: %s",
|
||||
state, strings.ToUpper(string(v.alert.Priority())), v.alert.Spike.Service, what))
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// --- small helpers ---------------------------------------------------------
|
||||
|
||||
func style(c lipgloss.TerminalColor, bold bool) lipgloss.Style {
|
||||
s := lipgloss.NewStyle().Foreground(c)
|
||||
if bold {
|
||||
s = s.Bold(true)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func fmtTime(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return "never"
|
||||
}
|
||||
return t.Format("15:04:05")
|
||||
}
|
||||
|
||||
func commas(n int) string {
|
||||
s := fmt.Sprintf("%d", n)
|
||||
if n < 1000 {
|
||||
return s
|
||||
}
|
||||
var parts []string
|
||||
for len(s) > 3 {
|
||||
parts = append([]string{s[len(s)-3:]}, parts...)
|
||||
s = s[:len(s)-3]
|
||||
}
|
||||
return s + "," + strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
func plural(n int) string {
|
||||
if n == 1 {
|
||||
return ""
|
||||
}
|
||||
return "s"
|
||||
}
|
||||
|
||||
// shortDur renders durations compactly: 90s -> "1m30s", 2h05m -> "2h5m".
|
||||
func shortDur(d time.Duration) string {
|
||||
s := d.Round(time.Second).String()
|
||||
if strings.HasSuffix(s, "m0s") {
|
||||
s = strings.TrimSuffix(s, "0s")
|
||||
}
|
||||
if strings.HasSuffix(s, "h0m") {
|
||||
s = strings.TrimSuffix(s, "0m")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// shortID abbreviates an issue fingerprint for the card's link chip.
|
||||
func shortID(id string) string {
|
||||
if n := utf8.RuneCountInString(id); n > 10 {
|
||||
return id[:10]
|
||||
}
|
||||
return id
|
||||
}
|
||||
Reference in new issue
Block a user