// issues-tui is a standalone terminal dashboard for current issues. It runs // the exact detection pipeline behind the webhook notifications // (alerts.LoadErrorEvents -> alerts.DetectAlerts -> the cooldown suppressor) // on a schedule against ClickHouse, and renders what is firing right now as // wide, short cards - built for an ultra-wide, low display. // // A pattern shows as PAGED when this tick would have sent a notification // (new or escalated past its cooldown) and as ongoing while it is still // firing but already paged once. When a service's spike disappears it is // reported as recovered for that scan. package main import ( "context" "flag" "fmt" "log" "os" "strings" "time" "github.com/ClickHouse/clickhouse-go/v2" tea "github.com/charmbracelet/bubbletea" "git.tornberg.me/go-otel/internal/alerts" "git.tornberg.me/go-otel/internal/features" ) func main() { // Detection defaults come from the same environment the server's // alerting runner reads, so the dashboard and the notifications cannot // drift apart; every flag overrides. acfg := alerts.ConfigFromEnv() addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port") database := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "ClickHouse database") user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "ClickHouse user") password := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "ClickHouse password") interval := flag.Duration("interval", acfg.Interval, "how often to rescan") window := flag.Duration("window", acfg.Window, "observation window") baselineWindows := flag.Int("baseline-windows", acfg.BaselineWindows, "preceding windows used as baseline") cooldown := flag.Duration("cooldown", acfg.Cooldown, "a paged pattern stays ongoing this long unless it doubles") minExcess := flag.Int("min-excess", acfg.MinExcess, "minimum paging excess to report a spike") multiple := flag.Float64("multiple", acfg.Multiple, "pattern must be novel or >= multiple * its baseline") lag := flag.Duration("lag", acfg.Lag, "ingest settle time: windows end this long before now") baseURL := flag.String("base-url", acfg.BaseURL, "UI base URL for issue links, e.g. https://otel.k6n.net") title := flag.String("title", "otel issues", "dashboard header title") once := flag.Bool("once", false, "scan once, print the dashboard to stdout and exit") flag.Parse() sc, err := newClickHouseScanner(*addr, *database, *user, *password, features.SpikeConfig{ Window: *window, BaselineWindows: *baselineWindows, Multiple: *multiple, MinExcess: *minExcess, }, *cooldown, *lag) if err != nil { log.Fatal(err) } defer sc.close() if *once { snap := sc.Scan(context.Background(), time.Now()) if snap.Err != nil { fmt.Fprintln(os.Stderr, "issues-tui:", snap.Err) os.Exit(1) } fmt.Println(renderSnapshot(snap, snapshotOptions{Width: 200, Height: 12, Title: *title, BaseURL: *baseURL})) return } p := tea.NewProgram(newModel(sc, modelConfig{ Interval: *interval, Title: *title, BaseURL: *baseURL, }), tea.WithAltScreen()) if _, err := p.Run(); err != nil { log.Fatal(err) } } func envOr(k, f string) string { if v := os.Getenv(k); v != "" { return v } return f } // scanner re-evaluates the notification pipeline on demand. All state // (cooldown suppressor, since an issue first fired) lives here, so Scan can // be called from the TUI, from -once mode or from tests with the same // behavior. type scanner struct { load alerts.Loader dataSince func(ctx context.Context) time.Time close func() cfg features.SpikeConfig lag time.Duration pager *features.Suppressor // active maps an issue key to the window end when it was first seen // firing; it mirrors what a person watching notifications would remember. active map[string]time.Time since time.Time sinceAt time.Time } func newClickHouseScanner(addr, database, user, password string, cfg features.SpikeConfig, cooldown, lag time.Duration) (*scanner, error) { conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{addr}, Auth: clickhouse.Auth{Database: database, Username: user, Password: password}}) if err != nil { return nil, err } return &scanner{ load: func(ctx context.Context, start, end time.Time) ([]features.Event, error) { return alerts.LoadErrorEvents(ctx, conn, database, start, end) }, dataSince: func(ctx context.Context) time.Time { return alerts.DataSince(ctx, conn, database) }, close: func() { _ = conn.Close() }, cfg: cfg, lag: lag, pager: features.NewSuppressor(cooldown), active: map[string]time.Time{}, }, nil } // issueView is one card on the dashboard: a firing spike reduced to an Alert // (the same model the notifications carry) plus display state. type issueView struct { alert alerts.Alert key string paged bool // this scan would have notified (new or escalated) firstSeen time.Time // window end when this issue first appeared } // recoveredIssue is an issue that fired on the previous scan but not now. type recoveredIssue struct { Service string FirstSeen time.Time LastSeen time.Time } // snapshot is one scan's result: everything firing right now. type snapshot struct { ScannedAt time.Time WindowEnd time.Time WindowStart time.Time Events int Issues []issueView Recovered []recoveredIssue Err error } // spikeKey identifies an issue across scans: a service plus the set of // patterns currently firing for it. func spikeKey(s features.Spike) string { sigs := make([]string, 0, len(s.Patterns)) for _, p := range s.Patterns { sigs = append(sigs, p.Signature) } return s.Service + "\x00" + strings.Join(sigs, "\x00") } func patternKey(service, signature string) string { return service + "\x00" + signature } // Scan runs one evaluation, mirroring the runner's Tick: load the window // ending lag ago plus its baseline, detect exactly what the server pages on, // and use a cooldown suppressor to tell a fresh page from an ongoing one. func (s *scanner) Scan(ctx context.Context, now time.Time) snapshot { cctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() end := now.Add(-s.lag).UTC() start := end.Add(-time.Duration(s.cfg.BaselineWindows+1) * s.cfg.Window) snap := snapshot{ScannedAt: now.UTC(), WindowEnd: end, WindowStart: start} if s.dataSince != nil && (s.since.IsZero() || now.Sub(s.sinceAt) > time.Hour) { // Mirrors the runner: refresh where data begins at most hourly, so // the warmup guard does not suppress alerts after a quiet period. if ds := s.dataSince(cctx); !ds.IsZero() { s.since, s.sinceAt, s.cfg.DataSince = ds, now, ds } } events, err := s.load(cctx, start, end) if err != nil { snap.Err = err return snap } snap.Events = len(events) firing := alerts.DetectAlerts(events, end, s.cfg) // The suppressor mutates only its own state and the copies Filter makes, // so firing keeps every pattern for display while paged marks what would // have been sent to Telegram / the webhook. pagedPatterns := map[string]bool{} for _, sp := range s.pager.Filter(firing) { for _, p := range sp.Patterns { pagedPatterns[patternKey(sp.Service, p.Signature)] = true } } nowActive := map[string]bool{} for _, sp := range firing { key := spikeKey(sp) nowActive[key] = true first, ok := s.active[key] if !ok { first = end s.active[key] = first } paged := false for _, p := range sp.Patterns { if pagedPatterns[patternKey(sp.Service, p.Signature)] { paged = true break } } snap.Issues = append(snap.Issues, issueView{alert: alerts.NewAlert(sp, end), key: key, paged: paged, firstSeen: first}) } for key, first := range s.active { if !nowActive[key] { delete(s.active, key) snap.Recovered = append(snap.Recovered, recoveredIssue{Service: serviceOfKey(key), FirstSeen: first, LastSeen: end}) } } sortIssues(snap.Issues) return snap } func serviceOfKey(key string) string { service, _, _ := strings.Cut(key, "\x00") return service } // sortIssues puts pages first, then the most urgent and largest spikes - the // order a person should read the display in. func sortIssues(issues []issueView) { // insertion sort keeps it stable without pulling in sort.Slice churn for // the tiny slices a dashboard shows. for i := 1; i < len(issues); i++ { for j := i; j > 0 && issueLess(issues[j], issues[j-1]); j-- { issues[j], issues[j-1] = issues[j-1], issues[j] } } } func issueLess(a, b issueView) bool { if a.paged != b.paged { return a.paged } ra, rb := prioRank(a.alert.Priority()), prioRank(b.alert.Priority()) if ra != rb { return ra < rb } return a.alert.Spike.Excess > b.alert.Spike.Excess } func prioRank(p alerts.Priority) int { switch p { case alerts.PriorityHigh: return 0 case alerts.PriorityMedium: return 1 } return 2 }