issues-tui: wide, short terminal dashboard for current issues
Standalone cmd that runs the same pipeline as the webhook notifications (alerts.LoadErrorEvents -> alerts.DetectAlerts -> cooldown suppressor) on a schedule and renders what is firing as a row of severity-colored cards - built for an ultra-wide, low display. A pattern shows PAGED when the scan would have notified and ongoing while suppressed; disappearing spikes are reported as recovered. -once prints one snapshot for cron/pipe use. Colors follow the otel-ui design-system tokens (dark and light).
This commit is contained in:
developer committed
1 parent
36f00d49b9
commit
a96e95212a
6 files changed
+1048
-78
No files matched your search
@@ -0,0 +1,249 @@
|
||||
// issues-tui is a standalone terminal dashboard for current issues. It runs
|
||||
// the exact detection pipeline behind the webhook notifications
|
||||
// (alerts.LoadErrorEvents -> alerts.DetectAlerts -> the cooldown suppressor)
|
||||
// on a schedule against ClickHouse, and renders what is firing right now as
|
||||
// wide, short cards - built for an ultra-wide, low display.
|
||||
//
|
||||
// A pattern shows as PAGED when this tick would have sent a notification
|
||||
// (new or escalated past its cooldown) and as ongoing while it is still
|
||||
// firing but already paged once. When a service's spike disappears it is
|
||||
// reported as recovered for that scan.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
|
||||
"git.tornberg.me/go-otel/internal/alerts"
|
||||
"git.tornberg.me/go-otel/internal/features"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Detection defaults come from the same environment the server's
|
||||
// alerting runner reads, so the dashboard and the notifications cannot
|
||||
// drift apart; every flag overrides.
|
||||
acfg := alerts.ConfigFromEnv()
|
||||
addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port")
|
||||
database := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "ClickHouse database")
|
||||
user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "ClickHouse user")
|
||||
password := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "ClickHouse password")
|
||||
interval := flag.Duration("interval", acfg.Interval, "how often to rescan")
|
||||
window := flag.Duration("window", acfg.Window, "observation window")
|
||||
baselineWindows := flag.Int("baseline-windows", acfg.BaselineWindows, "preceding windows used as baseline")
|
||||
cooldown := flag.Duration("cooldown", acfg.Cooldown, "a paged pattern stays ongoing this long unless it doubles")
|
||||
minExcess := flag.Int("min-excess", acfg.MinExcess, "minimum paging excess to report a spike")
|
||||
multiple := flag.Float64("multiple", acfg.Multiple, "pattern must be novel or >= multiple * its baseline")
|
||||
lag := flag.Duration("lag", acfg.Lag, "ingest settle time: windows end this long before now")
|
||||
baseURL := flag.String("base-url", acfg.BaseURL, "UI base URL for issue links, e.g. https://otel.k6n.net")
|
||||
title := flag.String("title", "otel issues", "dashboard header title")
|
||||
once := flag.Bool("once", false, "scan once, print the dashboard to stdout and exit")
|
||||
flag.Parse()
|
||||
|
||||
sc, err := newClickHouseScanner(*addr, *database, *user, *password, features.SpikeConfig{
|
||||
Window: *window, BaselineWindows: *baselineWindows, Multiple: *multiple, MinExcess: *minExcess,
|
||||
}, *cooldown, *lag)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer sc.close()
|
||||
|
||||
if *once {
|
||||
snap := sc.Scan(context.Background(), time.Now())
|
||||
if snap.Err != nil {
|
||||
fmt.Fprintln(os.Stderr, "issues-tui:", snap.Err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println(renderSnapshot(snap, snapshotOptions{Width: 200, Height: 12, Title: *title, BaseURL: *baseURL}))
|
||||
return
|
||||
}
|
||||
|
||||
p := tea.NewProgram(newModel(sc, modelConfig{
|
||||
Interval: *interval, Title: *title, BaseURL: *baseURL,
|
||||
}), tea.WithAltScreen())
|
||||
if _, err := p.Run(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func envOr(k, f string) string {
|
||||
if v := os.Getenv(k); v != "" {
|
||||
return v
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// scanner re-evaluates the notification pipeline on demand. All state
|
||||
// (cooldown suppressor, since an issue first fired) lives here, so Scan can
|
||||
// be called from the TUI, from -once mode or from tests with the same
|
||||
// behavior.
|
||||
type scanner struct {
|
||||
load alerts.Loader
|
||||
dataSince func(ctx context.Context) time.Time
|
||||
close func()
|
||||
cfg features.SpikeConfig
|
||||
lag time.Duration
|
||||
pager *features.Suppressor
|
||||
|
||||
// active maps an issue key to the window end when it was first seen
|
||||
// firing; it mirrors what a person watching notifications would remember.
|
||||
active map[string]time.Time
|
||||
|
||||
since time.Time
|
||||
sinceAt time.Time
|
||||
}
|
||||
|
||||
func newClickHouseScanner(addr, database, user, password string, cfg features.SpikeConfig, cooldown, lag time.Duration) (*scanner, error) {
|
||||
conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{addr}, Auth: clickhouse.Auth{Database: database, Username: user, Password: password}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &scanner{
|
||||
load: func(ctx context.Context, start, end time.Time) ([]features.Event, error) {
|
||||
return alerts.LoadErrorEvents(ctx, conn, database, start, end)
|
||||
},
|
||||
dataSince: func(ctx context.Context) time.Time { return alerts.DataSince(ctx, conn, database) },
|
||||
close: func() { _ = conn.Close() },
|
||||
cfg: cfg,
|
||||
lag: lag,
|
||||
pager: features.NewSuppressor(cooldown),
|
||||
active: map[string]time.Time{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// issueView is one card on the dashboard: a firing spike reduced to an Alert
|
||||
// (the same model the notifications carry) plus display state.
|
||||
type issueView struct {
|
||||
alert alerts.Alert
|
||||
key string
|
||||
paged bool // this scan would have notified (new or escalated)
|
||||
firstSeen time.Time // window end when this issue first appeared
|
||||
}
|
||||
|
||||
// recoveredIssue is an issue that fired on the previous scan but not now.
|
||||
type recoveredIssue struct {
|
||||
Service string
|
||||
FirstSeen time.Time
|
||||
LastSeen time.Time
|
||||
}
|
||||
|
||||
// snapshot is one scan's result: everything firing right now.
|
||||
type snapshot struct {
|
||||
ScannedAt time.Time
|
||||
WindowEnd time.Time
|
||||
WindowStart time.Time
|
||||
Events int
|
||||
Issues []issueView
|
||||
Recovered []recoveredIssue
|
||||
Err error
|
||||
}
|
||||
|
||||
// spikeKey identifies an issue across scans: a service plus the set of
|
||||
// patterns currently firing for it.
|
||||
func spikeKey(s features.Spike) string {
|
||||
sigs := make([]string, 0, len(s.Patterns))
|
||||
for _, p := range s.Patterns {
|
||||
sigs = append(sigs, p.Signature)
|
||||
}
|
||||
return s.Service + "\x00" + strings.Join(sigs, "\x00")
|
||||
}
|
||||
|
||||
// Scan runs one evaluation, mirroring the runner's Tick: load the window
|
||||
// ending lag ago plus its baseline, detect exactly what the server pages on,
|
||||
// and use a cooldown suppressor to tell a fresh page from an ongoing one.
|
||||
func (s *scanner) Scan(ctx context.Context, now time.Time) snapshot {
|
||||
cctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
end := now.Add(-s.lag).UTC()
|
||||
start := end.Add(-time.Duration(s.cfg.BaselineWindows+1) * s.cfg.Window)
|
||||
snap := snapshot{ScannedAt: now.UTC(), WindowEnd: end, WindowStart: start}
|
||||
|
||||
if s.dataSince != nil && (s.since.IsZero() || now.Sub(s.sinceAt) > time.Hour) {
|
||||
// Mirrors the runner: refresh where data begins at most hourly, so
|
||||
// the warmup guard does not suppress alerts after a quiet period.
|
||||
if ds := s.dataSince(cctx); !ds.IsZero() {
|
||||
s.since, s.sinceAt, s.cfg.DataSince = ds, now, ds
|
||||
}
|
||||
}
|
||||
|
||||
events, err := s.load(cctx, start, end)
|
||||
if err != nil {
|
||||
snap.Err = err
|
||||
return snap
|
||||
}
|
||||
snap.Events = len(events)
|
||||
|
||||
firing := alerts.DetectAlerts(events, end, s.cfg)
|
||||
// The suppressor mutates only its own state and the copies Filter makes,
|
||||
// so firing keeps every pattern for display while paged marks what would
|
||||
// have been sent to Telegram / the webhook.
|
||||
pagedKeys := map[string]bool{}
|
||||
for _, sp := range s.pager.Filter(firing) {
|
||||
pagedKeys[spikeKey(sp)] = true
|
||||
}
|
||||
|
||||
nowActive := map[string]bool{}
|
||||
for _, sp := range firing {
|
||||
key := spikeKey(sp)
|
||||
nowActive[key] = true
|
||||
first, ok := s.active[key]
|
||||
if !ok {
|
||||
first = end
|
||||
s.active[key] = first
|
||||
}
|
||||
snap.Issues = append(snap.Issues, issueView{alert: alerts.NewAlert(sp, end), key: key, paged: pagedKeys[key], firstSeen: first})
|
||||
}
|
||||
for key, first := range s.active {
|
||||
if !nowActive[key] {
|
||||
delete(s.active, key)
|
||||
snap.Recovered = append(snap.Recovered, recoveredIssue{Service: serviceOfKey(key), FirstSeen: first, LastSeen: end})
|
||||
}
|
||||
}
|
||||
sortIssues(snap.Issues)
|
||||
return snap
|
||||
}
|
||||
|
||||
func serviceOfKey(key string) string {
|
||||
service, _, _ := strings.Cut(key, "\x00")
|
||||
return service
|
||||
}
|
||||
|
||||
// sortIssues puts pages first, then the most urgent and largest spikes - the
|
||||
// order a person should read the display in.
|
||||
func sortIssues(issues []issueView) {
|
||||
// insertion sort keeps it stable without pulling in sort.Slice churn for
|
||||
// the tiny slices a dashboard shows.
|
||||
for i := 1; i < len(issues); i++ {
|
||||
for j := i; j > 0 && issueLess(issues[j], issues[j-1]); j-- {
|
||||
issues[j], issues[j-1] = issues[j-1], issues[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func issueLess(a, b issueView) bool {
|
||||
if a.paged != b.paged {
|
||||
return a.paged
|
||||
}
|
||||
ra, rb := prioRank(a.alert.Priority()), prioRank(b.alert.Priority())
|
||||
if ra != rb {
|
||||
return ra < rb
|
||||
}
|
||||
return a.alert.Spike.Excess > b.alert.Spike.Excess
|
||||
}
|
||||
|
||||
func prioRank(p alerts.Priority) int {
|
||||
switch p {
|
||||
case alerts.PriorityHigh:
|
||||
return 0
|
||||
case alerts.PriorityMedium:
|
||||
return 1
|
||||
}
|
||||
return 2
|
||||
}
|
||||
Reference in new issue
Block a user