289 lines
9.8 KiB
Go
289 lines
9.8 KiB
Go
// issues-tui is a standalone terminal dashboard for current issues. It runs
|
|
// the exact detection pipeline behind the webhook notifications
|
|
// (alerts.LoadErrorEvents -> alerts.DetectAlerts -> the cooldown suppressor)
|
|
// on a schedule against ClickHouse, and renders what is firing right now as
|
|
// wide, short cards - built for an ultra-wide, low display.
|
|
//
|
|
// A pattern shows as PAGED when this tick would have sent a notification
|
|
// (new or escalated past its cooldown) and as ongoing while it is still
|
|
// firing but already paged once. When a service's spike disappears it is
|
|
// reported as recovered for that scan, and the last firing issue stays on
|
|
// the display as resolved until something fires again.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/ClickHouse/clickhouse-go/v2"
|
|
tea "github.com/charmbracelet/bubbletea"
|
|
|
|
"git.tornberg.me/go-otel/internal/alerts"
|
|
"git.tornberg.me/go-otel/internal/features"
|
|
)
|
|
|
|
func main() {
|
|
// Detection defaults come from the same environment the server's
|
|
// alerting runner reads, so the dashboard and the notifications cannot
|
|
// drift apart; every flag overrides.
|
|
acfg := alerts.ConfigFromEnv()
|
|
addr := flag.String("addr", envOr("OTEL_CLICKHOUSE_ADDR", "127.0.0.1:9000"), "ClickHouse host:port")
|
|
database := flag.String("database", envOr("OTEL_CLICKHOUSE_DB", "otel"), "ClickHouse database")
|
|
user := flag.String("user", envOr("OTEL_CLICKHOUSE_USER", "default"), "ClickHouse user")
|
|
password := flag.String("password", os.Getenv("OTEL_CLICKHOUSE_PASSWORD"), "ClickHouse password")
|
|
interval := flag.Duration("interval", acfg.Interval, "how often to rescan")
|
|
window := flag.Duration("window", acfg.Window, "observation window")
|
|
baselineWindows := flag.Int("baseline-windows", acfg.BaselineWindows, "preceding windows used as baseline")
|
|
cooldown := flag.Duration("cooldown", acfg.Cooldown, "a paged pattern stays ongoing this long unless it doubles")
|
|
minExcess := flag.Int("min-excess", acfg.MinExcess, "minimum paging excess to report a spike")
|
|
multiple := flag.Float64("multiple", acfg.Multiple, "pattern must be novel or >= multiple * its baseline")
|
|
lag := flag.Duration("lag", acfg.Lag, "ingest settle time: windows end this long before now")
|
|
baseURL := flag.String("base-url", acfg.BaseURL, "UI base URL for issue links, e.g. https://otel.k6n.net")
|
|
title := flag.String("title", "otel issues", "dashboard header title")
|
|
once := flag.Bool("once", false, "scan once, print the dashboard to stdout and exit")
|
|
flag.Parse()
|
|
|
|
sc, err := newClickHouseScanner(*addr, *database, *user, *password, features.SpikeConfig{
|
|
Window: *window, BaselineWindows: *baselineWindows, Multiple: *multiple, MinExcess: *minExcess,
|
|
}, *cooldown, *lag)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
defer sc.close()
|
|
|
|
if *once {
|
|
snap := sc.Scan(context.Background(), time.Now())
|
|
if snap.Err != nil {
|
|
fmt.Fprintln(os.Stderr, "issues-tui:", snap.Err)
|
|
os.Exit(1)
|
|
}
|
|
fmt.Println(renderSnapshot(snap, snapshotOptions{Width: 200, Height: 12, Title: *title, BaseURL: *baseURL}))
|
|
return
|
|
}
|
|
|
|
p := tea.NewProgram(newModel(sc, modelConfig{
|
|
Interval: *interval, Title: *title, BaseURL: *baseURL,
|
|
}), tea.WithAltScreen())
|
|
if _, err := p.Run(); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func envOr(k, f string) string {
|
|
if v := os.Getenv(k); v != "" {
|
|
return v
|
|
}
|
|
return f
|
|
}
|
|
|
|
// scanner re-evaluates the notification pipeline on demand. All state
|
|
// (cooldown suppressor, since an issue first fired) lives here, so Scan can
|
|
// be called from the TUI, from -once mode or from tests with the same
|
|
// behavior.
|
|
type scanner struct {
|
|
load alerts.Loader
|
|
dataSince func(ctx context.Context) time.Time
|
|
close func()
|
|
cfg features.SpikeConfig
|
|
lag time.Duration
|
|
pager *features.Suppressor
|
|
|
|
// active maps an issue key to the window end when it was first seen
|
|
// firing; it mirrors what a person watching notifications would remember.
|
|
active map[string]time.Time
|
|
|
|
// lastActive is the most recent firing issue set; when a later scan
|
|
// finds nothing firing, it is kept on the display as resolved instead
|
|
// of flipping to ALL CLEAR.
|
|
lastActive []issueView
|
|
resolvedAt time.Time // window end when lastActive stopped firing
|
|
|
|
since time.Time
|
|
sinceAt time.Time
|
|
}
|
|
|
|
func newClickHouseScanner(addr, database, user, password string, cfg features.SpikeConfig, cooldown, lag time.Duration) (*scanner, error) {
|
|
conn, err := clickhouse.Open(&clickhouse.Options{Addr: []string{addr}, Auth: clickhouse.Auth{Database: database, Username: user, Password: password}})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &scanner{
|
|
load: func(ctx context.Context, start, end time.Time) ([]features.Event, error) {
|
|
return alerts.LoadErrorEvents(ctx, conn, database, start, end)
|
|
},
|
|
dataSince: func(ctx context.Context) time.Time { return alerts.DataSince(ctx, conn, database) },
|
|
close: func() { _ = conn.Close() },
|
|
cfg: cfg,
|
|
lag: lag,
|
|
pager: features.NewSuppressor(cooldown),
|
|
active: map[string]time.Time{},
|
|
}, nil
|
|
}
|
|
|
|
// issueView is one card on the dashboard: a firing spike reduced to an Alert
|
|
// (the same model the notifications carry) plus display state.
|
|
type issueView struct {
|
|
alert alerts.Alert
|
|
key string
|
|
paged bool // this scan would have notified (new or escalated)
|
|
firstSeen time.Time // window end when this issue first appeared
|
|
resolvedAt time.Time // window end when it stopped firing; renders as resolved
|
|
}
|
|
|
|
// recoveredIssue is an issue that fired on the previous scan but not now.
|
|
type recoveredIssue struct {
|
|
Service string
|
|
FirstSeen time.Time
|
|
LastSeen time.Time
|
|
}
|
|
|
|
// snapshot is one scan's result: everything firing right now, plus the last
|
|
// firing issue kept as resolved when nothing fires any more.
|
|
type snapshot struct {
|
|
ScannedAt time.Time
|
|
WindowEnd time.Time
|
|
WindowStart time.Time
|
|
Events int
|
|
Issues []issueView
|
|
Recovered []recoveredIssue
|
|
Resolved []issueView
|
|
Err error
|
|
}
|
|
|
|
// spikeKey identifies an issue across scans: a service plus the set of
|
|
// patterns currently firing for it.
|
|
func spikeKey(s features.Spike) string {
|
|
sigs := make([]string, 0, len(s.Patterns))
|
|
for _, p := range s.Patterns {
|
|
sigs = append(sigs, p.Signature)
|
|
}
|
|
return s.Service + "\x00" + strings.Join(sigs, "\x00")
|
|
}
|
|
|
|
func patternKey(service, signature string) string {
|
|
return service + "\x00" + signature
|
|
}
|
|
|
|
// Scan runs one evaluation, mirroring the runner's Tick: load the window
|
|
// ending lag ago plus its baseline, detect exactly what the server pages on,
|
|
// and use a cooldown suppressor to tell a fresh page from an ongoing one.
|
|
func (s *scanner) Scan(ctx context.Context, now time.Time) snapshot {
|
|
cctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
|
defer cancel()
|
|
end := now.Add(-s.lag).UTC()
|
|
start := end.Add(-time.Duration(s.cfg.BaselineWindows+1) * s.cfg.Window)
|
|
snap := snapshot{ScannedAt: now.UTC(), WindowEnd: end, WindowStart: start}
|
|
|
|
if s.dataSince != nil && (s.since.IsZero() || now.Sub(s.sinceAt) > time.Hour) {
|
|
// Mirrors the runner: refresh where data begins at most hourly, so
|
|
// the warmup guard does not suppress alerts after a quiet period.
|
|
if ds := s.dataSince(cctx); !ds.IsZero() {
|
|
s.since, s.sinceAt, s.cfg.DataSince = ds, now, ds
|
|
}
|
|
}
|
|
|
|
events, err := s.load(cctx, start, end)
|
|
if err != nil {
|
|
snap.Err = err
|
|
return snap
|
|
}
|
|
snap.Events = len(events)
|
|
|
|
firing := alerts.DetectAlerts(events, end, s.cfg)
|
|
// The suppressor mutates only its own state and the copies Filter makes,
|
|
// so firing keeps every pattern for display while paged marks what would
|
|
// have been sent to Telegram / the webhook.
|
|
pagedPatterns := map[string]bool{}
|
|
for _, sp := range s.pager.Filter(firing) {
|
|
for _, p := range sp.Patterns {
|
|
pagedPatterns[patternKey(sp.Service, p.Signature)] = true
|
|
}
|
|
}
|
|
|
|
nowActive := map[string]bool{}
|
|
for _, sp := range firing {
|
|
key := spikeKey(sp)
|
|
nowActive[key] = true
|
|
first, ok := s.active[key]
|
|
if !ok {
|
|
first = end
|
|
s.active[key] = first
|
|
}
|
|
paged := false
|
|
for _, p := range sp.Patterns {
|
|
if pagedPatterns[patternKey(sp.Service, p.Signature)] {
|
|
paged = true
|
|
break
|
|
}
|
|
}
|
|
snap.Issues = append(snap.Issues, issueView{alert: alerts.NewAlert(sp, end), key: key, paged: paged, firstSeen: first})
|
|
}
|
|
for key, first := range s.active {
|
|
if !nowActive[key] {
|
|
delete(s.active, key)
|
|
snap.Recovered = append(snap.Recovered, recoveredIssue{Service: serviceOfKey(key), FirstSeen: first, LastSeen: end})
|
|
}
|
|
}
|
|
// Keep the last firing issue on the display: once nothing fires any
|
|
// more, the cards stay up as resolved rather than the dashboard erasing
|
|
// the context with an ALL CLEAR panel.
|
|
if len(snap.Issues) > 0 {
|
|
s.lastActive = snap.Issues
|
|
s.resolvedAt = time.Time{}
|
|
} else if len(s.lastActive) > 0 {
|
|
if s.resolvedAt.IsZero() {
|
|
s.resolvedAt = end
|
|
}
|
|
for _, v := range s.lastActive {
|
|
v.resolvedAt = s.resolvedAt
|
|
snap.Resolved = append(snap.Resolved, v)
|
|
}
|
|
sortIssues(snap.Resolved)
|
|
}
|
|
sortIssues(snap.Issues)
|
|
return snap
|
|
}
|
|
|
|
func serviceOfKey(key string) string {
|
|
service, _, _ := strings.Cut(key, "\x00")
|
|
return service
|
|
}
|
|
|
|
// sortIssues puts pages first, then the most urgent and largest spikes - the
|
|
// order a person should read the display in.
|
|
func sortIssues(issues []issueView) {
|
|
// insertion sort keeps it stable without pulling in sort.Slice churn for
|
|
// the tiny slices a dashboard shows.
|
|
for i := 1; i < len(issues); i++ {
|
|
for j := i; j > 0 && issueLess(issues[j], issues[j-1]); j-- {
|
|
issues[j], issues[j-1] = issues[j-1], issues[j]
|
|
}
|
|
}
|
|
}
|
|
|
|
func issueLess(a, b issueView) bool {
|
|
if a.paged != b.paged {
|
|
return a.paged
|
|
}
|
|
ra, rb := prioRank(a.alert.Priority()), prioRank(b.alert.Priority())
|
|
if ra != rb {
|
|
return ra < rb
|
|
}
|
|
return a.alert.Spike.Excess > b.alert.Spike.Excess
|
|
}
|
|
|
|
func prioRank(p alerts.Priority) int {
|
|
switch p {
|
|
case alerts.PriorityHigh:
|
|
return 0
|
|
case alerts.PriorityMedium:
|
|
return 1
|
|
}
|
|
return 2
|
|
}
|