306 lines
12 KiB
Go
306 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/charmbracelet/lipgloss"
|
|
|
|
"git.tornberg.me/go-otel/internal/alerts"
|
|
"git.tornberg.me/go-otel/internal/features"
|
|
)
|
|
|
|
// fakeClock builds a scanner with a canned loader and no ClickHouse.
|
|
func fakeScanner(events []features.Event, dataSince time.Time) *scanner {
|
|
return &scanner{
|
|
load: func(context.Context, time.Time, time.Time) ([]features.Event, error) { return events, nil },
|
|
dataSince: func(context.Context) time.Time {
|
|
return dataSince
|
|
},
|
|
close: func() {},
|
|
cfg: features.SpikeConfig{Window: time.Minute, BaselineWindows: 2, Multiple: 3, MinExcess: 5},
|
|
pager: features.NewSuppressor(30 * time.Minute),
|
|
active: map[string]time.Time{},
|
|
}
|
|
}
|
|
|
|
func errs(n int, at time.Time) []features.Event {
|
|
events := make([]features.Event, 0, n)
|
|
for i := 0; i < n; i++ {
|
|
events = append(events, features.Event{
|
|
Timestamp: at,
|
|
Service: "checkout",
|
|
Kind: "span",
|
|
IsError: true,
|
|
StatusCode: 500,
|
|
Route: "/api/payments",
|
|
SpanName: "POST /api/payments",
|
|
Message: "payment gateway timeout",
|
|
PatternSignature: "checkout|500|/api/payments",
|
|
IssueFingerprint: "issue-checkout-500",
|
|
})
|
|
}
|
|
return events
|
|
}
|
|
|
|
// TestScannerPagesOnceThenOngoing mirrors the notification lifecycle: the
|
|
// first window past the threshold pages, later windows are ongoing until the
|
|
// count doubles (escalation) or the spike disappears.
|
|
func TestScannerPagesOnceThenOngoing(t *testing.T) {
|
|
base := time.Date(2025, 6, 1, 12, 0, 0, 0, time.UTC)
|
|
sc := fakeScanner(nil, base.Add(-time.Hour))
|
|
|
|
// The loader places the events inside whatever window is being
|
|
// evaluated, so every scan sees a live spike in its current window.
|
|
count := 12
|
|
sc.load = func(_ context.Context, _, end time.Time) ([]features.Event, error) {
|
|
return errs(count, end.Add(-30*time.Second)), nil
|
|
}
|
|
|
|
first := sc.Scan(context.Background(), base)
|
|
if len(first.Issues) != 1 || !first.Issues[0].paged {
|
|
t.Fatalf("first scan should page one issue, got %+v", first.Issues)
|
|
}
|
|
if first.Issues[0].alert.Priority() != alerts.PriorityHigh {
|
|
t.Fatalf("500 on a span should be high priority, got %s", first.Issues[0].alert.Priority())
|
|
}
|
|
if got := first.Issues[0].firstSeen; !got.Equal(first.WindowEnd) {
|
|
t.Fatalf("firstSeen should be the first window end, got %v", got)
|
|
}
|
|
|
|
// A little more traffic, still not double: ongoing, not paged again.
|
|
count = 14
|
|
second := sc.Scan(context.Background(), base.Add(time.Minute))
|
|
if len(second.Issues) != 1 || second.Issues[0].paged {
|
|
t.Fatalf("second scan should be ongoing, got %+v", second.Issues)
|
|
}
|
|
if got := second.Issues[0].firstSeen; !got.Equal(first.WindowEnd) {
|
|
t.Fatalf("ongoing issue should keep its first-seen time, got %v want %v", got, first.WindowEnd)
|
|
}
|
|
|
|
// Escalation: at least double the count pages again.
|
|
count = 30
|
|
third := sc.Scan(context.Background(), base.Add(2*time.Minute))
|
|
if len(third.Issues) != 1 || !third.Issues[0].paged {
|
|
t.Fatalf("doubled count should page again, got %+v", third.Issues)
|
|
}
|
|
|
|
// Recovery: the spike disappears and is reported once.
|
|
sc.load = func(_ context.Context, _, _ time.Time) ([]features.Event, error) { return nil, nil }
|
|
fourth := sc.Scan(context.Background(), base.Add(3*time.Minute))
|
|
if len(fourth.Issues) != 0 {
|
|
t.Fatalf("no events should mean no issues, got %+v", fourth.Issues)
|
|
}
|
|
if len(fourth.Recovered) != 1 || fourth.Recovered[0].Service != "checkout" {
|
|
t.Fatalf("expected checkout in recovered, got %+v", fourth.Recovered)
|
|
}
|
|
// The last issue stays on the display as resolved instead of ALL CLEAR.
|
|
if len(fourth.Resolved) != 1 || fourth.Resolved[0].alert.Spike.Service != "checkout" {
|
|
t.Fatalf("expected checkout kept as resolved, got %+v", fourth.Resolved)
|
|
}
|
|
if got := fourth.Resolved[0].resolvedAt; !got.Equal(fourth.WindowEnd) {
|
|
t.Fatalf("resolvedAt should be the window that stopped firing, got %v want %v", got, fourth.WindowEnd)
|
|
}
|
|
again := sc.Scan(context.Background(), base.Add(4*time.Minute))
|
|
if len(again.Recovered) != 0 {
|
|
t.Fatalf("recovery should be reported once, got %+v", again.Recovered)
|
|
}
|
|
if len(again.Resolved) != 1 || !again.Resolved[0].resolvedAt.Equal(fourth.Resolved[0].resolvedAt) {
|
|
t.Fatalf("resolved issue should persist with a stable resolved time, got %+v", again.Resolved)
|
|
}
|
|
}
|
|
|
|
func TestScannerAllClearAndError(t *testing.T) {
|
|
sc := fakeScanner(nil, time.Time{})
|
|
empty := sc.Scan(context.Background(), time.Now())
|
|
if len(empty.Issues) != 0 || empty.Err != nil {
|
|
t.Fatalf("empty loader should yield a clean snapshot, got %+v", empty)
|
|
}
|
|
if len(empty.Resolved) != 0 {
|
|
t.Fatalf("a fresh scanner has no last issue to keep, got %+v", empty.Resolved)
|
|
}
|
|
|
|
sc.load = func(context.Context, time.Time, time.Time) ([]features.Event, error) {
|
|
return nil, context.DeadlineExceeded
|
|
}
|
|
failed := sc.Scan(context.Background(), time.Now())
|
|
if failed.Err == nil {
|
|
t.Fatal("loader error should surface in the snapshot")
|
|
}
|
|
}
|
|
|
|
func TestGridLayoutFillsWideShortDisplays(t *testing.T) {
|
|
// 300 columns, one card row of height: six issues, one row of wide cards.
|
|
perRow, rows, shown, cardW := gridLayout(6, 300, 8)
|
|
if perRow != 6 || rows != 1 || shown != 6 {
|
|
t.Fatalf("want all 6 on one row, got perRow=%d rows=%d shown=%d", perRow, rows, shown)
|
|
}
|
|
if cardW < cardMinWidth || cardW > cardMaxWidth {
|
|
t.Fatalf("card width %d out of range", cardW)
|
|
}
|
|
|
|
// More issues than fit: cap at one row and let the footer count the rest.
|
|
perRow, rows, shown, _ = gridLayout(20, 300, 8)
|
|
if rows != 1 || shown != perRow {
|
|
t.Fatalf("want a single row capped at width, got rows=%d shown=%d perRow=%d", rows, shown, perRow)
|
|
}
|
|
|
|
// A taller display wraps to a second row.
|
|
_, rows, shown, _ = gridLayout(20, 300, 15)
|
|
if rows != 2 {
|
|
t.Fatalf("15 rows of height should fit two card rows, got %d", rows)
|
|
}
|
|
if shown != perRow*2 {
|
|
t.Fatalf("shown %d should fill two rows of %d", shown, perRow)
|
|
}
|
|
|
|
// Narrow display: at least one card still renders.
|
|
perRow, rows, shown, cardW = gridLayout(3, 36, 8)
|
|
if perRow < 1 || rows != 1 || shown != 1 || cardW < cardMinWidth {
|
|
t.Fatalf("narrow display should show one card, got perRow=%d rows=%d shown=%d cardW=%d", perRow, rows, shown, cardW)
|
|
}
|
|
}
|
|
|
|
func cardAt(t *testing.T, n int) (snapshot, snapshotOptions) {
|
|
t.Helper()
|
|
now := time.Date(2025, 6, 1, 12, 45, 0, 0, time.UTC)
|
|
sp := features.Spike{
|
|
Service: "checkout", WindowStart: now.Add(-5 * time.Minute), WindowEnd: now,
|
|
ErrorCount: 38, BaselineMean: 0.3, Excess: 37.7,
|
|
Patterns: []features.PatternDelta{{
|
|
Signature: "checkout|500|/api/payments", Count: 38, Excess: 37.7, Novel: true, Clients: 12,
|
|
Services: []string{"checkout", "payments-db"},
|
|
Example: features.Event{
|
|
Service: "checkout", Kind: "span", StatusCode: 500, Route: "/api/payments",
|
|
SpanName: "POST /api/payments", Message: "payment gateway timeout",
|
|
IssueFingerprint: "abcdef1234567890",
|
|
},
|
|
}},
|
|
}
|
|
snap := snapshot{ScannedAt: now, WindowStart: sp.WindowStart, WindowEnd: now, Events: 8213,
|
|
Issues: make([]issueView, n)}
|
|
for i := range snap.Issues {
|
|
snap.Issues[i] = issueView{alert: alerts.NewAlert(sp, now), key: sp.Service, paged: i == 0, firstSeen: now.Add(-12 * time.Minute)}
|
|
}
|
|
return snap, snapshotOptions{Width: 200, Height: 12, Title: "otel issues", BaseURL: "https://otel.k6n.net",
|
|
NextScanAt: now.Add(30 * time.Second), Now: now}
|
|
}
|
|
|
|
func TestRenderSnapshotShowsCards(t *testing.T) {
|
|
snap, o := cardAt(t, 8)
|
|
out := renderSnapshot(snap, o)
|
|
if !strings.Contains(out, "PAGED") || !strings.Contains(out, "checkout") {
|
|
t.Fatalf("dashboard should show the paged checkout card:\n%s", out)
|
|
}
|
|
if !strings.Contains(out, "HTTP 500") {
|
|
t.Fatalf("card should describe the failure:\n%s", out)
|
|
}
|
|
if lines := strings.Split(out, "\n"); len(lines) > 12 {
|
|
t.Fatalf("dashboard must respect the height budget, got %d lines", len(lines))
|
|
}
|
|
|
|
// The footer counts what the single card row could not show: five cards
|
|
// fit in 200 columns, three are hidden.
|
|
if !strings.Contains(out, "+3 more") {
|
|
t.Fatalf("footer should count hidden cards:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestRenderSnapshotAllClear(t *testing.T) {
|
|
now := time.Date(2025, 6, 1, 12, 45, 0, 0, time.UTC)
|
|
snap := snapshot{ScannedAt: now, WindowStart: now.Add(-5 * time.Minute), WindowEnd: now, Events: 8213}
|
|
out := renderSnapshot(snap, snapshotOptions{Width: 200, Height: 12, Title: "otel issues", Now: now})
|
|
if !strings.Contains(out, "ALL CLEAR") {
|
|
t.Fatalf("empty scan should show all clear:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestRenderSnapshotKeepsResolvedIssue(t *testing.T) {
|
|
now := time.Date(2025, 6, 1, 12, 45, 0, 0, time.UTC)
|
|
snap, o := cardAt(t, 1)
|
|
resolved := snap.Issues[0]
|
|
resolved.resolvedAt = now
|
|
snap.Issues = nil
|
|
snap.Resolved = []issueView{resolved}
|
|
out := renderSnapshot(snap, o)
|
|
if strings.Contains(out, "ALL CLEAR") {
|
|
t.Fatalf("a resolved issue should replace the ALL CLEAR panel:\n%s", out)
|
|
}
|
|
if !strings.Contains(out, "RESOLVED") || !strings.Contains(out, "checkout") {
|
|
t.Fatalf("dashboard should keep the last issue as resolved:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestRenderSnapshotScanError(t *testing.T) {
|
|
snap := snapshot{Err: context.DeadlineExceeded}
|
|
out := renderSnapshot(snap, snapshotOptions{Width: 200, Height: 12, Title: "otel issues", Now: time.Now()})
|
|
if !strings.Contains(out, "SCAN FAILED") {
|
|
t.Fatalf("failed scan should be visible:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestSortIssuesPagedFirst(t *testing.T) {
|
|
now := time.Now()
|
|
mk := func(service string, paged bool, prio alerts.Priority, excess float64) issueView {
|
|
pat := features.PatternDelta{Example: features.Event{Service: service, StatusCode: 500}}
|
|
if prio == alerts.PriorityLow {
|
|
pat.Example.StatusCode = 404
|
|
}
|
|
return issueView{paged: paged, alert: alerts.NewAlert(features.Spike{
|
|
Service: service, Excess: excess, Patterns: []features.PatternDelta{pat},
|
|
}, now)}
|
|
}
|
|
issues := []issueView{
|
|
mk("quiet", false, alerts.PriorityHigh, 40),
|
|
mk("loud", true, alerts.PriorityLow, 10),
|
|
mk("burning", false, alerts.PriorityHigh, 99),
|
|
}
|
|
sortIssues(issues)
|
|
if issues[0].alert.Spike.Service != "loud" {
|
|
t.Fatalf("paged issue should come first, got %s", issues[0].alert.Spike.Service)
|
|
}
|
|
if issues[1].alert.Spike.Service != "burning" || issues[2].alert.Spike.Service != "quiet" {
|
|
t.Fatalf("then by priority and excess, got %s, %s", issues[1].alert.Spike.Service, issues[2].alert.Spike.Service)
|
|
}
|
|
}
|
|
|
|
func TestBarAndMinimal(t *testing.T) {
|
|
out := bar(40, "left", "right")
|
|
if lipgloss.Width(out) != 40 {
|
|
t.Fatalf("bar should pad to width, got %q (%d)", out, lipgloss.Width(out))
|
|
}
|
|
if out2 := bar(8, "left", "right"); lipgloss.Width(out2) > 8 {
|
|
t.Fatalf("bar should truncate when both sides do not fit, got %q", out2)
|
|
}
|
|
|
|
snap := snapshot{Err: context.DeadlineExceeded}
|
|
if got := renderMinimal(snap, snapshotOptions{}); !strings.Contains(got, "SCAN FAILED") {
|
|
t.Fatalf("minimal render should surface scan errors, got %q", got)
|
|
}
|
|
}
|
|
|
|
func TestModelKeepsLastGoodSnapshotOnScanError(t *testing.T) {
|
|
m := newModel(nil, modelConfig{Interval: time.Minute})
|
|
good := snapshot{ScannedAt: time.Now(), Issues: []issueView{{}}}
|
|
updated, _ := m.Update(snapshotMsg(good))
|
|
m = updated.(model)
|
|
|
|
failed := snapshot{Err: context.DeadlineExceeded}
|
|
updated, _ = m.Update(snapshotMsg(failed))
|
|
m = updated.(model)
|
|
if len(m.snap.Issues) != 1 {
|
|
t.Fatalf("failed scan should not erase the last good snapshot: %+v", m.snap)
|
|
}
|
|
if m.scanErr == nil {
|
|
t.Fatal("failed scan should be visible separately from the last good snapshot")
|
|
}
|
|
|
|
updated, _ = m.Update(snapshotMsg(snapshot{ScannedAt: time.Now()}))
|
|
m = updated.(model)
|
|
if m.scanErr != nil {
|
|
t.Fatal("successful scan should clear the previous scan error")
|
|
}
|
|
}
|