343 lines
13 KiB
Go
343 lines
13 KiB
Go
package alerts
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.tornberg.me/go-otel/internal/features"
|
|
"git.tornberg.me/go-otel/internal/topology"
|
|
)
|
|
|
|
func burst(sig string, n int, at time.Time, status int, route string) []features.Event {
|
|
var out []features.Event
|
|
for i := 0; i < n; i++ {
|
|
out = append(out, features.Event{Timestamp: at.Add(time.Duration(i) * time.Second), Service: "api", Kind: "span", IsError: true,
|
|
PatternSignature: sig, StatusCode: status, Route: route, ClientIP: "203.0.113.7"})
|
|
}
|
|
return out
|
|
}
|
|
|
|
type fake struct {
|
|
mu sync.Mutex
|
|
sent []Alert
|
|
err error
|
|
}
|
|
|
|
func (f *fake) Name() string { return "fake" }
|
|
func (f *fake) Send(_ context.Context, a Alert) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.sent = append(f.sent, a)
|
|
return f.err
|
|
}
|
|
|
|
func newRunner(events []features.Event, n Notifier) *Runner {
|
|
cfg := Config{Interval: time.Minute, Window: 5 * time.Minute, BaselineWindows: 12, Cooldown: 30 * time.Minute, MinExcess: 5, Multiple: 3, Lag: 0}
|
|
return &Runner{
|
|
cfg: cfg, notifiers: []Notifier{n}, sup: features.NewSuppressor(cfg.Cooldown),
|
|
load: func(context.Context, time.Time, time.Time) ([]features.Event, error) { return events, nil },
|
|
dataSince: func(context.Context) time.Time { return time.Time{} },
|
|
}
|
|
}
|
|
|
|
func TestRunnerPrimesThenAlertsOnceAndFiltersProbes(t *testing.T) {
|
|
now := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
|
|
cur := now.Add(-2 * time.Minute)
|
|
f := &fake{}
|
|
// a real defect burst and a probe burst in the same window
|
|
events := append(burst("defect", 20, cur, 500, "/api/orders"), burst("probe", 50, cur, 404, "/.env")...)
|
|
r := newRunner(events, f)
|
|
|
|
r.Tick(context.Background(), now) // first tick primes: nothing is sent
|
|
if len(f.sent) != 0 {
|
|
t.Fatalf("primed tick sent %d alerts", len(f.sent))
|
|
}
|
|
// the same incident keeps going: suppressed by the primed cooldown
|
|
r.Tick(context.Background(), now.Add(time.Minute))
|
|
if len(f.sent) != 0 {
|
|
t.Fatalf("ongoing incident re-alerted: %d", len(f.sent))
|
|
}
|
|
// a new, different failure appears
|
|
r.load = func(context.Context, time.Time, time.Time) ([]features.Event, error) {
|
|
return append(events, burst("newdefect", 15, now.Add(time.Minute), 503, "/api/pay")...), nil
|
|
}
|
|
r.Tick(context.Background(), now.Add(4*time.Minute))
|
|
if len(f.sent) != 1 {
|
|
t.Fatalf("want 1 alert, got %d", len(f.sent))
|
|
}
|
|
txt := f.sent[0].Text("https://otel.example")
|
|
if strings.Contains(txt, "/.env") {
|
|
t.Fatalf("probe leaked into alert: %s", txt)
|
|
}
|
|
if !strings.Contains(txt, "/api/pay") || !strings.Contains(txt, "1 client") || !strings.HasSuffix(txt, "https://otel.example/issues") {
|
|
t.Fatalf("unexpected text: %s", txt)
|
|
}
|
|
if got := r.Status(); len(got.Alerts) != 1 || got.Channels[0] != "fake" {
|
|
t.Fatalf("status = %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestTelegramSendsAndNeverLeaksToken(t *testing.T) {
|
|
var gotPath string
|
|
var body map[string]any
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
gotPath = r.URL.Path
|
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
|
w.WriteHeader(200)
|
|
}))
|
|
tg := &Telegram{Token: "SECRETTOKEN", ChatID: "42", ThreadID: "7", API: srv.URL}
|
|
a := NewAlert(features.Spike{Service: "s", ErrorCount: 9}, time.Now())
|
|
if err := tg.Send(context.Background(), a); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if gotPath != "/botSECRETTOKEN/sendMessage" || body["chat_id"] != "42" || body["message_thread_id"] != float64(7) {
|
|
t.Fatalf("request wrong: %s %+v", gotPath, body)
|
|
}
|
|
srv.Close() // connection refused: the transport error contains the URL
|
|
err := tg.Send(context.Background(), a)
|
|
if err == nil || strings.Contains(err.Error(), "SECRETTOKEN") {
|
|
t.Fatalf("token leaked or no error: %v", err)
|
|
}
|
|
bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, `{"description":"bad token SECRETTOKEN"}`, 401)
|
|
}))
|
|
defer bad.Close()
|
|
err = (&Telegram{Token: "SECRETTOKEN", ChatID: "1", API: bad.URL}).Send(context.Background(), a)
|
|
if err == nil || strings.Contains(err.Error(), "SECRETTOKEN") || !strings.Contains(err.Error(), "401") {
|
|
t.Fatalf("error not scrubbed: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestTelegramTruncates(t *testing.T) {
|
|
var body map[string]any
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _ = json.NewDecoder(r.Body).Decode(&body) }))
|
|
defer srv.Close()
|
|
spike := features.Spike{Service: "s"}
|
|
for i := 0; i < 200; i++ {
|
|
spike.Patterns = append(spike.Patterns, features.PatternDelta{Count: 1, Example: features.Event{Message: strings.Repeat("x", 100)}})
|
|
}
|
|
if err := (&Telegram{Token: "t", ChatID: "1", API: srv.URL}).Send(context.Background(), NewAlert(spike, time.Now())); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
text := body["text"].(string)
|
|
if n := plainLen(text); n > 4096 {
|
|
t.Fatalf("message too long: %d", n)
|
|
}
|
|
if body["parse_mode"] != "HTML" || !strings.Contains(text, "more pattern") || strings.Count(text, "<b>") != strings.Count(text, "</b>") {
|
|
t.Fatalf("bad html message: %.200s", text)
|
|
}
|
|
}
|
|
|
|
func TestTelegramHTMLEscapesDynamicText(t *testing.T) {
|
|
at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
|
|
ev := burst("x", 12, at.Add(-2*time.Minute), 500, "/a<b>&c")
|
|
sp := DetectAlerts(ev, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5})
|
|
out := NewAlert(sp[0], at).HTML("https://otel.k6n.net", 4000)
|
|
if strings.Contains(out, "/a<b>") || !strings.Contains(out, "/a<b>&c") {
|
|
t.Fatalf("route not escaped:\n%s", out)
|
|
}
|
|
}
|
|
|
|
func TestScannerClientIsMutedEntirely(t *testing.T) {
|
|
at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
|
|
cur := at.Add(-2 * time.Minute)
|
|
scanner := func(route string, n int) []features.Event {
|
|
ev := burst("p-"+route, n, cur, 503, route)
|
|
for i := range ev {
|
|
ev[i].ClientIP = "198.51.100.9"
|
|
}
|
|
return ev
|
|
}
|
|
var events []features.Event
|
|
for _, r := range []string{"/.env", "/.cursorrules", "/.codex/auth.json"} {
|
|
events = append(events, scanner(r, 4)...)
|
|
}
|
|
events = append(events, scanner("/", 12)...) // the scanner's innocent-looking root request
|
|
if got := DetectAlerts(events, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5}); len(got) != 0 {
|
|
t.Fatalf("scanner traffic alerted: %+v", got)
|
|
}
|
|
// a different client with the same 503 on "/" still alerts
|
|
other := burst("real-503", 12, cur, 503, "/")
|
|
if got := DetectAlerts(other, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5}); len(got) != 1 {
|
|
t.Fatalf("real failure was muted: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestMinExcessAppliesAfterTriage(t *testing.T) {
|
|
at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
|
|
cur := at.Add(-2 * time.Minute)
|
|
// 200 probe errors + 6 real ones: total excess is large, actionable excess is 6 < 10
|
|
events := append(burst("probe", 200, cur, 404, "/.env"), burst("real", 6, cur, 503, "/api")...)
|
|
if got := DetectAlerts(events, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 10}); len(got) != 0 {
|
|
t.Fatalf("paged on excess that was mostly probes: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestClientErrorsRankBelowServerErrors(t *testing.T) {
|
|
at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
|
|
cur := at.Add(-2 * time.Minute)
|
|
cfg := features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 10}
|
|
// 40 x 422 discounted to 8 excess: below MinExcess 10, no page
|
|
if got := DetectAlerts(burst("bad-input", 40, cur, 422, "/api/order"), at, cfg); len(got) != 0 {
|
|
t.Fatalf("4xx alone paged: %+v", got)
|
|
}
|
|
// 5xx of the same volume does page
|
|
if got := DetectAlerts(burst("boom", 40, cur, 500, "/api/order"), at, cfg); len(got) != 1 {
|
|
t.Fatalf("5xx did not page")
|
|
}
|
|
// mixed: the 5xx is listed first and the alert is high priority
|
|
events := append(burst("bad-input", 40, cur, 422, "/api/order"), burst("boom", 8, cur, 500, "/api/pay")...)
|
|
got := DetectAlerts(events, at, cfg)
|
|
if len(got) != 1 {
|
|
t.Fatalf("mixed spike: %+v", got)
|
|
}
|
|
a := NewAlert(got[0], at)
|
|
if a.Priority() != PriorityHigh || a.Patterns[0].Priority != PriorityHigh || a.Patterns[1].Priority != PriorityLow {
|
|
t.Fatalf("priority order wrong: %+v", a.Patterns)
|
|
}
|
|
}
|
|
|
|
func TestAlertLinksToIssue(t *testing.T) {
|
|
at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
|
|
ev := burst("boom", 12, at.Add(-2*time.Minute), 500, "/api")
|
|
for i := range ev {
|
|
ev[i].IssueFingerprint = "00ab12cd34ef5678"
|
|
}
|
|
sp := DetectAlerts(ev, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5})
|
|
txt := NewAlert(sp[0], at).Text("https://otel.k6n.net/")
|
|
if !strings.Contains(txt, "https://otel.k6n.net/issues/00ab12cd34ef5678") {
|
|
t.Fatalf("no issue link:\n%s", txt)
|
|
}
|
|
}
|
|
|
|
func TestCrawlerClientErrorsNeverAlert(t *testing.T) {
|
|
at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
|
|
ev := burst("crawl", 100, at.Add(-2*time.Minute), 403, "/members")
|
|
for i := range ev {
|
|
ev[i].UAClass = "ai_crawler"
|
|
}
|
|
if got := DetectAlerts(ev, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5}); len(got) != 0 {
|
|
t.Fatalf("crawler 4xx alerted: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestAlertListsAffectedServices(t *testing.T) {
|
|
at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
|
|
ev := burst("boom", 12, at.Add(-2*time.Minute), 500, "/api")
|
|
for i := range ev {
|
|
ev[i].TraceServices = []string{"payments", "api"}
|
|
}
|
|
sp := DetectAlerts(ev, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5})
|
|
a := NewAlert(sp[0], at)
|
|
if got := strings.Join(a.Patterns[0].Services, ","); got != "api,payments" {
|
|
t.Fatalf("services = %q", got)
|
|
}
|
|
if !strings.Contains(a.Text(""), "Services with errors in the same requests: api, payments") {
|
|
t.Fatalf("text lacks services:\n%s", a.Text(""))
|
|
}
|
|
}
|
|
|
|
type fakeResolver struct{ got [2]string }
|
|
|
|
func (f *fakeResolver) Resolve(host, path string) []topology.Target {
|
|
f.got = [2]string{host, path}
|
|
return []topology.Target{
|
|
{Host: host, Path: "/api", Namespace: "shop", Service: "orders", Deployments: []string{"orders-v2"}},
|
|
{Host: host, Path: "/api", Namespace: "shop", Service: "gone", Missing: true},
|
|
}
|
|
}
|
|
|
|
func TestEdgeAlertShowsIngressBackends(t *testing.T) {
|
|
at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
|
|
ev := burst("bad-gw", 16, at.Add(-2*time.Minute), 502, "/api/order")
|
|
for i := range ev {
|
|
ev[i].Service, ev[i].Host = "nginx", "shop.example.com"
|
|
}
|
|
sp := DetectAlerts(ev, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5})
|
|
a := NewAlert(sp[0], at)
|
|
r := &fakeResolver{}
|
|
a.Enrich(r)
|
|
if r.got != [2]string{"shop.example.com", "/api/order"} {
|
|
t.Fatalf("resolved %v", r.got)
|
|
}
|
|
txt := a.Text("")
|
|
for _, want := range []string{"service shop/orders -> deployment orders-v2", "may not be instrumented", "kubectl -n shop", "this Service does not exist"} {
|
|
if !strings.Contains(txt, want) {
|
|
t.Fatalf("missing %q in:\n%s", want, txt)
|
|
}
|
|
}
|
|
// a non-edge service is never enriched
|
|
b := NewAlert(DetectAlerts(burst("x", 12, at.Add(-2*time.Minute), 500, "/"), at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5})[0], at)
|
|
b.Enrich(r)
|
|
if len(b.Patterns[0].Routes) != 0 {
|
|
t.Fatalf("non-edge alert enriched")
|
|
}
|
|
}
|
|
|
|
type fakePodResolver struct {
|
|
got []struct{ namespace, deployment string }
|
|
}
|
|
|
|
func (f *fakePodResolver) PodLogs(ctx context.Context, namespace, deployment string, tail int) topology.PodLog {
|
|
f.got = append(f.got, struct{ namespace, deployment string }{namespace, deployment})
|
|
switch deployment {
|
|
case "orders-v2":
|
|
return topology.PodLog{Pod: "orders-v2-7d8f9-abcde", Found: true, Lines: []string{"line1", "line2", "line3"}}
|
|
default:
|
|
return topology.PodLog{} // scaled to 0
|
|
}
|
|
}
|
|
|
|
func TestAlertShowsPodLogsOrScaledToZero(t *testing.T) {
|
|
at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
|
|
ev := burst("bad-gw", 16, at.Add(-2*time.Minute), 502, "/api/order")
|
|
for i := range ev {
|
|
ev[i].Service, ev[i].Host = "nginx", "shop.example.com"
|
|
}
|
|
sp := DetectAlerts(ev, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5})
|
|
a := NewAlert(sp[0], at)
|
|
a.Enrich(&fakeResolver{})
|
|
pr := &fakePodResolver{}
|
|
a.EnrichPods(context.Background(), pr)
|
|
|
|
txt := a.Text("")
|
|
if !strings.Contains(txt, "orders-v2: pod orders-v2-7d8f9-abcde, last 3 log line(s):") {
|
|
t.Fatalf("missing pod logs in:\n%s", txt)
|
|
}
|
|
for _, want := range []string{"line1", "line2", "line3"} {
|
|
if !strings.Contains(txt, want) {
|
|
t.Fatalf("missing log line %q in:\n%s", want, txt)
|
|
}
|
|
}
|
|
if len(pr.got) != 1 || pr.got[0].namespace != "shop" || pr.got[0].deployment != "orders-v2" {
|
|
t.Fatalf("unexpected lookups: %+v", pr.got)
|
|
}
|
|
}
|
|
|
|
func TestPodLineNoPod(t *testing.T) {
|
|
got := podLine(PodStatus{Deployment: "orders-v2", Found: false})
|
|
if got != "orders-v2: no pod running (scaled to 0?)" {
|
|
t.Fatalf("got %q", got)
|
|
}
|
|
}
|
|
|
|
func TestDropMuted(t *testing.T) {
|
|
mk := func(id string, excess float64) features.PatternDelta {
|
|
return features.PatternDelta{Excess: excess, Example: features.Event{IssueFingerprint: id}}
|
|
}
|
|
spikes := []features.Spike{
|
|
{Excess: 10, Patterns: []features.PatternDelta{mk("ignored", 6), mk("live", 4)}},
|
|
{Excess: 5, Patterns: []features.PatternDelta{mk("ignored", 5)}},
|
|
}
|
|
got := DropMuted(spikes, func(id string) bool { return id == "ignored" })
|
|
if len(got) != 1 || len(got[0].Patterns) != 1 || got[0].Patterns[0].Example.IssueFingerprint != "live" || got[0].Excess != 4 {
|
|
t.Fatalf("unexpected result: %+v", got)
|
|
}
|
|
}
|