include db for gitea build

internal/alerts/alerts_test.go

@@ -0,0 +1,130 @@
+package alerts
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/features"
+)
+
+func burst(sig string, n int, at time.Time, status int, route string) []features.Event {
+	var out []features.Event
+	for i := 0; i < n; i++ {
+		out = append(out, features.Event{Timestamp: at.Add(time.Duration(i) * time.Second), Service: "api", Kind: "span", IsError: true,
+			PatternSignature: sig, StatusCode: status, Route: route, ClientIP: "203.0.113.7"})
+	}
+	return out
+}
+
+type fake struct {
+	mu   sync.Mutex
+	sent []Alert
+	err  error
+}
+
+func (f *fake) Name() string { return "fake" }
+func (f *fake) Send(_ context.Context, a Alert) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.sent = append(f.sent, a)
+	return f.err
+}
+
+func newRunner(events []features.Event, n Notifier) *Runner {
+	cfg := Config{Interval: time.Minute, Window: 5 * time.Minute, BaselineWindows: 12, Cooldown: 30 * time.Minute, MinExcess: 5, Multiple: 3, Lag: 0}
+	return &Runner{
+		cfg: cfg, notifiers: []Notifier{n}, sup: features.NewSuppressor(cfg.Cooldown),
+		load:      func(context.Context, time.Time, time.Time) ([]features.Event, error) { return events, nil },
+		dataSince: func(context.Context) time.Time { return time.Time{} },
+	}
+}
+
+func TestRunnerPrimesThenAlertsOnceAndFiltersProbes(t *testing.T) {
+	now := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	cur := now.Add(-2 * time.Minute)
+	f := &fake{}
+	// a real defect burst and a probe burst in the same window
+	events := append(burst("defect", 20, cur, 500, "/api/orders"), burst("probe", 50, cur, 404, "/.env")...)
+	r := newRunner(events, f)
+
+	r.Tick(context.Background(), now) // first tick primes: nothing is sent
+	if len(f.sent) != 0 {
+		t.Fatalf("primed tick sent %d alerts", len(f.sent))
+	}
+	// the same incident keeps going: suppressed by the primed cooldown
+	r.Tick(context.Background(), now.Add(time.Minute))
+	if len(f.sent) != 0 {
+		t.Fatalf("ongoing incident re-alerted: %d", len(f.sent))
+	}
+	// a new, different failure appears
+	r.load = func(context.Context, time.Time, time.Time) ([]features.Event, error) {
+		return append(events, burst("newdefect", 15, now.Add(time.Minute), 503, "/api/pay")...), nil
+	}
+	r.Tick(context.Background(), now.Add(4*time.Minute))
+	if len(f.sent) != 1 {
+		t.Fatalf("want 1 alert, got %d", len(f.sent))
+	}
+	txt := f.sent[0].Text("https://otel.example")
+	if strings.Contains(txt, "/.env") {
+		t.Fatalf("probe leaked into alert: %s", txt)
+	}
+	if !strings.Contains(txt, "/api/pay") || !strings.Contains(txt, "1 client") || !strings.HasSuffix(txt, "https://otel.example/issues") {
+		t.Fatalf("unexpected text: %s", txt)
+	}
+	if got := r.Status(); len(got.Alerts) != 1 || got.Channels[0] != "fake" {
+		t.Fatalf("status = %+v", got)
+	}
+}
+
+func TestTelegramSendsAndNeverLeaksToken(t *testing.T) {
+	var gotPath string
+	var body map[string]any
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.Path
+		_ = json.NewDecoder(r.Body).Decode(&body)
+		w.WriteHeader(200)
+	}))
+	tg := &Telegram{Token: "SECRETTOKEN", ChatID: "42", ThreadID: "7", API: srv.URL}
+	a := NewAlert(features.Spike{Service: "s", ErrorCount: 9}, time.Now())
+	if err := tg.Send(context.Background(), a); err != nil {
+		t.Fatal(err)
+	}
+	if gotPath != "/botSECRETTOKEN/sendMessage" || body["chat_id"] != "42" || body["message_thread_id"] != float64(7) {
+		t.Fatalf("request wrong: %s %+v", gotPath, body)
+	}
+	srv.Close() // connection refused: the transport error contains the URL
+	err := tg.Send(context.Background(), a)
+	if err == nil || strings.Contains(err.Error(), "SECRETTOKEN") {
+		t.Fatalf("token leaked or no error: %v", err)
+	}
+	bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, `{"description":"bad token SECRETTOKEN"}`, 401)
+	}))
+	defer bad.Close()
+	err = (&Telegram{Token: "SECRETTOKEN", ChatID: "1", API: bad.URL}).Send(context.Background(), a)
+	if err == nil || strings.Contains(err.Error(), "SECRETTOKEN") || !strings.Contains(err.Error(), "401") {
+		t.Fatalf("error not scrubbed: %v", err)
+	}
+}
+
+func TestTelegramTruncates(t *testing.T) {
+	var body map[string]any
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _ = json.NewDecoder(r.Body).Decode(&body) }))
+	defer srv.Close()
+	spike := features.Spike{Service: "s"}
+	for i := 0; i < 200; i++ {
+		spike.Patterns = append(spike.Patterns, features.PatternDelta{Count: 1, Example: features.Event{Message: strings.Repeat("x", 100)}})
+	}
+	if err := (&Telegram{Token: "t", ChatID: "1", API: srv.URL}).Send(context.Background(), NewAlert(spike, time.Now())); err != nil {
+		t.Fatal(err)
+	}
+	if n := len([]rune(body["text"].(string))); n > 4096 {
+		t.Fatalf("message too long: %d", n)
+	}
+}

internal/alerts/filter.go

@@ -0,0 +1,39 @@
+package alerts
+
+import (
+	"git.tornberg.me/go-otel/internal/features"
+	"git.tornberg.me/go-otel/internal/triage"
+)
+
+// TriageOf classifies a pattern's example event with the deterministic rules.
+func TriageOf(e features.Event) triage.Result {
+	return triage.Classify(triage.Input{Service: e.Service, Kind: e.Kind, Status: e.StatusCode, Route: e.Route, Title: e.SpanName, Message: e.Message,
+		IsTimeout: e.IsTimeout, IsCancellation: e.IsCancellation, IsRetry: e.IsRetry})
+}
+
+// FilterActionable keeps only patterns whose category should page a human
+// (unless keepMuted), recomputing each spike's excess so a probe storm cannot
+// alert by itself. muted, when non-nil, counts what was dropped per category.
+func FilterActionable(spikes []features.Spike, keepMuted bool, muted map[triage.Category]int) []features.Spike {
+	var out []features.Spike
+	for _, sp := range spikes {
+		var kept []features.PatternDelta
+		sp.Excess = 0
+		for _, p := range sp.Patterns {
+			c := TriageOf(p.Example).Category
+			if !keepMuted && !c.Actionable() {
+				if muted != nil {
+					muted[c]++
+				}
+				continue
+			}
+			kept = append(kept, p)
+			sp.Excess += p.Excess
+		}
+		if len(kept) > 0 {
+			sp.Patterns = kept
+			out = append(out, sp)
+		}
+	}
+	return out
+}

internal/alerts/load.go

@@ -0,0 +1,134 @@
+// Package alerts turns the deterministic spike detector into notifications:
+// it loads recent error events, runs features.DetectSpikes, keeps only
+// actionable triage categories, suppresses repeats and posts what is left to
+// a Telegram chat and/or a generic webhook.
+package alerts
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+
+	"git.tornberg.me/go-otel/internal/features"
+	"git.tornberg.me/go-otel/internal/honeypot"
+	"git.tornberg.me/go-otel/internal/tracectx"
+	"git.tornberg.me/go-otel/internal/uaclass"
+)
+
+func chTime(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05.000000000") }
+
+// DataSince returns the earliest normalized row, the input to the detector's
+// warm-up guard. Zero means unknown (empty table or query failure).
+func DataSince(ctx context.Context, conn driver.Conn, database string) time.Time {
+	var t time.Time
+	if err := conn.QueryRow(ctx, fmt.Sprintf(`SELECT min(timestamp) FROM %s.normalized_events`, database)).Scan(&t); err != nil {
+		log.Printf("alerts: warm-up guard disabled: %v", err)
+		return time.Time{}
+	}
+	if t.Year() < 2000 { // empty table
+		return time.Time{}
+	}
+	return t
+}
+
+// LoadErrorEvents reads error events in [start, end) from normalized_events
+// and, when the trace columns exist, resolves duplicate hops and real client
+// IPs through each error's trace (see internal/tracectx).
+func LoadErrorEvents(ctx context.Context, conn driver.Conn, database string, start, end time.Time) ([]features.Event, error) {
+	// Trace columns exist only after the server has migrated the table.
+	var traceCols uint64
+	if err := conn.QueryRow(ctx, `SELECT count() FROM system.columns WHERE database = ? AND table = 'normalized_events' AND name IN ('span_id','parent_span_id','peer_ip','ua_class')`, database).Scan(&traceCols); err != nil {
+		return nil, err
+	}
+	withTrace := traceCols == 4
+	extra := "'' AS span_id, '' AS parent_span_id, '' AS peer_ip, '' AS ua_class, '' AS user_agent, '' AS trace_id"
+	if withTrace {
+		extra = "span_id, parent_span_id, peer_ip, ua_class, user_agent, trace_id"
+	}
+	rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT timestamp, service, release, kind, source, status_code, span_name, route, exception_type, message_normalized, pattern_signature, is_timeout, is_cancellation, is_retry, %s
+		FROM %s.normalized_events FINAL
+		WHERE is_error = 1 AND timestamp >= toDateTime64('%s', 9, 'UTC') AND timestamp < toDateTime64('%s', 9, 'UTC')
+		ORDER BY timestamp LIMIT 2000000`, extra, database, chTime(start), chTime(end)))
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []features.Event
+	var refs []tracectx.Key
+	var own []string // the span's own peer address, the fallback client
+	for rows.Next() {
+		e := features.Event{IsError: true}
+		var status uint16
+		var timeout, cancel, retry uint8
+		var spanID, parentID, peer, class, ua, traceID string
+		if err := rows.Scan(&e.Timestamp, &e.Service, &e.Release, &e.Kind, &e.Source, &status, &e.SpanName, &e.Route, &e.ExceptionType, &e.Message, &e.PatternSignature, &timeout, &cancel, &retry,
+			&spanID, &parentID, &peer, &class, &ua, &traceID); err != nil {
+			return nil, err
+		}
+		e.StatusCode = int(status)
+		e.IsTimeout, e.IsCancellation, e.IsRetry = timeout == 1, cancel == 1, retry == 1
+		e.TraceID, e.UAClass = traceID, class
+		if e.UAClass == "" && ua != "" {
+			e.UAClass = uaclass.Of(ua) // rows written before ua_class existed
+		}
+		out = append(out, e)
+		refs = append(refs, tracectx.Key{TraceID: traceID, SpanID: spanID})
+		own = append(own, peer)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	if withTrace && len(out) > 0 {
+		if err := resolve(ctx, conn, database, start, end, out, refs, own); err != nil {
+			// Unresolved events are still usable; only the dedupe is lost.
+			log.Printf("alerts: trace resolution skipped: %v", err)
+		}
+	}
+	return out, nil
+}
+
+// resolve loads every span of the traces that contain an error (ancestors are
+// usually not errors themselves) and copies the resolved client / duplicate /
+// root-cause facts onto the error events.
+func resolve(ctx context.Context, conn driver.Conn, database string, start, end time.Time, events []features.Event, refs []tracectx.Key, own []string) error {
+	rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT trace_id, span_id, parent_span_id, service, peer_ip, is_error, status_code
+		FROM %[1]s.normalized_events
+		WHERE kind = 'span' AND span_id != '' AND timestamp >= toDateTime64('%[2]s', 9, 'UTC') AND timestamp < toDateTime64('%[3]s', 9, 'UTC')
+		  AND trace_id IN (SELECT trace_id FROM %[1]s.normalized_events WHERE kind = 'span' AND is_error = 1 AND trace_id != ''
+		                   AND timestamp >= toDateTime64('%[2]s', 9, 'UTC') AND timestamp < toDateTime64('%[3]s', 9, 'UTC'))
+		LIMIT 5000000`, database, chTime(start.Add(-time.Minute)), chTime(end.Add(time.Minute))))
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+	var spans []tracectx.Span
+	for rows.Next() {
+		var s tracectx.Span
+		var isErr uint8
+		var status uint16
+		if err := rows.Scan(&s.TraceID, &s.SpanID, &s.ParentID, &s.Service, &s.PeerIP, &isErr, &status); err != nil {
+			return err
+		}
+		s.IsError, s.StatusCode, s.Edge = isErr == 1, int(status), tracectx.IsEdgeService(s.Service)
+		spans = append(spans, s)
+	}
+	if err := rows.Err(); err != nil {
+		return err
+	}
+	info := tracectx.Resolve(spans)
+	for i, ref := range refs {
+		in, ok := info[ref]
+		if !ok {
+			if !honeypot.Untrackable(own[i]) {
+				events[i].ClientIP = own[i]
+			}
+			continue
+		}
+		events[i].ClientIP, events[i].IsDuplicate, events[i].IsRootCause = in.ClientIP, in.IsDuplicate, in.IsRootCause
+		events[i].HasUpstream = in.UpstreamChildren > 0
+	}
+	return nil
+}

internal/alerts/notify.go

@@ -0,0 +1,235 @@
+package alerts
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/features"
+	"git.tornberg.me/go-otel/internal/uaclass"
+)
+
+// Alert is one notification: a spike reduced to its actionable patterns.
+type Alert struct {
+	ID       string          `json:"id"`
+	At       time.Time       `json:"at"`
+	Spike    features.Spike  `json:"spike"`
+	Patterns []PatternReport `json:"patterns"`
+	Test     bool            `json:"test,omitempty"`
+}
+
+// PatternReport is a spike pattern with its triage verdict, ready to show.
+type PatternReport struct {
+	Category string  `json:"category"`
+	Rule     string  `json:"rule"`
+	New      bool    `json:"new"`
+	Count    int     `json:"count"`
+	Baseline float64 `json:"baseline"`
+	Clients  int     `json:"clients,omitempty"`
+	UAClass  string  `json:"uaClass,omitempty"`
+	What     string  `json:"what"`
+}
+
+// Notifier delivers an alert somewhere. Implementations must not put secrets
+// in returned errors.
+type Notifier interface {
+	Name() string
+	Send(ctx context.Context, a Alert) error
+}
+
+// NewAlert builds an Alert from a spike, classifying each pattern.
+func NewAlert(spike features.Spike, at time.Time) Alert {
+	a := Alert{At: at, Spike: spike, ID: alertID(spike, at)}
+	for _, p := range spike.Patterns {
+		res := TriageOf(p.Example)
+		r := PatternReport{Category: string(res.Category), Rule: res.Rule, New: p.Novel, Count: p.Count, Baseline: p.BaselineMean, Clients: p.Clients, What: describe(p.Example)}
+		if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) {
+			r.UAClass = p.Example.UAClass
+		}
+		a.Patterns = append(a.Patterns, r)
+	}
+	return a
+}
+
+func alertID(s features.Spike, at time.Time) string {
+	return strconv.FormatInt(at.Unix(), 36) + "-" + strings.ReplaceAll(s.Service, " ", "_")
+}
+
+func describe(e features.Event) string {
+	parts := []string{}
+	if e.StatusCode != 0 {
+		parts = append(parts, "HTTP "+strconv.Itoa(e.StatusCode))
+	}
+	switch {
+	case e.Route != "":
+		parts = append(parts, e.Route)
+	case e.SpanName != "":
+		parts = append(parts, e.SpanName)
+	}
+	if e.ExceptionType != "" {
+		parts = append(parts, e.ExceptionType)
+	}
+	if e.Message != "" && e.Message != e.SpanName {
+		parts = append(parts, truncate(e.Message, 120))
+	}
+	if len(parts) == 0 {
+		return "(no detail)"
+	}
+	return strings.Join(parts, " | ")
+}
+
+func truncate(s string, n int) string {
+	r := []rune(s)
+	if len(r) <= n {
+		return s
+	}
+	return string(r[:n-1]) + "…"
+}
+
+// Text renders a plain-text message (no markup, so nothing needs escaping).
+func (a Alert) Text(baseURL string) string {
+	var b strings.Builder
+	prefix := "ALERT"
+	if a.Test {
+		prefix = "TEST"
+	}
+	fmt.Fprintf(&b, "%s %s: %d errors in %s (baseline %.1f/window, excess %.0f)\n",
+		prefix, a.Spike.Service, a.Spike.ErrorCount, a.Spike.WindowEnd.Sub(a.Spike.WindowStart).Round(time.Minute), a.Spike.BaselineMean, a.Spike.Excess)
+	for _, p := range a.Patterns {
+		tag := "rising"
+		if p.New {
+			tag = "NEW"
+		}
+		fmt.Fprintf(&b, "\n%s x%d (base %.1f) [%s]\n%s", tag, p.Count, p.Baseline, p.Category, p.What)
+		var extra []string
+		if p.Clients > 0 {
+			extra = append(extra, fmt.Sprintf("%d client%s", p.Clients, plural(p.Clients)))
+		}
+		if p.UAClass != "" {
+			extra = append(extra, p.UAClass)
+		}
+		if len(extra) > 0 {
+			fmt.Fprintf(&b, "\n(%s)", strings.Join(extra, ", "))
+		}
+		b.WriteString("\n")
+	}
+	if baseURL != "" {
+		fmt.Fprintf(&b, "\n%s/issues", strings.TrimRight(baseURL, "/"))
+	}
+	return strings.TrimSpace(b.String())
+}
+
+func plural(n int) string {
+	if n == 1 {
+		return ""
+	}
+	return "s"
+}
+
+// Telegram posts via the Bot API sendMessage method.
+type Telegram struct {
+	Token    string
+	ChatID   string
+	ThreadID string // optional forum topic
+	BaseURL  string // link appended to messages
+	API      string // override for tests; default https://api.telegram.org
+	Client   *http.Client
+}
+
+func (t *Telegram) Name() string { return "telegram" }
+
+// telegram limits a message to 4096 characters.
+const telegramMax = 4000
+
+func (t *Telegram) Send(ctx context.Context, a Alert) error {
+	api := t.API
+	if api == "" {
+		api = "https://api.telegram.org"
+	}
+	body := map[string]any{"chat_id": t.ChatID, "text": truncate(a.Text(t.BaseURL), telegramMax), "disable_web_page_preview": true}
+	if t.ThreadID != "" {
+		if id, err := strconv.Atoi(t.ThreadID); err == nil {
+			body["message_thread_id"] = id
+		}
+	}
+	return postJSON(ctx, t.client(), api+"/bot"+t.Token+"/sendMessage", nil, body, t.Token)
+}
+
+func (t *Telegram) client() *http.Client {
+	if t.Client != nil {
+		return t.Client
+	}
+	return &http.Client{Timeout: 15 * time.Second}
+}
+
+// Webhook posts {"text": ..., "alert": ...} to any URL, optionally with a
+// shared secret in the X-Alert-Secret header.
+type Webhook struct {
+	URL     string
+	Secret  string
+	BaseURL string
+	Client  *http.Client
+}
+
+func (w *Webhook) Name() string { return "webhook" }
+
+func (w *Webhook) Send(ctx context.Context, a Alert) error {
+	client := w.Client
+	if client == nil {
+		client = &http.Client{Timeout: 15 * time.Second}
+	}
+	headers := map[string]string{}
+	if w.Secret != "" {
+		headers["X-Alert-Secret"] = w.Secret
+	}
+	return postJSON(ctx, client, w.URL, headers, map[string]any{"text": a.Text(w.BaseURL), "alert": a}, w.URL, w.Secret)
+}
+
+// postJSON posts a JSON body. Errors are rebuilt without the request URL and
+// scrubbed of every secret, because both the Telegram token and webhook URLs
+// commonly carry credentials.
+func postJSON(ctx context.Context, client *http.Client, target string, headers map[string]string, body any, secrets ...string) error {
+	raw, err := json.Marshal(body)
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, bytes.NewReader(raw))
+	if err != nil {
+		return errors.New("invalid request")
+	}
+	req.Header.Set("Content-Type", "application/json")
+	for k, v := range headers {
+		req.Header.Set(k, v)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		var ue *url.Error
+		if errors.As(err, &ue) {
+			err = ue.Err // drop the URL the transport put in the message
+		}
+		return errors.New(scrub(err.Error(), secrets))
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode/100 != 2 {
+		detail, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
+		return fmt.Errorf("HTTP %d: %s", resp.StatusCode, scrub(strings.TrimSpace(string(detail)), secrets))
+	}
+	return nil
+}
+
+func scrub(s string, secrets []string) string {
+	for _, sec := range secrets {
+		if sec != "" {
+			s = strings.ReplaceAll(s, sec, "<redacted>")
+		}
+	}
+	return s
+}

internal/alerts/runner.go

@@ -0,0 +1,281 @@
+package alerts
+
+import (
+	"context"
+	"encoding/json"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+
+	"git.tornberg.me/go-otel/internal/features"
+)
+
+// Config is read from OTEL_ALERT_* environment variables. The runner starts
+// when at least one notifier is configured, or when OTEL_ALERT_DRY_RUN=true
+// (evaluate and record, send nothing).
+type Config struct {
+	DryRun          bool
+	Interval        time.Duration // how often the detector runs
+	Window          time.Duration // observation window
+	BaselineWindows int
+	Cooldown        time.Duration // suppress repeats of a pattern unless it doubles
+	MinExcess       int
+	Multiple        float64
+	Lag             time.Duration // ingest settle time: windows end this long before now
+	BaseURL         string        // UI link appended to messages, e.g. https://otel.k6n.net
+
+	TelegramToken, TelegramChatID, TelegramThreadID string
+	WebhookURL, WebhookSecret                       string
+}
+
+func ConfigFromEnv() Config {
+	return Config{
+		DryRun:          os.Getenv("OTEL_ALERT_DRY_RUN") == "true",
+		Interval:        envDuration("OTEL_ALERT_INTERVAL", time.Minute),
+		Window:          envDuration("OTEL_ALERT_WINDOW", 5*time.Minute),
+		BaselineWindows: envInt("OTEL_ALERT_BASELINE_WINDOWS", 36),
+		Cooldown:        envDuration("OTEL_ALERT_COOLDOWN", 30*time.Minute),
+		MinExcess:       envInt("OTEL_ALERT_MIN_EXCESS", 10),
+		Multiple:        3,
+		Lag:             envDuration("OTEL_ALERT_LAG", 30*time.Second),
+		BaseURL:         os.Getenv("OTEL_ALERT_BASE_URL"),
+
+		TelegramToken:    firstEnv("OTEL_ALERT_TELEGRAM_TOKEN", "TELEGRAM_KEY"),
+		TelegramChatID:   firstEnv("OTEL_ALERT_TELEGRAM_CHAT_ID", "TELEGRAM_CHATID"),
+		TelegramThreadID: os.Getenv("OTEL_ALERT_TELEGRAM_THREAD_ID"),
+		WebhookURL:       os.Getenv("OTEL_ALERT_WEBHOOK_URL"),
+		WebhookSecret:    os.Getenv("OTEL_ALERT_WEBHOOK_SECRET"),
+	}
+}
+
+// firstEnv returns the first non-empty variable; the plain TELEGRAM_* names are
+// what the repo's .env uses, the OTEL_ALERT_* ones are for the deployment.
+func firstEnv(keys ...string) string {
+	for _, k := range keys {
+		if v := os.Getenv(k); v != "" {
+			return v
+		}
+	}
+	return ""
+}
+
+func envDuration(k string, def time.Duration) time.Duration {
+	if d, err := time.ParseDuration(os.Getenv(k)); err == nil && d > 0 {
+		return d
+	}
+	return def
+}
+func envInt(k string, def int) int {
+	if n, err := strconv.Atoi(os.Getenv(k)); err == nil && n > 0 {
+		return n
+	}
+	return def
+}
+
+// Notifiers builds the configured delivery channels.
+func (c Config) Notifiers() []Notifier {
+	var out []Notifier
+	if c.TelegramToken != "" && c.TelegramChatID != "" {
+		out = append(out, &Telegram{Token: c.TelegramToken, ChatID: c.TelegramChatID, ThreadID: c.TelegramThreadID, BaseURL: c.BaseURL})
+	}
+	if c.WebhookURL != "" {
+		out = append(out, &Webhook{URL: c.WebhookURL, Secret: c.WebhookSecret, BaseURL: c.BaseURL})
+	}
+	return out
+}
+
+// Loader returns error events in [start, end); it is a field so tests can
+// feed events without ClickHouse.
+type Loader func(ctx context.Context, start, end time.Time) ([]features.Event, error)
+
+type Runner struct {
+	cfg       Config
+	load      Loader
+	dataSince func(ctx context.Context) time.Time
+	notifiers []Notifier
+	sup       *features.Suppressor
+
+	mu        sync.Mutex
+	primed    bool
+	recent    []Alert // newest last
+	lastRun   time.Time
+	lastError string
+	since     time.Time
+	sinceAt   time.Time
+}
+
+const maxRecent = 50
+
+// Start launches the runner against ClickHouse and returns it, or nil when
+// alerting is not configured.
+func Start(ctx context.Context, cfg Config, conn driver.Conn, database string) *Runner {
+	notifiers := cfg.Notifiers()
+	if len(notifiers) == 0 && !cfg.DryRun {
+		log.Printf("alerts: disabled (set OTEL_ALERT_TELEGRAM_TOKEN + OTEL_ALERT_TELEGRAM_CHAT_ID and/or OTEL_ALERT_WEBHOOK_URL)")
+		return nil
+	}
+	r := &Runner{
+		cfg:       cfg,
+		notifiers: notifiers,
+		sup:       features.NewSuppressor(cfg.Cooldown),
+		load: func(ctx context.Context, start, end time.Time) ([]features.Event, error) {
+			return LoadErrorEvents(ctx, conn, database, start, end)
+		},
+		dataSince: func(ctx context.Context) time.Time { return DataSince(ctx, conn, database) },
+	}
+	names := make([]string, 0, len(notifiers))
+	for _, n := range notifiers {
+		names = append(names, n.Name())
+	}
+	log.Printf("alerts: enabled every %s (window %s, baseline %d windows, dry-run=%v, channels=%v)", cfg.Interval, cfg.Window, cfg.BaselineWindows, cfg.DryRun, names)
+	go r.loop(ctx)
+	return r
+}
+
+func (r *Runner) loop(ctx context.Context) {
+	t := time.NewTicker(r.cfg.Interval)
+	defer t.Stop()
+	for {
+		r.Tick(ctx, time.Now())
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+		}
+	}
+}
+
+// Tick runs one evaluation. The first tick after a start only primes the
+// suppressor, so a deploy in the middle of an incident does not re-page it.
+func (r *Runner) Tick(ctx context.Context, now time.Time) {
+	cctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
+	defer cancel()
+	end := now.Add(-r.cfg.Lag).UTC()
+	start := end.Add(-time.Duration(r.cfg.BaselineWindows+1) * r.cfg.Window)
+	events, err := r.load(cctx, start, end)
+	r.mu.Lock()
+	r.lastRun = now
+	if err != nil {
+		r.lastError = err.Error()
+		r.mu.Unlock()
+		log.Printf("alerts: load failed: %v", err)
+		return
+	}
+	r.lastError = ""
+	if r.since.IsZero() || now.Sub(r.sinceAt) > time.Hour {
+		r.mu.Unlock()
+		since := r.dataSince(cctx)
+		r.mu.Lock()
+		r.since, r.sinceAt = since, now
+	}
+	since, primed := r.since, r.primed
+	r.mu.Unlock()
+
+	spikes := DetectAlerts(events, end, features.SpikeConfig{
+		Window: r.cfg.Window, BaselineWindows: r.cfg.BaselineWindows, Multiple: r.cfg.Multiple, MinExcess: r.cfg.MinExcess, DataSince: since,
+	})
+	spikes = r.sup.Filter(spikes)
+
+	r.mu.Lock()
+	r.primed = true
+	r.mu.Unlock()
+	if !primed {
+		if len(spikes) > 0 {
+			log.Printf("alerts: primed with %d ongoing spike(s), not sent", len(spikes))
+		}
+		return
+	}
+	for _, spike := range spikes {
+		r.deliver(cctx, NewAlert(spike, end))
+	}
+}
+
+// DetectAlerts is the detection half of a tick: spikes reduced to actionable
+// patterns. Exported so the CLI and the runner cannot drift apart.
+func DetectAlerts(events []features.Event, at time.Time, cfg features.SpikeConfig) []features.Spike {
+	return FilterActionable(features.DetectSpikes(events, at, cfg), false, nil)
+}
+
+func (r *Runner) deliver(ctx context.Context, a Alert) {
+	r.mu.Lock()
+	r.recent = append(r.recent, a)
+	if len(r.recent) > maxRecent {
+		r.recent = r.recent[len(r.recent)-maxRecent:]
+	}
+	r.mu.Unlock()
+	if r.cfg.DryRun {
+		log.Printf("alerts[dry-run]: %s", a.Text(r.cfg.BaseURL))
+		return
+	}
+	for _, n := range r.notifiers {
+		err := n.Send(ctx, a)
+		if err != nil {
+			time.Sleep(2 * time.Second)
+			err = n.Send(ctx, a)
+		}
+		if err != nil {
+			log.Printf("alerts: %s delivery failed for %s: %v", n.Name(), a.ID, err)
+		}
+	}
+}
+
+// Status is the JSON served by GET /api/alerts.
+type Status struct {
+	Enabled   bool      `json:"enabled"`
+	DryRun    bool      `json:"dryRun"`
+	Channels  []string  `json:"channels"`
+	LastRun   time.Time `json:"lastRun"`
+	LastError string    `json:"lastError,omitempty"`
+	Alerts    []Alert   `json:"alerts"`
+}
+
+func (r *Runner) Status() Status {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	s := Status{Enabled: true, DryRun: r.cfg.DryRun, LastRun: r.lastRun, LastError: r.lastError, Channels: []string{}}
+	for _, n := range r.notifiers {
+		s.Channels = append(s.Channels, n.Name())
+	}
+	s.Alerts = make([]Alert, len(r.recent))
+	for i, a := range r.recent { // newest first
+		s.Alerts[len(r.recent)-1-i] = a
+	}
+	return s
+}
+
+// HandleStatus serves recent alerts and runner health.
+func (r *Runner) HandleStatus(w http.ResponseWriter, req *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Cache-Control", "no-store")
+	_ = json.NewEncoder(w).Encode(r.Status())
+}
+
+// HandleTest sends a clearly marked test alert to every channel, so a new
+// token or chat id can be verified without waiting for a real incident.
+func (r *Runner) HandleTest(w http.ResponseWriter, req *http.Request) {
+	if req.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	now := time.Now().UTC()
+	a := NewAlert(features.Spike{Service: "alert-test", WindowStart: now.Add(-r.cfg.Window), WindowEnd: now, ErrorCount: 42, BaselineMean: 1, Excess: 41, Patterns: []features.PatternDelta{{
+		Signature: "test", Count: 42, BaselineMean: 1, Excess: 41, Novel: true, Clients: 3,
+		Example: features.Event{Service: "alert-test", Kind: "span", StatusCode: 500, Route: "/api/example", Message: "this is a test alert"},
+	}}}, now)
+	a.Test = true
+	results := map[string]string{}
+	for _, n := range r.notifiers {
+		if err := n.Send(req.Context(), a); err != nil {
+			results[n.Name()] = err.Error()
+		} else {
+			results[n.Name()] = "ok"
+		}
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(results)
+}