package alerts import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "strconv" "strings" "time" "git.tornberg.me/go-otel/internal/features" "git.tornberg.me/go-otel/internal/uaclass" ) // Alert is one notification: a spike reduced to its actionable patterns. type Alert struct { ID string `json:"id"` At time.Time `json:"at"` Spike features.Spike `json:"spike"` Patterns []PatternReport `json:"patterns"` Test bool `json:"test,omitempty"` } // PatternReport is a spike pattern with its triage verdict, ready to show. type PatternReport struct { Category string `json:"category"` Rule string `json:"rule"` New bool `json:"new"` Count int `json:"count"` Baseline float64 `json:"baseline"` Clients int `json:"clients,omitempty"` UAClass string `json:"uaClass,omitempty"` What string `json:"what"` } // Notifier delivers an alert somewhere. Implementations must not put secrets // in returned errors. type Notifier interface { Name() string Send(ctx context.Context, a Alert) error } // NewAlert builds an Alert from a spike, classifying each pattern. func NewAlert(spike features.Spike, at time.Time) Alert { a := Alert{At: at, Spike: spike, ID: alertID(spike, at)} for _, p := range spike.Patterns { res := TriageOf(p.Example) r := PatternReport{Category: string(res.Category), Rule: res.Rule, New: p.Novel, Count: p.Count, Baseline: p.BaselineMean, Clients: p.Clients, What: describe(p.Example)} if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) { r.UAClass = p.Example.UAClass } a.Patterns = append(a.Patterns, r) } return a } func alertID(s features.Spike, at time.Time) string { return strconv.FormatInt(at.Unix(), 36) + "-" + strings.ReplaceAll(s.Service, " ", "_") } func describe(e features.Event) string { parts := []string{} if e.StatusCode != 0 { parts = append(parts, "HTTP "+strconv.Itoa(e.StatusCode)) } switch { case e.Route != "": parts = append(parts, e.Route) case e.SpanName != "": parts = append(parts, e.SpanName) } if e.ExceptionType != "" { parts = append(parts, e.ExceptionType) } if e.Message != "" && e.Message != e.SpanName { parts = append(parts, truncate(e.Message, 120)) } if len(parts) == 0 { return "(no detail)" } return strings.Join(parts, " | ") } func truncate(s string, n int) string { r := []rune(s) if len(r) <= n { return s } return string(r[:n-1]) + "…" } // Text renders a plain-text message (no markup, so nothing needs escaping). func (a Alert) Text(baseURL string) string { var b strings.Builder prefix := "ALERT" if a.Test { prefix = "TEST" } fmt.Fprintf(&b, "%s %s: %d errors in %s (baseline %.1f/window, excess %.0f)\n", prefix, a.Spike.Service, a.Spike.ErrorCount, a.Spike.WindowEnd.Sub(a.Spike.WindowStart).Round(time.Minute), a.Spike.BaselineMean, a.Spike.Excess) for _, p := range a.Patterns { tag := "rising" if p.New { tag = "NEW" } fmt.Fprintf(&b, "\n%s x%d (base %.1f) [%s]\n%s", tag, p.Count, p.Baseline, p.Category, p.What) var extra []string if p.Clients > 0 { extra = append(extra, fmt.Sprintf("%d client%s", p.Clients, plural(p.Clients))) } if p.UAClass != "" { extra = append(extra, p.UAClass) } if len(extra) > 0 { fmt.Fprintf(&b, "\n(%s)", strings.Join(extra, ", ")) } b.WriteString("\n") } if baseURL != "" { fmt.Fprintf(&b, "\n%s/issues", strings.TrimRight(baseURL, "/")) } return strings.TrimSpace(b.String()) } func plural(n int) string { if n == 1 { return "" } return "s" } // Telegram posts via the Bot API sendMessage method. type Telegram struct { Token string ChatID string ThreadID string // optional forum topic BaseURL string // link appended to messages API string // override for tests; default https://api.telegram.org Client *http.Client } func (t *Telegram) Name() string { return "telegram" } // telegram limits a message to 4096 characters. const telegramMax = 4000 func (t *Telegram) Send(ctx context.Context, a Alert) error { api := t.API if api == "" { api = "https://api.telegram.org" } body := map[string]any{"chat_id": t.ChatID, "text": truncate(a.Text(t.BaseURL), telegramMax), "disable_web_page_preview": true} if t.ThreadID != "" { if id, err := strconv.Atoi(t.ThreadID); err == nil { body["message_thread_id"] = id } } return postJSON(ctx, t.client(), api+"/bot"+t.Token+"/sendMessage", nil, body, t.Token) } func (t *Telegram) client() *http.Client { if t.Client != nil { return t.Client } return &http.Client{Timeout: 15 * time.Second} } // Webhook posts {"text": ..., "alert": ...} to any URL, optionally with a // shared secret in the X-Alert-Secret header. type Webhook struct { URL string Secret string BaseURL string Client *http.Client } func (w *Webhook) Name() string { return "webhook" } func (w *Webhook) Send(ctx context.Context, a Alert) error { client := w.Client if client == nil { client = &http.Client{Timeout: 15 * time.Second} } headers := map[string]string{} if w.Secret != "" { headers["X-Alert-Secret"] = w.Secret } return postJSON(ctx, client, w.URL, headers, map[string]any{"text": a.Text(w.BaseURL), "alert": a}, w.URL, w.Secret) } // postJSON posts a JSON body. Errors are rebuilt without the request URL and // scrubbed of every secret, because both the Telegram token and webhook URLs // commonly carry credentials. func postJSON(ctx context.Context, client *http.Client, target string, headers map[string]string, body any, secrets ...string) error { raw, err := json.Marshal(body) if err != nil { return err } req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, bytes.NewReader(raw)) if err != nil { return errors.New("invalid request") } req.Header.Set("Content-Type", "application/json") for k, v := range headers { req.Header.Set(k, v) } resp, err := client.Do(req) if err != nil { var ue *url.Error if errors.As(err, &ue) { err = ue.Err // drop the URL the transport put in the message } return errors.New(scrub(err.Error(), secrets)) } defer resp.Body.Close() if resp.StatusCode/100 != 2 { detail, _ := io.ReadAll(io.LimitReader(resp.Body, 300)) return fmt.Errorf("HTTP %d: %s", resp.StatusCode, scrub(strings.TrimSpace(string(detail)), secrets)) } return nil } func scrub(s string, secrets []string) string { for _, sec := range secrets { if sec != "" { s = strings.ReplaceAll(s, sec, "") } } return s }