236 lines
6.4 KiB
Go
236 lines
6.4 KiB
Go
package alerts
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.tornberg.me/go-otel/internal/features"
|
|
"git.tornberg.me/go-otel/internal/uaclass"
|
|
)
|
|
|
|
// Alert is one notification: a spike reduced to its actionable patterns.
|
|
type Alert struct {
|
|
ID string `json:"id"`
|
|
At time.Time `json:"at"`
|
|
Spike features.Spike `json:"spike"`
|
|
Patterns []PatternReport `json:"patterns"`
|
|
Test bool `json:"test,omitempty"`
|
|
}
|
|
|
|
// PatternReport is a spike pattern with its triage verdict, ready to show.
|
|
type PatternReport struct {
|
|
Category string `json:"category"`
|
|
Rule string `json:"rule"`
|
|
New bool `json:"new"`
|
|
Count int `json:"count"`
|
|
Baseline float64 `json:"baseline"`
|
|
Clients int `json:"clients,omitempty"`
|
|
UAClass string `json:"uaClass,omitempty"`
|
|
What string `json:"what"`
|
|
}
|
|
|
|
// Notifier delivers an alert somewhere. Implementations must not put secrets
|
|
// in returned errors.
|
|
type Notifier interface {
|
|
Name() string
|
|
Send(ctx context.Context, a Alert) error
|
|
}
|
|
|
|
// NewAlert builds an Alert from a spike, classifying each pattern.
|
|
func NewAlert(spike features.Spike, at time.Time) Alert {
|
|
a := Alert{At: at, Spike: spike, ID: alertID(spike, at)}
|
|
for _, p := range spike.Patterns {
|
|
res := TriageOf(p.Example)
|
|
r := PatternReport{Category: string(res.Category), Rule: res.Rule, New: p.Novel, Count: p.Count, Baseline: p.BaselineMean, Clients: p.Clients, What: describe(p.Example)}
|
|
if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) {
|
|
r.UAClass = p.Example.UAClass
|
|
}
|
|
a.Patterns = append(a.Patterns, r)
|
|
}
|
|
return a
|
|
}
|
|
|
|
func alertID(s features.Spike, at time.Time) string {
|
|
return strconv.FormatInt(at.Unix(), 36) + "-" + strings.ReplaceAll(s.Service, " ", "_")
|
|
}
|
|
|
|
func describe(e features.Event) string {
|
|
parts := []string{}
|
|
if e.StatusCode != 0 {
|
|
parts = append(parts, "HTTP "+strconv.Itoa(e.StatusCode))
|
|
}
|
|
switch {
|
|
case e.Route != "":
|
|
parts = append(parts, e.Route)
|
|
case e.SpanName != "":
|
|
parts = append(parts, e.SpanName)
|
|
}
|
|
if e.ExceptionType != "" {
|
|
parts = append(parts, e.ExceptionType)
|
|
}
|
|
if e.Message != "" && e.Message != e.SpanName {
|
|
parts = append(parts, truncate(e.Message, 120))
|
|
}
|
|
if len(parts) == 0 {
|
|
return "(no detail)"
|
|
}
|
|
return strings.Join(parts, " | ")
|
|
}
|
|
|
|
func truncate(s string, n int) string {
|
|
r := []rune(s)
|
|
if len(r) <= n {
|
|
return s
|
|
}
|
|
return string(r[:n-1]) + "…"
|
|
}
|
|
|
|
// Text renders a plain-text message (no markup, so nothing needs escaping).
|
|
func (a Alert) Text(baseURL string) string {
|
|
var b strings.Builder
|
|
prefix := "ALERT"
|
|
if a.Test {
|
|
prefix = "TEST"
|
|
}
|
|
fmt.Fprintf(&b, "%s %s: %d errors in %s (baseline %.1f/window, excess %.0f)\n",
|
|
prefix, a.Spike.Service, a.Spike.ErrorCount, a.Spike.WindowEnd.Sub(a.Spike.WindowStart).Round(time.Minute), a.Spike.BaselineMean, a.Spike.Excess)
|
|
for _, p := range a.Patterns {
|
|
tag := "rising"
|
|
if p.New {
|
|
tag = "NEW"
|
|
}
|
|
fmt.Fprintf(&b, "\n%s x%d (base %.1f) [%s]\n%s", tag, p.Count, p.Baseline, p.Category, p.What)
|
|
var extra []string
|
|
if p.Clients > 0 {
|
|
extra = append(extra, fmt.Sprintf("%d client%s", p.Clients, plural(p.Clients)))
|
|
}
|
|
if p.UAClass != "" {
|
|
extra = append(extra, p.UAClass)
|
|
}
|
|
if len(extra) > 0 {
|
|
fmt.Fprintf(&b, "\n(%s)", strings.Join(extra, ", "))
|
|
}
|
|
b.WriteString("\n")
|
|
}
|
|
if baseURL != "" {
|
|
fmt.Fprintf(&b, "\n%s/issues", strings.TrimRight(baseURL, "/"))
|
|
}
|
|
return strings.TrimSpace(b.String())
|
|
}
|
|
|
|
func plural(n int) string {
|
|
if n == 1 {
|
|
return ""
|
|
}
|
|
return "s"
|
|
}
|
|
|
|
// Telegram posts via the Bot API sendMessage method.
|
|
type Telegram struct {
|
|
Token string
|
|
ChatID string
|
|
ThreadID string // optional forum topic
|
|
BaseURL string // link appended to messages
|
|
API string // override for tests; default https://api.telegram.org
|
|
Client *http.Client
|
|
}
|
|
|
|
func (t *Telegram) Name() string { return "telegram" }
|
|
|
|
// telegram limits a message to 4096 characters.
|
|
const telegramMax = 4000
|
|
|
|
func (t *Telegram) Send(ctx context.Context, a Alert) error {
|
|
api := t.API
|
|
if api == "" {
|
|
api = "https://api.telegram.org"
|
|
}
|
|
body := map[string]any{"chat_id": t.ChatID, "text": truncate(a.Text(t.BaseURL), telegramMax), "disable_web_page_preview": true}
|
|
if t.ThreadID != "" {
|
|
if id, err := strconv.Atoi(t.ThreadID); err == nil {
|
|
body["message_thread_id"] = id
|
|
}
|
|
}
|
|
return postJSON(ctx, t.client(), api+"/bot"+t.Token+"/sendMessage", nil, body, t.Token)
|
|
}
|
|
|
|
func (t *Telegram) client() *http.Client {
|
|
if t.Client != nil {
|
|
return t.Client
|
|
}
|
|
return &http.Client{Timeout: 15 * time.Second}
|
|
}
|
|
|
|
// Webhook posts {"text": ..., "alert": ...} to any URL, optionally with a
|
|
// shared secret in the X-Alert-Secret header.
|
|
type Webhook struct {
|
|
URL string
|
|
Secret string
|
|
BaseURL string
|
|
Client *http.Client
|
|
}
|
|
|
|
func (w *Webhook) Name() string { return "webhook" }
|
|
|
|
func (w *Webhook) Send(ctx context.Context, a Alert) error {
|
|
client := w.Client
|
|
if client == nil {
|
|
client = &http.Client{Timeout: 15 * time.Second}
|
|
}
|
|
headers := map[string]string{}
|
|
if w.Secret != "" {
|
|
headers["X-Alert-Secret"] = w.Secret
|
|
}
|
|
return postJSON(ctx, client, w.URL, headers, map[string]any{"text": a.Text(w.BaseURL), "alert": a}, w.URL, w.Secret)
|
|
}
|
|
|
|
// postJSON posts a JSON body. Errors are rebuilt without the request URL and
|
|
// scrubbed of every secret, because both the Telegram token and webhook URLs
|
|
// commonly carry credentials.
|
|
func postJSON(ctx context.Context, client *http.Client, target string, headers map[string]string, body any, secrets ...string) error {
|
|
raw, err := json.Marshal(body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, bytes.NewReader(raw))
|
|
if err != nil {
|
|
return errors.New("invalid request")
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
for k, v := range headers {
|
|
req.Header.Set(k, v)
|
|
}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
var ue *url.Error
|
|
if errors.As(err, &ue) {
|
|
err = ue.Err // drop the URL the transport put in the message
|
|
}
|
|
return errors.New(scrub(err.Error(), secrets))
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode/100 != 2 {
|
|
detail, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
|
|
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, scrub(strings.TrimSpace(string(detail)), secrets))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func scrub(s string, secrets []string) string {
|
|
for _, sec := range secrets {
|
|
if sec != "" {
|
|
s = strings.ReplaceAll(s, sec, "<redacted>")
|
|
}
|
|
}
|
|
return s
|
|
}
|