include db for gitea build
This commit is contained in:
mats committed
1 parent
8de4a2e010
commit
645f923175
33 files changed
+1490
-150
No files matched your search
@@ -0,0 +1,235 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.tornberg.me/go-otel/internal/features"
|
||||
"git.tornberg.me/go-otel/internal/uaclass"
|
||||
)
|
||||
|
||||
// Alert is one notification: a spike reduced to its actionable patterns.
|
||||
type Alert struct {
|
||||
ID string `json:"id"`
|
||||
At time.Time `json:"at"`
|
||||
Spike features.Spike `json:"spike"`
|
||||
Patterns []PatternReport `json:"patterns"`
|
||||
Test bool `json:"test,omitempty"`
|
||||
}
|
||||
|
||||
// PatternReport is a spike pattern with its triage verdict, ready to show.
|
||||
type PatternReport struct {
|
||||
Category string `json:"category"`
|
||||
Rule string `json:"rule"`
|
||||
New bool `json:"new"`
|
||||
Count int `json:"count"`
|
||||
Baseline float64 `json:"baseline"`
|
||||
Clients int `json:"clients,omitempty"`
|
||||
UAClass string `json:"uaClass,omitempty"`
|
||||
What string `json:"what"`
|
||||
}
|
||||
|
||||
// Notifier delivers an alert somewhere. Implementations must not put secrets
|
||||
// in returned errors.
|
||||
type Notifier interface {
|
||||
Name() string
|
||||
Send(ctx context.Context, a Alert) error
|
||||
}
|
||||
|
||||
// NewAlert builds an Alert from a spike, classifying each pattern.
|
||||
func NewAlert(spike features.Spike, at time.Time) Alert {
|
||||
a := Alert{At: at, Spike: spike, ID: alertID(spike, at)}
|
||||
for _, p := range spike.Patterns {
|
||||
res := TriageOf(p.Example)
|
||||
r := PatternReport{Category: string(res.Category), Rule: res.Rule, New: p.Novel, Count: p.Count, Baseline: p.BaselineMean, Clients: p.Clients, What: describe(p.Example)}
|
||||
if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) {
|
||||
r.UAClass = p.Example.UAClass
|
||||
}
|
||||
a.Patterns = append(a.Patterns, r)
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
func alertID(s features.Spike, at time.Time) string {
|
||||
return strconv.FormatInt(at.Unix(), 36) + "-" + strings.ReplaceAll(s.Service, " ", "_")
|
||||
}
|
||||
|
||||
func describe(e features.Event) string {
|
||||
parts := []string{}
|
||||
if e.StatusCode != 0 {
|
||||
parts = append(parts, "HTTP "+strconv.Itoa(e.StatusCode))
|
||||
}
|
||||
switch {
|
||||
case e.Route != "":
|
||||
parts = append(parts, e.Route)
|
||||
case e.SpanName != "":
|
||||
parts = append(parts, e.SpanName)
|
||||
}
|
||||
if e.ExceptionType != "" {
|
||||
parts = append(parts, e.ExceptionType)
|
||||
}
|
||||
if e.Message != "" && e.Message != e.SpanName {
|
||||
parts = append(parts, truncate(e.Message, 120))
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "(no detail)"
|
||||
}
|
||||
return strings.Join(parts, " | ")
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if len(r) <= n {
|
||||
return s
|
||||
}
|
||||
return string(r[:n-1]) + "…"
|
||||
}
|
||||
|
||||
// Text renders a plain-text message (no markup, so nothing needs escaping).
|
||||
func (a Alert) Text(baseURL string) string {
|
||||
var b strings.Builder
|
||||
prefix := "ALERT"
|
||||
if a.Test {
|
||||
prefix = "TEST"
|
||||
}
|
||||
fmt.Fprintf(&b, "%s %s: %d errors in %s (baseline %.1f/window, excess %.0f)\n",
|
||||
prefix, a.Spike.Service, a.Spike.ErrorCount, a.Spike.WindowEnd.Sub(a.Spike.WindowStart).Round(time.Minute), a.Spike.BaselineMean, a.Spike.Excess)
|
||||
for _, p := range a.Patterns {
|
||||
tag := "rising"
|
||||
if p.New {
|
||||
tag = "NEW"
|
||||
}
|
||||
fmt.Fprintf(&b, "\n%s x%d (base %.1f) [%s]\n%s", tag, p.Count, p.Baseline, p.Category, p.What)
|
||||
var extra []string
|
||||
if p.Clients > 0 {
|
||||
extra = append(extra, fmt.Sprintf("%d client%s", p.Clients, plural(p.Clients)))
|
||||
}
|
||||
if p.UAClass != "" {
|
||||
extra = append(extra, p.UAClass)
|
||||
}
|
||||
if len(extra) > 0 {
|
||||
fmt.Fprintf(&b, "\n(%s)", strings.Join(extra, ", "))
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
if baseURL != "" {
|
||||
fmt.Fprintf(&b, "\n%s/issues", strings.TrimRight(baseURL, "/"))
|
||||
}
|
||||
return strings.TrimSpace(b.String())
|
||||
}
|
||||
|
||||
func plural(n int) string {
|
||||
if n == 1 {
|
||||
return ""
|
||||
}
|
||||
return "s"
|
||||
}
|
||||
|
||||
// Telegram posts via the Bot API sendMessage method.
|
||||
type Telegram struct {
|
||||
Token string
|
||||
ChatID string
|
||||
ThreadID string // optional forum topic
|
||||
BaseURL string // link appended to messages
|
||||
API string // override for tests; default https://api.telegram.org
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
func (t *Telegram) Name() string { return "telegram" }
|
||||
|
||||
// telegram limits a message to 4096 characters.
|
||||
const telegramMax = 4000
|
||||
|
||||
func (t *Telegram) Send(ctx context.Context, a Alert) error {
|
||||
api := t.API
|
||||
if api == "" {
|
||||
api = "https://api.telegram.org"
|
||||
}
|
||||
body := map[string]any{"chat_id": t.ChatID, "text": truncate(a.Text(t.BaseURL), telegramMax), "disable_web_page_preview": true}
|
||||
if t.ThreadID != "" {
|
||||
if id, err := strconv.Atoi(t.ThreadID); err == nil {
|
||||
body["message_thread_id"] = id
|
||||
}
|
||||
}
|
||||
return postJSON(ctx, t.client(), api+"/bot"+t.Token+"/sendMessage", nil, body, t.Token)
|
||||
}
|
||||
|
||||
func (t *Telegram) client() *http.Client {
|
||||
if t.Client != nil {
|
||||
return t.Client
|
||||
}
|
||||
return &http.Client{Timeout: 15 * time.Second}
|
||||
}
|
||||
|
||||
// Webhook posts {"text": ..., "alert": ...} to any URL, optionally with a
|
||||
// shared secret in the X-Alert-Secret header.
|
||||
type Webhook struct {
|
||||
URL string
|
||||
Secret string
|
||||
BaseURL string
|
||||
Client *http.Client
|
||||
}
|
||||
|
||||
func (w *Webhook) Name() string { return "webhook" }
|
||||
|
||||
func (w *Webhook) Send(ctx context.Context, a Alert) error {
|
||||
client := w.Client
|
||||
if client == nil {
|
||||
client = &http.Client{Timeout: 15 * time.Second}
|
||||
}
|
||||
headers := map[string]string{}
|
||||
if w.Secret != "" {
|
||||
headers["X-Alert-Secret"] = w.Secret
|
||||
}
|
||||
return postJSON(ctx, client, w.URL, headers, map[string]any{"text": a.Text(w.BaseURL), "alert": a}, w.URL, w.Secret)
|
||||
}
|
||||
|
||||
// postJSON posts a JSON body. Errors are rebuilt without the request URL and
|
||||
// scrubbed of every secret, because both the Telegram token and webhook URLs
|
||||
// commonly carry credentials.
|
||||
func postJSON(ctx context.Context, client *http.Client, target string, headers map[string]string, body any, secrets ...string) error {
|
||||
raw, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return errors.New("invalid request")
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
var ue *url.Error
|
||||
if errors.As(err, &ue) {
|
||||
err = ue.Err // drop the URL the transport put in the message
|
||||
}
|
||||
return errors.New(scrub(err.Error(), secrets))
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
detail, _ := io.ReadAll(io.LimitReader(resp.Body, 300))
|
||||
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, scrub(strings.TrimSpace(string(detail)), secrets))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func scrub(s string, secrets []string) string {
|
||||
for _, sec := range secrets {
|
||||
if sec != "" {
|
||||
s = strings.ReplaceAll(s, sec, "<redacted>")
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
Reference in new issue
Block a user