include db for gitea build
This commit is contained in:
mats committed
1 parent
8de4a2e010
commit
645f923175
33 files changed
+1490
-150
No files matched your search
@@ -0,0 +1,281 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
|
||||
"git.tornberg.me/go-otel/internal/features"
|
||||
)
|
||||
|
||||
// Config is read from OTEL_ALERT_* environment variables. The runner starts
|
||||
// when at least one notifier is configured, or when OTEL_ALERT_DRY_RUN=true
|
||||
// (evaluate and record, send nothing).
|
||||
type Config struct {
|
||||
DryRun bool
|
||||
Interval time.Duration // how often the detector runs
|
||||
Window time.Duration // observation window
|
||||
BaselineWindows int
|
||||
Cooldown time.Duration // suppress repeats of a pattern unless it doubles
|
||||
MinExcess int
|
||||
Multiple float64
|
||||
Lag time.Duration // ingest settle time: windows end this long before now
|
||||
BaseURL string // UI link appended to messages, e.g. https://otel.k6n.net
|
||||
|
||||
TelegramToken, TelegramChatID, TelegramThreadID string
|
||||
WebhookURL, WebhookSecret string
|
||||
}
|
||||
|
||||
func ConfigFromEnv() Config {
|
||||
return Config{
|
||||
DryRun: os.Getenv("OTEL_ALERT_DRY_RUN") == "true",
|
||||
Interval: envDuration("OTEL_ALERT_INTERVAL", time.Minute),
|
||||
Window: envDuration("OTEL_ALERT_WINDOW", 5*time.Minute),
|
||||
BaselineWindows: envInt("OTEL_ALERT_BASELINE_WINDOWS", 36),
|
||||
Cooldown: envDuration("OTEL_ALERT_COOLDOWN", 30*time.Minute),
|
||||
MinExcess: envInt("OTEL_ALERT_MIN_EXCESS", 10),
|
||||
Multiple: 3,
|
||||
Lag: envDuration("OTEL_ALERT_LAG", 30*time.Second),
|
||||
BaseURL: os.Getenv("OTEL_ALERT_BASE_URL"),
|
||||
|
||||
TelegramToken: firstEnv("OTEL_ALERT_TELEGRAM_TOKEN", "TELEGRAM_KEY"),
|
||||
TelegramChatID: firstEnv("OTEL_ALERT_TELEGRAM_CHAT_ID", "TELEGRAM_CHATID"),
|
||||
TelegramThreadID: os.Getenv("OTEL_ALERT_TELEGRAM_THREAD_ID"),
|
||||
WebhookURL: os.Getenv("OTEL_ALERT_WEBHOOK_URL"),
|
||||
WebhookSecret: os.Getenv("OTEL_ALERT_WEBHOOK_SECRET"),
|
||||
}
|
||||
}
|
||||
|
||||
// firstEnv returns the first non-empty variable; the plain TELEGRAM_* names are
|
||||
// what the repo's .env uses, the OTEL_ALERT_* ones are for the deployment.
|
||||
func firstEnv(keys ...string) string {
|
||||
for _, k := range keys {
|
||||
if v := os.Getenv(k); v != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func envDuration(k string, def time.Duration) time.Duration {
|
||||
if d, err := time.ParseDuration(os.Getenv(k)); err == nil && d > 0 {
|
||||
return d
|
||||
}
|
||||
return def
|
||||
}
|
||||
func envInt(k string, def int) int {
|
||||
if n, err := strconv.Atoi(os.Getenv(k)); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// Notifiers builds the configured delivery channels.
|
||||
func (c Config) Notifiers() []Notifier {
|
||||
var out []Notifier
|
||||
if c.TelegramToken != "" && c.TelegramChatID != "" {
|
||||
out = append(out, &Telegram{Token: c.TelegramToken, ChatID: c.TelegramChatID, ThreadID: c.TelegramThreadID, BaseURL: c.BaseURL})
|
||||
}
|
||||
if c.WebhookURL != "" {
|
||||
out = append(out, &Webhook{URL: c.WebhookURL, Secret: c.WebhookSecret, BaseURL: c.BaseURL})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Loader returns error events in [start, end); it is a field so tests can
|
||||
// feed events without ClickHouse.
|
||||
type Loader func(ctx context.Context, start, end time.Time) ([]features.Event, error)
|
||||
|
||||
type Runner struct {
|
||||
cfg Config
|
||||
load Loader
|
||||
dataSince func(ctx context.Context) time.Time
|
||||
notifiers []Notifier
|
||||
sup *features.Suppressor
|
||||
|
||||
mu sync.Mutex
|
||||
primed bool
|
||||
recent []Alert // newest last
|
||||
lastRun time.Time
|
||||
lastError string
|
||||
since time.Time
|
||||
sinceAt time.Time
|
||||
}
|
||||
|
||||
const maxRecent = 50
|
||||
|
||||
// Start launches the runner against ClickHouse and returns it, or nil when
|
||||
// alerting is not configured.
|
||||
func Start(ctx context.Context, cfg Config, conn driver.Conn, database string) *Runner {
|
||||
notifiers := cfg.Notifiers()
|
||||
if len(notifiers) == 0 && !cfg.DryRun {
|
||||
log.Printf("alerts: disabled (set OTEL_ALERT_TELEGRAM_TOKEN + OTEL_ALERT_TELEGRAM_CHAT_ID and/or OTEL_ALERT_WEBHOOK_URL)")
|
||||
return nil
|
||||
}
|
||||
r := &Runner{
|
||||
cfg: cfg,
|
||||
notifiers: notifiers,
|
||||
sup: features.NewSuppressor(cfg.Cooldown),
|
||||
load: func(ctx context.Context, start, end time.Time) ([]features.Event, error) {
|
||||
return LoadErrorEvents(ctx, conn, database, start, end)
|
||||
},
|
||||
dataSince: func(ctx context.Context) time.Time { return DataSince(ctx, conn, database) },
|
||||
}
|
||||
names := make([]string, 0, len(notifiers))
|
||||
for _, n := range notifiers {
|
||||
names = append(names, n.Name())
|
||||
}
|
||||
log.Printf("alerts: enabled every %s (window %s, baseline %d windows, dry-run=%v, channels=%v)", cfg.Interval, cfg.Window, cfg.BaselineWindows, cfg.DryRun, names)
|
||||
go r.loop(ctx)
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *Runner) loop(ctx context.Context) {
|
||||
t := time.NewTicker(r.cfg.Interval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
r.Tick(ctx, time.Now())
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tick runs one evaluation. The first tick after a start only primes the
|
||||
// suppressor, so a deploy in the middle of an incident does not re-page it.
|
||||
func (r *Runner) Tick(ctx context.Context, now time.Time) {
|
||||
cctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
end := now.Add(-r.cfg.Lag).UTC()
|
||||
start := end.Add(-time.Duration(r.cfg.BaselineWindows+1) * r.cfg.Window)
|
||||
events, err := r.load(cctx, start, end)
|
||||
r.mu.Lock()
|
||||
r.lastRun = now
|
||||
if err != nil {
|
||||
r.lastError = err.Error()
|
||||
r.mu.Unlock()
|
||||
log.Printf("alerts: load failed: %v", err)
|
||||
return
|
||||
}
|
||||
r.lastError = ""
|
||||
if r.since.IsZero() || now.Sub(r.sinceAt) > time.Hour {
|
||||
r.mu.Unlock()
|
||||
since := r.dataSince(cctx)
|
||||
r.mu.Lock()
|
||||
r.since, r.sinceAt = since, now
|
||||
}
|
||||
since, primed := r.since, r.primed
|
||||
r.mu.Unlock()
|
||||
|
||||
spikes := DetectAlerts(events, end, features.SpikeConfig{
|
||||
Window: r.cfg.Window, BaselineWindows: r.cfg.BaselineWindows, Multiple: r.cfg.Multiple, MinExcess: r.cfg.MinExcess, DataSince: since,
|
||||
})
|
||||
spikes = r.sup.Filter(spikes)
|
||||
|
||||
r.mu.Lock()
|
||||
r.primed = true
|
||||
r.mu.Unlock()
|
||||
if !primed {
|
||||
if len(spikes) > 0 {
|
||||
log.Printf("alerts: primed with %d ongoing spike(s), not sent", len(spikes))
|
||||
}
|
||||
return
|
||||
}
|
||||
for _, spike := range spikes {
|
||||
r.deliver(cctx, NewAlert(spike, end))
|
||||
}
|
||||
}
|
||||
|
||||
// DetectAlerts is the detection half of a tick: spikes reduced to actionable
|
||||
// patterns. Exported so the CLI and the runner cannot drift apart.
|
||||
func DetectAlerts(events []features.Event, at time.Time, cfg features.SpikeConfig) []features.Spike {
|
||||
return FilterActionable(features.DetectSpikes(events, at, cfg), false, nil)
|
||||
}
|
||||
|
||||
func (r *Runner) deliver(ctx context.Context, a Alert) {
|
||||
r.mu.Lock()
|
||||
r.recent = append(r.recent, a)
|
||||
if len(r.recent) > maxRecent {
|
||||
r.recent = r.recent[len(r.recent)-maxRecent:]
|
||||
}
|
||||
r.mu.Unlock()
|
||||
if r.cfg.DryRun {
|
||||
log.Printf("alerts[dry-run]: %s", a.Text(r.cfg.BaseURL))
|
||||
return
|
||||
}
|
||||
for _, n := range r.notifiers {
|
||||
err := n.Send(ctx, a)
|
||||
if err != nil {
|
||||
time.Sleep(2 * time.Second)
|
||||
err = n.Send(ctx, a)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("alerts: %s delivery failed for %s: %v", n.Name(), a.ID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Status is the JSON served by GET /api/alerts.
|
||||
type Status struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
DryRun bool `json:"dryRun"`
|
||||
Channels []string `json:"channels"`
|
||||
LastRun time.Time `json:"lastRun"`
|
||||
LastError string `json:"lastError,omitempty"`
|
||||
Alerts []Alert `json:"alerts"`
|
||||
}
|
||||
|
||||
func (r *Runner) Status() Status {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
s := Status{Enabled: true, DryRun: r.cfg.DryRun, LastRun: r.lastRun, LastError: r.lastError, Channels: []string{}}
|
||||
for _, n := range r.notifiers {
|
||||
s.Channels = append(s.Channels, n.Name())
|
||||
}
|
||||
s.Alerts = make([]Alert, len(r.recent))
|
||||
for i, a := range r.recent { // newest first
|
||||
s.Alerts[len(r.recent)-1-i] = a
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// HandleStatus serves recent alerts and runner health.
|
||||
func (r *Runner) HandleStatus(w http.ResponseWriter, req *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
_ = json.NewEncoder(w).Encode(r.Status())
|
||||
}
|
||||
|
||||
// HandleTest sends a clearly marked test alert to every channel, so a new
|
||||
// token or chat id can be verified without waiting for a real incident.
|
||||
func (r *Runner) HandleTest(w http.ResponseWriter, req *http.Request) {
|
||||
if req.Method != http.MethodPost {
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
a := NewAlert(features.Spike{Service: "alert-test", WindowStart: now.Add(-r.cfg.Window), WindowEnd: now, ErrorCount: 42, BaselineMean: 1, Excess: 41, Patterns: []features.PatternDelta{{
|
||||
Signature: "test", Count: 42, BaselineMean: 1, Excess: 41, Novel: true, Clients: 3,
|
||||
Example: features.Event{Service: "alert-test", Kind: "span", StatusCode: 500, Route: "/api/example", Message: "this is a test alert"},
|
||||
}}}, now)
|
||||
a.Test = true
|
||||
results := map[string]string{}
|
||||
for _, n := range r.notifiers {
|
||||
if err := n.Send(req.Context(), a); err != nil {
|
||||
results[n.Name()] = err.Error()
|
||||
} else {
|
||||
results[n.Name()] = "ok"
|
||||
}
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(results)
|
||||
}
|
||||
Reference in new issue
Block a user