From 5d8bbcd756bbc83858eaecf67595e17516728935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats=20T=C3=B6rnberg?= Date: Wed, 23 Sep 2026 10:58:42 +0200 Subject: [PATCH] display notifications --- cmd/alert-test/main.go | 2 +- internal/alerts/aod.go | 93 +++++++++++++++++++++++++++++++++++++ internal/alerts/aod_test.go | 41 ++++++++++++++++ internal/alerts/runner.go | 5 ++ k8s/deployment.yaml | 3 ++ 5 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 internal/alerts/aod.go create mode 100644 internal/alerts/aod_test.go diff --git a/cmd/alert-test/main.go b/cmd/alert-test/main.go index 0ab2b94..221be32 100644 --- a/cmd/alert-test/main.go +++ b/cmd/alert-test/main.go @@ -17,7 +17,7 @@ func main() { cfg := alerts.ConfigFromEnv() notifiers := cfg.Notifiers() if len(notifiers) == 0 { - fmt.Fprintln(os.Stderr, "no channel configured (TELEGRAM_KEY + TELEGRAM_CHATID, or OTEL_ALERT_WEBHOOK_URL)") + fmt.Fprintln(os.Stderr, "no channel configured (Telegram, OTEL_ALERT_WEBHOOK_URL, or OTEL_ALERT_AOD_URL)") os.Exit(2) } now := time.Now().UTC() diff --git a/internal/alerts/aod.go b/internal/alerts/aod.go new file mode 100644 index 0000000..b684865 --- /dev/null +++ b/internal/alerts/aod.go @@ -0,0 +1,93 @@ +package alerts + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + "time" +) + +// AlwaysOnDisplay sends each alert to the session API used by the wall +// display. A session is created for the alert and then given a status and log +// entry so the display can show both the current incident and its details. +type AlwaysOnDisplay struct { + URL string + Client *http.Client +} + +func (a *AlwaysOnDisplay) Name() string { return "always-on-display" } + +type aodSession struct { + ID string `json:"id"` +} + +func (a *AlwaysOnDisplay) Send(ctx context.Context, alert Alert) error { + client := a.Client + if client == nil { + client = &http.Client{Timeout: 15 * time.Second} + } + base := strings.TrimRight(a.URL, "/") + + var session aodSession + name := fmt.Sprintf("%s alert: %s", strings.ToUpper(string(alert.Priority())), alert.Spike.Service) + if alert.Test { + name = "TEST alert: " + alert.Spike.Service + } + if err := postJSONDecode(ctx, client, base+"/api/sessions", nil, map[string]string{"name": name}, &session); err != nil { + return err + } + if session.ID == "" { + return fmt.Errorf("always-on-display: session response did not include an id") + } + + status := firstLine(alert.Text("")) + if err := postJSON(ctx, client, base+"/api/sessions/"+session.ID+"/status", nil, map[string]string{"status": status}); err != nil { + return err + } + level := "info" + if alert.Priority() == PriorityHigh { + level = "error" + } else if alert.Priority() == PriorityMedium { + level = "warning" + } + return postJSON(ctx, client, base+"/api/sessions/"+session.ID+"/log", nil, map[string]string{ + "level": level, + "message": alert.Text(""), + }) +} + +func firstLine(s string) string { + if i := strings.IndexByte(s, '\n'); i >= 0 { + return s[:i] + } + return s +} + +func postJSONDecode(ctx context.Context, client *http.Client, target string, headers map[string]string, body any, out any) error { + raw, err := json.Marshal(body) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, strings.NewReader(string(raw))) + if err != nil { + return fmt.Errorf("always-on-display: invalid request") + } + req.Header.Set("Content-Type", "application/json") + for k, v := range headers { + req.Header.Set(k, v) + } + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("always-on-display: request failed: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + return fmt.Errorf("always-on-display: HTTP %d", resp.StatusCode) + } + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("always-on-display: invalid session response: %w", err) + } + return nil +} diff --git a/internal/alerts/aod_test.go b/internal/alerts/aod_test.go new file mode 100644 index 0000000..2739054 --- /dev/null +++ b/internal/alerts/aod_test.go @@ -0,0 +1,41 @@ +package alerts + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "git.tornberg.me/go-otel/internal/features" +) + +func TestAlwaysOnDisplayCreatesAndUpdatesSession(t *testing.T) { + var paths []string + var bodies []map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatal(err) + } + bodies = append(bodies, body) + if r.URL.Path == "/api/sessions" { + _, _ = w.Write([]byte(`{"id":"session-1"}`)) + } + })) + defer srv.Close() + + a := NewAlert(features.Spike{Service: "orders", ErrorCount: 5}, time.Now()) + if err := (&AlwaysOnDisplay{URL: srv.URL}).Send(context.Background(), a); err != nil { + t.Fatal(err) + } + if strings.Join(paths, ",") != "/api/sessions,/api/sessions/session-1/status,/api/sessions/session-1/log" { + t.Fatalf("paths = %v", paths) + } + if bodies[0]["name"] != "LOW alert: orders" || bodies[1]["status"] == "" || bodies[2]["level"] != "info" { + t.Fatalf("bodies = %+v", bodies) + } +} diff --git a/internal/alerts/runner.go b/internal/alerts/runner.go index c3db39e..e2aecad 100644 --- a/internal/alerts/runner.go +++ b/internal/alerts/runner.go @@ -33,6 +33,7 @@ type Config struct { // Muted, when set, reports issues a person ignored; their spikes never alert. Muted func(issueID string) bool WebhookURL, WebhookSecret string + AODURL string } func ConfigFromEnv() Config { @@ -52,6 +53,7 @@ func ConfigFromEnv() Config { TelegramThreadID: os.Getenv("OTEL_ALERT_TELEGRAM_THREAD_ID"), WebhookURL: os.Getenv("OTEL_ALERT_WEBHOOK_URL"), WebhookSecret: os.Getenv("OTEL_ALERT_WEBHOOK_SECRET"), + AODURL: os.Getenv("OTEL_ALERT_AOD_URL"), } } @@ -88,6 +90,9 @@ func (c Config) Notifiers() []Notifier { if c.WebhookURL != "" { out = append(out, &Webhook{URL: c.WebhookURL, Secret: c.WebhookSecret, BaseURL: c.BaseURL}) } + if c.AODURL != "" { + out = append(out, &AlwaysOnDisplay{URL: c.AODURL}) + } return out } diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index d385b9c..be854f0 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -135,6 +135,9 @@ spec: optional: true - name: OTEL_ALERT_BASE_URL value: "https://otel.k6n.net" + # Optional Always-On Display session API. + - name: OTEL_ALERT_AOD_URL + value: "http://10.10.11.117:35851" - name: OTEL_STORE_WORKERS value: "4" # Optional rollout observation. The watcher is read-only and uses the