display notifications

cmd/alert-test/main.go

@@ -17,7 +17,7 @@ func main() {
 	cfg := alerts.ConfigFromEnv()
 	notifiers := cfg.Notifiers()
 	if len(notifiers) == 0 {
-		fmt.Fprintln(os.Stderr, "no channel configured (TELEGRAM_KEY + TELEGRAM_CHATID, or OTEL_ALERT_WEBHOOK_URL)")
+		fmt.Fprintln(os.Stderr, "no channel configured (Telegram, OTEL_ALERT_WEBHOOK_URL, or OTEL_ALERT_AOD_URL)")
 		os.Exit(2)
 	}
 	now := time.Now().UTC()

internal/alerts/aod.go

@@ -0,0 +1,93 @@
+package alerts
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// AlwaysOnDisplay sends each alert to the session API used by the wall
+// display. A session is created for the alert and then given a status and log
+// entry so the display can show both the current incident and its details.
+type AlwaysOnDisplay struct {
+	URL    string
+	Client *http.Client
+}
+
+func (a *AlwaysOnDisplay) Name() string { return "always-on-display" }
+
+type aodSession struct {
+	ID string `json:"id"`
+}
+
+func (a *AlwaysOnDisplay) Send(ctx context.Context, alert Alert) error {
+	client := a.Client
+	if client == nil {
+		client = &http.Client{Timeout: 15 * time.Second}
+	}
+	base := strings.TrimRight(a.URL, "/")
+
+	var session aodSession
+	name := fmt.Sprintf("%s alert: %s", strings.ToUpper(string(alert.Priority())), alert.Spike.Service)
+	if alert.Test {
+		name = "TEST alert: " + alert.Spike.Service
+	}
+	if err := postJSONDecode(ctx, client, base+"/api/sessions", nil, map[string]string{"name": name}, &session); err != nil {
+		return err
+	}
+	if session.ID == "" {
+		return fmt.Errorf("always-on-display: session response did not include an id")
+	}
+
+	status := firstLine(alert.Text(""))
+	if err := postJSON(ctx, client, base+"/api/sessions/"+session.ID+"/status", nil, map[string]string{"status": status}); err != nil {
+		return err
+	}
+	level := "info"
+	if alert.Priority() == PriorityHigh {
+		level = "error"
+	} else if alert.Priority() == PriorityMedium {
+		level = "warning"
+	}
+	return postJSON(ctx, client, base+"/api/sessions/"+session.ID+"/log", nil, map[string]string{
+		"level":   level,
+		"message": alert.Text(""),
+	})
+}
+
+func firstLine(s string) string {
+	if i := strings.IndexByte(s, '\n'); i >= 0 {
+		return s[:i]
+	}
+	return s
+}
+
+func postJSONDecode(ctx context.Context, client *http.Client, target string, headers map[string]string, body any, out any) error {
+	raw, err := json.Marshal(body)
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, strings.NewReader(string(raw)))
+	if err != nil {
+		return fmt.Errorf("always-on-display: invalid request")
+	}
+	req.Header.Set("Content-Type", "application/json")
+	for k, v := range headers {
+		req.Header.Set(k, v)
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("always-on-display: request failed: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode/100 != 2 {
+		return fmt.Errorf("always-on-display: HTTP %d", resp.StatusCode)
+	}
+	if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
+		return fmt.Errorf("always-on-display: invalid session response: %w", err)
+	}
+	return nil
+}

internal/alerts/aod_test.go

@@ -0,0 +1,41 @@
+package alerts
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/features"
+)
+
+func TestAlwaysOnDisplayCreatesAndUpdatesSession(t *testing.T) {
+	var paths []string
+	var bodies []map[string]any
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		paths = append(paths, r.URL.Path)
+		var body map[string]any
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			t.Fatal(err)
+		}
+		bodies = append(bodies, body)
+		if r.URL.Path == "/api/sessions" {
+			_, _ = w.Write([]byte(`{"id":"session-1"}`))
+		}
+	}))
+	defer srv.Close()
+
+	a := NewAlert(features.Spike{Service: "orders", ErrorCount: 5}, time.Now())
+	if err := (&AlwaysOnDisplay{URL: srv.URL}).Send(context.Background(), a); err != nil {
+		t.Fatal(err)
+	}
+	if strings.Join(paths, ",") != "/api/sessions,/api/sessions/session-1/status,/api/sessions/session-1/log" {
+		t.Fatalf("paths = %v", paths)
+	}
+	if bodies[0]["name"] != "LOW alert: orders" || bodies[1]["status"] == "" || bodies[2]["level"] != "info" {
+		t.Fatalf("bodies = %+v", bodies)
+	}
+}

internal/alerts/runner.go

@@ -33,6 +33,7 @@ type Config struct {
 	// Muted, when set, reports issues a person ignored; their spikes never alert.
 	Muted                     func(issueID string) bool
 	WebhookURL, WebhookSecret string
+	AODURL                    string
 }
 
 func ConfigFromEnv() Config {
@@ -52,6 +53,7 @@ func ConfigFromEnv() Config {
 		TelegramThreadID: os.Getenv("OTEL_ALERT_TELEGRAM_THREAD_ID"),
 		WebhookURL:       os.Getenv("OTEL_ALERT_WEBHOOK_URL"),
 		WebhookSecret:    os.Getenv("OTEL_ALERT_WEBHOOK_SECRET"),
+		AODURL:           os.Getenv("OTEL_ALERT_AOD_URL"),
 	}
 }
 
@@ -88,6 +90,9 @@ func (c Config) Notifiers() []Notifier {
 	if c.WebhookURL != "" {
 		out = append(out, &Webhook{URL: c.WebhookURL, Secret: c.WebhookSecret, BaseURL: c.BaseURL})
 	}
+	if c.AODURL != "" {
+		out = append(out, &AlwaysOnDisplay{URL: c.AODURL})
+	}
 	return out
 }
 

k8s/deployment.yaml

@@ -135,6 +135,9 @@ spec:
               optional: true
         - name: OTEL_ALERT_BASE_URL
           value: "https://otel.k6n.net"
+        # Optional Always-On Display session API.
+        - name: OTEL_ALERT_AOD_URL
+          value: "http://10.10.11.117:35851"
         - name: OTEL_STORE_WORKERS
           value: "4"
         # Optional rollout observation. The watcher is read-only and uses the