eval

internal/alerts/runner.go

@@ -34,6 +34,9 @@ type Config struct {
 	Muted                     func(issueID string) bool
 	WebhookURL, WebhookSecret string
 	AODURL                    string
+	// LayaURL, when set, asks laya (POST {LayaURL}/predict) for a shadow
+	// verdict on every alert and shows it in the notification.
+	LayaURL string
 }
 
 func ConfigFromEnv() Config {
@@ -54,6 +57,7 @@ func ConfigFromEnv() Config {
 		WebhookURL:       os.Getenv("OTEL_ALERT_WEBHOOK_URL"),
 		WebhookSecret:    os.Getenv("OTEL_ALERT_WEBHOOK_SECRET"),
 		AODURL:           os.Getenv("OTEL_ALERT_AOD_URL"),
+		LayaURL:          os.Getenv("OTEL_ALERT_LAYA_URL"),
 	}
 }
 
@@ -106,6 +110,7 @@ type Runner struct {
 	dataSince func(ctx context.Context) time.Time
 	notifiers []Notifier
 	resolver  Resolver // optional: ingress -> service -> deployment for edge errors
+	laya      *Laya    // optional: shadow verdict attached to each alert
 	sup       *features.Suppressor
 
 	mu        sync.Mutex
@@ -137,6 +142,10 @@ func Start(ctx context.Context, cfg Config, conn driver.Conn, database string, r
 		},
 		dataSince: func(ctx context.Context) time.Time { return DataSince(ctx, conn, database) },
 	}
+	if cfg.LayaURL != "" {
+		r.laya = &Laya{URL: cfg.LayaURL}
+		log.Printf("alerts: laya shadow verdicts from %s", cfg.LayaURL)
+	}
 	names := make([]string, 0, len(notifiers))
 	for _, n := range notifiers {
 		names = append(names, n.Name())
@@ -239,6 +248,7 @@ func (r *Runner) deliver(ctx context.Context, a Alert) {
 	if pr, ok := r.resolver.(PodResolver); ok {
 		a.EnrichPods(ctx, pr)
 	}
+	r.askLaya(ctx, &a)
 	r.mu.Lock()
 	r.recent = append(r.recent, a)
 	if len(r.recent) > maxRecent {
@@ -261,6 +271,23 @@ func (r *Runner) deliver(ctx context.Context, a Alert) {
 	}
 }
 
+// askLaya attaches laya's shadow verdict. It is bounded so a slow model
+// delays a page by seconds at most, and a failure only costs the verdict.
+func (r *Runner) askLaya(ctx context.Context, a *Alert) {
+	if r.laya == nil {
+		return
+	}
+	lctx, cancel := context.WithTimeout(ctx, 8*time.Second)
+	defer cancel()
+	v, err := r.laya.Classify(lctx, *a)
+	if err != nil {
+		log.Printf("alerts: laya verdict for %s failed: %v", a.ID, err)
+		return
+	}
+	a.Laya = v
+	log.Printf("alerts: laya[shadow] %s: %s", a.ID, v.Summary())
+}
+
 // Status is the JSON served by GET /api/alerts.
 type Status struct {
 	Enabled   bool      `json:"enabled"`
@@ -305,6 +332,7 @@ func (r *Runner) HandleTest(w http.ResponseWriter, req *http.Request) {
 		Example: features.Event{Service: "alert-test", Kind: "span", StatusCode: 500, Route: "/api/example", Message: "this is a test alert"},
 	}}}, now)
 	a.Test = true
+	r.askLaya(req.Context(), &a)
 	results := map[string]string{}
 	for _, n := range r.notifiers {
 		if err := n.Send(req.Context(), a); err != nil {