fix notifications

internal/alerts/notify.go

@@ -41,6 +41,7 @@ type PatternReport struct {
 	Path     string            `json:"path,omitempty"`
 	Services []string          `json:"services,omitempty"` // every service failing in these requests
 	Routes   []topology.Target `json:"routes,omitempty"`   // ingress -> service -> deployment behind an edge error
+	Pods     []PodStatus       `json:"pods,omitempty"`     // what each identified deployment's pod is doing right now
 	Rule     string            `json:"rule"`
 	New      bool              `json:"new"`
 	Count    int               `json:"count"`
@@ -50,6 +51,17 @@ type PatternReport struct {
 	What     string            `json:"what"`
 }
 
+// PodStatus is what was found when looking for a running pod behind a
+// deployment Enrich identified.
+type PodStatus struct {
+	Namespace  string   `json:"namespace"`
+	Deployment string   `json:"deployment"`
+	Pod        string   `json:"pod,omitempty"`
+	Lines      []string `json:"lines,omitempty"`
+	Found      bool     `json:"found"`
+	Err        string   `json:"err,omitempty"`
+}
+
 // Notifier delivers an alert somewhere. Implementations must not put secrets
 // in returned errors.
 type Notifier interface {
@@ -222,6 +234,9 @@ func (a Alert) Text(baseURL string) string {
 		for _, t := range p.Routes {
 			fmt.Fprintf(&b, "   %s\n", routeLine(t, p.Services))
 		}
+		for _, ps := range p.Pods {
+			fmt.Fprintf(&b, "   %s\n", podLine(ps))
+		}
 		fmt.Fprintf(&b, "   What to do: %s\n", advice(p.Category, p.Status))
 		if base != "" && p.IssueID != "" {
 			fmt.Fprintf(&b, "   %s/issues/%s\n", base, url.PathEscape(p.IssueID))
@@ -283,6 +298,9 @@ func (a Alert) HTML(baseURL string, max int) string {
 		for _, t := range p.Routes {
 			fmt.Fprintf(&s, "🧭 %s\n", esc(routeLine(t, p.Services)))
 		}
+		for _, ps := range p.Pods {
+			fmt.Fprintf(&s, "📜 %s\n", esc(podLine(ps)))
+		}
 		if p.Clients > 0 {
 			fmt.Fprintf(&s, "👥 %d client%s affected\n", p.Clients, plural(p.Clients))
 		}
@@ -437,3 +455,48 @@ func (a *Alert) Enrich(r Resolver) {
 		}
 	}
 }
+
+// podLogTail is how many trailing log lines a notification carries per pod.
+const podLogTail = 3
+
+// PodResolver looks up a running pod behind a namespace/deployment and its
+// most recent log lines.
+type PodResolver interface {
+	PodLogs(ctx context.Context, namespace, deployment string, tail int) topology.PodLog
+}
+
+// EnrichPods looks up a running pod for each deployment Enrich identified and
+// attaches its last few log lines, so the notification shows what the
+// backend is actually doing instead of just naming it. A deployment with no
+// pod right now - scaled to 0, most likely - is reported as such.
+func (a *Alert) EnrichPods(ctx context.Context, pr PodResolver) {
+	if pr == nil {
+		return
+	}
+	for i := range a.Patterns {
+		p := &a.Patterns[i]
+		for _, t := range p.Routes {
+			if t.Missing || t.External != "" {
+				continue
+			}
+			for _, d := range t.Deployments {
+				pl := pr.PodLogs(ctx, t.Namespace, d, podLogTail)
+				p.Pods = append(p.Pods, PodStatus{Namespace: t.Namespace, Deployment: d, Pod: pl.Pod, Lines: pl.Lines, Found: pl.Found, Err: pl.Err})
+			}
+		}
+	}
+}
+
+// podLine renders one deployment's pod status for a notification.
+func podLine(ps PodStatus) string {
+	switch {
+	case ps.Err != "":
+		return fmt.Sprintf("%s: could not check for a pod (%s)", ps.Deployment, ps.Err)
+	case !ps.Found:
+		return fmt.Sprintf("%s: no pod running (scaled to 0?)", ps.Deployment)
+	case len(ps.Lines) == 0:
+		return fmt.Sprintf("%s: pod %s, no recent log lines", ps.Deployment, ps.Pod)
+	default:
+		return fmt.Sprintf("%s: pod %s, last %d log line(s):\n     %s", ps.Deployment, ps.Pod, len(ps.Lines), strings.Join(ps.Lines, "\n     "))
+	}
+}