fixes

internal/alerts/alerts_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"git.tornberg.me/go-otel/internal/features"
+	"git.tornberg.me/go-otel/internal/topology"
 )
 
 func burst(sig string, n int, at time.Time, status int, route string) []features.Event {
@@ -124,9 +125,23 @@ func TestTelegramTruncates(t *testing.T) {
 	if err := (&Telegram{Token: "t", ChatID: "1", API: srv.URL}).Send(context.Background(), NewAlert(spike, time.Now())); err != nil {
 		t.Fatal(err)
 	}
-	if n := len([]rune(body["text"].(string))); n > 4096 {
+	text := body["text"].(string)
+	if n := plainLen(text); n > 4096 {
 		t.Fatalf("message too long: %d", n)
 	}
+	if body["parse_mode"] != "HTML" || !strings.Contains(text, "more pattern") || strings.Count(text, "<b>") != strings.Count(text, "</b>") {
+		t.Fatalf("bad html message: %.200s", text)
+	}
+}
+
+func TestTelegramHTMLEscapesDynamicText(t *testing.T) {
+	at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	ev := burst("x", 12, at.Add(-2*time.Minute), 500, "/a<b>&c")
+	sp := DetectAlerts(ev, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5})
+	out := NewAlert(sp[0], at).HTML("https://otel.k6n.net", 4000)
+	if strings.Contains(out, "/a<b>") || !strings.Contains(out, "/a&lt;b&gt;&amp;c") {
+		t.Fatalf("route not escaped:\n%s", out)
+	}
 }
 
 func TestScannerClientIsMutedEntirely(t *testing.T) {
@@ -163,3 +178,104 @@ func TestMinExcessAppliesAfterTriage(t *testing.T) {
 		t.Fatalf("paged on excess that was mostly probes: %+v", got)
 	}
 }
+
+func TestClientErrorsRankBelowServerErrors(t *testing.T) {
+	at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	cur := at.Add(-2 * time.Minute)
+	cfg := features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 10}
+	// 40 x 422 discounted to 8 excess: below MinExcess 10, no page
+	if got := DetectAlerts(burst("bad-input", 40, cur, 422, "/api/order"), at, cfg); len(got) != 0 {
+		t.Fatalf("4xx alone paged: %+v", got)
+	}
+	// 5xx of the same volume does page
+	if got := DetectAlerts(burst("boom", 40, cur, 500, "/api/order"), at, cfg); len(got) != 1 {
+		t.Fatalf("5xx did not page")
+	}
+	// mixed: the 5xx is listed first and the alert is high priority
+	events := append(burst("bad-input", 40, cur, 422, "/api/order"), burst("boom", 8, cur, 500, "/api/pay")...)
+	got := DetectAlerts(events, at, cfg)
+	if len(got) != 1 {
+		t.Fatalf("mixed spike: %+v", got)
+	}
+	a := NewAlert(got[0], at)
+	if a.Priority() != PriorityHigh || a.Patterns[0].Priority != PriorityHigh || a.Patterns[1].Priority != PriorityLow {
+		t.Fatalf("priority order wrong: %+v", a.Patterns)
+	}
+}
+
+func TestAlertLinksToIssue(t *testing.T) {
+	at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	ev := burst("boom", 12, at.Add(-2*time.Minute), 500, "/api")
+	for i := range ev {
+		ev[i].IssueFingerprint = "00ab12cd34ef5678"
+	}
+	sp := DetectAlerts(ev, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5})
+	txt := NewAlert(sp[0], at).Text("https://otel.k6n.net/")
+	if !strings.Contains(txt, "https://otel.k6n.net/issues/00ab12cd34ef5678") {
+		t.Fatalf("no issue link:\n%s", txt)
+	}
+}
+
+func TestCrawlerClientErrorsNeverAlert(t *testing.T) {
+	at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	ev := burst("crawl", 100, at.Add(-2*time.Minute), 403, "/members")
+	for i := range ev {
+		ev[i].UAClass = "ai_crawler"
+	}
+	if got := DetectAlerts(ev, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5}); len(got) != 0 {
+		t.Fatalf("crawler 4xx alerted: %+v", got)
+	}
+}
+
+func TestAlertListsAffectedServices(t *testing.T) {
+	at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	ev := burst("boom", 12, at.Add(-2*time.Minute), 500, "/api")
+	for i := range ev {
+		ev[i].TraceServices = []string{"payments", "api"}
+	}
+	sp := DetectAlerts(ev, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5})
+	a := NewAlert(sp[0], at)
+	if got := strings.Join(a.Patterns[0].Services, ","); got != "api,payments" {
+		t.Fatalf("services = %q", got)
+	}
+	if !strings.Contains(a.Text(""), "Services with errors in the same requests: api, payments") {
+		t.Fatalf("text lacks services:\n%s", a.Text(""))
+	}
+}
+
+type fakeResolver struct{ got [2]string }
+
+func (f *fakeResolver) Resolve(host, path string) []topology.Target {
+	f.got = [2]string{host, path}
+	return []topology.Target{
+		{Host: host, Path: "/api", Namespace: "shop", Service: "orders", Deployments: []string{"orders-v2"}},
+		{Host: host, Path: "/api", Namespace: "shop", Service: "gone", Missing: true},
+	}
+}
+
+func TestEdgeAlertShowsIngressBackends(t *testing.T) {
+	at := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC)
+	ev := burst("bad-gw", 16, at.Add(-2*time.Minute), 502, "/api/order")
+	for i := range ev {
+		ev[i].Service, ev[i].Host = "nginx", "shop.example.com"
+	}
+	sp := DetectAlerts(ev, at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5})
+	a := NewAlert(sp[0], at)
+	r := &fakeResolver{}
+	a.Enrich(r)
+	if r.got != [2]string{"shop.example.com", "/api/order"} {
+		t.Fatalf("resolved %v", r.got)
+	}
+	txt := a.Text("")
+	for _, want := range []string{"service shop/orders -> deployment orders-v2", "may not be instrumented", "kubectl -n shop", "this Service does not exist"} {
+		if !strings.Contains(txt, want) {
+			t.Fatalf("missing %q in:\n%s", want, txt)
+		}
+	}
+	// a non-edge service is never enriched
+	b := NewAlert(DetectAlerts(burst("x", 12, at.Add(-2*time.Minute), 500, "/"), at, features.SpikeConfig{Window: 5 * time.Minute, BaselineWindows: 12, MinExcess: 5})[0], at)
+	b.Enrich(r)
+	if len(b.Patterns[0].Routes) != 0 {
+		t.Fatalf("non-edge alert enriched")
+	}
+}

internal/alerts/filter.go

@@ -37,3 +37,44 @@ func FilterActionable(spikes []features.Spike, keepMuted bool, muted map[triage.
 	}
 	return out
 }
+
+// Priority ranks how urgently a pattern needs a human. Client errors (4xx)
+// are the caller's problem more often than ours, so they rank below server
+// failures and count for less when deciding whether to page.
+type Priority string
+
+const (
+	PriorityHigh   Priority = "high"
+	PriorityMedium Priority = "medium"
+	PriorityLow    Priority = "low"
+)
+
+// lowPriorityWeight is how much a low-priority pattern's excess counts toward
+// the paging threshold: five 4xx errors weigh as one 5xx.
+const lowPriorityWeight = 0.2
+
+// PriorityOf ranks an example event: 4xx is low; server failures (5xx, or a
+// service defect, saturation or unreachable dependency) are high.
+func PriorityOf(e features.Event, c triage.Category) Priority {
+	switch {
+	case e.StatusCode >= 400 && e.StatusCode < 500:
+		return PriorityLow
+	case e.StatusCode >= 500, c == triage.ServiceDefect, c == triage.CapacitySaturation, c == triage.DependencyUnreachable:
+		return PriorityHigh
+	}
+	return PriorityMedium
+}
+
+// PagingExcess is a spike's excess with low-priority patterns discounted, so a
+// pile of 4xx cannot page on its own but still tips a real 5xx spike.
+func PagingExcess(sp features.Spike) float64 {
+	var total float64
+	for _, p := range sp.Patterns {
+		if PriorityOf(p.Example, TriageOf(p.Example).Category) == PriorityLow {
+			total += p.Excess * lowPriorityWeight
+		} else {
+			total += p.Excess
+		}
+	}
+	return total
+}

internal/alerts/load.go

@@ -48,7 +48,7 @@ func LoadErrorEvents(ctx context.Context, conn driver.Conn, database string, sta
 	if withTrace {
 		extra = "span_id, parent_span_id, peer_ip, ua_class, user_agent, trace_id"
 	}
-	rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT timestamp, service, release, kind, source, status_code, span_name, route, exception_type, message_normalized, pattern_signature, is_timeout, is_cancellation, is_retry, %s
+	rows, err := conn.Query(ctx, fmt.Sprintf(`SELECT timestamp, service, release, kind, source, status_code, span_name, route, exception_type, message_normalized, pattern_signature, is_timeout, is_cancellation, is_retry, issue_fingerprint, host, %s
 		FROM %s.normalized_events FINAL
 		WHERE is_error = 1 AND timestamp >= toDateTime64('%s', 9, 'UTC') AND timestamp < toDateTime64('%s', 9, 'UTC')
 		ORDER BY timestamp LIMIT 2000000`, extra, database, chTime(start), chTime(end)))
@@ -64,7 +64,7 @@ func LoadErrorEvents(ctx context.Context, conn driver.Conn, database string, sta
 		var status uint16
 		var timeout, cancel, retry uint8
 		var spanID, parentID, peer, class, ua, traceID string
-		if err := rows.Scan(&e.Timestamp, &e.Service, &e.Release, &e.Kind, &e.Source, &status, &e.SpanName, &e.Route, &e.ExceptionType, &e.Message, &e.PatternSignature, &timeout, &cancel, &retry,
+		if err := rows.Scan(&e.Timestamp, &e.Service, &e.Release, &e.Kind, &e.Source, &status, &e.SpanName, &e.Route, &e.ExceptionType, &e.Message, &e.PatternSignature, &timeout, &cancel, &retry, &e.IssueFingerprint, &e.Host,
 			&spanID, &parentID, &peer, &class, &ua, &traceID); err != nil {
 			return nil, err
 		}
@@ -118,8 +118,20 @@ func resolve(ctx context.Context, conn driver.Conn, database string, start, end
 	if err := rows.Err(); err != nil {
 		return err
 	}
+	failing := map[string]map[string]struct{}{} // trace -> services with an error span
+	for _, sp := range spans {
+		if sp.IsError && sp.Service != "" {
+			if failing[sp.TraceID] == nil {
+				failing[sp.TraceID] = map[string]struct{}{}
+			}
+			failing[sp.TraceID][sp.Service] = struct{}{}
+		}
+	}
 	info := tracectx.Resolve(spans)
 	for i, ref := range refs {
+		for sv := range failing[ref.TraceID] {
+			events[i].TraceServices = append(events[i].TraceServices, sv)
+		}
 		in, ok := info[ref]
 		if !ok {
 			if !honeypot.Untrackable(own[i]) {

internal/alerts/notify.go

@@ -6,14 +6,19 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"html"
 	"io"
 	"net/http"
 	"net/url"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
 
 	"git.tornberg.me/go-otel/internal/features"
+	"git.tornberg.me/go-otel/internal/topology"
+	"git.tornberg.me/go-otel/internal/tracectx"
+	"git.tornberg.me/go-otel/internal/triage"
 	"git.tornberg.me/go-otel/internal/uaclass"
 )
 
@@ -28,14 +33,21 @@ type Alert struct {
 
 // PatternReport is a spike pattern with its triage verdict, ready to show.
 type PatternReport struct {
-	Category string  `json:"category"`
-	Rule     string  `json:"rule"`
-	New      bool    `json:"new"`
-	Count    int     `json:"count"`
-	Baseline float64 `json:"baseline"`
-	Clients  int     `json:"clients,omitempty"`
-	UAClass  string  `json:"uaClass,omitempty"`
-	What     string  `json:"what"`
+	Category string            `json:"category"`
+	Priority Priority          `json:"priority"`
+	IssueID  string            `json:"issueId,omitempty"`
+	Status   int               `json:"status,omitempty"`
+	Host     string            `json:"host,omitempty"`
+	Path     string            `json:"path,omitempty"`
+	Services []string          `json:"services,omitempty"` // every service failing in these requests
+	Routes   []topology.Target `json:"routes,omitempty"`   // ingress -> service -> deployment behind an edge error
+	Rule     string            `json:"rule"`
+	New      bool              `json:"new"`
+	Count    int               `json:"count"`
+	Baseline float64           `json:"baseline"`
+	Clients  int               `json:"clients,omitempty"`
+	UAClass  string            `json:"uaClass,omitempty"`
+	What     string            `json:"what"`
 }
 
 // Notifier delivers an alert somewhere. Implementations must not put secrets
@@ -50,15 +62,35 @@ func NewAlert(spike features.Spike, at time.Time) Alert {
 	a := Alert{At: at, Spike: spike, ID: alertID(spike, at)}
 	for _, p := range spike.Patterns {
 		res := TriageOf(p.Example)
-		r := PatternReport{Category: string(res.Category), Rule: res.Rule, New: p.Novel, Count: p.Count, Baseline: p.BaselineMean, Clients: p.Clients, What: describe(p.Example)}
+		r := PatternReport{Category: string(res.Category), Priority: PriorityOf(p.Example, res.Category), IssueID: p.Example.IssueFingerprint, Status: p.Example.StatusCode, Host: p.Example.Host, Path: p.Example.Route, Services: p.Services, Rule: res.Rule, New: p.Novel, Count: p.Count, Baseline: p.BaselineMean, Clients: p.Clients, What: describe(p.Example)}
 		if p.Example.UAClass != "" && uaclass.IsBot(p.Example.UAClass) {
 			r.UAClass = p.Example.UAClass
 		}
 		a.Patterns = append(a.Patterns, r)
 	}
+	// Most urgent first; the detector's excess order is kept within a priority.
+	sort.SliceStable(a.Patterns, func(i, j int) bool { return prioRank(a.Patterns[i].Priority) < prioRank(a.Patterns[j].Priority) })
 	return a
 }
 
+func prioRank(p Priority) int {
+	switch p {
+	case PriorityHigh:
+		return 0
+	case PriorityMedium:
+		return 1
+	}
+	return 2
+}
+
+// Priority is the most urgent priority among the alert's patterns.
+func (a Alert) Priority() Priority {
+	if len(a.Patterns) == 0 {
+		return PriorityLow
+	}
+	return a.Patterns[0].Priority
+}
+
 func alertID(s features.Spike, at time.Time) string {
 	return strconv.FormatInt(at.Unix(), 36) + "-" + strings.ReplaceAll(s.Service, " ", "_")
 }
@@ -81,7 +113,10 @@ func describe(e features.Event) string {
 		parts = append(parts, truncate(e.Message, 120))
 	}
 	if len(parts) == 0 {
-		return "(no detail)"
+		if e.Service != "" {
+			return "Errors in " + e.Service + " without a message, route or status (see the issue for the trace)"
+		}
+		return "Errors without a message, route or status"
 	}
 	return strings.Join(parts, " | ")
 }
@@ -94,35 +129,106 @@ func truncate(s string, n int) string {
 	return string(r[:n-1]) + "…"
 }
 
+// advice says what a human should do about a category. Wording is plain so it
+// can be acted on without knowing the detector's vocabulary.
+func advice(category string, status int) string {
+	if status >= 400 && status < 500 {
+		return "Client errors (4xx): usually bad requests or stale links, rarely our fault. Only act if the count keeps growing."
+	}
+	switch triage.Category(category) {
+	case triage.DependencyUnreachable:
+		return "Something this service depends on (database, upstream service or network) is failing. Check the health of the services listed here, starting with the deepest one."
+	case triage.CapacitySaturation:
+		return "Requests are timing out or resources are exhausted. Check load, connection pools and CPU/memory limits."
+	case triage.ServiceDefect:
+		return "Most likely a bug or crash in this service. Open the issue for the stack trace and check whether a recent release introduced it."
+	case triage.ClientCredentialIssue:
+		return "Callers are being rejected as unauthenticated. Check for an expired token or key."
+	}
+	return "Unclassified error. Open the issue and decide whether it matters."
+}
+
+// routeLine explains where an edge error was routed and, for backends that
+// reported no failing spans, how to check them directly: an uninstrumented
+// service is invisible to the trace, so it must be looked at by hand.
+func routeLine(t topology.Target, failing []string) string {
+	where := t.Host + t.Path + " -> service " + t.Namespace + "/" + t.Service
+	switch {
+	case t.Missing:
+		return where + ": this Service does not exist, so the ingress rule points at nothing."
+	case t.External != "":
+		return where + " -> external " + t.External + " (outside the cluster)."
+	case len(t.Deployments) == 0:
+		return where + " (no Deployment selected)."
+	}
+	var silent []string
+	for _, d := range t.Deployments {
+		if !contains(failing, d) && !contains(failing, t.Service) {
+			silent = append(silent, d)
+		}
+	}
+	line := where + " -> deployment " + strings.Join(t.Deployments, ", ")
+	if len(silent) > 0 {
+		line += ". No errors from it in the failing requests, it may not be instrumented: kubectl -n " + t.Namespace + " get pods -l <selector> / logs deploy/" + silent[0]
+	}
+	return line + "."
+}
+
+func contains(list []string, v string) bool {
+	for _, x := range list {
+		if x == v {
+			return true
+		}
+	}
+	return false
+}
+
+func verb(p PatternReport) string {
+	if p.New {
+		return "New error, never seen in the last few hours."
+	}
+	return fmt.Sprintf("Rising: %d now, normally about %.0f per window.", p.Count, p.Baseline)
+}
+
 // Text renders a plain-text message (no markup, so nothing needs escaping).
+// Each pattern says what failed, how it differs from normal, who and what is
+// affected, what to do, and where to look.
 func (a Alert) Text(baseURL string) string {
 	var b strings.Builder
-	prefix := "ALERT"
+	prefix := strings.ToUpper(string(a.Priority()))
 	if a.Test {
 		prefix = "TEST"
 	}
-	fmt.Fprintf(&b, "%s %s: %d errors in %s (baseline %.1f/window, excess %.0f)\n",
-		prefix, a.Spike.Service, a.Spike.ErrorCount, a.Spike.WindowEnd.Sub(a.Spike.WindowStart).Round(time.Minute), a.Spike.BaselineMean, a.Spike.Excess)
-	for _, p := range a.Patterns {
-		tag := "rising"
-		if p.New {
-			tag = "NEW"
+	window := a.Spike.WindowEnd.Sub(a.Spike.WindowStart).Round(time.Minute)
+	fmt.Fprintf(&b, "%s: %s is failing (%d errors in the last %s, normally about %.0f)\n",
+		prefix, a.Spike.Service, a.Spike.ErrorCount, window, a.Spike.BaselineMean)
+	base := strings.TrimRight(baseURL, "/")
+	for i, p := range a.Patterns {
+		fmt.Fprintf(&b, "\n%d. %s\n", i+1, p.What)
+		fmt.Fprintf(&b, "   %s\n", verb(p))
+		var who []string
+		if len(p.Services) > 1 {
+			who = append(who, "Services with errors in the same requests: "+strings.Join(p.Services, ", "))
 		}
-		fmt.Fprintf(&b, "\n%s x%d (base %.1f) [%s]\n%s", tag, p.Count, p.Baseline, p.Category, p.What)
-		var extra []string
 		if p.Clients > 0 {
-			extra = append(extra, fmt.Sprintf("%d client%s", p.Clients, plural(p.Clients)))
+			who = append(who, fmt.Sprintf("%d client%s affected", p.Clients, plural(p.Clients)))
 		}
 		if p.UAClass != "" {
-			extra = append(extra, p.UAClass)
+			who = append(who, "client looks like a bot ("+p.UAClass+")")
 		}
-		if len(extra) > 0 {
-			fmt.Fprintf(&b, "\n(%s)", strings.Join(extra, ", "))
+		for _, w := range who {
+			fmt.Fprintf(&b, "   %s.\n", w)
+		}
+		for _, t := range p.Routes {
+			fmt.Fprintf(&b, "   %s\n", routeLine(t, p.Services))
+		}
+		fmt.Fprintf(&b, "   What to do: %s\n", advice(p.Category, p.Status))
+		if base != "" && p.IssueID != "" {
+			fmt.Fprintf(&b, "   %s/issues/%s\n", base, url.PathEscape(p.IssueID))
 		}
-		b.WriteString("\n")
 	}
-	if baseURL != "" {
-		fmt.Fprintf(&b, "\n%s/issues", strings.TrimRight(baseURL, "/"))
+	if base != "" {
+		fmt.Fprintf(&b, "\nAll issues: %s/issues", base)
 	}
 	return strings.TrimSpace(b.String())
 }
@@ -134,6 +240,85 @@ func plural(n int) string {
 	return "s"
 }
 
+func priorityIcon(p Priority) string {
+	switch p {
+	case PriorityHigh:
+		return "🔴"
+	case PriorityMedium:
+		return "🟠"
+	}
+	return "🟡"
+}
+
+// HTML renders the alert for Telegram's HTML parse mode. Every dynamic value
+// is escaped, and patterns are added whole until the message would exceed max
+// characters, so a cut never lands inside a tag.
+func (a Alert) HTML(baseURL string, max int) string {
+	esc := html.EscapeString
+	window := a.Spike.WindowEnd.Sub(a.Spike.WindowStart).Round(time.Minute)
+	icon, label := priorityIcon(a.Priority()), strings.ToUpper(string(a.Priority()))
+	if a.Test {
+		icon, label = "🧪", "TEST"
+	}
+	head := fmt.Sprintf("%s <b>%s</b> · <b>%s</b> is failing\n%d errors in the last %s (normally about %.0f)\n",
+		icon, label, esc(a.Spike.Service), a.Spike.ErrorCount, window, a.Spike.BaselineMean)
+	base := strings.TrimRight(baseURL, "/")
+	foot := ""
+	if base != "" {
+		foot = fmt.Sprintf("\n<a href=\"%s\">All issues</a>", esc(base+"/issues"))
+	}
+	var b strings.Builder
+	b.WriteString(head)
+	for i, p := range a.Patterns {
+		var s strings.Builder
+		fmt.Fprintf(&s, "\n<b>%d.</b> <code>%s</code>\n", i+1, esc(p.What))
+		if p.New {
+			s.WriteString("🆕 New error, not seen in the last few hours\n")
+		} else {
+			fmt.Fprintf(&s, "📈 Rising: %d now, normally about %.0f per window\n", p.Count, p.Baseline)
+		}
+		if len(p.Services) > 1 {
+			fmt.Fprintf(&s, "🧩 Failing together: %s\n", esc(strings.Join(p.Services, ", ")))
+		}
+		for _, t := range p.Routes {
+			fmt.Fprintf(&s, "🧭 %s\n", esc(routeLine(t, p.Services)))
+		}
+		if p.Clients > 0 {
+			fmt.Fprintf(&s, "👥 %d client%s affected\n", p.Clients, plural(p.Clients))
+		}
+		if p.UAClass != "" {
+			fmt.Fprintf(&s, "🤖 Client looks like a bot (%s)\n", esc(p.UAClass))
+		}
+		fmt.Fprintf(&s, "🛠 <i>%s</i>\n", esc(advice(p.Category, p.Status)))
+		if base != "" && p.IssueID != "" {
+			fmt.Fprintf(&s, "🔗 <a href=\"%s\">Open issue</a>\n", esc(base+"/issues/"+url.PathEscape(p.IssueID)))
+		}
+		if plainLen(b.String())+plainLen(s.String())+plainLen(foot)+40 > max {
+			fmt.Fprintf(&b, "\n… and %d more pattern%s, see the issues list", len(a.Patterns)-i, plural(len(a.Patterns)-i))
+			break
+		}
+		b.WriteString(s.String())
+	}
+	b.WriteString(foot)
+	return strings.TrimSpace(b.String())
+}
+
+// plainLen counts what Telegram counts: characters after tags are parsed away.
+func plainLen(s string) int {
+	n, in := 0, false
+	for _, r := range html.UnescapeString(s) {
+		switch {
+		case r == '<':
+			in = true
+		case r == '>' && in:
+			in = false
+		case !in:
+			n++
+		}
+	}
+	return n
+}
+
 // Telegram posts via the Bot API sendMessage method.
 type Telegram struct {
 	Token    string
@@ -154,7 +339,7 @@ func (t *Telegram) Send(ctx context.Context, a Alert) error {
 	if api == "" {
 		api = "https://api.telegram.org"
 	}
-	body := map[string]any{"chat_id": t.ChatID, "text": truncate(a.Text(t.BaseURL), telegramMax), "disable_web_page_preview": true}
+	body := map[string]any{"chat_id": t.ChatID, "text": a.HTML(t.BaseURL, telegramMax), "parse_mode": "HTML", "disable_web_page_preview": true}
 	if t.ThreadID != "" {
 		if id, err := strconv.Atoi(t.ThreadID); err == nil {
 			body["message_thread_id"] = id
@@ -233,3 +418,22 @@ func scrub(s string, secrets []string) string {
 	}
 	return s
 }
+
+// Resolver maps a request host and path to the backends behind it.
+type Resolver interface {
+	Resolve(host, path string) []topology.Target
+}
+
+// Enrich attaches the ingress -> service -> deployment path to patterns of an
+// edge service (nginx, ingress-nginx, ...), whose errors say little about
+// which workload is at fault.
+func (a *Alert) Enrich(r Resolver) {
+	if r == nil || !tracectx.IsEdgeService(a.Spike.Service) {
+		return
+	}
+	for i := range a.Patterns {
+		if p := &a.Patterns[i]; p.Host != "" {
+			p.Routes = r.Resolve(p.Host, p.Path)
+		}
+	}
+}

internal/alerts/runner.go

@@ -98,6 +98,7 @@ type Runner struct {
 	load      Loader
 	dataSince func(ctx context.Context) time.Time
 	notifiers []Notifier
+	resolver  Resolver // optional: ingress -> service -> deployment for edge errors
 	sup       *features.Suppressor
 
 	mu        sync.Mutex
@@ -113,7 +114,7 @@ const maxRecent = 50
 
 // Start launches the runner against ClickHouse and returns it, or nil when
 // alerting is not configured.
-func Start(ctx context.Context, cfg Config, conn driver.Conn, database string) *Runner {
+func Start(ctx context.Context, cfg Config, conn driver.Conn, database string, resolver Resolver) *Runner {
 	notifiers := cfg.Notifiers()
 	if len(notifiers) == 0 && !cfg.DryRun {
 		log.Printf("alerts: disabled (set OTEL_ALERT_TELEGRAM_TOKEN + OTEL_ALERT_TELEGRAM_CHAT_ID and/or OTEL_ALERT_WEBHOOK_URL)")
@@ -122,6 +123,7 @@ func Start(ctx context.Context, cfg Config, conn driver.Conn, database string) *
 	r := &Runner{
 		cfg:       cfg,
 		notifiers: notifiers,
+		resolver:  resolver,
 		sup:       features.NewSuppressor(cfg.Cooldown),
 		load: func(ctx context.Context, start, end time.Time) ([]features.Event, error) {
 			return LoadErrorEvents(ctx, conn, database, start, end)
@@ -205,10 +207,11 @@ func DetectAlerts(events []features.Event, at time.Time, cfg features.SpikeConfi
 		minExcess = features.DefaultSpikeConfig().MinExcess
 	}
 	var out []features.Spike
-	// The excess threshold is applied after triage: a service whose total
-	// excess is mostly probes must not page on the little that is left.
+	// The excess threshold is applied after triage, with 4xx discounted: a
+	// service whose excess is mostly probes or client errors must not page on
+	// the little that is left.
 	for _, sp := range FilterActionable(features.DetectSpikes(events, at, cfg), false, nil) {
-		if sp.Excess >= float64(minExcess) {
+		if PagingExcess(sp) >= float64(minExcess) {
 			out = append(out, sp)
 		}
 	}
@@ -216,6 +219,7 @@ func DetectAlerts(events []features.Event, at time.Time, cfg features.SpikeConfi
 }
 
 func (r *Runner) deliver(ctx context.Context, a Alert) {
+	a.Enrich(r.resolver)
 	r.mu.Lock()
 	r.recent = append(r.recent, a)
 	if len(r.recent) > maxRecent {