more otel

internal/issues/tracker.go

@@ -5,9 +5,11 @@ package issues
 import (
 	"fmt"
 	"hash/fnv"
+	"regexp"
 	"sort"
 	"strings"
 	"sync"
+	"time"
 )
 
 // maxIssues bounds memory use: once exceeded, the least-recently-seen issue
@@ -17,6 +19,18 @@ const maxIssues = 2000
 // maxSamples bounds how many sample occurrences are kept per issue.
 const maxSamples = 5
 
+// trendWindow bounds how far back the per-hour trend buckets are kept.
+const trendWindow = 48 * time.Hour
+
+// Issue lifecycle states. New occurrences automatically reopen resolved
+// issues (a regression), but stay merged into ignored ones until the user
+// explicitly sets the state back to open.
+const (
+	StatusOpen     = "open"
+	StatusResolved = "resolved"
+	StatusIgnored  = "ignored"
+)
+
 // Sample is one occurrence of an issue.
 type Sample struct {
 	Timestamp int64  `json:"timestamp"`
@@ -26,38 +40,121 @@ type Sample struct {
 	// URL is the page URL an occurrence happened on, populated only for
 	// browser-originated (otel-rum) errors.
 	URL string `json:"url,omitempty"`
+	// Stacktrace is the exception.stacktrace attribute of the occurrence,
+	// when the source telemetry carried one.
+	Stacktrace         string `json:"stacktrace,omitempty"`
+	ResolvedStacktrace string `json:"resolvedStacktrace,omitempty"`
+	// HTTP request context, extracted from the span/log attributes when the
+	// error happened inside an HTTP handler (semconv v1.x old and new names
+	// are both handled by the extractor).
+	HTTPMethod string `json:"httpMethod,omitempty"`
+	HTTPRoute  string `json:"httpRoute,omitempty"`
+	HTTPStatus int    `json:"httpStatus,omitempty"`
+	// Client/release context.
+	UserAgent   string `json:"userAgent,omitempty"`
+	SourceIP    string `json:"sourceIp,omitempty"`
+	Release     string `json:"release,omitempty"`
+	Environment string `json:"environment,omitempty"`
+	// Breadcrumbs is the raw JSON captured by otel-rum.js before the error
+	// (fetch/console events), present only on frontend issues.
+	Breadcrumbs string `json:"breadcrumbs,omitempty"`
+	// Deterministic, non-PII features used for grouping and review.
+	NormalizedMessage string `json:"normalizedMessage,omitempty"`
+	NormalizedRoute   string `json:"normalizedRoute,omitempty"`
+	StatusClass       string `json:"statusClass,omitempty"`
+	StackSignature    string `json:"stackSignature,omitempty"`
 }
 
 // Issue is a deduplicated group of related errors, fingerprinted by kind,
-// service and title.
+// service and a normalized form of the title.
 type Issue struct {
 	ID               string   `json:"id"`
 	Title            string   `json:"title"`
-	Kind             string   `json:"kind"` // "span_error" or "log_error"
+	Kind             string   `json:"kind"` // "span_error", "log_error" or "frontend_error"
 	Service          string   `json:"service"`
 	Count            int      `json:"count"`
 	FirstSeen        int64    `json:"firstSeen"`
 	LastSeen         int64    `json:"lastSeen"`
 	AffectedServices []string `json:"affectedServices"`
 	Samples          []Sample `json:"samples"`
+	// Trend maps UTC hour buckets ("2006-01-02T15:00") to occurrence counts
+	// over the last trendWindow, for sparklines and spike detection.
+	Trend map[string]int `json:"trend,omitempty"`
+	// Status is the triage state: open, resolved or ignored.
+	Status     string `json:"status"`
+	ResolvedAt int64  `json:"resolvedAt,omitempty"`
+	// NormalizedTitle is the deterministic, redacted form used for exact grouping.
+	NormalizedTitle string `json:"normalizedTitle,omitempty"`
+	// PatternID groups related exact issues without merging their operational IDs.
+	PatternID        string `json:"patternId,omitempty"`
+	PatternSignature string `json:"patternSignature,omitempty"`
+}
+
+// SimilarIssue is a deterministic related-issue suggestion. Suggestions never
+// change issue IDs and include the reasons that produced the score.
+type SimilarIssue struct {
+	Issue    Issue    `json:"issue"`
+	Score    float64  `json:"score"`
+	Reasons  []string `json:"reasons"`
+	Decision string   `json:"decision,omitempty"`
+}
+
+// Pattern is a UI-friendly aggregate of exact issues.
+type Pattern struct {
+	ID              string   `json:"id"`
+	Signature       string   `json:"signature"`
+	Service         string   `json:"service"`
+	Kind            string   `json:"kind"`
+	IssueCount      int      `json:"issueCount"`
+	OccurrenceCount int      `json:"occurrenceCount"`
+	LastSeen        int64    `json:"lastSeen"`
+	IssueIDs        []string `json:"issueIds"`
 }
 
 // Tracker incrementally fingerprints and deduplicates errors as they're
 // recorded. It is safe for concurrent use.
 type Tracker struct {
-	mu     sync.Mutex
-	issues map[string]*Issue
+	mu       sync.Mutex
+	issues   map[string]*Issue
+	feedback map[string]string
+}
+
+// ListOptions filters and paginates the issue list. Issues remain ordered by
+// LastSeen descending; AfterLastSeen/AfterID form an opaque-to-callers cursor
+// boundary and make pagination stable when multiple issues share a timestamp.
+type ListOptions struct {
+	Service       string
+	Kind          string
+	Status        string
+	Title         string
+	SourceIP      string
+	Route         string
+	Since         int64
+	Until         int64
+	AfterLastSeen int64
+	AfterID       string
+	Limit         int
+}
+
+// ListPage is a filtered page of issues. HasMore indicates that another page
+// can be requested using the final issue as the cursor boundary.
+type ListPage struct {
+	Issues  []Issue
+	HasMore bool
 }
 
 // NewTracker creates an empty Tracker.
 func NewTracker() *Tracker {
-	return &Tracker{issues: make(map[string]*Issue)}
+	return &Tracker{issues: make(map[string]*Issue), feedback: make(map[string]string)}
 }
 
 // Record adds an occurrence of an error, creating a new issue or merging into
-// an existing one with a matching (kind, service, title) fingerprint.
+// an existing one with a matching (kind, service, normalized title)
+// fingerprint.
 func (t *Tracker) Record(kind, service, title string, sample Sample) {
-	id := fingerprint(kind, service, title)
+	normalizeSample(&sample, title)
+	id := fingerprint(kind, service, NormalizeTitle(title))
+	bucket := HourBucket(sample.Timestamp)
 
 	t.mu.Lock()
 	defer t.mu.Unlock()
@@ -68,24 +165,282 @@ func (t *Tracker) Record(kind, service, title string, sample Sample) {
 			t.evictOldestLocked()
 		}
 		issue = &Issue{
-			ID:        id,
-			Title:     title,
-			Kind:      kind,
-			Service:   service,
-			FirstSeen: sample.Timestamp,
+			ID:              id,
+			Title:           title,
+			Kind:            kind,
+			Service:         service,
+			FirstSeen:       sample.Timestamp,
+			Status:          StatusOpen,
+			NormalizedTitle: NormalizeTitle(title),
 		}
+		issue.PatternSignature = patternSignature(kind, service, title, sample)
+		issue.PatternID = fingerprint("pattern", issue.PatternSignature)
 		t.issues[id] = issue
 	}
 
 	issue.Count++
 	issue.LastSeen = sample.Timestamp
 	issue.AffectedServices = addUnique(issue.AffectedServices, service)
+	if issue.Trend == nil {
+		issue.Trend = make(map[string]int)
+	}
+	issue.Trend[bucket]++
+	pruneTrendLocked(issue.Trend, sample.Timestamp)
+	// A resolved issue that fires again is a regression: reopen it. Ignored
+	// issues keep their state (that's the point of ignoring a noisy one).
+	if issue.Status == StatusResolved {
+		issue.Status = StatusOpen
+		issue.ResolvedAt = 0
+	}
 	issue.Samples = append(issue.Samples, sample)
 	if len(issue.Samples) > maxSamples {
 		issue.Samples = issue.Samples[len(issue.Samples)-maxSamples:]
 	}
 }
 
+// Similar returns bounded, deterministic near-match suggestions for an issue.
+func (t *Tracker) Similar(id string, limit int) ([]SimilarIssue, bool) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	target, ok := t.issues[id]
+	if !ok {
+		return nil, false
+	}
+	if limit <= 0 || limit > 50 {
+		limit = 10
+	}
+	result := make([]SimilarIssue, 0)
+	for otherID, other := range t.issues {
+		if otherID == id || other.Service != target.Service || other.Kind != target.Kind {
+			continue
+		}
+		score, reasons := similarity(*target, *other)
+		if score < 0.45 {
+			continue
+		}
+		decision := t.feedback[feedbackKey(id, otherID)]
+		result = append(result, SimilarIssue{Issue: cloneIssue(other), Score: score, Reasons: reasons, Decision: decision})
+	}
+	sort.Slice(result, func(i, j int) bool {
+		if result[i].Score != result[j].Score {
+			return result[i].Score > result[j].Score
+		}
+		return result[i].Issue.ID < result[j].Issue.ID
+	})
+	if len(result) > limit {
+		result = result[:limit]
+	}
+	return result, true
+}
+
+// Patterns returns deterministic groups built from exact issue IDs.
+func (t *Tracker) Patterns() []Pattern {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	groups := make(map[string]*Pattern)
+	for _, issue := range t.issues {
+		p := groups[issue.PatternID]
+		if p == nil {
+			p = &Pattern{ID: issue.PatternID, Signature: issue.PatternSignature, Service: issue.Service, Kind: issue.Kind}
+			groups[issue.PatternID] = p
+		}
+		p.IssueCount++
+		p.OccurrenceCount += issue.Count
+		p.IssueIDs = append(p.IssueIDs, issue.ID)
+		if issue.LastSeen > p.LastSeen {
+			p.LastSeen = issue.LastSeen
+		}
+	}
+	result := make([]Pattern, 0, len(groups))
+	for _, p := range groups {
+		sort.Strings(p.IssueIDs)
+		result = append(result, *p)
+	}
+	sort.Slice(result, func(i, j int) bool {
+		if result[i].LastSeen != result[j].LastSeen {
+			return result[i].LastSeen > result[j].LastSeen
+		}
+		return result[i].ID < result[j].ID
+	})
+	return result
+}
+
+// SetSimilarityFeedback records a human label for later training data.
+func (t *Tracker) SetSimilarityFeedback(a, b, decision string) bool {
+	if a == "" || b == "" || a == b || (decision != "same" && decision != "different") {
+		return false
+	}
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if _, ok := t.issues[a]; !ok {
+		return false
+	}
+	if _, ok := t.issues[b]; !ok {
+		return false
+	}
+	t.feedback[feedbackKey(a, b)] = decision
+	return true
+}
+
+func feedbackKey(a, b string) string {
+	if a > b {
+		a, b = b, a
+	}
+	return a + "\x00" + b
+}
+
+func patternSignature(kind, service, title string, sample Sample) string {
+	route := sample.NormalizedRoute
+	status := sample.StatusClass
+	stack := sample.StackSignature
+	// The title is deliberately broad here: the exact normalized title remains
+	// the canonical issue identity, while this signature supports UI grouping.
+	title = normalizePatternTitle(title)
+	return strings.Join([]string{service, kind, route, status, title, stack}, "\x00")
+}
+
+func normalizeSample(sample *Sample, title string) {
+	sample.NormalizedMessage = NormalizeTitle(title)
+	sample.NormalizedRoute = NormalizeRoute(sample.HTTPRoute)
+	sample.StatusClass = statusClass(sample.HTTPStatus)
+	sample.StackSignature = firstStackFrame(sample.Stacktrace)
+}
+
+func normalizePatternTitle(title string) string {
+	s := NormalizeTitle(title)
+	s = regexp.MustCompile(`(?i)http\s+[1-5]\d{2}`).ReplaceAllString(s, "http <status>")
+	return s
+}
+
+func NormalizeRoute(route string) string {
+	route = strings.TrimSpace(strings.SplitN(route, "?", 2)[0])
+	if route == "" {
+		return ""
+	}
+	parts := strings.Split(route, "/")
+	for i, part := range parts {
+		if part == "" {
+			continue
+		}
+		if uuidRe.MatchString(part) || hexIDRe.MatchString(part) || numRe.MatchString(part) {
+			parts[i] = "{id}"
+		}
+	}
+	return strings.Join(parts, "/")
+}
+
+func statusClass(status int) string {
+	if status >= 500 && status < 600 {
+		return "5xx"
+	}
+	if status >= 400 && status < 500 {
+		return "4xx"
+	}
+	if status >= 300 && status < 400 {
+		return "3xx"
+	}
+	if status > 0 {
+		return "2xx"
+	}
+	return ""
+}
+
+func firstStackFrame(stack string) string {
+	if i := strings.IndexByte(stack, '\n'); i >= 0 {
+		stack = stack[:i]
+	}
+	return strings.TrimSpace(stack)
+}
+
+func similarity(a, b Issue) (float64, []string) {
+	score := 0.0
+	reasons := []string{}
+	if a.Service == b.Service {
+		score += .25
+	}
+	if a.Kind == b.Kind {
+		score += .15
+	}
+	aSample, bSample := latestSample(a), latestSample(b)
+	if NormalizeRoute(aSample.HTTPRoute) != "" && NormalizeRoute(aSample.HTTPRoute) == NormalizeRoute(bSample.HTTPRoute) {
+		score += .20
+		reasons = append(reasons, "same route template")
+	}
+	if statusClass(aSample.HTTPStatus) != "" && statusClass(aSample.HTTPStatus) == statusClass(bSample.HTTPStatus) {
+		score += .15
+		reasons = append(reasons, "same HTTP status class")
+	}
+	if firstStackFrame(aSample.Stacktrace) != "" && firstStackFrame(aSample.Stacktrace) == firstStackFrame(bSample.Stacktrace) {
+		score += .10
+		reasons = append(reasons, "same top stack frame")
+	}
+	if a.NormalizedTitle != "" && a.NormalizedTitle == b.NormalizedTitle {
+		score += .15
+		reasons = append(reasons, "same normalized message")
+	} else if tokenSimilarity(a.NormalizedTitle, b.NormalizedTitle) >= .5 {
+		score += .10
+		reasons = append(reasons, "similar normalized message")
+	}
+	if len(reasons) == 0 {
+		reasons = append(reasons, "same service and event kind")
+	}
+	return score, reasons
+}
+
+func latestSample(issue Issue) Sample {
+	if len(issue.Samples) == 0 {
+		return Sample{}
+	}
+	return issue.Samples[len(issue.Samples)-1]
+}
+
+func tokenSimilarity(a, b string) float64 {
+	aa, bb := strings.Fields(a), strings.Fields(b)
+	if len(aa) == 0 || len(bb) == 0 {
+		return 0
+	}
+	set := map[string]bool{}
+	for _, v := range aa {
+		set[v] = true
+	}
+	other := map[string]bool{}
+	for _, v := range bb {
+		other[v] = true
+	}
+	common := 0
+	for v := range other {
+		if set[v] {
+			common++
+		}
+	}
+	return float64(common) / float64(len(set)+len(other)-common)
+}
+
+// SetStatus updates the triage state of an issue. It returns the updated
+// issue, or false when the id is unknown or the status is invalid.
+func (t *Tracker) SetStatus(id, status string) (Issue, bool) {
+	switch status {
+	case StatusOpen, StatusResolved, StatusIgnored:
+	default:
+		return Issue{}, false
+	}
+
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	issue, ok := t.issues[id]
+	if !ok {
+		return Issue{}, false
+	}
+	issue.Status = status
+	if status == StatusOpen {
+		issue.ResolvedAt = 0
+	} else {
+		issue.ResolvedAt = time.Now().UnixNano()
+	}
+	return cloneIssue(issue), true
+}
+
 // evictOldestLocked removes the issue with the smallest LastSeen. Callers
 // must hold t.mu.
 func (t *Tracker) evictOldestLocked() {
@@ -107,22 +462,89 @@ func (t *Tracker) evictOldestLocked() {
 // List returns issues sorted by LastSeen descending (most recently seen
 // first). A non-positive limit returns all issues.
 func (t *Tracker) List(limit int) []Issue {
+	return t.ListFiltered(ListOptions{Limit: limit}).Issues
+}
+
+// ListFiltered returns a filtered, stable page of issues.
+func (t *Tracker) ListFiltered(opts ListOptions) ListPage {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
 	result := make([]Issue, 0, len(t.issues))
 	for _, issue := range t.issues {
+		if !matches(issue, opts) || beforeCursor(issue, opts) {
+			continue
+		}
 		result = append(result, cloneIssue(issue))
 	}
 
 	sort.Slice(result, func(i, j int) bool {
-		return result[i].LastSeen > result[j].LastSeen
+		if result[i].LastSeen != result[j].LastSeen {
+			return result[i].LastSeen > result[j].LastSeen
+		}
+		return result[i].ID > result[j].ID
 	})
 
-	if limit > 0 && len(result) > limit {
-		result = result[:limit]
+	limit := opts.Limit
+	if limit <= 0 {
+		limit = len(result)
 	}
-	return result
+	page := ListPage{Issues: result}
+	if len(page.Issues) > limit {
+		page.Issues = page.Issues[:limit]
+		page.HasMore = true
+	}
+	return page
+}
+
+func matches(issue *Issue, opts ListOptions) bool {
+	if opts.Service != "" && issue.Service != opts.Service {
+		return false
+	}
+	if opts.Kind != "" && issue.Kind != opts.Kind {
+		return false
+	}
+	if opts.Status != "" && issue.Status != opts.Status {
+		return false
+	}
+	if opts.Title != "" && !strings.Contains(strings.ToLower(issue.Title), strings.ToLower(opts.Title)) {
+		return false
+	}
+	if opts.Since != 0 && issue.LastSeen < opts.Since {
+		return false
+	}
+	if opts.Until != 0 && issue.LastSeen > opts.Until {
+		return false
+	}
+	if opts.SourceIP != "" && !issueHasSample(issue, func(sample Sample) bool { return sample.SourceIP == opts.SourceIP }) {
+		return false
+	}
+	if opts.Route != "" && !issueHasSample(issue, func(sample Sample) bool { return sample.HTTPRoute == opts.Route }) {
+		return false
+	}
+	return true
+}
+
+func issueHasSample(issue *Issue, predicate func(Sample) bool) bool {
+	for _, sample := range issue.Samples {
+		if predicate(sample) {
+			return true
+		}
+	}
+	return false
+}
+
+func beforeCursor(issue *Issue, opts ListOptions) bool {
+	if opts.AfterLastSeen == 0 {
+		return false
+	}
+	if issue.LastSeen < opts.AfterLastSeen {
+		return false
+	}
+	if issue.LastSeen > opts.AfterLastSeen {
+		return true
+	}
+	return issue.ID >= opts.AfterID
 }
 
 // Get returns a single issue by id.
@@ -137,15 +559,100 @@ func (t *Tracker) Get(id string) (Issue, bool) {
 	return cloneIssue(issue), true
 }
 
-// cloneIssue deep-copies the slice fields so returned issues don't share
+// cloneIssue deep-copies the slice/map fields so returned issues don't share
 // backing arrays with ones still being mutated by Record.
 func cloneIssue(i *Issue) Issue {
 	c := *i
 	c.AffectedServices = append([]string(nil), i.AffectedServices...)
 	c.Samples = append([]Sample(nil), i.Samples...)
+	if i.Trend != nil {
+		c.Trend = make(map[string]int, len(i.Trend))
+		for k, v := range i.Trend {
+			c.Trend[k] = v
+		}
+	}
 	return c
 }
 
+// HourBucket renders a unix-nanosecond timestamp as a UTC hour key used in
+// Issue.Trend. Keys sort lexicographically, so the UI can sort them as-is.
+func HourBucket(ts int64) string {
+	return time.Unix(0, ts).UTC().Format("2006-01-02T15:00")
+}
+
+// pruneTrendLocked drops trend buckets older than trendWindow. Callers must
+// hold the tracker lock.
+func pruneTrendLocked(trend map[string]int, now int64) {
+	if len(trend) <= 48 {
+		return
+	}
+	cutoff := HourBucket(time.Unix(0, now).Add(-trendWindow).UnixNano())
+	for k := range trend {
+		if k < cutoff {
+			delete(trend, k)
+		}
+	}
+}
+
+var (
+	uuidRe       = regexp.MustCompile(`(?i)\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b`)
+	hexIDRe      = regexp.MustCompile(`(?i)\b[0-9a-f]{8,}\b`)
+	httpStatusRe = regexp.MustCompile(`(?i)\bhttp\s+([1-5]\d{2})\b`)
+	numRe        = regexp.MustCompile(`\b\d+\b`)
+	spaceRe      = regexp.MustCompile(`\s+`)
+)
+
+// NormalizeTitle strips variable content (UUIDs, hex ids, numbers, whitespace,
+// casing) from an error title so that "User 123 not found" and "User 456 not
+// found" fingerprint into the same issue. The raw title is kept as the
+// display title of the first-seen occurrence. HTTP status codes (e.g. HTTP 404)
+// are preserved so different HTTP error types don't collapse together.
+func NormalizeTitle(title string) string {
+	s := uuidRe.ReplaceAllString(title, "<id>")
+	// Protect HTTP status codes from being replaced by <num>
+	s = httpStatusRe.ReplaceAllStringFunc(s, func(match string) string {
+		parts := strings.Fields(match)
+		if len(parts) != 2 {
+			return match
+		}
+		return "__HTTP_STATUS_" + encodeStatus(parts[1]) + "__"
+	})
+	s = numRe.ReplaceAllString(s, "<num>")
+	s = hexIDRe.ReplaceAllStringFunc(s, func(match string) string {
+		// Mixed hex runs (letters + digits, e.g. trace/span ids) become ids.
+		// Pure-digit runs were already handled by numRe above, and words
+		// spelled entirely from a-f (rare at this length) survive so distinct
+		// error messages don't collapse.
+		if strings.ContainsAny(match, "0123456789") {
+			return "<id>"
+		}
+		return match
+	})
+	s = regexp.MustCompile(`__HTTP_STATUS_([a-z]+)__`).ReplaceAllStringFunc(s, func(match string) string {
+		encoded := strings.TrimSuffix(strings.TrimPrefix(match, "__HTTP_STATUS_"), "__")
+		return "http " + decodeStatus(encoded)
+	})
+	s = strings.ReplaceAll(s, "__", "")
+	s = spaceRe.ReplaceAllString(s, " ")
+	return strings.ToLower(strings.TrimSpace(s))
+}
+
+func encodeStatus(code string) string {
+	var b strings.Builder
+	for _, digit := range code {
+		b.WriteByte('a' + byte(digit-'0'))
+	}
+	return b.String()
+}
+
+func decodeStatus(encoded string) string {
+	var b strings.Builder
+	for _, letter := range encoded {
+		b.WriteByte('0' + byte(letter-'a'))
+	}
+	return b.String()
+}
+
 func fingerprint(parts ...string) string {
 	h := fnv.New64a()
 	_, _ = h.Write([]byte(strings.Join(parts, "\x00")))

internal/issues/tracker_test.go

@@ -73,3 +73,157 @@ func TestTrackerListRespectsLimit(t *testing.T) {
 		t.Fatalf("expected limit to cap results at 2, got %d", len(list))
 	}
 }
+
+func TestTrackerFilteredCursorPagination(t *testing.T) {
+	tracker := NewTracker()
+	tracker.Record("span_error", "nginx", "GET /.env", Sample{Timestamp: 300, SourceIP: "203.0.113.10", HTTPRoute: "/.env"})
+	tracker.Record("span_error", "nginx", "GET /.git/HEAD", Sample{Timestamp: 200, SourceIP: "203.0.113.11", HTTPRoute: "/.git/HEAD"})
+	tracker.Record("log_error", "api", "database down", Sample{Timestamp: 100})
+
+	page := tracker.ListFiltered(ListOptions{Service: "nginx", Kind: "span_error", Route: "/.env", Limit: 1})
+	if len(page.Issues) != 1 || page.Issues[0].Title != "GET /.env" {
+		t.Fatalf("unexpected filtered page: %+v", page)
+	}
+	if page.HasMore {
+		t.Error("single matching issue should not have another page")
+	}
+
+	all := tracker.ListFiltered(ListOptions{Service: "nginx", Limit: 1})
+	if len(all.Issues) != 1 || !all.HasMore {
+		t.Fatalf("expected first paginated page with more results, got %+v", all)
+	}
+	next := tracker.ListFiltered(ListOptions{
+		Service:       "nginx",
+		AfterLastSeen: all.Issues[0].LastSeen,
+		AfterID:       all.Issues[0].ID,
+		Limit:         1,
+	})
+	if len(next.Issues) != 1 || next.Issues[0].Title != "GET /.git/HEAD" || next.HasMore {
+		t.Fatalf("unexpected cursor page: %+v", next)
+	}
+}
+
+func TestNormalizeTitleGroupsVariableMessages(t *testing.T) {
+	cases := [][3]string{
+		// same, same, different: [a, b, expectedEqual(a,b)]
+		{"User 123 not found", "User 456 not found", "user <num> not found"},
+		{"order 550e8400-e29b-41d4-a716-446655440000 failed", "order 550e8400-e29b-41d4-a716-446655440001 failed", "order <id> failed"},
+		{"request 0a1b2c3d4e5f6a7b timed out", "request ff11aa22bb33cc44 timed out", "request <id> timed out"},
+	}
+	for _, c := range cases {
+		if got := NormalizeTitle(c[0]); got != c[2] {
+			t.Errorf("NormalizeTitle(%q) = %q, want %q", c[0], got, c[2])
+		}
+		if NormalizeTitle(c[0]) != NormalizeTitle(c[1]) {
+			t.Errorf("expected %q and %q to normalize identically", c[0], c[1])
+		}
+	}
+	// Words spelled from hex letters without digits must survive.
+	if got := NormalizeTitle("bad failed deadbeef badge"); got != "bad failed deadbeef badge" {
+		t.Errorf("word-like hex incorrectly replaced: %q", got)
+	}
+}
+
+func TestTrackerNormalizedFingerprinting(t *testing.T) {
+	tracker := NewTracker()
+	tracker.Record("log_error", "api", "User 123 not found", Sample{Timestamp: 100, Message: "User 123 not found"})
+	tracker.Record("log_error", "api", "User 98765432 not found", Sample{Timestamp: 200, Message: "User 98765432 not found"})
+
+	list := tracker.List(0)
+	if len(list) != 1 {
+		t.Fatalf("expected variable messages to merge into 1 issue, got %d", len(list))
+	}
+	if list[0].Count != 2 {
+		t.Errorf("expected count 2, got %d", list[0].Count)
+	}
+	// The display title is the first-seen raw message.
+	if list[0].Title != "User 123 not found" {
+		t.Errorf("expected first-seen raw title kept, got %q", list[0].Title)
+	}
+}
+
+func TestTrackerTrendBuckets(t *testing.T) {
+	tracker := NewTracker()
+	// Two occurrences in the same hour.
+	tracker.Record("log_error", "api", "boom", Sample{Timestamp: 100})
+	tracker.Record("log_error", "api", "boom", Sample{Timestamp: 200})
+
+	issue, ok := tracker.Get(tracker.List(1)[0].ID)
+	if !ok {
+		t.Fatal("expected issue to exist")
+	}
+	bucket := HourBucket(100)
+	if issue.Trend[bucket] != 2 {
+		t.Errorf("expected trend[%s]=2, got %d", bucket, issue.Trend[bucket])
+	}
+}
+
+func TestTrackerStatusLifecycle(t *testing.T) {
+	tracker := NewTracker()
+	tracker.Record("log_error", "api", "boom", Sample{Timestamp: 100})
+	id := tracker.List(1)[0].ID
+
+	// Invalid status rejected.
+	if _, ok := tracker.SetStatus(id, "bogus"); ok {
+		t.Error("expected invalid status to be rejected")
+	}
+	// Unknown id rejected.
+	if _, ok := tracker.SetStatus("nope", StatusResolved); ok {
+		t.Error("expected unknown id to be rejected")
+	}
+	// Resolve.
+	resolved, ok := tracker.SetStatus(id, StatusResolved)
+	if !ok || resolved.Status != StatusResolved || resolved.ResolvedAt == 0 {
+		t.Fatalf("expected resolved issue, got %+v ok=%v", resolved, ok)
+	}
+	// A new occurrence reopens it (regression).
+	tracker.Record("log_error", "api", "boom", Sample{Timestamp: 200})
+	issue, _ := tracker.Get(id)
+	if issue.Status != StatusOpen || issue.ResolvedAt != 0 {
+		t.Errorf("expected regression to reopen issue, got status=%s resolvedAt=%d", issue.Status, issue.ResolvedAt)
+	}
+	// Ignored survives new occurrences.
+	if _, ok := tracker.SetStatus(id, StatusIgnored); !ok {
+		t.Fatal("expected ignore to succeed")
+	}
+	tracker.Record("log_error", "api", "boom", Sample{Timestamp: 300})
+	issue, _ = tracker.Get(id)
+	if issue.Status != StatusIgnored {
+		t.Errorf("expected ignored state to survive new occurrence, got %s", issue.Status)
+	}
+}
+
+func TestTrackerPatternsAndSimilarSuggestions(t *testing.T) {
+	tracker := NewTracker()
+	tracker.Record("span_error", "checkout", "HTTP 504: database timeout for order 123", Sample{
+		Timestamp: 100, HTTPRoute: "/orders/123", HTTPStatus: 504, Stacktrace: "db.go:42\nhandler.go:10",
+	})
+	tracker.Record("span_error", "checkout", "HTTP 502: database timeout for order 456", Sample{
+		Timestamp: 200, HTTPRoute: "/orders/456", HTTPStatus: 502, Stacktrace: "db.go:42\nhandler.go:10",
+	})
+	issuesList := tracker.List(0)
+	if len(issuesList) != 2 {
+		t.Fatalf("expected separate exact issues, got %d", len(issuesList))
+	}
+	if issuesList[0].PatternID == "" || issuesList[0].PatternID != issuesList[1].PatternID {
+		t.Fatalf("expected related HTTP variants to share a pattern: %+v", issuesList)
+	}
+	if issuesList[0].Samples[0].NormalizedRoute != "/orders/{id}" {
+		t.Fatalf("expected normalized route, got %q", issuesList[0].Samples[0].NormalizedRoute)
+	}
+	related, ok := tracker.Similar(issuesList[0].ID, 5)
+	if !ok || len(related) != 1 || related[0].Score < .45 {
+		t.Fatalf("expected deterministic related suggestion, got ok=%v related=%+v", ok, related)
+	}
+	if !tracker.SetSimilarityFeedback(issuesList[0].ID, related[0].Issue.ID, "same") {
+		t.Fatal("expected feedback to be accepted")
+	}
+	related, _ = tracker.Similar(issuesList[0].ID, 5)
+	if related[0].Decision != "same" {
+		t.Fatalf("expected feedback decision, got %q", related[0].Decision)
+	}
+	patterns := tracker.Patterns()
+	if len(patterns) != 1 || patterns[0].IssueCount != 2 || patterns[0].OccurrenceCount != 2 {
+		t.Fatalf("expected one two-issue pattern, got %+v", patterns)
+	}
+}