Derive deduplicated error issues from traces and logs (Phase 2 backend)

- Add internal/issues.Tracker: fingerprints (kind, service, title) into
  deduplicated Issues with count/firstSeen/lastSeen/affected services and
  bounded sample occurrences, evicting the least-recently-seen issue once
  a cap is reached.
- Add internal/collector/issue_extract.go to pull issues out of ingested
  OTLP data: spans with an error status (preferring exception.type/message
  from an "exception" event, falling back to the span status message), and
  logs at ERROR/FATAL severity (by severity number or text).
- Wire extraction into Collector.ConsumeTraces/ConsumeLogs and expose
  GET /api/issues and GET /api/issues/{id}.
- Fix a pre-existing data race in the collector test's mockStore (unsynced
  access to entries while addToHistory appends asynchronously), found via
  `go test -race` while adding the new issue-tracking tests.
- Update docs/PLAN.md, README.md and agents_example.md for the new API.

The UI "Issues" tab itself lives in the separate frontend project that
builds this server's dist/ bundle; this commit only covers the backend.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

internal/collector/collector.go

@@ -11,6 +11,7 @@ import (
 	"sync"
 	"time"
 
+	"git.tornberg.me/go-otel/internal/issues"
 	"git.tornberg.me/go-otel/internal/storage"
 	"git.tornberg.me/go-otel/internal/websocket"
 
@@ -30,6 +31,7 @@ type Collector struct {
 	subscribers  map[chan websocket.HistoryEntry]func(websocket.HistoryEntry) bool
 	subMutex     sync.Mutex
 	waitLimiter  *ipRateLimiter
+	issues       *issues.Tracker
 }
 
 // New creates a new Collector instance.
@@ -42,9 +44,20 @@ func New(broadcaster websocket.Broadcaster, historySize int, store storage.Store
 		subscribers: make(map[chan websocket.HistoryEntry]func(websocket.HistoryEntry) bool),
 		// Allow bursts of a few immediate waits, then steady-state ~1 req/sec per IP.
 		waitLimiter: newIPRateLimiter(1, 5),
+		issues:      issues.NewTracker(),
 	}
 }
 
+// ListIssues returns deduplicated error issues, most recently seen first.
+func (c *Collector) ListIssues(limit int) []issues.Issue {
+	return c.issues.List(limit)
+}
+
+// GetIssue returns a single issue by id.
+func (c *Collector) GetIssue(id string) (issues.Issue, bool) {
+	return c.issues.Get(id)
+}
+
 // SetBroadcaster updates the broadcaster used by the collector.
 func (c *Collector) SetBroadcaster(broadcaster websocket.Broadcaster) {
 	c.broadcaster = broadcaster
@@ -191,6 +204,7 @@ func (c *Collector) ConsumeTraces(ctx context.Context, td ptrace.Traces) error {
 
 	// Store and broadcast raw bytes as json.RawMessage to avoid unsafe interface{} usage
 		c.addToHistory(websocket.DataTypeTraces, buf)
+	extractSpanErrors(c.issues, td, time.Now().UnixNano())
 
 	entry := websocket.Entry{
 		Type: string(websocket.DataTypeTraces),
@@ -233,6 +247,7 @@ func (c *Collector) ConsumeLogs(ctx context.Context, ld plog.Logs) error {
 
 	// Store and broadcast raw bytes as json.RawMessage to avoid unsafe interface{} usage
 		c.addToHistory(websocket.DataTypeLogs, buf)
+	extractLogErrors(c.issues, ld, time.Now().UnixNano())
 
 	entry := websocket.Entry{
 		Type: string(websocket.DataTypeLogs),

internal/collector/collector_test.go

@@ -6,27 +6,51 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
 	"git.tornberg.me/go-otel/internal/storage"
 	"git.tornberg.me/go-otel/internal/websocket"
+	"go.opentelemetry.io/collector/pdata/pcommon"
+	"go.opentelemetry.io/collector/pdata/plog"
+	"go.opentelemetry.io/collector/pdata/ptrace"
 )
 
+type noopBroadcaster struct{}
+
+func (noopBroadcaster) Broadcast(entry *websocket.Entry) {}
+
+// mockStore is used by tests only; addToHistory appends to the store
+// asynchronously, so access must be synchronized even in single-threaded
+// test flows that poll for the async write to land.
 type mockStore struct {
+	mu      sync.Mutex
 	entries []websocket.HistoryEntry
 }
 
 func (m *mockStore) Append(ctx context.Context, entry websocket.HistoryEntry) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
 	m.entries = append(m.entries, entry)
 	return nil
 }
 
+func (m *mockStore) len() int {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return len(m.entries)
+}
+
 func (m *mockStore) GetHistory(ctx context.Context, limit int) ([]websocket.HistoryEntry, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
 	return m.entries, nil
 }
 
 func (m *mockStore) Search(ctx context.Context, opts storage.SearchOptions, limit int) ([]websocket.HistoryEntry, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
 	var filtered []websocket.HistoryEntry
 	for _, e := range m.entries {
 		if opts.Type != "" && e.Type != opts.Type {
@@ -53,7 +77,7 @@ func TestCollectorWithStore(t *testing.T) {
 		case <-timeout:
 			t.Fatal("timed out waiting for entry to be appended to store")
 		default:
-			if len(store.entries) > 0 {
+			if store.len() > 0 {
 				goto OK
 			}
 			time.Sleep(10 * time.Millisecond)
@@ -61,8 +85,9 @@ func TestCollectorWithStore(t *testing.T) {
 	}
 
 OK:
-	if store.entries[0].Type != "logs" {
-		t.Errorf("expected type 'logs', got %s", store.entries[0].Type)
+	entries, _ := store.GetHistory(context.Background(), 0)
+	if entries[0].Type != "logs" {
+		t.Errorf("expected type 'logs', got %s", entries[0].Type)
 	}
 }
 
@@ -167,3 +192,129 @@ func TestHandleWaitRateLimited(t *testing.T) {
 		t.Fatalf("expected rate limiting to eventually return 429, last code was %d", lastCode)
 	}
 }
+
+func TestConsumeTracesRecordsIssue(t *testing.T) {
+	coll := New(noopBroadcaster{}, 10, nil)
+
+	td := ptrace.NewTraces()
+	rs := td.ResourceSpans().AppendEmpty()
+	rs.Resource().Attributes().PutStr("service.name", "checkout")
+	span := rs.ScopeSpans().AppendEmpty().Spans().AppendEmpty()
+	span.SetName("charge-card")
+	span.Status().SetCode(ptrace.StatusCodeError)
+	span.Status().SetMessage("card declined")
+	span.SetTraceID(pcommon.TraceID{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16})
+
+	if err := coll.ConsumeTraces(context.Background(), td); err != nil {
+		t.Fatalf("ConsumeTraces failed: %v", err)
+	}
+
+	list := coll.ListIssues(10)
+	if len(list) != 1 {
+		t.Fatalf("expected 1 issue, got %d", len(list))
+	}
+	if list[0].Service != "checkout" {
+		t.Errorf("expected service 'checkout', got %s", list[0].Service)
+	}
+	if list[0].Kind != "span_error" {
+		t.Errorf("expected kind 'span_error', got %s", list[0].Kind)
+	}
+	if !strings.Contains(list[0].Title, "card declined") {
+		t.Errorf("expected title to contain status message, got %q", list[0].Title)
+	}
+	if list[0].Samples[0].TraceID == "" {
+		t.Errorf("expected sample to carry the trace id")
+	}
+
+	// A non-error span must not create an issue.
+	okSpan := rs.ScopeSpans().At(0).Spans().AppendEmpty()
+	okSpan.SetName("healthy-op")
+	okSpan.Status().SetCode(ptrace.StatusCodeOk)
+	if err := coll.ConsumeTraces(context.Background(), td); err != nil {
+		t.Fatalf("ConsumeTraces failed: %v", err)
+	}
+
+	// Re-sending the same error span dedupes into the existing issue.
+	list = coll.ListIssues(10)
+	if len(list) != 1 || list[0].Count != 2 {
+		t.Fatalf("expected deduped issue with count 2, got %+v", list)
+	}
+}
+
+func TestConsumeLogsRecordsIssue(t *testing.T) {
+	coll := New(noopBroadcaster{}, 10, nil)
+
+	ld := plog.NewLogs()
+	rl := ld.ResourceLogs().AppendEmpty()
+	rl.Resource().Attributes().PutStr("service.name", "api")
+	record := rl.ScopeLogs().AppendEmpty().LogRecords().AppendEmpty()
+	record.SetSeverityNumber(plog.SeverityNumberError)
+	record.Body().SetStr("panic: nil pointer dereference")
+
+	// An info-level record must not create an issue.
+	infoRecord := rl.ScopeLogs().At(0).LogRecords().AppendEmpty()
+	infoRecord.SetSeverityNumber(plog.SeverityNumberInfo)
+	infoRecord.Body().SetStr("request handled")
+
+	if err := coll.ConsumeLogs(context.Background(), ld); err != nil {
+		t.Fatalf("ConsumeLogs failed: %v", err)
+	}
+
+	list := coll.ListIssues(10)
+	if len(list) != 1 {
+		t.Fatalf("expected 1 issue, got %d", len(list))
+	}
+	if list[0].Kind != "log_error" {
+		t.Errorf("expected kind 'log_error', got %s", list[0].Kind)
+	}
+	if list[0].Title != "panic: nil pointer dereference" {
+		t.Errorf("unexpected title: %q", list[0].Title)
+	}
+}
+
+func TestHandleIssuesAndByID(t *testing.T) {
+	coll := New(noopBroadcaster{}, 10, nil)
+
+	ld := plog.NewLogs()
+	rl := ld.ResourceLogs().AppendEmpty()
+	rl.Resource().Attributes().PutStr("service.name", "api")
+	record := rl.ScopeLogs().AppendEmpty().LogRecords().AppendEmpty()
+	record.SetSeverityNumber(plog.SeverityNumberError)
+	record.Body().SetStr("boom")
+
+	if err := coll.ConsumeLogs(context.Background(), ld); err != nil {
+		t.Fatalf("ConsumeLogs failed: %v", err)
+	}
+
+	req := httptest.NewRequest(http.MethodGet, "/api/issues", nil)
+	rec := httptest.NewRecorder()
+	coll.HandleIssues(rec, req)
+
+	var list []map[string]any
+	if err := json.Unmarshal(rec.Body.Bytes(), &list); err != nil {
+		t.Fatalf("failed to decode issues response: %v", err)
+	}
+	if len(list) != 1 {
+		t.Fatalf("expected 1 issue in response, got %d", len(list))
+	}
+	id, _ := list[0]["id"].(string)
+	if id == "" {
+		t.Fatalf("expected issue to have an id, got %+v", list[0])
+	}
+
+	req = httptest.NewRequest(http.MethodGet, "/api/issues/"+id, nil)
+	req.SetPathValue("id", id)
+	rec = httptest.NewRecorder()
+	coll.HandleIssueByID(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("expected 200 for known issue id, got %d", rec.Code)
+	}
+
+	req = httptest.NewRequest(http.MethodGet, "/api/issues/missing", nil)
+	req.SetPathValue("id", "missing")
+	rec = httptest.NewRecorder()
+	coll.HandleIssueByID(rec, req)
+	if rec.Code != http.StatusNotFound {
+		t.Fatalf("expected 404 for unknown issue id, got %d", rec.Code)
+	}
+}

internal/collector/handlers.go

@@ -339,6 +339,46 @@ func (c *Collector) HandleTraceByID(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// HandleIssues returns deduplicated error issues derived from ingested
+// traces and logs, most recently seen first.
+func (c *Collector) HandleIssues(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	list := c.ListIssues(parseLimit(r, 100))
+
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(list); err != nil {
+		log.Printf("failed to encode issues: %v", err)
+		http.Error(w, "failed to encode issues", http.StatusInternalServerError)
+		return
+	}
+}
+
+// HandleIssueByID returns a single issue by id, including its samples.
+func (c *Collector) HandleIssueByID(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	id := r.PathValue("id")
+	issue, ok := c.GetIssue(id)
+	if !ok {
+		http.Error(w, "issue not found", http.StatusNotFound)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(issue); err != nil {
+		log.Printf("failed to encode issue: %v", err)
+		http.Error(w, "failed to encode issue", http.StatusInternalServerError)
+		return
+	}
+}
+
 // HandleHealth returns the health status of the collector.
 func (c *Collector) HandleHealth(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")

internal/collector/issue_extract.go

@@ -0,0 +1,161 @@
+package collector
+
+import (
+	"strings"
+
+	"git.tornberg.me/go-otel/internal/issues"
+	"go.opentelemetry.io/collector/pdata/pcommon"
+	"go.opentelemetry.io/collector/pdata/plog"
+	"go.opentelemetry.io/collector/pdata/ptrace"
+)
+
+// maxIssueTitleLen bounds how much of a log body is kept as an issue title,
+// so a single noisy log line can't blow up issue payload size.
+const maxIssueTitleLen = 200
+
+// extractSpanErrors scans traces for spans with an error status and records
+// one issue occurrence per error found.
+func extractSpanErrors(tracker *issues.Tracker, td ptrace.Traces, timestamp int64) {
+	resourceSpans := td.ResourceSpans()
+	for i := 0; i < resourceSpans.Len(); i++ {
+		rs := resourceSpans.At(i)
+		service := resourceServiceName(rs.Resource())
+
+		scopeSpans := rs.ScopeSpans()
+		for j := 0; j < scopeSpans.Len(); j++ {
+			spans := scopeSpans.At(j).Spans()
+			for k := 0; k < spans.Len(); k++ {
+				span := spans.At(k)
+				if span.Status().Code() != ptrace.StatusCodeError {
+					continue
+				}
+
+				title := spanErrorTitle(span)
+				tracker.Record("span_error", service, title, issues.Sample{
+					Timestamp: timestamp,
+					TraceID:   idString(span.TraceID()),
+					SpanID:    idString(span.SpanID()),
+					Message:   title,
+				})
+			}
+		}
+	}
+}
+
+// extractLogErrors scans logs for error/fatal severity records and records
+// one issue occurrence per error found.
+func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) {
+	resourceLogs := ld.ResourceLogs()
+	for i := 0; i < resourceLogs.Len(); i++ {
+		rl := resourceLogs.At(i)
+		service := resourceServiceName(rl.Resource())
+
+		scopeLogs := rl.ScopeLogs()
+		for j := 0; j < scopeLogs.Len(); j++ {
+			records := scopeLogs.At(j).LogRecords()
+			for k := 0; k < records.Len(); k++ {
+				record := records.At(k)
+				if !isErrorLog(record) {
+					continue
+				}
+
+				title := logErrorTitle(record)
+				tracker.Record("log_error", service, title, issues.Sample{
+					Timestamp: timestamp,
+					TraceID:   idString(record.TraceID()),
+					SpanID:    idString(record.SpanID()),
+					Message:   title,
+				})
+			}
+		}
+	}
+}
+
+func isErrorLog(record plog.LogRecord) bool {
+	if record.SeverityNumber() >= plog.SeverityNumberError {
+		return true
+	}
+	switch strings.ToUpper(strings.TrimSpace(record.SeverityText())) {
+	case "ERROR", "FATAL", "CRITICAL":
+		return true
+	default:
+		return false
+	}
+}
+
+func spanErrorTitle(span ptrace.Span) string {
+	events := span.Events()
+	for i := 0; i < events.Len(); i++ {
+		event := events.At(i)
+		if event.Name() != "exception" {
+			continue
+		}
+		excType := attrString(event.Attributes(), "exception.type")
+		excMsg := attrString(event.Attributes(), "exception.message")
+		switch {
+		case excType != "" && excMsg != "":
+			return excType + ": " + excMsg
+		case excMsg != "":
+			return excMsg
+		case excType != "":
+			return excType
+		}
+	}
+
+	if msg := span.Status().Message(); msg != "" {
+		return span.Name() + ": " + msg
+	}
+	return span.Name() + " (error)"
+}
+
+func logErrorTitle(record plog.LogRecord) string {
+	excType := attrString(record.Attributes(), "exception.type")
+	excMsg := attrString(record.Attributes(), "exception.message")
+	switch {
+	case excType != "" && excMsg != "":
+		return truncate(excType + ": " + excMsg)
+	case excMsg != "":
+		return truncate(excMsg)
+	}
+
+	body := strings.TrimSpace(record.Body().AsString())
+	if body == "" {
+		return "log error"
+	}
+	return truncate(body)
+}
+
+func truncate(s string) string {
+	if len(s) <= maxIssueTitleLen {
+		return s
+	}
+	return s[:maxIssueTitleLen] + "…"
+}
+
+func resourceServiceName(res pcommon.Resource) string {
+	if name := attrString(res.Attributes(), "service.name"); name != "" {
+		return name
+	}
+	return "unknown"
+}
+
+// attrString safely reads a string attribute, since a Value obtained from a
+// missing Map key is not valid to call methods on.
+func attrString(attrs pcommon.Map, key string) string {
+	if v, ok := attrs.Get(key); ok {
+		return v.AsString()
+	}
+	return ""
+}
+
+type traceOrSpanID interface {
+	IsEmpty() bool
+	String() string
+}
+
+func idString(id traceOrSpanID) string {
+	if id.IsEmpty() {
+		return ""
+	}
+	return id.String()
+}

internal/issues/tracker.go

@@ -0,0 +1,162 @@
+// Package issues derives deduplicated error "issues" from ingested telemetry,
+// so the web UI can show a triaged list instead of raw traces and logs.
+package issues
+
+import (
+	"fmt"
+	"hash/fnv"
+	"sort"
+	"strings"
+	"sync"
+)
+
+// maxIssues bounds memory use: once exceeded, the least-recently-seen issue
+// is evicted to make room for a newly seen distinct one.
+const maxIssues = 2000
+
+// maxSamples bounds how many sample occurrences are kept per issue.
+const maxSamples = 5
+
+// Sample is one occurrence of an issue.
+type Sample struct {
+	Timestamp int64  `json:"timestamp"`
+	TraceID   string `json:"traceId,omitempty"`
+	SpanID    string `json:"spanId,omitempty"`
+	Message   string `json:"message"`
+}
+
+// Issue is a deduplicated group of related errors, fingerprinted by kind,
+// service and title.
+type Issue struct {
+	ID               string   `json:"id"`
+	Title            string   `json:"title"`
+	Kind             string   `json:"kind"` // "span_error" or "log_error"
+	Service          string   `json:"service"`
+	Count            int      `json:"count"`
+	FirstSeen        int64    `json:"firstSeen"`
+	LastSeen         int64    `json:"lastSeen"`
+	AffectedServices []string `json:"affectedServices"`
+	Samples          []Sample `json:"samples"`
+}
+
+// Tracker incrementally fingerprints and deduplicates errors as they're
+// recorded. It is safe for concurrent use.
+type Tracker struct {
+	mu     sync.Mutex
+	issues map[string]*Issue
+}
+
+// NewTracker creates an empty Tracker.
+func NewTracker() *Tracker {
+	return &Tracker{issues: make(map[string]*Issue)}
+}
+
+// Record adds an occurrence of an error, creating a new issue or merging into
+// an existing one with a matching (kind, service, title) fingerprint.
+func (t *Tracker) Record(kind, service, title string, sample Sample) {
+	id := fingerprint(kind, service, title)
+
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	issue, ok := t.issues[id]
+	if !ok {
+		if len(t.issues) >= maxIssues {
+			t.evictOldestLocked()
+		}
+		issue = &Issue{
+			ID:        id,
+			Title:     title,
+			Kind:      kind,
+			Service:   service,
+			FirstSeen: sample.Timestamp,
+		}
+		t.issues[id] = issue
+	}
+
+	issue.Count++
+	issue.LastSeen = sample.Timestamp
+	issue.AffectedServices = addUnique(issue.AffectedServices, service)
+	issue.Samples = append(issue.Samples, sample)
+	if len(issue.Samples) > maxSamples {
+		issue.Samples = issue.Samples[len(issue.Samples)-maxSamples:]
+	}
+}
+
+// evictOldestLocked removes the issue with the smallest LastSeen. Callers
+// must hold t.mu.
+func (t *Tracker) evictOldestLocked() {
+	var oldestID string
+	var oldest int64
+	first := true
+	for id, issue := range t.issues {
+		if first || issue.LastSeen < oldest {
+			oldest = issue.LastSeen
+			oldestID = id
+			first = false
+		}
+	}
+	if oldestID != "" {
+		delete(t.issues, oldestID)
+	}
+}
+
+// List returns issues sorted by LastSeen descending (most recently seen
+// first). A non-positive limit returns all issues.
+func (t *Tracker) List(limit int) []Issue {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	result := make([]Issue, 0, len(t.issues))
+	for _, issue := range t.issues {
+		result = append(result, cloneIssue(issue))
+	}
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].LastSeen > result[j].LastSeen
+	})
+
+	if limit > 0 && len(result) > limit {
+		result = result[:limit]
+	}
+	return result
+}
+
+// Get returns a single issue by id.
+func (t *Tracker) Get(id string) (Issue, bool) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	issue, ok := t.issues[id]
+	if !ok {
+		return Issue{}, false
+	}
+	return cloneIssue(issue), true
+}
+
+// cloneIssue deep-copies the slice fields so returned issues don't share
+// backing arrays with ones still being mutated by Record.
+func cloneIssue(i *Issue) Issue {
+	c := *i
+	c.AffectedServices = append([]string(nil), i.AffectedServices...)
+	c.Samples = append([]Sample(nil), i.Samples...)
+	return c
+}
+
+func fingerprint(parts ...string) string {
+	h := fnv.New64a()
+	_, _ = h.Write([]byte(strings.Join(parts, "\x00")))
+	return fmt.Sprintf("%016x", h.Sum64())
+}
+
+func addUnique(list []string, v string) []string {
+	if v == "" {
+		return list
+	}
+	for _, existing := range list {
+		if existing == v {
+			return list
+		}
+	}
+	return append(list, v)
+}

internal/issues/tracker_test.go

@@ -0,0 +1,75 @@
+package issues
+
+import "testing"
+
+func TestTrackerDedupesByFingerprint(t *testing.T) {
+	tracker := NewTracker()
+
+	tracker.Record("span_error", "checkout", "charge-card: card declined", Sample{Timestamp: 100, TraceID: "t1"})
+	tracker.Record("span_error", "checkout", "charge-card: card declined", Sample{Timestamp: 200, TraceID: "t2"})
+	tracker.Record("span_error", "checkout", "different error", Sample{Timestamp: 150})
+
+	list := tracker.List(0)
+	if len(list) != 2 {
+		t.Fatalf("expected 2 distinct issues, got %d", len(list))
+	}
+
+	// Most recently seen first.
+	if list[0].Title != "charge-card: card declined" {
+		t.Errorf("expected most recent issue first, got %q", list[0].Title)
+	}
+	if list[0].Count != 2 {
+		t.Errorf("expected duplicate occurrences to merge into count 2, got %d", list[0].Count)
+	}
+	if list[0].FirstSeen != 100 || list[0].LastSeen != 200 {
+		t.Errorf("expected firstSeen=100 lastSeen=200, got firstSeen=%d lastSeen=%d", list[0].FirstSeen, list[0].LastSeen)
+	}
+	if len(list[0].Samples) != 2 {
+		t.Errorf("expected 2 samples, got %d", len(list[0].Samples))
+	}
+}
+
+func TestTrackerGet(t *testing.T) {
+	tracker := NewTracker()
+	tracker.Record("log_error", "api", "panic: nil pointer", Sample{Timestamp: 100})
+
+	list := tracker.List(0)
+	if len(list) != 1 {
+		t.Fatalf("expected 1 issue, got %d", len(list))
+	}
+
+	issue, ok := tracker.Get(list[0].ID)
+	if !ok {
+		t.Fatalf("expected to find issue %s", list[0].ID)
+	}
+	if issue.Service != "api" {
+		t.Errorf("expected service 'api', got %s", issue.Service)
+	}
+
+	if _, ok := tracker.Get("does-not-exist"); ok {
+		t.Errorf("expected lookup of unknown id to fail")
+	}
+}
+
+func TestTrackerDifferentServicesDoNotMerge(t *testing.T) {
+	tracker := NewTracker()
+	tracker.Record("span_error", "checkout", "timeout", Sample{Timestamp: 100})
+	tracker.Record("span_error", "billing", "timeout", Sample{Timestamp: 100})
+
+	list := tracker.List(0)
+	if len(list) != 2 {
+		t.Fatalf("expected same title on different services to stay distinct, got %d issues", len(list))
+	}
+}
+
+func TestTrackerListRespectsLimit(t *testing.T) {
+	tracker := NewTracker()
+	for i := 0; i < 5; i++ {
+		tracker.Record("log_error", "api", string(rune('a'+i)), Sample{Timestamp: int64(i)})
+	}
+
+	list := tracker.List(2)
+	if len(list) != 2 {
+		t.Fatalf("expected limit to cap results at 2, got %d", len(list))
+	}
+}