more otel

internal/collector/collector.go

@@ -7,11 +7,16 @@ import (
 	"context"
 	"encoding/json"
 	"log"
+	"os"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 
 	"git.tornberg.me/go-otel/internal/issues"
+	kube "git.tornberg.me/go-otel/internal/kubernetes"
+	"git.tornberg.me/go-otel/internal/prometheus"
+	"git.tornberg.me/go-otel/internal/sourcemaps"
 	"git.tornberg.me/go-otel/internal/storage"
 	"git.tornberg.me/go-otel/internal/websocket"
 
@@ -28,6 +33,7 @@ type Collector struct {
 	history      *ring.Ring
 	historyMutex sync.RWMutex
 	store        storage.Store
+	storeChan    chan websocket.HistoryEntry
 	subscribers  map[chan websocket.HistoryEntry]func(websocket.HistoryEntry) bool
 	subMutex     sync.Mutex
 	waitLimiter  *ipRateLimiter
@@ -35,15 +41,26 @@ type Collector struct {
 
 	ingestLimiter *ipRateLimiter
 	ingestKey     string
+
+	notifyIssuesFn func()
+
+	// statsQuerier is the ClickHouse store when configured; nil otherwise.
+	// The stats endpoints return 501 when it's nil so the UI can hide those
+	// views instead of erroring.
+	statsQuerier storage.StatsQuerier
+	sourceMaps   *sourcemaps.Resolver
+	kubeWatcher  *kube.Watcher
+	prometheus   *prometheus.Client
 }
 
 // New creates a new Collector instance.
 func New(broadcaster websocket.Broadcaster, historySize int, store storage.Store) *Collector {
-	return &Collector{
+	c := &Collector{
 		broadcaster: broadcaster,
 		historySize: historySize,
 		history:     ring.New(historySize),
 		store:       store,
+		storeChan:   make(chan websocket.HistoryEntry, 10000),
 		subscribers: make(map[chan websocket.HistoryEntry]func(websocket.HistoryEntry) bool),
 		// Allow bursts of a few immediate waits, then steady-state ~1 req/sec per IP.
 		waitLimiter: newIPRateLimiter(1, 5),
@@ -53,6 +70,26 @@ func New(broadcaster websocket.Broadcaster, historySize int, store storage.Store
 		// looping) from hammering the collector.
 		ingestLimiter: newIPRateLimiter(20, 50),
 	}
+
+	if store != nil {
+		// Bounded worker pool prevents concurrent ClickHouse INSERTs from
+		// multiplying query and merge memory under bursty ingest traffic.
+		numWorkers := 4
+		if value, err := strconv.Atoi(os.Getenv("OTEL_STORE_WORKERS")); err == nil && value > 0 && value <= 32 {
+			numWorkers = value
+		}
+		for i := 0; i < numWorkers; i++ {
+			go func() {
+				for entry := range c.storeChan {
+					if err := c.store.Append(context.Background(), entry); err != nil {
+						log.Printf("failed to append to store: %v", err)
+					}
+				}
+			}()
+		}
+	}
+
+	return c
 }
 
 // SetIngestKey configures an optional shared key that HTTP OTLP ingestion
@@ -64,19 +101,102 @@ func (c *Collector) SetIngestKey(key string) {
 	c.ingestKey = key
 }
 
+// SetStatsQuerier enables the aggregate stats endpoints (error trends,
+// release comparisons) when a store supporting them is configured.
+func (c *Collector) SetStatsQuerier(q storage.StatsQuerier) {
+	c.statsQuerier = q
+}
+
+func (c *Collector) SetKubernetesWatcher(w *kube.Watcher)     { c.kubeWatcher = w }
+func (c *Collector) SetPrometheusClient(p *prometheus.Client) { c.prometheus = p }
+func (c *Collector) ListRollouts() []kube.Rollout {
+	if c.kubeWatcher == nil {
+		return nil
+	}
+	return c.kubeWatcher.List()
+}
+func (c *Collector) GetRollout(id string) (kube.Rollout, bool) {
+	if c.kubeWatcher == nil {
+		return kube.Rollout{}, false
+	}
+	return c.kubeWatcher.Get(id)
+}
+func (c *Collector) KubernetesEnabled() bool { return c.kubeWatcher != nil && c.kubeWatcher.Enabled() }
+func (c *Collector) KubernetesError() string {
+	if c.kubeWatcher == nil {
+		return "kubernetes integration is not configured"
+	}
+	return c.kubeWatcher.Error()
+}
+
 // ListIssues returns deduplicated error issues, most recently seen first.
 func (c *Collector) ListIssues(limit int) []issues.Issue {
-	return c.issues.List(limit)
+	list := c.issues.ListFiltered(issues.ListOptions{Limit: limit}).Issues
+	if c.sourceMaps != nil {
+		for i := range list {
+			resolveIssue(&list[i], c.sourceMaps)
+		}
+	}
+	return list
+}
+
+// ListIssuesFiltered returns a filtered page of deduplicated issues.
+func (c *Collector) ListIssuesFiltered(opts issues.ListOptions) issues.ListPage {
+	page := c.issues.ListFiltered(opts)
+	if c.sourceMaps != nil {
+		for i := range page.Issues {
+			resolveIssue(&page.Issues[i], c.sourceMaps)
+		}
+	}
+	return page
 }
 
 // GetIssue returns a single issue by id.
 func (c *Collector) GetIssue(id string) (issues.Issue, bool) {
-	return c.issues.Get(id)
+	issue, ok := c.issues.Get(id)
+	if ok && c.sourceMaps != nil {
+		resolveIssue(&issue, c.sourceMaps)
+	}
+	return issue, ok
+}
+
+func (c *Collector) SetSourceMaps(resolver *sourcemaps.Resolver) { c.sourceMaps = resolver }
+
+func resolveIssue(issue *issues.Issue, resolver *sourcemaps.Resolver) {
+	for i := range issue.Samples {
+		issue.Samples[i].ResolvedStacktrace = resolver.ResolveStack(issue.Samples[i].Release, issue.Samples[i].Stacktrace)
+	}
+}
+
+// SetIssueStatus updates the triage state of an issue.
+func (c *Collector) SetIssueStatus(id, status string) (issues.Issue, bool) {
+	return c.issues.SetStatus(id, status)
+}
+
+func (c *Collector) SimilarIssues(id string, limit int) ([]issues.SimilarIssue, bool) {
+	return c.issues.Similar(id, limit)
+}
+
+func (c *Collector) ListPatterns() []issues.Pattern { return c.issues.Patterns() }
+
+func (c *Collector) SetSimilarityFeedback(a, b, decision string) bool {
+	return c.issues.SetSimilarityFeedback(a, b, decision)
 }
 
 // SetBroadcaster updates the broadcaster used by the collector.
 func (c *Collector) SetBroadcaster(broadcaster websocket.Broadcaster) {
 	c.broadcaster = broadcaster
+	if notifier, ok := broadcaster.(interface{ NotifyIssues() }); ok {
+		c.notifyIssuesFn = notifier.NotifyIssues
+	}
+}
+
+// notifyIssues pings WebSocket clients that new error telemetry arrived, so
+// the UI can refresh its Issues view without waiting for its poll interval.
+func (c *Collector) notifyIssues() {
+	if c.notifyIssuesFn != nil {
+		c.notifyIssuesFn()
+	}
 }
 
 func (c *Collector) addToHistory(dataType websocket.DataType, data json.RawMessage) {
@@ -92,12 +212,11 @@ func (c *Collector) addToHistory(dataType websocket.DataType, data json.RawMessa
 	c.historyMutex.Unlock()
 
 	if c.store != nil {
-		// Asynchronously append to store to avoid blocking ingestion
-		go func() {
-			if err := c.store.Append(context.Background(), entry); err != nil {
-				log.Printf("failed to append to store: %v", err)
-			}
-		}()
+		select {
+		case c.storeChan <- entry:
+		default:
+			log.Printf("collector: store write queue full, dropping entry to preserve ingestion")
+		}
 	}
 
 	c.notifySubscribers(entry)
@@ -219,8 +338,10 @@ func (c *Collector) ConsumeTraces(ctx context.Context, td ptrace.Traces) error {
 	}
 
 	// Store and broadcast raw bytes as json.RawMessage to avoid unsafe interface{} usage
-		c.addToHistory(websocket.DataTypeTraces, buf)
-	extractSpanErrors(c.issues, td, time.Now().UnixNano())
+	c.addToHistory(websocket.DataTypeTraces, buf)
+	if extractSpanErrors(c.issues, td, time.Now().UnixNano()) {
+		c.notifyIssues()
+	}
 
 	entry := websocket.Entry{
 		Type: string(websocket.DataTypeTraces),
@@ -241,7 +362,7 @@ func (c *Collector) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) erro
 	}
 
 	// Store and broadcast raw bytes as json.RawMessage to avoid unsafe interface{} usage
-		c.addToHistory(websocket.DataTypeMetrics, buf)
+	c.addToHistory(websocket.DataTypeMetrics, buf)
 
 	entry := websocket.Entry{
 		Type: string(websocket.DataTypeMetrics),
@@ -262,8 +383,10 @@ func (c *Collector) ConsumeLogs(ctx context.Context, ld plog.Logs) error {
 	}
 
 	// Store and broadcast raw bytes as json.RawMessage to avoid unsafe interface{} usage
-		c.addToHistory(websocket.DataTypeLogs, buf)
-	extractLogErrors(c.issues, ld, time.Now().UnixNano())
+	c.addToHistory(websocket.DataTypeLogs, buf)
+	if extractLogErrors(c.issues, ld, time.Now().UnixNano()) {
+		c.notifyIssues()
+	}
 
 	entry := websocket.Entry{
 		Type: string(websocket.DataTypeLogs),

internal/collector/collector_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"git.tornberg.me/go-otel/internal/issues"
 	"git.tornberg.me/go-otel/internal/storage"
 	"git.tornberg.me/go-otel/internal/websocket"
 	"go.opentelemetry.io/collector/pdata/pcommon"
@@ -242,6 +243,77 @@ func TestConsumeTracesRecordsIssue(t *testing.T) {
 	}
 }
 
+func TestConsumeTracesHTTPErrorsAndNaming(t *testing.T) {
+	coll := New(noopBroadcaster{}, 10, nil)
+
+	td := ptrace.NewTraces()
+
+	// 1. Nginx span with 404 and empty name (common for ingress/reverse proxy)
+	rs1 := td.ResourceSpans().AppendEmpty()
+	rs1.Resource().Attributes().PutStr("service.name", "nginx")
+	s1 := rs1.ScopeSpans().AppendEmpty().Spans().AppendEmpty()
+	s1.SetName("") // empty name
+	s1.Attributes().PutInt("http.status_code", 404)
+	s1.Attributes().PutStr("http.method", "GET")
+	s1.Attributes().PutStr("http.target", "/missing-path")
+
+	// 2. e2e-verify span with 500 and service name in span attributes
+	rs2 := td.ResourceSpans().AppendEmpty()
+	// Resource has no service.name, fallback to app attribute on span
+	s2 := rs2.ScopeSpans().AppendEmpty().Spans().AppendEmpty()
+	s2.SetName("test-endpoint")
+	s2.Attributes().PutStr("app.kubernetes.io/name", "e2e-verify")
+	s2.Attributes().PutInt("http.response.status_code", 500)
+	s2.Attributes().PutStr("http.request.method", "POST")
+	s2.Attributes().PutStr("http.route", "/api/v1/test")
+
+	if err := coll.ConsumeTraces(context.Background(), td); err != nil {
+		t.Fatalf("ConsumeTraces failed: %v", err)
+	}
+
+	issuesList := coll.ListIssues(10)
+	if len(issuesList) != 2 {
+		t.Fatalf("expected 2 distinct issues (404 and 500), got %d: %+v", len(issuesList), issuesList)
+	}
+
+	foundNginx := false
+	foundE2E := false
+
+	for _, iss := range issuesList {
+		if iss.Title == "(error)" {
+			t.Errorf("issue title must not be '(error)', got %q", iss.Title)
+		}
+		if iss.Service == "unknown" {
+			t.Errorf("service must not be 'unknown', got %s", iss.Service)
+		}
+		if iss.Service == "nginx" {
+			foundNginx = true
+			if !strings.Contains(iss.Title, "HTTP 404 (Not Found): GET /missing-path") {
+				t.Errorf("expected nginx 404 title, got %q", iss.Title)
+			}
+			if iss.Samples[0].HTTPStatus != 404 {
+				t.Errorf("expected sample HTTPStatus 404, got %d", iss.Samples[0].HTTPStatus)
+			}
+		}
+		if iss.Service == "e2e-verify" {
+			foundE2E = true
+			if !strings.Contains(iss.Title, "HTTP 500 (Internal Server Error): POST /api/v1/test") {
+				t.Errorf("expected e2e-verify 500 title, got %q", iss.Title)
+			}
+			if iss.Samples[0].HTTPStatus != 500 {
+				t.Errorf("expected sample HTTPStatus 500, got %d", iss.Samples[0].HTTPStatus)
+			}
+		}
+	}
+
+	if !foundNginx {
+		t.Errorf("expected to find nginx issue")
+	}
+	if !foundE2E {
+		t.Errorf("expected to find e2e-verify issue")
+	}
+}
+
 func TestConsumeLogsRecordsIssue(t *testing.T) {
 	coll := New(noopBroadcaster{}, 10, nil)
 
@@ -320,6 +392,41 @@ func TestHandleIssuesAndByID(t *testing.T) {
 	}
 }
 
+func TestHandleIssuesFiltersAndPaginates(t *testing.T) {
+	coll := New(noopBroadcaster{}, 10, nil)
+	coll.issues.Record("span_error", "nginx", "GET /.env", issues.Sample{Timestamp: 300, SourceIP: "203.0.113.10", HTTPRoute: "/.env"})
+	coll.issues.Record("span_error", "nginx", "GET /.git/HEAD", issues.Sample{Timestamp: 200, SourceIP: "203.0.113.11", HTTPRoute: "/.git/HEAD"})
+	coll.issues.Record("log_error", "api", "database down", issues.Sample{Timestamp: 100})
+
+	req := httptest.NewRequest(http.MethodGet, "/api/issues?service=nginx&kind=span_error&limit=1", nil)
+	rec := httptest.NewRecorder()
+	coll.HandleIssues(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("expected 200, got %d: %s", rec.Code, rec.Body.String())
+	}
+	var page []issues.Issue
+	if err := json.Unmarshal(rec.Body.Bytes(), &page); err != nil {
+		t.Fatalf("failed to decode issues page: %v", err)
+	}
+	if len(page) != 1 || page[0].Service != "nginx" {
+		t.Fatalf("unexpected filtered page: %+v", page)
+	}
+	cursor := rec.Header().Get("X-Next-Cursor")
+	if cursor == "" {
+		t.Fatal("expected next cursor header")
+	}
+
+	req = httptest.NewRequest(http.MethodGet, "/api/issues?service=nginx&limit=1&cursor="+cursor, nil)
+	rec = httptest.NewRecorder()
+	coll.HandleIssues(rec, req)
+	if err := json.Unmarshal(rec.Body.Bytes(), &page); err != nil {
+		t.Fatalf("failed to decode cursor page: %v", err)
+	}
+	if len(page) != 1 || page[0].Title != "GET /.git/HEAD" {
+		t.Fatalf("unexpected cursor page: %+v", page)
+	}
+}
+
 func TestHandleLogsIngestKey(t *testing.T) {
 	coll := New(noopBroadcaster{}, 10, nil)
 	coll.SetIngestKey("secret")

internal/collector/handlers.go

@@ -4,7 +4,9 @@ package collector
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"io"
 	"log"
 	"net/http"
@@ -12,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	"git.tornberg.me/go-otel/internal/issues"
 	"git.tornberg.me/go-otel/internal/storage"
 	"git.tornberg.me/go-otel/internal/websocket"
 	"go.opentelemetry.io/collector/pdata/plog"
@@ -378,35 +381,282 @@ func (c *Collector) HandleIssues(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	list := c.ListIssues(parseLimit(r, 100))
+	opts, err := parseIssueListOptions(r)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	page := c.ListIssuesFiltered(opts)
 
 	w.Header().Set("Content-Type", "application/json")
-	if err := json.NewEncoder(w).Encode(list); err != nil {
+	if page.HasMore && len(page.Issues) > 0 {
+		last := page.Issues[len(page.Issues)-1]
+		next := encodeIssueCursor(last.LastSeen, last.ID)
+		w.Header().Set("X-Next-Cursor", next)
+		nextQuery := r.URL.Query()
+		nextQuery.Set("cursor", next)
+		w.Header().Set("Link", fmt.Sprintf("<%s?%s>; rel=next", r.URL.Path, nextQuery.Encode()))
+	}
+	// Keep the response as a JSON array for compatibility. Pagination metadata
+	// is exposed in headers so existing UI clients do not need a migration.
+	if err := json.NewEncoder(w).Encode(page.Issues); err != nil {
 		log.Printf("failed to encode issues: %v", err)
 		http.Error(w, "failed to encode issues", http.StatusInternalServerError)
 		return
 	}
 }
 
+const maxIssuePageSize = 200
+
+type issueCursor struct {
+	LastSeen int64  `json:"lastSeen"`
+	ID       string `json:"id"`
+}
+
+func parseIssueListOptions(r *http.Request) (issues.ListOptions, error) {
+	q := r.URL.Query()
+	opts := issues.ListOptions{
+		Service:  q.Get("service"),
+		Kind:     q.Get("kind"),
+		Status:   q.Get("status"),
+		Title:    q.Get("q"),
+		SourceIP: q.Get("sourceIp"),
+		Route:    q.Get("route"),
+		Limit:    parseLimit(r, 100),
+	}
+	if opts.Limit <= 0 {
+		opts.Limit = 100
+	}
+	if opts.Limit > maxIssuePageSize {
+		opts.Limit = maxIssuePageSize
+	}
+	for name, dst := range map[string]*int64{"since": &opts.Since, "until": &opts.Until} {
+		if value := q.Get(name); value != "" {
+			parsed, err := strconv.ParseInt(value, 10, 64)
+			if err != nil || parsed < 0 {
+				return issues.ListOptions{}, fmt.Errorf("invalid %s; expected unix nanoseconds", name)
+			}
+			*dst = parsed
+		}
+	}
+	if opts.Since != 0 && opts.Until != 0 && opts.Since > opts.Until {
+		return issues.ListOptions{}, fmt.Errorf("since must be less than or equal to until")
+	}
+	if value := q.Get("cursor"); value != "" {
+		decoded, err := base64.RawURLEncoding.DecodeString(value)
+		if err != nil {
+			return issues.ListOptions{}, fmt.Errorf("invalid cursor")
+		}
+		var cursor issueCursor
+		if err := json.Unmarshal(decoded, &cursor); err != nil || cursor.LastSeen <= 0 || cursor.ID == "" {
+			return issues.ListOptions{}, fmt.Errorf("invalid cursor")
+		}
+		opts.AfterLastSeen = cursor.LastSeen
+		opts.AfterID = cursor.ID
+	}
+	return opts, nil
+}
+
+func encodeIssueCursor(lastSeen int64, id string) string {
+	data, _ := json.Marshal(issueCursor{LastSeen: lastSeen, ID: id})
+	return base64.RawURLEncoding.EncodeToString(data)
+}
+
+// HandleSourceMap accepts a JavaScript source map for a deployed release.
+// Multipart fields: release, source, file.
+func (c *Collector) HandleSourceMap(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if c.sourceMaps == nil {
+		http.Error(w, "source maps are not configured", http.StatusNotImplemented)
+		return
+	}
+	if err := r.ParseMultipartForm(50 << 20); err != nil {
+		http.Error(w, "invalid multipart body", http.StatusBadRequest)
+		return
+	}
+	release, source := r.FormValue("release"), r.FormValue("source")
+	file, _, err := r.FormFile("file")
+	if err != nil {
+		http.Error(w, "file is required", http.StatusBadRequest)
+		return
+	}
+	defer file.Close()
+	if err := c.sourceMaps.Upload(release, source, file); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	w.WriteHeader(http.StatusCreated)
+}
+
 // HandleIssueByID returns a single issue by id, including its samples.
 func (c *Collector) HandleIssueByID(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	switch r.Method {
+	case http.MethodGet:
+		issue, ok := c.GetIssue(id)
+		if !ok {
+			http.Error(w, "issue not found", http.StatusNotFound)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		if err := json.NewEncoder(w).Encode(issue); err != nil {
+			log.Printf("failed to encode issue: %v", err)
+			http.Error(w, "failed to encode issue", http.StatusInternalServerError)
+		}
+	case http.MethodPatch:
+		c.HandleIssueStatus(w, r, id)
+	default:
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+// HandlePatterns returns deterministic related-issue groups for visualization.
+func (c *Collector) HandlePatterns(w http.ResponseWriter, r *http.Request) {
 	if r.Method != http.MethodGet {
 		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
 		return
 	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(c.ListPatterns())
+}
 
-	id := r.PathValue("id")
-	issue, ok := c.GetIssue(id)
+// HandleRollouts returns normalized Kubernetes Deployment rollout history.
+func (c *Collector) HandleRollouts(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.KubernetesEnabled() {
+		jsonError(w, http.StatusNotImplemented, c.KubernetesError())
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(c.ListRollouts())
+}
+
+func (c *Collector) HandleRolloutByID(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.KubernetesEnabled() {
+		jsonError(w, http.StatusNotImplemented, c.KubernetesError())
+		return
+	}
+	rollout, ok := c.GetRollout(r.PathValue("id"))
+	if !ok {
+		http.Error(w, "rollout not found", http.StatusNotFound)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(rollout)
+}
+
+func (c *Collector) HandleRolloutImpact(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.KubernetesEnabled() {
+		jsonError(w, http.StatusNotImplemented, c.KubernetesError())
+		return
+	}
+	rollout, ok := c.GetRollout(r.PathValue("id"))
+	if !ok {
+		http.Error(w, "rollout not found", http.StatusNotFound)
+		return
+	}
+	response := map[string]interface{}{"configured": c.prometheus != nil, "rollout": rollout, "points": []interface{}{}}
+	if c.prometheus != nil {
+		start := time.Unix(0, rollout.StartedAt).Add(-15 * time.Minute)
+		end := time.Now()
+		if start.Before(end.Add(-24 * time.Hour)) {
+			start = end.Add(-24 * time.Hour)
+		}
+		points, err := c.prometheus.ErrorRate(r.Context(), rollout.Service, rollout.Release, start, end)
+		if err != nil {
+			jsonError(w, http.StatusBadGateway, err.Error())
+			return
+		}
+		response["points"] = points
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(response)
+}
+
+func jsonError(w http.ResponseWriter, status int, message string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(map[string]string{"error": message})
+}
+
+// HandleIssueSimilar returns explainable, non-merging related-issue suggestions.
+func (c *Collector) HandleIssueSimilar(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	limit := parseLimit(r, 10)
+	result, ok := c.SimilarIssues(r.PathValue("id"), limit)
 	if !ok {
 		http.Error(w, "issue not found", http.StatusNotFound)
 		return
 	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(result)
+}
+
+// HandleSimilarityFeedback records a human review label for a pair of issues.
+// Body: {"issueId":"...", "decision":"same|different"}.
+func (c *Collector) HandleSimilarityFeedback(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	var req struct {
+		IssueID  string `json:"issueId"`
+		Decision string `json:"decision"`
+	}
+	if err := json.NewDecoder(io.LimitReader(r.Body, 4096)).Decode(&req); err != nil {
+		http.Error(w, "invalid JSON body", http.StatusBadRequest)
+		return
+	}
+	if !c.SetSimilarityFeedback(r.PathValue("id"), req.IssueID, req.Decision) {
+		http.Error(w, "invalid issue pair or decision", http.StatusBadRequest)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]string{"status": "recorded", "decision": req.Decision})
+}
+
+// HandleIssueStatus updates the triage state (open/resolved/ignored) of an
+// issue. Body: {"status": "resolved"}.
+func (c *Collector) HandleIssueStatus(w http.ResponseWriter, r *http.Request, id string) {
+	var req struct {
+		Status string `json:"status"`
+	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, 4096))
+	if err != nil {
+		http.Error(w, "failed to read request body", http.StatusBadRequest)
+		return
+	}
+	if err := json.Unmarshal(body, &req); err != nil {
+		http.Error(w, "invalid JSON body", http.StatusBadRequest)
+		return
+	}
+
+	issue, ok := c.SetIssueStatus(id, req.Status)
+	if !ok {
+		http.Error(w, "issue not found or invalid status", http.StatusBadRequest)
+		return
+	}
 
 	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(issue); err != nil {
 		log.Printf("failed to encode issue: %v", err)
-		http.Error(w, "failed to encode issue", http.StatusInternalServerError)
-		return
 	}
 }
 
@@ -418,3 +668,102 @@ func (c *Collector) HandleHealth(w http.ResponseWriter, r *http.Request) {
 		log.Printf("failed to encode health check: %v", err)
 	}
 }
+
+// statsEnabled reports whether an aggregate-capable store is configured.
+func (c *Collector) statsEnabled() bool { return c.statsQuerier != nil }
+
+// statsNotEnabled writes the response used when the file store is active:
+// a stable 501 the UI treats as "view not available".
+func statsNotEnabled(w http.ResponseWriter) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusNotImplemented)
+	_ = json.NewEncoder(w).Encode(map[string]string{
+		"error": "stats require OTEL_CLICKHOUSE_ADDR (ClickHouse storage) to be configured",
+	})
+}
+
+// parseStatsCommon reads the shared service/hours params for stats endpoints.
+func parseStatsCommon(r *http.Request) (service string, hours int) {
+	service = r.URL.Query().Get("service")
+	hours = 72
+	if h, err := strconv.Atoi(r.URL.Query().Get("hours")); err == nil && h > 0 && h <= 24*90 {
+		hours = h
+	}
+	return service, hours
+}
+
+// HandleStatsErrors returns hourly/daily total and error counts for the
+// trends graphs. Only available with ClickHouse storage.
+func (c *Collector) HandleStatsErrors(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.statsEnabled() {
+		statsNotEnabled(w)
+		return
+	}
+
+	service, hours := parseStatsCommon(r)
+	daily := hours > 24*14 // beyond two weeks, daily buckets read better
+	points, err := c.statsQuerier.ErrorStats(r.Context(), service, hours, daily)
+	if err != nil {
+		log.Printf("failed to compute error stats: %v", err)
+		http.Error(w, "failed to compute error stats", http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{
+		"granularity": map[bool]string{true: "day", false: "hour"}[daily],
+		"points":      points,
+	})
+}
+
+// HandleStatsReleases returns per-release error-count time series so two
+// deploys can be compared line-by-line. Only available with ClickHouse.
+func (c *Collector) HandleStatsReleases(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.statsEnabled() {
+		statsNotEnabled(w)
+		return
+	}
+
+	service, hours := parseStatsCommon(r)
+	series, err := c.statsQuerier.ReleaseSeries(r.Context(), service, hours)
+	if err != nil {
+		log.Printf("failed to compute release stats: %v", err)
+		http.Error(w, "failed to compute release stats", http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{
+		"series": series,
+	})
+}
+
+// HandleStatsServices lists the distinct services present in the rollup.
+func (c *Collector) HandleStatsServices(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.statsEnabled() {
+		statsNotEnabled(w)
+		return
+	}
+
+	services, err := c.statsQuerier.Services(r.Context())
+	if err != nil {
+		log.Printf("failed to list services: %v", err)
+		http.Error(w, "failed to list services", http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{"services": services})
+}

internal/collector/issue_extract.go

@@ -1,6 +1,8 @@
 package collector
 
 import (
+	"fmt"
+	"strconv"
 	"strings"
 
 	"git.tornberg.me/go-otel/internal/issues"
@@ -9,46 +11,46 @@ import (
 	"go.opentelemetry.io/collector/pdata/ptrace"
 )
 
-// maxIssueTitleLen bounds how much of a log body is kept as an issue title,
-// so a single noisy log line can't blow up issue payload size.
+// maxIssueTitleLen bounds how much of a log body or span title is kept,
+// so a single noisy error can't blow up issue payload size.
 const maxIssueTitleLen = 200
 
-// extractSpanErrors scans traces for spans with an error status and records
-// one issue occurrence per error found.
-func extractSpanErrors(tracker *issues.Tracker, td ptrace.Traces, timestamp int64) {
+// extractSpanErrors scans traces for spans with an error status or error HTTP/gRPC
+// status code, recording one issue occurrence per error found. It reports
+// whether any error was seen, so the caller can notify live clients.
+func extractSpanErrors(tracker *issues.Tracker, td ptrace.Traces, timestamp int64) bool {
+	found := false
 	resourceSpans := td.ResourceSpans()
 	for i := 0; i < resourceSpans.Len(); i++ {
 		rs := resourceSpans.At(i)
-		service := resourceServiceName(rs.Resource())
 
 		scopeSpans := rs.ScopeSpans()
 		for j := 0; j < scopeSpans.Len(); j++ {
 			spans := scopeSpans.At(j).Spans()
 			for k := 0; k < spans.Len(); k++ {
 				span := spans.At(k)
-				if span.Status().Code() != ptrace.StatusCodeError {
+				if !isSpanError(span) {
 					continue
 				}
 
+				service := spanServiceName(rs.Resource(), span)
 				title := spanErrorTitle(span)
-				tracker.Record("span_error", service, title, issues.Sample{
-					Timestamp: timestamp,
-					TraceID:   idString(span.TraceID()),
-					SpanID:    idString(span.SpanID()),
-					Message:   title,
-				})
+				found = true
+				tracker.Record("span_error", service, title, spanSample(span, rs.Resource(), timestamp, title))
 			}
 		}
 	}
+	return found
 }
 
 // extractLogErrors scans logs for error/fatal severity records and records
-// one issue occurrence per error found.
-func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) {
+// one issue occurrence per error found. It reports whether any error was
+// seen, so the caller can notify live clients.
+func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) bool {
+	found := false
 	resourceLogs := ld.ResourceLogs()
 	for i := 0; i < resourceLogs.Len(); i++ {
 		rl := resourceLogs.At(i)
-		service := resourceServiceName(rl.Resource())
 
 		scopeLogs := rl.ScopeLogs()
 		for j := 0; j < scopeLogs.Len(); j++ {
@@ -59,23 +61,223 @@ func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) {
 					continue
 				}
 
+				service := logServiceName(rl.Resource(), record)
 				title := logErrorTitle(record)
+				found = true
 				kind := "log_error"
-				var url string
+				sample := logSample(record, rl.Resource(), timestamp, title)
 				if attrString(record.Attributes(), "source") == "frontend" {
 					kind = "frontend_error"
-					url = attrString(record.Attributes(), "browser.url")
+					sample.URL = attrString(record.Attributes(), "browser.url")
 				}
-				tracker.Record(kind, service, title, issues.Sample{
-					Timestamp: timestamp,
-					TraceID:   idString(record.TraceID()),
-					SpanID:    idString(record.SpanID()),
-					Message:   title,
-					URL:       url,
-				})
+				tracker.Record(kind, service, title, sample)
 			}
 		}
 	}
+	return found
+}
+
+// spanSample builds an enriched issue sample from a span's attributes:
+// stacktrace from exception events, HTTP request context from semconv
+// attributes (old and new names), and client/release metadata.
+func spanSample(span ptrace.Span, res pcommon.Resource, timestamp int64, title string) issues.Sample {
+	sample := issues.Sample{
+		Timestamp:   timestamp,
+		TraceID:     idString(span.TraceID()),
+		SpanID:      idString(span.SpanID()),
+		Message:     title,
+		HTTPMethod:  firstAttr(span.Attributes(), "http.request.method", "http.method"),
+		HTTPRoute:   firstAttr(span.Attributes(), "http.route", "http.target", "url.path"),
+		UserAgent:   firstAttr(span.Attributes(), "user_agent.original", "http.user_agent"),
+		SourceIP:    sourceIP(span.Attributes(), res.Attributes()),
+		Release:     firstAttr(span.Attributes(), "service.release"),
+		Environment: resourceEnvironment(res),
+	}
+	if status, ok := getHTTPStatusCode(span.Attributes()); ok {
+		sample.HTTPStatus = status
+	}
+	sample.Stacktrace = spanStacktrace(span)
+	return sample
+}
+
+// logSample builds an enriched issue sample from a log record's attributes.
+func logSample(record plog.LogRecord, res pcommon.Resource, timestamp int64, title string) issues.Sample {
+	sample := issues.Sample{
+		Timestamp:   timestamp,
+		TraceID:     idString(record.TraceID()),
+		SpanID:      idString(record.SpanID()),
+		Message:     title,
+		Stacktrace:  attrString(record.Attributes(), "exception.stacktrace"),
+		HTTPMethod:  firstAttr(record.Attributes(), "http.request.method", "http.method"),
+		HTTPRoute:   firstAttr(record.Attributes(), "http.route", "http.target", "url.path"),
+		UserAgent:   firstAttr(record.Attributes(), "user_agent.original", "http.user_agent", "browser.userAgent"),
+		SourceIP:    sourceIP(record.Attributes(), res.Attributes()),
+		Release:     firstAttr(record.Attributes(), "service.release"),
+		Environment: firstAttr(record.Attributes(), "service.environment"),
+		Breadcrumbs: attrString(record.Attributes(), "breadcrumbs"),
+	}
+	if status, ok := getHTTPStatusCode(record.Attributes()); ok {
+		sample.HTTPStatus = status
+	}
+	if sample.Environment == "" {
+		sample.Environment = resourceEnvironment(res)
+	}
+	return sample
+}
+
+// sourceIP extracts common client-IP attributes without trusting arbitrary
+// request headers. Instrumentation may attach these to either the span/log or
+// its resource.
+func sourceIP(attrs, resource pcommon.Map) string {
+	ip := firstAttr(attrs,
+		"client.address",
+		"client.ip",
+		"http.client_ip",
+		"net.peer.ip",
+		"network.peer.address",
+		"http.request.header.x-real-ip",
+		"http.request.header.x-forwarded-for",
+		"http.request.header.x_real_ip",
+		"http.request.header.x_forwarded_for",
+	)
+	if ip == "" {
+		ip = firstAttr(resource,
+			"client.address",
+			"client.ip",
+			"http.client_ip",
+			"net.peer.ip",
+			"network.peer.address",
+		)
+	}
+	return strings.TrimSpace(strings.SplitN(ip, ",", 2)[0])
+}
+
+// spanStacktrace returns exception.stacktrace from the span's exception
+// events, falling back to the span status message when it looks like an
+// exception ("type: message" is how spanErrorTitle renders one).
+func spanStacktrace(span ptrace.Span) string {
+	events := span.Events()
+	for i := 0; i < events.Len(); i++ {
+		event := events.At(i)
+		if event.Name() != "exception" {
+			continue
+		}
+		if st := attrString(event.Attributes(), "exception.stacktrace"); st != "" {
+			return truncate(st)
+		}
+	}
+	return ""
+}
+
+// firstAttr returns the first present attribute among the given keys.
+func firstAttr(attrs pcommon.Map, keys ...string) string {
+	for _, key := range keys {
+		if v, ok := attrs.Get(key); ok {
+			return v.AsString()
+		}
+	}
+	return ""
+}
+
+// getHTTPStatusCode extracts an HTTP status code from attributes, supporting
+// both new (http.response.status_code) and old (http.status_code) semconv keys,
+// stored as integer or string.
+func getHTTPStatusCode(attrs pcommon.Map) (int, bool) {
+	for _, key := range []string{"http.response.status_code", "http.status_code"} {
+		if v, ok := attrs.Get(key); ok {
+			switch v.Type() {
+			case pcommon.ValueTypeInt:
+				return int(v.Int()), true
+			case pcommon.ValueTypeStr:
+				if code, err := strconv.Atoi(v.Str()); err == nil {
+					return code, true
+				}
+			default:
+				if code, err := strconv.Atoi(v.AsString()); err == nil {
+					return code, true
+				}
+			}
+		}
+	}
+	return 0, false
+}
+
+// httpStatusText maps common HTTP error status codes to their canonical reason phrase.
+func httpStatusText(code int) string {
+	switch code {
+	case 400:
+		return "Bad Request"
+	case 401:
+		return "Unauthorized"
+	case 402:
+		return "Payment Required"
+	case 403:
+		return "Forbidden"
+	case 404:
+		return "Not Found"
+	case 405:
+		return "Method Not Allowed"
+	case 406:
+		return "Not Acceptable"
+	case 408:
+		return "Request Timeout"
+	case 409:
+		return "Conflict"
+	case 410:
+		return "Gone"
+	case 415:
+		return "Unsupported Media Type"
+	case 422:
+		return "Unprocessable Entity"
+	case 429:
+		return "Too Many Requests"
+	case 500:
+		return "Internal Server Error"
+	case 501:
+		return "Not Implemented"
+	case 502:
+		return "Bad Gateway"
+	case 503:
+		return "Service Unavailable"
+	case 504:
+		return "Gateway Timeout"
+	default:
+		if code >= 400 && code < 500 {
+			return "Client Error"
+		}
+		if code >= 500 && code < 600 {
+			return "Server Error"
+		}
+		return ""
+	}
+}
+
+// isSpanError determines if a span represents an error, checking:
+// 1. Explicit StatusCodeError
+// 2. HTTP response status >= 400
+// 3. gRPC error code (> 0)
+func isSpanError(span ptrace.Span) bool {
+	if span.Status().Code() == ptrace.StatusCodeError {
+		return true
+	}
+	if status, ok := getHTTPStatusCode(span.Attributes()); ok && status >= 400 {
+		return true
+	}
+	if grpcStr := firstAttr(span.Attributes(), "rpc.grpc.status_code"); grpcStr != "" {
+		if code, err := strconv.Atoi(grpcStr); err == nil && code > 0 {
+			return true
+		}
+	}
+	return false
+}
+
+// resourceEnvironment reads deployment.environment (new semconv) or
+// deployment.environment.name from a resource.
+func resourceEnvironment(res pcommon.Resource) string {
+	if v := attrString(res.Attributes(), "deployment.environment.name"); v != "" {
+		return v
+	}
+	return attrString(res.Attributes(), "deployment.environment")
 }
 
 func isErrorLog(record plog.LogRecord) bool {
@@ -91,6 +293,7 @@ func isErrorLog(record plog.LogRecord) bool {
 }
 
 func spanErrorTitle(span ptrace.Span) string {
+	// 1. Exception events take top priority (panics, exceptions with stacktrace).
 	events := span.Events()
 	for i := 0; i < events.Len(); i++ {
 		event := events.At(i)
@@ -109,10 +312,72 @@ func spanErrorTitle(span ptrace.Span) string {
 		}
 	}
 
-	if msg := span.Status().Message(); msg != "" {
-		return span.Name() + ": " + msg
+	// 2. HTTP error titles: "HTTP 404 (Not Found): GET /path" or "HTTP 500 (Internal Server Error): /api"
+	if status, ok := getHTTPStatusCode(span.Attributes()); ok && status >= 400 {
+		statusDesc := fmt.Sprintf("HTTP %d", status)
+		if txt := httpStatusText(status); txt != "" {
+			statusDesc = fmt.Sprintf("HTTP %d (%s)", status, txt)
+		}
+
+		method := firstAttr(span.Attributes(), "http.request.method", "http.method")
+		route := firstAttr(span.Attributes(), "http.route", "http.target", "url.path")
+
+		var endpoint string
+		if method != "" && route != "" {
+			endpoint = method + " " + route
+		} else if route != "" {
+			endpoint = route
+		} else if span.Name() != "" && !strings.EqualFold(span.Name(), "http") {
+			endpoint = span.Name()
+		}
+
+		statusMsg := span.Status().Message()
+		switch {
+		case endpoint != "" && statusMsg != "" && !strings.EqualFold(statusMsg, httpStatusText(status)):
+			return truncate(statusDesc + ": " + endpoint + " (" + statusMsg + ")")
+		case endpoint != "":
+			return truncate(statusDesc + ": " + endpoint)
+		case statusMsg != "":
+			return truncate(statusDesc + ": " + statusMsg)
+		default:
+			return statusDesc
+		}
 	}
-	return span.Name() + " (error)"
+
+	// 3. gRPC error titles
+	if grpcStr := firstAttr(span.Attributes(), "rpc.grpc.status_code"); grpcStr != "" {
+		if code, err := strconv.Atoi(grpcStr); err == nil && code > 0 {
+			method := firstAttr(span.Attributes(), "rpc.method")
+			service := firstAttr(span.Attributes(), "rpc.service")
+			target := span.Name()
+			if service != "" && method != "" {
+				target = service + "/" + method
+			}
+			msg := span.Status().Message()
+			if msg != "" {
+				return truncate(fmt.Sprintf("gRPC error %d: %s (%s)", code, target, msg))
+			}
+			if target != "" {
+				return truncate(fmt.Sprintf("gRPC error %d: %s", code, target))
+			}
+			return fmt.Sprintf("gRPC error %d", code)
+		}
+	}
+
+	// 4. Status message with span name
+	if msg := span.Status().Message(); msg != "" {
+		if span.Name() != "" {
+			return truncate(span.Name() + ": " + msg)
+		}
+		return truncate(msg)
+	}
+
+	// 5. Fallback to span name
+	if span.Name() != "" {
+		return truncate(span.Name() + " (error)")
+	}
+
+	return "span error"
 }
 
 func logErrorTitle(record plog.LogRecord) string {
@@ -139,8 +404,28 @@ func truncate(s string) string {
 	return s[:maxIssueTitleLen] + "…"
 }
 
+func spanServiceName(res pcommon.Resource, span ptrace.Span) string {
+	if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
+		return name
+	}
+	if name := firstAttr(span.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
+		return name
+	}
+	return "unknown"
+}
+
+func logServiceName(res pcommon.Resource, record plog.LogRecord) string {
+	if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
+		return name
+	}
+	if name := firstAttr(record.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
+		return name
+	}
+	return "unknown"
+}
+
 func resourceServiceName(res pcommon.Resource) string {
-	if name := attrString(res.Attributes(), "service.name"); name != "" {
+	if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
 		return name
 	}
 	return "unknown"