more otel

internal/auth/oidc.go

@@ -0,0 +1,380 @@
+package auth
+
+import (
+	"context"
+	"crypto"
+	"crypto/rsa"
+	"crypto/sha256"
+	"crypto/subtle"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"math/big"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"time"
+)
+
+type contextKey string
+
+// UserKey is the context key for authenticated user claims.
+const UserKey contextKey = "user"
+
+// Audience handles either a single string or an array of strings in JWT audience claim.
+type Audience []string
+
+func (a *Audience) UnmarshalJSON(b []byte) error {
+	if len(b) > 0 && b[0] == '"' {
+		var s string
+		if err := json.Unmarshal(b, &s); err != nil {
+			return err
+		}
+		*a = Audience{s}
+		return nil
+	}
+	return json.Unmarshal(b, (*[]string)(a))
+}
+
+func (a Audience) Contains(s string) bool {
+	for _, v := range a {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+// IDClaims represents verified claims in an OIDC token.
+type IDClaims struct {
+	Iss   string   `json:"iss"`
+	Sub   string   `json:"sub"`
+	Aud   Audience `json:"aud"`
+	Email string   `json:"email,omitempty"`
+	Name  string   `json:"name,omitempty"`
+	Roles []string `json:"roles,omitempty"`
+	Scope string   `json:"scope,omitempty"`
+	Exp   int64    `json:"exp"`
+}
+
+type jwksDoc struct {
+	Keys []struct {
+		Kid string `json:"kid"`
+		Kty string `json:"kty"`
+		Alg string `json:"alg"`
+		N   string `json:"n"`
+		E   string `json:"e"`
+	} `json:"keys"`
+}
+
+// OidcAuth provides OIDC token verification and authentication middleware.
+type OidcAuth struct {
+	issuer     string
+	client     string
+	disabled   bool
+	ingestKey  string
+	agentToken string
+
+	mu      sync.Mutex
+	keys    map[string]*rsa.PublicKey
+	fetched time.Time
+}
+
+// NewOidcAuth constructs a new OidcAuth instance.
+func NewOidcAuth(ingestKey string) *OidcAuth {
+	issuer := os.Getenv("OTEL_OIDC_ISSUER")
+	if issuer == "" {
+		issuer = "https://users.k6n.net"
+	}
+	client := os.Getenv("OTEL_OIDC_CLIENT_ID")
+	if client == "" {
+		client = "otel-debug"
+	}
+	disabled := os.Getenv("OTEL_AUTH_DISABLED") == "true"
+
+	return &OidcAuth{
+		issuer:     strings.TrimSuffix(issuer, "/"),
+		client:     client,
+		disabled:   disabled,
+		ingestKey:  ingestKey,
+		agentToken: os.Getenv("OTEL_AGENT_TOKEN"),
+	}
+}
+
+func (a *OidcAuth) Issuer() string {
+	return a.issuer
+}
+
+func (a *OidcAuth) ClientID() string {
+	return a.client
+}
+
+func (a *OidcAuth) IsDisabled() bool {
+	return a.disabled
+}
+
+func (a *OidcAuth) fetchKeys(ctx context.Context) (map[string]*rsa.PublicKey, error) {
+	url := a.issuer + "/oidc/v1/keys"
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("jwks endpoint returned %d", resp.StatusCode)
+	}
+
+	var doc jwksDoc
+	if err := json.NewDecoder(resp.Body).Decode(&doc); err != nil {
+		return nil, err
+	}
+
+	keys := make(map[string]*rsa.PublicKey, len(doc.Keys))
+	for _, k := range doc.Keys {
+		if k.Kty != "RSA" {
+			continue
+		}
+		n, err := base64.RawURLEncoding.DecodeString(k.N)
+		if err != nil {
+			continue
+		}
+		eBytes, err := base64.RawURLEncoding.DecodeString(k.E)
+		if err != nil || len(eBytes) > 4 {
+			continue
+		}
+		var eInt uint32
+		for _, b := range eBytes {
+			eInt = eInt<<8 | uint32(b)
+		}
+		keys[k.Kid] = &rsa.PublicKey{N: new(big.Int).SetBytes(n), E: int(eInt)}
+	}
+
+	if len(keys) == 0 {
+		return nil, errors.New("jwks contained no RSA keys")
+	}
+	return keys, nil
+}
+
+func (a *OidcAuth) publicKey(ctx context.Context, kid string) (*rsa.PublicKey, error) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+
+	if key, ok := a.keys[kid]; ok {
+		return key, nil
+	}
+
+	// Throttle refreshes to at most once per minute
+	if a.keys == nil || time.Since(a.fetched) > time.Minute {
+		keys, err := a.fetchKeys(ctx)
+		if err != nil {
+			return nil, err
+		}
+		a.keys = keys
+		a.fetched = time.Now()
+	}
+
+	key, ok := a.keys[kid]
+	if !ok {
+		return nil, fmt.Errorf("unknown key id %q", kid)
+	}
+	return key, nil
+}
+
+// VerifyToken verifies an RS256 JWT against the cached JWKS public keys.
+func (a *OidcAuth) VerifyToken(ctx context.Context, token string) (*IDClaims, error) {
+	parts := strings.Split(token, ".")
+	if len(parts) != 3 {
+		return nil, errors.New("malformed token")
+	}
+
+	var header struct {
+		Kid string `json:"kid"`
+		Alg string `json:"alg"`
+	}
+	h, err := base64.RawURLEncoding.DecodeString(parts[0])
+	if err != nil {
+		return nil, fmt.Errorf("decode header: %w", err)
+	}
+	if err := json.Unmarshal(h, &header); err != nil {
+		return nil, fmt.Errorf("parse header: %w", err)
+	}
+	if header.Alg != "RS256" {
+		return nil, fmt.Errorf("unexpected alg %q", header.Alg)
+	}
+
+	key, err := a.publicKey(ctx, header.Kid)
+	if err != nil {
+		return nil, err
+	}
+
+	sig, err := base64.RawURLEncoding.DecodeString(parts[2])
+	if err != nil {
+		return nil, fmt.Errorf("decode signature: %w", err)
+	}
+
+	sum := sha256.Sum256([]byte(parts[0] + "." + parts[1]))
+	if err := rsa.VerifyPKCS1v15(key, crypto.SHA256, sum[:], sig); err != nil {
+		return nil, errors.New("invalid signature")
+	}
+
+	payload, err := base64.RawURLEncoding.DecodeString(parts[1])
+	if err != nil {
+		return nil, fmt.Errorf("decode claims: %w", err)
+	}
+
+	var claims IDClaims
+	if err := json.Unmarshal(payload, &claims); err != nil {
+		return nil, fmt.Errorf("parse claims: %w", err)
+	}
+
+	if claims.Iss != a.issuer {
+		return nil, fmt.Errorf("unexpected issuer %q", claims.Iss)
+	}
+	if !claims.Aud.Contains(a.client) {
+		return nil, errors.New("token not issued for this client")
+	}
+	if claims.Exp != 0 && time.Now().Unix() > claims.Exp {
+		return nil, errors.New("token expired")
+	}
+
+	return &claims, nil
+}
+
+// HandleTokenExchange proxies the OIDC token exchange endpoint so the SPA
+// doesn't face CORS issues.
+func (a *OidcAuth) HandleTokenExchange(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "POST required", http.StatusMethodNotAllowed)
+		return
+	}
+	if err := r.ParseForm(); err != nil {
+		http.Error(w, "invalid form body", http.StatusBadRequest)
+		return
+	}
+
+	// Always ensure client_id is set
+	if r.PostForm.Get("client_id") == "" {
+		r.PostForm.Set("client_id", a.client)
+	}
+
+	resp, err := http.PostForm(a.issuer+"/oauth/v2/token", r.PostForm)
+	if err != nil {
+		log.Printf("token endpoint unreachable: %v", err)
+		http.Error(w, "token endpoint unreachable", http.StatusBadGateway)
+		return
+	}
+	defer resp.Body.Close()
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(resp.StatusCode)
+	_, _ = io.Copy(w, resp.Body)
+}
+
+// HandleConfig returns OIDC client configuration for the SPA frontend.
+func (a *OidcAuth) HandleConfig(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{
+		"issuer":   a.issuer,
+		"clientId": a.client,
+		"disabled": a.disabled,
+	})
+}
+
+// RequireAuth wraps an HTTP handler and ensures valid authentication.
+func (a *OidcAuth) RequireAuth(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if a.disabled {
+			next.ServeHTTP(w, r)
+			return
+		}
+
+		// 1. Allow if upstream proxy (ingress) already verified via auth-url
+		if authUser := r.Header.Get("X-Auth-User"); authUser != "" {
+			ctx := context.WithValue(r.Context(), UserKey, &IDClaims{Sub: authUser, Email: authUser})
+			next.ServeHTTP(w, r.WithContext(ctx))
+			return
+		}
+
+		// 2. Allow if valid ingest key is provided (for background automation / agents)
+		if a.ingestKey != "" {
+			k := r.Header.Get("X-Otel-Ingest-Key")
+			if k == "" {
+				k = r.URL.Query().Get("key")
+			}
+			if k != "" && k == a.ingestKey {
+				next.ServeHTTP(w, r)
+				return
+			}
+		}
+
+		// Service-to-service credential for debugging agents. It is separate
+		// from the ingest key because it grants access to protected debug data.
+		if a.agentToken != "" && constantTimeTokenMatch(r, a.agentToken) {
+			ctx := context.WithValue(r.Context(), UserKey, &IDClaims{Sub: "service-agent", Name: "debug agent"})
+			next.ServeHTTP(w, r.WithContext(ctx))
+			return
+		}
+
+		// 3. Check Bearer token (or access_token query param for WebSocket)
+		token := bearerToken(r)
+		if token != "" {
+			claims, err := a.VerifyToken(r.Context(), token)
+			if err == nil {
+				ctx := context.WithValue(r.Context(), UserKey, claims)
+				next.ServeHTTP(w, r.WithContext(ctx))
+				return
+			}
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusUnauthorized)
+			_ = json.NewEncoder(w).Encode(map[string]string{
+				"error": "invalid token: " + err.Error(),
+			})
+			return
+		}
+
+		// 4. Unauthorized
+		w.Header().Set("WWW-Authenticate", fmt.Sprintf(`Bearer realm="%s"`, a.client))
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusUnauthorized)
+		_ = json.NewEncoder(w).Encode(map[string]string{
+			"error": "authentication required",
+		})
+	})
+}
+
+func constantTimeTokenMatch(r *http.Request, expected string) bool {
+	token := r.Header.Get("X-Otel-Agent-Token")
+	if token == "" {
+		auth := r.Header.Get("Authorization")
+		if len(auth) > 7 && strings.EqualFold(auth[:7], "bearer ") {
+			token = strings.TrimSpace(auth[7:])
+		}
+	}
+	return len(token) == len(expected) && subtle.ConstantTimeCompare([]byte(token), []byte(expected)) == 1
+}
+
+func bearerToken(r *http.Request) string {
+	if t := r.URL.Query().Get("access_token"); t != "" {
+		return t
+	}
+	auth := r.Header.Get("Authorization")
+	if len(auth) > 7 && strings.EqualFold(auth[:7], "bearer ") {
+		return strings.TrimSpace(auth[7:])
+	}
+	return ""
+}
+
+// CurrentUser retrieves the authenticated user claims from the context, if present.
+func CurrentUser(ctx context.Context) *IDClaims {
+	claims, _ := ctx.Value(UserKey).(*IDClaims)
+	return claims
+}

internal/auth/oidc_test.go

@@ -0,0 +1,165 @@
+package auth
+
+import (
+	"context"
+	"crypto"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+func TestAudienceUnmarshal(t *testing.T) {
+	var a1 Audience
+	if err := json.Unmarshal([]byte(`"single-aud"`), &a1); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !a1.Contains("single-aud") || a1.Contains("other") {
+		t.Fatalf("unexpected audience content: %v", a1)
+	}
+
+	var a2 Audience
+	if err := json.Unmarshal([]byte(`["aud1", "aud2"]`), &a2); err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !a2.Contains("aud1") || !a2.Contains("aud2") || a2.Contains("aud3") {
+		t.Fatalf("unexpected audience content: %v", a2)
+	}
+}
+
+func TestVerifyToken(t *testing.T) {
+	privKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		t.Fatalf("failed to generate RSA key: %v", err)
+	}
+
+	auth := &OidcAuth{
+		issuer: "https://test.issuer",
+		client: "test-client",
+		keys: map[string]*rsa.PublicKey{
+			"key-1": &privKey.PublicKey,
+		},
+		fetched: time.Now(),
+	}
+
+	// Helper to mint test token
+	mintToken := func(kid string, claims IDClaims) string {
+		headerJSON, _ := json.Marshal(map[string]string{
+			"kid": kid,
+			"alg": "RS256",
+		})
+		claimsJSON, _ := json.Marshal(claims)
+
+		headerB64 := base64.RawURLEncoding.EncodeToString(headerJSON)
+		payloadB64 := base64.RawURLEncoding.EncodeToString(claimsJSON)
+
+		sum := sha256.Sum256([]byte(headerB64 + "." + payloadB64))
+		sig, _ := rsa.SignPKCS1v15(rand.Reader, privKey, crypto.SHA256, sum[:])
+		sigB64 := base64.RawURLEncoding.EncodeToString(sig)
+
+		return headerB64 + "." + payloadB64 + "." + sigB64
+	}
+
+	// Valid token
+	validToken := mintToken("key-1", IDClaims{
+		Iss: "https://test.issuer",
+		Sub: "user-123",
+		Aud: Audience{"test-client"},
+		Exp: time.Now().Add(time.Hour).Unix(),
+	})
+
+	claims, err := auth.VerifyToken(context.Background(), validToken)
+	if err != nil {
+		t.Fatalf("expected valid token to pass, got: %v", err)
+	}
+	if claims.Sub != "user-123" {
+		t.Fatalf("expected sub user-123, got %s", claims.Sub)
+	}
+
+	// Expired token
+	expiredToken := mintToken("key-1", IDClaims{
+		Iss: "https://test.issuer",
+		Sub: "user-123",
+		Aud: Audience{"test-client"},
+		Exp: time.Now().Add(-time.Hour).Unix(),
+	})
+	if _, err := auth.VerifyToken(context.Background(), expiredToken); err == nil {
+		t.Fatalf("expected expired token to fail")
+	}
+
+	// Wrong audience
+	wrongAudToken := mintToken("key-1", IDClaims{
+		Iss: "https://test.issuer",
+		Sub: "user-123",
+		Aud: Audience{"other-client"},
+		Exp: time.Now().Add(time.Hour).Unix(),
+	})
+	if _, err := auth.VerifyToken(context.Background(), wrongAudToken); err == nil {
+		t.Fatalf("expected wrong audience to fail")
+	}
+
+	// Wrong issuer
+	wrongIssToken := mintToken("key-1", IDClaims{
+		Iss: "https://wrong.issuer",
+		Sub: "user-123",
+		Aud: Audience{"test-client"},
+		Exp: time.Now().Add(time.Hour).Unix(),
+	})
+	if _, err := auth.VerifyToken(context.Background(), wrongIssToken); err == nil {
+		t.Fatalf("expected wrong issuer to fail")
+	}
+}
+
+func TestRequireAuthMiddleware(t *testing.T) {
+	auth := &OidcAuth{
+		issuer:     "https://test.issuer",
+		client:     "test-client",
+		ingestKey:  "secret-key",
+		agentToken: "agent-secret",
+	}
+
+	handler := auth.RequireAuth(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte("ok"))
+	}))
+
+	// 1. Unauthenticated request -> 401
+	req := httptest.NewRequest("GET", "/api/history", nil)
+	rec := httptest.NewRecorder()
+	handler.ServeHTTP(rec, req)
+	if rec.Code != http.StatusUnauthorized {
+		t.Fatalf("expected 401 for unauthenticated request, got %d", rec.Code)
+	}
+
+	// 2. Ingest key in header -> 200
+	reqKey := httptest.NewRequest("GET", "/api/history", nil)
+	reqKey.Header.Set("X-Otel-Ingest-Key", "secret-key")
+	recKey := httptest.NewRecorder()
+	handler.ServeHTTP(recKey, reqKey)
+	if recKey.Code != http.StatusOK {
+		t.Fatalf("expected 200 with valid ingest key, got %d", recKey.Code)
+	}
+
+	// 3. Dedicated service token in a header -> 200
+	reqAgent := httptest.NewRequest("GET", "/api/history", nil)
+	reqAgent.Header.Set("X-Otel-Agent-Token", "agent-secret")
+	recAgent := httptest.NewRecorder()
+	handler.ServeHTTP(recAgent, reqAgent)
+	if recAgent.Code != http.StatusOK {
+		t.Fatalf("expected 200 with valid agent token, got %d", recAgent.Code)
+	}
+
+	// 4. X-Auth-User from upstream ingress -> 200
+	reqAuthUser := httptest.NewRequest("GET", "/api/history", nil)
+	reqAuthUser.Header.Set("X-Auth-User", "mats@tornberg.me")
+	recAuthUser := httptest.NewRecorder()
+	handler.ServeHTTP(recAuthUser, reqAuthUser)
+	if recAuthUser.Code != http.StatusOK {
+		t.Fatalf("expected 200 with X-Auth-User, got %d", recAuthUser.Code)
+	}
+}

internal/collector/collector.go

@@ -7,11 +7,16 @@ import (
 	"context"
 	"encoding/json"
 	"log"
+	"os"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 
 	"git.tornberg.me/go-otel/internal/issues"
+	kube "git.tornberg.me/go-otel/internal/kubernetes"
+	"git.tornberg.me/go-otel/internal/prometheus"
+	"git.tornberg.me/go-otel/internal/sourcemaps"
 	"git.tornberg.me/go-otel/internal/storage"
 	"git.tornberg.me/go-otel/internal/websocket"
 
@@ -28,6 +33,7 @@ type Collector struct {
 	history      *ring.Ring
 	historyMutex sync.RWMutex
 	store        storage.Store
+	storeChan    chan websocket.HistoryEntry
 	subscribers  map[chan websocket.HistoryEntry]func(websocket.HistoryEntry) bool
 	subMutex     sync.Mutex
 	waitLimiter  *ipRateLimiter
@@ -35,15 +41,26 @@ type Collector struct {
 
 	ingestLimiter *ipRateLimiter
 	ingestKey     string
+
+	notifyIssuesFn func()
+
+	// statsQuerier is the ClickHouse store when configured; nil otherwise.
+	// The stats endpoints return 501 when it's nil so the UI can hide those
+	// views instead of erroring.
+	statsQuerier storage.StatsQuerier
+	sourceMaps   *sourcemaps.Resolver
+	kubeWatcher  *kube.Watcher
+	prometheus   *prometheus.Client
 }
 
 // New creates a new Collector instance.
 func New(broadcaster websocket.Broadcaster, historySize int, store storage.Store) *Collector {
-	return &Collector{
+	c := &Collector{
 		broadcaster: broadcaster,
 		historySize: historySize,
 		history:     ring.New(historySize),
 		store:       store,
+		storeChan:   make(chan websocket.HistoryEntry, 10000),
 		subscribers: make(map[chan websocket.HistoryEntry]func(websocket.HistoryEntry) bool),
 		// Allow bursts of a few immediate waits, then steady-state ~1 req/sec per IP.
 		waitLimiter: newIPRateLimiter(1, 5),
@@ -53,6 +70,26 @@ func New(broadcaster websocket.Broadcaster, historySize int, store storage.Store
 		// looping) from hammering the collector.
 		ingestLimiter: newIPRateLimiter(20, 50),
 	}
+
+	if store != nil {
+		// Bounded worker pool prevents concurrent ClickHouse INSERTs from
+		// multiplying query and merge memory under bursty ingest traffic.
+		numWorkers := 4
+		if value, err := strconv.Atoi(os.Getenv("OTEL_STORE_WORKERS")); err == nil && value > 0 && value <= 32 {
+			numWorkers = value
+		}
+		for i := 0; i < numWorkers; i++ {
+			go func() {
+				for entry := range c.storeChan {
+					if err := c.store.Append(context.Background(), entry); err != nil {
+						log.Printf("failed to append to store: %v", err)
+					}
+				}
+			}()
+		}
+	}
+
+	return c
 }
 
 // SetIngestKey configures an optional shared key that HTTP OTLP ingestion
@@ -64,19 +101,102 @@ func (c *Collector) SetIngestKey(key string) {
 	c.ingestKey = key
 }
 
+// SetStatsQuerier enables the aggregate stats endpoints (error trends,
+// release comparisons) when a store supporting them is configured.
+func (c *Collector) SetStatsQuerier(q storage.StatsQuerier) {
+	c.statsQuerier = q
+}
+
+func (c *Collector) SetKubernetesWatcher(w *kube.Watcher)     { c.kubeWatcher = w }
+func (c *Collector) SetPrometheusClient(p *prometheus.Client) { c.prometheus = p }
+func (c *Collector) ListRollouts() []kube.Rollout {
+	if c.kubeWatcher == nil {
+		return nil
+	}
+	return c.kubeWatcher.List()
+}
+func (c *Collector) GetRollout(id string) (kube.Rollout, bool) {
+	if c.kubeWatcher == nil {
+		return kube.Rollout{}, false
+	}
+	return c.kubeWatcher.Get(id)
+}
+func (c *Collector) KubernetesEnabled() bool { return c.kubeWatcher != nil && c.kubeWatcher.Enabled() }
+func (c *Collector) KubernetesError() string {
+	if c.kubeWatcher == nil {
+		return "kubernetes integration is not configured"
+	}
+	return c.kubeWatcher.Error()
+}
+
 // ListIssues returns deduplicated error issues, most recently seen first.
 func (c *Collector) ListIssues(limit int) []issues.Issue {
-	return c.issues.List(limit)
+	list := c.issues.ListFiltered(issues.ListOptions{Limit: limit}).Issues
+	if c.sourceMaps != nil {
+		for i := range list {
+			resolveIssue(&list[i], c.sourceMaps)
+		}
+	}
+	return list
+}
+
+// ListIssuesFiltered returns a filtered page of deduplicated issues.
+func (c *Collector) ListIssuesFiltered(opts issues.ListOptions) issues.ListPage {
+	page := c.issues.ListFiltered(opts)
+	if c.sourceMaps != nil {
+		for i := range page.Issues {
+			resolveIssue(&page.Issues[i], c.sourceMaps)
+		}
+	}
+	return page
 }
 
 // GetIssue returns a single issue by id.
 func (c *Collector) GetIssue(id string) (issues.Issue, bool) {
-	return c.issues.Get(id)
+	issue, ok := c.issues.Get(id)
+	if ok && c.sourceMaps != nil {
+		resolveIssue(&issue, c.sourceMaps)
+	}
+	return issue, ok
+}
+
+func (c *Collector) SetSourceMaps(resolver *sourcemaps.Resolver) { c.sourceMaps = resolver }
+
+func resolveIssue(issue *issues.Issue, resolver *sourcemaps.Resolver) {
+	for i := range issue.Samples {
+		issue.Samples[i].ResolvedStacktrace = resolver.ResolveStack(issue.Samples[i].Release, issue.Samples[i].Stacktrace)
+	}
+}
+
+// SetIssueStatus updates the triage state of an issue.
+func (c *Collector) SetIssueStatus(id, status string) (issues.Issue, bool) {
+	return c.issues.SetStatus(id, status)
+}
+
+func (c *Collector) SimilarIssues(id string, limit int) ([]issues.SimilarIssue, bool) {
+	return c.issues.Similar(id, limit)
+}
+
+func (c *Collector) ListPatterns() []issues.Pattern { return c.issues.Patterns() }
+
+func (c *Collector) SetSimilarityFeedback(a, b, decision string) bool {
+	return c.issues.SetSimilarityFeedback(a, b, decision)
 }
 
 // SetBroadcaster updates the broadcaster used by the collector.
 func (c *Collector) SetBroadcaster(broadcaster websocket.Broadcaster) {
 	c.broadcaster = broadcaster
+	if notifier, ok := broadcaster.(interface{ NotifyIssues() }); ok {
+		c.notifyIssuesFn = notifier.NotifyIssues
+	}
+}
+
+// notifyIssues pings WebSocket clients that new error telemetry arrived, so
+// the UI can refresh its Issues view without waiting for its poll interval.
+func (c *Collector) notifyIssues() {
+	if c.notifyIssuesFn != nil {
+		c.notifyIssuesFn()
+	}
 }
 
 func (c *Collector) addToHistory(dataType websocket.DataType, data json.RawMessage) {
@@ -92,12 +212,11 @@ func (c *Collector) addToHistory(dataType websocket.DataType, data json.RawMessa
 	c.historyMutex.Unlock()
 
 	if c.store != nil {
-		// Asynchronously append to store to avoid blocking ingestion
-		go func() {
-			if err := c.store.Append(context.Background(), entry); err != nil {
-				log.Printf("failed to append to store: %v", err)
-			}
-		}()
+		select {
+		case c.storeChan <- entry:
+		default:
+			log.Printf("collector: store write queue full, dropping entry to preserve ingestion")
+		}
 	}
 
 	c.notifySubscribers(entry)
@@ -219,8 +338,10 @@ func (c *Collector) ConsumeTraces(ctx context.Context, td ptrace.Traces) error {
 	}
 
 	// Store and broadcast raw bytes as json.RawMessage to avoid unsafe interface{} usage
-		c.addToHistory(websocket.DataTypeTraces, buf)
-	extractSpanErrors(c.issues, td, time.Now().UnixNano())
+	c.addToHistory(websocket.DataTypeTraces, buf)
+	if extractSpanErrors(c.issues, td, time.Now().UnixNano()) {
+		c.notifyIssues()
+	}
 
 	entry := websocket.Entry{
 		Type: string(websocket.DataTypeTraces),
@@ -241,7 +362,7 @@ func (c *Collector) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) erro
 	}
 
 	// Store and broadcast raw bytes as json.RawMessage to avoid unsafe interface{} usage
-		c.addToHistory(websocket.DataTypeMetrics, buf)
+	c.addToHistory(websocket.DataTypeMetrics, buf)
 
 	entry := websocket.Entry{
 		Type: string(websocket.DataTypeMetrics),
@@ -262,8 +383,10 @@ func (c *Collector) ConsumeLogs(ctx context.Context, ld plog.Logs) error {
 	}
 
 	// Store and broadcast raw bytes as json.RawMessage to avoid unsafe interface{} usage
-		c.addToHistory(websocket.DataTypeLogs, buf)
-	extractLogErrors(c.issues, ld, time.Now().UnixNano())
+	c.addToHistory(websocket.DataTypeLogs, buf)
+	if extractLogErrors(c.issues, ld, time.Now().UnixNano()) {
+		c.notifyIssues()
+	}
 
 	entry := websocket.Entry{
 		Type: string(websocket.DataTypeLogs),

internal/collector/collector_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"git.tornberg.me/go-otel/internal/issues"
 	"git.tornberg.me/go-otel/internal/storage"
 	"git.tornberg.me/go-otel/internal/websocket"
 	"go.opentelemetry.io/collector/pdata/pcommon"
@@ -242,6 +243,77 @@ func TestConsumeTracesRecordsIssue(t *testing.T) {
 	}
 }
 
+func TestConsumeTracesHTTPErrorsAndNaming(t *testing.T) {
+	coll := New(noopBroadcaster{}, 10, nil)
+
+	td := ptrace.NewTraces()
+
+	// 1. Nginx span with 404 and empty name (common for ingress/reverse proxy)
+	rs1 := td.ResourceSpans().AppendEmpty()
+	rs1.Resource().Attributes().PutStr("service.name", "nginx")
+	s1 := rs1.ScopeSpans().AppendEmpty().Spans().AppendEmpty()
+	s1.SetName("") // empty name
+	s1.Attributes().PutInt("http.status_code", 404)
+	s1.Attributes().PutStr("http.method", "GET")
+	s1.Attributes().PutStr("http.target", "/missing-path")
+
+	// 2. e2e-verify span with 500 and service name in span attributes
+	rs2 := td.ResourceSpans().AppendEmpty()
+	// Resource has no service.name, fallback to app attribute on span
+	s2 := rs2.ScopeSpans().AppendEmpty().Spans().AppendEmpty()
+	s2.SetName("test-endpoint")
+	s2.Attributes().PutStr("app.kubernetes.io/name", "e2e-verify")
+	s2.Attributes().PutInt("http.response.status_code", 500)
+	s2.Attributes().PutStr("http.request.method", "POST")
+	s2.Attributes().PutStr("http.route", "/api/v1/test")
+
+	if err := coll.ConsumeTraces(context.Background(), td); err != nil {
+		t.Fatalf("ConsumeTraces failed: %v", err)
+	}
+
+	issuesList := coll.ListIssues(10)
+	if len(issuesList) != 2 {
+		t.Fatalf("expected 2 distinct issues (404 and 500), got %d: %+v", len(issuesList), issuesList)
+	}
+
+	foundNginx := false
+	foundE2E := false
+
+	for _, iss := range issuesList {
+		if iss.Title == "(error)" {
+			t.Errorf("issue title must not be '(error)', got %q", iss.Title)
+		}
+		if iss.Service == "unknown" {
+			t.Errorf("service must not be 'unknown', got %s", iss.Service)
+		}
+		if iss.Service == "nginx" {
+			foundNginx = true
+			if !strings.Contains(iss.Title, "HTTP 404 (Not Found): GET /missing-path") {
+				t.Errorf("expected nginx 404 title, got %q", iss.Title)
+			}
+			if iss.Samples[0].HTTPStatus != 404 {
+				t.Errorf("expected sample HTTPStatus 404, got %d", iss.Samples[0].HTTPStatus)
+			}
+		}
+		if iss.Service == "e2e-verify" {
+			foundE2E = true
+			if !strings.Contains(iss.Title, "HTTP 500 (Internal Server Error): POST /api/v1/test") {
+				t.Errorf("expected e2e-verify 500 title, got %q", iss.Title)
+			}
+			if iss.Samples[0].HTTPStatus != 500 {
+				t.Errorf("expected sample HTTPStatus 500, got %d", iss.Samples[0].HTTPStatus)
+			}
+		}
+	}
+
+	if !foundNginx {
+		t.Errorf("expected to find nginx issue")
+	}
+	if !foundE2E {
+		t.Errorf("expected to find e2e-verify issue")
+	}
+}
+
 func TestConsumeLogsRecordsIssue(t *testing.T) {
 	coll := New(noopBroadcaster{}, 10, nil)
 
@@ -320,6 +392,41 @@ func TestHandleIssuesAndByID(t *testing.T) {
 	}
 }
 
+func TestHandleIssuesFiltersAndPaginates(t *testing.T) {
+	coll := New(noopBroadcaster{}, 10, nil)
+	coll.issues.Record("span_error", "nginx", "GET /.env", issues.Sample{Timestamp: 300, SourceIP: "203.0.113.10", HTTPRoute: "/.env"})
+	coll.issues.Record("span_error", "nginx", "GET /.git/HEAD", issues.Sample{Timestamp: 200, SourceIP: "203.0.113.11", HTTPRoute: "/.git/HEAD"})
+	coll.issues.Record("log_error", "api", "database down", issues.Sample{Timestamp: 100})
+
+	req := httptest.NewRequest(http.MethodGet, "/api/issues?service=nginx&kind=span_error&limit=1", nil)
+	rec := httptest.NewRecorder()
+	coll.HandleIssues(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("expected 200, got %d: %s", rec.Code, rec.Body.String())
+	}
+	var page []issues.Issue
+	if err := json.Unmarshal(rec.Body.Bytes(), &page); err != nil {
+		t.Fatalf("failed to decode issues page: %v", err)
+	}
+	if len(page) != 1 || page[0].Service != "nginx" {
+		t.Fatalf("unexpected filtered page: %+v", page)
+	}
+	cursor := rec.Header().Get("X-Next-Cursor")
+	if cursor == "" {
+		t.Fatal("expected next cursor header")
+	}
+
+	req = httptest.NewRequest(http.MethodGet, "/api/issues?service=nginx&limit=1&cursor="+cursor, nil)
+	rec = httptest.NewRecorder()
+	coll.HandleIssues(rec, req)
+	if err := json.Unmarshal(rec.Body.Bytes(), &page); err != nil {
+		t.Fatalf("failed to decode cursor page: %v", err)
+	}
+	if len(page) != 1 || page[0].Title != "GET /.git/HEAD" {
+		t.Fatalf("unexpected cursor page: %+v", page)
+	}
+}
+
 func TestHandleLogsIngestKey(t *testing.T) {
 	coll := New(noopBroadcaster{}, 10, nil)
 	coll.SetIngestKey("secret")

internal/collector/handlers.go

@@ -4,7 +4,9 @@ package collector
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"io"
 	"log"
 	"net/http"
@@ -12,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	"git.tornberg.me/go-otel/internal/issues"
 	"git.tornberg.me/go-otel/internal/storage"
 	"git.tornberg.me/go-otel/internal/websocket"
 	"go.opentelemetry.io/collector/pdata/plog"
@@ -378,35 +381,282 @@ func (c *Collector) HandleIssues(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	list := c.ListIssues(parseLimit(r, 100))
+	opts, err := parseIssueListOptions(r)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	page := c.ListIssuesFiltered(opts)
 
 	w.Header().Set("Content-Type", "application/json")
-	if err := json.NewEncoder(w).Encode(list); err != nil {
+	if page.HasMore && len(page.Issues) > 0 {
+		last := page.Issues[len(page.Issues)-1]
+		next := encodeIssueCursor(last.LastSeen, last.ID)
+		w.Header().Set("X-Next-Cursor", next)
+		nextQuery := r.URL.Query()
+		nextQuery.Set("cursor", next)
+		w.Header().Set("Link", fmt.Sprintf("<%s?%s>; rel=next", r.URL.Path, nextQuery.Encode()))
+	}
+	// Keep the response as a JSON array for compatibility. Pagination metadata
+	// is exposed in headers so existing UI clients do not need a migration.
+	if err := json.NewEncoder(w).Encode(page.Issues); err != nil {
 		log.Printf("failed to encode issues: %v", err)
 		http.Error(w, "failed to encode issues", http.StatusInternalServerError)
 		return
 	}
 }
 
+const maxIssuePageSize = 200
+
+type issueCursor struct {
+	LastSeen int64  `json:"lastSeen"`
+	ID       string `json:"id"`
+}
+
+func parseIssueListOptions(r *http.Request) (issues.ListOptions, error) {
+	q := r.URL.Query()
+	opts := issues.ListOptions{
+		Service:  q.Get("service"),
+		Kind:     q.Get("kind"),
+		Status:   q.Get("status"),
+		Title:    q.Get("q"),
+		SourceIP: q.Get("sourceIp"),
+		Route:    q.Get("route"),
+		Limit:    parseLimit(r, 100),
+	}
+	if opts.Limit <= 0 {
+		opts.Limit = 100
+	}
+	if opts.Limit > maxIssuePageSize {
+		opts.Limit = maxIssuePageSize
+	}
+	for name, dst := range map[string]*int64{"since": &opts.Since, "until": &opts.Until} {
+		if value := q.Get(name); value != "" {
+			parsed, err := strconv.ParseInt(value, 10, 64)
+			if err != nil || parsed < 0 {
+				return issues.ListOptions{}, fmt.Errorf("invalid %s; expected unix nanoseconds", name)
+			}
+			*dst = parsed
+		}
+	}
+	if opts.Since != 0 && opts.Until != 0 && opts.Since > opts.Until {
+		return issues.ListOptions{}, fmt.Errorf("since must be less than or equal to until")
+	}
+	if value := q.Get("cursor"); value != "" {
+		decoded, err := base64.RawURLEncoding.DecodeString(value)
+		if err != nil {
+			return issues.ListOptions{}, fmt.Errorf("invalid cursor")
+		}
+		var cursor issueCursor
+		if err := json.Unmarshal(decoded, &cursor); err != nil || cursor.LastSeen <= 0 || cursor.ID == "" {
+			return issues.ListOptions{}, fmt.Errorf("invalid cursor")
+		}
+		opts.AfterLastSeen = cursor.LastSeen
+		opts.AfterID = cursor.ID
+	}
+	return opts, nil
+}
+
+func encodeIssueCursor(lastSeen int64, id string) string {
+	data, _ := json.Marshal(issueCursor{LastSeen: lastSeen, ID: id})
+	return base64.RawURLEncoding.EncodeToString(data)
+}
+
+// HandleSourceMap accepts a JavaScript source map for a deployed release.
+// Multipart fields: release, source, file.
+func (c *Collector) HandleSourceMap(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if c.sourceMaps == nil {
+		http.Error(w, "source maps are not configured", http.StatusNotImplemented)
+		return
+	}
+	if err := r.ParseMultipartForm(50 << 20); err != nil {
+		http.Error(w, "invalid multipart body", http.StatusBadRequest)
+		return
+	}
+	release, source := r.FormValue("release"), r.FormValue("source")
+	file, _, err := r.FormFile("file")
+	if err != nil {
+		http.Error(w, "file is required", http.StatusBadRequest)
+		return
+	}
+	defer file.Close()
+	if err := c.sourceMaps.Upload(release, source, file); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	w.WriteHeader(http.StatusCreated)
+}
+
 // HandleIssueByID returns a single issue by id, including its samples.
 func (c *Collector) HandleIssueByID(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	switch r.Method {
+	case http.MethodGet:
+		issue, ok := c.GetIssue(id)
+		if !ok {
+			http.Error(w, "issue not found", http.StatusNotFound)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		if err := json.NewEncoder(w).Encode(issue); err != nil {
+			log.Printf("failed to encode issue: %v", err)
+			http.Error(w, "failed to encode issue", http.StatusInternalServerError)
+		}
+	case http.MethodPatch:
+		c.HandleIssueStatus(w, r, id)
+	default:
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+// HandlePatterns returns deterministic related-issue groups for visualization.
+func (c *Collector) HandlePatterns(w http.ResponseWriter, r *http.Request) {
 	if r.Method != http.MethodGet {
 		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
 		return
 	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(c.ListPatterns())
+}
 
-	id := r.PathValue("id")
-	issue, ok := c.GetIssue(id)
+// HandleRollouts returns normalized Kubernetes Deployment rollout history.
+func (c *Collector) HandleRollouts(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.KubernetesEnabled() {
+		jsonError(w, http.StatusNotImplemented, c.KubernetesError())
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(c.ListRollouts())
+}
+
+func (c *Collector) HandleRolloutByID(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.KubernetesEnabled() {
+		jsonError(w, http.StatusNotImplemented, c.KubernetesError())
+		return
+	}
+	rollout, ok := c.GetRollout(r.PathValue("id"))
+	if !ok {
+		http.Error(w, "rollout not found", http.StatusNotFound)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(rollout)
+}
+
+func (c *Collector) HandleRolloutImpact(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.KubernetesEnabled() {
+		jsonError(w, http.StatusNotImplemented, c.KubernetesError())
+		return
+	}
+	rollout, ok := c.GetRollout(r.PathValue("id"))
+	if !ok {
+		http.Error(w, "rollout not found", http.StatusNotFound)
+		return
+	}
+	response := map[string]interface{}{"configured": c.prometheus != nil, "rollout": rollout, "points": []interface{}{}}
+	if c.prometheus != nil {
+		start := time.Unix(0, rollout.StartedAt).Add(-15 * time.Minute)
+		end := time.Now()
+		if start.Before(end.Add(-24 * time.Hour)) {
+			start = end.Add(-24 * time.Hour)
+		}
+		points, err := c.prometheus.ErrorRate(r.Context(), rollout.Service, rollout.Release, start, end)
+		if err != nil {
+			jsonError(w, http.StatusBadGateway, err.Error())
+			return
+		}
+		response["points"] = points
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(response)
+}
+
+func jsonError(w http.ResponseWriter, status int, message string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	_ = json.NewEncoder(w).Encode(map[string]string{"error": message})
+}
+
+// HandleIssueSimilar returns explainable, non-merging related-issue suggestions.
+func (c *Collector) HandleIssueSimilar(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	limit := parseLimit(r, 10)
+	result, ok := c.SimilarIssues(r.PathValue("id"), limit)
 	if !ok {
 		http.Error(w, "issue not found", http.StatusNotFound)
 		return
 	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(result)
+}
+
+// HandleSimilarityFeedback records a human review label for a pair of issues.
+// Body: {"issueId":"...", "decision":"same|different"}.
+func (c *Collector) HandleSimilarityFeedback(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	var req struct {
+		IssueID  string `json:"issueId"`
+		Decision string `json:"decision"`
+	}
+	if err := json.NewDecoder(io.LimitReader(r.Body, 4096)).Decode(&req); err != nil {
+		http.Error(w, "invalid JSON body", http.StatusBadRequest)
+		return
+	}
+	if !c.SetSimilarityFeedback(r.PathValue("id"), req.IssueID, req.Decision) {
+		http.Error(w, "invalid issue pair or decision", http.StatusBadRequest)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]string{"status": "recorded", "decision": req.Decision})
+}
+
+// HandleIssueStatus updates the triage state (open/resolved/ignored) of an
+// issue. Body: {"status": "resolved"}.
+func (c *Collector) HandleIssueStatus(w http.ResponseWriter, r *http.Request, id string) {
+	var req struct {
+		Status string `json:"status"`
+	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, 4096))
+	if err != nil {
+		http.Error(w, "failed to read request body", http.StatusBadRequest)
+		return
+	}
+	if err := json.Unmarshal(body, &req); err != nil {
+		http.Error(w, "invalid JSON body", http.StatusBadRequest)
+		return
+	}
+
+	issue, ok := c.SetIssueStatus(id, req.Status)
+	if !ok {
+		http.Error(w, "issue not found or invalid status", http.StatusBadRequest)
+		return
+	}
 
 	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(issue); err != nil {
 		log.Printf("failed to encode issue: %v", err)
-		http.Error(w, "failed to encode issue", http.StatusInternalServerError)
-		return
 	}
 }
 
@@ -418,3 +668,102 @@ func (c *Collector) HandleHealth(w http.ResponseWriter, r *http.Request) {
 		log.Printf("failed to encode health check: %v", err)
 	}
 }
+
+// statsEnabled reports whether an aggregate-capable store is configured.
+func (c *Collector) statsEnabled() bool { return c.statsQuerier != nil }
+
+// statsNotEnabled writes the response used when the file store is active:
+// a stable 501 the UI treats as "view not available".
+func statsNotEnabled(w http.ResponseWriter) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusNotImplemented)
+	_ = json.NewEncoder(w).Encode(map[string]string{
+		"error": "stats require OTEL_CLICKHOUSE_ADDR (ClickHouse storage) to be configured",
+	})
+}
+
+// parseStatsCommon reads the shared service/hours params for stats endpoints.
+func parseStatsCommon(r *http.Request) (service string, hours int) {
+	service = r.URL.Query().Get("service")
+	hours = 72
+	if h, err := strconv.Atoi(r.URL.Query().Get("hours")); err == nil && h > 0 && h <= 24*90 {
+		hours = h
+	}
+	return service, hours
+}
+
+// HandleStatsErrors returns hourly/daily total and error counts for the
+// trends graphs. Only available with ClickHouse storage.
+func (c *Collector) HandleStatsErrors(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.statsEnabled() {
+		statsNotEnabled(w)
+		return
+	}
+
+	service, hours := parseStatsCommon(r)
+	daily := hours > 24*14 // beyond two weeks, daily buckets read better
+	points, err := c.statsQuerier.ErrorStats(r.Context(), service, hours, daily)
+	if err != nil {
+		log.Printf("failed to compute error stats: %v", err)
+		http.Error(w, "failed to compute error stats", http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{
+		"granularity": map[bool]string{true: "day", false: "hour"}[daily],
+		"points":      points,
+	})
+}
+
+// HandleStatsReleases returns per-release error-count time series so two
+// deploys can be compared line-by-line. Only available with ClickHouse.
+func (c *Collector) HandleStatsReleases(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.statsEnabled() {
+		statsNotEnabled(w)
+		return
+	}
+
+	service, hours := parseStatsCommon(r)
+	series, err := c.statsQuerier.ReleaseSeries(r.Context(), service, hours)
+	if err != nil {
+		log.Printf("failed to compute release stats: %v", err)
+		http.Error(w, "failed to compute release stats", http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{
+		"series": series,
+	})
+}
+
+// HandleStatsServices lists the distinct services present in the rollup.
+func (c *Collector) HandleStatsServices(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	if !c.statsEnabled() {
+		statsNotEnabled(w)
+		return
+	}
+
+	services, err := c.statsQuerier.Services(r.Context())
+	if err != nil {
+		log.Printf("failed to list services: %v", err)
+		http.Error(w, "failed to list services", http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{"services": services})
+}

internal/collector/issue_extract.go

@@ -1,6 +1,8 @@
 package collector
 
 import (
+	"fmt"
+	"strconv"
 	"strings"
 
 	"git.tornberg.me/go-otel/internal/issues"
@@ -9,46 +11,46 @@ import (
 	"go.opentelemetry.io/collector/pdata/ptrace"
 )
 
-// maxIssueTitleLen bounds how much of a log body is kept as an issue title,
-// so a single noisy log line can't blow up issue payload size.
+// maxIssueTitleLen bounds how much of a log body or span title is kept,
+// so a single noisy error can't blow up issue payload size.
 const maxIssueTitleLen = 200
 
-// extractSpanErrors scans traces for spans with an error status and records
-// one issue occurrence per error found.
-func extractSpanErrors(tracker *issues.Tracker, td ptrace.Traces, timestamp int64) {
+// extractSpanErrors scans traces for spans with an error status or error HTTP/gRPC
+// status code, recording one issue occurrence per error found. It reports
+// whether any error was seen, so the caller can notify live clients.
+func extractSpanErrors(tracker *issues.Tracker, td ptrace.Traces, timestamp int64) bool {
+	found := false
 	resourceSpans := td.ResourceSpans()
 	for i := 0; i < resourceSpans.Len(); i++ {
 		rs := resourceSpans.At(i)
-		service := resourceServiceName(rs.Resource())
 
 		scopeSpans := rs.ScopeSpans()
 		for j := 0; j < scopeSpans.Len(); j++ {
 			spans := scopeSpans.At(j).Spans()
 			for k := 0; k < spans.Len(); k++ {
 				span := spans.At(k)
-				if span.Status().Code() != ptrace.StatusCodeError {
+				if !isSpanError(span) {
 					continue
 				}
 
+				service := spanServiceName(rs.Resource(), span)
 				title := spanErrorTitle(span)
-				tracker.Record("span_error", service, title, issues.Sample{
-					Timestamp: timestamp,
-					TraceID:   idString(span.TraceID()),
-					SpanID:    idString(span.SpanID()),
-					Message:   title,
-				})
+				found = true
+				tracker.Record("span_error", service, title, spanSample(span, rs.Resource(), timestamp, title))
 			}
 		}
 	}
+	return found
 }
 
 // extractLogErrors scans logs for error/fatal severity records and records
-// one issue occurrence per error found.
-func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) {
+// one issue occurrence per error found. It reports whether any error was
+// seen, so the caller can notify live clients.
+func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) bool {
+	found := false
 	resourceLogs := ld.ResourceLogs()
 	for i := 0; i < resourceLogs.Len(); i++ {
 		rl := resourceLogs.At(i)
-		service := resourceServiceName(rl.Resource())
 
 		scopeLogs := rl.ScopeLogs()
 		for j := 0; j < scopeLogs.Len(); j++ {
@@ -59,23 +61,223 @@ func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) {
 					continue
 				}
 
+				service := logServiceName(rl.Resource(), record)
 				title := logErrorTitle(record)
+				found = true
 				kind := "log_error"
-				var url string
+				sample := logSample(record, rl.Resource(), timestamp, title)
 				if attrString(record.Attributes(), "source") == "frontend" {
 					kind = "frontend_error"
-					url = attrString(record.Attributes(), "browser.url")
+					sample.URL = attrString(record.Attributes(), "browser.url")
 				}
-				tracker.Record(kind, service, title, issues.Sample{
-					Timestamp: timestamp,
-					TraceID:   idString(record.TraceID()),
-					SpanID:    idString(record.SpanID()),
-					Message:   title,
-					URL:       url,
-				})
+				tracker.Record(kind, service, title, sample)
 			}
 		}
 	}
+	return found
+}
+
+// spanSample builds an enriched issue sample from a span's attributes:
+// stacktrace from exception events, HTTP request context from semconv
+// attributes (old and new names), and client/release metadata.
+func spanSample(span ptrace.Span, res pcommon.Resource, timestamp int64, title string) issues.Sample {
+	sample := issues.Sample{
+		Timestamp:   timestamp,
+		TraceID:     idString(span.TraceID()),
+		SpanID:      idString(span.SpanID()),
+		Message:     title,
+		HTTPMethod:  firstAttr(span.Attributes(), "http.request.method", "http.method"),
+		HTTPRoute:   firstAttr(span.Attributes(), "http.route", "http.target", "url.path"),
+		UserAgent:   firstAttr(span.Attributes(), "user_agent.original", "http.user_agent"),
+		SourceIP:    sourceIP(span.Attributes(), res.Attributes()),
+		Release:     firstAttr(span.Attributes(), "service.release"),
+		Environment: resourceEnvironment(res),
+	}
+	if status, ok := getHTTPStatusCode(span.Attributes()); ok {
+		sample.HTTPStatus = status
+	}
+	sample.Stacktrace = spanStacktrace(span)
+	return sample
+}
+
+// logSample builds an enriched issue sample from a log record's attributes.
+func logSample(record plog.LogRecord, res pcommon.Resource, timestamp int64, title string) issues.Sample {
+	sample := issues.Sample{
+		Timestamp:   timestamp,
+		TraceID:     idString(record.TraceID()),
+		SpanID:      idString(record.SpanID()),
+		Message:     title,
+		Stacktrace:  attrString(record.Attributes(), "exception.stacktrace"),
+		HTTPMethod:  firstAttr(record.Attributes(), "http.request.method", "http.method"),
+		HTTPRoute:   firstAttr(record.Attributes(), "http.route", "http.target", "url.path"),
+		UserAgent:   firstAttr(record.Attributes(), "user_agent.original", "http.user_agent", "browser.userAgent"),
+		SourceIP:    sourceIP(record.Attributes(), res.Attributes()),
+		Release:     firstAttr(record.Attributes(), "service.release"),
+		Environment: firstAttr(record.Attributes(), "service.environment"),
+		Breadcrumbs: attrString(record.Attributes(), "breadcrumbs"),
+	}
+	if status, ok := getHTTPStatusCode(record.Attributes()); ok {
+		sample.HTTPStatus = status
+	}
+	if sample.Environment == "" {
+		sample.Environment = resourceEnvironment(res)
+	}
+	return sample
+}
+
+// sourceIP extracts common client-IP attributes without trusting arbitrary
+// request headers. Instrumentation may attach these to either the span/log or
+// its resource.
+func sourceIP(attrs, resource pcommon.Map) string {
+	ip := firstAttr(attrs,
+		"client.address",
+		"client.ip",
+		"http.client_ip",
+		"net.peer.ip",
+		"network.peer.address",
+		"http.request.header.x-real-ip",
+		"http.request.header.x-forwarded-for",
+		"http.request.header.x_real_ip",
+		"http.request.header.x_forwarded_for",
+	)
+	if ip == "" {
+		ip = firstAttr(resource,
+			"client.address",
+			"client.ip",
+			"http.client_ip",
+			"net.peer.ip",
+			"network.peer.address",
+		)
+	}
+	return strings.TrimSpace(strings.SplitN(ip, ",", 2)[0])
+}
+
+// spanStacktrace returns exception.stacktrace from the span's exception
+// events, falling back to the span status message when it looks like an
+// exception ("type: message" is how spanErrorTitle renders one).
+func spanStacktrace(span ptrace.Span) string {
+	events := span.Events()
+	for i := 0; i < events.Len(); i++ {
+		event := events.At(i)
+		if event.Name() != "exception" {
+			continue
+		}
+		if st := attrString(event.Attributes(), "exception.stacktrace"); st != "" {
+			return truncate(st)
+		}
+	}
+	return ""
+}
+
+// firstAttr returns the first present attribute among the given keys.
+func firstAttr(attrs pcommon.Map, keys ...string) string {
+	for _, key := range keys {
+		if v, ok := attrs.Get(key); ok {
+			return v.AsString()
+		}
+	}
+	return ""
+}
+
+// getHTTPStatusCode extracts an HTTP status code from attributes, supporting
+// both new (http.response.status_code) and old (http.status_code) semconv keys,
+// stored as integer or string.
+func getHTTPStatusCode(attrs pcommon.Map) (int, bool) {
+	for _, key := range []string{"http.response.status_code", "http.status_code"} {
+		if v, ok := attrs.Get(key); ok {
+			switch v.Type() {
+			case pcommon.ValueTypeInt:
+				return int(v.Int()), true
+			case pcommon.ValueTypeStr:
+				if code, err := strconv.Atoi(v.Str()); err == nil {
+					return code, true
+				}
+			default:
+				if code, err := strconv.Atoi(v.AsString()); err == nil {
+					return code, true
+				}
+			}
+		}
+	}
+	return 0, false
+}
+
+// httpStatusText maps common HTTP error status codes to their canonical reason phrase.
+func httpStatusText(code int) string {
+	switch code {
+	case 400:
+		return "Bad Request"
+	case 401:
+		return "Unauthorized"
+	case 402:
+		return "Payment Required"
+	case 403:
+		return "Forbidden"
+	case 404:
+		return "Not Found"
+	case 405:
+		return "Method Not Allowed"
+	case 406:
+		return "Not Acceptable"
+	case 408:
+		return "Request Timeout"
+	case 409:
+		return "Conflict"
+	case 410:
+		return "Gone"
+	case 415:
+		return "Unsupported Media Type"
+	case 422:
+		return "Unprocessable Entity"
+	case 429:
+		return "Too Many Requests"
+	case 500:
+		return "Internal Server Error"
+	case 501:
+		return "Not Implemented"
+	case 502:
+		return "Bad Gateway"
+	case 503:
+		return "Service Unavailable"
+	case 504:
+		return "Gateway Timeout"
+	default:
+		if code >= 400 && code < 500 {
+			return "Client Error"
+		}
+		if code >= 500 && code < 600 {
+			return "Server Error"
+		}
+		return ""
+	}
+}
+
+// isSpanError determines if a span represents an error, checking:
+// 1. Explicit StatusCodeError
+// 2. HTTP response status >= 400
+// 3. gRPC error code (> 0)
+func isSpanError(span ptrace.Span) bool {
+	if span.Status().Code() == ptrace.StatusCodeError {
+		return true
+	}
+	if status, ok := getHTTPStatusCode(span.Attributes()); ok && status >= 400 {
+		return true
+	}
+	if grpcStr := firstAttr(span.Attributes(), "rpc.grpc.status_code"); grpcStr != "" {
+		if code, err := strconv.Atoi(grpcStr); err == nil && code > 0 {
+			return true
+		}
+	}
+	return false
+}
+
+// resourceEnvironment reads deployment.environment (new semconv) or
+// deployment.environment.name from a resource.
+func resourceEnvironment(res pcommon.Resource) string {
+	if v := attrString(res.Attributes(), "deployment.environment.name"); v != "" {
+		return v
+	}
+	return attrString(res.Attributes(), "deployment.environment")
 }
 
 func isErrorLog(record plog.LogRecord) bool {
@@ -91,6 +293,7 @@ func isErrorLog(record plog.LogRecord) bool {
 }
 
 func spanErrorTitle(span ptrace.Span) string {
+	// 1. Exception events take top priority (panics, exceptions with stacktrace).
 	events := span.Events()
 	for i := 0; i < events.Len(); i++ {
 		event := events.At(i)
@@ -109,10 +312,72 @@ func spanErrorTitle(span ptrace.Span) string {
 		}
 	}
 
-	if msg := span.Status().Message(); msg != "" {
-		return span.Name() + ": " + msg
+	// 2. HTTP error titles: "HTTP 404 (Not Found): GET /path" or "HTTP 500 (Internal Server Error): /api"
+	if status, ok := getHTTPStatusCode(span.Attributes()); ok && status >= 400 {
+		statusDesc := fmt.Sprintf("HTTP %d", status)
+		if txt := httpStatusText(status); txt != "" {
+			statusDesc = fmt.Sprintf("HTTP %d (%s)", status, txt)
+		}
+
+		method := firstAttr(span.Attributes(), "http.request.method", "http.method")
+		route := firstAttr(span.Attributes(), "http.route", "http.target", "url.path")
+
+		var endpoint string
+		if method != "" && route != "" {
+			endpoint = method + " " + route
+		} else if route != "" {
+			endpoint = route
+		} else if span.Name() != "" && !strings.EqualFold(span.Name(), "http") {
+			endpoint = span.Name()
+		}
+
+		statusMsg := span.Status().Message()
+		switch {
+		case endpoint != "" && statusMsg != "" && !strings.EqualFold(statusMsg, httpStatusText(status)):
+			return truncate(statusDesc + ": " + endpoint + " (" + statusMsg + ")")
+		case endpoint != "":
+			return truncate(statusDesc + ": " + endpoint)
+		case statusMsg != "":
+			return truncate(statusDesc + ": " + statusMsg)
+		default:
+			return statusDesc
+		}
 	}
-	return span.Name() + " (error)"
+
+	// 3. gRPC error titles
+	if grpcStr := firstAttr(span.Attributes(), "rpc.grpc.status_code"); grpcStr != "" {
+		if code, err := strconv.Atoi(grpcStr); err == nil && code > 0 {
+			method := firstAttr(span.Attributes(), "rpc.method")
+			service := firstAttr(span.Attributes(), "rpc.service")
+			target := span.Name()
+			if service != "" && method != "" {
+				target = service + "/" + method
+			}
+			msg := span.Status().Message()
+			if msg != "" {
+				return truncate(fmt.Sprintf("gRPC error %d: %s (%s)", code, target, msg))
+			}
+			if target != "" {
+				return truncate(fmt.Sprintf("gRPC error %d: %s", code, target))
+			}
+			return fmt.Sprintf("gRPC error %d", code)
+		}
+	}
+
+	// 4. Status message with span name
+	if msg := span.Status().Message(); msg != "" {
+		if span.Name() != "" {
+			return truncate(span.Name() + ": " + msg)
+		}
+		return truncate(msg)
+	}
+
+	// 5. Fallback to span name
+	if span.Name() != "" {
+		return truncate(span.Name() + " (error)")
+	}
+
+	return "span error"
 }
 
 func logErrorTitle(record plog.LogRecord) string {
@@ -139,8 +404,28 @@ func truncate(s string) string {
 	return s[:maxIssueTitleLen] + "…"
 }
 
+func spanServiceName(res pcommon.Resource, span ptrace.Span) string {
+	if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
+		return name
+	}
+	if name := firstAttr(span.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
+		return name
+	}
+	return "unknown"
+}
+
+func logServiceName(res pcommon.Resource, record plog.LogRecord) string {
+	if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
+		return name
+	}
+	if name := firstAttr(record.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
+		return name
+	}
+	return "unknown"
+}
+
 func resourceServiceName(res pcommon.Resource) string {
-	if name := attrString(res.Attributes(), "service.name"); name != "" {
+	if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
 		return name
 	}
 	return "unknown"

internal/honeypot/honeypot.go

@@ -0,0 +1,245 @@
+// Package honeypot tracks common vulnerability-scanner requests and serves an
+// inert decoy response. Its promoted IP list can be consumed by an ingress
+// controller or a separate synchronizer.
+package honeypot
+
+import (
+	"crypto/subtle"
+	"encoding/json"
+	"net"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	defaultThreshold = 3
+	defaultWindow    = 15 * time.Minute
+	defaultTTL       = 1 * time.Hour
+	maxTracked       = 10000
+)
+
+type Config struct {
+	Threshold int
+	Window    time.Duration
+	TTL       time.Duration
+	Delay     time.Duration
+	Token     string
+}
+
+type entry struct {
+	Count         int
+	FirstSeen     time.Time
+	LastSeen      time.Time
+	PromotedUntil time.Time
+}
+
+// IP describes an IP promoted for ingress honeypot routing.
+type IP struct {
+	Address       string    `json:"ip"`
+	Count         int       `json:"count"`
+	FirstSeen     time.Time `json:"first_seen"`
+	LastSeen      time.Time `json:"last_seen"`
+	PromotedUntil time.Time `json:"promoted_until"`
+}
+
+type Tracker struct {
+	threshold int
+	window    time.Duration
+	ttl       time.Duration
+	delay     time.Duration
+	token     string
+
+	mu      sync.Mutex
+	clients map[string]entry
+}
+
+func New(cfg Config) *Tracker {
+	if cfg.Threshold <= 0 {
+		cfg.Threshold = defaultThreshold
+	}
+	if cfg.Window <= 0 {
+		cfg.Window = defaultWindow
+	}
+	if cfg.TTL <= 0 {
+		cfg.TTL = defaultTTL
+	}
+	if cfg.Delay < 0 {
+		cfg.Delay = 0
+	}
+	if cfg.Delay > 10*time.Second {
+		cfg.Delay = 10 * time.Second
+	}
+	return &Tracker{threshold: cfg.Threshold, window: cfg.Window, ttl: cfg.TTL, delay: cfg.Delay, token: cfg.Token, clients: make(map[string]entry)}
+}
+
+func ConfigFromEnv() Config {
+	cfg := Config{Threshold: defaultThreshold, Window: defaultWindow, TTL: defaultTTL, Delay: 0, Token: os.Getenv("OTEL_HONEYPOT_TOKEN")}
+	if n, err := strconv.Atoi(os.Getenv("OTEL_HONEYPOT_THRESHOLD")); err == nil && n > 0 {
+		cfg.Threshold = n
+	}
+	if d, err := time.ParseDuration(os.Getenv("OTEL_HONEYPOT_WINDOW")); err == nil && d > 0 {
+		cfg.Window = d
+	}
+	if d, err := time.ParseDuration(os.Getenv("OTEL_HONEYPOT_TTL")); err == nil && d > 0 {
+		cfg.TTL = d
+	}
+	if d, err := time.ParseDuration(os.Getenv("OTEL_HONEYPOT_DELAY")); err == nil && d >= 0 {
+		cfg.Delay = d
+	}
+	return cfg
+}
+
+// Middleware records scanner paths and returns decoys. All other requests
+// continue to the wrapped application unchanged.
+func (t *Tracker) Middleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !isSuspiciousPath(r.URL.Path) {
+			next.ServeHTTP(w, r)
+			return
+		}
+		ip := clientIP(r)
+		promoted := t.record(ip)
+		if promoted && t.delay > 0 {
+			timer := time.NewTimer(t.delay)
+			select {
+			case <-timer.C:
+			case <-r.Context().Done():
+				if !timer.Stop() {
+					<-timer.C
+				}
+				return
+			}
+		}
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		w.Header().Set("Cache-Control", "no-store")
+		w.WriteHeader(http.StatusOK)
+		if strings.Contains(strings.ToLower(r.URL.Path), ".env") {
+			_, _ = w.Write([]byte("# decoy configuration - no real secrets\nAPP_ENV=production\nAPP_DEBUG=false\nDB_HOST=127.0.0.1\nDB_DATABASE=example_decoy\nDB_USERNAME=decoy_reader\nDB_PASSWORD=not-a-real-password\n"))
+			return
+		}
+		_, _ = w.Write([]byte("-- decoy diagnostic response\n-- no database connection was made\n-- DROP TABLE users;\n-- DROP TABLE sessions;\n-- inert text only\n"))
+	})
+}
+
+func (t *Tracker) record(ip string) bool {
+	now := time.Now()
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	state, ok := t.clients[ip]
+	if !ok || now.Sub(state.LastSeen) > t.window {
+		state = entry{FirstSeen: now}
+	}
+	state.Count++
+	state.LastSeen = now
+	if state.Count > t.threshold {
+		state.PromotedUntil = now.Add(t.ttl)
+	}
+	t.clients[ip] = state
+	if len(t.clients) > maxTracked {
+		t.evictOldest()
+	}
+	return !state.PromotedUntil.IsZero() && now.Before(state.PromotedUntil)
+}
+
+// PromotedIPs returns a snapshot suitable for an ingress synchronizer.
+func (t *Tracker) PromotedIPs() []IP {
+	now := time.Now()
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	result := make([]IP, 0)
+	for address, state := range t.clients {
+		if state.PromotedUntil.IsZero() || !now.Before(state.PromotedUntil) {
+			continue
+		}
+		result = append(result, IP{Address: address, Count: state.Count, FirstSeen: state.FirstSeen, LastSeen: state.LastSeen, PromotedUntil: state.PromotedUntil})
+	}
+	return result
+}
+
+// HandleIPs exposes the promoted list only when OTEL_HONEYPOT_TOKEN is set.
+// It is intended for a private ingress-side synchronizer, not public clients.
+func (t *Tracker) HandleIPs(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet || t.token == "" || !validToken(r, t.token) {
+		w.Header().Set("WWW-Authenticate", `Bearer realm="honeypot"`)
+		http.Error(w, "unauthorized", http.StatusUnauthorized)
+		return
+	}
+	t.writeIPs(w)
+}
+
+// HandleIPsForUser is used behind the application's normal authenticated UI
+// middleware. The internal machine endpoint remains protected by Token.
+func (t *Tracker) HandleIPsForUser(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	t.writeIPs(w)
+}
+
+func (t *Tracker) writeIPs(w http.ResponseWriter) {
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Cache-Control", "no-store")
+	_ = json.NewEncoder(w).Encode(t.PromotedIPs())
+}
+
+func validToken(r *http.Request, token string) bool {
+	value := strings.TrimSpace(strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer "))
+	if value == "" {
+		value = r.Header.Get("X-Honeypot-Token")
+	}
+	return len(value) == len(token) && subtle.ConstantTimeCompare([]byte(value), []byte(token)) == 1
+}
+
+func (t *Tracker) evictOldest() {
+	var address string
+	var oldest time.Time
+	for ip, state := range t.clients {
+		if address == "" || state.LastSeen.Before(oldest) {
+			address, oldest = ip, state.LastSeen
+		}
+	}
+	if address != "" {
+		delete(t.clients, address)
+	}
+}
+
+func isSuspiciousPath(path string) bool {
+	path = strings.ToLower(path)
+	for _, marker := range []string{
+		"/.env", "/.git/", "/.svn/", "/wp-", "/xmlrpc.php", "/phpmyadmin",
+		"/pma/", "/vendor/phpunit", "/eval-stdin.php", "/cgi-bin/", "/shell.php",
+		"/cmd.php", "/upload.php", "/v2/keys", "/var/task/", "/dashboard/env-", "/app.js",
+	} {
+		if strings.Contains(path, marker) {
+			return true
+		}
+	}
+	for _, suffix := range []string{".php", ".ini", ".bak", ".swp", ".yml", ".yaml", ".sql", ".json", ".config", ".cfg"} {
+		if strings.HasSuffix(path, suffix) {
+			return true
+		}
+	}
+	return false
+}
+
+func clientIP(r *http.Request) string {
+	// Prefer the single-value header normally overwritten by ingress-nginx.
+	// X-Forwarded-For can contain client-supplied values when a proxy appends
+	// rather than replaces it, so it is only a fallback.
+	if real := strings.TrimSpace(r.Header.Get("X-Real-IP")); real != "" {
+		return real
+	}
+	if forwarded := r.Header.Get("X-Forwarded-For"); forwarded != "" {
+		return strings.TrimSpace(strings.SplitN(forwarded, ",", 2)[0])
+	}
+	host, _, err := net.SplitHostPort(r.RemoteAddr)
+	if err == nil {
+		return host
+	}
+	return r.RemoteAddr
+}

internal/honeypot/honeypot_test.go

@@ -0,0 +1,41 @@
+package honeypot
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestPromotesAfterThresholdAndExpires(t *testing.T) {
+	tracker := New(Config{Threshold: 2, Window: time.Minute, TTL: time.Millisecond, Token: "secret"})
+	for i := 0; i < 3; i++ {
+		req := httptest.NewRequest(http.MethodGet, "https://example.test/.env", nil)
+		req.RemoteAddr = "192.0.2.10:1234"
+		tracker.Middleware(http.HandlerFunc(http.NotFound)).ServeHTTP(httptest.NewRecorder(), req)
+	}
+	if got := len(tracker.PromotedIPs()); got != 1 {
+		t.Fatalf("got %d promoted IPs, want 1", got)
+	}
+	time.Sleep(2 * time.Millisecond)
+	if got := len(tracker.PromotedIPs()); got != 0 {
+		t.Fatalf("got %d expired IPs, want 0", got)
+	}
+}
+
+func TestHandleIPsRequiresToken(t *testing.T) {
+	tracker := New(Config{Threshold: 1, Token: "secret"})
+	recorder := httptest.NewRecorder()
+	tracker.HandleIPs(recorder, httptest.NewRequest(http.MethodGet, "https://example.test/internal/honeypot/ips", nil))
+	if recorder.Code != http.StatusUnauthorized {
+		t.Fatalf("got %d, want 401", recorder.Code)
+	}
+	recorder = httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodGet, "https://example.test/internal/honeypot/ips", nil)
+	req.Header.Set("Authorization", "Bearer secret")
+	tracker.HandleIPs(recorder, req)
+	if recorder.Code != http.StatusOK || !strings.HasPrefix(recorder.Body.String(), "[]") {
+		t.Fatalf("got %d and %q", recorder.Code, recorder.Body.String())
+	}
+}

internal/issues/tracker.go

@@ -5,9 +5,11 @@ package issues
 import (
 	"fmt"
 	"hash/fnv"
+	"regexp"
 	"sort"
 	"strings"
 	"sync"
+	"time"
 )
 
 // maxIssues bounds memory use: once exceeded, the least-recently-seen issue
@@ -17,6 +19,18 @@ const maxIssues = 2000
 // maxSamples bounds how many sample occurrences are kept per issue.
 const maxSamples = 5
 
+// trendWindow bounds how far back the per-hour trend buckets are kept.
+const trendWindow = 48 * time.Hour
+
+// Issue lifecycle states. New occurrences automatically reopen resolved
+// issues (a regression), but stay merged into ignored ones until the user
+// explicitly sets the state back to open.
+const (
+	StatusOpen     = "open"
+	StatusResolved = "resolved"
+	StatusIgnored  = "ignored"
+)
+
 // Sample is one occurrence of an issue.
 type Sample struct {
 	Timestamp int64  `json:"timestamp"`
@@ -26,38 +40,121 @@ type Sample struct {
 	// URL is the page URL an occurrence happened on, populated only for
 	// browser-originated (otel-rum) errors.
 	URL string `json:"url,omitempty"`
+	// Stacktrace is the exception.stacktrace attribute of the occurrence,
+	// when the source telemetry carried one.
+	Stacktrace         string `json:"stacktrace,omitempty"`
+	ResolvedStacktrace string `json:"resolvedStacktrace,omitempty"`
+	// HTTP request context, extracted from the span/log attributes when the
+	// error happened inside an HTTP handler (semconv v1.x old and new names
+	// are both handled by the extractor).
+	HTTPMethod string `json:"httpMethod,omitempty"`
+	HTTPRoute  string `json:"httpRoute,omitempty"`
+	HTTPStatus int    `json:"httpStatus,omitempty"`
+	// Client/release context.
+	UserAgent   string `json:"userAgent,omitempty"`
+	SourceIP    string `json:"sourceIp,omitempty"`
+	Release     string `json:"release,omitempty"`
+	Environment string `json:"environment,omitempty"`
+	// Breadcrumbs is the raw JSON captured by otel-rum.js before the error
+	// (fetch/console events), present only on frontend issues.
+	Breadcrumbs string `json:"breadcrumbs,omitempty"`
+	// Deterministic, non-PII features used for grouping and review.
+	NormalizedMessage string `json:"normalizedMessage,omitempty"`
+	NormalizedRoute   string `json:"normalizedRoute,omitempty"`
+	StatusClass       string `json:"statusClass,omitempty"`
+	StackSignature    string `json:"stackSignature,omitempty"`
 }
 
 // Issue is a deduplicated group of related errors, fingerprinted by kind,
-// service and title.
+// service and a normalized form of the title.
 type Issue struct {
 	ID               string   `json:"id"`
 	Title            string   `json:"title"`
-	Kind             string   `json:"kind"` // "span_error" or "log_error"
+	Kind             string   `json:"kind"` // "span_error", "log_error" or "frontend_error"
 	Service          string   `json:"service"`
 	Count            int      `json:"count"`
 	FirstSeen        int64    `json:"firstSeen"`
 	LastSeen         int64    `json:"lastSeen"`
 	AffectedServices []string `json:"affectedServices"`
 	Samples          []Sample `json:"samples"`
+	// Trend maps UTC hour buckets ("2006-01-02T15:00") to occurrence counts
+	// over the last trendWindow, for sparklines and spike detection.
+	Trend map[string]int `json:"trend,omitempty"`
+	// Status is the triage state: open, resolved or ignored.
+	Status     string `json:"status"`
+	ResolvedAt int64  `json:"resolvedAt,omitempty"`
+	// NormalizedTitle is the deterministic, redacted form used for exact grouping.
+	NormalizedTitle string `json:"normalizedTitle,omitempty"`
+	// PatternID groups related exact issues without merging their operational IDs.
+	PatternID        string `json:"patternId,omitempty"`
+	PatternSignature string `json:"patternSignature,omitempty"`
+}
+
+// SimilarIssue is a deterministic related-issue suggestion. Suggestions never
+// change issue IDs and include the reasons that produced the score.
+type SimilarIssue struct {
+	Issue    Issue    `json:"issue"`
+	Score    float64  `json:"score"`
+	Reasons  []string `json:"reasons"`
+	Decision string   `json:"decision,omitempty"`
+}
+
+// Pattern is a UI-friendly aggregate of exact issues.
+type Pattern struct {
+	ID              string   `json:"id"`
+	Signature       string   `json:"signature"`
+	Service         string   `json:"service"`
+	Kind            string   `json:"kind"`
+	IssueCount      int      `json:"issueCount"`
+	OccurrenceCount int      `json:"occurrenceCount"`
+	LastSeen        int64    `json:"lastSeen"`
+	IssueIDs        []string `json:"issueIds"`
 }
 
 // Tracker incrementally fingerprints and deduplicates errors as they're
 // recorded. It is safe for concurrent use.
 type Tracker struct {
-	mu     sync.Mutex
-	issues map[string]*Issue
+	mu       sync.Mutex
+	issues   map[string]*Issue
+	feedback map[string]string
+}
+
+// ListOptions filters and paginates the issue list. Issues remain ordered by
+// LastSeen descending; AfterLastSeen/AfterID form an opaque-to-callers cursor
+// boundary and make pagination stable when multiple issues share a timestamp.
+type ListOptions struct {
+	Service       string
+	Kind          string
+	Status        string
+	Title         string
+	SourceIP      string
+	Route         string
+	Since         int64
+	Until         int64
+	AfterLastSeen int64
+	AfterID       string
+	Limit         int
+}
+
+// ListPage is a filtered page of issues. HasMore indicates that another page
+// can be requested using the final issue as the cursor boundary.
+type ListPage struct {
+	Issues  []Issue
+	HasMore bool
 }
 
 // NewTracker creates an empty Tracker.
 func NewTracker() *Tracker {
-	return &Tracker{issues: make(map[string]*Issue)}
+	return &Tracker{issues: make(map[string]*Issue), feedback: make(map[string]string)}
 }
 
 // Record adds an occurrence of an error, creating a new issue or merging into
-// an existing one with a matching (kind, service, title) fingerprint.
+// an existing one with a matching (kind, service, normalized title)
+// fingerprint.
 func (t *Tracker) Record(kind, service, title string, sample Sample) {
-	id := fingerprint(kind, service, title)
+	normalizeSample(&sample, title)
+	id := fingerprint(kind, service, NormalizeTitle(title))
+	bucket := HourBucket(sample.Timestamp)
 
 	t.mu.Lock()
 	defer t.mu.Unlock()
@@ -68,24 +165,282 @@ func (t *Tracker) Record(kind, service, title string, sample Sample) {
 			t.evictOldestLocked()
 		}
 		issue = &Issue{
-			ID:        id,
-			Title:     title,
-			Kind:      kind,
-			Service:   service,
-			FirstSeen: sample.Timestamp,
+			ID:              id,
+			Title:           title,
+			Kind:            kind,
+			Service:         service,
+			FirstSeen:       sample.Timestamp,
+			Status:          StatusOpen,
+			NormalizedTitle: NormalizeTitle(title),
 		}
+		issue.PatternSignature = patternSignature(kind, service, title, sample)
+		issue.PatternID = fingerprint("pattern", issue.PatternSignature)
 		t.issues[id] = issue
 	}
 
 	issue.Count++
 	issue.LastSeen = sample.Timestamp
 	issue.AffectedServices = addUnique(issue.AffectedServices, service)
+	if issue.Trend == nil {
+		issue.Trend = make(map[string]int)
+	}
+	issue.Trend[bucket]++
+	pruneTrendLocked(issue.Trend, sample.Timestamp)
+	// A resolved issue that fires again is a regression: reopen it. Ignored
+	// issues keep their state (that's the point of ignoring a noisy one).
+	if issue.Status == StatusResolved {
+		issue.Status = StatusOpen
+		issue.ResolvedAt = 0
+	}
 	issue.Samples = append(issue.Samples, sample)
 	if len(issue.Samples) > maxSamples {
 		issue.Samples = issue.Samples[len(issue.Samples)-maxSamples:]
 	}
 }
 
+// Similar returns bounded, deterministic near-match suggestions for an issue.
+func (t *Tracker) Similar(id string, limit int) ([]SimilarIssue, bool) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	target, ok := t.issues[id]
+	if !ok {
+		return nil, false
+	}
+	if limit <= 0 || limit > 50 {
+		limit = 10
+	}
+	result := make([]SimilarIssue, 0)
+	for otherID, other := range t.issues {
+		if otherID == id || other.Service != target.Service || other.Kind != target.Kind {
+			continue
+		}
+		score, reasons := similarity(*target, *other)
+		if score < 0.45 {
+			continue
+		}
+		decision := t.feedback[feedbackKey(id, otherID)]
+		result = append(result, SimilarIssue{Issue: cloneIssue(other), Score: score, Reasons: reasons, Decision: decision})
+	}
+	sort.Slice(result, func(i, j int) bool {
+		if result[i].Score != result[j].Score {
+			return result[i].Score > result[j].Score
+		}
+		return result[i].Issue.ID < result[j].Issue.ID
+	})
+	if len(result) > limit {
+		result = result[:limit]
+	}
+	return result, true
+}
+
+// Patterns returns deterministic groups built from exact issue IDs.
+func (t *Tracker) Patterns() []Pattern {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	groups := make(map[string]*Pattern)
+	for _, issue := range t.issues {
+		p := groups[issue.PatternID]
+		if p == nil {
+			p = &Pattern{ID: issue.PatternID, Signature: issue.PatternSignature, Service: issue.Service, Kind: issue.Kind}
+			groups[issue.PatternID] = p
+		}
+		p.IssueCount++
+		p.OccurrenceCount += issue.Count
+		p.IssueIDs = append(p.IssueIDs, issue.ID)
+		if issue.LastSeen > p.LastSeen {
+			p.LastSeen = issue.LastSeen
+		}
+	}
+	result := make([]Pattern, 0, len(groups))
+	for _, p := range groups {
+		sort.Strings(p.IssueIDs)
+		result = append(result, *p)
+	}
+	sort.Slice(result, func(i, j int) bool {
+		if result[i].LastSeen != result[j].LastSeen {
+			return result[i].LastSeen > result[j].LastSeen
+		}
+		return result[i].ID < result[j].ID
+	})
+	return result
+}
+
+// SetSimilarityFeedback records a human label for later training data.
+func (t *Tracker) SetSimilarityFeedback(a, b, decision string) bool {
+	if a == "" || b == "" || a == b || (decision != "same" && decision != "different") {
+		return false
+	}
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if _, ok := t.issues[a]; !ok {
+		return false
+	}
+	if _, ok := t.issues[b]; !ok {
+		return false
+	}
+	t.feedback[feedbackKey(a, b)] = decision
+	return true
+}
+
+func feedbackKey(a, b string) string {
+	if a > b {
+		a, b = b, a
+	}
+	return a + "\x00" + b
+}
+
+func patternSignature(kind, service, title string, sample Sample) string {
+	route := sample.NormalizedRoute
+	status := sample.StatusClass
+	stack := sample.StackSignature
+	// The title is deliberately broad here: the exact normalized title remains
+	// the canonical issue identity, while this signature supports UI grouping.
+	title = normalizePatternTitle(title)
+	return strings.Join([]string{service, kind, route, status, title, stack}, "\x00")
+}
+
+func normalizeSample(sample *Sample, title string) {
+	sample.NormalizedMessage = NormalizeTitle(title)
+	sample.NormalizedRoute = NormalizeRoute(sample.HTTPRoute)
+	sample.StatusClass = statusClass(sample.HTTPStatus)
+	sample.StackSignature = firstStackFrame(sample.Stacktrace)
+}
+
+func normalizePatternTitle(title string) string {
+	s := NormalizeTitle(title)
+	s = regexp.MustCompile(`(?i)http\s+[1-5]\d{2}`).ReplaceAllString(s, "http <status>")
+	return s
+}
+
+func NormalizeRoute(route string) string {
+	route = strings.TrimSpace(strings.SplitN(route, "?", 2)[0])
+	if route == "" {
+		return ""
+	}
+	parts := strings.Split(route, "/")
+	for i, part := range parts {
+		if part == "" {
+			continue
+		}
+		if uuidRe.MatchString(part) || hexIDRe.MatchString(part) || numRe.MatchString(part) {
+			parts[i] = "{id}"
+		}
+	}
+	return strings.Join(parts, "/")
+}
+
+func statusClass(status int) string {
+	if status >= 500 && status < 600 {
+		return "5xx"
+	}
+	if status >= 400 && status < 500 {
+		return "4xx"
+	}
+	if status >= 300 && status < 400 {
+		return "3xx"
+	}
+	if status > 0 {
+		return "2xx"
+	}
+	return ""
+}
+
+func firstStackFrame(stack string) string {
+	if i := strings.IndexByte(stack, '\n'); i >= 0 {
+		stack = stack[:i]
+	}
+	return strings.TrimSpace(stack)
+}
+
+func similarity(a, b Issue) (float64, []string) {
+	score := 0.0
+	reasons := []string{}
+	if a.Service == b.Service {
+		score += .25
+	}
+	if a.Kind == b.Kind {
+		score += .15
+	}
+	aSample, bSample := latestSample(a), latestSample(b)
+	if NormalizeRoute(aSample.HTTPRoute) != "" && NormalizeRoute(aSample.HTTPRoute) == NormalizeRoute(bSample.HTTPRoute) {
+		score += .20
+		reasons = append(reasons, "same route template")
+	}
+	if statusClass(aSample.HTTPStatus) != "" && statusClass(aSample.HTTPStatus) == statusClass(bSample.HTTPStatus) {
+		score += .15
+		reasons = append(reasons, "same HTTP status class")
+	}
+	if firstStackFrame(aSample.Stacktrace) != "" && firstStackFrame(aSample.Stacktrace) == firstStackFrame(bSample.Stacktrace) {
+		score += .10
+		reasons = append(reasons, "same top stack frame")
+	}
+	if a.NormalizedTitle != "" && a.NormalizedTitle == b.NormalizedTitle {
+		score += .15
+		reasons = append(reasons, "same normalized message")
+	} else if tokenSimilarity(a.NormalizedTitle, b.NormalizedTitle) >= .5 {
+		score += .10
+		reasons = append(reasons, "similar normalized message")
+	}
+	if len(reasons) == 0 {
+		reasons = append(reasons, "same service and event kind")
+	}
+	return score, reasons
+}
+
+func latestSample(issue Issue) Sample {
+	if len(issue.Samples) == 0 {
+		return Sample{}
+	}
+	return issue.Samples[len(issue.Samples)-1]
+}
+
+func tokenSimilarity(a, b string) float64 {
+	aa, bb := strings.Fields(a), strings.Fields(b)
+	if len(aa) == 0 || len(bb) == 0 {
+		return 0
+	}
+	set := map[string]bool{}
+	for _, v := range aa {
+		set[v] = true
+	}
+	other := map[string]bool{}
+	for _, v := range bb {
+		other[v] = true
+	}
+	common := 0
+	for v := range other {
+		if set[v] {
+			common++
+		}
+	}
+	return float64(common) / float64(len(set)+len(other)-common)
+}
+
+// SetStatus updates the triage state of an issue. It returns the updated
+// issue, or false when the id is unknown or the status is invalid.
+func (t *Tracker) SetStatus(id, status string) (Issue, bool) {
+	switch status {
+	case StatusOpen, StatusResolved, StatusIgnored:
+	default:
+		return Issue{}, false
+	}
+
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	issue, ok := t.issues[id]
+	if !ok {
+		return Issue{}, false
+	}
+	issue.Status = status
+	if status == StatusOpen {
+		issue.ResolvedAt = 0
+	} else {
+		issue.ResolvedAt = time.Now().UnixNano()
+	}
+	return cloneIssue(issue), true
+}
+
 // evictOldestLocked removes the issue with the smallest LastSeen. Callers
 // must hold t.mu.
 func (t *Tracker) evictOldestLocked() {
@@ -107,22 +462,89 @@ func (t *Tracker) evictOldestLocked() {
 // List returns issues sorted by LastSeen descending (most recently seen
 // first). A non-positive limit returns all issues.
 func (t *Tracker) List(limit int) []Issue {
+	return t.ListFiltered(ListOptions{Limit: limit}).Issues
+}
+
+// ListFiltered returns a filtered, stable page of issues.
+func (t *Tracker) ListFiltered(opts ListOptions) ListPage {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
 	result := make([]Issue, 0, len(t.issues))
 	for _, issue := range t.issues {
+		if !matches(issue, opts) || beforeCursor(issue, opts) {
+			continue
+		}
 		result = append(result, cloneIssue(issue))
 	}
 
 	sort.Slice(result, func(i, j int) bool {
-		return result[i].LastSeen > result[j].LastSeen
+		if result[i].LastSeen != result[j].LastSeen {
+			return result[i].LastSeen > result[j].LastSeen
+		}
+		return result[i].ID > result[j].ID
 	})
 
-	if limit > 0 && len(result) > limit {
-		result = result[:limit]
+	limit := opts.Limit
+	if limit <= 0 {
+		limit = len(result)
 	}
-	return result
+	page := ListPage{Issues: result}
+	if len(page.Issues) > limit {
+		page.Issues = page.Issues[:limit]
+		page.HasMore = true
+	}
+	return page
+}
+
+func matches(issue *Issue, opts ListOptions) bool {
+	if opts.Service != "" && issue.Service != opts.Service {
+		return false
+	}
+	if opts.Kind != "" && issue.Kind != opts.Kind {
+		return false
+	}
+	if opts.Status != "" && issue.Status != opts.Status {
+		return false
+	}
+	if opts.Title != "" && !strings.Contains(strings.ToLower(issue.Title), strings.ToLower(opts.Title)) {
+		return false
+	}
+	if opts.Since != 0 && issue.LastSeen < opts.Since {
+		return false
+	}
+	if opts.Until != 0 && issue.LastSeen > opts.Until {
+		return false
+	}
+	if opts.SourceIP != "" && !issueHasSample(issue, func(sample Sample) bool { return sample.SourceIP == opts.SourceIP }) {
+		return false
+	}
+	if opts.Route != "" && !issueHasSample(issue, func(sample Sample) bool { return sample.HTTPRoute == opts.Route }) {
+		return false
+	}
+	return true
+}
+
+func issueHasSample(issue *Issue, predicate func(Sample) bool) bool {
+	for _, sample := range issue.Samples {
+		if predicate(sample) {
+			return true
+		}
+	}
+	return false
+}
+
+func beforeCursor(issue *Issue, opts ListOptions) bool {
+	if opts.AfterLastSeen == 0 {
+		return false
+	}
+	if issue.LastSeen < opts.AfterLastSeen {
+		return false
+	}
+	if issue.LastSeen > opts.AfterLastSeen {
+		return true
+	}
+	return issue.ID >= opts.AfterID
 }
 
 // Get returns a single issue by id.
@@ -137,15 +559,100 @@ func (t *Tracker) Get(id string) (Issue, bool) {
 	return cloneIssue(issue), true
 }
 
-// cloneIssue deep-copies the slice fields so returned issues don't share
+// cloneIssue deep-copies the slice/map fields so returned issues don't share
 // backing arrays with ones still being mutated by Record.
 func cloneIssue(i *Issue) Issue {
 	c := *i
 	c.AffectedServices = append([]string(nil), i.AffectedServices...)
 	c.Samples = append([]Sample(nil), i.Samples...)
+	if i.Trend != nil {
+		c.Trend = make(map[string]int, len(i.Trend))
+		for k, v := range i.Trend {
+			c.Trend[k] = v
+		}
+	}
 	return c
 }
 
+// HourBucket renders a unix-nanosecond timestamp as a UTC hour key used in
+// Issue.Trend. Keys sort lexicographically, so the UI can sort them as-is.
+func HourBucket(ts int64) string {
+	return time.Unix(0, ts).UTC().Format("2006-01-02T15:00")
+}
+
+// pruneTrendLocked drops trend buckets older than trendWindow. Callers must
+// hold the tracker lock.
+func pruneTrendLocked(trend map[string]int, now int64) {
+	if len(trend) <= 48 {
+		return
+	}
+	cutoff := HourBucket(time.Unix(0, now).Add(-trendWindow).UnixNano())
+	for k := range trend {
+		if k < cutoff {
+			delete(trend, k)
+		}
+	}
+}
+
+var (
+	uuidRe       = regexp.MustCompile(`(?i)\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b`)
+	hexIDRe      = regexp.MustCompile(`(?i)\b[0-9a-f]{8,}\b`)
+	httpStatusRe = regexp.MustCompile(`(?i)\bhttp\s+([1-5]\d{2})\b`)
+	numRe        = regexp.MustCompile(`\b\d+\b`)
+	spaceRe      = regexp.MustCompile(`\s+`)
+)
+
+// NormalizeTitle strips variable content (UUIDs, hex ids, numbers, whitespace,
+// casing) from an error title so that "User 123 not found" and "User 456 not
+// found" fingerprint into the same issue. The raw title is kept as the
+// display title of the first-seen occurrence. HTTP status codes (e.g. HTTP 404)
+// are preserved so different HTTP error types don't collapse together.
+func NormalizeTitle(title string) string {
+	s := uuidRe.ReplaceAllString(title, "<id>")
+	// Protect HTTP status codes from being replaced by <num>
+	s = httpStatusRe.ReplaceAllStringFunc(s, func(match string) string {
+		parts := strings.Fields(match)
+		if len(parts) != 2 {
+			return match
+		}
+		return "__HTTP_STATUS_" + encodeStatus(parts[1]) + "__"
+	})
+	s = numRe.ReplaceAllString(s, "<num>")
+	s = hexIDRe.ReplaceAllStringFunc(s, func(match string) string {
+		// Mixed hex runs (letters + digits, e.g. trace/span ids) become ids.
+		// Pure-digit runs were already handled by numRe above, and words
+		// spelled entirely from a-f (rare at this length) survive so distinct
+		// error messages don't collapse.
+		if strings.ContainsAny(match, "0123456789") {
+			return "<id>"
+		}
+		return match
+	})
+	s = regexp.MustCompile(`__HTTP_STATUS_([a-z]+)__`).ReplaceAllStringFunc(s, func(match string) string {
+		encoded := strings.TrimSuffix(strings.TrimPrefix(match, "__HTTP_STATUS_"), "__")
+		return "http " + decodeStatus(encoded)
+	})
+	s = strings.ReplaceAll(s, "__", "")
+	s = spaceRe.ReplaceAllString(s, " ")
+	return strings.ToLower(strings.TrimSpace(s))
+}
+
+func encodeStatus(code string) string {
+	var b strings.Builder
+	for _, digit := range code {
+		b.WriteByte('a' + byte(digit-'0'))
+	}
+	return b.String()
+}
+
+func decodeStatus(encoded string) string {
+	var b strings.Builder
+	for _, letter := range encoded {
+		b.WriteByte('0' + byte(letter-'a'))
+	}
+	return b.String()
+}
+
 func fingerprint(parts ...string) string {
 	h := fnv.New64a()
 	_, _ = h.Write([]byte(strings.Join(parts, "\x00")))

internal/issues/tracker_test.go

@@ -73,3 +73,157 @@ func TestTrackerListRespectsLimit(t *testing.T) {
 		t.Fatalf("expected limit to cap results at 2, got %d", len(list))
 	}
 }
+
+func TestTrackerFilteredCursorPagination(t *testing.T) {
+	tracker := NewTracker()
+	tracker.Record("span_error", "nginx", "GET /.env", Sample{Timestamp: 300, SourceIP: "203.0.113.10", HTTPRoute: "/.env"})
+	tracker.Record("span_error", "nginx", "GET /.git/HEAD", Sample{Timestamp: 200, SourceIP: "203.0.113.11", HTTPRoute: "/.git/HEAD"})
+	tracker.Record("log_error", "api", "database down", Sample{Timestamp: 100})
+
+	page := tracker.ListFiltered(ListOptions{Service: "nginx", Kind: "span_error", Route: "/.env", Limit: 1})
+	if len(page.Issues) != 1 || page.Issues[0].Title != "GET /.env" {
+		t.Fatalf("unexpected filtered page: %+v", page)
+	}
+	if page.HasMore {
+		t.Error("single matching issue should not have another page")
+	}
+
+	all := tracker.ListFiltered(ListOptions{Service: "nginx", Limit: 1})
+	if len(all.Issues) != 1 || !all.HasMore {
+		t.Fatalf("expected first paginated page with more results, got %+v", all)
+	}
+	next := tracker.ListFiltered(ListOptions{
+		Service:       "nginx",
+		AfterLastSeen: all.Issues[0].LastSeen,
+		AfterID:       all.Issues[0].ID,
+		Limit:         1,
+	})
+	if len(next.Issues) != 1 || next.Issues[0].Title != "GET /.git/HEAD" || next.HasMore {
+		t.Fatalf("unexpected cursor page: %+v", next)
+	}
+}
+
+func TestNormalizeTitleGroupsVariableMessages(t *testing.T) {
+	cases := [][3]string{
+		// same, same, different: [a, b, expectedEqual(a,b)]
+		{"User 123 not found", "User 456 not found", "user <num> not found"},
+		{"order 550e8400-e29b-41d4-a716-446655440000 failed", "order 550e8400-e29b-41d4-a716-446655440001 failed", "order <id> failed"},
+		{"request 0a1b2c3d4e5f6a7b timed out", "request ff11aa22bb33cc44 timed out", "request <id> timed out"},
+	}
+	for _, c := range cases {
+		if got := NormalizeTitle(c[0]); got != c[2] {
+			t.Errorf("NormalizeTitle(%q) = %q, want %q", c[0], got, c[2])
+		}
+		if NormalizeTitle(c[0]) != NormalizeTitle(c[1]) {
+			t.Errorf("expected %q and %q to normalize identically", c[0], c[1])
+		}
+	}
+	// Words spelled from hex letters without digits must survive.
+	if got := NormalizeTitle("bad failed deadbeef badge"); got != "bad failed deadbeef badge" {
+		t.Errorf("word-like hex incorrectly replaced: %q", got)
+	}
+}
+
+func TestTrackerNormalizedFingerprinting(t *testing.T) {
+	tracker := NewTracker()
+	tracker.Record("log_error", "api", "User 123 not found", Sample{Timestamp: 100, Message: "User 123 not found"})
+	tracker.Record("log_error", "api", "User 98765432 not found", Sample{Timestamp: 200, Message: "User 98765432 not found"})
+
+	list := tracker.List(0)
+	if len(list) != 1 {
+		t.Fatalf("expected variable messages to merge into 1 issue, got %d", len(list))
+	}
+	if list[0].Count != 2 {
+		t.Errorf("expected count 2, got %d", list[0].Count)
+	}
+	// The display title is the first-seen raw message.
+	if list[0].Title != "User 123 not found" {
+		t.Errorf("expected first-seen raw title kept, got %q", list[0].Title)
+	}
+}
+
+func TestTrackerTrendBuckets(t *testing.T) {
+	tracker := NewTracker()
+	// Two occurrences in the same hour.
+	tracker.Record("log_error", "api", "boom", Sample{Timestamp: 100})
+	tracker.Record("log_error", "api", "boom", Sample{Timestamp: 200})
+
+	issue, ok := tracker.Get(tracker.List(1)[0].ID)
+	if !ok {
+		t.Fatal("expected issue to exist")
+	}
+	bucket := HourBucket(100)
+	if issue.Trend[bucket] != 2 {
+		t.Errorf("expected trend[%s]=2, got %d", bucket, issue.Trend[bucket])
+	}
+}
+
+func TestTrackerStatusLifecycle(t *testing.T) {
+	tracker := NewTracker()
+	tracker.Record("log_error", "api", "boom", Sample{Timestamp: 100})
+	id := tracker.List(1)[0].ID
+
+	// Invalid status rejected.
+	if _, ok := tracker.SetStatus(id, "bogus"); ok {
+		t.Error("expected invalid status to be rejected")
+	}
+	// Unknown id rejected.
+	if _, ok := tracker.SetStatus("nope", StatusResolved); ok {
+		t.Error("expected unknown id to be rejected")
+	}
+	// Resolve.
+	resolved, ok := tracker.SetStatus(id, StatusResolved)
+	if !ok || resolved.Status != StatusResolved || resolved.ResolvedAt == 0 {
+		t.Fatalf("expected resolved issue, got %+v ok=%v", resolved, ok)
+	}
+	// A new occurrence reopens it (regression).
+	tracker.Record("log_error", "api", "boom", Sample{Timestamp: 200})
+	issue, _ := tracker.Get(id)
+	if issue.Status != StatusOpen || issue.ResolvedAt != 0 {
+		t.Errorf("expected regression to reopen issue, got status=%s resolvedAt=%d", issue.Status, issue.ResolvedAt)
+	}
+	// Ignored survives new occurrences.
+	if _, ok := tracker.SetStatus(id, StatusIgnored); !ok {
+		t.Fatal("expected ignore to succeed")
+	}
+	tracker.Record("log_error", "api", "boom", Sample{Timestamp: 300})
+	issue, _ = tracker.Get(id)
+	if issue.Status != StatusIgnored {
+		t.Errorf("expected ignored state to survive new occurrence, got %s", issue.Status)
+	}
+}
+
+func TestTrackerPatternsAndSimilarSuggestions(t *testing.T) {
+	tracker := NewTracker()
+	tracker.Record("span_error", "checkout", "HTTP 504: database timeout for order 123", Sample{
+		Timestamp: 100, HTTPRoute: "/orders/123", HTTPStatus: 504, Stacktrace: "db.go:42\nhandler.go:10",
+	})
+	tracker.Record("span_error", "checkout", "HTTP 502: database timeout for order 456", Sample{
+		Timestamp: 200, HTTPRoute: "/orders/456", HTTPStatus: 502, Stacktrace: "db.go:42\nhandler.go:10",
+	})
+	issuesList := tracker.List(0)
+	if len(issuesList) != 2 {
+		t.Fatalf("expected separate exact issues, got %d", len(issuesList))
+	}
+	if issuesList[0].PatternID == "" || issuesList[0].PatternID != issuesList[1].PatternID {
+		t.Fatalf("expected related HTTP variants to share a pattern: %+v", issuesList)
+	}
+	if issuesList[0].Samples[0].NormalizedRoute != "/orders/{id}" {
+		t.Fatalf("expected normalized route, got %q", issuesList[0].Samples[0].NormalizedRoute)
+	}
+	related, ok := tracker.Similar(issuesList[0].ID, 5)
+	if !ok || len(related) != 1 || related[0].Score < .45 {
+		t.Fatalf("expected deterministic related suggestion, got ok=%v related=%+v", ok, related)
+	}
+	if !tracker.SetSimilarityFeedback(issuesList[0].ID, related[0].Issue.ID, "same") {
+		t.Fatal("expected feedback to be accepted")
+	}
+	related, _ = tracker.Similar(issuesList[0].ID, 5)
+	if related[0].Decision != "same" {
+		t.Fatalf("expected feedback decision, got %q", related[0].Decision)
+	}
+	patterns := tracker.Patterns()
+	if len(patterns) != 1 || patterns[0].IssueCount != 2 || patterns[0].OccurrenceCount != 2 {
+		t.Fatalf("expected one two-issue pattern, got %+v", patterns)
+	}
+}

internal/kubernetes/types.go

@@ -0,0 +1,23 @@
+package kubernetes
+
+// Rollout is the normalized, UI-safe view of a Kubernetes Deployment rollout.
+type Rollout struct {
+	ID                string `json:"id"`
+	Namespace         string `json:"namespace"`
+	Workload          string `json:"workload"`
+	Service           string `json:"service,omitempty"`
+	Release           string `json:"release,omitempty"`
+	Revision          string `json:"revision,omitempty"`
+	Image             string `json:"image,omitempty"`
+	Phase             string `json:"phase"`
+	Reason            string `json:"reason,omitempty"`
+	Message           string `json:"message,omitempty"`
+	StartedAt         int64  `json:"startedAt"`
+	CompletedAt       int64  `json:"completedAt,omitempty"`
+	LastSeen          int64  `json:"lastSeen"`
+	Generation        int64  `json:"generation"`
+	DesiredReplicas   int32  `json:"desiredReplicas"`
+	UpdatedReplicas   int32  `json:"updatedReplicas"`
+	AvailableReplicas int32  `json:"availableReplicas"`
+	ReadyReplicas     int32  `json:"readyReplicas"`
+}

internal/kubernetes/watcher.go

@@ -0,0 +1,239 @@
+// Package kubernetes observes read-only workload state for rollout correlation.
+package kubernetes
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+const maxRollouts = 500
+
+type Watcher struct {
+	mu       sync.RWMutex
+	rollouts map[string]Rollout
+	enabled  bool
+	err      string
+}
+
+// NewFromEnv enables observation in-cluster, or with KUBECONFIG locally. It is
+// disabled by default outside Kubernetes so local development needs no kubeconfig.
+func NewFromEnv(ctx context.Context) (*Watcher, error) {
+	if strings.EqualFold(os.Getenv("OTEL_KUBERNETES_ENABLED"), "false") {
+		return nil, nil
+	}
+	config, err := kubeConfig()
+	if err != nil {
+		if os.Getenv("OTEL_KUBERNETES_ENABLED") == "true" {
+			return nil, err
+		}
+		return nil, nil
+	}
+	client, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("create kubernetes client: %w", err)
+	}
+
+	w := &Watcher{rollouts: make(map[string]Rollout), enabled: true}
+	namespace := os.Getenv("OTEL_KUBERNETES_NAMESPACE")
+	selector := os.Getenv("OTEL_KUBERNETES_LABEL_SELECTOR")
+	factory := informers.NewSharedInformerFactoryWithOptions(client, 30*time.Second,
+		informers.WithTweakListOptions(func(options *metav1.ListOptions) {
+			options.LabelSelector = selector
+		}),
+	)
+	if namespace != "" {
+		factory = informers.NewSharedInformerFactoryWithOptions(client, 30*time.Second,
+			informers.WithNamespace(namespace), informers.WithTweakListOptions(func(options *metav1.ListOptions) {
+				options.LabelSelector = selector
+			}))
+	}
+	informer := factory.Apps().V1().Deployments().Informer()
+	_, _ = informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc:    func(obj interface{}) { w.observe(obj) },
+		UpdateFunc: func(_, obj interface{}) { w.observe(obj) },
+	})
+	go func() {
+		factory.Start(ctx.Done())
+		if !cache.WaitForCacheSync(ctx.Done(), informer.HasSynced) {
+			w.setError("deployment informer did not sync")
+			return
+		}
+		<-ctx.Done()
+	}()
+	return w, nil
+}
+
+func kubeConfig() (*rest.Config, error) {
+	if config, err := rest.InClusterConfig(); err == nil {
+		return config, nil
+	}
+	kubeconfig := os.Getenv("KUBECONFIG")
+	if kubeconfig == "" {
+		home, err := os.UserHomeDir()
+		if err != nil {
+			return nil, err
+		}
+		kubeconfig = home + string(os.PathSeparator) + ".kube" + string(os.PathSeparator) + "config"
+	}
+	return clientcmd.BuildConfigFromFlags("", kubeconfig)
+}
+
+func (w *Watcher) observe(obj interface{}) {
+	deployment, ok := obj.(*appsv1.Deployment)
+	if !ok {
+		return
+	}
+	now := time.Now().UnixNano()
+	revision := deployment.Annotations["deployment.kubernetes.io/revision"]
+	if revision == "" {
+		revision = strconv.FormatInt(deployment.Generation, 10)
+	}
+	key := deployment.Namespace + "/" + deployment.Name + "/" + revision
+	phase, reason, message, completed := rolloutStatus(deployment)
+	started := now
+	for _, condition := range deployment.Status.Conditions {
+		if condition.Type == appsv1.DeploymentProgressing && !condition.LastUpdateTime.IsZero() {
+			started = condition.LastUpdateTime.UnixNano()
+		}
+		if condition.Type == appsv1.DeploymentAvailable && !condition.LastUpdateTime.IsZero() && completed {
+			completed = condition.LastUpdateTime.UnixNano() != 0
+		}
+	}
+	if old, exists := w.rollouts[key]; exists {
+		started = old.StartedAt
+		if old.CompletedAt != 0 {
+			completed = true
+		}
+	}
+	rollout := Rollout{
+		ID: rolloutID(key), Namespace: deployment.Namespace, Workload: deployment.Name,
+		Service:  metadataValue(deployment, "app.kubernetes.io/name", "app", "service.name"),
+		Release:  metadataValue(deployment, "otel.k6n.net/release", "app.kubernetes.io/version", "service.release"),
+		Revision: revision, Image: deploymentImage(deployment), Phase: phase, Reason: reason,
+		Message: message, StartedAt: started, LastSeen: now, Generation: deployment.Generation,
+		DesiredReplicas: desiredReplicas(deployment), UpdatedReplicas: deployment.Status.UpdatedReplicas,
+		AvailableReplicas: deployment.Status.AvailableReplicas, ReadyReplicas: deployment.Status.ReadyReplicas,
+	}
+	if completed {
+		rollout.CompletedAt = now
+	}
+	w.mu.Lock()
+	w.rollouts[key] = rollout
+	if len(w.rollouts) > maxRollouts {
+		w.evictOldestLocked()
+	}
+	w.mu.Unlock()
+}
+
+func (w *Watcher) evictOldestLocked() {
+	var oldest string
+	var ts int64
+	for key, item := range w.rollouts {
+		if oldest == "" || item.LastSeen < ts {
+			oldest, ts = key, item.LastSeen
+		}
+	}
+	if oldest != "" {
+		delete(w.rollouts, oldest)
+	}
+}
+func (w *Watcher) setError(message string) { w.mu.Lock(); defer w.mu.Unlock(); w.err = message }
+
+func (w *Watcher) List() []Rollout {
+	w.mu.RLock()
+	defer w.mu.RUnlock()
+	result := make([]Rollout, 0, len(w.rollouts))
+	for _, item := range w.rollouts {
+		result = append(result, item)
+	}
+	sort.Slice(result, func(i, j int) bool { return result[i].LastSeen > result[j].LastSeen })
+	return result
+}
+func (w *Watcher) Get(id string) (Rollout, bool) {
+	w.mu.RLock()
+	defer w.mu.RUnlock()
+	for _, item := range w.rollouts {
+		if item.ID == id {
+			return item, true
+		}
+	}
+	return Rollout{}, false
+}
+func (w *Watcher) Enabled() bool { return w != nil && w.enabled }
+func (w *Watcher) Error() string {
+	if w == nil {
+		return "kubernetes integration is not configured"
+	}
+	w.mu.RLock()
+	defer w.mu.RUnlock()
+	return w.err
+}
+
+func rolloutID(key string) string {
+	sum := sha256.Sum256([]byte(key))
+	return hex.EncodeToString(sum[:8])
+}
+func metadataValue(deployment *appsv1.Deployment, keys ...string) string {
+	for _, key := range keys {
+		if value := deployment.Labels[key]; value != "" {
+			return value
+		}
+		if value := deployment.Annotations[key]; value != "" {
+			return value
+		}
+		if value := deployment.Spec.Template.Labels[key]; value != "" {
+			return value
+		}
+		if value := deployment.Spec.Template.Annotations[key]; value != "" {
+			return value
+		}
+	}
+	return ""
+}
+func deploymentImage(deployment *appsv1.Deployment) string {
+	if len(deployment.Spec.Template.Spec.Containers) == 0 {
+		return ""
+	}
+	return deployment.Spec.Template.Spec.Containers[0].Image
+}
+func desiredReplicas(deployment *appsv1.Deployment) int32 {
+	if deployment.Spec.Replicas == nil {
+		return 1
+	}
+	return *deployment.Spec.Replicas
+}
+
+func rolloutStatus(deployment *appsv1.Deployment) (phase, reason, message string, completed bool) {
+	phase = "progressing"
+	for _, condition := range deployment.Status.Conditions {
+		switch condition.Type {
+		case appsv1.DeploymentProgressing:
+			reason, message = condition.Reason, condition.Message
+			if condition.Status == corev1.ConditionFalse {
+				return "failed", reason, message, false
+			}
+		case appsv1.DeploymentAvailable:
+			if condition.Status == corev1.ConditionTrue && deployment.Status.UpdatedReplicas == desiredReplicas(deployment) && deployment.Status.AvailableReplicas >= desiredReplicas(deployment) {
+				return "completed", reason, message, true
+			}
+		}
+	}
+	return phase, reason, message, false
+}

internal/prometheus/client.go

@@ -0,0 +1,103 @@
+// Package prometheus provides optional Prometheus range queries for rollout impact.
+package prometheus
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type Point struct {
+	Timestamp float64 `json:"timestamp"`
+	Value     string  `json:"value"`
+}
+
+type Client struct {
+	baseURL       string
+	queryTemplate string
+	httpClient    *http.Client
+}
+
+func NewFromEnv() *Client {
+	baseURL := strings.TrimRight(os.Getenv("OTEL_PROMETHEUS_URL"), "/")
+	if baseURL == "" {
+		return nil
+	}
+	query := os.Getenv("OTEL_PROMETHEUS_ERROR_QUERY")
+	if query == "" {
+		query = `sum(rate(http_server_requests_total{service_name=%q,http_response_status_code=~"5.."}[5m]))`
+	}
+	return &Client{baseURL: baseURL, queryTemplate: query, httpClient: &http.Client{Timeout: 10 * time.Second}}
+}
+
+func (c *Client) Configured() bool { return c != nil }
+
+func (c *Client) ErrorRate(ctx context.Context, service, release string, start, end time.Time) ([]Point, error) {
+	if c == nil {
+		return nil, fmt.Errorf("prometheus is not configured")
+	}
+	query := renderQuery(c.queryTemplate, service, release)
+	params := url.Values{
+		"query": {query},
+		"start": {strconv.FormatInt(start.Unix(), 10)},
+		"end":   {strconv.FormatInt(end.Unix(), 10)},
+		"step":  {"300"},
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/api/v1/query_range?"+params.Encode(), nil)
+	if err != nil {
+		return nil, err
+	}
+	response, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+	if response.StatusCode/100 != 2 {
+		return nil, fmt.Errorf("prometheus returned HTTP %s", response.Status)
+	}
+	var payload struct {
+		Status string `json:"status"`
+		Error  string `json:"error"`
+		Data   struct {
+			Result []struct {
+				Values [][]json.RawMessage `json:"values"`
+			} `json:"result"`
+		} `json:"data"`
+	}
+	if err := json.NewDecoder(response.Body).Decode(&payload); err != nil {
+		return nil, err
+	}
+	if payload.Status != "success" {
+		return nil, fmt.Errorf("prometheus query failed: %s", payload.Error)
+	}
+	points := make([]Point, 0)
+	for _, series := range payload.Data.Result {
+		for _, value := range series.Values {
+			if len(value) != 2 {
+				continue
+			}
+			var timestamp float64
+			var sample string
+			if json.Unmarshal(value[0], &timestamp) == nil && json.Unmarshal(value[1], &sample) == nil {
+				points = append(points, Point{Timestamp: timestamp, Value: sample})
+			}
+		}
+	}
+	return points, nil
+}
+
+func renderQuery(template, service, release string) string {
+	values := []string{strconv.Quote(service), strconv.Quote(release)}
+	for _, value := range values {
+		if strings.Contains(template, "%q") {
+			template = strings.Replace(template, "%q", value, 1)
+		}
+	}
+	return template
+}

internal/sourcemaps/store.go

@@ -0,0 +1,106 @@
+package sourcemaps
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+
+	sm "github.com/go-sourcemap/sourcemap"
+)
+
+var frameRE = regexp.MustCompile(`(.*?)(https?://[^ )]+|[^ )]+):(\d+):(\d+)(\)?\s*)$`)
+
+type Resolver struct {
+	dir  string
+	mu   sync.RWMutex
+	maps map[string]*sm.Consumer
+}
+
+func New(dir string) (*Resolver, error) {
+	if dir == "" {
+		dir = "./sourcemaps"
+	}
+	if err := os.MkdirAll(dir, 0750); err != nil {
+		return nil, err
+	}
+	return &Resolver{dir: dir, maps: make(map[string]*sm.Consumer)}, nil
+}
+
+func key(release, source string) string { return release + "\x00" + normalize(source) }
+
+func normalize(source string) string {
+	source = strings.Split(source, "?")[0]
+	if i := strings.Index(source, "/assets/"); i >= 0 {
+		return source[i:]
+	}
+	return strings.TrimPrefix(source, "./")
+}
+
+func (r *Resolver) Upload(release, source string, body io.Reader) error {
+	if strings.TrimSpace(release) == "" || strings.TrimSpace(source) == "" {
+		return fmt.Errorf("release and source are required")
+	}
+	b, err := io.ReadAll(io.LimitReader(body, 50<<20))
+	if err != nil {
+		return err
+	}
+	consumer, err := sm.Parse(source+".map", b)
+	if err != nil {
+		return fmt.Errorf("invalid source map: %w", err)
+	}
+	name := filepath.Join(r.dir, safe(release)+"-"+safe(normalize(source))+".map")
+	if err := os.WriteFile(name, b, 0600); err != nil {
+		return err
+	}
+	r.mu.Lock()
+	r.maps[key(release, source)] = consumer
+	r.mu.Unlock()
+	return nil
+}
+
+func safe(s string) string {
+	s = strings.ReplaceAll(s, "/", "_")
+	s = strings.ReplaceAll(s, "\\", "_")
+	return strings.Map(func(r rune) rune {
+		if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || strings.ContainsRune("._-", r) {
+			return r
+		}
+		return '_'
+	}, s)
+}
+
+func (r *Resolver) ResolveStack(release, stack string) string {
+	if release == "" || stack == "" {
+		return stack
+	}
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	lines := strings.Split(stack, "\n")
+	for i, line := range lines {
+		m := frameRE.FindStringSubmatch(line)
+		if m == nil {
+			continue
+		}
+		genLine, _ := strconv.Atoi(m[3])
+		genCol, _ := strconv.Atoi(m[4])
+		consumer := r.maps[key(release, m[2])]
+		if consumer == nil {
+			continue
+		}
+		source, name, srcLine, srcCol, ok := consumer.Source(genLine, genCol)
+		if !ok {
+			continue
+		}
+		label := source
+		if name != "" {
+			label += " (" + name + ")"
+		}
+		lines[i] = fmt.Sprintf("%s%s:%d:%d%s  [mapped from %s:%d:%d]", m[1], label, srcLine, srcCol, m[5], m[2], genLine, genCol)
+	}
+	return strings.Join(lines, "\n")
+}

internal/storage/clickhouse.go

@@ -0,0 +1,528 @@
+package storage
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+
+	"git.tornberg.me/go-otel/internal/websocket"
+)
+
+// ClickHouseStore implements the Store interface backed by ClickHouse, so
+// history search stays fast and unbounded (the FileStore rescans the whole
+// log file on every query) and aggregate time-series - error trends, release
+// comparisons - can be computed server-side instead of in the browser.
+type ClickHouseStore struct {
+	conn driver.Conn
+	db   string
+}
+
+// ClickHouseConfig controls how the store connects.
+type ClickHouseConfig struct {
+	Addr            string        // host:port, native protocol (default port 9000)
+	Database        string        // defaults to "otel"
+	Username        string        // defaults to "default"
+	Password        string
+	MaxOpenConns    int           // defaults to 50
+	MaxIdleConns    int           // defaults to 25
+	DialTimeout     time.Duration // defaults to 30s (controls connection acquire timeout in driver pool)
+	ConnMaxLifetime time.Duration // defaults to 10m
+}
+
+func parseEnvInt(keys []string, def int) int {
+	for _, k := range keys {
+		if v := os.Getenv(k); v != "" {
+			if n, err := strconv.Atoi(v); err == nil && n > 0 {
+				return n
+			}
+		}
+	}
+	return def
+}
+
+func parseEnvDuration(keys []string, def time.Duration) time.Duration {
+	for _, k := range keys {
+		if v := os.Getenv(k); v != "" {
+			if d, err := time.ParseDuration(v); err == nil && d > 0 {
+				return d
+			}
+		}
+	}
+	return def
+}
+
+// NewClickHouseStore connects, creates the schema if needed, and returns a
+// ready store. Non-table errors during schema init are logged, not fatal:
+// raw storage works even if the rollup view can't be created.
+func NewClickHouseStore(ctx context.Context, cfg ClickHouseConfig) (*ClickHouseStore, error) {
+	if cfg.Database == "" {
+		cfg.Database = "otel"
+	}
+	if cfg.Username == "" {
+		cfg.Username = "default"
+	}
+
+	maxOpen := cfg.MaxOpenConns
+	if maxOpen <= 0 {
+		maxOpen = parseEnvInt([]string{"OTEL_CLICKHOUSE_MAX_OPEN_CONNS", "CLICKHOUSE_MAX_OPEN_CONNS"}, 50)
+	}
+	maxIdle := cfg.MaxIdleConns
+	if maxIdle <= 0 {
+		maxIdle = parseEnvInt([]string{"OTEL_CLICKHOUSE_MAX_IDLE_CONNS", "CLICKHOUSE_MAX_IDLE_CONNS"}, 25)
+	}
+	dialTimeout := cfg.DialTimeout
+	if dialTimeout <= 0 {
+		dialTimeout = parseEnvDuration([]string{"OTEL_CLICKHOUSE_DIAL_TIMEOUT", "CLICKHOUSE_DIAL_TIMEOUT"}, 30*time.Second)
+	}
+	connMaxLifetime := cfg.ConnMaxLifetime
+	if connMaxLifetime <= 0 {
+		connMaxLifetime = parseEnvDuration([]string{"OTEL_CLICKHOUSE_CONN_MAX_LIFETIME", "CLICKHOUSE_CONN_MAX_LIFETIME"}, 10*time.Minute)
+	}
+
+	// 1. Connect to default database with short timeout to ensure the target database exists.
+	bootConn, err := clickhouse.Open(&clickhouse.Options{
+		Addr: []string{cfg.Addr},
+		Auth: clickhouse.Auth{
+			Database: "default",
+			Username: cfg.Username,
+			Password: cfg.Password,
+		},
+		DialTimeout: 5 * time.Second,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("clickhouse dial %s: %w", cfg.Addr, err)
+	}
+	if err := bootConn.Ping(ctx); err != nil {
+		_ = bootConn.Close()
+		return nil, fmt.Errorf("clickhouse ping %s: %w", cfg.Addr, err)
+	}
+
+	if err := bootConn.Exec(ctx,
+		fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", cfg.Database)); err != nil {
+		_ = bootConn.Close()
+		return nil, fmt.Errorf("clickhouse create database %s: %w", cfg.Database, err)
+	}
+	// Close bootstrap connection so it does not leak idle connections.
+	_ = bootConn.Close()
+
+	// 2. Open production connection pool targeting the actual database with full pool settings.
+	conn, err := clickhouse.Open(&clickhouse.Options{
+		Addr: []string{cfg.Addr},
+		Auth: clickhouse.Auth{
+			Database: cfg.Database,
+			Username: cfg.Username,
+			Password: cfg.Password,
+		},
+		MaxOpenConns:     maxOpen,
+		MaxIdleConns:     maxIdle,
+		DialTimeout:      dialTimeout,
+		ConnMaxLifetime:  connMaxLifetime,
+		ConnOpenStrategy: clickhouse.ConnOpenRoundRobin,
+		ClientInfo: clickhouse.ClientInfo{
+			Products: []struct {
+				Name    string
+				Version string
+			}{
+				{Name: "go-otel", Version: "1.0"},
+			},
+		},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("clickhouse open %s/%s: %w", cfg.Addr, cfg.Database, err)
+	}
+	if err := conn.Ping(ctx); err != nil {
+		_ = conn.Close()
+		return nil, fmt.Errorf("clickhouse ping %s/%s: %w", cfg.Addr, cfg.Database, err)
+	}
+
+	s := &ClickHouseStore{conn: conn, db: cfg.Database}
+	if err := s.initSchema(ctx); err != nil {
+		_ = conn.Close()
+		return nil, err
+	}
+	return s, nil
+}
+
+// initSchema creates the raw entries table and the materialized hourly
+// rollup used by the stats endpoints. Idempotent.
+func (s *ClickHouseStore) initSchema(ctx context.Context) error {
+	if err := s.conn.Exec(ctx, fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.entries (
+		timestamp DateTime64(9, 'UTC') NOT NULL,
+		type      LowCardinality(String) NOT NULL,
+		service   LowCardinality(String) NOT NULL DEFAULT 'unknown',
+		is_error  UInt8 NOT NULL DEFAULT 0,
+		release   LowCardinality(String) NOT NULL DEFAULT '',
+		trace_id  String NOT NULL DEFAULT '',
+		data      String NOT NULL
+	) ENGINE = MergeTree
+	PARTITION BY toYYYYMM(timestamp)
+	ORDER BY (type, timestamp)
+	TTL toDateTime(timestamp) + INTERVAL 90 DAY`, s.db)); err != nil {
+		return fmt.Errorf("clickhouse create entries: %w", err)
+	}
+
+	// Hourly per-service/release rollup maintained on insert. is_error is a
+	// dimension (0/1), so errors = sum(count) WHERE is_error=1 and
+	// total = sum(count) unfiltered.
+	if err := s.conn.Exec(ctx, fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.hourly (
+		hour     DateTime('UTC') NOT NULL,
+		type     LowCardinality(String) NOT NULL,
+		service  LowCardinality(String) NOT NULL,
+		release  LowCardinality(String) NOT NULL DEFAULT '',
+		is_error UInt8 NOT NULL DEFAULT 0,
+		count    UInt64
+	) ENGINE = SummingMergeTree
+	ORDER BY (hour, type, service, release, is_error)`, s.db)); err != nil {
+		return fmt.Errorf("clickhouse create hourly: %w", err)
+	}
+
+	mv := fmt.Sprintf(`CREATE MATERIALIZED VIEW IF NOT EXISTS %s.hourly_mv TO %s.hourly AS
+		SELECT
+			toStartOfHour(timestamp) AS hour,
+			type,
+			service,
+			release,
+			is_error,
+			count() AS count
+		FROM %s.entries
+		GROUP BY hour, type, service, release, is_error`, s.db, s.db, s.db)
+	if err := s.conn.Exec(ctx, mv); err != nil {
+		// Raw storage works without the rollup; stats endpoints will just be
+		// empty until this succeeds.
+		fmt.Printf("clickhouse: failed to create rollup view (stats endpoints stay empty): %v\n", err)
+	}
+	return nil
+}
+
+// chPayload extracts rollup columns from the marshaled OTLP JSON. Field
+// structs are deliberately minimal so unmarshal cost stays low.
+type chPayload struct {
+	ResourceSpans []chResourceSpan `json:"resourceSpans"`
+	ResourceLogs  []chResourceLog  `json:"resourceLogs"`
+}
+
+type chResourceSpan struct {
+	Resource   chResource `json:"resource"`
+	ScopeSpans []struct {
+		Spans []chSpan `json:"spans"`
+	} `json:"scopeSpans"`
+}
+
+type chResourceLog struct {
+	Resource  chResource `json:"resource"`
+	ScopeLogs []struct {
+		LogRecords []chLogRecord `json:"logRecords"`
+	} `json:"scopeLogs"`
+}
+
+type chResource struct {
+	Attributes []chAttr `json:"attributes"`
+}
+
+type chAttr struct {
+	Key   string `json:"key"`
+	Value struct {
+		StringValue string `json:"stringValue"`
+	} `json:"value"`
+}
+
+type chSpan struct {
+	TraceID string `json:"traceId"`
+	Status  struct {
+		Code int `json:"code"`
+	} `json:"status"`
+	Attributes []chAttr `json:"attributes"`
+}
+
+type chLogRecord struct {
+	SeverityNumber int  `json:"severityNumber"`
+	SeverityText   string `json:"severityText"`
+	TraceID        string `json:"traceId"`
+	Attributes     []chAttr `json:"attributes"`
+}
+
+// extractHead pulls (service, is_error, release, trace_id) out of a raw OTLP
+// JSON payload for the rollup columns. Best-effort: on parse failure the
+// entry is still stored with defaults.
+func extractHead(data []byte) (service string, isErr bool, release, traceID string) {
+	service, release = "unknown", ""
+	var p chPayload
+	if err := json.Unmarshal(data, &p); err != nil {
+		return service, false, release, traceID
+	}
+
+	attrString := func(attrs []chAttr, key string) string {
+		for _, a := range attrs {
+			if a.Key == key {
+				return a.Value.StringValue
+			}
+		}
+		return ""
+	}
+
+	// Traces: error if any span has status code 2 (StatusCodeError).
+	for _, rs := range p.ResourceSpans {
+		if s := attrString(rs.Resource.Attributes, "service.name"); s != "" && service == "unknown" {
+			service = s
+		}
+		if r := attrString(rs.Resource.Attributes, "service.release"); r != "" && release == "" {
+			release = r
+		}
+		for _, ss := range rs.ScopeSpans {
+			for _, span := range ss.Spans {
+				if span.TraceID != "" && traceID == "" {
+					traceID = span.TraceID
+				}
+				if span.Status.Code == 2 {
+					isErr = true
+				}
+				if r := attrString(span.Attributes, "service.release"); r != "" && release == "" {
+					release = r
+				}
+			}
+		}
+	}
+
+	// Logs: error if severity number >= 17 (Error) or text is error-ish.
+	for _, rl := range p.ResourceLogs {
+		if s := attrString(rl.Resource.Attributes, "service.name"); s != "" && service == "unknown" {
+			service = s
+		}
+		if r := attrString(rl.Resource.Attributes, "service.release"); r != "" && release == "" {
+			release = r
+		}
+		for _, sl := range rl.ScopeLogs {
+			for _, rec := range sl.LogRecords {
+				if rec.TraceID != "" && traceID == "" {
+					traceID = rec.TraceID
+				}
+				if r := attrString(rec.Attributes, "service.release"); r != "" && release == "" {
+					release = r
+				}
+				if rec.SeverityNumber >= 17 {
+					isErr = true
+				}
+				switch strings.ToUpper(strings.TrimSpace(rec.SeverityText)) {
+				case "ERROR", "FATAL", "CRITICAL":
+					isErr = true
+				}
+			}
+		}
+	}
+	return service, isErr, release, traceID
+}
+
+// Append stores one history entry with its extracted rollup columns.
+// AsyncInsert lets ClickHouse batch the writes server-side. The severity
+// column is intentionally omitted: one payload can hold many records with
+// differing severities, so error-ness is a per-entry flag instead.
+func (s *ClickHouseStore) Append(ctx context.Context, entry websocket.HistoryEntry) error {
+	service, isErr, release, traceID := extractHead(entry.Data)
+
+	ts := time.Unix(0, entry.Timestamp).UTC()
+	err := s.conn.AsyncInsert(ctx, fmt.Sprintf(
+		`INSERT INTO %s.entries
+		 (timestamp, type, service, is_error, release, trace_id, data)
+		 VALUES (?, ?, ?, ?, ?, ?, ?)`, s.db),
+		false,
+		ts, entry.Type, service, boolToUint8(isErr), release, traceID, string(entry.Data),
+	)
+	if err != nil {
+		return fmt.Errorf("clickhouse append: %w", err)
+	}
+	return nil
+}
+
+func boolToUint8(b bool) uint8 {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+// GetHistory returns the most recent limit entries, newest first.
+func (s *ClickHouseStore) GetHistory(ctx context.Context, limit int) ([]websocket.HistoryEntry, error) {
+	return s.Search(ctx, SearchOptions{}, limit)
+}
+
+// Search implements the same semantics as FileStore.Search: case-insensitive
+// substring query over the raw payload, type/time filters, newest first,
+// capped at limit.
+func (s *ClickHouseStore) Search(ctx context.Context, opts SearchOptions, limit int) ([]websocket.HistoryEntry, error) {
+	if limit <= 0 {
+		limit = 50
+	}
+
+	where := []string{"1=1"}
+	args := []interface{}{}
+	if opts.Query != "" {
+		where = append(where, "positionCaseInsensitive(data, ?) > 0")
+		args = append(args, opts.Query)
+	}
+	if opts.Type != "" {
+		where = append(where, "type = ?")
+		args = append(args, opts.Type)
+	}
+	if opts.Since != 0 {
+		where = append(where, "timestamp >= ?")
+		args = append(args, time.Unix(0, opts.Since).UTC())
+	}
+	if opts.Until != 0 {
+		where = append(where, "timestamp <= ?")
+		args = append(args, time.Unix(0, opts.Until).UTC())
+	}
+
+	query := fmt.Sprintf(
+		`SELECT timestamp, type, data
+		 FROM %s.entries
+		 WHERE %s
+		 ORDER BY timestamp DESC
+		 LIMIT ?`, s.db, strings.Join(where, " AND "))
+	args = append(args, limit)
+
+	rows, err := s.conn.Query(ctx, query, args...)
+	if err != nil {
+		return nil, fmt.Errorf("clickhouse search: %w", err)
+	}
+	defer rows.Close()
+
+	var results []websocket.HistoryEntry
+	for rows.Next() {
+		var (
+			ts  time.Time
+			typ string
+			raw string
+		)
+		if err := rows.Scan(&ts, &typ, &raw); err != nil {
+			return nil, err
+		}
+		results = append(results, websocket.HistoryEntry{
+			Type:      typ,
+			Timestamp: ts.UnixNano(),
+			Data:      json.RawMessage(raw),
+		})
+	}
+	return results, rows.Err()
+}
+
+// StatsPoint is one bucket of a stats time-series.
+type StatsPoint struct {
+	Bucket string `json:"bucket"`
+	Total  uint64 `json:"total"`
+	Errors uint64 `json:"errors"`
+}
+
+// ErrorStats returns hourly (or daily) total/error entry counts over the
+// lookback window, for the trends graphs. Empty buckets are absent; the UI
+// fills gaps for display.
+func (s *ClickHouseStore) ErrorStats(ctx context.Context, service string, hours int, daily bool) ([]StatsPoint, error) {
+	bucket := "toStartOfHour(hour)"
+	if daily {
+		bucket = "toStartOfDay(hour)"
+	}
+	where := []string{"hour >= ?"}
+	args := []interface{}{time.Now().UTC().Add(-time.Duration(hours) * time.Hour)}
+	if service != "" {
+		where = append(where, "service = ?")
+		args = append(args, service)
+	}
+
+	query := fmt.Sprintf(
+		`SELECT
+			toString(%s) AS bucket,
+			sum(count) AS total,
+			sumIf(count, is_error = 1) AS errors
+		 FROM %s.hourly
+		 WHERE %s
+		 GROUP BY bucket
+		 ORDER BY bucket`, bucket, s.db, strings.Join(where, " AND "))
+
+	rows, err := s.conn.Query(ctx, query, args...)
+	if err != nil {
+		return nil, fmt.Errorf("clickhouse error stats: %w", err)
+	}
+	defer rows.Close()
+
+	var out []StatsPoint
+	for rows.Next() {
+		var p StatsPoint
+		if err := rows.Scan(&p.Bucket, &p.Total, &p.Errors); err != nil {
+			return nil, err
+		}
+		out = append(out, p)
+	}
+	return out, rows.Err()
+}
+
+// ReleaseSeries returns one error-count series per release:
+// {release -> [{bucket, total, errors}]}, so the UI can overlay deploys.
+func (s *ClickHouseStore) ReleaseSeries(ctx context.Context, service string, hours int) (map[string][]StatsPoint, error) {
+	where := []string{"hour >= ?", "release != ''"}
+	args := []interface{}{time.Now().UTC().Add(-time.Duration(hours) * time.Hour)}
+	if service != "" {
+		where = append(where, "service = ?")
+		args = append(args, service)
+	}
+
+	query := fmt.Sprintf(
+		`SELECT
+			release,
+			toString(toStartOfHour(hour)) AS bucket,
+			sum(count) AS total,
+			sumIf(count, is_error = 1) AS errors
+		 FROM %s.hourly
+		 WHERE %s
+		 GROUP BY release, bucket
+		 ORDER BY bucket`, s.db, strings.Join(where, " AND "))
+
+	rows, err := s.conn.Query(ctx, query, args...)
+	if err != nil {
+		return nil, fmt.Errorf("clickhouse release series: %w", err)
+	}
+	defer rows.Close()
+
+	out := map[string][]StatsPoint{}
+	for rows.Next() {
+		var (
+			release string
+			p       StatsPoint
+		)
+		if err := rows.Scan(&release, &p.Bucket, &p.Total, &p.Errors); err != nil {
+			return nil, err
+		}
+		out[release] = append(out[release], p)
+	}
+	return out, rows.Err()
+}
+
+// Services returns the distinct services seen in the rollup, for filter UIs.
+func (s *ClickHouseStore) Services(ctx context.Context) ([]string, error) {
+	rows, err := s.conn.Query(ctx, fmt.Sprintf(
+		`SELECT DISTINCT service FROM %s.hourly ORDER BY service`, s.db))
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var out []string
+	for rows.Next() {
+		var svc string
+		if err := rows.Scan(&svc); err != nil {
+			return nil, err
+		}
+		out = append(out, svc)
+	}
+	return out, rows.Err()
+}
+
+// Close closes the underlying connection.
+func (s *ClickHouseStore) Close() error {
+	return s.conn.Close()
+}

internal/storage/clickhouse_test.go

@@ -0,0 +1,121 @@
+package storage
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"testing"
+	"time"
+
+	"git.tornberg.me/go-otel/internal/websocket"
+)
+
+// TestClickHouseStoreIntegration is skipped unless OTEL_CLICKHOUSE_TEST_ADDR
+// points at a disposable ClickHouse (native port 9000). It exercises the real
+// SQL: schema creation, Append, rollup-triggered stats and Search filters.
+//
+// Placeholder round-trips through AsyncInsert use wait_for_async_insert so
+// reads below see the writes without sleeps.
+func TestClickHouseStoreIntegration(t *testing.T) {
+	addr := os.Getenv("OTEL_CLICKHOUSE_TEST_ADDR")
+	if addr == "" {
+		t.Skip("set OTEL_CLICKHOUSE_TEST_ADDR=host:9000 (with a disposable DB) to run")
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// Unique database per run so repeated runs don't collide.
+	db := "otel_test_" + time.Now().UTC().Format("150405")
+	cfg := ClickHouseConfig{Addr: addr, Database: db, Username: os.Getenv("OTEL_CLICKHOUSE_TEST_USER"), Password: os.Getenv("OTEL_CLICKHOUSE_TEST_PASSWORD")}
+	s, err := NewClickHouseStore(ctx, cfg)
+	if err != nil {
+		t.Skipf("ClickHouse not reachable at %s: %v", addr, err)
+	}
+	defer func() {
+		_ = s.conn.Exec(ctx, "DROP DATABASE IF EXISTS "+db)
+		_ = s.Close()
+	}()
+
+	now := time.Now().UTC()
+	mk := func(minsAgo int, typ, service string, isErr bool, release string) websocket.HistoryEntry {
+		raw, _ := json.Marshal(map[string]interface{}{"n": minsAgo})
+		return websocket.HistoryEntry{
+			Type:      typ,
+			Timestamp: now.Add(-time.Duration(minsAgo) * time.Minute).UnixNano(),
+			Data:      raw,
+		}
+	}
+	entries := []websocket.HistoryEntry{
+		mk(200, "logs", "api", true, "v1.3"),
+		mk(190, "logs", "api", false, "v1.3"),
+		mk(100, "logs", "api", true, "v1.4"),
+		mk(90, "logs", "api", false, "v1.4"),
+		mk(30, "logs", "api", false, "v1.4"),
+	}
+	for i, e := range entries {
+		if err := s.Append(ctx, e); err != nil {
+			t.Fatalf("Append[%d]: %v", i, err)
+		}
+	}
+	// Flush async insert buffer before reading.
+	if err := s.conn.AsyncInsert(ctx, "INSERT INTO FUNCTION null('x UInt8') VALUES (0)", true); err != nil {
+		t.Fatalf("flush: %v", err)
+	}
+
+	// Search: newest first, limit respected.
+	res, err := s.Search(ctx, SearchOptions{Type: "logs"}, 3)
+	if err != nil {
+		t.Fatalf("Search: %v", err)
+	}
+	if len(res) != 3 {
+		t.Fatalf("expected 3 search results, got %d", len(res))
+	}
+	if res[0].Timestamp < res[2].Timestamp {
+		t.Errorf("expected newest first")
+	}
+
+	// Search: substring filter.
+	res, err = s.Search(ctx, SearchOptions{Query: "\"n\":200"}, 10)
+	if err != nil {
+		t.Fatalf("Search query: %v", err)
+	}
+	if len(res) != 1 {
+		t.Errorf("expected substring filter to match 1 entry, got %d", len(res))
+	}
+
+	// ErrorStats via the rollup MV: two hours with data, 2 errors total.
+	points, err := s.ErrorStats(ctx, "api", 24, false)
+	if err != nil {
+		t.Fatalf("ErrorStats: %v", err)
+	}
+	errSum, totSum := uint64(0), uint64(0)
+	for _, p := range points {
+		errSum += p.Errors
+		totSum += p.Total
+	}
+	if errSum != 2 || totSum != 5 {
+		t.Errorf("expected rollup totals errors=2 total=5, got errors=%d total=%d", errSum, totSum)
+	}
+
+	// ReleaseSeries: v1.3 has 2 entries (1 error), v1.4 has 3 (1 error).
+	series, err := s.ReleaseSeries(ctx, "api", 24)
+	if err != nil {
+		t.Fatalf("ReleaseSeries: %v", err)
+	}
+	var v13, v14 uint64
+	for _, p := range series["v1.3"] {
+		v13 += p.Total
+	}
+	for _, p := range series["v1.4"] {
+		v14 += p.Total
+	}
+	if v13 != 2 || v14 != 3 {
+		t.Errorf("expected per-release totals v1.3=2 v1.4=3, got %d/%d", v13, v14)
+	}
+
+	// Services.
+	svcs, err := s.Services(ctx)
+	if err != nil || len(svcs) == 0 || svcs[0] != "api" {
+		t.Errorf("expected services [api], got %v (err=%v)", svcs, err)
+	}
+}

internal/storage/store.go

@@ -26,3 +26,15 @@ type Store interface {
 	GetHistory(ctx context.Context, limit int) ([]websocket.HistoryEntry, error)
 	Search(ctx context.Context, opts SearchOptions, limit int) ([]websocket.HistoryEntry, error)
 }
+
+// StatsQuerier is implemented by stores that can answer aggregate
+// time-series questions (currently ClickHouseStore). The stats HTTP
+// endpoints are only enabled when the configured store implements it.
+type StatsQuerier interface {
+	// ErrorStats returns hourly/daily total and error entry counts.
+	ErrorStats(ctx context.Context, service string, hours int, daily bool) ([]StatsPoint, error)
+	// ReleaseSeries returns per-release error-count time series.
+	ReleaseSeries(ctx context.Context, service string, hours int) (map[string][]StatsPoint, error)
+	// Services lists distinct services seen in the rollup.
+	Services(ctx context.Context) ([]string, error)
+}

internal/websocket/server.go

@@ -38,6 +38,28 @@ type Entry struct {
 	Data json.RawMessage `json:"data"`
 }
 
+// issuesNotification is broadcast when new error telemetry has been ingested,
+// so UI clients can refresh their Issues view immediately. It deliberately
+// carries no payload - clients re-fetch GET /api/issues.
+type issuesNotification struct {
+	Type string `json:"type"` // always "issues"
+}
+
+// NotifyIssues broadcasts an issues-refresh hint to all connected clients.
+func (s *Server) NotifyIssues() {
+	msg, err := json.Marshal(issuesNotification{Type: "issues"})
+	if err != nil {
+		return
+	}
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	for client := range s.clients {
+		if err := client.WriteMessage(websocket.TextMessage, msg); err != nil {
+			log.Printf("write error: %v", err)
+		}
+	}
+}
+
 // Broadcaster interface for sending telemetry updates to all connected clients.
 type Broadcaster interface {
 	Broadcast(data *Entry)