go-otel/internal/issues/tracker.go

// Package issues derives deduplicated error "issues" from ingested telemetry,
// so the web UI can show a triaged list instead of raw traces and logs.
package issues

import (
	"fmt"
	"hash/fnv"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"
)

// maxIssues bounds memory use: once exceeded, the least-recently-seen issue
// is evicted to make room for a newly seen distinct one.
const maxIssues = 2000

// maxSamples bounds how many sample occurrences are kept per issue.
const maxSamples = 5

// trendWindow bounds how far back the per-hour trend buckets are kept.
const trendWindow = 48 * time.Hour

// Issue lifecycle states. New occurrences automatically reopen resolved
// issues (a regression), but stay merged into ignored ones until the user
// explicitly sets the state back to open.
const (
	StatusOpen     = "open"
	StatusResolved = "resolved"
	StatusIgnored  = "ignored"
)

// Sample is one occurrence of an issue.
type Sample struct {
	Timestamp int64  `json:"timestamp"`
	TraceID   string `json:"traceId,omitempty"`
	SpanID    string `json:"spanId,omitempty"`
	Message   string `json:"message"`
	// URL is the page URL an occurrence happened on, populated only for
	// browser-originated (otel-rum) errors.
	URL string `json:"url,omitempty"`
	// Stacktrace is the exception.stacktrace attribute of the occurrence,
	// when the source telemetry carried one.
	Stacktrace         string `json:"stacktrace,omitempty"`
	ResolvedStacktrace string `json:"resolvedStacktrace,omitempty"`
	// HTTP request context, extracted from the span/log attributes when the
	// error happened inside an HTTP handler (semconv v1.x old and new names
	// are both handled by the extractor).
	HTTPMethod string `json:"httpMethod,omitempty"`
	HTTPRoute  string `json:"httpRoute,omitempty"`
	HTTPStatus int    `json:"httpStatus,omitempty"`
	// Client/release context.
	UserAgent   string `json:"userAgent,omitempty"`
	SourceIP    string `json:"sourceIp,omitempty"`
	Release     string `json:"release,omitempty"`
	Environment string `json:"environment,omitempty"`
	// Breadcrumbs is the raw JSON captured by otel-rum.js before the error
	// (fetch/console events), present only on frontend issues.
	Breadcrumbs string `json:"breadcrumbs,omitempty"`
	// Deterministic, non-PII features used for grouping and review.
	NormalizedMessage string `json:"normalizedMessage,omitempty"`
	NormalizedRoute   string `json:"normalizedRoute,omitempty"`
	StatusClass       string `json:"statusClass,omitempty"`
	StackSignature    string `json:"stackSignature,omitempty"`
}

// Issue is a deduplicated group of related errors, fingerprinted by kind,
// service and a normalized form of the title.
type Issue struct {
	ID               string   `json:"id"`
	Title            string   `json:"title"`
	Kind             string   `json:"kind"` // "span_error", "log_error" or "frontend_error"
	Service          string   `json:"service"`
	Count            int      `json:"count"`
	FirstSeen        int64    `json:"firstSeen"`
	LastSeen         int64    `json:"lastSeen"`
	AffectedServices []string `json:"affectedServices"`
	Samples          []Sample `json:"samples"`
	// Trend maps UTC hour buckets ("2006-01-02T15:00") to occurrence counts
	// over the last trendWindow, for sparklines and spike detection.
	Trend map[string]int `json:"trend,omitempty"`
	// Status is the triage state: open, resolved or ignored.
	Status     string `json:"status"`
	ResolvedAt int64  `json:"resolvedAt,omitempty"`
	// NormalizedTitle is the deterministic, redacted form used for exact grouping.
	NormalizedTitle string `json:"normalizedTitle,omitempty"`
	// PatternID groups related exact issues without merging their operational IDs.
	PatternID        string `json:"patternId,omitempty"`
	PatternSignature string `json:"patternSignature,omitempty"`
}

// SimilarIssue is a deterministic related-issue suggestion. Suggestions never
// change issue IDs and include the reasons that produced the score.
type SimilarIssue struct {
	Issue    Issue    `json:"issue"`
	Score    float64  `json:"score"`
	Reasons  []string `json:"reasons"`
	Decision string   `json:"decision,omitempty"`
}

// Pattern is a UI-friendly aggregate of exact issues.
type Pattern struct {
	ID              string   `json:"id"`
	Signature       string   `json:"signature"`
	Service         string   `json:"service"`
	Kind            string   `json:"kind"`
	IssueCount      int      `json:"issueCount"`
	OccurrenceCount int      `json:"occurrenceCount"`
	LastSeen        int64    `json:"lastSeen"`
	IssueIDs        []string `json:"issueIds"`
}

// Tracker incrementally fingerprints and deduplicates errors as they're
// recorded. It is safe for concurrent use.
type Tracker struct {
	mu       sync.Mutex
	issues   map[string]*Issue
	feedback map[string]string
}

// ListOptions filters and paginates the issue list. Issues remain ordered by
// LastSeen descending; AfterLastSeen/AfterID form an opaque-to-callers cursor
// boundary and make pagination stable when multiple issues share a timestamp.
type ListOptions struct {
	Service       string
	Kind          string
	Status        string
	Title         string
	SourceIP      string
	Route         string
	Since         int64
	Until         int64
	AfterLastSeen int64
	AfterID       string
	Limit         int
}

// ListPage is a filtered page of issues. HasMore indicates that another page
// can be requested using the final issue as the cursor boundary.
type ListPage struct {
	Issues  []Issue
	HasMore bool
}

// NewTracker creates an empty Tracker.
func NewTracker() *Tracker {
	return &Tracker{issues: make(map[string]*Issue), feedback: make(map[string]string)}
}

// Record adds an occurrence of an error, creating a new issue or merging into
// an existing one with a matching (kind, service, normalized title)
// fingerprint.
func (t *Tracker) Record(kind, service, title string, sample Sample) {
	normalizeSample(&sample, title)
	id := fingerprint(kind, service, NormalizeTitle(title))
	bucket := HourBucket(sample.Timestamp)

	t.mu.Lock()
	defer t.mu.Unlock()

	issue, ok := t.issues[id]
	if !ok {
		if len(t.issues) >= maxIssues {
			t.evictOldestLocked()
		}
		issue = &Issue{
			ID:              id,
			Title:           title,
			Kind:            kind,
			Service:         service,
			FirstSeen:       sample.Timestamp,
			Status:          StatusOpen,
			NormalizedTitle: NormalizeTitle(title),
		}
		issue.PatternSignature = patternSignature(kind, service, title, sample)
		issue.PatternID = fingerprint("pattern", issue.PatternSignature)
		t.issues[id] = issue
	}

	issue.Count++
	issue.LastSeen = sample.Timestamp
	issue.AffectedServices = addUnique(issue.AffectedServices, service)
	if issue.Trend == nil {
		issue.Trend = make(map[string]int)
	}
	issue.Trend[bucket]++
	pruneTrendLocked(issue.Trend, sample.Timestamp)
	// A resolved issue that fires again is a regression: reopen it. Ignored
	// issues keep their state (that's the point of ignoring a noisy one).
	if issue.Status == StatusResolved {
		issue.Status = StatusOpen
		issue.ResolvedAt = 0
	}
	issue.Samples = append(issue.Samples, sample)
	if len(issue.Samples) > maxSamples {
		issue.Samples = issue.Samples[len(issue.Samples)-maxSamples:]
	}
}

// Similar returns bounded, deterministic near-match suggestions for an issue.
func (t *Tracker) Similar(id string, limit int) ([]SimilarIssue, bool) {
	t.mu.Lock()
	defer t.mu.Unlock()
	target, ok := t.issues[id]
	if !ok {
		return nil, false
	}
	if limit <= 0 || limit > 50 {
		limit = 10
	}
	result := make([]SimilarIssue, 0)
	for otherID, other := range t.issues {
		if otherID == id || other.Service != target.Service || other.Kind != target.Kind {
			continue
		}
		score, reasons := similarity(*target, *other)
		if score < 0.45 {
			continue
		}
		decision := t.feedback[feedbackKey(id, otherID)]
		result = append(result, SimilarIssue{Issue: cloneIssue(other), Score: score, Reasons: reasons, Decision: decision})
	}
	sort.Slice(result, func(i, j int) bool {
		if result[i].Score != result[j].Score {
			return result[i].Score > result[j].Score
		}
		return result[i].Issue.ID < result[j].Issue.ID
	})
	if len(result) > limit {
		result = result[:limit]
	}
	return result, true
}

// Patterns returns deterministic groups built from exact issue IDs.
func (t *Tracker) Patterns() []Pattern {
	t.mu.Lock()
	defer t.mu.Unlock()
	groups := make(map[string]*Pattern)
	for _, issue := range t.issues {
		p := groups[issue.PatternID]
		if p == nil {
			p = &Pattern{ID: issue.PatternID, Signature: issue.PatternSignature, Service: issue.Service, Kind: issue.Kind}
			groups[issue.PatternID] = p
		}
		p.IssueCount++
		p.OccurrenceCount += issue.Count
		p.IssueIDs = append(p.IssueIDs, issue.ID)
		if issue.LastSeen > p.LastSeen {
			p.LastSeen = issue.LastSeen
		}
	}
	result := make([]Pattern, 0, len(groups))
	for _, p := range groups {
		sort.Strings(p.IssueIDs)
		result = append(result, *p)
	}
	sort.Slice(result, func(i, j int) bool {
		if result[i].LastSeen != result[j].LastSeen {
			return result[i].LastSeen > result[j].LastSeen
		}
		return result[i].ID < result[j].ID
	})
	return result
}

// SetSimilarityFeedback records a human label for later training data.
func (t *Tracker) SetSimilarityFeedback(a, b, decision string) bool {
	if a == "" || b == "" || a == b || (decision != "same" && decision != "different") {
		return false
	}
	t.mu.Lock()
	defer t.mu.Unlock()
	if _, ok := t.issues[a]; !ok {
		return false
	}
	if _, ok := t.issues[b]; !ok {
		return false
	}
	t.feedback[feedbackKey(a, b)] = decision
	return true
}

func feedbackKey(a, b string) string {
	if a > b {
		a, b = b, a
	}
	return a + "\x00" + b
}

func patternSignature(kind, service, title string, sample Sample) string {
	route := sample.NormalizedRoute
	status := sample.StatusClass
	stack := sample.StackSignature
	// The title is deliberately broad here: the exact normalized title remains
	// the canonical issue identity, while this signature supports UI grouping.
	title = normalizePatternTitle(title)
	return strings.Join([]string{service, kind, route, status, title, stack}, "\x00")
}

func normalizeSample(sample *Sample, title string) {
	sample.NormalizedMessage = NormalizeTitle(title)
	sample.NormalizedRoute = NormalizeRoute(sample.HTTPRoute)
	sample.StatusClass = statusClass(sample.HTTPStatus)
	sample.StackSignature = firstStackFrame(sample.Stacktrace)
}

func normalizePatternTitle(title string) string {
	s := NormalizeTitle(title)
	s = regexp.MustCompile(`(?i)http\s+[1-5]\d{2}`).ReplaceAllString(s, "http <status>")
	return s
}

func NormalizeRoute(route string) string {
	route = strings.TrimSpace(strings.SplitN(route, "?", 2)[0])
	if route == "" {
		return ""
	}
	parts := strings.Split(route, "/")
	for i, part := range parts {
		if part == "" {
			continue
		}
		if uuidRe.MatchString(part) || hexIDRe.MatchString(part) || numRe.MatchString(part) {
			parts[i] = "{id}"
		}
	}
	return strings.Join(parts, "/")
}

func statusClass(status int) string {
	if status >= 500 && status < 600 {
		return "5xx"
	}
	if status >= 400 && status < 500 {
		return "4xx"
	}
	if status >= 300 && status < 400 {
		return "3xx"
	}
	if status > 0 {
		return "2xx"
	}
	return ""
}

func firstStackFrame(stack string) string {
	if i := strings.IndexByte(stack, '\n'); i >= 0 {
		stack = stack[:i]
	}
	return strings.TrimSpace(stack)
}

func similarity(a, b Issue) (float64, []string) {
	score := 0.0
	reasons := []string{}
	if a.Service == b.Service {
		score += .25
	}
	if a.Kind == b.Kind {
		score += .15
	}
	aSample, bSample := latestSample(a), latestSample(b)
	if NormalizeRoute(aSample.HTTPRoute) != "" && NormalizeRoute(aSample.HTTPRoute) == NormalizeRoute(bSample.HTTPRoute) {
		score += .20
		reasons = append(reasons, "same route template")
	}
	if statusClass(aSample.HTTPStatus) != "" && statusClass(aSample.HTTPStatus) == statusClass(bSample.HTTPStatus) {
		score += .15
		reasons = append(reasons, "same HTTP status class")
	}
	if firstStackFrame(aSample.Stacktrace) != "" && firstStackFrame(aSample.Stacktrace) == firstStackFrame(bSample.Stacktrace) {
		score += .10
		reasons = append(reasons, "same top stack frame")
	}
	if a.NormalizedTitle != "" && a.NormalizedTitle == b.NormalizedTitle {
		score += .15
		reasons = append(reasons, "same normalized message")
	} else if tokenSimilarity(a.NormalizedTitle, b.NormalizedTitle) >= .5 {
		score += .10
		reasons = append(reasons, "similar normalized message")
	}
	if len(reasons) == 0 {
		reasons = append(reasons, "same service and event kind")
	}
	return score, reasons
}

func latestSample(issue Issue) Sample {
	if len(issue.Samples) == 0 {
		return Sample{}
	}
	return issue.Samples[len(issue.Samples)-1]
}

func tokenSimilarity(a, b string) float64 {
	aa, bb := strings.Fields(a), strings.Fields(b)
	if len(aa) == 0 || len(bb) == 0 {
		return 0
	}
	set := map[string]bool{}
	for _, v := range aa {
		set[v] = true
	}
	other := map[string]bool{}
	for _, v := range bb {
		other[v] = true
	}
	common := 0
	for v := range other {
		if set[v] {
			common++
		}
	}
	return float64(common) / float64(len(set)+len(other)-common)
}

// SetStatus updates the triage state of an issue. It returns the updated
// issue, or false when the id is unknown or the status is invalid.
func (t *Tracker) SetStatus(id, status string) (Issue, bool) {
	switch status {
	case StatusOpen, StatusResolved, StatusIgnored:
	default:
		return Issue{}, false
	}

	t.mu.Lock()
	defer t.mu.Unlock()

	issue, ok := t.issues[id]
	if !ok {
		return Issue{}, false
	}
	issue.Status = status
	if status == StatusOpen {
		issue.ResolvedAt = 0
	} else {
		issue.ResolvedAt = time.Now().UnixNano()
	}
	return cloneIssue(issue), true
}

// evictOldestLocked removes the issue with the smallest LastSeen. Callers
// must hold t.mu.
func (t *Tracker) evictOldestLocked() {
	var oldestID string
	var oldest int64
	first := true
	for id, issue := range t.issues {
		if first || issue.LastSeen < oldest {
			oldest = issue.LastSeen
			oldestID = id
			first = false
		}
	}
	if oldestID != "" {
		delete(t.issues, oldestID)
	}
}

// List returns issues sorted by LastSeen descending (most recently seen
// first). A non-positive limit returns all issues.
func (t *Tracker) List(limit int) []Issue {
	return t.ListFiltered(ListOptions{Limit: limit}).Issues
}

// ListFiltered returns a filtered, stable page of issues.
func (t *Tracker) ListFiltered(opts ListOptions) ListPage {
	t.mu.Lock()
	defer t.mu.Unlock()

	result := make([]Issue, 0, len(t.issues))
	for _, issue := range t.issues {
		if !matches(issue, opts) || beforeCursor(issue, opts) {
			continue
		}
		result = append(result, cloneIssue(issue))
	}

	sort.Slice(result, func(i, j int) bool {
		if result[i].LastSeen != result[j].LastSeen {
			return result[i].LastSeen > result[j].LastSeen
		}
		return result[i].ID > result[j].ID
	})

	limit := opts.Limit
	if limit <= 0 {
		limit = len(result)
	}
	page := ListPage{Issues: result}
	if len(page.Issues) > limit {
		page.Issues = page.Issues[:limit]
		page.HasMore = true
	}
	return page
}

func matches(issue *Issue, opts ListOptions) bool {
	if opts.Service != "" && issue.Service != opts.Service {
		return false
	}
	if opts.Kind != "" && issue.Kind != opts.Kind {
		return false
	}
	if opts.Status != "" && issue.Status != opts.Status {
		return false
	}
	if opts.Title != "" && !strings.Contains(strings.ToLower(issue.Title), strings.ToLower(opts.Title)) {
		return false
	}
	if opts.Since != 0 && issue.LastSeen < opts.Since {
		return false
	}
	if opts.Until != 0 && issue.LastSeen > opts.Until {
		return false
	}
	if opts.SourceIP != "" && !issueHasSample(issue, func(sample Sample) bool { return sample.SourceIP == opts.SourceIP }) {
		return false
	}
	if opts.Route != "" && !issueHasSample(issue, func(sample Sample) bool { return sample.HTTPRoute == opts.Route }) {
		return false
	}
	return true
}

func issueHasSample(issue *Issue, predicate func(Sample) bool) bool {
	for _, sample := range issue.Samples {
		if predicate(sample) {
			return true
		}
	}
	return false
}

func beforeCursor(issue *Issue, opts ListOptions) bool {
	if opts.AfterLastSeen == 0 {
		return false
	}
	if issue.LastSeen < opts.AfterLastSeen {
		return false
	}
	if issue.LastSeen > opts.AfterLastSeen {
		return true
	}
	return issue.ID >= opts.AfterID
}

// Get returns a single issue by id.
func (t *Tracker) Get(id string) (Issue, bool) {
	t.mu.Lock()
	defer t.mu.Unlock()

	issue, ok := t.issues[id]
	if !ok {
		return Issue{}, false
	}
	return cloneIssue(issue), true
}

// cloneIssue deep-copies the slice/map fields so returned issues don't share
// backing arrays with ones still being mutated by Record.
func cloneIssue(i *Issue) Issue {
	c := *i
	c.AffectedServices = append([]string(nil), i.AffectedServices...)
	c.Samples = append([]Sample(nil), i.Samples...)
	if i.Trend != nil {
		c.Trend = make(map[string]int, len(i.Trend))
		for k, v := range i.Trend {
			c.Trend[k] = v
		}
	}
	return c
}

// HourBucket renders a unix-nanosecond timestamp as a UTC hour key used in
// Issue.Trend. Keys sort lexicographically, so the UI can sort them as-is.
func HourBucket(ts int64) string {
	return time.Unix(0, ts).UTC().Format("2006-01-02T15:00")
}

// pruneTrendLocked drops trend buckets older than trendWindow. Callers must
// hold the tracker lock.
func pruneTrendLocked(trend map[string]int, now int64) {
	if len(trend) <= 48 {
		return
	}
	cutoff := HourBucket(time.Unix(0, now).Add(-trendWindow).UnixNano())
	for k := range trend {
		if k < cutoff {
			delete(trend, k)
		}
	}
}

var (
	uuidRe       = regexp.MustCompile(`(?i)\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b`)
	hexIDRe      = regexp.MustCompile(`(?i)\b[0-9a-f]{8,}\b`)
	httpStatusRe = regexp.MustCompile(`(?i)\bhttp\s+([1-5]\d{2})\b`)
	numRe        = regexp.MustCompile(`\b\d+\b`)
	spaceRe      = regexp.MustCompile(`\s+`)
)

// NormalizeTitle strips variable content (UUIDs, hex ids, numbers, whitespace,
// casing) from an error title so that "User 123 not found" and "User 456 not
// found" fingerprint into the same issue. The raw title is kept as the
// display title of the first-seen occurrence. HTTP status codes (e.g. HTTP 404)
// are preserved so different HTTP error types don't collapse together.
func NormalizeTitle(title string) string {
	s := uuidRe.ReplaceAllString(title, "<id>")
	// Protect HTTP status codes from being replaced by <num>
	s = httpStatusRe.ReplaceAllStringFunc(s, func(match string) string {
		parts := strings.Fields(match)
		if len(parts) != 2 {
			return match
		}
		return "__HTTP_STATUS_" + encodeStatus(parts[1]) + "__"
	})
	s = numRe.ReplaceAllString(s, "<num>")
	s = hexIDRe.ReplaceAllStringFunc(s, func(match string) string {
		// Mixed hex runs (letters + digits, e.g. trace/span ids) become ids.
		// Pure-digit runs were already handled by numRe above, and words
		// spelled entirely from a-f (rare at this length) survive so distinct
		// error messages don't collapse.
		if strings.ContainsAny(match, "0123456789") {
			return "<id>"
		}
		return match
	})
	s = regexp.MustCompile(`__HTTP_STATUS_([a-z]+)__`).ReplaceAllStringFunc(s, func(match string) string {
		encoded := strings.TrimSuffix(strings.TrimPrefix(match, "__HTTP_STATUS_"), "__")
		return "http " + decodeStatus(encoded)
	})
	s = strings.ReplaceAll(s, "__", "")
	s = spaceRe.ReplaceAllString(s, " ")
	return strings.ToLower(strings.TrimSpace(s))
}

func encodeStatus(code string) string {
	var b strings.Builder
	for _, digit := range code {
		b.WriteByte('a' + byte(digit-'0'))
	}
	return b.String()
}

func decodeStatus(encoded string) string {
	var b strings.Builder
	for _, letter := range encoded {
		b.WriteByte('0' + byte(letter-'a'))
	}
	return b.String()
}

func fingerprint(parts ...string) string {
	h := fnv.New64a()
	_, _ = h.Write([]byte(strings.Join(parts, "\x00")))
	return fmt.Sprintf("%016x", h.Sum64())
}

func addUnique(list []string, v string) []string {
	if v == "" {
		return list
	}
	for _, existing := range list {
		if existing == v {
			return list
		}
	}
	return append(list, v)
}