// Package issues derives deduplicated error "issues" from ingested telemetry, // so the web UI can show a triaged list instead of raw traces and logs. package issues import ( "fmt" "hash/fnv" "regexp" "sort" "strings" "sync" "time" ) // maxIssues bounds memory use: once exceeded, the least-recently-seen issue // is evicted to make room for a newly seen distinct one. const maxIssues = 2000 // maxSamples bounds how many sample occurrences are kept per issue. const maxSamples = 5 // trendWindow bounds how far back the per-hour trend buckets are kept. const trendWindow = 48 * time.Hour // Issue lifecycle states. New occurrences automatically reopen resolved // issues (a regression), but stay merged into ignored ones until the user // explicitly sets the state back to open. const ( StatusOpen = "open" StatusResolved = "resolved" StatusIgnored = "ignored" ) // Sample is one occurrence of an issue. type Sample struct { Timestamp int64 `json:"timestamp"` TraceID string `json:"traceId,omitempty"` SpanID string `json:"spanId,omitempty"` Message string `json:"message"` // URL is the page URL an occurrence happened on, populated only for // browser-originated (otel-rum) errors. URL string `json:"url,omitempty"` // Stacktrace is the exception.stacktrace attribute of the occurrence, // when the source telemetry carried one. Stacktrace string `json:"stacktrace,omitempty"` ResolvedStacktrace string `json:"resolvedStacktrace,omitempty"` // HTTP request context, extracted from the span/log attributes when the // error happened inside an HTTP handler (semconv v1.x old and new names // are both handled by the extractor). HTTPMethod string `json:"httpMethod,omitempty"` HTTPRoute string `json:"httpRoute,omitempty"` HTTPStatus int `json:"httpStatus,omitempty"` // Client/release context. UserAgent string `json:"userAgent,omitempty"` SourceIP string `json:"sourceIp,omitempty"` Release string `json:"release,omitempty"` Environment string `json:"environment,omitempty"` // Breadcrumbs is the raw JSON captured by otel-rum.js before the error // (fetch/console events), present only on frontend issues. Breadcrumbs string `json:"breadcrumbs,omitempty"` // Deterministic, non-PII features used for grouping and review. NormalizedMessage string `json:"normalizedMessage,omitempty"` NormalizedRoute string `json:"normalizedRoute,omitempty"` StatusClass string `json:"statusClass,omitempty"` StackSignature string `json:"stackSignature,omitempty"` } // Issue is a deduplicated group of related errors, fingerprinted by kind, // service and a normalized form of the title. type Issue struct { ID string `json:"id"` Title string `json:"title"` Kind string `json:"kind"` // "span_error", "log_error" or "frontend_error" Service string `json:"service"` Count int `json:"count"` FirstSeen int64 `json:"firstSeen"` LastSeen int64 `json:"lastSeen"` AffectedServices []string `json:"affectedServices"` Samples []Sample `json:"samples"` // Trend maps UTC hour buckets ("2006-01-02T15:00") to occurrence counts // over the last trendWindow, for sparklines and spike detection. Trend map[string]int `json:"trend,omitempty"` // Status is the triage state: open, resolved or ignored. Status string `json:"status"` ResolvedAt int64 `json:"resolvedAt,omitempty"` // NormalizedTitle is the deterministic, redacted form used for exact grouping. NormalizedTitle string `json:"normalizedTitle,omitempty"` // PatternID groups related exact issues without merging their operational IDs. PatternID string `json:"patternId,omitempty"` PatternSignature string `json:"patternSignature,omitempty"` } // SimilarIssue is a deterministic related-issue suggestion. Suggestions never // change issue IDs and include the reasons that produced the score. type SimilarIssue struct { Issue Issue `json:"issue"` Score float64 `json:"score"` Reasons []string `json:"reasons"` Decision string `json:"decision,omitempty"` } // Pattern is a UI-friendly aggregate of exact issues. type Pattern struct { ID string `json:"id"` Signature string `json:"signature"` Service string `json:"service"` Kind string `json:"kind"` IssueCount int `json:"issueCount"` OccurrenceCount int `json:"occurrenceCount"` LastSeen int64 `json:"lastSeen"` IssueIDs []string `json:"issueIds"` } // Tracker incrementally fingerprints and deduplicates errors as they're // recorded. It is safe for concurrent use. type Tracker struct { mu sync.Mutex issues map[string]*Issue feedback map[string]string } // ListOptions filters and paginates the issue list. Issues remain ordered by // LastSeen descending; AfterLastSeen/AfterID form an opaque-to-callers cursor // boundary and make pagination stable when multiple issues share a timestamp. type ListOptions struct { Service string Kind string Status string Title string SourceIP string Route string Since int64 Until int64 AfterLastSeen int64 AfterID string Limit int } // ListPage is a filtered page of issues. HasMore indicates that another page // can be requested using the final issue as the cursor boundary. type ListPage struct { Issues []Issue HasMore bool } // NewTracker creates an empty Tracker. func NewTracker() *Tracker { return &Tracker{issues: make(map[string]*Issue), feedback: make(map[string]string)} } // Record adds an occurrence of an error, creating a new issue or merging into // an existing one with a matching (kind, service, normalized title) // fingerprint. func (t *Tracker) Record(kind, service, title string, sample Sample) { normalizeSample(&sample, title) id := Fingerprint(kind, service, NormalizeTitle(title)) bucket := HourBucket(sample.Timestamp) t.mu.Lock() defer t.mu.Unlock() issue, ok := t.issues[id] if !ok { if len(t.issues) >= maxIssues { t.evictOldestLocked() } issue = &Issue{ ID: id, Title: title, Kind: kind, Service: service, FirstSeen: sample.Timestamp, Status: StatusOpen, NormalizedTitle: NormalizeTitle(title), } issue.PatternSignature = patternSignature(kind, service, title, sample) issue.PatternID = Fingerprint("pattern", issue.PatternSignature) t.issues[id] = issue } issue.Count++ issue.LastSeen = sample.Timestamp issue.AffectedServices = addUnique(issue.AffectedServices, service) if issue.Trend == nil { issue.Trend = make(map[string]int) } issue.Trend[bucket]++ pruneTrendLocked(issue.Trend, sample.Timestamp) // A resolved issue that fires again is a regression: reopen it. Ignored // issues keep their state (that's the point of ignoring a noisy one). if issue.Status == StatusResolved { issue.Status = StatusOpen issue.ResolvedAt = 0 } issue.Samples = append(issue.Samples, sample) if len(issue.Samples) > maxSamples { issue.Samples = issue.Samples[len(issue.Samples)-maxSamples:] } } // Similar returns bounded, deterministic near-match suggestions for an issue. func (t *Tracker) Similar(id string, limit int) ([]SimilarIssue, bool) { t.mu.Lock() defer t.mu.Unlock() target, ok := t.issues[id] if !ok { return nil, false } if limit <= 0 || limit > 50 { limit = 10 } result := make([]SimilarIssue, 0) for otherID, other := range t.issues { if otherID == id || other.Service != target.Service || other.Kind != target.Kind { continue } score, reasons := similarity(*target, *other) if score < 0.45 { continue } decision := t.feedback[feedbackKey(id, otherID)] result = append(result, SimilarIssue{Issue: cloneIssue(other), Score: score, Reasons: reasons, Decision: decision}) } sort.Slice(result, func(i, j int) bool { if result[i].Score != result[j].Score { return result[i].Score > result[j].Score } return result[i].Issue.ID < result[j].Issue.ID }) if len(result) > limit { result = result[:limit] } return result, true } // Patterns returns deterministic groups built from exact issue IDs. func (t *Tracker) Patterns() []Pattern { t.mu.Lock() defer t.mu.Unlock() groups := make(map[string]*Pattern) for _, issue := range t.issues { p := groups[issue.PatternID] if p == nil { p = &Pattern{ID: issue.PatternID, Signature: issue.PatternSignature, Service: issue.Service, Kind: issue.Kind} groups[issue.PatternID] = p } p.IssueCount++ p.OccurrenceCount += issue.Count p.IssueIDs = append(p.IssueIDs, issue.ID) if issue.LastSeen > p.LastSeen { p.LastSeen = issue.LastSeen } } result := make([]Pattern, 0, len(groups)) for _, p := range groups { sort.Strings(p.IssueIDs) result = append(result, *p) } sort.Slice(result, func(i, j int) bool { if result[i].LastSeen != result[j].LastSeen { return result[i].LastSeen > result[j].LastSeen } return result[i].ID < result[j].ID }) return result } // SetSimilarityFeedback records a human label for later training data. func (t *Tracker) SetSimilarityFeedback(a, b, decision string) bool { if a == "" || b == "" || a == b || (decision != "same" && decision != "different") { return false } t.mu.Lock() defer t.mu.Unlock() if _, ok := t.issues[a]; !ok { return false } if _, ok := t.issues[b]; !ok { return false } t.feedback[feedbackKey(a, b)] = decision return true } func feedbackKey(a, b string) string { if a > b { a, b = b, a } return a + "\x00" + b } func patternSignature(kind, service, title string, sample Sample) string { route := sample.NormalizedRoute status := sample.StatusClass stack := sample.StackSignature // The title is deliberately broad here: the exact normalized title remains // the canonical issue identity, while this signature supports UI grouping. title = normalizePatternTitle(title) return strings.Join([]string{service, kind, route, status, title, stack}, "\x00") } func normalizeSample(sample *Sample, title string) { sample.NormalizedMessage = NormalizeTitle(title) sample.NormalizedRoute = NormalizeRoute(sample.HTTPRoute) sample.StatusClass = statusClass(sample.HTTPStatus) sample.StackSignature = firstStackFrame(sample.Stacktrace) } func normalizePatternTitle(title string) string { s := NormalizeTitle(title) s = regexp.MustCompile(`(?i)http\s+[1-5]\d{2}`).ReplaceAllString(s, "http ") return s } func NormalizeRoute(route string) string { route = strings.TrimSpace(strings.SplitN(route, "?", 2)[0]) if route == "" { return "" } parts := strings.Split(route, "/") for i, part := range parts { if part == "" { continue } if uuidRe.MatchString(part) || hexIDRe.MatchString(part) || numRe.MatchString(part) { parts[i] = "{id}" } } return strings.Join(parts, "/") } func statusClass(status int) string { if status >= 500 && status < 600 { return "5xx" } if status >= 400 && status < 500 { return "4xx" } if status >= 300 && status < 400 { return "3xx" } if status > 0 { return "2xx" } return "" } func firstStackFrame(stack string) string { if i := strings.IndexByte(stack, '\n'); i >= 0 { stack = stack[:i] } return strings.TrimSpace(stack) } func similarity(a, b Issue) (float64, []string) { score := 0.0 reasons := []string{} if a.Service == b.Service { score += .25 } if a.Kind == b.Kind { score += .15 } aSample, bSample := latestSample(a), latestSample(b) if NormalizeRoute(aSample.HTTPRoute) != "" && NormalizeRoute(aSample.HTTPRoute) == NormalizeRoute(bSample.HTTPRoute) { score += .20 reasons = append(reasons, "same route template") } if statusClass(aSample.HTTPStatus) != "" && statusClass(aSample.HTTPStatus) == statusClass(bSample.HTTPStatus) { score += .15 reasons = append(reasons, "same HTTP status class") } if firstStackFrame(aSample.Stacktrace) != "" && firstStackFrame(aSample.Stacktrace) == firstStackFrame(bSample.Stacktrace) { score += .10 reasons = append(reasons, "same top stack frame") } if a.NormalizedTitle != "" && a.NormalizedTitle == b.NormalizedTitle { score += .15 reasons = append(reasons, "same normalized message") } else if tokenSimilarity(a.NormalizedTitle, b.NormalizedTitle) >= .5 { score += .10 reasons = append(reasons, "similar normalized message") } if len(reasons) == 0 { reasons = append(reasons, "same service and event kind") } return score, reasons } func latestSample(issue Issue) Sample { if len(issue.Samples) == 0 { return Sample{} } return issue.Samples[len(issue.Samples)-1] } func tokenSimilarity(a, b string) float64 { aa, bb := strings.Fields(a), strings.Fields(b) if len(aa) == 0 || len(bb) == 0 { return 0 } set := map[string]bool{} for _, v := range aa { set[v] = true } other := map[string]bool{} for _, v := range bb { other[v] = true } common := 0 for v := range other { if set[v] { common++ } } return float64(common) / float64(len(set)+len(other)-common) } // SetStatus updates the triage state of an issue. It returns the updated // issue, or false when the id is unknown or the status is invalid. func (t *Tracker) SetStatus(id, status string) (Issue, bool) { switch status { case StatusOpen, StatusResolved, StatusIgnored: default: return Issue{}, false } t.mu.Lock() defer t.mu.Unlock() issue, ok := t.issues[id] if !ok { return Issue{}, false } issue.Status = status if status == StatusOpen { issue.ResolvedAt = 0 } else { issue.ResolvedAt = time.Now().UnixNano() } return cloneIssue(issue), true } // evictOldestLocked removes the issue with the smallest LastSeen. Callers // must hold t.mu. func (t *Tracker) evictOldestLocked() { var oldestID string var oldest int64 first := true for id, issue := range t.issues { if first || issue.LastSeen < oldest { oldest = issue.LastSeen oldestID = id first = false } } if oldestID != "" { delete(t.issues, oldestID) } } // List returns issues sorted by LastSeen descending (most recently seen // first). A non-positive limit returns all issues. func (t *Tracker) List(limit int) []Issue { return t.ListFiltered(ListOptions{Limit: limit}).Issues } // ListFiltered returns a filtered, stable page of issues. func (t *Tracker) ListFiltered(opts ListOptions) ListPage { t.mu.Lock() defer t.mu.Unlock() result := make([]Issue, 0, len(t.issues)) for _, issue := range t.issues { if !matches(issue, opts) || beforeCursor(issue, opts) { continue } result = append(result, cloneIssue(issue)) } sort.Slice(result, func(i, j int) bool { if result[i].LastSeen != result[j].LastSeen { return result[i].LastSeen > result[j].LastSeen } return result[i].ID > result[j].ID }) limit := opts.Limit if limit <= 0 { limit = len(result) } page := ListPage{Issues: result} if len(page.Issues) > limit { page.Issues = page.Issues[:limit] page.HasMore = true } return page } func matches(issue *Issue, opts ListOptions) bool { if opts.Service != "" && issue.Service != opts.Service { return false } if opts.Kind != "" && issue.Kind != opts.Kind { return false } if opts.Status != "" && issue.Status != opts.Status { return false } if opts.Title != "" && !strings.Contains(strings.ToLower(issue.Title), strings.ToLower(opts.Title)) { return false } if opts.Since != 0 && issue.LastSeen < opts.Since { return false } if opts.Until != 0 && issue.LastSeen > opts.Until { return false } if opts.SourceIP != "" && !issueHasSample(issue, func(sample Sample) bool { return sample.SourceIP == opts.SourceIP }) { return false } if opts.Route != "" && !issueHasSample(issue, func(sample Sample) bool { return sample.HTTPRoute == opts.Route }) { return false } return true } func issueHasSample(issue *Issue, predicate func(Sample) bool) bool { for _, sample := range issue.Samples { if predicate(sample) { return true } } return false } func beforeCursor(issue *Issue, opts ListOptions) bool { if opts.AfterLastSeen == 0 { return false } if issue.LastSeen < opts.AfterLastSeen { return false } if issue.LastSeen > opts.AfterLastSeen { return true } return issue.ID >= opts.AfterID } // Get returns a single issue by id. func (t *Tracker) Get(id string) (Issue, bool) { t.mu.Lock() defer t.mu.Unlock() issue, ok := t.issues[id] if !ok { return Issue{}, false } return cloneIssue(issue), true } // cloneIssue deep-copies the slice/map fields so returned issues don't share // backing arrays with ones still being mutated by Record. func cloneIssue(i *Issue) Issue { c := *i c.AffectedServices = append([]string(nil), i.AffectedServices...) c.Samples = append([]Sample(nil), i.Samples...) if i.Trend != nil { c.Trend = make(map[string]int, len(i.Trend)) for k, v := range i.Trend { c.Trend[k] = v } } return c } // HourBucket renders a unix-nanosecond timestamp as a UTC hour key used in // Issue.Trend. Keys sort lexicographically, so the UI can sort them as-is. func HourBucket(ts int64) string { return time.Unix(0, ts).UTC().Format("2006-01-02T15:00") } // pruneTrendLocked drops trend buckets older than trendWindow. Callers must // hold the tracker lock. func pruneTrendLocked(trend map[string]int, now int64) { if len(trend) <= 48 { return } cutoff := HourBucket(time.Unix(0, now).Add(-trendWindow).UnixNano()) for k := range trend { if k < cutoff { delete(trend, k) } } } var ( uuidRe = regexp.MustCompile(`(?i)\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b`) hexIDRe = regexp.MustCompile(`(?i)\b[0-9a-f]{8,}\b`) httpStatusRe = regexp.MustCompile(`(?i)\bhttp\s+([1-5]\d{2})\b`) numRe = regexp.MustCompile(`\b\d+\b`) spaceRe = regexp.MustCompile(`\s+`) ) // NormalizeTitle strips variable content (UUIDs, hex ids, numbers, whitespace, // casing) from an error title so that "User 123 not found" and "User 456 not // found" fingerprint into the same issue. The raw title is kept as the // display title of the first-seen occurrence. HTTP status codes (e.g. HTTP 404) // are preserved so different HTTP error types don't collapse together. func NormalizeTitle(title string) string { s := uuidRe.ReplaceAllString(title, "") // Protect HTTP status codes from being replaced by s = httpStatusRe.ReplaceAllStringFunc(s, func(match string) string { parts := strings.Fields(match) if len(parts) != 2 { return match } return "__HTTP_STATUS_" + encodeStatus(parts[1]) + "__" }) s = numRe.ReplaceAllString(s, "") s = hexIDRe.ReplaceAllStringFunc(s, func(match string) string { // Mixed hex runs (letters + digits, e.g. trace/span ids) become ids. // Pure-digit runs were already handled by numRe above, and words // spelled entirely from a-f (rare at this length) survive so distinct // error messages don't collapse. if strings.ContainsAny(match, "0123456789") { return "" } return match }) s = regexp.MustCompile(`__HTTP_STATUS_([a-z]+)__`).ReplaceAllStringFunc(s, func(match string) string { encoded := strings.TrimSuffix(strings.TrimPrefix(match, "__HTTP_STATUS_"), "__") return "http " + decodeStatus(encoded) }) s = strings.ReplaceAll(s, "__", "") s = spaceRe.ReplaceAllString(s, " ") return strings.ToLower(strings.TrimSpace(s)) } func encodeStatus(code string) string { var b strings.Builder for _, digit := range code { b.WriteByte('a' + byte(digit-'0')) } return b.String() } func decodeStatus(encoded string) string { var b strings.Builder for _, letter := range encoded { b.WriteByte('0' + byte(letter-'a')) } return b.String() } // Fingerprint creates the stable opaque identifier used for canonical issue // grouping. Normalization uses it for error rows so offline feature exports // can count the same issue identity as the live issue tracker. func Fingerprint(parts ...string) string { h := fnv.New64a() _, _ = h.Write([]byte(strings.Join(parts, "\x00"))) return fmt.Sprintf("%016x", h.Sum64()) } func addUnique(list []string, v string) []string { if v == "" { return list } for _, existing := range list { if existing == v { return list } } return append(list, v) }