Files
mats 0052822e49
Build and Publish / Metadata (push) Successful in 9s
Build and Publish / BuildAndDeployAmd64 (push) Successful in 2m35s
fix
2026-09-22 22:09:32 +02:00

770 lines
23 KiB
Go

// Package issues derives deduplicated error "issues" from ingested telemetry,
// so the web UI can show a triaged list instead of raw traces and logs.
package issues
import (
"context"
"fmt"
"hash/fnv"
"log"
"regexp"
"sort"
"strings"
"sync"
"time"
)
// maxIssues bounds memory use: once exceeded, the least-recently-seen issue
// is evicted to make room for a newly seen distinct one.
const maxIssues = 2000
// maxSamples bounds how many sample occurrences are kept per issue.
const maxSamples = 5
// trendWindow bounds how far back the per-hour trend buckets are kept.
const trendWindow = 48 * time.Hour
// Issue lifecycle states. New occurrences automatically reopen resolved
// issues (a regression), but stay merged into ignored ones until the user
// explicitly sets the state back to open.
const (
StatusOpen = "open"
StatusResolved = "resolved"
StatusIgnored = "ignored"
)
// Sample is one occurrence of an issue.
type Sample struct {
Timestamp int64 `json:"timestamp"`
TraceID string `json:"traceId,omitempty"`
SpanID string `json:"spanId,omitempty"`
Message string `json:"message"`
// URL is the page URL an occurrence happened on, populated only for
// browser-originated (otel-rum) errors.
URL string `json:"url,omitempty"`
// Stacktrace is the exception.stacktrace attribute of the occurrence,
// when the source telemetry carried one.
Stacktrace string `json:"stacktrace,omitempty"`
ResolvedStacktrace string `json:"resolvedStacktrace,omitempty"`
// HTTP request context, extracted from the span/log attributes when the
// error happened inside an HTTP handler (semconv v1.x old and new names
// are both handled by the extractor).
HTTPMethod string `json:"httpMethod,omitempty"`
HTTPRoute string `json:"httpRoute,omitempty"`
HTTPStatus int `json:"httpStatus,omitempty"`
// Client/release context.
UserAgent string `json:"userAgent,omitempty"`
// UAClass is derived from UserAgent when issues are served (see
// internal/uaclass); it is not part of the stored sample.
UAClass string `json:"uaClass,omitempty"`
SourceIP string `json:"sourceIp,omitempty"`
Release string `json:"release,omitempty"`
Environment string `json:"environment,omitempty"`
// Breadcrumbs is the raw JSON captured by otel-rum.js before the error
// (fetch/console events), present only on frontend issues.
Breadcrumbs string `json:"breadcrumbs,omitempty"`
// Deterministic, non-PII features used for grouping and review.
NormalizedMessage string `json:"normalizedMessage,omitempty"`
NormalizedRoute string `json:"normalizedRoute,omitempty"`
StatusClass string `json:"statusClass,omitempty"`
StackSignature string `json:"stackSignature,omitempty"`
}
// Issue is a deduplicated group of related errors, fingerprinted by kind,
// service and a normalized form of the title.
type Issue struct {
ID string `json:"id"`
Title string `json:"title"`
Kind string `json:"kind"` // "span_error", "log_error" or "frontend_error"
Service string `json:"service"`
Count int `json:"count"`
FirstSeen int64 `json:"firstSeen"`
LastSeen int64 `json:"lastSeen"`
AffectedServices []string `json:"affectedServices"`
Samples []Sample `json:"samples"`
// Trend maps UTC hour buckets ("2006-01-02T15:00") to occurrence counts
// over the last trendWindow, for sparklines and spike detection.
Trend map[string]int `json:"trend,omitempty"`
// Status is the triage state: open, resolved or ignored.
Status string `json:"status"`
ResolvedAt int64 `json:"resolvedAt,omitempty"`
// RegressedAt is when a resolved issue last fired again and was reopened.
RegressedAt int64 `json:"regressedAt,omitempty"`
// NormalizedTitle is the deterministic, redacted form used for exact grouping.
NormalizedTitle string `json:"normalizedTitle,omitempty"`
// PatternID groups related exact issues without merging their operational IDs.
PatternID string `json:"patternId,omitempty"`
PatternSignature string `json:"patternSignature,omitempty"`
}
// SimilarIssue is a deterministic related-issue suggestion. Suggestions never
// change issue IDs and include the reasons that produced the score.
type SimilarIssue struct {
Issue Issue `json:"issue"`
Score float64 `json:"score"`
Reasons []string `json:"reasons"`
Decision string `json:"decision,omitempty"`
}
// Pattern is a UI-friendly aggregate of exact issues.
type Pattern struct {
ID string `json:"id"`
Signature string `json:"signature"`
Service string `json:"service"`
Kind string `json:"kind"`
IssueCount int `json:"issueCount"`
OccurrenceCount int `json:"occurrenceCount"`
LastSeen int64 `json:"lastSeen"`
IssueIDs []string `json:"issueIds"`
}
// Tracker incrementally fingerprints and deduplicates errors as they're
// recorded. It is safe for concurrent use.
type Tracker struct {
mu sync.Mutex
issues map[string]*Issue
// triage holds persisted resolved/ignored decisions by issue id, applied
// when an issue is (re)created after a restart.
triage map[string]TriageState
statusStore StatusStore
feedback map[string]string
sink OccurrenceSink
}
// OccurrenceSink receives every recorded occurrence (already normalized) so
// it can be persisted beyond the bounded in-memory sample window. It is
// called outside the tracker lock and must not block.
type OccurrenceSink func(issueID, kind, service string, sample Sample)
// SetOccurrenceSink registers the sink; nil disables it.
func (t *Tracker) SetOccurrenceSink(fn OccurrenceSink) {
t.mu.Lock()
t.sink = fn
t.mu.Unlock()
}
// ListOptions filters and paginates the issue list. Issues remain ordered by
// LastSeen descending; AfterLastSeen/AfterID form an opaque-to-callers cursor
// boundary and make pagination stable when multiple issues share a timestamp.
type ListOptions struct {
Service string
Kind string
Status string
Title string
SourceIP string
Route string
Since int64
Until int64
AfterLastSeen int64
AfterID string
Limit int
}
// ListPage is a filtered page of issues. HasMore indicates that another page
// can be requested using the final issue as the cursor boundary.
type ListPage struct {
Issues []Issue
HasMore bool
}
// NewTracker creates an empty Tracker.
func NewTracker() *Tracker {
return &Tracker{issues: make(map[string]*Issue), feedback: make(map[string]string), triage: make(map[string]TriageState)}
}
// SetStatusStore loads persisted triage decisions and saves future ones.
func (t *Tracker) SetStatusStore(ctx context.Context, store StatusStore) error {
states, err := store.Load(ctx)
if err != nil {
return err
}
t.mu.Lock()
defer t.mu.Unlock()
t.statusStore = store
for id, st := range states {
t.triage[id] = st
if issue, ok := t.issues[id]; ok {
issue.Status, issue.ResolvedAt = st.Status, st.ResolvedAt
}
}
return nil
}
// IsIgnored reports whether a person ignored the issue. It also answers for
// issues not yet recreated since a restart, which alerting needs.
func (t *Tracker) IsIgnored(id string) bool {
t.mu.Lock()
defer t.mu.Unlock()
return t.triage[id].Status == StatusIgnored
}
// persistLocked records a decision. The write happens off the lock.
func (t *Tracker) persistLocked(id string, st TriageState) {
if st.Status == StatusOpen {
delete(t.triage, id)
} else {
t.triage[id] = st
}
if t.statusStore == nil {
return
}
store := t.statusStore
go func() {
if err := store.Save(context.Background(), id, st); err != nil {
log.Printf("failed to persist issue status: %v", err)
}
}()
}
// Record adds an occurrence of an error, creating a new issue or merging into
// an existing one with a matching (kind, service, normalized title)
// fingerprint.
func (t *Tracker) Record(kind, service, title string, sample Sample) {
normalizeSample(&sample, title)
id, sink := t.record(kind, service, title, sample)
if sink != nil {
sink(id, kind, service, sample)
}
}
// Restore records an occurrence loaded from the persistent occurrence store.
// It deliberately does not invoke the occurrence sink, avoiding duplicate
// writes while rebuilding the in-memory index during startup.
func (t *Tracker) Restore(kind, service, title string, sample Sample) {
normalizeSample(&sample, title)
_, _ = t.record(kind, service, title, sample)
}
func (t *Tracker) record(kind, service, title string, sample Sample) (string, OccurrenceSink) {
id := Fingerprint(kind, service, NormalizeTitle(title))
bucket := HourBucket(sample.Timestamp)
t.mu.Lock()
defer t.mu.Unlock()
issue, ok := t.issues[id]
if !ok {
if len(t.issues) >= maxIssues {
t.evictOldestLocked()
}
issue = &Issue{
ID: id,
Title: title,
Kind: kind,
Service: service,
FirstSeen: sample.Timestamp,
Status: StatusOpen,
NormalizedTitle: NormalizeTitle(title),
}
if st, ok := t.triage[id]; ok {
issue.Status, issue.ResolvedAt = st.Status, st.ResolvedAt
}
issue.PatternSignature = patternSignature(kind, service, title, sample)
issue.PatternID = Fingerprint("pattern", issue.PatternSignature)
t.issues[id] = issue
}
issue.Count++
issue.LastSeen = sample.Timestamp
issue.AffectedServices = addUnique(issue.AffectedServices, service)
if issue.Trend == nil {
issue.Trend = make(map[string]int)
}
issue.Trend[bucket]++
pruneTrendLocked(issue.Trend, sample.Timestamp)
// A resolved issue that fires again is a regression: reopen it. Ignored
// issues keep their state (that's the point of ignoring a noisy one).
if issue.Status == StatusResolved {
issue.Status = StatusOpen
issue.ResolvedAt = 0
issue.RegressedAt = sample.Timestamp
t.persistLocked(id, TriageState{Status: StatusOpen})
}
issue.Samples = append(issue.Samples, sample)
if len(issue.Samples) > maxSamples {
issue.Samples = issue.Samples[len(issue.Samples)-maxSamples:]
}
return id, t.sink
}
// Similar returns bounded, deterministic near-match suggestions for an issue.
func (t *Tracker) Similar(id string, limit int) ([]SimilarIssue, bool) {
t.mu.Lock()
defer t.mu.Unlock()
target, ok := t.issues[id]
if !ok {
return nil, false
}
if limit <= 0 || limit > 50 {
limit = 10
}
result := make([]SimilarIssue, 0)
for otherID, other := range t.issues {
if otherID == id || other.Service != target.Service || other.Kind != target.Kind {
continue
}
score, reasons := similarity(*target, *other)
if score < 0.45 {
continue
}
decision := t.feedback[feedbackKey(id, otherID)]
result = append(result, SimilarIssue{Issue: cloneIssue(other), Score: score, Reasons: reasons, Decision: decision})
}
sort.Slice(result, func(i, j int) bool {
if result[i].Score != result[j].Score {
return result[i].Score > result[j].Score
}
return result[i].Issue.ID < result[j].Issue.ID
})
if len(result) > limit {
result = result[:limit]
}
return result, true
}
// Patterns returns deterministic groups built from exact issue IDs.
func (t *Tracker) Patterns() []Pattern {
t.mu.Lock()
defer t.mu.Unlock()
groups := make(map[string]*Pattern)
for _, issue := range t.issues {
p := groups[issue.PatternID]
if p == nil {
p = &Pattern{ID: issue.PatternID, Signature: issue.PatternSignature, Service: issue.Service, Kind: issue.Kind}
groups[issue.PatternID] = p
}
p.IssueCount++
p.OccurrenceCount += issue.Count
p.IssueIDs = append(p.IssueIDs, issue.ID)
if issue.LastSeen > p.LastSeen {
p.LastSeen = issue.LastSeen
}
}
result := make([]Pattern, 0, len(groups))
for _, p := range groups {
sort.Strings(p.IssueIDs)
result = append(result, *p)
}
sort.Slice(result, func(i, j int) bool {
if result[i].LastSeen != result[j].LastSeen {
return result[i].LastSeen > result[j].LastSeen
}
return result[i].ID < result[j].ID
})
return result
}
// SetSimilarityFeedback records a human label for later training data.
func (t *Tracker) SetSimilarityFeedback(a, b, decision string) bool {
if a == "" || b == "" || a == b || (decision != "same" && decision != "different") {
return false
}
t.mu.Lock()
defer t.mu.Unlock()
if _, ok := t.issues[a]; !ok {
return false
}
if _, ok := t.issues[b]; !ok {
return false
}
t.feedback[feedbackKey(a, b)] = decision
return true
}
func feedbackKey(a, b string) string {
if a > b {
a, b = b, a
}
return a + "\x00" + b
}
func patternSignature(kind, service, title string, sample Sample) string {
route := sample.NormalizedRoute
status := sample.StatusClass
stack := sample.StackSignature
// The title is deliberately broad here: the exact normalized title remains
// the canonical issue identity, while this signature supports UI grouping.
title = normalizePatternTitle(title)
return strings.Join([]string{service, kind, route, status, title, stack}, "\x00")
}
func normalizeSample(sample *Sample, title string) {
sample.NormalizedMessage = NormalizeTitle(title)
sample.NormalizedRoute = NormalizeRoute(sample.HTTPRoute)
sample.StatusClass = statusClass(sample.HTTPStatus)
sample.StackSignature = firstStackFrame(sample.Stacktrace)
}
func normalizePatternTitle(title string) string {
s := NormalizeTitle(title)
s = regexp.MustCompile(`(?i)http\s+[1-5]\d{2}`).ReplaceAllString(s, "http <status>")
return s
}
func NormalizeRoute(route string) string {
route = strings.TrimSpace(strings.SplitN(route, "?", 2)[0])
if route == "" {
return ""
}
parts := strings.Split(route, "/")
for i, part := range parts {
if part == "" {
continue
}
if uuidRe.MatchString(part) || hexIDRe.MatchString(part) || numRe.MatchString(part) || prefixedIDRe.MatchString(part) {
parts[i] = "{id}"
}
}
return strings.Join(parts, "/")
}
func statusClass(status int) string {
if status >= 500 && status < 600 {
return "5xx"
}
if status >= 400 && status < 500 {
return "4xx"
}
if status >= 300 && status < 400 {
return "3xx"
}
if status > 0 {
return "2xx"
}
return ""
}
func firstStackFrame(stack string) string {
if i := strings.IndexByte(stack, '\n'); i >= 0 {
stack = stack[:i]
}
return strings.TrimSpace(stack)
}
func similarity(a, b Issue) (float64, []string) {
score := 0.0
reasons := []string{}
if a.Service == b.Service {
score += .25
}
if a.Kind == b.Kind {
score += .15
}
aSample, bSample := latestSample(a), latestSample(b)
if NormalizeRoute(aSample.HTTPRoute) != "" && NormalizeRoute(aSample.HTTPRoute) == NormalizeRoute(bSample.HTTPRoute) {
score += .20
reasons = append(reasons, "same route template")
}
if statusClass(aSample.HTTPStatus) != "" && statusClass(aSample.HTTPStatus) == statusClass(bSample.HTTPStatus) {
score += .15
reasons = append(reasons, "same HTTP status class")
}
if firstStackFrame(aSample.Stacktrace) != "" && firstStackFrame(aSample.Stacktrace) == firstStackFrame(bSample.Stacktrace) {
score += .10
reasons = append(reasons, "same top stack frame")
}
if a.NormalizedTitle != "" && a.NormalizedTitle == b.NormalizedTitle {
score += .15
reasons = append(reasons, "same normalized message")
} else if tokenSimilarity(a.NormalizedTitle, b.NormalizedTitle) >= .5 {
score += .10
reasons = append(reasons, "similar normalized message")
}
if len(reasons) == 0 {
reasons = append(reasons, "same service and event kind")
}
return score, reasons
}
func latestSample(issue Issue) Sample {
if len(issue.Samples) == 0 {
return Sample{}
}
return issue.Samples[len(issue.Samples)-1]
}
func tokenSimilarity(a, b string) float64 {
aa, bb := strings.Fields(a), strings.Fields(b)
if len(aa) == 0 || len(bb) == 0 {
return 0
}
set := map[string]bool{}
for _, v := range aa {
set[v] = true
}
other := map[string]bool{}
for _, v := range bb {
other[v] = true
}
common := 0
for v := range other {
if set[v] {
common++
}
}
return float64(common) / float64(len(set)+len(other)-common)
}
// SetStatus updates the triage state of an issue. It returns the updated
// issue, or false when the id is unknown or the status is invalid.
func (t *Tracker) SetStatus(id, status string) (Issue, bool) {
switch status {
case StatusOpen, StatusResolved, StatusIgnored:
default:
return Issue{}, false
}
t.mu.Lock()
defer t.mu.Unlock()
issue, ok := t.issues[id]
if !ok {
return Issue{}, false
}
issue.Status = status
if status == StatusOpen {
issue.ResolvedAt = 0
} else {
issue.ResolvedAt = time.Now().UnixNano()
}
issue.RegressedAt = 0
t.persistLocked(id, TriageState{Status: status, ResolvedAt: issue.ResolvedAt})
return cloneIssue(issue), true
}
// evictOldestLocked removes the issue with the smallest LastSeen. Callers
// must hold t.mu.
func (t *Tracker) evictOldestLocked() {
var oldestID string
var oldest int64
first := true
for id, issue := range t.issues {
if first || issue.LastSeen < oldest {
oldest = issue.LastSeen
oldestID = id
first = false
}
}
if oldestID != "" {
delete(t.issues, oldestID)
}
}
// List returns issues sorted by LastSeen descending (most recently seen
// first). A non-positive limit returns all issues.
func (t *Tracker) List(limit int) []Issue {
return t.ListFiltered(ListOptions{Limit: limit}).Issues
}
// ListFiltered returns a filtered, stable page of issues.
func (t *Tracker) ListFiltered(opts ListOptions) ListPage {
t.mu.Lock()
defer t.mu.Unlock()
result := make([]Issue, 0, len(t.issues))
for _, issue := range t.issues {
if !matches(issue, opts) || beforeCursor(issue, opts) {
continue
}
result = append(result, cloneIssue(issue))
}
sort.Slice(result, func(i, j int) bool {
if result[i].LastSeen != result[j].LastSeen {
return result[i].LastSeen > result[j].LastSeen
}
return result[i].ID > result[j].ID
})
limit := opts.Limit
if limit <= 0 {
limit = len(result)
}
page := ListPage{Issues: result}
if len(page.Issues) > limit {
page.Issues = page.Issues[:limit]
page.HasMore = true
}
return page
}
func matches(issue *Issue, opts ListOptions) bool {
if opts.Service != "" && issue.Service != opts.Service {
return false
}
if opts.Kind != "" && issue.Kind != opts.Kind {
return false
}
if opts.Status != "" && issue.Status != opts.Status {
return false
}
if opts.Title != "" && !strings.Contains(strings.ToLower(issue.Title), strings.ToLower(opts.Title)) {
return false
}
if opts.Since != 0 && issue.LastSeen < opts.Since {
return false
}
if opts.Until != 0 && issue.LastSeen > opts.Until {
return false
}
if opts.SourceIP != "" && !issueHasSample(issue, func(sample Sample) bool { return sample.SourceIP == opts.SourceIP }) {
return false
}
if opts.Route != "" && !issueHasSample(issue, func(sample Sample) bool { return sample.HTTPRoute == opts.Route }) {
return false
}
return true
}
func issueHasSample(issue *Issue, predicate func(Sample) bool) bool {
for _, sample := range issue.Samples {
if predicate(sample) {
return true
}
}
return false
}
func beforeCursor(issue *Issue, opts ListOptions) bool {
if opts.AfterLastSeen == 0 {
return false
}
if issue.LastSeen < opts.AfterLastSeen {
return false
}
if issue.LastSeen > opts.AfterLastSeen {
return true
}
return issue.ID >= opts.AfterID
}
// Get returns a single issue by id.
func (t *Tracker) Get(id string) (Issue, bool) {
t.mu.Lock()
defer t.mu.Unlock()
issue, ok := t.issues[id]
if !ok {
return Issue{}, false
}
return cloneIssue(issue), true
}
// cloneIssue deep-copies the slice/map fields so returned issues don't share
// backing arrays with ones still being mutated by Record.
func cloneIssue(i *Issue) Issue {
c := *i
c.AffectedServices = append([]string(nil), i.AffectedServices...)
c.Samples = append([]Sample(nil), i.Samples...)
if i.Trend != nil {
c.Trend = make(map[string]int, len(i.Trend))
for k, v := range i.Trend {
c.Trend[k] = v
}
}
return c
}
// HourBucket renders a unix-nanosecond timestamp as a UTC hour key used in
// Issue.Trend. Keys sort lexicographically, so the UI can sort them as-is.
func HourBucket(ts int64) string {
return time.Unix(0, ts).UTC().Format("2006-01-02T15:00")
}
// pruneTrendLocked drops trend buckets older than trendWindow. Callers must
// hold the tracker lock.
func pruneTrendLocked(trend map[string]int, now int64) {
if len(trend) <= 48 {
return
}
cutoff := HourBucket(time.Unix(0, now).Add(-trendWindow).UnixNano())
for k := range trend {
if k < cutoff {
delete(trend, k)
}
}
}
var (
uuidRe = regexp.MustCompile(`(?i)\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b`)
hexIDRe = regexp.MustCompile(`(?i)\b[0-9a-f]{8,}\b`)
httpStatusRe = regexp.MustCompile(`(?i)\bhttp\s+([1-5]\d{2})\b`)
numRe = regexp.MustCompile(`\b\d+\b`)
// A route segment that is a short prefix plus a long hex id (term_ab31…,
// sess-9f2c…): \b does not fire between "_" and a hex digit, so hexIDRe misses it.
prefixedIDRe = regexp.MustCompile(`(?i)^[a-z]{1,12}[_-][0-9a-f]{8,}$`)
spaceRe = regexp.MustCompile(`\s+`)
)
// NormalizeTitle strips variable content (UUIDs, hex ids, numbers, whitespace,
// casing) from an error title so that "User 123 not found" and "User 456 not
// found" fingerprint into the same issue. The raw title is kept as the
// display title of the first-seen occurrence. HTTP status codes (e.g. HTTP 404)
// are preserved so different HTTP error types don't collapse together.
func NormalizeTitle(title string) string {
s := uuidRe.ReplaceAllString(title, "<id>")
// Protect HTTP status codes from being replaced by <num>
s = httpStatusRe.ReplaceAllStringFunc(s, func(match string) string {
parts := strings.Fields(match)
if len(parts) != 2 {
return match
}
return "__HTTP_STATUS_" + encodeStatus(parts[1]) + "__"
})
s = numRe.ReplaceAllString(s, "<num>")
s = hexIDRe.ReplaceAllStringFunc(s, func(match string) string {
// Mixed hex runs (letters + digits, e.g. trace/span ids) become ids.
// Pure-digit runs were already handled by numRe above, and words
// spelled entirely from a-f (rare at this length) survive so distinct
// error messages don't collapse.
if strings.ContainsAny(match, "0123456789") {
return "<id>"
}
return match
})
s = regexp.MustCompile(`__HTTP_STATUS_([a-z]+)__`).ReplaceAllStringFunc(s, func(match string) string {
encoded := strings.TrimSuffix(strings.TrimPrefix(match, "__HTTP_STATUS_"), "__")
return "http " + decodeStatus(encoded)
})
s = strings.ReplaceAll(s, "__", "")
s = spaceRe.ReplaceAllString(s, " ")
return strings.ToLower(strings.TrimSpace(s))
}
func encodeStatus(code string) string {
var b strings.Builder
for _, digit := range code {
b.WriteByte('a' + byte(digit-'0'))
}
return b.String()
}
func decodeStatus(encoded string) string {
var b strings.Builder
for _, letter := range encoded {
b.WriteByte('0' + byte(letter-'a'))
}
return b.String()
}
// Fingerprint creates the stable opaque identifier used for canonical issue
// grouping. Normalization uses it for error rows so offline feature exports
// can count the same issue identity as the live issue tracker.
func Fingerprint(parts ...string) string {
h := fnv.New64a()
_, _ = h.Write([]byte(strings.Join(parts, "\x00")))
return fmt.Sprintf("%016x", h.Sum64())
}
func addUnique(list []string, v string) []string {
if v == "" {
return list
}
for _, existing := range list {
if existing == v {
return list
}
}
return append(list, v)
}