Derive deduplicated error issues from traces and logs (Phase 2 backend)
- Add internal/issues.Tracker: fingerprints (kind, service, title) into
deduplicated Issues with count/firstSeen/lastSeen/affected services and
bounded sample occurrences, evicting the least-recently-seen issue once
a cap is reached.
- Add internal/collector/issue_extract.go to pull issues out of ingested
OTLP data: spans with an error status (preferring exception.type/message
from an "exception" event, falling back to the span status message), and
logs at ERROR/FATAL severity (by severity number or text).
- Wire extraction into Collector.ConsumeTraces/ConsumeLogs and expose
GET /api/issues and GET /api/issues/{id}.
- Fix a pre-existing data race in the collector test's mockStore (unsynced
access to entries while addToHistory appends asynchronously), found via
`go test -race` while adding the new issue-tracking tests.
- Update docs/PLAN.md, README.md and agents_example.md for the new API.
The UI "Issues" tab itself lives in the separate frontend project that
builds this server's dist/ bundle; this commit only covers the backend.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f77b89a779
commit
0d274ff89c
10 files changed
+627
-6
No files matched your search
@@ -0,0 +1,162 @@
|
||||
// Package issues derives deduplicated error "issues" from ingested telemetry,
|
||||
// so the web UI can show a triaged list instead of raw traces and logs.
|
||||
package issues
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// maxIssues bounds memory use: once exceeded, the least-recently-seen issue
|
||||
// is evicted to make room for a newly seen distinct one.
|
||||
const maxIssues = 2000
|
||||
|
||||
// maxSamples bounds how many sample occurrences are kept per issue.
|
||||
const maxSamples = 5
|
||||
|
||||
// Sample is one occurrence of an issue.
|
||||
type Sample struct {
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
TraceID string `json:"traceId,omitempty"`
|
||||
SpanID string `json:"spanId,omitempty"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// Issue is a deduplicated group of related errors, fingerprinted by kind,
|
||||
// service and title.
|
||||
type Issue struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Kind string `json:"kind"` // "span_error" or "log_error"
|
||||
Service string `json:"service"`
|
||||
Count int `json:"count"`
|
||||
FirstSeen int64 `json:"firstSeen"`
|
||||
LastSeen int64 `json:"lastSeen"`
|
||||
AffectedServices []string `json:"affectedServices"`
|
||||
Samples []Sample `json:"samples"`
|
||||
}
|
||||
|
||||
// Tracker incrementally fingerprints and deduplicates errors as they're
|
||||
// recorded. It is safe for concurrent use.
|
||||
type Tracker struct {
|
||||
mu sync.Mutex
|
||||
issues map[string]*Issue
|
||||
}
|
||||
|
||||
// NewTracker creates an empty Tracker.
|
||||
func NewTracker() *Tracker {
|
||||
return &Tracker{issues: make(map[string]*Issue)}
|
||||
}
|
||||
|
||||
// Record adds an occurrence of an error, creating a new issue or merging into
|
||||
// an existing one with a matching (kind, service, title) fingerprint.
|
||||
func (t *Tracker) Record(kind, service, title string, sample Sample) {
|
||||
id := fingerprint(kind, service, title)
|
||||
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
issue, ok := t.issues[id]
|
||||
if !ok {
|
||||
if len(t.issues) >= maxIssues {
|
||||
t.evictOldestLocked()
|
||||
}
|
||||
issue = &Issue{
|
||||
ID: id,
|
||||
Title: title,
|
||||
Kind: kind,
|
||||
Service: service,
|
||||
FirstSeen: sample.Timestamp,
|
||||
}
|
||||
t.issues[id] = issue
|
||||
}
|
||||
|
||||
issue.Count++
|
||||
issue.LastSeen = sample.Timestamp
|
||||
issue.AffectedServices = addUnique(issue.AffectedServices, service)
|
||||
issue.Samples = append(issue.Samples, sample)
|
||||
if len(issue.Samples) > maxSamples {
|
||||
issue.Samples = issue.Samples[len(issue.Samples)-maxSamples:]
|
||||
}
|
||||
}
|
||||
|
||||
// evictOldestLocked removes the issue with the smallest LastSeen. Callers
|
||||
// must hold t.mu.
|
||||
func (t *Tracker) evictOldestLocked() {
|
||||
var oldestID string
|
||||
var oldest int64
|
||||
first := true
|
||||
for id, issue := range t.issues {
|
||||
if first || issue.LastSeen < oldest {
|
||||
oldest = issue.LastSeen
|
||||
oldestID = id
|
||||
first = false
|
||||
}
|
||||
}
|
||||
if oldestID != "" {
|
||||
delete(t.issues, oldestID)
|
||||
}
|
||||
}
|
||||
|
||||
// List returns issues sorted by LastSeen descending (most recently seen
|
||||
// first). A non-positive limit returns all issues.
|
||||
func (t *Tracker) List(limit int) []Issue {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
result := make([]Issue, 0, len(t.issues))
|
||||
for _, issue := range t.issues {
|
||||
result = append(result, cloneIssue(issue))
|
||||
}
|
||||
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
return result[i].LastSeen > result[j].LastSeen
|
||||
})
|
||||
|
||||
if limit > 0 && len(result) > limit {
|
||||
result = result[:limit]
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// Get returns a single issue by id.
|
||||
func (t *Tracker) Get(id string) (Issue, bool) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
issue, ok := t.issues[id]
|
||||
if !ok {
|
||||
return Issue{}, false
|
||||
}
|
||||
return cloneIssue(issue), true
|
||||
}
|
||||
|
||||
// cloneIssue deep-copies the slice fields so returned issues don't share
|
||||
// backing arrays with ones still being mutated by Record.
|
||||
func cloneIssue(i *Issue) Issue {
|
||||
c := *i
|
||||
c.AffectedServices = append([]string(nil), i.AffectedServices...)
|
||||
c.Samples = append([]Sample(nil), i.Samples...)
|
||||
return c
|
||||
}
|
||||
|
||||
func fingerprint(parts ...string) string {
|
||||
h := fnv.New64a()
|
||||
_, _ = h.Write([]byte(strings.Join(parts, "\x00")))
|
||||
return fmt.Sprintf("%016x", h.Sum64())
|
||||
}
|
||||
|
||||
func addUnique(list []string, v string) []string {
|
||||
if v == "" {
|
||||
return list
|
||||
}
|
||||
for _, existing := range list {
|
||||
if existing == v {
|
||||
return list
|
||||
}
|
||||
}
|
||||
return append(list, v)
|
||||
}
|
||||
Reference in new issue
Block a user