- Add internal/issues.Tracker: fingerprints (kind, service, title) into
deduplicated Issues with count/firstSeen/lastSeen/affected services and
bounded sample occurrences, evicting the least-recently-seen issue once
a cap is reached.
- Add internal/collector/issue_extract.go to pull issues out of ingested
OTLP data: spans with an error status (preferring exception.type/message
from an "exception" event, falling back to the span status message), and
logs at ERROR/FATAL severity (by severity number or text).
- Wire extraction into Collector.ConsumeTraces/ConsumeLogs and expose
GET /api/issues and GET /api/issues/{id}.
- Fix a pre-existing data race in the collector test's mockStore (unsynced
access to entries while addToHistory appends asynchronously), found via
`go test -race` while adding the new issue-tracking tests.
- Update docs/PLAN.md, README.md and agents_example.md for the new API.
The UI "Issues" tab itself lives in the separate frontend project that
builds this server's dist/ bundle; this commit only covers the backend.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
162 lines
4.0 KiB
Go
162 lines
4.0 KiB
Go
package collector
|
|
|
|
import (
|
|
"strings"
|
|
|
|
"git.tornberg.me/go-otel/internal/issues"
|
|
"go.opentelemetry.io/collector/pdata/pcommon"
|
|
"go.opentelemetry.io/collector/pdata/plog"
|
|
"go.opentelemetry.io/collector/pdata/ptrace"
|
|
)
|
|
|
|
// maxIssueTitleLen bounds how much of a log body is kept as an issue title,
|
|
// so a single noisy log line can't blow up issue payload size.
|
|
const maxIssueTitleLen = 200
|
|
|
|
// extractSpanErrors scans traces for spans with an error status and records
|
|
// one issue occurrence per error found.
|
|
func extractSpanErrors(tracker *issues.Tracker, td ptrace.Traces, timestamp int64) {
|
|
resourceSpans := td.ResourceSpans()
|
|
for i := 0; i < resourceSpans.Len(); i++ {
|
|
rs := resourceSpans.At(i)
|
|
service := resourceServiceName(rs.Resource())
|
|
|
|
scopeSpans := rs.ScopeSpans()
|
|
for j := 0; j < scopeSpans.Len(); j++ {
|
|
spans := scopeSpans.At(j).Spans()
|
|
for k := 0; k < spans.Len(); k++ {
|
|
span := spans.At(k)
|
|
if span.Status().Code() != ptrace.StatusCodeError {
|
|
continue
|
|
}
|
|
|
|
title := spanErrorTitle(span)
|
|
tracker.Record("span_error", service, title, issues.Sample{
|
|
Timestamp: timestamp,
|
|
TraceID: idString(span.TraceID()),
|
|
SpanID: idString(span.SpanID()),
|
|
Message: title,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// extractLogErrors scans logs for error/fatal severity records and records
|
|
// one issue occurrence per error found.
|
|
func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) {
|
|
resourceLogs := ld.ResourceLogs()
|
|
for i := 0; i < resourceLogs.Len(); i++ {
|
|
rl := resourceLogs.At(i)
|
|
service := resourceServiceName(rl.Resource())
|
|
|
|
scopeLogs := rl.ScopeLogs()
|
|
for j := 0; j < scopeLogs.Len(); j++ {
|
|
records := scopeLogs.At(j).LogRecords()
|
|
for k := 0; k < records.Len(); k++ {
|
|
record := records.At(k)
|
|
if !isErrorLog(record) {
|
|
continue
|
|
}
|
|
|
|
title := logErrorTitle(record)
|
|
tracker.Record("log_error", service, title, issues.Sample{
|
|
Timestamp: timestamp,
|
|
TraceID: idString(record.TraceID()),
|
|
SpanID: idString(record.SpanID()),
|
|
Message: title,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func isErrorLog(record plog.LogRecord) bool {
|
|
if record.SeverityNumber() >= plog.SeverityNumberError {
|
|
return true
|
|
}
|
|
switch strings.ToUpper(strings.TrimSpace(record.SeverityText())) {
|
|
case "ERROR", "FATAL", "CRITICAL":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func spanErrorTitle(span ptrace.Span) string {
|
|
events := span.Events()
|
|
for i := 0; i < events.Len(); i++ {
|
|
event := events.At(i)
|
|
if event.Name() != "exception" {
|
|
continue
|
|
}
|
|
excType := attrString(event.Attributes(), "exception.type")
|
|
excMsg := attrString(event.Attributes(), "exception.message")
|
|
switch {
|
|
case excType != "" && excMsg != "":
|
|
return excType + ": " + excMsg
|
|
case excMsg != "":
|
|
return excMsg
|
|
case excType != "":
|
|
return excType
|
|
}
|
|
}
|
|
|
|
if msg := span.Status().Message(); msg != "" {
|
|
return span.Name() + ": " + msg
|
|
}
|
|
return span.Name() + " (error)"
|
|
}
|
|
|
|
func logErrorTitle(record plog.LogRecord) string {
|
|
excType := attrString(record.Attributes(), "exception.type")
|
|
excMsg := attrString(record.Attributes(), "exception.message")
|
|
switch {
|
|
case excType != "" && excMsg != "":
|
|
return truncate(excType + ": " + excMsg)
|
|
case excMsg != "":
|
|
return truncate(excMsg)
|
|
}
|
|
|
|
body := strings.TrimSpace(record.Body().AsString())
|
|
if body == "" {
|
|
return "log error"
|
|
}
|
|
return truncate(body)
|
|
}
|
|
|
|
func truncate(s string) string {
|
|
if len(s) <= maxIssueTitleLen {
|
|
return s
|
|
}
|
|
return s[:maxIssueTitleLen] + "…"
|
|
}
|
|
|
|
func resourceServiceName(res pcommon.Resource) string {
|
|
if name := attrString(res.Attributes(), "service.name"); name != "" {
|
|
return name
|
|
}
|
|
return "unknown"
|
|
}
|
|
|
|
// attrString safely reads a string attribute, since a Value obtained from a
|
|
// missing Map key is not valid to call methods on.
|
|
func attrString(attrs pcommon.Map, key string) string {
|
|
if v, ok := attrs.Get(key); ok {
|
|
return v.AsString()
|
|
}
|
|
return ""
|
|
}
|
|
|
|
type traceOrSpanID interface {
|
|
IsEmpty() bool
|
|
String() string
|
|
}
|
|
|
|
func idString(id traceOrSpanID) string {
|
|
if id.IsEmpty() {
|
|
return ""
|
|
}
|
|
return id.String()
|
|
}
|