package collector import ( "fmt" "strconv" "strings" "git.tornberg.me/go-otel/internal/httpattr" "git.tornberg.me/go-otel/internal/issues" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" "go.opentelemetry.io/collector/pdata/ptrace" ) // maxIssueTitleLen bounds how much of a log body or span title is kept, // so a single noisy error can't blow up issue payload size. const maxIssueTitleLen = 200 // extractSpanErrors scans traces for spans with an error status or error HTTP/gRPC // status code, recording one issue occurrence per error found. It reports // whether any error was seen, so the caller can notify live clients. func extractSpanErrors(tracker *issues.Tracker, td ptrace.Traces, timestamp int64) bool { found := false resourceSpans := td.ResourceSpans() for i := 0; i < resourceSpans.Len(); i++ { rs := resourceSpans.At(i) scopeSpans := rs.ScopeSpans() for j := 0; j < scopeSpans.Len(); j++ { spans := scopeSpans.At(j).Spans() for k := 0; k < spans.Len(); k++ { span := spans.At(k) if !isSpanError(span) { continue } service := spanServiceName(rs.Resource(), span) title := spanErrorTitle(span) found = true tracker.Record("span_error", service, title, spanSample(span, rs.Resource(), timestamp, title)) } } } return found } func recordSuspiciousTraceIPs(recorder HoneypotRecorder, td ptrace.Traces) { if recorder == nil { return } resourceSpans := td.ResourceSpans() for i := 0; i < resourceSpans.Len(); i++ { rs := resourceSpans.At(i) scopeSpans := rs.ScopeSpans() for j := 0; j < scopeSpans.Len(); j++ { spans := scopeSpans.At(j).Spans() for k := 0; k < spans.Len(); k++ { span := spans.At(k) _ = recorder.RecordTelemetry(sourceIP(span.Attributes(), rs.Resource().Attributes()), httpRoute(span.Attributes())) } } } } func recordSuspiciousLogIPs(recorder HoneypotRecorder, ld plog.Logs) { if recorder == nil { return } resourceLogs := ld.ResourceLogs() for i := 0; i < resourceLogs.Len(); i++ { rl := resourceLogs.At(i) scopeLogs := rl.ScopeLogs() for j := 0; j < scopeLogs.Len(); j++ { records := scopeLogs.At(j).LogRecords() for k := 0; k < records.Len(); k++ { record := records.At(k) _ = recorder.RecordTelemetry(sourceIP(record.Attributes(), rl.Resource().Attributes()), httpRoute(record.Attributes())) } } } } // extractLogErrors scans logs for error/fatal severity records and records // one issue occurrence per error found. It reports whether any error was // seen, so the caller can notify live clients. func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) bool { found := false resourceLogs := ld.ResourceLogs() for i := 0; i < resourceLogs.Len(); i++ { rl := resourceLogs.At(i) scopeLogs := rl.ScopeLogs() for j := 0; j < scopeLogs.Len(); j++ { records := scopeLogs.At(j).LogRecords() for k := 0; k < records.Len(); k++ { record := records.At(k) if !isErrorLog(record) { continue } service := logServiceName(rl.Resource(), record) title := logErrorTitle(record) found = true kind := "log_error" sample := logSample(record, rl.Resource(), timestamp, title) if attrString(record.Attributes(), "source") == "frontend" { kind = "frontend_error" sample.URL = attrString(record.Attributes(), "browser.url") } tracker.Record(kind, service, title, sample) } } } return found } // spanSample builds an enriched issue sample from a span's attributes: // stacktrace from exception events, HTTP request context from semconv // attributes (old and new names), and client/release metadata. func spanSample(span ptrace.Span, res pcommon.Resource, timestamp int64, title string) issues.Sample { sample := issues.Sample{ Timestamp: timestamp, TraceID: idString(span.TraceID()), SpanID: idString(span.SpanID()), Message: title, HTTPMethod: firstAttr(span.Attributes(), "http.request.method", "http.method"), HTTPRoute: httpRoute(span.Attributes()), UserAgent: firstAttr(span.Attributes(), "user_agent.original", "http.user_agent"), SourceIP: sourceIP(span.Attributes(), res.Attributes()), Release: firstAttr(span.Attributes(), "service.release"), Environment: resourceEnvironment(res), } if status, ok := getHTTPStatusCode(span.Attributes()); ok { sample.HTTPStatus = status } sample.Stacktrace = spanStacktrace(span) return sample } // logSample builds an enriched issue sample from a log record's attributes. func logSample(record plog.LogRecord, res pcommon.Resource, timestamp int64, title string) issues.Sample { sample := issues.Sample{ Timestamp: timestamp, TraceID: idString(record.TraceID()), SpanID: idString(record.SpanID()), Message: title, Stacktrace: attrString(record.Attributes(), "exception.stacktrace"), HTTPMethod: firstAttr(record.Attributes(), "http.request.method", "http.method"), HTTPRoute: httpRoute(record.Attributes()), UserAgent: firstAttr(record.Attributes(), "user_agent.original", "http.user_agent", "browser.userAgent"), SourceIP: sourceIP(record.Attributes(), res.Attributes()), Release: firstAttr(record.Attributes(), "service.release"), Environment: firstAttr(record.Attributes(), "service.environment"), Breadcrumbs: attrString(record.Attributes(), "breadcrumbs"), } if status, ok := getHTTPStatusCode(record.Attributes()); ok { sample.HTTPStatus = status } if sample.Environment == "" { sample.Environment = resourceEnvironment(res) } return sample } // sourceIP extracts common client-IP attributes without trusting arbitrary // request headers. Instrumentation may attach these to either the span/log or // its resource. func sourceIP(attrs, resource pcommon.Map) string { ip := firstAttr(attrs, "http.request.header.x-real-ip", "http.request.header.x_real_ip", "http.request.header.x-forwarded-for", "http.request.header.x_forwarded_for", "client.address", "client.ip", "http.client_ip", "network.peer.address", "net.peer.ip", ) if ip == "" { ip = firstAttr(resource, "http.request.header.x-real-ip", "http.request.header.x_real_ip", "http.request.header.x-forwarded-for", "http.request.header.x_forwarded_for", "client.address", "client.ip", "http.client_ip", "network.peer.address", "net.peer.ip", ) } return strings.TrimSpace(strings.SplitN(ip, ",", 2)[0]) } // spanStacktrace returns exception.stacktrace from the span's exception // events, falling back to the span status message when it looks like an // exception ("type: message" is how spanErrorTitle renders one). func spanStacktrace(span ptrace.Span) string { events := span.Events() for i := 0; i < events.Len(); i++ { event := events.At(i) if event.Name() != "exception" { continue } if st := attrString(event.Attributes(), "exception.stacktrace"); st != "" { return truncate(st) } } return "" } // firstAttr returns the first present attribute among the given keys. func firstAttr(attrs pcommon.Map, keys ...string) string { for _, key := range keys { if v, ok := attrs.Get(key); ok { return v.AsString() } } return "" } // httpRoute prefers the concrete URL path when instrumentation reports "/" // as a generic router mount. This keeps API endpoints distinct in issue // fingerprints and avoids collapsing unrelated requests into one root route. func httpRoute(attrs pcommon.Map) string { ep := httpattr.Extract(func(keys ...string) string { return firstAttr(attrs, keys...) }) if ep.Route != "" { return ep.Route } return ep.Path } // getHTTPStatusCode extracts an HTTP status code from attributes, supporting // both new (http.response.status_code) and old (http.status_code) semconv keys, // stored as integer or string. func getHTTPStatusCode(attrs pcommon.Map) (int, bool) { for _, key := range []string{"http.response.status_code", "http.status_code"} { if v, ok := attrs.Get(key); ok { switch v.Type() { case pcommon.ValueTypeInt: return int(v.Int()), true case pcommon.ValueTypeStr: if code, err := strconv.Atoi(v.Str()); err == nil { return code, true } default: if code, err := strconv.Atoi(v.AsString()); err == nil { return code, true } } } } return 0, false } // httpStatusText maps common HTTP error status codes to their canonical reason phrase. func httpStatusText(code int) string { switch code { case 400: return "Bad Request" case 401: return "Unauthorized" case 402: return "Payment Required" case 403: return "Forbidden" case 404: return "Not Found" case 405: return "Method Not Allowed" case 406: return "Not Acceptable" case 408: return "Request Timeout" case 409: return "Conflict" case 410: return "Gone" case 415: return "Unsupported Media Type" case 422: return "Unprocessable Entity" case 429: return "Too Many Requests" case 500: return "Internal Server Error" case 501: return "Not Implemented" case 502: return "Bad Gateway" case 503: return "Service Unavailable" case 504: return "Gateway Timeout" default: if code >= 400 && code < 500 { return "Client Error" } if code >= 500 && code < 600 { return "Server Error" } return "" } } // isSpanError determines if a span represents an error, checking: // 1. Explicit StatusCodeError // 2. HTTP response status >= 400 // 3. gRPC error code (> 0) func isSpanError(span ptrace.Span) bool { if span.Status().Code() == ptrace.StatusCodeError { return true } if status, ok := getHTTPStatusCode(span.Attributes()); ok && status >= 400 { return true } if grpcStr := firstAttr(span.Attributes(), "rpc.grpc.status_code"); grpcStr != "" { if code, err := strconv.Atoi(grpcStr); err == nil && code > 0 { return true } } return false } // resourceEnvironment reads deployment.environment (new semconv) or // deployment.environment.name from a resource. func resourceEnvironment(res pcommon.Resource) string { if v := attrString(res.Attributes(), "deployment.environment.name"); v != "" { return v } return attrString(res.Attributes(), "deployment.environment") } func isErrorLog(record plog.LogRecord) bool { if record.SeverityNumber() >= plog.SeverityNumberError { return true } switch strings.ToUpper(strings.TrimSpace(record.SeverityText())) { case "ERROR", "FATAL", "CRITICAL": return true default: return false } } func spanErrorTitle(span ptrace.Span) string { // 1. Exception events take top priority (panics, exceptions with stacktrace). events := span.Events() for i := 0; i < events.Len(); i++ { event := events.At(i) if event.Name() != "exception" { continue } excType := attrString(event.Attributes(), "exception.type") excMsg := attrString(event.Attributes(), "exception.message") switch { case excType != "" && excMsg != "": return excType + ": " + excMsg case excMsg != "": return excMsg case excType != "": return excType } } // 2. HTTP error titles: "HTTP 404 (Not Found): GET /path" or "HTTP 500 (Internal Server Error): /api" if status, ok := getHTTPStatusCode(span.Attributes()); ok && status >= 400 { statusDesc := fmt.Sprintf("HTTP %d", status) if txt := httpStatusText(status); txt != "" { statusDesc = fmt.Sprintf("HTTP %d (%s)", status, txt) } method := firstAttr(span.Attributes(), "http.request.method", "http.method") route := httpRoute(span.Attributes()) var endpoint string if method != "" && route != "" { endpoint = method + " " + route } else if route != "" { endpoint = route } else if span.Name() != "" && !strings.EqualFold(span.Name(), "http") { endpoint = span.Name() } statusMsg := span.Status().Message() switch { case endpoint != "" && statusMsg != "" && !strings.EqualFold(statusMsg, httpStatusText(status)): return truncate(statusDesc + ": " + endpoint + " (" + statusMsg + ")") case endpoint != "": return truncate(statusDesc + ": " + endpoint) case statusMsg != "": return truncate(statusDesc + ": " + statusMsg) default: return statusDesc } } // 3. gRPC error titles if grpcStr := firstAttr(span.Attributes(), "rpc.grpc.status_code"); grpcStr != "" { if code, err := strconv.Atoi(grpcStr); err == nil && code > 0 { method := firstAttr(span.Attributes(), "rpc.method") service := firstAttr(span.Attributes(), "rpc.service") target := span.Name() if service != "" && method != "" { target = service + "/" + method } msg := span.Status().Message() if msg != "" { return truncate(fmt.Sprintf("gRPC error %d: %s (%s)", code, target, msg)) } if target != "" { return truncate(fmt.Sprintf("gRPC error %d: %s", code, target)) } return fmt.Sprintf("gRPC error %d", code) } } // 4. Status message with span name if msg := span.Status().Message(); msg != "" { if span.Name() != "" { return truncate(span.Name() + ": " + msg) } return truncate(msg) } // 5. Fallback to span name if span.Name() != "" { return truncate(span.Name() + " (error)") } return "span error" } func logErrorTitle(record plog.LogRecord) string { excType := attrString(record.Attributes(), "exception.type") excMsg := attrString(record.Attributes(), "exception.message") switch { case excType != "" && excMsg != "": return truncate(excType + ": " + excMsg) case excMsg != "": return truncate(excMsg) } body := strings.TrimSpace(record.Body().AsString()) if body == "" { return "log error" } return truncate(body) } func truncate(s string) string { if len(s) <= maxIssueTitleLen { return s } return s[:maxIssueTitleLen] + "…" } func spanServiceName(res pcommon.Resource, span ptrace.Span) string { if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" { return name } if name := firstAttr(span.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" { return name } return "unknown" } func logServiceName(res pcommon.Resource, record plog.LogRecord) string { if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" { return name } if name := firstAttr(record.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" { return name } return "unknown" } func resourceServiceName(res pcommon.Resource) string { if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" { return name } return "unknown" } // attrString safely reads a string attribute, since a Value obtained from a // missing Map key is not valid to call methods on. func attrString(attrs pcommon.Map, key string) string { if v, ok := attrs.Get(key); ok { return v.AsString() } return "" } type traceOrSpanID interface { IsEmpty() bool String() string } func idString(id traceOrSpanID) string { if id.IsEmpty() { return "" } return id.String() }