more otel

internal/collector/issue_extract.go

@@ -1,6 +1,8 @@
 package collector
 
 import (
+	"fmt"
+	"strconv"
 	"strings"
 
 	"git.tornberg.me/go-otel/internal/issues"
@@ -9,46 +11,46 @@ import (
 	"go.opentelemetry.io/collector/pdata/ptrace"
 )
 
-// maxIssueTitleLen bounds how much of a log body is kept as an issue title,
-// so a single noisy log line can't blow up issue payload size.
+// maxIssueTitleLen bounds how much of a log body or span title is kept,
+// so a single noisy error can't blow up issue payload size.
 const maxIssueTitleLen = 200
 
-// extractSpanErrors scans traces for spans with an error status and records
-// one issue occurrence per error found.
-func extractSpanErrors(tracker *issues.Tracker, td ptrace.Traces, timestamp int64) {
+// extractSpanErrors scans traces for spans with an error status or error HTTP/gRPC
+// status code, recording one issue occurrence per error found. It reports
+// whether any error was seen, so the caller can notify live clients.
+func extractSpanErrors(tracker *issues.Tracker, td ptrace.Traces, timestamp int64) bool {
+	found := false
 	resourceSpans := td.ResourceSpans()
 	for i := 0; i < resourceSpans.Len(); i++ {
 		rs := resourceSpans.At(i)
-		service := resourceServiceName(rs.Resource())
 
 		scopeSpans := rs.ScopeSpans()
 		for j := 0; j < scopeSpans.Len(); j++ {
 			spans := scopeSpans.At(j).Spans()
 			for k := 0; k < spans.Len(); k++ {
 				span := spans.At(k)
-				if span.Status().Code() != ptrace.StatusCodeError {
+				if !isSpanError(span) {
 					continue
 				}
 
+				service := spanServiceName(rs.Resource(), span)
 				title := spanErrorTitle(span)
-				tracker.Record("span_error", service, title, issues.Sample{
-					Timestamp: timestamp,
-					TraceID:   idString(span.TraceID()),
-					SpanID:    idString(span.SpanID()),
-					Message:   title,
-				})
+				found = true
+				tracker.Record("span_error", service, title, spanSample(span, rs.Resource(), timestamp, title))
 			}
 		}
 	}
+	return found
 }
 
 // extractLogErrors scans logs for error/fatal severity records and records
-// one issue occurrence per error found.
-func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) {
+// one issue occurrence per error found. It reports whether any error was
+// seen, so the caller can notify live clients.
+func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) bool {
+	found := false
 	resourceLogs := ld.ResourceLogs()
 	for i := 0; i < resourceLogs.Len(); i++ {
 		rl := resourceLogs.At(i)
-		service := resourceServiceName(rl.Resource())
 
 		scopeLogs := rl.ScopeLogs()
 		for j := 0; j < scopeLogs.Len(); j++ {
@@ -59,23 +61,223 @@ func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) {
 					continue
 				}
 
+				service := logServiceName(rl.Resource(), record)
 				title := logErrorTitle(record)
+				found = true
 				kind := "log_error"
-				var url string
+				sample := logSample(record, rl.Resource(), timestamp, title)
 				if attrString(record.Attributes(), "source") == "frontend" {
 					kind = "frontend_error"
-					url = attrString(record.Attributes(), "browser.url")
+					sample.URL = attrString(record.Attributes(), "browser.url")
 				}
-				tracker.Record(kind, service, title, issues.Sample{
-					Timestamp: timestamp,
-					TraceID:   idString(record.TraceID()),
-					SpanID:    idString(record.SpanID()),
-					Message:   title,
-					URL:       url,
-				})
+				tracker.Record(kind, service, title, sample)
 			}
 		}
 	}
+	return found
+}
+
+// spanSample builds an enriched issue sample from a span's attributes:
+// stacktrace from exception events, HTTP request context from semconv
+// attributes (old and new names), and client/release metadata.
+func spanSample(span ptrace.Span, res pcommon.Resource, timestamp int64, title string) issues.Sample {
+	sample := issues.Sample{
+		Timestamp:   timestamp,
+		TraceID:     idString(span.TraceID()),
+		SpanID:      idString(span.SpanID()),
+		Message:     title,
+		HTTPMethod:  firstAttr(span.Attributes(), "http.request.method", "http.method"),
+		HTTPRoute:   firstAttr(span.Attributes(), "http.route", "http.target", "url.path"),
+		UserAgent:   firstAttr(span.Attributes(), "user_agent.original", "http.user_agent"),
+		SourceIP:    sourceIP(span.Attributes(), res.Attributes()),
+		Release:     firstAttr(span.Attributes(), "service.release"),
+		Environment: resourceEnvironment(res),
+	}
+	if status, ok := getHTTPStatusCode(span.Attributes()); ok {
+		sample.HTTPStatus = status
+	}
+	sample.Stacktrace = spanStacktrace(span)
+	return sample
+}
+
+// logSample builds an enriched issue sample from a log record's attributes.
+func logSample(record plog.LogRecord, res pcommon.Resource, timestamp int64, title string) issues.Sample {
+	sample := issues.Sample{
+		Timestamp:   timestamp,
+		TraceID:     idString(record.TraceID()),
+		SpanID:      idString(record.SpanID()),
+		Message:     title,
+		Stacktrace:  attrString(record.Attributes(), "exception.stacktrace"),
+		HTTPMethod:  firstAttr(record.Attributes(), "http.request.method", "http.method"),
+		HTTPRoute:   firstAttr(record.Attributes(), "http.route", "http.target", "url.path"),
+		UserAgent:   firstAttr(record.Attributes(), "user_agent.original", "http.user_agent", "browser.userAgent"),
+		SourceIP:    sourceIP(record.Attributes(), res.Attributes()),
+		Release:     firstAttr(record.Attributes(), "service.release"),
+		Environment: firstAttr(record.Attributes(), "service.environment"),
+		Breadcrumbs: attrString(record.Attributes(), "breadcrumbs"),
+	}
+	if status, ok := getHTTPStatusCode(record.Attributes()); ok {
+		sample.HTTPStatus = status
+	}
+	if sample.Environment == "" {
+		sample.Environment = resourceEnvironment(res)
+	}
+	return sample
+}
+
+// sourceIP extracts common client-IP attributes without trusting arbitrary
+// request headers. Instrumentation may attach these to either the span/log or
+// its resource.
+func sourceIP(attrs, resource pcommon.Map) string {
+	ip := firstAttr(attrs,
+		"client.address",
+		"client.ip",
+		"http.client_ip",
+		"net.peer.ip",
+		"network.peer.address",
+		"http.request.header.x-real-ip",
+		"http.request.header.x-forwarded-for",
+		"http.request.header.x_real_ip",
+		"http.request.header.x_forwarded_for",
+	)
+	if ip == "" {
+		ip = firstAttr(resource,
+			"client.address",
+			"client.ip",
+			"http.client_ip",
+			"net.peer.ip",
+			"network.peer.address",
+		)
+	}
+	return strings.TrimSpace(strings.SplitN(ip, ",", 2)[0])
+}
+
+// spanStacktrace returns exception.stacktrace from the span's exception
+// events, falling back to the span status message when it looks like an
+// exception ("type: message" is how spanErrorTitle renders one).
+func spanStacktrace(span ptrace.Span) string {
+	events := span.Events()
+	for i := 0; i < events.Len(); i++ {
+		event := events.At(i)
+		if event.Name() != "exception" {
+			continue
+		}
+		if st := attrString(event.Attributes(), "exception.stacktrace"); st != "" {
+			return truncate(st)
+		}
+	}
+	return ""
+}
+
+// firstAttr returns the first present attribute among the given keys.
+func firstAttr(attrs pcommon.Map, keys ...string) string {
+	for _, key := range keys {
+		if v, ok := attrs.Get(key); ok {
+			return v.AsString()
+		}
+	}
+	return ""
+}
+
+// getHTTPStatusCode extracts an HTTP status code from attributes, supporting
+// both new (http.response.status_code) and old (http.status_code) semconv keys,
+// stored as integer or string.
+func getHTTPStatusCode(attrs pcommon.Map) (int, bool) {
+	for _, key := range []string{"http.response.status_code", "http.status_code"} {
+		if v, ok := attrs.Get(key); ok {
+			switch v.Type() {
+			case pcommon.ValueTypeInt:
+				return int(v.Int()), true
+			case pcommon.ValueTypeStr:
+				if code, err := strconv.Atoi(v.Str()); err == nil {
+					return code, true
+				}
+			default:
+				if code, err := strconv.Atoi(v.AsString()); err == nil {
+					return code, true
+				}
+			}
+		}
+	}
+	return 0, false
+}
+
+// httpStatusText maps common HTTP error status codes to their canonical reason phrase.
+func httpStatusText(code int) string {
+	switch code {
+	case 400:
+		return "Bad Request"
+	case 401:
+		return "Unauthorized"
+	case 402:
+		return "Payment Required"
+	case 403:
+		return "Forbidden"
+	case 404:
+		return "Not Found"
+	case 405:
+		return "Method Not Allowed"
+	case 406:
+		return "Not Acceptable"
+	case 408:
+		return "Request Timeout"
+	case 409:
+		return "Conflict"
+	case 410:
+		return "Gone"
+	case 415:
+		return "Unsupported Media Type"
+	case 422:
+		return "Unprocessable Entity"
+	case 429:
+		return "Too Many Requests"
+	case 500:
+		return "Internal Server Error"
+	case 501:
+		return "Not Implemented"
+	case 502:
+		return "Bad Gateway"
+	case 503:
+		return "Service Unavailable"
+	case 504:
+		return "Gateway Timeout"
+	default:
+		if code >= 400 && code < 500 {
+			return "Client Error"
+		}
+		if code >= 500 && code < 600 {
+			return "Server Error"
+		}
+		return ""
+	}
+}
+
+// isSpanError determines if a span represents an error, checking:
+// 1. Explicit StatusCodeError
+// 2. HTTP response status >= 400
+// 3. gRPC error code (> 0)
+func isSpanError(span ptrace.Span) bool {
+	if span.Status().Code() == ptrace.StatusCodeError {
+		return true
+	}
+	if status, ok := getHTTPStatusCode(span.Attributes()); ok && status >= 400 {
+		return true
+	}
+	if grpcStr := firstAttr(span.Attributes(), "rpc.grpc.status_code"); grpcStr != "" {
+		if code, err := strconv.Atoi(grpcStr); err == nil && code > 0 {
+			return true
+		}
+	}
+	return false
+}
+
+// resourceEnvironment reads deployment.environment (new semconv) or
+// deployment.environment.name from a resource.
+func resourceEnvironment(res pcommon.Resource) string {
+	if v := attrString(res.Attributes(), "deployment.environment.name"); v != "" {
+		return v
+	}
+	return attrString(res.Attributes(), "deployment.environment")
 }
 
 func isErrorLog(record plog.LogRecord) bool {
@@ -91,6 +293,7 @@ func isErrorLog(record plog.LogRecord) bool {
 }
 
 func spanErrorTitle(span ptrace.Span) string {
+	// 1. Exception events take top priority (panics, exceptions with stacktrace).
 	events := span.Events()
 	for i := 0; i < events.Len(); i++ {
 		event := events.At(i)
@@ -109,10 +312,72 @@ func spanErrorTitle(span ptrace.Span) string {
 		}
 	}
 
-	if msg := span.Status().Message(); msg != "" {
-		return span.Name() + ": " + msg
+	// 2. HTTP error titles: "HTTP 404 (Not Found): GET /path" or "HTTP 500 (Internal Server Error): /api"
+	if status, ok := getHTTPStatusCode(span.Attributes()); ok && status >= 400 {
+		statusDesc := fmt.Sprintf("HTTP %d", status)
+		if txt := httpStatusText(status); txt != "" {
+			statusDesc = fmt.Sprintf("HTTP %d (%s)", status, txt)
+		}
+
+		method := firstAttr(span.Attributes(), "http.request.method", "http.method")
+		route := firstAttr(span.Attributes(), "http.route", "http.target", "url.path")
+
+		var endpoint string
+		if method != "" && route != "" {
+			endpoint = method + " " + route
+		} else if route != "" {
+			endpoint = route
+		} else if span.Name() != "" && !strings.EqualFold(span.Name(), "http") {
+			endpoint = span.Name()
+		}
+
+		statusMsg := span.Status().Message()
+		switch {
+		case endpoint != "" && statusMsg != "" && !strings.EqualFold(statusMsg, httpStatusText(status)):
+			return truncate(statusDesc + ": " + endpoint + " (" + statusMsg + ")")
+		case endpoint != "":
+			return truncate(statusDesc + ": " + endpoint)
+		case statusMsg != "":
+			return truncate(statusDesc + ": " + statusMsg)
+		default:
+			return statusDesc
+		}
 	}
-	return span.Name() + " (error)"
+
+	// 3. gRPC error titles
+	if grpcStr := firstAttr(span.Attributes(), "rpc.grpc.status_code"); grpcStr != "" {
+		if code, err := strconv.Atoi(grpcStr); err == nil && code > 0 {
+			method := firstAttr(span.Attributes(), "rpc.method")
+			service := firstAttr(span.Attributes(), "rpc.service")
+			target := span.Name()
+			if service != "" && method != "" {
+				target = service + "/" + method
+			}
+			msg := span.Status().Message()
+			if msg != "" {
+				return truncate(fmt.Sprintf("gRPC error %d: %s (%s)", code, target, msg))
+			}
+			if target != "" {
+				return truncate(fmt.Sprintf("gRPC error %d: %s", code, target))
+			}
+			return fmt.Sprintf("gRPC error %d", code)
+		}
+	}
+
+	// 4. Status message with span name
+	if msg := span.Status().Message(); msg != "" {
+		if span.Name() != "" {
+			return truncate(span.Name() + ": " + msg)
+		}
+		return truncate(msg)
+	}
+
+	// 5. Fallback to span name
+	if span.Name() != "" {
+		return truncate(span.Name() + " (error)")
+	}
+
+	return "span error"
 }
 
 func logErrorTitle(record plog.LogRecord) string {
@@ -139,8 +404,28 @@ func truncate(s string) string {
 	return s[:maxIssueTitleLen] + "…"
 }
 
+func spanServiceName(res pcommon.Resource, span ptrace.Span) string {
+	if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
+		return name
+	}
+	if name := firstAttr(span.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
+		return name
+	}
+	return "unknown"
+}
+
+func logServiceName(res pcommon.Resource, record plog.LogRecord) string {
+	if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
+		return name
+	}
+	if name := firstAttr(record.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
+		return name
+	}
+	return "unknown"
+}
+
 func resourceServiceName(res pcommon.Resource) string {
-	if name := attrString(res.Attributes(), "service.name"); name != "" {
+	if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
 		return name
 	}
 	return "unknown"