506 lines
15 KiB
Go
506 lines
15 KiB
Go
package collector
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"git.tornberg.me/go-otel/internal/httpattr"
|
|
"git.tornberg.me/go-otel/internal/issues"
|
|
"go.opentelemetry.io/collector/pdata/pcommon"
|
|
"go.opentelemetry.io/collector/pdata/plog"
|
|
"go.opentelemetry.io/collector/pdata/ptrace"
|
|
)
|
|
|
|
// maxIssueTitleLen bounds how much of a log body or span title is kept,
|
|
// so a single noisy error can't blow up issue payload size.
|
|
const maxIssueTitleLen = 200
|
|
|
|
// extractSpanErrors scans traces for spans with an error status or error HTTP/gRPC
|
|
// status code, recording one issue occurrence per error found. It reports
|
|
// whether any error was seen, so the caller can notify live clients.
|
|
func extractSpanErrors(tracker *issues.Tracker, td ptrace.Traces, timestamp int64) bool {
|
|
found := false
|
|
resourceSpans := td.ResourceSpans()
|
|
for i := 0; i < resourceSpans.Len(); i++ {
|
|
rs := resourceSpans.At(i)
|
|
|
|
scopeSpans := rs.ScopeSpans()
|
|
for j := 0; j < scopeSpans.Len(); j++ {
|
|
spans := scopeSpans.At(j).Spans()
|
|
for k := 0; k < spans.Len(); k++ {
|
|
span := spans.At(k)
|
|
if !isSpanError(span) {
|
|
continue
|
|
}
|
|
|
|
service := spanServiceName(rs.Resource(), span)
|
|
title := spanErrorTitle(span)
|
|
found = true
|
|
tracker.Record("span_error", service, title, spanSample(span, rs.Resource(), timestamp, title))
|
|
}
|
|
}
|
|
}
|
|
return found
|
|
}
|
|
|
|
func recordSuspiciousTraceIPs(recorder HoneypotRecorder, td ptrace.Traces) {
|
|
if recorder == nil {
|
|
return
|
|
}
|
|
resourceSpans := td.ResourceSpans()
|
|
for i := 0; i < resourceSpans.Len(); i++ {
|
|
rs := resourceSpans.At(i)
|
|
scopeSpans := rs.ScopeSpans()
|
|
for j := 0; j < scopeSpans.Len(); j++ {
|
|
spans := scopeSpans.At(j).Spans()
|
|
for k := 0; k < spans.Len(); k++ {
|
|
span := spans.At(k)
|
|
_ = recorder.RecordTelemetry(sourceIP(span.Attributes(), rs.Resource().Attributes()), httpRoute(span.Attributes()))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func recordSuspiciousLogIPs(recorder HoneypotRecorder, ld plog.Logs) {
|
|
if recorder == nil {
|
|
return
|
|
}
|
|
resourceLogs := ld.ResourceLogs()
|
|
for i := 0; i < resourceLogs.Len(); i++ {
|
|
rl := resourceLogs.At(i)
|
|
scopeLogs := rl.ScopeLogs()
|
|
for j := 0; j < scopeLogs.Len(); j++ {
|
|
records := scopeLogs.At(j).LogRecords()
|
|
for k := 0; k < records.Len(); k++ {
|
|
record := records.At(k)
|
|
_ = recorder.RecordTelemetry(sourceIP(record.Attributes(), rl.Resource().Attributes()), httpRoute(record.Attributes()))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// extractLogErrors scans logs for error/fatal severity records and records
|
|
// one issue occurrence per error found. It reports whether any error was
|
|
// seen, so the caller can notify live clients.
|
|
func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) bool {
|
|
found := false
|
|
resourceLogs := ld.ResourceLogs()
|
|
for i := 0; i < resourceLogs.Len(); i++ {
|
|
rl := resourceLogs.At(i)
|
|
|
|
scopeLogs := rl.ScopeLogs()
|
|
for j := 0; j < scopeLogs.Len(); j++ {
|
|
records := scopeLogs.At(j).LogRecords()
|
|
for k := 0; k < records.Len(); k++ {
|
|
record := records.At(k)
|
|
if !isErrorLog(record) {
|
|
continue
|
|
}
|
|
|
|
service := logServiceName(rl.Resource(), record)
|
|
title := logErrorTitle(record)
|
|
found = true
|
|
kind := "log_error"
|
|
sample := logSample(record, rl.Resource(), timestamp, title)
|
|
if attrString(record.Attributes(), "source") == "frontend" {
|
|
kind = "frontend_error"
|
|
sample.URL = attrString(record.Attributes(), "browser.url")
|
|
}
|
|
tracker.Record(kind, service, title, sample)
|
|
}
|
|
}
|
|
}
|
|
return found
|
|
}
|
|
|
|
// spanSample builds an enriched issue sample from a span's attributes:
|
|
// stacktrace from exception events, HTTP request context from semconv
|
|
// attributes (old and new names), and client/release metadata.
|
|
func spanSample(span ptrace.Span, res pcommon.Resource, timestamp int64, title string) issues.Sample {
|
|
sample := issues.Sample{
|
|
Timestamp: timestamp,
|
|
TraceID: idString(span.TraceID()),
|
|
SpanID: idString(span.SpanID()),
|
|
Message: title,
|
|
HTTPMethod: firstAttr(span.Attributes(), "http.request.method", "http.method"),
|
|
HTTPRoute: httpRoute(span.Attributes()),
|
|
UserAgent: firstAttr(span.Attributes(), "user_agent.original", "http.user_agent"),
|
|
SourceIP: sourceIP(span.Attributes(), res.Attributes()),
|
|
Release: firstAttr(span.Attributes(), "service.release"),
|
|
Environment: resourceEnvironment(res),
|
|
}
|
|
if status, ok := getHTTPStatusCode(span.Attributes()); ok {
|
|
sample.HTTPStatus = status
|
|
}
|
|
sample.Stacktrace = spanStacktrace(span)
|
|
return sample
|
|
}
|
|
|
|
// logSample builds an enriched issue sample from a log record's attributes.
|
|
func logSample(record plog.LogRecord, res pcommon.Resource, timestamp int64, title string) issues.Sample {
|
|
sample := issues.Sample{
|
|
Timestamp: timestamp,
|
|
TraceID: idString(record.TraceID()),
|
|
SpanID: idString(record.SpanID()),
|
|
Message: title,
|
|
Stacktrace: attrString(record.Attributes(), "exception.stacktrace"),
|
|
HTTPMethod: firstAttr(record.Attributes(), "http.request.method", "http.method"),
|
|
HTTPRoute: httpRoute(record.Attributes()),
|
|
UserAgent: firstAttr(record.Attributes(), "user_agent.original", "http.user_agent", "browser.userAgent"),
|
|
SourceIP: sourceIP(record.Attributes(), res.Attributes()),
|
|
Release: firstAttr(record.Attributes(), "service.release"),
|
|
Environment: firstAttr(record.Attributes(), "service.environment"),
|
|
Breadcrumbs: attrString(record.Attributes(), "breadcrumbs"),
|
|
}
|
|
if status, ok := getHTTPStatusCode(record.Attributes()); ok {
|
|
sample.HTTPStatus = status
|
|
}
|
|
if sample.Environment == "" {
|
|
sample.Environment = resourceEnvironment(res)
|
|
}
|
|
return sample
|
|
}
|
|
|
|
// sourceIP extracts common client-IP attributes without trusting arbitrary
|
|
// request headers. Instrumentation may attach these to either the span/log or
|
|
// its resource.
|
|
func sourceIP(attrs, resource pcommon.Map) string {
|
|
ip := firstAttr(attrs,
|
|
"http.request.header.x-real-ip",
|
|
"http.request.header.x_real_ip",
|
|
"http.request.header.x-forwarded-for",
|
|
"http.request.header.x_forwarded_for",
|
|
"client.address",
|
|
"client.ip",
|
|
"http.client_ip",
|
|
"network.peer.address",
|
|
"net.peer.ip",
|
|
)
|
|
if ip == "" {
|
|
ip = firstAttr(resource,
|
|
"http.request.header.x-real-ip",
|
|
"http.request.header.x_real_ip",
|
|
"http.request.header.x-forwarded-for",
|
|
"http.request.header.x_forwarded_for",
|
|
"client.address",
|
|
"client.ip",
|
|
"http.client_ip",
|
|
"network.peer.address",
|
|
"net.peer.ip",
|
|
)
|
|
}
|
|
return strings.TrimSpace(strings.SplitN(ip, ",", 2)[0])
|
|
}
|
|
|
|
// spanStacktrace returns exception.stacktrace from the span's exception
|
|
// events, falling back to the span status message when it looks like an
|
|
// exception ("type: message" is how spanErrorTitle renders one).
|
|
func spanStacktrace(span ptrace.Span) string {
|
|
events := span.Events()
|
|
for i := 0; i < events.Len(); i++ {
|
|
event := events.At(i)
|
|
if event.Name() != "exception" {
|
|
continue
|
|
}
|
|
if st := attrString(event.Attributes(), "exception.stacktrace"); st != "" {
|
|
return truncate(st)
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// firstAttr returns the first present attribute among the given keys.
|
|
func firstAttr(attrs pcommon.Map, keys ...string) string {
|
|
for _, key := range keys {
|
|
if v, ok := attrs.Get(key); ok {
|
|
return v.AsString()
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// httpRoute prefers the concrete URL path when instrumentation reports "/"
|
|
// as a generic router mount. This keeps API endpoints distinct in issue
|
|
// fingerprints and avoids collapsing unrelated requests into one root route.
|
|
func httpRoute(attrs pcommon.Map) string {
|
|
ep := httpattr.Extract(func(keys ...string) string { return firstAttr(attrs, keys...) })
|
|
if ep.Route != "" {
|
|
return ep.Route
|
|
}
|
|
return ep.Path
|
|
}
|
|
|
|
// getHTTPStatusCode extracts an HTTP status code from attributes, supporting
|
|
// both new (http.response.status_code) and old (http.status_code) semconv keys,
|
|
// stored as integer or string.
|
|
func getHTTPStatusCode(attrs pcommon.Map) (int, bool) {
|
|
for _, key := range []string{"http.response.status_code", "http.status_code"} {
|
|
if v, ok := attrs.Get(key); ok {
|
|
switch v.Type() {
|
|
case pcommon.ValueTypeInt:
|
|
return int(v.Int()), true
|
|
case pcommon.ValueTypeStr:
|
|
if code, err := strconv.Atoi(v.Str()); err == nil {
|
|
return code, true
|
|
}
|
|
default:
|
|
if code, err := strconv.Atoi(v.AsString()); err == nil {
|
|
return code, true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
// httpStatusText maps common HTTP error status codes to their canonical reason phrase.
|
|
func httpStatusText(code int) string {
|
|
switch code {
|
|
case 400:
|
|
return "Bad Request"
|
|
case 401:
|
|
return "Unauthorized"
|
|
case 402:
|
|
return "Payment Required"
|
|
case 403:
|
|
return "Forbidden"
|
|
case 404:
|
|
return "Not Found"
|
|
case 405:
|
|
return "Method Not Allowed"
|
|
case 406:
|
|
return "Not Acceptable"
|
|
case 408:
|
|
return "Request Timeout"
|
|
case 409:
|
|
return "Conflict"
|
|
case 410:
|
|
return "Gone"
|
|
case 415:
|
|
return "Unsupported Media Type"
|
|
case 422:
|
|
return "Unprocessable Entity"
|
|
case 429:
|
|
return "Too Many Requests"
|
|
case 500:
|
|
return "Internal Server Error"
|
|
case 501:
|
|
return "Not Implemented"
|
|
case 502:
|
|
return "Bad Gateway"
|
|
case 503:
|
|
return "Service Unavailable"
|
|
case 504:
|
|
return "Gateway Timeout"
|
|
default:
|
|
if code >= 400 && code < 500 {
|
|
return "Client Error"
|
|
}
|
|
if code >= 500 && code < 600 {
|
|
return "Server Error"
|
|
}
|
|
return ""
|
|
}
|
|
}
|
|
|
|
// isSpanError determines if a span represents an error, checking:
|
|
// 1. Explicit StatusCodeError
|
|
// 2. HTTP response status >= 400
|
|
// 3. gRPC error code (> 0)
|
|
func isSpanError(span ptrace.Span) bool {
|
|
if span.Status().Code() == ptrace.StatusCodeError {
|
|
return true
|
|
}
|
|
if status, ok := getHTTPStatusCode(span.Attributes()); ok && status >= 400 {
|
|
return true
|
|
}
|
|
if grpcStr := firstAttr(span.Attributes(), "rpc.grpc.status_code"); grpcStr != "" {
|
|
if code, err := strconv.Atoi(grpcStr); err == nil && code > 0 {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// resourceEnvironment reads deployment.environment (new semconv) or
|
|
// deployment.environment.name from a resource.
|
|
func resourceEnvironment(res pcommon.Resource) string {
|
|
if v := attrString(res.Attributes(), "deployment.environment.name"); v != "" {
|
|
return v
|
|
}
|
|
return attrString(res.Attributes(), "deployment.environment")
|
|
}
|
|
|
|
func isErrorLog(record plog.LogRecord) bool {
|
|
if record.SeverityNumber() >= plog.SeverityNumberError {
|
|
return true
|
|
}
|
|
switch strings.ToUpper(strings.TrimSpace(record.SeverityText())) {
|
|
case "ERROR", "FATAL", "CRITICAL":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func spanErrorTitle(span ptrace.Span) string {
|
|
// 1. Exception events take top priority (panics, exceptions with stacktrace).
|
|
events := span.Events()
|
|
for i := 0; i < events.Len(); i++ {
|
|
event := events.At(i)
|
|
if event.Name() != "exception" {
|
|
continue
|
|
}
|
|
excType := attrString(event.Attributes(), "exception.type")
|
|
excMsg := attrString(event.Attributes(), "exception.message")
|
|
switch {
|
|
case excType != "" && excMsg != "":
|
|
return excType + ": " + excMsg
|
|
case excMsg != "":
|
|
return excMsg
|
|
case excType != "":
|
|
return excType
|
|
}
|
|
}
|
|
|
|
// 2. HTTP error titles: "HTTP 404 (Not Found): GET /path" or "HTTP 500 (Internal Server Error): /api"
|
|
if status, ok := getHTTPStatusCode(span.Attributes()); ok && status >= 400 {
|
|
statusDesc := fmt.Sprintf("HTTP %d", status)
|
|
if txt := httpStatusText(status); txt != "" {
|
|
statusDesc = fmt.Sprintf("HTTP %d (%s)", status, txt)
|
|
}
|
|
|
|
method := firstAttr(span.Attributes(), "http.request.method", "http.method")
|
|
route := httpRoute(span.Attributes())
|
|
|
|
var endpoint string
|
|
if method != "" && route != "" {
|
|
endpoint = method + " " + route
|
|
} else if route != "" {
|
|
endpoint = route
|
|
} else if span.Name() != "" && !strings.EqualFold(span.Name(), "http") {
|
|
endpoint = span.Name()
|
|
}
|
|
|
|
statusMsg := span.Status().Message()
|
|
switch {
|
|
case endpoint != "" && statusMsg != "" && !strings.EqualFold(statusMsg, httpStatusText(status)):
|
|
return truncate(statusDesc + ": " + endpoint + " (" + statusMsg + ")")
|
|
case endpoint != "":
|
|
return truncate(statusDesc + ": " + endpoint)
|
|
case statusMsg != "":
|
|
return truncate(statusDesc + ": " + statusMsg)
|
|
default:
|
|
return statusDesc
|
|
}
|
|
}
|
|
|
|
// 3. gRPC error titles
|
|
if grpcStr := firstAttr(span.Attributes(), "rpc.grpc.status_code"); grpcStr != "" {
|
|
if code, err := strconv.Atoi(grpcStr); err == nil && code > 0 {
|
|
method := firstAttr(span.Attributes(), "rpc.method")
|
|
service := firstAttr(span.Attributes(), "rpc.service")
|
|
target := span.Name()
|
|
if service != "" && method != "" {
|
|
target = service + "/" + method
|
|
}
|
|
msg := span.Status().Message()
|
|
if msg != "" {
|
|
return truncate(fmt.Sprintf("gRPC error %d: %s (%s)", code, target, msg))
|
|
}
|
|
if target != "" {
|
|
return truncate(fmt.Sprintf("gRPC error %d: %s", code, target))
|
|
}
|
|
return fmt.Sprintf("gRPC error %d", code)
|
|
}
|
|
}
|
|
|
|
// 4. Status message with span name
|
|
if msg := span.Status().Message(); msg != "" {
|
|
if span.Name() != "" {
|
|
return truncate(span.Name() + ": " + msg)
|
|
}
|
|
return truncate(msg)
|
|
}
|
|
|
|
// 5. Fallback to span name
|
|
if span.Name() != "" {
|
|
return truncate(span.Name() + " (error)")
|
|
}
|
|
|
|
return "span error"
|
|
}
|
|
|
|
func logErrorTitle(record plog.LogRecord) string {
|
|
excType := attrString(record.Attributes(), "exception.type")
|
|
excMsg := attrString(record.Attributes(), "exception.message")
|
|
switch {
|
|
case excType != "" && excMsg != "":
|
|
return truncate(excType + ": " + excMsg)
|
|
case excMsg != "":
|
|
return truncate(excMsg)
|
|
}
|
|
|
|
body := strings.TrimSpace(record.Body().AsString())
|
|
if body == "" {
|
|
return "log error"
|
|
}
|
|
return truncate(body)
|
|
}
|
|
|
|
func truncate(s string) string {
|
|
if len(s) <= maxIssueTitleLen {
|
|
return s
|
|
}
|
|
return s[:maxIssueTitleLen] + "…"
|
|
}
|
|
|
|
func spanServiceName(res pcommon.Resource, span ptrace.Span) string {
|
|
if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
|
|
return name
|
|
}
|
|
if name := firstAttr(span.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
|
|
return name
|
|
}
|
|
return "unknown"
|
|
}
|
|
|
|
func logServiceName(res pcommon.Resource, record plog.LogRecord) string {
|
|
if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
|
|
return name
|
|
}
|
|
if name := firstAttr(record.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
|
|
return name
|
|
}
|
|
return "unknown"
|
|
}
|
|
|
|
func resourceServiceName(res pcommon.Resource) string {
|
|
if name := firstAttr(res.Attributes(), "service.name", "app.kubernetes.io/name", "app", "name"); name != "" && name != "unknown" {
|
|
return name
|
|
}
|
|
return "unknown"
|
|
}
|
|
|
|
// attrString safely reads a string attribute, since a Value obtained from a
|
|
// missing Map key is not valid to call methods on.
|
|
func attrString(attrs pcommon.Map, key string) string {
|
|
if v, ok := attrs.Get(key); ok {
|
|
return v.AsString()
|
|
}
|
|
return ""
|
|
}
|
|
|
|
type traceOrSpanID interface {
|
|
IsEmpty() bool
|
|
String() string
|
|
}
|
|
|
|
func idString(id traceOrSpanID) string {
|
|
if id.IsEmpty() {
|
|
return ""
|
|
}
|
|
return id.String()
|
|
}
|