// Package tracectx resolves parent/child relationships between spans that // arrive in different OTLP envelopes. One user request is exported as several // spans by several processes (ingress outer span, ingress inner span on a // loopback hop, the application span), so a single span is a poor unit for // both display and modelling. Resolution therefore works on a set of spans // loaded together (a time window, or every span of the traces of interest) and // is independent of arrival order. package tracectx import "git.tornberg.me/go-otel/internal/honeypot" // Span is the subset of a normalized span needed for resolution. type Span struct { TraceID string SpanID string ParentID string Service string PeerIP string // immediate peer address as recorded on the span IsError bool StatusCode int Edge bool // ingress/proxy service (see IsEdgeService) } // Key identifies a span inside a resolved set. type Key struct{ TraceID, SpanID string } // Info is what resolution learned about one span. type Info struct { // ClientIP is the first public peer address on the span or its ancestors. // It is empty when every hop is internal (LAN client, NAT/firewall). ClientIP string // ClientHops is 0 when ClientIP came from the span itself, N when it came // from the Nth ancestor. A non-zero value is what fixes loopback hops. ClientHops int // EdgePeer is the peer address of the outermost edge span, public or not. EdgePeer string RootSpan string // outermost ancestor present in the set Depth int // ancestors present in the set // Children counts direct children; UpstreamChildren counts those in a // different service. UpstreamChildren == 0 on an edge span means the proxy // answered on its own (probe, rate limit, unrouted host). Children int UpstreamChildren int // ErrorChildren counts direct children that are errors. ErrorChildren int // IsRootCause marks an error span with no erroring child: the deepest // place the failure is visible. IsRootCause bool // IsDuplicate marks an error that repeats the failure of an ancestor (same // status code, or no status on either side). The outermost error owns the // request; counting the duplicates inflates error rates by the number of // hops a request passed through. IsDuplicate bool // ParentMissing is true when the span names a parent that is not in the set // (client-supplied traceparent, or a service that was not exporting). ParentMissing bool } const maxHops = 32 // IsEdgeService reports whether a service name is an ingress/proxy layer. func IsEdgeService(service string) bool { switch service { case "nginx", "ingress-nginx", "envoy", "traefik", "haproxy", "gateway": return true } return false } // Resolve derives Info for every span. Spans without a trace or span id are // ignored. Cycles and pathological depth are cut off after maxHops. func Resolve(spans []Span) map[Key]Info { byKey := make(map[Key]Span, len(spans)) for _, s := range spans { if s.TraceID == "" || s.SpanID == "" { continue } byKey[Key{s.TraceID, s.SpanID}] = s } children := map[Key][]Key{} for k, s := range byKey { if s.ParentID != "" { p := Key{s.TraceID, s.ParentID} children[p] = append(children[p], k) } } out := make(map[Key]Info, len(byKey)) for k, s := range byKey { info := Info{RootSpan: s.SpanID} if !honeypot.Untrackable(s.PeerIP) { info.ClientIP = s.PeerIP } if s.Edge && s.PeerIP != "" { info.EdgePeer = s.PeerIP } cur := s for hops := 1; cur.ParentID != "" && hops <= maxHops; hops++ { parent, ok := byKey[Key{cur.TraceID, cur.ParentID}] if !ok { if hops == 1 { info.ParentMissing = true } break } info.Depth = hops info.RootSpan = parent.SpanID if info.ClientIP == "" && !honeypot.Untrackable(parent.PeerIP) { info.ClientIP, info.ClientHops = parent.PeerIP, hops } if parent.Edge && parent.PeerIP != "" { info.EdgePeer = parent.PeerIP // keep walking: outermost wins } if s.IsError && parent.IsError && info.Depth > 0 && sameFailure(s, parent) { info.IsDuplicate = true } cur = parent } for _, ck := range children[k] { c := byKey[ck] info.Children++ if c.Service != s.Service { info.UpstreamChildren++ } if c.IsError { info.ErrorChildren++ } } info.IsRootCause = s.IsError && info.ErrorChildren == 0 out[k] = info } return out } func sameFailure(a, b Span) bool { return a.StatusCode == b.StatusCode }