include db for gitea build

cmd/bad-actors/main.go

@@ -17,14 +17,19 @@ import (
 	"github.com/ClickHouse/clickhouse-go/v2"
 
 	"git.tornberg.me/go-otel/internal/honeypot"
+	"git.tornberg.me/go-otel/internal/tracectx"
 	"git.tornberg.me/go-otel/internal/triage"
+	"git.tornberg.me/go-otel/internal/uaclass"
 )
 
 type otlp struct {
 	ResourceSpans []struct {
 		ScopeSpans []struct {
 			Spans []struct {
-				Attributes []struct {
+				TraceID      string `json:"traceId"`
+				SpanID       string `json:"spanId"`
+				ParentSpanID string `json:"parentSpanId"`
+				Attributes   []struct {
 					Key   string                     `json:"key"`
 					Value map[string]json.RawMessage `json:"value"`
 				} `json:"attributes"`
@@ -76,8 +81,15 @@ func main() {
 	}
 	defer rows.Close()
 
-	actors := map[string]*actor{}
-	spans := 0
+	// Pass 1 collects every span; pass 2 resolves parents, because the outer
+	// nginx span (real client) and the inner loopback span are exported in
+	// different envelopes and arrive in either order.
+	type record struct {
+		key                  tracectx.Key
+		ip, path, status, ua string
+	}
+	var records []record
+	var all []tracectx.Span
 	for rows.Next() {
 		var data string
 		if err := rows.Scan(&data); err != nil {
@@ -99,25 +111,9 @@ func main() {
 						}
 					}
 					ip := strings.TrimSpace(strings.SplitN(first(at, "http.request.header.x-real-ip", "http.request.header.x-forwarded-for", "client.address", "net.peer.ip"), ",", 2)[0])
-					if ip == "" {
-						continue
-					}
-					spans++
 					path := strings.SplitN(first(at, "http.target", "url.path", "http.route"), "?", 2)[0]
-					a := actors[ip]
-					if a == nil {
-						a = &actor{IP: ip, Paths: map[string]struct{}{}, Statuses: map[string]int{}}
-						actors[ip] = a
-					}
-					a.Total++
-					if a.UserAgent == "" {
-						a.UserAgent = first(at, "http.user_agent")
-					}
-					if triage.IsProbePath(path) {
-						a.Probes++
-						a.Paths[path] = struct{}{}
-						a.Statuses[first(at, "http.status_code")]++
-					}
+					all = append(all, tracectx.Span{TraceID: sp.TraceID, SpanID: sp.SpanID, ParentID: sp.ParentSpanID, Service: *service, PeerIP: ip, Edge: true})
+					records = append(records, record{key: tracectx.Key{TraceID: sp.TraceID, SpanID: sp.SpanID}, ip: ip, path: path, status: first(at, "http.status_code"), ua: first(at, "http.user_agent")})
 				}
 			}
 		}
@@ -126,6 +122,39 @@ func main() {
 		log.Fatal(err)
 	}
 
+	resolved := tracectx.Resolve(all)
+	actors := map[string]*actor{}
+	spans, recovered, recoveredProbes := 0, 0, 0
+	for _, r := range records {
+		ip := r.ip
+		if in, ok := resolved[r.key]; ok && in.ClientIP != "" && in.ClientHops > 0 && internal(ip) {
+			ip = in.ClientIP // hop behind a proxy: use the real client from an ancestor
+			recovered++
+			if triage.IsProbePath(r.path) {
+				recoveredProbes++
+			}
+		}
+		if ip == "" {
+			continue
+		}
+		spans++
+		a := actors[ip]
+		if a == nil {
+			a = &actor{IP: ip, Paths: map[string]struct{}{}, Statuses: map[string]int{}}
+			actors[ip] = a
+		}
+		a.Total++
+		if a.UserAgent == "" {
+			a.UserAgent = r.ua
+		}
+		if triage.IsProbePath(r.path) {
+			a.Probes++
+			a.Paths[r.path] = struct{}{}
+			a.Statuses[r.status]++
+		}
+	}
+	fmt.Printf("resolved %d hop spans to their real client via the parent span (%d of them probes)\n", recovered, recoveredProbes)
+
 	var list []*actor
 	totalProbes, blockableProbes, blockable, skippedInternal, skippedInternalProbes := 0, 0, 0, 0, 0
 	for _, a := range actors {
@@ -149,12 +178,12 @@ func main() {
 	fmt.Printf("%d spans with an IP, %d distinct IPs, %d probe requests\n", spans, len(actors), totalProbes)
 	fmt.Printf("blockable (public; >=%d probes/>=%d paths/>=%.0f%% probes, or >=%d probes over >=%d paths): %d IPs covering %d probes (%.0f%%)\n", *minProbes, *minPaths, *minShare*100, *broadProbes, *broadPaths, blockable, blockableProbes, pct(blockableProbes, totalProbes))
 	fmt.Printf("internal/loopback (never blocked): %d IPs, %d probes (%.0f%%) - real client is hidden behind a proxy hop\n\n", skippedInternal, skippedInternalProbes, pct(skippedInternalProbes, totalProbes))
-	fmt.Printf("%-40s %6s %6s %6s  %s\n", "ip", "probes", "total", "paths", "verdict / user-agent / statuses")
+	fmt.Printf("%-40s %6s %6s %6s  %-14s %s\n", "ip", "probes", "total", "paths", "class", "verdict / user-agent / statuses")
 	for i, a := range list {
 		if i >= *top {
 			break
 		}
-		fmt.Printf("%-40s %6d %6d %6d  %.60s %v\n", a.IP, a.Probes, a.Total, len(a.Paths), a.UserAgent, a.Statuses)
+		fmt.Printf("%-40s %6d %6d %6d  %-14s %.60s %v\n", a.IP, a.Probes, a.Total, len(a.Paths), uaclass.Of(strings.TrimPrefix(strings.TrimPrefix(a.UserAgent, "[BLOCK] "), "[INTERNAL] ")), a.UserAgent, a.Statuses)
 	}
 }