all the changes

internal/honeypot/honeypot.go

@@ -8,11 +8,14 @@ import (
 	"encoding/json"
 	"net"
 	"net/http"
+	"net/netip"
 	"os"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
+
+	"git.tornberg.me/go-otel/internal/triage"
 )
 
 const (
@@ -20,6 +23,7 @@ const (
 	defaultWindow    = 15 * time.Minute
 	defaultTTL       = 1 * time.Hour
 	maxTracked       = 10000
+	maxPathsTracked  = 64
 )
 
 type Config struct {
@@ -28,9 +32,14 @@ type Config struct {
 	TTL       time.Duration
 	Delay     time.Duration
 	Token     string
+	// MinPaths is how many distinct suspicious paths an IP must request
+	// before it is promoted. A legitimate client repeatedly fetching one
+	// odd path (a manifest.json) is not a scanner. Zero means 1.
+	MinPaths int
 }
 
 type entry struct {
+	Paths         map[string]struct{}
 	Count         int
 	FirstSeen     time.Time
 	LastSeen      time.Time
@@ -41,6 +50,7 @@ type entry struct {
 type IP struct {
 	Address       string    `json:"ip"`
 	Count         int       `json:"count"`
+	Paths         int       `json:"paths"`
 	FirstSeen     time.Time `json:"first_seen"`
 	LastSeen      time.Time `json:"last_seen"`
 	PromotedUntil time.Time `json:"promoted_until"`
@@ -52,6 +62,7 @@ type Tracker struct {
 	ttl       time.Duration
 	delay     time.Duration
 	token     string
+	minPaths  int
 
 	mu      sync.Mutex
 	clients map[string]entry
@@ -73,11 +84,17 @@ func New(cfg Config) *Tracker {
 	if cfg.Delay > 10*time.Second {
 		cfg.Delay = 10 * time.Second
 	}
-	return &Tracker{threshold: cfg.Threshold, window: cfg.Window, ttl: cfg.TTL, delay: cfg.Delay, token: cfg.Token, clients: make(map[string]entry)}
+	if cfg.MinPaths < 1 {
+		cfg.MinPaths = 1
+	}
+	return &Tracker{threshold: cfg.Threshold, window: cfg.Window, ttl: cfg.TTL, delay: cfg.Delay, token: cfg.Token, minPaths: cfg.MinPaths, clients: make(map[string]entry)}
 }
 
 func ConfigFromEnv() Config {
-	cfg := Config{Threshold: defaultThreshold, Window: defaultWindow, TTL: defaultTTL, Delay: 0, Token: os.Getenv("OTEL_HONEYPOT_TOKEN")}
+	cfg := Config{Threshold: defaultThreshold, Window: defaultWindow, TTL: defaultTTL, Delay: 0, Token: os.Getenv("OTEL_HONEYPOT_TOKEN"), MinPaths: 3}
+	if n, err := strconv.Atoi(os.Getenv("OTEL_HONEYPOT_MIN_PATHS")); err == nil && n > 0 {
+		cfg.MinPaths = n
+	}
 	if n, err := strconv.Atoi(os.Getenv("OTEL_HONEYPOT_THRESHOLD")); err == nil && n > 0 {
 		cfg.Threshold = n
 	}
@@ -102,7 +119,7 @@ func (t *Tracker) Middleware(next http.Handler) http.Handler {
 			return
 		}
 		ip := clientIP(r)
-		promoted := t.record(ip)
+		promoted := t.record(ip, r.URL.Path)
 		if promoted && t.delay > 0 {
 			timer := time.NewTimer(t.delay)
 			select {
@@ -125,17 +142,55 @@ func (t *Tracker) Middleware(next http.Handler) http.Handler {
 	})
 }
 
-func (t *Tracker) record(ip string) bool {
+// Untrackable reports addresses that must never be tracked or blocked:
+// loopback, private, link-local, multicast, unspecified or unparsable. Behind
+// a proxy hop the observed peer is often 127.0.0.1 or a node address, and
+// promoting it would cut off the ingress itself.
+func Untrackable(ip string) bool {
+	addr, err := netip.ParseAddr(strings.TrimSpace(ip))
+	if err != nil {
+		return true
+	}
+	addr = addr.Unmap()
+	if addr.IsLoopback() || addr.IsPrivate() || addr.IsLinkLocalUnicast() || addr.IsUnspecified() || addr.IsMulticast() {
+		return true
+	}
+	for _, p := range reservedPrefixes {
+		if p.Contains(addr) {
+			return true
+		}
+	}
+	return false
+}
+
+// reservedPrefixes are ranges the standard library helpers do not cover but
+// that are never a remote scanner: carrier-grade NAT (also used by Tailscale
+// and mobile carriers), IETF protocol assignments, benchmarking, and the
+// reserved/broadcast block.
+var reservedPrefixes = []netip.Prefix{
+	netip.MustParsePrefix("100.64.0.0/10"),
+	netip.MustParsePrefix("192.0.0.0/24"),
+	netip.MustParsePrefix("198.18.0.0/15"),
+	netip.MustParsePrefix("240.0.0.0/4"),
+}
+
+func (t *Tracker) record(ip, path string) bool {
+	if Untrackable(ip) {
+		return false
+	}
 	now := time.Now()
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	state, ok := t.clients[ip]
 	if !ok || now.Sub(state.LastSeen) > t.window {
-		state = entry{FirstSeen: now}
+		state = entry{FirstSeen: now, Paths: map[string]struct{}{}}
 	}
 	state.Count++
 	state.LastSeen = now
-	if state.Count >= t.threshold {
+	if len(state.Paths) < maxPathsTracked {
+		state.Paths[strings.ToLower(path)] = struct{}{}
+	}
+	if state.Count >= t.threshold && len(state.Paths) >= t.minPaths {
 		state.PromotedUntil = now.Add(t.ttl)
 	}
 	t.clients[ip] = state
@@ -152,10 +207,10 @@ func (t *Tracker) record(ip string) bool {
 // promoted-IP feed.
 func (t *Tracker) RecordTelemetry(ip, path string) bool {
 	ip = strings.TrimSpace(strings.SplitN(ip, ",", 2)[0])
-	if ip == "" || !isSuspiciousPath(path) {
+	if ip == "" || !(isSuspiciousPath(path) || triage.IsProbePath(path)) {
 		return false
 	}
-	return t.record(ip)
+	return t.record(ip, path)
 }
 
 // PromotedIPs returns a snapshot suitable for an ingress synchronizer.
@@ -168,7 +223,7 @@ func (t *Tracker) PromotedIPs() []IP {
 		if state.PromotedUntil.IsZero() || !now.Before(state.PromotedUntil) {
 			continue
 		}
-		result = append(result, IP{Address: address, Count: state.Count, FirstSeen: state.FirstSeen, LastSeen: state.LastSeen, PromotedUntil: state.PromotedUntil})
+		result = append(result, IP{Address: address, Count: state.Count, Paths: len(state.Paths), FirstSeen: state.FirstSeen, LastSeen: state.LastSeen, PromotedUntil: state.PromotedUntil})
 	}
 	return result
 }

internal/honeypot/honeypot_test.go

@@ -10,8 +10,8 @@ import (
 
 func TestPromotesAfterThresholdAndExpires(t *testing.T) {
 	tracker := New(Config{Threshold: 2, Window: time.Minute, TTL: time.Millisecond, Token: "secret"})
-	for i := 0; i < 3; i++ {
-		req := httptest.NewRequest(http.MethodGet, "https://example.test/.env", nil)
+	for _, path := range []string{"/.env", "/.git/config", "/wp-login.php"} {
+		req := httptest.NewRequest(http.MethodGet, "https://example.test"+path, nil)
 		req.RemoteAddr = "192.0.2.10:1234"
 		tracker.Middleware(http.HandlerFunc(http.NotFound)).ServeHTTP(httptest.NewRecorder(), req)
 	}
@@ -56,3 +56,29 @@ func TestRecordTelemetryPromotesTheObservedSourceIP(t *testing.T) {
 		t.Fatalf("unexpected promoted IPs: %+v", ips)
 	}
 }
+
+func TestNeverTracksInternalAddresses(t *testing.T) {
+	tracker := New(Config{Threshold: 1, TTL: time.Minute})
+	for _, ip := range []string{"127.0.0.1", "::1", "10.244.4.1", "192.168.1.5", "169.254.1.1", "100.64.0.1", "100.127.255.254", "198.18.0.1", "240.0.0.1", "255.255.255.255", "::ffff:10.1.2.3", "not-an-ip", ""} {
+		if tracker.RecordTelemetry(ip, "/.env") {
+			t.Fatalf("%q must never be promoted", ip)
+		}
+	}
+	if got := len(tracker.PromotedIPs()); got != 0 {
+		t.Fatalf("promoted %d internal addresses", got)
+	}
+}
+
+func TestMinPathsRequiresBreadth(t *testing.T) {
+	tracker := New(Config{Threshold: 2, TTL: time.Minute, MinPaths: 3})
+	for i := 0; i < 5; i++ {
+		tracker.RecordTelemetry("203.0.113.9", "/manifest.json")
+	}
+	if len(tracker.PromotedIPs()) != 0 {
+		t.Fatal("one repeated path must not promote")
+	}
+	tracker.RecordTelemetry("203.0.113.9", "/.env")
+	if !tracker.RecordTelemetry("203.0.113.9", "/wp-login.php") {
+		t.Fatal("three distinct paths must promote")
+	}
+}