all the changes

internal/blocker/blocker.go

@@ -0,0 +1,383 @@
+// Package blocker turns the honeypot's promoted-IP feed into ingress-nginx
+// `block-cidrs` entries.
+//
+// Safety properties, all deliberate:
+//   - dry-run is the default; nothing is written unless DryRun is false;
+//   - it only manages entries it added itself (tracked in an annotation on the
+//     ConfigMap), so manually configured block-cidrs are preserved untouched;
+//   - loopback/private/link-local addresses and an operator allowlist are never
+//     blocked;
+//   - blocks expire, the set is capped, growth per tick is limited, and writes
+//     are rate-limited because every ConfigMap change reloads nginx.
+package blocker
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"net/netip"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+
+	"git.tornberg.me/go-otel/internal/honeypot"
+)
+
+const AnnotationManaged = "otel-debug.k6n.net/blocked"
+
+type Source interface {
+	PromotedIPs() []honeypot.IP
+}
+
+type Config struct {
+	Enabled       bool
+	DryRun        bool
+	Namespace     string
+	ConfigMap     string
+	Key           string
+	Interval      time.Duration // how often the feed is evaluated
+	MinWrite      time.Duration // minimum gap between ConfigMap writes
+	BlockTTL      time.Duration // how long a block lasts after last sighting
+	MinProbes     int           // per-window probe hits required to block
+	MinPaths      int           // distinct probe paths required to block
+	BroadProbes   int           // hits+paths that block regardless of MinProbes
+	BroadPaths    int
+	MaxEntries    int // cap on entries this blocker manages
+	MaxNewPerTick int // limit on newly added entries per evaluation
+	Allow         []netip.Prefix
+}
+
+func DefaultConfig() Config {
+	return Config{
+		DryRun: true, Namespace: "ingress-nginx", ConfigMap: "ingress-nginx-controller", Key: "block-cidrs",
+		Interval: time.Minute, MinWrite: 5 * time.Minute, BlockTTL: 24 * time.Hour,
+		MinProbes: 10, MinPaths: 3, BroadProbes: 100, BroadPaths: 50, MaxEntries: 500, MaxNewPerTick: 20,
+	}
+}
+
+// ConfigFromEnv reads OTEL_BLOCKER_*. Enabled defaults to false and DryRun to
+// true; both must be changed on purpose.
+func ConfigFromEnv() Config {
+	c := DefaultConfig()
+	c.Enabled = strings.EqualFold(os.Getenv("OTEL_BLOCKER_ENABLED"), "true")
+	if v := os.Getenv("OTEL_BLOCKER_DRY_RUN"); v != "" {
+		c.DryRun = !strings.EqualFold(v, "false")
+	}
+	str := func(env string, dst *string) {
+		if v := os.Getenv(env); v != "" {
+			*dst = v
+		}
+	}
+	num := func(env string, dst *int) {
+		if n, err := strconv.Atoi(os.Getenv(env)); err == nil && n > 0 {
+			*dst = n
+		}
+	}
+	dur := func(env string, dst *time.Duration) {
+		if d, err := time.ParseDuration(os.Getenv(env)); err == nil && d > 0 {
+			*dst = d
+		}
+	}
+	str("OTEL_BLOCKER_NAMESPACE", &c.Namespace)
+	str("OTEL_BLOCKER_CONFIGMAP", &c.ConfigMap)
+	str("OTEL_BLOCKER_KEY", &c.Key)
+	num("OTEL_BLOCKER_MIN_PROBES", &c.MinProbes)
+	num("OTEL_BLOCKER_MIN_PATHS", &c.MinPaths)
+	num("OTEL_BLOCKER_MAX_ENTRIES", &c.MaxEntries)
+	dur("OTEL_BLOCKER_INTERVAL", &c.Interval)
+	dur("OTEL_BLOCKER_MIN_WRITE", &c.MinWrite)
+	dur("OTEL_BLOCKER_TTL", &c.BlockTTL)
+	for _, item := range strings.Split(os.Getenv("OTEL_BLOCKER_ALLOW"), ",") {
+		item = strings.TrimSpace(item)
+		if item == "" {
+			continue
+		}
+		if p, err := netip.ParsePrefix(item); err == nil {
+			c.Allow = append(c.Allow, p)
+		} else if a, err := netip.ParseAddr(item); err == nil {
+			c.Allow = append(c.Allow, netip.PrefixFrom(a, a.BitLen()))
+		} else {
+			log.Printf("blocker: ignoring invalid OTEL_BLOCKER_ALLOW entry %q", item)
+		}
+	}
+	return c
+}
+
+// Start builds a client (in-cluster, or KUBECONFIG locally) and runs the
+// blocker in the background. It returns nil, nil when disabled.
+func Start(ctx context.Context, cfg Config, source Source) (*Blocker, error) {
+	if !cfg.Enabled {
+		return nil, nil
+	}
+	rc, err := rest.InClusterConfig()
+	if err != nil {
+		kubeconfig := os.Getenv("KUBECONFIG")
+		if kubeconfig == "" {
+			home, herr := os.UserHomeDir()
+			if herr != nil {
+				return nil, herr
+			}
+			kubeconfig = home + "/.kube/config"
+		}
+		if rc, err = clientcmd.BuildConfigFromFlags("", kubeconfig); err != nil {
+			return nil, fmt.Errorf("blocker kubernetes config: %w", err)
+		}
+	}
+	client, err := kubernetes.NewForConfig(rc)
+	if err != nil {
+		return nil, fmt.Errorf("blocker kubernetes client: %w", err)
+	}
+	b := New(cfg, client, source)
+	go b.Run(ctx)
+	return b, nil
+}
+
+type Blocker struct {
+	cfg    Config
+	client kubernetes.Interface
+	source Source
+	now    func() time.Time
+
+	mu        sync.Mutex
+	lastWrite time.Time
+	lastErr   string
+	last      Status
+}
+
+// Status is a snapshot for the UI and for tests.
+type Status struct {
+	Enabled    bool             `json:"enabled"`
+	DryRun     bool             `json:"dryRun"`
+	Evaluated  time.Time        `json:"evaluated"`
+	Candidates []string         `json:"candidates"`
+	Managed    map[string]int64 `json:"managed"` // ip -> unix expiry
+	Wrote      bool             `json:"wrote"`
+	LastWrite  time.Time        `json:"lastWrite"`
+	Error      string           `json:"error,omitempty"`
+}
+
+func New(cfg Config, client kubernetes.Interface, source Source) *Blocker {
+	d := DefaultConfig()
+	if cfg.Interval <= 0 {
+		cfg.Interval = d.Interval
+	}
+	if cfg.MinWrite <= 0 {
+		cfg.MinWrite = d.MinWrite
+	}
+	if cfg.BlockTTL <= 0 {
+		cfg.BlockTTL = d.BlockTTL
+	}
+	if cfg.MaxEntries <= 0 {
+		cfg.MaxEntries = d.MaxEntries
+	}
+	if cfg.MaxNewPerTick <= 0 {
+		cfg.MaxNewPerTick = d.MaxNewPerTick
+	}
+	if cfg.MinProbes <= 0 {
+		cfg.MinProbes = d.MinProbes
+	}
+	if cfg.MinPaths <= 0 {
+		cfg.MinPaths = d.MinPaths
+	}
+	if cfg.BroadProbes <= 0 {
+		cfg.BroadProbes = d.BroadProbes
+	}
+	if cfg.BroadPaths <= 0 {
+		cfg.BroadPaths = d.BroadPaths
+	}
+	return &Blocker{cfg: cfg, client: client, source: source, now: time.Now}
+}
+
+// Run evaluates the feed until ctx is cancelled.
+func (b *Blocker) Run(ctx context.Context) {
+	log.Printf("blocker: started (dryRun=%t, %s/%s key=%s)", b.cfg.DryRun, b.cfg.Namespace, b.cfg.ConfigMap, b.cfg.Key)
+	t := time.NewTicker(b.cfg.Interval)
+	defer t.Stop()
+	for {
+		if err := b.Tick(ctx); err != nil {
+			log.Printf("blocker: %v", err)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+		}
+	}
+}
+
+// Blockable applies the policy to one promoted IP.
+func (b *Blocker) Blockable(ip honeypot.IP) bool {
+	if honeypot.Untrackable(ip.Address) {
+		return false
+	}
+	addr, err := netip.ParseAddr(ip.Address)
+	if err != nil {
+		return false
+	}
+	for _, p := range b.cfg.Allow {
+		if p.Contains(addr) {
+			return false
+		}
+	}
+	return (ip.Count >= b.cfg.MinProbes && ip.Paths >= b.cfg.MinPaths) ||
+		(ip.Count >= b.cfg.BroadProbes && ip.Paths >= b.cfg.BroadPaths)
+}
+
+// Tick runs one evaluation. It is exported so tests and a manual trigger can
+// drive it without waiting on the timer.
+func (b *Blocker) Tick(ctx context.Context) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	now := b.now()
+	status := Status{Enabled: true, DryRun: b.cfg.DryRun, Evaluated: now, Managed: map[string]int64{}, LastWrite: b.lastWrite}
+	defer func() { b.last = status }()
+
+	cms := b.client.CoreV1().ConfigMaps(b.cfg.Namespace)
+	cm, err := cms.Get(ctx, b.cfg.ConfigMap, metav1.GetOptions{})
+	if err != nil {
+		b.lastErr = err.Error()
+		status.Error = b.lastErr
+		return fmt.Errorf("get configmap: %w", err)
+	}
+
+	previous := map[string]int64{}
+	if raw := cm.Annotations[AnnotationManaged]; raw != "" {
+		if err := json.Unmarshal([]byte(raw), &previous); err != nil {
+			previous = map[string]int64{} // unreadable state: treat as empty rather than guess
+		}
+	}
+	managed := map[string]int64{}
+	for ip, exp := range previous {
+		if exp > now.Unix() {
+			managed[ip] = exp
+		}
+	}
+
+	added := 0
+	for _, cand := range b.source.PromotedIPs() {
+		if !b.Blockable(cand) {
+			continue
+		}
+		status.Candidates = append(status.Candidates, cand.Address)
+		_, exists := managed[cand.Address]
+		if !exists {
+			if added >= b.cfg.MaxNewPerTick {
+				continue
+			}
+			added++
+		}
+		managed[cand.Address] = now.Add(b.cfg.BlockTTL).Unix()
+	}
+	sort.Strings(status.Candidates)
+	for len(managed) > b.cfg.MaxEntries { // drop the soonest-expiring first
+		var oldest string
+		for ip, exp := range managed {
+			if oldest == "" || exp < managed[oldest] || (exp == managed[oldest] && ip < oldest) {
+				oldest = ip
+			}
+		}
+		delete(managed, oldest)
+	}
+	status.Managed = managed
+
+	current := splitCIDRs(cm.Data[b.cfg.Key])
+	manual := make([]string, 0, len(current))
+	for _, entry := range current {
+		if _, wasOurs := previous[entry]; !wasOurs {
+			manual = append(manual, entry)
+		}
+	}
+	desired := append([]string(nil), manual...)
+	for ip := range managed {
+		desired = append(desired, ip)
+	}
+	desired = dedupSorted(desired)
+	changed := strings.Join(desired, ",") != strings.Join(dedupSorted(current), ",")
+
+	if !changed {
+		return nil
+	}
+	if b.cfg.DryRun {
+		log.Printf("blocker[dry-run]: would set %s to %d entries (%d managed): %v", b.cfg.Key, len(desired), len(managed), managedList(managed))
+		return nil
+	}
+	if !b.lastWrite.IsZero() && now.Sub(b.lastWrite) < b.cfg.MinWrite {
+		return nil // rate-limited; the change is picked up on a later tick
+	}
+	state, _ := json.Marshal(managed)
+	if cm.Annotations == nil {
+		cm.Annotations = map[string]string{}
+	}
+	if cm.Data == nil {
+		cm.Data = map[string]string{}
+	}
+	cm.Annotations[AnnotationManaged] = string(state)
+	cm.Data[b.cfg.Key] = strings.Join(desired, ",")
+	if len(desired) == 0 {
+		delete(cm.Data, b.cfg.Key)
+	}
+	if _, err := cms.Update(ctx, cm, metav1.UpdateOptions{}); err != nil {
+		b.lastErr = err.Error()
+		status.Error = b.lastErr
+		return fmt.Errorf("update configmap: %w", err)
+	}
+	b.lastWrite, b.lastErr = now, ""
+	status.Wrote, status.LastWrite = true, now
+	log.Printf("blocker: updated %s: %d entries (%d managed)", b.cfg.Key, len(desired), len(managed))
+	return nil
+}
+
+// HandleStatus serves the last evaluation. Register it behind authentication.
+func (b *Blocker) HandleStatus(w http.ResponseWriter, r *http.Request) {
+	b.mu.Lock()
+	s := b.last
+	b.mu.Unlock()
+	if s.Evaluated.IsZero() {
+		s = Status{Enabled: true, DryRun: b.cfg.DryRun, Managed: map[string]int64{}}
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Cache-Control", "no-store")
+	_ = json.NewEncoder(w).Encode(s)
+}
+
+func splitCIDRs(v string) []string {
+	var out []string
+	for _, p := range strings.Split(v, ",") {
+		if p = strings.TrimSpace(p); p != "" {
+			out = append(out, p)
+		}
+	}
+	return out
+}
+
+func dedupSorted(in []string) []string {
+	seen := map[string]struct{}{}
+	out := make([]string, 0, len(in))
+	for _, v := range in {
+		if _, ok := seen[v]; !ok {
+			seen[v] = struct{}{}
+			out = append(out, v)
+		}
+	}
+	sort.Strings(out)
+	return out
+}
+
+func managedList(m map[string]int64) []string {
+	out := make([]string, 0, len(m))
+	for ip := range m {
+		out = append(out, ip)
+	}
+	sort.Strings(out)
+	return out
+}

internal/blocker/blocker_test.go

@@ -0,0 +1,153 @@
+package blocker
+
+import (
+	"context"
+	"encoding/json"
+	"net/netip"
+	"strings"
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+
+	"git.tornberg.me/go-otel/internal/honeypot"
+)
+
+type feed []honeypot.IP
+
+func (f feed) PromotedIPs() []honeypot.IP { return f }
+
+func setup(t *testing.T, cfg Config, data map[string]string, src Source) (*Blocker, *fake.Clientset, *time.Time) {
+	t.Helper()
+	client := fake.NewSimpleClientset(&corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{Namespace: "ingress-nginx", Name: "ingress-nginx-controller"},
+		Data:       data,
+	})
+	b := New(cfg, client, src)
+	clock := time.Date(2026, 9, 19, 12, 0, 0, 0, time.UTC)
+	b.now = func() time.Time { return clock }
+	return b, client, &clock
+}
+
+func current(t *testing.T, c *fake.Clientset) *corev1.ConfigMap {
+	t.Helper()
+	cm, err := c.CoreV1().ConfigMaps("ingress-nginx").Get(context.Background(), "ingress-nginx-controller", metav1.GetOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	return cm
+}
+
+func scanner(ip string) honeypot.IP { return honeypot.IP{Address: ip, Count: 40, Paths: 12} }
+
+func liveConfig() Config {
+	c := DefaultConfig()
+	c.DryRun = false
+	return c
+}
+
+func TestDryRunWritesNothing(t *testing.T) {
+	b, client, _ := setup(t, DefaultConfig(), map[string]string{"use-gzip": "true"}, feed{scanner("203.0.113.5")})
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	cm := current(t, client)
+	if _, ok := cm.Data["block-cidrs"]; ok || cm.Annotations[AnnotationManaged] != "" {
+		t.Fatalf("dry run wrote to the configmap: %+v %+v", cm.Data, cm.Annotations)
+	}
+	if len(b.last.Candidates) != 1 {
+		t.Fatalf("dry run should still report candidates: %+v", b.last)
+	}
+}
+
+func TestBlocksScannerAndPreservesManualEntries(t *testing.T) {
+	b, client, _ := setup(t, liveConfig(), map[string]string{"block-cidrs": "198.51.100.0/24", "use-gzip": "true"}, feed{scanner("203.0.113.5")})
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	cm := current(t, client)
+	if got := cm.Data["block-cidrs"]; got != "198.51.100.0/24,203.0.113.5" {
+		t.Fatalf("block-cidrs = %q", got)
+	}
+	if cm.Data["use-gzip"] != "true" {
+		t.Fatal("unrelated keys must be untouched")
+	}
+	var state map[string]int64
+	if err := json.Unmarshal([]byte(cm.Annotations[AnnotationManaged]), &state); err != nil || len(state) != 1 {
+		t.Fatalf("annotation = %q (%v)", cm.Annotations[AnnotationManaged], err)
+	}
+}
+
+func TestNeverBlocksInternalOrAllowlisted(t *testing.T) {
+	cfg := liveConfig()
+	cfg.Allow = []netip.Prefix{netip.MustParsePrefix("203.0.113.0/24")}
+	b, client, _ := setup(t, cfg, nil, feed{scanner("127.0.0.1"), scanner("10.244.4.1"), scanner("192.168.1.9"), scanner("203.0.113.77")})
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if v := current(t, client).Data["block-cidrs"]; v != "" {
+		t.Fatalf("blocked %q", v)
+	}
+}
+
+func TestThresholdsKeepOutOccasionalVisitors(t *testing.T) {
+	b, client, _ := setup(t, liveConfig(), nil, feed{
+		{Address: "203.0.113.1", Count: 4, Paths: 4},    // too few hits
+		{Address: "203.0.113.2", Count: 500, Paths: 1},  // one path hammered
+		{Address: "203.0.113.3", Count: 200, Paths: 60}, // broad scanner
+	})
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if v := current(t, client).Data["block-cidrs"]; v != "203.0.113.3" {
+		t.Fatalf("block-cidrs = %q", v)
+	}
+}
+
+func TestExpiryRemovesOnlyManagedEntries(t *testing.T) {
+	src := feed{scanner("203.0.113.5")}
+	b, client, clock := setup(t, liveConfig(), map[string]string{"block-cidrs": "198.51.100.7"}, src)
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	b.source = feed{} // scanner goes quiet
+	*clock = clock.Add(25 * time.Hour)
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if v := current(t, client).Data["block-cidrs"]; v != "198.51.100.7" {
+		t.Fatalf("after expiry block-cidrs = %q, want only the manual entry", v)
+	}
+}
+
+func TestWritesAreRateLimitedAndNewPerTickCapped(t *testing.T) {
+	cfg := liveConfig()
+	cfg.MaxNewPerTick = 2
+	var many feed
+	for _, ip := range []string{"203.0.113.1", "203.0.113.2", "203.0.113.3", "203.0.113.4"} {
+		many = append(many, scanner(ip))
+	}
+	b, client, clock := setup(t, cfg, nil, many)
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if n := len(strings.Split(current(t, client).Data["block-cidrs"], ",")); n != 2 {
+		t.Fatalf("first tick added %d, want 2", n)
+	}
+	*clock = clock.Add(time.Minute) // inside MinWrite
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if n := len(strings.Split(current(t, client).Data["block-cidrs"], ",")); n != 2 {
+		t.Fatalf("write was not rate limited: %d entries", n)
+	}
+	*clock = clock.Add(10 * time.Minute)
+	if err := b.Tick(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if n := len(strings.Split(current(t, client).Data["block-cidrs"], ",")); n != 4 {
+		t.Fatalf("after the interval got %d entries, want 4", n)
+	}
+}

internal/collector/collector_test.go

@@ -392,6 +392,47 @@ func TestHandleIssuesAndByID(t *testing.T) {
 	}
 }
 
+func TestIssueTrendsUseTelemetryTimestamps(t *testing.T) {
+	coll := New(noopBroadcaster{}, 10, nil)
+
+	base := time.Date(2026, 9, 19, 10, 0, 0, 0, time.UTC)
+	ld := plog.NewLogs()
+	rl := ld.ResourceLogs().AppendEmpty()
+	rl.Resource().Attributes().PutStr("service.name", "api")
+	records := rl.ScopeLogs().AppendEmpty().LogRecords()
+	first := records.AppendEmpty()
+	first.SetTimestamp(pcommon.NewTimestampFromTime(base))
+	first.SetSeverityNumber(plog.SeverityNumberError)
+	first.Body().SetStr("first failure")
+	second := records.AppendEmpty()
+	second.SetTimestamp(pcommon.NewTimestampFromTime(base.Add(time.Hour)))
+	second.SetSeverityNumber(plog.SeverityNumberError)
+	second.Body().SetStr("second failure")
+
+	if err := coll.ConsumeLogs(context.Background(), ld); err != nil {
+		t.Fatalf("ConsumeLogs failed: %v", err)
+	}
+
+	list := coll.ListIssues(0)
+	if len(list) != 2 {
+		t.Fatalf("expected two issues, got %d", len(list))
+	}
+	for _, issue := range list {
+		if len(issue.Trend) != 1 {
+			t.Fatalf("expected one trend bucket for %q, got %+v", issue.Title, issue.Trend)
+		}
+	}
+	expected := map[string]string{
+		"first failure":  issues.HourBucket(base.UnixNano()),
+		"second failure": issues.HourBucket(base.Add(time.Hour).UnixNano()),
+	}
+	for _, issue := range list {
+		if _, ok := issue.Trend[expected[issue.Title]]; !ok {
+			t.Fatalf("telemetry timestamp was not used for %q: %+v", issue.Title, issue.Trend)
+		}
+	}
+}
+
 func TestHandleMLDetectorTest(t *testing.T) {
 	coll := New(nil, 10, nil)
 	req := httptest.NewRequest(http.MethodPost, "/api/ml/test-spike", nil)

internal/collector/issue_extract.go

@@ -37,7 +37,14 @@ func extractSpanErrors(tracker *issues.Tracker, td ptrace.Traces, timestamp int6
 				service := spanServiceName(rs.Resource(), span)
 				title := spanErrorTitle(span)
 				found = true
-				tracker.Record("span_error", service, title, spanSample(span, rs.Resource(), timestamp, title))
+				occurrenceTimestamp := span.StartTimestamp()
+				if occurrenceTimestamp == 0 {
+					occurrenceTimestamp = span.EndTimestamp()
+				}
+				if occurrenceTimestamp == 0 {
+					occurrenceTimestamp = pcommon.Timestamp(timestamp)
+				}
+				tracker.Record("span_error", service, title, spanSample(span, rs.Resource(), int64(occurrenceTimestamp), title))
 			}
 		}
 	}
@@ -102,7 +109,14 @@ func extractLogErrors(tracker *issues.Tracker, ld plog.Logs, timestamp int64) bo
 				title := logErrorTitle(record)
 				found = true
 				kind := "log_error"
-				sample := logSample(record, rl.Resource(), timestamp, title)
+				occurrenceTimestamp := record.Timestamp()
+				if occurrenceTimestamp == 0 {
+					occurrenceTimestamp = record.ObservedTimestamp()
+				}
+				if occurrenceTimestamp == 0 {
+					occurrenceTimestamp = pcommon.Timestamp(timestamp)
+				}
+				sample := logSample(record, rl.Resource(), int64(occurrenceTimestamp), title)
 				if attrString(record.Attributes(), "source") == "frontend" {
 					kind = "frontend_error"
 					sample.URL = attrString(record.Attributes(), "browser.url")

internal/honeypot/honeypot.go

@@ -8,11 +8,14 @@ import (
 	"encoding/json"
 	"net"
 	"net/http"
+	"net/netip"
 	"os"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
+
+	"git.tornberg.me/go-otel/internal/triage"
 )
 
 const (
@@ -20,6 +23,7 @@ const (
 	defaultWindow    = 15 * time.Minute
 	defaultTTL       = 1 * time.Hour
 	maxTracked       = 10000
+	maxPathsTracked  = 64
 )
 
 type Config struct {
@@ -28,9 +32,14 @@ type Config struct {
 	TTL       time.Duration
 	Delay     time.Duration
 	Token     string
+	// MinPaths is how many distinct suspicious paths an IP must request
+	// before it is promoted. A legitimate client repeatedly fetching one
+	// odd path (a manifest.json) is not a scanner. Zero means 1.
+	MinPaths int
 }
 
 type entry struct {
+	Paths         map[string]struct{}
 	Count         int
 	FirstSeen     time.Time
 	LastSeen      time.Time
@@ -41,6 +50,7 @@ type entry struct {
 type IP struct {
 	Address       string    `json:"ip"`
 	Count         int       `json:"count"`
+	Paths         int       `json:"paths"`
 	FirstSeen     time.Time `json:"first_seen"`
 	LastSeen      time.Time `json:"last_seen"`
 	PromotedUntil time.Time `json:"promoted_until"`
@@ -52,6 +62,7 @@ type Tracker struct {
 	ttl       time.Duration
 	delay     time.Duration
 	token     string
+	minPaths  int
 
 	mu      sync.Mutex
 	clients map[string]entry
@@ -73,11 +84,17 @@ func New(cfg Config) *Tracker {
 	if cfg.Delay > 10*time.Second {
 		cfg.Delay = 10 * time.Second
 	}
-	return &Tracker{threshold: cfg.Threshold, window: cfg.Window, ttl: cfg.TTL, delay: cfg.Delay, token: cfg.Token, clients: make(map[string]entry)}
+	if cfg.MinPaths < 1 {
+		cfg.MinPaths = 1
+	}
+	return &Tracker{threshold: cfg.Threshold, window: cfg.Window, ttl: cfg.TTL, delay: cfg.Delay, token: cfg.Token, minPaths: cfg.MinPaths, clients: make(map[string]entry)}
 }
 
 func ConfigFromEnv() Config {
-	cfg := Config{Threshold: defaultThreshold, Window: defaultWindow, TTL: defaultTTL, Delay: 0, Token: os.Getenv("OTEL_HONEYPOT_TOKEN")}
+	cfg := Config{Threshold: defaultThreshold, Window: defaultWindow, TTL: defaultTTL, Delay: 0, Token: os.Getenv("OTEL_HONEYPOT_TOKEN"), MinPaths: 3}
+	if n, err := strconv.Atoi(os.Getenv("OTEL_HONEYPOT_MIN_PATHS")); err == nil && n > 0 {
+		cfg.MinPaths = n
+	}
 	if n, err := strconv.Atoi(os.Getenv("OTEL_HONEYPOT_THRESHOLD")); err == nil && n > 0 {
 		cfg.Threshold = n
 	}
@@ -102,7 +119,7 @@ func (t *Tracker) Middleware(next http.Handler) http.Handler {
 			return
 		}
 		ip := clientIP(r)
-		promoted := t.record(ip)
+		promoted := t.record(ip, r.URL.Path)
 		if promoted && t.delay > 0 {
 			timer := time.NewTimer(t.delay)
 			select {
@@ -125,17 +142,55 @@ func (t *Tracker) Middleware(next http.Handler) http.Handler {
 	})
 }
 
-func (t *Tracker) record(ip string) bool {
+// Untrackable reports addresses that must never be tracked or blocked:
+// loopback, private, link-local, multicast, unspecified or unparsable. Behind
+// a proxy hop the observed peer is often 127.0.0.1 or a node address, and
+// promoting it would cut off the ingress itself.
+func Untrackable(ip string) bool {
+	addr, err := netip.ParseAddr(strings.TrimSpace(ip))
+	if err != nil {
+		return true
+	}
+	addr = addr.Unmap()
+	if addr.IsLoopback() || addr.IsPrivate() || addr.IsLinkLocalUnicast() || addr.IsUnspecified() || addr.IsMulticast() {
+		return true
+	}
+	for _, p := range reservedPrefixes {
+		if p.Contains(addr) {
+			return true
+		}
+	}
+	return false
+}
+
+// reservedPrefixes are ranges the standard library helpers do not cover but
+// that are never a remote scanner: carrier-grade NAT (also used by Tailscale
+// and mobile carriers), IETF protocol assignments, benchmarking, and the
+// reserved/broadcast block.
+var reservedPrefixes = []netip.Prefix{
+	netip.MustParsePrefix("100.64.0.0/10"),
+	netip.MustParsePrefix("192.0.0.0/24"),
+	netip.MustParsePrefix("198.18.0.0/15"),
+	netip.MustParsePrefix("240.0.0.0/4"),
+}
+
+func (t *Tracker) record(ip, path string) bool {
+	if Untrackable(ip) {
+		return false
+	}
 	now := time.Now()
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	state, ok := t.clients[ip]
 	if !ok || now.Sub(state.LastSeen) > t.window {
-		state = entry{FirstSeen: now}
+		state = entry{FirstSeen: now, Paths: map[string]struct{}{}}
 	}
 	state.Count++
 	state.LastSeen = now
-	if state.Count >= t.threshold {
+	if len(state.Paths) < maxPathsTracked {
+		state.Paths[strings.ToLower(path)] = struct{}{}
+	}
+	if state.Count >= t.threshold && len(state.Paths) >= t.minPaths {
 		state.PromotedUntil = now.Add(t.ttl)
 	}
 	t.clients[ip] = state
@@ -152,10 +207,10 @@ func (t *Tracker) record(ip string) bool {
 // promoted-IP feed.
 func (t *Tracker) RecordTelemetry(ip, path string) bool {
 	ip = strings.TrimSpace(strings.SplitN(ip, ",", 2)[0])
-	if ip == "" || !isSuspiciousPath(path) {
+	if ip == "" || !(isSuspiciousPath(path) || triage.IsProbePath(path)) {
 		return false
 	}
-	return t.record(ip)
+	return t.record(ip, path)
 }
 
 // PromotedIPs returns a snapshot suitable for an ingress synchronizer.
@@ -168,7 +223,7 @@ func (t *Tracker) PromotedIPs() []IP {
 		if state.PromotedUntil.IsZero() || !now.Before(state.PromotedUntil) {
 			continue
 		}
-		result = append(result, IP{Address: address, Count: state.Count, FirstSeen: state.FirstSeen, LastSeen: state.LastSeen, PromotedUntil: state.PromotedUntil})
+		result = append(result, IP{Address: address, Count: state.Count, Paths: len(state.Paths), FirstSeen: state.FirstSeen, LastSeen: state.LastSeen, PromotedUntil: state.PromotedUntil})
 	}
 	return result
 }

internal/honeypot/honeypot_test.go

@@ -10,8 +10,8 @@ import (
 
 func TestPromotesAfterThresholdAndExpires(t *testing.T) {
 	tracker := New(Config{Threshold: 2, Window: time.Minute, TTL: time.Millisecond, Token: "secret"})
-	for i := 0; i < 3; i++ {
-		req := httptest.NewRequest(http.MethodGet, "https://example.test/.env", nil)
+	for _, path := range []string{"/.env", "/.git/config", "/wp-login.php"} {
+		req := httptest.NewRequest(http.MethodGet, "https://example.test"+path, nil)
 		req.RemoteAddr = "192.0.2.10:1234"
 		tracker.Middleware(http.HandlerFunc(http.NotFound)).ServeHTTP(httptest.NewRecorder(), req)
 	}
@@ -56,3 +56,29 @@ func TestRecordTelemetryPromotesTheObservedSourceIP(t *testing.T) {
 		t.Fatalf("unexpected promoted IPs: %+v", ips)
 	}
 }
+
+func TestNeverTracksInternalAddresses(t *testing.T) {
+	tracker := New(Config{Threshold: 1, TTL: time.Minute})
+	for _, ip := range []string{"127.0.0.1", "::1", "10.244.4.1", "192.168.1.5", "169.254.1.1", "100.64.0.1", "100.127.255.254", "198.18.0.1", "240.0.0.1", "255.255.255.255", "::ffff:10.1.2.3", "not-an-ip", ""} {
+		if tracker.RecordTelemetry(ip, "/.env") {
+			t.Fatalf("%q must never be promoted", ip)
+		}
+	}
+	if got := len(tracker.PromotedIPs()); got != 0 {
+		t.Fatalf("promoted %d internal addresses", got)
+	}
+}
+
+func TestMinPathsRequiresBreadth(t *testing.T) {
+	tracker := New(Config{Threshold: 2, TTL: time.Minute, MinPaths: 3})
+	for i := 0; i < 5; i++ {
+		tracker.RecordTelemetry("203.0.113.9", "/manifest.json")
+	}
+	if len(tracker.PromotedIPs()) != 0 {
+		t.Fatal("one repeated path must not promote")
+	}
+	tracker.RecordTelemetry("203.0.113.9", "/.env")
+	if !tracker.RecordTelemetry("203.0.113.9", "/wp-login.php") {
+		t.Fatal("three distinct paths must promote")
+	}
+}

internal/storage/clickhouse.go

@@ -153,6 +153,9 @@ func NewClickHouseStore(ctx context.Context, cfg ClickHouseConfig) (*ClickHouseS
 // initSchema creates the raw entries table and the materialized hourly
 // rollup used by the stats endpoints. Idempotent.
 func (s *ClickHouseStore) initSchema(ctx context.Context) error {
+	if err := EnsureNormalizedSchema(ctx, s.conn, s.db); err != nil {
+		return fmt.Errorf("normalized schema: %w", err)
+	}
 	if err := s.conn.Exec(ctx, fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.entries (
 		timestamp DateTime64(9, 'UTC') NOT NULL,
 		type      LowCardinality(String) NOT NULL,
@@ -361,6 +364,7 @@ func (s *ClickHouseStore) Append(ctx context.Context, entry websocket.HistoryEnt
 	if err := s.appendSpanFacts(ctx, ts, facts); err != nil {
 		return fmt.Errorf("clickhouse span facts: %w", err)
 	}
+	s.appendNormalized(ctx, entry)
 	return nil
 }
 

internal/storage/normalized.go

@@ -0,0 +1,107 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+
+	"git.tornberg.me/go-otel/internal/normalization"
+	"git.tornberg.me/go-otel/internal/websocket"
+)
+
+// normalizedColumns is the explicit insert column list. Tables migrated with
+// ALTER ADD COLUMN have a different physical column order than a fresh
+// CREATE, so positional inserts are not safe.
+const normalizedColumns = `event_id, entry_type, event_index, ingest_timestamp, timestamp, service, release, environment,
+	kind, source, is_error, status_class, status_code, duration_ms, route, span_name, metric_name, metric_type, metric_unit,
+	metric_value, metric_count, metric_sum, metric_min, metric_max, exception_type, message_normalized, stack_signature,
+	trace_id, issue_fingerprint, is_retry, is_timeout, is_cancellation, exact_signature, pattern_signature, version,
+	method, host, user_agent, origin_host, referer_host, fetch_site, fetch_mode`
+
+// NormalizedInsertSQL is the INSERT prefix for a normalized_events batch.
+func NormalizedInsertSQL(db string) string {
+	return fmt.Sprintf("INSERT INTO %s.normalized_events (%s)", db, normalizedColumns)
+}
+
+// EnsureNormalizedSchema creates normalized_events and applies the additive
+// migrations. Idempotent.
+func EnsureNormalizedSchema(ctx context.Context, conn driver.Conn, db string) error {
+	query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.normalized_events (
+		event_id String,
+		entry_type LowCardinality(String),
+		event_index UInt32,
+		ingest_timestamp DateTime64(9, 'UTC'),
+		timestamp DateTime64(9, 'UTC'),
+		service LowCardinality(String), release LowCardinality(String), environment LowCardinality(String),
+		kind LowCardinality(String), source LowCardinality(String), is_error UInt8,
+		status_class LowCardinality(String), status_code UInt16, duration_ms Float64,
+		route String, span_name String, metric_name String, metric_type LowCardinality(String), metric_unit String,
+		metric_value Float64, metric_count UInt64, metric_sum Float64, metric_min Float64, metric_max Float64,
+		exception_type String, message_normalized String, stack_signature String, trace_id String, issue_fingerprint String,
+		is_retry UInt8, is_timeout UInt8, is_cancellation UInt8,
+		exact_signature String, pattern_signature String, version UInt8 DEFAULT 1,
+		method String DEFAULT '', host String DEFAULT '', user_agent String DEFAULT '',
+		origin_host String DEFAULT '', referer_host String DEFAULT '', fetch_site String DEFAULT '', fetch_mode String DEFAULT ''
+	) ENGINE = ReplacingMergeTree(version)
+	PARTITION BY toYYYYMM(timestamp)
+	ORDER BY (service, kind, pattern_signature, timestamp, event_id)
+	TTL toDateTime(timestamp) + INTERVAL 180 DAY`, db)
+	if err := conn.Exec(ctx, query); err != nil {
+		return err
+	}
+	// Developer tables predate these columns; historical rows get safe defaults.
+	for _, col := range []string{
+		"issue_fingerprint String DEFAULT ''", "is_retry UInt8 DEFAULT 0", "is_timeout UInt8 DEFAULT 0",
+		"is_cancellation UInt8 DEFAULT 0", "method String DEFAULT ''", "host String DEFAULT ''",
+		"user_agent String DEFAULT ''", "origin_host String DEFAULT ''", "referer_host String DEFAULT ''",
+		"fetch_site String DEFAULT ''", "fetch_mode String DEFAULT ''",
+	} {
+		if err := conn.Exec(ctx, fmt.Sprintf("ALTER TABLE %s.normalized_events ADD COLUMN IF NOT EXISTS %s", db, col)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// AppendNormalizedEvent adds one event to a batch prepared with NormalizedInsertSQL.
+func AppendNormalizedEvent(batch driver.Batch, e normalization.Event) error {
+	return batch.Append(
+		e.EventID, e.EntryType, e.EventIndex, time.Unix(0, e.IngestTimestamp), time.Unix(0, e.Timestamp),
+		e.Service, e.Release, e.Environment, e.Kind, e.Source, boolToUint8(e.IsError),
+		e.StatusClass, uint16(e.StatusCode), e.DurationMS, e.Route, e.SpanName, e.MetricName, e.MetricType, e.MetricUnit,
+		e.MetricValue, e.MetricCount, e.MetricSum, e.MetricMin, e.MetricMax,
+		e.ExceptionType, e.MessageNormalized, e.StackSignature, e.TraceID, e.IssueFingerprint,
+		boolToUint8(e.IsRetry), boolToUint8(e.IsTimeout), boolToUint8(e.IsCancellation), e.ExactSignature, e.PatternSignature,
+		uint8(1), e.Method, e.Host, e.UserAgent, e.OriginHost, e.RefererHost, e.FetchSite, e.FetchMode,
+	)
+}
+
+// appendNormalized derives and stores normalized rows for one raw entry.
+// The raw entry is the source of truth, so a failure here is logged and never
+// fails ingest; the backfill tool can rebuild any gap.
+func (s *ClickHouseStore) appendNormalized(ctx context.Context, entry websocket.HistoryEntry) {
+	events, err := normalization.NormalizeEntry(entry.Type, entry.Timestamp, entry.Data)
+	if err != nil || len(events) == 0 {
+		return
+	}
+	ctx = clickhouse.Context(ctx, clickhouse.WithSettings(clickhouse.Settings{"async_insert": 1, "wait_for_async_insert": 0}))
+	batch, err := s.conn.PrepareBatch(ctx, NormalizedInsertSQL(s.db))
+	if err != nil {
+		log.Printf("normalized insert prepare: %v", err)
+		return
+	}
+	for _, e := range events {
+		if err := AppendNormalizedEvent(batch, e); err != nil {
+			_ = batch.Abort()
+			log.Printf("normalized insert append: %v", err)
+			return
+		}
+	}
+	if err := batch.Send(); err != nil {
+		log.Printf("normalized insert send: %v", err)
+	}
+}

internal/triage/triage.go

@@ -0,0 +1,107 @@
+// Package triage assigns a cause-level category to a normalized error
+// pattern. The categories are deliberately not restatements of status or
+// severity: they answer "why did this happen and who should care".
+//
+// Rules are deterministic and ordered; the first match wins. Structured flags
+// (timeout, cancellation, retry) are preferred over message text.
+package triage
+
+import (
+	"regexp"
+	"strings"
+)
+
+type Category string
+
+const (
+	ScannerProbe          Category = "scanner_probe"
+	DependencyUnreachable Category = "dependency_unreachable"
+	ClientCredentialIssue Category = "client_credential_issue"
+	StaleReference        Category = "stale_reference"
+	ServiceDefect         Category = "service_defect"
+	CapacitySaturation    Category = "capacity_saturation"
+	ExpectedLifecycle     Category = "expected_lifecycle"
+	NeedsReview           Category = "needs_review"
+)
+
+// Input is the non-PII slice of a normalized event the rules look at.
+type Input struct {
+	Service        string
+	Kind           string
+	Status         int
+	Route          string
+	Title          string
+	Message        string
+	IsTimeout      bool
+	IsCancellation bool
+	IsRetry        bool
+}
+
+type Result struct {
+	Category Category `json:"category"`
+	Rule     string   `json:"rule"` // which rule fired, for review and tuning
+}
+
+var (
+	// Probe families: secrets/dotfiles, config and backup file types, source
+	// control, CMS/admin paths, path traversal. Deliberately path-only.
+	probeRoute = regexp.MustCompile(`(?i)(\.env\b|\.git\b|\.htpasswd|\.htaccess|\.npmrc|\.claude|\.docker|\.kube|\.aws|\.ssh|` +
+		`wp-|/wp/|/wp$|wordpress|phpmyadmin|phpinfo|cgi-bin|xmlrpc|userfiles|@fs/|etc/passwd|proc/self|\.\./|%2e%2e|` +
+		`\.php\b|credentials|secrets?\b|firebase|serviceaccount|/config(/|\.)|settings\.(js|json)$|bootstrap\.properties|` +
+		`\.(ya?ml|toml|properties|sql|bak|old|swp|zip|tar|tgz|gz|ini|conf|cfg)$)`)
+	bugText  = regexp.MustCompile(`(?i)panic|nil pointer|index out of range|typeerror|referenceerror|deadlock|unhandled|chunkloaderror|segfault`)
+	capText  = regexp.MustCompile(`(?i)timeout|deadline exceeded|\bpool\b|memory limit|\boom(killed)?\b|too many|queue full|exhaust`)
+	depText  = regexp.MustCompile(`(?i)dial|connection refused|\beof\b|no such host|upstream|unavailable|broken pipe|reset by peer`)
+	lifeText = regexp.MustCompile(`(?i)healthz|readyz|/ping|keepalive|vital|\bcls\b|ttfb|fcp|lcp|snapshot|replay|unchanged|session\.load`)
+)
+
+// IsProbePath reports whether a request path looks like an automated scan.
+// It is the single definition shared by triage and IP-blocking analysis.
+func IsProbePath(path string) bool { return probeRoute.MatchString(path) }
+
+// Classify returns the category for one pattern's example event.
+func Classify(in Input) Result {
+	blob := strings.Join([]string{in.Title, in.Message, in.Route}, " ")
+	switch {
+	case probeRoute.MatchString(in.Route):
+		return Result{ScannerProbe, "probe-route"}
+	case in.IsCancellation:
+		return Result{ExpectedLifecycle, "cancellation-flag"}
+	case in.IsRetry:
+		return Result{ExpectedLifecycle, "retry-flag"}
+	case bugText.MatchString(blob):
+		return Result{ServiceDefect, "bug-text"}
+	case in.IsTimeout:
+		return Result{CapacitySaturation, "timeout-flag"}
+	case capText.MatchString(blob):
+		return Result{CapacitySaturation, "capacity-text"}
+	case depText.MatchString(blob):
+		return Result{DependencyUnreachable, "dependency-text"}
+	case in.Status == 401:
+		return Result{ClientCredentialIssue, "status-401"}
+	case in.Status == 502 || in.Status == 503 || in.Status == 504:
+		return Result{DependencyUnreachable, "status-5xx-upstream"}
+	case in.Status >= 500:
+		return Result{ServiceDefect, "status-5xx"}
+	case in.Status == 404 || in.Status == 410:
+		return Result{StaleReference, "status-404"}
+	case lifeText.MatchString(blob):
+		return Result{ExpectedLifecycle, "lifecycle-text"}
+	case in.Status == 499 || in.Status == 200 || in.Status == 202 || in.Status == 204 || (in.Status >= 300 && in.Status < 400):
+		return Result{ExpectedLifecycle, "benign-status"}
+	case in.Status == 0 && (in.Kind == "span" || in.Kind == "log"):
+		return Result{ExpectedLifecycle, "no-status-signal"}
+	}
+	return Result{NeedsReview, "no-rule"}
+}
+
+// Actionable reports whether alerts in this category should page a human by
+// default. Probes are a security channel rather than an ops one, and the rest
+// are expected noise or harmless.
+func (c Category) Actionable() bool {
+	switch c {
+	case ScannerProbe, StaleReference, ExpectedLifecycle:
+		return false
+	}
+	return true
+}

internal/triage/triage_test.go

@@ -0,0 +1,43 @@
+package triage
+
+import "testing"
+
+func TestClassify(t *testing.T) {
+	cases := []struct {
+		name string
+		in   Input
+		want Category
+	}{
+		{"wp 500 is a probe not an outage", Input{Status: 500, Route: "/wp"}, ScannerProbe},
+		{"php 405 is a probe", Input{Status: 405, Route: "/index.php"}, ScannerProbe},
+		{"ses.env", Input{Status: 404, Route: "/ses.env"}, ScannerProbe},
+		{"yaml config 302", Input{Status: 302, Route: "/config/aws.yml"}, ScannerProbe},
+		{"traversal", Input{Status: 404, Route: "/api/w/admins/get_log_file/../../../../proc/self/environ"}, ScannerProbe},
+		{"credentials file", Input{Status: 404, Route: "/firebase-credentials.json"}, ScannerProbe},
+		{"matrix rooms is not OOM", Input{Status: 200, Route: "/_matrix/client/r0/rooms/x/messages"}, ExpectedLifecycle},
+		{"real api 404 stays stale", Input{Status: 404, Route: "/api/config"}, StaleReference},
+		{"env 404", Input{Status: 404, Route: "/.env"}, ScannerProbe},
+		{"git config 502", Input{Status: 502, Route: "/.git/config"}, ScannerProbe},
+		{"poller 401", Input{Status: 401, Route: "/api/repos/available"}, ClientCredentialIssue},
+		{"upstream 503", Input{Status: 503, Route: "/store/regions"}, DependencyUnreachable},
+		{"real 500", Input{Status: 500, Route: "/"}, ServiceDefect},
+		{"panic in message", Input{Message: "panic: nil pointer dereference"}, ServiceDefect},
+		{"timeout flag", Input{IsTimeout: true, Message: "query"}, CapacitySaturation},
+		{"dial error", Input{Message: "*websocket.dialerror: connection refused"}, DependencyUnreachable},
+		{"missing commit", Input{Status: 404, Route: "/mats/repo/commit/{id}"}, StaleReference},
+		{"cancellation", Input{IsCancellation: true, Message: "context canceled"}, ExpectedLifecycle},
+		{"health", Input{Kind: "span", Status: 200, Route: "/readyz"}, ExpectedLifecycle},
+		{"unknown", Input{Kind: "metric", Status: 418}, NeedsReview},
+	}
+	for _, c := range cases {
+		if got := Classify(c.in); got.Category != c.want {
+			t.Errorf("%s: got %s (%s), want %s", c.name, got.Category, got.Rule, c.want)
+		}
+	}
+}
+
+func TestProbeBeatsLifecycleFlags(t *testing.T) {
+	if got := Classify(Input{Route: "/.env", IsRetry: true}); got.Category != ScannerProbe {
+		t.Fatalf("got %s", got.Category)
+	}
+}