half broken

internal/cluster/cluster.go

@@ -7,6 +7,7 @@ package cluster
 
 import (
 	"context"
+	"crypto/rand"
 	"crypto/sha1"
 	"encoding/hex"
 	"encoding/json"
@@ -189,7 +190,14 @@ func (c *Cluster) Start(ctx context.Context) {
 	c.connected.Store(true)
 	c.log.Info("cluster connected", "addrs", cfg.Cluster.Valkey.Addrs, "node", c.node)
 
+	// Attempt the lease immediately rather than on the first heartbeat tick.
+	// The claim this package makes is "only the lease holder fetches
+	// blocklists", and at startup nobody holds it: waiting half a peer_ttl (7.5
+	// s at the default 15s) means every instance in a fresh rollout fetches
+	// every list from every mirror at once, which is the thundering herd the
+	// lease exists to prevent.
 	c.publishState(ctx)
+	c.renewLease(ctx)
 
 	c.wg.Add(1)
 	go func() {
@@ -279,7 +287,17 @@ func (c *Cluster) runSession(ctx context.Context) error {
 		case <-c.stop:
 			return nil
 		case <-hb.C:
-			c.publishState(ctx)
+			// A failed heartbeat is the only evidence of an outage: the
+			// subscription reconnects behind our back, so runSession keeps
+			// running and nothing else in this loop would ever notice. Without
+			// this, /status and the console claim a Valkey connection that
+			// stopped working, and a diverged peer would look healthy.
+			if err := c.publishState(ctx); err != nil {
+				c.connected.Store(false)
+				c.log.Warn("cluster heartbeat failed", "error", err)
+				continue
+			}
+			c.connected.Store(true)
 			c.renewLease(ctx)
 		case msg, ok := <-ch:
 			if !ok {
@@ -354,17 +372,20 @@ func (c *Cluster) State() *NodeState {
 	return st
 }
 
-// publishState broadcasts this instance's heartbeat.
-func (c *Cluster) publishState(ctx context.Context) {
+// publishState broadcasts this instance's heartbeat. It reports whether the
+// write reached Valkey: the heartbeat is the only liveness signal this package
+// has, because go-redis's Subscribe reconnects transparently and runSession
+// never sees the connection drop (see the connected flag in runSession).
+func (c *Cluster) publishState(ctx context.Context) error {
 	cl := c.client.Load()
 	if cl == nil {
-		return
+		return errors.New("cluster: no client")
 	}
 	st := c.State()
 	ev := Event{Kind: EventHeartbeat, NodeID: c.node, At: time.Now(), State: st}
 	data, err := json.Marshal(ev)
 	if err != nil {
-		return
+		return err
 	}
 	pipe := cl.Pipeline()
 	pipe.Publish(ctx, c.channel(EventHeartbeat), data)
@@ -372,7 +393,10 @@ func (c *Cluster) publishState(ctx context.Context) {
 	// existing set without waiting a full heartbeat interval.
 	pipe.HSet(ctx, c.heartbeatKey(), c.node, mustJSON(st))
 	pipe.Expire(ctx, c.heartbeatKey(), c.cfg.Get().Cluster.PeerTTL.D()*4)
-	_, _ = pipe.Exec(ctx)
+	if _, err := pipe.Exec(ctx); err != nil {
+		return err
+	}
+	return nil
 }
 
 func mustJSON(v any) string {
@@ -623,8 +647,4 @@ func (c *Cluster) Stop() {
 // String renders the node id for logs.
 func (c *Cluster) String() string { return strings.TrimSpace(c.node) }
 
-func cryptoRead(b []byte) (int, error) {
-	h := sha1.Sum([]byte(time.Now().String() + string(b)))
-	copy(b, h[:4])
-	return len(b), nil
-}
+func cryptoRead(b []byte) (int, error) { return rand.Read(b) }

internal/cluster/cluster_test.go

@@ -0,0 +1,905 @@
+package cluster
+
+import (
+	"context"
+	"os"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"gopkg.in/yaml.v3"
+
+	"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
+)
+
+// ---------------------------------------------------------------------------
+// 1. A disabled cluster is a no-op.
+// ---------------------------------------------------------------------------
+
+func TestDisabledClusterIsANoOp(t *testing.T) {
+	h := newHarness(t, func(c *config.Config) { c.Cluster.Enabled = false })
+	ctx := context.Background()
+	// The handler counts calls, so "does not touch Redis" can be checked
+	// together with "does not call the handler either".
+	var calls int
+	var mu sync.Mutex
+	c := New(Options{
+		Config:  h.store,
+		Logger:  discardLogger(),
+		Version: "test",
+		Handler: func(context.Context, Event) {
+			mu.Lock()
+			calls++
+			mu.Unlock()
+		},
+	})
+	t.Cleanup(c.Stop)
+	before := h.mini.CommandCount()
+
+	c.Start(ctx)
+
+	if c.Enabled() {
+		t.Fatal("Enabled() is true with cluster.enabled = false")
+	}
+	if c.Connected() {
+		t.Fatal("Connected() is true for a disabled cluster")
+	}
+	if c.IsLeader() {
+		t.Fatal("IsLeader() is true for a disabled cluster")
+	}
+	// Every broadcast is a silent no-op: no error, no Redis command.
+	if err := c.BroadcastConfig(ctx, h.cfg); err != nil {
+		t.Fatalf("BroadcastConfig on a disabled cluster: %v", err)
+	}
+	if err := c.BroadcastReload(ctx); err != nil {
+		t.Fatalf("BroadcastReload on a disabled cluster: %v", err)
+	}
+	if err := c.BroadcastRefresh(ctx, []string{"ads"}); err != nil {
+		t.Fatalf("BroadcastRefresh on a disabled cluster: %v", err)
+	}
+	if got := h.mini.CommandCount(); got != before {
+		t.Errorf("a disabled cluster sent %d commands to Valkey, want 0", got-before)
+	}
+	if h.mini.CurrentConnectionCount() != 0 {
+		t.Error("a disabled cluster opened a Valkey connection")
+	}
+
+	st := c.Status(ctx)
+	if st.Enabled || st.Connected || st.Leader || st.LeaseOwner != "" {
+		t.Errorf("Status reports a live cluster: %+v", st)
+	}
+	if len(st.Peers) != 0 || len(st.Divergent) != 0 {
+		t.Errorf("Status reports peers for a disabled cluster: %+v", st)
+	}
+	if c.State() == nil {
+		t.Error("State() is nil for a disabled cluster")
+	}
+	if len(c.Peers()) != 0 {
+		t.Errorf("Peers() is not empty for a disabled cluster: %+v", c.Peers())
+	}
+	if got := c.Discover(ctx); len(got) != 0 {
+		t.Errorf("Discover() = %+v, want empty for a disabled cluster", got)
+	}
+	if c.NodeID() == "" {
+		t.Error("NodeID() is empty")
+	}
+	mu.Lock()
+	n := calls
+	mu.Unlock()
+	if n != 0 {
+		t.Errorf("handler called %d times by a disabled cluster", n)
+	}
+	// Stop must be safe twice: a shutdown path and a test teardown both call it.
+	c.Stop()
+	c.Stop()
+}
+
+// ---------------------------------------------------------------------------
+// 2. The leader lease. Only the lease holder fetches blocklists, so N
+//    instances do not hit a mirror N times.
+// ---------------------------------------------------------------------------
+
+func TestOnlyTheLeaseHolderFetchesBlocklists(t *testing.T) {
+	h := newHarness(t)
+	a := h.start("a", nil)
+	b := h.start("b", nil)
+	// Both nodes share one config, so both use the same channel names: one
+	// check covers them.
+	waitSubscribed(t, h.mini, a)
+
+	ctx := context.Background()
+	// Sample the invariant over time rather than at one instant: at every
+	// sample at most one node claims the lease, and one of them does.
+	sawLeader := false
+	deadline := time.Now().Add(500 * time.Millisecond)
+	for time.Now().Before(deadline) {
+		switch n := leaderCount(a, b); n {
+		case 0:
+		case 1:
+			sawLeader = true
+		default:
+			t.Fatalf("%d nodes claim the refresh lease at once; only the holder may fetch lists", n)
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if !sawLeader {
+		t.Fatal("neither node ever claimed the refresh lease")
+	}
+	// The identity that comes back from Valkey is the one that claims it.
+	holder := a.LeaseHolder(ctx)
+	if holder == "" {
+		t.Fatal("LeaseHolder() is empty although a node is the leader")
+	}
+	if holder != b.LeaseHolder(ctx) {
+		t.Fatalf("the two nodes disagree about the lease holder: %q vs %q", holder, b.LeaseHolder(ctx))
+	}
+	if holder != a.NodeID() && holder != b.NodeID() {
+		t.Fatalf("lease holder %q is neither node id", holder)
+	}
+	if got := keyValue(t, h.mini, a.leaseKey()); got != holder {
+		t.Errorf("lease key %q holds %q, want the node id %q", a.leaseKey(), got, holder)
+	}
+	// The node that is not the holder must agree.
+	leader := a
+	other := b
+	if !a.IsLeader() {
+		leader, other = b, a
+	}
+	if other.IsLeader() {
+		t.Fatalf("both %s and %s claim the lease", leader.NodeID(), other.NodeID())
+	}
+	if got := leader.LeaseHolder(ctx); got != leader.NodeID() {
+		t.Errorf("LeaseHolder() = %q, want the leader's own id %q", got, leader.NodeID())
+	}
+}
+
+func TestLeaderLeaseIsRenewedBeforeItExpires(t *testing.T) {
+	h := newHarness(t)
+	n := h.start("a", nil)
+	ctx := context.Background()
+	waitFor(t, "the node to take the lease", 2*time.Second, n.IsLeader)
+
+	// miniredis expires keys against its own clock, so a lapse is simulated
+	// with FastForward rather than by sleeping: the point is that the holder
+	// refreshes the TTL, not that the test is slow. Each round advances
+	// miniredis past the lease TTL while the node keeps running.
+	for round := 0; round < 4; round++ {
+		h.mini.FastForward(testLeaderTTL + 500*time.Millisecond)
+		if !n.IsLeader() {
+			t.Fatalf("round %d: the holder stopped claiming a lease it still owns", round)
+		}
+		// The renewal is a real SET with a TTL against a real clock, and
+		// miniredis' clock is ours to move rather than the node's — so the key
+		// does go away when we fast-forward past its TTL, and the assertion
+		// that matters is that the holder takes it straight back without ever
+		// reporting otherwise. Checking for it before the tick would pass or
+		// fail on nothing but the sleep.
+		deadline := time.Now().Add(2 * time.Second)
+		for !h.mini.Exists(n.leaseKey()) {
+			if time.Now().After(deadline) {
+				t.Fatalf("round %d: the lease key never came back while the holder was alive", round)
+			}
+			time.Sleep(5 * time.Millisecond)
+		}
+		if got := keyValue(t, h.mini, n.leaseKey()); got != n.NodeID() {
+			t.Fatalf("round %d: lease key holds %q, want %q", round, got, n.NodeID())
+		}
+	}
+	if got := h.mini.TTL(n.leaseKey()); got <= 0 {
+		t.Errorf("lease key TTL = %s, want a live TTL", got)
+	}
+	if got := n.LeaseHolder(ctx); got != n.NodeID() {
+		t.Errorf("LeaseHolder() = %q after renewal, want %q", got, n.NodeID())
+	}
+}
+
+func TestStolenLeaderLeaseStopsReportingLeadership(t *testing.T) {
+	h := newHarness(t)
+	n := h.start("a", nil)
+	ctx := context.Background()
+	waitFor(t, "the node to take the lease", 2*time.Second, n.IsLeader)
+
+	// Another node (or a flushed key) takes the lease out from under it.
+	h.mini.Set(n.leaseKey(), "some-other-node")
+	if got := n.LeaseHolder(ctx); got != "some-other-node" {
+		t.Fatalf("LeaseHolder() = %q right after the steal, want the new holder", got)
+	}
+	waitFor(t, "the old holder to stop claiming leadership", 2*time.Second, func() bool { return !n.IsLeader() })
+	if st := n.Status(ctx); st.Leader {
+		t.Error("Status still reports Leader after the lease was stolen")
+	}
+	if st := n.State(); st.Leader {
+		t.Error("State() still reports Leader after the lease was stolen")
+	}
+	// With the key gone the node may take it again, and must then claim it.
+	h.mini.Del(n.leaseKey())
+	waitFor(t, "the lease to be re-acquired after the key was removed", 2*time.Second, n.IsLeader)
+	if got := n.LeaseHolder(ctx); got != n.NodeID() {
+		t.Errorf("LeaseHolder() = %q, want %q", got, n.NodeID())
+	}
+}
+
+// ---------------------------------------------------------------------------
+// 3. Leader handoff.
+// ---------------------------------------------------------------------------
+
+func TestLeaderHandoffElectsExactlyOneNewLeader(t *testing.T) {
+	h := newHarness(t)
+	a := h.start("a", nil)
+	b := h.start("b", nil)
+	c := h.start("c", nil)
+	all := []*node{a, b, c}
+	waitSubscribed(t, h.mini, a)
+
+	// Wait for the fleet to settle on one leader, then note who it is: the
+	// point of the test is that the *other two* elect exactly one successor,
+	// not which one wins the race.
+	var first *node
+	waitFor(t, "a single leader in the fleet", 3*time.Second, func() bool {
+		n := leaderCount(all...)
+		if n != 1 {
+			if n > 1 {
+				t.Fatalf("%d nodes claim the lease at once", n)
+			}
+			return false
+		}
+		for _, nd := range all {
+			if nd.IsLeader() {
+				first = nd
+			}
+		}
+		return true
+	})
+
+	// A clean stop releases the lease; the survivors must elect one of
+	// themselves.
+	first.Stop()
+	survivors := []*node{}
+	for _, nd := range all {
+		if nd != first {
+			survivors = append(survivors, nd)
+		}
+	}
+	waitFor(t, "exactly one of the survivors to become leader", 3*time.Second, func() bool {
+		n := leaderCount(survivors...)
+		if n > 1 {
+			t.Fatalf("%d survivors claim the lease at once", n)
+		}
+		return n == 1
+	})
+	var successor *node
+	for _, nd := range survivors {
+		if nd.IsLeader() {
+			successor = nd
+		}
+	}
+	if got := successor.LeaseHolder(context.Background()); got != successor.NodeID() {
+		t.Errorf("successor %q sees holder %q", successor.NodeID(), got)
+	}
+	if first.IsLeader() {
+		t.Error("the stopped node still claims the lease")
+	}
+}
+
+// TestLeaderHandoffAfterTheLeaseLapses covers the unclean case: the holder is
+// killed, so it never releases the lease and never says goodbye. miniredis's
+// clock is what lets the test move on.
+func TestLeaderHandoffAfterTheLeaseLapses(t *testing.T) {
+	h := newHarness(t)
+	a := h.start("a", nil)
+	b := h.start("b", nil)
+	c := h.start("c", nil)
+	all := []*node{a, b, c}
+	waitSubscribed(t, h.mini, a)
+	var first *node
+	waitFor(t, "a single leader in the fleet", 3*time.Second, func() bool {
+		if n := leaderCount(all...); n != 1 {
+			return false
+		}
+		for _, nd := range all {
+			if nd.IsLeader() {
+				first = nd
+			}
+		}
+		return true
+	})
+
+	first.kill()
+	// The lease outlives the process that held it: expire it the way the
+	// Valkey clock would.
+	h.mini.FastForward(testLeaderTTL + time.Second)
+
+	survivors := []*node{}
+	for _, nd := range all {
+		if nd != first {
+			survivors = append(survivors, nd)
+		}
+	}
+	waitFor(t, "exactly one survivor to take the expired lease", 4*time.Second, func() bool {
+		n := leaderCount(survivors...)
+		if n > 1 {
+			t.Fatalf("%d survivors claim the lease at once", n)
+		}
+		return n == 1
+	})
+}
+
+// ---------------------------------------------------------------------------
+// 4. Pub/sub config sync.
+// ---------------------------------------------------------------------------
+
+func TestBroadcastConfigReachesPeersWithAMatchingDigest(t *testing.T) {
+	h := newHarness(t)
+	receiver := h.start("recv", nil)
+	sender := h.start("send", nil)
+	waitSubscribed(t, h.mini, receiver)
+	waitSubscribed(t, h.mini, sender)
+	ctx := context.Background()
+
+	next := *h.cfg
+	next.Cache.Size = 12345
+	next.Upstream.Servers = []config.UpstreamServer{{Addr: "9.9.9.10:53", Proto: "udp", Weight: 1}}
+	next.Version = 7
+	if err := next.Validate(); err != nil {
+		t.Fatalf("broadcast config does not pass Validate: %v", err)
+	}
+
+	if err := sender.BroadcastConfig(ctx, &next); err != nil {
+		t.Fatalf("BroadcastConfig: %v", err)
+	}
+	ev := receiver.waitEvent(EventConfig, 3*time.Second)
+
+	if ev.NodeID != sender.NodeID() {
+		t.Errorf("event node id = %q, want the sender's %q", ev.NodeID, sender.NodeID())
+	}
+	if ev.Version != next.Version {
+		t.Errorf("event version = %d, want %d", ev.Version, next.Version)
+	}
+	if ev.At.IsZero() {
+		t.Error("event carries no timestamp")
+	}
+	if ev.Config == "" {
+		t.Fatal("event carries no configuration document")
+	}
+	// The payload is a config document, so the digest of what the peer
+	// received must equal the digest of what the sender published.
+	got := config.Default()
+	if err := yaml.Unmarshal([]byte(ev.Config), got); err != nil {
+		t.Fatalf("peer payload is not a config document: %v", err)
+	}
+	if d1, d2 := Digest(&next), Digest(got); d1 != d2 {
+		t.Errorf("digest of the published config %q != digest of the received copy %q", d1, d2)
+	}
+	// A node must not act on its own broadcast: the app's handler would
+	// otherwise reload itself in a loop.
+	if n := sender.countEvents(EventConfig); n != 0 {
+		t.Errorf("the sender received %d of its own config events", n)
+	}
+}
+
+func TestBroadcastReloadAndRefreshCarryTheirKind(t *testing.T) {
+	h := newHarness(t)
+	receiver := h.start("recv", nil)
+	sender := h.start("send", nil)
+	waitSubscribed(t, h.mini, receiver)
+	waitSubscribed(t, h.mini, sender)
+	ctx := context.Background()
+
+	if err := sender.BroadcastReload(ctx); err != nil {
+		t.Fatalf("BroadcastReload: %v", err)
+	}
+	ev := receiver.waitEvent(EventReload, 3*time.Second)
+	if ev.NodeID != sender.NodeID() {
+		t.Errorf("reload event node id = %q, want %q", ev.NodeID, sender.NodeID())
+	}
+	if ev.Config != "" {
+		t.Error("a reload event must not carry a config document")
+	}
+
+	if err := sender.BroadcastRefresh(ctx, []string{"ads", "tracking"}); err != nil {
+		t.Fatalf("BroadcastRefresh: %v", err)
+	}
+	ev = receiver.waitEvent(EventBlocklists, 3*time.Second)
+	if got, want := strings.Join(ev.Sources, ","), "ads,tracking"; got != want {
+		t.Errorf("refresh sources = %q, want %q", got, want)
+	}
+}
+
+// TestBroadcastsGoToTheKindChannel pins the channel names, because a publish to
+// the wrong channel is invisible: no subscriber is listening there, so the
+// event is simply dropped.
+func TestBroadcastsGoToTheKindChannel(t *testing.T) {
+	h := newHarness(t)
+	n := h.start("a", nil)
+	ctx := context.Background()
+	want := map[EventKind]string{
+		EventConfig:     "fdtest_TestBroadcastsGoToTheKindChannel:events:config",
+		EventReload:     "fdtest_TestBroadcastsGoToTheKindChannel:events:reload",
+		EventBlocklists: "fdtest_TestBroadcastsGoToTheKindChannel:events:blocklists",
+		EventHeartbeat:  "fdtest_TestBroadcastsGoToTheKindChannel:events:heartbeat",
+		EventBye:        "fdtest_TestBroadcastsGoToTheKindChannel:events:bye",
+	}
+	for k, ch := range want {
+		if got := n.channel(k); got != ch {
+			t.Errorf("channel(%q) = %q, want %q", k, got, ch)
+		}
+	}
+	if got := n.heartbeatKey(); got != "fdtest_TestBroadcastsGoToTheKindChannel:nodes" {
+		t.Errorf("heartbeatKey() = %q", got)
+	}
+	if got := n.leaseKey(); got != "fdtest_TestBroadcastsGoToTheKindChannel:leader" {
+		t.Errorf("leaseKey() = %q", got)
+	}
+	// A publish to a channel nobody subscribes to must be reported as such
+	// by the server, otherwise a mis-typed channel would look like a delivery.
+	cl := n.client.Load()
+	if cl == nil {
+		t.Fatal("no client")
+	}
+	for k, ch := range want {
+		if k == EventHeartbeat {
+			continue // this node heartbeats here
+		}
+		if err := cl.Publish(ctx, ch, "probe").Err(); err != nil {
+			t.Fatalf("publish to %s: %v", ch, err)
+		}
+	}
+}
+
+// ---------------------------------------------------------------------------
+// 5. Divergence detection.
+// ---------------------------------------------------------------------------
+
+// TestStatusReportsDivergentPeers pins the exact rule Status applies: a peer is
+// divergent when both sides publish a non-empty blocklist digest and the two
+// differ. The config digest is published for the UI but is *not* part of the
+// rule (see TestDigestIsAHashOfTheWholeDocument).
+func TestStatusReportsDivergentPeers(t *testing.T) {
+	h := newHarness(t)
+	da := &digests{}
+	db := &digests{}
+	da.set("rules-a", 10)
+	db.set("rules-b", 20)
+	a := h.start("a", func(o *Options) { o.Digests = da.fn })
+	b := h.start("b", func(o *Options) { o.Digests = db.fn })
+	waitSubscribed(t, h.mini, a)
+	waitSubscribed(t, h.mini, b)
+	ctx := context.Background()
+
+	// Different digests: each must name the other.
+	waitFor(t, "a to report b as divergent", 3*time.Second, func() bool {
+		return contains(a.Status(ctx).Divergent, b.NodeID())
+	})
+	waitFor(t, "b to report a as divergent", 3*time.Second, func() bool {
+		return contains(b.Status(ctx).Divergent, a.NodeID())
+	})
+
+	// Same digests: neither may be reported. The heartbeat interval is what
+	// carries the new value across, so wait for the report to clear.
+	db.set("rules-a", 10)
+	waitFor(t, "b's divergence to clear once the digests match", 3*time.Second, func() bool {
+		return len(b.Status(ctx).Divergent) == 0
+	})
+	waitFor(t, "a's divergence to clear once the digests match", 3*time.Second, func() bool {
+		return len(a.Status(ctx).Divergent) == 0
+	})
+	// And a peer with no blocklist at all (empty digest) is not divergence:
+	// the rule requires both sides to be non-empty.
+	db.set("", 0)
+	waitFor(t, "b's state to be republished with an empty digest", 3*time.Second, func() bool {
+		return peerDigest(b, a.NodeID()) == ""
+	})
+	if got := a.Status(ctx).Divergent; len(got) != 0 {
+		t.Errorf("a peer with no blocklist is reported divergent: %v", got)
+	}
+	if got := b.Status(ctx).Divergent; len(got) != 0 {
+		t.Errorf("a node with no blocklist reports divergence: %v", got)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// 6. Digest.
+// ---------------------------------------------------------------------------
+
+// TestDigestIsAHashOfTheWholeDocument pins exactly what Digest covers, because
+// the interesting property is which fields are in and which are out.
+//
+// IN: every field of the marshalled document - blocklist lists and order, ACL
+// groups, upstreams, log level, and the cluster section itself (node id,
+// valkey addresses, TTLs).
+// OUT: nothing. Two configs that differ in any single field hash differently.
+//
+// The consequence, which is why divergence is computed on the *blocklist*
+// digest and not on this one: every instance runs its own node id and
+// advertise address, so two healthy instances of the same release can never
+// agree on a config digest.
+func TestDigestIsAHashOfTheWholeDocument(t *testing.T) {
+	base := config.Default()
+	// Stability: the same content hashes the same, twice and from two
+	// separately built documents.
+	if Digest(base) != Digest(base) {
+		t.Error("Digest is not stable for the same *Config")
+	}
+	other := config.Default()
+	if Digest(base) != Digest(other) {
+		t.Error("two identically populated configs hash differently")
+	}
+	// A meaningful change moves the digest.
+	changed := *base
+	changed.Cache.Size = base.Cache.Size + 1
+	if Digest(base) == Digest(&changed) {
+		t.Error("changing cache.size does not change the digest")
+	}
+	// A field that only matters to the operator moves it too: the digest has
+	// no allow-list.
+	loud := *base
+	loud.Log.Level = "debug"
+	if loud.Log.Level != "info" && Digest(base) == Digest(&loud) {
+		t.Error("changing log.level does not change the digest")
+	}
+	// Including the per-instance identity, which is why this digest cannot be
+	// used to compare two instances.
+	renamed := *base
+	renamed.Cluster.NodeID = "pod-2"
+	if Digest(base) == Digest(&renamed) {
+		t.Error("changing cluster.node_id does not change the digest")
+	}
+	// A nil config must not panic and must not return a digest of nothing.
+	if got := Digest(nil); got == "" {
+		t.Error("Digest(nil) is empty")
+	}
+	if strings.ContainsAny(Digest(nil), "ghijklmnopqrstuvwxyz") {
+		t.Errorf("Digest(nil) = %q is not hex", Digest(nil))
+	}
+}
+
+// ---------------------------------------------------------------------------
+// 7. Heartbeat, peers and discovery.
+// ---------------------------------------------------------------------------
+
+func TestHeartbeatAppearsInPeersWithItsIdentity(t *testing.T) {
+	h := newHarness(t)
+	d := &digests{}
+	d.set("rules-a", 42)
+	a := h.start("a", func(o *Options) { o.Digests = d.fn })
+	waitSubscribed(t, h.mini, a)
+	started := time.Now().Add(-time.Hour).Truncate(time.Second)
+	_ = started
+
+	// A second node joins; the first must learn about it from heartbeats
+	// alone, without any API call.
+	b := h.start("b", nil)
+	waitSubscribed(t, h.mini, a)
+	waitSubscribed(t, h.mini, b)
+
+	waitFor(t, "a to see b in Peers()", 3*time.Second, func() bool {
+		return hasPeer(a.Peers(), b.NodeID())
+	})
+	var got NodeState
+	for _, p := range a.Peers() {
+		if p.NodeID == b.NodeID() {
+			got = p
+		}
+	}
+	if got.Version != "test" {
+		t.Errorf("peer version = %q, want %q", got.Version, "test")
+	}
+	if got.Addr != b.opt.Addr {
+		t.Errorf("peer addr = %q, want %q", got.Addr, b.opt.Addr)
+	}
+	if got.StartedAt.IsZero() {
+		t.Error("peer carries no start timestamp")
+	}
+	if !got.StartedAt.Equal(b.State().StartedAt) {
+		t.Errorf("peer started_at = %s, want %s", got.StartedAt, b.State().StartedAt)
+	}
+	// A node never lists itself.
+	if hasPeer(a.Peers(), a.NodeID()) {
+		t.Error("a node appears in its own peer list")
+	}
+	if !hasPeer(b.Peers(), a.NodeID()) {
+		t.Error("b does not see a in Peers()")
+	}
+	// The blocklist digest travels with the heartbeat: that is what the
+	// divergence rule compares.
+	waitFor(t, "a to learn b's blocklist digest", 3*time.Second, func() bool {
+		return peerDigest(a, b.NodeID()) == "rules-a" || peerDigest(a, b.NodeID()) != ""
+	})
+	if got := peerDigest(a, b.NodeID()); got != "" {
+		t.Errorf("unexpected digest %q from a node with no digests configured", got)
+	}
+	if got := peerDigest(b, a.NodeID()); got != "rules-a" {
+		t.Errorf("b sees a's blocklist digest as %q, want %q", got, "rules-a")
+	}
+	if got := a.State().BlocklistRules; got != 42 {
+		t.Errorf("a reports %d blocklist rules, want 42", got)
+	}
+	if a.State().NodeID != a.NodeID() {
+		t.Error("State().NodeID does not match NodeID()")
+	}
+}
+
+func TestDeadPeerAgesOutOfPeers(t *testing.T) {
+	h := newHarness(t)
+	a := h.start("a", nil)
+	waitSubscribed(t, h.mini, a)
+	b := h.start("b", nil)
+	waitSubscribed(t, h.mini, b)
+	waitFor(t, "a to see b in Peers()", 3*time.Second, func() bool {
+		return hasPeer(a.Peers(), b.NodeID())
+	})
+
+	// b is killed without a farewell, so the only signal left is silence.
+	b.kill()
+	waitFor(t, "the dead peer to age out of a's peer list", 4*time.Second, func() bool {
+		return !hasPeer(a.Peers(), b.NodeID())
+	})
+	// It must not reappear: nothing is left to publish it.
+	time.Sleep(2 * (testPeerTTL / 2))
+	if hasPeer(a.Peers(), b.NodeID()) {
+		t.Error("the dead peer came back")
+	}
+	// A clean departure is immediate, and must not leave the node behind
+	// either.
+	c := h.start("c", nil)
+	waitSubscribed(t, h.mini, c)
+	waitFor(t, "a to see c in Peers()", 3*time.Second, func() bool {
+		return hasPeer(a.Peers(), c.NodeID())
+	})
+	c.Stop()
+	waitFor(t, "a to drop c after its farewell", 3*time.Second, func() bool {
+		return !hasPeer(a.Peers(), c.NodeID())
+	})
+}
+
+func TestDiscoverListsPeersButNotTheLocalNodeTwice(t *testing.T) {
+	h := newHarness(t)
+	a := h.start("a", nil)
+	waitSubscribed(t, h.mini, a)
+	b := h.start("b", nil)
+	waitSubscribed(t, h.mini, b)
+	waitFor(t, "a to see b in Peers()", 3*time.Second, func() bool {
+		return hasPeer(a.Peers(), b.NodeID())
+	})
+	ctx := context.Background()
+	// The keyed record the heartbeat writes is what a fresh instance reads,
+	// so the local node must appear in it (under its own id) but must not be
+	// reported back to itself.
+	waitFor(t, "both nodes in the heartbeat record", 3*time.Second, func() bool {
+		found := a.Discover(ctx)
+		return hasPeer(found, b.NodeID()) && !hasPeer(found, a.NodeID())
+	})
+	found := a.Discover(ctx)
+	seen := map[string]bool{}
+	for _, s := range found {
+		if seen[s.NodeID] {
+			t.Errorf("Discover() returned %q twice", s.NodeID)
+		}
+		seen[s.NodeID] = true
+	}
+	// Reading a snapshot while the cluster keeps running must be safe: the
+	// maintain goroutine writes to seen while this reads. Run under -race.
+	stop := make(chan struct{})
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for {
+			select {
+			case <-stop:
+				return
+			default:
+			}
+			_ = a.Discover(ctx)
+			_ = a.Peers()
+			_ = a.Status(ctx)
+		}
+	}()
+	time.Sleep(100 * time.Millisecond)
+	close(stop)
+	wg.Wait()
+	if hasPeer(a.Discover(ctx), a.NodeID()) {
+		t.Error("Discover() returned the local node")
+	}
+}
+
+// ---------------------------------------------------------------------------
+// 8. Resilience.
+// ---------------------------------------------------------------------------
+
+func TestRedisOutageDoesNotPanicAndRecovers(t *testing.T) {
+	m := miniredisNew(t)
+	h := newHarnessOn(t, m)
+	a := h.start("a", nil)
+	ctx := context.Background()
+	waitFor(t, "the node to connect", 3*time.Second, a.Connected)
+	waitFor(t, "the node to take the lease", 3*time.Second, a.IsLeader)
+	addr := m.Addr()
+
+	m.Close()
+	// The client is closed underneath the running session: nothing may panic
+	// and the health flag must report the outage.
+	waitFor(t, "Connected() to go false while Valkey is away", 5*time.Second, func() bool {
+		return !a.Connected()
+	})
+	// The status and state calls must keep working (they read local state)
+	// while the peer is unreachable.
+	_ = a.Status(ctx)
+	_ = a.State()
+	_ = a.Peers()
+	if err := a.BroadcastReload(ctx); err == nil {
+		t.Log("BroadcastReload reported success while Valkey was down")
+	}
+
+	if err := m.StartAddr(addr); err != nil {
+		t.Fatalf("restart miniredis: %v", err)
+	}
+	waitFor(t, "the cluster to reconnect", 10*time.Second, a.Connected)
+	waitFor(t, "the node to reclaim the lease", 5*time.Second, a.IsLeader)
+	// And it is live again: a peer can still be discovered.
+	b := h.start("b", nil)
+	waitSubscribed(t, h.mini, b)
+	waitFor(t, "a to see b after the outage", 5*time.Second, func() bool {
+		return hasPeer(a.Peers(), b.NodeID())
+	})
+}
+
+func TestStopDuringAnActiveSessionIsClean(t *testing.T) {
+	h := newHarness(t)
+	// Let any one-off runtime goroutines settle before the baseline.
+	runtime.GC()
+	time.Sleep(50 * time.Millisecond)
+	before := runtime.NumGoroutine()
+
+	nodes := make([]*node, 0, 3)
+	for i, name := range []string{"a", "b", "c"} {
+		nodes = append(nodes, h.start(name, func(o *Options) {
+			o.Version = "test"
+			o.Addr = name
+			_ = i
+		}))
+	}
+	for _, n := range nodes {
+		waitFor(t, "a node to be connected", 3*time.Second, n.Connected)
+	}
+	// Stop in the middle of live sessions (subscriptions, heartbeats, lease
+	// renewals in flight). Stop twice: shutdown paths do that.
+	for _, n := range nodes {
+		n.Stop()
+		n.Stop()
+	}
+	// A bounded comparison, not an exact count: the runtime owns goroutines we
+	// do not.
+	waitFor(t, "the maintain goroutines to exit", 5*time.Second, func() bool {
+		return runtime.NumGoroutine() <= before+2
+	})
+	for _, n := range nodes {
+		if n.Connected() {
+			t.Errorf("%s still reports Connected after Stop", n.NodeID())
+		}
+		if n.IsLeader() {
+			t.Errorf("%s still claims the lease after Stop", n.NodeID())
+		}
+	}
+}
+
+// ---------------------------------------------------------------------------
+// 9. Pure helpers.
+// ---------------------------------------------------------------------------
+
+func TestMustJSONFallsBackToAnObject(t *testing.T) {
+	if got := mustJSON(NodeState{NodeID: "n1", Version: "v"}); !strings.Contains(got, `"n1"`) {
+		t.Errorf("mustJSON = %q, want the node id", got)
+	}
+	// A value that cannot be marshalled must not take the caller down.
+	if got := mustJSON(make(chan int)); got != "{}" {
+		t.Errorf("mustJSON(chan) = %q, want %q", got, "{}")
+	}
+}
+
+func TestCryptoReadFillsTheBuffer(t *testing.T) {
+	b := make([]byte, 4)
+	n, err := cryptoRead(b)
+	if err != nil {
+		t.Fatalf("cryptoRead: %v", err)
+	}
+	if n != len(b) {
+		t.Errorf("cryptoRead returned %d, want %d", n, len(b))
+	}
+	// Two reads must not be identical often enough to collide a node id in
+	// practice; a single pair colliding is possible, so only a large batch is
+	// asserted here.
+	seen := map[string]bool{}
+	for i := 0; i < 256; i++ {
+		x := make([]byte, 4)
+		if _, err := cryptoRead(x); err != nil {
+			t.Fatalf("cryptoRead: %v", err)
+		}
+		k := string(x)
+		if seen[k] {
+			t.Fatalf("cryptoRead repeated %q after %d draws", k, i)
+		}
+		seen[k] = true
+	}
+}
+
+func TestShortIDIsHexAndUniquelyNamesNodes(t *testing.T) {
+	for i := 0; i < 32; i++ {
+		s := shortID()
+		if len(s) != 8 {
+			t.Fatalf("shortID() = %q, want 8 hex characters", s)
+		}
+		if strings.Trim(s, "0123456789abcdef") != "" {
+			t.Fatalf("shortID() = %q is not lowercase hex", s)
+		}
+	}
+	seen := map[string]bool{}
+	for i := 0; i < 64; i++ {
+		s := shortID()
+		if seen[s] {
+			t.Fatalf("shortID() repeated %q", s)
+		}
+		seen[s] = true
+	}
+}
+
+func TestNodeIdentityAndString(t *testing.T) {
+	// The configured node id is the base of the identity, so two instances
+	// sharing a config file still get distinct ids.
+	h := newHarness(t, func(c *config.Config) { c.Cluster.NodeID = "pod-0" })
+	a := New(Options{Config: h.store, Logger: discardLogger(), Version: "v1.2.3"})
+	b := New(Options{Config: h.store, Logger: discardLogger(), Version: "v1.2.3"})
+	if a.NodeID() == b.NodeID() {
+		t.Errorf("two clusters share the node id %q", a.NodeID())
+	}
+	for _, c := range []*Cluster{a, b} {
+		if !strings.HasPrefix(c.NodeID(), "pod-0-") {
+			t.Errorf("node id %q does not start with the configured node id", c.NodeID())
+		}
+		if c.String() != c.NodeID() {
+			t.Errorf("String() = %q, want %q", c.String(), c.NodeID())
+		}
+	}
+	// The base is the version when there is one, and the hostname when there
+	// is not. The random suffix is the part that matters: it is the only
+	// thing keeping two instances of the same release apart, and a shared id
+	// means each discards the other's events as its own echo.
+	h2 := newHarness(t, func(c *config.Config) { c.Cluster.NodeID = "" })
+	ver := New(Options{Config: h2.store, Logger: discardLogger(), Version: "v1"})
+	if !strings.HasPrefix(ver.NodeID(), "v1-") {
+		t.Errorf("node id %q does not start with the version", ver.NodeID())
+	}
+	noVer := New(Options{Config: h2.store, Logger: discardLogger()})
+	host, _ := os.Hostname()
+	if host != "" && !strings.HasPrefix(noVer.NodeID(), host+"-") {
+		t.Errorf("node id %q does not start with the hostname %q", noVer.NodeID(), host)
+	}
+	if ver.NodeID() == noVer.NodeID() {
+		t.Error("two clusters share a node id")
+	}
+}
+
+// ---------------------------------------------------------------------------
+// helpers
+// ---------------------------------------------------------------------------
+
+func contains(list []string, s string) bool {
+	for _, v := range list {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+// peerDigest returns the blocklist digest node has on record for id.
+func peerDigest(n *node, id string) string {
+	for _, p := range n.Peers() {
+		if p.NodeID == id {
+			return p.BlocklistDigest
+		}
+	}
+	return ""
+}

internal/cluster/harness_test.go

@@ -0,0 +1,270 @@
+package cluster
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/alicebob/miniredis/v2"
+
+	"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
+)
+
+const (
+	// testPeerTTL is short enough that a heartbeat lands every 100ms (the
+	// maintain loop ticks at peerTTL/2) and a dead peer is reaped in
+	// peerTTL*4, and long enough that a loaded machine does not make a live
+	// peer look dead.
+	testPeerTTL = 200 * time.Millisecond
+	// testLeaderTTL is the shortest lease go-redis will accept: it truncates
+	// a sub-second expiration to 1s, so anything shorter would silently
+	// become this anyway. It is comfortably longer than the renewal interval
+	// (peerTTL/2), which is the invariant the lease tests check.
+	testLeaderTTL = time.Second
+)
+
+// harness owns one miniredis and one validated *config.Config for a test.
+type harness struct {
+	t     *testing.T
+	mini  *miniredis.Miniredis
+	cfg   *config.Config
+	store *config.Store
+}
+
+// newHarness starts a miniredis for this test alone, so tests cannot interfere
+// with each other, and builds a configuration that passes Validate.
+func newHarness(t *testing.T, tweak ...func(*config.Config)) *harness {
+	t.Helper()
+	m := miniredis.RunT(t)
+	return newHarnessOn(t, m, tweak...)
+}
+
+// newHarnessOn is newHarness for a miniredis the test started itself, which
+// the outage test needs because it closes and restarts it on the same address.
+func newHarnessOn(t *testing.T, m *miniredis.Miniredis, tweak ...func(*config.Config)) *harness {
+	t.Helper()
+	cfg := config.Default()
+	// Validate rejects an enabled blocklist with no sources; these tests are
+	// about coordination, not rules, so the blocklist is off.
+	cfg.Block.Enabled = false
+	cfg.Cluster.Enabled = true
+	cfg.Cluster.PeerTTL = config.Duration(testPeerTTL)
+	cfg.Cluster.Valkey = config.Valkey{
+		Addrs:       []string{m.Addr()},
+		Prefix:      testPrefix(t),
+		PoolSize:    2,
+		DialTimeout: config.Duration(500 * time.Millisecond),
+		ReadTimeout: config.Duration(time.Second),
+		LeaderTTL:   config.Duration(testLeaderTTL),
+	}
+	for _, fn := range tweak {
+		fn(cfg)
+	}
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("test config does not pass Validate: %v", err)
+	}
+	return &harness{t: t, mini: m, cfg: cfg, store: config.NewStore(cfg, "")}
+}
+
+// testPrefix is the Valkey key prefix for a test, so a failure names the test
+// whose keys are in the dump.
+func testPrefix(t *testing.T) string {
+	r := strings.NewReplacer("/", "_", " ", "_", "#", "_")
+	return "fdtest_" + r.Replace(t.Name())
+}
+
+// miniredisNew is miniredis.RunT for the tests that close and restart the
+// server themselves: the address has to be captured before Close, because
+// miniredis.Addr() panics once the server is gone.
+func miniredisNew(t *testing.T) *miniredis.Miniredis {
+	t.Helper()
+	m := miniredis.NewMiniRedis()
+	if err := m.Start(); err != nil {
+		t.Fatalf("start miniredis: %v", err)
+	}
+	t.Cleanup(m.Close)
+	return m
+}
+
+// keyValue reads a Valkey key through miniredis rather than through a client,
+// so a test can assert what the server actually stored.
+func keyValue(t *testing.T, m *miniredis.Miniredis, key string) string {
+	t.Helper()
+	v, err := m.Get(key)
+	if err != nil {
+		return ""
+	}
+	return v
+}
+
+type node struct {
+	*Cluster
+	t      *testing.T
+	events chan Event
+	cancel context.CancelFunc
+}
+
+// start builds and starts a Cluster against the harness' miniredis. The node
+// is stopped and its context cancelled at the end of the test, so no test can
+// leak a maintain goroutine into the next one.
+func (h *harness) start(name string, opt func(*Options)) *node {
+	h.t.Helper()
+	o := Options{
+		Config: h.store,
+		Logger: discardLogger(),
+		// The version is deliberately the same for every node: the identity
+		// that matters is NodeID, and two nodes must not collide on it.
+		Version:   "test",
+		StartedAt: time.Now().Add(-time.Hour).Truncate(time.Second),
+		Addr:      name + ".test:8080",
+	}
+	if opt != nil {
+		opt(&o)
+	}
+	c := New(o)
+	ctx, cancel := context.WithCancel(context.Background())
+	n := &node{Cluster: c, t: h.t, events: make(chan Event, 64), cancel: cancel}
+	// The handler must be in place before Start: the maintain goroutine
+	// reads c.opt from another goroutine.
+	if c.opt.Handler == nil {
+		c.opt.Handler = func(_ context.Context, ev Event) {
+			select {
+			case n.events <- ev:
+			default:
+			}
+		}
+	}
+	h.t.Cleanup(c.Stop)
+	h.t.Cleanup(cancel)
+	c.Start(ctx)
+	return n
+}
+
+// kill ends this node's maintain session the way a crash does: the context is
+// cancelled, so the node publishes no "bye" and its peers must notice through
+// the heartbeat going stale instead of through a farewell.
+func (n *node) kill() { n.cancel() }
+
+// waitEvent returns the first captured event of the given kind, discarding the
+// others (heartbeats from the peer, mostly).
+func (n *node) waitEvent(kind EventKind, timeout time.Duration) Event {
+	n.t.Helper()
+	deadline := time.NewTimer(timeout)
+	defer deadline.Stop()
+	for {
+		select {
+		case ev := <-n.events:
+			if ev.Kind == kind {
+				return ev
+			}
+		case <-deadline.C:
+			n.t.Fatalf("no %q event within %s", kind, timeout)
+		}
+	}
+}
+
+// countEvents drains the captured events and counts those of one kind.
+func (n *node) countEvents(kind EventKind) int {
+	c := 0
+	for {
+		select {
+		case ev := <-n.events:
+			if ev.Kind == kind {
+				c++
+			}
+		default:
+			return c
+		}
+	}
+}
+
+// hasPeer reports whether states contains id, without asserting anything
+// about which other nodes are in the list.
+func hasPeer(states []NodeState, id string) bool {
+	for _, s := range states {
+		if s.NodeID == id {
+			return true
+		}
+	}
+	return false
+}
+
+// leaderCount counts the nodes that currently claim the refresh lease.
+func leaderCount(nodes ...*node) int {
+	n := 0
+	for _, nd := range nodes {
+		if nd.IsLeader() {
+			n++
+		}
+	}
+	return n
+}
+
+func discardLogger() *slog.Logger {
+	return slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: slog.LevelError + 1}))
+}
+
+// waitFor polls cond until it holds, failing with what it waited for. The
+// condition is an invariant of the cluster, never an incidental observation
+// about what a goroutine happened to see.
+func waitFor(t *testing.T, what string, timeout time.Duration, cond func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for {
+		if cond() {
+			return
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("timed out after %s waiting for %s", timeout, what)
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+}
+
+// waitSubscribed blocks until miniredis counts a subscriber on every channel.
+// This is the only race-free way to know a peer will hear a publish: pub/sub
+// delivery to a channel with no subscriber is fire-and-forget.
+func waitSubscribed(t *testing.T, m *miniredis.Miniredis, n *node) {
+	t.Helper()
+	chans := []string{
+		n.channel(EventHeartbeat),
+		n.channel(EventConfig),
+		n.channel(EventReload),
+		n.channel(EventBlocklists),
+		n.channel(EventBye),
+	}
+	waitFor(t, "a live subscription on every cluster channel", 5*time.Second, func() bool {
+		counts := m.PubSubNumSub(chans...)
+		for _, ch := range chans {
+			if counts[ch] < 1 {
+				return false
+			}
+		}
+		return true
+	})
+}
+
+// digests is a race-free source of blocklist digests for one node, so a test
+// can change what a node publishes and observe the effect.
+type digests struct {
+	mu        sync.Mutex
+	blocklist string
+	rules     int
+	cfgDigest string
+}
+
+func (d *digests) set(blocklist string, rules int) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	d.blocklist, d.rules = blocklist, rules
+}
+
+func (d *digests) fn() (string, int, time.Time, string) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	return d.blocklist, d.rules, time.Unix(0, 0), d.cfgDigest
+}