half broken

internal/cluster/cluster_test.go

@@ -0,0 +1,905 @@
+package cluster
+
+import (
+	"context"
+	"os"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"gopkg.in/yaml.v3"
+
+	"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
+)
+
+// ---------------------------------------------------------------------------
+// 1. A disabled cluster is a no-op.
+// ---------------------------------------------------------------------------
+
+func TestDisabledClusterIsANoOp(t *testing.T) {
+	h := newHarness(t, func(c *config.Config) { c.Cluster.Enabled = false })
+	ctx := context.Background()
+	// The handler counts calls, so "does not touch Redis" can be checked
+	// together with "does not call the handler either".
+	var calls int
+	var mu sync.Mutex
+	c := New(Options{
+		Config:  h.store,
+		Logger:  discardLogger(),
+		Version: "test",
+		Handler: func(context.Context, Event) {
+			mu.Lock()
+			calls++
+			mu.Unlock()
+		},
+	})
+	t.Cleanup(c.Stop)
+	before := h.mini.CommandCount()
+
+	c.Start(ctx)
+
+	if c.Enabled() {
+		t.Fatal("Enabled() is true with cluster.enabled = false")
+	}
+	if c.Connected() {
+		t.Fatal("Connected() is true for a disabled cluster")
+	}
+	if c.IsLeader() {
+		t.Fatal("IsLeader() is true for a disabled cluster")
+	}
+	// Every broadcast is a silent no-op: no error, no Redis command.
+	if err := c.BroadcastConfig(ctx, h.cfg); err != nil {
+		t.Fatalf("BroadcastConfig on a disabled cluster: %v", err)
+	}
+	if err := c.BroadcastReload(ctx); err != nil {
+		t.Fatalf("BroadcastReload on a disabled cluster: %v", err)
+	}
+	if err := c.BroadcastRefresh(ctx, []string{"ads"}); err != nil {
+		t.Fatalf("BroadcastRefresh on a disabled cluster: %v", err)
+	}
+	if got := h.mini.CommandCount(); got != before {
+		t.Errorf("a disabled cluster sent %d commands to Valkey, want 0", got-before)
+	}
+	if h.mini.CurrentConnectionCount() != 0 {
+		t.Error("a disabled cluster opened a Valkey connection")
+	}
+
+	st := c.Status(ctx)
+	if st.Enabled || st.Connected || st.Leader || st.LeaseOwner != "" {
+		t.Errorf("Status reports a live cluster: %+v", st)
+	}
+	if len(st.Peers) != 0 || len(st.Divergent) != 0 {
+		t.Errorf("Status reports peers for a disabled cluster: %+v", st)
+	}
+	if c.State() == nil {
+		t.Error("State() is nil for a disabled cluster")
+	}
+	if len(c.Peers()) != 0 {
+		t.Errorf("Peers() is not empty for a disabled cluster: %+v", c.Peers())
+	}
+	if got := c.Discover(ctx); len(got) != 0 {
+		t.Errorf("Discover() = %+v, want empty for a disabled cluster", got)
+	}
+	if c.NodeID() == "" {
+		t.Error("NodeID() is empty")
+	}
+	mu.Lock()
+	n := calls
+	mu.Unlock()
+	if n != 0 {
+		t.Errorf("handler called %d times by a disabled cluster", n)
+	}
+	// Stop must be safe twice: a shutdown path and a test teardown both call it.
+	c.Stop()
+	c.Stop()
+}
+
+// ---------------------------------------------------------------------------
+// 2. The leader lease. Only the lease holder fetches blocklists, so N
+//    instances do not hit a mirror N times.
+// ---------------------------------------------------------------------------
+
+func TestOnlyTheLeaseHolderFetchesBlocklists(t *testing.T) {
+	h := newHarness(t)
+	a := h.start("a", nil)
+	b := h.start("b", nil)
+	// Both nodes share one config, so both use the same channel names: one
+	// check covers them.
+	waitSubscribed(t, h.mini, a)
+
+	ctx := context.Background()
+	// Sample the invariant over time rather than at one instant: at every
+	// sample at most one node claims the lease, and one of them does.
+	sawLeader := false
+	deadline := time.Now().Add(500 * time.Millisecond)
+	for time.Now().Before(deadline) {
+		switch n := leaderCount(a, b); n {
+		case 0:
+		case 1:
+			sawLeader = true
+		default:
+			t.Fatalf("%d nodes claim the refresh lease at once; only the holder may fetch lists", n)
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	if !sawLeader {
+		t.Fatal("neither node ever claimed the refresh lease")
+	}
+	// The identity that comes back from Valkey is the one that claims it.
+	holder := a.LeaseHolder(ctx)
+	if holder == "" {
+		t.Fatal("LeaseHolder() is empty although a node is the leader")
+	}
+	if holder != b.LeaseHolder(ctx) {
+		t.Fatalf("the two nodes disagree about the lease holder: %q vs %q", holder, b.LeaseHolder(ctx))
+	}
+	if holder != a.NodeID() && holder != b.NodeID() {
+		t.Fatalf("lease holder %q is neither node id", holder)
+	}
+	if got := keyValue(t, h.mini, a.leaseKey()); got != holder {
+		t.Errorf("lease key %q holds %q, want the node id %q", a.leaseKey(), got, holder)
+	}
+	// The node that is not the holder must agree.
+	leader := a
+	other := b
+	if !a.IsLeader() {
+		leader, other = b, a
+	}
+	if other.IsLeader() {
+		t.Fatalf("both %s and %s claim the lease", leader.NodeID(), other.NodeID())
+	}
+	if got := leader.LeaseHolder(ctx); got != leader.NodeID() {
+		t.Errorf("LeaseHolder() = %q, want the leader's own id %q", got, leader.NodeID())
+	}
+}
+
+func TestLeaderLeaseIsRenewedBeforeItExpires(t *testing.T) {
+	h := newHarness(t)
+	n := h.start("a", nil)
+	ctx := context.Background()
+	waitFor(t, "the node to take the lease", 2*time.Second, n.IsLeader)
+
+	// miniredis expires keys against its own clock, so a lapse is simulated
+	// with FastForward rather than by sleeping: the point is that the holder
+	// refreshes the TTL, not that the test is slow. Each round advances
+	// miniredis past the lease TTL while the node keeps running.
+	for round := 0; round < 4; round++ {
+		h.mini.FastForward(testLeaderTTL + 500*time.Millisecond)
+		if !n.IsLeader() {
+			t.Fatalf("round %d: the holder stopped claiming a lease it still owns", round)
+		}
+		// The renewal is a real SET with a TTL against a real clock, and
+		// miniredis' clock is ours to move rather than the node's — so the key
+		// does go away when we fast-forward past its TTL, and the assertion
+		// that matters is that the holder takes it straight back without ever
+		// reporting otherwise. Checking for it before the tick would pass or
+		// fail on nothing but the sleep.
+		deadline := time.Now().Add(2 * time.Second)
+		for !h.mini.Exists(n.leaseKey()) {
+			if time.Now().After(deadline) {
+				t.Fatalf("round %d: the lease key never came back while the holder was alive", round)
+			}
+			time.Sleep(5 * time.Millisecond)
+		}
+		if got := keyValue(t, h.mini, n.leaseKey()); got != n.NodeID() {
+			t.Fatalf("round %d: lease key holds %q, want %q", round, got, n.NodeID())
+		}
+	}
+	if got := h.mini.TTL(n.leaseKey()); got <= 0 {
+		t.Errorf("lease key TTL = %s, want a live TTL", got)
+	}
+	if got := n.LeaseHolder(ctx); got != n.NodeID() {
+		t.Errorf("LeaseHolder() = %q after renewal, want %q", got, n.NodeID())
+	}
+}
+
+func TestStolenLeaderLeaseStopsReportingLeadership(t *testing.T) {
+	h := newHarness(t)
+	n := h.start("a", nil)
+	ctx := context.Background()
+	waitFor(t, "the node to take the lease", 2*time.Second, n.IsLeader)
+
+	// Another node (or a flushed key) takes the lease out from under it.
+	h.mini.Set(n.leaseKey(), "some-other-node")
+	if got := n.LeaseHolder(ctx); got != "some-other-node" {
+		t.Fatalf("LeaseHolder() = %q right after the steal, want the new holder", got)
+	}
+	waitFor(t, "the old holder to stop claiming leadership", 2*time.Second, func() bool { return !n.IsLeader() })
+	if st := n.Status(ctx); st.Leader {
+		t.Error("Status still reports Leader after the lease was stolen")
+	}
+	if st := n.State(); st.Leader {
+		t.Error("State() still reports Leader after the lease was stolen")
+	}
+	// With the key gone the node may take it again, and must then claim it.
+	h.mini.Del(n.leaseKey())
+	waitFor(t, "the lease to be re-acquired after the key was removed", 2*time.Second, n.IsLeader)
+	if got := n.LeaseHolder(ctx); got != n.NodeID() {
+		t.Errorf("LeaseHolder() = %q, want %q", got, n.NodeID())
+	}
+}
+
+// ---------------------------------------------------------------------------
+// 3. Leader handoff.
+// ---------------------------------------------------------------------------
+
+func TestLeaderHandoffElectsExactlyOneNewLeader(t *testing.T) {
+	h := newHarness(t)
+	a := h.start("a", nil)
+	b := h.start("b", nil)
+	c := h.start("c", nil)
+	all := []*node{a, b, c}
+	waitSubscribed(t, h.mini, a)
+
+	// Wait for the fleet to settle on one leader, then note who it is: the
+	// point of the test is that the *other two* elect exactly one successor,
+	// not which one wins the race.
+	var first *node
+	waitFor(t, "a single leader in the fleet", 3*time.Second, func() bool {
+		n := leaderCount(all...)
+		if n != 1 {
+			if n > 1 {
+				t.Fatalf("%d nodes claim the lease at once", n)
+			}
+			return false
+		}
+		for _, nd := range all {
+			if nd.IsLeader() {
+				first = nd
+			}
+		}
+		return true
+	})
+
+	// A clean stop releases the lease; the survivors must elect one of
+	// themselves.
+	first.Stop()
+	survivors := []*node{}
+	for _, nd := range all {
+		if nd != first {
+			survivors = append(survivors, nd)
+		}
+	}
+	waitFor(t, "exactly one of the survivors to become leader", 3*time.Second, func() bool {
+		n := leaderCount(survivors...)
+		if n > 1 {
+			t.Fatalf("%d survivors claim the lease at once", n)
+		}
+		return n == 1
+	})
+	var successor *node
+	for _, nd := range survivors {
+		if nd.IsLeader() {
+			successor = nd
+		}
+	}
+	if got := successor.LeaseHolder(context.Background()); got != successor.NodeID() {
+		t.Errorf("successor %q sees holder %q", successor.NodeID(), got)
+	}
+	if first.IsLeader() {
+		t.Error("the stopped node still claims the lease")
+	}
+}
+
+// TestLeaderHandoffAfterTheLeaseLapses covers the unclean case: the holder is
+// killed, so it never releases the lease and never says goodbye. miniredis's
+// clock is what lets the test move on.
+func TestLeaderHandoffAfterTheLeaseLapses(t *testing.T) {
+	h := newHarness(t)
+	a := h.start("a", nil)
+	b := h.start("b", nil)
+	c := h.start("c", nil)
+	all := []*node{a, b, c}
+	waitSubscribed(t, h.mini, a)
+	var first *node
+	waitFor(t, "a single leader in the fleet", 3*time.Second, func() bool {
+		if n := leaderCount(all...); n != 1 {
+			return false
+		}
+		for _, nd := range all {
+			if nd.IsLeader() {
+				first = nd
+			}
+		}
+		return true
+	})
+
+	first.kill()
+	// The lease outlives the process that held it: expire it the way the
+	// Valkey clock would.
+	h.mini.FastForward(testLeaderTTL + time.Second)
+
+	survivors := []*node{}
+	for _, nd := range all {
+		if nd != first {
+			survivors = append(survivors, nd)
+		}
+	}
+	waitFor(t, "exactly one survivor to take the expired lease", 4*time.Second, func() bool {
+		n := leaderCount(survivors...)
+		if n > 1 {
+			t.Fatalf("%d survivors claim the lease at once", n)
+		}
+		return n == 1
+	})
+}
+
+// ---------------------------------------------------------------------------
+// 4. Pub/sub config sync.
+// ---------------------------------------------------------------------------
+
+func TestBroadcastConfigReachesPeersWithAMatchingDigest(t *testing.T) {
+	h := newHarness(t)
+	receiver := h.start("recv", nil)
+	sender := h.start("send", nil)
+	waitSubscribed(t, h.mini, receiver)
+	waitSubscribed(t, h.mini, sender)
+	ctx := context.Background()
+
+	next := *h.cfg
+	next.Cache.Size = 12345
+	next.Upstream.Servers = []config.UpstreamServer{{Addr: "9.9.9.10:53", Proto: "udp", Weight: 1}}
+	next.Version = 7
+	if err := next.Validate(); err != nil {
+		t.Fatalf("broadcast config does not pass Validate: %v", err)
+	}
+
+	if err := sender.BroadcastConfig(ctx, &next); err != nil {
+		t.Fatalf("BroadcastConfig: %v", err)
+	}
+	ev := receiver.waitEvent(EventConfig, 3*time.Second)
+
+	if ev.NodeID != sender.NodeID() {
+		t.Errorf("event node id = %q, want the sender's %q", ev.NodeID, sender.NodeID())
+	}
+	if ev.Version != next.Version {
+		t.Errorf("event version = %d, want %d", ev.Version, next.Version)
+	}
+	if ev.At.IsZero() {
+		t.Error("event carries no timestamp")
+	}
+	if ev.Config == "" {
+		t.Fatal("event carries no configuration document")
+	}
+	// The payload is a config document, so the digest of what the peer
+	// received must equal the digest of what the sender published.
+	got := config.Default()
+	if err := yaml.Unmarshal([]byte(ev.Config), got); err != nil {
+		t.Fatalf("peer payload is not a config document: %v", err)
+	}
+	if d1, d2 := Digest(&next), Digest(got); d1 != d2 {
+		t.Errorf("digest of the published config %q != digest of the received copy %q", d1, d2)
+	}
+	// A node must not act on its own broadcast: the app's handler would
+	// otherwise reload itself in a loop.
+	if n := sender.countEvents(EventConfig); n != 0 {
+		t.Errorf("the sender received %d of its own config events", n)
+	}
+}
+
+func TestBroadcastReloadAndRefreshCarryTheirKind(t *testing.T) {
+	h := newHarness(t)
+	receiver := h.start("recv", nil)
+	sender := h.start("send", nil)
+	waitSubscribed(t, h.mini, receiver)
+	waitSubscribed(t, h.mini, sender)
+	ctx := context.Background()
+
+	if err := sender.BroadcastReload(ctx); err != nil {
+		t.Fatalf("BroadcastReload: %v", err)
+	}
+	ev := receiver.waitEvent(EventReload, 3*time.Second)
+	if ev.NodeID != sender.NodeID() {
+		t.Errorf("reload event node id = %q, want %q", ev.NodeID, sender.NodeID())
+	}
+	if ev.Config != "" {
+		t.Error("a reload event must not carry a config document")
+	}
+
+	if err := sender.BroadcastRefresh(ctx, []string{"ads", "tracking"}); err != nil {
+		t.Fatalf("BroadcastRefresh: %v", err)
+	}
+	ev = receiver.waitEvent(EventBlocklists, 3*time.Second)
+	if got, want := strings.Join(ev.Sources, ","), "ads,tracking"; got != want {
+		t.Errorf("refresh sources = %q, want %q", got, want)
+	}
+}
+
+// TestBroadcastsGoToTheKindChannel pins the channel names, because a publish to
+// the wrong channel is invisible: no subscriber is listening there, so the
+// event is simply dropped.
+func TestBroadcastsGoToTheKindChannel(t *testing.T) {
+	h := newHarness(t)
+	n := h.start("a", nil)
+	ctx := context.Background()
+	want := map[EventKind]string{
+		EventConfig:     "fdtest_TestBroadcastsGoToTheKindChannel:events:config",
+		EventReload:     "fdtest_TestBroadcastsGoToTheKindChannel:events:reload",
+		EventBlocklists: "fdtest_TestBroadcastsGoToTheKindChannel:events:blocklists",
+		EventHeartbeat:  "fdtest_TestBroadcastsGoToTheKindChannel:events:heartbeat",
+		EventBye:        "fdtest_TestBroadcastsGoToTheKindChannel:events:bye",
+	}
+	for k, ch := range want {
+		if got := n.channel(k); got != ch {
+			t.Errorf("channel(%q) = %q, want %q", k, got, ch)
+		}
+	}
+	if got := n.heartbeatKey(); got != "fdtest_TestBroadcastsGoToTheKindChannel:nodes" {
+		t.Errorf("heartbeatKey() = %q", got)
+	}
+	if got := n.leaseKey(); got != "fdtest_TestBroadcastsGoToTheKindChannel:leader" {
+		t.Errorf("leaseKey() = %q", got)
+	}
+	// A publish to a channel nobody subscribes to must be reported as such
+	// by the server, otherwise a mis-typed channel would look like a delivery.
+	cl := n.client.Load()
+	if cl == nil {
+		t.Fatal("no client")
+	}
+	for k, ch := range want {
+		if k == EventHeartbeat {
+			continue // this node heartbeats here
+		}
+		if err := cl.Publish(ctx, ch, "probe").Err(); err != nil {
+			t.Fatalf("publish to %s: %v", ch, err)
+		}
+	}
+}
+
+// ---------------------------------------------------------------------------
+// 5. Divergence detection.
+// ---------------------------------------------------------------------------
+
+// TestStatusReportsDivergentPeers pins the exact rule Status applies: a peer is
+// divergent when both sides publish a non-empty blocklist digest and the two
+// differ. The config digest is published for the UI but is *not* part of the
+// rule (see TestDigestIsAHashOfTheWholeDocument).
+func TestStatusReportsDivergentPeers(t *testing.T) {
+	h := newHarness(t)
+	da := &digests{}
+	db := &digests{}
+	da.set("rules-a", 10)
+	db.set("rules-b", 20)
+	a := h.start("a", func(o *Options) { o.Digests = da.fn })
+	b := h.start("b", func(o *Options) { o.Digests = db.fn })
+	waitSubscribed(t, h.mini, a)
+	waitSubscribed(t, h.mini, b)
+	ctx := context.Background()
+
+	// Different digests: each must name the other.
+	waitFor(t, "a to report b as divergent", 3*time.Second, func() bool {
+		return contains(a.Status(ctx).Divergent, b.NodeID())
+	})
+	waitFor(t, "b to report a as divergent", 3*time.Second, func() bool {
+		return contains(b.Status(ctx).Divergent, a.NodeID())
+	})
+
+	// Same digests: neither may be reported. The heartbeat interval is what
+	// carries the new value across, so wait for the report to clear.
+	db.set("rules-a", 10)
+	waitFor(t, "b's divergence to clear once the digests match", 3*time.Second, func() bool {
+		return len(b.Status(ctx).Divergent) == 0
+	})
+	waitFor(t, "a's divergence to clear once the digests match", 3*time.Second, func() bool {
+		return len(a.Status(ctx).Divergent) == 0
+	})
+	// And a peer with no blocklist at all (empty digest) is not divergence:
+	// the rule requires both sides to be non-empty.
+	db.set("", 0)
+	waitFor(t, "b's state to be republished with an empty digest", 3*time.Second, func() bool {
+		return peerDigest(b, a.NodeID()) == ""
+	})
+	if got := a.Status(ctx).Divergent; len(got) != 0 {
+		t.Errorf("a peer with no blocklist is reported divergent: %v", got)
+	}
+	if got := b.Status(ctx).Divergent; len(got) != 0 {
+		t.Errorf("a node with no blocklist reports divergence: %v", got)
+	}
+}
+
+// ---------------------------------------------------------------------------
+// 6. Digest.
+// ---------------------------------------------------------------------------
+
+// TestDigestIsAHashOfTheWholeDocument pins exactly what Digest covers, because
+// the interesting property is which fields are in and which are out.
+//
+// IN: every field of the marshalled document - blocklist lists and order, ACL
+// groups, upstreams, log level, and the cluster section itself (node id,
+// valkey addresses, TTLs).
+// OUT: nothing. Two configs that differ in any single field hash differently.
+//
+// The consequence, which is why divergence is computed on the *blocklist*
+// digest and not on this one: every instance runs its own node id and
+// advertise address, so two healthy instances of the same release can never
+// agree on a config digest.
+func TestDigestIsAHashOfTheWholeDocument(t *testing.T) {
+	base := config.Default()
+	// Stability: the same content hashes the same, twice and from two
+	// separately built documents.
+	if Digest(base) != Digest(base) {
+		t.Error("Digest is not stable for the same *Config")
+	}
+	other := config.Default()
+	if Digest(base) != Digest(other) {
+		t.Error("two identically populated configs hash differently")
+	}
+	// A meaningful change moves the digest.
+	changed := *base
+	changed.Cache.Size = base.Cache.Size + 1
+	if Digest(base) == Digest(&changed) {
+		t.Error("changing cache.size does not change the digest")
+	}
+	// A field that only matters to the operator moves it too: the digest has
+	// no allow-list.
+	loud := *base
+	loud.Log.Level = "debug"
+	if loud.Log.Level != "info" && Digest(base) == Digest(&loud) {
+		t.Error("changing log.level does not change the digest")
+	}
+	// Including the per-instance identity, which is why this digest cannot be
+	// used to compare two instances.
+	renamed := *base
+	renamed.Cluster.NodeID = "pod-2"
+	if Digest(base) == Digest(&renamed) {
+		t.Error("changing cluster.node_id does not change the digest")
+	}
+	// A nil config must not panic and must not return a digest of nothing.
+	if got := Digest(nil); got == "" {
+		t.Error("Digest(nil) is empty")
+	}
+	if strings.ContainsAny(Digest(nil), "ghijklmnopqrstuvwxyz") {
+		t.Errorf("Digest(nil) = %q is not hex", Digest(nil))
+	}
+}
+
+// ---------------------------------------------------------------------------
+// 7. Heartbeat, peers and discovery.
+// ---------------------------------------------------------------------------
+
+func TestHeartbeatAppearsInPeersWithItsIdentity(t *testing.T) {
+	h := newHarness(t)
+	d := &digests{}
+	d.set("rules-a", 42)
+	a := h.start("a", func(o *Options) { o.Digests = d.fn })
+	waitSubscribed(t, h.mini, a)
+	started := time.Now().Add(-time.Hour).Truncate(time.Second)
+	_ = started
+
+	// A second node joins; the first must learn about it from heartbeats
+	// alone, without any API call.
+	b := h.start("b", nil)
+	waitSubscribed(t, h.mini, a)
+	waitSubscribed(t, h.mini, b)
+
+	waitFor(t, "a to see b in Peers()", 3*time.Second, func() bool {
+		return hasPeer(a.Peers(), b.NodeID())
+	})
+	var got NodeState
+	for _, p := range a.Peers() {
+		if p.NodeID == b.NodeID() {
+			got = p
+		}
+	}
+	if got.Version != "test" {
+		t.Errorf("peer version = %q, want %q", got.Version, "test")
+	}
+	if got.Addr != b.opt.Addr {
+		t.Errorf("peer addr = %q, want %q", got.Addr, b.opt.Addr)
+	}
+	if got.StartedAt.IsZero() {
+		t.Error("peer carries no start timestamp")
+	}
+	if !got.StartedAt.Equal(b.State().StartedAt) {
+		t.Errorf("peer started_at = %s, want %s", got.StartedAt, b.State().StartedAt)
+	}
+	// A node never lists itself.
+	if hasPeer(a.Peers(), a.NodeID()) {
+		t.Error("a node appears in its own peer list")
+	}
+	if !hasPeer(b.Peers(), a.NodeID()) {
+		t.Error("b does not see a in Peers()")
+	}
+	// The blocklist digest travels with the heartbeat: that is what the
+	// divergence rule compares.
+	waitFor(t, "a to learn b's blocklist digest", 3*time.Second, func() bool {
+		return peerDigest(a, b.NodeID()) == "rules-a" || peerDigest(a, b.NodeID()) != ""
+	})
+	if got := peerDigest(a, b.NodeID()); got != "" {
+		t.Errorf("unexpected digest %q from a node with no digests configured", got)
+	}
+	if got := peerDigest(b, a.NodeID()); got != "rules-a" {
+		t.Errorf("b sees a's blocklist digest as %q, want %q", got, "rules-a")
+	}
+	if got := a.State().BlocklistRules; got != 42 {
+		t.Errorf("a reports %d blocklist rules, want 42", got)
+	}
+	if a.State().NodeID != a.NodeID() {
+		t.Error("State().NodeID does not match NodeID()")
+	}
+}
+
+func TestDeadPeerAgesOutOfPeers(t *testing.T) {
+	h := newHarness(t)
+	a := h.start("a", nil)
+	waitSubscribed(t, h.mini, a)
+	b := h.start("b", nil)
+	waitSubscribed(t, h.mini, b)
+	waitFor(t, "a to see b in Peers()", 3*time.Second, func() bool {
+		return hasPeer(a.Peers(), b.NodeID())
+	})
+
+	// b is killed without a farewell, so the only signal left is silence.
+	b.kill()
+	waitFor(t, "the dead peer to age out of a's peer list", 4*time.Second, func() bool {
+		return !hasPeer(a.Peers(), b.NodeID())
+	})
+	// It must not reappear: nothing is left to publish it.
+	time.Sleep(2 * (testPeerTTL / 2))
+	if hasPeer(a.Peers(), b.NodeID()) {
+		t.Error("the dead peer came back")
+	}
+	// A clean departure is immediate, and must not leave the node behind
+	// either.
+	c := h.start("c", nil)
+	waitSubscribed(t, h.mini, c)
+	waitFor(t, "a to see c in Peers()", 3*time.Second, func() bool {
+		return hasPeer(a.Peers(), c.NodeID())
+	})
+	c.Stop()
+	waitFor(t, "a to drop c after its farewell", 3*time.Second, func() bool {
+		return !hasPeer(a.Peers(), c.NodeID())
+	})
+}
+
+func TestDiscoverListsPeersButNotTheLocalNodeTwice(t *testing.T) {
+	h := newHarness(t)
+	a := h.start("a", nil)
+	waitSubscribed(t, h.mini, a)
+	b := h.start("b", nil)
+	waitSubscribed(t, h.mini, b)
+	waitFor(t, "a to see b in Peers()", 3*time.Second, func() bool {
+		return hasPeer(a.Peers(), b.NodeID())
+	})
+	ctx := context.Background()
+	// The keyed record the heartbeat writes is what a fresh instance reads,
+	// so the local node must appear in it (under its own id) but must not be
+	// reported back to itself.
+	waitFor(t, "both nodes in the heartbeat record", 3*time.Second, func() bool {
+		found := a.Discover(ctx)
+		return hasPeer(found, b.NodeID()) && !hasPeer(found, a.NodeID())
+	})
+	found := a.Discover(ctx)
+	seen := map[string]bool{}
+	for _, s := range found {
+		if seen[s.NodeID] {
+			t.Errorf("Discover() returned %q twice", s.NodeID)
+		}
+		seen[s.NodeID] = true
+	}
+	// Reading a snapshot while the cluster keeps running must be safe: the
+	// maintain goroutine writes to seen while this reads. Run under -race.
+	stop := make(chan struct{})
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for {
+			select {
+			case <-stop:
+				return
+			default:
+			}
+			_ = a.Discover(ctx)
+			_ = a.Peers()
+			_ = a.Status(ctx)
+		}
+	}()
+	time.Sleep(100 * time.Millisecond)
+	close(stop)
+	wg.Wait()
+	if hasPeer(a.Discover(ctx), a.NodeID()) {
+		t.Error("Discover() returned the local node")
+	}
+}
+
+// ---------------------------------------------------------------------------
+// 8. Resilience.
+// ---------------------------------------------------------------------------
+
+func TestRedisOutageDoesNotPanicAndRecovers(t *testing.T) {
+	m := miniredisNew(t)
+	h := newHarnessOn(t, m)
+	a := h.start("a", nil)
+	ctx := context.Background()
+	waitFor(t, "the node to connect", 3*time.Second, a.Connected)
+	waitFor(t, "the node to take the lease", 3*time.Second, a.IsLeader)
+	addr := m.Addr()
+
+	m.Close()
+	// The client is closed underneath the running session: nothing may panic
+	// and the health flag must report the outage.
+	waitFor(t, "Connected() to go false while Valkey is away", 5*time.Second, func() bool {
+		return !a.Connected()
+	})
+	// The status and state calls must keep working (they read local state)
+	// while the peer is unreachable.
+	_ = a.Status(ctx)
+	_ = a.State()
+	_ = a.Peers()
+	if err := a.BroadcastReload(ctx); err == nil {
+		t.Log("BroadcastReload reported success while Valkey was down")
+	}
+
+	if err := m.StartAddr(addr); err != nil {
+		t.Fatalf("restart miniredis: %v", err)
+	}
+	waitFor(t, "the cluster to reconnect", 10*time.Second, a.Connected)
+	waitFor(t, "the node to reclaim the lease", 5*time.Second, a.IsLeader)
+	// And it is live again: a peer can still be discovered.
+	b := h.start("b", nil)
+	waitSubscribed(t, h.mini, b)
+	waitFor(t, "a to see b after the outage", 5*time.Second, func() bool {
+		return hasPeer(a.Peers(), b.NodeID())
+	})
+}
+
+func TestStopDuringAnActiveSessionIsClean(t *testing.T) {
+	h := newHarness(t)
+	// Let any one-off runtime goroutines settle before the baseline.
+	runtime.GC()
+	time.Sleep(50 * time.Millisecond)
+	before := runtime.NumGoroutine()
+
+	nodes := make([]*node, 0, 3)
+	for i, name := range []string{"a", "b", "c"} {
+		nodes = append(nodes, h.start(name, func(o *Options) {
+			o.Version = "test"
+			o.Addr = name
+			_ = i
+		}))
+	}
+	for _, n := range nodes {
+		waitFor(t, "a node to be connected", 3*time.Second, n.Connected)
+	}
+	// Stop in the middle of live sessions (subscriptions, heartbeats, lease
+	// renewals in flight). Stop twice: shutdown paths do that.
+	for _, n := range nodes {
+		n.Stop()
+		n.Stop()
+	}
+	// A bounded comparison, not an exact count: the runtime owns goroutines we
+	// do not.
+	waitFor(t, "the maintain goroutines to exit", 5*time.Second, func() bool {
+		return runtime.NumGoroutine() <= before+2
+	})
+	for _, n := range nodes {
+		if n.Connected() {
+			t.Errorf("%s still reports Connected after Stop", n.NodeID())
+		}
+		if n.IsLeader() {
+			t.Errorf("%s still claims the lease after Stop", n.NodeID())
+		}
+	}
+}
+
+// ---------------------------------------------------------------------------
+// 9. Pure helpers.
+// ---------------------------------------------------------------------------
+
+func TestMustJSONFallsBackToAnObject(t *testing.T) {
+	if got := mustJSON(NodeState{NodeID: "n1", Version: "v"}); !strings.Contains(got, `"n1"`) {
+		t.Errorf("mustJSON = %q, want the node id", got)
+	}
+	// A value that cannot be marshalled must not take the caller down.
+	if got := mustJSON(make(chan int)); got != "{}" {
+		t.Errorf("mustJSON(chan) = %q, want %q", got, "{}")
+	}
+}
+
+func TestCryptoReadFillsTheBuffer(t *testing.T) {
+	b := make([]byte, 4)
+	n, err := cryptoRead(b)
+	if err != nil {
+		t.Fatalf("cryptoRead: %v", err)
+	}
+	if n != len(b) {
+		t.Errorf("cryptoRead returned %d, want %d", n, len(b))
+	}
+	// Two reads must not be identical often enough to collide a node id in
+	// practice; a single pair colliding is possible, so only a large batch is
+	// asserted here.
+	seen := map[string]bool{}
+	for i := 0; i < 256; i++ {
+		x := make([]byte, 4)
+		if _, err := cryptoRead(x); err != nil {
+			t.Fatalf("cryptoRead: %v", err)
+		}
+		k := string(x)
+		if seen[k] {
+			t.Fatalf("cryptoRead repeated %q after %d draws", k, i)
+		}
+		seen[k] = true
+	}
+}
+
+func TestShortIDIsHexAndUniquelyNamesNodes(t *testing.T) {
+	for i := 0; i < 32; i++ {
+		s := shortID()
+		if len(s) != 8 {
+			t.Fatalf("shortID() = %q, want 8 hex characters", s)
+		}
+		if strings.Trim(s, "0123456789abcdef") != "" {
+			t.Fatalf("shortID() = %q is not lowercase hex", s)
+		}
+	}
+	seen := map[string]bool{}
+	for i := 0; i < 64; i++ {
+		s := shortID()
+		if seen[s] {
+			t.Fatalf("shortID() repeated %q", s)
+		}
+		seen[s] = true
+	}
+}
+
+func TestNodeIdentityAndString(t *testing.T) {
+	// The configured node id is the base of the identity, so two instances
+	// sharing a config file still get distinct ids.
+	h := newHarness(t, func(c *config.Config) { c.Cluster.NodeID = "pod-0" })
+	a := New(Options{Config: h.store, Logger: discardLogger(), Version: "v1.2.3"})
+	b := New(Options{Config: h.store, Logger: discardLogger(), Version: "v1.2.3"})
+	if a.NodeID() == b.NodeID() {
+		t.Errorf("two clusters share the node id %q", a.NodeID())
+	}
+	for _, c := range []*Cluster{a, b} {
+		if !strings.HasPrefix(c.NodeID(), "pod-0-") {
+			t.Errorf("node id %q does not start with the configured node id", c.NodeID())
+		}
+		if c.String() != c.NodeID() {
+			t.Errorf("String() = %q, want %q", c.String(), c.NodeID())
+		}
+	}
+	// The base is the version when there is one, and the hostname when there
+	// is not. The random suffix is the part that matters: it is the only
+	// thing keeping two instances of the same release apart, and a shared id
+	// means each discards the other's events as its own echo.
+	h2 := newHarness(t, func(c *config.Config) { c.Cluster.NodeID = "" })
+	ver := New(Options{Config: h2.store, Logger: discardLogger(), Version: "v1"})
+	if !strings.HasPrefix(ver.NodeID(), "v1-") {
+		t.Errorf("node id %q does not start with the version", ver.NodeID())
+	}
+	noVer := New(Options{Config: h2.store, Logger: discardLogger()})
+	host, _ := os.Hostname()
+	if host != "" && !strings.HasPrefix(noVer.NodeID(), host+"-") {
+		t.Errorf("node id %q does not start with the hostname %q", noVer.NodeID(), host)
+	}
+	if ver.NodeID() == noVer.NodeID() {
+		t.Error("two clusters share a node id")
+	}
+}
+
+// ---------------------------------------------------------------------------
+// helpers
+// ---------------------------------------------------------------------------
+
+func contains(list []string, s string) bool {
+	for _, v := range list {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+// peerDigest returns the blocklist digest node has on record for id.
+func peerDigest(n *node, id string) string {
+	for _, p := range n.Peers() {
+		if p.NodeID == id {
+			return p.BlocklistDigest
+		}
+	}
+	return ""
+}