fancy-dns/internal/cluster/cluster_test.go

package cluster

import (
	"context"
	"os"
	"runtime"
	"strings"
	"sync"
	"testing"
	"time"

	"gopkg.in/yaml.v3"

	"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
)

// ---------------------------------------------------------------------------
// 1. A disabled cluster is a no-op.
// ---------------------------------------------------------------------------

func TestDisabledClusterIsANoOp(t *testing.T) {
	h := newHarness(t, func(c *config.Config) { c.Cluster.Enabled = false })
	ctx := context.Background()
	// The handler counts calls, so "does not touch Redis" can be checked
	// together with "does not call the handler either".
	var calls int
	var mu sync.Mutex
	c := New(Options{
		Config:  h.store,
		Logger:  discardLogger(),
		Version: "test",
		Handler: func(context.Context, Event) {
			mu.Lock()
			calls++
			mu.Unlock()
		},
	})
	t.Cleanup(c.Stop)
	before := h.mini.CommandCount()

	c.Start(ctx)

	if c.Enabled() {
		t.Fatal("Enabled() is true with cluster.enabled = false")
	}
	if c.Connected() {
		t.Fatal("Connected() is true for a disabled cluster")
	}
	if c.IsLeader() {
		t.Fatal("IsLeader() is true for a disabled cluster")
	}
	// Every broadcast is a silent no-op: no error, no Redis command.
	if err := c.BroadcastConfig(ctx, h.cfg); err != nil {
		t.Fatalf("BroadcastConfig on a disabled cluster: %v", err)
	}
	if err := c.BroadcastReload(ctx); err != nil {
		t.Fatalf("BroadcastReload on a disabled cluster: %v", err)
	}
	if err := c.BroadcastRefresh(ctx, []string{"ads"}); err != nil {
		t.Fatalf("BroadcastRefresh on a disabled cluster: %v", err)
	}
	if got := h.mini.CommandCount(); got != before {
		t.Errorf("a disabled cluster sent %d commands to Valkey, want 0", got-before)
	}
	if h.mini.CurrentConnectionCount() != 0 {
		t.Error("a disabled cluster opened a Valkey connection")
	}

	st := c.Status(ctx)
	if st.Enabled || st.Connected || st.Leader || st.LeaseOwner != "" {
		t.Errorf("Status reports a live cluster: %+v", st)
	}
	if len(st.Peers) != 0 || len(st.Divergent) != 0 {
		t.Errorf("Status reports peers for a disabled cluster: %+v", st)
	}
	if c.State() == nil {
		t.Error("State() is nil for a disabled cluster")
	}
	if len(c.Peers()) != 0 {
		t.Errorf("Peers() is not empty for a disabled cluster: %+v", c.Peers())
	}
	if got := c.Discover(ctx); len(got) != 0 {
		t.Errorf("Discover() = %+v, want empty for a disabled cluster", got)
	}
	if c.NodeID() == "" {
		t.Error("NodeID() is empty")
	}
	mu.Lock()
	n := calls
	mu.Unlock()
	if n != 0 {
		t.Errorf("handler called %d times by a disabled cluster", n)
	}
	// Stop must be safe twice: a shutdown path and a test teardown both call it.
	c.Stop()
	c.Stop()
}

// ---------------------------------------------------------------------------
// 2. The leader lease. Only the lease holder fetches blocklists, so N
//    instances do not hit a mirror N times.
// ---------------------------------------------------------------------------

func TestOnlyTheLeaseHolderFetchesBlocklists(t *testing.T) {
	h := newHarness(t)
	a := h.start("a", nil)
	b := h.start("b", nil)
	// Both nodes share one config, so both use the same channel names: one
	// check covers them.
	waitSubscribed(t, h.mini, a)

	ctx := context.Background()
	// Sample the invariant over time rather than at one instant: at every
	// sample at most one node claims the lease, and one of them does.
	sawLeader := false
	deadline := time.Now().Add(500 * time.Millisecond)
	for time.Now().Before(deadline) {
		switch n := leaderCount(a, b); n {
		case 0:
		case 1:
			sawLeader = true
		default:
			t.Fatalf("%d nodes claim the refresh lease at once; only the holder may fetch lists", n)
		}
		time.Sleep(10 * time.Millisecond)
	}
	if !sawLeader {
		t.Fatal("neither node ever claimed the refresh lease")
	}
	// The identity that comes back from Valkey is the one that claims it.
	holder := a.LeaseHolder(ctx)
	if holder == "" {
		t.Fatal("LeaseHolder() is empty although a node is the leader")
	}
	if holder != b.LeaseHolder(ctx) {
		t.Fatalf("the two nodes disagree about the lease holder: %q vs %q", holder, b.LeaseHolder(ctx))
	}
	if holder != a.NodeID() && holder != b.NodeID() {
		t.Fatalf("lease holder %q is neither node id", holder)
	}
	if got := keyValue(t, h.mini, a.leaseKey()); got != holder {
		t.Errorf("lease key %q holds %q, want the node id %q", a.leaseKey(), got, holder)
	}
	// The node that is not the holder must agree.
	leader := a
	other := b
	if !a.IsLeader() {
		leader, other = b, a
	}
	if other.IsLeader() {
		t.Fatalf("both %s and %s claim the lease", leader.NodeID(), other.NodeID())
	}
	if got := leader.LeaseHolder(ctx); got != leader.NodeID() {
		t.Errorf("LeaseHolder() = %q, want the leader's own id %q", got, leader.NodeID())
	}
}

func TestLeaderLeaseIsRenewedBeforeItExpires(t *testing.T) {
	h := newHarness(t)
	n := h.start("a", nil)
	ctx := context.Background()
	waitFor(t, "the node to take the lease", 2*time.Second, n.IsLeader)

	// miniredis expires keys against its own clock, so a lapse is simulated
	// with FastForward rather than by sleeping: the point is that the holder
	// refreshes the TTL, not that the test is slow. Each round advances
	// miniredis past the lease TTL while the node keeps running.
	for round := 0; round < 4; round++ {
		h.mini.FastForward(testLeaderTTL + 500*time.Millisecond)
		if !n.IsLeader() {
			t.Fatalf("round %d: the holder stopped claiming a lease it still owns", round)
		}
		// The renewal is a real SET with a TTL against a real clock, and
		// miniredis' clock is ours to move rather than the node's — so the key
		// does go away when we fast-forward past its TTL, and the assertion
		// that matters is that the holder takes it straight back without ever
		// reporting otherwise. Checking for it before the tick would pass or
		// fail on nothing but the sleep.
		deadline := time.Now().Add(2 * time.Second)
		for !h.mini.Exists(n.leaseKey()) {
			if time.Now().After(deadline) {
				t.Fatalf("round %d: the lease key never came back while the holder was alive", round)
			}
			time.Sleep(5 * time.Millisecond)
		}
		if got := keyValue(t, h.mini, n.leaseKey()); got != n.NodeID() {
			t.Fatalf("round %d: lease key holds %q, want %q", round, got, n.NodeID())
		}
	}
	if got := h.mini.TTL(n.leaseKey()); got <= 0 {
		t.Errorf("lease key TTL = %s, want a live TTL", got)
	}
	if got := n.LeaseHolder(ctx); got != n.NodeID() {
		t.Errorf("LeaseHolder() = %q after renewal, want %q", got, n.NodeID())
	}
}

func TestStolenLeaderLeaseStopsReportingLeadership(t *testing.T) {
	h := newHarness(t)
	n := h.start("a", nil)
	ctx := context.Background()
	waitFor(t, "the node to take the lease", 2*time.Second, n.IsLeader)

	// Another node (or a flushed key) takes the lease out from under it.
	h.mini.Set(n.leaseKey(), "some-other-node")
	if got := n.LeaseHolder(ctx); got != "some-other-node" {
		t.Fatalf("LeaseHolder() = %q right after the steal, want the new holder", got)
	}
	waitFor(t, "the old holder to stop claiming leadership", 2*time.Second, func() bool { return !n.IsLeader() })
	if st := n.Status(ctx); st.Leader {
		t.Error("Status still reports Leader after the lease was stolen")
	}
	if st := n.State(); st.Leader {
		t.Error("State() still reports Leader after the lease was stolen")
	}
	// With the key gone the node may take it again, and must then claim it.
	h.mini.Del(n.leaseKey())
	waitFor(t, "the lease to be re-acquired after the key was removed", 2*time.Second, n.IsLeader)
	if got := n.LeaseHolder(ctx); got != n.NodeID() {
		t.Errorf("LeaseHolder() = %q, want %q", got, n.NodeID())
	}
}

// ---------------------------------------------------------------------------
// 3. Leader handoff.
// ---------------------------------------------------------------------------

func TestLeaderHandoffElectsExactlyOneNewLeader(t *testing.T) {
	h := newHarness(t)
	a := h.start("a", nil)
	b := h.start("b", nil)
	c := h.start("c", nil)
	all := []*node{a, b, c}
	waitSubscribed(t, h.mini, a)

	// Wait for the fleet to settle on one leader, then note who it is: the
	// point of the test is that the *other two* elect exactly one successor,
	// not which one wins the race.
	var first *node
	waitFor(t, "a single leader in the fleet", 3*time.Second, func() bool {
		n := leaderCount(all...)
		if n != 1 {
			if n > 1 {
				t.Fatalf("%d nodes claim the lease at once", n)
			}
			return false
		}
		for _, nd := range all {
			if nd.IsLeader() {
				first = nd
			}
		}
		return true
	})

	// A clean stop releases the lease; the survivors must elect one of
	// themselves.
	first.Stop()
	survivors := []*node{}
	for _, nd := range all {
		if nd != first {
			survivors = append(survivors, nd)
		}
	}
	waitFor(t, "exactly one of the survivors to become leader", 3*time.Second, func() bool {
		n := leaderCount(survivors...)
		if n > 1 {
			t.Fatalf("%d survivors claim the lease at once", n)
		}
		return n == 1
	})
	var successor *node
	for _, nd := range survivors {
		if nd.IsLeader() {
			successor = nd
		}
	}
	if got := successor.LeaseHolder(context.Background()); got != successor.NodeID() {
		t.Errorf("successor %q sees holder %q", successor.NodeID(), got)
	}
	if first.IsLeader() {
		t.Error("the stopped node still claims the lease")
	}
}

// TestLeaderHandoffAfterTheLeaseLapses covers the unclean case: the holder is
// killed, so it never releases the lease and never says goodbye. miniredis's
// clock is what lets the test move on.
func TestLeaderHandoffAfterTheLeaseLapses(t *testing.T) {
	h := newHarness(t)
	a := h.start("a", nil)
	b := h.start("b", nil)
	c := h.start("c", nil)
	all := []*node{a, b, c}
	waitSubscribed(t, h.mini, a)
	var first *node
	waitFor(t, "a single leader in the fleet", 3*time.Second, func() bool {
		if n := leaderCount(all...); n != 1 {
			return false
		}
		for _, nd := range all {
			if nd.IsLeader() {
				first = nd
			}
		}
		return true
	})

	first.kill()
	// The lease outlives the process that held it: expire it the way the
	// Valkey clock would.
	h.mini.FastForward(testLeaderTTL + time.Second)

	survivors := []*node{}
	for _, nd := range all {
		if nd != first {
			survivors = append(survivors, nd)
		}
	}
	waitFor(t, "exactly one survivor to take the expired lease", 4*time.Second, func() bool {
		n := leaderCount(survivors...)
		if n > 1 {
			t.Fatalf("%d survivors claim the lease at once", n)
		}
		return n == 1
	})
}

// ---------------------------------------------------------------------------
// 4. Pub/sub config sync.
// ---------------------------------------------------------------------------

func TestBroadcastConfigReachesPeersWithAMatchingDigest(t *testing.T) {
	h := newHarness(t)
	receiver := h.start("recv", nil)
	sender := h.start("send", nil)
	waitSubscribed(t, h.mini, receiver)
	waitSubscribed(t, h.mini, sender)
	ctx := context.Background()

	next := *h.cfg
	next.Cache.Size = 12345
	next.Upstream.Servers = []config.UpstreamServer{{Addr: "9.9.9.10:53", Proto: "udp", Weight: 1}}
	next.Version = 7
	if err := next.Validate(); err != nil {
		t.Fatalf("broadcast config does not pass Validate: %v", err)
	}

	if err := sender.BroadcastConfig(ctx, &next); err != nil {
		t.Fatalf("BroadcastConfig: %v", err)
	}
	ev := receiver.waitEvent(EventConfig, 3*time.Second)

	if ev.NodeID != sender.NodeID() {
		t.Errorf("event node id = %q, want the sender's %q", ev.NodeID, sender.NodeID())
	}
	if ev.Version != next.Version {
		t.Errorf("event version = %d, want %d", ev.Version, next.Version)
	}
	if ev.At.IsZero() {
		t.Error("event carries no timestamp")
	}
	if ev.Config == "" {
		t.Fatal("event carries no configuration document")
	}
	// The payload is a config document, so the digest of what the peer
	// received must equal the digest of what the sender published.
	got := config.Default()
	if err := yaml.Unmarshal([]byte(ev.Config), got); err != nil {
		t.Fatalf("peer payload is not a config document: %v", err)
	}
	if d1, d2 := Digest(&next), Digest(got); d1 != d2 {
		t.Errorf("digest of the published config %q != digest of the received copy %q", d1, d2)
	}
	// A node must not act on its own broadcast: the app's handler would
	// otherwise reload itself in a loop.
	if n := sender.countEvents(EventConfig); n != 0 {
		t.Errorf("the sender received %d of its own config events", n)
	}
}

func TestBroadcastReloadAndRefreshCarryTheirKind(t *testing.T) {
	h := newHarness(t)
	receiver := h.start("recv", nil)
	sender := h.start("send", nil)
	waitSubscribed(t, h.mini, receiver)
	waitSubscribed(t, h.mini, sender)
	ctx := context.Background()

	if err := sender.BroadcastReload(ctx); err != nil {
		t.Fatalf("BroadcastReload: %v", err)
	}
	ev := receiver.waitEvent(EventReload, 3*time.Second)
	if ev.NodeID != sender.NodeID() {
		t.Errorf("reload event node id = %q, want %q", ev.NodeID, sender.NodeID())
	}
	if ev.Config != "" {
		t.Error("a reload event must not carry a config document")
	}

	if err := sender.BroadcastRefresh(ctx, []string{"ads", "tracking"}); err != nil {
		t.Fatalf("BroadcastRefresh: %v", err)
	}
	ev = receiver.waitEvent(EventBlocklists, 3*time.Second)
	if got, want := strings.Join(ev.Sources, ","), "ads,tracking"; got != want {
		t.Errorf("refresh sources = %q, want %q", got, want)
	}
}

// TestBroadcastsGoToTheKindChannel pins the channel names, because a publish to
// the wrong channel is invisible: no subscriber is listening there, so the
// event is simply dropped.
func TestBroadcastsGoToTheKindChannel(t *testing.T) {
	h := newHarness(t)
	n := h.start("a", nil)
	ctx := context.Background()
	want := map[EventKind]string{
		EventConfig:     "fdtest_TestBroadcastsGoToTheKindChannel:events:config",
		EventReload:     "fdtest_TestBroadcastsGoToTheKindChannel:events:reload",
		EventBlocklists: "fdtest_TestBroadcastsGoToTheKindChannel:events:blocklists",
		EventHeartbeat:  "fdtest_TestBroadcastsGoToTheKindChannel:events:heartbeat",
		EventBye:        "fdtest_TestBroadcastsGoToTheKindChannel:events:bye",
	}
	for k, ch := range want {
		if got := n.channel(k); got != ch {
			t.Errorf("channel(%q) = %q, want %q", k, got, ch)
		}
	}
	if got := n.heartbeatKey(); got != "fdtest_TestBroadcastsGoToTheKindChannel:nodes" {
		t.Errorf("heartbeatKey() = %q", got)
	}
	if got := n.leaseKey(); got != "fdtest_TestBroadcastsGoToTheKindChannel:leader" {
		t.Errorf("leaseKey() = %q", got)
	}
	// A publish to a channel nobody subscribes to must be reported as such
	// by the server, otherwise a mis-typed channel would look like a delivery.
	cl := n.client.Load()
	if cl == nil {
		t.Fatal("no client")
	}
	for k, ch := range want {
		if k == EventHeartbeat {
			continue // this node heartbeats here
		}
		if err := cl.Publish(ctx, ch, "probe").Err(); err != nil {
			t.Fatalf("publish to %s: %v", ch, err)
		}
	}
}

// ---------------------------------------------------------------------------
// 5. Divergence detection.
// ---------------------------------------------------------------------------

// TestStatusReportsDivergentPeers pins the exact rule Status applies: a peer is
// divergent when both sides publish a non-empty blocklist digest and the two
// differ. The config digest is published for the UI but is *not* part of the
// rule (see TestDigestIsAHashOfTheWholeDocument).
func TestStatusReportsDivergentPeers(t *testing.T) {
	h := newHarness(t)
	da := &digests{}
	db := &digests{}
	da.set("rules-a", 10)
	db.set("rules-b", 20)
	a := h.start("a", func(o *Options) { o.Digests = da.fn })
	b := h.start("b", func(o *Options) { o.Digests = db.fn })
	waitSubscribed(t, h.mini, a)
	waitSubscribed(t, h.mini, b)
	ctx := context.Background()

	// Different digests: each must name the other.
	waitFor(t, "a to report b as divergent", 3*time.Second, func() bool {
		return contains(a.Status(ctx).Divergent, b.NodeID())
	})
	waitFor(t, "b to report a as divergent", 3*time.Second, func() bool {
		return contains(b.Status(ctx).Divergent, a.NodeID())
	})

	// Same digests: neither may be reported. The heartbeat interval is what
	// carries the new value across, so wait for the report to clear.
	db.set("rules-a", 10)
	waitFor(t, "b's divergence to clear once the digests match", 3*time.Second, func() bool {
		return len(b.Status(ctx).Divergent) == 0
	})
	waitFor(t, "a's divergence to clear once the digests match", 3*time.Second, func() bool {
		return len(a.Status(ctx).Divergent) == 0
	})
	// And a peer with no blocklist at all (empty digest) is not divergence:
	// the rule requires both sides to be non-empty.
	db.set("", 0)
	waitFor(t, "a's view of b to be republished with an empty digest", 3*time.Second, func() bool {
		return peerDigest(a, b.NodeID()) == ""
	})
	if got := a.Status(ctx).Divergent; len(got) != 0 {
		t.Errorf("a peer with no blocklist is reported divergent: %v", got)
	}
	if got := b.Status(ctx).Divergent; len(got) != 0 {
		t.Errorf("a node with no blocklist reports divergence: %v", got)
	}
}

// ---------------------------------------------------------------------------
// 6. Digest.
// ---------------------------------------------------------------------------

// TestDigestIsAHashOfTheWholeDocument pins exactly what Digest covers, because
// the interesting property is which fields are in and which are out.
//
// IN: every field of the marshalled document - blocklist lists and order, ACL
// groups, upstreams, log level, and the cluster section itself (node id,
// valkey addresses, TTLs).
// OUT: nothing. Two configs that differ in any single field hash differently.
//
// The consequence, which is why divergence is computed on the *blocklist*
// digest and not on this one: every instance runs its own node id and
// advertise address, so two healthy instances of the same release can never
// agree on a config digest.
func TestDigestIsAHashOfTheWholeDocument(t *testing.T) {
	base := config.Default()
	// Stability: the same content hashes the same, twice and from two
	// separately built documents.
	if Digest(base) != Digest(base) {
		t.Error("Digest is not stable for the same *Config")
	}
	other := config.Default()
	if Digest(base) != Digest(other) {
		t.Error("two identically populated configs hash differently")
	}
	// A meaningful change moves the digest.
	changed := *base
	changed.Cache.Size = base.Cache.Size + 1
	if Digest(base) == Digest(&changed) {
		t.Error("changing cache.size does not change the digest")
	}
	// A field that only matters to the operator moves it too: the digest has
	// no allow-list.
	loud := *base
	loud.Log.Level = "debug"
	if loud.Log.Level != "info" && Digest(base) == Digest(&loud) {
		t.Error("changing log.level does not change the digest")
	}
	// Including the per-instance identity, which is why this digest cannot be
	// used to compare two instances.
	renamed := *base
	renamed.Cluster.NodeID = "pod-2"
	if Digest(base) == Digest(&renamed) {
		t.Error("changing cluster.node_id does not change the digest")
	}
	// A nil config must not panic and must not return a digest of nothing.
	if got := Digest(nil); got == "" {
		t.Error("Digest(nil) is empty")
	}
	if strings.ContainsAny(Digest(nil), "ghijklmnopqrstuvwxyz") {
		t.Errorf("Digest(nil) = %q is not hex", Digest(nil))
	}
}

// ---------------------------------------------------------------------------
// 7. Heartbeat, peers and discovery.
// ---------------------------------------------------------------------------

func TestHeartbeatAppearsInPeersWithItsIdentity(t *testing.T) {
	h := newHarness(t)
	d := &digests{}
	d.set("rules-a", 42)
	a := h.start("a", func(o *Options) { o.Digests = d.fn })
	waitSubscribed(t, h.mini, a)
	started := time.Now().Add(-time.Hour).Truncate(time.Second)
	_ = started

	// A second node joins; the first must learn about it from heartbeats
	// alone, without any API call.
	b := h.start("b", nil)
	waitSubscribed(t, h.mini, a)
	waitSubscribed(t, h.mini, b)

	waitFor(t, "a to see b in Peers()", 3*time.Second, func() bool {
		return hasPeer(a.Peers(), b.NodeID())
	})
	var got NodeState
	for _, p := range a.Peers() {
		if p.NodeID == b.NodeID() {
			got = p
		}
	}
	if got.Version != "test" {
		t.Errorf("peer version = %q, want %q", got.Version, "test")
	}
	if got.Addr != b.opt.Addr {
		t.Errorf("peer addr = %q, want %q", got.Addr, b.opt.Addr)
	}
	if got.StartedAt.IsZero() {
		t.Error("peer carries no start timestamp")
	}
	if !got.StartedAt.Equal(b.State().StartedAt) {
		t.Errorf("peer started_at = %s, want %s", got.StartedAt, b.State().StartedAt)
	}
	// A node never lists itself.
	if hasPeer(a.Peers(), a.NodeID()) {
		t.Error("a node appears in its own peer list")
	}
	// The other direction has to be waited for rather than read: a joined
	// *after* a's heartbeat was already in flight, so a hearing b is
	// immediate while b hears a only on a's next tick. Reading b here is a
	// race that loses almost every run, and it has nothing to do with the
	// behaviour under test.
	waitFor(t, "b to see a in Peers()", 3*time.Second, func() bool {
		return hasPeer(b.Peers(), a.NodeID())
	})
	// The blocklist digest travels with the heartbeat: that is what the
	// divergence rule compares. b has no digests configured, so a's view of
	// b is legitimately empty and b's view of a is the one that must arrive.
	waitFor(t, "b to learn a's blocklist digest", 3*time.Second, func() bool {
		return peerDigest(b, a.NodeID()) == "rules-a"
	})
	if got := peerDigest(a, b.NodeID()); got != "" {
		t.Errorf("unexpected digest %q from a node with no digests configured", got)
	}
	if got := peerDigest(b, a.NodeID()); got != "rules-a" {
		t.Errorf("b sees a's blocklist digest as %q, want %q", got, "rules-a")
	}
	if got := a.State().BlocklistRules; got != 42 {
		t.Errorf("a reports %d blocklist rules, want 42", got)
	}
	if a.State().NodeID != a.NodeID() {
		t.Error("State().NodeID does not match NodeID()")
	}
}

func TestDeadPeerAgesOutOfPeers(t *testing.T) {
	h := newHarness(t)
	a := h.start("a", nil)
	waitSubscribed(t, h.mini, a)
	b := h.start("b", nil)
	waitSubscribed(t, h.mini, b)
	waitFor(t, "a to see b in Peers()", 3*time.Second, func() bool {
		return hasPeer(a.Peers(), b.NodeID())
	})

	// b is killed without a farewell, so the only signal left is silence.
	b.kill()
	waitFor(t, "the dead peer to age out of a's peer list", 4*time.Second, func() bool {
		return !hasPeer(a.Peers(), b.NodeID())
	})
	// It must not reappear: nothing is left to publish it.
	time.Sleep(2 * (testPeerTTL / 2))
	if hasPeer(a.Peers(), b.NodeID()) {
		t.Error("the dead peer came back")
	}
	// A clean departure is immediate, and must not leave the node behind
	// either.
	c := h.start("c", nil)
	waitSubscribed(t, h.mini, c)
	waitFor(t, "a to see c in Peers()", 3*time.Second, func() bool {
		return hasPeer(a.Peers(), c.NodeID())
	})
	c.Stop()
	waitFor(t, "a to drop c after its farewell", 3*time.Second, func() bool {
		return !hasPeer(a.Peers(), c.NodeID())
	})
}

func TestDiscoverListsPeersButNotTheLocalNodeTwice(t *testing.T) {
	h := newHarness(t)
	a := h.start("a", nil)
	waitSubscribed(t, h.mini, a)
	b := h.start("b", nil)
	waitSubscribed(t, h.mini, b)
	waitFor(t, "a to see b in Peers()", 3*time.Second, func() bool {
		return hasPeer(a.Peers(), b.NodeID())
	})
	ctx := context.Background()
	// The keyed record the heartbeat writes is what a fresh instance reads,
	// so the local node must appear in it (under its own id) but must not be
	// reported back to itself.
	waitFor(t, "both nodes in the heartbeat record", 3*time.Second, func() bool {
		found := a.Discover(ctx)
		return hasPeer(found, b.NodeID()) && !hasPeer(found, a.NodeID())
	})
	found := a.Discover(ctx)
	seen := map[string]bool{}
	for _, s := range found {
		if seen[s.NodeID] {
			t.Errorf("Discover() returned %q twice", s.NodeID)
		}
		seen[s.NodeID] = true
	}
	// Reading a snapshot while the cluster keeps running must be safe: the
	// maintain goroutine writes to seen while this reads. Run under -race.
	stop := make(chan struct{})
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			select {
			case <-stop:
				return
			default:
			}
			_ = a.Discover(ctx)
			_ = a.Peers()
			_ = a.Status(ctx)
		}
	}()
	time.Sleep(100 * time.Millisecond)
	close(stop)
	wg.Wait()
	if hasPeer(a.Discover(ctx), a.NodeID()) {
		t.Error("Discover() returned the local node")
	}
}

// ---------------------------------------------------------------------------
// 8. Resilience.
// ---------------------------------------------------------------------------

func TestRedisOutageDoesNotPanicAndRecovers(t *testing.T) {
	m := miniredisNew(t)
	h := newHarnessOn(t, m)
	a := h.start("a", nil)
	ctx := context.Background()
	waitFor(t, "the node to connect", 3*time.Second, a.Connected)
	waitFor(t, "the node to take the lease", 3*time.Second, a.IsLeader)
	addr := m.Addr()

	m.Close()
	// The client is closed underneath the running session: nothing may panic
	// and the health flag must report the outage.
	waitFor(t, "Connected() to go false while Valkey is away", 5*time.Second, func() bool {
		return !a.Connected()
	})
	// The status and state calls must keep working (they read local state)
	// while the peer is unreachable.
	_ = a.Status(ctx)
	_ = a.State()
	_ = a.Peers()
	if err := a.BroadcastReload(ctx); err == nil {
		t.Log("BroadcastReload reported success while Valkey was down")
	}

	if err := m.StartAddr(addr); err != nil {
		t.Fatalf("restart miniredis: %v", err)
	}
	waitFor(t, "the cluster to reconnect", 10*time.Second, a.Connected)
	waitFor(t, "the node to reclaim the lease", 5*time.Second, a.IsLeader)
	// And it is live again: a peer can still be discovered.
	b := h.start("b", nil)
	waitSubscribed(t, h.mini, b)
	waitFor(t, "a to see b after the outage", 5*time.Second, func() bool {
		return hasPeer(a.Peers(), b.NodeID())
	})
}

func TestStopDuringAnActiveSessionIsClean(t *testing.T) {
	h := newHarness(t)
	// Let any one-off runtime goroutines settle before the baseline.
	runtime.GC()
	time.Sleep(50 * time.Millisecond)
	before := runtime.NumGoroutine()

	nodes := make([]*node, 0, 3)
	for i, name := range []string{"a", "b", "c"} {
		nodes = append(nodes, h.start(name, func(o *Options) {
			o.Version = "test"
			o.Addr = name
			_ = i
		}))
	}
	for _, n := range nodes {
		waitFor(t, "a node to be connected", 3*time.Second, n.Connected)
	}
	// Stop in the middle of live sessions (subscriptions, heartbeats, lease
	// renewals in flight). Stop twice: shutdown paths do that.
	for _, n := range nodes {
		n.Stop()
		n.Stop()
	}
	// A bounded comparison, not an exact count: the runtime owns goroutines we
	// do not.
	waitFor(t, "the maintain goroutines to exit", 5*time.Second, func() bool {
		return runtime.NumGoroutine() <= before+2
	})
	for _, n := range nodes {
		if n.Connected() {
			t.Errorf("%s still reports Connected after Stop", n.NodeID())
		}
		if n.IsLeader() {
			t.Errorf("%s still claims the lease after Stop", n.NodeID())
		}
	}
}

// ---------------------------------------------------------------------------
// 9. Pure helpers.
// ---------------------------------------------------------------------------

func TestMustJSONFallsBackToAnObject(t *testing.T) {
	if got := mustJSON(NodeState{NodeID: "n1", Version: "v"}); !strings.Contains(got, `"n1"`) {
		t.Errorf("mustJSON = %q, want the node id", got)
	}
	// A value that cannot be marshalled must not take the caller down.
	if got := mustJSON(make(chan int)); got != "{}" {
		t.Errorf("mustJSON(chan) = %q, want %q", got, "{}")
	}
}

func TestCryptoReadFillsTheBuffer(t *testing.T) {
	b := make([]byte, 4)
	n, err := cryptoRead(b)
	if err != nil {
		t.Fatalf("cryptoRead: %v", err)
	}
	if n != len(b) {
		t.Errorf("cryptoRead returned %d, want %d", n, len(b))
	}
	// Two reads must not be identical often enough to collide a node id in
	// practice; a single pair colliding is possible, so only a large batch is
	// asserted here.
	seen := map[string]bool{}
	for i := 0; i < 256; i++ {
		x := make([]byte, 4)
		if _, err := cryptoRead(x); err != nil {
			t.Fatalf("cryptoRead: %v", err)
		}
		k := string(x)
		if seen[k] {
			t.Fatalf("cryptoRead repeated %q after %d draws", k, i)
		}
		seen[k] = true
	}
}

func TestShortIDIsHexAndUniquelyNamesNodes(t *testing.T) {
	for i := 0; i < 32; i++ {
		s := shortID()
		if len(s) != 8 {
			t.Fatalf("shortID() = %q, want 8 hex characters", s)
		}
		if strings.Trim(s, "0123456789abcdef") != "" {
			t.Fatalf("shortID() = %q is not lowercase hex", s)
		}
	}
	seen := map[string]bool{}
	for i := 0; i < 64; i++ {
		s := shortID()
		if seen[s] {
			t.Fatalf("shortID() repeated %q", s)
		}
		seen[s] = true
	}
}

func TestNodeIdentityAndString(t *testing.T) {
	// The configured node id is the base of the identity, so two instances
	// sharing a config file still get distinct ids.
	h := newHarness(t, func(c *config.Config) { c.Cluster.NodeID = "pod-0" })
	a := New(Options{Config: h.store, Logger: discardLogger(), Version: "v1.2.3"})
	b := New(Options{Config: h.store, Logger: discardLogger(), Version: "v1.2.3"})
	if a.NodeID() == b.NodeID() {
		t.Errorf("two clusters share the node id %q", a.NodeID())
	}
	for _, c := range []*Cluster{a, b} {
		if !strings.HasPrefix(c.NodeID(), "pod-0-") {
			t.Errorf("node id %q does not start with the configured node id", c.NodeID())
		}
		if c.String() != c.NodeID() {
			t.Errorf("String() = %q, want %q", c.String(), c.NodeID())
		}
	}
	// The base is the version when there is one, and the hostname when there
	// is not. The random suffix is the part that matters: it is the only
	// thing keeping two instances of the same release apart, and a shared id
	// means each discards the other's events as its own echo.
	h2 := newHarness(t, func(c *config.Config) { c.Cluster.NodeID = "" })
	ver := New(Options{Config: h2.store, Logger: discardLogger(), Version: "v1"})
	if !strings.HasPrefix(ver.NodeID(), "v1-") {
		t.Errorf("node id %q does not start with the version", ver.NodeID())
	}
	noVer := New(Options{Config: h2.store, Logger: discardLogger()})
	host, _ := os.Hostname()
	if host != "" && !strings.HasPrefix(noVer.NodeID(), host+"-") {
		t.Errorf("node id %q does not start with the hostname %q", noVer.NodeID(), host)
	}
	if ver.NodeID() == noVer.NodeID() {
		t.Error("two clusters share a node id")
	}
}

// ---------------------------------------------------------------------------
// helpers
// ---------------------------------------------------------------------------

func contains(list []string, s string) bool {
	for _, v := range list {
		if v == s {
			return true
		}
	}
	return false
}

// peerDigest returns the blocklist digest node has on record for id.
func peerDigest(n *node, id string) string {
	for _, p := range n.Peers() {
		if p.NodeID == id {
			return p.BlocklistDigest
		}
	}
	return ""
}