half broken

internal/cluster/cluster.go

@@ -7,6 +7,7 @@ package cluster
 
 import (
 	"context"
+	"crypto/rand"
 	"crypto/sha1"
 	"encoding/hex"
 	"encoding/json"
@@ -189,7 +190,14 @@ func (c *Cluster) Start(ctx context.Context) {
 	c.connected.Store(true)
 	c.log.Info("cluster connected", "addrs", cfg.Cluster.Valkey.Addrs, "node", c.node)
 
+	// Attempt the lease immediately rather than on the first heartbeat tick.
+	// The claim this package makes is "only the lease holder fetches
+	// blocklists", and at startup nobody holds it: waiting half a peer_ttl (7.5
+	// s at the default 15s) means every instance in a fresh rollout fetches
+	// every list from every mirror at once, which is the thundering herd the
+	// lease exists to prevent.
 	c.publishState(ctx)
+	c.renewLease(ctx)
 
 	c.wg.Add(1)
 	go func() {
@@ -279,7 +287,17 @@ func (c *Cluster) runSession(ctx context.Context) error {
 		case <-c.stop:
 			return nil
 		case <-hb.C:
-			c.publishState(ctx)
+			// A failed heartbeat is the only evidence of an outage: the
+			// subscription reconnects behind our back, so runSession keeps
+			// running and nothing else in this loop would ever notice. Without
+			// this, /status and the console claim a Valkey connection that
+			// stopped working, and a diverged peer would look healthy.
+			if err := c.publishState(ctx); err != nil {
+				c.connected.Store(false)
+				c.log.Warn("cluster heartbeat failed", "error", err)
+				continue
+			}
+			c.connected.Store(true)
 			c.renewLease(ctx)
 		case msg, ok := <-ch:
 			if !ok {
@@ -354,17 +372,20 @@ func (c *Cluster) State() *NodeState {
 	return st
 }
 
-// publishState broadcasts this instance's heartbeat.
-func (c *Cluster) publishState(ctx context.Context) {
+// publishState broadcasts this instance's heartbeat. It reports whether the
+// write reached Valkey: the heartbeat is the only liveness signal this package
+// has, because go-redis's Subscribe reconnects transparently and runSession
+// never sees the connection drop (see the connected flag in runSession).
+func (c *Cluster) publishState(ctx context.Context) error {
 	cl := c.client.Load()
 	if cl == nil {
-		return
+		return errors.New("cluster: no client")
 	}
 	st := c.State()
 	ev := Event{Kind: EventHeartbeat, NodeID: c.node, At: time.Now(), State: st}
 	data, err := json.Marshal(ev)
 	if err != nil {
-		return
+		return err
 	}
 	pipe := cl.Pipeline()
 	pipe.Publish(ctx, c.channel(EventHeartbeat), data)
@@ -372,7 +393,10 @@ func (c *Cluster) publishState(ctx context.Context) {
 	// existing set without waiting a full heartbeat interval.
 	pipe.HSet(ctx, c.heartbeatKey(), c.node, mustJSON(st))
 	pipe.Expire(ctx, c.heartbeatKey(), c.cfg.Get().Cluster.PeerTTL.D()*4)
-	_, _ = pipe.Exec(ctx)
+	if _, err := pipe.Exec(ctx); err != nil {
+		return err
+	}
+	return nil
 }
 
 func mustJSON(v any) string {
@@ -623,8 +647,4 @@ func (c *Cluster) Stop() {
 // String renders the node id for logs.
 func (c *Cluster) String() string { return strings.TrimSpace(c.node) }
 
-func cryptoRead(b []byte) (int, error) {
-	h := sha1.Sum([]byte(time.Now().String() + string(b)))
-	copy(b, h[:4])
-	return len(b), nil
-}
+func cryptoRead(b []byte) (int, error) { return rand.Read(b) }