update

pkg/actor/simple_grain_pool.go

@@ -45,6 +45,16 @@ type SimpleGrainPool[V any] struct {
 	purgeTicker *time.Ticker
 }
 
+// noLocalStamp is the sentinel value used in the parallel []int64
+// lastChanges slice to mark an id for which the broadcaster has no
+// local grain (e.g. TakeOwnership called before anyone has loaded the
+// grain, or a custom Host implementation that doesn't carry a stamp).
+// Receivers treat this sentinel as "no arbitration possible" and fall
+// back to the pre-arbitration "always accept remote" verdict so the
+// behaviour is identical to running an older binary that didn't carry
+// stamps at all.
+const noLocalStamp int64 = -1
+
 type GrainPoolConfig[V any] struct {
 	Hostname         string
 	Spawn            func(ctx context.Context, id uint64) (Grain[V], error)
@@ -114,17 +124,20 @@ func (p *SimpleGrainPool[V]) RemoveListener(listener LogListener) {
 func (p *SimpleGrainPool[V]) purge() {
 	purgeLimit := time.Now().Add(-p.ttl)
 	purgedIds := make([]uint64, 0, len(p.grains))
+	purgedStamps := make([]int64, 0, len(p.grains))
 	p.localMu.Lock()
 	evicted := 0
 	for id, grain := range p.grains {
 		if grain.GetLastAccess().Before(purgeLimit) {
-			purgedIds = append(purgedIds, id)
 			if p.destroy != nil {
 				if err := p.destroy(grain); err != nil {
 					log.Printf("failed to destroy grain %d: %v", id, err)
 				}
 			}
-
+			// Capture the lastChange stamp BEFORE delete so peers
+			// receive an honest per-id eviction record on the wire.
+			purgedIds = append(purgedIds, id)
+			purgedStamps = append(purgedStamps, grain.GetLastChange().UnixNano())
 			delete(p.grains, id)
 			evicted++
 		}
@@ -135,7 +148,7 @@ func (p *SimpleGrainPool[V]) purge() {
 		p.metrics.SetPoolStats(remaining, p.poolSize)
 	}
 	p.forAllHosts(func(remote Host[V]) {
-		remote.AnnounceExpiry(purgedIds)
+		remote.AnnounceExpiry(purgedIds, purgedStamps)
 	})
 
 }
@@ -161,7 +174,14 @@ func (p *SimpleGrainPool[V]) GetLocalIds() []uint64 {
 	return ids
 }
 
-func (p *SimpleGrainPool[V]) HandleRemoteExpiry(host string, ids []uint64) error {
+func (p *SimpleGrainPool[V]) HandleRemoteExpiry(host string, ids []uint64, _ []int64) error {
+	// lastChanges is informational on this path; expiry is unilateral
+	// from the announcer's perspective — we just drop the id from any
+	// remoteOwners mapping so future cross-pod forwards stop hitting a
+	// dead peer. The parallel stamp slice is kept in the signature for
+	// wire symmetry with AnnounceOwnership and so a future hardening
+	// pass can use stamps to detect ghost-evictions (e.g. peer evicted
+	// with a stamp newer than our last seen Apply result).
 	p.remoteMu.Lock()
 	defer p.remoteMu.Unlock()
 	for _, id := range ids {
@@ -170,7 +190,7 @@ func (p *SimpleGrainPool[V]) HandleRemoteExpiry(host string, ids []uint64) error
 	return nil
 }
 
-func (p *SimpleGrainPool[V]) HandleOwnershipChange(host string, ids []uint64) error {
+func (p *SimpleGrainPool[V]) HandleOwnershipChange(host string, ids []uint64, lastChanges []int64) error {
 	p.remoteMu.RLock()
 	remoteHost, exists := p.remoteHosts[host]
 	p.remoteMu.RUnlock()
@@ -186,15 +206,71 @@ func (p *SimpleGrainPool[V]) HandleOwnershipChange(host string, ids []uint64) er
 	// p.localMu holder.
 	p.remoteMu.Lock()
 	p.localMu.Lock()
-	for _, id := range ids {
-		log.Printf("Handling ownership change for cart %d to host %s", id, host)
+	var accepted, kept, legacy int
+	for i, id := range ids {
+		// remoteStamp is the announcer's lastChange at broadcast time,
+		// or noLocalStamp (-1) if the announcer didn't have a local
+		// grain (legacy / TakeOwnership path). The
+		// 0..len(lastChanges)-1 slice is indexed parallel to ids; out
+		// of range falls back to noLocalStamp (= "no arbitration").
+		var remoteStamp int64 = noLocalStamp
+		if i < len(lastChanges) {
+			remoteStamp = lastChanges[i]
+		}
+		var localStamp int64 = noLocalStamp
+		var hasLocal bool
+		if g, ok := p.grains[id]; ok {
+			localStamp = g.GetLastChange().UnixNano()
+			hasLocal = true
+		}
+
+		// First-spawn-wins arbitration. Five distinct cases, ordered:
+		//   1) remoteStamp == noLocalStamp  → announcer didn't carry a
+		//      real stamp (legacy or TakeOwnership). Keep current
+		//      "always accept remote" behaviour so mixed-version
+		//      rollouts and unbroadcast-advertised TakeOwnership don't
+		//      regress.
+		//   2) localStamp  == noLocalStamp  → we have no grain at all.
+		//      Accept remote so the next read forwards to the right
+		//      owner instead of us spawning a parallel projection.
+		//   3) stamps differ → first-spawn-wins. The pod that spawned
+		//      the grain earlier (smaller stamp) owns it; its broadcast
+		//      is authoritative. The newer pod's projection is
+		//      redundant — we drop our local copy and defer to remote
+		//      so future OwnerHost lookups route correctly.
+		//   4) stamps equal  → deterministic tie-break: lexicographically
+		//      smaller hostname wins (both pods compute the same
+		//      verdict, so the cold-cache first-touch cannot flip-flop).
+		var keepLocal bool
+		switch {
+		case remoteStamp == noLocalStamp:
+			keepLocal = false
+			if hasLocal {
+				legacy++
+			}
+		case localStamp == noLocalStamp:
+			keepLocal = false
+		case localStamp != remoteStamp:
+			keepLocal = localStamp < remoteStamp
+		default:
+			keepLocal = p.hostname < host
+		}
+		if keepLocal {
+			kept++
+			continue
+		}
 		delete(p.grains, id)
 		p.remoteOwners[id] = remoteHost
+		accepted++
 	}
 	remaining := len(p.grains)
 	p.localMu.Unlock()
 	p.remoteMu.Unlock()
 	p.metrics.SetPoolStats(remaining, p.poolSize)
+	if kept > 0 || accepted > 0 || legacy > 0 {
+		log.Printf("HandleOwnershipChange from %s: accepted_remote=%d kept_local=%d legacy_no_stamp=%d",
+			host, accepted, kept, legacy)
+	}
 	return nil
 }
 
@@ -418,8 +494,27 @@ func (p *SimpleGrainPool[V]) broadcastOwnership(ids []uint64) {
 		return
 	}
 
+	// Capture each id's grain.lastChange UnixNano stamp under the local
+	// read-lock so peers receive an honest oracle for the
+	// first-spawn-wins arbitration. For ids that don't have a local
+	// grain (e.g. TakeOwnership called before anyone has loaded the
+	// grain, or a caller pushing ids into the pool from outside the
+	// spawn path), substitute the noLocalStamp sentinel — receivers see
+	// -1 and fall back to "always accept remote", preserving the
+	// pre-arbitration semantics for that case.
+	p.localMu.RLock()
+	stamps := make([]int64, 0, len(ids))
+	for _, id := range ids {
+		if g, ok := p.grains[id]; ok {
+			stamps = append(stamps, g.GetLastChange().UnixNano())
+		} else {
+			stamps = append(stamps, noLocalStamp)
+		}
+	}
+	p.localMu.RUnlock()
+
 	p.forAllHosts(func(rh Host[V]) {
-		rh.AnnounceOwnership(p.hostname, ids)
+		rh.AnnounceOwnership(p.hostname, ids, stamps)
 	})
 	log.Printf("%s taking ownership of %d ids", p.hostname, len(ids))
 	// go p.statsUpdate()