metrics

pkg/actor/simple_grain_pool.go

@@ -28,6 +28,12 @@ type SimpleGrainPool[V any] struct {
 	// grain processes a single message at a time (the actor guarantee).
 	grainLocks *keyedMutex
 
+	// metrics is the prometheus instrumentation wired in by NewMetrics
+	// at composition time. Nil is fine — every touch site nil-checks
+	// first, so the existing tests (which construct a pool without
+	// metrics) keep working.
+	metrics *Metrics
+
 	// Cluster coordination --------------------------------------------------
 	hostname     string
 	remoteMu     sync.RWMutex
@@ -51,6 +57,13 @@ type GrainPoolConfig[V any] struct {
 	PoolSize         int
 	MutationRegistry MutationRegistry
 	Storage          LogStorage[V]
+	// Metrics is the prometheus instrumentation to register pool-level
+	// counters, gauges, and histograms with. Nil-safe: when unset, all
+	// touch sites are no-ops. Use actor.NewMetrics("cart") (or the
+	// matching service prefix) to construct the per-service metrics
+	// struct, and pair it with DiskStorage.SetMetrics so the same
+	// Metrics instance covers both pool and event-log metrics.
+	Metrics *Metrics
 }
 
 func NewSimpleGrainPool[V any](config GrainPoolConfig[V]) (*SimpleGrainPool[V], error) {
@@ -64,6 +77,7 @@ func NewSimpleGrainPool[V any](config GrainPoolConfig[V]) (*SimpleGrainPool[V],
 		ttl:              config.TTL,
 		poolSize:         config.PoolSize,
 		hostname:         config.Hostname,
+		metrics:          config.Metrics,
 		remoteOwners:     make(map[uint64]Host[V]),
 		remoteHosts:      make(map[string]Host[V]),
 		grainLocks:       newKeyedMutex(),
@@ -76,6 +90,11 @@ func NewSimpleGrainPool[V any](config GrainPoolConfig[V]) (*SimpleGrainPool[V],
 		}
 	}()
 
+	// Publish the initial pool stats so the dashboard has a number to
+	// draw on first scrape, rather than waiting for the first grain
+	// spawn / eviction to bump the gauges.
+	p.metrics.SetPoolStats(0, p.poolSize)
+
 	return p, nil
 }
 
@@ -96,6 +115,7 @@ func (p *SimpleGrainPool[V]) purge() {
 	purgeLimit := time.Now().Add(-p.ttl)
 	purgedIds := make([]uint64, 0, len(p.grains))
 	p.localMu.Lock()
+	evicted := 0
 	for id, grain := range p.grains {
 		if grain.GetLastAccess().Before(purgeLimit) {
 			purgedIds = append(purgedIds, id)
@@ -106,9 +126,14 @@ func (p *SimpleGrainPool[V]) purge() {
 			}
 
 			delete(p.grains, id)
+			evicted++
 		}
 	}
+	remaining := len(p.grains)
 	p.localMu.Unlock()
+	if evicted > 0 {
+		p.metrics.SetPoolStats(remaining, p.poolSize)
+	}
 	p.forAllHosts(func(remote Host[V]) {
 		remote.AnnounceExpiry(purgedIds)
 	})
@@ -156,15 +181,20 @@ func (p *SimpleGrainPool[V]) HandleOwnershipChange(host string, ids []uint64) er
 		}
 		remoteHost = createdHost
 	}
+	// Lock order: remoteMu before localMu. Unlock in reverse order so
+	// the localMu gauge read below cannot deadlock against a concurrent
+	// p.localMu holder.
 	p.remoteMu.Lock()
-	defer p.remoteMu.Unlock()
 	p.localMu.Lock()
-	defer p.localMu.Unlock()
 	for _, id := range ids {
 		log.Printf("Handling ownership change for cart %d to host %s", id, host)
 		delete(p.grains, id)
 		p.remoteOwners[id] = remoteHost
 	}
+	remaining := len(p.grains)
+	p.localMu.Unlock()
+	p.remoteMu.Unlock()
+	p.metrics.SetPoolStats(remaining, p.poolSize)
 	return nil
 }
 
@@ -209,8 +239,9 @@ func (p *SimpleGrainPool[V]) AddRemote(host string) (Host[V], error) {
 		return existing, nil
 	}
 	p.remoteHosts[host] = remote
+	count := len(p.remoteHosts)
 	p.remoteMu.Unlock()
-	// connectedRemotes.Set(float64(p.RemoteCount()))
+	p.metrics.SetConnectedRemotes(count)
 
 	log.Printf("Connected to remote host %s", host)
 	go p.pingLoop(remote)
@@ -250,6 +281,7 @@ func (p *SimpleGrainPool[V]) RemoveHost(host string) {
 			delete(p.remoteOwners, id)
 		}
 	}
+	remoteCount := len(p.remoteHosts)
 	log.Printf("Removing host %s, grains: %d", host, count)
 	p.remoteMu.Unlock()
 
@@ -257,7 +289,7 @@ func (p *SimpleGrainPool[V]) RemoveHost(host string) {
 	if exists {
 		remote.Close()
 	}
-	// connectedRemotes.Set(float64(p.RemoteCount()))
+	p.metrics.SetConnectedRemotes(remoteCount)
 }
 
 func (p *SimpleGrainPool[V]) RemoteCount() int {
@@ -332,7 +364,7 @@ func (p *SimpleGrainPool[V]) Negotiate(otherHosts []string) {
 }
 
 func (p *SimpleGrainPool[V]) SendNegotiation() {
-	//negotiationCount.Inc()
+	p.metrics.IncNegotiation()
 
 	p.remoteMu.RLock()
 	hosts := make([]string, 0, len(p.remoteHosts)+1)
@@ -405,10 +437,13 @@ func (p *SimpleGrainPool[V]) getOrClaimGrain(ctx context.Context, id uint64) (Gr
 	if err != nil {
 		return nil, err
 	}
+	p.metrics.IncSpawn()
 	go p.broadcastOwnership([]uint64{id})
 	p.localMu.Lock()
 	p.grains[id] = grain
+	remaining := len(p.grains)
 	p.localMu.Unlock()
+	p.metrics.SetPoolStats(remaining, p.poolSize)
 
 	return grain, nil
 }
@@ -417,7 +452,20 @@ func (p *SimpleGrainPool[V]) getOrClaimGrain(ctx context.Context, id uint64) (Gr
 // var ErrNotOwner = fmt.Errorf("not owner")
 
 // Apply applies a mutation to a grain.
-func (p *SimpleGrainPool[V]) Apply(ctx context.Context, id uint64, mutation ...proto.Message) (*MutationResult[V], error) {
+func (p *SimpleGrainPool[V]) Apply(ctx context.Context, id uint64, mutation ...proto.Message) (result *MutationResult[V], err error) {
+	// The metric closure fires exactly once on every return path: the
+	// latency / counters cover the full Apply (lock acquire + spawn +
+	// handler + storage enqueue), and the failure counter only bumps
+	// when the call returned a non-nil error. `len(mutation)` is the
+	// number of proto messages in this Apply call — a single call can
+	// batch several messages (SetCartItems sends ClearCart + N AddItem
+	// together), and the counter is incremented by that count so the
+	// dashboard's `rate(cart_mutations_total[1m])` reports messages
+	// per second rather than calls per second, matching the original
+	// `grainMutations.Add(len(data.Mutations))` semantics.
+	done := p.metrics.MutationStarted(len(mutation))
+	defer func() { done(err) }()
+
 	// Serialize all access to this grain: spawn, mutation handlers and the
 	// final state read happen atomically with respect to other callers of the
 	// same id. Different ids never contend.
@@ -443,19 +491,20 @@ func (p *SimpleGrainPool[V]) Apply(ctx context.Context, id uint64, mutation ...p
 	for _, listener := range p.listeners {
 		go listener.AppendMutations(id, mutations...)
 	}
-	result, err := grain.GetCurrentState()
+	state, err := grain.GetCurrentState()
 	if err != nil {
 		return nil, err
 	}
 
 	return &MutationResult[V]{
-		Result:    *result,
+		Result:    *state,
 		Mutations: mutations,
 	}, nil
 }
 
 // Get returns the current state of a grain.
-func (p *SimpleGrainPool[V]) Get(ctx context.Context, id uint64) (*V, error) {
+func (p *SimpleGrainPool[V]) Get(ctx context.Context, id uint64) (result *V, err error) {
+	p.metrics.IncLookup()
 	unlock := p.grainLocks.lock(id)
 	defer unlock()