more stuff

synced-pool.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"reflect"
 	"sync"
 	"time"
 
@@ -105,6 +106,27 @@ var (
 		Name: "cart_ring_host_share",
 		Help: "Fractional share of ring vnodes per host",
 	}, []string{"host"})
+
+	cartMutationsTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "cart_mutations_total",
+		Help: "Total number of cart state mutations applied (local + remote routed).",
+	})
+
+	cartMutationFailuresTotal = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "cart_mutation_failures_total",
+		Help: "Total number of failed cart state mutations (local apply errors or remote routing failures).",
+	})
+
+	cartMutationLatencySeconds = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "cart_mutation_latency_seconds",
+		Help:    "Latency of cart mutations (successful or failed) in seconds.",
+		Buckets: prometheus.DefBuckets,
+	}, []string{"mutation"})
+
+	cartActiveGrains = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "cart_active_grains",
+		Help: "Number of active (resident) local grains.",
+	})
 )
 
 func NewSyncedPool(local *GrainLocalPool, hostname string, discovery Discovery) (*SyncedPool, error) {
@@ -564,7 +586,33 @@ func (p *SyncedPool) Apply(id CartId, mutation interface{}) (*CartGrain, error)
 	if err != nil {
 		return nil, err
 	}
-	return grain.Apply(mutation, false)
+	start := time.Now()
+	result, applyErr := grain.Apply(mutation, false)
+
+	// Derive mutation type label (strip pointer)
+	mutationType := "unknown"
+	if mutation != nil {
+		if t := reflect.TypeOf(mutation); t != nil {
+			if t.Kind() == reflect.Ptr {
+				t = t.Elem()
+			}
+			if t.Name() != "" {
+				mutationType = t.Name()
+			}
+		}
+	}
+	cartMutationLatencySeconds.WithLabelValues(mutationType).Observe(time.Since(start).Seconds())
+
+	if applyErr == nil && result != nil {
+		cartMutationsTotal.Inc()
+		if p.ownerHostFor(id) == p.Hostname {
+			// Update active grains gauge only for local ownership
+			cartActiveGrains.Set(float64(p.local.DebugGrainCount()))
+		}
+	} else if applyErr != nil {
+		cartMutationFailuresTotal.Inc()
+	}
+	return result, applyErr
 }
 
 // Get returns current state of a grain (local or remote).