package actor import ( "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) // Metrics bundles every prometheus metric the actor package emits. It is // constructed per service (cart / checkout / profile / order) with a // service-specific prefix so the metric names do not collide when more // than one actor-pool service is scraped into the same Prometheus. // // All touch sites are nil-safe: the pool / storage / registry code wraps // each counter / histogram call in a nil-check on the *Metrics pointer, // so the existing tests (which construct a pool or storage without // metrics) keep working without a forced dependency on this struct. // // Metric names match the panels in grafana_dashboard_cart.json exactly. // The dashboard had `connected_remotes` (no prefix) as a typo; the // source-of-truth metric exported here is `_connected_remotes` // and the dashboard JSON is updated in the same change to match. type Metrics struct { // Pool — set by SimpleGrainPool. ActiveGrains prometheus.Gauge GrainsInPool prometheus.Gauge PoolUsage prometheus.Gauge ConnectedRemotes prometheus.Gauge GrainSpawnedTotal prometheus.Counter GrainLookupsTotal prometheus.Counter MutationsTotal prometheus.Counter MutationFailuresTotal prometheus.Counter MutationLatency prometheus.Histogram RemoteNegotiationTotal prometheus.Counter // Event log — set by DiskStorage and StateStorage. EventLogAppends prometheus.Counter EventLogBytesWritten prometheus.Counter EventLogFilesExisting prometheus.Gauge EventLogLastAppendUnix prometheus.Gauge EventLogReplayTotal prometheus.Counter EventLogReplayFailures prometheus.Counter EventLogReplayDuration prometheus.Histogram EventLogUnknownTypes prometheus.Counter EventLogMutationErrors prometheus.Counter } // NewMetrics registers every actor metric in the default Prometheus // registry (the one promhttp.Handler() exposes on /metrics) with the // service-specific prefix. The prefix is the same string the binary uses // as its service name (e.g. "cart", "checkout", "profile", "order"), // which keeps the metric names aligned with the per-service labels in // grafana_dashboard_cart.json. func NewMetrics(prefix string) *Metrics { return &Metrics{ ActiveGrains: promauto.NewGauge(prometheus.GaugeOpts{ Name: prefix + "_active_grains", Help: "Number of grains currently held in the local pool.", }), GrainsInPool: promauto.NewGauge(prometheus.GaugeOpts{ Name: prefix + "_grains_in_pool", Help: "Alias of active_grains: the dashboard shows both as a sanity check.", }), PoolUsage: promauto.NewGauge(prometheus.GaugeOpts{ Name: prefix + "_grain_pool_usage", Help: "Local pool fill ratio: len(grains) / poolSize (range 0..1).", }), ConnectedRemotes: promauto.NewGauge(prometheus.GaugeOpts{ Name: prefix + "_connected_remotes", Help: "Number of remote actor-pool hosts currently connected via gRPC.", }), GrainSpawnedTotal: promauto.NewCounter(prometheus.CounterOpts{ Name: prefix + "_grain_spawned_total", Help: "Total number of grains spawned (loaded from disk or freshly created).", }), GrainLookupsTotal: promauto.NewCounter(prometheus.CounterOpts{ Name: prefix + "_grain_lookups_total", Help: "Total number of pool.Get calls (state reads).", }), MutationsTotal: promauto.NewCounter(prometheus.CounterOpts{ Name: prefix + "_mutations_total", Help: "Total number of mutations applied across all grains (one per proto message, not per call).", }), MutationFailuresTotal: promauto.NewCounter(prometheus.CounterOpts{ Name: prefix + "_mutation_failures_total", Help: "Total number of pool.Apply calls that returned an error.", }), // Latency buckets cover the typical pool.Apply (handler + registry // apply + storage enqueue): sub-ms in the hot path, hundreds of ms // under load. Tweak if a hot mutation type moves outside this range. MutationLatency: promauto.NewHistogram(prometheus.HistogramOpts{ Name: prefix + "_mutation_latency_seconds", Help: "Wall-clock duration of a pool.Apply call (lock + spawn + handler + storage enqueue).", Buckets: prometheus.ExponentialBuckets(0.0005, 2, 14), // 0.5ms ... ~8s }), RemoteNegotiationTotal: promauto.NewCounter(prometheus.CounterOpts{ Name: prefix + "_remote_negotiation_total", Help: "Total number of cluster-negotiation rounds initiated by this pod.", }), EventLogAppends: promauto.NewCounter(prometheus.CounterOpts{ Name: prefix + "_event_log_appends_total", Help: "Total number of event-log line appends (one per persisted mutation message).", }), EventLogBytesWritten: promauto.NewCounter(prometheus.CounterOpts{ Name: prefix + "_event_log_bytes_written_total", Help: "Total bytes written to per-grain .events.log files.", }), EventLogFilesExisting: promauto.NewGauge(prometheus.GaugeOpts{ Name: prefix + "_event_log_files_existing", Help: "Number of per-grain .events.log files currently on disk in the storage directory.", }), EventLogLastAppendUnix: promauto.NewGauge(prometheus.GaugeOpts{ Name: prefix + "_event_log_last_append_unix", Help: "Unix timestamp of the last successful event-log append.", }), EventLogReplayTotal: promauto.NewCounter(prometheus.CounterOpts{ Name: prefix + "_event_log_replay_total", Help: "Total number of successful event-log replays (one per grain loaded from disk).", }), EventLogReplayFailures: promauto.NewCounter(prometheus.CounterOpts{ Name: prefix + "_event_log_replay_failures_total", Help: "Total number of failed event-log replays (open error, JSON parse error, or handler error).", }), EventLogReplayDuration: promauto.NewHistogram(prometheus.HistogramOpts{ Name: prefix + "_event_log_replay_duration_seconds", Help: "Wall-clock time to replay a single grain's event log from disk.", Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), // 1ms ... ~16s }), EventLogUnknownTypes: promauto.NewCounter(prometheus.CounterOpts{ Name: prefix + "_event_log_unknown_types_total", Help: "Total number of event-log entries with a type the registry does not recognise.", }), EventLogMutationErrors: promauto.NewCounter(prometheus.CounterOpts{ Name: prefix + "_event_log_mutation_errors_total", Help: "Total number of mutation-handler errors observed during event-log replay.", }), } } // MutationStarted returns a closure to be deferred at the top of // pool.Apply so the latency observation / counters fire exactly once // per call regardless of which return path is taken. mutationCount // is the number of proto messages in the Apply call (a single call // can batch several messages — SetCartItems, for example, sends // ClearCart + N AddItem together); the counter is incremented by // that count so the dashboard's `rate(cart_mutations_total[1m])` // reports "messages per second" rather than "calls per second", // matching the original `grainMutations.Add(len(data.Mutations))` // semantics. The returned closure is nil-safe — pass a nil *Metrics // in and the call is a no-op. func (m *Metrics) MutationStarted(mutationCount int) func(err error) { if m == nil { return func(error) {} } start := time.Now() return func(err error) { if mutationCount > 0 { m.MutationsTotal.Add(float64(mutationCount)) } if err != nil { m.MutationFailuresTotal.Inc() } m.MutationLatency.Observe(time.Since(start).Seconds()) } } // SetPoolStats updates the three pool-level gauges (grains_in_pool, // active_grains, pool_usage) in one call. Safe to invoke under the // pool's localMu — it does not take any lock itself, only prometheus // atomic counters. func (m *Metrics) SetPoolStats(grains, poolSize int) { if m == nil { return } m.GrainsInPool.Set(float64(grains)) m.ActiveGrains.Set(float64(grains)) if poolSize > 0 { m.PoolUsage.Set(float64(grains) / float64(poolSize)) } else { m.PoolUsage.Set(0) } } // IncSpawn bumps the grain-spawned counter. Nil-safe. func (m *Metrics) IncSpawn() { if m == nil { return } m.GrainSpawnedTotal.Inc() } // IncLookup bumps the grain-lookups counter. Nil-safe. func (m *Metrics) IncLookup() { if m == nil { return } m.GrainLookupsTotal.Inc() } // IncNegotiation bumps the remote-negotiation counter. Nil-safe. func (m *Metrics) IncNegotiation() { if m == nil { return } m.RemoteNegotiationTotal.Inc() } // SetConnectedRemotes updates the connected-remotes gauge. Nil-safe. func (m *Metrics) SetConnectedRemotes(n int) { if m == nil { return } m.ConnectedRemotes.Set(float64(n)) }