metrics
This commit is contained in:
@@ -0,0 +1,216 @@
|
||||
package actor
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
// Metrics bundles every prometheus metric the actor package emits. It is
|
||||
// constructed per service (cart / checkout / profile / order) with a
|
||||
// service-specific prefix so the metric names do not collide when more
|
||||
// than one actor-pool service is scraped into the same Prometheus.
|
||||
//
|
||||
// All touch sites are nil-safe: the pool / storage / registry code wraps
|
||||
// each counter / histogram call in a nil-check on the *Metrics pointer,
|
||||
// so the existing tests (which construct a pool or storage without
|
||||
// metrics) keep working without a forced dependency on this struct.
|
||||
//
|
||||
// Metric names match the panels in grafana_dashboard_cart.json exactly.
|
||||
// The dashboard had `connected_remotes` (no prefix) as a typo; the
|
||||
// source-of-truth metric exported here is `<prefix>_connected_remotes`
|
||||
// and the dashboard JSON is updated in the same change to match.
|
||||
type Metrics struct {
|
||||
// Pool — set by SimpleGrainPool.
|
||||
ActiveGrains prometheus.Gauge
|
||||
GrainsInPool prometheus.Gauge
|
||||
PoolUsage prometheus.Gauge
|
||||
ConnectedRemotes prometheus.Gauge
|
||||
GrainSpawnedTotal prometheus.Counter
|
||||
GrainLookupsTotal prometheus.Counter
|
||||
MutationsTotal prometheus.Counter
|
||||
MutationFailuresTotal prometheus.Counter
|
||||
MutationLatency prometheus.Histogram
|
||||
RemoteNegotiationTotal prometheus.Counter
|
||||
|
||||
// Event log — set by DiskStorage and StateStorage.
|
||||
EventLogAppends prometheus.Counter
|
||||
EventLogBytesWritten prometheus.Counter
|
||||
EventLogFilesExisting prometheus.Gauge
|
||||
EventLogLastAppendUnix prometheus.Gauge
|
||||
EventLogReplayTotal prometheus.Counter
|
||||
EventLogReplayFailures prometheus.Counter
|
||||
EventLogReplayDuration prometheus.Histogram
|
||||
EventLogUnknownTypes prometheus.Counter
|
||||
EventLogMutationErrors prometheus.Counter
|
||||
}
|
||||
|
||||
// NewMetrics registers every actor metric in the default Prometheus
|
||||
// registry (the one promhttp.Handler() exposes on /metrics) with the
|
||||
// service-specific prefix. The prefix is the same string the binary uses
|
||||
// as its service name (e.g. "cart", "checkout", "profile", "order"),
|
||||
// which keeps the metric names aligned with the per-service labels in
|
||||
// grafana_dashboard_cart.json.
|
||||
func NewMetrics(prefix string) *Metrics {
|
||||
return &Metrics{
|
||||
ActiveGrains: promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Name: prefix + "_active_grains",
|
||||
Help: "Number of grains currently held in the local pool.",
|
||||
}),
|
||||
GrainsInPool: promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Name: prefix + "_grains_in_pool",
|
||||
Help: "Alias of active_grains: the dashboard shows both as a sanity check.",
|
||||
}),
|
||||
PoolUsage: promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Name: prefix + "_grain_pool_usage",
|
||||
Help: "Local pool fill ratio: len(grains) / poolSize (range 0..1).",
|
||||
}),
|
||||
ConnectedRemotes: promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Name: prefix + "_connected_remotes",
|
||||
Help: "Number of remote actor-pool hosts currently connected via gRPC.",
|
||||
}),
|
||||
GrainSpawnedTotal: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: prefix + "_grain_spawned_total",
|
||||
Help: "Total number of grains spawned (loaded from disk or freshly created).",
|
||||
}),
|
||||
GrainLookupsTotal: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: prefix + "_grain_lookups_total",
|
||||
Help: "Total number of pool.Get calls (state reads).",
|
||||
}),
|
||||
MutationsTotal: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: prefix + "_mutations_total",
|
||||
Help: "Total number of mutations applied across all grains (one per proto message, not per call).",
|
||||
}),
|
||||
MutationFailuresTotal: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: prefix + "_mutation_failures_total",
|
||||
Help: "Total number of pool.Apply calls that returned an error.",
|
||||
}),
|
||||
// Latency buckets cover the typical pool.Apply (handler + registry
|
||||
// apply + storage enqueue): sub-ms in the hot path, hundreds of ms
|
||||
// under load. Tweak if a hot mutation type moves outside this range.
|
||||
MutationLatency: promauto.NewHistogram(prometheus.HistogramOpts{
|
||||
Name: prefix + "_mutation_latency_seconds",
|
||||
Help: "Wall-clock duration of a pool.Apply call (lock + spawn + handler + storage enqueue).",
|
||||
Buckets: prometheus.ExponentialBuckets(0.0005, 2, 14), // 0.5ms ... ~8s
|
||||
}),
|
||||
RemoteNegotiationTotal: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: prefix + "_remote_negotiation_total",
|
||||
Help: "Total number of cluster-negotiation rounds initiated by this pod.",
|
||||
}),
|
||||
|
||||
EventLogAppends: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: prefix + "_event_log_appends_total",
|
||||
Help: "Total number of event-log line appends (one per persisted mutation message).",
|
||||
}),
|
||||
EventLogBytesWritten: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: prefix + "_event_log_bytes_written_total",
|
||||
Help: "Total bytes written to per-grain .events.log files.",
|
||||
}),
|
||||
EventLogFilesExisting: promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Name: prefix + "_event_log_files_existing",
|
||||
Help: "Number of per-grain .events.log files currently on disk in the storage directory.",
|
||||
}),
|
||||
EventLogLastAppendUnix: promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Name: prefix + "_event_log_last_append_unix",
|
||||
Help: "Unix timestamp of the last successful event-log append.",
|
||||
}),
|
||||
EventLogReplayTotal: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: prefix + "_event_log_replay_total",
|
||||
Help: "Total number of successful event-log replays (one per grain loaded from disk).",
|
||||
}),
|
||||
EventLogReplayFailures: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: prefix + "_event_log_replay_failures_total",
|
||||
Help: "Total number of failed event-log replays (open error, JSON parse error, or handler error).",
|
||||
}),
|
||||
EventLogReplayDuration: promauto.NewHistogram(prometheus.HistogramOpts{
|
||||
Name: prefix + "_event_log_replay_duration_seconds",
|
||||
Help: "Wall-clock time to replay a single grain's event log from disk.",
|
||||
Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), // 1ms ... ~16s
|
||||
}),
|
||||
EventLogUnknownTypes: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: prefix + "_event_log_unknown_types_total",
|
||||
Help: "Total number of event-log entries with a type the registry does not recognise.",
|
||||
}),
|
||||
EventLogMutationErrors: promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: prefix + "_event_log_mutation_errors_total",
|
||||
Help: "Total number of mutation-handler errors observed during event-log replay.",
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
// MutationStarted returns a closure to be deferred at the top of
|
||||
// pool.Apply so the latency observation / counters fire exactly once
|
||||
// per call regardless of which return path is taken. mutationCount
|
||||
// is the number of proto messages in the Apply call (a single call
|
||||
// can batch several messages — SetCartItems, for example, sends
|
||||
// ClearCart + N AddItem together); the counter is incremented by
|
||||
// that count so the dashboard's `rate(cart_mutations_total[1m])`
|
||||
// reports "messages per second" rather than "calls per second",
|
||||
// matching the original `grainMutations.Add(len(data.Mutations))`
|
||||
// semantics. The returned closure is nil-safe — pass a nil *Metrics
|
||||
// in and the call is a no-op.
|
||||
func (m *Metrics) MutationStarted(mutationCount int) func(err error) {
|
||||
if m == nil {
|
||||
return func(error) {}
|
||||
}
|
||||
start := time.Now()
|
||||
return func(err error) {
|
||||
if mutationCount > 0 {
|
||||
m.MutationsTotal.Add(float64(mutationCount))
|
||||
}
|
||||
if err != nil {
|
||||
m.MutationFailuresTotal.Inc()
|
||||
}
|
||||
m.MutationLatency.Observe(time.Since(start).Seconds())
|
||||
}
|
||||
}
|
||||
|
||||
// SetPoolStats updates the three pool-level gauges (grains_in_pool,
|
||||
// active_grains, pool_usage) in one call. Safe to invoke under the
|
||||
// pool's localMu — it does not take any lock itself, only prometheus
|
||||
// atomic counters.
|
||||
func (m *Metrics) SetPoolStats(grains, poolSize int) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.GrainsInPool.Set(float64(grains))
|
||||
m.ActiveGrains.Set(float64(grains))
|
||||
if poolSize > 0 {
|
||||
m.PoolUsage.Set(float64(grains) / float64(poolSize))
|
||||
} else {
|
||||
m.PoolUsage.Set(0)
|
||||
}
|
||||
}
|
||||
|
||||
// IncSpawn bumps the grain-spawned counter. Nil-safe.
|
||||
func (m *Metrics) IncSpawn() {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.GrainSpawnedTotal.Inc()
|
||||
}
|
||||
|
||||
// IncLookup bumps the grain-lookups counter. Nil-safe.
|
||||
func (m *Metrics) IncLookup() {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.GrainLookupsTotal.Inc()
|
||||
}
|
||||
|
||||
// IncNegotiation bumps the remote-negotiation counter. Nil-safe.
|
||||
func (m *Metrics) IncNegotiation() {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.RemoteNegotiationTotal.Inc()
|
||||
}
|
||||
|
||||
// SetConnectedRemotes updates the connected-remotes gauge. Nil-safe.
|
||||
func (m *Metrics) SetConnectedRemotes(n int) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
m.ConnectedRemotes.Set(float64(n))
|
||||
}
|
||||
Reference in New Issue
Block a user