217 lines
8.5 KiB
Go
217 lines
8.5 KiB
Go
package actor
|
|
|
|
import (
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/promauto"
|
|
)
|
|
|
|
// Metrics bundles every prometheus metric the actor package emits. It is
|
|
// constructed per service (cart / checkout / profile / order) with a
|
|
// service-specific prefix so the metric names do not collide when more
|
|
// than one actor-pool service is scraped into the same Prometheus.
|
|
//
|
|
// All touch sites are nil-safe: the pool / storage / registry code wraps
|
|
// each counter / histogram call in a nil-check on the *Metrics pointer,
|
|
// so the existing tests (which construct a pool or storage without
|
|
// metrics) keep working without a forced dependency on this struct.
|
|
//
|
|
// Metric names match the panels in grafana_dashboard_cart.json exactly.
|
|
// The dashboard had `connected_remotes` (no prefix) as a typo; the
|
|
// source-of-truth metric exported here is `<prefix>_connected_remotes`
|
|
// and the dashboard JSON is updated in the same change to match.
|
|
type Metrics struct {
|
|
// Pool — set by SimpleGrainPool.
|
|
ActiveGrains prometheus.Gauge
|
|
GrainsInPool prometheus.Gauge
|
|
PoolUsage prometheus.Gauge
|
|
ConnectedRemotes prometheus.Gauge
|
|
GrainSpawnedTotal prometheus.Counter
|
|
GrainLookupsTotal prometheus.Counter
|
|
MutationsTotal prometheus.Counter
|
|
MutationFailuresTotal prometheus.Counter
|
|
MutationLatency prometheus.Histogram
|
|
RemoteNegotiationTotal prometheus.Counter
|
|
|
|
// Event log — set by DiskStorage and StateStorage.
|
|
EventLogAppends prometheus.Counter
|
|
EventLogBytesWritten prometheus.Counter
|
|
EventLogFilesExisting prometheus.Gauge
|
|
EventLogLastAppendUnix prometheus.Gauge
|
|
EventLogReplayTotal prometheus.Counter
|
|
EventLogReplayFailures prometheus.Counter
|
|
EventLogReplayDuration prometheus.Histogram
|
|
EventLogUnknownTypes prometheus.Counter
|
|
EventLogMutationErrors prometheus.Counter
|
|
}
|
|
|
|
// NewMetrics registers every actor metric in the default Prometheus
|
|
// registry (the one promhttp.Handler() exposes on /metrics) with the
|
|
// service-specific prefix. The prefix is the same string the binary uses
|
|
// as its service name (e.g. "cart", "checkout", "profile", "order"),
|
|
// which keeps the metric names aligned with the per-service labels in
|
|
// grafana_dashboard_cart.json.
|
|
func NewMetrics(prefix string) *Metrics {
|
|
return &Metrics{
|
|
ActiveGrains: promauto.NewGauge(prometheus.GaugeOpts{
|
|
Name: prefix + "_active_grains",
|
|
Help: "Number of grains currently held in the local pool.",
|
|
}),
|
|
GrainsInPool: promauto.NewGauge(prometheus.GaugeOpts{
|
|
Name: prefix + "_grains_in_pool",
|
|
Help: "Alias of active_grains: the dashboard shows both as a sanity check.",
|
|
}),
|
|
PoolUsage: promauto.NewGauge(prometheus.GaugeOpts{
|
|
Name: prefix + "_grain_pool_usage",
|
|
Help: "Local pool fill ratio: len(grains) / poolSize (range 0..1).",
|
|
}),
|
|
ConnectedRemotes: promauto.NewGauge(prometheus.GaugeOpts{
|
|
Name: prefix + "_connected_remotes",
|
|
Help: "Number of remote actor-pool hosts currently connected via gRPC.",
|
|
}),
|
|
GrainSpawnedTotal: promauto.NewCounter(prometheus.CounterOpts{
|
|
Name: prefix + "_grain_spawned_total",
|
|
Help: "Total number of grains spawned (loaded from disk or freshly created).",
|
|
}),
|
|
GrainLookupsTotal: promauto.NewCounter(prometheus.CounterOpts{
|
|
Name: prefix + "_grain_lookups_total",
|
|
Help: "Total number of pool.Get calls (state reads).",
|
|
}),
|
|
MutationsTotal: promauto.NewCounter(prometheus.CounterOpts{
|
|
Name: prefix + "_mutations_total",
|
|
Help: "Total number of mutations applied across all grains (one per proto message, not per call).",
|
|
}),
|
|
MutationFailuresTotal: promauto.NewCounter(prometheus.CounterOpts{
|
|
Name: prefix + "_mutation_failures_total",
|
|
Help: "Total number of pool.Apply calls that returned an error.",
|
|
}),
|
|
// Latency buckets cover the typical pool.Apply (handler + registry
|
|
// apply + storage enqueue): sub-ms in the hot path, hundreds of ms
|
|
// under load. Tweak if a hot mutation type moves outside this range.
|
|
MutationLatency: promauto.NewHistogram(prometheus.HistogramOpts{
|
|
Name: prefix + "_mutation_latency_seconds",
|
|
Help: "Wall-clock duration of a pool.Apply call (lock + spawn + handler + storage enqueue).",
|
|
Buckets: prometheus.ExponentialBuckets(0.0005, 2, 14), // 0.5ms ... ~8s
|
|
}),
|
|
RemoteNegotiationTotal: promauto.NewCounter(prometheus.CounterOpts{
|
|
Name: prefix + "_remote_negotiation_total",
|
|
Help: "Total number of cluster-negotiation rounds initiated by this pod.",
|
|
}),
|
|
|
|
EventLogAppends: promauto.NewCounter(prometheus.CounterOpts{
|
|
Name: prefix + "_event_log_appends_total",
|
|
Help: "Total number of event-log line appends (one per persisted mutation message).",
|
|
}),
|
|
EventLogBytesWritten: promauto.NewCounter(prometheus.CounterOpts{
|
|
Name: prefix + "_event_log_bytes_written_total",
|
|
Help: "Total bytes written to per-grain .events.log files.",
|
|
}),
|
|
EventLogFilesExisting: promauto.NewGauge(prometheus.GaugeOpts{
|
|
Name: prefix + "_event_log_files_existing",
|
|
Help: "Number of per-grain .events.log files currently on disk in the storage directory.",
|
|
}),
|
|
EventLogLastAppendUnix: promauto.NewGauge(prometheus.GaugeOpts{
|
|
Name: prefix + "_event_log_last_append_unix",
|
|
Help: "Unix timestamp of the last successful event-log append.",
|
|
}),
|
|
EventLogReplayTotal: promauto.NewCounter(prometheus.CounterOpts{
|
|
Name: prefix + "_event_log_replay_total",
|
|
Help: "Total number of successful event-log replays (one per grain loaded from disk).",
|
|
}),
|
|
EventLogReplayFailures: promauto.NewCounter(prometheus.CounterOpts{
|
|
Name: prefix + "_event_log_replay_failures_total",
|
|
Help: "Total number of failed event-log replays (open error, JSON parse error, or handler error).",
|
|
}),
|
|
EventLogReplayDuration: promauto.NewHistogram(prometheus.HistogramOpts{
|
|
Name: prefix + "_event_log_replay_duration_seconds",
|
|
Help: "Wall-clock time to replay a single grain's event log from disk.",
|
|
Buckets: prometheus.ExponentialBuckets(0.001, 2, 14), // 1ms ... ~16s
|
|
}),
|
|
EventLogUnknownTypes: promauto.NewCounter(prometheus.CounterOpts{
|
|
Name: prefix + "_event_log_unknown_types_total",
|
|
Help: "Total number of event-log entries with a type the registry does not recognise.",
|
|
}),
|
|
EventLogMutationErrors: promauto.NewCounter(prometheus.CounterOpts{
|
|
Name: prefix + "_event_log_mutation_errors_total",
|
|
Help: "Total number of mutation-handler errors observed during event-log replay.",
|
|
}),
|
|
}
|
|
}
|
|
|
|
// MutationStarted returns a closure to be deferred at the top of
|
|
// pool.Apply so the latency observation / counters fire exactly once
|
|
// per call regardless of which return path is taken. mutationCount
|
|
// is the number of proto messages in the Apply call (a single call
|
|
// can batch several messages — SetCartItems, for example, sends
|
|
// ClearCart + N AddItem together); the counter is incremented by
|
|
// that count so the dashboard's `rate(cart_mutations_total[1m])`
|
|
// reports "messages per second" rather than "calls per second",
|
|
// matching the original `grainMutations.Add(len(data.Mutations))`
|
|
// semantics. The returned closure is nil-safe — pass a nil *Metrics
|
|
// in and the call is a no-op.
|
|
func (m *Metrics) MutationStarted(mutationCount int) func(err error) {
|
|
if m == nil {
|
|
return func(error) {}
|
|
}
|
|
start := time.Now()
|
|
return func(err error) {
|
|
if mutationCount > 0 {
|
|
m.MutationsTotal.Add(float64(mutationCount))
|
|
}
|
|
if err != nil {
|
|
m.MutationFailuresTotal.Inc()
|
|
}
|
|
m.MutationLatency.Observe(time.Since(start).Seconds())
|
|
}
|
|
}
|
|
|
|
// SetPoolStats updates the three pool-level gauges (grains_in_pool,
|
|
// active_grains, pool_usage) in one call. Safe to invoke under the
|
|
// pool's localMu — it does not take any lock itself, only prometheus
|
|
// atomic counters.
|
|
func (m *Metrics) SetPoolStats(grains, poolSize int) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
m.GrainsInPool.Set(float64(grains))
|
|
m.ActiveGrains.Set(float64(grains))
|
|
if poolSize > 0 {
|
|
m.PoolUsage.Set(float64(grains) / float64(poolSize))
|
|
} else {
|
|
m.PoolUsage.Set(0)
|
|
}
|
|
}
|
|
|
|
// IncSpawn bumps the grain-spawned counter. Nil-safe.
|
|
func (m *Metrics) IncSpawn() {
|
|
if m == nil {
|
|
return
|
|
}
|
|
m.GrainSpawnedTotal.Inc()
|
|
}
|
|
|
|
// IncLookup bumps the grain-lookups counter. Nil-safe.
|
|
func (m *Metrics) IncLookup() {
|
|
if m == nil {
|
|
return
|
|
}
|
|
m.GrainLookupsTotal.Inc()
|
|
}
|
|
|
|
// IncNegotiation bumps the remote-negotiation counter. Nil-safe.
|
|
func (m *Metrics) IncNegotiation() {
|
|
if m == nil {
|
|
return
|
|
}
|
|
m.RemoteNegotiationTotal.Inc()
|
|
}
|
|
|
|
// SetConnectedRemotes updates the connected-remotes gauge. Nil-safe.
|
|
func (m *Metrics) SetConnectedRemotes(n int) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
m.ConnectedRemotes.Set(float64(n))
|
|
}
|