fancy-dns/internal/metrics/metrics.go

// Package metrics holds the Prometheus collectors and the in-process
// time-series buffers that back the admin UI's charts.
//
// The UI needs finer-grained series than Prometheus's default histogram gives,
// so a small fixed-bucket ring per metric is kept in memory: cheap, bounded,
// and enough to draw a dashboard without shipping raw samples to a browser.
package metrics

import (
	"sync"
	"sync/atomic"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/collectors"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"net/http"
)

// Namespace prefixes every metric name.
const Namespace = "fancydns"

// Counters for the request pipeline.
var (
	queriesTotal   prometheus.Counter
	blockedTotal   prometheus.Counter
	cacheHits      prometheus.Counter
	cacheMisses    prometheus.Counter
	cacheStaleHits prometheus.Counter
	coalesced      prometheus.Counter
	prefetches     prometheus.Counter
	logDropped     prometheus.Counter
	groupQueries   *prometheus.CounterVec
	rcodes         *prometheus.CounterVec
	upstreamErrors prometheus.Counter
	dnsErrors      prometheus.Counter
	dropsTotal     prometheus.Counter
	refusedTotal   prometheus.Counter
)

// Gauges for live state.
var (
	upstreamHealth *prometheus.GaugeVec
	concurrentNow  prometheus.Gauge
	listCount      prometheus.Gauge
	listBytes      prometheus.Gauge
	aclGroups      prometheus.Gauge
	localRecords   prometheus.Gauge
	up             prometheus.Gauge
)

// Histograms for latency, with buckets that matter for DNS: sub-millisecond
// cache hits, a few milliseconds for a local answer, tens for an upstream.
var (
	queryDuration *prometheus.HistogramVec
	upstreamDur   *prometheus.HistogramVec
	parseDuration prometheus.Histogram
)

var registerOnce sync.Once

// Registry is the registry served on /metrics.
var Registry = prometheus.NewRegistry()

// initCollectors registers everything exactly once. It is safe to call from
// several components.
func initCollectors() {
	registerOnce.Do(func() {
		queriesTotal = prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: Namespace, Name: "queries_total",
			Help: "Total DNS queries received across all transports.",
		})
		blockedTotal = prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: Namespace, Name: "blocked_total",
			Help: "Queries answered with a blocklist response.",
		})
		cacheHits = prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: Namespace, Name: "cache_hits_total",
			Help: "Cache hits (fresh).",
		})
		cacheMisses = prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: Namespace, Name: "cache_misses_total",
			Help: "Cache misses that required an upstream lookup.",
		})
		cacheStaleHits = prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: Namespace, Name: "cache_stale_hits_total",
			Help: "Answers served from the serve-stale window.",
		})
		coalesced = prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: Namespace, Name: "upstream_coalesced_total",
			Help: "Cache misses answered by sharing another query's in-flight upstream exchange.",
		})
		prefetches = prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: Namespace, Name: "cache_prefetch_total",
			Help: "Background refreshes started for an entry that was stale or close to expiry.",
		})
		logDropped = prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: Namespace, Name: "access_log_dropped_total",
			Help: "Access-log lines discarded because the log stream could not keep up.",
		})
		groupQueries = prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: Namespace, Name: "group_queries_total",
			Help: "Queries by ACL group and outcome. The group is empty-named \"default\" when no group matched.",
		}, []string{"group", "outcome"})
		rcodes = prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: Namespace, Name: "responses_total",
			Help: "Responses by RCODE.",
		}, []string{"rcode"})
		upstreamErrors = prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: Namespace, Name: "upstream_errors_total",
			Help: "Upstream exchange failures.",
		})
		dnsErrors = prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: Namespace, Name: "errors_total",
			Help: "Responses with a non-success RCODE.",
		})
		dropsTotal = prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: Namespace, Name: "dropped_total",
			Help: "Queries dropped without a response (blocklist drop or overload).",
		})
		refusedTotal = prometheus.NewCounter(prometheus.CounterOpts{
			Namespace: Namespace, Name: "refused_total",
			Help: "Queries answered REFUSED by ACL policy.",
		})
		upstreamHealth = prometheus.NewGaugeVec(prometheus.GaugeOpts{
			Namespace: Namespace, Name: "upstream_health",
			Help: "1 when an upstream is up, 0 when down.",
		}, []string{"addr", "proto"})
		concurrentNow = prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: Namespace, Name: "concurrent_queries",
			Help: "Queries currently in flight.",
		})
		listCount = prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: Namespace, Name: "blocklist_rules",
			Help: "Total compiled blocklist rules.",
		})
		listBytes = prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: Namespace, Name: "blocklist_source_bytes",
			Help: "Bytes of source data compiled into the blocklist.",
		})
		aclGroups = prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: Namespace, Name: "acl_groups",
			Help: "Number of configured ACL groups.",
		})
		localRecords = prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: Namespace, Name: "local_records",
			Help: "Records in the local zone store.",
		})
		up = prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: Namespace, Name: "up",
			Help: "1 if the process is serving, 0 otherwise.",
		})
		queryDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
			Namespace: Namespace, Name: "query_duration_seconds",
			Help:    "End-to-end query handling time.",
			Buckets: []float64{0.0002, 0.0005, 0.001, 0.002, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5},
		}, []string{"transport", "outcome"})
		upstreamDur = prometheus.NewHistogramVec(prometheus.HistogramOpts{
			Namespace: Namespace, Name: "upstream_duration_seconds",
			Help:    "Time spent talking to a specific upstream.",
			Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2},
		}, []string{"addr"})
		parseDuration = prometheus.NewHistogram(prometheus.HistogramOpts{
			Namespace: Namespace, Name: "blocklist_compile_seconds",
			Help:    "Time to fetch and compile all blocklist sources.",
			Buckets: []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30},
		})

		Registry.MustRegister(
			queriesTotal, blockedTotal, cacheHits, cacheMisses, cacheStaleHits, coalesced, prefetches, logDropped, groupQueries, rcodes,
			upstreamErrors, dnsErrors, dropsTotal, refusedTotal, upstreamHealth,
			concurrentNow, listCount, listBytes, aclGroups, localRecords, up,
			queryDuration, upstreamDur, parseDuration,
			collectors.NewGoCollector(),
			collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
		)
	})
}

// Outcome labels for queryDuration.
const (
	OutcomeCache    = "cache"
	OutcomeBlocked  = "blocked"
	OutcomeLocal    = "local"
	OutcomeUpstream = "upstream"
	OutcomeError    = "error"
	OutcomeRefused  = "refused"
	OutcomeDropped  = "dropped"
)

// Record is the single entry point for a pipeline outcome. Both the Prometheus
// counters and the in-process atomics the dashboard reads are bumped here, so
// the two views can never disagree.
//
// Callers must not also call CountBlocked/CountError/etc: that would double
// count. Those helpers exist for the few places that report an outcome without
// going through Record.
func Record(transport, outcome string, d time.Duration) {
	initCollectors()
	queriesTotal.Inc()
	queries.Add(1)
	if d > 0 {
		queryDuration.WithLabelValues(transport, outcome).Observe(d.Seconds())
	}
	switch outcome {
	case OutcomeCache:
		cacheHits.Inc()
		hits.Add(1)
	case OutcomeBlocked:
		blockedTotal.Inc()
		blocked.Add(1)
	case OutcomeError:
		dnsErrors.Inc()
		errs.Add(1)
	case OutcomeDropped:
		dropsTotal.Inc()
		drops.Add(1)
		// A drop is both a drop and an error for the operator's purposes.
		errs.Add(1)
	case OutcomeRefused:
		refusedTotal.Inc()
		refused.Add(1)
	}
}

// RecordMiss counts a cache miss. A miss is not an error: it just means the
// cache could not answer, so the error counter does not move. The in-process
// counter does, like every other outcome: the dashboard reads it and must
// never disagree with the Prometheus series.
func RecordMiss() { initCollectors(); cacheMisses.Inc(); misses.Add(1) }

// RecordStaleHit counts a serve-stale hit.
func RecordStaleHit() { initCollectors(); cacheStaleHits.Inc(); staleHits.Add(1) }

// RecordCoalesced counts a query that shared an in-flight upstream exchange.
func RecordCoalesced() { initCollectors(); coalesced.Inc() }

// RecordAccessLogDropped counts an access-log line that was discarded.
func RecordAccessLogDropped() { initCollectors(); logDropped.Inc() }

// RecordGroupQuery counts a finished query against its ACL group.
func RecordGroupQuery(group, outcome string) {
	initCollectors()
	if group == "" {
		group = "default"
	}
	groupQueries.WithLabelValues(group, outcome).Inc()
}

// RecordRcode counts a response by RCODE name.
func RecordRcode(rcode string) { initCollectors(); rcodes.WithLabelValues(rcode).Inc() }

// RecordPrefetch counts a background refresh.
func RecordPrefetch() { initCollectors(); prefetches.Inc() }

// RecordUpstreamError counts an upstream failure.
func RecordUpstreamError() { initCollectors(); upstreamErrors.Inc() }

// RecordUpstreamLatency observes a single upstream exchange.
func RecordUpstreamLatency(addr string, d time.Duration) {
	initCollectors()
	upstreamDur.WithLabelValues(addr).Observe(d.Seconds())
}

// SetConcurrent publishes the in-flight gauge.
func SetConcurrent(n int) { initCollectors(); concurrentNow.Set(float64(n)) }

// SetUp publishes the liveness gauge.
func SetUp(v bool) {
	initCollectors()
	if v {
		up.Set(1)
		return
	}
	up.Set(0)
}

// SetUpstreamHealth publishes per-upstream health.
func SetUpstreamHealth(addr, proto string, healthy bool) {
	initCollectors()
	v := 0.0
	if healthy {
		v = 1
	}
	upstreamHealth.WithLabelValues(addr, proto).Set(v)
}

// SetBlocklistStats publishes the compiled rule count and source bytes.
func SetBlocklistStats(rules int, sourceBytes int64) {
	initCollectors()
	listCount.Set(float64(rules))
	listBytes.Set(float64(sourceBytes))
}

// SetACLGroups publishes the group count.
func SetACLGroups(n int) { initCollectors(); aclGroups.Set(float64(n)) }

// SetLocalRecords publishes the local zone record count.
func SetLocalRecords(n int) { initCollectors(); localRecords.Set(float64(n)) }

// RecordBlocklistCompile observes a full compile pass.
func RecordBlocklistCompile(d time.Duration) {
	initCollectors()
	parseDuration.Observe(d.Seconds())
}

// Handler returns the HTTP handler for /metrics.
func Handler() http.Handler {
	initCollectors()
	return promhttp.HandlerFor(Registry, promhttp.HandlerOpts{
		Registry:            Registry,
		EnableOpenMetrics:   true,
		MaxRequestsInFlight: 4,
	})
}

// Totals is a point-in-time snapshot of the counters, used by the dashboard.
type Totals struct {
	Queries    uint64
	Blocked    uint64
	Hits       uint64
	Misses     uint64
	StaleHits  uint64
	Errors     uint64
	Drops      uint64
	Refused    uint64
	Concurrent int64
}

var (
	queries   atomic.Uint64
	blocked   atomic.Uint64
	hits      atomic.Uint64
	misses    atomic.Uint64
	staleHits atomic.Uint64
	errs      atomic.Uint64
	drops     atomic.Uint64
	refused   atomic.Uint64
	inflight  atomic.Int64
)

// TotalsNow returns the current totals. These mirror the Prometheus counters
// (both are incremented together) so the dashboard does not have to scrape.
func TotalsNow() Totals {
	return Totals{
		Queries:    queries.Load(),
		Blocked:    blocked.Load(),
		Hits:       hits.Load(),
		Misses:     misses.Load(),
		StaleHits:  staleHits.Load(),
		Errors:     errs.Load(),
		Drops:      drops.Load(),
		Refused:    refused.Load(),
		Concurrent: inflight.Load(),
	}
}

func bumpCounter(c *atomic.Uint64) func() {
	c.Add(1)
	return func() { c.Add(^uint64(0)) }
}

// BeginQuery marks a query as in flight and returns its completion func.
func BeginQuery() func() {
	inflight.Add(1)
	queries.Add(1)
	return func() { inflight.Add(-1) }
}

// CountBlocked increments the blocked counter.
func CountBlocked() { blocked.Add(1) }

// CountError increments the error counter.
func CountError() { errs.Add(1) }

// CountDrop increments the drop counter.
func CountDrop() { drops.Add(1) }

// CountRefused increments the refused counter.
func CountRefused() { refused.Add(1) }