Make the resolver pure DNS: JSON access log on stdout instead of a log store
The resolver kept its own query-log store (a bbolt ring, a live ring with SSE fanout, rolling per-domain counters and dashboard time-series) for the console. In a multi-node deployment each pod held a different slice of that, so every view disagreed with itself, and the cost sat on the query path. Charts belong in Grafana and log search in the log collector, so the store is gone. internal/accesslog writes one JSON line per notable query to stdout. The default is what was blocked and what failed (blocked, redirected, SERVFAIL, dropped, refused); log_cached and sample_rate add ordinary answers for a debugging session. Lines go through a bounded buffer to a single writer goroutine and are dropped, and counted in fancydns_access_log_dropped_total, when the stream cannot keep up, so a stalled log pipe can never stall a query. The config key is access_log (the old qlog block is ignored, like any unknown key). The metrics gain what the dashboard used to compute: fancydns_group_queries_total by ACL group and outcome and fancydns_responses_total by RCODE. Removed with the store: /api/dashboard, /api/logs, /api/logs/stream, /api/logs/clear, /api/cache/top, the series rings, and the cache's per-name hit table, which took a mutex and built a string on every cache hit. Tests that read the old log now read the access-log lines; the contract test drops the removed endpoints. The web console still calls them and is changed in the next commit. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
matsandClaude Sonnet 5.5 committed
1 parent
dd1117fd91
commit
c6c5ca4ff7
31 files changed
+818
-3389
No files matched your search
+31
-147
@@ -7,7 +7,6 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@@ -30,6 +29,9 @@ var (
|
||||
cacheStaleHits prometheus.Counter
|
||||
coalesced prometheus.Counter
|
||||
prefetches prometheus.Counter
|
||||
logDropped prometheus.Counter
|
||||
groupQueries *prometheus.CounterVec
|
||||
rcodes *prometheus.CounterVec
|
||||
upstreamErrors prometheus.Counter
|
||||
dnsErrors prometheus.Counter
|
||||
dropsTotal prometheus.Counter
|
||||
@@ -92,6 +94,18 @@ func initCollectors() {
|
||||
Namespace: Namespace, Name: "cache_prefetch_total",
|
||||
Help: "Background refreshes started for an entry that was stale or close to expiry.",
|
||||
})
|
||||
logDropped = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: Namespace, Name: "access_log_dropped_total",
|
||||
Help: "Access-log lines discarded because the log stream could not keep up.",
|
||||
})
|
||||
groupQueries = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: Namespace, Name: "group_queries_total",
|
||||
Help: "Queries by ACL group and outcome. The group is empty-named \"default\" when no group matched.",
|
||||
}, []string{"group", "outcome"})
|
||||
rcodes = prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: Namespace, Name: "responses_total",
|
||||
Help: "Responses by RCODE.",
|
||||
}, []string{"rcode"})
|
||||
upstreamErrors = prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: Namespace, Name: "upstream_errors_total",
|
||||
Help: "Upstream exchange failures.",
|
||||
@@ -153,7 +167,7 @@ func initCollectors() {
|
||||
})
|
||||
|
||||
Registry.MustRegister(
|
||||
queriesTotal, blockedTotal, cacheHits, cacheMisses, cacheStaleHits, coalesced, prefetches,
|
||||
queriesTotal, blockedTotal, cacheHits, cacheMisses, cacheStaleHits, coalesced, prefetches, logDropped, groupQueries, rcodes,
|
||||
upstreamErrors, dnsErrors, dropsTotal, refusedTotal, upstreamHealth,
|
||||
concurrentNow, listCount, listBytes, aclGroups, localRecords, up,
|
||||
queryDuration, upstreamDur, parseDuration,
|
||||
@@ -221,6 +235,21 @@ func RecordStaleHit() { initCollectors(); cacheStaleHits.Inc(); staleHits.Add(1)
|
||||
// RecordCoalesced counts a query that shared an in-flight upstream exchange.
|
||||
func RecordCoalesced() { initCollectors(); coalesced.Inc() }
|
||||
|
||||
// RecordAccessLogDropped counts an access-log line that was discarded.
|
||||
func RecordAccessLogDropped() { initCollectors(); logDropped.Inc() }
|
||||
|
||||
// RecordGroupQuery counts a finished query against its ACL group.
|
||||
func RecordGroupQuery(group, outcome string) {
|
||||
initCollectors()
|
||||
if group == "" {
|
||||
group = "default"
|
||||
}
|
||||
groupQueries.WithLabelValues(group, outcome).Inc()
|
||||
}
|
||||
|
||||
// RecordRcode counts a response by RCODE name.
|
||||
func RecordRcode(rcode string) { initCollectors(); rcodes.WithLabelValues(rcode).Inc() }
|
||||
|
||||
// RecordPrefetch counts a background refresh.
|
||||
func RecordPrefetch() { initCollectors(); prefetches.Inc() }
|
||||
|
||||
@@ -349,148 +378,3 @@ func CountDrop() { drops.Add(1) }
|
||||
|
||||
// CountRefused increments the refused counter.
|
||||
func CountRefused() { refused.Add(1) }
|
||||
|
||||
// Series is a fixed-window time series for the UI charts.
|
||||
type Series struct {
|
||||
// Buckets are the sample timestamps (epoch seconds), oldest first.
|
||||
Buckets []int64 `json:"t"`
|
||||
Values []int64 `json:"v"`
|
||||
// ResolutionMS is the bucket width.
|
||||
ResolutionMS int64 `json:"resolution_ms"`
|
||||
}
|
||||
|
||||
// Ring is a fixed-bucket counter ring: a circular array of counts indexed by
|
||||
// bucket number. Lock-free on the write path, which matters because it is hit
|
||||
// once per DNS query.
|
||||
type Ring struct {
|
||||
buckets []atomic.Int64
|
||||
res time.Duration
|
||||
start atomic.Int64 // first bucket timestamp, epoch seconds
|
||||
pos atomic.Uint32
|
||||
sealed atomic.Bool
|
||||
count int
|
||||
}
|
||||
|
||||
// NewRing builds a ring of n buckets at the given resolution.
|
||||
func NewRing(n int, res time.Duration) *Ring {
|
||||
if n <= 0 {
|
||||
n = 120
|
||||
}
|
||||
if res <= 0 {
|
||||
res = 5 * time.Second
|
||||
}
|
||||
r := &Ring{buckets: make([]atomic.Int64, n), res: res, count: n}
|
||||
r.start.Store(time.Now().Unix())
|
||||
return r
|
||||
}
|
||||
|
||||
// Inc bumps the current bucket by 1, rotating when the window advances.
|
||||
func (r *Ring) Inc() { r.Add(1) }
|
||||
|
||||
// Add bumps the current bucket.
|
||||
func (r *Ring) Add(delta int64) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now().Unix()
|
||||
bucket := now / int64(r.res.Seconds())
|
||||
start := r.start.Load()
|
||||
// If more than count buckets elapsed, we skipped a full window: reset
|
||||
// everything so a long idle period does not report phantom zeros followed
|
||||
// by a sudden spike.
|
||||
if bucket-start >= int64(r.count) {
|
||||
if r.start.CompareAndSwap(start, bucket-int64(r.count)+1) {
|
||||
for i := range r.buckets {
|
||||
r.buckets[i].Store(0)
|
||||
}
|
||||
}
|
||||
start = r.start.Load()
|
||||
}
|
||||
if bucket < start {
|
||||
return
|
||||
}
|
||||
idx := int(bucket - start)
|
||||
if idx >= r.count {
|
||||
idx = r.count - 1
|
||||
}
|
||||
// Store at the position implied by the bucket so readers can map index to
|
||||
// time without extra bookkeeping.
|
||||
r.buckets[idx%r.count].Add(delta)
|
||||
}
|
||||
|
||||
// AddAt bumps an explicit bucket (used by tests).
|
||||
func (r *Ring) AddAt(bucket int64, delta int64) {
|
||||
start := r.start.Load()
|
||||
idx := int(bucket - start)
|
||||
if idx < 0 || idx >= r.count {
|
||||
return
|
||||
}
|
||||
r.buckets[idx%r.count].Add(delta)
|
||||
}
|
||||
|
||||
// Series snapshots the ring into aligned arrays.
|
||||
func (r *Ring) Snapshot() ([]int64, []int64) {
|
||||
if r == nil {
|
||||
return nil, nil
|
||||
}
|
||||
start := r.start.Load()
|
||||
now := time.Now().Unix()
|
||||
step := int64(r.res.Seconds())
|
||||
ts := make([]int64, r.count)
|
||||
vs := make([]int64, r.count)
|
||||
for i := 0; i < r.count; i++ {
|
||||
b := start + int64(i)
|
||||
ts[i] = b * step
|
||||
vs[i] = r.buckets[i%r.count].Load()
|
||||
}
|
||||
// Trim leading buckets that are in the future (ring not yet filled) so the
|
||||
// chart starts at "now" rather than at zero time.
|
||||
if now > ts[len(ts)-1] {
|
||||
shift := int((now-ts[len(ts)-1])/step) + 1
|
||||
if shift >= len(ts) {
|
||||
return []int64{now}, []int64{0}
|
||||
}
|
||||
ts = append(ts[:0], ts[shift:]...)
|
||||
vs = append(vs[:0], vs[shift:]...)
|
||||
}
|
||||
return ts, vs
|
||||
}
|
||||
|
||||
// Series is the snapshot form.
|
||||
func (r *Ring) Series() Series {
|
||||
ts, vs := r.Snapshot()
|
||||
return Series{Buckets: ts, Values: vs, ResolutionMS: r.res.Milliseconds()}
|
||||
}
|
||||
|
||||
// LCG is a tiny deterministic PRNG for jitter, so the test suite does not
|
||||
// depend on the global rand.
|
||||
type LCG struct{ s uint64 }
|
||||
|
||||
func NewLCG(seed uint64) *LCG { return &LCG{s: seed | 1} }
|
||||
|
||||
func (l *LCG) Next() uint64 {
|
||||
l.s = l.s*6364136223846793005 + 1442695040888963407
|
||||
return l.s >> 11
|
||||
}
|
||||
|
||||
func (l *LCG) Float() float64 { return float64(l.Next()) / float64(1<<53) }
|
||||
|
||||
// SafeFloat converts an int count to a float for charting, guarding NaN.
|
||||
func SafeFloat(n int64) float64 {
|
||||
if n <= 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(n)
|
||||
}
|
||||
|
||||
// Percent computes a safe percentage.
|
||||
func Percent(part, total int64) float64 {
|
||||
if total <= 0 {
|
||||
return 0
|
||||
}
|
||||
p := float64(part) / float64(total) * 100
|
||||
if math.IsNaN(p) || math.IsInf(p, 0) {
|
||||
return 0
|
||||
}
|
||||
return p
|
||||
}
|
||||
Reference in new issue
Block a user