Add a Grafana dashboard for the resolver's metrics

With traffic charts gone from the console, deploy/grafana/fancy-dns.json is
where they live: queries by outcome, transport and client group, latency
quantiles, RCODEs, cache hits/misses/stale, the upstream round trips saved by
coalescing and early refresh, upstream health and latency, blocklist and zone
sizes, and (in a Loki row) the blocked/failed-query access log. A node variable
spreads or narrows every panel, and the per-node panel shows whether load is
even across pods.

A test reads the dashboard and fails if a panel queries a metric the resolver
does not export, so a rename cannot silently empty a panel.

The ingress no longer needs the SSE buffering/timeout annotations, and the
statefulset comments no longer describe a query log on the volume.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

deploy/k8s/10-deployment.yaml

@@ -11,8 +11,8 @@ metadata:
 spec:
   serviceName: fancy-dns
   # A StatefulSet rather than a Deployment because each pod owns a PVC (the
-  # cache snapshot and the query log are per-pod state worth keeping across a
-  # reschedule). The flip side: the HPA does not work on a StatefulSet — the
+  # cache snapshot and downloaded blocklists are per-pod state worth keeping
+  # across a reschedule). The flip side: the HPA does not work on a StatefulSet — the
   # controller and the autoscaler fight over the replica count — so scaling
   # means adding nodes and replicas here, not CPU targets.
   replicas: 1
@@ -170,16 +170,16 @@ spec:
         - name: tmp
           emptyDir: {}
   volumeClaimTemplates:
-    # Cache snapshot (cache.persist.path) and query log (qlog.path) live here,
-    # along with downloaded blocklists. ReadWriteOnce is per-pod: a pod keeps
+    # Cache snapshot (cache.persist.path) lives here, along with downloaded
+    # blocklists. The resolver stores no query log: the access log goes to stdout. ReadWriteOnce is per-pod: a pod keeps
     # its own cache across a reschedule, which is the whole point of the
     # snapshot, but it cannot be shared between replicas.
     #
     # Sizing: cache.size is an entry count, not bytes, so disk usage has to be
     # estimated. A packed A/AAAA answer with a small authority section is
     # roughly 300-600 bytes on the wire and nearer 1 kB resident, so 500k
-    # entries lands around 500 MB; qlog is the other consumer (see
-    # qlog.max_entries). 4Gi leaves room for both plus a re-fetched list.
+    # entries lands around 500 MB. 4Gi leaves room for that plus a re-fetched
+    # list.
     - metadata:
         name: data
       spec:

deploy/k8s/30-ingress.yaml

@@ -13,13 +13,6 @@ metadata:
     k8s-helper.io/icon: "network"
     nginx.ingress.kubernetes.io/backend-protocol: "HTTP"
     cert-manager.io/cluster-issuer: letsencrypt-prod
-    # SSE (the live query log) is the reason for these three. With response
-    # buffering on, nginx holds events until the buffer fills and the log page
-    # looks frozen; with a 60s read timeout the stream is cut every minute and
-    # the page reconnects in a loop.
-    nginx.ingress.kubernetes.io/proxy-buffering: "off"
-    nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
-    nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
 spec:
   ingressClassName: nginx
   rules:

internal/metrics/dashboard_test.go

@@ -0,0 +1,47 @@
+package metrics
+
+import (
+	"os"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// TestGrafanaDashboardOnlyUsesMetricsThatExist keeps deploy/grafana/fancy-dns.json
+// from drifting away from the collectors: a renamed metric would otherwise
+// leave a panel empty and nothing would say why.
+func TestGrafanaDashboardOnlyUsesMetricsThatExist(t *testing.T) {
+	initCollectors()
+	descs := make(chan *prometheus.Desc, 256)
+	go func() { Registry.Describe(descs); close(descs) }()
+	known := map[string]bool{}
+	nameRe := regexp.MustCompile(`fqName: "([a-z0-9_]+)"`)
+	for d := range descs {
+		if m := nameRe.FindStringSubmatch(d.String()); m != nil {
+			known[m[1]] = true
+		}
+	}
+	if !known["fancydns_queries_total"] {
+		t.Fatalf("collector names were not read; got %v", known)
+	}
+
+	raw, err := os.ReadFile("../../deploy/grafana/fancy-dns.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	used := regexp.MustCompile(`fancydns_[a-z0-9_]+`).FindAllString(string(raw), -1)
+	if len(used) == 0 {
+		t.Fatal("the dashboard references no fancydns metrics")
+	}
+	for _, name := range used {
+		base := name
+		for _, suffix := range []string{"_bucket", "_count", "_sum"} {
+			base = strings.TrimSuffix(base, suffix)
+		}
+		if !known[base] {
+			t.Errorf("the dashboard queries %s, which the resolver does not export", name)
+		}
+	}
+}