diff --git a/deploy/grafana/fancy-dns.json b/deploy/grafana/fancy-dns.json new file mode 100644 index 0000000..8a10aef --- /dev/null +++ b/deploy/grafana/fancy-dns.json @@ -0,0 +1,1157 @@ +{ + "uid": "fancy-dns", + "title": "fancy-dns", + "tags": [ + "dns", + "fancy-dns" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "description": "Traffic, cache, upstream health and blocked/failed queries for fancy-dns. Needs the resolver's /metrics; the last row needs Loki for the access log.", + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "type": "datasource", + "query": "prometheus", + "current": {} + }, + { + "name": "DS_LOKI", + "label": "Loki", + "type": "datasource", + "query": "loki", + "current": {} + }, + { + "name": "instance", + "label": "Node", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values(fancydns_up, instance)", + "refId": "q" + }, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "refresh": 2 + }, + { + "name": "action", + "label": "Action", + "type": "custom", + "multi": true, + "includeAll": true, + "allValue": ".*", + "query": "block,redirect,error,drop,refused,allow,cached", + "current": { + "text": "All", + "value": "$__all" + } + }, + { + "name": "logselector", + "label": "Log stream selector", + "type": "textbox", + "query": "{container=\"fancy-dns\"}", + "description": "The Loki selector for the resolver's pods; adjust to your labels." + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Queries / s", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fancydns_queries_total{instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 2, + "type": "stat", + "title": "Blocked", + "description": "Share of queries answered from a blocklist.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 4, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "100 * sum(rate(fancydns_blocked_total{instance=~\"$instance\"}[$__rate_interval])) / clamp_min(sum(rate(fancydns_queries_total{instance=~\"$instance\"}[$__rate_interval])), 1e-9)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 3, + "type": "stat", + "title": "Cache hit ratio", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 8, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fancydns_cache_hits_total{instance=~\"$instance\"}[$__rate_interval])) / clamp_min(sum(rate(fancydns_cache_hits_total{instance=~\"$instance\"}[$__rate_interval])) + sum(rate(fancydns_cache_misses_total{instance=~\"$instance\"}[$__rate_interval])), 1e-9)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.4 + }, + { + "color": "green", + "value": 0.7 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 4, + "type": "stat", + "title": "p95 latency", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(fancydns_query_duration_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 5, + "type": "stat", + "title": "Errors / s", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 16, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fancydns_errors_total{instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 6, + "type": "stat", + "title": "Nodes up", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 20, + "y": 0, + "w": 4, + "h": 4 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "count(fancydns_up == 1)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "area" + } + }, + { + "id": 7, + "type": "row", + "title": "Traffic", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 4, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 8, + "type": "timeseries", + "title": "Queries by outcome", + "description": "allow/upstream, cache, local, blocked, error, refused.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 0, + "y": 5, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (outcome) (rate(fancydns_query_duration_seconds_count{instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "fillOpacity": 12, + "lineWidth": 1, + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 9, + "type": "timeseries", + "title": "Queries by transport", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 12, + "y": 5, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (transport) (rate(fancydns_query_duration_seconds_count{instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{transport}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "fillOpacity": 12, + "lineWidth": 1, + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Latency", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 0, + "y": 13, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.5, sum by (le) (rate(fancydns_query_duration_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))", + "refId": "A", + "legendFormat": "p50" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(fancydns_query_duration_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))", + "refId": "B", + "legendFormat": "p95" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(fancydns_query_duration_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))", + "refId": "C", + "legendFormat": "p99" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "fillOpacity": 12, + "lineWidth": 1 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 11, + "type": "timeseries", + "title": "Responses by RCODE", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 12, + "y": 13, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (rcode) (rate(fancydns_responses_total{instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{rcode}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "fillOpacity": 12, + "lineWidth": 1, + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Queries by node", + "description": "Whether the load is spread evenly across the resolver pods.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 0, + "y": 21, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (instance) (rate(fancydns_queries_total{instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "fillOpacity": 12, + "lineWidth": 1 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 13, + "type": "timeseries", + "title": "Queries by client group", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 12, + "y": 21, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum by (group, outcome) (rate(fancydns_group_queries_total{instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{group}} / {{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "fillOpacity": 12, + "lineWidth": 1 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 14, + "type": "row", + "title": "Cache", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 29, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 15, + "type": "timeseries", + "title": "Cache hits, misses and stale answers", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 0, + "y": 30, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fancydns_cache_hits_total{instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "hits" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fancydns_cache_misses_total{instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "misses" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fancydns_cache_stale_hits_total{instance=~\"$instance\"}[$__rate_interval]))", + "refId": "C", + "legendFormat": "served stale" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "fillOpacity": 12, + "lineWidth": 1 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 16, + "type": "timeseries", + "title": "Upstream round trips saved", + "description": "Coalesced: identical misses that waited for one upstream exchange. Refreshes: hot entries replaced before they expired.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 12, + "y": 30, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fancydns_upstream_coalesced_total{instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "shared with an in-flight query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fancydns_cache_prefetch_total{instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "background refreshes" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "fillOpacity": 12, + "lineWidth": 1 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 17, + "type": "row", + "title": "Upstreams", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 38, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 18, + "type": "timeseries", + "title": "Upstream health", + "description": "1 up, 0 down.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 0, + "y": 39, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "min by (addr) (fancydns_upstream_health{instance=~\"$instance\"})", + "refId": "A", + "legendFormat": "{{addr}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 12, + "lineWidth": 1 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 19, + "type": "timeseries", + "title": "Upstream latency p95", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 8, + "y": 39, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "histogram_quantile(0.95, sum by (le, addr) (rate(fancydns_upstream_duration_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))", + "refId": "A", + "legendFormat": "{{addr}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "fillOpacity": 12, + "lineWidth": 1 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 20, + "type": "timeseries", + "title": "Upstream errors", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 16, + "y": 39, + "w": 8, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fancydns_upstream_errors_total{instance=~\"$instance\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "errors" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "fillOpacity": 12, + "lineWidth": 1 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 21, + "type": "row", + "title": "State", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 47, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 22, + "type": "timeseries", + "title": "Blocklist rules and local records", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 0, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(fancydns_blocklist_rules{instance=~\"$instance\"})", + "refId": "A", + "legendFormat": "blocklist rules" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "max(fancydns_local_records{instance=~\"$instance\"})", + "refId": "B", + "legendFormat": "local zone records" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 12, + "lineWidth": 1 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 23, + "type": "timeseries", + "title": "In flight, and access-log lines dropped", + "description": "Dropped lines mean the pod's log stream could not keep up; DNS is unaffected.", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "x": 12, + "y": 48, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(fancydns_concurrent_queries{instance=~\"$instance\"})", + "refId": "A", + "legendFormat": "queries in flight" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "sum(rate(fancydns_access_log_dropped_total{instance=~\"$instance\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "log lines dropped / s" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "fillOpacity": 12, + "lineWidth": 1 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 24, + "type": "row", + "title": "Blocked and failed queries (needs Loki)", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 56, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 25, + "type": "logs", + "title": "Access log", + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "x": 0, + "y": 57, + "w": 24, + "h": 12 + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "$logselector | json | log=\"access\" | action=~\"$action\"", + "refId": "A" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": false, + "enableLogDetails": true, + "sortOrder": "Descending" + } + } + ] +} diff --git a/deploy/k8s/10-deployment.yaml b/deploy/k8s/10-deployment.yaml index 322dd3f..724d813 100644 --- a/deploy/k8s/10-deployment.yaml +++ b/deploy/k8s/10-deployment.yaml @@ -11,8 +11,8 @@ metadata: spec: serviceName: fancy-dns # A StatefulSet rather than a Deployment because each pod owns a PVC (the - # cache snapshot and the query log are per-pod state worth keeping across a - # reschedule). The flip side: the HPA does not work on a StatefulSet — the + # cache snapshot and downloaded blocklists are per-pod state worth keeping + # across a reschedule). The flip side: the HPA does not work on a StatefulSet — the # controller and the autoscaler fight over the replica count — so scaling # means adding nodes and replicas here, not CPU targets. replicas: 1 @@ -170,16 +170,16 @@ spec: - name: tmp emptyDir: {} volumeClaimTemplates: - # Cache snapshot (cache.persist.path) and query log (qlog.path) live here, - # along with downloaded blocklists. ReadWriteOnce is per-pod: a pod keeps + # Cache snapshot (cache.persist.path) lives here, along with downloaded + # blocklists. The resolver stores no query log: the access log goes to stdout. ReadWriteOnce is per-pod: a pod keeps # its own cache across a reschedule, which is the whole point of the # snapshot, but it cannot be shared between replicas. # # Sizing: cache.size is an entry count, not bytes, so disk usage has to be # estimated. A packed A/AAAA answer with a small authority section is # roughly 300-600 bytes on the wire and nearer 1 kB resident, so 500k - # entries lands around 500 MB; qlog is the other consumer (see - # qlog.max_entries). 4Gi leaves room for both plus a re-fetched list. + # entries lands around 500 MB. 4Gi leaves room for that plus a re-fetched + # list. - metadata: name: data spec: diff --git a/deploy/k8s/30-ingress.yaml b/deploy/k8s/30-ingress.yaml index 8e8de67..11316d3 100644 --- a/deploy/k8s/30-ingress.yaml +++ b/deploy/k8s/30-ingress.yaml @@ -13,13 +13,6 @@ metadata: k8s-helper.io/icon: "network" nginx.ingress.kubernetes.io/backend-protocol: "HTTP" cert-manager.io/cluster-issuer: letsencrypt-prod - # SSE (the live query log) is the reason for these three. With response - # buffering on, nginx holds events until the buffer fills and the log page - # looks frozen; with a 60s read timeout the stream is cut every minute and - # the page reconnects in a loop. - nginx.ingress.kubernetes.io/proxy-buffering: "off" - nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" - nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" spec: ingressClassName: nginx rules: diff --git a/internal/metrics/dashboard_test.go b/internal/metrics/dashboard_test.go new file mode 100644 index 0000000..46878c9 --- /dev/null +++ b/internal/metrics/dashboard_test.go @@ -0,0 +1,47 @@ +package metrics + +import ( + "os" + "regexp" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" +) + +// TestGrafanaDashboardOnlyUsesMetricsThatExist keeps deploy/grafana/fancy-dns.json +// from drifting away from the collectors: a renamed metric would otherwise +// leave a panel empty and nothing would say why. +func TestGrafanaDashboardOnlyUsesMetricsThatExist(t *testing.T) { + initCollectors() + descs := make(chan *prometheus.Desc, 256) + go func() { Registry.Describe(descs); close(descs) }() + known := map[string]bool{} + nameRe := regexp.MustCompile(`fqName: "([a-z0-9_]+)"`) + for d := range descs { + if m := nameRe.FindStringSubmatch(d.String()); m != nil { + known[m[1]] = true + } + } + if !known["fancydns_queries_total"] { + t.Fatalf("collector names were not read; got %v", known) + } + + raw, err := os.ReadFile("../../deploy/grafana/fancy-dns.json") + if err != nil { + t.Fatal(err) + } + used := regexp.MustCompile(`fancydns_[a-z0-9_]+`).FindAllString(string(raw), -1) + if len(used) == 0 { + t.Fatal("the dashboard references no fancydns metrics") + } + for _, name := range used { + base := name + for _, suffix := range []string{"_bucket", "_count", "_sum"} { + base = strings.TrimSuffix(base, suffix) + } + if !known[base] { + t.Errorf("the dashboard queries %s, which the resolver does not export", name) + } + } +}