Add a read-only Kubernetes source for zones, client groups and blocklists

The runtime can now take ClientGroup, BlockList and DNSZone objects
(fancydns.k6n.net/v1alpha1) from a cluster with -kube, from a pod's service
account or a kubeconfig alike. internal/kubesource watches them with dynamic
informers, converts and validates each object on its own, and publishes an
immutable snapshot. A bad object is skipped and keeps its last good version,
so one broken edit cannot remove a running policy; an invalid merged config is
dropped whole. The last snapshot is saved so a restart without apiserver access
still serves.

config.Store gains an overlay: Base() is what the YAML says, Get() is base plus
cluster objects. Everything that writes the config back (mutateConfig,
SetAdminAddr, the settings editor) now starts from Base(), otherwise cluster
objects would be copied into the file and shadow their own objects. YAML wins
every name collision. putACL and toggleBlocklist refuse to edit cluster-owned
objects. BlockList has no path field and refuses file:// URLs so a namespaced
object cannot pick files the runtime reads or writes. DNSZone is rendered
through the zone writer's own prove-by-reparse step, with the SOA serial taken
from metadata.generation so every replica renders identical bytes.

qlog: Flush reset the pending slice to [:0], so entries logged while it was
still storing the batch were appended over the same backing array. The race
detector found it through the new Kubernetes integration test. pending is now
set to nil, with a regression test that fails without the change.

CRD manifests, RBAC and examples are in deploy/k8s/crds; tests pin the schemas
to the Go spec structs so the two cannot drift.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

internal/app/app.go

@@ -16,6 +16,7 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"git.k8s-git-k6n.net/mats/fancy-dns/internal/acl"
@@ -25,6 +26,7 @@ import (
 	"git.k8s-git-k6n.net/mats/fancy-dns/internal/cluster"
 	"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
 	"git.k8s-git-k6n.net/mats/fancy-dns/internal/engine"
+	"git.k8s-git-k6n.net/mats/fancy-dns/internal/kubesource"
 	"git.k8s-git-k6n.net/mats/fancy-dns/internal/kubezones"
 	"git.k8s-git-k6n.net/mats/fancy-dns/internal/localzone"
 	"git.k8s-git-k6n.net/mats/fancy-dns/internal/metrics"
@@ -47,6 +49,8 @@ type Options struct {
 	// LogLevel and LogFormat override the config's logging settings.
 	LogLevel  string
 	LogFormat string
+	// Kube turns on the Kubernetes source for groups, lists and zones.
+	Kube KubeOptions
 }
 
 // App is the running process.
@@ -65,6 +69,10 @@ type App struct {
 	local      *localzone.Holder
 	// zones is the Kubernetes-sourced zone client.
 	zones *kubezones.Client
+	// kube watches the cluster's fancy-dns objects when enabled; kubeSnap is
+	// what the config overlay merges in.
+	kube     *kubesource.Source
+	kubeSnap atomic.Pointer[kubesource.Snapshot]
 	// managedZones is the console-managed zone directory: the only zone data
 	// the admin API may rewrite. An empty directory is what makes the
 	// console's Zones page read-only.
@@ -167,6 +175,10 @@ func (a *App) Start(ctx context.Context) error {
 	if err := a.startLogger(cfg); err != nil {
 		return err
 	}
+	if err := a.startKube(ctx); err != nil {
+		return err
+	}
+	cfg = a.cfg.Get() // the overlay may have changed it
 	a.startBlocklist()
 	a.startLocalZones(cfg)
 	a.startUpstream(cfg)
@@ -369,6 +381,9 @@ func (a *App) reloadLocalZones() {
 	if a.managedZones.Exists() {
 		paths = append(paths, a.managedZones.Dir())
 	}
+	if dir := a.kubeZoneDir(); dir != "" {
+		paths = append(paths, dir)
+	}
 	if len(paths) == 0 {
 		a.local.Set(nil)
 		metrics.SetLocalRecords(0)
@@ -866,7 +881,7 @@ func (a *App) Log() *slog.Logger { return a.log }
 // SetAdminAddr overrides the admin API listen address without touching the
 // config file, which is what a container platform's -admin-addr flag wants.
 func (a *App) SetAdminAddr(addr string) error {
-	next := a.cfg.Get()
+	next := a.cfg.Base()
 	cp := *next
 	cp.API = next.API
 	cp.API.Listen = []string{addr}

internal/app/harness_test.go

@@ -89,7 +89,7 @@ type harness struct {
 // can make a realistic single-value edit instead of inventing a minimal stub.
 func (h *harness) OriginalConfig() string { return h.config }
 
-func newHarness(t *testing.T, mutate func(*config.Config)) *harness {
+func newHarness(t *testing.T, mutate func(*config.Config), opts ...func(*Options)) *harness {
 	t.Helper()
 	dir := t.TempDir()
 	up := newTestUpstream(t)
@@ -159,7 +159,11 @@ func newHarness(t *testing.T, mutate func(*config.Config)) *harness {
 		t.Fatal(err)
 	}
 
-	a, err := New(Options{ConfigPath: path, Version: "test"})
+	opt := Options{ConfigPath: path, Version: "test"}
+	for _, o := range opts {
+		o(&opt)
+	}
+	a, err := New(opt)
 	if err != nil {
 		t.Fatalf("new app: %v", err)
 	}

internal/app/hooks.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"net/netip"
+	"reflect"
 	"sort"
 	"strings"
 	"sync"
@@ -431,6 +432,9 @@ func (a *App) toggleBlocklist(name string, enabled bool) error {
 	if a.blocks == nil {
 		return fmt.Errorf("blocklists are disabled")
 	}
+	if a.clusterOwned("list", name) {
+		return fmt.Errorf("blocklist %q is a Kubernetes BlockList; set spec.disabled on the object instead", name)
+	}
 	err := a.mutateConfig(func(c *config.Config) error {
 		found := false
 		for i := range c.Block.Lists {
@@ -459,7 +463,10 @@ func (a *App) toggleBlocklist(name string, enabled bool) error {
 // mutateConfig clones, mutates, validates and commits the config, returning
 // the validation error when the change is rejected.
 func (a *App) mutateConfig(fn func(*config.Config) error) error {
-	cur := a.cfg.Get()
+	// Start from the base, not the effective config: the effective one holds
+	// the cluster's groups and lists, and persisting those would copy them
+	// into the YAML file where they would shadow their own objects.
+	cur := a.cfg.Base()
 	next := *cur
 	// A shallow copy is not enough: the slices and maps we touch need their own
 	// backing arrays, or a rejected change would corrupt the live config.
@@ -767,6 +774,28 @@ func (a *App) getACL() any {
 
 // putACL validates and persists new ACL groups.
 func (a *App) putACL(_ context.Context, groups []config.Group, def string) (int64, error) {
+	// The console shows the effective list, so a save sends the cluster's
+	// groups back along with the file's. Unchanged ones are dropped here (the
+	// cluster still owns them); a changed one is refused, because the write
+	// would land in the YAML and silently shadow the object.
+	if snap := a.kubeSnap.Load(); snap != nil {
+		owned := make(map[string]config.Group, len(snap.Groups))
+		for _, g := range snap.Groups {
+			owned[g.Name] = g
+		}
+		kept := groups[:0:0]
+		for _, g := range groups {
+			cg, ok := owned[g.Name]
+			if !ok {
+				kept = append(kept, g)
+				continue
+			}
+			if !reflect.DeepEqual(g, cg) {
+				return 0, fmt.Errorf("group %q is a Kubernetes ClientGroup; edit the object instead", g.Name)
+			}
+		}
+		groups = kept
+	}
 	err := a.mutateConfig(func(c *config.Config) error {
 		c.ACL.Groups = groups
 		if def != "" {

internal/app/kube.go

@@ -0,0 +1,200 @@
+package app
+
+import (
+	"context"
+	"path/filepath"
+	"time"
+
+	"k8s.io/client-go/dynamic"
+
+	"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
+	"git.k8s-git-k6n.net/mats/fancy-dns/internal/kubesource"
+)
+
+// KubeOptions turn on the Kubernetes source: ClientGroup, BlockList and
+// DNSZone objects are watched and merged into the configuration. The runtime
+// only reads; it works the same in a pod and on a workstation.
+type KubeOptions struct {
+	Enabled bool
+	// Kubeconfig and Context pick credentials; both empty means the pod's
+	// service account when there is one, else the default kubeconfig.
+	Kubeconfig string
+	Context    string
+	// Namespace restricts the watch; empty means the pod's own namespace
+	// in-cluster, else every namespace.
+	Namespace string
+	// SyncTimeout bounds how long startup waits for the first listing before
+	// falling back to the last saved snapshot. Zero means 30 seconds.
+	SyncTimeout time.Duration
+	// Client replaces the one built from the credentials; for tests.
+	Client dynamic.Interface
+}
+
+const (
+	kubeDegraded  = "kubernetes: not synced, serving the last saved cluster configuration"
+	kubeRejected  = "kubernetes: objects rejected"
+	kubeNoSnap    = "kubernetes: not synced and no saved snapshot, cluster configuration is empty"
+	kubeInvalid   = "kubernetes: the merged configuration is invalid, cluster objects are ignored"
+	kubeZoneFail  = "kubernetes: cluster zones could not be written, serving the previous ones"
+	defaultKubeTO = 30 * time.Second
+)
+
+func (a *App) kubeZoneDir() string {
+	if a.kube == nil {
+		return ""
+	}
+	return filepath.Join(a.DataDir(), "kube-zones")
+}
+
+func (a *App) kubeSnapshotPath() string { return filepath.Join(a.DataDir(), "kube-snapshot.json") }
+
+// startKube connects to the cluster and installs the overlay. It runs before
+// the components that read the configuration, so the first blocklist compile
+// and the first zone load already include the cluster's objects.
+func (a *App) startKube(ctx context.Context) error {
+	o := a.opt.Kube
+	if !o.Enabled {
+		return nil
+	}
+	ns := o.Namespace
+	if ns == "" {
+		ns = kubesource.Namespace()
+	}
+	client, from := o.Client, "injected client"
+	if client == nil {
+		var err error
+		client, from, err = kubesource.NewClient(kubesource.ClientOptions{Kubeconfig: o.Kubeconfig, Context: o.Context})
+		if err != nil {
+			return err
+		}
+	}
+	src, err := kubesource.New(kubesource.Options{Client: client, Namespace: ns, Logger: a.log})
+	if err != nil {
+		return err
+	}
+	a.kube = src
+	a.log.Info("watching Kubernetes for fancy-dns objects", "credentials", from, "namespace", ns)
+
+	a.cfg.SetOverlay(a.kubeOverlay)
+	go func() { _ = src.Run(ctx) }()
+
+	wait := o.SyncTimeout
+	if wait <= 0 {
+		wait = defaultKubeTO
+	}
+	syncCtx, cancel := context.WithTimeout(ctx, wait)
+	defer cancel()
+	if err := src.WaitSynced(syncCtx); err != nil {
+		// A cluster that cannot be reached must not stop DNS from coming up:
+		// resolve from what was last known and keep watching in the background.
+		snap, lerr := kubesource.Load(a.kubeSnapshotPath())
+		switch {
+		case lerr != nil:
+			a.log.Error("saved cluster snapshot is unreadable", "error", lerr)
+			a.markDegraded(kubeNoSnap)
+		case snap == nil:
+			a.markDegraded(kubeNoSnap)
+		default:
+			a.kubeSnap.Store(snap)
+			a.markDegraded(kubeDegraded)
+		}
+		a.log.Warn("Kubernetes did not sync in time, continuing without it", "waited", wait.String(), "error", err)
+		a.applyKube(a.kubeSnap.Load(), false)
+	} else {
+		a.applyKube(src.Snapshot(), true)
+	}
+	// Later changes arrive here. Registered after the first apply so the
+	// startup path above is the only one that runs before the listeners exist.
+	src.OnChange(func(sn *kubesource.Snapshot) { a.applyKube(sn, true) })
+	return nil
+}
+
+// applyKube makes snap the cluster's contribution to the configuration.
+func (a *App) applyKube(snap *kubesource.Snapshot, live bool) {
+	if snap == nil {
+		a.cfg.Reapply()
+		return
+	}
+	prev := a.kubeSnap.Load()
+	a.kubeSnap.Store(snap)
+	if live {
+		a.clearDegraded(kubeDegraded)
+		a.clearDegraded(kubeNoSnap)
+		if err := kubesource.Save(a.kubeSnapshotPath(), snap); err != nil {
+			a.log.Warn("could not save the cluster snapshot", "error", err)
+		}
+		if len(snap.Problems) > 0 {
+			a.markDegraded(kubeRejected)
+		} else {
+			a.clearDegraded(kubeRejected)
+		}
+	}
+	if _, err := kubesource.SyncZoneDir(a.kubeZoneDir(), snap.Zones); err != nil {
+		a.markDegraded(kubeZoneFail)
+		a.log.Error("could not write cluster zones, keeping the previous ones", "error", err)
+	} else {
+		a.clearDegraded(kubeZoneFail)
+	}
+	a.cfg.Reapply()
+	if prev == nil {
+		return
+	}
+	// Reapply reloaded the zones and rebuilt the ACL. Cached answers were
+	// produced under the old policy, so they go, as they do for any ACL,
+	// blocklist or zone edit.
+	a.invalidateBlocklistCache()
+	if a.blocks != nil {
+		go func() {
+			if _, err := a.blocks.RefreshAll(context.Background()); err != nil {
+				a.log.Warn("blocklist refresh after cluster change failed", "error", err)
+			}
+		}()
+	}
+}
+
+// kubeOverlay is the Store overlay: the file's configuration plus the
+// cluster's objects. A merge that does not validate is dropped whole and the
+// file's configuration stays in force, because half of a policy is worse than
+// the last whole one.
+func (a *App) kubeOverlay(base *config.Config) *config.Config {
+	snap := a.kubeSnap.Load()
+	if snap == nil {
+		return base
+	}
+	merged, dropped := kubesource.Merge(base, snap)
+	for _, d := range dropped {
+		a.log.Warn("cluster object ignored", "object", d)
+	}
+	check := *merged
+	if err := check.Validate(); err != nil {
+		a.log.Error("cluster configuration does not validate, ignoring it", "error", err)
+		a.markDegraded(kubeInvalid)
+		return base
+	}
+	a.clearDegraded(kubeInvalid)
+	return merged
+}
+
+// clusterOwned reports whether a group or list name comes from the cluster,
+// and so cannot be edited through the console.
+func (a *App) clusterOwned(kind, name string) bool {
+	snap := a.kubeSnap.Load()
+	if snap == nil {
+		return false
+	}
+	switch kind {
+	case "group":
+		for _, g := range snap.Groups {
+			if g.Name == name {
+				return true
+			}
+		}
+	case "list":
+		for _, l := range snap.Lists {
+			if l.Name == name {
+				return true
+			}
+		}
+	}
+	return false
+}

internal/app/kube_test.go

@@ -0,0 +1,118 @@
+package app
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/miekg/dns"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	dynfake "k8s.io/client-go/dynamic/fake"
+
+	"git.k8s-git-k6n.net/mats/fancy-dns/internal/kubesource"
+)
+
+func kubeObj(kind, name string, spec map[string]any) *unstructured.Unstructured {
+	return &unstructured.Unstructured{Object: map[string]any{
+		"apiVersion": kubesource.APIGroup + "/" + kubesource.Version,
+		"kind":       kind,
+		"metadata":   map[string]any{"name": name, "namespace": "dns", "generation": int64(1)},
+		"spec":       spec,
+	}}
+}
+
+func kubeClient() *dynfake.FakeDynamicClient {
+	return dynfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(),
+		map[schema.GroupVersionResource]string{
+			kubesource.ZoneGVR:  "DNSZoneList",
+			kubesource.GroupGVR: "ClientGroupList",
+			kubesource.ListGVR:  "BlockListList",
+		})
+}
+
+func eventually(t *testing.T, what string, ok func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		if ok() {
+			return
+		}
+		time.Sleep(25 * time.Millisecond)
+	}
+	t.Fatalf("timed out waiting for %s", what)
+}
+
+// TestKubeObjectsReachTheResolver is the point of the Kubernetes source: an
+// object applied to the cluster changes what this resolver answers, with no
+// reload, and deleting it takes the change back.
+func TestKubeObjectsReachTheResolver(t *testing.T) {
+	client := kubeClient()
+	h := newHarness(t, nil, func(o *Options) {
+		o.Kube = KubeOptions{Enabled: true, Namespace: "dns", Client: client, SyncTimeout: 5 * time.Second}
+	})
+	ctx := context.Background()
+	zones := client.Resource(kubesource.ZoneGVR).Namespace("dns")
+	groups := client.Resource(kubesource.GroupGVR).Namespace("dns")
+
+	answer := func() string {
+		m := h.query("www.kube.test", dns.TypeA)
+		if len(m.Answer) == 1 {
+			if a, ok := m.Answer[0].(*dns.A); ok && m.Authoritative {
+				return a.A.String()
+			}
+		}
+		return ""
+	}
+
+	if _, err := zones.Create(ctx, kubeObj("DNSZone", "kube", map[string]any{
+		"zone": "kube.test",
+		"records": []any{
+			map[string]any{"name": "@", "type": "NS", "value": "ns1.kube.test."},
+			map[string]any{"name": "ns1", "type": "A", "value": "10.1.2.1"},
+			map[string]any{"name": "www", "type": "A", "value": "10.1.2.3"},
+		},
+	}), metav1.CreateOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	eventually(t, "the DNSZone to be served", func() bool { return answer() == "10.1.2.3" })
+
+	// The cluster's data never lands in the YAML the console writes back.
+	if got := len(h.app.cfg.Base().ACL.Groups); got != 0 {
+		t.Fatalf("base has %d groups", got)
+	}
+
+	if _, err := groups.Create(ctx, kubeObj("ClientGroup", "no-loopback", map[string]any{
+		"policy": "deny", "match": map[string]any{"cidrs": []any{"127.0.0.0/8"}},
+	}), metav1.CreateOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	eventually(t, "the ClientGroup to refuse the client", func() bool {
+		return h.query("www.example.test", dns.TypeA).Rcode == dns.RcodeRefused
+	})
+	if len(h.app.cfg.Base().ACL.Groups) != 0 || len(h.app.cfg.Get().ACL.Groups) != 1 {
+		t.Fatalf("base=%d effective=%d groups", len(h.app.cfg.Base().ACL.Groups), len(h.app.cfg.Get().ACL.Groups))
+	}
+
+	// The console cannot overwrite a cluster-owned group.
+	if _, err := h.app.putACL(ctx, nil, ""); err != nil {
+		t.Fatalf("saving the console's own (empty) list: %v", err)
+	}
+	if len(h.app.cfg.Get().ACL.Groups) != 1 {
+		t.Fatal("saving the file's groups removed the cluster's")
+	}
+
+	if err := groups.Delete(ctx, "no-loopback", metav1.DeleteOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	eventually(t, "the deleted ClientGroup to stop applying", func() bool {
+		return h.query("www.example.test", dns.TypeA).Rcode == dns.RcodeSuccess
+	})
+
+	if err := zones.Delete(ctx, "kube", metav1.DeleteOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	eventually(t, "the deleted DNSZone to stop being authoritative", func() bool { return answer() == "" })
+}