Add fancy-dns-console migrate to convert an existing deployment into cluster objects

The settings of the ConfigMap-and-YAML deployment have to become objects before
the mechanisms that held them can go. The command reads what the old console
wrote (the zone--<apex>.zone keys of the zones ConfigMap and the
address-overrides RPZ), and optionally the resolver's config.yaml, and converts:

  zone master files            -> DNSZone
  address-overrides RPZ        -> AddressOverride, one per name
  URL blocklists               -> BlockList
  allow_entries/block_entries  -> DomainRule inline-allow / inline-block
  acl.groups                   -> ClientGroup

By default it only prints manifests, which can be reviewed or committed; -apply
creates or updates the objects. It never deletes or edits the ConfigMaps or the
config. Each conversion is validated with the code the resolver uses to read the
objects, and anything it cannot represent faithfully is listed and left alone:
a zone with a record type the API does not write, a list that is a file on the
pod, a name that is not a legal object name, an RPZ line that is not an A or
AAAA override. Tests apply a conversion of the live data to a fake cluster and
read it back through the resolver's source, and run the command against fake
ConfigMaps, offline from files, and with the ConfigMaps missing.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

internal/migrate/migrate.go

@@ -0,0 +1,297 @@
+// Package migrate turns what a ConfigMap-and-YAML deployment held into fancy-dns
+// objects: zone master files become DNSZone, the address-overrides RPZ becomes
+// AddressOverride, URL blocklists become BlockList, inline allow/block entries
+// become DomainRule, and ACL groups become ClientGroup.
+//
+// It converts, validates with the same code the resolver uses to read the
+// objects, and reports what it could not carry over. It never deletes anything
+// and never guesses: an input it cannot represent faithfully is listed and left
+// where it is.
+package migrate
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/yaml"
+
+	"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
+	"git.k8s-git-k6n.net/mats/fancy-dns/internal/kubeedit"
+	"git.k8s-git-k6n.net/mats/fancy-dns/internal/kubesource"
+	"git.k8s-git-k6n.net/mats/fancy-dns/internal/localzone"
+)
+
+// Inputs is everything to convert. Any part may be empty.
+type Inputs struct {
+	// ZoneFiles maps an apex ("home.example.com") to its master file.
+	ZoneFiles map[string]string
+	// RPZ is the address-overrides RPZ text.
+	RPZ string
+	// Config is the resolver's YAML configuration.
+	Config *config.Config
+	// Namespace is where the objects will live.
+	Namespace string
+}
+
+// Result is the converted objects and the account of what was not converted.
+type Result struct {
+	Objects []*unstructured.Unstructured
+	// Notes say what was converted and what to remove from the config afterwards.
+	Notes []string
+	// Skipped are inputs that could not be represented, each with the reason.
+	Skipped []string
+}
+
+const managedBy = "fancy-dns-migrate"
+
+// Convert does the whole conversion.
+func Convert(in Inputs) Result {
+	var r Result
+	r.zones(in)
+	r.overrides(in)
+	if in.Config != nil {
+		r.config(in)
+	}
+	sort.SliceStable(r.Objects, func(i, j int) bool {
+		a, b := r.Objects[i], r.Objects[j]
+		if a.GetKind() != b.GetKind() {
+			return a.GetKind() < b.GetKind()
+		}
+		return a.GetName() < b.GetName()
+	})
+	return r
+}
+
+func (r *Result) add(kind, name, ns string, spec any) {
+	m, err := toMap(spec)
+	if err != nil {
+		r.Skipped = append(r.Skipped, fmt.Sprintf("%s %s: %v", kind, name, err))
+		return
+	}
+	u := &unstructured.Unstructured{Object: map[string]any{
+		"apiVersion": kubesource.APIGroup + "/" + kubesource.Version,
+		"kind":       kind,
+		"metadata": map[string]any{
+			"name": name, "namespace": ns,
+			"labels": map[string]any{"app.kubernetes.io/managed-by": managedBy},
+		},
+		"spec": m,
+	}}
+	r.Objects = append(r.Objects, u)
+}
+
+func (r *Result) skip(format string, args ...any) {
+	r.Skipped = append(r.Skipped, fmt.Sprintf(format, args...))
+}
+func (r *Result) note(format string, args ...any) {
+	r.Notes = append(r.Notes, fmt.Sprintf(format, args...))
+}
+
+// zones converts master files.
+func (r *Result) zones(in Inputs) {
+	apexes := make([]string, 0, len(in.ZoneFiles))
+	for a := range in.ZoneFiles {
+		apexes = append(apexes, a)
+	}
+	sort.Strings(apexes)
+	for _, name := range apexes {
+		apex, err := localzone.NormalizeOrigin(name)
+		if err != nil {
+			r.skip("zone file %q: %v", name, err)
+			continue
+		}
+		zf, err := localzone.ParseZoneFile(apex, in.ZoneFiles[name])
+		if err != nil {
+			r.skip("zone file %q: does not parse: %v", name, err)
+			continue
+		}
+		spec := kubesource.DNSZoneSpec{Zone: strings.TrimSuffix(apex, "."), TTL: zf.TTL, SOA: zf.Soa}
+		for _, rec := range zf.Records {
+			rel := "@"
+			if rec.Name != apex {
+				rel = strings.TrimSuffix(rec.Name, "."+apex)
+			}
+			ttl := rec.TTL
+			if ttl == zf.TTL {
+				ttl = 0 // inherits the zone's, as it did in the file
+			}
+			spec.Records = append(spec.Records, kubesource.ZoneRecord{Name: rel, Type: rec.Type, TTL: ttl, Value: rec.Value})
+		}
+		if _, err := kubesource.CheckZone(spec); err != nil {
+			r.skip("zone %s: not representable as a DNSZone (%v); it stays a master file", apex, err)
+			continue
+		}
+		objName := strings.TrimSuffix(apex, ".")
+		r.add("DNSZone", objName, in.Namespace, spec)
+		r.note("zone %s: %d records -> DNSZone/%s", apex, len(spec.Records), objName)
+	}
+}
+
+// overrides converts the address-overrides RPZ.
+func (r *Result) overrides(in Inputs) {
+	if strings.TrimSpace(in.RPZ) == "" {
+		return
+	}
+	byName := map[string][]string{}
+	for i, line := range strings.Split(in.RPZ, "\n") {
+		if j := strings.IndexAny(line, ";#"); j >= 0 {
+			line = line[:j]
+		}
+		f := strings.Fields(line)
+		if len(f) == 0 || strings.HasPrefix(f[0], "$") {
+			continue
+		}
+		// "name [ttl] [IN] A|AAAA address"
+		rest := f[1:]
+		for len(rest) > 0 && (strings.EqualFold(rest[0], "IN") || isNumber(rest[0])) {
+			rest = rest[1:]
+		}
+		if len(rest) < 2 || !(strings.EqualFold(rest[0], "A") || strings.EqualFold(rest[0], "AAAA")) {
+			r.skip("address overrides line %d %q: only A and AAAA overrides are converted", i+1, strings.TrimSpace(line))
+			continue
+		}
+		name := strings.TrimSuffix(strings.ToLower(f[0]), ".")
+		byName[name] = append(byName[name], rest[1])
+	}
+	names := make([]string, 0, len(byName))
+	for n := range byName {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	for _, n := range names {
+		spec := kubesource.AddressOverrideSpec{Domain: n, Addresses: byName[n]}
+		o, err := kubesource.CheckOverride(spec)
+		if err != nil {
+			r.skip("address override %s: %v", n, err)
+			continue
+		}
+		r.add("AddressOverride", kubeedit.ObjectName(o.Domain), in.Namespace,
+			kubesource.AddressOverrideSpec{Domain: o.Domain, Addresses: o.Addresses})
+	}
+	if len(names) > 0 {
+		r.note("address overrides: %d names -> AddressOverride objects", len(names))
+	}
+}
+
+func isNumber(s string) bool {
+	for _, c := range s {
+		if c < '0' || c > '9' {
+			return false
+		}
+	}
+	return s != ""
+}
+
+// config converts the parts of the resolver's YAML that cluster objects replace.
+func (r *Result) config(in Inputs) {
+	cfg := in.Config
+	for _, l := range cfg.Block.Lists {
+		switch {
+		case l.URL == "" && strings.EqualFold(l.Format, "rpz") && strings.Contains(l.Path, "address-overrides"):
+			r.note("list %q is the address-overrides file; the AddressOverride objects replace it, remove it from blocklist.lists", l.Name)
+		case l.URL == "":
+			r.skip("list %q: it is a file on the pod (%s), which a cluster object cannot point at; host it at a URL or keep it in the config", l.Name, l.Path)
+		case !strings.HasPrefix(l.URL, "http://") && !strings.HasPrefix(l.URL, "https://"):
+			r.skip("list %q: url %q is not http(s)", l.Name, l.URL)
+		default:
+			spec := kubesource.BlockListSpec{
+				URL: l.URL, Format: l.Format, Mirror: l.Mirror, Disabled: !l.Enabled,
+				MaxBytes: l.MaxBytes, Categories: l.Categories, Comment: l.Comment,
+				InsecureSkipTLSVerify: l.InsecureSkipTLSVerify,
+			}
+			if l.Refresh > 0 {
+				spec.Refresh = time.Duration(l.Refresh).String()
+			}
+			// The config loader fills these in; only a value the operator chose
+			// is worth writing down.
+			if l.Timeout > 0 && l.Timeout != cfg.Server.Timeouts.List {
+				spec.Timeout = time.Duration(l.Timeout).String()
+			}
+			if spec.MaxBytes == 512<<20 {
+				spec.MaxBytes = 0
+			}
+			if _, err := kubesource.CheckList(l.Name, spec); err != nil {
+				r.skip("list %q: %v", l.Name, err)
+				continue
+			}
+			if name := kubeedit.ObjectName(l.Name); name != l.Name {
+				r.note("list %q: object name will be %q (not a legal Kubernetes name as written)", l.Name, name)
+				r.add("BlockList", name, in.Namespace, spec)
+			} else {
+				r.add("BlockList", l.Name, in.Namespace, spec)
+			}
+			r.note("list %q -> BlockList; remove it from blocklist.lists", l.Name)
+		}
+	}
+	for _, rules := range []struct {
+		action  string
+		entries []string
+	}{{"block", cfg.Block.BlockEntries}, {"allow", cfg.Block.AllowEntries}} {
+		if len(rules.entries) == 0 {
+			continue
+		}
+		var ok []string
+		for _, e := range rules.entries {
+			e = strings.TrimSpace(e)
+			if e == "" || strings.HasPrefix(e, "#") {
+				continue
+			}
+			if _, err := kubesource.CheckRules(kubesource.DomainRuleSpec{Action: rules.action, Names: []string{e}}); err != nil {
+				r.skip("%s entry %q: %v", rules.action, e, err)
+				continue
+			}
+			ok = append(ok, e)
+		}
+		if len(ok) == 0 {
+			continue
+		}
+		r.add("DomainRule", "inline-"+rules.action, in.Namespace, kubesource.DomainRuleSpec{
+			Action: rules.action, Names: ok, Comment: "Migrated from blocklist." + rules.action + "_entries",
+		})
+		r.note("%d %s entries -> DomainRule/inline-%s; remove blocklist.%s_entries", len(ok), rules.action, rules.action, rules.action)
+	}
+	for i, g := range cfg.ACL.Groups {
+		spec := kubeedit.SpecFromGroup(g, int32(i+1)*10)
+		if _, err := kubesource.CheckGroup(g.Name, spec); err != nil {
+			r.skip("acl group %q: %v", g.Name, err)
+			continue
+		}
+		if name := g.Name; kubeedit.ObjectName(name) != name {
+			r.skip("acl group %q: not a legal Kubernetes object name; rename it", g.Name)
+			continue
+		}
+		r.add("ClientGroup", g.Name, in.Namespace, spec)
+	}
+	if len(cfg.ACL.Groups) > 0 {
+		r.note("%d acl groups -> ClientGroup; remove acl.groups from the config (default_policy stays)", len(cfg.ACL.Groups))
+	}
+}
+
+// YAML renders the objects as a multi-document manifest.
+func YAML(objs []*unstructured.Unstructured) ([]byte, error) {
+	var b strings.Builder
+	for i, o := range objs {
+		out, err := yaml.Marshal(o.Object)
+		if err != nil {
+			return nil, err
+		}
+		if i > 0 {
+			b.WriteString("---\n")
+		}
+		b.Write(out)
+	}
+	return []byte(b.String()), nil
+}
+
+// toMap converts a spec struct to unstructured form. ToUnstructured wants a
+// pointer to the concrete type.
+func toMap(spec any) (map[string]any, error) {
+	v := reflect.New(reflect.TypeOf(spec))
+	v.Elem().Set(reflect.ValueOf(spec))
+	return runtime.DefaultUnstructuredConverter.ToUnstructured(v.Interface())
+}

internal/migrate/migrate_test.go

@@ -0,0 +1,168 @@
+package migrate
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	dynfake "k8s.io/client-go/dynamic/fake"
+
+	"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
+	"git.k8s-git-k6n.net/mats/fancy-dns/internal/kubeedit"
+	"git.k8s-git-k6n.net/mats/fancy-dns/internal/kubesource"
+)
+
+// The data the live deployment actually has.
+const liveRPZ = `$TTL 60
+tornberg.me A 10.10.3.10
+knatofs.se A 10.10.3.10
+k6n.net A 10.10.3.10
+slask.local A 10.10.3.10
+iamu.dev A 10.10.3.10
+`
+
+const homeZone = `$TTL 300
+$ORIGIN home.example.com.
+@   IN SOA ns1.home.example.com. hostmaster.home.example.com. 2024010101 3600 600 604800 300
+@   IN NS  ns1.home.example.com.
+ns1 IN A   10.0.0.53
+nas 60 IN A 10.0.0.20
+mail IN MX 10 mx.home.example.com.
+txt IN TXT "v=spf1 -all"
+*.apps IN A 10.0.0.30
+`
+
+func liveConfig(t *testing.T) *config.Config {
+	t.Helper()
+	cfg := config.Default()
+	cfg.Block.Lists = []config.ListSource{
+		{Name: "edge-router-x-address-overrides", Format: "rpz", Path: "/etc/fancy-dns/rpz/address-overrides.rpz", Enabled: true},
+		{Name: "haas", Format: "plain", URL: "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts",
+			Path: "/var/lib/fancy-dns/lists/hosts", Enabled: true, Refresh: config.Duration(24 * time.Hour), Categories: []string{"ads", "tracking"}},
+		{Name: "local file", Format: "hosts", Path: "/etc/x/hosts", Enabled: true},
+	}
+	cfg.Block.BlockEntries = []string{"ads.example.com", "*.tracker.example", "bad name"}
+	cfg.Block.AllowEntries = []string{"good.example"}
+	cfg.ACL.Groups = []config.Group{
+		{Name: "kids", Policy: "block_only", Match: config.Match{CIDRs: []string{"10.0.20.0/24"}}},
+		{Name: "Bad Name", Policy: "allow", Match: config.Match{CIDRs: []string{"10.0.30.0/24"}}},
+	}
+	return cfg
+}
+
+func TestConvertEverythingTheDeploymentHolds(t *testing.T) {
+	res := Convert(Inputs{
+		ZoneFiles: map[string]string{"home.example.com": homeZone, "broken.example": "not a zone @@@ ("},
+		RPZ:       liveRPZ + "bad.test CNAME x.example.\n",
+		Config:    liveConfig(t),
+		Namespace: "dns",
+	})
+	kinds := map[string]int{}
+	for _, o := range res.Objects {
+		kinds[o.GetKind()]++
+		if o.GetNamespace() != "dns" || o.GetLabels()["app.kubernetes.io/managed-by"] != managedBy {
+			t.Errorf("%s/%s: namespace/labels = %q %v", o.GetKind(), o.GetName(), o.GetNamespace(), o.GetLabels())
+		}
+	}
+	want := map[string]int{"DNSZone": 1, "AddressOverride": 5, "BlockList": 1, "DomainRule": 2, "ClientGroup": 1}
+	for k, n := range want {
+		if kinds[k] != n {
+			t.Errorf("%s: %d objects, want %d (%v)", k, kinds[k], n, kinds)
+		}
+	}
+	skipped := strings.Join(res.Skipped, "\n")
+	for _, must := range []string{"broken.example", "bad.test", "local file", "bad name", "Bad Name"} {
+		if !strings.Contains(skipped, must) {
+			t.Errorf("%q should have been reported as skipped:\n%s", must, skipped)
+		}
+	}
+	notes := strings.Join(res.Notes, "\n")
+	for _, must := range []string{"edge-router-x-address-overrides", "haas", "blocklist.block_entries"} {
+		if !strings.Contains(notes, must) {
+			t.Errorf("notes should mention %q:\n%s", must, notes)
+		}
+	}
+}
+
+// TestConvertedObjectsAreWhatTheResolverAccepts applies the result to a cluster
+// and reads it back through the resolver's own source: nothing converted may be
+// rejected there, and the data must come out the same.
+func TestConvertedObjectsAreWhatTheResolverAccepts(t *testing.T) {
+	res := Convert(Inputs{
+		ZoneFiles: map[string]string{"home.example.com": homeZone},
+		RPZ:       liveRPZ,
+		Config:    liveConfig(t),
+		Namespace: "dns",
+	})
+	client := dynfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(),
+		map[schema.GroupVersionResource]string{
+			kubesource.ZoneGVR: "DNSZoneList", kubesource.GroupGVR: "ClientGroupList", kubesource.ListGVR: "BlockListList",
+			kubesource.OverrideGVR: "AddressOverrideList", kubesource.RuleGVR: "DomainRuleList",
+		})
+	ed := kubeedit.New(client, "dns")
+	for _, o := range res.Objects {
+		if err := ed.Apply(context.Background(), o, "migrate"); err != nil {
+			t.Fatalf("apply %s/%s: %v", o.GetKind(), o.GetName(), err)
+		}
+	}
+
+	src, err := kubesource.New(kubesource.Options{Client: client, Namespace: "dns", Debounce: 10 * time.Millisecond})
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := make(chan *kubesource.Snapshot, 4)
+	src.OnChange(func(s *kubesource.Snapshot) { got <- s })
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go func() { _ = src.Run(ctx) }()
+	var sn *kubesource.Snapshot
+	select {
+	case sn = <-got:
+	case <-time.After(5 * time.Second):
+		t.Fatal("no snapshot")
+	}
+
+	if len(sn.Problems) != 0 {
+		t.Fatalf("the resolver rejected migrated objects: %+v", sn.Problems)
+	}
+	if len(sn.Overrides) != 5 || sn.Overrides[0].Domain != "iamu.dev" || sn.Overrides[0].Addresses[0] != "10.10.3.10" {
+		t.Errorf("overrides = %+v", sn.Overrides)
+	}
+	if len(sn.Groups) != 1 || sn.Groups[0].Name != "kids" || sn.Groups[0].Policy != "block_only" {
+		t.Errorf("groups = %+v", sn.Groups)
+	}
+	if len(sn.Lists) != 1 || sn.Lists[0].Name != "haas" || !sn.Lists[0].Enabled || sn.Lists[0].Path != "" {
+		t.Errorf("lists = %+v", sn.Lists)
+	}
+	if strings.Join(sn.BlockNames, ",") != "ads.example.com,*.tracker.example" || strings.Join(sn.AllowNames, ",") != "good.example" {
+		t.Errorf("rules = %v / %v", sn.BlockNames, sn.AllowNames)
+	}
+	text := sn.Zones["home.example.com."]
+	if text == "" {
+		t.Fatalf("zones = %v", sn.Zones)
+	}
+	for _, line := range []string{"ns1", "10.0.0.20", "mx.home.example.com.", "v=spf1 -all", "*.apps", "IN SOA"} {
+		if !strings.Contains(text, line) {
+			t.Errorf("rendered zone lacks %q:\n%s", line, text)
+		}
+	}
+	// nas had its own TTL (60); the rest inherit the zone's 300.
+	if !strings.Contains(text, "nas") || !strings.Contains(text, " 1m IN A ") && !strings.Contains(text, " 60 IN A ") {
+		t.Errorf("nas lost its TTL:\n%s", text)
+	}
+}
+
+func TestYAMLRoundTrips(t *testing.T) {
+	res := Convert(Inputs{RPZ: liveRPZ, Namespace: "dns"})
+	out, err := YAML(res.Objects)
+	if err != nil {
+		t.Fatal(err)
+	}
+	s := string(out)
+	if strings.Count(s, "---") != 4 || !strings.Contains(s, "kind: AddressOverride") || !strings.Contains(s, "namespace: dns") {
+		t.Fatalf("YAML:\n%s", s)
+	}
+}