diff --git a/cmd/fancy-dns-console/main.go b/cmd/fancy-dns-console/main.go index abf4acc..355d7f3 100644 --- a/cmd/fancy-dns-console/main.go +++ b/cmd/fancy-dns-console/main.go @@ -28,6 +28,17 @@ import ( ) func main() { + // `fancy-dns-console migrate ...` converts an existing ConfigMap-and-YAML + // deployment into cluster objects; see migrate.go. + if len(os.Args) > 1 && os.Args[1] == "migrate" { + if err := migrateMain(os.Args[2:], os.Stdout, os.Stderr); err != nil { + if !errors.Is(err, flag.ErrHelp) { + fmt.Fprintln(os.Stderr, "fancy-dns-console migrate:", err) + } + os.Exit(1) + } + return + } var ( configPath = flag.String("config", envOr("FANCYDNS_CONSOLE_CONFIG", "/etc/fancy-dns/console.yaml"), "console configuration (api: listen, auth, ui)") runtimeURL = flag.String("runtime-url", os.Getenv("FANCYDNS_RUNTIME_URL"), "admin API of the resolver, e.g. http://fancy-dns.dns:9053") diff --git a/cmd/fancy-dns-console/migrate.go b/cmd/fancy-dns-console/migrate.go new file mode 100644 index 0000000..26177f1 --- /dev/null +++ b/cmd/fancy-dns-console/migrate.go @@ -0,0 +1,188 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + + "git.k8s-git-k6n.net/mats/fancy-dns/internal/config" + "git.k8s-git-k6n.net/mats/fancy-dns/internal/kubeedit" + "git.k8s-git-k6n.net/mats/fancy-dns/internal/kubesource" + "git.k8s-git-k6n.net/mats/fancy-dns/internal/migrate" +) + +var configMaps = schema.GroupVersionResource{Version: "v1", Resource: "configmaps"} + +// newClient builds the cluster client; a test replaces it. +var newClient = func(o kubesource.ClientOptions) (dynamic.Interface, error) { + c, _, err := kubesource.NewClient(o) + return c, err +} + +// migrateMain converts an existing ConfigMap-and-YAML deployment into cluster +// objects. By default it only prints manifests, so the result can be reviewed +// or committed; -apply writes them. It changes nothing else. +func migrateMain(args []string, stdout, stderr io.Writer) error { + fs := flag.NewFlagSet("migrate", flag.ContinueOnError) + fs.SetOutput(stderr) + var ( + ns = fs.String("kube-namespace", os.Getenv("FANCYDNS_KUBE_NAMESPACE"), "namespace of the ConfigMaps and of the new objects (default: the pod's own)") + kubeconfig = fs.String("kubeconfig", os.Getenv("FANCYDNS_KUBECONFIG"), "kubeconfig (default: in-cluster, then $KUBECONFIG, then ~/.kube/config)") + kubeContext = fs.String("kube-context", os.Getenv("FANCYDNS_KUBE_CONTEXT"), "kubeconfig context") + zonesCM = fs.String("zones-configmap", "fancy-dns-zones", "ConfigMap holding the zone--.zone files (\"\" to skip)") + rpzCM = fs.String("rpz-configmap", "fancy-dns-rpz", "ConfigMap holding the address-overrides RPZ (\"\" to skip)") + rpzKey = fs.String("rpz-key", "address-overrides.rpz", "key of the RPZ in that ConfigMap") + zonesDir = fs.String("zones-dir", "", "read zone files from this directory (.zone) instead of the ConfigMap") + rpzFile = fs.String("rpz-file", "", "read the address-overrides RPZ from this file instead of the ConfigMap") + cfgPath = fs.String("config", "", "the resolver's config.yaml, to convert its URL lists, inline entries and ACL groups") + apply = fs.Bool("apply", false, "create the objects in the cluster (default: print the manifests)") + ) + if err := fs.Parse(args); err != nil { + return err + } + + in := migrate.Inputs{Namespace: *ns, ZoneFiles: map[string]string{}} + if in.Namespace == "" { + in.Namespace = kubesource.Namespace() + } + if in.Namespace == "" { + return errors.New("-kube-namespace is required when not running in a pod") + } + + needCluster := *apply || (*zonesDir == "" && *zonesCM != "") || (*rpzFile == "" && *rpzCM != "") + var client dynamic.Interface + if needCluster { + var err error + client, err = newClient(kubesource.ClientOptions{Kubeconfig: *kubeconfig, Context: *kubeContext}) + if err != nil { + return err + } + } + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + var notes []string + if *zonesDir != "" { + files, err := readZoneDir(*zonesDir) + if err != nil { + return err + } + in.ZoneFiles = files + } else if *zonesCM != "" { + data, found, err := configMapData(ctx, client, in.Namespace, *zonesCM) + if err != nil { + return err + } + if !found { + notes = append(notes, fmt.Sprintf("ConfigMap %s/%s does not exist: no zone files to convert", in.Namespace, *zonesCM)) + } + for k, v := range data { + if strings.HasPrefix(k, "zone--") && strings.HasSuffix(k, ".zone") { + in.ZoneFiles[strings.TrimSuffix(strings.TrimPrefix(k, "zone--"), ".zone")] = v + } + } + } + if *rpzFile != "" { + b, err := os.ReadFile(*rpzFile) + if err != nil { + return err + } + in.RPZ = string(b) + } else if *rpzCM != "" { + data, found, err := configMapData(ctx, client, in.Namespace, *rpzCM) + if err != nil { + return err + } + if !found { + notes = append(notes, fmt.Sprintf("ConfigMap %s/%s does not exist: no address overrides to convert", in.Namespace, *rpzCM)) + } + in.RPZ = data[*rpzKey] + } + if *cfgPath != "" { + cfg, err := config.Load(*cfgPath) + if err != nil { + return fmt.Errorf("%w\n(the config expands ${VAR} references; set any variable it names, its value is not used)", err) + } + in.Config = cfg + } + + res := migrate.Convert(in) + for _, n := range notes { + fmt.Fprintln(stderr, "note:", n) + } + for _, n := range res.Notes { + fmt.Fprintln(stderr, "converted:", n) + } + for _, s := range res.Skipped { + fmt.Fprintln(stderr, "NOT converted:", s) + } + if len(res.Objects) == 0 { + fmt.Fprintln(stderr, "nothing to migrate") + return nil + } + if !*apply { + out, err := migrate.YAML(res.Objects) + if err != nil { + return err + } + _, err = stdout.Write(out) + fmt.Fprintf(stderr, "\n%d objects printed. Review them, then run again with -apply, or `kubectl apply -f -`.\n", len(res.Objects)) + return err + } + ed := kubeedit.New(client, in.Namespace) + for _, o := range res.Objects { + if err := ed.Apply(ctx, o, "fancy-dns-migrate"); err != nil { + return fmt.Errorf("apply %s/%s: %w", o.GetKind(), o.GetName(), err) + } + fmt.Fprintf(stderr, "applied %s/%s\n", o.GetKind(), o.GetName()) + } + fmt.Fprintf(stderr, "\n%d objects applied in namespace %s. The old ConfigMaps were not touched.\n", len(res.Objects), in.Namespace) + return nil +} + +func readZoneDir(dir string) (map[string]string, error) { + paths, err := filepath.Glob(filepath.Join(dir, "*.zone")) + if err != nil { + return nil, err + } + out := map[string]string{} + for _, p := range paths { + b, err := os.ReadFile(p) + if err != nil { + return nil, err + } + name := strings.TrimSuffix(filepath.Base(p), ".zone") + out[strings.TrimPrefix(name, "zone--")] = string(b) + } + return out, nil +} + +// configMapData returns a ConfigMap's data, and whether the ConfigMap exists. +func configMapData(ctx context.Context, c dynamic.Interface, ns, name string) (map[string]string, bool, error) { + u, err := c.Resource(configMaps).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil, false, nil + } + if err != nil { + return nil, false, fmt.Errorf("read ConfigMap %s/%s: %w", ns, name, err) + } + raw, _ := u.Object["data"].(map[string]any) + out := make(map[string]string, len(raw)) + for k, v := range raw { + if s, ok := v.(string); ok { + out[k] = s + } + } + return out, true, nil +} diff --git a/cmd/fancy-dns-console/migrate_test.go b/cmd/fancy-dns-console/migrate_test.go new file mode 100644 index 0000000..3eae394 --- /dev/null +++ b/cmd/fancy-dns-console/migrate_test.go @@ -0,0 +1,140 @@ +package main + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + dynfake "k8s.io/client-go/dynamic/fake" + + "git.k8s-git-k6n.net/mats/fancy-dns/internal/kubesource" +) + +const rpz = "$TTL 60\ntornberg.me A 10.10.3.10\nk6n.net A 10.10.3.10\n" + +const zone = `$TTL 300 +@ IN SOA ns1.home.example.com. hostmaster.home.example.com. 1 3600 600 604800 300 +@ IN NS ns1.home.example.com. +ns1 IN A 10.0.0.53 +nas IN A 10.0.0.20 +` + +func configMap(ns, name string, data map[string]any) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", "kind": "ConfigMap", + "metadata": map[string]any{"name": name, "namespace": ns}, + "data": data, + }} +} + +func fakeCluster(objs ...runtime.Object) *dynfake.FakeDynamicClient { + return dynfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), + map[schema.GroupVersionResource]string{ + configMaps: "ConfigMapList", + kubesource.ZoneGVR: "DNSZoneList", + kubesource.GroupGVR: "ClientGroupList", + kubesource.ListGVR: "BlockListList", + kubesource.OverrideGVR: "AddressOverrideList", + kubesource.RuleGVR: "DomainRuleList", + }, objs...) +} + +func useClient(t *testing.T, c dynamic.Interface) { + t.Helper() + old := newClient + newClient = func(kubesource.ClientOptions) (dynamic.Interface, error) { return c, nil } + t.Cleanup(func() { newClient = old }) +} + +// TestMigrateFromTheClusterThenApply reads the two ConfigMaps the old console +// wrote, prints manifests without touching anything, and then applies them. +func TestMigrateFromTheClusterThenApply(t *testing.T) { + c := fakeCluster( + configMap("dns", "fancy-dns-zones", map[string]any{"zone--home.example.com.zone": zone, "unrelated": "x"}), + configMap("dns", "fancy-dns-rpz", map[string]any{"address-overrides.rpz": rpz}), + ) + useClient(t, c) + + var out, errb bytes.Buffer + if err := migrateMain([]string{"-kube-namespace", "dns"}, &out, &errb); err != nil { + t.Fatalf("dry run: %v\n%s", err, errb.String()) + } + for _, want := range []string{"kind: DNSZone", "kind: AddressOverride", "domain: tornberg.me", "zone: home.example.com"} { + if !strings.Contains(out.String(), want) { + t.Errorf("manifest lacks %q:\n%s", want, out.String()) + } + } + if l, _ := c.Resource(kubesource.OverrideGVR).Namespace("dns").List(context.Background(), metav1.ListOptions{}); len(l.Items) != 0 { + t.Fatal("the dry run wrote objects") + } + + out.Reset() + errb.Reset() + if err := migrateMain([]string{"-kube-namespace", "dns", "-apply"}, &out, &errb); err != nil { + t.Fatalf("apply: %v\n%s", err, errb.String()) + } + if l, _ := c.Resource(kubesource.OverrideGVR).Namespace("dns").List(context.Background(), metav1.ListOptions{}); len(l.Items) != 2 { + t.Fatalf("%d overrides applied, want 2", len(l.Items)) + } + if l, _ := c.Resource(kubesource.ZoneGVR).Namespace("dns").List(context.Background(), metav1.ListOptions{}); len(l.Items) != 1 { + t.Fatalf("%d zones applied, want 1", len(l.Items)) + } + if !strings.Contains(errb.String(), "old ConfigMaps were not touched") { + t.Errorf("no reassurance in the report:\n%s", errb.String()) + } + // The ConfigMaps are exactly as they were. + cm, err := c.Resource(configMaps).Namespace("dns").Get(context.Background(), "fancy-dns-rpz", metav1.GetOptions{}) + if err != nil || cm.Object["data"].(map[string]any)["address-overrides.rpz"] != rpz { + t.Fatalf("the source ConfigMap changed: %v %v", cm, err) + } + // Running it again changes nothing (create-or-update). + if err := migrateMain([]string{"-kube-namespace", "dns", "-apply"}, &out, &errb); err != nil { + t.Fatalf("second apply: %v", err) + } +} + +func TestMigrateOfflineFromFiles(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "home.example.com.zone"), []byte(zone), 0o600); err != nil { + t.Fatal(err) + } + rpzPath := filepath.Join(t.TempDir(), "a.rpz") + if err := os.WriteFile(rpzPath, []byte(rpz), 0o600); err != nil { + t.Fatal(err) + } + // No cluster client is built when nothing is read from one. + old := newClient + newClient = func(kubesource.ClientOptions) (dynamic.Interface, error) { + t.Fatal("a cluster client was built for an offline run") + return nil, nil + } + defer func() { newClient = old }() + + var out, errb bytes.Buffer + err := migrateMain([]string{"-kube-namespace", "dns", "-zones-dir", dir, "-rpz-file", rpzPath, "-zones-configmap", "", "-rpz-configmap", ""}, &out, &errb) + if err != nil { + t.Fatalf("%v\n%s", err, errb.String()) + } + if !strings.Contains(out.String(), "kind: DNSZone") || !strings.Contains(out.String(), "kind: AddressOverride") { + t.Fatalf("manifest:\n%s", out.String()) + } +} + +func TestMigrateMissingConfigMapsIsNotAnError(t *testing.T) { + useClient(t, fakeCluster()) + var out, errb bytes.Buffer + if err := migrateMain([]string{"-kube-namespace", "dns"}, &out, &errb); err != nil { + t.Fatalf("%v", err) + } + if !strings.Contains(errb.String(), "does not exist") || !strings.Contains(errb.String(), "nothing to migrate") { + t.Fatalf("report:\n%s", errb.String()) + } +} diff --git a/internal/kubeedit/apply.go b/internal/kubeedit/apply.go new file mode 100644 index 0000000..8f30c13 --- /dev/null +++ b/internal/kubeedit/apply.go @@ -0,0 +1,44 @@ +package kubeedit + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + + "git.k8s-git-k6n.net/mats/fancy-dns/internal/kubesource" +) + +// ResourceFor maps a fancy-dns kind to its resource. +func ResourceFor(kind string) (schema.GroupVersionResource, bool) { + switch kind { + case "DNSZone": + return kubesource.ZoneGVR, true + case "ClientGroup": + return kubesource.GroupGVR, true + case "BlockList": + return kubesource.ListGVR, true + case "AddressOverride": + return kubesource.OverrideGVR, true + case "DomainRule": + return kubesource.RuleGVR, true + } + return schema.GroupVersionResource{}, false +} + +// Apply creates the object or replaces its spec, whatever the kind. It is for +// tools that already hold a validated object (the migration); the console's +// own edits use the typed methods. +func (e *Editor) Apply(ctx context.Context, u *unstructured.Unstructured, actor string) error { + gvr, ok := ResourceFor(u.GetKind()) + if !ok { + return fmt.Errorf("%s is not a fancy-dns kind", u.GetKind()) + } + spec, _ := u.Object["spec"].(map[string]any) + if spec == nil { + return fmt.Errorf("%s %s has no spec", u.GetKind(), u.GetName()) + } + _, err := e.upsertRaw(ctx, gvr, u.GetKind(), u.GetName(), actor, spec, u.GetLabels()) + return err +} diff --git a/internal/kubeedit/edit.go b/internal/kubeedit/edit.go index 84e1b0f..87a278e 100644 --- a/internal/kubeedit/edit.go +++ b/internal/kubeedit/edit.go @@ -153,3 +153,37 @@ func (e *Editor) list(ctx context.Context, gvr schema.GroupVersionResource) ([]u sort.Slice(l.Items, func(i, j int) bool { return l.Items[i].GetName() < l.Items[j].GetName() }) return l.Items, nil } + +// upsertRaw is upsert for a spec that is already unstructured. +func (e *Editor) upsertRaw(ctx context.Context, gvr schema.GroupVersionResource, kind, name, actor string, spec map[string]any, labels map[string]string) (*unstructured.Unstructured, error) { + var out *unstructured.Unstructured + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + cur, err := e.res(gvr).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + obj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": kubesource.APIGroup + "/" + kubesource.Version, + "kind": kind, + "metadata": map[string]any{"name": name, "namespace": e.ns}, + "spec": spec, + }} + stamp(obj, actor) + if len(labels) > 0 { + merged := obj.GetLabels() + for k, v := range labels { + merged[k] = v + } + obj.SetLabels(merged) + } + out, err = e.res(gvr).Create(ctx, obj, metav1.CreateOptions{}) + return err + } + if err != nil { + return err + } + cur.Object["spec"] = spec + stamp(cur, actor) + out, err = e.res(gvr).Update(ctx, cur, metav1.UpdateOptions{}) + return err + }) + return out, err +} diff --git a/internal/kubeedit/overrides.go b/internal/kubeedit/overrides.go index b23dd84..9fe1dc1 100644 --- a/internal/kubeedit/overrides.go +++ b/internal/kubeedit/overrides.go @@ -15,6 +15,9 @@ import ( // objectName turns a DNS name into a Kubernetes object name. Most names are // already one; a name that is not (an underscore, say) is cleaned and given a // short hash so two different names cannot collide. +// ObjectName is the Kubernetes object name for a DNS name. +func ObjectName(domain string) string { return objectName(domain) } + func objectName(domain string) string { var b strings.Builder changed := false diff --git a/internal/localzone/render.go b/internal/localzone/render.go index 751b950..68313ba 100644 --- a/internal/localzone/render.go +++ b/internal/localzone/render.go @@ -3,6 +3,8 @@ package localzone import ( "fmt" "strings" + + "git.k8s-git-k6n.net/mats/fancy-dns/internal/zonefile" ) // RenderZone renders a complete zone from declared records, for sources that @@ -76,3 +78,35 @@ func CanonicalRecord(origin string, in Record) (Record, error) { rec.TTL = in.TTL // 0 still means "the zone's default" return rec, nil } + +// ZoneFile is a master file read back into the form a DNSZone holds it in. +type ZoneFile struct { + // TTL is the file's $TTL, or the default when it has none. + TTL uint32 + // Soa is the SOA rdata ("ns mbox serial refresh retry expire minimum"), or + // "" when the file has none. + Soa string + Records []Record +} + +// ParseZoneFile reads an RFC 1035 master file for the zone origin. Records come +// back canonical (absolute owner, uppercase type, rdata as the parser prints +// it) and without the SOA, which is returned separately. It is the reading half +// of a migration from master files to DNSZone objects. +func ParseZoneFile(origin, text string) (ZoneFile, error) { + origin, err := NormalizeOrigin(origin) + if err != nil { + return ZoneFile{}, fmt.Errorf("zone name: %w", err) + } + z, err := zonefile.Parse(strings.NewReader(text), zonefile.Options{Origin: origin, File: origin + "zone"}) + if err != nil { + return ZoneFile{}, err + } + f := fileStateFromZone(z, origin) + out := ZoneFile{TTL: f.ttl, Records: f.recs} + if f.soa != nil { + out.Soa = fmt.Sprintf("%s %s %d %d %d %d %d", f.soa.Ns, f.soa.Mbox, + f.soa.Serial, f.soa.Refresh, f.soa.Retry, f.soa.Expire, f.soa.Minttl) + } + return out, nil +} diff --git a/internal/migrate/migrate.go b/internal/migrate/migrate.go new file mode 100644 index 0000000..419c42d --- /dev/null +++ b/internal/migrate/migrate.go @@ -0,0 +1,297 @@ +// Package migrate turns what a ConfigMap-and-YAML deployment held into fancy-dns +// objects: zone master files become DNSZone, the address-overrides RPZ becomes +// AddressOverride, URL blocklists become BlockList, inline allow/block entries +// become DomainRule, and ACL groups become ClientGroup. +// +// It converts, validates with the same code the resolver uses to read the +// objects, and reports what it could not carry over. It never deletes anything +// and never guesses: an input it cannot represent faithfully is listed and left +// where it is. +package migrate + +import ( + "fmt" + "reflect" + "sort" + "strings" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/yaml" + + "git.k8s-git-k6n.net/mats/fancy-dns/internal/config" + "git.k8s-git-k6n.net/mats/fancy-dns/internal/kubeedit" + "git.k8s-git-k6n.net/mats/fancy-dns/internal/kubesource" + "git.k8s-git-k6n.net/mats/fancy-dns/internal/localzone" +) + +// Inputs is everything to convert. Any part may be empty. +type Inputs struct { + // ZoneFiles maps an apex ("home.example.com") to its master file. + ZoneFiles map[string]string + // RPZ is the address-overrides RPZ text. + RPZ string + // Config is the resolver's YAML configuration. + Config *config.Config + // Namespace is where the objects will live. + Namespace string +} + +// Result is the converted objects and the account of what was not converted. +type Result struct { + Objects []*unstructured.Unstructured + // Notes say what was converted and what to remove from the config afterwards. + Notes []string + // Skipped are inputs that could not be represented, each with the reason. + Skipped []string +} + +const managedBy = "fancy-dns-migrate" + +// Convert does the whole conversion. +func Convert(in Inputs) Result { + var r Result + r.zones(in) + r.overrides(in) + if in.Config != nil { + r.config(in) + } + sort.SliceStable(r.Objects, func(i, j int) bool { + a, b := r.Objects[i], r.Objects[j] + if a.GetKind() != b.GetKind() { + return a.GetKind() < b.GetKind() + } + return a.GetName() < b.GetName() + }) + return r +} + +func (r *Result) add(kind, name, ns string, spec any) { + m, err := toMap(spec) + if err != nil { + r.Skipped = append(r.Skipped, fmt.Sprintf("%s %s: %v", kind, name, err)) + return + } + u := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": kubesource.APIGroup + "/" + kubesource.Version, + "kind": kind, + "metadata": map[string]any{ + "name": name, "namespace": ns, + "labels": map[string]any{"app.kubernetes.io/managed-by": managedBy}, + }, + "spec": m, + }} + r.Objects = append(r.Objects, u) +} + +func (r *Result) skip(format string, args ...any) { + r.Skipped = append(r.Skipped, fmt.Sprintf(format, args...)) +} +func (r *Result) note(format string, args ...any) { + r.Notes = append(r.Notes, fmt.Sprintf(format, args...)) +} + +// zones converts master files. +func (r *Result) zones(in Inputs) { + apexes := make([]string, 0, len(in.ZoneFiles)) + for a := range in.ZoneFiles { + apexes = append(apexes, a) + } + sort.Strings(apexes) + for _, name := range apexes { + apex, err := localzone.NormalizeOrigin(name) + if err != nil { + r.skip("zone file %q: %v", name, err) + continue + } + zf, err := localzone.ParseZoneFile(apex, in.ZoneFiles[name]) + if err != nil { + r.skip("zone file %q: does not parse: %v", name, err) + continue + } + spec := kubesource.DNSZoneSpec{Zone: strings.TrimSuffix(apex, "."), TTL: zf.TTL, SOA: zf.Soa} + for _, rec := range zf.Records { + rel := "@" + if rec.Name != apex { + rel = strings.TrimSuffix(rec.Name, "."+apex) + } + ttl := rec.TTL + if ttl == zf.TTL { + ttl = 0 // inherits the zone's, as it did in the file + } + spec.Records = append(spec.Records, kubesource.ZoneRecord{Name: rel, Type: rec.Type, TTL: ttl, Value: rec.Value}) + } + if _, err := kubesource.CheckZone(spec); err != nil { + r.skip("zone %s: not representable as a DNSZone (%v); it stays a master file", apex, err) + continue + } + objName := strings.TrimSuffix(apex, ".") + r.add("DNSZone", objName, in.Namespace, spec) + r.note("zone %s: %d records -> DNSZone/%s", apex, len(spec.Records), objName) + } +} + +// overrides converts the address-overrides RPZ. +func (r *Result) overrides(in Inputs) { + if strings.TrimSpace(in.RPZ) == "" { + return + } + byName := map[string][]string{} + for i, line := range strings.Split(in.RPZ, "\n") { + if j := strings.IndexAny(line, ";#"); j >= 0 { + line = line[:j] + } + f := strings.Fields(line) + if len(f) == 0 || strings.HasPrefix(f[0], "$") { + continue + } + // "name [ttl] [IN] A|AAAA address" + rest := f[1:] + for len(rest) > 0 && (strings.EqualFold(rest[0], "IN") || isNumber(rest[0])) { + rest = rest[1:] + } + if len(rest) < 2 || !(strings.EqualFold(rest[0], "A") || strings.EqualFold(rest[0], "AAAA")) { + r.skip("address overrides line %d %q: only A and AAAA overrides are converted", i+1, strings.TrimSpace(line)) + continue + } + name := strings.TrimSuffix(strings.ToLower(f[0]), ".") + byName[name] = append(byName[name], rest[1]) + } + names := make([]string, 0, len(byName)) + for n := range byName { + names = append(names, n) + } + sort.Strings(names) + for _, n := range names { + spec := kubesource.AddressOverrideSpec{Domain: n, Addresses: byName[n]} + o, err := kubesource.CheckOverride(spec) + if err != nil { + r.skip("address override %s: %v", n, err) + continue + } + r.add("AddressOverride", kubeedit.ObjectName(o.Domain), in.Namespace, + kubesource.AddressOverrideSpec{Domain: o.Domain, Addresses: o.Addresses}) + } + if len(names) > 0 { + r.note("address overrides: %d names -> AddressOverride objects", len(names)) + } +} + +func isNumber(s string) bool { + for _, c := range s { + if c < '0' || c > '9' { + return false + } + } + return s != "" +} + +// config converts the parts of the resolver's YAML that cluster objects replace. +func (r *Result) config(in Inputs) { + cfg := in.Config + for _, l := range cfg.Block.Lists { + switch { + case l.URL == "" && strings.EqualFold(l.Format, "rpz") && strings.Contains(l.Path, "address-overrides"): + r.note("list %q is the address-overrides file; the AddressOverride objects replace it, remove it from blocklist.lists", l.Name) + case l.URL == "": + r.skip("list %q: it is a file on the pod (%s), which a cluster object cannot point at; host it at a URL or keep it in the config", l.Name, l.Path) + case !strings.HasPrefix(l.URL, "http://") && !strings.HasPrefix(l.URL, "https://"): + r.skip("list %q: url %q is not http(s)", l.Name, l.URL) + default: + spec := kubesource.BlockListSpec{ + URL: l.URL, Format: l.Format, Mirror: l.Mirror, Disabled: !l.Enabled, + MaxBytes: l.MaxBytes, Categories: l.Categories, Comment: l.Comment, + InsecureSkipTLSVerify: l.InsecureSkipTLSVerify, + } + if l.Refresh > 0 { + spec.Refresh = time.Duration(l.Refresh).String() + } + // The config loader fills these in; only a value the operator chose + // is worth writing down. + if l.Timeout > 0 && l.Timeout != cfg.Server.Timeouts.List { + spec.Timeout = time.Duration(l.Timeout).String() + } + if spec.MaxBytes == 512<<20 { + spec.MaxBytes = 0 + } + if _, err := kubesource.CheckList(l.Name, spec); err != nil { + r.skip("list %q: %v", l.Name, err) + continue + } + if name := kubeedit.ObjectName(l.Name); name != l.Name { + r.note("list %q: object name will be %q (not a legal Kubernetes name as written)", l.Name, name) + r.add("BlockList", name, in.Namespace, spec) + } else { + r.add("BlockList", l.Name, in.Namespace, spec) + } + r.note("list %q -> BlockList; remove it from blocklist.lists", l.Name) + } + } + for _, rules := range []struct { + action string + entries []string + }{{"block", cfg.Block.BlockEntries}, {"allow", cfg.Block.AllowEntries}} { + if len(rules.entries) == 0 { + continue + } + var ok []string + for _, e := range rules.entries { + e = strings.TrimSpace(e) + if e == "" || strings.HasPrefix(e, "#") { + continue + } + if _, err := kubesource.CheckRules(kubesource.DomainRuleSpec{Action: rules.action, Names: []string{e}}); err != nil { + r.skip("%s entry %q: %v", rules.action, e, err) + continue + } + ok = append(ok, e) + } + if len(ok) == 0 { + continue + } + r.add("DomainRule", "inline-"+rules.action, in.Namespace, kubesource.DomainRuleSpec{ + Action: rules.action, Names: ok, Comment: "Migrated from blocklist." + rules.action + "_entries", + }) + r.note("%d %s entries -> DomainRule/inline-%s; remove blocklist.%s_entries", len(ok), rules.action, rules.action, rules.action) + } + for i, g := range cfg.ACL.Groups { + spec := kubeedit.SpecFromGroup(g, int32(i+1)*10) + if _, err := kubesource.CheckGroup(g.Name, spec); err != nil { + r.skip("acl group %q: %v", g.Name, err) + continue + } + if name := g.Name; kubeedit.ObjectName(name) != name { + r.skip("acl group %q: not a legal Kubernetes object name; rename it", g.Name) + continue + } + r.add("ClientGroup", g.Name, in.Namespace, spec) + } + if len(cfg.ACL.Groups) > 0 { + r.note("%d acl groups -> ClientGroup; remove acl.groups from the config (default_policy stays)", len(cfg.ACL.Groups)) + } +} + +// YAML renders the objects as a multi-document manifest. +func YAML(objs []*unstructured.Unstructured) ([]byte, error) { + var b strings.Builder + for i, o := range objs { + out, err := yaml.Marshal(o.Object) + if err != nil { + return nil, err + } + if i > 0 { + b.WriteString("---\n") + } + b.Write(out) + } + return []byte(b.String()), nil +} + +// toMap converts a spec struct to unstructured form. ToUnstructured wants a +// pointer to the concrete type. +func toMap(spec any) (map[string]any, error) { + v := reflect.New(reflect.TypeOf(spec)) + v.Elem().Set(reflect.ValueOf(spec)) + return runtime.DefaultUnstructuredConverter.ToUnstructured(v.Interface()) +} diff --git a/internal/migrate/migrate_test.go b/internal/migrate/migrate_test.go new file mode 100644 index 0000000..bd0dde8 --- /dev/null +++ b/internal/migrate/migrate_test.go @@ -0,0 +1,168 @@ +package migrate + +import ( + "context" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynfake "k8s.io/client-go/dynamic/fake" + + "git.k8s-git-k6n.net/mats/fancy-dns/internal/config" + "git.k8s-git-k6n.net/mats/fancy-dns/internal/kubeedit" + "git.k8s-git-k6n.net/mats/fancy-dns/internal/kubesource" +) + +// The data the live deployment actually has. +const liveRPZ = `$TTL 60 +tornberg.me A 10.10.3.10 +knatofs.se A 10.10.3.10 +k6n.net A 10.10.3.10 +slask.local A 10.10.3.10 +iamu.dev A 10.10.3.10 +` + +const homeZone = `$TTL 300 +$ORIGIN home.example.com. +@ IN SOA ns1.home.example.com. hostmaster.home.example.com. 2024010101 3600 600 604800 300 +@ IN NS ns1.home.example.com. +ns1 IN A 10.0.0.53 +nas 60 IN A 10.0.0.20 +mail IN MX 10 mx.home.example.com. +txt IN TXT "v=spf1 -all" +*.apps IN A 10.0.0.30 +` + +func liveConfig(t *testing.T) *config.Config { + t.Helper() + cfg := config.Default() + cfg.Block.Lists = []config.ListSource{ + {Name: "edge-router-x-address-overrides", Format: "rpz", Path: "/etc/fancy-dns/rpz/address-overrides.rpz", Enabled: true}, + {Name: "haas", Format: "plain", URL: "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts", + Path: "/var/lib/fancy-dns/lists/hosts", Enabled: true, Refresh: config.Duration(24 * time.Hour), Categories: []string{"ads", "tracking"}}, + {Name: "local file", Format: "hosts", Path: "/etc/x/hosts", Enabled: true}, + } + cfg.Block.BlockEntries = []string{"ads.example.com", "*.tracker.example", "bad name"} + cfg.Block.AllowEntries = []string{"good.example"} + cfg.ACL.Groups = []config.Group{ + {Name: "kids", Policy: "block_only", Match: config.Match{CIDRs: []string{"10.0.20.0/24"}}}, + {Name: "Bad Name", Policy: "allow", Match: config.Match{CIDRs: []string{"10.0.30.0/24"}}}, + } + return cfg +} + +func TestConvertEverythingTheDeploymentHolds(t *testing.T) { + res := Convert(Inputs{ + ZoneFiles: map[string]string{"home.example.com": homeZone, "broken.example": "not a zone @@@ ("}, + RPZ: liveRPZ + "bad.test CNAME x.example.\n", + Config: liveConfig(t), + Namespace: "dns", + }) + kinds := map[string]int{} + for _, o := range res.Objects { + kinds[o.GetKind()]++ + if o.GetNamespace() != "dns" || o.GetLabels()["app.kubernetes.io/managed-by"] != managedBy { + t.Errorf("%s/%s: namespace/labels = %q %v", o.GetKind(), o.GetName(), o.GetNamespace(), o.GetLabels()) + } + } + want := map[string]int{"DNSZone": 1, "AddressOverride": 5, "BlockList": 1, "DomainRule": 2, "ClientGroup": 1} + for k, n := range want { + if kinds[k] != n { + t.Errorf("%s: %d objects, want %d (%v)", k, kinds[k], n, kinds) + } + } + skipped := strings.Join(res.Skipped, "\n") + for _, must := range []string{"broken.example", "bad.test", "local file", "bad name", "Bad Name"} { + if !strings.Contains(skipped, must) { + t.Errorf("%q should have been reported as skipped:\n%s", must, skipped) + } + } + notes := strings.Join(res.Notes, "\n") + for _, must := range []string{"edge-router-x-address-overrides", "haas", "blocklist.block_entries"} { + if !strings.Contains(notes, must) { + t.Errorf("notes should mention %q:\n%s", must, notes) + } + } +} + +// TestConvertedObjectsAreWhatTheResolverAccepts applies the result to a cluster +// and reads it back through the resolver's own source: nothing converted may be +// rejected there, and the data must come out the same. +func TestConvertedObjectsAreWhatTheResolverAccepts(t *testing.T) { + res := Convert(Inputs{ + ZoneFiles: map[string]string{"home.example.com": homeZone}, + RPZ: liveRPZ, + Config: liveConfig(t), + Namespace: "dns", + }) + client := dynfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), + map[schema.GroupVersionResource]string{ + kubesource.ZoneGVR: "DNSZoneList", kubesource.GroupGVR: "ClientGroupList", kubesource.ListGVR: "BlockListList", + kubesource.OverrideGVR: "AddressOverrideList", kubesource.RuleGVR: "DomainRuleList", + }) + ed := kubeedit.New(client, "dns") + for _, o := range res.Objects { + if err := ed.Apply(context.Background(), o, "migrate"); err != nil { + t.Fatalf("apply %s/%s: %v", o.GetKind(), o.GetName(), err) + } + } + + src, err := kubesource.New(kubesource.Options{Client: client, Namespace: "dns", Debounce: 10 * time.Millisecond}) + if err != nil { + t.Fatal(err) + } + got := make(chan *kubesource.Snapshot, 4) + src.OnChange(func(s *kubesource.Snapshot) { got <- s }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = src.Run(ctx) }() + var sn *kubesource.Snapshot + select { + case sn = <-got: + case <-time.After(5 * time.Second): + t.Fatal("no snapshot") + } + + if len(sn.Problems) != 0 { + t.Fatalf("the resolver rejected migrated objects: %+v", sn.Problems) + } + if len(sn.Overrides) != 5 || sn.Overrides[0].Domain != "iamu.dev" || sn.Overrides[0].Addresses[0] != "10.10.3.10" { + t.Errorf("overrides = %+v", sn.Overrides) + } + if len(sn.Groups) != 1 || sn.Groups[0].Name != "kids" || sn.Groups[0].Policy != "block_only" { + t.Errorf("groups = %+v", sn.Groups) + } + if len(sn.Lists) != 1 || sn.Lists[0].Name != "haas" || !sn.Lists[0].Enabled || sn.Lists[0].Path != "" { + t.Errorf("lists = %+v", sn.Lists) + } + if strings.Join(sn.BlockNames, ",") != "ads.example.com,*.tracker.example" || strings.Join(sn.AllowNames, ",") != "good.example" { + t.Errorf("rules = %v / %v", sn.BlockNames, sn.AllowNames) + } + text := sn.Zones["home.example.com."] + if text == "" { + t.Fatalf("zones = %v", sn.Zones) + } + for _, line := range []string{"ns1", "10.0.0.20", "mx.home.example.com.", "v=spf1 -all", "*.apps", "IN SOA"} { + if !strings.Contains(text, line) { + t.Errorf("rendered zone lacks %q:\n%s", line, text) + } + } + // nas had its own TTL (60); the rest inherit the zone's 300. + if !strings.Contains(text, "nas") || !strings.Contains(text, " 1m IN A ") && !strings.Contains(text, " 60 IN A ") { + t.Errorf("nas lost its TTL:\n%s", text) + } +} + +func TestYAMLRoundTrips(t *testing.T) { + res := Convert(Inputs{RPZ: liveRPZ, Namespace: "dns"}) + out, err := YAML(res.Objects) + if err != nil { + t.Fatal(err) + } + s := string(out) + if strings.Count(s, "---") != 4 || !strings.Contains(s, "kind: AddressOverride") || !strings.Contains(s, "namespace: dns") { + t.Fatalf("YAML:\n%s", s) + } +}