Add a read-only Kubernetes source for zones, client groups and blocklists

The runtime can now take ClientGroup, BlockList and DNSZone objects
(fancydns.k6n.net/v1alpha1) from a cluster with -kube, from a pod's service
account or a kubeconfig alike. internal/kubesource watches them with dynamic
informers, converts and validates each object on its own, and publishes an
immutable snapshot. A bad object is skipped and keeps its last good version,
so one broken edit cannot remove a running policy; an invalid merged config is
dropped whole. The last snapshot is saved so a restart without apiserver access
still serves.

config.Store gains an overlay: Base() is what the YAML says, Get() is base plus
cluster objects. Everything that writes the config back (mutateConfig,
SetAdminAddr, the settings editor) now starts from Base(), otherwise cluster
objects would be copied into the file and shadow their own objects. YAML wins
every name collision. putACL and toggleBlocklist refuse to edit cluster-owned
objects. BlockList has no path field and refuses file:// URLs so a namespaced
object cannot pick files the runtime reads or writes. DNSZone is rendered
through the zone writer's own prove-by-reparse step, with the SOA serial taken
from metadata.generation so every replica renders identical bytes.

qlog: Flush reset the pending slice to [:0], so entries logged while it was
still storing the batch were appended over the same backing array. The race
detector found it through the new Kubernetes integration test. pending is now
set to nil, with a regression test that fails without the change.

CRD manifests, RBAC and examples are in deploy/k8s/crds; tests pin the schemas
to the Go spec structs so the two cannot drift.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
matsandClaude Sonnet 5.5 committed 2026-10-16 08:35:36 +02:00
1 parent 340b4eaf80
commit 4d38945d94
27 files changed
+2250 -19

No files matched your search

+66
View File
@@ -0,0 +1,66 @@
package kubesource
import (
"fmt"
"os"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
)
// ClientOptions say where the cluster is.
type ClientOptions struct {
// Kubeconfig is a kubeconfig path. Empty means: the in-cluster service
// account if there is one, else the standard loading rules ($KUBECONFIG,
// then ~/.kube/config). The same binary therefore runs in a pod and on a
// workstation with no flag at all.
Kubeconfig string
// Context selects a kubeconfig context; empty is the file's current one.
Context string
}
// Namespace returns the namespace to watch when none was configured: the
// pod's own when running in-cluster, else "" (every namespace the credentials
// can list).
func Namespace() string {
b, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
if err != nil {
return ""
}
return string(b)
}
// NewClient builds a dynamic client and reports which source it used.
func NewClient(o ClientOptions) (dynamic.Interface, string, error) {
cfg, from, err := restConfig(o)
if err != nil {
return nil, "", err
}
// The runtime lists and watches a handful of resources; the defaults
// (5 QPS) are fine for that, but a hung apiserver must not hang us.
cfg.UserAgent = "fancy-dns"
c, err := dynamic.NewForConfig(cfg)
if err != nil {
return nil, "", fmt.Errorf("kubernetes client: %w", err)
}
return c, from, nil
}
func restConfig(o ClientOptions) (*rest.Config, string, error) {
if o.Kubeconfig == "" && o.Context == "" {
if cfg, err := rest.InClusterConfig(); err == nil {
return cfg, "in-cluster service account", nil
}
}
rules := clientcmd.NewDefaultClientConfigLoadingRules()
if o.Kubeconfig != "" {
rules.ExplicitPath = o.Kubeconfig
}
cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
rules, &clientcmd.ConfigOverrides{CurrentContext: o.Context}).ClientConfig()
if err != nil {
return nil, "", fmt.Errorf("no usable Kubernetes credentials (in-cluster or kubeconfig): %w", err)
}
return cfg, "kubeconfig", nil
}
+191
View File
@@ -0,0 +1,191 @@
package kubesource
import (
"fmt"
"net"
"strings"
"time"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
"git.k8s-git-k6n.net/mats/fancy-dns/internal/localzone"
)
// maxZoneRecords bounds one DNSZone. An object is capped near 1.5 MiB by etcd
// anyway; this keeps a hostile or runaway one from costing a render.
const maxZoneRecords = 5000
// group is a converted ClientGroup with its ordering key.
type group struct {
cfg config.Group
priority int32
}
// zone is a converted DNSZone: the apex and the rendered master file.
type zone struct {
apex string
text string
}
func spec(u *unstructured.Unstructured, into any) error {
raw, ok := u.Object["spec"].(map[string]any)
if !ok {
return fmt.Errorf("the object has no spec")
}
return runtime.DefaultUnstructuredConverter.FromUnstructured(raw, into)
}
func convertGroup(u *unstructured.Unstructured) (group, error) {
var s ClientGroupSpec
if err := spec(u, &s); err != nil {
return group{}, err
}
g := config.Group{
Name: u.GetName(),
Comment: s.Comment,
Match: s.Match,
Policy: s.Policy,
AllowDomains: s.AllowDomains,
DenyDomains: s.DenyDomains,
SkipBlocklist: s.SkipBlocklist,
SkipCache: s.SkipCache,
RateLimit: s.RateLimit,
MaxConcurrent: s.MaxConcurrent,
LogQueries: s.LogQueries,
Upstreams: s.Upstreams,
}
switch g.Policy {
case "allow", "deny", "block_only":
default:
return group{}, fmt.Errorf("policy must be allow, deny or block_only (got %q)", g.Policy)
}
for _, cidr := range g.Match.CIDRs {
if _, _, err := net.ParseCIDR(cidr); err != nil {
return group{}, fmt.Errorf("bad CIDR %q", cidr)
}
}
// Copy before normalising: the slice belongs to the decoded object.
g.Match.PTRSuffixes = append([]string(nil), g.Match.PTRSuffixes...)
for i, s := range g.Match.PTRSuffixes {
if !strings.HasSuffix(s, ".") {
g.Match.PTRSuffixes[i] = s + "."
}
}
if !g.Match.Any && len(g.Match.CIDRs) == 0 && len(g.Match.PTRSuffixes) == 0 && len(g.Match.Tags) == 0 {
return group{}, fmt.Errorf("match selects no clients: set cidrs, ptr_suffixes, tags or any")
}
if g.Match.Any && len(g.Match.CIDRs) == 0 && len(g.Match.PTRSuffixes) == 0 && len(g.Match.Tags) == 0 {
// config.Validate refuses this shape for every group. Refusing it here
// rejects one object instead of the whole merged configuration.
return group{}, fmt.Errorf("match.any alone is not accepted; use a catch-all in the YAML acl.groups")
}
if g.RateLimit < 0 || g.MaxConcurrent < 0 {
return group{}, fmt.Errorf("rate_limit and max_concurrent must not be negative")
}
return group{cfg: g, priority: s.Priority}, nil
}
func convertList(u *unstructured.Unstructured) (config.ListSource, error) {
var s BlockListSpec
if err := spec(u, &s); err != nil {
return config.ListSource{}, err
}
l := config.ListSource{
Name: u.GetName(),
Format: s.Format,
URL: s.URL,
Mirror: s.Mirror,
Enabled: !s.Disabled,
MaxBytes: s.MaxBytes,
Categories: s.Categories,
Comment: s.Comment,
InsecureSkipTLSVerify: s.InsecureSkipTLSVerify,
}
if !isHTTP(l.URL) {
return config.ListSource{}, fmt.Errorf("url must be http(s) (got %q)", l.URL)
}
for _, m := range l.Mirror {
if !isHTTP(m) {
return config.ListSource{}, fmt.Errorf("mirror must be http(s) (got %q)", m)
}
}
switch l.Format {
case "", "auto", "hosts", "adblock", "plain", "rpz":
default:
return config.ListSource{}, fmt.Errorf("unknown format %q", l.Format)
}
var err error
if l.Refresh, err = duration(s.Refresh); err != nil {
return config.ListSource{}, fmt.Errorf("refresh: %w", err)
}
if l.Timeout, err = duration(s.Timeout); err != nil {
return config.ListSource{}, fmt.Errorf("timeout: %w", err)
}
if l.MaxBytes < 0 {
return config.ListSource{}, fmt.Errorf("max_bytes must not be negative")
}
return l, nil
}
func isHTTP(u string) bool {
return strings.HasPrefix(u, "https://") || strings.HasPrefix(u, "http://")
}
func duration(s string) (config.Duration, error) {
if s == "" {
return 0, nil
}
d, err := time.ParseDuration(s)
if err != nil {
return 0, err
}
if d < 0 {
return 0, fmt.Errorf("must not be negative")
}
return config.Duration(d), nil
}
func convertZone(u *unstructured.Unstructured) (zone, error) {
var s DNSZoneSpec
if err := spec(u, &s); err != nil {
return zone{}, err
}
if s.Zone == "" {
return zone{}, fmt.Errorf("spec.zone is required")
}
if len(s.Records) > maxZoneRecords {
return zone{}, fmt.Errorf("%d records exceeds the limit of %d", len(s.Records), maxZoneRecords)
}
apex, err := localzone.NormalizeOrigin(s.Zone)
if err != nil {
return zone{}, fmt.Errorf("spec.zone: %w", err)
}
var ns []string
recs := make([]localzone.Record, 0, len(s.Records))
for _, r := range s.Records {
name := r.Name
if name == "@" {
name = apex
}
recs = append(recs, localzone.Record{Name: name, Type: r.Type, TTL: r.TTL, Value: r.Value})
if strings.EqualFold(r.Type, "NS") && name == apex {
ns = append(ns, r.Value)
}
}
if s.SOA == "" && len(ns) == 0 {
return zone{}, fmt.Errorf("a zone needs an apex NS record (name \"@\") or an explicit soa")
}
// The generation changes exactly when the spec does, so every replica
// renders the same bytes for the same object version.
serial := uint32(u.GetGeneration())
if serial == 0 {
serial = 1
}
text, err := localzone.RenderZone(apex, localzone.ZoneDefaults{TTL: s.TTL, Soa: s.SOA}, recs, serial)
if err != nil {
return zone{}, err
}
return zone{apex: apex, text: text}, nil
}
+44
View File
@@ -0,0 +1,44 @@
package kubesource
import (
"os"
"path/filepath"
"sort"
"strings"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
)
type schemaGVR = schema.GroupVersionResource
var (
metav1UpdateOptions = metav1.UpdateOptions{}
metav1DeleteOptions = metav1.DeleteOptions{}
)
func metav1Time(sec int64) metav1.Time { return metav1.NewTime(time.Unix(sec, 0)) }
func writeTemp(t *testing.T, z zone) []string {
t.Helper()
dir := t.TempDir()
if _, err := SyncZoneDir(dir, map[string]string{z.apex: z.text}); err != nil {
t.Fatal(err)
}
return []string{dir}
}
func fileNames(t *testing.T, dir string) string {
t.Helper()
m, _ := filepath.Glob(filepath.Join(dir, "*"))
var out []string
for _, p := range m {
if _, err := os.Stat(p); err == nil {
out = append(out, filepath.Base(p))
}
}
sort.Strings(out)
return strings.Join(out, ",")
}
+117
View File
@@ -0,0 +1,117 @@
package kubesource
import (
"bytes"
"io"
"os"
"reflect"
"sort"
"strings"
"testing"
"gopkg.in/yaml.v3"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
func yamlDocs(t *testing.T, path string) []map[string]any {
t.Helper()
b, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var out []map[string]any
dec := yaml.NewDecoder(bytes.NewReader(b))
for {
var m map[string]any
if err := dec.Decode(&m); err == io.EOF {
break
} else if err != nil {
t.Fatalf("%s: %v", path, err)
}
if m != nil {
out = append(out, m)
}
}
return out
}
func jsonFields(typ reflect.Type) []string {
var out []string
for i := 0; i < typ.NumField(); i++ {
tag := strings.Split(typ.Field(i).Tag.Get("json"), ",")[0]
if tag != "" && tag != "-" {
out = append(out, tag)
}
}
sort.Strings(out)
return out
}
// TestCRDSchemasMatchTheGoSpecs keeps deploy/k8s/crds/crds.yaml and the spec
// structs from drifting: a field the schema lacks is pruned by the apiserver
// before the runtime ever sees it, and a schema field the struct lacks is
// accepted and silently ignored.
func TestCRDSchemasMatchTheGoSpecs(t *testing.T) {
want := map[string]reflect.Type{
"DNSZone": reflect.TypeOf(DNSZoneSpec{}),
"ClientGroup": reflect.TypeOf(ClientGroupSpec{}),
"BlockList": reflect.TypeOf(BlockListSpec{}),
}
seen := 0
for _, doc := range yamlDocs(t, "../../deploy/k8s/crds/crds.yaml") {
names := doc["spec"].(map[string]any)["names"].(map[string]any)
kind := names["kind"].(string)
typ, ok := want[kind]
if !ok {
t.Errorf("unexpected CRD kind %s", kind)
continue
}
seen++
versions := doc["spec"].(map[string]any)["versions"].([]any)
schema := versions[0].(map[string]any)["schema"].(map[string]any)["openAPIV3Schema"].(map[string]any)
props := schema["properties"].(map[string]any)["spec"].(map[string]any)["properties"].(map[string]any)
var got []string
for k := range props {
got = append(got, k)
}
sort.Strings(got)
if !reflect.DeepEqual(got, jsonFields(typ)) {
t.Errorf("%s: schema fields %v, Go fields %v", kind, got, jsonFields(typ))
}
if plural := names["plural"].(string); plural != map[string]string{
"DNSZone": ZoneGVR.Resource, "ClientGroup": GroupGVR.Resource, "BlockList": ListGVR.Resource,
}[kind] {
t.Errorf("%s: plural %q does not match the GVR the runtime watches", kind, plural)
}
}
if seen != len(want) {
t.Errorf("saw %d CRDs, want %d", seen, len(want))
}
}
// TestExampleObjectsConvert makes sure the shipped examples are accepted by
// the runtime, so the first thing an operator copies is not a rejection.
func TestExampleObjectsConvert(t *testing.T) {
n := 0
for _, doc := range yamlDocs(t, "../../deploy/k8s/crds/examples.yaml") {
u := &unstructured.Unstructured{Object: doc}
var err error
switch u.GetKind() {
case "DNSZone":
_, err = convertZone(u)
case "ClientGroup":
_, err = convertGroup(u)
case "BlockList":
_, err = convertList(u)
default:
t.Fatalf("unknown kind %s", u.GetKind())
}
if err != nil {
t.Errorf("%s %s: %v", u.GetKind(), u.GetName(), err)
}
n++
}
if n != 3 {
t.Fatalf("converted %d examples, want 3", n)
}
}
+183
View File
@@ -0,0 +1,183 @@
package kubesource
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
)
func signature(s *Snapshot) string {
b, _ := json.Marshal(struct {
G []config.Group
L []config.ListSource
Z map[string]string
}{s.Groups, s.Lists, s.Zones})
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:])
}
// Merge returns base with the cluster's groups and lists added; base is not
// modified. Precedence is deliberate: whatever the YAML declares is the
// operator's and wins. YAML groups are evaluated before cluster groups, and a
// cluster list that reuses a YAML list's name is dropped. The names of what was
// dropped are returned so the caller can say so.
func Merge(base *config.Config, snap *Snapshot) (*config.Config, []string) {
if snap == nil {
return base, nil
}
next := *base
next.ACL = base.ACL
next.ACL.Groups = append([]config.Group(nil), base.ACL.Groups...)
next.Block = base.Block
next.Block.Lists = append([]config.ListSource(nil), base.Block.Lists...)
var dropped []string
groups := map[string]bool{}
for _, g := range base.ACL.Groups {
groups[g.Name] = true
}
for _, g := range snap.Groups {
if groups[g.Name] {
dropped = append(dropped, "ClientGroup "+g.Name+" (name used in the YAML)")
continue
}
next.ACL.Groups = append(next.ACL.Groups, g)
}
lists := map[string]bool{}
for _, l := range base.Block.Lists {
lists[l.Name] = true
}
for _, l := range snap.Lists {
if lists[l.Name] {
dropped = append(dropped, "BlockList "+l.Name+" (name used in the YAML)")
continue
}
// The same defaults config.applyDefaults gives a YAML list.
if l.Format == "" {
l.Format = "auto"
}
if l.Refresh == 0 {
l.Refresh = config.Duration(24 * time.Hour)
}
if l.Timeout <= 0 {
l.Timeout = base.Server.Timeouts.List
}
if l.MaxBytes <= 0 {
l.MaxBytes = 512 << 20
}
next.Block.Lists = append(next.Block.Lists, l)
}
return &next, dropped
}
// SyncZoneDir makes dir hold exactly one <apex>.zone file per zone, written
// atomically, and removes the files of zones that no longer exist. It reports
// whether anything on disk changed. Files it did not write (anything that is
// not *.zone) are left alone.
func SyncZoneDir(dir string, zones map[string]string) (bool, error) {
if err := os.MkdirAll(dir, 0o755); err != nil {
return false, fmt.Errorf("zone directory %s: %w", dir, err)
}
changed := false
want := map[string]bool{}
for apex, text := range zones {
name := zoneFile(apex)
want[name] = true
path := filepath.Join(dir, name)
if old, err := os.ReadFile(path); err == nil && string(old) == text {
continue
}
tmp, err := os.CreateTemp(dir, ".kube-zone-*")
if err != nil {
return changed, err
}
_, werr := tmp.WriteString(text)
cerr := tmp.Close()
if werr != nil || cerr != nil {
_ = os.Remove(tmp.Name())
return changed, fmt.Errorf("write %s: %w", name, firstErr(werr, cerr))
}
if err := os.Chmod(tmp.Name(), 0o644); err != nil {
_ = os.Remove(tmp.Name())
return changed, err
}
if err := os.Rename(tmp.Name(), path); err != nil {
_ = os.Remove(tmp.Name())
return changed, err
}
changed = true
}
entries, err := os.ReadDir(dir)
if err != nil {
return changed, err
}
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ".zone") || want[e.Name()] {
continue
}
if err := os.Remove(filepath.Join(dir, e.Name())); err != nil {
return changed, err
}
changed = true
}
return changed, nil
}
// zoneFile is the file name for an apex. The apex has been normalised by
// NormalizeName, so it holds only DNS name characters; the wildcard label is
// the one that is not file-safe on every platform.
func zoneFile(apex string) string {
n := strings.TrimSuffix(apex, ".")
if n == "" {
n = "root"
}
return strings.ReplaceAll(n, "*", "_wild") + ".zone"
}
func firstErr(errs ...error) error {
for _, e := range errs {
if e != nil {
return e
}
}
return nil
}
// Save writes the snapshot so a restart that cannot reach the apiserver still
// has the last known policy. Problems are not saved.
func Save(path string, s *Snapshot) error {
b, err := json.Marshal(s)
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
return config.WriteFileAtomic(path, b, 0o644)
}
// Load reads a snapshot written by Save. A missing file is (nil, nil).
func Load(path string) (*Snapshot, error) {
b, err := os.ReadFile(path)
if os.IsNotExist(err) {
return nil, nil
}
if err != nil {
return nil, err
}
var s Snapshot
if err := json.Unmarshal(b, &s); err != nil {
return nil, fmt.Errorf("%s: %w", path, err)
}
if s.Zones == nil {
s.Zones = map[string]string{}
}
return &s, nil
}
+327
View File
@@ -0,0 +1,327 @@
package kubesource
import (
"context"
"errors"
"fmt"
"log/slog"
"sort"
"sync"
"sync/atomic"
"time"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/dynamic/dynamicinformer"
"k8s.io/client-go/tools/cache"
"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
)
// Problem is one object the source could not use.
type Problem struct {
Kind string `json:"kind"`
Namespace string `json:"namespace"`
Name string `json:"name"`
Message string `json:"message"`
// KeptLastGood is true when an earlier valid version of the object is
// still in force.
KeptLastGood bool `json:"kept_last_good"`
}
// Snapshot is everything the cluster currently declares, already converted and
// validated per object. It is immutable once published.
type Snapshot struct {
// Revision increases with every rebuild that changed content.
Revision uint64 `json:"revision"`
// Groups are in evaluation order.
Groups []config.Group `json:"groups"`
Lists []config.ListSource `json:"lists"`
// Zones maps an apex to its rendered master file.
Zones map[string]string `json:"zones"`
Problems []Problem `json:"problems,omitempty"`
}
// Options configure a Source.
type Options struct {
// Client is the dynamic client; build one with NewClient.
Client dynamic.Interface
// Namespace restricts the watch; "" watches every namespace.
Namespace string
// Debounce coalesces bursts of events into one rebuild.
Debounce time.Duration
Resync time.Duration
Logger *slog.Logger
}
// Source watches the custom resources and publishes Snapshots.
type Source struct {
opt Options
log *slog.Logger
cur atomic.Pointer[Snapshot]
trigger chan struct{}
synced chan struct{}
syncOnce sync.Once
mu sync.Mutex
onChange []func(*Snapshot)
// good holds the last valid conversion of each object, so one edit that
// breaks an object does not remove its policy from a running resolver.
good map[string]any
sig string
rev uint64
}
// New returns a Source. Nothing runs until Run.
func New(opt Options) (*Source, error) {
if opt.Client == nil {
return nil, errors.New("kubesource: a client is required")
}
if opt.Debounce <= 0 {
opt.Debounce = 250 * time.Millisecond
}
if opt.Resync <= 0 {
opt.Resync = 10 * time.Minute
}
if opt.Logger == nil {
opt.Logger = slog.Default()
}
s := &Source{
opt: opt,
log: opt.Logger.With("component", "kubesource"),
trigger: make(chan struct{}, 1),
synced: make(chan struct{}),
good: map[string]any{},
}
return s, nil
}
// Snapshot is the latest published snapshot, or nil before the first sync.
func (s *Source) Snapshot() *Snapshot { return s.cur.Load() }
// OnChange registers fn to run, in order, after each snapshot that differs
// from the previous one.
func (s *Source) OnChange(fn func(*Snapshot)) {
s.mu.Lock()
s.onChange = append(s.onChange, fn)
s.mu.Unlock()
}
// WaitSynced blocks until every informer has listed once, or ctx ends. A
// cluster without the CRDs installed never syncs, so callers bound the wait.
func (s *Source) WaitSynced(ctx context.Context) error {
select {
case <-s.synced:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
// Run starts the informers and rebuilds snapshots until ctx ends.
func (s *Source) Run(ctx context.Context) error {
factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(s.opt.Client, s.opt.Resync, s.opt.Namespace, nil)
informers := make(map[schema.GroupVersionResource]cache.SharedIndexInformer, len(allGVRs))
handler := cache.ResourceEventHandlerFuncs{
AddFunc: func(any) { s.poke() },
UpdateFunc: func(_, _ any) { s.poke() },
DeleteFunc: func(any) { s.poke() },
}
var syncs []cache.InformerSynced
for _, gvr := range allGVRs {
inf := factory.ForResource(gvr).Informer()
if _, err := inf.AddEventHandler(handler); err != nil {
return fmt.Errorf("watch %s: %w", gvr.Resource, err)
}
_ = inf.SetWatchErrorHandler(func(_ *cache.Reflector, err error) {
s.log.Warn("watch failed, will retry", "resource", gvr.Resource, "error", err)
})
informers[gvr] = inf
syncs = append(syncs, inf.HasSynced)
}
factory.Start(ctx.Done())
go func() {
if cache.WaitForCacheSync(ctx.Done(), syncs...) {
s.rebuild(informers)
s.syncOnce.Do(func() { close(s.synced) })
}
}()
for {
select {
case <-ctx.Done():
factory.Shutdown()
return nil
case <-s.trigger:
}
// Debounce: let a burst (a kubectl apply of a directory) land first.
t := time.NewTimer(s.opt.Debounce)
select {
case <-ctx.Done():
t.Stop()
factory.Shutdown()
return nil
case <-t.C:
}
select {
case <-s.synced:
s.rebuild(informers)
default:
// Not synced yet: the sync goroutine does the first rebuild, and
// a partial list must not be published as the cluster's state.
}
}
}
func (s *Source) poke() {
select {
case s.trigger <- struct{}{}:
default:
}
}
func (s *Source) objects(inf cache.SharedIndexInformer) []*unstructured.Unstructured {
raw := inf.GetStore().List()
out := make([]*unstructured.Unstructured, 0, len(raw))
for _, o := range raw {
if u, ok := o.(*unstructured.Unstructured); ok && u.GetDeletionTimestamp() == nil {
out = append(out, u)
}
}
// A stable order makes duplicate resolution and the snapshot signature
// deterministic regardless of the informer's map iteration.
sort.Slice(out, func(i, j int) bool {
a, b := out[i], out[j]
if at, bt := a.GetCreationTimestamp(), b.GetCreationTimestamp(); !at.Equal(&bt) {
return at.Before(&bt)
}
if a.GetNamespace() != b.GetNamespace() {
return a.GetNamespace() < b.GetNamespace()
}
return a.GetName() < b.GetName()
})
return out
}
func (s *Source) rebuild(informers map[schema.GroupVersionResource]cache.SharedIndexInformer) {
s.mu.Lock()
snap := s.build(
s.objects(informers[GroupGVR]),
s.objects(informers[ListGVR]),
s.objects(informers[ZoneGVR]),
)
sig := signature(snap)
changed := sig != s.sig
if changed {
s.sig = sig
s.rev++
snap.Revision = s.rev
s.cur.Store(snap)
} else if prev := s.cur.Load(); prev != nil {
// Content is the same; problems may still have changed, so refresh
// them without notifying anyone.
snap.Revision = prev.Revision
s.cur.Store(snap)
}
hooks := append([]func(*Snapshot){}, s.onChange...)
s.mu.Unlock()
if !changed {
return
}
s.log.Info("cluster configuration changed", "revision", snap.Revision,
"groups", len(snap.Groups), "lists", len(snap.Lists), "zones", len(snap.Zones),
"rejected", len(snap.Problems))
for _, fn := range hooks {
fn(snap)
}
}
// build converts every object. Caller holds s.mu.
func (s *Source) build(groups, lists, zones []*unstructured.Unstructured) *Snapshot {
snap := &Snapshot{Zones: map[string]string{}}
live := map[string]bool{}
problem := func(kind string, u *unstructured.Unstructured, msg string, kept bool) {
snap.Problems = append(snap.Problems, Problem{
Kind: kind, Namespace: u.GetNamespace(), Name: u.GetName(), Message: msg, KeptLastGood: kept,
})
s.log.Warn("rejected object", "kind", kind, "namespace", u.GetNamespace(), "name", u.GetName(),
"error", msg, "kept_last_good", kept)
}
// resolve converts one object, falling back to its last good version.
resolve := func(kind string, u *unstructured.Unstructured, conv func() (any, error)) (any, bool) {
key := kind + "/" + u.GetNamespace() + "/" + u.GetName()
live[key] = true
v, err := conv()
if err == nil {
s.good[key] = v
return v, true
}
old, ok := s.good[key]
problem(kind, u, err.Error(), ok)
return old, ok
}
seenGroup := map[string]bool{}
var gs []group
for _, u := range groups {
v, ok := resolve("ClientGroup", u, func() (any, error) { return convertGroup(u) })
if !ok {
continue
}
g := v.(group)
if seenGroup[g.cfg.Name] {
problem("ClientGroup", u, "another ClientGroup already uses this name", false)
continue
}
seenGroup[g.cfg.Name] = true
gs = append(gs, g)
}
sort.SliceStable(gs, func(i, j int) bool {
if gs[i].priority != gs[j].priority {
return gs[i].priority < gs[j].priority
}
return gs[i].cfg.Name < gs[j].cfg.Name
})
for _, g := range gs {
snap.Groups = append(snap.Groups, g.cfg)
}
seenList := map[string]bool{}
for _, u := range lists {
v, ok := resolve("BlockList", u, func() (any, error) { return convertList(u) })
if !ok {
continue
}
l := v.(config.ListSource)
if seenList[l.Name] {
problem("BlockList", u, "another BlockList already uses this name", false)
continue
}
seenList[l.Name] = true
snap.Lists = append(snap.Lists, l)
}
for _, u := range zones {
v, ok := resolve("DNSZone", u, func() (any, error) { return convertZone(u) })
if !ok {
continue
}
z := v.(zone)
if _, dup := snap.Zones[z.apex]; dup {
// Objects are ordered oldest first, so the earliest claim wins.
problem("DNSZone", u, "another DNSZone already declares "+z.apex, false)
continue
}
snap.Zones[z.apex] = z.text
}
for key := range s.good {
if !live[key] {
delete(s.good, key)
}
}
return snap
}
+296
View File
@@ -0,0 +1,296 @@
package kubesource
import (
"context"
"strings"
"testing"
"time"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
dynfake "k8s.io/client-go/dynamic/fake"
"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
"git.k8s-git-k6n.net/mats/fancy-dns/internal/localzone"
"git.k8s-git-k6n.net/mats/fancy-dns/internal/zonefile"
)
func obj(kind, name string, spec map[string]any) *unstructured.Unstructured {
return &unstructured.Unstructured{Object: map[string]any{
"apiVersion": APIGroup + "/" + Version,
"kind": kind,
"metadata": map[string]any{"name": name, "namespace": "dns", "generation": int64(3)},
"spec": spec,
}}
}
func cgroup(name string, spec map[string]any) *unstructured.Unstructured {
return obj("ClientGroup", name, spec)
}
func fakeClient(objs ...runtime.Object) *dynfake.FakeDynamicClient {
return dynfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(),
map[schemaGVR]string{
ZoneGVR: "DNSZoneList",
GroupGVR: "ClientGroupList",
ListGVR: "BlockListList",
}, objs...)
}
func start(t *testing.T, objs ...runtime.Object) (*Source, *dynfake.FakeDynamicClient, chan *Snapshot) {
t.Helper()
c := fakeClient(objs...)
s, err := New(Options{Client: c, Namespace: "dns", Debounce: 10 * time.Millisecond})
if err != nil {
t.Fatal(err)
}
ch := make(chan *Snapshot, 16)
s.OnChange(func(sn *Snapshot) { ch <- sn })
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { _ = s.Run(ctx); close(done) }()
t.Cleanup(func() { cancel(); <-done })
return s, c, ch
}
func next(t *testing.T, ch chan *Snapshot) *Snapshot {
t.Helper()
select {
case sn := <-ch:
return sn
case <-time.After(5 * time.Second):
t.Fatal("no snapshot published")
return nil
}
}
var cidrMatch = map[string]any{"cidrs": []any{"10.0.0.0/8"}}
func TestGroupsAreOrderedByPriorityThenName(t *testing.T) {
_, _, ch := start(t,
cgroup("b-kids", map[string]any{"priority": int64(10), "policy": "block_only", "match": cidrMatch}),
cgroup("a-kids", map[string]any{"priority": int64(10), "policy": "deny", "match": cidrMatch}),
cgroup("first", map[string]any{"priority": int64(1), "policy": "allow", "match": cidrMatch}),
)
sn := next(t, ch)
var names []string
for _, g := range sn.Groups {
names = append(names, g.Name)
}
if got := strings.Join(names, ","); got != "first,a-kids,b-kids" {
t.Fatalf("order = %s", got)
}
}
func TestRejectedObjectIsReportedAndOthersSurvive(t *testing.T) {
_, _, ch := start(t,
cgroup("good", map[string]any{"policy": "allow", "match": cidrMatch}),
cgroup("badpolicy", map[string]any{"policy": "maybe", "match": cidrMatch}),
cgroup("badcidr", map[string]any{"policy": "deny", "match": map[string]any{"cidrs": []any{"10.0.0.0/33"}}}),
cgroup("empty", map[string]any{"policy": "deny", "match": map[string]any{}}),
)
sn := next(t, ch)
if len(sn.Groups) != 1 || sn.Groups[0].Name != "good" {
t.Fatalf("groups = %+v", sn.Groups)
}
if len(sn.Problems) != 3 {
t.Fatalf("problems = %+v", sn.Problems)
}
}
func TestBrokenEditKeepsLastGoodVersion(t *testing.T) {
_, c, ch := start(t, cgroup("kids", map[string]any{"policy": "deny", "match": cidrMatch}))
if sn := next(t, ch); sn.Groups[0].Policy != "deny" {
t.Fatalf("first = %+v", sn.Groups)
}
// A bad edit changes nothing in force, so it publishes nothing...
bad := cgroup("kids", map[string]any{"policy": "bogus", "match": cidrMatch})
if _, err := c.Resource(GroupGVR).Namespace("dns").Update(context.Background(), bad, metav1UpdateOptions); err != nil {
t.Fatal(err)
}
select {
case sn := <-ch:
t.Fatalf("a rejected edit published a snapshot: %+v", sn.Groups)
case <-time.After(300 * time.Millisecond):
}
// ...and a good edit goes through.
good := cgroup("kids", map[string]any{"policy": "block_only", "match": cidrMatch})
if _, err := c.Resource(GroupGVR).Namespace("dns").Update(context.Background(), good, metav1UpdateOptions); err != nil {
t.Fatal(err)
}
if sn := next(t, ch); sn.Groups[0].Policy != "block_only" {
t.Fatalf("after good edit = %+v", sn.Groups)
}
}
func TestDeleteRemovesGroup(t *testing.T) {
_, c, ch := start(t, cgroup("kids", map[string]any{"policy": "deny", "match": cidrMatch}))
next(t, ch)
if err := c.Resource(GroupGVR).Namespace("dns").Delete(context.Background(), "kids", metav1DeleteOptions); err != nil {
t.Fatal(err)
}
if sn := next(t, ch); len(sn.Groups) != 0 {
t.Fatalf("groups = %+v", sn.Groups)
}
}
func TestBlockListConversion(t *testing.T) {
for name, tc := range map[string]struct {
spec map[string]any
ok bool
}{
"https": {map[string]any{"url": "https://example.com/hosts", "format": "hosts", "refresh": "6h"}, true},
"file url": {map[string]any{"url": "file:///etc/shadow"}, false},
"no url": {map[string]any{"format": "hosts"}, false},
"bad format": {map[string]any{"url": "https://e.com/x", "format": "yaml"}, false},
"bad refresh": {map[string]any{"url": "https://e.com/x", "refresh": "soon"}, false},
"bad mirror": {map[string]any{"url": "https://e.com/x", "mirror": []any{"file:///x"}}, false},
} {
t.Run(name, func(t *testing.T) {
l, err := convertList(obj("BlockList", "ads", tc.spec))
if (err == nil) != tc.ok {
t.Fatalf("err = %v", err)
}
if tc.ok && (!l.Enabled || l.Path != "" || l.Refresh != config.Duration(6*time.Hour)) {
t.Fatalf("list = %+v", l)
}
})
}
}
func zoneSpec(records ...any) map[string]any {
return map[string]any{"zone": "Home.Example.com", "records": records}
}
func rec(name, typ, value string) map[string]any {
return map[string]any{"name": name, "type": typ, "value": value}
}
func TestZoneRendersAndRoundTrips(t *testing.T) {
z, err := convertZone(obj("DNSZone", "home", zoneSpec(
rec("@", "NS", "ns1.home.example.com."),
rec("ns1", "A", "10.0.0.1"),
rec("nas", "A", "10.0.0.2"),
rec("*.apps", "A", "10.0.0.3"),
)))
if err != nil {
t.Fatal(err)
}
if z.apex != "home.example.com." {
t.Fatalf("apex = %q", z.apex)
}
parsed, err := zonefile.Parse(strings.NewReader(z.text), zonefile.Options{Origin: z.apex, File: "t"})
if err != nil {
t.Fatalf("rendered zone does not parse: %v\n%s", err, z.text)
}
if parsed.SOA == nil || parsed.SOA.Serial != 3 {
t.Fatalf("SOA serial should be the object generation: %+v", parsed.SOA)
}
// Same input, same bytes: replicas must agree.
again, _ := convertZone(obj("DNSZone", "home", zoneSpec(
rec("@", "NS", "ns1.home.example.com."),
rec("ns1", "A", "10.0.0.1"),
rec("nas", "A", "10.0.0.2"),
rec("*.apps", "A", "10.0.0.3"),
)))
if again.text != z.text {
t.Fatal("rendering is not deterministic")
}
store, err := localzone.Load(writeTemp(t, z), localzone.Options{})
if err != nil {
t.Fatal(err)
}
if store.Count() == 0 {
t.Fatal("store loaded no records")
}
}
func TestZoneRejections(t *testing.T) {
for name, spec := range map[string]map[string]any{
"no apex NS": zoneSpec(rec("nas", "A", "10.0.0.2")),
"bad value": zoneSpec(rec("@", "NS", "ns1.home.example.com."), rec("nas", "A", "not-an-ip")),
"outside zone": zoneSpec(rec("@", "NS", "ns1.home.example.com."), rec("x.other.org.", "A", "10.0.0.2")),
"duplicate": zoneSpec(rec("@", "NS", "ns1.home.example.com."), rec("a", "A", "10.0.0.2"), rec("a", "A", "10.0.0.2")),
"under delegated": zoneSpec(rec("@", "NS", "ns1.home.example.com."), rec("sub", "NS", "ns.else.net."), rec("x.sub", "A", "10.0.0.2")),
"soa as record": zoneSpec(rec("@", "NS", "ns1.home.example.com."), rec("@", "SOA", "a. b. 1 2 3 4 5")),
"no zone": {"records": []any{}},
} {
t.Run(name, func(t *testing.T) {
if _, err := convertZone(obj("DNSZone", "z", spec)); err == nil {
t.Fatal("expected a rejection")
}
})
}
}
func TestDuplicateApexOldestWins(t *testing.T) {
a := obj("DNSZone", "a", zoneSpec(rec("@", "NS", "ns1.home.example.com.")))
b := obj("DNSZone", "b", zoneSpec(rec("@", "NS", "ns2.home.example.com.")))
a.SetCreationTimestamp(metav1Time(1000))
b.SetCreationTimestamp(metav1Time(2000))
_, _, ch := start(t, b, a)
sn := next(t, ch)
if len(sn.Zones) != 1 || !strings.Contains(sn.Zones["home.example.com."], "ns1.home.example.com.") {
t.Fatalf("zones = %v", sn.Zones)
}
if len(sn.Problems) != 1 || sn.Problems[0].Name != "b" {
t.Fatalf("problems = %+v", sn.Problems)
}
}
func TestMergePrecedenceAndDefaults(t *testing.T) {
base := config.Default()
base.ACL.Groups = []config.Group{{Name: "yaml", Policy: "allow", Match: config.Match{CIDRs: []string{"192.168.0.0/16"}}}}
base.Block.Lists = []config.ListSource{{Name: "ads", URL: "https://a/x", Enabled: true}}
snap := &Snapshot{
Groups: []config.Group{{Name: "yaml", Policy: "deny"}, {Name: "crd", Policy: "deny"}},
Lists: []config.ListSource{{Name: "ads", URL: "https://b/x"}, {Name: "crd-list", URL: "https://c/x", Enabled: true}},
}
merged, dropped := Merge(base, snap)
if len(merged.ACL.Groups) != 2 || merged.ACL.Groups[0].Policy != "allow" || merged.ACL.Groups[1].Name != "crd" {
t.Fatalf("groups = %+v", merged.ACL.Groups)
}
if len(merged.Block.Lists) != 2 || merged.Block.Lists[0].URL != "https://a/x" {
t.Fatalf("lists = %+v", merged.Block.Lists)
}
if got := merged.Block.Lists[1]; got.Format != "auto" || got.Refresh == 0 || got.MaxBytes == 0 {
t.Fatalf("defaults not applied: %+v", got)
}
if len(dropped) != 2 {
t.Fatalf("dropped = %v", dropped)
}
if len(base.ACL.Groups) != 1 || len(base.Block.Lists) != 1 {
t.Fatal("Merge modified its input")
}
}
func TestSyncZoneDir(t *testing.T) {
dir := t.TempDir()
if ch, err := SyncZoneDir(dir, map[string]string{"a.example.": "x", "b.example.": "y"}); err != nil || !ch {
t.Fatalf("first sync: %v %v", ch, err)
}
if ch, err := SyncZoneDir(dir, map[string]string{"a.example.": "x", "b.example.": "y"}); err != nil || ch {
t.Fatalf("idempotent sync: %v %v", ch, err)
}
if ch, err := SyncZoneDir(dir, map[string]string{"a.example.": "x"}); err != nil || !ch {
t.Fatalf("removal sync: %v %v", ch, err)
}
if got := fileNames(t, dir); got != "a.example.zone" {
t.Fatalf("files = %s", got)
}
}
func TestSnapshotSaveLoad(t *testing.T) {
path := t.TempDir() + "/sub/snap.json"
in := &Snapshot{Revision: 4, Groups: []config.Group{{Name: "g", Policy: "allow"}}, Zones: map[string]string{"a.": "z"}}
if err := Save(path, in); err != nil {
t.Fatal(err)
}
out, err := Load(path)
if err != nil || out.Revision != 4 || out.Groups[0].Name != "g" || out.Zones["a."] != "z" {
t.Fatalf("%+v %v", out, err)
}
if none, err := Load(path + ".missing"); none != nil || err != nil {
t.Fatalf("missing file: %v %v", none, err)
}
}
+83
View File
@@ -0,0 +1,83 @@
// Package kubesource turns fancy-dns custom resources (DNSZone, ClientGroup,
// BlockList) into the runtime's own configuration types. It is read-only: the
// runtime watches the cluster and never writes to it.
//
// Objects are decoded from unstructured data into the plain structs below, so
// there is no generated clientset or deepcopy code to keep in step with the
// CRD manifests in deploy/k8s/crds.
package kubesource
import (
"k8s.io/apimachinery/pkg/runtime/schema"
"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
)
// APIGroup and Version name the custom resources.
const (
APIGroup = "fancydns.k6n.net"
Version = "v1alpha1"
)
// The resources the runtime watches.
var (
ZoneGVR = schema.GroupVersionResource{Group: APIGroup, Version: Version, Resource: "dnszones"}
GroupGVR = schema.GroupVersionResource{Group: APIGroup, Version: Version, Resource: "clientgroups"}
ListGVR = schema.GroupVersionResource{Group: APIGroup, Version: Version, Resource: "blocklists"}
allGVRs = []schema.GroupVersionResource{ZoneGVR, GroupGVR, ListGVR}
kindNames = map[string]string{ZoneGVR.Resource: "DNSZone", GroupGVR.Resource: "ClientGroup", ListGVR.Resource: "BlockList"}
)
// DNSZoneSpec declares one authoritative zone. The zone's name is spec.zone,
// not metadata.name, because a DNS name is not always a legal object name.
type DNSZoneSpec struct {
Zone string `json:"zone"`
// TTL is the zone's $TTL and SOA minimum; 0 means the writer's default.
TTL uint32 `json:"ttl,omitempty"`
// SOA is the SOA rdata ("ns1.example.com. hostmaster.example.com. ..."),
// or empty to synthesise one from the first apex NS record.
SOA string `json:"soa,omitempty"`
Records []ZoneRecord `json:"records,omitempty"`
}
// ZoneRecord is one resource record. Value is presentation-format RDATA.
type ZoneRecord struct {
Name string `json:"name"`
Type string `json:"type"`
TTL uint32 `json:"ttl,omitempty"`
Value string `json:"value"`
}
// ClientGroupSpec mirrors config.Group. The group's name is metadata.name.
// Groups are evaluated first-match-wins, so Priority orders them: lower runs
// first, ties break on name.
type ClientGroupSpec struct {
Priority int32 `json:"priority,omitempty"`
Comment string `json:"comment,omitempty"`
Match config.Match `json:"match"`
Policy string `json:"policy"`
AllowDomains []string `json:"allow_domains,omitempty"`
DenyDomains []string `json:"deny_domains,omitempty"`
SkipBlocklist bool `json:"skip_blocklist,omitempty"`
SkipCache bool `json:"skip_cache,omitempty"`
RateLimit int `json:"rate_limit,omitempty"`
MaxConcurrent int `json:"max_concurrent,omitempty"`
LogQueries bool `json:"log_queries,omitempty"`
Upstreams []string `json:"upstreams,omitempty"`
}
// BlockListSpec mirrors config.ListSource for a remote list. There is no path
// field on purpose: a namespace-scoped object must not be able to name a file
// the runtime reads or writes. file:// URLs are refused for the same reason.
type BlockListSpec struct {
URL string `json:"url"`
Format string `json:"format,omitempty"`
Mirror []string `json:"mirror,omitempty"`
Disabled bool `json:"disabled,omitempty"`
Refresh string `json:"refresh,omitempty"`
Timeout string `json:"timeout,omitempty"`
MaxBytes int64 `json:"max_bytes,omitempty"`
Categories []string `json:"categories,omitempty"`
Comment string `json:"comment,omitempty"`
InsecureSkipTLSVerify bool `json:"insecure_skip_tls_verify,omitempty"`
}