fancy-dns/internal/kubesource/source.go

package kubesource

import (
	"context"
	"errors"
	"fmt"
	"log/slog"
	"sort"
	"sync"
	"sync/atomic"
	"time"

	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/dynamic/dynamicinformer"
	"k8s.io/client-go/tools/cache"

	"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
)

// Problem is one object the source could not use.
type Problem struct {
	Kind      string `json:"kind"`
	Namespace string `json:"namespace"`
	Name      string `json:"name"`
	Message   string `json:"message"`
	// KeptLastGood is true when an earlier valid version of the object is
	// still in force.
	KeptLastGood bool `json:"kept_last_good"`
}

// Snapshot is everything the cluster currently declares, already converted and
// validated per object. It is immutable once published.
type Snapshot struct {
	// Revision increases with every rebuild that changed content.
	Revision uint64 `json:"revision"`
	// Groups are in evaluation order.
	Groups []config.Group      `json:"groups"`
	Lists  []config.ListSource `json:"lists"`
	// Zones maps an apex to its rendered master file.
	Zones map[string]string `json:"zones"`
	// Overrides are the address overrides, sorted by name.
	Overrides []Override `json:"overrides,omitempty"`
	// BlockNames and AllowNames are the inline domain rules from all
	// DomainRule objects.
	BlockNames []string  `json:"block_names,omitempty"`
	AllowNames []string  `json:"allow_names,omitempty"`
	Problems   []Problem `json:"problems,omitempty"`
}

// Options configure a Source.
type Options struct {
	// Client is the dynamic client; build one with NewClient.
	Client dynamic.Interface
	// Namespace restricts the watch; "" watches every namespace.
	Namespace string
	// Debounce coalesces bursts of events into one rebuild.
	Debounce time.Duration
	Resync   time.Duration
	Logger   *slog.Logger
}

// Source watches the custom resources and publishes Snapshots.
type Source struct {
	opt      Options
	log      *slog.Logger
	cur      atomic.Pointer[Snapshot]
	trigger  chan struct{}
	synced   chan struct{}
	syncOnce sync.Once

	mu       sync.Mutex
	onChange []func(*Snapshot)
	// good holds the last valid conversion of each object, so one edit that
	// breaks an object does not remove its policy from a running resolver.
	good map[string]any
	sig  string
	rev  uint64
}

// New returns a Source. Nothing runs until Run.
func New(opt Options) (*Source, error) {
	if opt.Client == nil {
		return nil, errors.New("kubesource: a client is required")
	}
	if opt.Debounce <= 0 {
		opt.Debounce = 250 * time.Millisecond
	}
	if opt.Resync <= 0 {
		opt.Resync = 10 * time.Minute
	}
	if opt.Logger == nil {
		opt.Logger = slog.Default()
	}
	s := &Source{
		opt:     opt,
		log:     opt.Logger.With("component", "kubesource"),
		trigger: make(chan struct{}, 1),
		synced:  make(chan struct{}),
		good:    map[string]any{},
	}
	return s, nil
}

// Snapshot is the latest published snapshot, or nil before the first sync.
func (s *Source) Snapshot() *Snapshot { return s.cur.Load() }

// OnChange registers fn to run, in order, after each snapshot that differs
// from the previous one.
func (s *Source) OnChange(fn func(*Snapshot)) {
	s.mu.Lock()
	s.onChange = append(s.onChange, fn)
	s.mu.Unlock()
}

// WaitSynced blocks until every informer has listed once, or ctx ends. A
// cluster without the CRDs installed never syncs, so callers bound the wait.
func (s *Source) WaitSynced(ctx context.Context) error {
	select {
	case <-s.synced:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}

// Run starts the informers and rebuilds snapshots until ctx ends.
func (s *Source) Run(ctx context.Context) error {
	factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(s.opt.Client, s.opt.Resync, s.opt.Namespace, nil)
	informers := make(map[schema.GroupVersionResource]cache.SharedIndexInformer, len(allGVRs))
	handler := cache.ResourceEventHandlerFuncs{
		AddFunc:    func(any) { s.poke() },
		UpdateFunc: func(_, _ any) { s.poke() },
		DeleteFunc: func(any) { s.poke() },
	}
	var syncs []cache.InformerSynced
	for _, gvr := range allGVRs {
		inf := factory.ForResource(gvr).Informer()
		if _, err := inf.AddEventHandler(handler); err != nil {
			return fmt.Errorf("watch %s: %w", gvr.Resource, err)
		}
		_ = inf.SetWatchErrorHandler(func(_ *cache.Reflector, err error) {
			s.log.Warn("watch failed, will retry", "resource", gvr.Resource, "error", err)
		})
		informers[gvr] = inf
		syncs = append(syncs, inf.HasSynced)
	}
	factory.Start(ctx.Done())

	go func() {
		if cache.WaitForCacheSync(ctx.Done(), syncs...) {
			s.rebuild(informers)
			s.syncOnce.Do(func() { close(s.synced) })
		}
	}()

	for {
		select {
		case <-ctx.Done():
			factory.Shutdown()
			return nil
		case <-s.trigger:
		}
		// Debounce: let a burst (a kubectl apply of a directory) land first.
		t := time.NewTimer(s.opt.Debounce)
		select {
		case <-ctx.Done():
			t.Stop()
			factory.Shutdown()
			return nil
		case <-t.C:
		}
		select {
		case <-s.synced:
			s.rebuild(informers)
		default:
			// Not synced yet: the sync goroutine does the first rebuild, and
			// a partial list must not be published as the cluster's state.
		}
	}
}

func (s *Source) poke() {
	select {
	case s.trigger <- struct{}{}:
	default:
	}
}

func (s *Source) objects(inf cache.SharedIndexInformer) []*unstructured.Unstructured {
	raw := inf.GetStore().List()
	out := make([]*unstructured.Unstructured, 0, len(raw))
	for _, o := range raw {
		if u, ok := o.(*unstructured.Unstructured); ok && u.GetDeletionTimestamp() == nil {
			out = append(out, u)
		}
	}
	// A stable order makes duplicate resolution and the snapshot signature
	// deterministic regardless of the informer's map iteration.
	sort.Slice(out, func(i, j int) bool {
		a, b := out[i], out[j]
		if at, bt := a.GetCreationTimestamp(), b.GetCreationTimestamp(); !at.Equal(&bt) {
			return at.Before(&bt)
		}
		if a.GetNamespace() != b.GetNamespace() {
			return a.GetNamespace() < b.GetNamespace()
		}
		return a.GetName() < b.GetName()
	})
	return out
}

func (s *Source) rebuild(informers map[schema.GroupVersionResource]cache.SharedIndexInformer) {
	s.mu.Lock()
	in := make(map[schema.GroupVersionResource][]*unstructured.Unstructured, len(informers))
	for gvr, inf := range informers {
		in[gvr] = s.objects(inf)
	}
	snap := s.build(in)
	sig := signature(snap)
	changed := sig != s.sig
	if changed {
		s.sig = sig
		s.rev++
		snap.Revision = s.rev
		s.cur.Store(snap)
	} else if prev := s.cur.Load(); prev != nil {
		// Content is the same; problems may still have changed, so refresh
		// them without notifying anyone.
		snap.Revision = prev.Revision
		s.cur.Store(snap)
	}
	hooks := append([]func(*Snapshot){}, s.onChange...)
	s.mu.Unlock()
	if !changed {
		return
	}
	s.log.Info("cluster configuration changed", "revision", snap.Revision,
		"groups", len(snap.Groups), "lists", len(snap.Lists), "zones", len(snap.Zones),
		"overrides", len(snap.Overrides), "block_names", len(snap.BlockNames), "allow_names", len(snap.AllowNames),
		"rejected", len(snap.Problems))
	for _, fn := range hooks {
		fn(snap)
	}
}

// build converts every object. Caller holds s.mu.
func (s *Source) build(in map[schema.GroupVersionResource][]*unstructured.Unstructured) *Snapshot {
	groups, lists, zones := in[GroupGVR], in[ListGVR], in[ZoneGVR]
	snap := &Snapshot{Zones: map[string]string{}}
	live := map[string]bool{}

	problem := func(kind string, u *unstructured.Unstructured, msg string, kept bool) {
		snap.Problems = append(snap.Problems, Problem{
			Kind: kind, Namespace: u.GetNamespace(), Name: u.GetName(), Message: msg, KeptLastGood: kept,
		})
		s.log.Warn("rejected object", "kind", kind, "namespace", u.GetNamespace(), "name", u.GetName(),
			"error", msg, "kept_last_good", kept)
	}
	// resolve converts one object, falling back to its last good version.
	resolve := func(kind string, u *unstructured.Unstructured, conv func() (any, error)) (any, bool) {
		key := kind + "/" + u.GetNamespace() + "/" + u.GetName()
		live[key] = true
		v, err := conv()
		if err == nil {
			s.good[key] = v
			return v, true
		}
		old, ok := s.good[key]
		problem(kind, u, err.Error(), ok)
		return old, ok
	}

	seenGroup := map[string]bool{}
	var gs []group
	for _, u := range groups {
		v, ok := resolve("ClientGroup", u, func() (any, error) { return convertGroup(u) })
		if !ok {
			continue
		}
		g := v.(group)
		if seenGroup[g.cfg.Name] {
			problem("ClientGroup", u, "another ClientGroup already uses this name", false)
			continue
		}
		seenGroup[g.cfg.Name] = true
		gs = append(gs, g)
	}
	sort.SliceStable(gs, func(i, j int) bool {
		if gs[i].priority != gs[j].priority {
			return gs[i].priority < gs[j].priority
		}
		return gs[i].cfg.Name < gs[j].cfg.Name
	})
	for _, g := range gs {
		snap.Groups = append(snap.Groups, g.cfg)
	}

	seenList := map[string]bool{}
	for _, u := range lists {
		v, ok := resolve("BlockList", u, func() (any, error) { return convertList(u) })
		if !ok {
			continue
		}
		l := v.(config.ListSource)
		if seenList[l.Name] {
			problem("BlockList", u, "another BlockList already uses this name", false)
			continue
		}
		seenList[l.Name] = true
		snap.Lists = append(snap.Lists, l)
	}

	for _, u := range zones {
		v, ok := resolve("DNSZone", u, func() (any, error) { return convertZone(u) })
		if !ok {
			continue
		}
		z := v.(zone)
		if _, dup := snap.Zones[z.apex]; dup {
			// Objects are ordered oldest first, so the earliest claim wins.
			problem("DNSZone", u, "another DNSZone already declares "+z.apex, false)
			continue
		}
		snap.Zones[z.apex] = z.text
	}

	seenOverride := map[string]bool{}
	for _, u := range in[OverrideGVR] {
		v, ok := resolve("AddressOverride", u, func() (any, error) { return convertOverride(u) })
		if !ok {
			continue
		}
		o := v.(Override)
		if seenOverride[o.Domain] {
			problem("AddressOverride", u, "another AddressOverride already covers "+o.Domain, false)
			continue
		}
		seenOverride[o.Domain] = true
		snap.Overrides = append(snap.Overrides, o)
	}
	sort.Slice(snap.Overrides, func(i, j int) bool { return snap.Overrides[i].Domain < snap.Overrides[j].Domain })

	for _, u := range in[RuleGVR] {
		v, ok := resolve("DomainRule", u, func() (any, error) { return convertRules(u) })
		if !ok {
			continue
		}
		r := v.(Rules)
		if r.Action == "block" {
			snap.BlockNames = append(snap.BlockNames, r.Names...)
		} else {
			snap.AllowNames = append(snap.AllowNames, r.Names...)
		}
	}

	for key := range s.good {
		if !live[key] {
			delete(s.good, key)
		}
	}
	return snap
}