Add a read-only Kubernetes source for zones, client groups and blocklists
The runtime can now take ClientGroup, BlockList and DNSZone objects (fancydns.k6n.net/v1alpha1) from a cluster with -kube, from a pod's service account or a kubeconfig alike. internal/kubesource watches them with dynamic informers, converts and validates each object on its own, and publishes an immutable snapshot. A bad object is skipped and keeps its last good version, so one broken edit cannot remove a running policy; an invalid merged config is dropped whole. The last snapshot is saved so a restart without apiserver access still serves. config.Store gains an overlay: Base() is what the YAML says, Get() is base plus cluster objects. Everything that writes the config back (mutateConfig, SetAdminAddr, the settings editor) now starts from Base(), otherwise cluster objects would be copied into the file and shadow their own objects. YAML wins every name collision. putACL and toggleBlocklist refuse to edit cluster-owned objects. BlockList has no path field and refuses file:// URLs so a namespaced object cannot pick files the runtime reads or writes. DNSZone is rendered through the zone writer's own prove-by-reparse step, with the SOA serial taken from metadata.generation so every replica renders identical bytes. qlog: Flush reset the pending slice to [:0], so entries logged while it was still storing the batch were appended over the same backing array. The race detector found it through the new Kubernetes integration test. pending is now set to nil, with a regression test that fails without the change. CRD manifests, RBAC and examples are in deploy/k8s/crds; tests pin the schemas to the Go spec structs so the two cannot drift. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
matsandClaude Sonnet 5.5 committed
1 parent
340b4eaf80
commit
4d38945d94
27 files changed
+2250
-19
No files matched your search
@@ -0,0 +1,327 @@
|
||||
package kubesource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/client-go/dynamic"
|
||||
"k8s.io/client-go/dynamic/dynamicinformer"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
|
||||
"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
|
||||
)
|
||||
|
||||
// Problem is one object the source could not use.
|
||||
type Problem struct {
|
||||
Kind string `json:"kind"`
|
||||
Namespace string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
Message string `json:"message"`
|
||||
// KeptLastGood is true when an earlier valid version of the object is
|
||||
// still in force.
|
||||
KeptLastGood bool `json:"kept_last_good"`
|
||||
}
|
||||
|
||||
// Snapshot is everything the cluster currently declares, already converted and
|
||||
// validated per object. It is immutable once published.
|
||||
type Snapshot struct {
|
||||
// Revision increases with every rebuild that changed content.
|
||||
Revision uint64 `json:"revision"`
|
||||
// Groups are in evaluation order.
|
||||
Groups []config.Group `json:"groups"`
|
||||
Lists []config.ListSource `json:"lists"`
|
||||
// Zones maps an apex to its rendered master file.
|
||||
Zones map[string]string `json:"zones"`
|
||||
Problems []Problem `json:"problems,omitempty"`
|
||||
}
|
||||
|
||||
// Options configure a Source.
|
||||
type Options struct {
|
||||
// Client is the dynamic client; build one with NewClient.
|
||||
Client dynamic.Interface
|
||||
// Namespace restricts the watch; "" watches every namespace.
|
||||
Namespace string
|
||||
// Debounce coalesces bursts of events into one rebuild.
|
||||
Debounce time.Duration
|
||||
Resync time.Duration
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
// Source watches the custom resources and publishes Snapshots.
|
||||
type Source struct {
|
||||
opt Options
|
||||
log *slog.Logger
|
||||
cur atomic.Pointer[Snapshot]
|
||||
trigger chan struct{}
|
||||
synced chan struct{}
|
||||
syncOnce sync.Once
|
||||
|
||||
mu sync.Mutex
|
||||
onChange []func(*Snapshot)
|
||||
// good holds the last valid conversion of each object, so one edit that
|
||||
// breaks an object does not remove its policy from a running resolver.
|
||||
good map[string]any
|
||||
sig string
|
||||
rev uint64
|
||||
}
|
||||
|
||||
// New returns a Source. Nothing runs until Run.
|
||||
func New(opt Options) (*Source, error) {
|
||||
if opt.Client == nil {
|
||||
return nil, errors.New("kubesource: a client is required")
|
||||
}
|
||||
if opt.Debounce <= 0 {
|
||||
opt.Debounce = 250 * time.Millisecond
|
||||
}
|
||||
if opt.Resync <= 0 {
|
||||
opt.Resync = 10 * time.Minute
|
||||
}
|
||||
if opt.Logger == nil {
|
||||
opt.Logger = slog.Default()
|
||||
}
|
||||
s := &Source{
|
||||
opt: opt,
|
||||
log: opt.Logger.With("component", "kubesource"),
|
||||
trigger: make(chan struct{}, 1),
|
||||
synced: make(chan struct{}),
|
||||
good: map[string]any{},
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Snapshot is the latest published snapshot, or nil before the first sync.
|
||||
func (s *Source) Snapshot() *Snapshot { return s.cur.Load() }
|
||||
|
||||
// OnChange registers fn to run, in order, after each snapshot that differs
|
||||
// from the previous one.
|
||||
func (s *Source) OnChange(fn func(*Snapshot)) {
|
||||
s.mu.Lock()
|
||||
s.onChange = append(s.onChange, fn)
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// WaitSynced blocks until every informer has listed once, or ctx ends. A
|
||||
// cluster without the CRDs installed never syncs, so callers bound the wait.
|
||||
func (s *Source) WaitSynced(ctx context.Context) error {
|
||||
select {
|
||||
case <-s.synced:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// Run starts the informers and rebuilds snapshots until ctx ends.
|
||||
func (s *Source) Run(ctx context.Context) error {
|
||||
factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(s.opt.Client, s.opt.Resync, s.opt.Namespace, nil)
|
||||
informers := make(map[schema.GroupVersionResource]cache.SharedIndexInformer, len(allGVRs))
|
||||
handler := cache.ResourceEventHandlerFuncs{
|
||||
AddFunc: func(any) { s.poke() },
|
||||
UpdateFunc: func(_, _ any) { s.poke() },
|
||||
DeleteFunc: func(any) { s.poke() },
|
||||
}
|
||||
var syncs []cache.InformerSynced
|
||||
for _, gvr := range allGVRs {
|
||||
inf := factory.ForResource(gvr).Informer()
|
||||
if _, err := inf.AddEventHandler(handler); err != nil {
|
||||
return fmt.Errorf("watch %s: %w", gvr.Resource, err)
|
||||
}
|
||||
_ = inf.SetWatchErrorHandler(func(_ *cache.Reflector, err error) {
|
||||
s.log.Warn("watch failed, will retry", "resource", gvr.Resource, "error", err)
|
||||
})
|
||||
informers[gvr] = inf
|
||||
syncs = append(syncs, inf.HasSynced)
|
||||
}
|
||||
factory.Start(ctx.Done())
|
||||
|
||||
go func() {
|
||||
if cache.WaitForCacheSync(ctx.Done(), syncs...) {
|
||||
s.rebuild(informers)
|
||||
s.syncOnce.Do(func() { close(s.synced) })
|
||||
}
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
factory.Shutdown()
|
||||
return nil
|
||||
case <-s.trigger:
|
||||
}
|
||||
// Debounce: let a burst (a kubectl apply of a directory) land first.
|
||||
t := time.NewTimer(s.opt.Debounce)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
t.Stop()
|
||||
factory.Shutdown()
|
||||
return nil
|
||||
case <-t.C:
|
||||
}
|
||||
select {
|
||||
case <-s.synced:
|
||||
s.rebuild(informers)
|
||||
default:
|
||||
// Not synced yet: the sync goroutine does the first rebuild, and
|
||||
// a partial list must not be published as the cluster's state.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Source) poke() {
|
||||
select {
|
||||
case s.trigger <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Source) objects(inf cache.SharedIndexInformer) []*unstructured.Unstructured {
|
||||
raw := inf.GetStore().List()
|
||||
out := make([]*unstructured.Unstructured, 0, len(raw))
|
||||
for _, o := range raw {
|
||||
if u, ok := o.(*unstructured.Unstructured); ok && u.GetDeletionTimestamp() == nil {
|
||||
out = append(out, u)
|
||||
}
|
||||
}
|
||||
// A stable order makes duplicate resolution and the snapshot signature
|
||||
// deterministic regardless of the informer's map iteration.
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
a, b := out[i], out[j]
|
||||
if at, bt := a.GetCreationTimestamp(), b.GetCreationTimestamp(); !at.Equal(&bt) {
|
||||
return at.Before(&bt)
|
||||
}
|
||||
if a.GetNamespace() != b.GetNamespace() {
|
||||
return a.GetNamespace() < b.GetNamespace()
|
||||
}
|
||||
return a.GetName() < b.GetName()
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Source) rebuild(informers map[schema.GroupVersionResource]cache.SharedIndexInformer) {
|
||||
s.mu.Lock()
|
||||
snap := s.build(
|
||||
s.objects(informers[GroupGVR]),
|
||||
s.objects(informers[ListGVR]),
|
||||
s.objects(informers[ZoneGVR]),
|
||||
)
|
||||
sig := signature(snap)
|
||||
changed := sig != s.sig
|
||||
if changed {
|
||||
s.sig = sig
|
||||
s.rev++
|
||||
snap.Revision = s.rev
|
||||
s.cur.Store(snap)
|
||||
} else if prev := s.cur.Load(); prev != nil {
|
||||
// Content is the same; problems may still have changed, so refresh
|
||||
// them without notifying anyone.
|
||||
snap.Revision = prev.Revision
|
||||
s.cur.Store(snap)
|
||||
}
|
||||
hooks := append([]func(*Snapshot){}, s.onChange...)
|
||||
s.mu.Unlock()
|
||||
if !changed {
|
||||
return
|
||||
}
|
||||
s.log.Info("cluster configuration changed", "revision", snap.Revision,
|
||||
"groups", len(snap.Groups), "lists", len(snap.Lists), "zones", len(snap.Zones),
|
||||
"rejected", len(snap.Problems))
|
||||
for _, fn := range hooks {
|
||||
fn(snap)
|
||||
}
|
||||
}
|
||||
|
||||
// build converts every object. Caller holds s.mu.
|
||||
func (s *Source) build(groups, lists, zones []*unstructured.Unstructured) *Snapshot {
|
||||
snap := &Snapshot{Zones: map[string]string{}}
|
||||
live := map[string]bool{}
|
||||
|
||||
problem := func(kind string, u *unstructured.Unstructured, msg string, kept bool) {
|
||||
snap.Problems = append(snap.Problems, Problem{
|
||||
Kind: kind, Namespace: u.GetNamespace(), Name: u.GetName(), Message: msg, KeptLastGood: kept,
|
||||
})
|
||||
s.log.Warn("rejected object", "kind", kind, "namespace", u.GetNamespace(), "name", u.GetName(),
|
||||
"error", msg, "kept_last_good", kept)
|
||||
}
|
||||
// resolve converts one object, falling back to its last good version.
|
||||
resolve := func(kind string, u *unstructured.Unstructured, conv func() (any, error)) (any, bool) {
|
||||
key := kind + "/" + u.GetNamespace() + "/" + u.GetName()
|
||||
live[key] = true
|
||||
v, err := conv()
|
||||
if err == nil {
|
||||
s.good[key] = v
|
||||
return v, true
|
||||
}
|
||||
old, ok := s.good[key]
|
||||
problem(kind, u, err.Error(), ok)
|
||||
return old, ok
|
||||
}
|
||||
|
||||
seenGroup := map[string]bool{}
|
||||
var gs []group
|
||||
for _, u := range groups {
|
||||
v, ok := resolve("ClientGroup", u, func() (any, error) { return convertGroup(u) })
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
g := v.(group)
|
||||
if seenGroup[g.cfg.Name] {
|
||||
problem("ClientGroup", u, "another ClientGroup already uses this name", false)
|
||||
continue
|
||||
}
|
||||
seenGroup[g.cfg.Name] = true
|
||||
gs = append(gs, g)
|
||||
}
|
||||
sort.SliceStable(gs, func(i, j int) bool {
|
||||
if gs[i].priority != gs[j].priority {
|
||||
return gs[i].priority < gs[j].priority
|
||||
}
|
||||
return gs[i].cfg.Name < gs[j].cfg.Name
|
||||
})
|
||||
for _, g := range gs {
|
||||
snap.Groups = append(snap.Groups, g.cfg)
|
||||
}
|
||||
|
||||
seenList := map[string]bool{}
|
||||
for _, u := range lists {
|
||||
v, ok := resolve("BlockList", u, func() (any, error) { return convertList(u) })
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
l := v.(config.ListSource)
|
||||
if seenList[l.Name] {
|
||||
problem("BlockList", u, "another BlockList already uses this name", false)
|
||||
continue
|
||||
}
|
||||
seenList[l.Name] = true
|
||||
snap.Lists = append(snap.Lists, l)
|
||||
}
|
||||
|
||||
for _, u := range zones {
|
||||
v, ok := resolve("DNSZone", u, func() (any, error) { return convertZone(u) })
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
z := v.(zone)
|
||||
if _, dup := snap.Zones[z.apex]; dup {
|
||||
// Objects are ordered oldest first, so the earliest claim wins.
|
||||
problem("DNSZone", u, "another DNSZone already declares "+z.apex, false)
|
||||
continue
|
||||
}
|
||||
snap.Zones[z.apex] = z.text
|
||||
}
|
||||
|
||||
for key := range s.good {
|
||||
if !live[key] {
|
||||
delete(s.good, key)
|
||||
}
|
||||
}
|
||||
return snap
|
||||
}
|
||||
Reference in new issue
Block a user