fancy-dns/internal/app/integration_test.go

package app

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/miekg/dns"

	"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
)

// zonesPayload is the slice of GET /api/zones this test reads. It is declared
// here rather than imported so the assertions name the fields the console
// depends on; the contract test in this package pins the whole key set.
type zonesPayload struct {
	Zones []struct {
		Origin      string `json:"origin"`
		Managed     bool   `json:"managed"`
		Serial      uint32 `json:"serial"`
		Records     []any  `json:"records"`
		RecordCount int    `json:"record_count"`
		Delegations []struct {
			Name    string   `json:"name"`
			Targets []string `json:"targets"`
			Managed bool     `json:"managed"`
		} `json:"delegations"`
	} `json:"zones"`
	Files      []string `json:"files"`
	ManagedDir string   `json:"managed_dir"`
	Writable   bool     `json:"writable"`
	Types      []string `json:"types"`
	Records    int      `json:"records"`
}

// TestZonesAreReadOnlyOnTheResolver: the resolver lists the zones it serves and
// writes none of them. Editing a zone is the console's job and happens on the
// DNSZone objects (see console_e2e_test.go); every write route on the
// resolver answers "not available" and changes nothing.
func TestZonesAreReadOnlyOnTheResolver(t *testing.T) {
	h := newHarness(t, nil)
	code, body := h.get("/api/zones")
	if code != http.StatusOK {
		t.Fatalf("GET /api/zones = %d: %s", code, body)
	}
	var zones zonesPayload
	if err := json.Unmarshal(body, &zones); err != nil {
		t.Fatalf("zones: %v: %s", err, body)
	}
	if zones.Writable || zones.ManagedDir != "" {
		t.Errorf("writable=%v managed_dir=%q on a resolver with no Kubernetes source", zones.Writable, zones.ManagedDir)
	}
	if len(zones.Zones) != 1 || zones.Zones[0].Origin != "local." || zones.Zones[0].Managed {
		t.Errorf("zones = %+v, want only the operator's master file, not managed", zones.Zones)
	}
	for _, w := range []struct{ method, path, body string }{
		{http.MethodPost, "/api/zones/records", `{"zone":"console.test.","name":"console.test.","type":"NS","value":"ns1.console.test."}`},
		{http.MethodPut, "/api/zones/records", `{"zone":"console.test.","name":"www","type":"A","value":"10.0.0.1"}`},
		{http.MethodDelete, "/api/zones/records?zone=local.&name=ns.local.&type=A", ""},
		{http.MethodPost, "/api/zones/delegations", `{"zone":"console.test.","name":"sub","targets":["ns1.sub.console.test."]}`},
		{http.MethodDelete, "/api/zones/delegations?zone=local.&name=sub.local.", ""},
		{http.MethodDelete, "/api/zones?zone=local.", ""},
		{http.MethodGet, "/api/local-zones", ""},
	} {
		code, out := h.do(w.method, w.path, w.body)
		if code != http.StatusNotImplemented && code != http.StatusNotFound && code != http.StatusMethodNotAllowed {
			t.Errorf("%s %s = %d, want it refused: %s", w.method, w.path, code, out)
		}
	}
	if m := h.query("ns.local", dns.TypeA); len(m.Answer) != 1 {
		t.Errorf("the operator's zone stopped answering after the refused writes: %v", m)
	}
}

// TestResolveUpstream covers the happy path: a name nobody blocks is forwarded
// to the upstream and the answer is cached.
func TestResolveUpstream(t *testing.T) {
	h := newHarness(t, nil)
	res := h.query("www.example.test", dns.TypeA)
	if res.Rcode != dns.RcodeSuccess {
		t.Fatalf("rcode = %s, want NOERROR", dns.RcodeToString[res.Rcode])
	}
	if len(res.Answer) == 0 {
		t.Fatal("no answer records")
	}
	if got := res.Answer[0].(*dns.A).A.String(); got != "203.0.113.7" {
		t.Errorf("answer = %s, want 203.0.113.7", got)
	}
	if h.up.hitCount("www.example.test.") == 0 {
		t.Error("upstream was never asked")
	}
	// Second query must come from cache: the upstream counter must not move.
	before := h.up.hitCount("www.example.test.")
	h.query("www.example.test", dns.TypeA)
	if after := h.up.hitCount("www.example.test."); after != before {
		t.Errorf("upstream hit count moved from %d to %d: the cache did not serve the repeat query", before, after)
	}
}

// TestBlockedNames covers every blocklist source shape: hosts-file entries
// (apex plus subdomains), RPZ wildcards and inline rules.
func TestBlockedNames(t *testing.T) {
	h := newHarness(t, nil)
	cases := []string{
		"ads.example.com",      // hosts file
		"deep.ads.example.com", // hosts file, subdomain
		"tracker.example.net",
		"inline-blocked.example.com",
		"anything.inline-sub.example.com",
	}
	for _, name := range cases {
		res := h.query(name, dns.TypeA)
		if res.Rcode != dns.RcodeNameError {
			t.Errorf("%s: rcode = %s, want NXDOMAIN", name, dns.RcodeToString[res.Rcode])
		}
	}
	// A blocked name must never reach the upstream.
	if n := h.up.hitCount("ads.example.com."); n != 0 {
		t.Errorf("blocked name reached the upstream %d times", n)
	}
}

// TestAllowOverridesBlock checks that an inline allow entry beats a list rule.
func TestAllowOverridesBlock(t *testing.T) {
	h := newHarness(t, nil)
	res := h.query("allow.example.com", dns.TypeA)
	// The fake upstream only knows *.test., so an allowed-but-unresolvable name
	// must come back as NXDOMAIN *from upstream*, which means it got past the
	// blocklist. Assert on the upstream counter instead of the rcode.
	if h.up.hitCount("allow.example.com.") == 0 {
		t.Error("allowed name did not reach the upstream, so the allow rule lost to a block rule")
	}
	_ = res
}

// TestLocalZone covers the local master file: exact, wildcard and NODATA.
func TestLocalZone(t *testing.T) {
	h := newHarness(t, nil)
	res := h.query("ns.local", dns.TypeA)
	if len(res.Answer) != 1 {
		t.Fatalf("want 1 answer for ns.local, got %d", len(res.Answer))
	}
	if got := res.Answer[0].(*dns.A).A.String(); got != "192.168.1.2" {
		t.Errorf("ns.local = %s, want 192.168.1.2", got)
	}
	if !res.Authoritative {
		t.Error("local zone answer should be authoritative")
	}
	// Wildcard covers a name one label below the wildcard's parent.
	w := h.query("anything.wild.local", dns.TypeA)
	if len(w.Answer) == 0 || w.Answer[0].(*dns.A).A.String() != "192.168.1.50" {
		t.Errorf("wildcard answer wrong: %v", w.Answer)
	}
	// NODATA: the name exists but has no A record.
	n := h.query("deep.svc.local", dns.TypeAAAA)
	if n.Rcode != dns.RcodeSuccess {
		t.Errorf("NODATA rcode = %s, want NOERROR", dns.RcodeToString[n.Rcode])
	}
	if len(n.Answer) != 0 {
		t.Errorf("NODATA returned %d answers", len(n.Answer))
	}
	// NXDOMAIN inside a zone we are authoritative for.
	x := h.query("nope.local", dns.TypeA)
	if x.Rcode != dns.RcodeNameError {
		t.Errorf("in-zone NXDOMAIN rcode = %s", dns.RcodeToString[x.Rcode])
	}
}

// TestACLDenyGroup checks that a deny group refuses before anything else,
// including a name the local zone could answer.
func TestACLDenyGroup(t *testing.T) {
	h := newHarness(t, func(c *config.Config) {
		c.ACL.Groups = []config.Group{{
			Name:    "deny-all",
			Match:   config.Match{CIDRs: []string{"127.0.0.0/8"}},
			Policy:  "deny",
			Comment: "test",
		}}
	})
	res := h.query("www.example.test", dns.TypeA)
	if res.Rcode != dns.RcodeRefused {
		t.Errorf("rcode = %s, want REFUSED", dns.RcodeToString[res.Rcode])
	}
	if h.up.hitCount("www.example.test.") != 0 {
		t.Error("a denied client still reached the upstream")
	}
}

// TestACLAllowDomains checks a group's allow_domains acts as a filter.
func TestACLAllowDomains(t *testing.T) {
	h := newHarness(t, func(c *config.Config) {
		c.ACL.Groups = []config.Group{{
			Name:         "only-test",
			Match:        config.Match{CIDRs: []string{"127.0.0.0/8"}},
			Policy:       "allow",
			AllowDomains: []string{"*.test"},
		}}
	})
	// Inside the allow list: resolves.
	if r := h.query("www.example.test", dns.TypeA); r.Rcode != dns.RcodeSuccess {
		t.Errorf("allowed domain rcode = %s", dns.RcodeToString[r.Rcode])
	}
	// Outside it: blocked by the group.
	if r := h.query("ads.example.com", dns.TypeA); r.Rcode != dns.RcodeNameError {
		t.Errorf("domain outside allow_domains rcode = %s, want NXDOMAIN", dns.RcodeToString[r.Rcode])
	}
}

// TestACLSkipBlocklist checks a group can opt out of blocklist filtering.
func TestACLSkipBlocklist(t *testing.T) {
	h := newHarness(t, func(c *config.Config) {
		c.ACL.Groups = []config.Group{{
			Name:          "trusted",
			Match:         config.Match{CIDRs: []string{"127.0.0.0/8"}},
			Policy:        "allow",
			SkipBlocklist: true,
		}}
	})
	h.query("ads.example.com", dns.TypeA)
	if h.up.hitCount("ads.example.com.") == 0 {
		t.Error("skip_blocklist group still blocked the name")
	}
}

// TestBlockActions covers the non-default blocklist responses.
func TestBlockActions(t *testing.T) {
	t.Run("zero", func(t *testing.T) {
		h := newHarness(t, func(c *config.Config) {
			c.Block.DefaultAction = "zero"
			c.Block.ResponseIP = "0.0.0.0"
		})
		res := h.query("ads.example.com", dns.TypeA)
		if res.Rcode != dns.RcodeSuccess {
			t.Fatalf("rcode = %s, want NOERROR", dns.RcodeToString[res.Rcode])
		}
		if len(res.Answer) != 1 || res.Answer[0].(*dns.A).A.String() != "0.0.0.0" {
			t.Errorf("sinkhole answer = %v", res.Answer)
		}
		// AAAA for the same name must get the v6 sinkhole, not the v4 one.
		res6 := h.query("ads.example.com", dns.TypeAAAA)
		if len(res6.Answer) != 1 || res6.Answer[0].(*dns.AAAA).AAAA.String() != "::" {
			t.Errorf("v6 sinkhole answer = %v", res6.Answer)
		}
	})
	t.Run("refused", func(t *testing.T) {
		h := newHarness(t, func(c *config.Config) { c.Block.DefaultAction = "refused" })
		if r := h.query("ads.example.com", dns.TypeA); r.Rcode != dns.RcodeRefused {
			t.Errorf("rcode = %s, want REFUSED", dns.RcodeToString[r.Rcode])
		}
	})
	t.Run("redirect", func(t *testing.T) {
		h := newHarness(t, func(c *config.Config) {
			c.Block.DefaultAction = "redirect"
			c.Block.RedirectName = "blockpage.local"
		})
		res := h.query("ads.example.com", dns.TypeA)
		if len(res.Answer) == 0 {
			t.Fatal("redirect produced no answer")
		}
		cname, ok := res.Answer[0].(*dns.CNAME)
		if !ok {
			t.Fatalf("first answer is %T, want CNAME", res.Answer[0])
		}
		if cname.Target != "blockpage.local." {
			t.Errorf("cname target = %s", cname.Target)
		}
	})
}

// TestMalformedQueries covers the input validation path.
func TestMalformedQueries(t *testing.T) {
	h := newHarness(t, nil)
	// Zero questions: the server must answer FORMERR, not crash or hang.
	m := new(dns.Msg)
	m.Id = dns.Id()
	m.Question = nil
	c := &dns.Client{Timeout: 3 * time.Second}
	res, _, err := c.Exchange(m, h.addr)
	if err != nil {
		t.Fatalf("exchange: %v", err)
	}
	if res.Rcode != dns.RcodeFormatError {
		t.Errorf("rcode = %s, want FORMERR", dns.RcodeToString[res.Rcode])
	}
}

// TestTCPSameAnswer checks TCP returns the same answer as UDP, and that a large
// UDP answer is truncated rather than dropped.
func TestTCPSameAnswer(t *testing.T) {
	h := newHarness(t, nil)
	m := new(dns.Msg)
	m.SetQuestion("www.example.test.", dns.TypeA)
	m.RecursionDesired = true
	tcp := &dns.Client{Net: "tcp", Timeout: 5 * time.Second}
	res, _, err := tcp.Exchange(m, h.addr)
	if err != nil {
		t.Fatalf("tcp exchange: %v", err)
	}
	if res.Rcode != dns.RcodeSuccess || len(res.Answer) == 0 {
		t.Errorf("tcp answer wrong: rcode=%s answers=%d", dns.RcodeToString[res.Rcode], len(res.Answer))
	}
}

// TestAPIAuth checks the admin API refuses unauthenticated reads and accepts
// the configured key.
func TestAPIAuth(t *testing.T) {
	h := newHarness(t, nil)
	// No key: 401.
	res, err := httpGet("http://"+h.api+"/api/whoami", "")
	if err != nil {
		t.Fatal(err)
	}
	if res.status != 401 {
		t.Errorf("unauthenticated /whoami = %d, want 401", res.status)
	}
	res, err = httpGet("http://"+h.api+"/api/whoami", "wrong-key")
	if err != nil {
		t.Fatal(err)
	}
	if res.status != 401 {
		t.Errorf("bad key /whoami = %d, want 401", res.status)
	}
	res, err = httpGet("http://"+h.api+"/api/whoami", h.key)
	if err != nil {
		t.Fatal(err)
	}
	if res.status != 200 {
		t.Errorf("good key /whoami = %d, want 200", res.status)
	}
	var who struct {
		Name     string `json:"name"`
		CanWrite bool   `json:"can_write"`
	}
	if err := json.Unmarshal(res.body, &who); err != nil {
		t.Fatal(err)
	}
	if who.Name != "rw-1" || !who.CanWrite {
		t.Errorf("whoami = %+v", who)
	}
	// Health is deliberately unauthenticated for probes.
	res, err = httpGet("http://"+h.api+"/api/healthz", "")
	if err != nil {
		t.Fatal(err)
	}
	if res.status != 200 {
		t.Errorf("/healthz = %d, want 200", res.status)
	}
}

// TestReadOnlyKeyCannotWrite checks a read-only key is refused on mutations.
func TestReadOnlyKeyCannotWrite(t *testing.T) {
	h := newHarness(t, func(c *config.Config) {
		c.API.Auth.ReadOnlyAPIKeys = []string{"ro-key"}
	})
	res, err := httpRequest("POST", "http://"+h.api+"/api/cache/flush", "ro-key", "")
	if err != nil {
		t.Fatal(err)
	}
	if res.status != 403 {
		t.Errorf("read-only flush = %d, want 403", res.status)
	}
	res, err = httpRequest("POST", "http://"+h.api+"/api/cache/flush", h.key, "")
	if err != nil {
		t.Fatal(err)
	}
	if res.status != 200 {
		t.Errorf("read-write flush = %d, want 200", res.status)
	}
}

// TestDomainAddDeleteRoundTrip is the operator's core loop: add a rule, see it
// take effect on a live query, then remove it.
func TestDomainAddDeleteRoundTrip(t *testing.T) {
	h := newHarness(t, nil)
	// The name lives under .test. so the fake upstream resolves it: that way
	// NOERROR unambiguously means "not blocked" and NXDOMAIN unambiguously
	// means "the blocklist answered", with no third possibility to confuse the
	// assertion.
	const name = "newly-blocked.example.test"
	before := h.up.hitCount(dns.Fqdn(name))
	if r := h.query(name, dns.TypeA); r.Rcode != dns.RcodeSuccess {
		t.Fatalf("precondition: rcode = %s, want NOERROR", dns.RcodeToString[r.Rcode])
	}
	if h.up.hitCount(dns.Fqdn(name)) == before {
		t.Error("precondition: the upstream was not consulted")
	}

	code, body := h.do("POST", "/api/domains", `{"name":"`+name+`","action":"block"}`)
	if code != 200 {
		t.Fatalf("add domain = %d: %s", code, body)
	}
	if r := h.query(name, dns.TypeA); r.Rcode != dns.RcodeNameError {
		t.Errorf("after add, rcode = %s, want NXDOMAIN", dns.RcodeToString[r.Rcode])
	}
	// The rule must survive a cache flush, i.e. it is really in the config and
	// not just a transient matcher state.
	if code, _ := h.do("POST", "/api/cache/flush", ""); code != 200 {
		t.Fatalf("flush = %d", code)
	}
	if r := h.query(name, dns.TypeA); r.Rcode != dns.RcodeNameError {
		t.Errorf("after flush, rcode = %s, want NXDOMAIN", dns.RcodeToString[r.Rcode])
	}

	code, body = h.do("DELETE", "/api/domains/"+name, "")
	if code != 200 {
		t.Fatalf("delete domain = %d: %s", code, body)
	}
	if r := h.query(name, dns.TypeA); r.Rcode != dns.RcodeSuccess {
		t.Errorf("after delete, rcode = %s, want NOERROR", dns.RcodeToString[r.Rcode])
	}
}

// TestDomainAddInvalid checks the API rejects a bad payload with 400 rather
// than writing a broken rule.
func TestDomainAddInvalid(t *testing.T) {
	h := newHarness(t, nil)
	code, _ := h.do("POST", "/api/domains", `{"name":"","action":"block"}`)
	if code != 400 {
		t.Errorf("empty name = %d, want 400", code)
	}
	code, _ = h.do("POST", "/api/domains", `{"name":"x.example.com","action":"nonsense"}`)
	if code != 400 {
		t.Errorf("bad action = %d, want 400", code)
	}
}

// TestACLUpdateRoundTrip checks the clients editor's write path and that a
// saved group is enforced immediately.
// The console's "Test" button goes through the live pipeline, so this pins
// the whole path: a name the fake upstream answers must come back with the
// answer records, and one it does not must come back as a result (a 200 with
// an rcode) rather than as a failed request.
func TestDNSTestEndpoint(t *testing.T) {
	h := newHarness(t, nil)

	code, body := h.do(http.MethodPost, "/api/dns/test", `{"name":"www.example.test","type":"A"}`)
	if code != http.StatusOK {
		t.Fatalf("POST /api/dns/test = %d: %s", code, body)
	}
	var hit struct {
		Name    string   `json:"name"`
		Type    string   `json:"type"`
		RCode   string   `json:"rcode"`
		Action  string   `json:"action"`
		Answers []string `json:"answers"`
	}
	if err := json.Unmarshal(body, &hit); err != nil {
		t.Fatal(err)
	}
	if hit.Name != "www.example.test." || hit.Type != "A" {
		t.Errorf("canonical name = %q type = %q", hit.Name, hit.Type)
	}
	if hit.RCode != "NOERROR" {
		t.Fatalf("rcode = %q, want NOERROR: %s", hit.RCode, body)
	}
	if len(hit.Answers) != 1 || !strings.Contains(hit.Answers[0], "203.0.113.7") {
		t.Errorf("answers = %v, want the upstream A record", hit.Answers)
	}

	// A name the upstream does not hold is still a successful test.
	code, body = h.do(http.MethodPost, "/api/dns/test", `{"name":"nothing.example.com","type":"A"}`)
	if code != http.StatusOK {
		t.Fatalf("POST /api/dns/test (miss) = %d: %s", code, body)
	}
	if !strings.Contains(string(body), "NXDOMAIN") {
		t.Errorf("a miss is not reported as a result: %s", body)
	}

	// An unknown record type is a bad request the caller can fix, so it comes
	// back as an error in the payload rather than as a bogus lookup.
	code, body = h.do(http.MethodPost, "/api/dns/test", `{"name":"www.example.test","type":"NOTATYPE"}`)
	if code != http.StatusOK {
		t.Fatalf("POST /api/dns/test (bad type) = %d: %s", code, body)
	}
	if !strings.Contains(string(body), "error") {
		t.Errorf("a bad type is not reported: %s", body)
	}
}

func TestACLUpdateRoundTrip(t *testing.T) {
	h := newHarness(t, nil)
	// Write a deny group through the API.
	body := `{"groups":[{"name":"nobody","match":{"cidrs":["127.0.0.0/8"]},"policy":"deny","comment":"test"}],"default_policy":"allow"}`
	code, out := h.do("PUT", "/api/clients", body)
	if code != 200 {
		t.Fatalf("put clients = %d: %s", code, out)
	}
	if r := h.query("www.example.test", dns.TypeA); r.Rcode != dns.RcodeRefused {
		t.Errorf("after save, rcode = %s, want REFUSED", dns.RcodeToString[r.Rcode])
	}
	// Read it back.
	code, out = h.get("/api/clients")
	if code != 200 {
		t.Fatalf("get clients = %d", code)
	}
	if !strings.Contains(string(out), "nobody") {
		t.Error("saved group is missing from the read-back")
	}
	// An invalid group must be rejected with 400 and not applied.
	bad := `{"groups":[{"name":"x","match":{"cidrs":["not-a-cidr"]},"policy":"deny"}],"default_policy":"allow"}`
	code, _ = h.do("PUT", "/api/clients", bad)
	if code != 400 {
		t.Errorf("bad CIDR = %d, want 400", code)
	}
}

// TestUpstreamUpdate checks the upstreams editor round-trips and rebinds the
// pool without a restart.
func TestUpstreamUpdate(t *testing.T) {
	h := newHarness(t, nil)
	body := fmt.Sprintf(`{"policy":"failover","servers":[{"addr":%q,"proto":"udp","weight":1}],"max_attempts":1,"breaker_threshold":5,"healthcheck_interval_s":0}`, h.up.addr)
	code, out := h.do("PUT", "/api/upstreams", body)
	if code != 200 {
		t.Fatalf("put upstreams = %d: %s", code, out)
	}
	code, out = h.get("/api/upstreams")
	if code != 200 {
		t.Fatalf("get upstreams = %d", code)
	}
	if !strings.Contains(string(out), "failover") {
		t.Error("policy did not persist")
	}
	// Still resolves after the rebind.
	if r := h.query("www.example.test", dns.TypeA); r.Rcode != dns.RcodeSuccess {
		t.Errorf("after upstream update, rcode = %s", dns.RcodeToString[r.Rcode])
	}
	// Invalid config rejected.
	code, _ = h.do("PUT", "/api/upstreams", `{"policy":"nonsense","servers":[]}`)
	if code != 400 {
		t.Errorf("bad policy = %d, want 400", code)
	}
}

// TestSettingsValidateAndSave covers the config editor: validate, save, and
// confirm the file on disk changed.
func TestSettingsValidateAndSave(t *testing.T) {
	h := newHarness(t, nil)
	code, out := h.get("/api/settings")
	if code != 200 {
		t.Fatalf("get settings = %d", code)
	}
	var s struct {
		ConfigYAML string `json:"config_yaml"`
		Path       string `json:"path"`
		Writable   bool   `json:"writable"`
	}
	if err := json.Unmarshal(out, &s); err != nil {
		t.Fatal(err)
	}
	if !s.Writable {
		t.Error("config file should be writable in the test harness")
	}
	// A valid edit round-trips.
	edited := strings.Replace(s.ConfigYAML, "max_concurrent: 1024", "max_concurrent: 2048", 1)
	if edited == s.ConfigYAML {
		t.Fatal("test could not find the setting it wanted to edit")
	}
	payload, _ := json.Marshal(map[string]string{"config_yaml": edited})
	code, out = h.do("POST", "/api/settings/validate", string(payload))
	if code != 200 {
		t.Fatalf("validate = %d: %s", code, out)
	}
	var v struct {
		OK     bool `json:"ok"`
		Errors []struct {
			Field   string `json:"field"`
			Message string `json:"message"`
		} `json:"errors"`
	}
	if err := json.Unmarshal(out, &v); err != nil {
		t.Fatal(err)
	}
	if !v.OK {
		t.Errorf("valid config reported errors: %+v", v.Errors)
	}
	code, out = h.do("PUT", "/api/settings", string(payload))
	if code != 200 {
		t.Fatalf("save settings = %d: %s", code, out)
	}
	// The file on disk must now contain the edit.
	onDisk := readFileString(t, s.Path)
	if !strings.Contains(onDisk, "max_concurrent: 2048") {
		t.Error("config file on disk was not updated")
	}
	// An invalid document must be refused and the file left alone.
	badPayload, _ := json.Marshal(map[string]string{"config_yaml": "server:\n  max_concurrent: -5\n"})
	code, _ = h.do("PUT", "/api/settings", string(badPayload))
	if code != 400 {
		t.Errorf("invalid save = %d, want 400", code)
	}
	if readFileString(t, s.Path) != onDisk {
		t.Error("a rejected config write still modified the file")
	}
}

// TestBlocklistToggle checks a disabled list stops applying and its cached
// answers are dropped.
func TestBlocklistToggle(t *testing.T) {
	h := newHarness(t, nil)
	const name = "blocked.example.test"
	if r := h.query(name, dns.TypeA); r.Rcode != dns.RcodeNameError {
		t.Fatal("precondition: name should be blocked")
	}
	code, out := h.do("POST", "/api/blocklists/toggle", `{"name":"test-hosts","enabled":false}`)
	if code != 200 {
		t.Fatalf("toggle = %d: %s", code, out)
	}
	// The recompile runs in the background, so poll rather than assume.
	waitFor(t, 5*time.Second, func() bool {
		return h.query(name, dns.TypeA).Rcode == dns.RcodeSuccess
	})
	// Re-enabling must block it again: a one-way toggle would be a nasty
	// surprise.
	if code, _ := h.do("POST", "/api/blocklists/toggle", `{"name":"test-hosts","enabled":true}`); code != 200 {
		t.Fatalf("re-enable = %d", code)
	}
	waitFor(t, 5*time.Second, func() bool {
		return h.query(name, dns.TypeA).Rcode == dns.RcodeNameError
	})
}

// TestStatus checks the status and health endpoints carry the sections the
// console's status page reads.
func TestStatus(t *testing.T) {
	h := newHarness(t, nil)
	h.query("www.example.test", dns.TypeA)

	code, out := h.get("/api/healthz")
	if code != 200 || !strings.Contains(string(out), `"status":"ok"`) {
		t.Fatalf("healthz = %d %s", code, out)
	}
	code, out = h.get("/api/status")
	if code != 200 {
		t.Fatalf("status = %d", code)
	}
	var st struct {
		Components map[string]json.RawMessage `json:"components"`
	}
	if err := json.Unmarshal(out, &st); err != nil {
		t.Fatal(err)
	}
	for _, want := range []string{"listeners", "blocklist", "cache", "upstream", "access_log"} {
		if _, ok := st.Components[want]; !ok {
			t.Errorf("status has no %q section", want)
		}
	}
}

// TestCacheEndpoints covers the cache stats and flush.
func TestCacheEndpoints(t *testing.T) {
	h := newHarness(t, nil)
	h.query("www.example.test", dns.TypeA)
	code, out := h.get("/api/cache")
	if code != 200 {
		t.Fatalf("cache = %d", code)
	}
	var c struct {
		Entries int     `json:"entries"`
		Shards  int     `json:"shards"`
		HitRate float64 `json:"hit_rate"`
	}
	if err := json.Unmarshal(out, &c); err != nil {
		t.Fatal(err)
	}
	if c.Shards <= 0 {
		t.Error("shards is zero")
	}
	if code, _ := h.do("POST", "/api/cache/flush", ""); code != 200 {
		t.Errorf("flush = %d, want 200", code)
	}
}

// TestReloadFromDisk checks SIGHUP-style reload applies a hand-edited file and
// reports a broken one without dropping the running config.
func TestReloadFromDisk(t *testing.T) {
	h := newHarness(t, nil)
	// Break the file, then reload: must fail and keep serving.
	broken := strings.Replace(h.OriginalConfig(), "max_concurrent: 1024", "max_concurrent: not-a-number", 1)
	if broken == h.OriginalConfig() {
		t.Fatal("test could not find the setting it wanted to break")
	}
	writeFileString(t, h.app.cfg.Path(), broken)
	if err := h.app.Reload(); err == nil {
		t.Fatal("reload of a broken config should fail")
	}
	if h.app.cfg.LastError() == "" {
		t.Error("a failed reload should record the error for the health endpoint")
	}
	// The server keeps working with the previous config.
	if r := h.query("www.example.test", dns.TypeA); r.Rcode != dns.RcodeSuccess {
		t.Errorf("after a failed reload, rcode = %s", dns.RcodeToString[r.Rcode])
	}
	// Fix the file and reload: should succeed and clear the error. The whole
	// document is rewritten from the harness's own config, changing only the
	// broken value, so the reload is testing a realistic edit rather than a
	// minimal stub.
	fixed := strings.Replace(h.OriginalConfig(), "max_concurrent: 1024", "max_concurrent: 512", 1)
	if fixed == h.OriginalConfig() {
		t.Fatal("test could not find the setting it wanted to edit")
	}
	writeFileString(t, h.app.cfg.Path(), fixed)
	if err := h.app.Reload(); err != nil {
		t.Fatalf("reload of a fixed config failed: %v", err)
	}
	if h.app.cfg.LastError() != "" {
		t.Error("a successful reload should clear the recorded error")
	}
}

// TestMetricsEndpoint checks Prometheus output is served and protected.
func TestMetricsEndpoint(t *testing.T) {
	h := newHarness(t, nil)
	res, err := httpGet("http://"+h.api+"/api/metrics", "")
	if err != nil {
		t.Fatal(err)
	}
	if res.status != 200 {
		t.Errorf("unprotected /metrics = %d, want 200", res.status)
	}
	if !strings.Contains(string(res.body), "fancydns_queries_total") {
		t.Error("metrics output has no fancydns_queries_total")
	}
	// With protection on, an unauthenticated scrape must fail.
	h2 := newHarness(t, func(c *config.Config) { c.Metrics.Protect = true })
	res, err = httpGet("http://"+h2.api+"/api/metrics", "")
	if err != nil {
		t.Fatal(err)
	}
	if res.status != 401 {
		t.Errorf("protected /metrics without a key = %d, want 401", res.status)
	}
	res, err = httpGet("http://"+h2.api+"/api/metrics", h2.key)
	if err != nil {
		t.Fatal(err)
	}
	if res.status != 200 {
		t.Errorf("protected /metrics with a key = %d, want 200", res.status)
	}
}

// TestConcurrentQueries hammers the server from many goroutines to catch races
// and to make sure the shared state is actually safe. Run with -race.
func TestConcurrentQueries(t *testing.T) {
	h := newHarness(t, nil)
	const workers, each = 16, 25
	done := make(chan error, workers)
	for w := 0; w < workers; w++ {
		go func(w int) {
			m := new(dns.Msg)
			m.SetQuestion("www.example.test.", dns.TypeA)
			m.RecursionDesired = true
			c := &dns.Client{Net: "udp", Timeout: 5 * time.Second}
			for i := 0; i < each; i++ {
				if _, _, err := c.Exchange(m, h.addr); err != nil {
					done <- err
					return
				}
			}
			done <- nil
		}(w)
	}
	for w := 0; w < workers; w++ {
		if err := <-done; err != nil {
			t.Fatalf("concurrent query failed: %v", err)
		}
	}
	// Also exercise the API concurrently, since it reads the same state.
	apiDone := make(chan error, workers)
	for w := 0; w < workers; w++ {
		go func() {
			res, err := httpGet("http://"+h.api+"/api/status", h.key)
			if err != nil {
				apiDone <- err
				return
			}
			if res.status != 200 {
				apiDone <- fmt.Errorf("status = %d", res.status)
				return
			}
			apiDone <- nil
		}()
	}
	for w := 0; w < workers; w++ {
		if err := <-apiDone; err != nil {
			t.Errorf("concurrent api call failed: %v", err)
		}
	}
}

// TestConfigWritesFollowAllowConfigWrite: with api.allow_config_write off, which
// is how the cluster deployment runs, nothing may rewrite the config file, and
// the console is told so (config.writable) instead of offering editors whose
// every save fails. The upstream editor used to ignore the setting.
func TestConfigWritesFollowAllowConfigWrite(t *testing.T) {
	h := newHarness(t, func(c *config.Config) { c.API.AllowConfigWrite = false })
	before := h.OriginalConfig()

	_, body := h.get("/api/status")
	var st struct {
		Config struct {
			Writable bool `json:"writable"`
		} `json:"config"`
	}
	if err := json.Unmarshal(body, &st); err != nil || st.Config.Writable {
		t.Errorf("status config.writable = %v (%v), want false", st.Config.Writable, err)
	}
	_, body = h.get("/api/settings")
	var settings struct {
		Writable bool `json:"writable"`
	}
	if err := json.Unmarshal(body, &settings); err != nil || settings.Writable {
		t.Errorf("settings writable = %v (%v), want false", settings.Writable, err)
	}

	up := fmt.Sprintf(`{"policy":"failover","servers":[{"addr":%q,"proto":"udp","weight":1}],"max_attempts":1,"breaker_threshold":5,"healthcheck_interval_s":0}`, h.up.addr)
	if code, out := h.do("PUT", "/api/upstreams", up); code != http.StatusBadRequest || !strings.Contains(string(out), "allow_config_write") {
		t.Errorf("PUT /api/upstreams = %d %s, want a 400 naming the setting", code, out)
	}
	if code, _ := h.do("PUT", "/api/clients", `{"groups":[]}`); code != http.StatusBadRequest {
		t.Errorf("PUT /api/clients = %d, want 400", code)
	}
	if code, _ := h.do("PUT", "/api/settings", `{"config_yaml":"x: 1"}`); code != http.StatusForbidden {
		t.Errorf("PUT /api/settings = %d, want 403", code)
	}
	raw, err := os.ReadFile(filepath.Join(h.dir, "config.yaml"))
	if err != nil || string(raw) != before {
		t.Errorf("the config file changed (%v)", err)
	}
}