package server import ( "bytes" "context" "io" "net" "sync" "testing" "time" "github.com/miekg/dns" ) // Shutdown is the only part of this package that can lose a client's query, // so it is tested as behaviour rather than as a sequence: the socket is // released, Stop is idempotent, and a query that is already inside the // pipeline is still answered. The last one is the invariant that matters — a // resolver that answers "connection closed" to a query it had already accepted // makes every client retry against somewhere else. // holdOnce parks every handler that arrives from now on and returns a release // func that is safe to call twice, so a failing test can unblock the gate in // its cleanup without a double-close panic. func holdOnce(t *testing.T, f *fakeHandler) func() { t.Helper() release := f.hold() var once sync.Once out := func() { once.Do(release) } t.Cleanup(out) return out } // startStop calls srv.Stop in the background and returns the channel it will // report on. stopWithin is that plus a bounded wait; the in-flight test needs // the gap, because it has to watch the shutdown overlap a parked query instead // of waiting for it to finish. func startStop(srv *Server, ctxTimeout time.Duration) <-chan error { done := make(chan error, 1) go func() { ctx, cancel := context.WithTimeout(context.Background(), ctxTimeout) defer cancel() done <- srv.Stop(ctx) }() return done } // awaitStop waits for a Stop started by startStop. func awaitStop(t *testing.T, done <-chan error, d time.Duration) error { t.Helper() select { case err := <-done: return err case <-time.After(d): t.Fatalf("Stop did not return within %s", d) return nil } } // stopWithin calls srv.Stop and fails the test if it has not returned within d. // A hang inside a test binary is reported as a package timeout, which says // nothing about which test caused it; bounding every Stop call here makes the // failure name the test. func stopWithin(t *testing.T, srv *Server, d time.Duration) error { t.Helper() return awaitStop(t, startStop(srv, 5*time.Second), d) } // bindFree tries to bind network/addr and closes the socket again. It reports // the error rather than failing so a caller can retry: a socket that was // closed a moment ago can still be in TIME_WAIT, and the point of every use // here is that a *second* bind of the port works, not that the first one does. func bindFree(network, addr string) error { if network == "udp" { pc, err := net.ListenPacket("udp", addr) if err != nil { return err } return pc.Close() } ln, err := net.Listen("tcp", addr) if err != nil { return err } return ln.Close() } // waitPortFree blocks until network/addr can be bound. Use it only when nothing // else holds the port: a connected client's own local socket is on that same // port, so a bind probe can never succeed while a connection is open. The // in-flight test uses waitPortRefused for exactly that reason. func waitPortFree(t *testing.T, network, addr, what string) { t.Helper() deadline := time.Now().Add(2 * time.Second) for { err := bindFree(network, addr) if err == nil { return } if time.Now().After(deadline) { t.Fatalf("%s: %s %s is still bound 2s after Stop was called (%v)", what, network, addr, err) } time.Sleep(2 * time.Millisecond) } } // waitPortRefused blocks until a dial to addr is refused, which is how the // in-flight test observes that Stop has reached the point where it closes the // listeners — before it waits for its goroutines. A refused dial is the right // probe there: the parked query holds a *connection* to that port, so binding // it would fail for a reason that has nothing to do with Stop. func waitPortRefused(t *testing.T, network, addr, what string) { t.Helper() deadline := time.Now().Add(2 * time.Second) for { conn, err := net.DialTimeout(network, addr, 200*time.Millisecond) if err != nil { return } conn.Close() if time.Now().After(deadline) { t.Fatalf("%s: %s %s still accepts connections 2s after Stop was called", what, network, addr) } time.Sleep(2 * time.Millisecond) } } // TestStopReleasesEveryListenerPort: an operator who restarts fancy-dns — or // hands the port to something else — must be able to bind it again. It also // empties the listener snapshot, because that is what the admin API and the // cluster node state read: a listener still reported as running after the // process stopped serving on it sends the health check the wrong answer. func TestStopReleasesEveryListenerPort(t *testing.T) { h := newHarness(t, nil) addrs := map[string]string{} for _, l := range h.srv.Listeners() { addrs[l.Transport] = l.Addr } if len(addrs) != 4 { t.Fatalf("fixture bound %d listeners, want 4: %+v", len(addrs), addrs) } if err := stopWithin(t, h.srv, 2*time.Second); err != nil { t.Fatalf("Stop: %v", err) } for transport, addr := range addrs { network := "tcp" if transport == "udp" { network = "udp" } // One retry: TIME_WAIT is a property of the socket, not a listener // still being held, and failing on it would make this a flake. var err error for i := 0; i < 2; i++ { if err = bindFree(network, addr); err == nil { break } time.Sleep(20 * time.Millisecond) } if err != nil { t.Errorf("%s %s could not be bound after Stop: %v", network, addr, err) } } if got := h.srv.Listeners(); len(got) != 0 { t.Errorf("Listeners() = %+v after Stop, want empty", got) } } // TestStopIsIdempotent: Stop is called from signal handling, from an admin // endpoint and from a test cleanup, so more than one of them can reach a // server that is already down. A second call must be a no-op returning nil — // not a panic on a half-drained listener list, and not a wait on goroutines // that are already gone. The same is true of a server that never started at // all, which is what a config error produces. func TestStopIsIdempotent(t *testing.T) { h := newHarness(t, nil) if err := stopWithin(t, h.srv, 2*time.Second); err != nil { t.Fatalf("first Stop: %v", err) } // The code clears started before it closes anything, so the second call // short-circuits on that flag and never re-closes a listener or re-waits // on the wait group. Pinned: a second Stop that returned the first Stop's // error, or waited again, would be a bug in its own right. if err := stopWithin(t, h.srv, 2*time.Second); err != nil { t.Errorf("second Stop = %v, want nil", err) } if err := stopWithin(t, h.srv, 2*time.Second); err != nil { t.Errorf("third Stop = %v, want nil", err) } // A server that never started: nothing to close, and nothing to say. if err := stopWithin(t, New(Options{}), 2*time.Second); err != nil { t.Errorf("Stop on a server that never started = %v, want nil", err) } } // TestStopWithAnAlreadyCancelledContextReturnsPromptly: the context bounds how // long Stop waits, and a caller that has already given up must not be made to // wait for a listener to close. Only the bound is asserted — never a // duration. The return *value* is deliberately not asserted: Stop races its // own wait group against ctx.Done() and takes whichever is ready first, so the // same call can return nil or context.Canceled depending on how the shutdown // went. Asserting either would pin the scheduler, not the code. // // What IS asserted is that the listeners are released even then: closing them // is not the part the context is allowed to interrupt, because that is the // work the next process start needs. func TestStopWithAnAlreadyCancelledContextReturnsPromptly(t *testing.T) { h := newHarness(t, nil) udp := h.addr("udp") ctx, cancel := context.WithCancel(context.Background()) cancel() done := make(chan error, 1) go func() { done <- h.srv.Stop(ctx) }() select { case <-done: case <-time.After(2 * time.Second): t.Fatal("Stop with an already-cancelled context did not return within 2s") } waitPortFree(t, "udp", udp, "a Stop whose context was already cancelled") } // TestInFlightQueryIsAnsweredAcrossStop is the invariant in this file. A // query that has been accepted and is inside the pipeline belongs to the // client: closing the listener must not make it vanish. The client here has // its query parked in the handler while Stop runs, so if the shutdown path // abandoned in-flight work the client would see a closed connection (TCP) or a // 5xx/EOF (DoH) and every client would retry elsewhere. // // The test proves Stop is really in flight before the handler is released: it // waits for the listener's port to become bindable, which Stop does before it // waits for its goroutines. Without that ordering the test would pass even if // the shutdown never overlapped the query. func TestInFlightQueryIsAnsweredAcrossStop(t *testing.T) { t.Run("tcp", func(t *testing.T) { h := newHarness(t, nil) release := holdOnce(t, h.fake) conn, err := net.DialTimeout("tcp", h.addr("tcp"), 2*time.Second) if err != nil { t.Fatalf("dial: %v", err) } t.Cleanup(func() { _ = conn.Close() }) _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) co := &dns.Conn{Conn: conn} if err := co.WriteMsg(question("inflight-shutdown.test.", dns.TypeA)); err != nil { t.Fatalf("write: %v", err) } h.fake.waitArrived(t) // Stop runs in the background and the test watches the overlap: it // closes the listener before it waits for its goroutines, so a refused // dial is proof that the shutdown is running *while* the query is // still parked in the handler. Without that ordering the test would // pass even if the shutdown never overlapped the query at all. stopped := startStop(h.srv, 5*time.Second) waitPortRefused(t, "tcp", h.addr("tcp"), "Stop closing the TCP listener") release() res, err := co.ReadMsg() if err != nil { t.Fatalf("the in-flight query was lost across Stop: %v", err) } if a := answerAddr(res); a != "203.0.113.9" { t.Errorf("A = %q, want 203.0.113.9 (reply: %v)", a, res) } if err := awaitStop(t, stopped, 3*time.Second); err != nil { t.Errorf("Stop: %v", err) } // Stop waits for the connection goroutines, so by the time it returns // nothing is still inside the engine. A shutdown that returned while a // query was in flight would leave the process exiting with work // running. if got := h.srv.Stats().Inflight; got != 0 { t.Errorf("Stats().Inflight = %d after Stop returned, want 0", got) } }) t.Run("doh", func(t *testing.T) { h := newHarness(t, nil) release := holdOnce(t, h.fake) cl := h.dohClient(false) type reply struct { status int body []byte err error } got := make(chan reply, 1) // The request is issued from its own goroutine: it does not come back // until the handler is released, which happens after Stop has started. // h.doHPOST is not usable here because it calls t.Fatalf, and a // Fatalf from a non-test goroutine does not stop the test. go func() { wire, err := question("inflight-shutdown.test.", dns.TypeA).Pack() if err != nil { got <- reply{err: err} return } res, err := cl.Post(h.dohURL(h.base()), contentTypeDNS, bytes.NewReader(wire)) if err != nil { got <- reply{err: err} return } defer res.Body.Close() body, _ := io.ReadAll(res.Body) got <- reply{status: res.StatusCode, body: body} }() h.fake.waitArrived(t) // The DoH path is the interesting one: http.Server.Shutdown waits for // the in-flight request rather than cancelling it, so Stop has to // block until the answer has been written. It must still return, and // the client must still get its answer. The probe is the same as on // the TCP side: the listener is closed before Stop waits. stopped := startStop(h.srv, 5*time.Second) waitPortRefused(t, "tcp", h.addr("doh"), "Stop closing the DoH listener") release() select { case r := <-got: if r.err != nil { t.Fatalf("the in-flight DoH request failed across Stop: %v", r.err) } if r.status != 200 { t.Fatalf("status = %d, want 200: %s", r.status, r.body) } if a := answerAddr(unpackDoH(t, r.body)); a != "203.0.113.9" { t.Errorf("A = %q, want 203.0.113.9", a) } case <-time.After(3 * time.Second): t.Fatal("the in-flight DoH request never came back") } if err := awaitStop(t, stopped, 3*time.Second); err != nil { t.Errorf("Stop: %v", err) } if got := h.srv.Stats().Inflight; got != 0 { t.Errorf("Stats().Inflight = %d after Stop returned, want 0", got) } }) } // TestStopWithNoQueryInFlightReturnsAndStartsAgain: an idle Stop has nothing // to wait for, so it returns immediately — the bound here is generous because // the point is "does not hang", not "is fast". Then the process has to be able // to serve again: the same Server is started a second time, binds four fresh // listeners and answers. The counters deliberately survive the restart: /status // is a lifetime view, and a counter that reset on restart would make the // console's time series jump to zero every time the pod rolled. func TestStopWithNoQueryInFlightReturnsAndStartsAgain(t *testing.T) { h := newHarness(t, nil) before := h.addr("udp") if err := stopWithin(t, h.srv, 2*time.Second); err != nil { t.Fatalf("Stop: %v", err) } if err := h.srv.Start(context.Background()); err != nil { t.Fatalf("Start after Stop: %v", err) } if got := len(h.srv.Listeners()); got != 4 { t.Errorf("Listeners() after the restart has %d entries, want 4", got) } res := h.ask("udp", "after-restart.test.", dns.TypeA) if a := answerAddr(res); a != "203.0.113.9" { t.Errorf("A after the restart = %q, want 203.0.113.9", a) } if got := h.fake.last().name; got != "after-restart.test." { t.Errorf("the restarted server did not reach the engine: last question %q", got) } if got := h.srv.ByTransport()["udp"].Queries; got != 1 { t.Errorf("udp.Queries = %d after the restart, want 1 (counters are a lifetime total)", got) } // The new listeners are different sockets: the harness binds port 0, so a // restart is only possible because Stop gave the old ones back. Asserting // it also keeps the test honest about which server answered. if after := h.addr("udp"); after == before { t.Logf("udp rebound the same address %s; port 0 makes this possible but unlikely", after) } // And the harness's own cleanup still stops the restarted server without // complaint, which is the t.Cleanup path every test in this package takes. if got := h.srv.Stats().Inflight; got != 0 { t.Errorf("Stats().Inflight = %d, want 0", got) } }