Coalesce identical misses, refuse non-QUERY opcodes, refresh hot entries early, check the answered question

Concurrent cache misses for one name each went upstream. After a restart, a
cache flush or a cluster-object change that is a burst of identical queries
exactly when the cache is empty. Identical exchanges (same name, type, class,
DO bit, pinned upstream set and forwarded client subnet) now share one: the
first query runs it on its own goroutine and context, the rest wait and get
private copies, and the leader alone writes the cache. If the leader's client
leaves, waiters take over instead of failing with its cancellation. Shared
answers are counted in fancydns_upstream_coalesced_total.

validate() accepted OpcodeUpdate, so a dynamic update was forwarded upstream as
an ordinary lookup, and every other opcode got FORMERR. A recursive resolver
implements QUERY only: anything else is now NOTIMP (RFC 1035 4.1.1), checked
before the question count since STATUS carries no question.

Prefetch only fired when a stale entry was served, so the first client after a
hot name expired got a TTL-1 stale answer. A fresh entry is now refreshed in
the background when it has been served at least twice and under a tenth of its
lifetime remains; the hit counter restarts with each stored entry, so a name
nobody comes back for stops being refreshed. Counted in
fancydns_cache_prefetch_total.

miekg/dns matches only the message ID of an upstream reply, so a forged reply
with the right ID for a different question was accepted and cached under the
name that was asked. exchangeOne now requires the reply to carry the question
that was sent (an empty section only for FORMERR, SERVFAIL, NOTIMP and
REFUSED). A mismatch fails the attempt without counting against the upstream,
so a spoofer cannot trip the circuit breaker.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

internal/engine/engine.go

@@ -130,6 +130,8 @@ type Engine struct {
 
 	// inflight is the current in-flight count for the gauge.
 	inflight atomic.Int64
+	// flights coalesces identical concurrent upstream exchanges.
+	flights flightGroup
 	// dropped counts queries rejected by the concurrency cap.
 	dropped atomic.Uint64
 
@@ -283,6 +285,9 @@ func (e *Engine) Handle(ctx context.Context, req Request) Response {
 	// else looks at it.
 	if err := validate(q); err != nil {
 		res.Msg = formErr(q)
+		if err == errNotImplemented {
+			res.Msg = notImpl(q)
+		}
 		res.Action = qlog.ActionError
 		e.finish(req, res, start, decision)
 		return res
@@ -433,7 +438,14 @@ func (e *Engine) Handle(ctx context.Context, req Request) Response {
 	}
 	fwd := e.prepareUpstream(q, req)
 	upStart := time.Now()
-	up, addr, err := pool.Exchange(ctx, fwd, decision.Upstreams, time.Duration(e.upstreamTimeout.Load()))
+	// Identical misses in flight share one upstream exchange. The leader does
+	// the work (and the cache write below, via the shared path); the rest wait.
+	up, addr, shared, err := e.flights.do(ctx, upstreamFlightKey(fwd, decision.Upstreams), func() (*dns.Msg, string, error) {
+		return pool.Exchange(ctx, fwd, decision.Upstreams, time.Duration(e.upstreamTimeout.Load()))
+	})
+	if shared {
+		metrics.RecordCoalesced()
+	}
 	res.Duration = time.Since(start)
 	if err != nil || up == nil {
 		metrics.RecordUpstreamError()
@@ -442,7 +454,9 @@ func (e *Engine) Handle(ctx context.Context, req Request) Response {
 		e.finish(req, res, start, decision)
 		return res
 	}
-	metrics.RecordUpstreamLatency(addr, time.Since(upStart))
+	if !shared {
+		metrics.RecordUpstreamLatency(addr, time.Since(upStart))
+	}
 	// finalize is what makes the answer safe to put on the wire: it echoes the
 	// client's question and id verbatim and truncates to the client's
 	// advertised buffer. Without it an uncached multi-kilobyte answer left
@@ -464,7 +478,7 @@ func (e *Engine) Handle(ctx context.Context, req Request) Response {
 		key := cache.QueryKey(qname, qtype, q.Question[0].Qclass, cache.Do(q), 0)
 		// Cache the untruncated upstream answer so a small client later does
 		// not poison the entry with a truncated copy.
-		if !up.Truncated {
+		if !up.Truncated && !shared {
 			e.cache.PutResponse(key, up, addr)
 		}
 	}
@@ -912,10 +926,14 @@ func validate(q *dns.Msg) error {
 	if q.Response {
 		return errMalformed
 	}
-	if len(q.Question) != 1 {
-		return errMalformed
+	// A recursive resolver implements QUERY and nothing else. UPDATE and
+	// NOTIFY are for authoritative servers: forwarding them upstream as lookups
+	// is wrong, and so is answering FORMERR (RFC 1035 §4.1.1: NOTIMP).
+	// Checked before the question count: STATUS carries no question.
+	if q.Opcode != dns.OpcodeQuery {
+		return errNotImplemented
 	}
-	if q.Opcode != dns.OpcodeQuery && q.Opcode != dns.OpcodeUpdate {
+	if len(q.Question) != 1 {
 		return errMalformed
 	}
 	name := q.Question[0].Name
@@ -925,7 +943,10 @@ func validate(q *dns.Msg) error {
 	return nil
 }
 
-var errMalformed = errString("malformed query")
+var (
+	errMalformed      = errString("malformed query")
+	errNotImplemented = errString("opcode not implemented")
+)
 
 type errString string
 
@@ -937,6 +958,12 @@ func formErr(q *dns.Msg) *dns.Msg {
 	return m
 }
 
+func notImpl(q *dns.Msg) *dns.Msg {
+	m := new(dns.Msg)
+	m.SetRcode(q, dns.RcodeNotImplemented)
+	return m
+}
+
 func servFail(q *dns.Msg, reason string) *dns.Msg {
 	m := new(dns.Msg)
 	m.SetRcode(q, dns.RcodeServerFailure)
@@ -1200,3 +1227,22 @@ func (e *Engine) SetFlusher(fn func()) {
 	}
 	e.flusher.Store(&fn)
 }
+
+// upstreamFlightKey names the exchange fwd would make, so two queries share an
+// upstream round trip only when the upstream would give them the same answer.
+func upstreamFlightKey(fwd *dns.Msg, ups []string) flightKey {
+	qn := fwd.Question[0]
+	k := flightKey{
+		name: strings.ToLower(qn.Name), qtype: qn.Qtype, qclass: qn.Qclass,
+		ups: strings.Join(ups, ","),
+	}
+	if o := fwd.IsEdns0(); o != nil {
+		k.do = o.Do()
+		for _, opt := range o.Option {
+			if sub, ok := opt.(*dns.EDNS0_SUBNET); ok {
+				k.ecs = sub.String()
+			}
+		}
+	}
+	return k
+}