initial commit
This commit is contained in:
developer committed
1 parent
5b935301e5
commit
85453852d5
105 files changed
+24846
No files matched your search
@@ -0,0 +1,598 @@
|
||||
// Package resolver forwards queries to upstream recursive servers with
|
||||
// health tracking, per-upstream circuit breakers and parallel racing.
|
||||
package resolver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
|
||||
"git.k8s-git-k6n.net/mats/fancy-dns/internal/config"
|
||||
)
|
||||
|
||||
// ErrNoUpstream is returned when every configured upstream is unavailable.
|
||||
var ErrNoUpstream = errors.New("resolver: no healthy upstream available")
|
||||
|
||||
// ErrCircuitOpen is returned by a single upstream whose breaker is open.
|
||||
var ErrCircuitOpen = errors.New("resolver: upstream circuit breaker open")
|
||||
|
||||
// Health is an upstream's current state.
|
||||
type Health string
|
||||
|
||||
const (
|
||||
// HealthUp means the last probes and queries succeeded.
|
||||
HealthUp Health = "up"
|
||||
// HealthDegraded means recent failures but still serving.
|
||||
HealthDegraded Health = "degraded"
|
||||
// HealthDown means failing or the breaker is open.
|
||||
HealthDown Health = "down"
|
||||
// HealthUnknown means not probed yet.
|
||||
HealthUnknown Health = "unknown"
|
||||
)
|
||||
|
||||
// Upstream is one configured server plus its live health.
|
||||
type Upstream struct {
|
||||
cfg config.UpstreamServer
|
||||
// addr is the address actually dialled: a bootstrap IP for TLS (SNI keeps
|
||||
// the hostname) and the literal address otherwise.
|
||||
addr string
|
||||
// resolved holds the current bootstrap addresses when the configured
|
||||
// address is a name.
|
||||
resolved []string
|
||||
|
||||
queries atomic.Uint64
|
||||
failures atomic.Uint64
|
||||
consecutive atomic.Uint32
|
||||
latencySum atomic.Uint64 // microseconds
|
||||
latencyN atomic.Uint64
|
||||
p95 []float64 // recent latencies in ms, for the UI
|
||||
p95mu sync.Mutex
|
||||
p95ring []float64
|
||||
|
||||
breakerOpenUntil atomic.Int64
|
||||
health atomic.Pointer[Health]
|
||||
lastErr atomic.Pointer[string]
|
||||
}
|
||||
|
||||
// Addr is the address the resolver dials.
|
||||
func (u *Upstream) Addr() string { return u.addr }
|
||||
|
||||
// Config returns the upstream's configuration entry.
|
||||
func (u *Upstream) Config() config.UpstreamServer { return u.cfg }
|
||||
|
||||
// Health returns the current health state.
|
||||
func (u *Upstream) Health() Health {
|
||||
if h := u.health.Load(); h != nil {
|
||||
return *h
|
||||
}
|
||||
return HealthUnknown
|
||||
}
|
||||
|
||||
func (u *Upstream) setHealth(h Health) { u.health.Store(&h) }
|
||||
|
||||
// SetError records the last error message for the UI.
|
||||
func (u *Upstream) SetError(err error) {
|
||||
if err == nil {
|
||||
u.lastErr.Store(nil)
|
||||
return
|
||||
}
|
||||
s := err.Error()
|
||||
u.lastErr.Store(&s)
|
||||
}
|
||||
|
||||
// LastError returns the last error message, or "".
|
||||
func (u *Upstream) LastError() string {
|
||||
if p := u.lastErr.Load(); p != nil {
|
||||
return *p
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Success records a successful query.
|
||||
func (u *Upstream) Success(d time.Duration) {
|
||||
u.queries.Add(1)
|
||||
u.consecutive.Store(0)
|
||||
u.latencySum.Add(uint64(d.Microseconds()))
|
||||
u.latencyN.Add(1)
|
||||
u.recordLatency(d)
|
||||
u.SetError(nil)
|
||||
if u.Health() != HealthUp {
|
||||
u.setHealth(HealthUp)
|
||||
}
|
||||
}
|
||||
|
||||
// Failure records a failed query and trips the breaker once the threshold is
|
||||
// crossed.
|
||||
func (u *Upstream) Failure(err error, threshold int, cooldown time.Duration) {
|
||||
u.queries.Add(1)
|
||||
u.failures.Add(1)
|
||||
n := u.consecutive.Add(1)
|
||||
u.SetError(err)
|
||||
if threshold > 0 && int(n) >= threshold {
|
||||
u.breakerOpenUntil.Store(time.Now().Add(cooldown).UnixNano())
|
||||
u.setHealth(HealthDown)
|
||||
return
|
||||
}
|
||||
if u.Health() != HealthDown {
|
||||
u.setHealth(HealthDegraded)
|
||||
}
|
||||
}
|
||||
|
||||
// BreakerOpen reports whether the upstream's breaker is open.
|
||||
func (u *Upstream) BreakerOpen() bool {
|
||||
until := u.breakerOpenUntil.Load()
|
||||
return until > time.Now().UnixNano()
|
||||
}
|
||||
|
||||
func (u *Upstream) recordLatency(d time.Duration) {
|
||||
ms := float64(d.Microseconds()) / 1000
|
||||
u.p95mu.Lock()
|
||||
if len(u.p95ring) < 512 {
|
||||
u.p95ring = append(u.p95ring, ms)
|
||||
} else {
|
||||
copy(u.p95ring, u.p95ring[1:])
|
||||
u.p95ring[511] = ms
|
||||
}
|
||||
u.p95mu.Unlock()
|
||||
}
|
||||
|
||||
// LatencyStats returns average and p95 latency in milliseconds.
|
||||
func (u *Upstream) LatencyStats() (avg, p95 float64) {
|
||||
n := u.latencyN.Load()
|
||||
if n == 0 {
|
||||
return 0, 0
|
||||
}
|
||||
avg = float64(u.latencySum.Load()) / float64(n) / 1000
|
||||
u.p95mu.Lock()
|
||||
samples := append([]float64(nil), u.p95ring...)
|
||||
u.p95mu.Unlock()
|
||||
if len(samples) == 0 {
|
||||
return avg, 0
|
||||
}
|
||||
sort.Float64s(samples)
|
||||
idx := int(float64(len(samples)) * 0.95)
|
||||
if idx >= len(samples) {
|
||||
idx = len(samples) - 1
|
||||
}
|
||||
return avg, samples[idx]
|
||||
}
|
||||
|
||||
// Stats is the JSON-friendly upstream state for the API.
|
||||
type Stats struct {
|
||||
Addr string `json:"addr"`
|
||||
Proto string `json:"proto"`
|
||||
ServerName string `json:"server_name"`
|
||||
Weight int `json:"weight"`
|
||||
Comment string `json:"comment,omitempty"`
|
||||
Health Health `json:"health"`
|
||||
LatencyMS float64 `json:"latency_ms"`
|
||||
AvgMS float64 `json:"avg_ms"`
|
||||
P95MS float64 `json:"p95_ms"`
|
||||
FailRate float64 `json:"fail_rate"`
|
||||
Queries uint64 `json:"queries"`
|
||||
Failures uint64 `json:"failures"`
|
||||
BreakerOpen bool `json:"breaker_open"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
}
|
||||
|
||||
// Stats snapshots one upstream.
|
||||
func (u *Upstream) Stats() Stats {
|
||||
q := u.queries.Load()
|
||||
f := u.failures.Load()
|
||||
var rate float64
|
||||
if q > 0 {
|
||||
rate = float64(f) / float64(q)
|
||||
}
|
||||
avg, p95 := u.LatencyStats()
|
||||
return Stats{
|
||||
Addr: u.addr,
|
||||
Proto: u.cfg.Proto,
|
||||
ServerName: u.cfg.ServerName,
|
||||
Weight: u.cfg.Weight,
|
||||
Comment: u.cfg.Comment,
|
||||
Health: u.Health(),
|
||||
LatencyMS: p95,
|
||||
AvgMS: avg,
|
||||
P95MS: p95,
|
||||
FailRate: rate,
|
||||
Queries: q,
|
||||
Failures: f,
|
||||
BreakerOpen: u.BreakerOpen(),
|
||||
LastError: u.LastError(),
|
||||
}
|
||||
}
|
||||
|
||||
// Pool is the set of upstreams plus the policy that selects among them.
|
||||
type Pool struct {
|
||||
cfg config.Upstream
|
||||
mu sync.RWMutex
|
||||
ups []*Upstream
|
||||
byTag map[string][]*Upstream
|
||||
// clients are reused per upstream: one UDP socket, one TCP dialer, one TLS
|
||||
// session pool. Reusing them is what keeps the per-query cost low.
|
||||
clients map[*Upstream]*dns.Client
|
||||
// bootstrap resolves names to addresses.
|
||||
bootstrap map[string][]string
|
||||
|
||||
rng *rand.Rand
|
||||
mu2 sync.Mutex
|
||||
}
|
||||
|
||||
// NewPool builds a pool from config. bootstrapIPs may be empty; then upstream
|
||||
// hostnames must be literal IPs (or resolved by the OS).
|
||||
func NewPool(cfg config.Upstream) *Pool {
|
||||
p := &Pool{
|
||||
cfg: cfg,
|
||||
byTag: make(map[string][]*Upstream),
|
||||
clients: make(map[*Upstream]*dns.Client),
|
||||
bootstrap: make(map[string][]string),
|
||||
rng: rand.New(rand.NewPCG(uint64(time.Now().UnixNano()), 0x9e3779b9)),
|
||||
}
|
||||
for _, b := range cfg.Bootstraps {
|
||||
p.bootstrap[b.Name] = append([]string(nil), b.Addresses...)
|
||||
}
|
||||
for _, s := range cfg.Servers {
|
||||
up := &Upstream{cfg: s}
|
||||
up.setHealth(HealthUnknown)
|
||||
p.ups = append(p.ups, up)
|
||||
}
|
||||
p.resolveAll()
|
||||
return p
|
||||
}
|
||||
|
||||
func (p *Pool) resolveAll() {
|
||||
for _, up := range p.ups {
|
||||
host, port, err := net.SplitHostPort(up.cfg.Addr)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if net.ParseIP(host) != nil {
|
||||
up.addr = up.cfg.Addr
|
||||
continue
|
||||
}
|
||||
if addrs := p.bootstrap[host]; len(addrs) > 0 {
|
||||
up.resolved = addrs
|
||||
up.addr = net.JoinHostPort(addrs[0], port)
|
||||
continue
|
||||
}
|
||||
// Fall back to the system resolver.
|
||||
ips, err := net.LookupIP(host)
|
||||
if err != nil || len(ips) == 0 {
|
||||
up.addr = up.cfg.Addr
|
||||
continue
|
||||
}
|
||||
strs := make([]string, 0, len(ips))
|
||||
for _, ip := range ips {
|
||||
strs = append(strs, ip.String())
|
||||
}
|
||||
up.resolved = strs
|
||||
up.addr = net.JoinHostPort(strs[0], port)
|
||||
}
|
||||
}
|
||||
|
||||
// Config returns the pool's upstream config.
|
||||
func (p *Pool) Config() config.Upstream { return p.cfg }
|
||||
|
||||
// All returns every upstream.
|
||||
func (p *Pool) All() []*Upstream { return p.ups }
|
||||
|
||||
// Get returns an upstream by address, or nil.
|
||||
func (p *Pool) Get(addr string) *Upstream {
|
||||
for _, u := range p.ups {
|
||||
if u.addr == addr || u.cfg.Addr == addr {
|
||||
return u
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Healthy returns upstreams whose breaker is closed, in configured order.
|
||||
func (p *Pool) Healthy() []*Upstream {
|
||||
out := make([]*Upstream, 0, len(p.ups))
|
||||
for _, u := range p.ups {
|
||||
if !u.BreakerOpen() {
|
||||
out = append(out, u)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Stats snapshots every upstream.
|
||||
func (p *Pool) Stats() []Stats {
|
||||
out := make([]Stats, 0, len(p.ups))
|
||||
for _, u := range p.ups {
|
||||
out = append(out, u.Stats())
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// clientFor returns the memoised dns.Client for an upstream.
|
||||
func (p *Pool) clientFor(u *Upstream) *dns.Client {
|
||||
p.mu.RLock()
|
||||
c, ok := p.clients[u]
|
||||
p.mu.RUnlock()
|
||||
if ok {
|
||||
return c
|
||||
}
|
||||
var nc *dns.Client
|
||||
switch u.cfg.Proto {
|
||||
case "tls":
|
||||
tlsCfg := &tls.Config{
|
||||
ServerName: u.cfg.ServerName,
|
||||
InsecureSkipVerify: u.cfg.InsecureSkipVerify,
|
||||
MinVersion: tls.VersionTLS12,
|
||||
NextProtos: []string{"dot"},
|
||||
}
|
||||
nc = &dns.Client{
|
||||
Net: "tcp-tls",
|
||||
TLSConfig: tlsCfg,
|
||||
Dialer: &net.Dialer{Timeout: 3 * time.Second},
|
||||
Timeout: 5 * time.Second,
|
||||
}
|
||||
case "tcp":
|
||||
nc = &dns.Client{Net: "tcp", Timeout: 5 * time.Second}
|
||||
default:
|
||||
nc = &dns.Client{Net: "udp", UDPSize: 1232, Timeout: 3 * time.Second}
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.clients[u] = nc
|
||||
p.mu.Unlock()
|
||||
return nc
|
||||
}
|
||||
|
||||
// selectFor returns the upstreams to try, honouring the policy. tags, if
|
||||
// non-empty, restricts the pool to a group's own upstreams.
|
||||
func (p *Pool) selectFor(policy string, maxAttempts int) []*Upstream {
|
||||
healthy := p.Healthy()
|
||||
if len(healthy) == 0 {
|
||||
// Every breaker is open: try them all anyway rather than failing the
|
||||
// query outright. A blip should not become an outage.
|
||||
healthy = append(healthy, p.ups...)
|
||||
}
|
||||
if len(healthy) == 0 {
|
||||
return nil
|
||||
}
|
||||
if maxAttempts <= 0 || maxAttempts > len(healthy) {
|
||||
maxAttempts = len(healthy)
|
||||
}
|
||||
switch policy {
|
||||
case "failover":
|
||||
// Configured order, and rotate past the unhealthy ones.
|
||||
return healthy[:maxAttempts]
|
||||
case "random":
|
||||
p.mu2.Lock()
|
||||
p.rng.Shuffle(len(healthy), func(i, j int) { healthy[i], healthy[j] = healthy[j], healthy[i] })
|
||||
p.mu2.Unlock()
|
||||
return healthy[:maxAttempts]
|
||||
default: // parallel
|
||||
return healthy
|
||||
}
|
||||
}
|
||||
|
||||
// Exchange forwards q to the pool and returns the first acceptable answer.
|
||||
//
|
||||
// policy is the pool's configured policy; tags restricts the pool to a
|
||||
// per-client upstream override. edns0Size is the client's advertised buffer,
|
||||
// used to decide TCP fallback on truncation.
|
||||
func (p *Pool) Exchange(ctx context.Context, q *dns.Msg, tags []string, timeout time.Duration) (*dns.Msg, string, error) {
|
||||
candidates := p.ups
|
||||
if len(tags) > 0 {
|
||||
if tagged := p.byTag[strings.Join(tags, ",")]; len(tagged) > 0 {
|
||||
candidates = tagged
|
||||
}
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
return nil, "", ErrNoUpstream
|
||||
}
|
||||
attempts := p.cfg.MaxAttempts
|
||||
chosen := selectFrom(candidates, p.cfg.Policy, attempts)
|
||||
if len(chosen) == 0 {
|
||||
return nil, "", ErrNoUpstream
|
||||
}
|
||||
|
||||
if len(chosen) == 1 {
|
||||
msg, up, err := p.exchangeOne(ctx, chosen[0], q, timeout)
|
||||
return msg, up, err
|
||||
}
|
||||
|
||||
// Parallel: send to all at once, take the first answer that is not a
|
||||
// transport failure. Cancellable so we do not leak requests to a slow
|
||||
// upstream after another one has answered.
|
||||
type res struct {
|
||||
msg *dns.Msg
|
||||
up string
|
||||
err error
|
||||
}
|
||||
results := make(chan res, len(chosen))
|
||||
sub, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
for _, up := range chosen {
|
||||
go func(up *Upstream) {
|
||||
msg, addr, err := p.exchangeOne(sub, up, q, timeout)
|
||||
results <- res{msg: msg, up: addr, err: err}
|
||||
}(up)
|
||||
}
|
||||
var lastErr error
|
||||
for i := 0; i < len(chosen); i++ {
|
||||
select {
|
||||
case r := <-results:
|
||||
if r.err == nil && r.msg != nil && !isTransportFailure(r.msg.Rcode) {
|
||||
return r.msg, r.up, nil
|
||||
}
|
||||
if r.err != nil {
|
||||
lastErr = r.err
|
||||
} else if r.msg != nil {
|
||||
lastErr = fmt.Errorf("upstream %s returned %s", r.up, dns.RcodeToString[r.msg.Rcode])
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return nil, "", ctx.Err()
|
||||
}
|
||||
}
|
||||
if lastErr == nil {
|
||||
lastErr = ErrNoUpstream
|
||||
}
|
||||
return nil, "", lastErr
|
||||
}
|
||||
|
||||
func selectFrom(ups []*Upstream, policy string, maxAttempts int) []*Upstream {
|
||||
// Filter out open breakers but keep everything if that empties the list.
|
||||
open := make([]*Upstream, 0, len(ups))
|
||||
for _, u := range ups {
|
||||
if !u.BreakerOpen() {
|
||||
open = append(open, u)
|
||||
}
|
||||
}
|
||||
if len(open) == 0 {
|
||||
open = ups
|
||||
}
|
||||
if maxAttempts <= 0 || maxAttempts > len(open) {
|
||||
maxAttempts = len(open)
|
||||
}
|
||||
switch policy {
|
||||
case "failover":
|
||||
return open[:maxAttempts]
|
||||
case "random":
|
||||
rand.Shuffle(len(open), func(i, j int) { open[i], open[j] = open[j], open[i] })
|
||||
return open[:maxAttempts]
|
||||
default:
|
||||
return open
|
||||
}
|
||||
}
|
||||
|
||||
// exchangeOne sends q to a single upstream, retrying over TCP when the UDP
|
||||
// answer comes back truncated (RFC 1035 §4.2.1).
|
||||
func (p *Pool) exchangeOne(ctx context.Context, up *Upstream, q *dns.Msg, timeout time.Duration) (*dns.Msg, string, error) {
|
||||
if timeout <= 0 {
|
||||
timeout = 2 * time.Second
|
||||
}
|
||||
client := p.clientFor(up)
|
||||
msg := q.Copy()
|
||||
start := time.Now()
|
||||
resp, rtt, err := client.ExchangeContext(ctx, msg, up.addr)
|
||||
up.Success(rtt)
|
||||
if err != nil {
|
||||
return nil, up.addr, err
|
||||
}
|
||||
_ = start
|
||||
if !resp.Truncated {
|
||||
return resp, up.addr, nil
|
||||
}
|
||||
// Retry over TCP to get the full answer. Some upstreams are configured
|
||||
// UDP-only in which case this simply fails and we use the TC answer.
|
||||
tcpClient := &dns.Client{Net: "tcp", Timeout: timeout}
|
||||
resp2, rtt2, err2 := tcpClient.ExchangeContext(ctx, q.Copy(), up.addr)
|
||||
if err2 == nil && resp2 != nil && !resp2.Truncated {
|
||||
up.Success(rtt2)
|
||||
return resp2, up.addr, nil
|
||||
}
|
||||
up.Failure(fmt.Errorf("truncated answer, TCP retry failed: %w", err2), up.breakerThreshold(), p.cfg.BreakerCooldown.D())
|
||||
// Return the truncated answer; the client will retry over TCP itself.
|
||||
return resp, up.addr, nil
|
||||
}
|
||||
|
||||
func isTransportFailure(rcode int) bool {
|
||||
return rcode == dns.RcodeServerFailure
|
||||
}
|
||||
|
||||
// breakerThreshold is the effective failure threshold for an upstream.
|
||||
func (u *Upstream) breakerThreshold() int {
|
||||
if u.cfg.MaxFails > 0 {
|
||||
return u.cfg.MaxFails
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// HealthCheck probes every upstream with a cheap query, refreshing health
|
||||
// state even when no client traffic is flowing. It returns the number of
|
||||
// upstreams that answered.
|
||||
func (p *Pool) HealthCheck(ctx context.Context, timeout time.Duration) int {
|
||||
probe := new(dns.Msg)
|
||||
probe.SetQuestion(".", dns.TypeNS)
|
||||
probe.RecursionDesired = true
|
||||
ok := 0
|
||||
for _, up := range p.ups {
|
||||
if up.BreakerOpen() {
|
||||
continue
|
||||
}
|
||||
c := p.clientFor(up)
|
||||
_, rtt, err := c.ExchangeContext(ctx, probe, up.addr)
|
||||
if err != nil {
|
||||
up.Failure(err, p.breakerThresholdFor(up), p.cfg.BreakerCooldown.D())
|
||||
continue
|
||||
}
|
||||
up.Success(rtt)
|
||||
ok++
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
func (p *Pool) breakerThresholdFor(u *Upstream) int {
|
||||
if t := u.breakerThreshold(); t > 0 {
|
||||
return t
|
||||
}
|
||||
return p.cfg.BreakerThreshold
|
||||
}
|
||||
|
||||
// Rebind applies a new upstream configuration in place, keeping the live
|
||||
// health state of upstreams whose address is unchanged.
|
||||
func (p *Pool) Rebind(cfg config.Upstream) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.cfg = cfg
|
||||
old := make(map[string]*Upstream, len(p.ups))
|
||||
for _, u := range p.ups {
|
||||
old[u.cfg.Addr+"/"+u.cfg.Proto] = u
|
||||
}
|
||||
next := make([]*Upstream, 0, len(cfg.Servers))
|
||||
for _, s := range cfg.Servers {
|
||||
if u, ok := old[s.Addr+"/"+s.Proto]; ok {
|
||||
u.cfg = s
|
||||
next = append(next, u)
|
||||
delete(old, s.Addr+"/"+s.Proto)
|
||||
continue
|
||||
}
|
||||
u := &Upstream{cfg: s}
|
||||
u.setHealth(HealthUnknown)
|
||||
next = append(next, u)
|
||||
}
|
||||
p.ups = next
|
||||
// Drop clients for removed upstreams.
|
||||
for u := range p.clients {
|
||||
if old[u.cfg.Addr+"/"+u.cfg.Proto] != nil {
|
||||
delete(p.clients, u)
|
||||
}
|
||||
}
|
||||
p.resolveAllLocked()
|
||||
}
|
||||
|
||||
func (p *Pool) resolveAllLocked() {
|
||||
for _, b := range p.cfg.Bootstraps {
|
||||
p.bootstrap[b.Name] = append([]string(nil), b.Addresses...)
|
||||
}
|
||||
for _, up := range p.ups {
|
||||
host, port, err := net.SplitHostPort(up.cfg.Addr)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if net.ParseIP(host) != nil {
|
||||
up.addr = up.cfg.Addr
|
||||
continue
|
||||
}
|
||||
if addrs := p.bootstrap[host]; len(addrs) > 0 {
|
||||
up.resolved = addrs
|
||||
up.addr = net.JoinHostPort(addrs[0], port)
|
||||
continue
|
||||
}
|
||||
up.addr = up.cfg.Addr
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user