|
| 1 | +package dns |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "fmt" |
| 6 | + "net" |
| 7 | + "strings" |
| 8 | + "sync" |
| 9 | + "time" |
| 10 | +) |
| 11 | + |
| 12 | +// Pool health thresholds (#69). |
| 13 | +const ( |
| 14 | + poolHealthMinLookups = 20 // lookups before a resolver can be benched |
| 15 | + poolHealthFailPct = 50 // failure share that benches a resolver |
| 16 | + poolBenchFor = 30 * time.Second // how long a benched resolver sits out |
| 17 | +) |
| 18 | + |
| 19 | +// PoolResolver is one member of a Pool. |
| 20 | +type PoolResolver struct { |
| 21 | + Addr string |
| 22 | + resolver *net.Resolver |
| 23 | + |
| 24 | + mu sync.Mutex |
| 25 | + stats ResolverStats |
| 26 | + window int // lookups since the last health reset |
| 27 | + windowKO int // failures in that window |
| 28 | + until time.Time |
| 29 | +} |
| 30 | + |
| 31 | +// ResolverStats is one pool member's accounting for the run-quality report. |
| 32 | +type ResolverStats struct { |
| 33 | + Addr string `json:"address"` |
| 34 | + Lookups int64 `json:"lookups"` |
| 35 | + Found int64 `json:"found"` |
| 36 | + NXDomain int64 `json:"nxdomain"` |
| 37 | + Failed int64 `json:"failed"` // timeout, refused or other |
| 38 | + Benched int64 `json:"benched"` // times it was taken out of rotation |
| 39 | +} |
| 40 | + |
| 41 | +// Pool spreads lookups over several resolvers, round-robin over the healthy |
| 42 | +// ones. A resolver whose recent failure rate passes poolHealthFailPct is |
| 43 | +// benched for poolBenchFor, then given another chance. |
| 44 | +type Pool struct { |
| 45 | + members []*PoolResolver |
| 46 | + mu sync.Mutex |
| 47 | + next int |
| 48 | + now func() time.Time |
| 49 | +} |
| 50 | + |
| 51 | +// ParseResolverList parses resolver addresses, one per line or entry, as |
| 52 | +// ip or ip:port (port 53 by default). Blank lines and # comments are skipped. |
| 53 | +func ParseResolverList(lines []string) ([]string, error) { |
| 54 | + var out []string |
| 55 | + seen := map[string]bool{} |
| 56 | + for _, line := range lines { |
| 57 | + s := strings.TrimSpace(line) |
| 58 | + if s == "" || strings.HasPrefix(s, "#") { |
| 59 | + continue |
| 60 | + } |
| 61 | + if net.ParseIP(s) != nil { |
| 62 | + s = net.JoinHostPort(s, "53") |
| 63 | + } |
| 64 | + host, port, err := net.SplitHostPort(s) |
| 65 | + if err != nil || net.ParseIP(host) == nil || port == "" { |
| 66 | + return nil, fmt.Errorf("invalid resolver %q: want ip or ip:port", line) |
| 67 | + } |
| 68 | + if !seen[s] { |
| 69 | + seen[s] = true |
| 70 | + out = append(out, s) |
| 71 | + } |
| 72 | + } |
| 73 | + return out, nil |
| 74 | +} |
| 75 | + |
| 76 | +// NewPool builds a pool of resolvers from NewResolver. |
| 77 | +func NewPool(addrs []string, timeout time.Duration) *Pool { |
| 78 | + p := &Pool{now: time.Now} |
| 79 | + for _, a := range addrs { |
| 80 | + p.members = append(p.members, &PoolResolver{Addr: a, resolver: NewResolver(timeout, a), stats: ResolverStats{Addr: a}}) |
| 81 | + } |
| 82 | + return p |
| 83 | +} |
| 84 | + |
| 85 | +// Size returns the number of resolvers in the pool. |
| 86 | +func (p *Pool) Size() int { return len(p.members) } |
| 87 | + |
| 88 | +// pick returns the next healthy resolver, skipping avoid when possible. When |
| 89 | +// every resolver is benched, the one whose bench ends soonest is used. |
| 90 | +func (p *Pool) pick(avoid *PoolResolver) *PoolResolver { |
| 91 | + p.mu.Lock() |
| 92 | + defer p.mu.Unlock() |
| 93 | + now := p.now() |
| 94 | + n := len(p.members) |
| 95 | + var fallback *PoolResolver |
| 96 | + var fallbackUntil time.Time |
| 97 | + for i := 0; i < n; i++ { |
| 98 | + m := p.members[(p.next+i)%n] |
| 99 | + m.mu.Lock() |
| 100 | + until := m.until |
| 101 | + m.mu.Unlock() |
| 102 | + if now.Before(until) { |
| 103 | + if fallback == nil || until.Before(fallbackUntil) { |
| 104 | + fallback, fallbackUntil = m, until |
| 105 | + } |
| 106 | + continue |
| 107 | + } |
| 108 | + if m == avoid && n > 1 { |
| 109 | + continue |
| 110 | + } |
| 111 | + p.next = (p.next + i + 1) % n |
| 112 | + return m |
| 113 | + } |
| 114 | + if fallback != nil { |
| 115 | + return fallback |
| 116 | + } |
| 117 | + return avoid |
| 118 | +} |
| 119 | + |
| 120 | +// report records a lookup's outcome and benches the resolver if it is failing. |
| 121 | +func (p *Pool) report(m *PoolResolver, o Outcome) { |
| 122 | + if o == OutcomeCanceled { |
| 123 | + return |
| 124 | + } |
| 125 | + m.mu.Lock() |
| 126 | + defer m.mu.Unlock() |
| 127 | + m.stats.Lookups++ |
| 128 | + m.window++ |
| 129 | + switch o { |
| 130 | + case OutcomeFound: |
| 131 | + m.stats.Found++ |
| 132 | + case OutcomeNXDomain: |
| 133 | + m.stats.NXDomain++ |
| 134 | + default: |
| 135 | + m.stats.Failed++ |
| 136 | + m.windowKO++ |
| 137 | + } |
| 138 | + if m.window >= poolHealthMinLookups { |
| 139 | + if m.windowKO*100 > m.window*poolHealthFailPct { |
| 140 | + m.until = p.now().Add(poolBenchFor) |
| 141 | + m.stats.Benched++ |
| 142 | + } |
| 143 | + m.window, m.windowKO = 0, 0 |
| 144 | + } |
| 145 | +} |
| 146 | + |
| 147 | +// Resolve looks domain up through the pool, up to attempts times, each |
| 148 | +// attempt on a different healthy resolver where possible. It returns the |
| 149 | +// records, the outcome of the last attempt, and the resolver that answered. |
| 150 | +func (p *Pool) Resolve(ctx context.Context, domain string, timeout time.Duration, logf Logf, attempts int, types []string) ([]Record, Outcome, string) { |
| 151 | + var last *PoolResolver |
| 152 | + outcome := OutcomeOther |
| 153 | + for attempt := 0; attempt < max(attempts, 1); attempt++ { |
| 154 | + if ctx.Err() != nil { |
| 155 | + return nil, OutcomeCanceled, "" |
| 156 | + } |
| 157 | + m := p.pick(last) |
| 158 | + last = m |
| 159 | + var recs []Record |
| 160 | + recs, outcome = ResolveDomainWithRetry(ctx, m.resolver, domain, timeout, logf, 1, types) |
| 161 | + p.report(m, outcome) |
| 162 | + switch outcome { |
| 163 | + case OutcomeFound: |
| 164 | + return recs, outcome, m.Addr |
| 165 | + case OutcomeNXDomain, OutcomeCanceled: |
| 166 | + return nil, outcome, m.Addr |
| 167 | + } |
| 168 | + } |
| 169 | + return nil, outcome, "" |
| 170 | +} |
| 171 | + |
| 172 | +// Stats returns a snapshot of every member's accounting, in pool order. |
| 173 | +func (p *Pool) Stats() []ResolverStats { |
| 174 | + out := make([]ResolverStats, 0, len(p.members)) |
| 175 | + for _, m := range p.members { |
| 176 | + m.mu.Lock() |
| 177 | + out = append(out, m.stats) |
| 178 | + m.mu.Unlock() |
| 179 | + } |
| 180 | + return out |
| 181 | +} |
0 commit comments