|
| 1 | +package scan |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "fmt" |
| 6 | + "strings" |
| 7 | + "sync/atomic" |
| 8 | + "time" |
| 9 | +) |
| 10 | + |
| 11 | +// dispatcher owns candidate generation for one scan (#159): the expansion |
| 12 | +// queue, the visited set (loop and duplicate protection), the resume |
| 13 | +// watermark, the -max-queries and -max-duration budgets, and the count of |
| 14 | +// dispatched jobs not yet completed. Its methods are plain state changes, |
| 15 | +// called only from run's goroutine (or a test), so they need no locking. |
| 16 | +// |
| 17 | +// Candidates are generated lazily. The queue holds expansions (a parent plus |
| 18 | +// a cursor into Entries) rather than one job per name, so depth-1 names are |
| 19 | +// built only as they are dispatched and a found parent costs one enqueue |
| 20 | +// instead of len(Entries) (#49). Once a budget is reached the remaining |
| 21 | +// candidates are counted, never built or recorded (#51). |
| 22 | +type dispatcher struct { |
| 23 | + cfg Config |
| 24 | + n int // len(cfg.Entries) |
| 25 | + |
| 26 | + visited map[string]struct{} |
| 27 | + frontier []expansion |
| 28 | + head int // first unconsumed expansion in frontier |
| 29 | + remaining int // candidates not yet generated |
| 30 | + |
| 31 | + // Resume point: rootDone[i-start] is set once entry i is settled (looked |
| 32 | + // up, or skipped as a duplicate or out of scope); watermark is the first |
| 33 | + // index not yet settled. |
| 34 | + start int |
| 35 | + rootDone []bool |
| 36 | + watermark int |
| 37 | + |
| 38 | + admitted int // jobs handed out (or held) and counted in the total |
| 39 | + skipped int // candidates a budget kept from being tested |
| 40 | + excluded int // candidates -exclude kept out of scope |
| 41 | + capped bool |
| 42 | + expired bool // the budget that capped admission was the deadline |
| 43 | + pending int // dispatched jobs not yet completed |
| 44 | +} |
| 45 | + |
| 46 | +func newDispatcher(cfg Config, maxDepth int) *dispatcher { |
| 47 | + n := len(cfg.Entries) |
| 48 | + start := min(max(cfg.ResumeFrom, 0), n) |
| 49 | + d := &dispatcher{ |
| 50 | + cfg: cfg, |
| 51 | + n: n, |
| 52 | + visited: make(map[string]struct{}, n), |
| 53 | + frontier: []expansion{{parent: cfg.Domain, depth: 1, next: start}}, |
| 54 | + remaining: n - start, |
| 55 | + start: start, |
| 56 | + rootDone: make([]bool, n-start), |
| 57 | + watermark: start, |
| 58 | + } |
| 59 | + // Resumed recursive scans re-expand the names found before the |
| 60 | + // interrupt, whose children may not all have been tested (#73). |
| 61 | + if cfg.Recursive { |
| 62 | + rootLabels := strings.Count(cfg.Domain, ".") + 1 |
| 63 | + for _, p := range cfg.ResumeParents { |
| 64 | + p = strings.ToLower(strings.TrimSuffix(p, ".")) |
| 65 | + if !strings.HasSuffix(p, "."+cfg.Domain) { |
| 66 | + continue |
| 67 | + } |
| 68 | + if depth := strings.Count(p, ".") + 1 - rootLabels; depth >= 1 && depth < maxDepth { |
| 69 | + d.frontier = append(d.frontier, expansion{parent: p, depth: depth + 1}) |
| 70 | + d.remaining += n |
| 71 | + } |
| 72 | + } |
| 73 | + } |
| 74 | + return d |
| 75 | +} |
| 76 | + |
| 77 | +// settle marks wordlist entry i as done and advances the resume watermark. |
| 78 | +// i is -1 for jobs below depth 1, which do not affect it. |
| 79 | +func (d *dispatcher) settle(i int) { |
| 80 | + if i < d.start { |
| 81 | + return |
| 82 | + } |
| 83 | + d.rootDone[i-d.start] = true |
| 84 | + for d.watermark < d.n && d.rootDone[d.watermark-d.start] { |
| 85 | + d.watermark++ |
| 86 | + } |
| 87 | +} |
| 88 | + |
| 89 | +// stopAdmitting ends admission for a budget: every candidate not yet |
| 90 | +// generated, and every child a later find would add, is skipped. |
| 91 | +func (d *dispatcher) stopAdmitting() { |
| 92 | + d.skipped += d.remaining |
| 93 | + d.remaining = 0 |
| 94 | + d.capped = true |
| 95 | + d.frontier, d.head = nil, 0 |
| 96 | +} |
| 97 | + |
| 98 | +// total is the progress total: jobs admitted plus candidates not yet |
| 99 | +// generated, at most -max-queries. |
| 100 | +func (d *dispatcher) total() int64 { |
| 101 | + t := d.admitted + d.remaining |
| 102 | + if d.cfg.MaxQueries > 0 && t > d.cfg.MaxQueries { |
| 103 | + t = d.cfg.MaxQueries |
| 104 | + } |
| 105 | + return int64(t) |
| 106 | +} |
| 107 | + |
| 108 | +// next generates the next admissible job, if any. |
| 109 | +func (d *dispatcher) next() (job, bool) { |
| 110 | + for d.head < len(d.frontier) { |
| 111 | + e := &d.frontier[d.head] |
| 112 | + if e.next == d.n { |
| 113 | + d.head++ |
| 114 | + // Compact only once the consumed prefix is at least half the |
| 115 | + // slice, so each element is copied O(1) times. |
| 116 | + if d.head == len(d.frontier) { |
| 117 | + d.frontier, d.head = d.frontier[:0], 0 |
| 118 | + } else if d.head*2 >= len(d.frontier) { |
| 119 | + d.frontier, d.head = append(d.frontier[:0], d.frontier[d.head:]...), 0 |
| 120 | + } |
| 121 | + continue |
| 122 | + } |
| 123 | + if d.cfg.pastDeadline() { |
| 124 | + d.expired = true |
| 125 | + d.stopAdmitting() |
| 126 | + return job{}, false |
| 127 | + } |
| 128 | + if d.cfg.MaxQueries > 0 && d.admitted >= d.cfg.MaxQueries { |
| 129 | + d.stopAdmitting() |
| 130 | + return job{}, false |
| 131 | + } |
| 132 | + entry := d.cfg.Entries[e.next] |
| 133 | + e.next++ |
| 134 | + d.remaining-- |
| 135 | + name := entry + "." + e.parent |
| 136 | + // Key by the name relative to the root domain. For depth 1 that is |
| 137 | + // the entry itself, so no extra string is retained. |
| 138 | + key := entry |
| 139 | + if e.depth > 1 { |
| 140 | + key = name[:len(name)-len(d.cfg.Domain)-1] |
| 141 | + } |
| 142 | + root := -1 |
| 143 | + if e.depth == 1 { |
| 144 | + root = e.next - 1 |
| 145 | + } |
| 146 | + if _, dup := d.visited[key]; dup { |
| 147 | + d.settle(root) |
| 148 | + continue |
| 149 | + } |
| 150 | + d.visited[key] = struct{}{} |
| 151 | + if d.cfg.scope.excluded(name) { |
| 152 | + d.excluded++ |
| 153 | + d.settle(root) |
| 154 | + continue |
| 155 | + } |
| 156 | + d.admitted++ |
| 157 | + return job{domain: name, depth: e.depth, root: root}, true |
| 158 | + } |
| 159 | + return job{}, false |
| 160 | +} |
| 161 | + |
| 162 | +// expand queues the children of a found parent, unless a budget has stopped |
| 163 | +// admission or every child would be out of scope; then they are only counted. |
| 164 | +func (d *dispatcher) expand(parent job) { |
| 165 | + switch { |
| 166 | + case d.capped: |
| 167 | + d.skipped += d.n |
| 168 | + case d.cfg.scope.subtreeExcluded(parent.domain): |
| 169 | + d.excluded += d.n |
| 170 | + default: |
| 171 | + d.frontier = append(d.frontier, expansion{parent: parent.domain, depth: parent.depth + 1}) |
| 172 | + d.remaining += d.n |
| 173 | + } |
| 174 | +} |
| 175 | + |
| 176 | +// expire applies the deadline when its timer fires (#190). held reports |
| 177 | +// whether a job was admitted but not yet handed to a worker; it is skipped. |
| 178 | +func (d *dispatcher) expire(held bool) { |
| 179 | + if d.capped { |
| 180 | + return |
| 181 | + } |
| 182 | + d.expired = true |
| 183 | + if held { |
| 184 | + d.admitted-- |
| 185 | + d.skipped++ |
| 186 | + } |
| 187 | + d.stopAdmitting() |
| 188 | +} |
| 189 | + |
| 190 | +// complete records a worker's report on a dispatched job. |
| 191 | +func (d *dispatcher) complete(c completion) { |
| 192 | + d.pending-- |
| 193 | + if c.skipped { |
| 194 | + // Taken after the deadline and never looked up. |
| 195 | + d.skipped++ |
| 196 | + d.admitted-- |
| 197 | + if !d.capped { |
| 198 | + d.expired = true |
| 199 | + d.stopAdmitting() |
| 200 | + } |
| 201 | + } |
| 202 | + if c.finished { |
| 203 | + d.settle(c.root) |
| 204 | + } |
| 205 | +} |
| 206 | + |
| 207 | +// finish stores the dispatcher's counts in stats and announces skipped |
| 208 | +// candidates. A deadline that left nothing untested changed nothing, so it |
| 209 | +// does not mark the scan expired. |
| 210 | +func (d *dispatcher) finish(ctx context.Context, stats *counters, events chan<- Event) { |
| 211 | + stats.skipped.Store(int64(d.skipped)) |
| 212 | + stats.rootDone.Store(int64(d.watermark)) |
| 213 | + stats.excluded.Store(int64(d.excluded)) |
| 214 | + d.expired = d.expired && d.skipped > 0 |
| 215 | + stats.expired.Store(d.expired) |
| 216 | + if d.skipped == 0 { |
| 217 | + return |
| 218 | + } |
| 219 | + msg := fmt.Sprintf("query cap reached ({max_queries} %d); skipped %d additional jobs", d.cfg.MaxQueries, d.skipped) |
| 220 | + if d.expired { |
| 221 | + msg = fmt.Sprintf("time budget reached ({max_duration}); skipped %d additional jobs", d.skipped) |
| 222 | + } |
| 223 | + select { |
| 224 | + case events <- Event{Kind: EventNotice, Notice: NoticeCap, Message: msg}: |
| 225 | + case <-ctx.Done(): |
| 226 | + } |
| 227 | +} |
| 228 | + |
| 229 | +// run hands jobs to workers until no candidate is left and every dispatched |
| 230 | +// job has completed, or ctx is cancelled; then it closes jobs. Workers send |
| 231 | +// found parents on enqueue (before reporting on completed), and run expands |
| 232 | +// them. total receives the progress total as it changes. |
| 233 | +func (d *dispatcher) run(ctx context.Context, jobs chan<- job, enqueue <-chan job, completed <-chan completion, |
| 234 | + total *int64, stats *counters, events chan<- Event) { |
| 235 | + defer close(jobs) |
| 236 | + // The deadline timer wakes the dispatcher while it waits on a busy |
| 237 | + // worker pool, so the budget holds even if no job completes (#190). |
| 238 | + var deadline <-chan time.Time |
| 239 | + if !d.cfg.Deadline.IsZero() { |
| 240 | + timer := time.NewTimer(time.Until(d.cfg.Deadline)) |
| 241 | + defer timer.Stop() |
| 242 | + deadline = timer.C |
| 243 | + } |
| 244 | + publish := func() { atomic.StoreInt64(total, d.total()) } |
| 245 | + |
| 246 | + var ready job |
| 247 | + haveReady := false |
| 248 | + for { |
| 249 | + if !haveReady { |
| 250 | + ready, haveReady = d.next() |
| 251 | + publish() |
| 252 | + } |
| 253 | + if !haveReady && d.pending == 0 { |
| 254 | + d.finish(ctx, stats, events) |
| 255 | + return |
| 256 | + } |
| 257 | + var out chan<- job |
| 258 | + if haveReady { |
| 259 | + out = jobs |
| 260 | + } |
| 261 | + select { |
| 262 | + case <-ctx.Done(): |
| 263 | + d.finish(ctx, stats, events) |
| 264 | + return |
| 265 | + case parent := <-enqueue: |
| 266 | + d.expand(parent) |
| 267 | + publish() |
| 268 | + case out <- ready: |
| 269 | + haveReady = false |
| 270 | + d.pending++ |
| 271 | + case <-deadline: |
| 272 | + deadline = nil |
| 273 | + d.expire(haveReady) |
| 274 | + haveReady = false |
| 275 | + publish() |
| 276 | + case c := <-completed: |
| 277 | + d.complete(c) |
| 278 | + publish() |
| 279 | + } |
| 280 | + } |
| 281 | +} |
0 commit comments