Skip to content

Commit 6e7ad89

Browse files
TMHSDigitalclaude
andcommitted
refactor(scan): lift the dispatcher out of Run; processJob takes 2 args (#159)
The dispatcher (expansion queue, visited set, resume watermark, -max-queries and -max-duration budgets, pending count) moves from a 150-line closure in Run into a dispatcher type in dispatcher.go. Its methods (next, settle, expand, expire, complete, total, finish) are plain state changes called from run's goroutine, so dispatcher_test.go covers dedupe, exclude, cap, deadline, resume and expansion without goroutines, workers or DNS. Run's goroutine is now one line. The state every worker shares (counters, events, enqueue, guard, wildcard fingerprint and cache, limiter) is bundled into scanState; processJob and checkReliability are its methods, and processJob takes (ctx, job). No behaviour change: the scan, pkg/subenum and TUI suites pass under -race, and BenchmarkRun1M is unchanged (2.11 s before, 2.17 s after). Fixes #159 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LpkeXSPwrzd7ZCuX3no7gD
1 parent f47f318 commit 6e7ad89

3 files changed

Lines changed: 486 additions & 231 deletions

File tree

‎internal/scan/dispatcher.go‎

Lines changed: 281 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,281 @@
1+
package scan
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strings"
7+
"sync/atomic"
8+
"time"
9+
)
10+
11+
// dispatcher owns candidate generation for one scan (#159): the expansion
12+
// queue, the visited set (loop and duplicate protection), the resume
13+
// watermark, the -max-queries and -max-duration budgets, and the count of
14+
// dispatched jobs not yet completed. Its methods are plain state changes,
15+
// called only from run's goroutine (or a test), so they need no locking.
16+
//
17+
// Candidates are generated lazily. The queue holds expansions (a parent plus
18+
// a cursor into Entries) rather than one job per name, so depth-1 names are
19+
// built only as they are dispatched and a found parent costs one enqueue
20+
// instead of len(Entries) (#49). Once a budget is reached the remaining
21+
// candidates are counted, never built or recorded (#51).
22+
type dispatcher struct {
23+
cfg Config
24+
n int // len(cfg.Entries)
25+
26+
visited map[string]struct{}
27+
frontier []expansion
28+
head int // first unconsumed expansion in frontier
29+
remaining int // candidates not yet generated
30+
31+
// Resume point: rootDone[i-start] is set once entry i is settled (looked
32+
// up, or skipped as a duplicate or out of scope); watermark is the first
33+
// index not yet settled.
34+
start int
35+
rootDone []bool
36+
watermark int
37+
38+
admitted int // jobs handed out (or held) and counted in the total
39+
skipped int // candidates a budget kept from being tested
40+
excluded int // candidates -exclude kept out of scope
41+
capped bool
42+
expired bool // the budget that capped admission was the deadline
43+
pending int // dispatched jobs not yet completed
44+
}
45+
46+
func newDispatcher(cfg Config, maxDepth int) *dispatcher {
47+
n := len(cfg.Entries)
48+
start := min(max(cfg.ResumeFrom, 0), n)
49+
d := &dispatcher{
50+
cfg: cfg,
51+
n: n,
52+
visited: make(map[string]struct{}, n),
53+
frontier: []expansion{{parent: cfg.Domain, depth: 1, next: start}},
54+
remaining: n - start,
55+
start: start,
56+
rootDone: make([]bool, n-start),
57+
watermark: start,
58+
}
59+
// Resumed recursive scans re-expand the names found before the
60+
// interrupt, whose children may not all have been tested (#73).
61+
if cfg.Recursive {
62+
rootLabels := strings.Count(cfg.Domain, ".") + 1
63+
for _, p := range cfg.ResumeParents {
64+
p = strings.ToLower(strings.TrimSuffix(p, "."))
65+
if !strings.HasSuffix(p, "."+cfg.Domain) {
66+
continue
67+
}
68+
if depth := strings.Count(p, ".") + 1 - rootLabels; depth >= 1 && depth < maxDepth {
69+
d.frontier = append(d.frontier, expansion{parent: p, depth: depth + 1})
70+
d.remaining += n
71+
}
72+
}
73+
}
74+
return d
75+
}
76+
77+
// settle marks wordlist entry i as done and advances the resume watermark.
78+
// i is -1 for jobs below depth 1, which do not affect it.
79+
func (d *dispatcher) settle(i int) {
80+
if i < d.start {
81+
return
82+
}
83+
d.rootDone[i-d.start] = true
84+
for d.watermark < d.n && d.rootDone[d.watermark-d.start] {
85+
d.watermark++
86+
}
87+
}
88+
89+
// stopAdmitting ends admission for a budget: every candidate not yet
90+
// generated, and every child a later find would add, is skipped.
91+
func (d *dispatcher) stopAdmitting() {
92+
d.skipped += d.remaining
93+
d.remaining = 0
94+
d.capped = true
95+
d.frontier, d.head = nil, 0
96+
}
97+
98+
// total is the progress total: jobs admitted plus candidates not yet
99+
// generated, at most -max-queries.
100+
func (d *dispatcher) total() int64 {
101+
t := d.admitted + d.remaining
102+
if d.cfg.MaxQueries > 0 && t > d.cfg.MaxQueries {
103+
t = d.cfg.MaxQueries
104+
}
105+
return int64(t)
106+
}
107+
108+
// next generates the next admissible job, if any.
109+
func (d *dispatcher) next() (job, bool) {
110+
for d.head < len(d.frontier) {
111+
e := &d.frontier[d.head]
112+
if e.next == d.n {
113+
d.head++
114+
// Compact only once the consumed prefix is at least half the
115+
// slice, so each element is copied O(1) times.
116+
if d.head == len(d.frontier) {
117+
d.frontier, d.head = d.frontier[:0], 0
118+
} else if d.head*2 >= len(d.frontier) {
119+
d.frontier, d.head = append(d.frontier[:0], d.frontier[d.head:]...), 0
120+
}
121+
continue
122+
}
123+
if d.cfg.pastDeadline() {
124+
d.expired = true
125+
d.stopAdmitting()
126+
return job{}, false
127+
}
128+
if d.cfg.MaxQueries > 0 && d.admitted >= d.cfg.MaxQueries {
129+
d.stopAdmitting()
130+
return job{}, false
131+
}
132+
entry := d.cfg.Entries[e.next]
133+
e.next++
134+
d.remaining--
135+
name := entry + "." + e.parent
136+
// Key by the name relative to the root domain. For depth 1 that is
137+
// the entry itself, so no extra string is retained.
138+
key := entry
139+
if e.depth > 1 {
140+
key = name[:len(name)-len(d.cfg.Domain)-1]
141+
}
142+
root := -1
143+
if e.depth == 1 {
144+
root = e.next - 1
145+
}
146+
if _, dup := d.visited[key]; dup {
147+
d.settle(root)
148+
continue
149+
}
150+
d.visited[key] = struct{}{}
151+
if d.cfg.scope.excluded(name) {
152+
d.excluded++
153+
d.settle(root)
154+
continue
155+
}
156+
d.admitted++
157+
return job{domain: name, depth: e.depth, root: root}, true
158+
}
159+
return job{}, false
160+
}
161+
162+
// expand queues the children of a found parent, unless a budget has stopped
163+
// admission or every child would be out of scope; then they are only counted.
164+
func (d *dispatcher) expand(parent job) {
165+
switch {
166+
case d.capped:
167+
d.skipped += d.n
168+
case d.cfg.scope.subtreeExcluded(parent.domain):
169+
d.excluded += d.n
170+
default:
171+
d.frontier = append(d.frontier, expansion{parent: parent.domain, depth: parent.depth + 1})
172+
d.remaining += d.n
173+
}
174+
}
175+
176+
// expire applies the deadline when its timer fires (#190). held reports
177+
// whether a job was admitted but not yet handed to a worker; it is skipped.
178+
func (d *dispatcher) expire(held bool) {
179+
if d.capped {
180+
return
181+
}
182+
d.expired = true
183+
if held {
184+
d.admitted--
185+
d.skipped++
186+
}
187+
d.stopAdmitting()
188+
}
189+
190+
// complete records a worker's report on a dispatched job.
191+
func (d *dispatcher) complete(c completion) {
192+
d.pending--
193+
if c.skipped {
194+
// Taken after the deadline and never looked up.
195+
d.skipped++
196+
d.admitted--
197+
if !d.capped {
198+
d.expired = true
199+
d.stopAdmitting()
200+
}
201+
}
202+
if c.finished {
203+
d.settle(c.root)
204+
}
205+
}
206+
207+
// finish stores the dispatcher's counts in stats and announces skipped
208+
// candidates. A deadline that left nothing untested changed nothing, so it
209+
// does not mark the scan expired.
210+
func (d *dispatcher) finish(ctx context.Context, stats *counters, events chan<- Event) {
211+
stats.skipped.Store(int64(d.skipped))
212+
stats.rootDone.Store(int64(d.watermark))
213+
stats.excluded.Store(int64(d.excluded))
214+
d.expired = d.expired && d.skipped > 0
215+
stats.expired.Store(d.expired)
216+
if d.skipped == 0 {
217+
return
218+
}
219+
msg := fmt.Sprintf("query cap reached ({max_queries} %d); skipped %d additional jobs", d.cfg.MaxQueries, d.skipped)
220+
if d.expired {
221+
msg = fmt.Sprintf("time budget reached ({max_duration}); skipped %d additional jobs", d.skipped)
222+
}
223+
select {
224+
case events <- Event{Kind: EventNotice, Notice: NoticeCap, Message: msg}:
225+
case <-ctx.Done():
226+
}
227+
}
228+
229+
// run hands jobs to workers until no candidate is left and every dispatched
230+
// job has completed, or ctx is cancelled; then it closes jobs. Workers send
231+
// found parents on enqueue (before reporting on completed), and run expands
232+
// them. total receives the progress total as it changes.
233+
func (d *dispatcher) run(ctx context.Context, jobs chan<- job, enqueue <-chan job, completed <-chan completion,
234+
total *int64, stats *counters, events chan<- Event) {
235+
defer close(jobs)
236+
// The deadline timer wakes the dispatcher while it waits on a busy
237+
// worker pool, so the budget holds even if no job completes (#190).
238+
var deadline <-chan time.Time
239+
if !d.cfg.Deadline.IsZero() {
240+
timer := time.NewTimer(time.Until(d.cfg.Deadline))
241+
defer timer.Stop()
242+
deadline = timer.C
243+
}
244+
publish := func() { atomic.StoreInt64(total, d.total()) }
245+
246+
var ready job
247+
haveReady := false
248+
for {
249+
if !haveReady {
250+
ready, haveReady = d.next()
251+
publish()
252+
}
253+
if !haveReady && d.pending == 0 {
254+
d.finish(ctx, stats, events)
255+
return
256+
}
257+
var out chan<- job
258+
if haveReady {
259+
out = jobs
260+
}
261+
select {
262+
case <-ctx.Done():
263+
d.finish(ctx, stats, events)
264+
return
265+
case parent := <-enqueue:
266+
d.expand(parent)
267+
publish()
268+
case out <- ready:
269+
haveReady = false
270+
d.pending++
271+
case <-deadline:
272+
deadline = nil
273+
d.expire(haveReady)
274+
haveReady = false
275+
publish()
276+
case c := <-completed:
277+
d.complete(c)
278+
publish()
279+
}
280+
}
281+
}

0 commit comments

Comments
 (0)