Skip to content

Commit 1fcd5bc

Browse files
TMHSDigitalclaude
andcommitted
perf(scan): hand jobs out in chunks over buffered channels (#184)
Every job crossed two unbuffered channels (jobs and completed), so 50 workers spent most of a cheap scan in channel lock contention: a 1M-name simulated scan took about 1.6 s, 65% of it in semasleep and lock2. The dispatcher now admits up to maxChunk (8) jobs at a time, fewer when there are few names left per worker so a short wordlist still reaches every worker, and sends them as one slice over a buffered jobs channel. A worker reports a chunk's completions as one slice over a buffered completed channel, and still offers it after an interrupt when there is room; on cancellation the dispatcher counts the reports already buffered before computing the resume point. Deadline skips, -max-queries and EventDone ordering are unchanged. BenchmarkRun1M (simulate, 50 workers): about 0.4 s instead of 1.6 s. A resumed scan may now repeat up to one chunk per worker. Fixes #184 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent e79739a commit 1fcd5bc

4 files changed

Lines changed: 113 additions & 41 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
2626
- `-diff-records` makes `-diff` compare the records of names found again, so a CNAME re-pointed at an unclaimed service or a moved address is reported as `changed` (`~ name` in text; `"change": "changed"` with `previous_records` in JSON and JSONL), counted in the report's `diff.changed`, and exits 4. When either side has a CNAME only the alias targets are compared, so CDN address rotation is not a change. The previous file must be JSON, JSONL or CSV. The GitHub Action gains a `changed` output (#189).
2727

2828
### Changed
29+
- The scan dispatcher hands jobs to workers in small chunks (up to 8, fewer when there are few names per worker) over buffered channels, and each worker reports a chunk's jobs at once, instead of two unbuffered hand-offs per job. A 1M-name simulated scan at 50 workers (`make bench`) takes about 0.4 s instead of 1.6 s. A resumed scan may repeat up to one chunk per worker instead of one lookup per worker (#184).
2930
- Live UDP scans reuse sockets instead of opening and closing one per query, which capped throughput near 10k queries per second with nearly all CPU in socket teardown. A 100k-name scan against a loopback server now takes about 2 s instead of 12 s at `-t 200`. A late answer to an earlier query on a reused socket is ignored, as before, and cannot make a lookup count as refused (#180).
3031
- `-format json` streams its array instead of holding every result and then encoding the whole document, so its peak memory matches `-format jsonl` (a 300k-result run: about 200 MB instead of 420 MB). The output is byte for byte the same (#181).
3132
- Tests no longer flake under load: lab end-to-end runs use a 5 s timeout with 3 attempts and fail with a clear message if the lab 2 wildcard probe times out, the resolver-pool spread test counts lookups per member instead of timing the scan (the timing view is now `BenchmarkPoolSpreadsLoad`), and CI runs tests in shuffled order (#167).

‎internal/scan/dispatcher.go‎

Lines changed: 60 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -173,20 +173,38 @@ func (d *dispatcher) expand(parent job) {
173173
}
174174
}
175175

176-
// expire applies the deadline when its timer fires (#190). held reports
177-
// whether a job was admitted but not yet handed to a worker; it is skipped.
178-
func (d *dispatcher) expire(held bool) {
176+
// expire applies the deadline when its timer fires (#190). held is how many
177+
// jobs were admitted but not yet handed to a worker; they are skipped.
178+
func (d *dispatcher) expire(held int) {
179179
if d.capped {
180180
return
181181
}
182182
d.expired = true
183-
if held {
184-
d.admitted--
185-
d.skipped++
186-
}
183+
d.admitted -= held
184+
d.skipped += held
187185
d.stopAdmitting()
188186
}
189187

188+
// maxChunk caps how many jobs a worker takes at once (#184).
189+
const maxChunk = 8
190+
191+
// nextChunk admits up to one job per worker's share of what is left, at
192+
// most maxChunk: handing jobs out in chunks cuts the channel traffic per
193+
// job, and sizing the chunk by what is left keeps a short wordlist spread
194+
// over every worker.
195+
func (d *dispatcher) nextChunk(workers int) []job {
196+
size := min(maxChunk, max(1, d.remaining/max(1, workers)))
197+
var chunk []job
198+
for len(chunk) < size {
199+
j, ok := d.next()
200+
if !ok {
201+
break
202+
}
203+
chunk = append(chunk, j)
204+
}
205+
return chunk
206+
}
207+
190208
// complete records a worker's report on a dispatched job.
191209
func (d *dispatcher) complete(c completion) {
192210
d.pending--
@@ -204,6 +222,13 @@ func (d *dispatcher) complete(c completion) {
204222
}
205223
}
206224

225+
// completeAll records one worker's report on a chunk.
226+
func (d *dispatcher) completeAll(batch []completion) {
227+
for _, c := range batch {
228+
d.complete(c)
229+
}
230+
}
231+
207232
// finish stores the dispatcher's counts in stats and announces skipped
208233
// candidates. A deadline that left nothing untested changed nothing, so it
209234
// does not mark the scan expired.
@@ -226,11 +251,12 @@ func (d *dispatcher) finish(ctx context.Context, stats *counters, events chan<-
226251
}
227252
}
228253

229-
// run hands jobs to workers until no candidate is left and every dispatched
230-
// job has completed, or ctx is cancelled; then it closes jobs. Workers send
231-
// found parents on enqueue (before reporting on completed), and run expands
232-
// them. total receives the progress total as it changes.
233-
func (d *dispatcher) run(ctx context.Context, jobs chan<- job, enqueue <-chan job, completed <-chan completion,
254+
// run hands jobs to workers, in chunks (see nextChunk), until no candidate
255+
// is left and every dispatched job has completed, or ctx is cancelled; then
256+
// it closes jobs. Workers send found parents on enqueue (before reporting on
257+
// completed), and run expands them. total receives the progress total as it
258+
// changes.
259+
func (d *dispatcher) run(ctx context.Context, workers int, jobs chan<- []job, enqueue <-chan job, completed <-chan []completion,
234260
total *int64, stats *counters, events chan<- Event) {
235261
defer close(jobs)
236262
// The deadline timer wakes the dispatcher while it waits on a busy
@@ -243,38 +269,47 @@ func (d *dispatcher) run(ctx context.Context, jobs chan<- job, enqueue <-chan jo
243269
}
244270
publish := func() { atomic.StoreInt64(total, d.total()) }
245271

246-
var ready job
247-
haveReady := false
272+
var ready []job
248273
for {
249-
if !haveReady {
250-
ready, haveReady = d.next()
274+
if len(ready) == 0 {
275+
ready = d.nextChunk(workers)
251276
publish()
252277
}
253-
if !haveReady && d.pending == 0 {
278+
if len(ready) == 0 && d.pending == 0 {
254279
d.finish(ctx, stats, events)
255280
return
256281
}
257-
var out chan<- job
258-
if haveReady {
282+
var out chan<- []job
283+
if len(ready) > 0 {
259284
out = jobs
260285
}
261286
select {
262287
case <-ctx.Done():
288+
// Count the reports already buffered, so an interrupt keeps
289+
// the resume point as far as the finished jobs allow.
290+
for drained := false; !drained; {
291+
select {
292+
case batch := <-completed:
293+
d.completeAll(batch)
294+
default:
295+
drained = true
296+
}
297+
}
263298
d.finish(ctx, stats, events)
264299
return
265300
case parent := <-enqueue:
266301
d.expand(parent)
267302
publish()
268303
case out <- ready:
269-
haveReady = false
270-
d.pending++
304+
d.pending += len(ready)
305+
ready = nil
271306
case <-deadline:
272307
deadline = nil
273-
d.expire(haveReady)
274-
haveReady = false
308+
d.expire(len(ready))
309+
ready = nil
275310
publish()
276-
case c := <-completed:
277-
d.complete(c)
311+
case batch := <-completed:
312+
d.completeAll(batch)
278313
publish()
279314
}
280315
}

‎internal/scan/dispatcher_test.go‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,28 @@ func testDispatcher(cfg Config, maxDepth int) *dispatcher {
1717
return newDispatcher(cfg, maxDepth)
1818
}
1919

20+
// TestNextChunkSize covers #184: chunks are at most maxChunk jobs, and a
21+
// short wordlist goes out one job at a time so every worker gets some.
22+
func TestNextChunkSize(t *testing.T) {
23+
big := testDispatcher(Config{Entries: makeEntries(10000)}, 1)
24+
if got := len(big.nextChunk(50)); got != maxChunk {
25+
t.Errorf("10000 entries, 50 workers: chunk of %d, want %d", got, maxChunk)
26+
}
27+
small := testDispatcher(Config{Entries: makeEntries(20)}, 1)
28+
for i := 0; i < 20; i++ {
29+
if got := len(small.nextChunk(100)); got != 1 {
30+
t.Fatalf("20 entries, 100 workers: chunk %d has %d jobs, want 1", i, got)
31+
}
32+
}
33+
if got := small.nextChunk(100); got != nil {
34+
t.Errorf("exhausted dispatcher returned %v", got)
35+
}
36+
mid := testDispatcher(Config{Entries: makeEntries(30)}, 1)
37+
if got := len(mid.nextChunk(10)); got != 3 {
38+
t.Errorf("30 entries, 10 workers: chunk of %d, want 3", got)
39+
}
40+
}
41+
2042
// drain takes every job next will give, without completing any.
2143
func drain(d *dispatcher) []string {
2244
var names []string
@@ -144,7 +166,7 @@ func TestDispatcherDeadline(t *testing.T) {
144166
if _, ok := d.next(); !ok {
145167
t.Fatal("no job")
146168
}
147-
d.expire(true)
169+
d.expire(1)
148170
if !d.expired || d.skipped != 4 || d.admitted != 0 || d.total() != 0 {
149171
t.Errorf("expired %v skipped %d admitted %d total %d, want true 4 0 0", d.expired, d.skipped, d.admitted, d.total())
150172
}

‎internal/scan/runner.go‎

Lines changed: 29 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,8 @@ type Stats struct {
147147
Internal int64 // results with a non-public address (Result.Internal, #218)
148148
// RootDone is the resume point: every wordlist entry before this index
149149
// has been fully looked up (#73). Entries at or after it may also be
150-
// done; resuming repeats at most a worker pool's worth of lookups.
150+
// done; resuming repeats at most one chunk (maxChunk jobs) per worker,
151+
// since workers report their jobs a chunk at a time (#184).
151152
RootDone int64
152153
Aborted bool // the reliability guard cancelled the scan
153154
Expired bool // Deadline (-max-duration) stopped admission; Skipped counts what it left
@@ -683,9 +684,13 @@ func Run(ctx context.Context, cfg Config, events chan<- Event) {
683684
// outstanding work and closes jobs only once every dispatched job has
684685
// completed. Workers send found parents on enqueue (before signalling
685686
// completed) and the dispatcher expands them into depth-capped children.
686-
jobs := make(chan job)
687+
// Jobs go out in small chunks and both hand-off channels are buffered,
688+
// so workers and the dispatcher rarely block on each other (#184); each
689+
// job still reports its own completion as soon as it is done.
690+
workers := workerCount(cfg, maxDepth)
691+
jobs := make(chan []job, handoffPerWorker*workers)
687692
enqueue := make(chan job)
688-
completed := make(chan completion)
693+
completed := make(chan []completion, handoffPerWorker*workers)
689694

690695
// Progress ticker - fires every second.
691696
// tickerDone signals the goroutine to stop; tickerStopped confirms it has
@@ -718,7 +723,7 @@ func Run(ctx context.Context, cfg Config, events chan<- Event) {
718723

719724
// The dispatcher hands out candidates and closes jobs once every
720725
// dispatched job has completed, or ctx is cancelled (see dispatcher).
721-
go newDispatcher(cfg, maxDepth).run(ctx, jobs, enqueue, completed, &total, &stats, events)
726+
go newDispatcher(cfg, maxDepth).run(ctx, workers, jobs, enqueue, completed, &total, &stats, events)
722727

723728
// Worker pool.
724729
st := &scanState{
@@ -734,25 +739,31 @@ func Run(ctx context.Context, cfg Config, events chan<- Event) {
734739
}
735740
st.auto = newAutoRate(limiter, sent)
736741
}
737-
workers := workerCount(cfg, maxDepth)
738742
for i := 0; i < workers; i++ {
739743
wg.Add(1)
740744
go func() {
741745
defer wg.Done()
742-
for j := range jobs {
743-
if cfg.pastDeadline() {
744-
// Taken after the budget ran out: skip it, like the
746+
for chunk := range jobs {
747+
done := make([]completion, 0, len(chunk))
748+
for _, j := range chunk {
749+
c := completion{root: j.root, skipped: true}
750+
// One taken after the budget ran out is skipped, like the
745751
// candidates the dispatcher never handed out (#190).
746-
select {
747-
case completed <- completion{root: j.root, skipped: true}:
748-
case <-ctx.Done():
752+
if !cfg.pastDeadline() {
753+
c = completion{root: j.root, finished: st.processJob(ctx, j)}
749754
}
750-
continue
755+
done = append(done, c)
751756
}
752-
finished := st.processJob(ctx, j)
757+
// The buffer holds several reports per worker, so this send
758+
// rarely waits, and after an interrupt a finished chunk is
759+
// still reported when there is room, for the resume point.
753760
select {
754-
case completed <- completion{root: j.root, finished: finished}:
755-
case <-ctx.Done():
761+
case completed <- done:
762+
default:
763+
select {
764+
case completed <- done:
765+
case <-ctx.Done():
766+
}
756767
}
757768
}
758769
}()
@@ -808,6 +819,9 @@ type scanState struct {
808819
auto *autoRate // the AutoRate controller; nil without AutoRate (#220)
809820
}
810821

822+
// handoffPerWorker sizes the jobs and completed buffers, per worker (#184).
823+
const handoffPerWorker = 4
824+
811825
// observe feeds one lookup to the AutoRate controller and reports a rate
812826
// change as a notice.
813827
func (s *scanState) observe(failed bool) {

0 commit comments

Comments
 (0)