Skip to content

Commit 09b2475

Browse files
TMHSDigitalclaude
andcommitted
Cut wordlist memory: stream -w, skip visited when flat (#248)
Loading a 5M-line wordlist peaked around 740 MB. Three copies go: - wordlists are normalized while they are read (wordlist.Preparer, EachLine, EachLineOf), so the raw lines are never all held; - a non-recursive scan keeps no dispatcher visited set, since every caller passes unique entries; only recursion can repeat a name; - passive sources and the zone transfer add to one candidates set per target instead of each rebuilding a map of the whole wordlist. Measured on 5M lines with -simulate -max-queries 1: peak working set 706/742/767 MB before, 576/559/559 MB after, same run time. Refs #248 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent da85fde commit 09b2475

8 files changed

Lines changed: 190 additions & 88 deletions

File tree

‎internal/scan/dispatcher.go‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,26 @@ type dispatcher struct {
4343
pending int // dispatched jobs not yet completed
4444
}
4545

46+
// visitedSet is the dispatcher's visited set: a map for recursive scans,
47+
// nil otherwise.
48+
func visitedSet(recursive bool, n int) map[string]struct{} {
49+
if !recursive {
50+
return nil
51+
}
52+
return make(map[string]struct{}, n)
53+
}
54+
4655
func newDispatcher(cfg Config, maxDepth int) *dispatcher {
4756
n := len(cfg.Entries)
4857
start := min(max(cfg.ResumeFrom, 0), n)
4958
d := &dispatcher{
50-
cfg: cfg,
51-
n: n,
52-
visited: make(map[string]struct{}, n),
59+
cfg: cfg,
60+
n: n,
61+
// Entries are unique (every caller deduplicates them), so only a
62+
// recursive scan, whose levels can reach one name twice, needs to
63+
// remember what it visited. Without it a multi-million-entry list
64+
// skips a map as large as the list (#248).
65+
visited: visitedSet(cfg.Recursive, n),
5366
frontier: []expansion{{parent: cfg.Domain, depth: 1, next: start}},
5467
remaining: n - start,
5568
start: start,
@@ -143,11 +156,13 @@ func (d *dispatcher) next() (job, bool) {
143156
if e.depth == 1 {
144157
root = e.next - 1
145158
}
146-
if _, dup := d.visited[key]; dup {
147-
d.settle(root)
148-
continue
159+
if d.visited != nil {
160+
if _, dup := d.visited[key]; dup {
161+
d.settle(root)
162+
continue
163+
}
164+
d.visited[key] = struct{}{}
149165
}
150-
d.visited[key] = struct{}{}
151166
if d.cfg.scope.excluded(name) {
152167
d.excluded++
153168
d.settle(root)

‎internal/scan/dispatcher_test.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,14 @@ func drain(d *dispatcher) []string {
5151
}
5252
}
5353

54+
// A recursive scan can reach one name twice, so it dedupes; a flat scan
55+
// relies on its entries being unique already and keeps no visited set
56+
// (#248).
5457
func TestDispatcherDedupesAndSettlesDuplicates(t *testing.T) {
55-
d := testDispatcher(Config{Entries: []string{"www", "mail", "www", "api"}}, 1)
58+
if flat := testDispatcher(Config{Entries: []string{"www"}}, 1); flat.visited != nil {
59+
t.Error("a non-recursive scan allocated a visited set")
60+
}
61+
d := testDispatcher(Config{Entries: []string{"www", "mail", "www", "api"}, Recursive: true}, 1)
5662
got := drain(d)
5763
if want := []string{"www.example.com", "mail.example.com", "api.example.com"}; !slices.Equal(got, want) {
5864
t.Fatalf("jobs %v, want %v", got, want)

‎internal/scan/runner.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,10 @@ const (
2424

2525
// Config holds all parameters needed to run a subdomain scan.
2626
type Config struct {
27-
Domain string
27+
Domain string
28+
// Entries are the labels to try, unique and normalized (wordlist.Build
29+
// and Prepare guarantee both); a non-recursive scan does not dedupe them
30+
// again (#248).
2831
Entries []string
2932
Concurrency int
3033
Timeout time.Duration

‎internal/wordlist/reader.go‎

Lines changed: 73 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -74,32 +74,44 @@ func IsComment(line string) bool {
7474

7575
// ReadLines reads every line of path, or of standard input when path is "-".
7676
func ReadLines(path string) ([]string, error) {
77-
var r io.Reader
77+
var lines []string
78+
err := EachLineOf(path, func(line string) { lines = append(lines, line) })
79+
return lines, err
80+
}
81+
82+
// EachLineOf calls fn with each line of path, or of standard input when
83+
// path is "-", without holding the file in memory (#248).
84+
func EachLineOf(path string, fn func(string)) error {
7885
if path == Stdin {
79-
r = os.Stdin
80-
} else {
81-
f, err := os.Open(path)
82-
if err != nil {
83-
return nil, err
84-
}
85-
defer func() { _ = f.Close() }()
86-
r = f
86+
return EachLine(os.Stdin, fn)
87+
}
88+
f, err := os.Open(path)
89+
if err != nil {
90+
return err
8791
}
88-
return ReadLinesFrom(r)
92+
defer func() { _ = f.Close() }()
93+
return EachLine(f, fn)
8994
}
9095

9196
// ReadLinesFrom is ReadLines for an already open reader, such as the
9297
// wordlist embedded in the binary.
9398
func ReadLinesFrom(r io.Reader) ([]string, error) {
9499
var lines []string
100+
err := EachLine(r, func(line string) { lines = append(lines, line) })
101+
return lines, err
102+
}
103+
104+
// EachLine calls fn with each line of r, as ReadLinesFrom would return it.
105+
func EachLine(r io.Reader, fn func(string)) error {
95106
br := bufio.NewReaderSize(r, 64<<10)
107+
first := true
96108
for {
97109
chunk, isPrefix, err := br.ReadLine()
98110
if err == io.EOF {
99-
return lines, nil
111+
return nil
100112
}
101113
if err != nil {
102-
return lines, err
114+
return err
103115
}
104116
// A line longer than any valid name (a merged list gone wrong, a
105117
// binary file) is truncated and drained rather than failing the whole
@@ -109,16 +121,17 @@ func ReadLinesFrom(r io.Reader) ([]string, error) {
109121
_, isPrefix, err = br.ReadLine()
110122
}
111123
if err != nil && err != io.EOF {
112-
return lines, err
124+
return err
113125
}
114-
if len(lines) == 0 {
126+
if first {
115127
// Windows editors often prepend a UTF-8 byte order mark, which
116128
// TrimSpace does not remove; it would invalidate the first entry.
117129
line = strings.TrimPrefix(line, "\ufeff")
130+
first = false
118131
}
119-
lines = append(lines, line)
132+
fn(line)
120133
if err == io.EOF {
121-
return lines, nil
134+
return nil
122135
}
123136
}
124137
}
@@ -144,26 +157,52 @@ type Prepared struct {
144157

145158
// Prepare normalizes raw wordlist lines; see Build.
146159
func Prepare(lines []string) *Prepared {
147-
p := &Prepared{}
148-
index := make(map[string]int)
160+
pr := NewPreparer()
149161
for _, line := range lines {
150-
if IsComment(line) {
151-
continue
152-
}
153-
entry, ok := Normalize(line)
154-
if !ok {
155-
p.invalid++
156-
continue
157-
}
158-
if i, exists := index[entry]; exists {
159-
p.counts[i]++
160-
continue
161-
}
162-
index[entry] = len(p.entries)
163-
p.entries = append(p.entries, entry)
164-
p.counts = append(p.counts, 1)
165-
p.maxLen = max(p.maxLen, len(entry))
162+
pr.Add(line)
166163
}
164+
return pr.Prepared()
165+
}
166+
167+
// Preparer builds a Prepared one line at a time, so a large wordlist can be
168+
// normalized while it is read instead of first being held as raw lines
169+
// (#248).
170+
type Preparer struct {
171+
p *Prepared
172+
index map[string]int
173+
}
174+
175+
// NewPreparer returns an empty Preparer.
176+
func NewPreparer() *Preparer {
177+
return &Preparer{p: &Prepared{}, index: make(map[string]int)}
178+
}
179+
180+
// Add takes one raw wordlist line.
181+
func (pr *Preparer) Add(line string) {
182+
if IsComment(line) {
183+
return
184+
}
185+
entry, ok := Normalize(line)
186+
p := pr.p
187+
if !ok {
188+
p.invalid++
189+
return
190+
}
191+
if i, exists := pr.index[entry]; exists {
192+
p.counts[i]++
193+
return
194+
}
195+
pr.index[entry] = len(p.entries)
196+
p.entries = append(p.entries, entry)
197+
p.counts = append(p.counts, 1)
198+
p.maxLen = max(p.maxLen, len(entry))
199+
}
200+
201+
// Prepared returns the result and releases the Preparer's index; the
202+
// Preparer must not be used afterwards.
203+
func (pr *Preparer) Prepared() *Prepared {
204+
p := pr.p
205+
pr.p, pr.index = nil, nil
167206
return p
168207
}
169208

‎main_ct.go‎

Lines changed: 34 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -100,41 +100,50 @@ func sourceByName(sources []targetSource, name string) targetSource {
100100
return targetSource{}
101101
}
102102

103-
// withCTEntries appends the source names that are not already wordlist
104-
// entries, after them, so the wordlist's own resume positions stay put.
103+
// candidates gathers a target's candidate entries and -permute seeds as the
104+
// passive sources and the zone transfer add names to the wordlist's (#248).
105+
// The set of entries is built once, at the first names, however many
106+
// sources follow; it used to be rebuilt per source, a map as large as the
107+
// wordlist each time.
108+
type candidates struct {
109+
target string
110+
entries []string // the wordlist's, then the new names in order
111+
seeds map[string]struct{} // -seeds names plus every added name, full
112+
have map[string]bool // entries, once a source has names
113+
owned bool // seeds is this target's copy, not the shared -seeds map
114+
}
115+
116+
// add appends the names that are not already entries, after them, so the
117+
// wordlist's own resume positions stay put, and returns how many were new.
105118
// Names too long for the target are dropped by wordlist.Build.
106-
func withCTEntries(entries, names []string, target string) ([]string, int) {
119+
func (c *candidates) add(names []string) int {
107120
if len(names) == 0 {
108-
return entries, 0
121+
return 0
109122
}
110-
extra, _, _ := wordlist.Build(names, target)
111-
have := make(map[string]bool, len(entries))
112-
for _, e := range entries {
113-
have[e] = true
123+
if c.have == nil {
124+
c.have = make(map[string]bool, len(c.entries)+len(names))
125+
for _, e := range c.entries {
126+
c.have[e] = true
127+
}
114128
}
115-
out := entries
129+
extra, _, _ := wordlist.Build(names, c.target)
116130
added := 0
117131
for _, e := range extra {
118-
if !have[e] {
119-
out = append(out, e)
132+
if !c.have[e] {
133+
c.have[e] = true
134+
c.entries = append(c.entries, e)
120135
added++
121136
}
122137
}
123-
return out, added
124-
}
125-
126-
// withCTSeeds returns the -seeds names plus the source names, as full
127-
// names, for the -permute pass.
128-
func withCTSeeds(seeds map[string]struct{}, names []string, target string) map[string]struct{} {
129-
if len(names) == 0 {
130-
return seeds
131-
}
132-
out := make(map[string]struct{}, len(seeds)+len(names))
133-
for s := range seeds {
134-
out[s] = struct{}{}
138+
if !c.owned {
139+
own := make(map[string]struct{}, len(c.seeds)+len(names))
140+
for s := range c.seeds {
141+
own[s] = struct{}{}
142+
}
143+
c.seeds, c.owned = own, true
135144
}
136145
for _, n := range names {
137-
out[n+"."+target] = struct{}{}
146+
c.seeds[n+"."+c.target] = struct{}{}
138147
}
139-
return out
148+
return added
140149
}

‎main_ct_test.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"net/http/httptest"
77
"os"
88
"path/filepath"
9+
"slices"
910
"strings"
1011
"testing"
1112

@@ -136,6 +137,29 @@ func TestE2EPassiveSources(t *testing.T) {
136137
}
137138
}
138139

140+
// TestCandidates covers #248's shared set: names from several sources are
141+
// added once, after the wordlist, and the shared -seeds map is never
142+
// written to.
143+
func TestCandidates(t *testing.T) {
144+
shared := map[string]struct{}{"seed.example.com": {}}
145+
c := &candidates{target: "example.com", entries: []string{"www", "mail"}, seeds: shared}
146+
if n := c.add(nil); n != 0 || len(c.seeds) != 1 {
147+
t.Fatalf("no names: added %d, seeds %v", n, c.seeds)
148+
}
149+
if n := c.add([]string{"api", "www", "bad name!"}); n != 1 {
150+
t.Errorf("first source added %d, want 1 (api)", n)
151+
}
152+
if n := c.add([]string{"api", "old", "OLD"}); n != 1 {
153+
t.Errorf("second source added %d, want 1 (old)", n)
154+
}
155+
if want := []string{"www", "mail", "api", "old"}; !slices.Equal(c.entries, want) {
156+
t.Errorf("entries = %v, want %v", c.entries, want)
157+
}
158+
if _, ok := c.seeds["old.example.com"]; !ok || len(shared) != 1 {
159+
t.Errorf("seeds = %v (shared %v): want the names added to a copy only", c.seeds, shared)
160+
}
161+
}
162+
139163
// TestE2ELabRefusesSources: passive sources reach external services, so lab
140164
// mode refuses them, and simulation ignores them with a note.
141165
func TestE2ELabRefusesSources(t *testing.T) {

‎main_session.go‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -392,14 +392,14 @@ func (s *session) loadInputs() bool {
392392
}
393393
out.Info("Resuming the interrupted run of: subenum %s", strings.Join(s.args, " "))
394394
}
395-
wordLines, err := readWordlist(f, &s.in)
395+
// Normalized once, while it is read, for every target; per target only
396+
// the name-length limit differs (#182, #248).
397+
words, err := prepareWordlist(f, &s.in)
396398
if err != nil {
397399
out.Error("reading wordlist file: %v", err)
398400
return false
399401
}
400-
// Normalized once for every target; per target only the name-length
401-
// limit differs (#182).
402-
s.words = wordlist.Prepare(wordLines)
402+
s.words = words
403403
if probe, _, _ := s.words.For(""); len(probe) == 0 {
404404
out.Error("wordlist %s has no valid entries", f.wordlists.describe())
405405
return false
@@ -629,14 +629,13 @@ func (s *session) scanOneTarget(ctx context.Context, i int, target string) (next
629629
s.targetResults = resultLog{full: s.format != output.FormatText || s.f.showRecords || s.diff != nil}
630630
s.entryDigests[i] = entryDigest(entries)
631631
saved := s.resume.target(target)
632-
seeds := s.seeds
633632
// Passive sources (-ct, -sources) propose names; each is still looked up
634-
// like a wordlist entry (#130, #193).
633+
// like a wordlist entry (#130, #193). One set of the entries serves
634+
// every source and the zone transfer (#248).
635+
cands := &candidates{target: target, entries: entries, seeds: s.seeds}
635636
sourceInfo := passiveNames(ctx, s.in.sources, target, saved, out)
636637
for _, src := range sourceInfo {
637-
var added int
638-
entries, added = withCTEntries(entries, src.names, target)
639-
seeds = withCTSeeds(seeds, src.names, target)
638+
added := cands.add(src.names)
640639
if src.fetched {
641640
out.Info("%s: %d names under %s, %d of them new candidates", passive.Labels[src.source], len(src.names), target, added)
642641
}
@@ -646,13 +645,12 @@ func (s *session) scanOneTarget(ctx context.Context, i int, target string) (next
646645
// Transferred names are candidates like -ct's: each is still looked
647646
// up, so only names that resolve are reported (#85).
648647
axfrInfo = zoneTransfer(ctx, s.f, target, saved, out)
649-
var added int
650-
entries, added = withCTEntries(entries, axfrInfo.names, target)
651-
seeds = withCTSeeds(seeds, axfrInfo.names, target)
648+
added := cands.add(axfrInfo.names)
652649
if len(axfrInfo.allowed) > 0 {
653650
out.Info("Zone transfer: %d names under %s, %d of them new candidates", len(axfrInfo.names), target, added)
654651
}
655652
}
653+
entries, seeds := cands.entries, cands.seeds
656654
pass := scanPass{domain: target, entries: entries, maxQueries: s.f.maxQueries}
657655
var mainSaved *resumePass
658656
restoredHits := 0 // permutation results restored from the state

0 commit comments

Comments
 (0)