Skip to content

Commit e8a1ea4

Browse files
TMHSDigitalclaude
andcommitted
fix(tools): make wordlist-gen deterministic and fail on write errors
-combine iterated a map, so entry order changed on every run; write and close errors were printed to stdout and the tool exited 0; TLD stripping used a fixed list that mishandled ccTLDs such as .co.uk; and there were no tests. Generate into an ordered slice, strip the last label plus a known second-level label, skip prefix-prefix combinations, exit non-zero on write failure, move per-entry output behind -v, and add tests. Fixes #45 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 724f4fe commit e8a1ea4

3 files changed

Lines changed: 198 additions & 104 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1919
- TUI scan view: the viewport height is clamped to at least one row, so terminals shorter than the layout no longer produce a negative height (#37, ROADMAP N6). The layout reserve also accounts for the hint margin, which previously overflowed the screen by one line.
2020
- TUI scan view: wildcard and error notices are capped at the latest three (with a "+N earlier notices" line) and the viewport shrinks to fit them, so recursive scans no longer push the progress bar and status off-screen. Ctrl+C on a finished scan now quits instead of relabelling it "Aborted". The results buffer is capped at 10,000 lines and re-rendered on progress ticks once past 200 results, so large scans no longer slow the UI and back-pressure the workers. `~/` paths in the wordlist and output fields are expanded (#43).
2121
- `-v` lines no longer splice into the progress line. The `dns` package no longer writes to stderr; lookups log through `scan.Config.Logf` (the CLI passes `output.Writer.Info`), and `Info`/`Error` now take the writer mutex and blank the in-place progress line before printing (#36, ROADMAP N5).
22+
- `tools/wordlist-gen` output is deterministic: `-combine` no longer iterates a Go map, so the same flags give a byte-identical file. Write errors go to stderr and exit non-zero instead of being swallowed, suffixes such as `.co.uk` are stripped correctly, a prefix is no longer combined with itself, per-entry logging moved behind `-v`, and the generator has unit tests (#45).
2223

2324
### Added
2425
- `-format jsonl` streams one JSON object per resolved subdomain, so structured output can be piped live (#34).

‎tools/wordlist-gen.go‎

Lines changed: 126 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -1,139 +1,161 @@
1-
// Copyright (c) 2025 TM Hospitality Strategies
1+
// Copyright (c) 2026 TM Hospitality Strategies
22
//
3-
// Tool for generating custom wordlists for subdomain enumeration
4-
// Educational purposes only
3+
// Tool for generating custom wordlists for subdomain enumeration.
4+
// For authorized use only.
55

66
package main
77

88
import (
99
"bufio"
1010
"flag"
1111
"fmt"
12+
"io"
1213
"os"
1314
"strings"
1415
)
1516

17+
// commonPrefixes are frequently seen subdomain labels, emitted first when
18+
// -common is set.
19+
var commonPrefixes = []string{
20+
"www", "mail", "remote", "blog", "webmail", "server", "ns1", "ns2",
21+
"smtp", "secure", "vpn", "m", "shop", "ftp", "mail2", "test",
22+
"portal", "admin", "host", "api", "dev", "web", "cloud", "email",
23+
"apps", "support", "app", "staging", "proxy", "beta", "gateway",
24+
"cdn", "auth", "intranet", "mobile", "sso", "help", "docs",
25+
}
26+
27+
// secondLevelSuffixes are labels that sit under a ccTLD as a registry
28+
// namespace (acme.co.uk, acme.com.au), so they are stripped along with it.
29+
var secondLevelSuffixes = map[string]bool{
30+
"co": true, "com": true, "net": true, "org": true, "gov": true,
31+
"edu": true, "ac": true, "or": true, "ne": true, "go": true,
32+
}
33+
34+
type options struct {
35+
common bool
36+
domain string
37+
combine string
38+
}
39+
40+
// domainTerms extracts candidate words from a domain: the public suffix is
41+
// dropped (the last label, plus a second-level label such as "co" in
42+
// acme.co.uk) and the rest is split on dots, hyphens and underscores. Terms
43+
// shorter than three characters are skipped.
44+
func domainTerms(domain string) []string {
45+
labels := strings.Split(strings.ToLower(strings.Trim(domain, ".")), ".")
46+
if len(labels) > 1 {
47+
labels = labels[:len(labels)-1]
48+
}
49+
if len(labels) > 1 && secondLevelSuffixes[labels[len(labels)-1]] {
50+
labels = labels[:len(labels)-1]
51+
}
52+
var terms []string
53+
for _, part := range strings.FieldsFunc(strings.Join(labels, "."), func(r rune) bool {
54+
return r == '.' || r == '-' || r == '_'
55+
}) {
56+
if len(part) > 2 {
57+
terms = append(terms, part)
58+
}
59+
}
60+
return terms
61+
}
62+
63+
// generate returns the wordlist in a stable order: common prefixes, then
64+
// domain terms, then each -combine prefix followed by its prefix-word
65+
// combinations. Duplicates keep their first position, so the same options
66+
// always produce byte-identical output.
67+
func generate(o options) []string {
68+
var out []string
69+
seen := make(map[string]bool)
70+
add := func(w string) {
71+
w = strings.ToLower(strings.TrimSpace(w))
72+
if w == "" || seen[w] {
73+
return
74+
}
75+
seen[w] = true
76+
out = append(out, w)
77+
}
78+
79+
if o.common {
80+
for _, p := range commonPrefixes {
81+
add(p)
82+
}
83+
}
84+
if o.domain != "" {
85+
for _, t := range domainTerms(o.domain) {
86+
add(t)
87+
}
88+
}
89+
if o.combine != "" {
90+
base := append([]string(nil), out...)
91+
for _, prefix := range strings.Split(o.combine, ",") {
92+
prefix = strings.ToLower(strings.TrimSpace(prefix))
93+
if prefix == "" {
94+
continue
95+
}
96+
add(prefix)
97+
for _, word := range base {
98+
if word != prefix { // no "shop-shop"
99+
add(prefix + "-" + word)
100+
}
101+
}
102+
}
103+
}
104+
return out
105+
}
106+
107+
// writeWordlist writes one word per line and reports the first write error.
108+
func writeWordlist(w io.Writer, words []string) error {
109+
bw := bufio.NewWriter(w)
110+
for _, word := range words {
111+
if _, err := fmt.Fprintln(bw, word); err != nil {
112+
return err
113+
}
114+
}
115+
return bw.Flush()
116+
}
117+
16118
func main() {
17-
// Command-line flags
18119
outputFile := flag.String("o", "wordlist.txt", "Path to output wordlist file")
19120
combineWith := flag.String("combine", "", "Combine each word with these prefixes (comma-separated)")
20121
addCommon := flag.Bool("common", true, "Add common subdomain prefixes")
21122
domainInfo := flag.String("domain", "", "Domain to extract potential subdomains from (e.g., company-name.com -> company, name)")
123+
verbose := flag.Bool("v", false, "Print every generated entry")
22124
flag.Parse()
23125

24-
// Validate output file
25126
if *outputFile == "" {
26-
fmt.Println("Error: Output file cannot be empty")
127+
fmt.Fprintln(os.Stderr, "Error: output file cannot be empty")
27128
os.Exit(1)
28129
}
29130

30-
// Create output file
131+
words := generate(options{common: *addCommon, domain: *domainInfo, combine: *combineWith})
132+
31133
file, err := os.Create(*outputFile)
32134
if err != nil {
33-
fmt.Printf("Error creating output file: %v\n", err)
135+
fmt.Fprintf(os.Stderr, "Error creating output file: %v\n", err)
34136
os.Exit(1)
35137
}
36-
37-
writer := bufio.NewWriter(file)
38-
defer func() {
39-
if flushErr := writer.Flush(); flushErr != nil {
40-
fmt.Printf("Error flushing writer: %v\n", flushErr)
41-
}
42-
if closeErr := file.Close(); closeErr != nil {
43-
fmt.Printf("Error closing file: %v\n", closeErr)
44-
}
45-
}()
46-
47-
// Track added words to avoid duplicates
48-
addedWords := make(map[string]bool)
49-
50-
// Add common subdomain prefixes if requested
51-
if *addCommon {
52-
fmt.Println("Adding common subdomain prefixes...")
53-
commonPrefixes := []string{
54-
"www", "mail", "remote", "blog", "webmail", "server", "ns1", "ns2",
55-
"smtp", "secure", "vpn", "m", "shop", "ftp", "mail2", "test",
56-
"portal", "admin", "host", "api", "dev", "web", "cloud", "email",
57-
"apps", "support", "app", "staging", "proxy", "beta", "gateway",
58-
"cdn", "auth", "intranet", "mobile", "sso", "help", "docs",
59-
}
60-
61-
for _, prefix := range commonPrefixes {
62-
if !addedWords[prefix] {
63-
fmt.Fprintln(writer, prefix)
64-
addedWords[prefix] = true
65-
fmt.Printf("Added: %s\n", prefix)
66-
}
67-
}
138+
writeErr := writeWordlist(file, words)
139+
closeErr := file.Close()
140+
if writeErr != nil || closeErr != nil {
141+
fmt.Fprintf(os.Stderr, "Error writing %s: %v\n", *outputFile, firstErr(writeErr, closeErr))
142+
os.Exit(1)
68143
}
69144

70-
// Process domain-specific terms if provided
71-
if *domainInfo != "" {
72-
fmt.Printf("Extracting terms from domain: %s\n", *domainInfo)
73-
74-
// Remove TLD and split by separators
75-
domain := *domainInfo
76-
77-
// Remove common TLDs
78-
domain = strings.TrimSuffix(domain, ".com")
79-
domain = strings.TrimSuffix(domain, ".org")
80-
domain = strings.TrimSuffix(domain, ".net")
81-
domain = strings.TrimSuffix(domain, ".edu")
82-
domain = strings.TrimSuffix(domain, ".gov")
83-
domain = strings.TrimSuffix(domain, ".io")
84-
85-
// Split by common separators
86-
parts := strings.FieldsFunc(domain, func(r rune) bool {
87-
return r == '.' || r == '-' || r == '_'
88-
})
89-
90-
for _, part := range parts {
91-
if len(part) > 2 && !addedWords[part] { // Only add if length > 2
92-
fmt.Fprintln(writer, part)
93-
addedWords[part] = true
94-
fmt.Printf("Added: %s\n", part)
95-
}
145+
if *verbose {
146+
for _, w := range words {
147+
fmt.Println(w)
96148
}
97149
}
150+
fmt.Fprintf(os.Stderr, "Wordlist generated at %s with %d unique entries\n", *outputFile, len(words))
151+
fmt.Fprintln(os.Stderr, "NOTE: Only use this tool to generate wordlists for domains you have explicit permission to test.")
152+
}
98153

99-
// Process combination prefixes if provided
100-
if *combineWith != "" {
101-
prefixes := strings.Split(*combineWith, ",")
102-
fmt.Printf("Combining with prefixes: %v\n", prefixes)
103-
104-
// Create a temporary copy of addedWords to iterate over
105-
wordsCopy := make([]string, 0, len(addedWords))
106-
for word := range addedWords {
107-
wordsCopy = append(wordsCopy, word)
108-
}
109-
110-
// Create combinations
111-
for _, prefix := range prefixes {
112-
prefix = strings.TrimSpace(prefix)
113-
if prefix == "" {
114-
continue
115-
}
116-
117-
// Add the prefix itself
118-
if !addedWords[prefix] {
119-
fmt.Fprintln(writer, prefix)
120-
addedWords[prefix] = true
121-
fmt.Printf("Added: %s\n", prefix)
122-
}
123-
124-
// Create combinations with existing words
125-
for _, word := range wordsCopy {
126-
combination := prefix + "-" + word
127-
if !addedWords[combination] {
128-
fmt.Fprintln(writer, combination)
129-
addedWords[combination] = true
130-
fmt.Printf("Added: %s\n", combination)
131-
}
132-
}
154+
func firstErr(errs ...error) error {
155+
for _, err := range errs {
156+
if err != nil {
157+
return err
133158
}
134159
}
135-
136-
// Count total words
137-
fmt.Printf("\nWordlist generated at %s with %d unique entries\n", *outputFile, len(addedWords))
138-
fmt.Println("\nNOTE: Only use this tool to generate wordlists for domains you have explicit permission to test.")
160+
return nil
139161
}

‎tools/wordlist-gen_test.go‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
package main
2+
3+
import (
4+
"errors"
5+
"reflect"
6+
"strings"
7+
"testing"
8+
)
9+
10+
// TestGenerateDeterministic covers #45: -combine used to iterate a map, so the
11+
// order changed on every run.
12+
func TestGenerateDeterministic(t *testing.T) {
13+
o := options{common: true, domain: "acme-corp.com", combine: "qa,uat"}
14+
first := generate(o)
15+
for i := 0; i < 20; i++ {
16+
if got := generate(o); !reflect.DeepEqual(got, first) {
17+
t.Fatalf("run %d produced a different order", i)
18+
}
19+
}
20+
// Combinations follow the order of the base words.
21+
idx := map[string]int{}
22+
for i, w := range first {
23+
idx[w] = i
24+
}
25+
order := []string{"www", "acme", "qa", "qa-www", "qa-mail", "qa-acme", "uat", "uat-www"}
26+
for i := 1; i < len(order); i++ {
27+
if idx[order[i-1]] >= idx[order[i]] {
28+
t.Errorf("%q should come before %q", order[i-1], order[i])
29+
}
30+
}
31+
}
32+
33+
func TestGenerateDedupAndCase(t *testing.T) {
34+
got := generate(options{domain: "Shop.Example.com", combine: "WWW, shop ,"})
35+
want := []string{"shop", "example", "www", "www-shop", "www-example", "shop-example"}
36+
if !reflect.DeepEqual(got, want) {
37+
t.Errorf("generate = %q, want %q", got, want)
38+
}
39+
}
40+
41+
func TestDomainTerms(t *testing.T) {
42+
tests := map[string][]string{
43+
"acme-corp.com": {"acme", "corp"},
44+
"acme.co.uk": {"acme"},
45+
"shop.acme.com.au": {"shop", "acme"},
46+
"big_name.dev.io": {"big", "name", "dev"},
47+
"acme.technology": {"acme"},
48+
"x.io": nil,
49+
"single": {"single"},
50+
"api-gateway.example": {"api", "gateway"},
51+
}
52+
for in, want := range tests {
53+
if got := domainTerms(in); !reflect.DeepEqual(got, want) {
54+
t.Errorf("domainTerms(%q) = %q, want %q", in, got, want)
55+
}
56+
}
57+
}
58+
59+
type failingWriter struct{}
60+
61+
func (failingWriter) Write([]byte) (int, error) { return 0, errors.New("disk full") }
62+
63+
func TestWriteWordlistReportsErrors(t *testing.T) {
64+
if err := writeWordlist(failingWriter{}, []string{"a", "b"}); err == nil {
65+
t.Fatal("expected write error")
66+
}
67+
var sb strings.Builder
68+
if err := writeWordlist(&sb, []string{"a", "b"}); err != nil || sb.String() != "a\nb\n" {
69+
t.Fatalf("got %q, %v", sb.String(), err)
70+
}
71+
}

0 commit comments

Comments
 (0)