Skip to content

Commit 3dc97d0

Browse files
TMHSDigitalclaude
andcommitted
Add wordlist-gen -from to learn labels from earlier results (#423)
wordlist-gen built lists only from common prefixes, the domain's own terms and -combine words, so an organisation's naming habits found on one domain could not seed the list for the next. Add -from (repeatable) and -min-count: each results file is read with the parser -diff and -seeds use, the labels below each name's apex are counted, and they are written first, most frequent first, ahead of the common prefixes, which are not repeated. The results-file parser moves from internal/cli to internal/diff (diff.Parse, diff.Load) so the tool can share it; the CLI's parseBaseline now wraps it and its tests are unchanged. Fixes #423 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent f967e6a commit 3dc97d0

5 files changed

Lines changed: 273 additions & 79 deletions

File tree

‎internal/cli/diff.go‎

Lines changed: 6 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@ package cli
22

33
import (
44
"context"
5-
"encoding/csv"
6-
"encoding/json"
75
"errors"
86
"flag"
97
"fmt"
@@ -62,79 +60,14 @@ func loadBaseline(path string) (baseline, error) {
6260
return parseBaseline(data)
6361
}
6462

65-
// parseBaseline is loadBaseline on a results file's contents.
63+
// parseBaseline is loadBaseline on a results file's contents; the parser
64+
// lives in internal/diff, which wordlist-gen's -from shares (#423).
6665
func parseBaseline(data []byte) (baseline, error) {
67-
b := baseline{names: map[string]struct{}{}, records: map[string][]dns.Record{}}
68-
add := func(name string, records ...dns.Record) string {
69-
name = diff.Name(name)
70-
if name != "" {
71-
b.names[name] = struct{}{}
72-
for _, r := range records {
73-
if r.Type != "" {
74-
b.records[name] = append(b.records[name], r)
75-
}
76-
}
77-
}
78-
return name
79-
}
80-
text := strings.TrimPrefix(string(data), string(rune(0xFEFF)))
81-
trimmed := strings.TrimSpace(text)
82-
83-
if strings.HasPrefix(trimmed, "[") {
84-
var results []output.Result
85-
if err := json.Unmarshal([]byte(trimmed), &results); err != nil {
86-
return baseline{}, err
87-
}
88-
for _, r := range results {
89-
add(r.Subdomain, r.Records...)
90-
}
91-
b.hasRecords = true
92-
return b, nil
93-
}
94-
if strings.HasPrefix(trimmed, "subdomain,") {
95-
rows, err := csv.NewReader(strings.NewReader(trimmed)).ReadAll()
96-
if err != nil {
97-
return baseline{}, err
98-
}
99-
for _, row := range rows[1:] {
100-
switch {
101-
case len(row) >= 3:
102-
add(row[0], dns.Record{Type: row[1], Value: row[2]})
103-
case len(row) > 0:
104-
add(row[0])
105-
}
106-
}
107-
b.hasRecords = true
108-
return b, nil
109-
}
110-
for _, line := range strings.Split(text, "\n") {
111-
line = strings.TrimSpace(line)
112-
switch {
113-
case line == "" || strings.HasPrefix(line, "#"):
114-
continue
115-
case strings.HasPrefix(line, "{"):
116-
var r struct {
117-
Type string `json:"type"`
118-
Subdomain string `json:"subdomain"`
119-
Records []dns.Record `json:"records"`
120-
}
121-
if err := json.Unmarshal([]byte(line), &r); err != nil {
122-
return baseline{}, err
123-
}
124-
if r.Type != "summary" {
125-
add(r.Subdomain, r.Records...)
126-
b.hasRecords = true
127-
}
128-
default:
129-
for _, p := range []string{"Found (SIMULATED):", "Found:", "+ ", "- ", "~ "} {
130-
line = strings.TrimSpace(strings.TrimPrefix(line, p))
131-
}
132-
if fields := strings.Fields(line); len(fields) > 0 {
133-
add(fields[0])
134-
}
135-
}
66+
d, err := diff.Parse(data)
67+
if err != nil {
68+
return baseline{}, err
13669
}
137-
return b, nil
70+
return baseline{names: d.Names, records: d.Records, hasRecords: d.HasRecords}, nil
13871
}
13972

14073
const diffUsage = "Usage: subenum diff [-records] [-type A,AAAA,CNAME] [-format text|json|jsonl|csv] <previous> <current>"

‎internal/diff/parse.go‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
package diff
2+
3+
import (
4+
"encoding/csv"
5+
"encoding/json"
6+
"os"
7+
"strings"
8+
9+
"github.com/TMHSDigital/subenum/internal/dns"
10+
)
11+
12+
// Baseline is a results file read back: its names, and the records behind
13+
// them when the format carries records (JSON, JSONL, CSV; not text).
14+
type Baseline struct {
15+
Names map[string]struct{}
16+
Records map[string][]dns.Record
17+
HasRecords bool
18+
}
19+
20+
// Load reads a subenum results file in any output format; see Parse.
21+
func Load(path string) (Baseline, error) {
22+
data, err := os.ReadFile(path)
23+
if err != nil {
24+
return Baseline{}, err
25+
}
26+
return Parse(data)
27+
}
28+
29+
// Parse reads the names of a subenum results file in any output format:
30+
// JSONL (summary lines are skipped), a JSON array, CSV, or text (bare names,
31+
// "Found: name", -show-records lines, or -diff's +/-/~ lines). The CLI's
32+
// -diff and -seeds and wordlist-gen's -from share it (#423).
33+
func Parse(data []byte) (Baseline, error) {
34+
b := Baseline{Names: map[string]struct{}{}, Records: map[string][]dns.Record{}}
35+
add := func(name string, records ...dns.Record) {
36+
name = Name(name)
37+
if name == "" {
38+
return
39+
}
40+
b.Names[name] = struct{}{}
41+
for _, r := range records {
42+
if r.Type != "" {
43+
b.Records[name] = append(b.Records[name], r)
44+
}
45+
}
46+
}
47+
text := strings.TrimPrefix(string(data), string(rune(0xFEFF)))
48+
trimmed := strings.TrimSpace(text)
49+
50+
type result struct {
51+
Type string `json:"type"`
52+
Subdomain string `json:"subdomain"`
53+
Records []dns.Record `json:"records"`
54+
}
55+
if strings.HasPrefix(trimmed, "[") {
56+
var results []result
57+
if err := json.Unmarshal([]byte(trimmed), &results); err != nil {
58+
return Baseline{}, err
59+
}
60+
for _, r := range results {
61+
add(r.Subdomain, r.Records...)
62+
}
63+
b.HasRecords = true
64+
return b, nil
65+
}
66+
if strings.HasPrefix(trimmed, "subdomain,") {
67+
rows, err := csv.NewReader(strings.NewReader(trimmed)).ReadAll()
68+
if err != nil {
69+
return Baseline{}, err
70+
}
71+
for _, row := range rows[1:] {
72+
switch {
73+
case len(row) >= 3:
74+
add(row[0], dns.Record{Type: row[1], Value: row[2]})
75+
case len(row) > 0:
76+
add(row[0])
77+
}
78+
}
79+
b.HasRecords = true
80+
return b, nil
81+
}
82+
for _, line := range strings.Split(text, "\n") {
83+
line = strings.TrimSpace(line)
84+
switch {
85+
case line == "" || strings.HasPrefix(line, "#"):
86+
continue
87+
case strings.HasPrefix(line, "{"):
88+
var r result
89+
if err := json.Unmarshal([]byte(line), &r); err != nil {
90+
return Baseline{}, err
91+
}
92+
if r.Type != "summary" {
93+
add(r.Subdomain, r.Records...)
94+
b.HasRecords = true
95+
}
96+
default:
97+
for _, p := range []string{"Found (SIMULATED):", "Found:", "+ ", "- ", "~ "} {
98+
line = strings.TrimSpace(strings.TrimPrefix(line, p))
99+
}
100+
if fields := strings.Fields(line); len(fields) > 0 {
101+
add(fields[0])
102+
}
103+
}
104+
}
105+
return b, nil
106+
}

‎tools/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,13 @@ go build -buildvcs=false -o wordlist-gen wordlist-gen.go
3030

3131
# Add custom combinations
3232
./wordlist-gen -domain company-name.com -combine "dev,test,staging" -f -o company-wordlist.txt
33+
34+
# Learn an organisation's naming habits from earlier results
35+
./wordlist-gen -from results.jsonl -from old-scan.txt -min-count 2 -o learned.txt
3336
```
3437

38+
`-from` reads earlier subenum results in any output format (text, JSON, JSONL or CSV, the formats `-diff` reads), takes the labels of each name below its apex (`api.k8s.example.com` gives `api` and `k8s`), and writes them first, most frequent first, ahead of the common prefixes, which are not repeated. The apex is `-domain` when a name ends in it, and otherwise the registrable domain (`example.com`, `acme.co.uk`).
39+
3540
### Parameters
3641

3742
| Flag | Description | Default |
@@ -41,6 +46,8 @@ go build -buildvcs=false -o wordlist-gen wordlist-gen.go
4146
| `-common` | Include common subdomain prefixes | `true` |
4247
| `-domain` | Domain to extract terms from | `""` (empty) |
4348
| `-combine` | Comma-separated prefixes to combine with other terms; each must be a valid DNS label | `""` (empty) |
49+
| `-from` | Earlier subenum results file to learn labels from, most frequent first; repeatable | none |
50+
| `-min-count` | With `-from`, drop labels seen fewer than this many times | `1` |
4451
| `-v` | Print every generated entry to stdout | `false` |
4552

4653
### Example

‎tools/wordlist-gen.go‎

Lines changed: 83 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,11 @@ import (
1414
"fmt"
1515
"io"
1616
"os"
17+
"sort"
1718
"strings"
1819

1920
"github.com/TMHSDigital/subenum/internal/cliargs"
21+
"github.com/TMHSDigital/subenum/internal/diff"
2022
"github.com/TMHSDigital/subenum/internal/wordlist"
2123
)
2224

@@ -41,6 +43,64 @@ type options struct {
4143
common bool
4244
domain string
4345
combine string
46+
learned []string // labels from earlier results (-from), most frequent first
47+
}
48+
49+
// stringList is a repeatable string flag.
50+
type stringList []string
51+
52+
func (l *stringList) String() string { return strings.Join(*l, ",") }
53+
func (l *stringList) Set(s string) error { *l = append(*l, s); return nil }
54+
55+
// learnLabels reads earlier subenum results files, in any output format, and
56+
// returns the labels of their names below each name's apex, most frequent
57+
// first, then alphabetically (#423). The apex is domain when a name ends in
58+
// it, and otherwise the registrable domain as domainTerms sees it: the last
59+
// label, a second-level label such as "co", and the organisation's label.
60+
// Labels seen fewer than minCount times are dropped.
61+
func learnLabels(paths []string, domain string, minCount int) ([]string, error) {
62+
domain = strings.ToLower(strings.Trim(domain, "."))
63+
counts := map[string]int{}
64+
for _, path := range paths {
65+
b, err := diff.Load(path)
66+
if err != nil {
67+
return nil, fmt.Errorf("reading -from %s: %w", path, err)
68+
}
69+
for name := range b.Names {
70+
for _, label := range labelsBelowApex(name, domain) {
71+
counts[label]++
72+
}
73+
}
74+
}
75+
labels := make([]string, 0, len(counts))
76+
for label, n := range counts {
77+
if n >= minCount {
78+
labels = append(labels, label)
79+
}
80+
}
81+
sort.Slice(labels, func(i, j int) bool {
82+
if counts[labels[i]] != counts[labels[j]] {
83+
return counts[labels[i]] > counts[labels[j]]
84+
}
85+
return labels[i] < labels[j]
86+
})
87+
return labels, nil
88+
}
89+
90+
// labelsBelowApex splits name into the labels left of its apex.
91+
func labelsBelowApex(name, domain string) []string {
92+
if domain != "" && strings.HasSuffix(name, "."+domain) {
93+
return strings.Split(strings.TrimSuffix(name, "."+domain), ".")
94+
}
95+
labels := strings.Split(name, ".")
96+
apex := 2 // the organisation's label and the TLD
97+
if len(labels) > 2 && secondLevelSuffixes[labels[len(labels)-2]] {
98+
apex = 3
99+
}
100+
if len(labels) <= apex {
101+
return nil
102+
}
103+
return labels[:len(labels)-apex]
44104
}
45105

46106
// domainTerms extracts candidate words from a domain: the public suffix is
@@ -66,10 +126,11 @@ func domainTerms(domain string) []string {
66126
return terms
67127
}
68128

69-
// generate returns the wordlist in a stable order: common prefixes, then
70-
// domain terms, then each -combine prefix followed by its prefix-word
71-
// combinations. Duplicates keep their first position, so the same options
72-
// always produce byte-identical output.
129+
// generate returns the wordlist in a stable order: labels learned from
130+
// earlier results, then common prefixes, then domain terms, then each
131+
// -combine prefix followed by its prefix-word combinations. Duplicates keep
132+
// their first position, so the same options always produce byte-identical
133+
// output.
73134
func generate(o options) []string {
74135
var out []string
75136
seen := make(map[string]bool)
@@ -87,6 +148,9 @@ func generate(o options) []string {
87148
out = append(out, w)
88149
}
89150

151+
for _, l := range o.learned {
152+
add(l)
153+
}
90154
if o.common {
91155
for _, p := range commonPrefixes {
92156
add(p)
@@ -154,6 +218,9 @@ func run(args []string, stdout, stderr io.Writer) int {
154218
addCommon := fs.Bool("common", true, "Add common subdomain prefixes")
155219
domainInfo := fs.String("domain", "", "Domain to extract potential subdomains from (e.g., company-name.com -> company, name)")
156220
verbose := fs.Bool("v", false, "Print every generated entry")
221+
var from stringList
222+
fs.Var(&from, "from", "Earlier subenum results file (text, JSON, JSONL or CSV) to learn labels from, most frequent first; repeatable")
223+
minCount := fs.Int("min-count", 1, "With -from, drop labels seen fewer than this many times")
157224

158225
positionals, err := cliargs.Parse(fs, args)
159226
if err != nil {
@@ -178,9 +245,19 @@ func run(args []string, stdout, stderr io.Writer) int {
178245
return 2
179246
}
180247

181-
words := generate(options{common: *addCommon, domain: *domainInfo, combine: *combineWith})
248+
if *minCount < 1 {
249+
_, _ = fmt.Fprintf(stderr, "Error: -min-count must be at least 1, got %d\n", *minCount)
250+
return 2
251+
}
252+
learned, err := learnLabels(from, *domainInfo, *minCount)
253+
if err != nil {
254+
_, _ = fmt.Fprintf(stderr, "Error: %v\n", err)
255+
return 1
256+
}
257+
258+
words := generate(options{common: *addCommon, domain: *domainInfo, combine: *combineWith, learned: learned})
182259
if len(words) == 0 {
183-
_, _ = fmt.Fprintln(stderr, "Error: no entries generated; enable -common or pass -domain or -combine")
260+
_, _ = fmt.Fprintln(stderr, "Error: no entries generated; enable -common or pass -domain, -combine or -from")
184261
return 1
185262
}
186263

0 commit comments

Comments
 (0)