|
| 1 | +package cli |
| 2 | + |
| 3 | +import ( |
| 4 | + "encoding/json" |
| 5 | + "fmt" |
| 6 | + "os" |
| 7 | + "path/filepath" |
| 8 | + "strings" |
| 9 | + "sync" |
| 10 | + "testing" |
| 11 | + |
| 12 | + "golang.org/x/net/dns/dnsmessage" |
| 13 | + |
| 14 | + "github.com/TMHSDigital/subenum/internal/dnstest" |
| 15 | + "github.com/TMHSDigital/subenum/internal/output" |
| 16 | + "github.com/TMHSDigital/subenum/internal/scan" |
| 17 | + "github.com/TMHSDigital/subenum/internal/wordlist" |
| 18 | +) |
| 19 | + |
| 20 | +// TestE2EResumeSkipsFinishedTarget covers #304: resuming a -dL run restores a |
| 21 | +// target that finished before the interrupt without scanning it again, and |
| 22 | +// continues the interrupted one from its saved position. |
| 23 | +func TestE2EResumeSkipsFinishedTarget(t *testing.T) { |
| 24 | + t.Parallel() |
| 25 | + wl := writeFile(t, "wl.txt", e2eWords) |
| 26 | + dl := writeFile(t, "domains.txt", "a.example\nb.example\n") |
| 27 | + args := []string{"-simulate", "-seed", "5", "-hit-rate", "100", "-t", "1", "-progress=false", "-format", "jsonl", "-w", wl, "-dL", dl} |
| 28 | + entries, _, _ := wordlist.Build(strings.Split(e2eWords, "\n"), "a.example") |
| 29 | + total := int64(len(entries)) |
| 30 | + saved := func(domain string, n int) []output.Result { |
| 31 | + var results []output.Result |
| 32 | + for _, e := range entries[:n] { |
| 33 | + results = append(results, output.Result{Result: scan.Result{Subdomain: e + "." + domain}}) |
| 34 | + } |
| 35 | + return results |
| 36 | + } |
| 37 | + // a.example is finished with three names, fewer than a scan at -hit-rate |
| 38 | + // 100 gives, and its saved position is left at the start: only done says |
| 39 | + // it needs no scan, and a second scan of it would show as extra names. |
| 40 | + path := writeState(t, resumeState{Args: args, Targets: []resumeTarget{ |
| 41 | + {Domain: "a.example", Done: true, Processed: total, Total: total, |
| 42 | + Results: saved("a.example", 3), Stats: scan.Stats{Found: 3, NXDomain: total - 3}}, |
| 43 | + {Domain: "b.example", RootDone: 2, Processed: 2, Total: total, |
| 44 | + Results: saved("b.example", 2), Stats: scan.Stats{Found: 2}}, |
| 45 | + }}) |
| 46 | + code, out, stderr := runCLIStreams(t, "-resume", path) |
| 47 | + if code != 0 { |
| 48 | + t.Fatalf("exit %d\n%s", code, stderr) |
| 49 | + } |
| 50 | + results, summary := jsonlRecords(t, out) |
| 51 | + seen := map[string]int{} |
| 52 | + for _, r := range results { |
| 53 | + seen[r.Subdomain]++ |
| 54 | + } |
| 55 | + want := map[string]int{} |
| 56 | + for _, r := range saved("a.example", 3) { |
| 57 | + want[r.Subdomain] = 1 |
| 58 | + } |
| 59 | + for _, r := range saved("b.example", len(entries)) { |
| 60 | + want[r.Subdomain] = 1 |
| 61 | + } |
| 62 | + if fmt.Sprint(seen) != fmt.Sprint(want) { |
| 63 | + t.Errorf("results = %v, want each of %v once", seen, want) |
| 64 | + } |
| 65 | + for _, msg := range []string{ |
| 66 | + "a.example was finished before the interrupt; 3 results restored", |
| 67 | + fmt.Sprintf("Resuming b.example at wordlist entry 2 of %d (2 results restored)", len(entries)), |
| 68 | + } { |
| 69 | + if !strings.Contains(stderr, msg) { |
| 70 | + t.Errorf("stderr lacks %q:\n%s", msg, stderr) |
| 71 | + } |
| 72 | + } |
| 73 | + if strings.Contains(stderr, "Resuming a.example") { |
| 74 | + t.Errorf("the finished target was scanned again:\n%s", stderr) |
| 75 | + } |
| 76 | + if len(summary.Targets) != 2 { |
| 77 | + t.Fatalf("summary has %d targets, want 2", len(summary.Targets)) |
| 78 | + } |
| 79 | + if a := summary.Targets[0]; a.Status != "ok" || a.Found != 3 || a.Processed != total { |
| 80 | + t.Errorf("a.example: status %q, found %d, processed %d; want ok, 3, %d", a.Status, a.Found, a.Processed, total) |
| 81 | + } |
| 82 | + if b := summary.Targets[1]; b.Status != "ok" || b.Found != total || b.Processed != total { |
| 83 | + t.Errorf("b.example: status %q, found %d, processed %d; want ok, %d, %d", b.Status, b.Found, b.Processed, total, total) |
| 84 | + } |
| 85 | + if _, err := os.Stat(path); !os.IsNotExist(err) { |
| 86 | + t.Errorf("the finished run's state file was not removed (err %v)", err) |
| 87 | + } |
| 88 | +} |
| 89 | + |
| 90 | +// TestE2EResolverAbortSkipsRemainingTargets covers #304: when the |
| 91 | +// reliability guard aborts one target of a -dL run, the targets after it are |
| 92 | +// skipped without a query, and the run fails; with -no-abort every target is |
| 93 | +// scanned. |
| 94 | +func TestE2EResolverAbortSkipsRemainingTargets(t *testing.T) { |
| 95 | + t.Parallel() |
| 96 | + domains := []string{"one.example", "two.example", "three.example"} |
| 97 | + // Enough failing names for the guard to act on. |
| 98 | + var words strings.Builder |
| 99 | + for i := 0; i < 250; i++ { |
| 100 | + fmt.Fprintf(&words, "host%d\n", i) |
| 101 | + } |
| 102 | + wl := writeFile(t, "wl.txt", words.String()) |
| 103 | + dl := writeFile(t, "domains.txt", strings.Join(domains, "\n")+"\n") |
| 104 | + |
| 105 | + // run scans the list against a resolver that answers for each apex and |
| 106 | + // fails every wordlist name. It returns the exit code, stderr, each |
| 107 | + // target's status and the number of queries the server saw per target. |
| 108 | + run := func(t *testing.T, extra ...string) (int, string, []string, map[string]int) { |
| 109 | + t.Helper() |
| 110 | + var mu sync.Mutex |
| 111 | + queries := map[string]int{} |
| 112 | + srv := dnstest.Start(t, func(q dnstest.Query) dnstest.Reply { |
| 113 | + for _, d := range domains { |
| 114 | + if q.Name != d && !strings.HasSuffix(q.Name, "."+d) { |
| 115 | + continue |
| 116 | + } |
| 117 | + mu.Lock() |
| 118 | + queries[d]++ |
| 119 | + mu.Unlock() |
| 120 | + switch { |
| 121 | + case q.Name != d && strings.HasPrefix(q.Name, "host"): |
| 122 | + return dnstest.ServFail |
| 123 | + case q.Name != d: |
| 124 | + return dnstest.NXDomain |
| 125 | + case q.Type == dnsmessage.TypeA: |
| 126 | + return dnstest.Reply{A: []string{"192.0.2.10"}} |
| 127 | + } |
| 128 | + return dnstest.Reply{} |
| 129 | + } |
| 130 | + return dnstest.NXDomain |
| 131 | + }) |
| 132 | + dir := t.TempDir() |
| 133 | + stats := filepath.Join(dir, "run.json") |
| 134 | + args := append(localArgs("-dns-server", srv.Addr, "-type", "A", "-progress=false", |
| 135 | + "-state", filepath.Join(dir, "state.json"), "-stats", stats, "-w", wl, "-dL", dl), extra...) |
| 136 | + code, _, stderr := runCLIStreams(t, args...) |
| 137 | + var summary output.Summary |
| 138 | + if data, err := os.ReadFile(stats); err != nil || json.Unmarshal(data, &summary) != nil { |
| 139 | + t.Fatalf("reading -stats: %v\n%s", err, stderr) |
| 140 | + } |
| 141 | + var status []string |
| 142 | + for _, tg := range summary.Targets { |
| 143 | + status = append(status, tg.Status) |
| 144 | + } |
| 145 | + mu.Lock() |
| 146 | + defer mu.Unlock() |
| 147 | + return code, stderr, status, queries |
| 148 | + } |
| 149 | + |
| 150 | + t.Run("abort", func(t *testing.T) { |
| 151 | + t.Parallel() |
| 152 | + code, stderr, status, queries := run(t) |
| 153 | + if code != exitFailure { |
| 154 | + t.Errorf("exit %d, want %d (every target failed or was skipped)\n%s", code, exitFailure, stderr) |
| 155 | + } |
| 156 | + if !strings.Contains(stderr, "skipping the 2 remaining targets") { |
| 157 | + t.Errorf("stderr does not say the remaining targets were skipped:\n%s", stderr) |
| 158 | + } |
| 159 | + if got := strings.Join(status, ","); got != "failed,skipped,skipped" { |
| 160 | + t.Errorf("target statuses = %s, want failed,skipped,skipped", got) |
| 161 | + } |
| 162 | + if queries["one.example"] == 0 || queries["two.example"] != 0 || queries["three.example"] != 0 { |
| 163 | + t.Errorf("queries per target = %v, want some for one.example only", queries) |
| 164 | + } |
| 165 | + }) |
| 166 | + t.Run("no-abort", func(t *testing.T) { |
| 167 | + t.Parallel() |
| 168 | + code, stderr, status, queries := run(t, "-no-abort") |
| 169 | + if code != exitOK { |
| 170 | + t.Errorf("exit %d, want %d\n%s", code, exitOK, stderr) |
| 171 | + } |
| 172 | + if strings.Contains(stderr, "remaining targets") { |
| 173 | + t.Errorf("targets were skipped under -no-abort:\n%s", stderr) |
| 174 | + } |
| 175 | + if got := strings.Join(status, ","); got != "ok,ok,ok" { |
| 176 | + t.Errorf("target statuses = %s, want ok,ok,ok", got) |
| 177 | + } |
| 178 | + for _, d := range domains { |
| 179 | + if queries[d] == 0 { |
| 180 | + t.Errorf("%s got no queries under -no-abort: %v", d, queries) |
| 181 | + } |
| 182 | + } |
| 183 | + }) |
| 184 | +} |
| 185 | + |
| 186 | +// TestE2EReportFileWriteErrors covers #304: a -stats, -sarif or -inconclusive |
| 187 | +// file that cannot be written is reported and fails the run, after the |
| 188 | +// results were printed. |
| 189 | +func TestE2EReportFileWriteErrors(t *testing.T) { |
| 190 | + t.Parallel() |
| 191 | + wl := writeFile(t, "wl.txt", e2eWords) |
| 192 | + for _, flag := range []string{"-stats", "-sarif", "-inconclusive"} { |
| 193 | + t.Run(flag, func(t *testing.T) { |
| 194 | + t.Parallel() |
| 195 | + // A directory cannot be written as a file. |
| 196 | + code, out, stderr := runCLIStreams(t, "-simulate", "-hit-rate", "100", "-progress=false", |
| 197 | + "-w", wl, flag, t.TempDir(), "example.com") |
| 198 | + if code != exitFailure { |
| 199 | + t.Errorf("exit %d, want %d", code, exitFailure) |
| 200 | + } |
| 201 | + if want := "writing " + flag + " file"; strings.Count(stderr, want) != 1 { |
| 202 | + t.Errorf("stderr should have %q once:\n%s", want, stderr) |
| 203 | + } |
| 204 | + if got := len(strings.Fields(out)); got != 6 || !strings.Contains(out, "www.example.com") { |
| 205 | + t.Errorf("results were not printed in full (%d names):\n%s", got, out) |
| 206 | + } |
| 207 | + }) |
| 208 | + } |
| 209 | +} |
| 210 | + |
| 211 | +// TestE2EOutputFileCommitFailure covers #304 (and #30): when the finished |
| 212 | +// results cannot replace the -o path, the error is printed once, the run |
| 213 | +// exits 1 and no temporary file is left. |
| 214 | +func TestE2EOutputFileCommitFailure(t *testing.T) { |
| 215 | + t.Parallel() |
| 216 | + wl := writeFile(t, "wl.txt", e2eWords) |
| 217 | + dir := t.TempDir() |
| 218 | + outPath := filepath.Join(dir, "out.txt") |
| 219 | + // The path becomes a directory once the scan is under way, after the |
| 220 | + // temporary file beside it was opened. |
| 221 | + var mkdirErr error |
| 222 | + stdout := &lineTrigger{n: 1, fire: func() { mkdirErr = os.Mkdir(outPath, 0o700) }} |
| 223 | + var stderr lockedBuffer |
| 224 | + code := runWith(runArgs{ |
| 225 | + args: []string{"-simulate", "-hit-rate", "100", "-t", "1", "-progress=false", "-w", wl, "-o", outPath, "example.com"}, |
| 226 | + stdin: strings.NewReader(""), stdout: stdout, stderr: &stderr, |
| 227 | + }) |
| 228 | + if mkdirErr != nil { |
| 229 | + t.Fatal(mkdirErr) |
| 230 | + } |
| 231 | + if fi, err := os.Stat(outPath); err != nil || !fi.IsDir() { |
| 232 | + t.Fatalf("the -o path was not replaced by a directory during the scan (err %v)", err) |
| 233 | + } |
| 234 | + if code != exitFailure { |
| 235 | + t.Errorf("exit %d, want %d", code, exitFailure) |
| 236 | + } |
| 237 | + if got := strings.Count(stderr.String(), "writing output file"); got != 1 { |
| 238 | + t.Errorf("\"writing output file\" printed %d times, want once:\n%s", got, stderr.String()) |
| 239 | + } |
| 240 | + if left, _ := os.ReadDir(dir); len(left) != 1 { |
| 241 | + t.Errorf("files left beside the -o path: %v", left) |
| 242 | + } |
| 243 | +} |
| 244 | + |
| 245 | +// TestE2ESeedsImplyPermute covers #304: -seeds alone turns the permutation |
| 246 | +// pass on, and its candidates come from the seed names, here one that is not |
| 247 | +// in the wordlist. |
| 248 | +func TestE2ESeedsImplyPermute(t *testing.T) { |
| 249 | + t.Parallel() |
| 250 | + wl := writeFile(t, "wl.txt", "www\nmail\n") |
| 251 | + seeds := writeFile(t, "prev.jsonl", `{"subdomain":"api.example.com"}`+"\n") |
| 252 | + args := []string{"-simulate", "-hit-rate", "100", "-progress=false", "-format", "jsonl", "-w", wl} |
| 253 | + |
| 254 | + code, out := runCLI(t, "", append(args, "example.com")...) |
| 255 | + if code != 0 { |
| 256 | + t.Fatalf("without -seeds: exit %d", code) |
| 257 | + } |
| 258 | + if _, summary := jsonlRecords(t, out); len(summary.Targets) != 1 || summary.Targets[0].Permutation != nil { |
| 259 | + t.Fatalf("without -seeds: summary targets = %+v, want one without a permutation pass", summary.Targets) |
| 260 | + } |
| 261 | + |
| 262 | + code, out = runCLI(t, "", append(args, "-seeds", seeds, "example.com")...) |
| 263 | + if code != 0 { |
| 264 | + t.Fatalf("with -seeds: exit %d", code) |
| 265 | + } |
| 266 | + _, summary := jsonlRecords(t, out) |
| 267 | + if len(summary.Targets) != 1 || summary.Targets[0].Permutation == nil { |
| 268 | + t.Fatalf("with -seeds: summary targets = %+v, want a permutation pass", summary.Targets) |
| 269 | + } |
| 270 | + p := summary.Targets[0].Permutation |
| 271 | + // The pass's seeds are the two names the main pass found and the one |
| 272 | + // from the file. |
| 273 | + entries, found := []string{"www", "mail"}, []string{"www.example.com", "mail.example.com"} |
| 274 | + without := candidatesFor("example.com", entries, found) |
| 275 | + want := candidatesFor("example.com", entries, append(found, "api.example.com")) |
| 276 | + if len(want) <= len(without) { |
| 277 | + t.Fatalf("api adds no permutation candidates (%d with it, %d without); the test needs another seed", len(want), len(without)) |
| 278 | + } |
| 279 | + if p.Seeds != 3 || p.Candidates != len(want) || p.Processed != int64(len(want)) { |
| 280 | + t.Errorf("permutation = %+v, want 3 seeds and %d candidates, all processed", *p, len(want)) |
| 281 | + } |
| 282 | +} |
0 commit comments