Skip to content

Commit 9ab8e60

Browse files
TMHSDigitalclaude
andcommitted
Retry busy passive sources and ask them concurrently (#271)
A single 429 or 5xx from crt.sh or the Wayback Machine cost a scan all of that source's names, and sources ran one after another, each with a 60-second limit, for every -dL target. A new internal/retryhttp sends their requests, retrying 429, 502, 503, 504 and transport errors twice (2s and 5s backoff, or Retry-After capped at 30s) while the lookup's time lasts. The CLI and pkg/subenum now ask a target's sources at the same time and keep results and warnings in source order. Fixes #271 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 5d50c2e commit 9ab8e60

11 files changed

Lines changed: 366 additions & 48 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -455,7 +455,7 @@ The run-quality report counts them per target (`ct_names`). If crt.sh is down or
455455
subenum -sources ct,wayback example.com
456456
```
457457

458-
The report has one `sources` entry per source, with the names it added or the `error` it hit; a failed source is a warning, and the scan goes on with the others. Simulation ignores `-sources`, lab mode refuses it, and a resumed scan reuses what each source returned.
458+
The sources of a target are asked at the same time, and a busy answer (429, 502, 503, 504) or a dropped connection is retried twice with a short backoff, honouring `Retry-After`, within the source's 60-second limit. The report has one `sources` entry per source, with the names it added or the `error` it hit; a failed source is a warning, and the scan goes on with the others. Simulation ignores `-sources`, lab mode refuses it, and a resumed scan reuses what each source returned.
459459

460460
### Zone transfers: the whole zone in one query
461461

‎internal/ct/ct.go‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"strings"
1616
"time"
1717

18+
"github.com/TMHSDigital/subenum/internal/retryhttp"
1819
"github.com/TMHSDigital/subenum/internal/wordlist"
1920
)
2021

@@ -62,25 +63,24 @@ func (c Client) Names(ctx context.Context, domain string) ([]string, error) {
6263
defer cancel()
6364

6465
q := url.Values{"q": {"%." + domain}, "output": {"json"}, "deduplicate": {"Y"}}
65-
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"?"+q.Encode(), nil)
66+
resp, err := retryhttp.Get(ctx, hc, "certificate transparency lookup", func(ctx context.Context) (*http.Request, error) {
67+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"?"+q.Encode(), nil)
68+
if err != nil {
69+
return nil, err
70+
}
71+
if c.UserAgent != "" {
72+
req.Header.Set("User-Agent", c.UserAgent)
73+
}
74+
req.Header.Set("Accept", "application/json")
75+
return req, nil
76+
})
6677
if err != nil {
6778
return nil, err
6879
}
69-
if c.UserAgent != "" {
70-
req.Header.Set("User-Agent", c.UserAgent)
71-
}
72-
req.Header.Set("Accept", "application/json")
73-
resp, err := hc.Do(req)
74-
if err != nil {
75-
return nil, fmt.Errorf("certificate transparency lookup: %w", err)
76-
}
7780
defer func() { _ = resp.Body.Close() }()
78-
if resp.StatusCode != http.StatusOK {
79-
return nil, fmt.Errorf("certificate transparency lookup: %s returned %s", req.URL.Host, resp.Status)
80-
}
8181
var entries []entry
8282
if err := json.NewDecoder(io.LimitReader(resp.Body, maxBody)).Decode(&entries); err != nil {
83-
return nil, fmt.Errorf("certificate transparency lookup: reading %s's answer: %w", req.URL.Host, err)
83+
return nil, fmt.Errorf("certificate transparency lookup: reading %s's answer: %w", resp.Request.URL.Host, err)
8484
}
8585
return relativeNames(entries, domain), nil
8686
}

‎internal/ct/ct_test.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,15 @@ import (
1111
"sync/atomic"
1212
"testing"
1313
"time"
14+
15+
"github.com/TMHSDigital/subenum/internal/retryhttp"
1416
)
1517

18+
// Failing test servers are retried at once, not after a real backoff.
19+
func init() {
20+
retryhttp.Sleep = func(ctx context.Context, _ time.Duration) error { return ctx.Err() }
21+
}
22+
1623
func TestNames(t *testing.T) {
1724
var query string
1825
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

‎internal/passive/passive_test.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,16 @@ import (
77
"slices"
88
"strings"
99
"testing"
10+
"time"
11+
12+
"github.com/TMHSDigital/subenum/internal/retryhttp"
1013
)
1114

15+
// Failing test servers are retried at once, not after a real backoff.
16+
func init() {
17+
retryhttp.Sleep = func(ctx context.Context, _ time.Duration) error { return ctx.Err() }
18+
}
19+
1220
func TestParseList(t *testing.T) {
1321
got, err := ParseList(" Wayback, ct ,,ct")
1422
if err != nil || !slices.Equal(got, []string{"ct", "wayback"}) {

‎internal/passive/wayback.go‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"strings"
1313
"time"
1414

15+
"github.com/TMHSDigital/subenum/internal/retryhttp"
1516
"github.com/TMHSDigital/subenum/internal/wordlist"
1617
)
1718

@@ -64,24 +65,23 @@ func (w Wayback) Names(ctx context.Context, domain string) ([]string, error) {
6465
"collapse": {"urlkey"},
6566
"limit": {strconv.Itoa(waybackLimit)},
6667
}
67-
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"?"+q.Encode(), nil)
68+
resp, err := retryhttp.Get(ctx, hc, "wayback machine lookup", func(ctx context.Context) (*http.Request, error) {
69+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"?"+q.Encode(), nil)
70+
if err != nil {
71+
return nil, err
72+
}
73+
if w.UserAgent != "" {
74+
req.Header.Set("User-Agent", w.UserAgent)
75+
}
76+
return req, nil
77+
})
6878
if err != nil {
6979
return nil, err
7080
}
71-
if w.UserAgent != "" {
72-
req.Header.Set("User-Agent", w.UserAgent)
73-
}
74-
resp, err := hc.Do(req)
75-
if err != nil {
76-
return nil, fmt.Errorf("wayback machine lookup: %w", err)
77-
}
7881
defer func() { _ = resp.Body.Close() }()
79-
if resp.StatusCode != http.StatusOK {
80-
return nil, fmt.Errorf("wayback machine lookup: %s returned %s", req.URL.Host, resp.Status)
81-
}
8282
names, err := hostNames(io.LimitReader(resp.Body, waybackMaxBody), domain)
8383
if err != nil {
84-
return nil, fmt.Errorf("wayback machine lookup: reading %s's answer: %w", req.URL.Host, err)
84+
return nil, fmt.Errorf("wayback machine lookup: reading %s's answer: %w", resp.Request.URL.Host, err)
8585
}
8686
return names, nil
8787
}

‎internal/retryhttp/retryhttp.go‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// Package retryhttp sends the passive sources' HTTP requests, retrying the
2+
// failures a busy public service gives back (#271): 429, 502, 503 and 504,
3+
// and transport errors such as a reset connection. crt.sh in particular
4+
// often answers 502 or 503 under load, and one failed request used to cost a
5+
// scan all of that source's names.
6+
package retryhttp
7+
8+
import (
9+
"context"
10+
"fmt"
11+
"net/http"
12+
"strconv"
13+
"time"
14+
)
15+
16+
// Retries is how many times a failed request is sent again.
17+
const Retries = 2
18+
19+
// maxWait caps one wait, Retry-After included, so a long Retry-After does
20+
// not stall a scan.
21+
const maxWait = 30 * time.Second
22+
23+
// backoff is the wait before each retry when the server names none.
24+
var backoff = []time.Duration{2 * time.Second, 5 * time.Second}
25+
26+
// Sleep waits for d or until ctx ends. Tests of code that reaches a failing
27+
// test server replace it so retries do not wait for real.
28+
var Sleep = func(ctx context.Context, d time.Duration) error {
29+
t := time.NewTimer(d)
30+
defer t.Stop()
31+
select {
32+
case <-t.C:
33+
return nil
34+
case <-ctx.Done():
35+
return ctx.Err()
36+
}
37+
}
38+
39+
// retryable reports whether a response status is worth another try.
40+
func retryable(code int) bool {
41+
switch code {
42+
case http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
43+
return true
44+
}
45+
return false
46+
}
47+
48+
// Get sends a GET request built by newReq through hc, up to Retries more
49+
// times while it fails in a retryable way and ctx lasts. It returns the
50+
// first 200 response, whose body the caller closes, or the last failure as
51+
// an error naming the host and status. what prefixes errors, such as
52+
// "wayback machine lookup".
53+
func Get(ctx context.Context, hc *http.Client, what string, newReq func(context.Context) (*http.Request, error)) (*http.Response, error) {
54+
var last error
55+
for attempt := 0; ; attempt++ {
56+
req, err := newReq(ctx)
57+
if err != nil {
58+
return nil, err
59+
}
60+
resp, err := hc.Do(req)
61+
var wait time.Duration
62+
switch {
63+
case err != nil:
64+
if ctx.Err() != nil {
65+
return nil, fmt.Errorf("%s: %w", what, err)
66+
}
67+
last = fmt.Errorf("%s: %w", what, err)
68+
case resp.StatusCode == http.StatusOK:
69+
return resp, nil
70+
default:
71+
last = fmt.Errorf("%s: %s returned %s", what, req.URL.Host, resp.Status)
72+
_ = resp.Body.Close()
73+
if !retryable(resp.StatusCode) {
74+
return nil, last
75+
}
76+
wait = retryAfter(resp.Header.Get("Retry-After"), time.Now())
77+
}
78+
if attempt == Retries {
79+
return nil, last
80+
}
81+
if wait <= 0 {
82+
wait = backoff[min(attempt, len(backoff)-1)]
83+
}
84+
wait = min(wait, maxWait)
85+
if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < wait {
86+
return nil, last // the lookup's time runs out before the retry could go
87+
}
88+
if Sleep(ctx, wait) != nil {
89+
return nil, last
90+
}
91+
}
92+
}
93+
94+
// retryAfter reads a Retry-After header, in seconds or as an HTTP date; zero
95+
// when absent or unreadable.
96+
func retryAfter(v string, now time.Time) time.Duration {
97+
if v == "" {
98+
return 0
99+
}
100+
if s, err := strconv.Atoi(v); err == nil && s > 0 {
101+
return time.Duration(s) * time.Second
102+
}
103+
if t, err := http.ParseTime(v); err == nil {
104+
return t.Sub(now)
105+
}
106+
return 0
107+
}
Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
package retryhttp
2+
3+
import (
4+
"context"
5+
"net/http"
6+
"net/http/httptest"
7+
"strings"
8+
"sync/atomic"
9+
"testing"
10+
"time"
11+
)
12+
13+
// server answers each request with the next status in codes (the last one
14+
// repeats), counting the requests, and records the waits Get asks for.
15+
func server(t *testing.T, header http.Header, codes ...int) (url string, hits *atomic.Int32, waits *[]time.Duration) {
16+
t.Helper()
17+
hits = new(atomic.Int32)
18+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
19+
n := int(hits.Add(1)) - 1
20+
for k, v := range header {
21+
w.Header()[k] = v
22+
}
23+
w.WriteHeader(codes[min(n, len(codes)-1)])
24+
}))
25+
t.Cleanup(srv.Close)
26+
waits = new([]time.Duration)
27+
saved := Sleep
28+
t.Cleanup(func() { Sleep = saved })
29+
Sleep = func(_ context.Context, d time.Duration) error {
30+
*waits = append(*waits, d)
31+
return nil
32+
}
33+
return srv.URL, hits, waits
34+
}
35+
36+
func get(ctx context.Context, url string) (*http.Response, error) {
37+
return Get(ctx, http.DefaultClient, "test lookup", func(ctx context.Context) (*http.Request, error) {
38+
return http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
39+
})
40+
}
41+
42+
func TestGetRetriesBusyAnswers(t *testing.T) {
43+
url, hits, waits := server(t, nil, http.StatusServiceUnavailable, http.StatusBadGateway, http.StatusOK)
44+
resp, err := get(context.Background(), url)
45+
if err != nil {
46+
t.Fatalf("Get: %v", err)
47+
}
48+
_ = resp.Body.Close()
49+
if hits.Load() != 3 || len(*waits) != 2 || (*waits)[0] != backoff[0] || (*waits)[1] != backoff[1] {
50+
t.Errorf("%d requests, waits %v; want 3 requests with waits %v", hits.Load(), *waits, backoff)
51+
}
52+
}
53+
54+
func TestGetGivesUpAfterRetries(t *testing.T) {
55+
url, hits, _ := server(t, nil, http.StatusTooManyRequests)
56+
if _, err := get(context.Background(), url); err == nil || !strings.Contains(err.Error(), "429") || !strings.HasPrefix(err.Error(), "test lookup: ") {
57+
t.Errorf("err = %v, want the last 429 named", err)
58+
}
59+
if hits.Load() != Retries+1 {
60+
t.Errorf("%d requests, want %d", hits.Load(), Retries+1)
61+
}
62+
}
63+
64+
func TestGetDoesNotRetryOtherStatuses(t *testing.T) {
65+
url, hits, _ := server(t, nil, http.StatusNotFound, http.StatusOK)
66+
if _, err := get(context.Background(), url); err == nil || !strings.Contains(err.Error(), "404") {
67+
t.Errorf("err = %v, want the 404", err)
68+
}
69+
if hits.Load() != 1 {
70+
t.Errorf("a 404 was retried: %d requests", hits.Load())
71+
}
72+
}
73+
74+
func TestGetHonoursRetryAfter(t *testing.T) {
75+
url, _, waits := server(t, http.Header{"Retry-After": {"7"}}, http.StatusTooManyRequests, http.StatusOK)
76+
resp, err := get(context.Background(), url)
77+
if err != nil {
78+
t.Fatal(err)
79+
}
80+
_ = resp.Body.Close()
81+
if len(*waits) != 1 || (*waits)[0] != 7*time.Second {
82+
t.Errorf("waits = %v, want [7s]", *waits)
83+
}
84+
85+
url, hits, waits := server(t, http.Header{"Retry-After": {"3600"}}, http.StatusTooManyRequests, http.StatusOK)
86+
resp, err = get(context.Background(), url)
87+
if err != nil {
88+
t.Fatal(err)
89+
}
90+
_ = resp.Body.Close()
91+
if hits.Load() != 2 || (*waits)[0] != maxWait {
92+
t.Errorf("a long Retry-After: waits %v, want capped at %v", *waits, maxWait)
93+
}
94+
}
95+
96+
func TestGetStopsAtTheDeadline(t *testing.T) {
97+
url, hits, _ := server(t, nil, http.StatusServiceUnavailable, http.StatusOK)
98+
ctx, cancel := context.WithTimeout(context.Background(), time.Second) // shorter than the first backoff
99+
defer cancel()
100+
if _, err := get(ctx, url); err == nil {
101+
t.Error("Get retried past its deadline")
102+
}
103+
if hits.Load() != 1 {
104+
t.Errorf("%d requests, want 1: the retry could not finish in time", hits.Load())
105+
}
106+
}
107+
108+
func TestRetryAfter(t *testing.T) {
109+
now := time.Date(2026, 11, 13, 12, 0, 0, 0, time.UTC)
110+
for v, want := range map[string]time.Duration{
111+
"": 0,
112+
"12": 12 * time.Second,
113+
"-3": 0,
114+
"soon": 0,
115+
"Fri, 13 Nov 2026 12:00:10 GMT": 10 * time.Second,
116+
} {
117+
if got := retryAfter(v, now); got != want {
118+
t.Errorf("retryAfter(%q) = %v, want %v", v, got, want)
119+
}
120+
}
121+
}

‎main_config_test.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,16 @@ package main
22

33
import (
44
"bytes"
5+
"context"
56
"flag"
67
"os"
78
"path/filepath"
89
"slices"
910
"strings"
1011
"testing"
12+
"time"
1113

14+
"github.com/TMHSDigital/subenum/internal/retryhttp"
1215
"github.com/TMHSDigital/subenum/internal/scan"
1316
"github.com/TMHSDigital/subenum/internal/tui"
1417
)
@@ -24,6 +27,8 @@ func TestMain(m *testing.M) {
2427
_ = os.Setenv(configEnv, filepath.Join(os.TempDir(), "subenum-test-no-such-config.json"))
2528
// An interrupted run saves resume state; keep it out of the package dir.
2629
_ = os.Setenv("SUBENUM_STATE", filepath.Join(os.TempDir(), "subenum-test-resume.json"))
30+
// Passive-source tests use failing local servers; retry them at once.
31+
retryhttp.Sleep = func(ctx context.Context, _ time.Duration) error { return ctx.Err() }
2732
os.Exit(m.Run())
2833
}
2934

0 commit comments

Comments
 (0)