Skip to content

Commit 566655b

Browse files
Fix #409: CT and Wayback drop every parsed name when the body read fail…
Closes #409. Prepared by AgentLab (an AI agent system).
1 parent dd5b156 commit 566655b

4 files changed

Lines changed: 66 additions & 3 deletions

File tree

‎internal/ct/ct.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,11 @@ func (c Client) Names(ctx context.Context, domain string) ([]string, error) {
8787
// cut instead of losing them all (#251).
8888
return sortedNames(set), fmt.Errorf("%w: the answer reached the %d MiB read limit", ErrTruncated, maxBody>>20)
8989
}
90+
if body.err != nil && len(set) > 0 {
91+
// The body read failed midway (a timeout, a reset connection):
92+
// keep the names parsed before the cut, as for the limit (#409).
93+
return sortedNames(set), fmt.Errorf("%w: the answer was cut off: %v", ErrTruncated, body.err)
94+
}
9095
return nil, fmt.Errorf("certificate transparency lookup: reading %s's answer: %w", resp.Request.URL.Host, err)
9196
}
9297
return sortedNames(set), nil
@@ -124,6 +129,7 @@ type limitReader struct {
124129
r io.Reader
125130
n int64
126131
hit bool
132+
err error // a read error other than io.EOF, such as a timeout
127133
}
128134

129135
func (l *limitReader) Read(p []byte) (int, error) {
@@ -136,6 +142,9 @@ func (l *limitReader) Read(p []byte) (int, error) {
136142
}
137143
n, err := l.r.Read(p)
138144
l.n -= int64(n)
145+
if err != nil && !errors.Is(err, io.EOF) {
146+
l.err = err
147+
}
139148
return n, err
140149
}
141150

‎internal/ct/ct_test.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,30 @@ func TestNamesOversizedBody(t *testing.T) {
147147
}
148148
}
149149

150+
// TestNamesStalledBody covers #409: a body that stalls past the timeout
151+
// keeps the names parsed before it, with ErrTruncated.
152+
func TestNamesStalledBody(t *testing.T) {
153+
release := make(chan struct{})
154+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
155+
_, _ = io.WriteString(w, `[{"name_value":"a.example.com"},`)
156+
w.(http.Flusher).Flush()
157+
select {
158+
case <-r.Context().Done():
159+
case <-release:
160+
}
161+
}))
162+
defer srv.Close()
163+
defer close(release)
164+
165+
got, err := names(context.Background(), srv, 200*time.Millisecond)
166+
if !errors.Is(err, ErrTruncated) {
167+
t.Errorf("error = %v, want one wrapping ErrTruncated", err)
168+
}
169+
if want := []string{"a"}; !slices.Equal(got, want) {
170+
t.Errorf("Names = %v, want the names before the stall %v", got, want)
171+
}
172+
}
173+
150174
func TestNamesNotAnArray(t *testing.T) {
151175
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
152176
_, _ = w.Write([]byte(`{"name_value": "a.example.com"}`))

‎internal/passive/passive_test.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,28 @@ func TestWaybackTransportError(t *testing.T) {
194194
}
195195
}
196196

197+
// TestWaybackStalledBody covers #409: a body that stalls past the timeout
198+
// keeps the names read before it, with a Truncated error.
199+
func TestWaybackStalledBody(t *testing.T) {
200+
release := make(chan struct{})
201+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
202+
_, _ = w.Write([]byte("http://a.example.com/\n"))
203+
w.(http.Flusher).Flush()
204+
select {
205+
case <-r.Context().Done():
206+
case <-release:
207+
}
208+
}))
209+
defer srv.Close()
210+
defer close(release)
211+
212+
names, err := Wayback{BaseURL: srv.URL, Timeout: 200 * time.Millisecond}.Names(context.Background(), "example.com")
213+
var tr *Truncated
214+
if !errors.As(err, &tr) || tr.Source != "wayback" || !slices.Equal(names, []string{"a"}) {
215+
t.Errorf("names %q, err %v; want [a] and a wayback Truncated error", names, err)
216+
}
217+
}
218+
197219
// FuzzHostNames: any answer parses without panicking, and every name it
198220
// gives is a valid entry under the domain (#259).
199221
func FuzzHostNames(f *testing.F) {

‎internal/passive/wayback.go‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,11 @@ func (w Wayback) Names(ctx context.Context, domain string) ([]string, error) {
8383
body := &io.LimitedReader{R: resp.Body, N: waybackMaxBody}
8484
names, lines, err := hostNames(body, domain)
8585
if err != nil {
86+
if len(names) > 0 {
87+
// The body read failed midway (a timeout, a reset connection):
88+
// keep the names read before the cut (#409).
89+
return names, &Truncated{Source: "wayback", Why: fmt.Sprintf("the answer was cut off: %v", err)}
90+
}
8691
return nil, fmt.Errorf("wayback machine lookup: reading %s's answer: %w", resp.Request.URL.Host, err)
8792
}
8893
switch {
@@ -99,16 +104,19 @@ func (w Wayback) Names(ctx context.Context, domain string) ([]string, error) {
99104
const waybackMaxLine = 1 << 20
100105

101106
// hostNames reads one archived URL per line and returns the hosts under
102-
// domain, relative to it, and how many lines it read.
107+
// domain, relative to it, and how many lines it read. On a read error it
108+
// returns the hosts read before it, with the error (#409).
103109
func hostNames(r io.Reader, domain string) ([]string, int, error) {
104110
domain = strings.ToLower(strings.TrimSuffix(domain, "."))
105111
set := map[string]bool{}
106112
br := bufio.NewReaderSize(r, 64<<10)
107113
lines := 0
114+
var readErr error
108115
for {
109116
raw, tooLong, err := readURLLine(br)
110117
if err != nil && !errors.Is(err, io.EOF) {
111-
return nil, lines, err
118+
readErr = err // the partial line before the error is dropped
119+
break
112120
}
113121
line := strings.TrimSpace(string(raw))
114122
if tooLong {
@@ -126,7 +134,7 @@ func hostNames(r io.Reader, domain string) ([]string, int, error) {
126134
out = append(out, n)
127135
}
128136
sort.Strings(out)
129-
return out, lines, nil
137+
return out, lines, readErr
130138
}
131139

132140
// readURLLine reads one line, without its newline. A line over

0 commit comments

Comments
 (0)