Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 49 additions & 20 deletions internal/download/download.go
Original file line number Diff line number Diff line change
Expand Up @@ -354,50 +354,79 @@ func (c *Client) File(ctx context.Context, o FileOptions) error {
return nil
}

// Range reads bytes [offset, offset+length) of a file of total bytes with one range request. The answer must be 206
// with exactly that Content-Range; anything else (a server that ignores Range answers 200 with the whole file) is an
// Error of Kind NoRange, and its body is not read.
// Range reads a small range into memory. Use RangeTo for entries that can be large.
func (c *Client) Range(ctx context.Context, rawURL string, offset, length, total int64, what string) ([]byte, error) {
if offset < 0 || length <= 0 || offset+length > total {
return nil, &Error{Message: fmt.Sprintf("要的区间不对(%d+%d / %d)", offset, length, total)}
var b bytes.Buffer
if err := c.RangeTo(ctx, rawURL, offset, length, total, what, &b); err != nil {
return nil, err
}
return b.Bytes(), nil
}

// RangeTo writes bytes [offset, offset+length) to dst using a fixed-size buffer. The answer must be 206 with exactly
// that Content-Range and byte count. Rejected headers leave dst untouched. A failed transfer may leave a prefix;
// the caller must reset dst before retrying. Writer errors are returned unchanged and must not be retried as network errors.
func (c *Client) RangeTo(ctx context.Context, rawURL string, offset, length, total int64, what string, dst io.Writer) error {
if offset < 0 || length <= 0 || offset > total || length > total-offset {
return &Error{Message: fmt.Sprintf("要的区间不对(%d+%d / %d)", offset, length, total)}
}
end := offset + length - 1
resp, err := c.get(ctx, rawURL, fmt.Sprintf("bytes=%d-%d", offset, end), what)
if err != nil {
return nil, err
return err
}
defer resp.close()
want := fmt.Sprintf("bytes %d-%d/%d", offset, end, total)
if got := resp.Header.Get("Content-Range"); resp.StatusCode != http.StatusPartialContent || got != want {
return nil, &Error{Message: fmt.Sprintf("服务器没按区间回(HTTP %d,Content-Range: %q,要的是 %s)", resp.StatusCode, got, want),
return &Error{Message: fmt.Sprintf("服务器没按区间回(HTTP %d,Content-Range: %q,要的是 %s)", resp.StatusCode, got, want),
Kind: NoRange, Status: resp.StatusCode}
}
if resp.ContentLength >= 0 && resp.ContentLength != length {
return nil, sizeError("区间大小和预期不符(服务器给 %d 字节,应为 %d 字节)", resp.ContentLength, length)
return sizeError("区间大小和预期不符(服务器给 %d 字节,应为 %d 字节)", resp.ContentLength, length)
}
buf := make([]byte, length+1) // one more to notice a longer body
done := 0
buf := make([]byte, 64<<10)
var done int64
for {
n, rerr := resp.read(buf[done:], c.stallTimeout())
done += n
if done > int(length) {
return nil, sizeError("区间大小和预期不符(收到的比 %d 字节多)", length)
if ctx.Err() != nil {
return canceled(ctx)
}
// One extra byte detects a chunked response that runs past the requested range, without writing it.
next := int64(len(buf))
if length-done < next {
next = length - done + 1
}
n, rerr := resp.read(buf[:next], c.stallTimeout())
if int64(n) > length-done {
return sizeError("区间大小和预期不符(收到的比 %d 字节多)", length)
}
if n > 0 {
written, werr := dst.Write(buf[:n])
if werr != nil {
return werr
}
if written != n {
return io.ErrShortWrite
}
done += int64(n)
}
if rerr == io.EOF {
break
}
if rerr != nil {
if ctx.Err() != nil {
return nil, canceled(ctx)
return canceled(ctx)
}
if resp.fired.Load() {
return nil, timeoutError("超时:%s(收数据,已收 %d 字节),%s 秒内没有进展", what, done, seconds(c.stallTimeout()))
return timeoutError("超时:%s(收数据,已收 %d 字节),%s 秒内没有进展", what, done, seconds(c.stallTimeout()))
}
return nil, &Error{Message: "收数据出错:" + NetError(rerr, "")}
return &Error{Message: "收数据出错:" + NetError(rerr, "")}
}
}
if done != int(length) {
return nil, &Error{Message: fmt.Sprintf("区间没收全:应为 %d 字节,实际 %d 字节", length, done)}
if ctx.Err() != nil {
return canceled(ctx)
}
return buf[:length], nil
if done != length {
return &Error{Message: fmt.Sprintf("区间没收全:应为 %d 字节,实际 %d 字节", length, done)}
}
return nil
}
166 changes: 166 additions & 0 deletions internal/download/range_stream_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
// Copyright 2026 jinghero
// SPDX-License-Identifier: AGPL-3.0-or-later

package download

import (
"bytes"
"context"
"errors"
"fmt"
"io"
"math"
"net/http"
"net/http/httptest"
"runtime"
"strings"
"testing"
"time"

"modavd/internal/errs"
)

type rangeWriter struct {
count, max int
err error
cancel context.CancelFunc
}

func (w *rangeWriter) Write(p []byte) (int, error) {
w.count += len(p)
w.max = max(w.max, len(p))
if w.cancel != nil {
w.cancel()
}
if w.err != nil {
return 0, w.err
}
return len(p), nil
}

func TestRangeToStreamsWithBoundedMemory(t *testing.T) {
const size = 12 << 20
// A generated body has no size-dependent allocation to hide the client's allocation cost.
body := &watchedBody{Reader: io.LimitReader(zeroReader{}, size)}
d := Client{HTTP: &http.Client{Transport: roundTrip(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: 206, Header: http.Header{"Content-Range": {fmt.Sprintf("bytes 0-%d/%d", size-1, size)}}, ContentLength: size, Body: body}, nil
})}}
w := &rangeWriter{}
runtime.GC()
var before, after runtime.MemStats
runtime.ReadMemStats(&before)
err := d.RangeTo(context.Background(), "https://example.invalid/pkg", 0, size, size, "test", w)
runtime.ReadMemStats(&after)
allocated := after.TotalAlloc - before.TotalAlloc
if err != nil || w.count != size || w.max > 64<<10 || !body.closed || allocated > 2<<20 {
t.Fatalf("stream: bytes %d, max chunk %d, allocated %d, closed %v, error %v", w.count, w.max, allocated, body.closed, err)
}
t.Logf("%d bytes transferred, max write %d, allocated %d", size, w.max, allocated)
}

type zeroReader struct{}

func (zeroReader) Read(p []byte) (int, error) { clear(p); return len(p), nil }

func TestRangeToProtocol(t *testing.T) {
for _, tc := range []struct {
name, cr, body, encoding string
status int
length int64
kind Kind
want string
}{
{"valid", "bytes 2-4/10", "cde", "", 206, 3, Network, "cde"},
{"ignored", "", "abcdefghij", "", 200, 10, NoRange, ""},
{"other range", "bytes 2-5/10", "cdef", "", 206, 4, NoRange, ""},
{"wrong total", "bytes 2-4/11", "cde", "", 206, 3, NoRange, ""},
{"wrong length", "bytes 2-4/10", "cd", "", 206, 2, Size, ""},
{"encoding", "bytes 2-4/10", "cde", "gzip", 206, 3, Encoding, ""},
{"short", "bytes 2-4/10", "cd", "", 206, -1, Network, "cd"},
{"long", "bytes 2-4/10", "cdef", "", 206, -1, Size, ""},
} {
t.Run(tc.name, func(t *testing.T) {
body := &watchedBody{Reader: strings.NewReader(tc.body)}
d := Client{HTTP: &http.Client{Transport: roundTrip(func(req *http.Request) (*http.Response, error) {
if req.Header.Get("Range") != "bytes=2-4" || req.Header.Get("Accept-Encoding") != "identity" {
t.Errorf("headers: %v", req.Header)
}
return &http.Response{StatusCode: tc.status, Header: http.Header{"Content-Range": {tc.cr}, "Content-Encoding": {tc.encoding}}, ContentLength: tc.length, Body: body}, nil
})}}
var dst bytes.Buffer
err := d.RangeTo(context.Background(), "https://example.invalid/pkg", 2, 3, 10, "test", &dst)
var de *Error
if tc.name == "valid" {
if err != nil {
t.Fatal(err)
}
} else if !errors.As(err, &de) || de.Kind != tc.kind {
t.Fatalf("want kind %d, got %v", tc.kind, err)
}
if dst.String() != tc.want || !body.closed {
t.Fatalf("output %q, closed %v", dst.String(), body.closed)
}
})
}
}

func TestRangeToLocalErrorAndCancel(t *testing.T) {
local := errors.New("disk full")
for _, cancelWrite := range []bool{false, true} {
ctx, cancel := context.WithCancel(context.Background())
body := &watchedBody{Reader: strings.NewReader("abc")}
d := Client{HTTP: &http.Client{Transport: roundTrip(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: 206, Header: http.Header{"Content-Range": {"bytes 0-2/3"}}, ContentLength: 3, Body: body}, nil
})}}
w := &rangeWriter{err: local}
if cancelWrite {
w.err, w.cancel = nil, cancel
}
err := d.RangeTo(ctx, "https://example.invalid/pkg", 0, 3, 3, "test", w)
cancel()
if (!cancelWrite && !errors.Is(err, local)) || (cancelWrite && errs.CodeOf(err) != errs.Canceled) || !body.closed {
t.Fatalf("cancel %v: closed %v, error %v", cancelWrite, body.closed, err)
}
}
}

func TestRangeToStallClosesResponse(t *testing.T) {
closed := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Range", "bytes 0-5/6")
w.Header().Set("Content-Length", "6")
w.WriteHeader(206)
io.WriteString(w, "abc")
w.(http.Flusher).Flush()
<-r.Context().Done()
close(closed)
}))
defer srv.Close()
d := Client{HTTP: NewHTTPClient(0), StallTimeout: 30 * time.Millisecond}
err := d.RangeTo(context.Background(), srv.URL, 0, 6, 6, "test", io.Discard)
var de *Error
if !errors.As(err, &de) || de.Kind != Timeout {
t.Fatalf("want timeout, got %v", err)
}
select {
case <-closed:
case <-time.After(time.Second):
t.Fatal("response not canceled")
}
}

func TestRangeToOverflow(t *testing.T) {
requests := 0
d := Client{HTTP: &http.Client{Transport: roundTrip(func(*http.Request) (*http.Response, error) {
requests++
return &http.Response{StatusCode: 200, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(""))}, nil
})}}
for _, bounds := range [][3]int64{{math.MaxInt64 - 1, 5, math.MaxInt64}, {-1, 1, 10}, {0, 0, 10}} {
if err := d.RangeTo(context.Background(), "https://example.invalid/pkg", bounds[0], bounds[1], bounds[2], "test", io.Discard); err == nil {
t.Fatal("invalid range accepted")
}
}
if requests != 0 {
t.Fatal("invalid range sent a request")
}
}
4 changes: 2 additions & 2 deletions internal/update/caller_settings_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,8 +116,8 @@ func TestCallerSettingsOverride(t *testing.T) {
t.Fatalf("status: %+v", st)
}
st, err := u.Install(context.Background())
// every file changed between the two builds: GitHub's range path falls back to the whole package (second notice)
if err != nil || !st.Updated || len(st.Notices) != 2 || !strings.Contains(st.Notices[0], "填入的 CDK 错误") || !strings.Contains(st.Notices[1], "变了的文件太多") {
// The normal GitHub range threshold adds no notice; the bad CDK still does.
if err != nil || !st.Updated || len(st.Notices) != 1 || !strings.Contains(st.Notices[0], "填入的 CDK 错误") {
t.Fatalf("install: %+v %v", st, err)
}
if len(seen) == 0 || seen[len(seen)-1]["cdk"] != cdk {
Expand Down
Loading
Loading