From 95281f042b6dc3e3041a08f74e4530eaea1eaeba Mon Sep 17 00:00:00 2001 From: gspark-etri Date: Thu, 22 Oct 2026 13:31:52 +0900 Subject: [PATCH] onggi diff --ranges: changed byte ranges per file (#46) Co-Authored-By: Claude Opus 5.5 --- cmd/onggi/store.go | 10 ++++++++-- pkg/store/diff.go | 14 +++++++++++++- pkg/store/diff_test.go | 24 ++++++++++++++++++++---- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/cmd/onggi/store.go b/cmd/onggi/store.go index 36a7f3c..734742c 100644 --- a/cmd/onggi/store.go +++ b/cmd/onggi/store.go @@ -365,8 +365,9 @@ func diff(args []string, stdout, stderr io.Writer) int { fs.SetOutput(stderr) sf := addStoreFlags(fs) asJSON := fs.Bool("json", false, "print the diff as JSON") + ranges := fs.Bool("ranges", false, "list the changed byte ranges of each file (with --json: [offset, length] pairs)") if err := fs.Parse(args); err != nil || fs.NArg() != 2 { - fmt.Fprintln(stderr, "usage: onggi diff --store S [--json] ") + fmt.Fprintln(stderr, "usage: onggi diff --store S [--json] [--ranges] ") return 2 } ctx := context.Background() @@ -382,7 +383,7 @@ func diff(args []string, stdout, stderr io.Writer) int { return 1 } } - st, err := s.Diff(ctx, ds[0], ds[1]) + st, err := s.Diff(ctx, ds[0], ds[1], store.DiffOptions{Ranges: *ranges}) if err != nil { fmt.Fprintln(stderr, "onggi:", err) return 1 @@ -412,5 +413,10 @@ func diff(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%d\n", f.Path, f.Base, human(f.Size), human(f.Same), human(f.Moved), human(f.New), f.ChangedRuns) } tw.Flush() + for _, f := range st.Files { + for _, r := range f.Changed { + fmt.Fprintf(stdout, "changed %s offset %d length %d (%s)\n", f.Path, r[0], r[1], human(r[1])) + } + } return 0 } diff --git a/pkg/store/diff.go b/pkg/store/diff.go index b412dbb..4ccb270 100644 --- a/pkg/store/diff.go +++ b/pkg/store/diff.go @@ -89,6 +89,15 @@ type FileDiff struct { SameBlocks int64 `json:"same_blocks"` // bytes in whole BlockSize blocks that are all Same ChangedRuns int `json:"changed_runs"` // maximal runs of bytes that are not Same LargestRun int64 `json:"largest_run"` + // Changed lists the changed runs as [offset, length] when DiffOptions.Ranges is set (#46): + // every other byte is at the same offset in Base. A runtime that holds Base can update only + // these ranges. + Changed [][2]int64 `json:"changed,omitempty"` +} + +// DiffOptions controls a diff. +type DiffOptions struct { + Ranges bool // list the changed runs of each file } // DiffStats sums a diff over the files. @@ -150,7 +159,7 @@ func abs(x int64) int64 { } // Diff compares artifact m against base, both in s. -func (s *Store) Diff(ctx context.Context, base, m digest.Digest) (DiffStats, error) { +func (s *Store) Diff(ctx context.Context, base, m digest.Digest, o DiffOptions) (DiffStats, error) { var st DiffStats bm, err := s.ReadManifest(ctx, base) if err != nil { @@ -214,6 +223,9 @@ func (s *Store) Diff(ctx context.Context, base, m digest.Digest) (DiffStats, err return } fd.ChangedRuns++ + if o.Ranges { + fd.Changed = append(fd.Changed, [2]int64{changedFrom, end - changedFrom}) + } fd.LargestRun = max(fd.LargestRun, end-changedFrom) changedFrom = -1 } diff --git a/pkg/store/diff_test.go b/pkg/store/diff_test.go index 9ed13f0..c413f0a 100644 --- a/pkg/store/diff_test.go +++ b/pkg/store/diff_test.go @@ -1,6 +1,7 @@ package store import ( + "bytes" "context" "path/filepath" "testing" @@ -28,7 +29,7 @@ func TestDiff(t *testing.T) { base := pushDir(t, s, map[string][]byte{"decode_B2.rbln": w, "config.json": []byte(`{"batch": 2}`)}) // identical content under another name: all Same, paired by kind - st, err := s.Diff(ctx, base, pushDir(t, s, map[string][]byte{"decode_B8.rbln": w, "config.json": []byte(`{"batch": 2}`)})) + st, err := s.Diff(ctx, base, pushDir(t, s, map[string][]byte{"decode_B8.rbln": w, "config.json": []byte(`{"batch": 2}`)}), DiffOptions{}) if err != nil { t.Fatal(err) } @@ -42,7 +43,7 @@ func TestDiff(t *testing.T) { // one region overwritten in place v := append([]byte(nil), w...) copy(v[3<<20:], random(22, 200<<10)) - st, err = s.Diff(ctx, base, pushDir(t, s, map[string][]byte{"decode_B2.rbln": v})) + st, err = s.Diff(ctx, base, pushDir(t, s, map[string][]byte{"decode_B2.rbln": v}), DiffOptions{Ranges: true}) if err != nil { t.Fatal(err) } @@ -57,15 +58,30 @@ func TestDiff(t *testing.T) { t.Fatalf("no new stored bytes: %+v", st) } + // the ranges cover exactly the bytes that are not at the same offset, in order (#46) + var covered, end int64 + for _, r := range f.Changed { + if r[0] < end || r[1] <= 0 || r[0]+r[1] > f.Size { + t.Fatalf("ranges out of order or out of the file: %v", f.Changed) + } + covered, end = covered+r[1], r[0]+r[1] + } + if len(f.Changed) != f.ChangedRuns || covered != f.Size-f.Same || int64(len(v)) != f.Size { + t.Fatalf("ranges %v cover %d B, want %d", f.Changed, covered, f.Size-f.Same) + } + if !bytes.Equal(v[:f.Changed[0][0]], w[:f.Changed[0][0]]) { + t.Fatal("bytes before the first changed range differ from the base") + } + // the same bytes rearranged in 32 KiB blocks: most found elsewhere in the base, little at the // same offset; sub-chunks that straddle block boundaries are new - st, err = s.Diff(ctx, base, pushDir(t, s, map[string][]byte{"decode_B2.rbln": shuffled(w, 23)})) + st, err = s.Diff(ctx, base, pushDir(t, s, map[string][]byte{"decode_B2.rbln": shuffled(w, 23)}), DiffOptions{}) if err != nil { t.Fatal(err) } f = st.Files[0] t.Logf("rearranged: %+v", f) - if f.Same+f.Moved+f.New != f.Size || f.Moved < f.Size/2 || f.Same > f.Size/10 { + if f.Same+f.Moved+f.New != f.Size || f.Moved < f.Size/2 || f.Same > f.Size/10 || f.Changed != nil { t.Fatalf("rearranged: %+v", f) } }