onggi diff --ranges: changed byte ranges per file (#46)

cmd/onggi/store.go

@@ -365,8 +365,9 @@ func diff(args []string, stdout, stderr io.Writer) int {
 	fs.SetOutput(stderr)
 	sf := addStoreFlags(fs)
 	asJSON := fs.Bool("json", false, "print the diff as JSON")
+	ranges := fs.Bool("ranges", false, "list the changed byte ranges of each file (with --json: [offset, length] pairs)")
 	if err := fs.Parse(args); err != nil || fs.NArg() != 2 {
-		fmt.Fprintln(stderr, "usage: onggi diff --store S [--json] <base ref|sha256:...> <ref|sha256:...>")
+		fmt.Fprintln(stderr, "usage: onggi diff --store S [--json] [--ranges] <base ref|sha256:...> <ref|sha256:...>")
 		return 2
 	}
 	ctx := context.Background()
@@ -382,7 +383,7 @@ func diff(args []string, stdout, stderr io.Writer) int {
 			return 1
 		}
 	}
-	st, err := s.Diff(ctx, ds[0], ds[1])
+	st, err := s.Diff(ctx, ds[0], ds[1], store.DiffOptions{Ranges: *ranges})
 	if err != nil {
 		fmt.Fprintln(stderr, "onggi:", err)
 		return 1
@@ -412,5 +413,10 @@ func diff(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%d\n", f.Path, f.Base, human(f.Size), human(f.Same), human(f.Moved), human(f.New), f.ChangedRuns)
 	}
 	tw.Flush()
+	for _, f := range st.Files {
+		for _, r := range f.Changed {
+			fmt.Fprintf(stdout, "changed %s offset %d length %d (%s)\n", f.Path, r[0], r[1], human(r[1]))
+		}
+	}
 	return 0
 }

pkg/store/diff.go

@@ -89,6 +89,15 @@ type FileDiff struct {
 	SameBlocks  int64  `json:"same_blocks"`  // bytes in whole BlockSize blocks that are all Same
 	ChangedRuns int    `json:"changed_runs"` // maximal runs of bytes that are not Same
 	LargestRun  int64  `json:"largest_run"`
+	// Changed lists the changed runs as [offset, length] when DiffOptions.Ranges is set (#46):
+	// every other byte is at the same offset in Base. A runtime that holds Base can update only
+	// these ranges.
+	Changed [][2]int64 `json:"changed,omitempty"`
+}
+
+// DiffOptions controls a diff.
+type DiffOptions struct {
+	Ranges bool // list the changed runs of each file
 }
 
 // DiffStats sums a diff over the files.
@@ -150,7 +159,7 @@ func abs(x int64) int64 {
 }
 
 // Diff compares artifact m against base, both in s.
-func (s *Store) Diff(ctx context.Context, base, m digest.Digest) (DiffStats, error) {
+func (s *Store) Diff(ctx context.Context, base, m digest.Digest, o DiffOptions) (DiffStats, error) {
 	var st DiffStats
 	bm, err := s.ReadManifest(ctx, base)
 	if err != nil {
@@ -214,6 +223,9 @@ func (s *Store) Diff(ctx context.Context, base, m digest.Digest) (DiffStats, err
 				return
 			}
 			fd.ChangedRuns++
+			if o.Ranges {
+				fd.Changed = append(fd.Changed, [2]int64{changedFrom, end - changedFrom})
+			}
 			fd.LargestRun = max(fd.LargestRun, end-changedFrom)
 			changedFrom = -1
 		}

pkg/store/diff_test.go

@@ -1,6 +1,7 @@
 package store
 
 import (
+	"bytes"
 	"context"
 	"path/filepath"
 	"testing"
@@ -28,7 +29,7 @@ func TestDiff(t *testing.T) {
 	base := pushDir(t, s, map[string][]byte{"decode_B2.rbln": w, "config.json": []byte(`{"batch": 2}`)})
 
 	// identical content under another name: all Same, paired by kind
-	st, err := s.Diff(ctx, base, pushDir(t, s, map[string][]byte{"decode_B8.rbln": w, "config.json": []byte(`{"batch": 2}`)}))
+	st, err := s.Diff(ctx, base, pushDir(t, s, map[string][]byte{"decode_B8.rbln": w, "config.json": []byte(`{"batch": 2}`)}), DiffOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -42,7 +43,7 @@ func TestDiff(t *testing.T) {
 	// one region overwritten in place
 	v := append([]byte(nil), w...)
 	copy(v[3<<20:], random(22, 200<<10))
-	st, err = s.Diff(ctx, base, pushDir(t, s, map[string][]byte{"decode_B2.rbln": v}))
+	st, err = s.Diff(ctx, base, pushDir(t, s, map[string][]byte{"decode_B2.rbln": v}), DiffOptions{Ranges: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -57,15 +58,30 @@ func TestDiff(t *testing.T) {
 		t.Fatalf("no new stored bytes: %+v", st)
 	}
 
+	// the ranges cover exactly the bytes that are not at the same offset, in order (#46)
+	var covered, end int64
+	for _, r := range f.Changed {
+		if r[0] < end || r[1] <= 0 || r[0]+r[1] > f.Size {
+			t.Fatalf("ranges out of order or out of the file: %v", f.Changed)
+		}
+		covered, end = covered+r[1], r[0]+r[1]
+	}
+	if len(f.Changed) != f.ChangedRuns || covered != f.Size-f.Same || int64(len(v)) != f.Size {
+		t.Fatalf("ranges %v cover %d B, want %d", f.Changed, covered, f.Size-f.Same)
+	}
+	if !bytes.Equal(v[:f.Changed[0][0]], w[:f.Changed[0][0]]) {
+		t.Fatal("bytes before the first changed range differ from the base")
+	}
+
 	// the same bytes rearranged in 32 KiB blocks: most found elsewhere in the base, little at the
 	// same offset; sub-chunks that straddle block boundaries are new
-	st, err = s.Diff(ctx, base, pushDir(t, s, map[string][]byte{"decode_B2.rbln": shuffled(w, 23)}))
+	st, err = s.Diff(ctx, base, pushDir(t, s, map[string][]byte{"decode_B2.rbln": shuffled(w, 23)}), DiffOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
 	f = st.Files[0]
 	t.Logf("rearranged: %+v", f)
-	if f.Same+f.Moved+f.New != f.Size || f.Moved < f.Size/2 || f.Same > f.Size/10 {
+	if f.Same+f.Moved+f.New != f.Size || f.Moved < f.Size/2 || f.Same > f.Size/10 || f.Changed != nil {
 		t.Fatalf("rearranged: %+v", f)
 	}
 }