diff --git a/merkledag/importer/batch.go b/merkledag/importer/batch.go new file mode 100644 index 0000000..8948d74 --- /dev/null +++ b/merkledag/importer/batch.go @@ -0,0 +1,259 @@ +package importer + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/fs" + "os" + "time" + + mdag "github.com/ipfs/boxo/ipld/merkledag" + unixfs "github.com/ipfs/boxo/ipld/unixfs" + ipld "github.com/ipfs/go-ipld-format" +) + +// Batch stages directory changes for a basic UnixFS Editor. File objects are +// written by PutFile; Commit writes only the final affected directory versions, +// bottom-up, and advances the Editor's Root after all writes succeed. A failed +// batch may leave immutable file or directory objects in Store. It is not a +// storage transaction, and neither Batch nor Editor supports concurrent use. +type Batch struct { + editor *Editor + baseRoot string + root *batchDirectory + failed error + closed bool +} + +type batchDirectory struct { + node *mdag.ProtoNode + children map[string]*batchDirectory + dirty bool +} + +// BeginBatch starts an isolated directory overlay at the current Root. Discard +// the Batch to abandon it. HAMT and adaptive directory layouts are unsupported. +func (e *Editor) BeginBatch() (*Batch, error) { + if e == nil || e.flat || e.opts.DirLayout != DirLayoutBasic { + return nil, fmt.Errorf("directory batching requires a basic UnixFS editor") + } + root, err := e.batchDirectory(e.root) + if err != nil { + return nil, err + } + return &Batch{editor: e, baseRoot: e.Root(), root: root}, nil +} + +func (e *Editor) batchDirectory(node ipld.Node) (*batchDirectory, error) { + dirty := node == nil + if node == nil { + dir, err := e.newDirectory() + if err != nil { + return nil, err + } + node, err = dir.GetNode() + if err != nil { + return nil, err + } + } + proto, ok := node.(*mdag.ProtoNode) + if !ok { + return nil, fmt.Errorf("path component %s is not a basic UnixFS directory", node.Cid()) + } + data, err := unixfs.FSNodeFromBytes(proto.Data()) + if err != nil || data.Type() != unixfs.TDirectory { + return nil, fmt.Errorf("path component %s is not a basic UnixFS directory", node.Cid()) + } + return &batchDirectory{node: proto.Copy().(*mdag.ProtoNode), children: make(map[string]*batchDirectory), dirty: dirty}, nil +} + +func (b *Batch) ready(ctx context.Context) error { + if b == nil || b.closed { + return fmt.Errorf("directory batch is closed") + } + if b.failed != nil { + return fmt.Errorf("directory batch failed: %w", b.failed) + } + return ctx.Err() +} + +// PutFile writes a file DAG and stages its directory binding. Replacing an +// existing entry follows Editor.PutFile semantics; intermediate directory +// versions are never sent to Store. +func (b *Batch) PutFile(ctx context.Context, filePath string, data []byte, mode fs.FileMode) (err error) { + if err := b.ready(ctx); err != nil { + return err + } + defer func() { + if err != nil { + b.failed = err + } + }() + clean, parts, err := cleanPathParts(filePath) + if err != nil { + return err + } + importer := pathImporter{dag: b.editor.dag, opts: b.editor.opts, build: b.editor.build} + file, err := importer.importFileReader(ctx, clean, bytes.NewReader(data), virtualFileMode(mode), time.Time{}) + if err != nil { + return err + } + return b.put(ctx, b.root, parts, file) +} + +func (b *Batch) child(ctx context.Context, dir *batchDirectory, name string, create bool) (*batchDirectory, error) { + if child, ok := dir.children[name]; ok { + return child, nil + } + var node ipld.Node + link, err := dir.node.GetNodeLink(name) + if errors.Is(err, mdag.ErrLinkNotFound) { + if !create { + return nil, fmt.Errorf("%s: %w", name, os.ErrNotExist) + } + } else if err != nil { + return nil, err + } else { + node, err = b.editor.dag.Get(ctx, link.Cid) + if err != nil { + return nil, err + } + } + child, err := b.editor.batchDirectory(node) + if err != nil { + return nil, err + } + dir.children[name] = child + return child, nil +} + +func (b *Batch) put(ctx context.Context, dir *batchDirectory, parts []string, file ipld.Node) error { + if err := ctx.Err(); err != nil { + return err + } + name := parts[0] + if len(parts) == 1 { + if err := removeBatchLink(dir.node, name); err != nil { + return err + } + if err := dir.node.AddNodeLink(name, file); err != nil { + return err + } + delete(dir.children, name) + } else { + child, err := b.child(ctx, dir, name, true) + if err != nil { + return err + } + if err := b.put(ctx, child, parts[1:], file); err != nil { + return err + } + } + dir.dirty = true + return nil +} + +// RemoveFile removes a staged or existing binding and prunes empty ancestors. +// A directory that disappears within the batch is not written as an empty +// intermediate object. The root directory itself is always retained. +func (b *Batch) RemoveFile(ctx context.Context, filePath string) (err error) { + if err := b.ready(ctx); err != nil { + return err + } + defer func() { + if err != nil { + b.failed = err + } + }() + _, parts, err := cleanPathParts(filePath) + if err != nil { + return err + } + return b.remove(ctx, b.root, parts) +} + +func (b *Batch) remove(ctx context.Context, dir *batchDirectory, parts []string) error { + if err := ctx.Err(); err != nil { + return err + } + name := parts[0] + if len(parts) == 1 { + _, linkErr := dir.node.GetNodeLink(name) + if linkErr != nil && dir.children[name] == nil { + return fmt.Errorf("%s: %w", name, os.ErrNotExist) + } + if err := removeBatchLink(dir.node, name); err != nil { + return err + } + delete(dir.children, name) + } else { + child, err := b.child(ctx, dir, name, false) + if err != nil { + return err + } + if err := b.remove(ctx, child, parts[1:]); err != nil { + return err + } + if len(child.node.Links()) == 0 && len(child.children) == 0 { + if err := removeBatchLink(dir.node, name); err != nil { + return err + } + delete(dir.children, name) + } + } + dir.dirty = true + return nil +} + +func removeBatchLink(node *mdag.ProtoNode, name string) error { + err := node.RemoveNodeLink(name) + if errors.Is(err, mdag.ErrLinkNotFound) { + return nil + } + return err +} + +// Commit closes the batch and publishes its Root after writing each surviving +// affected directory once. A changed Editor Root rejects a stale batch. +func (b *Batch) Commit(ctx context.Context) error { + if err := b.ready(ctx); err != nil { + return err + } + defer func() { b.closed, b.root = true, nil }() + if b.editor.Root() != b.baseRoot { + return fmt.Errorf("editor root changed while directory batch was open") + } + root, err := b.flush(ctx, b.root) + if err != nil { + return err + } + b.editor.root = root + return nil +} + +func (b *Batch) flush(ctx context.Context, dir *batchDirectory) (*mdag.ProtoNode, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if !dir.dirty { + return dir.node, nil + } + for _, name := range sortedKeys(dir.children) { + child, err := b.flush(ctx, dir.children[name]) + if err != nil { + return nil, err + } + if err := removeBatchLink(dir.node, name); err != nil { + return nil, err + } + if err := dir.node.AddNodeLink(name, child); err != nil { + return nil, err + } + } + if err := b.editor.dag.Add(ctx, dir.node); err != nil { + return nil, err + } + return dir.node, nil +} diff --git a/merkledag/importer/batch_test.go b/merkledag/importer/batch_test.go new file mode 100644 index 0000000..a8c08a1 --- /dev/null +++ b/merkledag/importer/batch_test.go @@ -0,0 +1,221 @@ +package importer + +import ( + "context" + "errors" + "testing" + + casmemory "github.com/dewebprotocol/malt-client/internal/cas/memory" + mdag "github.com/ipfs/boxo/ipld/merkledag" + unixfs "github.com/ipfs/boxo/ipld/unixfs" + cid "github.com/ipfs/go-cid" +) + +type batchCountingStore struct { + *casmemory.Store + directories []cid.Cid + fail bool +} + +func (s *batchCountingStore) PutWithCodec(ctx context.Context, data []byte, codec uint64) (cid.Cid, error) { + directory := false + if codec == cid.DagProtobuf { + if node, err := mdag.DecodeProtobuf(data); err == nil { + if fsnode, err := unixfs.FSNodeFromBytes(node.Data()); err == nil { + directory = fsnode.Type() == unixfs.TDirectory + } + } + } + if directory && s.fail { + return cid.Undef, errors.New("injected directory write failure") + } + key, err := s.Store.PutWithCodec(ctx, data, codec) + if err == nil && directory { + s.directories = append(s.directories, key) + } + return key, err +} + +func batchFixture(t *testing.T, files map[string]string) (*Editor, *batchCountingStore) { + t.Helper() + store := &batchCountingStore{Store: casmemory.New()} + editor, err := NewEditor(store, Options{DirLayout: DirLayoutBasic, ChunkSize: 4, PreserveFileMode: true}) + if err != nil { + t.Fatal(err) + } + batch, err := editor.BeginBatch() + if err != nil { + t.Fatal(err) + } + for _, path := range sortedKeys(files) { + if err := batch.PutFile(t.Context(), path, []byte(files[path]), 0o644); err != nil { + t.Fatal(err) + } + } + if err := batch.Commit(t.Context()); err != nil { + t.Fatal(err) + } + return editor, store +} + +func assertBatchRoot(t *testing.T, editor *Editor, files map[string]string) { + t.Helper() + // The independently exercised single-file editor constructs the same final + // UnixFS graph, including file modes, links, sizes and directory encoding. + oracle, err := NewEditor(casmemory.New(), editor.opts) + if err != nil { + t.Fatal(err) + } + for _, path := range sortedKeys(files) { + if err := oracle.PutFile(t.Context(), path, []byte(files[path]), 0o644); err != nil { + t.Fatal(err) + } + } + if err := oracle.EnsureRoot(t.Context()); err != nil { + t.Fatal(err) + } + if editor.Root() != oracle.Root() { + t.Fatalf("batch root %s differs from independent final-tree root %s", editor.Root(), oracle.Root()) + } +} + +func TestBatchWritesOnlyFinalDirectories(t *testing.T) { + files := map[string]string{"dir/a": "old-a", "dir/b": "old-b", "untouched/file": "stable"} + editor, store := batchFixture(t, files) + if len(store.directories) != 3 { + t.Fatalf("initial directory writes = %d, want dir, untouched and root (3)", len(store.directories)) + } + before := editor.Root() + store.directories = nil + batch, err := editor.BeginBatch() + if err != nil { + t.Fatal(err) + } + for _, path := range []string{"dir/a", "dir/b"} { + files[path] = "new-" + path + if err := batch.PutFile(t.Context(), path, []byte(files[path]), 0o644); err != nil { + t.Fatal(err) + } + } + if len(store.directories) != 0 || editor.Root() != before { + t.Fatal("uncommitted batch wrote directories or advanced the root") + } + if err := batch.Commit(t.Context()); err != nil { + t.Fatal(err) + } + if len(store.directories) != 2 { + t.Fatalf("directory writes = %d, want final dir and root (2)", len(store.directories)) + } + assertBatchRoot(t, editor, files) + old, err := cid.Parse(before) + if err != nil { + t.Fatal(err) + } + if _, err := store.Get(t.Context(), old); err != nil { + t.Fatalf("old immutable root was not retained: %v", err) + } + store.directories = nil + noop, err := editor.BeginBatch() + if err != nil { + t.Fatal(err) + } + if err := noop.Commit(t.Context()); err != nil || len(store.directories) != 0 { + t.Fatalf("no-op wrote directories: %v, %v", store.directories, err) + } + if err := noop.Commit(t.Context()); err == nil { + t.Fatal("closed batch committed twice") + } +} + +func TestBatchPrunesDirectoriesAndHandlesTypeTransitions(t *testing.T) { + files := map[string]string{"a": "old-a", "b/c": "old-c", "untouched/file": "stable"} + editor, store := batchFixture(t, files) + for _, step := range []struct { + remove []string + put map[string]string + writes int + }{ + {[]string{"a", "b/c"}, map[string]string{"a/x": "new-x", "b": "new-b"}, 2}, + {[]string{"a/x", "b"}, map[string]string{"a": "final-a", "b/c": "final-c"}, 2}, + {[]string{"a", "b/c", "untouched/file"}, nil, 1}, + {nil, map[string]string{"deep/path/a": "a", "deep/path/b": "b"}, 3}, + {[]string{"deep/path/a", "deep/path/b"}, nil, 1}, + } { + store.directories = nil + batch, err := editor.BeginBatch() + if err != nil { + t.Fatal(err) + } + for _, path := range step.remove { + if err := batch.RemoveFile(t.Context(), path); err != nil { + t.Fatal(err) + } + delete(files, path) + } + for _, path := range sortedKeys(step.put) { + if err := batch.PutFile(t.Context(), path, []byte(step.put[path]), 0o644); err != nil { + t.Fatal(err) + } + files[path] = step.put[path] + } + if err := batch.Commit(t.Context()); err != nil { + t.Fatal(err) + } + if len(store.directories) != step.writes { + t.Fatalf("directory writes = %d, want %d", len(store.directories), step.writes) + } + assertBatchRoot(t, editor, files) + } +} + +func TestBatchRejectsFailedAndStaleUpdates(t *testing.T) { + for _, failure := range []string{"missing", "path", "file-ancestor", "storage", "stale", "cancelled"} { + t.Run(failure, func(t *testing.T) { + editor, store := batchFixture(t, map[string]string{"file": "original"}) + batch, err := editor.BeginBatch() + if err != nil { + t.Fatal(err) + } + ctx := t.Context() + switch failure { + case "missing": + err = batch.RemoveFile(ctx, "missing/file") + case "path": + err = batch.PutFile(ctx, "../escape", []byte("bad"), 0o644) + case "file-ancestor": + err = batch.PutFile(ctx, "file/child", []byte("bad"), 0o644) + default: + if err := batch.PutFile(ctx, "file", []byte("updated"), 0o644); err != nil { + t.Fatal(err) + } + if failure == "storage" { + store.fail = true + } else if failure == "stale" { + if err := editor.PutFile(ctx, "other", []byte("other"), 0o644); err != nil { + t.Fatal(err) + } + } else { + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + } + } + if (failure == "missing" || failure == "path" || failure == "file-ancestor") && err == nil { + t.Fatal("invalid mutation succeeded") + } + before := editor.Root() + if err := batch.Commit(ctx); err == nil || editor.Root() != before { + t.Fatalf("failed commit changed root or succeeded: %v", err) + } + }) + } + for _, layout := range []string{DirLayoutHAMT, DirLayoutAdaptive} { + editor, err := NewEditor(casmemory.New(), Options{DirLayout: layout}) + if err != nil { + t.Fatal(err) + } + if _, err := editor.BeginBatch(); err == nil { + t.Fatalf("batch accepted unsupported layout %s", layout) + } + } +}