Support reusable flat UnixFS inputs and incremental node capabilities

docs/node-api.md

@@ -98,3 +98,15 @@ authentication store, or a new replication engine. Current local-only execution
 provides CAS; native backup/mount/write-back still compose Gateway or hybrid
 capabilities. New local executors and network transports can implement the same
 contracts without changing Core or synchronization's authority over local roots.
+
+## Local authentication computation
+
+The narrow `AuthenticationBuilder`, `AuthenticationUpdater` and
+`AuthenticationSnapshot` ports let an in-process node build original state,
+apply Core's checked `authentication.Delta`, and return a Root-checked input
+snapshot without exporting complete authentication vectors on every mutation.
+Gateway's public native storage service implements them. They introduce no HTTP
+update endpoint, publication permission or portable transition proof. Callers
+across a trust boundary verify returned state/proofs with Core at their selected
+Root. UnixFS owns projection and file encoding; node implementations own durable
+relations and reconstructible caches.

docs/unixfs-manifest.md

@@ -31,8 +31,7 @@ V2 uses these canonical encoding rules:
 - every entry has exactly `name` followed by `type`;
 - `type` is exactly `dir` or `file`;
 - `name` is one lossless UnixFS path segment: it is non-empty, is neither `.`
-  nor `..`, does not start with the reserved `@` prefix, contains no NUL, `/`,
-  or `\`, and has no leading or trailing Unicode whitespace or U+FEFF;
+  nor `..`, and contains no NUL, `/`, or `\`;
 - entries are unique and sorted by their names' UTF-8 bytes;
 - no insignificant whitespace is emitted;
 - printable Unicode is emitted directly, the short JSON escapes are used for
@@ -60,3 +59,16 @@ manifests, their `0x310001` codec, and the early raw-CID fallback are removed.
 The runtime does not infer file/directory type from a target Root layout.
 Experimental old trees must be rebuilt from their original application data;
 changing a CID prefix does not authenticate a V2 manifest.
+
+The manifest is a lossless payload, so names starting with `@` and boundary
+whitespace are preserved. Interactive staging continues to enforce its stricter
+canonical path policy at the application entry point. Flat repository imports
+use `model.FlatPathLabel`: the empty path maps to `@payload`; a source path
+beginning with `@` is escaped with a leading `/`. Other paths retain their full
+relative spelling. Only the ArcSet is flat; directory payloads never enumerate
+descendants recursively or include child CIDs.
+
+`WriteChunkedFile` stores raw chunks and uses `model.EncodeFileMetadata` for the
+CID-bound `malt.unixfs.chunks/1` payload. Its optional `mode` preserves filesystem
+permission changes; count remains authenticated Positional metadata. CAS receipts
+are checked before the original inputs reach `nodeapi.AuthenticationBuilder`.

nodeapi/ports.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"errors"
 
+	"github.com/dewebprotocol/malt-core/engine"
 	"github.com/dewebprotocol/malt-core/protocol"
+	"github.com/dewebprotocol/malt-core/sdk/authentication"
 	cid "github.com/ipfs/go-cid"
 )
 
@@ -102,6 +104,25 @@ type Authentication interface {
 	Authenticate(context.Context, protocol.AuthenticationRequest) (*protocol.AuthenticationResult, error)
 }
 
+// AuthenticationBuilder computes and stores one input view in a local node.
+// This is an in-process storage capability, not a wire update proof or authority
+// to publish. Remote clients still submit and verify exact candidates.
+type AuthenticationBuilder interface {
+	BuildAuthentication(context.Context, engine.State) (cid.Cid, error)
+}
+
+// AuthenticationUpdater applies Core's checked delta against a selected Root.
+// The returned root is operational output, never implicit client acceptance.
+type AuthenticationUpdater interface {
+	ApplyAuthentication(context.Context, cid.Cid, authentication.Delta) (cid.Cid, error)
+}
+
+// AuthenticationSnapshot returns original inputs checked at the selected Root.
+// Callers crossing a trust boundary verify this state locally with Core.
+type AuthenticationSnapshot interface {
+	SnapshotAuthentication(context.Context, cid.Cid) (engine.State, error)
+}
+
 // AuthenticationWriter transfers complete candidate materialization. It has
 // no authority to publish or accept a root. Candidate lookup returns a non-nil
 // complete candidate for exactly the requested Root. Materialization validates

unixfs/authentication_reader.go

@@ -33,7 +33,24 @@ func (r *verifiedReader) authenticate(ctx context.Context, q protocol.Authentica
 	return &protocol.AuthenticationVerification{Request: q, Result: *result}, nil
 }
 func (r *verifiedReader) resolveSegments(ctx context.Context, root cid.Cid, segments []string) (*Resolution, error) {
+	return r.resolveProjection(ctx, root, segments, false)
+}
+
+// Payload projection is explicit so a real @payload path cannot alias it.
+func (r *verifiedReader) resolvePayloadSegments(ctx context.Context, root cid.Cid, segments []string) (*Resolution, error) {
+	return r.resolveProjection(ctx, root, segments, true)
+}
+
+func (r *verifiedReader) parseReadPath(path string) ([]string, error) {
+	if r.layoutKind == LayoutFlatV1 {
+		return unixfsmodel.ParseFlatPath(path)
+	}
+	return unixfsmodel.ParsePath(path)
+}
+
+func (r *verifiedReader) resolveProjection(ctx context.Context, root cid.Cid, segments []string, payload bool) (*Resolution, error) {
 	key := stagedProjectionKey(root, segments)
+	key.payload = payload
 	if r.stagedResolutions != nil {
 		if cached := r.stagedResolutions[key]; cached != nil {
 			return cached, nil
@@ -46,20 +63,22 @@ func (r *verifiedReader) resolveSegments(ctx context.Context, root cid.Cid, segm
 	}
 	if r.layoutKind != LayoutRootedV1 {
 		// Flat and hybrid layouts authenticate one opaque full-path label.
-		// A terminal payload projection is a separate ordinary payload-label step.
-		count := len(segments)
-		payload := count > 0 && segments[count-1] == "@payload"
-		if payload {
-			count--
-		}
 		steps = [][]byte{}
-		if count > 0 {
-			steps = append(steps, []byte(strings.Join(segments[:count], "/")))
-		}
-		if payload {
-			steps = append(steps, []byte("@payload"))
+		if len(segments) > 0 {
+			label := []byte(strings.Join(segments, "/"))
+			if r.layoutKind == LayoutFlatV1 {
+				var err error
+				label, err = unixfsmodel.FlatPathLabel(string(label))
+				if err != nil {
+					return nil, err
+				}
+			}
+			steps = append(steps, label)
 		}
 	}
+	if payload {
+		steps = append(steps, []byte("@payload"))
+	}
 	q := protocol.AuthenticationRequest{Profile: protocol.AuthenticationPathProfile, Root: root.String(), Steps: steps, Operation: "resolve"}
 	verified, err := r.authenticate(ctx, q)
 	if err != nil {

unixfs/chunked_file.go

@@ -0,0 +1,57 @@
+package unixfs
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/dewebprotocol/malt-client/nodeapi"
+	model "github.com/dewebprotocol/malt-client/unixfs/model"
+	"github.com/dewebprotocol/malt-core/auth/coordinate"
+	"github.com/dewebprotocol/malt-core/derivation"
+	"github.com/dewebprotocol/malt-core/engine"
+	"github.com/dewebprotocol/malt-core/maltcid"
+	cid "github.com/ipfs/go-cid"
+)
+
+// FileBlockStore distinguishes body chunks from the metadata document without
+// requiring observers to recognize or reimplement application serialization.
+// Both kinds use raw CIDs. The boolean is true only for the metadata document.
+type FileBlockStore interface {
+	PutFileBlock(context.Context, []byte, bool) (cid.Cid, error)
+}
+
+// WriteChunkedFile stores fixed-size raw chunks and authenticated file metadata.
+// The caller supplies CAS and node capabilities; UnixFS owns file serialization.
+func WriteChunkedFile(ctx context.Context, blocks FileBlockStore, roots nodeapi.AuthenticationBuilder, data []byte, mode uint32, chunkSize int, profile maltcid.ProfileID) (cid.Cid, error) {
+	if err := ctx.Err(); err != nil {
+		return cid.Undef, err
+	}
+	if chunkSize <= 0 {
+		return cid.Undef, fmt.Errorf("chunk size must be positive")
+	}
+	entries := []engine.Entry{}
+	for start := 0; start < len(data); {
+		end := start + min(chunkSize, len(data)-start)
+		key, err := blocks.PutFileBlock(ctx, data[start:end], false)
+		if err != nil {
+			return cid.Undef, err
+		}
+		if expected, err := nodeapi.CIDForBlock(nodeapi.Block{Data: data[start:end], Codec: cid.Raw}); err != nil || !expected.Equals(key) {
+			return cid.Undef, nodeapi.ErrCorruptedBlock
+		}
+		entries = append(entries, engine.Entry{Label: coordinate.EncodeIndex(uint64(len(entries))), Target: key})
+		start = end
+	}
+	metadata, err := model.EncodeFileMetadata(uint64(len(data)), uint64(chunkSize), uint64(len(entries)), mode)
+	if err != nil {
+		return cid.Undef, err
+	}
+	payload, err := blocks.PutFileBlock(ctx, metadata, true)
+	if err != nil {
+		return cid.Undef, err
+	}
+	if expected, err := nodeapi.CIDForBlock(nodeapi.Block{Data: metadata, Codec: cid.Raw}); err != nil || !expected.Equals(payload) {
+		return cid.Undef, nodeapi.ErrCorruptedBlock
+	}
+	return roots.BuildAuthentication(ctx, engine.State{Descriptor: maltcid.RootDescriptor{Layout: maltcid.Positional, DerivationProfile: uint8(derivation.Direct), Profile: profile}, PayloadCID: payload, Entries: entries})
+}

unixfs/chunked_file_test.go

@@ -0,0 +1,83 @@
+package unixfs
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/dewebprotocol/malt-client/nodeapi"
+	model "github.com/dewebprotocol/malt-client/unixfs/model"
+	"github.com/dewebprotocol/malt-core/auth/commitment/ipa"
+	"github.com/dewebprotocol/malt-core/engine"
+	"github.com/dewebprotocol/malt-core/maltcid"
+	"github.com/dewebprotocol/malt-core/sdk/authentication"
+	cid "github.com/ipfs/go-cid"
+)
+
+type fileCapture struct {
+	state    engine.State
+	bodies   [][]byte
+	metadata []byte
+	corrupt  bool
+	builds   int
+	engine   *engine.Engine
+}
+
+func (f *fileCapture) PutFileBlock(_ context.Context, data []byte, metadata bool) (cid.Cid, error) {
+	if f.corrupt {
+		return cid.MustParse("bafkqaaa"), nil
+	}
+	if metadata {
+		f.metadata = append([]byte{}, data...)
+	} else {
+		f.bodies = append(f.bodies, append([]byte{}, data...))
+	}
+	return nodeapi.CIDForBlock(nodeapi.Block{Data: data, Codec: cid.Raw})
+}
+func (f *fileCapture) BuildAuthentication(ctx context.Context, state engine.State) (cid.Cid, error) {
+	f.state = state
+	f.builds++
+	w, err := authentication.BuildWriter(ctx, f.engine, state)
+	if err != nil {
+		return cid.Undef, err
+	}
+	return w.Root(), nil
+}
+
+func TestChunkedFileUsesRuntimeGeometryAndChecksCASReceipts(t *testing.T) {
+	scheme, err := ipa.NewCommitterScheme(ipa.ProfileDirect)
+	if err != nil {
+		t.Fatal(err)
+	}
+	registry := engine.NewRegistry()
+	if err := registry.Register(scheme); err != nil {
+		t.Fatal(err)
+	}
+	for _, data := range [][]byte{nil, []byte("123456789")} {
+		capture := &fileCapture{engine: engine.New(registry)}
+		root, err := WriteChunkedFile(t.Context(), capture, capture, data, 0o755, 4, maltcid.IPA256)
+		if err != nil || !maltcid.IsMaltCid(root) {
+			t.Fatal(err)
+		}
+		count := (len(data) + 3) / 4
+		if len(capture.bodies) != count || len(capture.state.Entries) != count {
+			t.Fatal("chunk count")
+		}
+		metadata, err := model.ParseChunkMetadata(capture.state.PayloadCID, capture.metadata, uint64(count))
+		if err != nil || metadata.TotalSize != uint64(len(data)) || metadata.Mode == nil || *metadata.Mode != 0o755 {
+			t.Fatal("metadata", err)
+		}
+		bad := &fileCapture{corrupt: true}
+		if _, err := WriteChunkedFile(t.Context(), bad, bad, data, 0o755, 4, maltcid.IPA256); !errors.Is(err, nodeapi.ErrCorruptedBlock) || bad.builds != 0 {
+			t.Fatal("unbound CAS receipt accepted", err)
+		}
+	}
+}
+
+func TestFlatProjectionEscapesOnlyTheLeadingReservedNamespace(t *testing.T) {
+	a := cid.MustParse("bafkqaaa")
+	bindings := ProjectDirectoryBindings(LayoutFlatV1, []DirectoryChild{{Name: "@payload", Root: a}, {Name: "src", Directory: true, Manifest: a, Descendants: map[string]cid.Cid{"/@scope/file": a}}})
+	if !bindings["/@payload"].Equals(a) || !bindings["src/@scope/file"].Equals(a) || !bindings["src"].Equals(a) || len(bindings) != 3 {
+		t.Fatal(bindings)
+	}
+}

unixfs/flat_reserved_read_test.go

@@ -0,0 +1,83 @@
+package unixfs_test
+
+import (
+	"testing"
+
+	unixfs "github.com/dewebprotocol/malt-client/unixfs"
+	model "github.com/dewebprotocol/malt-client/unixfs/model"
+	cid "github.com/ipfs/go-cid"
+)
+
+func TestFlatReaderDistinguishesLiteralPayloadNameFromMetadata(t *testing.T) {
+	remote := newRealRemote(t)
+	manifest := func(entries []model.DirectoryEntry) cid.Cid {
+		t.Helper()
+		body, err := model.EncodeDirectoryManifest(entries)
+		if err != nil {
+			t.Fatal(err)
+		}
+		key, err := remote.PutWithCodec(t.Context(), body.Data, body.Codec)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return key
+	}
+	bindings := map[string]string{
+		"@payload": manifest([]model.DirectoryEntry{
+			{Name: "@payload", Type: model.DirectoryEntryTypeFile},
+			{Name: "@scope", Type: model.DirectoryEntryTypeDir},
+			{Name: "src", Type: model.DirectoryEntryTypeDir},
+		}).String(),
+		"/@scope": manifest([]model.DirectoryEntry{{Name: "file", Type: model.DirectoryEntryTypeFile}}).String(),
+		"src":     manifest([]model.DirectoryEntry{{Name: "@payload", Type: model.DirectoryEntryTypeFile}}).String(),
+	}
+	paths := []string{"@payload", "@scope/file", "src/@payload"}
+	for _, path := range paths {
+		key, err := remote.Put(t.Context(), []byte(path))
+		if err != nil {
+			t.Fatal(err)
+		}
+		label, err := model.FlatPathLabel(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		bindings[string(label)] = key.String()
+	}
+	root, err := remote.UpdateStagedRoot(t.Context(), cid.Undef, bindings)
+	if err != nil {
+		t.Fatal(err)
+	}
+	opts := unixfs.ReaderOptions{Remote: remote, Blocks: remote, Layout: unixfs.LayoutFlatV1}
+	reader, err := unixfs.NewReader(opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	statter, err := unixfs.NewStagedPathStatter(opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, path := range paths {
+		t.Run(path, func(t *testing.T) {
+			resolved, err := reader.Resolve(t.Context(), root, path)
+			if err != nil {
+				t.Fatal(err)
+			}
+			stat, err := reader.(unixfs.LookupReader).Lookup(t.Context(), root, path)
+			if err != nil || stat.Kind != unixfs.StagedKindFile || !stat.Payload.Equals(resolved.Target) {
+				t.Fatalf("lookup: %+v, %v", stat, err)
+			}
+			staged, err := statter.StatStagedPath(t.Context(), root.String(), path)
+			if err != nil || staged.Kind != unixfs.StagedKindFile || staged.Key != resolved.Target.String() {
+				t.Fatalf("staged stat: %+v, %v", staged, err)
+			}
+			read, err := reader.ReadFile(t.Context(), root, path)
+			if err != nil || string(keptBody(read)) != path {
+				t.Fatalf("read: %q, %v", keptBody(read), err)
+			}
+		})
+	}
+	stat, err := reader.Stat(t.Context(), root, "")
+	if err != nil || stat.Kind != unixfs.StagedKindDirectory || len(stat.Entries) != 3 {
+		t.Fatalf("root manifest: %+v, %v", stat, err)
+	}
+}

unixfs/model/chunk_metadata.go

@@ -16,6 +16,17 @@ type ChunkMetadata struct {
 	Profile   string `json:"profile"`
 	ChunkSize uint64 `json:"chunk_size,string"`
 	TotalSize uint64 `json:"total_size,string"`
+	// Mode is optional filesystem metadata, bound by the same payload CID.
+	Mode *uint32 `json:"mode,omitempty"`
+}
+
+// EncodeFileMetadata preserves mode-only updates as authenticated writes.
+func EncodeFileMetadata(total, chunk, count uint64, mode uint32) ([]byte, error) {
+	m := ChunkMetadata{Profile: ChunkMetadataProfile, ChunkSize: chunk, TotalSize: total, Mode: &mode}
+	if err := m.Validate(count); err != nil {
+		return nil, err
+	}
+	return json.Marshal(m)
 }
 
 func (m ChunkMetadata) Validate(count uint64) error {