From 209ce7d5ae0404703b8c377241e415a60f229db6 Mon Sep 17 00:00:00 2001 From: ChefBingbong <133646395+ChefBingbong@users.noreply.github.com> Date: Wed, 13 Aug 2025 19:09:34 +0100 Subject: [PATCH 1/8] fix: optimize duplicate writing in leafnode creation (#493) * fix: optimise duplicate writing in leafnode creation * chore: apply same temp array rewriting in packedUintNum64sToLeafNodes * chore: only anntend to specific dev comment before feedback --- .../persistent-merkle-tree/src/packedNode.ts | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/packages/persistent-merkle-tree/src/packedNode.ts b/packages/persistent-merkle-tree/src/packedNode.ts index d2f4a239..a8df64f8 100644 --- a/packages/persistent-merkle-tree/src/packedNode.ts +++ b/packages/persistent-merkle-tree/src/packedNode.ts @@ -45,7 +45,7 @@ export function packedUintNum64sToLeafNodes(values: number[]): LeafNode[] { */ export function packedRootsBytesToLeafNodes(dataView: DataView, start: number, end: number): Node[] { const size = end - start; - + // If the offset in data is not a multiple of 4, Uint32Array can't be used // > start offset of Uint32Array should be a multiple of 4 // NOTE: Performance tests show that using a DataView is as fast as Uint32Array @@ -53,9 +53,7 @@ export function packedRootsBytesToLeafNodes(dataView: DataView, start: number, e const fullNodeCount = Math.floor(size / 32); const leafNodes = new Array(Math.ceil(size / 32)); - // Efficiently construct the tree writing to hashObjects directly - - // TODO: Optimize, with this approach each h property is written twice + // Efficiently construct the tree writing to hashObjects directly for (let i = 0; i < fullNodeCount; i++) { const offset = start + i * 32; leafNodes[i] = new LeafNode( @@ -70,28 +68,37 @@ export function packedRootsBytesToLeafNodes(dataView: DataView, start: number, e ); } - // Consider that the last node may only include partial data +// Consider that the last node may only include partial data const remainderBytes = size % 32; - // Last node + // Instead of creating a LeafNode with zeros and then overwriting some properties, we do a + // single write in the constructor: We pass all eight hValues to the LeafNode constructor. if (remainderBytes > 0) { - const node = new LeafNode(0, 0, 0, 0, 0, 0, 0, 0); - leafNodes[fullNodeCount] = node; - - // Loop to dynamically copy the full h values + const offset = start + fullNodeCount * 32; const fullHCount = Math.floor(remainderBytes / 4); - for (let h = 0; h < fullHCount; h++) { - setNodeH(node, h, dataView.getInt32(start + fullNodeCount * 32 + h * 4, true)); + const remainderUint32 = remainderBytes % 4; + const hValues = new Array(8).fill(0); // Temporary array initialized to zeros + + // Set fully available h values + for (let i = 0; i < fullHCount; i++) { + hValues[i] = dataView.getInt32(offset + i * 4, true); } - const remainderUint32 = size % 4; + // Set partial h value if there are remaining bytes if (remainderUint32 > 0) { let h = 0; - for (let i = 0; i < remainderUint32; i++) { - h |= dataView.getUint8(start + size - remainderUint32 + i) << (i * 8); + const partialOffset = offset + fullHCount * 4; + for (let j = 0; j < remainderUint32; j++) { + h |= dataView.getUint8(partialOffset + j) << (j * 8); } - setNodeH(node, fullHCount, h); + hValues[fullHCount] = h; } + + // Create the partial node with all h values set once + leafNodes[fullNodeCount] = new LeafNode( + hValues[0], hValues[1], hValues[2], hValues[3], + hValues[4], hValues[5], hValues[6], hValues[7] + ); } return leafNodes; From 05153494f6cd04898e42ad1d98ae1cf3f708aca6 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Wed, 13 Aug 2025 20:12:03 +0200 Subject: [PATCH 2/8] Fix formatting --- .../persistent-merkle-tree/src/packedNode.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/persistent-merkle-tree/src/packedNode.ts b/packages/persistent-merkle-tree/src/packedNode.ts index a8df64f8..bbebf991 100644 --- a/packages/persistent-merkle-tree/src/packedNode.ts +++ b/packages/persistent-merkle-tree/src/packedNode.ts @@ -45,7 +45,7 @@ export function packedUintNum64sToLeafNodes(values: number[]): LeafNode[] { */ export function packedRootsBytesToLeafNodes(dataView: DataView, start: number, end: number): Node[] { const size = end - start; - + // If the offset in data is not a multiple of 4, Uint32Array can't be used // > start offset of Uint32Array should be a multiple of 4 // NOTE: Performance tests show that using a DataView is as fast as Uint32Array @@ -53,7 +53,7 @@ export function packedRootsBytesToLeafNodes(dataView: DataView, start: number, e const fullNodeCount = Math.floor(size / 32); const leafNodes = new Array(Math.ceil(size / 32)); - // Efficiently construct the tree writing to hashObjects directly + // Efficiently construct the tree writing to hashObjects directly for (let i = 0; i < fullNodeCount; i++) { const offset = start + i * 32; leafNodes[i] = new LeafNode( @@ -68,7 +68,7 @@ export function packedRootsBytesToLeafNodes(dataView: DataView, start: number, e ); } -// Consider that the last node may only include partial data + // Consider that the last node may only include partial data const remainderBytes = size % 32; // Instead of creating a LeafNode with zeros and then overwriting some properties, we do a @@ -96,8 +96,14 @@ export function packedRootsBytesToLeafNodes(dataView: DataView, start: number, e // Create the partial node with all h values set once leafNodes[fullNodeCount] = new LeafNode( - hValues[0], hValues[1], hValues[2], hValues[3], - hValues[4], hValues[5], hValues[6], hValues[7] + hValues[0], + hValues[1], + hValues[2], + hValues[3], + hValues[4], + hValues[5], + hValues[6], + hValues[7] ); } From 0f5f575aefd2f7b1a8b24dfc58a674c678e4aea4 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Wed, 13 Aug 2025 20:19:45 +0200 Subject: [PATCH 3/8] Use uint32 instead int32 for leaf nodes --- .../persistent-merkle-tree/src/packedNode.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/persistent-merkle-tree/src/packedNode.ts b/packages/persistent-merkle-tree/src/packedNode.ts index bbebf991..a2964454 100644 --- a/packages/persistent-merkle-tree/src/packedNode.ts +++ b/packages/persistent-merkle-tree/src/packedNode.ts @@ -57,14 +57,14 @@ export function packedRootsBytesToLeafNodes(dataView: DataView, start: number, e for (let i = 0; i < fullNodeCount; i++) { const offset = start + i * 32; leafNodes[i] = new LeafNode( - dataView.getInt32(offset + 0, true), - dataView.getInt32(offset + 4, true), - dataView.getInt32(offset + 8, true), - dataView.getInt32(offset + 12, true), - dataView.getInt32(offset + 16, true), - dataView.getInt32(offset + 20, true), - dataView.getInt32(offset + 24, true), - dataView.getInt32(offset + 28, true) + dataView.getUint32(offset + 0, true), + dataView.getUint32(offset + 4, true), + dataView.getUint32(offset + 8, true), + dataView.getUint32(offset + 12, true), + dataView.getUint32(offset + 16, true), + dataView.getUint32(offset + 20, true), + dataView.getUint32(offset + 24, true), + dataView.getUint32(offset + 28, true) ); } From 0a70b7278ff6ae71ff524c268174a9eaefe3072f Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Wed, 13 Aug 2025 20:27:37 +0200 Subject: [PATCH 4/8] Rename variables to clarify execution --- .../persistent-merkle-tree/src/packedNode.ts | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/packages/persistent-merkle-tree/src/packedNode.ts b/packages/persistent-merkle-tree/src/packedNode.ts index a2964454..e41ab01c 100644 --- a/packages/persistent-merkle-tree/src/packedNode.ts +++ b/packages/persistent-merkle-tree/src/packedNode.ts @@ -45,17 +45,18 @@ export function packedUintNum64sToLeafNodes(values: number[]): LeafNode[] { */ export function packedRootsBytesToLeafNodes(dataView: DataView, start: number, end: number): Node[] { const size = end - start; + const fullLeafBytes = 32; // If the offset in data is not a multiple of 4, Uint32Array can't be used // > start offset of Uint32Array should be a multiple of 4 // NOTE: Performance tests show that using a DataView is as fast as Uint32Array - const fullNodeCount = Math.floor(size / 32); - const leafNodes = new Array(Math.ceil(size / 32)); + const fullNodeCount = Math.floor(size / fullLeafBytes); + const leafNodes = new Array(Math.ceil(size / fullLeafBytes)); // Efficiently construct the tree writing to hashObjects directly for (let i = 0; i < fullNodeCount; i++) { - const offset = start + i * 32; + const offset = start + i * fullLeafBytes; leafNodes[i] = new LeafNode( dataView.getUint32(offset + 0, true), dataView.getUint32(offset + 4, true), @@ -68,30 +69,30 @@ export function packedRootsBytesToLeafNodes(dataView: DataView, start: number, e ); } - // Consider that the last node may only include partial data - const remainderBytes = size % 32; // Instead of creating a LeafNode with zeros and then overwriting some properties, we do a // single write in the constructor: We pass all eight hValues to the LeafNode constructor. + const remainderBytes = size % fullLeafBytes; if (remainderBytes > 0) { - const offset = start + fullNodeCount * 32; - const fullHCount = Math.floor(remainderBytes / 4); - const remainderUint32 = remainderBytes % 4; - const hValues = new Array(8).fill(0); // Temporary array initialized to zeros - - // Set fully available h values - for (let i = 0; i < fullHCount; i++) { - hValues[i] = dataView.getInt32(offset + i * 4, true); + const offset = start + fullNodeCount * fullLeafBytes; + // Precompute final h values once + const hValues = [0, 0, 0, 0, 0, 0, 0, 0]; + + // Whole 4-byte words we can take directly + const fullWordCount = Math.floor(remainderBytes / 4); + for (let i = 0; i < fullWordCount; i++) { + hValues[i] = dataView.getUint32(offset + i * 4, true); } - // Set partial h value if there are remaining bytes - if (remainderUint32 > 0) { + // Remaining bytes that form a partial word + const remainderByteCount = remainderBytes % 4; + if (remainderByteCount > 0) { let h = 0; - const partialOffset = offset + fullHCount * 4; - for (let j = 0; j < remainderUint32; j++) { + const partialOffset = offset + fullWordCount * 4; + for (let j = 0; j < remainderByteCount; j++) { h |= dataView.getUint8(partialOffset + j) << (j * 8); } - hValues[fullHCount] = h; + hValues[fullWordCount] = h; } // Create the partial node with all h values set once From a281f120ea7b33f40eefa800cab86b8e457ce464 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Wed, 13 Aug 2025 20:37:14 +0200 Subject: [PATCH 5/8] Optimize the pre-init lead node values to --- .../persistent-merkle-tree/src/packedNode.ts | 50 +++++++++++++------ 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/packages/persistent-merkle-tree/src/packedNode.ts b/packages/persistent-merkle-tree/src/packedNode.ts index e41ab01c..1defaafa 100644 --- a/packages/persistent-merkle-tree/src/packedNode.ts +++ b/packages/persistent-merkle-tree/src/packedNode.ts @@ -1,4 +1,4 @@ -import {LeafNode, Node, getNodeH, setNodeH} from "./node.ts"; +import {LeafNode, Node, getNodeH} from "./node.ts"; import {subtreeFillToContents} from "./subtree.ts"; const NUMBER_2_POW_32 = 2 ** 32; @@ -20,23 +20,41 @@ export function packedRootsBytesToNode(depth: number, dataView: DataView, start: * |------|------|------|------|------|------|------|------| */ export function packedUintNum64sToLeafNodes(values: number[]): LeafNode[] { - const leafNodes = new Array(Math.ceil(values.length / 4)); - for (let i = 0; i < values.length; i++) { - const nodeIndex = Math.floor(i / 4); - const leafNode = leafNodes[nodeIndex] ?? new LeafNode(0, 0, 0, 0, 0, 0, 0, 0); - const vIndex = i % 4; - const hIndex = 2 * vIndex; - const value = values[i]; - // same logic to UintNumberType.value_serializeToBytes() for 8 bytes - if (value === Infinity) { - setNodeH(leafNode, hIndex, 0xffffffff); - setNodeH(leafNode, hIndex + 1, 0xffffffff); - } else { - setNodeH(leafNode, hIndex, value & 0xffffffff); - setNodeH(leafNode, hIndex + 1, (value / NUMBER_2_POW_32) & 0xffffffff); + if (values.length === 0) return []; + + const leaves = Math.ceil(values.length / 4); + const leafNodes = new Array(leaves); + + let i = 0; // index into values + for (let nodeIndex = 0; nodeIndex < leaves; nodeIndex++) { + // Pre-fill with zeros; we’ll assign the used slots below. + let h0 = 0, h1 = 0, h2 = 0, h3 = 0, h4 = 0, h5 = 0, h6 = 0, h7 = 0; + + // Up to 4 uint64 numbers per leaf → 8 x uint32 words (lo,hi) pairs + for (let slot = 0; slot < 4 && i < values.length; slot++, i++) { + const value = values[i]; + let lo: number, hi: number; + + if (value === Infinity) { + lo = 0xffffffff; + hi = 0xffffffff; + } else { + // low 32 bits (unsigned) and high 32 bits (unsigned) + lo = (value >>> 0) >>> 0; + hi = (Math.floor(value / NUMBER_2_POW_32) >>> 0) >>> 0; + } + + switch (slot) { + case 0: h0 = lo; h1 = hi; break; + case 1: h2 = lo; h3 = hi; break; + case 2: h4 = lo; h5 = hi; break; + case 3: h6 = lo; h7 = hi; break; + } } - leafNodes[nodeIndex] = leafNode; + + leafNodes[nodeIndex] = new LeafNode(h0, h1, h2, h3, h4, h5, h6, h7); } + return leafNodes; } From 0fc9cb23ee244b19945ee1063728e1582db41f20 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Wed, 13 Aug 2025 20:37:40 +0200 Subject: [PATCH 6/8] Lint the code --- .../persistent-merkle-tree/src/packedNode.ts | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/packages/persistent-merkle-tree/src/packedNode.ts b/packages/persistent-merkle-tree/src/packedNode.ts index 1defaafa..b9fd25a8 100644 --- a/packages/persistent-merkle-tree/src/packedNode.ts +++ b/packages/persistent-merkle-tree/src/packedNode.ts @@ -28,7 +28,14 @@ export function packedUintNum64sToLeafNodes(values: number[]): LeafNode[] { let i = 0; // index into values for (let nodeIndex = 0; nodeIndex < leaves; nodeIndex++) { // Pre-fill with zeros; we’ll assign the used slots below. - let h0 = 0, h1 = 0, h2 = 0, h3 = 0, h4 = 0, h5 = 0, h6 = 0, h7 = 0; + let h0 = 0, + h1 = 0, + h2 = 0, + h3 = 0, + h4 = 0, + h5 = 0, + h6 = 0, + h7 = 0; // Up to 4 uint64 numbers per leaf → 8 x uint32 words (lo,hi) pairs for (let slot = 0; slot < 4 && i < values.length; slot++, i++) { @@ -45,10 +52,22 @@ export function packedUintNum64sToLeafNodes(values: number[]): LeafNode[] { } switch (slot) { - case 0: h0 = lo; h1 = hi; break; - case 1: h2 = lo; h3 = hi; break; - case 2: h4 = lo; h5 = hi; break; - case 3: h6 = lo; h7 = hi; break; + case 0: + h0 = lo; + h1 = hi; + break; + case 1: + h2 = lo; + h3 = hi; + break; + case 2: + h4 = lo; + h5 = hi; + break; + case 3: + h6 = lo; + h7 = hi; + break; } } @@ -87,7 +106,6 @@ export function packedRootsBytesToLeafNodes(dataView: DataView, start: number, e ); } - // Instead of creating a LeafNode with zeros and then overwriting some properties, we do a // single write in the constructor: We pass all eight hValues to the LeafNode constructor. const remainderBytes = size % fullLeafBytes; From 2a945e4e196bec5353362893ab1111b1b1f07f48 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Wed, 13 Aug 2025 20:58:12 +0200 Subject: [PATCH 7/8] Use the original int32 to get values from the view --- .../persistent-merkle-tree/src/packedNode.ts | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/persistent-merkle-tree/src/packedNode.ts b/packages/persistent-merkle-tree/src/packedNode.ts index b9fd25a8..bce25cdc 100644 --- a/packages/persistent-merkle-tree/src/packedNode.ts +++ b/packages/persistent-merkle-tree/src/packedNode.ts @@ -20,6 +20,7 @@ export function packedRootsBytesToNode(depth: number, dataView: DataView, start: * |------|------|------|------|------|------|------|------| */ export function packedUintNum64sToLeafNodes(values: number[]): LeafNode[] { + console.log("%%%%%%%%% packedUintNum64sToLeafNodes called with", values.length, "values"); if (values.length === 0) return []; const leaves = Math.ceil(values.length / 4); @@ -81,6 +82,7 @@ export function packedUintNum64sToLeafNodes(values: number[]): LeafNode[] { * Optimized deserialization of linear bytes to consecutive leaf nodes */ export function packedRootsBytesToLeafNodes(dataView: DataView, start: number, end: number): Node[] { + console.log("%%%%%%%%% packedRootsBytesToLeafNodes called with", {start, end}); const size = end - start; const fullLeafBytes = 32; @@ -95,14 +97,14 @@ export function packedRootsBytesToLeafNodes(dataView: DataView, start: number, e for (let i = 0; i < fullNodeCount; i++) { const offset = start + i * fullLeafBytes; leafNodes[i] = new LeafNode( - dataView.getUint32(offset + 0, true), - dataView.getUint32(offset + 4, true), - dataView.getUint32(offset + 8, true), - dataView.getUint32(offset + 12, true), - dataView.getUint32(offset + 16, true), - dataView.getUint32(offset + 20, true), - dataView.getUint32(offset + 24, true), - dataView.getUint32(offset + 28, true) + dataView.getInt32(offset + 0, true), + dataView.getInt32(offset + 4, true), + dataView.getInt32(offset + 8, true), + dataView.getInt32(offset + 12, true), + dataView.getInt32(offset + 16, true), + dataView.getInt32(offset + 20, true), + dataView.getInt32(offset + 24, true), + dataView.getInt32(offset + 28, true) ); } @@ -117,7 +119,7 @@ export function packedRootsBytesToLeafNodes(dataView: DataView, start: number, e // Whole 4-byte words we can take directly const fullWordCount = Math.floor(remainderBytes / 4); for (let i = 0; i < fullWordCount; i++) { - hValues[i] = dataView.getUint32(offset + i * 4, true); + hValues[i] = dataView.getInt32(offset + i * 4, true); } // Remaining bytes that form a partial word From 092d795fe0342dc99656a911acaf52ceab37a30c Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Wed, 13 Aug 2025 21:00:16 +0200 Subject: [PATCH 8/8] Remove debug logs --- packages/persistent-merkle-tree/src/packedNode.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/persistent-merkle-tree/src/packedNode.ts b/packages/persistent-merkle-tree/src/packedNode.ts index bce25cdc..a9a0317d 100644 --- a/packages/persistent-merkle-tree/src/packedNode.ts +++ b/packages/persistent-merkle-tree/src/packedNode.ts @@ -20,7 +20,6 @@ export function packedRootsBytesToNode(depth: number, dataView: DataView, start: * |------|------|------|------|------|------|------|------| */ export function packedUintNum64sToLeafNodes(values: number[]): LeafNode[] { - console.log("%%%%%%%%% packedUintNum64sToLeafNodes called with", values.length, "values"); if (values.length === 0) return []; const leaves = Math.ceil(values.length / 4); @@ -82,7 +81,6 @@ export function packedUintNum64sToLeafNodes(values: number[]): LeafNode[] { * Optimized deserialization of linear bytes to consecutive leaf nodes */ export function packedRootsBytesToLeafNodes(dataView: DataView, start: number, end: number): Node[] { - console.log("%%%%%%%%% packedRootsBytesToLeafNodes called with", {start, end}); const size = end - start; const fullLeafBytes = 32;