fix: derive bundle proofs from cached numbered children

src/dev_bundler_task.erl

@@ -79,12 +79,7 @@ execute_task(#task{type = build_proofs, data = CommittedTX, opts = Opts} = Task)
     try
         ?event(debug_bundler, log_task(executing_task, Task, [])),
         % Calculate chunks and proofs
-        TX = hb_message:convert(
-            CommittedTX, <<"tx@1.0">>, <<"structured@1.0">>, Opts),
-        Data = TX#tx.data,
-        DataRoot = TX#tx.data_root,
-        DataSize = TX#tx.data_size,
-        Mode = ar_tx:chunking_mode(TX#tx.format),
+        {Data, DataRoot, DataSize, Mode} = proof_data(CommittedTX, Opts),
         Chunks = ar_tx:chunk_binary(Mode, ?DATA_CHUNK_SIZE, Data),
         ?event(bundler_short, {building_proofs,
             {bundle, Task#task.bundle_id},
@@ -199,6 +194,57 @@ data_items_to_tx(Items, Opts) ->
         data = List
     }).
 
+proof_data(CommittedTX, Opts) ->
+    case bundle_data(CommittedTX, Opts) of
+        {ok, Data} ->
+            Mode = ar_tx:chunking_mode(2),
+            {Data, ar_tx:data_root(Mode, Data), byte_size(Data), Mode};
+        not_found ->
+            TX = hb_message:convert(
+                CommittedTX, <<"tx@1.0">>, <<"structured@1.0">>, Opts),
+            Mode = ar_tx:chunking_mode(TX#tx.format),
+            {TX#tx.data, TX#tx.data_root, TX#tx.data_size, Mode}
+    end.
+
+bundle_data(CommittedTX, Opts) when is_map(CommittedTX) ->
+    case numbered_items(CommittedTX) of
+        [] ->
+            not_found;
+        Items ->
+            TXs = [
+                hb_message:convert(
+                    Item,
+                    <<"ans104@1.0">>,
+                    <<"structured@1.0">>,
+                    Opts
+                )
+            || Item <- Items],
+            {undefined, Data} = ar_bundles:serialize_bundle(list, TXs, false),
+            {ok, Data}
+    end;
+bundle_data(_CommittedTX, _Opts) ->
+    not_found.
+
+numbered_items(Message) ->
+    [
+        maps:get(Key, Message)
+    ||
+        Key <- lists:sort(
+            fun(A, B) -> binary_to_integer(A) =< binary_to_integer(B) end,
+            lists:filter(
+                fun(Key) ->
+                    try
+                        _ = binary_to_integer(Key),
+                        true
+                    catch
+                        _:_ -> false
+                    end
+                end,
+                maps:keys(hb_private:reset(Message))
+            )
+        )
+    ].
+
 get_price(DataSize, Opts) ->
     hb_ao:resolve(
         #{ <<"device">> => <<"arweave@2.9">> },
@@ -316,6 +362,82 @@ build_signed_tx_test() ->
         hb_mock_server:stop(ServerHandle)
     end.
 
+post_tx_and_build_proofs_bytes_match_for_nested_body_test() ->
+    Anchor = rand:bytes(32),
+    Price = 12345,
+    {ServerHandle, NodeOpts} = hb_mock_server:start_arweave_gateway(#{
+        price => {200, integer_to_binary(Price)},
+        tx_anchor => {200, hb_util:encode(Anchor)}
+    }),
+    TestOpts = NodeOpts#{
+        <<"priv-wallet">> => ar_wallet:new(),
+        <<"store">> => hb_test_utils:test_store()
+    },
+    try
+        Item = hb_message:commit(
+            #{
+                <<"body">> => #{
+                    <<"event">> => <<"is_admissible">>,
+                    <<"reference">> => <<"ref">>,
+                    <<"status-class">> => <<"success">>
+                },
+                <<"data-protocol">> => <<"ao">>,
+                <<"path">> => <<"compute">>,
+                <<"type">> => <<"Assignment">>,
+                <<"variant">> => <<"ao.N.1">>
+            },
+            TestOpts,
+            #{ <<"device">> => <<"ans104@1.0">>, <<"bundle">> => false }
+        ),
+        {ok, SignedTX} = build_signed_tx([Item], TestOpts),
+        ?assert(ar_tx:verify(SignedTX)),
+        CommittedTX = hb_message:convert(
+            SignedTX,
+            #{ <<"device">> => <<"structured@1.0">>, <<"bundle">> => true },
+            #{ <<"device">> => <<"tx@1.0">>, <<"bundle">> => true },
+            TestOpts
+        ),
+        ok = dev_bundler_cache:write_tx(CommittedTX, [Item], TestOpts),
+        TXID = hb_message:id(CommittedTX, signed, TestOpts),
+        CachedTX = dev_bundler_cache:load_tx(TXID, TestOpts),
+        {ProofData, ProofRoot, ProofSize, _Mode} =
+            proof_data(CachedTX, TestOpts),
+        ProofTX = SignedTX#tx{
+            data = ProofData,
+            data_size = ProofSize,
+            data_root = ProofRoot
+        },
+        ?assertEqual(SignedTX#tx.data_size, ProofTX#tx.data_size),
+        ?assertEqual(SignedTX#tx.data_root, ProofTX#tx.data_root),
+        ?assertEqual(SignedTX#tx.data, ProofTX#tx.data),
+        SignedBundle = ar_bundles:deserialize(SignedTX),
+        ProofBundle = ar_bundles:deserialize(ProofTX),
+        [SignedChild] =
+            hb_util:numbered_keys_to_list(SignedBundle#tx.data, TestOpts),
+        [ProofChild] =
+            hb_util:numbered_keys_to_list(ProofBundle#tx.data, TestOpts),
+        ?assert(ar_bundles:verify_item(SignedChild)),
+        ?assert(ar_bundles:verify_item(ProofChild)),
+        ?assertEqual(SignedChild, ProofChild),
+        ProofStructuredChild = hb_message:convert(
+            ProofChild, <<"structured@1.0">>, <<"ans104@1.0">>, TestOpts),
+        ?assertEqual(
+            maps:get(<<"commitments">>, Item),
+            maps:get(<<"commitments">>, ProofStructuredChild)
+        ),
+        {ok, Proofs} = execute_task(#task{
+            bundle_id = make_ref(),
+            type = build_proofs,
+            data = CachedTX,
+            opts = TestOpts
+        }),
+        [FirstProof | _] = Proofs,
+        ?assertEqual(SignedTX#tx.data_size, maps:get(data_size, FirstProof)),
+        ?assertEqual(SignedTX#tx.data_root, maps:get(data_root, FirstProof))
+    after
+        hb_mock_server:stop(ServerHandle)
+    end.
+
 build_signed_tx_on_arbundles_js_test() ->
     Anchor = rand:bytes(32),
     Price = 12345,

src/hb_cache.erl

@@ -191,7 +191,8 @@ list(Path, Opts) when is_map(Opts) and not is_map_key(<<"store-module">>, Opts)
 list(Path, Store) ->
     list(Path, Store, #{}).
 list(Path, Store, Opts) ->
-    case hb_store:read(Store, Path, Opts) of
+    case hb_store:list(Store, Path, Opts) of
+        {ok, Names} -> Names;
         {composite, Names} -> Names;
         _ -> []
     end.
@@ -1199,6 +1200,18 @@ write_with_only_read_only_store_test() ->
     ?assertMatch({ok, _}, write(<<"some-binary-payload">>, Opts)),
     ?assertMatch({ok, _}, write(#{ <<"hello">> => <<"world">> }, Opts)).
 
+list_lmdb_children_without_group_marker_test() ->
+    Store = hb_test_utils:test_store(hb_store_lmdb, <<"legacy-list">>),
+    Opts = #{ <<"store">> => Store },
+    try
+        hb_store:reset(Store),
+        ok = hb_store:write(Store, #{ <<"legacy/1">> => <<"one">> }, Opts),
+        ok = hb_store:write(Store, #{ <<"legacy/2">> => <<"two">> }, Opts),
+        ?assertEqual([<<"1">>, <<"2">>], lists:sort(list(<<"legacy">>, Opts)))
+    after
+        hb_store:reset(Store)
+    end.
+
 %% @doc Run a specific test with a given store module.
 run_test() ->
     Store = hb_test_utils:test_store(),

src/hb_store_lmdb.erl

@@ -377,13 +377,14 @@ scope(_) -> scope().
 %% @param Path Binary prefix to search for
 %% @returns {ok, [Key]} list of matching keys, {error, Reason} on failure
 list(Opts, #{ <<"list">> := Path }, _NodeOpts) ->
-    case read_resolved(Opts, hb_path:to_binary(Path)) of
+    PathBin = hb_path:to_binary(Path),
+    case read_resolved(Opts, PathBin) of
         {ok, ResolvedPath, <<"group">>} ->
             list_children(Opts, ResolvedPath);
         {ok, _ResolvedPath, _Value} ->
             {error, not_found};
         not_found ->
-            {error, not_found}
+            list_children(Opts, PathBin)
     end.
 
 list_children(Opts, ResolvedPath) ->